[fio.git] / t / nvmept.py

#!/usr/bin/env python3
"""
# nvmept.py
#
# Test fio's io_uring_cmd ioengine with NVMe pass-through commands.
#
# USAGE
# see python3 nvmept.py --help
#
# EXAMPLES
# python3 t/nvmept.py --dut /dev/ng0n1
# python3 t/nvmept.py --dut /dev/ng1n1 -f ./fio
#
# REQUIREMENTS
# Python 3.6
#
"""
import os
import sys
import json
import time
import locale
import argparse
import subprocess
from pathlib import Path

class FioTest():
    """fio test."""

    def __init__(self, artifact_root, test_opts, debug):
        """
        artifact_root   root directory for artifacts (subdirectory will be created under here)
        test            test specification
        """
        self.artifact_root = artifact_root
        self.test_opts = test_opts
        self.debug = debug
        self.filename_stub = None
        self.filenames = {}
        self.json_data = None

        self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
                                     f"{self.test_opts['test_id']:03d}"))
        if not os.path.exists(self.test_dir):
            os.mkdir(self.test_dir)

        self.filename_stub = f"pt{self.test_opts['test_id']:03d}"
        self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
        self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
        self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
        self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
        self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")

    def run_fio(self, fio_path):
        """Run a test."""

        fio_args = [
            "--name=nvmept",
            "--ioengine=io_uring_cmd",
            "--cmd_type=nvme",
            "--iodepth=8",
            "--iodepth_batch=4",
            "--iodepth_batch_complete=4",
            f"--filename={self.test_opts['filename']}",
            f"--rw={self.test_opts['rw']}",
            f"--output={self.filenames['output']}",
            f"--output-format={self.test_opts['output-format']}",
        ]
        for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
                    'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
                    'time_based', 'runtime', 'verify', 'io_size']:
            if opt in self.test_opts:
                option = f"--{opt}={self.test_opts[opt]}"
                fio_args.append(option)

        command = [fio_path] + fio_args
        with open(self.filenames['command'], "w+",
                  encoding=locale.getpreferredencoding()) as command_file:
            command_file.write(" ".join(command))

        passed = True

        try:
            with open(self.filenames['stdout'], "w+",
                      encoding=locale.getpreferredencoding()) as stdout_file, \
                open(self.filenames['stderr'], "w+",
                     encoding=locale.getpreferredencoding()) as stderr_file, \
                open(self.filenames['exitcode'], "w+",
                     encoding=locale.getpreferredencoding()) as exitcode_file:
                proc = None
                # Avoid using subprocess.run() here because when a timeout occurs,
                # fio will be stopped with SIGKILL. This does not give fio a
                # chance to clean up and means that child processes may continue
                # running and submitting IO.
                proc = subprocess.Popen(command,
                                        stdout=stdout_file,
                                        stderr=stderr_file,
                                        cwd=self.test_dir,
                                        universal_newlines=True)
                proc.communicate(timeout=300)
                exitcode_file.write(f'{proc.returncode}\n')
                passed &= (proc.returncode == 0)
        except subprocess.TimeoutExpired:
            proc.terminate()
            proc.communicate()
            assert proc.poll()
            print("Timeout expired")
            passed = False
        except Exception:
            if proc:
                if not proc.poll():
                    proc.terminate()
                    proc.communicate()
            print(f"Exception: {sys.exc_info()}")
            passed = False

        if passed:
            if 'output-format' in self.test_opts and 'json' in \
                    self.test_opts['output-format']:
                if not self.get_json():
                    print('Unable to decode JSON data')
                    passed = False

        return passed

    def get_json(self):
        """Convert fio JSON output into a python JSON object"""

        filename = self.filenames['output']
        with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
            file_data = file.read()

        #
        # Sometimes fio informational messages are included at the top of the
        # JSON output, especially under Windows. Try to decode output as JSON
        # data, lopping off up to the first four lines
        #
        lines = file_data.splitlines()
        for i in range(5):
            file_data = '\n'.join(lines[i:])
            try:
                self.json_data = json.loads(file_data)
            except json.JSONDecodeError:
                continue
            else:
                return True

        return False

    @staticmethod
    def check_empty(job):
        """
        Make sure JSON data is empty.

        Some data structures should be empty. This function makes sure that they are.

        job         JSON object that we need to check for emptiness
        """

        return job['total_ios'] == 0 and \
                job['slat_ns']['N'] == 0 and \
                job['clat_ns']['N'] == 0 and \
                job['lat_ns']['N'] == 0

    def check_all_ddirs(self, ddir_nonzero, job):
        """
        Iterate over the data directions and check whether each is
        appropriately empty or not.
        """

        retval = True
        ddirlist = ['read', 'write', 'trim']

        for ddir in ddirlist:
            if ddir in ddir_nonzero:
                if self.check_empty(job[ddir]):
                    print(f"Unexpected zero {ddir} data found in output")
                    retval = False
            else:
                if not self.check_empty(job[ddir]):
                    print(f"Unexpected {ddir} data found in output")
                    retval = False

        return retval

    def check(self):
        """Check test output."""

        raise NotImplementedError()


class PTTest(FioTest):
    """
    NVMe pass-through test class. Check to make sure output for selected data
    direction(s) is non-zero and that zero data appears for other directions.
    """

    def check(self):
        if 'rw' not in self.test_opts:
            return True

        job = self.json_data['jobs'][0]
        retval = True

        if self.test_opts['rw'] in ['read', 'randread']:
            retval = self.check_all_ddirs(['read'], job)
        elif self.test_opts['rw'] in ['write', 'randwrite']:
            if 'verify' not in self.test_opts:
                retval = self.check_all_ddirs(['write'], job)
            else:
                retval = self.check_all_ddirs(['read', 'write'], job)
        elif self.test_opts['rw'] in ['trim', 'randtrim']:
            retval = self.check_all_ddirs(['trim'], job)
        elif self.test_opts['rw'] in ['readwrite', 'randrw']:
            retval = self.check_all_ddirs(['read', 'write'], job)
        elif self.test_opts['rw'] in ['trimwrite', 'randtrimwrite']:
            retval = self.check_all_ddirs(['trim', 'write'], job)
        else:
            print(f"Unhandled rw value {self.test_opts['rw']}")
            retval = False

        return retval


def parse_args():
    """Parse command-line arguments."""

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
    parser.add_argument('-s', '--skip', nargs='+', type=int,
                        help='list of test(s) to skip')
    parser.add_argument('-o', '--run-only', nargs='+', type=int,
                        help='list of test(s) to run, skipping all others')
    parser.add_argument('--dut', help='target NVMe character device to test '
                        '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
    args = parser.parse_args()

    return args


def main():
    """Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands."""

    args = parse_args()

    artifact_root = args.artifact_root if args.artifact_root else \
        f"nvmept-test-{time.strftime('%Y%m%d-%H%M%S')}"
    os.mkdir(artifact_root)
    print(f"Artifact directory is {artifact_root}")

    if args.fio:
        fio = str(Path(args.fio).absolute())
    else:
        fio = 'fio'
    print(f"fio path is {fio}")

    test_list = [
        {
            "test_id": 1,
            "rw": 'read',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 2,
            "rw": 'randread',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 3,
            "rw": 'write',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 4,
            "rw": 'randwrite',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 5,
            "rw": 'trim',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 6,
            "rw": 'randtrim',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 7,
            "rw": 'write',
            "io_size": 1024*1024,
            "verify": "crc32c",
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 8,
            "rw": 'randwrite',
            "io_size": 1024*1024,
            "verify": "crc32c",
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 9,
            "rw": 'readwrite',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 10,
            "rw": 'randrw',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 11,
            "rw": 'trimwrite',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 12,
            "rw": 'randtrimwrite',
            "timebased": 1,
            "runtime": 3,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 13,
            "rw": 'randread',
            "timebased": 1,
            "runtime": 3,
            "fixedbufs": 1,
            "nonvectored": 1,
            "force_async": 1,
            "registerfiles": 1,
            "sqthread_poll": 1,
            "output-format": "json",
            "test_obj": PTTest,
        },
        {
            "test_id": 14,
            "rw": 'randwrite',
            "timebased": 1,
            "runtime": 3,
            "fixedbufs": 1,
            "nonvectored": 1,
            "force_async": 1,
            "registerfiles": 1,
            "sqthread_poll": 1,
            "output-format": "json",
            "test_obj": PTTest,
        },
    ]

    passed = 0
    failed = 0
    skipped = 0

    for test in test_list:
        if (args.skip and test['test_id'] in args.skip) or \
           (args.run_only and test['test_id'] not in args.run_only):
            skipped = skipped + 1
            outcome = 'SKIPPED (User request)'
        else:
            test['filename'] = args.dut
            test_obj = test['test_obj'](artifact_root, test, args.debug)
            status = test_obj.run_fio(fio)
            if status:
                status = test_obj.check()
            if status:
                passed = passed + 1
                outcome = 'PASSED'
            else:
                failed = failed + 1
                outcome = 'FAILED'

        print(f"**********Test {test['test_id']} {outcome}**********")

    print(f"{passed} tests passed, {failed} failed, {skipped} skipped")

    sys.exit(failed)


if __name__ == '__main__':
    main()
Commit	Line	Data
b03ed937 VF	1	#!/usr/bin/env python3
	2	"""
	3	# nvmept.py
	4	#
	5	# Test fio's io_uring_cmd ioengine with NVMe pass-through commands.
	6	#
	7	# USAGE
	8	# see python3 nvmept.py --help
	9	#
	10	# EXAMPLES
	11	# python3 t/nvmept.py --dut /dev/ng0n1
	12	# python3 t/nvmept.py --dut /dev/ng1n1 -f ./fio
	13	#
	14	# REQUIREMENTS
	15	# Python 3.6
	16	#
	17	"""
	18	import os
	19	import sys
	20	import json
	21	import time
	22	import locale
	23	import argparse
	24	import subprocess
	25	from pathlib import Path
	26
	27	class FioTest():
	28	"""fio test."""
	29
	30	def __init__(self, artifact_root, test_opts, debug):
	31	"""
	32	artifact_root root directory for artifacts (subdirectory will be created under here)
	33	test test specification
	34	"""
	35	self.artifact_root = artifact_root
	36	self.test_opts = test_opts
	37	self.debug = debug
	38	self.filename_stub = None
	39	self.filenames = {}
	40	self.json_data = None
	41
	42	self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
	43	f"{self.test_opts['test_id']:03d}"))
	44	if not os.path.exists(self.test_dir):
	45	os.mkdir(self.test_dir)
	46
	47	self.filename_stub = f"pt{self.test_opts['test_id']:03d}"
	48	self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
	49	self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
	50	self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
	51	self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
	52	self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")
	53
	54	def run_fio(self, fio_path):
	55	"""Run a test."""
	56
	57	fio_args = [
	58	"--name=nvmept",
	59	"--ioengine=io_uring_cmd",
	60	"--cmd_type=nvme",
	61	"--iodepth=8",
	62	"--iodepth_batch=4",
	63	"--iodepth_batch_complete=4",
	64	f"--filename={self.test_opts['filename']}",
65	f"--rw={self.test_opts['rw']}",
66	f"--output={self.filenames['output']}",
67	f"--output-format={self.test_opts['output-format']}",
68	]
69	for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
70	'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
71	'time_based', 'runtime', 'verify', 'io_size']:
72	if opt in self.test_opts:
73	option = f"--{opt}={self.test_opts[opt]}"
74	fio_args.append(option)
75
76	command = [fio_path] + fio_args
77	with open(self.filenames['command'], "w+",
78	encoding=locale.getpreferredencoding()) as command_file:
79	command_file.write(" ".join(command))
80
81	passed = True
82
83	try:
84	with open(self.filenames['stdout'], "w+",
85	encoding=locale.getpreferredencoding()) as stdout_file, \
86	open(self.filenames['stderr'], "w+",
87	encoding=locale.getpreferredencoding()) as stderr_file, \
88	open(self.filenames['exitcode'], "w+",
89	encoding=locale.getpreferredencoding()) as exitcode_file:
90	proc = None
91	# Avoid using subprocess.run() here because when a timeout occurs,
92	# fio will be stopped with SIGKILL. This does not give fio a
93	# chance to clean up and means that child processes may continue
94	# running and submitting IO.
95	proc = subprocess.Popen(command,
96	stdout=stdout_file,
97	stderr=stderr_file,
98	cwd=self.test_dir,
99	universal_newlines=True)
100	proc.communicate(timeout=300)
101	exitcode_file.write(f'{proc.returncode}\n')
102	passed &= (proc.returncode == 0)
103	except subprocess.TimeoutExpired:
104	proc.terminate()
105	proc.communicate()
106	assert proc.poll()
107	print("Timeout expired")
108	passed = False
109	except Exception:
110	if proc:
111	if not proc.poll():
112	proc.terminate()
113	proc.communicate()
114	print(f"Exception: {sys.exc_info()}")
115	passed = False
116
117	if passed:
118	if 'output-format' in self.test_opts and 'json' in \
119	self.test_opts['output-format']:
120	if not self.get_json():
121	print('Unable to decode JSON data')
122	passed = False
123
124	return passed
125
126	def get_json(self):
127	"""Convert fio JSON output into a python JSON object"""
128
129	filename = self.filenames['output']
130	with open(filename, 'r', encoding=locale.getpreferredencoding()) as file:
131	file_data = file.read()
132
133	#
134	# Sometimes fio informational messages are included at the top of the
135	# JSON output, especially under Windows. Try to decode output as JSON
136	# data, lopping off up to the first four lines
137	#
138	lines = file_data.splitlines()
139	for i in range(5):
140	file_data = '\n'.join(lines[i:])
141	try:
142	self.json_data = json.loads(file_data)
143	except json.JSONDecodeError:
144	continue
145	else:
146	return True
147
148	return False
149
150	@staticmethod
151	def check_empty(job):
152	"""
153	Make sure JSON data is empty.
154
155	Some data structures should be empty. This function makes sure that they are.
156
157	job JSON object that we need to check for emptiness
158	"""
159
160	return job['total_ios'] == 0 and \
161	job['slat_ns']['N'] == 0 and \
162	job['clat_ns']['N'] == 0 and \
163	job['lat_ns']['N'] == 0
164
165	def check_all_ddirs(self, ddir_nonzero, job):
166	"""
167	Iterate over the data directions and check whether each is
168	appropriately empty or not.
169	"""
170
171	retval = True
172	ddirlist = ['read', 'write', 'trim']
173
174	for ddir in ddirlist:
175	if ddir in ddir_nonzero:
176	if self.check_empty(job[ddir]):
177	print(f"Unexpected zero {ddir} data found in output")
178	retval = False
179	else:
180	if not self.check_empty(job[ddir]):
181	print(f"Unexpected {ddir} data found in output")
182	retval = False
183
184	return retval
185
186	def check(self):
187	"""Check test output."""
188
189	raise NotImplementedError()
190
191
192	class PTTest(FioTest):
193	"""
194	NVMe pass-through test class. Check to make sure output for selected data
195	direction(s) is non-zero and that zero data appears for other directions.
196	"""
197
198	def check(self):
199	if 'rw' not in self.test_opts:
200	return True
201
202	job = self.json_data['jobs'][0]
203	retval = True
204
205	if self.test_opts['rw'] in ['read', 'randread']:
206	retval = self.check_all_ddirs(['read'], job)
207	elif self.test_opts['rw'] in ['write', 'randwrite']:
208	if 'verify' not in self.test_opts:
209	retval = self.check_all_ddirs(['write'], job)
210	else:
211	retval = self.check_all_ddirs(['read', 'write'], job)
212	elif self.test_opts['rw'] in ['trim', 'randtrim']:
213	retval = self.check_all_ddirs(['trim'], job)
214	elif self.test_opts['rw'] in ['readwrite', 'randrw']:
215	retval = self.check_all_ddirs(['read', 'write'], job)
216	elif self.test_opts['rw'] in ['trimwrite', 'randtrimwrite']:
217	retval = self.check_all_ddirs(['trim', 'write'], job)
218	else:
219	print(f"Unhandled rw value {self.test_opts['rw']}")
220	retval = False
221
222	return retval
223
224
225	def parse_args():
226	"""Parse command-line arguments."""
227
228	parser = argparse.ArgumentParser()
229	parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
230	parser.add_argument('-a', '--artifact-root', help='artifact root directory')
231	parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
232	parser.add_argument('-s', '--skip', nargs='+', type=int,
233	help='list of test(s) to skip')
234	parser.add_argument('-o', '--run-only', nargs='+', type=int,
235	help='list of test(s) to run, skipping all others')
236	parser.add_argument('--dut', help='target NVMe character device to test '
237	'(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
238	args = parser.parse_args()
239
240	return args
241
242
243	def main():
244	"""Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands."""
245
246	args = parse_args()
247
248	artifact_root = args.artifact_root if args.artifact_root else \
249	f"nvmept-test-{time.strftime('%Y%m%d-%H%M%S')}"
250	os.mkdir(artifact_root)
251	print(f"Artifact directory is {artifact_root}")
252
253	if args.fio:
254	fio = str(Path(args.fio).absolute())
255	else:
256	fio = 'fio'
257	print(f"fio path is {fio}")
258
259	test_list = [
260	{
261	"test_id": 1,
262	"rw": 'read',
263	"timebased": 1,
264	"runtime": 3,
265	"output-format": "json",
266	"test_obj": PTTest,
267	},
268	{
269	"test_id": 2,
270	"rw": 'randread',
271	"timebased": 1,
272	"runtime": 3,
273	"output-format": "json",
274	"test_obj": PTTest,
275	},
276	{
277	"test_id": 3,
278	"rw": 'write',
279	"timebased": 1,
280	"runtime": 3,
281	"output-format": "json",
282	"test_obj": PTTest,
283	},
284	{
285	"test_id": 4,
286	"rw": 'randwrite',
287	"timebased": 1,
288	"runtime": 3,
289	"output-format": "json",
290	"test_obj": PTTest,
291	},
292	{
293	"test_id": 5,
294	"rw": 'trim',
295	"timebased": 1,
296	"runtime": 3,
297	"output-format": "json",
298	"test_obj": PTTest,
299	},
300	{
301	"test_id": 6,
302	"rw": 'randtrim',
303	"timebased": 1,
304	"runtime": 3,
305	"output-format": "json",
306	"test_obj": PTTest,
307	},
308	{
309	"test_id": 7,
310	"rw": 'write',
311	"io_size": 1024*1024,
312	"verify": "crc32c",
313	"output-format": "json",
314	"test_obj": PTTest,
315	},
316	{
317	"test_id": 8,
318	"rw": 'randwrite',
319	"io_size": 1024*1024,
320	"verify": "crc32c",
321	"output-format": "json",
322	"test_obj": PTTest,
323	},
324	{
325	"test_id": 9,
326	"rw": 'readwrite',
327	"timebased": 1,
328	"runtime": 3,
329	"output-format": "json",
330	"test_obj": PTTest,
331	},
332	{
333	"test_id": 10,
334	"rw": 'randrw',
335	"timebased": 1,
336	"runtime": 3,
337	"output-format": "json",
338	"test_obj": PTTest,
339	},
340	{
341	"test_id": 11,
342	"rw": 'trimwrite',
343	"timebased": 1,
344	"runtime": 3,
345	"output-format": "json",
346	"test_obj": PTTest,
347	},
348	{
349	"test_id": 12,
350	"rw": 'randtrimwrite',
351	"timebased": 1,
352	"runtime": 3,
353	"output-format": "json",
354	"test_obj": PTTest,
355	},
356	{
357	"test_id": 13,
358	"rw": 'randread',
359	"timebased": 1,
360	"runtime": 3,
361	"fixedbufs": 1,
362	"nonvectored": 1,
363	"force_async": 1,
364	"registerfiles": 1,
365	"sqthread_poll": 1,
366	"output-format": "json",
367	"test_obj": PTTest,
368	},
369	{
370	"test_id": 14,
371	"rw": 'randwrite',
372	"timebased": 1,
373	"runtime": 3,
374	"fixedbufs": 1,
375	"nonvectored": 1,
376	"force_async": 1,
377	"registerfiles": 1,
378	"sqthread_poll": 1,
379	"output-format": "json",
380	"test_obj": PTTest,
381	},
382	]
383
384	passed = 0
385	failed = 0
386	skipped = 0
387
388	for test in test_list:
389	if (args.skip and test['test_id'] in args.skip) or \
390	(args.run_only and test['test_id'] not in args.run_only):
391	skipped = skipped + 1
392	outcome = 'SKIPPED (User request)'
393	else:
394	test['filename'] = args.dut
395	test_obj = test['test_obj'](artifact_root, test, args.debug)
396	status = test_obj.run_fio(fio)
397	if status:
398	status = test_obj.check()
399	if status:
400	passed = passed + 1
401	outcome = 'PASSED'
402	else:
403	failed = failed + 1
404	outcome = 'FAILED'
405
406	print(f"********Test {test['test_id']} {outcome}********")
407
408	print(f"{passed} tests passed, {failed} failed, {skipped} skipped")
409
410	sys.exit(failed)
411
412
413	if __name__ == '__main__':
414	main()