[fio.git] / t / random_seed.py

#!/usr/bin/env python3
"""
# random_seed.py
#
# Test fio's random seed options.
#
# - make sure that randseed overrides randrepeat and allrandrepeat
# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
#
# USAGE
# see python3 random_seed.py --help
#
# EXAMPLES
# python3 t/random_seed.py
# python3 t/random_seed.py -f ./fio
#
# REQUIREMENTS
# Python 3.6
#
"""
import os
import sys
import time
import locale
import argparse
import subprocess
from pathlib import Path

class FioRandTest():
    """fio random seed test."""

    def __init__(self, artifact_root, test_options, debug):
        """
        artifact_root   root directory for artifacts (subdirectory will be created under here)
        test            test specification
        """
        self.artifact_root = artifact_root
        self.test_options = test_options
        self.debug = debug
        self.filename_stub = None
        self.filenames = {}

        self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
                                     f"{self.test_options['test_id']:03d}"))
        if not os.path.exists(self.test_dir):
            os.mkdir(self.test_dir)

        self.filename_stub = f"random{self.test_options['test_id']:03d}"
        self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
        self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
        self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
        self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
        self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")

    def run_fio(self, fio_path):
        """Run a test."""

        fio_args = [
            "--debug=random",
            "--name=random_seed",
            "--ioengine=null",
            "--filesize=32k",
            "--rw=randread",
            f"--output={self.filenames['output']}",
        ]
        for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
            if opt in self.test_options:
                option = f"--{opt}={self.test_options[opt]}"
                fio_args.append(option)

        command = [fio_path] + fio_args
        with open(self.filenames['command'], "w+", encoding=locale.getpreferredencoding()) as command_file:
            command_file.write(" ".join(command))

        passed = True

        try:
            with open(self.filenames['stdout'], "w+", encoding=locale.getpreferredencoding()) as stdout_file, \
                open(self.filenames['stderr'], "w+", encoding=locale.getpreferredencoding()) as stderr_file, \
                open(self.filenames['exitcode'], "w+", encoding=locale.getpreferredencoding()) as exitcode_file:
                proc = None
                # Avoid using subprocess.run() here because when a timeout occurs,
                # fio will be stopped with SIGKILL. This does not give fio a
                # chance to clean up and means that child processes may continue
                # running and submitting IO.
                proc = subprocess.Popen(command,
                                        stdout=stdout_file,
                                        stderr=stderr_file,
                                        cwd=self.test_dir,
                                        universal_newlines=True)
                proc.communicate(timeout=300)
                exitcode_file.write(f'{proc.returncode}\n')
                passed &= (proc.returncode == 0)
        except subprocess.TimeoutExpired:
            proc.terminate()
            proc.communicate()
            assert proc.poll()
            print("Timeout expired")
            passed = False
        except Exception:
            if proc:
                if not proc.poll():
                    proc.terminate()
                    proc.communicate()
            print(f"Exception: {sys.exc_info()}")
            passed = False

        return passed

    def get_rand_seeds(self):
        """Collect random seeds from --debug=random output."""
        with open(self.filenames['output'], "r", encoding=locale.getpreferredencoding()) as out_file:
            file_data = out_file.read()

            offsets = 0
            for line in file_data.split('\n'):
                if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
                    tokens = line.split('=')
                    offsets = int(tokens[len(tokens)-1])
                    break

            if offsets == 0:
                pass
                # find an exception to throw

            seed_list = []
            for line in file_data.split('\n'):
                if 'random' not in line:
                    continue
                if 'rand_seeds[' in line:
                    tokens = line.split('=')
                    seed = int(tokens[-1])
                    seed_list.append(seed)
                    # assume that seeds are in order

            return seed_list

    def check(self):
        """Check test output."""

        raise NotImplementedError()


class TestRR(FioRandTest):
    """
    Test object for [all]randrepeat. If run for the first time just collect the
    seeds. For later runs make sure the seeds match or do not match those
    previously collected.
    """
    # one set of seeds is for randrepeat=0 and the other is for randrepeat=1
    seeds = { 0: None, 1: None }

    def check(self):
        """Check output for allrandrepeat=1."""

        retval = True
        opt = 'randrepeat' if 'randrepeat' in self.test_options else 'allrandrepeat'
        rr = self.test_options[opt]
        rand_seeds = self.get_rand_seeds()

        if not TestRR.seeds[rr]:
            TestRR.seeds[rr] = rand_seeds
            if self.debug:
                print(f"TestRR: saving rand_seeds for [a]rr={rr}")
        else:
            if rr:
                if TestRR.seeds[1] != rand_seeds:
                    retval = False
                    print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
                else:
                    if self.debug:
                        print(f"TestRR: seeds correctly match for [a]rr={rr}")
                if TestRR.seeds[0] == rand_seeds:
                    retval = False
                    print("TestRR: seeds unexpectedly match those from system RNG")
            else:
                if TestRR.seeds[0] == rand_seeds:
                    retval = False
                    print(f"TestRR: unexpected seed match for [a]rr={rr}")
                else:
                    if self.debug:
                        print(f"TestRR: seeds correctly don't match for [a]rr={rr}")
                if TestRR.seeds[1] == rand_seeds:
                    retval = False
                    print(f"TestRR: random seeds unexpectedly match those from [a]rr=1")

        return retval


class TestRS(FioRandTest):
    """
    Test object when randseed=something controls the generated seeds. If run
    for the first time for a given randseed just collect the seeds. For later
    runs with the same seed make sure the seeds are the same as those
    previously collected.
    """
    seeds = {}

    def check(self):
        """Check output for randseed=something."""

        retval = True
        rand_seeds = self.get_rand_seeds()
        randseed = self.test_options['randseed']

        if self.debug:
            print("randseed = ", randseed)

        if randseed not in TestRS.seeds:
            TestRS.seeds[randseed] = rand_seeds
            if self.debug:
                print("TestRS: saving rand_seeds")
        else:
            if TestRS.seeds[randseed] != rand_seeds:
                retval = False
                print("TestRS: seeds don't match when they should")
            else:
                if self.debug:
                    print("TestRS: seeds correctly match")

        # Now try to find seeds generated using a different randseed and make
        # sure they *don't* match
        for key in TestRS.seeds:
            if key != randseed:
                if TestRS.seeds[key] == rand_seeds:
                    retval = False
                    print("TestRS: randseeds differ but generated seeds match.")
                else:
                    if self.debug:
                        print("TestRS: randseeds differ and generated seeds also differ.")

        return retval


def parse_args():
    """Parse command-line arguments."""

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
    parser.add_argument('-s', '--skip', nargs='+', type=int,
                        help='list of test(s) to skip')
    parser.add_argument('-o', '--run-only', nargs='+', type=int,
                        help='list of test(s) to run, skipping all others')
    args = parser.parse_args()

    return args


def main():
    """Run tests of fio random seed options"""

    args = parse_args()

    artifact_root = args.artifact_root if args.artifact_root else \
        f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
    os.mkdir(artifact_root)
    print(f"Artifact directory is {artifact_root}")

    if args.fio:
        fio = str(Path(args.fio).absolute())
    else:
        fio = 'fio'
    print(f"fio path is {fio}")

    test_list = [
        {
            "test_id": 1,
            "randrepeat": 0,
            "test_obj": TestRR,
        },
        {
            "test_id": 2,
            "randrepeat": 0,
            "test_obj": TestRR,
        },
        {
            "test_id": 3,
            "randrepeat": 1,
            "test_obj": TestRR,
        },
        {
            "test_id": 4,
            "randrepeat": 1,
            "test_obj": TestRR,
        },
        {
            "test_id": 5,
            "allrandrepeat": 0,
            "test_obj": TestRR,
        },
        {
            "test_id": 6,
            "allrandrepeat": 0,
            "test_obj": TestRR,
        },
        {
            "test_id": 7,
            "allrandrepeat": 1,
            "test_obj": TestRR,
        },
        {
            "test_id": 8,
            "allrandrepeat": 1,
            "test_obj": TestRR,
        },
        {
            "test_id": 9,
            "randrepeat": 0,
            "randseed": "12345",
            "test_obj": TestRS,
        },
        {
            "test_id": 10,
            "randrepeat": 0,
            "randseed": "12345",
            "test_obj": TestRS,
        },
        {
            "test_id": 11,
            "randrepeat": 1,
            "randseed": "12345",
            "test_obj": TestRS,
        },
        {
            "test_id": 12,
            "allrandrepeat": 0,
            "randseed": "12345",
            "test_obj": TestRS,
        },
        {
            "test_id": 13,
            "allrandrepeat": 1,
            "randseed": "12345",
            "test_obj": TestRS,
        },
        {
            "test_id": 14,
            "randrepeat": 0,
            "randseed": "67890",
            "test_obj": TestRS,
        },
        {
            "test_id": 15,
            "randrepeat": 1,
            "randseed": "67890",
            "test_obj": TestRS,
        },
        {
            "test_id": 16,
            "allrandrepeat": 0,
            "randseed": "67890",
            "test_obj": TestRS,
        },
        {
            "test_id": 17,
            "allrandrepeat": 1,
            "randseed": "67890",
            "test_obj": TestRS,
        },
    ]

    passed = 0
    failed = 0
    skipped = 0

    for test in test_list:
        if (args.skip and test['test_id'] in args.skip) or \
           (args.run_only and test['test_id'] not in args.run_only):
            skipped = skipped + 1
            outcome = 'SKIPPED (User request)'
        else:
            test_obj = test['test_obj'](artifact_root, test, args.debug)
            status = test_obj.run_fio(fio)
            if status:
                status = test_obj.check()
            if status:
                passed = passed + 1
                outcome = 'PASSED'
            else:
                failed = failed + 1
                outcome = 'FAILED'

        print(f"**********Test {test['test_id']} {outcome}**********")

    print(f"{passed} tests passed, {failed} failed, {skipped} skipped")

    sys.exit(failed)


if __name__ == '__main__':
    main()
Commit	Line	Data
c994fa62 VF	1	#!/usr/bin/env python3
	2	"""
	3	# random_seed.py
	4	#
	5	# Test fio's random seed options.
	6	#
	7	# - make sure that randseed overrides randrepeat and allrandrepeat
	8	# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
	9	# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
	10	#
	11	# USAGE
	12	# see python3 random_seed.py --help
	13	#
	14	# EXAMPLES
	15	# python3 t/random_seed.py
	16	# python3 t/random_seed.py -f ./fio
	17	#
	18	# REQUIREMENTS
	19	# Python 3.6
	20	#
	21	"""
	22	import os
	23	import sys
	24	import time
	25	import locale
	26	import argparse
	27	import subprocess
	28	from pathlib import Path
	29
	30	class FioRandTest():
	31	"""fio random seed test."""
	32
	33	def __init__(self, artifact_root, test_options, debug):
	34	"""
	35	artifact_root root directory for artifacts (subdirectory will be created under here)
	36	test test specification
	37	"""
	38	self.artifact_root = artifact_root
	39	self.test_options = test_options
	40	self.debug = debug
	41	self.filename_stub = None
	42	self.filenames = {}
	43
	44	self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
	45	f"{self.test_options['test_id']:03d}"))
	46	if not os.path.exists(self.test_dir):
	47	os.mkdir(self.test_dir)
	48
	49	self.filename_stub = f"random{self.test_options['test_id']:03d}"
	50	self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
	51	self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
	52	self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
	53	self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
	54	self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")
	55
	56	def run_fio(self, fio_path):
	57	"""Run a test."""
	58
	59	fio_args = [
	60	"--debug=random",
	61	"--name=random_seed",
	62	"--ioengine=null",
	63	"--filesize=32k",
	64	"--rw=randread",
65	f"--output={self.filenames['output']}",
66	]
67	for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
68	if opt in self.test_options:
69	option = f"--{opt}={self.test_options[opt]}"
70	fio_args.append(option)
71
72	command = [fio_path] + fio_args
73	with open(self.filenames['command'], "w+", encoding=locale.getpreferredencoding()) as command_file:
74	command_file.write(" ".join(command))
75
76	passed = True
77
78	try:
79	with open(self.filenames['stdout'], "w+", encoding=locale.getpreferredencoding()) as stdout_file, \
80	open(self.filenames['stderr'], "w+", encoding=locale.getpreferredencoding()) as stderr_file, \
81	open(self.filenames['exitcode'], "w+", encoding=locale.getpreferredencoding()) as exitcode_file:
82	proc = None
83	# Avoid using subprocess.run() here because when a timeout occurs,
84	# fio will be stopped with SIGKILL. This does not give fio a
85	# chance to clean up and means that child processes may continue
86	# running and submitting IO.
87	proc = subprocess.Popen(command,
88	stdout=stdout_file,
89	stderr=stderr_file,
90	cwd=self.test_dir,
91	universal_newlines=True)
92	proc.communicate(timeout=300)
93	exitcode_file.write(f'{proc.returncode}\n')
94	passed &= (proc.returncode == 0)
95	except subprocess.TimeoutExpired:
96	proc.terminate()
97	proc.communicate()
98	assert proc.poll()
99	print("Timeout expired")
100	passed = False
101	except Exception:
102	if proc:
103	if not proc.poll():
104	proc.terminate()
105	proc.communicate()
106	print(f"Exception: {sys.exc_info()}")
107	passed = False
108
109	return passed
110
111	def get_rand_seeds(self):
112	"""Collect random seeds from --debug=random output."""
113	with open(self.filenames['output'], "r", encoding=locale.getpreferredencoding()) as out_file:
114	file_data = out_file.read()
115
116	offsets = 0
117	for line in file_data.split('\n'):
118	if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
119	tokens = line.split('=')
120	offsets = int(tokens[len(tokens)-1])
121	break
122
123	if offsets == 0:
124	pass
125	# find an exception to throw
126
127	seed_list = []
128	for line in file_data.split('\n'):
129	if 'random' not in line:
130	continue
131	if 'rand_seeds[' in line:
132	tokens = line.split('=')
133	seed = int(tokens[-1])
134	seed_list.append(seed)
135	# assume that seeds are in order
136
137	return seed_list
138
139	def check(self):
140	"""Check test output."""
141
142	raise NotImplementedError()
143
144
145	class TestRR(FioRandTest):
146	"""
147	Test object for [all]randrepeat. If run for the first time just collect the
148	seeds. For later runs make sure the seeds match or do not match those
149	previously collected.
150	"""
151	# one set of seeds is for randrepeat=0 and the other is for randrepeat=1
152	seeds = { 0: None, 1: None }
153
154	def check(self):
155	"""Check output for allrandrepeat=1."""
156
157	retval = True
158	opt = 'randrepeat' if 'randrepeat' in self.test_options else 'allrandrepeat'
159	rr = self.test_options[opt]
160	rand_seeds = self.get_rand_seeds()
161
162	if not TestRR.seeds[rr]:
163	TestRR.seeds[rr] = rand_seeds
164	if self.debug:
165	print(f"TestRR: saving rand_seeds for [a]rr={rr}")
166	else:
167	if rr:
168	if TestRR.seeds[1] != rand_seeds:
169	retval = False
170	print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
171	else:
172	if self.debug:
173	print(f"TestRR: seeds correctly match for [a]rr={rr}")
174	if TestRR.seeds[0] == rand_seeds:
175	retval = False
176	print("TestRR: seeds unexpectedly match those from system RNG")
177	else:
178	if TestRR.seeds[0] == rand_seeds:
179	retval = False
180	print(f"TestRR: unexpected seed match for [a]rr={rr}")
181	else:
182	if self.debug:
183	print(f"TestRR: seeds correctly don't match for [a]rr={rr}")
184	if TestRR.seeds[1] == rand_seeds:
185	retval = False
186	print(f"TestRR: random seeds unexpectedly match those from [a]rr=1")
187
188	return retval
189
190
191	class TestRS(FioRandTest):
192	"""
193	Test object when randseed=something controls the generated seeds. If run
194	for the first time for a given randseed just collect the seeds. For later
195	runs with the same seed make sure the seeds are the same as those
196	previously collected.
197	"""
198	seeds = {}
199
200	def check(self):
201	"""Check output for randseed=something."""
202
203	retval = True
204	rand_seeds = self.get_rand_seeds()
205	randseed = self.test_options['randseed']
206
207	if self.debug:
208	print("randseed = ", randseed)
209
210	if randseed not in TestRS.seeds:
211	TestRS.seeds[randseed] = rand_seeds
212	if self.debug:
213	print("TestRS: saving rand_seeds")
214	else:
215	if TestRS.seeds[randseed] != rand_seeds:
216	retval = False
217	print("TestRS: seeds don't match when they should")
218	else:
219	if self.debug:
220	print("TestRS: seeds correctly match")
221
222	# Now try to find seeds generated using a different randseed and make
223	# sure they don't match
224	for key in TestRS.seeds:
225	if key != randseed:
226	if TestRS.seeds[key] == rand_seeds:
227	retval = False
228	print("TestRS: randseeds differ but generated seeds match.")
229	else:
230	if self.debug:
231	print("TestRS: randseeds differ and generated seeds also differ.")
232
233	return retval
234
235
236	def parse_args():
237	"""Parse command-line arguments."""
238
239	parser = argparse.ArgumentParser()
240	parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
241	parser.add_argument('-a', '--artifact-root', help='artifact root directory')
242	parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
243	parser.add_argument('-s', '--skip', nargs='+', type=int,
244	help='list of test(s) to skip')
245	parser.add_argument('-o', '--run-only', nargs='+', type=int,
246	help='list of test(s) to run, skipping all others')
247	args = parser.parse_args()
248
249	return args
250
251
252	def main():
253	"""Run tests of fio random seed options"""
254
255	args = parse_args()
256
257	artifact_root = args.artifact_root if args.artifact_root else \
258	f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
259	os.mkdir(artifact_root)
260	print(f"Artifact directory is {artifact_root}")
261
262	if args.fio:
263	fio = str(Path(args.fio).absolute())
264	else:
265	fio = 'fio'
266	print(f"fio path is {fio}")
267
268	test_list = [
269	{
270	"test_id": 1,
271	"randrepeat": 0,
272	"test_obj": TestRR,
273	},
274	{
275	"test_id": 2,
276	"randrepeat": 0,
277	"test_obj": TestRR,
278	},
279	{
280	"test_id": 3,
281	"randrepeat": 1,
282	"test_obj": TestRR,
283	},
284	{
285	"test_id": 4,
286	"randrepeat": 1,
287	"test_obj": TestRR,
288	},
289	{
290	"test_id": 5,
291	"allrandrepeat": 0,
292	"test_obj": TestRR,
293	},
294	{
295	"test_id": 6,
296	"allrandrepeat": 0,
297	"test_obj": TestRR,
298	},
299	{
300	"test_id": 7,
301	"allrandrepeat": 1,
302	"test_obj": TestRR,
303	},
304	{
305	"test_id": 8,
306	"allrandrepeat": 1,
307	"test_obj": TestRR,
308	},
309	{
310	"test_id": 9,
311	"randrepeat": 0,
312	"randseed": "12345",
313	"test_obj": TestRS,
314	},
315	{
316	"test_id": 10,
317	"randrepeat": 0,
318	"randseed": "12345",
319	"test_obj": TestRS,
320	},
321	{
322	"test_id": 11,
323	"randrepeat": 1,
324	"randseed": "12345",
325	"test_obj": TestRS,
326	},
327	{
328	"test_id": 12,
329	"allrandrepeat": 0,
330	"randseed": "12345",
331	"test_obj": TestRS,
332	},
333	{
334	"test_id": 13,
335	"allrandrepeat": 1,
336	"randseed": "12345",
337	"test_obj": TestRS,
338	},
339	{
340	"test_id": 14,
341	"randrepeat": 0,
342	"randseed": "67890",
343	"test_obj": TestRS,
344	},
345	{
346	"test_id": 15,
347	"randrepeat": 1,
348	"randseed": "67890",
349	"test_obj": TestRS,
350	},
351	{
352	"test_id": 16,
353	"allrandrepeat": 0,
354	"randseed": "67890",
355	"test_obj": TestRS,
356	},
357	{
358	"test_id": 17,
359	"allrandrepeat": 1,
360	"randseed": "67890",
361	"test_obj": TestRS,
362	},
363	]
364
365	passed = 0
366	failed = 0
367	skipped = 0
368
369	for test in test_list:
370	if (args.skip and test['test_id'] in args.skip) or \
371	(args.run_only and test['test_id'] not in args.run_only):
372	skipped = skipped + 1
373	outcome = 'SKIPPED (User request)'
374	else:
375	test_obj = test['test_obj'](artifact_root, test, args.debug)
376	status = test_obj.run_fio(fio)
377	if status:
378	status = test_obj.check()
379	if status:
380	passed = passed + 1
381	outcome = 'PASSED'
382	else:
383	failed = failed + 1
384	outcome = 'FAILED'
385
386	print(f"********Test {test['test_id']} {outcome}********")
387
388	print(f"{passed} tests passed, {failed} failed, {skipped} skipped")
389
390	sys.exit(failed)
391
392
393	if __name__ == '__main__':
394	main()