[fio.git] / t / random_seed.py

#!/usr/bin/env python3
"""
# random_seed.py
#
# Test fio's random seed options.
#
# - make sure that randseed overrides randrepeat and allrandrepeat
# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
#
# USAGE
# see python3 random_seed.py --help
#
# EXAMPLES
# python3 t/random_seed.py
# python3 t/random_seed.py -f ./fio
#
# REQUIREMENTS
# Python 3.6
#
"""
import os
import sys
import time
import locale
import logging
import argparse
from pathlib import Path
from fiotestlib import FioJobCmdTest, run_fio_tests

class FioRandTest(FioJobCmdTest):
    """fio random seed test."""

    def setup(self, parameters):
        """Setup the test."""

        fio_args = [
            "--debug=random",
            "--name=random_seed",
            "--ioengine=null",
            "--filesize=32k",
            "--rw=randread",
            f"--output={self.filenames['output']}",
        ]
        for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
            if opt in self.fio_opts:
                option = f"--{opt}={self.fio_opts[opt]}"
                fio_args.append(option)

        super().setup(fio_args)

    def get_rand_seeds(self):
        """Collect random seeds from --debug=random output."""
        with open(self.filenames['output'], "r",
                  encoding=locale.getpreferredencoding()) as out_file:
            file_data = out_file.read()

            offsets = 0
            for line in file_data.split('\n'):
                if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
                    tokens = line.split('=')
                    offsets = int(tokens[len(tokens)-1])
                    break

            if offsets == 0:
                pass
                # find an exception to throw

            seed_list = []
            for line in file_data.split('\n'):
                if 'random' not in line:
                    continue
                if 'rand_seeds[' in line:
                    tokens = line.split('=')
                    seed = int(tokens[-1])
                    seed_list.append(seed)
                    # assume that seeds are in order

            return seed_list


class TestRR(FioRandTest):
    """
    Test object for [all]randrepeat. If run for the first time just collect the
    seeds. For later runs make sure the seeds match or do not match those
    previously collected.
    """
    # one set of seeds is for randrepeat=0 and the other is for randrepeat=1
    seeds = { 0: None, 1: None }

    def check_result(self):
        """Check output for allrandrepeat=1."""

        super().check_result()
        if not self.passed:
            return

        opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
        rr = self.fio_opts[opt]
        rand_seeds = self.get_rand_seeds()

        if not TestRR.seeds[rr]:
            TestRR.seeds[rr] = rand_seeds
            logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
        else:
            if rr:
                if TestRR.seeds[1] != rand_seeds:
                    self.passed = False
                    print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
                else:
                    logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
                if TestRR.seeds[0] == rand_seeds:
                    self.passed = False
                    print("TestRR: seeds unexpectedly match those from system RNG")
            else:
                if TestRR.seeds[0] == rand_seeds:
                    self.passed = False
                    print(f"TestRR: unexpected seed match for [a]rr={rr}")
                else:
                    logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
                if TestRR.seeds[1] == rand_seeds:
                    self.passed = False
                    print("TestRR: random seeds unexpectedly match those from [a]rr=1")


class TestRS(FioRandTest):
    """
    Test object when randseed=something controls the generated seeds. If run
    for the first time for a given randseed just collect the seeds. For later
    runs with the same seed make sure the seeds are the same as those
    previously collected.
    """
    seeds = {}

    def check_result(self):
        """Check output for randseed=something."""

        super().check_result()
        if not self.passed:
            return

        rand_seeds = self.get_rand_seeds()
        randseed = self.fio_opts['randseed']

        logging.debug("randseed = %s", randseed)

        if randseed not in TestRS.seeds:
            TestRS.seeds[randseed] = rand_seeds
            logging.debug("TestRS: saving rand_seeds")
        else:
            if TestRS.seeds[randseed] != rand_seeds:
                self.passed = False
                print("TestRS: seeds don't match when they should")
            else:
                logging.debug("TestRS: seeds correctly match")

        # Now try to find seeds generated using a different randseed and make
        # sure they *don't* match
        for key, value in TestRS.seeds.items():
            if key != randseed:
                if value == rand_seeds:
                    self.passed = False
                    print("TestRS: randseeds differ but generated seeds match.")
                else:
                    logging.debug("TestRS: randseeds differ and generated seeds also differ.")


def parse_args():
    """Parse command-line arguments."""

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
    parser.add_argument('-s', '--skip', nargs='+', type=int,
                        help='list of test(s) to skip')
    parser.add_argument('-o', '--run-only', nargs='+', type=int,
                        help='list of test(s) to run, skipping all others')
    args = parser.parse_args()

    return args


def main():
    """Run tests of fio random seed options"""

    args = parse_args()

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    artifact_root = args.artifact_root if args.artifact_root else \
        f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
    os.mkdir(artifact_root)
    print(f"Artifact directory is {artifact_root}")

    if args.fio:
        fio_path = str(Path(args.fio).absolute())
    else:
        fio_path = 'fio'
    print(f"fio path is {fio_path}")

    test_list = [
        {
            "test_id": 1,
            "fio_opts": {
                "randrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 2,
            "fio_opts": {
                "randrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 3,
            "fio_opts": {
                "randrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 4,
            "fio_opts": {
                "randrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 5,
            "fio_opts": {
                "allrandrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 6,
            "fio_opts": {
                "allrandrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 7,
            "fio_opts": {
                "allrandrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 8,
            "fio_opts": {
                "allrandrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 9,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 10,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 11,
            "fio_opts": {
                "randrepeat": 1,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 12,
            "fio_opts": {
                "allrandrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 13,
            "fio_opts": {
                "allrandrepeat": 1,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 14,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 15,
            "fio_opts": {
                "randrepeat": 1,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 16,
            "fio_opts": {
                "allrandrepeat": 0,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 17,
            "fio_opts": {
                "allrandrepeat": 1,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
    ]

    test_env = {
              'fio_path': fio_path,
              'fio_root': str(Path(__file__).absolute().parent.parent),
              'artifact_root': artifact_root,
              'basename': 'random',
              }

    _, failed, _ = run_fio_tests(test_list, test_env, args)
    sys.exit(failed)


if __name__ == '__main__':
    main()
Commit	Line	Data
c994fa62 VF	1	#!/usr/bin/env python3
	2	"""
	3	# random_seed.py
	4	#
	5	# Test fio's random seed options.
	6	#
	7	# - make sure that randseed overrides randrepeat and allrandrepeat
	8	# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
	9	# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
	10	#
	11	# USAGE
	12	# see python3 random_seed.py --help
	13	#
	14	# EXAMPLES
	15	# python3 t/random_seed.py
	16	# python3 t/random_seed.py -f ./fio
	17	#
	18	# REQUIREMENTS
	19	# Python 3.6
	20	#
	21	"""
	22	import os
	23	import sys
	24	import time
	25	import locale
86131edc	26	import logging
c994fa62	27	import argparse
c994fa62	28	from pathlib import Path
1017a7d3	29	from fiotestlib import FioJobCmdTest, run_fio_tests
c994fa62	30
1017a7d3	31	class FioRandTest(FioJobCmdTest):
c994fa62 VF	32	"""fio random seed test."""
c994fa62 VF	33
1017a7d3 VF	34	def setup(self, parameters):
1017a7d3 VF	35	"""Setup the test."""
c994fa62 VF	36
	37	fio_args = [
	38	"--debug=random",
	39	"--name=random_seed",
	40	"--ioengine=null",
	41	"--filesize=32k",
	42	"--rw=randread",
	43	f"--output={self.filenames['output']}",
	44	]
	45	for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
1017a7d3 VF	46	if opt in self.fio_opts:
1017a7d3 VF	47	option = f"--{opt}={self.fio_opts[opt]}"
c994fa62 VF	48	fio_args.append(option)
c994fa62 VF	49
1017a7d3	50	super().setup(fio_args)
c994fa62 VF	51
	52	def get_rand_seeds(self):
	53	"""Collect random seeds from --debug=random output."""
cab6c7d1 VF	54	with open(self.filenames['output'], "r",
cab6c7d1 VF	55	encoding=locale.getpreferredencoding()) as out_file:
c994fa62 VF	56	file_data = out_file.read()
	57
	58	offsets = 0
	59	for line in file_data.split('\n'):
	60	if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
	61	tokens = line.split('=')
	62	offsets = int(tokens[len(tokens)-1])
	63	break
	64
	65	if offsets == 0:
	66	pass
	67	# find an exception to throw
	68
	69	seed_list = []
	70	for line in file_data.split('\n'):
	71	if 'random' not in line:
	72	continue
	73	if 'rand_seeds[' in line:
	74	tokens = line.split('=')
	75	seed = int(tokens[-1])
	76	seed_list.append(seed)
	77	# assume that seeds are in order
	78
	79	return seed_list
	80
c994fa62 VF	81
	82	class TestRR(FioRandTest):
	83	"""
	84	Test object for [all]randrepeat. If run for the first time just collect the
	85	seeds. For later runs make sure the seeds match or do not match those
	86	previously collected.
	87	"""
	88	# one set of seeds is for randrepeat=0 and the other is for randrepeat=1
	89	seeds = { 0: None, 1: None }
	90
1017a7d3	91	def check_result(self):
c994fa62 VF	92	"""Check output for allrandrepeat=1."""
c994fa62 VF	93
6f2b92cf VF	94	super().check_result()
	95	if not self.passed:
	96	return
	97
1017a7d3 VF	98	opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
1017a7d3 VF	99	rr = self.fio_opts[opt]
c994fa62 VF	100	rand_seeds = self.get_rand_seeds()
	101
	102	if not TestRR.seeds[rr]:
	103	TestRR.seeds[rr] = rand_seeds
cab6c7d1	104	logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
c994fa62 VF	105	else:
	106	if rr:
	107	if TestRR.seeds[1] != rand_seeds:
1017a7d3	108	self.passed = False
c994fa62 VF	109	print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
c994fa62 VF	110	else:
cab6c7d1	111	logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
c994fa62	112	if TestRR.seeds[0] == rand_seeds:
1017a7d3	113	self.passed = False
c994fa62 VF	114	print("TestRR: seeds unexpectedly match those from system RNG")
	115	else:
	116	if TestRR.seeds[0] == rand_seeds:
1017a7d3	117	self.passed = False
c994fa62 VF	118	print(f"TestRR: unexpected seed match for [a]rr={rr}")
c994fa62 VF	119	else:
cab6c7d1	120	logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
c994fa62	121	if TestRR.seeds[1] == rand_seeds:
1017a7d3	122	self.passed = False
cab6c7d1	123	print("TestRR: random seeds unexpectedly match those from [a]rr=1")
c994fa62	124
c994fa62 VF	125
	126	class TestRS(FioRandTest):
	127	"""
	128	Test object when randseed=something controls the generated seeds. If run
	129	for the first time for a given randseed just collect the seeds. For later
	130	runs with the same seed make sure the seeds are the same as those
	131	previously collected.
	132	"""
	133	seeds = {}
	134
1017a7d3	135	def check_result(self):
c994fa62 VF	136	"""Check output for randseed=something."""
c994fa62 VF	137
6f2b92cf VF	138	super().check_result()
	139	if not self.passed:
	140	return
	141
c994fa62	142	rand_seeds = self.get_rand_seeds()
1017a7d3	143	randseed = self.fio_opts['randseed']
c994fa62	144
cab6c7d1	145	logging.debug("randseed = %s", randseed)
c994fa62 VF	146
	147	if randseed not in TestRS.seeds:
	148	TestRS.seeds[randseed] = rand_seeds
86131edc	149	logging.debug("TestRS: saving rand_seeds")
c994fa62 VF	150	else:
c994fa62 VF	151	if TestRS.seeds[randseed] != rand_seeds:
1017a7d3	152	self.passed = False
c994fa62 VF	153	print("TestRS: seeds don't match when they should")
c994fa62 VF	154	else:
86131edc	155	logging.debug("TestRS: seeds correctly match")
c994fa62 VF	156
	157	# Now try to find seeds generated using a different randseed and make
	158	# sure they don't match
cab6c7d1	159	for key, value in TestRS.seeds.items():
c994fa62	160	if key != randseed:
cab6c7d1	161	if value == rand_seeds:
1017a7d3	162	self.passed = False
c994fa62 VF	163	print("TestRS: randseeds differ but generated seeds match.")
c994fa62 VF	164	else:
86131edc	165	logging.debug("TestRS: randseeds differ and generated seeds also differ.")
c994fa62 VF	166
	167
	168	def parse_args():
	169	"""Parse command-line arguments."""
	170
	171	parser = argparse.ArgumentParser()
	172	parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
	173	parser.add_argument('-a', '--artifact-root', help='artifact root directory')
	174	parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
	175	parser.add_argument('-s', '--skip', nargs='+', type=int,
	176	help='list of test(s) to skip')
	177	parser.add_argument('-o', '--run-only', nargs='+', type=int,
	178	help='list of test(s) to run, skipping all others')
	179	args = parser.parse_args()
	180
	181	return args
	182
	183
	184	def main():
	185	"""Run tests of fio random seed options"""
	186
	187	args = parse_args()
	188
86131edc VF	189	if args.debug:
	190	logging.basicConfig(level=logging.DEBUG)
	191	else:
	192	logging.basicConfig(level=logging.INFO)
	193
c994fa62 VF	194	artifact_root = args.artifact_root if args.artifact_root else \
	195	f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
	196	os.mkdir(artifact_root)
	197	print(f"Artifact directory is {artifact_root}")
	198
	199	if args.fio:
1017a7d3	200	fio_path = str(Path(args.fio).absolute())
c994fa62	201	else:
1017a7d3 VF	202	fio_path = 'fio'
1017a7d3 VF	203	print(f"fio path is {fio_path}")
c994fa62 VF	204
	205	test_list = [
	206	{
	207	"test_id": 1,
1017a7d3 VF	208	"fio_opts": {
	209	"randrepeat": 0,
	210	},
	211	"test_class": TestRR,
c994fa62 VF	212	},
	213	{
	214	"test_id": 2,
1017a7d3 VF	215	"fio_opts": {
	216	"randrepeat": 0,
	217	},
	218	"test_class": TestRR,
c994fa62 VF	219	},
	220	{
	221	"test_id": 3,
1017a7d3 VF	222	"fio_opts": {
	223	"randrepeat": 1,
	224	},
	225	"test_class": TestRR,
c994fa62 VF	226	},
	227	{
	228	"test_id": 4,
1017a7d3 VF	229	"fio_opts": {
	230	"randrepeat": 1,
	231	},
	232	"test_class": TestRR,
c994fa62 VF	233	},
	234	{
	235	"test_id": 5,
1017a7d3 VF	236	"fio_opts": {
	237	"allrandrepeat": 0,
	238	},
	239	"test_class": TestRR,
c994fa62 VF	240	},
	241	{
	242	"test_id": 6,
1017a7d3 VF	243	"fio_opts": {
	244	"allrandrepeat": 0,
	245	},
	246	"test_class": TestRR,
c994fa62 VF	247	},
	248	{
	249	"test_id": 7,
1017a7d3 VF	250	"fio_opts": {
	251	"allrandrepeat": 1,
	252	},
	253	"test_class": TestRR,
c994fa62 VF	254	},
	255	{
	256	"test_id": 8,
1017a7d3 VF	257	"fio_opts": {
	258	"allrandrepeat": 1,
	259	},
	260	"test_class": TestRR,
c994fa62 VF	261	},
	262	{
	263	"test_id": 9,
1017a7d3 VF	264	"fio_opts": {
	265	"randrepeat": 0,
	266	"randseed": "12345",
	267	},
	268	"test_class": TestRS,
c994fa62 VF	269	},
	270	{
	271	"test_id": 10,
1017a7d3 VF	272	"fio_opts": {
	273	"randrepeat": 0,
	274	"randseed": "12345",
	275	},
	276	"test_class": TestRS,
c994fa62 VF	277	},
	278	{
	279	"test_id": 11,
1017a7d3 VF	280	"fio_opts": {
	281	"randrepeat": 1,
	282	"randseed": "12345",
	283	},
	284	"test_class": TestRS,
c994fa62 VF	285	},
	286	{
	287	"test_id": 12,
1017a7d3 VF	288	"fio_opts": {
	289	"allrandrepeat": 0,
	290	"randseed": "12345",
	291	},
	292	"test_class": TestRS,
c994fa62 VF	293	},
	294	{
	295	"test_id": 13,
1017a7d3 VF	296	"fio_opts": {
	297	"allrandrepeat": 1,
	298	"randseed": "12345",
	299	},
	300	"test_class": TestRS,
c994fa62 VF	301	},
	302	{
	303	"test_id": 14,
1017a7d3 VF	304	"fio_opts": {
	305	"randrepeat": 0,
	306	"randseed": "67890",
	307	},
	308	"test_class": TestRS,
c994fa62 VF	309	},
	310	{
	311	"test_id": 15,
1017a7d3 VF	312	"fio_opts": {
	313	"randrepeat": 1,
	314	"randseed": "67890",
	315	},
	316	"test_class": TestRS,
c994fa62 VF	317	},
	318	{
	319	"test_id": 16,
1017a7d3 VF	320	"fio_opts": {
	321	"allrandrepeat": 0,
	322	"randseed": "67890",
	323	},
	324	"test_class": TestRS,
c994fa62 VF	325	},
	326	{
	327	"test_id": 17,
1017a7d3 VF	328	"fio_opts": {
	329	"allrandrepeat": 1,
	330	"randseed": "67890",
	331	},
	332	"test_class": TestRS,
c994fa62 VF	333	},
	334	]
	335
1017a7d3 VF	336	test_env = {
	337	'fio_path': fio_path,
	338	'fio_root': str(Path(__file__).absolute().parent.parent),
	339	'artifact_root': artifact_root,
	340	'basename': 'random',
	341	}
c994fa62	342
1017a7d3	343	_, failed, _ = run_fio_tests(test_list, test_env, args)
c994fa62 VF	344	sys.exit(failed)
	345
	346
	347	if __name__ == '__main__':
	348	main()