[fio.git] / t / random_seed.py

#!/usr/bin/env python3
"""
# random_seed.py
#
# Test fio's random seed options.
#
# - make sure that randseed overrides randrepeat and allrandrepeat
# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
#
# USAGE
# see python3 random_seed.py --help
#
# EXAMPLES
# python3 t/random_seed.py
# python3 t/random_seed.py -f ./fio
#
# REQUIREMENTS
# Python 3.6
#
"""
import os
import sys
import time
import locale
import logging
import argparse
from pathlib import Path
from fiotestlib import FioJobCmdTest, run_fio_tests

class FioRandTest(FioJobCmdTest):
    """fio random seed test."""

    def setup(self, parameters):
        """Setup the test."""

        fio_args = [
            "--debug=random",
            "--name=random_seed",
            "--ioengine=null",
            "--filesize=32k",
            "--rw=randread",
            f"--output={self.filenames['output']}",
        ]
        for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
            if opt in self.fio_opts:
                option = f"--{opt}={self.fio_opts[opt]}"
                fio_args.append(option)

        super().setup(fio_args)

    def get_rand_seeds(self):
        """Collect random seeds from --debug=random output."""
        with open(self.filenames['output'], "r",
                  encoding=locale.getpreferredencoding()) as out_file:
            file_data = out_file.read()

            offsets = 0
            for line in file_data.split('\n'):
                if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
                    tokens = line.split('=')
                    offsets = int(tokens[len(tokens)-1])
                    break

            if offsets == 0:
                pass
                # find an exception to throw

            seed_list = []
            for line in file_data.split('\n'):
                if 'random' not in line:
                    continue
                if 'rand_seeds[' in line:
                    tokens = line.split('=')
                    seed = int(tokens[-1])
                    seed_list.append(seed)
                    # assume that seeds are in order

            return seed_list


class TestRR(FioRandTest):
    """
    Test object for [all]randrepeat. If run for the first time just collect the
    seeds. For later runs make sure the seeds match or do not match those
    previously collected.
    """
    # one set of seeds is for randrepeat=0 and the other is for randrepeat=1
    seeds = { 0: None, 1: None }

    def check_result(self):
        """Check output for allrandrepeat=1."""

        opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
        rr = self.fio_opts[opt]
        rand_seeds = self.get_rand_seeds()

        if not TestRR.seeds[rr]:
            TestRR.seeds[rr] = rand_seeds
            logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
        else:
            if rr:
                if TestRR.seeds[1] != rand_seeds:
                    self.passed = False
                    print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
                else:
                    logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
                if TestRR.seeds[0] == rand_seeds:
                    self.passed = False
                    print("TestRR: seeds unexpectedly match those from system RNG")
            else:
                if TestRR.seeds[0] == rand_seeds:
                    self.passed = False
                    print(f"TestRR: unexpected seed match for [a]rr={rr}")
                else:
                    logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
                if TestRR.seeds[1] == rand_seeds:
                    self.passed = False
                    print("TestRR: random seeds unexpectedly match those from [a]rr=1")


class TestRS(FioRandTest):
    """
    Test object when randseed=something controls the generated seeds. If run
    for the first time for a given randseed just collect the seeds. For later
    runs with the same seed make sure the seeds are the same as those
    previously collected.
    """
    seeds = {}

    def check_result(self):
        """Check output for randseed=something."""

        rand_seeds = self.get_rand_seeds()
        randseed = self.fio_opts['randseed']

        logging.debug("randseed = %s", randseed)

        if randseed not in TestRS.seeds:
            TestRS.seeds[randseed] = rand_seeds
            logging.debug("TestRS: saving rand_seeds")
        else:
            if TestRS.seeds[randseed] != rand_seeds:
                self.passed = False
                print("TestRS: seeds don't match when they should")
            else:
                logging.debug("TestRS: seeds correctly match")

        # Now try to find seeds generated using a different randseed and make
        # sure they *don't* match
        for key, value in TestRS.seeds.items():
            if key != randseed:
                if value == rand_seeds:
                    self.passed = False
                    print("TestRS: randseeds differ but generated seeds match.")
                else:
                    logging.debug("TestRS: randseeds differ and generated seeds also differ.")


def parse_args():
    """Parse command-line arguments."""

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
    parser.add_argument('-s', '--skip', nargs='+', type=int,
                        help='list of test(s) to skip')
    parser.add_argument('-o', '--run-only', nargs='+', type=int,
                        help='list of test(s) to run, skipping all others')
    args = parser.parse_args()

    return args


def main():
    """Run tests of fio random seed options"""

    args = parse_args()

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    artifact_root = args.artifact_root if args.artifact_root else \
        f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
    os.mkdir(artifact_root)
    print(f"Artifact directory is {artifact_root}")

    if args.fio:
        fio_path = str(Path(args.fio).absolute())
    else:
        fio_path = 'fio'
    print(f"fio path is {fio_path}")

    test_list = [
        {
            "test_id": 1,
            "fio_opts": {
                "randrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 2,
            "fio_opts": {
                "randrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 3,
            "fio_opts": {
                "randrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 4,
            "fio_opts": {
                "randrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 5,
            "fio_opts": {
                "allrandrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 6,
            "fio_opts": {
                "allrandrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 7,
            "fio_opts": {
                "allrandrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 8,
            "fio_opts": {
                "allrandrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 9,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 10,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 11,
            "fio_opts": {
                "randrepeat": 1,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 12,
            "fio_opts": {
                "allrandrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 13,
            "fio_opts": {
                "allrandrepeat": 1,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 14,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 15,
            "fio_opts": {
                "randrepeat": 1,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 16,
            "fio_opts": {
                "allrandrepeat": 0,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 17,
            "fio_opts": {
                "allrandrepeat": 1,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
    ]

    test_env = {
              'fio_path': fio_path,
              'fio_root': str(Path(__file__).absolute().parent.parent),
              'artifact_root': artifact_root,
              'basename': 'random',
              }

    _, failed, _ = run_fio_tests(test_list, test_env, args)
    sys.exit(failed)


if __name__ == '__main__':
    main()
Commit	Line	Data
c994fa62 VF	1	#!/usr/bin/env python3
	2	"""
	3	# random_seed.py
	4	#
	5	# Test fio's random seed options.
	6	#
	7	# - make sure that randseed overrides randrepeat and allrandrepeat
	8	# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
	9	# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
	10	#
	11	# USAGE
	12	# see python3 random_seed.py --help
	13	#
	14	# EXAMPLES
	15	# python3 t/random_seed.py
	16	# python3 t/random_seed.py -f ./fio
	17	#
	18	# REQUIREMENTS
	19	# Python 3.6
	20	#
	21	"""
	22	import os
	23	import sys
	24	import time
	25	import locale
86131edc	26	import logging
c994fa62	27	import argparse
c994fa62	28	from pathlib import Path
1017a7d3	29	from fiotestlib import FioJobCmdTest, run_fio_tests
c994fa62	30
1017a7d3	31	class FioRandTest(FioJobCmdTest):
c994fa62 VF	32	"""fio random seed test."""
c994fa62 VF	33
1017a7d3 VF	34	def setup(self, parameters):
1017a7d3 VF	35	"""Setup the test."""
c994fa62 VF	36
	37	fio_args = [
	38	"--debug=random",
	39	"--name=random_seed",
	40	"--ioengine=null",
	41	"--filesize=32k",
	42	"--rw=randread",
	43	f"--output={self.filenames['output']}",
	44	]
	45	for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
1017a7d3 VF	46	if opt in self.fio_opts:
1017a7d3 VF	47	option = f"--{opt}={self.fio_opts[opt]}"
c994fa62 VF	48	fio_args.append(option)
c994fa62 VF	49
1017a7d3	50	super().setup(fio_args)
c994fa62 VF	51
	52	def get_rand_seeds(self):
	53	"""Collect random seeds from --debug=random output."""
cab6c7d1 VF	54	with open(self.filenames['output'], "r",
cab6c7d1 VF	55	encoding=locale.getpreferredencoding()) as out_file:
c994fa62 VF	56	file_data = out_file.read()
	57
	58	offsets = 0
	59	for line in file_data.split('\n'):
	60	if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
	61	tokens = line.split('=')
	62	offsets = int(tokens[len(tokens)-1])
	63	break
	64
	65	if offsets == 0:
	66	pass
	67	# find an exception to throw
	68
	69	seed_list = []
	70	for line in file_data.split('\n'):
	71	if 'random' not in line:
	72	continue
	73	if 'rand_seeds[' in line:
	74	tokens = line.split('=')
	75	seed = int(tokens[-1])
	76	seed_list.append(seed)
	77	# assume that seeds are in order
	78
	79	return seed_list
	80
c994fa62 VF	81
	82	class TestRR(FioRandTest):
	83	"""
	84	Test object for [all]randrepeat. If run for the first time just collect the
	85	seeds. For later runs make sure the seeds match or do not match those
	86	previously collected.
	87	"""
	88	# one set of seeds is for randrepeat=0 and the other is for randrepeat=1
	89	seeds = { 0: None, 1: None }
	90
1017a7d3	91	def check_result(self):
c994fa62 VF	92	"""Check output for allrandrepeat=1."""
c994fa62 VF	93
1017a7d3 VF	94	opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
1017a7d3 VF	95	rr = self.fio_opts[opt]
c994fa62 VF	96	rand_seeds = self.get_rand_seeds()
	97
	98	if not TestRR.seeds[rr]:
	99	TestRR.seeds[rr] = rand_seeds
cab6c7d1	100	logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
c994fa62 VF	101	else:
	102	if rr:
	103	if TestRR.seeds[1] != rand_seeds:
1017a7d3	104	self.passed = False
c994fa62 VF	105	print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
c994fa62 VF	106	else:
cab6c7d1	107	logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
c994fa62	108	if TestRR.seeds[0] == rand_seeds:
1017a7d3	109	self.passed = False
c994fa62 VF	110	print("TestRR: seeds unexpectedly match those from system RNG")
	111	else:
	112	if TestRR.seeds[0] == rand_seeds:
1017a7d3	113	self.passed = False
c994fa62 VF	114	print(f"TestRR: unexpected seed match for [a]rr={rr}")
c994fa62 VF	115	else:
cab6c7d1	116	logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
c994fa62	117	if TestRR.seeds[1] == rand_seeds:
1017a7d3	118	self.passed = False
cab6c7d1	119	print("TestRR: random seeds unexpectedly match those from [a]rr=1")
c994fa62	120
c994fa62 VF	121
	122	class TestRS(FioRandTest):
	123	"""
	124	Test object when randseed=something controls the generated seeds. If run
	125	for the first time for a given randseed just collect the seeds. For later
	126	runs with the same seed make sure the seeds are the same as those
	127	previously collected.
	128	"""
	129	seeds = {}
	130
1017a7d3	131	def check_result(self):
c994fa62 VF	132	"""Check output for randseed=something."""
c994fa62 VF	133
c994fa62	134	rand_seeds = self.get_rand_seeds()
1017a7d3	135	randseed = self.fio_opts['randseed']
c994fa62	136
cab6c7d1	137	logging.debug("randseed = %s", randseed)
c994fa62 VF	138
	139	if randseed not in TestRS.seeds:
	140	TestRS.seeds[randseed] = rand_seeds
86131edc	141	logging.debug("TestRS: saving rand_seeds")
c994fa62 VF	142	else:
c994fa62 VF	143	if TestRS.seeds[randseed] != rand_seeds:
1017a7d3	144	self.passed = False
c994fa62 VF	145	print("TestRS: seeds don't match when they should")
c994fa62 VF	146	else:
86131edc	147	logging.debug("TestRS: seeds correctly match")
c994fa62 VF	148
	149	# Now try to find seeds generated using a different randseed and make
	150	# sure they don't match
cab6c7d1	151	for key, value in TestRS.seeds.items():
c994fa62	152	if key != randseed:
cab6c7d1	153	if value == rand_seeds:
1017a7d3	154	self.passed = False
c994fa62 VF	155	print("TestRS: randseeds differ but generated seeds match.")
c994fa62 VF	156	else:
86131edc	157	logging.debug("TestRS: randseeds differ and generated seeds also differ.")
c994fa62 VF	158
	159
	160	def parse_args():
	161	"""Parse command-line arguments."""
	162
	163	parser = argparse.ArgumentParser()
	164	parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
	165	parser.add_argument('-a', '--artifact-root', help='artifact root directory')
	166	parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
	167	parser.add_argument('-s', '--skip', nargs='+', type=int,
	168	help='list of test(s) to skip')
	169	parser.add_argument('-o', '--run-only', nargs='+', type=int,
	170	help='list of test(s) to run, skipping all others')
	171	args = parser.parse_args()
	172
	173	return args
	174
	175
	176	def main():
	177	"""Run tests of fio random seed options"""
	178
	179	args = parse_args()
	180
86131edc VF	181	if args.debug:
	182	logging.basicConfig(level=logging.DEBUG)
	183	else:
	184	logging.basicConfig(level=logging.INFO)
	185
c994fa62 VF	186	artifact_root = args.artifact_root if args.artifact_root else \
	187	f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
	188	os.mkdir(artifact_root)
	189	print(f"Artifact directory is {artifact_root}")
	190
	191	if args.fio:
1017a7d3	192	fio_path = str(Path(args.fio).absolute())
c994fa62	193	else:
1017a7d3 VF	194	fio_path = 'fio'
1017a7d3 VF	195	print(f"fio path is {fio_path}")
c994fa62 VF	196
	197	test_list = [
	198	{
	199	"test_id": 1,
1017a7d3 VF	200	"fio_opts": {
	201	"randrepeat": 0,
	202	},
	203	"test_class": TestRR,
c994fa62 VF	204	},
	205	{
	206	"test_id": 2,
1017a7d3 VF	207	"fio_opts": {
	208	"randrepeat": 0,
	209	},
	210	"test_class": TestRR,
c994fa62 VF	211	},
	212	{
	213	"test_id": 3,
1017a7d3 VF	214	"fio_opts": {
	215	"randrepeat": 1,
	216	},
	217	"test_class": TestRR,
c994fa62 VF	218	},
	219	{
	220	"test_id": 4,
1017a7d3 VF	221	"fio_opts": {
	222	"randrepeat": 1,
	223	},
	224	"test_class": TestRR,
c994fa62 VF	225	},
	226	{
	227	"test_id": 5,
1017a7d3 VF	228	"fio_opts": {
	229	"allrandrepeat": 0,
	230	},
	231	"test_class": TestRR,
c994fa62 VF	232	},
	233	{
	234	"test_id": 6,
1017a7d3 VF	235	"fio_opts": {
	236	"allrandrepeat": 0,
	237	},
	238	"test_class": TestRR,
c994fa62 VF	239	},
	240	{
	241	"test_id": 7,
1017a7d3 VF	242	"fio_opts": {
	243	"allrandrepeat": 1,
	244	},
	245	"test_class": TestRR,
c994fa62 VF	246	},
	247	{
	248	"test_id": 8,
1017a7d3 VF	249	"fio_opts": {
	250	"allrandrepeat": 1,
	251	},
	252	"test_class": TestRR,
c994fa62 VF	253	},
	254	{
	255	"test_id": 9,
1017a7d3 VF	256	"fio_opts": {
	257	"randrepeat": 0,
	258	"randseed": "12345",
	259	},
	260	"test_class": TestRS,
c994fa62 VF	261	},
	262	{
	263	"test_id": 10,
1017a7d3 VF	264	"fio_opts": {
	265	"randrepeat": 0,
	266	"randseed": "12345",
	267	},
	268	"test_class": TestRS,
c994fa62 VF	269	},
	270	{
	271	"test_id": 11,
1017a7d3 VF	272	"fio_opts": {
	273	"randrepeat": 1,
	274	"randseed": "12345",
	275	},
	276	"test_class": TestRS,
c994fa62 VF	277	},
	278	{
	279	"test_id": 12,
1017a7d3 VF	280	"fio_opts": {
	281	"allrandrepeat": 0,
	282	"randseed": "12345",
	283	},
	284	"test_class": TestRS,
c994fa62 VF	285	},
	286	{
	287	"test_id": 13,
1017a7d3 VF	288	"fio_opts": {
	289	"allrandrepeat": 1,
	290	"randseed": "12345",
	291	},
	292	"test_class": TestRS,
c994fa62 VF	293	},
	294	{
	295	"test_id": 14,
1017a7d3 VF	296	"fio_opts": {
	297	"randrepeat": 0,
	298	"randseed": "67890",
	299	},
	300	"test_class": TestRS,
c994fa62 VF	301	},
	302	{
	303	"test_id": 15,
1017a7d3 VF	304	"fio_opts": {
	305	"randrepeat": 1,
	306	"randseed": "67890",
	307	},
	308	"test_class": TestRS,
c994fa62 VF	309	},
	310	{
	311	"test_id": 16,
1017a7d3 VF	312	"fio_opts": {
	313	"allrandrepeat": 0,
	314	"randseed": "67890",
	315	},
	316	"test_class": TestRS,
c994fa62 VF	317	},
	318	{
	319	"test_id": 17,
1017a7d3 VF	320	"fio_opts": {
	321	"allrandrepeat": 1,
	322	"randseed": "67890",
	323	},
	324	"test_class": TestRS,
c994fa62 VF	325	},
	326	]
	327
1017a7d3 VF	328	test_env = {
	329	'fio_path': fio_path,
	330	'fio_root': str(Path(__file__).absolute().parent.parent),
	331	'artifact_root': artifact_root,
	332	'basename': 'random',
	333	}
c994fa62	334
1017a7d3	335	_, failed, _ = run_fio_tests(test_list, test_env, args)
c994fa62 VF	336	sys.exit(failed)
	337
	338
	339	if __name__ == '__main__':
	340	main()