[fio.git] / t / random_seed.py

#!/usr/bin/env python3
"""
# random_seed.py
#
# Test fio's random seed options.
#
# - make sure that randseed overrides randrepeat and allrandrepeat
# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
#
# USAGE
# see python3 random_seed.py --help
#
# EXAMPLES
# python3 t/random_seed.py
# python3 t/random_seed.py -f ./fio
#
# REQUIREMENTS
# Python 3.6
#
"""
import os
import sys
import time
import locale
import logging
import argparse
from pathlib import Path
from fiotestlib import FioJobCmdTest, run_fio_tests

class FioRandTest(FioJobCmdTest):
    """fio random seed test."""

    def setup(self, parameters):
        """Setup the test."""

        fio_args = [
            "--debug=random",
            "--name=random_seed",
            "--ioengine=null",
            "--filesize=32k",
            "--rw=randread",
            f"--output={self.filenames['output']}",
        ]
        for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
            if opt in self.fio_opts:
                option = f"--{opt}={self.fio_opts[opt]}"
                fio_args.append(option)

        super().setup(fio_args)

    def get_rand_seeds(self):
        """Collect random seeds from --debug=random output."""
        with open(self.filenames['output'], "r",
                  encoding=locale.getpreferredencoding()) as out_file:
            file_data = out_file.read()

            offsets = 0
            for line in file_data.split('\n'):
                if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
                    tokens = line.split('=')
                    offsets = int(tokens[len(tokens)-1])
                    break

            if offsets == 0:
                pass
                # find an exception to throw

            seed_list = []
            for line in file_data.split('\n'):
                if 'random' not in line:
                    continue
                if 'rand_seeds[' in line:
                    tokens = line.split('=')
                    seed = int(tokens[-1])
                    seed_list.append(seed)
                    # assume that seeds are in order

            return seed_list


class TestRR(FioRandTest):
    """
    Test object for [all]randrepeat. If run for the first time just collect the
    seeds. For later runs make sure the seeds match or do not match those
    previously collected.
    """
    # one set of seeds is for randrepeat=0 and the other is for randrepeat=1
    seeds = { 0: None, 1: None }

    def check_result(self):
        """Check output for allrandrepeat=1."""

        super().check_result()
        if not self.passed:
            return

        opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
        rr = self.fio_opts[opt]
        rand_seeds = self.get_rand_seeds()

        if not TestRR.seeds[rr]:
            TestRR.seeds[rr] = rand_seeds
            logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
        else:
            if rr:
                if TestRR.seeds[1] != rand_seeds:
                    self.passed = False
                    print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
                else:
                    logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
                if TestRR.seeds[0] == rand_seeds:
                    self.passed = False
                    print("TestRR: seeds unexpectedly match those from system RNG")
            else:
                if TestRR.seeds[0] == rand_seeds:
                    self.passed = False
                    print(f"TestRR: unexpected seed match for [a]rr={rr}")
                else:
                    logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
                if TestRR.seeds[1] == rand_seeds:
                    self.passed = False
                    print("TestRR: random seeds unexpectedly match those from [a]rr=1")


class TestRS(FioRandTest):
    """
    Test object when randseed=something controls the generated seeds. If run
    for the first time for a given randseed just collect the seeds. For later
    runs with the same seed make sure the seeds are the same as those
    previously collected.
    """
    seeds = {}

    def check_result(self):
        """Check output for randseed=something."""

        super().check_result()
        if not self.passed:
            return

        rand_seeds = self.get_rand_seeds()
        randseed = self.fio_opts['randseed']

        logging.debug("randseed = %s", randseed)

        if randseed not in TestRS.seeds:
            TestRS.seeds[randseed] = rand_seeds
            logging.debug("TestRS: saving rand_seeds")
        else:
            if TestRS.seeds[randseed] != rand_seeds:
                self.passed = False
                print("TestRS: seeds don't match when they should")
            else:
                logging.debug("TestRS: seeds correctly match")

        # Now try to find seeds generated using a different randseed and make
        # sure they *don't* match
        for key, value in TestRS.seeds.items():
            if key != randseed:
                if value == rand_seeds:
                    self.passed = False
                    print("TestRS: randseeds differ but generated seeds match.")
                else:
                    logging.debug("TestRS: randseeds differ and generated seeds also differ.")


def parse_args():
    """Parse command-line arguments."""

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
    parser.add_argument('-a', '--artifact-root', help='artifact root directory')
    parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
    parser.add_argument('-s', '--skip', nargs='+', type=int,
                        help='list of test(s) to skip')
    parser.add_argument('-o', '--run-only', nargs='+', type=int,
                        help='list of test(s) to run, skipping all others')
    args = parser.parse_args()

    return args


def main():
    """Run tests of fio random seed options"""

    args = parse_args()

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    artifact_root = args.artifact_root if args.artifact_root else \
        f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
    os.mkdir(artifact_root)
    print(f"Artifact directory is {artifact_root}")

    if args.fio:
        fio_path = str(Path(args.fio).absolute())
    else:
        fio_path = 'fio'
    print(f"fio path is {fio_path}")

    test_list = [
        {
            "test_id": 1,
            "fio_opts": {
                "randrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 2,
            "fio_opts": {
                "randrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 3,
            "fio_opts": {
                "randrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 4,
            "fio_opts": {
                "randrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 5,
            "fio_opts": {
                "allrandrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 6,
            "fio_opts": {
                "allrandrepeat": 0,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 7,
            "fio_opts": {
                "allrandrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 8,
            "fio_opts": {
                "allrandrepeat": 1,
                },
            "test_class": TestRR,
        },
        {
            "test_id": 9,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 10,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 11,
            "fio_opts": {
                "randrepeat": 1,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 12,
            "fio_opts": {
                "allrandrepeat": 0,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 13,
            "fio_opts": {
                "allrandrepeat": 1,
                "randseed": "12345",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 14,
            "fio_opts": {
                "randrepeat": 0,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 15,
            "fio_opts": {
                "randrepeat": 1,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 16,
            "fio_opts": {
                "allrandrepeat": 0,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
        {
            "test_id": 17,
            "fio_opts": {
                "allrandrepeat": 1,
                "randseed": "67890",
                },
            "test_class": TestRS,
        },
    ]

    test_env = {
              'fio_path': fio_path,
              'fio_root': str(Path(__file__).absolute().parent.parent),
              'artifact_root': artifact_root,
              'basename': 'random',
              }

    _, failed, _ = run_fio_tests(test_list, test_env, args)
    sys.exit(failed)


if __name__ == '__main__':
    main()
Commit	Line	Data
	1	#!/usr/bin/env python3
	2	"""
	3	# random_seed.py
	4	#
	5	# Test fio's random seed options.
	6	#
	7	# - make sure that randseed overrides randrepeat and allrandrepeat
	8	# - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
	9	# - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
	10	#
	11	# USAGE
	12	# see python3 random_seed.py --help
	13	#
	14	# EXAMPLES
	15	# python3 t/random_seed.py
	16	# python3 t/random_seed.py -f ./fio
	17	#
	18	# REQUIREMENTS
	19	# Python 3.6
	20	#
	21	"""
	22	import os
	23	import sys
	24	import time
	25	import locale
	26	import logging
	27	import argparse
	28	from pathlib import Path
	29	from fiotestlib import FioJobCmdTest, run_fio_tests
	30
	31	class FioRandTest(FioJobCmdTest):
	32	"""fio random seed test."""
	33
	34	def setup(self, parameters):
	35	"""Setup the test."""
	36
	37	fio_args = [
	38	"--debug=random",
	39	"--name=random_seed",
	40	"--ioengine=null",
	41	"--filesize=32k",
	42	"--rw=randread",
	43	f"--output={self.filenames['output']}",
	44	]
	45	for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
	46	if opt in self.fio_opts:
	47	option = f"--{opt}={self.fio_opts[opt]}"
	48	fio_args.append(option)
	49
	50	super().setup(fio_args)
	51
	52	def get_rand_seeds(self):
	53	"""Collect random seeds from --debug=random output."""
	54	with open(self.filenames['output'], "r",
	55	encoding=locale.getpreferredencoding()) as out_file:
	56	file_data = out_file.read()
	57
	58	offsets = 0
	59	for line in file_data.split('\n'):
	60	if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
	61	tokens = line.split('=')
	62	offsets = int(tokens[len(tokens)-1])
	63	break
	64
	65	if offsets == 0:
	66	pass
	67	# find an exception to throw
	68
	69	seed_list = []
	70	for line in file_data.split('\n'):
	71	if 'random' not in line:
	72	continue
	73	if 'rand_seeds[' in line:
	74	tokens = line.split('=')
	75	seed = int(tokens[-1])
	76	seed_list.append(seed)
	77	# assume that seeds are in order
	78
	79	return seed_list
	80
	81
	82	class TestRR(FioRandTest):
	83	"""
	84	Test object for [all]randrepeat. If run for the first time just collect the
	85	seeds. For later runs make sure the seeds match or do not match those
	86	previously collected.
	87	"""
	88	# one set of seeds is for randrepeat=0 and the other is for randrepeat=1
	89	seeds = { 0: None, 1: None }
	90
	91	def check_result(self):
	92	"""Check output for allrandrepeat=1."""
	93
	94	super().check_result()
	95	if not self.passed:
	96	return
	97
	98	opt = 'randrepeat' if 'randrepeat' in self.fio_opts else 'allrandrepeat'
	99	rr = self.fio_opts[opt]
	100	rand_seeds = self.get_rand_seeds()
	101
	102	if not TestRR.seeds[rr]:
	103	TestRR.seeds[rr] = rand_seeds
	104	logging.debug("TestRR: saving rand_seeds for [a]rr=%d", rr)
	105	else:
	106	if rr:
	107	if TestRR.seeds[1] != rand_seeds:
	108	self.passed = False
	109	print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
	110	else:
	111	logging.debug("TestRR: seeds correctly match for [a]rr=%d", rr)
	112	if TestRR.seeds[0] == rand_seeds:
	113	self.passed = False
	114	print("TestRR: seeds unexpectedly match those from system RNG")
	115	else:
	116	if TestRR.seeds[0] == rand_seeds:
	117	self.passed = False
	118	print(f"TestRR: unexpected seed match for [a]rr={rr}")
	119	else:
	120	logging.debug("TestRR: seeds correctly don't match for [a]rr=%d", rr)
	121	if TestRR.seeds[1] == rand_seeds:
	122	self.passed = False
	123	print("TestRR: random seeds unexpectedly match those from [a]rr=1")
	124
	125
	126	class TestRS(FioRandTest):
	127	"""
	128	Test object when randseed=something controls the generated seeds. If run
	129	for the first time for a given randseed just collect the seeds. For later
	130	runs with the same seed make sure the seeds are the same as those
	131	previously collected.
	132	"""
	133	seeds = {}
	134
	135	def check_result(self):
	136	"""Check output for randseed=something."""
	137
	138	super().check_result()
	139	if not self.passed:
	140	return
	141
	142	rand_seeds = self.get_rand_seeds()
	143	randseed = self.fio_opts['randseed']
	144
	145	logging.debug("randseed = %s", randseed)
	146
	147	if randseed not in TestRS.seeds:
	148	TestRS.seeds[randseed] = rand_seeds
	149	logging.debug("TestRS: saving rand_seeds")
	150	else:
	151	if TestRS.seeds[randseed] != rand_seeds:
	152	self.passed = False
	153	print("TestRS: seeds don't match when they should")
	154	else:
	155	logging.debug("TestRS: seeds correctly match")
	156
	157	# Now try to find seeds generated using a different randseed and make
	158	# sure they don't match
	159	for key, value in TestRS.seeds.items():
	160	if key != randseed:
	161	if value == rand_seeds:
	162	self.passed = False
	163	print("TestRS: randseeds differ but generated seeds match.")
	164	else:
	165	logging.debug("TestRS: randseeds differ and generated seeds also differ.")
	166
	167
	168	def parse_args():
	169	"""Parse command-line arguments."""
	170
	171	parser = argparse.ArgumentParser()
	172	parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
	173	parser.add_argument('-a', '--artifact-root', help='artifact root directory')
	174	parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
	175	parser.add_argument('-s', '--skip', nargs='+', type=int,
	176	help='list of test(s) to skip')
	177	parser.add_argument('-o', '--run-only', nargs='+', type=int,
	178	help='list of test(s) to run, skipping all others')
	179	args = parser.parse_args()
	180
	181	return args
	182
	183
	184	def main():
	185	"""Run tests of fio random seed options"""
	186
	187	args = parse_args()
	188
	189	if args.debug:
	190	logging.basicConfig(level=logging.DEBUG)
	191	else:
	192	logging.basicConfig(level=logging.INFO)
	193
	194	artifact_root = args.artifact_root if args.artifact_root else \
	195	f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
	196	os.mkdir(artifact_root)
	197	print(f"Artifact directory is {artifact_root}")
	198
	199	if args.fio:
	200	fio_path = str(Path(args.fio).absolute())
	201	else:
	202	fio_path = 'fio'
	203	print(f"fio path is {fio_path}")
	204
	205	test_list = [
	206	{
	207	"test_id": 1,
	208	"fio_opts": {
	209	"randrepeat": 0,
	210	},
	211	"test_class": TestRR,
	212	},
	213	{
	214	"test_id": 2,
	215	"fio_opts": {
	216	"randrepeat": 0,
	217	},
	218	"test_class": TestRR,
	219	},
	220	{
	221	"test_id": 3,
	222	"fio_opts": {
	223	"randrepeat": 1,
	224	},
	225	"test_class": TestRR,
	226	},
	227	{
	228	"test_id": 4,
	229	"fio_opts": {
	230	"randrepeat": 1,
	231	},
	232	"test_class": TestRR,
	233	},
	234	{
	235	"test_id": 5,
	236	"fio_opts": {
	237	"allrandrepeat": 0,
	238	},
	239	"test_class": TestRR,
	240	},
	241	{
	242	"test_id": 6,
	243	"fio_opts": {
	244	"allrandrepeat": 0,
	245	},
	246	"test_class": TestRR,
	247	},
	248	{
	249	"test_id": 7,
	250	"fio_opts": {
	251	"allrandrepeat": 1,
	252	},
	253	"test_class": TestRR,
	254	},
	255	{
	256	"test_id": 8,
	257	"fio_opts": {
	258	"allrandrepeat": 1,
	259	},
	260	"test_class": TestRR,
	261	},
	262	{
	263	"test_id": 9,
	264	"fio_opts": {
	265	"randrepeat": 0,
	266	"randseed": "12345",
	267	},
	268	"test_class": TestRS,
	269	},
	270	{
	271	"test_id": 10,
	272	"fio_opts": {
	273	"randrepeat": 0,
	274	"randseed": "12345",
	275	},
	276	"test_class": TestRS,
	277	},
	278	{
	279	"test_id": 11,
	280	"fio_opts": {
	281	"randrepeat": 1,
	282	"randseed": "12345",
	283	},
	284	"test_class": TestRS,
	285	},
	286	{
	287	"test_id": 12,
	288	"fio_opts": {
	289	"allrandrepeat": 0,
	290	"randseed": "12345",
	291	},
	292	"test_class": TestRS,
	293	},
	294	{
	295	"test_id": 13,
	296	"fio_opts": {
	297	"allrandrepeat": 1,
	298	"randseed": "12345",
	299	},
	300	"test_class": TestRS,
	301	},
	302	{
	303	"test_id": 14,
	304	"fio_opts": {
	305	"randrepeat": 0,
	306	"randseed": "67890",
	307	},
	308	"test_class": TestRS,
	309	},
	310	{
	311	"test_id": 15,
	312	"fio_opts": {
	313	"randrepeat": 1,
	314	"randseed": "67890",
	315	},
	316	"test_class": TestRS,
	317	},
	318	{
	319	"test_id": 16,
	320	"fio_opts": {
	321	"allrandrepeat": 0,
	322	"randseed": "67890",
	323	},
	324	"test_class": TestRS,
	325	},
	326	{
	327	"test_id": 17,
	328	"fio_opts": {
	329	"allrandrepeat": 1,
	330	"randseed": "67890",
	331	},
	332	"test_class": TestRS,
	333	},
	334	]
	335
	336	test_env = {
	337	'fio_path': fio_path,
	338	'fio_root': str(Path(__file__).absolute().parent.parent),
	339	'artifact_root': artifact_root,
	340	'basename': 'random',
	341	}
	342
	343	_, failed, _ = run_fio_tests(test_list, test_env, args)
	344	sys.exit(failed)
	345
	346
	347	if __name__ == '__main__':
	348	main()