[fio.git] / t / strided.py

#!/usr/bin/python
# Note: this script is python2 and python3 compatible.
#
# strided.py
#
# Test zonemode=strided. This uses the null ioengine when no file is
# specified. If a file is specified, use it for randdom read testing.
# Some of the zoneranges in the tests are 16MiB. So when using a file
# a minimum size of 32MiB is recommended.
#
# USAGE
# python strided.py fio-executable [-f file/device]
#
# EXAMPLES
# python t/strided.py ./fio
# python t/strided.py ./fio -f /dev/sda
# dd if=/dev/zero of=temp bs=1M count=32
# python t/strided.py ./fio -f temp
#
# REQUIREMENTS
# Python 2.6+
#
# ===TEST MATRIX===
#
# --zonemode=strided, zoneskip unset
#   w/ randommap and LFSR
#       zonesize=zonerange  all blocks in zonerange touched
#       zonesize>zonerange  all blocks touched and roll-over back into zone
#       zonesize<zonerange  all blocks inside zone
#
#   w/o randommap       all blocks inside zone
#

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import argparse
import subprocess


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable (e.g., ./fio)')
    parser.add_argument('-f', '--filename', help="file/device to test")
    args = parser.parse_args()

    return args


def run_fio(fio, test, index):
    filename = "strided"
    fio_args = [
                "--max-jobs=16",
                "--name=strided",
                "--zonemode=strided",
                "--log_offset=1",
                "--randrepeat=0",
                "--rw=randread",
                "--write_iops_log={0}{1:03d}".format(filename, index),
                "--output={0}{1:03d}.out".format(filename, index),
                "--zonerange={zonerange}".format(**test),
                "--zonesize={zonesize}".format(**test),
                "--bs={bs}".format(**test),
               ]
    if 'norandommap' in test:
        fio_args.append('--norandommap')
    if 'random_generator' in test:
        fio_args.append('--random_generator={random_generator}'.format(**test))
    if 'offset' in test:
        fio_args.append('--offset={offset}'.format(**test))
    if 'filename' in test:
        fio_args.append('--filename={filename}'.format(**test))
        fio_args.append('--filesize={filesize})'.format(**test))
    else:
        fio_args.append('--ioengine=null')
        fio_args.append('--size={size}'.format(**test))
        fio_args.append('--io_size={io_size}'.format(**test))
        fio_args.append('--filesize={size})'.format(**test))

    output = subprocess.check_output([fio] + fio_args, universal_newlines=True)

    f = open("{0}{1:03d}_iops.1.log".format(filename, index), "r")
    log = f.read()
    f.close()

    return log


def check_output(iops_log, test):
    zonestart = 0 if 'offset' not in test else test['offset']
    iospersize = test['zonesize'] / test['bs']
    iosperrange = test['zonerange'] / test['bs']
    iosperzone = 0
    lines = iops_log.split('\n')
    zoneset = set()

    for line in lines:
        if len(line) == 0:
            continue

        if iosperzone == iospersize:
            # time to move to a new zone
            iosperzone = 0
            zoneset = set()
            zonestart += test['zonerange']
            if zonestart >= test['filesize']:
                zonestart = 0 if 'offset' not in test else test['offset']

        iosperzone = iosperzone + 1
        tokens = line.split(',')
        offset = int(tokens[4])
        if offset < zonestart or offset >= zonestart + test['zonerange']:
            print("Offset {0} outside of zone starting at {1}".format(
                    offset, zonestart))
            return False

        # skip next section if norandommap is enabled with no
        # random_generator or with a random_generator != lfsr
        if 'norandommap' in test:
            if 'random_generator' in test:
                if test['random_generator'] != 'lfsr':
                    continue
            else:
                continue

        # we either have a random map enabled or we
        # are using an LFSR
        # so all blocks should be unique and we should have
        # covered the entire zone when iosperzone % iosperrange == 0
        block = (offset - zonestart) / test['bs']
        if block in zoneset:
            print("Offset {0} in zone already touched".format(offset))
            return False

        zoneset.add(block)
        if iosperzone % iosperrange == 0:
            if len(zoneset) != iosperrange:
                print("Expected {0} blocks in zone but only saw {1}".format(
                        iosperrange, len(zoneset)))
                return False
            zoneset = set()

    return True


if __name__ == '__main__':
    args = parse_args()

    tests = [   # randommap enabled
                {
                    "zonerange": 4096,
                    "zonesize": 4096,
                    "bs": 4096,
                    "offset": 8*4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "zonerange": 4096,
                    "zonesize": 4096,
                    "bs": 4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "zonerange": 16*1024*1024,
                    "zonesize": 16*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                {
                    "zonerange": 4096,
                    "zonesize": 4*4096,
                    "bs": 4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "zonerange": 16*1024*1024,
                    "zonesize": 32*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                {
                    "zonerange": 8192,
                    "zonesize": 4096,
                    "bs": 4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "zonerange": 16*1024*1024,
                    "zonesize": 8*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                # lfsr
                {
                    "random_generator": "lfsr",
                    "zonerange": 4096*1024,
                    "zonesize": 4096*1024,
                    "bs": 4096,
                    "offset": 8*4096*1024,
                    "size": 16*4096*1024,
                    "io_size": 16*4096*1024,
                },
                {
                    "random_generator": "lfsr",
                    "zonerange": 4096*1024,
                    "zonesize": 4096*1024,
                    "bs": 4096,
                    "size": 16*4096*1024,
                    "io_size": 16*4096*1024,
                },
                {
                    "random_generator": "lfsr",
                    "zonerange": 16*1024*1024,
                    "zonesize": 16*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                {
                    "random_generator": "lfsr",
                    "zonerange": 4096*1024,
                    "zonesize": 4*4096*1024,
                    "bs": 4096,
                    "size": 16*4096*1024,
                    "io_size": 16*4096*1024,
                },
                {
                    "random_generator": "lfsr",
                    "zonerange": 16*1024*1024,
                    "zonesize": 32*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                {
                    "random_generator": "lfsr",
                    "zonerange": 8192*1024,
                    "zonesize": 4096*1024,
                    "bs": 4096,
                    "size": 16*4096*1024,
                    "io_size": 16*4096*1024,
                },
                {
                    "random_generator": "lfsr",
                    "zonerange": 16*1024*1024,
                    "zonesize": 8*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                # norandommap
                {
                    "norandommap": 1,
                    "zonerange": 4096,
                    "zonesize": 4096,
                    "bs": 4096,
                    "offset": 8*4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "norandommap": 1,
                    "zonerange": 4096,
                    "zonesize": 4096,
                    "bs": 4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "norandommap": 1,
                    "zonerange": 16*1024*1024,
                    "zonesize": 16*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                {
                    "norandommap": 1,
                    "zonerange": 4096,
                    "zonesize": 8192,
                    "bs": 4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "norandommap": 1,
                    "zonerange": 16*1024*1024,
                    "zonesize": 32*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*204,
                },
                {
                    "norandommap": 1,
                    "zonerange": 8192,
                    "zonesize": 4096,
                    "bs": 4096,
                    "size": 16*4096,
                    "io_size": 16*4096,
                },
                {
                    "norandommap": 1,
                    "zonerange": 16*1024*1024,
                    "zonesize": 8*1024*1024,
                    "bs": 4096,
                    "size": 256*1024*1024,
                    "io_size": 256*1024*1024,
                },

            ]

    index = 1
    passed = 0
    failed = 0

    if args.filename:
        statinfo = os.stat(args.filename)
        filesize = statinfo.st_size
        if filesize == 0:
            f = os.open(args.filename, os.O_RDONLY)
            filesize = os.lseek(f, 0, os.SEEK_END)
            os.close(f)

    for test in tests:
        if args.filename:
            test['filename'] = args.filename
            test['filesize'] = filesize
        else:
            test['filesize'] = test['size']
        iops_log = run_fio(args.fio, test, index)
        status = check_output(iops_log, test)
        print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
        if status:
            passed = passed + 1
        else:
            failed = failed + 1
        index = index + 1

    print("{0} tests passed, {1} failed".format(passed, failed))

    sys.exit(failed)
Commit	Line	Data
6d5af490 VF	1	#!/usr/bin/python
	2	# Note: this script is python2 and python3 compatible.
	3	#
	4	# strided.py
	5	#
	6	# Test zonemode=strided. This uses the null ioengine when no file is
	7	# specified. If a file is specified, use it for randdom read testing.
	8	# Some of the zoneranges in the tests are 16MiB. So when using a file
	9	# a minimum size of 32MiB is recommended.
	10	#
	11	# USAGE
	12	# python strided.py fio-executable [-f file/device]
	13	#
	14	# EXAMPLES
	15	# python t/strided.py ./fio
	16	# python t/strided.py ./fio -f /dev/sda
	17	# dd if=/dev/zero of=temp bs=1M count=32
	18	# python t/strided.py ./fio -f temp
	19	#
	20	# REQUIREMENTS
	21	# Python 2.6+
	22	#
	23	# ===TEST MATRIX===
	24	#
fa9fd914	25	# --zonemode=strided, zoneskip unset
6d5af490 VF	26	# w/ randommap and LFSR
	27	# zonesize=zonerange all blocks in zonerange touched
	28	# zonesize>zonerange all blocks touched and roll-over back into zone
	29	# zonesize<zonerange all blocks inside zone
	30	#
	31	# w/o randommap all blocks inside zone
	32	#
	33
	34	from __future__ import absolute_import
	35	from __future__ import print_function
	36	import os
	37	import sys
	38	import argparse
	39	import subprocess
	40
	41
	42	def parse_args():
	43	parser = argparse.ArgumentParser()
	44	parser.add_argument('fio',
	45	help='path to fio executable (e.g., ./fio)')
	46	parser.add_argument('-f', '--filename', help="file/device to test")
	47	args = parser.parse_args()
	48
	49	return args
	50
	51
	52	def run_fio(fio, test, index):
	53	filename = "strided"
	54	fio_args = [
771dbb52	55	"--max-jobs=16",
6d5af490 VF	56	"--name=strided",
	57	"--zonemode=strided",
	58	"--log_offset=1",
	59	"--randrepeat=0",
	60	"--rw=randread",
6d5af490 VF	61	"--write_iops_log={0}{1:03d}".format(filename, index),
	62	"--output={0}{1:03d}.out".format(filename, index),
	63	"--zonerange={zonerange}".format(**test),
	64	"--zonesize={zonesize}".format(**test),
	65	"--bs={bs}".format(**test),
	66	]
	67	if 'norandommap' in test:
	68	fio_args.append('--norandommap')
	69	if 'random_generator' in test:
	70	fio_args.append('--random_generator={random_generator}'.format(**test))
	71	if 'offset' in test:
	72	fio_args.append('--offset={offset}'.format(**test))
	73	if 'filename' in test:
	74	fio_args.append('--filename={filename}'.format(**test))
	75	fio_args.append('--filesize={filesize})'.format(**test))
	76	else:
	77	fio_args.append('--ioengine=null')
	78	fio_args.append('--size={size}'.format(**test))
	79	fio_args.append('--io_size={io_size}'.format(**test))
	80	fio_args.append('--filesize={size})'.format(**test))
	81
	82	output = subprocess.check_output([fio] + fio_args, universal_newlines=True)
	83
	84	f = open("{0}{1:03d}_iops.1.log".format(filename, index), "r")
	85	log = f.read()
	86	f.close()
	87
	88	return log
	89
	90
	91	def check_output(iops_log, test):
	92	zonestart = 0 if 'offset' not in test else test['offset']
	93	iospersize = test['zonesize'] / test['bs']
	94	iosperrange = test['zonerange'] / test['bs']
	95	iosperzone = 0
	96	lines = iops_log.split('\n')
	97	zoneset = set()
	98
	99	for line in lines:
	100	if len(line) == 0:
	101	continue
	102
	103	if iosperzone == iospersize:
	104	# time to move to a new zone
	105	iosperzone = 0
	106	zoneset = set()
	107	zonestart += test['zonerange']
	108	if zonestart >= test['filesize']:
	109	zonestart = 0 if 'offset' not in test else test['offset']
	110
	111	iosperzone = iosperzone + 1
	112	tokens = line.split(',')
	113	offset = int(tokens[4])
	114	if offset < zonestart or offset >= zonestart + test['zonerange']:
	115	print("Offset {0} outside of zone starting at {1}".format(
	116	offset, zonestart))
	117	return False
	118
	119	# skip next section if norandommap is enabled with no
	120	# random_generator or with a random_generator != lfsr
	121	if 'norandommap' in test:
	122	if 'random_generator' in test:
	123	if test['random_generator'] != 'lfsr':
	124	continue
125	else:
126	continue
127
128	# we either have a random map enabled or we
129	# are using an LFSR
130	# so all blocks should be unique and we should have
131	# covered the entire zone when iosperzone % iosperrange == 0
132	block = (offset - zonestart) / test['bs']
133	if block in zoneset:
134	print("Offset {0} in zone already touched".format(offset))
135	return False
136
137	zoneset.add(block)
138	if iosperzone % iosperrange == 0:
139	if len(zoneset) != iosperrange:
140	print("Expected {0} blocks in zone but only saw {1}".format(
141	iosperrange, len(zoneset)))
142	return False
143	zoneset = set()
144
145	return True
146
147
148	if __name__ == '__main__':
149	args = parse_args()
150
151	tests = [ # randommap enabled
152	{
153	"zonerange": 4096,
154	"zonesize": 4096,
155	"bs": 4096,
156	"offset": 8*4096,
157	"size": 16*4096,
158	"io_size": 16*4096,
159	},
160	{
161	"zonerange": 4096,
162	"zonesize": 4096,
163	"bs": 4096,
164	"size": 16*4096,
165	"io_size": 16*4096,
166	},
167	{
168	"zonerange": 1610241024,
169	"zonesize": 1610241024,
170	"bs": 4096,
171	"size": 25610241024,
172	"io_size": 2561024204,
173	},
174	{
175	"zonerange": 4096,
176	"zonesize": 4*4096,
177	"bs": 4096,
178	"size": 16*4096,
179	"io_size": 16*4096,
180	},
181	{
182	"zonerange": 1610241024,
183	"zonesize": 3210241024,
184	"bs": 4096,
185	"size": 25610241024,
186	"io_size": 2561024204,
187	},
188	{
189	"zonerange": 8192,
190	"zonesize": 4096,
191	"bs": 4096,
192	"size": 16*4096,
193	"io_size": 16*4096,
194	},
195	{
196	"zonerange": 1610241024,
197	"zonesize": 810241024,
198	"bs": 4096,
199	"size": 25610241024,
200	"io_size": 2561024204,
201	},
202	# lfsr
203	{
204	"random_generator": "lfsr",
022a8946 VF	205	"zonerange": 4096*1024,
022a8946 VF	206	"zonesize": 4096*1024,
6d5af490	207	"bs": 4096,
022a8946 VF	208	"offset": 840961024,
	209	"size": 1640961024,
	210	"io_size": 1640961024,
6d5af490 VF	211	},
	212	{
	213	"random_generator": "lfsr",
022a8946 VF	214	"zonerange": 4096*1024,
022a8946 VF	215	"zonesize": 4096*1024,
6d5af490	216	"bs": 4096,
022a8946 VF	217	"size": 1640961024,
022a8946 VF	218	"io_size": 1640961024,
6d5af490 VF	219	},
	220	{
	221	"random_generator": "lfsr",
	222	"zonerange": 1610241024,
	223	"zonesize": 1610241024,
	224	"bs": 4096,
	225	"size": 25610241024,
	226	"io_size": 2561024204,
	227	},
	228	{
	229	"random_generator": "lfsr",
022a8946 VF	230	"zonerange": 4096*1024,
022a8946 VF	231	"zonesize": 440961024,
6d5af490	232	"bs": 4096,
022a8946 VF	233	"size": 1640961024,
022a8946 VF	234	"io_size": 1640961024,
6d5af490 VF	235	},
	236	{
	237	"random_generator": "lfsr",
	238	"zonerange": 1610241024,
	239	"zonesize": 3210241024,
	240	"bs": 4096,
	241	"size": 25610241024,
	242	"io_size": 2561024204,
	243	},
	244	{
	245	"random_generator": "lfsr",
022a8946 VF	246	"zonerange": 8192*1024,
022a8946 VF	247	"zonesize": 4096*1024,
6d5af490	248	"bs": 4096,
022a8946 VF	249	"size": 1640961024,
022a8946 VF	250	"io_size": 1640961024,
6d5af490 VF	251	},
	252	{
	253	"random_generator": "lfsr",
	254	"zonerange": 1610241024,
	255	"zonesize": 810241024,
	256	"bs": 4096,
	257	"size": 25610241024,
	258	"io_size": 2561024204,
	259	},
	260	# norandommap
	261	{
	262	"norandommap": 1,
	263	"zonerange": 4096,
	264	"zonesize": 4096,
	265	"bs": 4096,
	266	"offset": 8*4096,
	267	"size": 16*4096,
	268	"io_size": 16*4096,
	269	},
	270	{
	271	"norandommap": 1,
	272	"zonerange": 4096,
	273	"zonesize": 4096,
	274	"bs": 4096,
	275	"size": 16*4096,
	276	"io_size": 16*4096,
	277	},
	278	{
	279	"norandommap": 1,
	280	"zonerange": 1610241024,
	281	"zonesize": 1610241024,
	282	"bs": 4096,
	283	"size": 25610241024,
	284	"io_size": 2561024204,
	285	},
	286	{
	287	"norandommap": 1,
	288	"zonerange": 4096,
	289	"zonesize": 8192,
	290	"bs": 4096,
	291	"size": 16*4096,
	292	"io_size": 16*4096,
	293	},
	294	{
	295	"norandommap": 1,
	296	"zonerange": 1610241024,
	297	"zonesize": 3210241024,
	298	"bs": 4096,
	299	"size": 25610241024,
	300	"io_size": 2561024204,
	301	},
	302	{
	303	"norandommap": 1,
	304	"zonerange": 8192,
	305	"zonesize": 4096,
	306	"bs": 4096,
	307	"size": 16*4096,
	308	"io_size": 16*4096,
	309	},
	310	{
	311	"norandommap": 1,
	312	"zonerange": 1610241024,
	313	"zonesize": 810241024,
	314	"bs": 4096,
315	"size": 25610241024,
022a8946	316	"io_size": 25610241024,
6d5af490 VF	317	},
	318
	319	]
	320
	321	index = 1
	322	passed = 0
	323	failed = 0
	324
	325	if args.filename:
	326	statinfo = os.stat(args.filename)
	327	filesize = statinfo.st_size
	328	if filesize == 0:
	329	f = os.open(args.filename, os.O_RDONLY)
	330	filesize = os.lseek(f, 0, os.SEEK_END)
	331	os.close(f)
	332
	333	for test in tests:
	334	if args.filename:
	335	test['filename'] = args.filename
	336	test['filesize'] = filesize
	337	else:
	338	test['filesize'] = test['size']
	339	iops_log = run_fio(args.fio, test, index)
	340	status = check_output(iops_log, test)
	341	print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
	342	if status:
	343	passed = passed + 1
	344	else:
	345	failed = failed + 1
	346	index = index + 1
	347
	348	print("{0} tests passed, {1} failed".format(passed, failed))
	349
	350	sys.exit(failed)