t/strided.py

   1 #!/usr/bin/env python3
   2 #
   3 # strided.py
   4 #
   5 # Test zonemode=strided. This uses the null ioengine when no file is
   6 # specified. If a file is specified, use it for randdom read testing.
   7 # Some of the zoneranges in the tests are 16MiB. So when using a file
   8 # a minimum size of 32MiB is recommended.
   9 #
  10 # USAGE
  11 # python strided.py fio-executable [-f file/device]
  12 #
  13 # EXAMPLES
  14 # python t/strided.py ./fio
  15 # python t/strided.py ./fio -f /dev/sda
  16 # dd if=/dev/zero of=temp bs=1M count=32
  17 # python t/strided.py ./fio -f temp
  18 #
  19 # REQUIREMENTS
  20 # Python 2.6+
  21 #
  22 # ===TEST MATRIX===
  23 #
  24 # --zonemode=strided, zoneskip unset
  25 #   w/ randommap and LFSR
  26 #       zonesize=zonerange  all blocks in zonerange touched
  27 #       zonesize>zonerange  all blocks touched and roll-over back into zone
  28 #       zonesize<zonerange  all blocks inside zone
  29 #
  30 #   w/o randommap       all blocks inside zone
  31 #
  32
  33 from __future__ import absolute_import
  34 from __future__ import print_function
  35 import os
  36 import sys
  37 import argparse
  38 import subprocess
  39
  40
  41 def parse_args():
  42     parser = argparse.ArgumentParser()
  43     parser.add_argument('fio',
  44                         help='path to fio executable (e.g., ./fio)')
  45     parser.add_argument('-f', '--filename', help="file/device to test")
  46     args = parser.parse_args()
  47
  48     return args
  49
  50
  51 def run_fio(fio, test, index):
  52     filename = "strided"
  53     fio_args = [
  54                 "--max-jobs=16",
  55                 "--name=strided",
  56                 "--zonemode=strided",
  57                 "--log_offset=1",
  58                 "--randrepeat=0",
  59                 "--rw=randread",
  60                 "--write_iops_log={0}{1:03d}".format(filename, index),
  61                 "--output={0}{1:03d}.out".format(filename, index),
  62                 "--zonerange={zonerange}".format(**test),
  63                 "--zonesize={zonesize}".format(**test),
  64                 "--bs={bs}".format(**test),
  65                ]
  66     if 'norandommap' in test:
  67         fio_args.append('--norandommap')
  68     if 'random_generator' in test:
  69         fio_args.append('--random_generator={random_generator}'.format(**test))
  70     if 'offset' in test:
  71         fio_args.append('--offset={offset}'.format(**test))
  72     if 'filename' in test:
  73         fio_args.append('--filename={filename}'.format(**test))
  74         fio_args.append('--filesize={filesize})'.format(**test))
  75     else:
  76         fio_args.append('--ioengine=null')
  77         fio_args.append('--size={size}'.format(**test))
  78         fio_args.append('--io_size={io_size}'.format(**test))
  79         fio_args.append('--filesize={size})'.format(**test))
  80
  81     output = subprocess.check_output([fio] + fio_args, universal_newlines=True)
  82
  83     f = open("{0}{1:03d}_iops.1.log".format(filename, index), "r")
  84     log = f.read()
  85     f.close()
  86
  87     return log
  88
  89
  90 def check_output(iops_log, test):
  91     zonestart = 0 if 'offset' not in test else test['offset']
  92     iospersize = test['zonesize'] / test['bs']
  93     iosperrange = test['zonerange'] / test['bs']
  94     iosperzone = 0
  95     lines = iops_log.split('\n')
  96     zoneset = set()
  97
  98     for line in lines:
  99         if len(line) == 0:
 100             continue
 101
 102         if iosperzone == iospersize:
 103             # time to move to a new zone
 104             iosperzone = 0
 105             zoneset = set()
 106             zonestart += test['zonerange']
 107             if zonestart >= test['filesize']:
 108                 zonestart = 0 if 'offset' not in test else test['offset']
 109
 110         iosperzone = iosperzone + 1
 111         tokens = line.split(',')
 112         offset = int(tokens[4])
 113         if offset < zonestart or offset >= zonestart + test['zonerange']:
 114             print("Offset {0} outside of zone starting at {1}".format(
 115                     offset, zonestart))
 116             return False
 117
 118         # skip next section if norandommap is enabled with no
 119         # random_generator or with a random_generator != lfsr
 120         if 'norandommap' in test:
 121             if 'random_generator' in test:
 122                 if test['random_generator'] != 'lfsr':
 123                     continue
 124             else:
 125                 continue
 126
 127         # we either have a random map enabled or we
 128         # are using an LFSR
 129         # so all blocks should be unique and we should have
 130         # covered the entire zone when iosperzone % iosperrange == 0
 131         block = (offset - zonestart) / test['bs']
 132         if block in zoneset:
 133             print("Offset {0} in zone already touched".format(offset))
 134             return False
 135
 136         zoneset.add(block)
 137         if iosperzone % iosperrange == 0:
 138             if len(zoneset) != iosperrange:
 139                 print("Expected {0} blocks in zone but only saw {1}".format(
 140                         iosperrange, len(zoneset)))
 141                 return False
 142             zoneset = set()
 143
 144     return True
 145
 146
 147 if __name__ == '__main__':
 148     args = parse_args()
 149
 150     tests = [   # randommap enabled
 151                 {
 152                     "zonerange": 4096,
 153                     "zonesize": 4096,
 154                     "bs": 4096,
 155                     "offset": 8*4096,
 156                     "size": 16*4096,
 157                     "io_size": 16*4096,
 158                 },
 159                 {
 160                     "zonerange": 4096,
 161                     "zonesize": 4096,
 162                     "bs": 4096,
 163                     "size": 16*4096,
 164                     "io_size": 16*4096,
 165                 },
 166                 {
 167                     "zonerange": 16*1024*1024,
 168                     "zonesize": 16*1024*1024,
 169                     "bs": 4096,
 170                     "size": 256*1024*1024,
 171                     "io_size": 256*1024*204,
 172                 },
 173                 {
 174                     "zonerange": 4096,
 175                     "zonesize": 4*4096,
 176                     "bs": 4096,
 177                     "size": 16*4096,
 178                     "io_size": 16*4096,
 179                 },
 180                 {
 181                     "zonerange": 16*1024*1024,
 182                     "zonesize": 32*1024*1024,
 183                     "bs": 4096,
 184                     "size": 256*1024*1024,
 185                     "io_size": 256*1024*204,
 186                 },
 187                 {
 188                     "zonerange": 8192,
 189                     "zonesize": 4096,
 190                     "bs": 4096,
 191                     "size": 16*4096,
 192                     "io_size": 16*4096,
 193                 },
 194                 {
 195                     "zonerange": 16*1024*1024,
 196                     "zonesize": 8*1024*1024,
 197                     "bs": 4096,
 198                     "size": 256*1024*1024,
 199                     "io_size": 256*1024*204,
 200                 },
 201                 # lfsr
 202                 {
 203                     "random_generator": "lfsr",
 204                     "zonerange": 4096*1024,
 205                     "zonesize": 4096*1024,
 206                     "bs": 4096,
 207                     "offset": 8*4096*1024,
 208                     "size": 16*4096*1024,
 209                     "io_size": 16*4096*1024,
 210                 },
 211                 {
 212                     "random_generator": "lfsr",
 213                     "zonerange": 4096*1024,
 214                     "zonesize": 4096*1024,
 215                     "bs": 4096,
 216                     "size": 16*4096*1024,
 217                     "io_size": 16*4096*1024,
 218                 },
 219                 {
 220                     "random_generator": "lfsr",
 221                     "zonerange": 16*1024*1024,
 222                     "zonesize": 16*1024*1024,
 223                     "bs": 4096,
 224                     "size": 256*1024*1024,
 225                     "io_size": 256*1024*204,
 226                 },
 227                 {
 228                     "random_generator": "lfsr",
 229                     "zonerange": 4096*1024,
 230                     "zonesize": 4*4096*1024,
 231                     "bs": 4096,
 232                     "size": 16*4096*1024,
 233                     "io_size": 16*4096*1024,
 234                 },
 235                 {
 236                     "random_generator": "lfsr",
 237                     "zonerange": 16*1024*1024,
 238                     "zonesize": 32*1024*1024,
 239                     "bs": 4096,
 240                     "size": 256*1024*1024,
 241                     "io_size": 256*1024*204,
 242                 },
 243                 {
 244                     "random_generator": "lfsr",
 245                     "zonerange": 8192*1024,
 246                     "zonesize": 4096*1024,
 247                     "bs": 4096,
 248                     "size": 16*4096*1024,
 249                     "io_size": 16*4096*1024,
 250                 },
 251                 {
 252                     "random_generator": "lfsr",
 253                     "zonerange": 16*1024*1024,
 254                     "zonesize": 8*1024*1024,
 255                     "bs": 4096,
 256                     "size": 256*1024*1024,
 257                     "io_size": 256*1024*204,
 258                 },
 259                 # norandommap
 260                 {
 261                     "norandommap": 1,
 262                     "zonerange": 4096,
 263                     "zonesize": 4096,
 264                     "bs": 4096,
 265                     "offset": 8*4096,
 266                     "size": 16*4096,
 267                     "io_size": 16*4096,
 268                 },
 269                 {
 270                     "norandommap": 1,
 271                     "zonerange": 4096,
 272                     "zonesize": 4096,
 273                     "bs": 4096,
 274                     "size": 16*4096,
 275                     "io_size": 16*4096,
 276                 },
 277                 {
 278                     "norandommap": 1,
 279                     "zonerange": 16*1024*1024,
 280                     "zonesize": 16*1024*1024,
 281                     "bs": 4096,
 282                     "size": 256*1024*1024,
 283                     "io_size": 256*1024*204,
 284                 },
 285                 {
 286                     "norandommap": 1,
 287                     "zonerange": 4096,
 288                     "zonesize": 8192,
 289                     "bs": 4096,
 290                     "size": 16*4096,
 291                     "io_size": 16*4096,
 292                 },
 293                 {
 294                     "norandommap": 1,
 295                     "zonerange": 16*1024*1024,
 296                     "zonesize": 32*1024*1024,
 297                     "bs": 4096,
 298                     "size": 256*1024*1024,
 299                     "io_size": 256*1024*204,
 300                 },
 301                 {
 302                     "norandommap": 1,
 303                     "zonerange": 8192,
 304                     "zonesize": 4096,
 305                     "bs": 4096,
 306                     "size": 16*4096,
 307                     "io_size": 16*4096,
 308                 },
 309                 {
 310                     "norandommap": 1,
 311                     "zonerange": 16*1024*1024,
 312                     "zonesize": 8*1024*1024,
 313                     "bs": 4096,
 314                     "size": 256*1024*1024,
 315                     "io_size": 256*1024*1024,
 316                 },
 317
 318             ]
 319
 320     index = 1
 321     passed = 0
 322     failed = 0
 323
 324     if args.filename:
 325         statinfo = os.stat(args.filename)
 326         filesize = statinfo.st_size
 327         if filesize == 0:
 328             f = os.open(args.filename, os.O_RDONLY)
 329             filesize = os.lseek(f, 0, os.SEEK_END)
 330             os.close(f)
 331
 332     for test in tests:
 333         if args.filename:
 334             test['filename'] = args.filename
 335             test['filesize'] = filesize
 336         else:
 337             test['filesize'] = test['size']
 338         iops_log = run_fio(args.fio, test, index)
 339         status = check_output(iops_log, test)
 340         print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
 341         if status:
 342             passed = passed + 1
 343         else:
 344             failed = failed + 1
 345         index = index + 1
 346
 347     print("{0} tests passed, {1} failed".format(passed, failed))
 348
 349     sys.exit(failed)