t/strided.py

   1 #!/usr/bin/python
   2 # Note: this script is python2 and python3 compatible.
   3 #
   4 # strided.py
   5 #
   6 # Test zonemode=strided. This uses the null ioengine when no file is
   7 # specified. If a file is specified, use it for randdom read testing.
   8 # Some of the zoneranges in the tests are 16MiB. So when using a file
   9 # a minimum size of 32MiB is recommended.
  10 #
  11 # USAGE
  12 # python strided.py fio-executable [-f file/device]
  13 #
  14 # EXAMPLES
  15 # python t/strided.py ./fio
  16 # python t/strided.py ./fio -f /dev/sda
  17 # dd if=/dev/zero of=temp bs=1M count=32
  18 # python t/strided.py ./fio -f temp
  19 #
  20 # REQUIREMENTS
  21 # Python 2.6+
  22 #
  23 # ===TEST MATRIX===
  24 #
  25 # --zonemode=strided, zoneskip >= 0
  26 #   w/ randommap and LFSR
  27 #       zonesize=zonerange  all blocks in zonerange touched
  28 #       zonesize>zonerange  all blocks touched and roll-over back into zone
  29 #       zonesize<zonerange  all blocks inside zone
  30 #
  31 #   w/o randommap       all blocks inside zone
  32 #
  33
  34 from __future__ import absolute_import
  35 from __future__ import print_function
  36 import os
  37 import sys
  38 import argparse
  39 import subprocess
  40
  41
  42 def parse_args():
  43     parser = argparse.ArgumentParser()
  44     parser.add_argument('fio',
  45                         help='path to fio executable (e.g., ./fio)')
  46     parser.add_argument('-f', '--filename', help="file/device to test")
  47     args = parser.parse_args()
  48
  49     return args
  50
  51
  52 def run_fio(fio, test, index):
  53     filename = "strided"
  54     fio_args = [
  55                 "--name=strided",
  56                 "--zonemode=strided",
  57                 "--log_offset=1",
  58                 "--randrepeat=0",
  59                 "--rw=randread",
  60                 "--zoneskip=0",
  61                 "--write_iops_log={0}{1:03d}".format(filename, index),
  62                 "--output={0}{1:03d}.out".format(filename, index),
  63                 "--zonerange={zonerange}".format(**test),
  64                 "--zonesize={zonesize}".format(**test),
  65                 "--bs={bs}".format(**test),
  66                ]
  67     if 'norandommap' in test:
  68         fio_args.append('--norandommap')
  69     if 'random_generator' in test:
  70         fio_args.append('--random_generator={random_generator}'.format(**test))
  71     if 'offset' in test:
  72         fio_args.append('--offset={offset}'.format(**test))
  73     if 'filename' in test:
  74         fio_args.append('--filename={filename}'.format(**test))
  75         fio_args.append('--filesize={filesize})'.format(**test))
  76     else:
  77         fio_args.append('--ioengine=null')
  78         fio_args.append('--size={size}'.format(**test))
  79         fio_args.append('--io_size={io_size}'.format(**test))
  80         fio_args.append('--filesize={size})'.format(**test))
  81
  82     output = subprocess.check_output([fio] + fio_args, universal_newlines=True)
  83
  84     f = open("{0}{1:03d}_iops.1.log".format(filename, index), "r")
  85     log = f.read()
  86     f.close()
  87
  88     return log
  89
  90
  91 def check_output(iops_log, test):
  92     zonestart = 0 if 'offset' not in test else test['offset']
  93     iospersize = test['zonesize'] / test['bs']
  94     iosperrange = test['zonerange'] / test['bs']
  95     iosperzone = 0
  96     lines = iops_log.split('\n')
  97     zoneset = set()
  98
  99     for line in lines:
 100         if len(line) == 0:
 101             continue
 102
 103         if iosperzone == iospersize:
 104             # time to move to a new zone
 105             iosperzone = 0
 106             zoneset = set()
 107             zonestart += test['zonerange']
 108             if zonestart >= test['filesize']:
 109                 zonestart = 0 if 'offset' not in test else test['offset']
 110
 111         iosperzone = iosperzone + 1
 112         tokens = line.split(',')
 113         offset = int(tokens[4])
 114         if offset < zonestart or offset >= zonestart + test['zonerange']:
 115             print("Offset {0} outside of zone starting at {1}".format(
 116                     offset, zonestart))
 117             return False
 118
 119         # skip next section if norandommap is enabled with no
 120         # random_generator or with a random_generator != lfsr
 121         if 'norandommap' in test:
 122             if 'random_generator' in test:
 123                 if test['random_generator'] != 'lfsr':
 124                     continue
 125             else:
 126                 continue
 127
 128         # we either have a random map enabled or we
 129         # are using an LFSR
 130         # so all blocks should be unique and we should have
 131         # covered the entire zone when iosperzone % iosperrange == 0
 132         block = (offset - zonestart) / test['bs']
 133         if block in zoneset:
 134             print("Offset {0} in zone already touched".format(offset))
 135             return False
 136
 137         zoneset.add(block)
 138         if iosperzone % iosperrange == 0:
 139             if len(zoneset) != iosperrange:
 140                 print("Expected {0} blocks in zone but only saw {1}".format(
 141                         iosperrange, len(zoneset)))
 142                 return False
 143             zoneset = set()
 144
 145     return True
 146
 147
 148 if __name__ == '__main__':
 149     args = parse_args()
 150
 151     tests = [   # randommap enabled
 152                 {
 153                     "zonerange": 4096,
 154                     "zonesize": 4096,
 155                     "bs": 4096,
 156                     "offset": 8*4096,
 157                     "size": 16*4096,
 158                     "io_size": 16*4096,
 159                 },
 160                 {
 161                     "zonerange": 4096,
 162                     "zonesize": 4096,
 163                     "bs": 4096,
 164                     "size": 16*4096,
 165                     "io_size": 16*4096,
 166                 },
 167                 {
 168                     "zonerange": 16*1024*1024,
 169                     "zonesize": 16*1024*1024,
 170                     "bs": 4096,
 171                     "size": 256*1024*1024,
 172                     "io_size": 256*1024*204,
 173                 },
 174                 {
 175                     "zonerange": 4096,
 176                     "zonesize": 4*4096,
 177                     "bs": 4096,
 178                     "size": 16*4096,
 179                     "io_size": 16*4096,
 180                 },
 181                 {
 182                     "zonerange": 16*1024*1024,
 183                     "zonesize": 32*1024*1024,
 184                     "bs": 4096,
 185                     "size": 256*1024*1024,
 186                     "io_size": 256*1024*204,
 187                 },
 188                 {
 189                     "zonerange": 8192,
 190                     "zonesize": 4096,
 191                     "bs": 4096,
 192                     "size": 16*4096,
 193                     "io_size": 16*4096,
 194                 },
 195                 {
 196                     "zonerange": 16*1024*1024,
 197                     "zonesize": 8*1024*1024,
 198                     "bs": 4096,
 199                     "size": 256*1024*1024,
 200                     "io_size": 256*1024*204,
 201                 },
 202                 # lfsr
 203                 {
 204                     "random_generator": "lfsr",
 205                     "zonerange": 4096*1024,
 206                     "zonesize": 4096*1024,
 207                     "bs": 4096,
 208                     "offset": 8*4096*1024,
 209                     "size": 16*4096*1024,
 210                     "io_size": 16*4096*1024,
 211                 },
 212                 {
 213                     "random_generator": "lfsr",
 214                     "zonerange": 4096*1024,
 215                     "zonesize": 4096*1024,
 216                     "bs": 4096,
 217                     "size": 16*4096*1024,
 218                     "io_size": 16*4096*1024,
 219                 },
 220                 {
 221                     "random_generator": "lfsr",
 222                     "zonerange": 16*1024*1024,
 223                     "zonesize": 16*1024*1024,
 224                     "bs": 4096,
 225                     "size": 256*1024*1024,
 226                     "io_size": 256*1024*204,
 227                 },
 228                 {
 229                     "random_generator": "lfsr",
 230                     "zonerange": 4096*1024,
 231                     "zonesize": 4*4096*1024,
 232                     "bs": 4096,
 233                     "size": 16*4096*1024,
 234                     "io_size": 16*4096*1024,
 235                 },
 236                 {
 237                     "random_generator": "lfsr",
 238                     "zonerange": 16*1024*1024,
 239                     "zonesize": 32*1024*1024,
 240                     "bs": 4096,
 241                     "size": 256*1024*1024,
 242                     "io_size": 256*1024*204,
 243                 },
 244                 {
 245                     "random_generator": "lfsr",
 246                     "zonerange": 8192*1024,
 247                     "zonesize": 4096*1024,
 248                     "bs": 4096,
 249                     "size": 16*4096*1024,
 250                     "io_size": 16*4096*1024,
 251                 },
 252                 {
 253                     "random_generator": "lfsr",
 254                     "zonerange": 16*1024*1024,
 255                     "zonesize": 8*1024*1024,
 256                     "bs": 4096,
 257                     "size": 256*1024*1024,
 258                     "io_size": 256*1024*204,
 259                 },
 260                 # norandommap
 261                 {
 262                     "norandommap": 1,
 263                     "zonerange": 4096,
 264                     "zonesize": 4096,
 265                     "bs": 4096,
 266                     "offset": 8*4096,
 267                     "size": 16*4096,
 268                     "io_size": 16*4096,
 269                 },
 270                 {
 271                     "norandommap": 1,
 272                     "zonerange": 4096,
 273                     "zonesize": 4096,
 274                     "bs": 4096,
 275                     "size": 16*4096,
 276                     "io_size": 16*4096,
 277                 },
 278                 {
 279                     "norandommap": 1,
 280                     "zonerange": 16*1024*1024,
 281                     "zonesize": 16*1024*1024,
 282                     "bs": 4096,
 283                     "size": 256*1024*1024,
 284                     "io_size": 256*1024*204,
 285                 },
 286                 {
 287                     "norandommap": 1,
 288                     "zonerange": 4096,
 289                     "zonesize": 8192,
 290                     "bs": 4096,
 291                     "size": 16*4096,
 292                     "io_size": 16*4096,
 293                 },
 294                 {
 295                     "norandommap": 1,
 296                     "zonerange": 16*1024*1024,
 297                     "zonesize": 32*1024*1024,
 298                     "bs": 4096,
 299                     "size": 256*1024*1024,
 300                     "io_size": 256*1024*204,
 301                 },
 302                 {
 303                     "norandommap": 1,
 304                     "zonerange": 8192,
 305                     "zonesize": 4096,
 306                     "bs": 4096,
 307                     "size": 16*4096,
 308                     "io_size": 16*4096,
 309                 },
 310                 {
 311                     "norandommap": 1,
 312                     "zonerange": 16*1024*1024,
 313                     "zonesize": 8*1024*1024,
 314                     "bs": 4096,
 315                     "size": 256*1024*1024,
 316                     "io_size": 256*1024*1024,
 317                 },
 318
 319             ]
 320
 321     index = 1
 322     passed = 0
 323     failed = 0
 324
 325     if args.filename:
 326         statinfo = os.stat(args.filename)
 327         filesize = statinfo.st_size
 328         if filesize == 0:
 329             f = os.open(args.filename, os.O_RDONLY)
 330             filesize = os.lseek(f, 0, os.SEEK_END)
 331             os.close(f)
 332
 333     for test in tests:
 334         if args.filename:
 335             test['filename'] = args.filename
 336             test['filesize'] = filesize
 337         else:
 338             test['filesize'] = test['size']
 339         iops_log = run_fio(args.fio, test, index)
 340         status = check_output(iops_log, test)
 341         print("Test {0} {1}".format(index, ("PASSED" if status else "FAILED")))
 342         if status:
 343             passed = passed + 1
 344         else:
 345             failed = failed + 1
 346         index = index + 1
 347
 348     print("{0} tests passed, {1} failed".format(passed, failed))
 349
 350     sys.exit(failed)