t/random_seed.py

   1 #!/usr/bin/env python3
   2 """
   3 # random_seed.py
   4 #
   5 # Test fio's random seed options.
   6 #
   7 # - make sure that randseed overrides randrepeat and allrandrepeat
   8 # - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set
   9 # - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set
  10 #
  11 # USAGE
  12 # see python3 random_seed.py --help
  13 #
  14 # EXAMPLES
  15 # python3 t/random_seed.py
  16 # python3 t/random_seed.py -f ./fio
  17 #
  18 # REQUIREMENTS
  19 # Python 3.6
  20 #
  21 """
  22 import os
  23 import sys
  24 import time
  25 import locale
  26 import argparse
  27 import subprocess
  28 from pathlib import Path
  29
  30 class FioRandTest():
  31     """fio random seed test."""
  32
  33     def __init__(self, artifact_root, test_options, debug):
  34         """
  35         artifact_root   root directory for artifacts (subdirectory will be created under here)
  36         test            test specification
  37         """
  38         self.artifact_root = artifact_root
  39         self.test_options = test_options
  40         self.debug = debug
  41         self.filename_stub = None
  42         self.filenames = {}
  43
  44         self.test_dir = os.path.abspath(os.path.join(self.artifact_root,
  45                                      f"{self.test_options['test_id']:03d}"))
  46         if not os.path.exists(self.test_dir):
  47             os.mkdir(self.test_dir)
  48
  49         self.filename_stub = f"random{self.test_options['test_id']:03d}"
  50         self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command")
  51         self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout")
  52         self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr")
  53         self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode")
  54         self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output")
  55
  56     def run_fio(self, fio_path):
  57         """Run a test."""
  58
  59         fio_args = [
  60             "--debug=random",
  61             "--name=random_seed",
  62             "--ioengine=null",
  63             "--filesize=32k",
  64             "--rw=randread",
  65             f"--output={self.filenames['output']}",
  66         ]
  67         for opt in ['randseed', 'randrepeat', 'allrandrepeat']:
  68             if opt in self.test_options:
  69                 option = f"--{opt}={self.test_options[opt]}"
  70                 fio_args.append(option)
  71
  72         command = [fio_path] + fio_args
  73         with open(self.filenames['command'], "w+", encoding=locale.getpreferredencoding()) as command_file:
  74             command_file.write(" ".join(command))
  75
  76         passed = True
  77
  78         try:
  79             with open(self.filenames['stdout'], "w+", encoding=locale.getpreferredencoding()) as stdout_file, \
  80                 open(self.filenames['stderr'], "w+", encoding=locale.getpreferredencoding()) as stderr_file, \
  81                 open(self.filenames['exitcode'], "w+", encoding=locale.getpreferredencoding()) as exitcode_file:
  82                 proc = None
  83                 # Avoid using subprocess.run() here because when a timeout occurs,
  84                 # fio will be stopped with SIGKILL. This does not give fio a
  85                 # chance to clean up and means that child processes may continue
  86                 # running and submitting IO.
  87                 proc = subprocess.Popen(command,
  88                                         stdout=stdout_file,
  89                                         stderr=stderr_file,
  90                                         cwd=self.test_dir,
  91                                         universal_newlines=True)
  92                 proc.communicate(timeout=300)
  93                 exitcode_file.write(f'{proc.returncode}\n')
  94                 passed &= (proc.returncode == 0)
  95         except subprocess.TimeoutExpired:
  96             proc.terminate()
  97             proc.communicate()
  98             assert proc.poll()
  99             print("Timeout expired")
 100             passed = False
 101         except Exception:
 102             if proc:
 103                 if not proc.poll():
 104                     proc.terminate()
 105                     proc.communicate()
 106             print(f"Exception: {sys.exc_info()}")
 107             passed = False
 108
 109         return passed
 110
 111     def get_rand_seeds(self):
 112         """Collect random seeds from --debug=random output."""
 113         with open(self.filenames['output'], "r", encoding=locale.getpreferredencoding()) as out_file:
 114             file_data = out_file.read()
 115
 116             offsets = 0
 117             for line in file_data.split('\n'):
 118                 if 'random' in line and 'FIO_RAND_NR_OFFS=' in line:
 119                     tokens = line.split('=')
 120                     offsets = int(tokens[len(tokens)-1])
 121                     break
 122
 123             if offsets == 0:
 124                 pass
 125                 # find an exception to throw
 126
 127             seed_list = []
 128             for line in file_data.split('\n'):
 129                 if 'random' not in line:
 130                     continue
 131                 if 'rand_seeds[' in line:
 132                     tokens = line.split('=')
 133                     seed = int(tokens[-1])
 134                     seed_list.append(seed)
 135                     # assume that seeds are in order
 136
 137             return seed_list
 138
 139     def check(self):
 140         """Check test output."""
 141
 142         raise NotImplementedError()
 143
 144
 145 class TestRR(FioRandTest):
 146     """
 147     Test object for [all]randrepeat. If run for the first time just collect the
 148     seeds. For later runs make sure the seeds match or do not match those
 149     previously collected.
 150     """
 151     # one set of seeds is for randrepeat=0 and the other is for randrepeat=1
 152     seeds = { 0: None, 1: None }
 153
 154     def check(self):
 155         """Check output for allrandrepeat=1."""
 156
 157         retval = True
 158         opt = 'randrepeat' if 'randrepeat' in self.test_options else 'allrandrepeat'
 159         rr = self.test_options[opt]
 160         rand_seeds = self.get_rand_seeds()
 161
 162         if not TestRR.seeds[rr]:
 163             TestRR.seeds[rr] = rand_seeds
 164             if self.debug:
 165                 print(f"TestRR: saving rand_seeds for [a]rr={rr}")
 166         else:
 167             if rr:
 168                 if TestRR.seeds[1] != rand_seeds:
 169                     retval = False
 170                     print(f"TestRR: unexpected seed mismatch for [a]rr={rr}")
 171                 else:
 172                     if self.debug:
 173                         print(f"TestRR: seeds correctly match for [a]rr={rr}")
 174                 if TestRR.seeds[0] == rand_seeds:
 175                     retval = False
 176                     print("TestRR: seeds unexpectedly match those from system RNG")
 177             else:
 178                 if TestRR.seeds[0] == rand_seeds:
 179                     retval = False
 180                     print(f"TestRR: unexpected seed match for [a]rr={rr}")
 181                 else:
 182                     if self.debug:
 183                         print(f"TestRR: seeds correctly don't match for [a]rr={rr}")
 184                 if TestRR.seeds[1] == rand_seeds:
 185                     retval = False
 186                     print(f"TestRR: random seeds unexpectedly match those from [a]rr=1")
 187
 188         return retval
 189
 190
 191 class TestRS(FioRandTest):
 192     """
 193     Test object when randseed=something controls the generated seeds. If run
 194     for the first time for a given randseed just collect the seeds. For later
 195     runs with the same seed make sure the seeds are the same as those
 196     previously collected.
 197     """
 198     seeds = {}
 199
 200     def check(self):
 201         """Check output for randseed=something."""
 202
 203         retval = True
 204         rand_seeds = self.get_rand_seeds()
 205         randseed = self.test_options['randseed']
 206
 207         if self.debug:
 208             print("randseed = ", randseed)
 209
 210         if randseed not in TestRS.seeds:
 211             TestRS.seeds[randseed] = rand_seeds
 212             if self.debug:
 213                 print("TestRS: saving rand_seeds")
 214         else:
 215             if TestRS.seeds[randseed] != rand_seeds:
 216                 retval = False
 217                 print("TestRS: seeds don't match when they should")
 218             else:
 219                 if self.debug:
 220                     print("TestRS: seeds correctly match")
 221
 222         # Now try to find seeds generated using a different randseed and make
 223         # sure they *don't* match
 224         for key in TestRS.seeds:
 225             if key != randseed:
 226                 if TestRS.seeds[key] == rand_seeds:
 227                     retval = False
 228                     print("TestRS: randseeds differ but generated seeds match.")
 229                 else:
 230                     if self.debug:
 231                         print("TestRS: randseeds differ and generated seeds also differ.")
 232
 233         return retval
 234
 235
 236 def parse_args():
 237     """Parse command-line arguments."""
 238
 239     parser = argparse.ArgumentParser()
 240     parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
 241     parser.add_argument('-a', '--artifact-root', help='artifact root directory')
 242     parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
 243     parser.add_argument('-s', '--skip', nargs='+', type=int,
 244                         help='list of test(s) to skip')
 245     parser.add_argument('-o', '--run-only', nargs='+', type=int,
 246                         help='list of test(s) to run, skipping all others')
 247     args = parser.parse_args()
 248
 249     return args
 250
 251
 252 def main():
 253     """Run tests of fio random seed options"""
 254
 255     args = parse_args()
 256
 257     artifact_root = args.artifact_root if args.artifact_root else \
 258         f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}"
 259     os.mkdir(artifact_root)
 260     print(f"Artifact directory is {artifact_root}")
 261
 262     if args.fio:
 263         fio = str(Path(args.fio).absolute())
 264     else:
 265         fio = 'fio'
 266     print(f"fio path is {fio}")
 267
 268     test_list = [
 269         {
 270             "test_id": 1,
 271             "randrepeat": 0,
 272             "test_obj": TestRR,
 273         },
 274         {
 275             "test_id": 2,
 276             "randrepeat": 0,
 277             "test_obj": TestRR,
 278         },
 279         {
 280             "test_id": 3,
 281             "randrepeat": 1,
 282             "test_obj": TestRR,
 283         },
 284         {
 285             "test_id": 4,
 286             "randrepeat": 1,
 287             "test_obj": TestRR,
 288         },
 289         {
 290             "test_id": 5,
 291             "allrandrepeat": 0,
 292             "test_obj": TestRR,
 293         },
 294         {
 295             "test_id": 6,
 296             "allrandrepeat": 0,
 297             "test_obj": TestRR,
 298         },
 299         {
 300             "test_id": 7,
 301             "allrandrepeat": 1,
 302             "test_obj": TestRR,
 303         },
 304         {
 305             "test_id": 8,
 306             "allrandrepeat": 1,
 307             "test_obj": TestRR,
 308         },
 309         {
 310             "test_id": 9,
 311             "randrepeat": 0,
 312             "randseed": "12345",
 313             "test_obj": TestRS,
 314         },
 315         {
 316             "test_id": 10,
 317             "randrepeat": 0,
 318             "randseed": "12345",
 319             "test_obj": TestRS,
 320         },
 321         {
 322             "test_id": 11,
 323             "randrepeat": 1,
 324             "randseed": "12345",
 325             "test_obj": TestRS,
 326         },
 327         {
 328             "test_id": 12,
 329             "allrandrepeat": 0,
 330             "randseed": "12345",
 331             "test_obj": TestRS,
 332         },
 333         {
 334             "test_id": 13,
 335             "allrandrepeat": 1,
 336             "randseed": "12345",
 337             "test_obj": TestRS,
 338         },
 339         {
 340             "test_id": 14,
 341             "randrepeat": 0,
 342             "randseed": "67890",
 343             "test_obj": TestRS,
 344         },
 345         {
 346             "test_id": 15,
 347             "randrepeat": 1,
 348             "randseed": "67890",
 349             "test_obj": TestRS,
 350         },
 351         {
 352             "test_id": 16,
 353             "allrandrepeat": 0,
 354             "randseed": "67890",
 355             "test_obj": TestRS,
 356         },
 357         {
 358             "test_id": 17,
 359             "allrandrepeat": 1,
 360             "randseed": "67890",
 361             "test_obj": TestRS,
 362         },
 363     ]
 364
 365     passed = 0
 366     failed = 0
 367     skipped = 0
 368
 369     for test in test_list:
 370         if (args.skip and test['test_id'] in args.skip) or \
 371            (args.run_only and test['test_id'] not in args.run_only):
 372             skipped = skipped + 1
 373             outcome = 'SKIPPED (User request)'
 374         else:
 375             test_obj = test['test_obj'](artifact_root, test, args.debug)
 376             status = test_obj.run_fio(fio)
 377             if status:
 378                 status = test_obj.check()
 379             if status:
 380                 passed = passed + 1
 381                 outcome = 'PASSED'
 382             else:
 383                 failed = failed + 1
 384                 outcome = 'FAILED'
 385
 386         print(f"**********Test {test['test_id']} {outcome}**********")
 387
 388     print(f"{passed} tests passed, {failed} failed, {skipped} skipped")
 389
 390     sys.exit(failed)
 391
 392
 393 if __name__ == '__main__':
 394     main()