Commit | Line | Data |
---|---|---|
c994fa62 VF |
1 | #!/usr/bin/env python3 |
2 | """ | |
3 | # random_seed.py | |
4 | # | |
5 | # Test fio's random seed options. | |
6 | # | |
7 | # - make sure that randseed overrides randrepeat and allrandrepeat | |
8 | # - make sure that seeds differ across invocations when [all]randrepeat=0 and randseed is not set | |
9 | # - make sure that seeds are always the same when [all]randrepeat=1 and randseed is not set | |
10 | # | |
11 | # USAGE | |
12 | # see python3 random_seed.py --help | |
13 | # | |
14 | # EXAMPLES | |
15 | # python3 t/random_seed.py | |
16 | # python3 t/random_seed.py -f ./fio | |
17 | # | |
18 | # REQUIREMENTS | |
19 | # Python 3.6 | |
20 | # | |
21 | """ | |
22 | import os | |
23 | import sys | |
24 | import time | |
25 | import locale | |
26 | import argparse | |
27 | import subprocess | |
28 | from pathlib import Path | |
29 | ||
30 | class FioRandTest(): | |
31 | """fio random seed test.""" | |
32 | ||
33 | def __init__(self, artifact_root, test_options, debug): | |
34 | """ | |
35 | artifact_root root directory for artifacts (subdirectory will be created under here) | |
36 | test test specification | |
37 | """ | |
38 | self.artifact_root = artifact_root | |
39 | self.test_options = test_options | |
40 | self.debug = debug | |
41 | self.filename_stub = None | |
42 | self.filenames = {} | |
43 | ||
44 | self.test_dir = os.path.abspath(os.path.join(self.artifact_root, | |
45 | f"{self.test_options['test_id']:03d}")) | |
46 | if not os.path.exists(self.test_dir): | |
47 | os.mkdir(self.test_dir) | |
48 | ||
49 | self.filename_stub = f"random{self.test_options['test_id']:03d}" | |
50 | self.filenames['command'] = os.path.join(self.test_dir, f"{self.filename_stub}.command") | |
51 | self.filenames['stdout'] = os.path.join(self.test_dir, f"{self.filename_stub}.stdout") | |
52 | self.filenames['stderr'] = os.path.join(self.test_dir, f"{self.filename_stub}.stderr") | |
53 | self.filenames['exitcode'] = os.path.join(self.test_dir, f"{self.filename_stub}.exitcode") | |
54 | self.filenames['output'] = os.path.join(self.test_dir, f"{self.filename_stub}.output") | |
55 | ||
56 | def run_fio(self, fio_path): | |
57 | """Run a test.""" | |
58 | ||
59 | fio_args = [ | |
60 | "--debug=random", | |
61 | "--name=random_seed", | |
62 | "--ioengine=null", | |
63 | "--filesize=32k", | |
64 | "--rw=randread", | |
65 | f"--output={self.filenames['output']}", | |
66 | ] | |
67 | for opt in ['randseed', 'randrepeat', 'allrandrepeat']: | |
68 | if opt in self.test_options: | |
69 | option = f"--{opt}={self.test_options[opt]}" | |
70 | fio_args.append(option) | |
71 | ||
72 | command = [fio_path] + fio_args | |
73 | with open(self.filenames['command'], "w+", encoding=locale.getpreferredencoding()) as command_file: | |
74 | command_file.write(" ".join(command)) | |
75 | ||
76 | passed = True | |
77 | ||
78 | try: | |
79 | with open(self.filenames['stdout'], "w+", encoding=locale.getpreferredencoding()) as stdout_file, \ | |
80 | open(self.filenames['stderr'], "w+", encoding=locale.getpreferredencoding()) as stderr_file, \ | |
81 | open(self.filenames['exitcode'], "w+", encoding=locale.getpreferredencoding()) as exitcode_file: | |
82 | proc = None | |
83 | # Avoid using subprocess.run() here because when a timeout occurs, | |
84 | # fio will be stopped with SIGKILL. This does not give fio a | |
85 | # chance to clean up and means that child processes may continue | |
86 | # running and submitting IO. | |
87 | proc = subprocess.Popen(command, | |
88 | stdout=stdout_file, | |
89 | stderr=stderr_file, | |
90 | cwd=self.test_dir, | |
91 | universal_newlines=True) | |
92 | proc.communicate(timeout=300) | |
93 | exitcode_file.write(f'{proc.returncode}\n') | |
94 | passed &= (proc.returncode == 0) | |
95 | except subprocess.TimeoutExpired: | |
96 | proc.terminate() | |
97 | proc.communicate() | |
98 | assert proc.poll() | |
99 | print("Timeout expired") | |
100 | passed = False | |
101 | except Exception: | |
102 | if proc: | |
103 | if not proc.poll(): | |
104 | proc.terminate() | |
105 | proc.communicate() | |
106 | print(f"Exception: {sys.exc_info()}") | |
107 | passed = False | |
108 | ||
109 | return passed | |
110 | ||
111 | def get_rand_seeds(self): | |
112 | """Collect random seeds from --debug=random output.""" | |
113 | with open(self.filenames['output'], "r", encoding=locale.getpreferredencoding()) as out_file: | |
114 | file_data = out_file.read() | |
115 | ||
116 | offsets = 0 | |
117 | for line in file_data.split('\n'): | |
118 | if 'random' in line and 'FIO_RAND_NR_OFFS=' in line: | |
119 | tokens = line.split('=') | |
120 | offsets = int(tokens[len(tokens)-1]) | |
121 | break | |
122 | ||
123 | if offsets == 0: | |
124 | pass | |
125 | # find an exception to throw | |
126 | ||
127 | seed_list = [] | |
128 | for line in file_data.split('\n'): | |
129 | if 'random' not in line: | |
130 | continue | |
131 | if 'rand_seeds[' in line: | |
132 | tokens = line.split('=') | |
133 | seed = int(tokens[-1]) | |
134 | seed_list.append(seed) | |
135 | # assume that seeds are in order | |
136 | ||
137 | return seed_list | |
138 | ||
139 | def check(self): | |
140 | """Check test output.""" | |
141 | ||
142 | raise NotImplementedError() | |
143 | ||
144 | ||
145 | class TestRR(FioRandTest): | |
146 | """ | |
147 | Test object for [all]randrepeat. If run for the first time just collect the | |
148 | seeds. For later runs make sure the seeds match or do not match those | |
149 | previously collected. | |
150 | """ | |
151 | # one set of seeds is for randrepeat=0 and the other is for randrepeat=1 | |
152 | seeds = { 0: None, 1: None } | |
153 | ||
154 | def check(self): | |
155 | """Check output for allrandrepeat=1.""" | |
156 | ||
157 | retval = True | |
158 | opt = 'randrepeat' if 'randrepeat' in self.test_options else 'allrandrepeat' | |
159 | rr = self.test_options[opt] | |
160 | rand_seeds = self.get_rand_seeds() | |
161 | ||
162 | if not TestRR.seeds[rr]: | |
163 | TestRR.seeds[rr] = rand_seeds | |
164 | if self.debug: | |
165 | print(f"TestRR: saving rand_seeds for [a]rr={rr}") | |
166 | else: | |
167 | if rr: | |
168 | if TestRR.seeds[1] != rand_seeds: | |
169 | retval = False | |
170 | print(f"TestRR: unexpected seed mismatch for [a]rr={rr}") | |
171 | else: | |
172 | if self.debug: | |
173 | print(f"TestRR: seeds correctly match for [a]rr={rr}") | |
174 | if TestRR.seeds[0] == rand_seeds: | |
175 | retval = False | |
176 | print("TestRR: seeds unexpectedly match those from system RNG") | |
177 | else: | |
178 | if TestRR.seeds[0] == rand_seeds: | |
179 | retval = False | |
180 | print(f"TestRR: unexpected seed match for [a]rr={rr}") | |
181 | else: | |
182 | if self.debug: | |
183 | print(f"TestRR: seeds correctly don't match for [a]rr={rr}") | |
184 | if TestRR.seeds[1] == rand_seeds: | |
185 | retval = False | |
186 | print(f"TestRR: random seeds unexpectedly match those from [a]rr=1") | |
187 | ||
188 | return retval | |
189 | ||
190 | ||
191 | class TestRS(FioRandTest): | |
192 | """ | |
193 | Test object when randseed=something controls the generated seeds. If run | |
194 | for the first time for a given randseed just collect the seeds. For later | |
195 | runs with the same seed make sure the seeds are the same as those | |
196 | previously collected. | |
197 | """ | |
198 | seeds = {} | |
199 | ||
200 | def check(self): | |
201 | """Check output for randseed=something.""" | |
202 | ||
203 | retval = True | |
204 | rand_seeds = self.get_rand_seeds() | |
205 | randseed = self.test_options['randseed'] | |
206 | ||
207 | if self.debug: | |
208 | print("randseed = ", randseed) | |
209 | ||
210 | if randseed not in TestRS.seeds: | |
211 | TestRS.seeds[randseed] = rand_seeds | |
212 | if self.debug: | |
213 | print("TestRS: saving rand_seeds") | |
214 | else: | |
215 | if TestRS.seeds[randseed] != rand_seeds: | |
216 | retval = False | |
217 | print("TestRS: seeds don't match when they should") | |
218 | else: | |
219 | if self.debug: | |
220 | print("TestRS: seeds correctly match") | |
221 | ||
222 | # Now try to find seeds generated using a different randseed and make | |
223 | # sure they *don't* match | |
224 | for key in TestRS.seeds: | |
225 | if key != randseed: | |
226 | if TestRS.seeds[key] == rand_seeds: | |
227 | retval = False | |
228 | print("TestRS: randseeds differ but generated seeds match.") | |
229 | else: | |
230 | if self.debug: | |
231 | print("TestRS: randseeds differ and generated seeds also differ.") | |
232 | ||
233 | return retval | |
234 | ||
235 | ||
236 | def parse_args(): | |
237 | """Parse command-line arguments.""" | |
238 | ||
239 | parser = argparse.ArgumentParser() | |
240 | parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)') | |
241 | parser.add_argument('-a', '--artifact-root', help='artifact root directory') | |
242 | parser.add_argument('-d', '--debug', help='enable debug output', action='store_true') | |
243 | parser.add_argument('-s', '--skip', nargs='+', type=int, | |
244 | help='list of test(s) to skip') | |
245 | parser.add_argument('-o', '--run-only', nargs='+', type=int, | |
246 | help='list of test(s) to run, skipping all others') | |
247 | args = parser.parse_args() | |
248 | ||
249 | return args | |
250 | ||
251 | ||
252 | def main(): | |
253 | """Run tests of fio random seed options""" | |
254 | ||
255 | args = parse_args() | |
256 | ||
257 | artifact_root = args.artifact_root if args.artifact_root else \ | |
258 | f"random-seed-test-{time.strftime('%Y%m%d-%H%M%S')}" | |
259 | os.mkdir(artifact_root) | |
260 | print(f"Artifact directory is {artifact_root}") | |
261 | ||
262 | if args.fio: | |
263 | fio = str(Path(args.fio).absolute()) | |
264 | else: | |
265 | fio = 'fio' | |
266 | print(f"fio path is {fio}") | |
267 | ||
268 | test_list = [ | |
269 | { | |
270 | "test_id": 1, | |
271 | "randrepeat": 0, | |
272 | "test_obj": TestRR, | |
273 | }, | |
274 | { | |
275 | "test_id": 2, | |
276 | "randrepeat": 0, | |
277 | "test_obj": TestRR, | |
278 | }, | |
279 | { | |
280 | "test_id": 3, | |
281 | "randrepeat": 1, | |
282 | "test_obj": TestRR, | |
283 | }, | |
284 | { | |
285 | "test_id": 4, | |
286 | "randrepeat": 1, | |
287 | "test_obj": TestRR, | |
288 | }, | |
289 | { | |
290 | "test_id": 5, | |
291 | "allrandrepeat": 0, | |
292 | "test_obj": TestRR, | |
293 | }, | |
294 | { | |
295 | "test_id": 6, | |
296 | "allrandrepeat": 0, | |
297 | "test_obj": TestRR, | |
298 | }, | |
299 | { | |
300 | "test_id": 7, | |
301 | "allrandrepeat": 1, | |
302 | "test_obj": TestRR, | |
303 | }, | |
304 | { | |
305 | "test_id": 8, | |
306 | "allrandrepeat": 1, | |
307 | "test_obj": TestRR, | |
308 | }, | |
309 | { | |
310 | "test_id": 9, | |
311 | "randrepeat": 0, | |
312 | "randseed": "12345", | |
313 | "test_obj": TestRS, | |
314 | }, | |
315 | { | |
316 | "test_id": 10, | |
317 | "randrepeat": 0, | |
318 | "randseed": "12345", | |
319 | "test_obj": TestRS, | |
320 | }, | |
321 | { | |
322 | "test_id": 11, | |
323 | "randrepeat": 1, | |
324 | "randseed": "12345", | |
325 | "test_obj": TestRS, | |
326 | }, | |
327 | { | |
328 | "test_id": 12, | |
329 | "allrandrepeat": 0, | |
330 | "randseed": "12345", | |
331 | "test_obj": TestRS, | |
332 | }, | |
333 | { | |
334 | "test_id": 13, | |
335 | "allrandrepeat": 1, | |
336 | "randseed": "12345", | |
337 | "test_obj": TestRS, | |
338 | }, | |
339 | { | |
340 | "test_id": 14, | |
341 | "randrepeat": 0, | |
342 | "randseed": "67890", | |
343 | "test_obj": TestRS, | |
344 | }, | |
345 | { | |
346 | "test_id": 15, | |
347 | "randrepeat": 1, | |
348 | "randseed": "67890", | |
349 | "test_obj": TestRS, | |
350 | }, | |
351 | { | |
352 | "test_id": 16, | |
353 | "allrandrepeat": 0, | |
354 | "randseed": "67890", | |
355 | "test_obj": TestRS, | |
356 | }, | |
357 | { | |
358 | "test_id": 17, | |
359 | "allrandrepeat": 1, | |
360 | "randseed": "67890", | |
361 | "test_obj": TestRS, | |
362 | }, | |
363 | ] | |
364 | ||
365 | passed = 0 | |
366 | failed = 0 | |
367 | skipped = 0 | |
368 | ||
369 | for test in test_list: | |
370 | if (args.skip and test['test_id'] in args.skip) or \ | |
371 | (args.run_only and test['test_id'] not in args.run_only): | |
372 | skipped = skipped + 1 | |
373 | outcome = 'SKIPPED (User request)' | |
374 | else: | |
375 | test_obj = test['test_obj'](artifact_root, test, args.debug) | |
376 | status = test_obj.run_fio(fio) | |
377 | if status: | |
378 | status = test_obj.check() | |
379 | if status: | |
380 | passed = passed + 1 | |
381 | outcome = 'PASSED' | |
382 | else: | |
383 | failed = failed + 1 | |
384 | outcome = 'FAILED' | |
385 | ||
386 | print(f"**********Test {test['test_id']} {outcome}**********") | |
387 | ||
388 | print(f"{passed} tests passed, {failed} failed, {skipped} skipped") | |
389 | ||
390 | sys.exit(failed) | |
391 | ||
392 | ||
393 | if __name__ == '__main__': | |
394 | main() |