projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'testing' of https://github.com/vincentkfu/fio
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
17# # git clone [fio-repository]
18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
24# - Python 3
25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
42# TODO automatically detect dependencies and skip tests accordingly
43#
44
45import os
46import sys
47import json
48import time
49import logging
50import argparse
51import subprocess
52from pathlib import Path
53
54
55class FioTest(object):
56 """Base for all fio tests."""
57
58 def __init__(self, exe_path, parameters, success):
59 self.exe_path = exe_path
60 self.parameters = parameters
61 self.success = success
62 self.output = {}
63 self.artifact_root = None
64 self.testnum = None
65 self.test_dir = None
66 self.passed = True
67 self.failure_reason = ''
68
69 def setup(self, artifact_root, testnum):
70 self.artifact_root = artifact_root
71 self.testnum = testnum
72 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
73 if not os.path.exists(self.test_dir):
74 os.mkdir(self.test_dir)
75
76 self.command_file = os.path.join(
77 self.test_dir,
78 "{0}.command".format(os.path.basename(self.exe_path)))
79 self.stdout_file = os.path.join(
80 self.test_dir,
81 "{0}.stdout".format(os.path.basename(self.exe_path)))
82 self.stderr_file = os.path.join(
83 self.test_dir,
84 "{0}.stderr".format(os.path.basename(self.exe_path)))
85 self.exticode_file = os.path.join(
86 self.test_dir,
87 "{0}.exitcode".format(os.path.basename(self.exe_path)))
88
89 def run(self):
90 raise NotImplementedError()
91
92 def check_result(self):
93 raise NotImplementedError()
94
95
96class FioExeTest(FioTest):
97 """Test consists of an executable binary or script"""
98
99 def __init__(self, exe_path, parameters, success):
100 """Construct a FioExeTest which is a FioTest consisting of an
101 executable binary or script.
102
103 exe_path: location of executable binary or script
104 parameters: list of parameters for executable
105 success: Definition of test success
106 """
107
108 FioTest.__init__(self, exe_path, parameters, success)
109
110 def setup(self, artifact_root, testnum):
111 super(FioExeTest, self).setup(artifact_root, testnum)
112
113 def run(self):
114 if self.parameters:
115 command = [self.exe_path] + self.parameters
116 else:
117 command = [self.exe_path]
118 command_file = open(self.command_file, "w+")
119 command_file.write("%s\n" % command)
120 command_file.close()
121
122 stdout_file = open(self.stdout_file, "w+")
123 stderr_file = open(self.stderr_file, "w+")
124 exticode_file = open(self.exticode_file, "w+")
125 try:
126 # Avoid using subprocess.run() here because when a timeout occurs,
127 # fio will be stopped with SIGKILL. This does not give fio a
128 # chance to clean up and means that child processes may continue
129 # running and submitting IO.
130 proc = subprocess.Popen(command,
131 stdout=stdout_file,
132 stderr=stderr_file,
133 cwd=self.test_dir,
134 universal_newlines=True)
135 proc.communicate(timeout=self.success['timeout'])
136 exticode_file.write('{0}\n'.format(proc.returncode))
137 logging.debug("return code: %d" % proc.returncode)
138 self.output['proc'] = proc
139 except subprocess.TimeoutExpired:
140 proc.terminate()
141 proc.communicate()
142 assert proc.poll()
143 self.output['failure'] = 'timeout'
144 except Exception:
145 if not proc.poll():
146 proc.terminate()
147 proc.communicate()
148 self.output['failure'] = 'exception'
149 self.output['exc_info'] = sys.exc_info()
150 finally:
151 stdout_file.close()
152 stderr_file.close()
153 exticode_file.close()
154
155 def check_result(self):
156 if 'proc' not in self.output:
157 if self.output['failure'] == 'timeout':
158 self.failure_reason = "{0} timeout,".format(self.failure_reason)
159 else:
160 assert self.output['failure'] == 'exception'
161 self.failure_reason = '{0} exception: {1}, {2}'.format(
162 self.failure_reason, self.output['exc_info'][0],
163 self.output['exc_info'][1])
164
165 self.passed = False
166 return
167
168 if 'zero_return' in self.success:
169 if self.success['zero_return']:
170 if self.output['proc'].returncode != 0:
171 self.passed = False
172 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
173 else:
174 if self.output['proc'].returncode == 0:
175 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
176 self.passed = False
177
178 if 'stderr_empty' in self.success:
179 stderr_size = os.path.getsize(self.stderr_file)
180 if self.success['stderr_empty']:
181 if stderr_size != 0:
182 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
183 self.passed = False
184 else:
185 if stderr_size == 0:
186 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
187 self.passed = False
188
189
190class FioJobTest(FioExeTest):
191 """Test consists of a fio job"""
192
193 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
194 fio_pre_success=None, output_format="normal"):
195 """Construct a FioJobTest which is a FioExeTest consisting of a
196 single fio job file with an optional setup step.
197
198 fio_path: location of fio executable
199 fio_job: location of fio job file
200 success: Definition of test success
201 fio_pre_job: fio job for preconditioning
202 fio_pre_success: Definition of test success for fio precon job
203 output_format: normal (default), json, jsonplus, or terse
204 """
205
206 self.fio_job = fio_job
207 self.fio_pre_job = fio_pre_job
208 self.fio_pre_success = fio_pre_success if fio_pre_success else success
209 self.output_format = output_format
210 self.precon_failed = False
211 self.json_data = None
212 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
213 self.fio_args = [
214 "--output-format={0}".format(self.output_format),
215 "--output={0}".format(self.fio_output),
216 self.fio_job,
217 ]
218 FioExeTest.__init__(self, fio_path, self.fio_args, success)
219
220 def setup(self, artifact_root, testnum):
221 super(FioJobTest, self).setup(artifact_root, testnum)
222
223 self.command_file = os.path.join(
224 self.test_dir,
225 "{0}.command".format(os.path.basename(self.fio_job)))
226 self.stdout_file = os.path.join(
227 self.test_dir,
228 "{0}.stdout".format(os.path.basename(self.fio_job)))
229 self.stderr_file = os.path.join(
230 self.test_dir,
231 "{0}.stderr".format(os.path.basename(self.fio_job)))
232 self.exticode_file = os.path.join(
233 self.test_dir,
234 "{0}.exitcode".format(os.path.basename(self.fio_job)))
235
236 def run_pre_job(self):
237 precon = FioJobTest(self.exe_path, self.fio_pre_job,
238 self.fio_pre_success,
239 output_format=self.output_format)
240 precon.setup(self.artifact_root, self.testnum)
241 precon.run()
242 precon.check_result()
243 self.precon_failed = not precon.passed
244 self.failure_reason = precon.failure_reason
245
246 def run(self):
247 if self.fio_pre_job:
248 self.run_pre_job()
249
250 if not self.precon_failed:
251 super(FioJobTest, self).run()
252 else:
253 logging.debug("precondition step failed")
254
255 def check_result(self):
256 if self.precon_failed:
257 self.passed = False
258 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
259 return
260
261 super(FioJobTest, self).check_result()
262
263 if 'json' in self.output_format:
264 output_file = open(os.path.join(self.test_dir, self.fio_output), "r")
265 file_data = output_file.read()
266 output_file.close()
267 try:
268 self.json_data = json.loads(file_data)
269 except json.JSONDecodeError:
270 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
271 self.passed = False
272
273
274class FioJobTest_t0005(FioJobTest):
275 """Test consists of fio test job t0005
276 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
277
278 def check_result(self):
279 super(FioJobTest_t0005, self).check_result()
280
281 if not self.passed:
282 return
283
284 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
285 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
286 self.passed = False
287 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
288 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
289 self.passed = False
290
291
292class FioJobTest_t0006(FioJobTest):
293 """Test consists of fio test job t0006
294 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
295
296 def check_result(self):
297 super(FioJobTest_t0006, self).check_result()
298
299 if not self.passed:
300 return
301
302 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
303 / self.json_data['jobs'][0]['write']['io_kbytes']
304 logging.debug("ratio: %f" % ratio)
305 if ratio < 1.99 or ratio > 2.01:
306 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
307 self.passed = False
308
309
310class FioJobTest_t0007(FioJobTest):
311 """Test consists of fio test job t0007
312 Confirm that read['io_kbytes'] = 87040"""
313
314 def check_result(self):
315 super(FioJobTest_t0007, self).check_result()
316
317 if not self.passed:
318 return
319
320 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
321 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
322 self.passed = False
323
324
325class FioJobTest_t0008(FioJobTest):
326 """Test consists of fio test job t0008
327 Confirm that read['io_kbytes'] = 32768 and that
328 write['io_kbytes'] ~ 16568
329
330 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
331 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
332
333 def check_result(self):
334 super(FioJobTest_t0008, self).check_result()
335
336 if not self.passed:
337 return
338
339 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
340 logging.debug("ratio: %f" % ratio)
341
342 if ratio < 0.99 or ratio > 1.01:
343 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
344 self.passed = False
345 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
346 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
347 self.passed = False
348
349
350class FioJobTest_t0009(FioJobTest):
351 """Test consists of fio test job t0009
352 Confirm that runtime >= 60s"""
353
354 def check_result(self):
355 super(FioJobTest_t0009, self).check_result()
356
357 if not self.passed:
358 return
359
360 logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed'])
361
362 if self.json_data['jobs'][0]['elapsed'] < 60:
363 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
364 self.passed = False
365
366
367class FioJobTest_t0011(FioJobTest):
368 """Test consists of fio test job t0009
369 Confirm that job0 iops == 1000
370 and that job1_iops / job0_iops ~ 8
371 With two runs of fio-3.16 I observed a ratio of 8.3"""
372
373 def check_result(self):
374 super(FioJobTest_t0011, self).check_result()
375
376 if not self.passed:
377 return
378
379 iops1 = self.json_data['jobs'][0]['read']['iops']
380 iops2 = self.json_data['jobs'][1]['read']['iops']
381 ratio = iops2 / iops1
382 logging.debug("ratio: %f" % ratio)
383
384 if iops1 < 999 or iops1 > 1001:
385 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
386 self.passed = False
387
388 if ratio < 7 or ratio > 9:
389 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
390 self.passed = False
391
392
393SUCCESS_DEFAULT = {
394 'zero_return': True,
395 'stderr_empty': True,
396 'timeout': 300,
397 }
398SUCCESS_NONZERO = {
399 'zero_return': False,
400 'stderr_empty': False,
401 'timeout': 300,
402 }
403SUCCESS_STDERR = {
404 'zero_return': True,
405 'stderr_empty': False,
406 'timeout': 300,
407 }
408TEST_LIST = [
409 {
410 'test_id': 1,
411 'test_class': FioJobTest,
412 'job': 't0001-52c58027.fio',
413 'success': SUCCESS_DEFAULT,
414 'pre_job': None,
415 'pre_success': None,
416 },
417 {
418 'test_id': 2,
419 'test_class': FioJobTest,
420 'job': 't0002-13af05ae-post.fio',
421 'success': SUCCESS_DEFAULT,
422 'pre_job': 't0002-13af05ae-pre.fio',
423 'pre_success': None,
424 },
425 {
426 'test_id': 3,
427 'test_class': FioJobTest,
428 'job': 't0003-0ae2c6e1-post.fio',
429 'success': SUCCESS_NONZERO,
430 'pre_job': 't0003-0ae2c6e1-pre.fio',
431 'pre_success': SUCCESS_DEFAULT,
432 },
433 {
434 'test_id': 4,
435 'test_class': FioJobTest,
436 'job': 't0004-8a99fdf6.fio',
437 'success': SUCCESS_DEFAULT,
438 'pre_job': None,
439 'pre_success': None,
440 },
441 {
442 'test_id': 5,
443 'test_class': FioJobTest_t0005,
444 'job': 't0005-f7078f7b.fio',
445 'success': SUCCESS_DEFAULT,
446 'pre_job': None,
447 'pre_success': None,
448 'output_format': 'json',
449 },
450 {
451 'test_id': 6,
452 'test_class': FioJobTest_t0006,
453 'job': 't0006-82af2a7c.fio',
454 'success': SUCCESS_DEFAULT,
455 'pre_job': None,
456 'pre_success': None,
457 'output_format': 'json',
458 },
459 {
460 'test_id': 7,
461 'test_class': FioJobTest_t0007,
462 'job': 't0007-37cf9e3c.fio',
463 'success': SUCCESS_DEFAULT,
464 'pre_job': None,
465 'pre_success': None,
466 'output_format': 'json',
467 },
468 {
469 'test_id': 8,
470 'test_class': FioJobTest_t0008,
471 'job': 't0008-ae2fafc8.fio',
472 'success': SUCCESS_DEFAULT,
473 'pre_job': None,
474 'pre_success': None,
475 'output_format': 'json',
476 },
477 {
478 'test_id': 9,
479 'test_class': FioJobTest_t0009,
480 'job': 't0009-f8b0bd10.fio',
481 'success': SUCCESS_DEFAULT,
482 'pre_job': None,
483 'pre_success': None,
484 'output_format': 'json',
485 },
486 {
487 'test_id': 10,
488 'test_class': FioJobTest,
489 'job': 't0010-b7aae4ba.fio',
490 'success': SUCCESS_DEFAULT,
491 'pre_job': None,
492 'pre_success': None,
493 },
494 {
495 'test_id': 11,
496 'test_class': FioJobTest_t0011,
497 'job': 't0011-5d2788d5.fio',
498 'success': SUCCESS_DEFAULT,
499 'pre_job': None,
500 'pre_success': None,
501 'output_format': 'json',
502 },
503 {
504 'test_id': 1000,
505 'test_class': FioExeTest,
506 'exe': 't/axmap',
507 'parameters': None,
508 'success': SUCCESS_DEFAULT,
509 },
510 {
511 'test_id': 1001,
512 'test_class': FioExeTest,
513 'exe': 't/ieee754',
514 'parameters': None,
515 'success': SUCCESS_DEFAULT,
516 },
517 {
518 'test_id': 1002,
519 'test_class': FioExeTest,
520 'exe': 't/lfsr-test',
521 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
522 'success': SUCCESS_STDERR,
523 },
524 {
525 'test_id': 1003,
526 'test_class': FioExeTest,
527 'exe': 't/readonly.py',
528 'parameters': ['-f', '{fio_path}'],
529 'success': SUCCESS_DEFAULT,
530 },
531 {
532 'test_id': 1004,
533 'test_class': FioExeTest,
534 'exe': 't/steadystate_tests.py',
535 'parameters': ['{fio_path}'],
536 'success': SUCCESS_DEFAULT,
537 },
538 {
539 'test_id': 1005,
540 'test_class': FioExeTest,
541 'exe': 't/stest',
542 'parameters': None,
543 'success': SUCCESS_STDERR,
544 },
545 {
546 'test_id': 1006,
547 'test_class': FioExeTest,
548 'exe': 't/strided.py',
549 'parameters': ['{fio_path}'],
550 'success': SUCCESS_DEFAULT,
551 },
552 {
553 'test_id': 1007,
554 'test_class': FioExeTest,
555 'exe': 't/zbd/run-tests-against-regular-nullb',
556 'parameters': None,
557 'success': SUCCESS_DEFAULT,
558 },
559 {
560 'test_id': 1008,
561 'test_class': FioExeTest,
562 'exe': 't/zbd/run-tests-against-zoned-nullb',
563 'parameters': None,
564 'success': SUCCESS_DEFAULT,
565 },
566 {
567 'test_id': 1009,
568 'test_class': FioExeTest,
569 'exe': 'unittests/unittest',
570 'parameters': None,
571 'success': SUCCESS_DEFAULT,
572 },
573]
574
575
576def parse_args():
577 parser = argparse.ArgumentParser()
578 parser.add_argument('-r', '--fio-root',
579 help='fio root path')
580 parser.add_argument('-f', '--fio',
581 help='path to fio executable (e.g., ./fio)')
582 parser.add_argument('-a', '--artifact-root',
583 help='artifact root directory')
584 parser.add_argument('-s', '--skip', nargs='+', type=int,
585 help='list of test(s) to skip')
586 parser.add_argument('-o', '--run-only', nargs='+', type=int,
587 help='list of test(s) to run, skipping all others')
588 args = parser.parse_args()
589
590 return args
591
592
593def main():
594 logging.basicConfig(level=logging.INFO)
595
596 args = parse_args()
597 if args.fio_root:
598 fio_root = args.fio_root
599 else:
600 fio_root = Path(__file__).absolute().parent.parent
601 logging.debug("fio_root: %s" % fio_root)
602
603 if args.fio:
604 fio_path = args.fio
605 else:
606 fio_path = os.path.join(fio_root, "fio")
607 logging.debug("fio_path: %s" % fio_path)
608
609 artifact_root = args.artifact_root if args.artifact_root else \
610 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
611 os.mkdir(artifact_root)
612 print("Artifact directory is %s" % artifact_root)
613
614 passed = 0
615 failed = 0
616 skipped = 0
617
618 for config in TEST_LIST:
619 if (args.skip and config['test_id'] in args.skip) or \
620 (args.run_only and config['test_id'] not in args.run_only):
621 skipped = skipped + 1
622 print("Test {0} SKIPPED".format(config['test_id']))
623 continue
624
625 if issubclass(config['test_class'], FioJobTest):
626 if config['pre_job']:
627 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
628 config['pre_job'])
629 else:
630 fio_pre_job = None
631 if config['pre_success']:
632 fio_pre_success = config['pre_success']
633 else:
634 fio_pre_success = None
635 if 'output_format' in config:
636 output_format = config['output_format']
637 else:
638 output_format = 'normal'
639 test = config['test_class'](
640 fio_path,
641 os.path.join(fio_root, 't', 'jobs', config['job']),
642 config['success'],
643 fio_pre_job=fio_pre_job,
644 fio_pre_success=fio_pre_success,
645 output_format=output_format)
646 elif issubclass(config['test_class'], FioExeTest):
647 exe_path = os.path.join(fio_root, config['exe'])
648 if config['parameters']:
649 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
650 else:
651 parameters = None
652 test = config['test_class'](exe_path, parameters,
653 config['success'])
654 else:
655 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
656 failed = failed + 1
657 continue
658
659 test.setup(artifact_root, config['test_id'])
660 test.run()
661 test.check_result()
662 if test.passed:
663 result = "PASSED"
664 passed = passed + 1
665 else:
666 result = "FAILED: {0}".format(test.failure_reason)
667 failed = failed + 1
668 print("Test {0} {1}".format(config['test_id'], result))
669
670 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
671
672 sys.exit(failed)
673
674
675if __name__ == '__main__':
676 main()
fio - Flexible IO Tester