projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
t/run-fio-tests: improve error handling
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
b048455f 17# # git clone git://git.kernel.dk/fio.git
df1eaa36
VF		 18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
24# - Python 3
25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
42# TODO automatically detect dependencies and skip tests accordingly
43#
44
45import os
46import sys
47import json
48import time
49import logging
50import argparse
51import subprocess
52from pathlib import Path
53
54
55class FioTest(object):
56 """Base for all fio tests."""
57
58 def __init__(self, exe_path, parameters, success):
59 self.exe_path = exe_path
60 self.parameters = parameters
61 self.success = success
62 self.output = {}
63 self.artifact_root = None
64 self.testnum = None
65 self.test_dir = None
66 self.passed = True
67 self.failure_reason = ''
68
69 def setup(self, artifact_root, testnum):
70 self.artifact_root = artifact_root
71 self.testnum = testnum
72 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
73 if not os.path.exists(self.test_dir):
74 os.mkdir(self.test_dir)
75
76 self.command_file = os.path.join(
77 self.test_dir,
78 "{0}.command".format(os.path.basename(self.exe_path)))
79 self.stdout_file = os.path.join(
80 self.test_dir,
81 "{0}.stdout".format(os.path.basename(self.exe_path)))
82 self.stderr_file = os.path.join(
83 self.test_dir,
84 "{0}.stderr".format(os.path.basename(self.exe_path)))
85 self.exticode_file = os.path.join(
86 self.test_dir,
87 "{0}.exitcode".format(os.path.basename(self.exe_path)))
88
89 def run(self):
90 raise NotImplementedError()
91
92 def check_result(self):
93 raise NotImplementedError()
94
95
96class FioExeTest(FioTest):
97 """Test consists of an executable binary or script"""
98
99 def __init__(self, exe_path, parameters, success):
100 """Construct a FioExeTest which is a FioTest consisting of an
101 executable binary or script.
102
103 exe_path: location of executable binary or script
104 parameters: list of parameters for executable
105 success: Definition of test success
106 """
107
108 FioTest.__init__(self, exe_path, parameters, success)
109
110 def setup(self, artifact_root, testnum):
111 super(FioExeTest, self).setup(artifact_root, testnum)
112
113 def run(self):
114 if self.parameters:
115 command = [self.exe_path] + self.parameters
116 else:
117 command = [self.exe_path]
118 command_file = open(self.command_file, "w+")
119 command_file.write("%s\n" % command)
120 command_file.close()
121
122 stdout_file = open(self.stdout_file, "w+")
123 stderr_file = open(self.stderr_file, "w+")
124 exticode_file = open(self.exticode_file, "w+")
125 try:
b048455f 126 proc = None
df1eaa36
VF		 127 # Avoid using subprocess.run() here because when a timeout occurs,
128 # fio will be stopped with SIGKILL. This does not give fio a
129 # chance to clean up and means that child processes may continue
130 # running and submitting IO.
131 proc = subprocess.Popen(command,
132 stdout=stdout_file,
133 stderr=stderr_file,
134 cwd=self.test_dir,
135 universal_newlines=True)
136 proc.communicate(timeout=self.success['timeout'])
137 exticode_file.write('{0}\n'.format(proc.returncode))
138 logging.debug("return code: %d" % proc.returncode)
139 self.output['proc'] = proc
140 except subprocess.TimeoutExpired:
141 proc.terminate()
142 proc.communicate()
143 assert proc.poll()
144 self.output['failure'] = 'timeout'
145 except Exception:
b048455f
VF		 146 if proc:
147 if not proc.poll():
148 proc.terminate()
149 proc.communicate()
df1eaa36
VF		 150 self.output['failure'] = 'exception'
151 self.output['exc_info'] = sys.exc_info()
152 finally:
153 stdout_file.close()
154 stderr_file.close()
155 exticode_file.close()
156
157 def check_result(self):
158 if 'proc' not in self.output:
159 if self.output['failure'] == 'timeout':
160 self.failure_reason = "{0} timeout,".format(self.failure_reason)
161 else:
162 assert self.output['failure'] == 'exception'
163 self.failure_reason = '{0} exception: {1}, {2}'.format(
164 self.failure_reason, self.output['exc_info'][0],
165 self.output['exc_info'][1])
166
167 self.passed = False
168 return
169
170 if 'zero_return' in self.success:
171 if self.success['zero_return']:
172 if self.output['proc'].returncode != 0:
173 self.passed = False
174 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
175 else:
176 if self.output['proc'].returncode == 0:
177 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
178 self.passed = False
179
180 if 'stderr_empty' in self.success:
181 stderr_size = os.path.getsize(self.stderr_file)
182 if self.success['stderr_empty']:
183 if stderr_size != 0:
184 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
185 self.passed = False
186 else:
187 if stderr_size == 0:
188 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
189 self.passed = False
190
191
192class FioJobTest(FioExeTest):
193 """Test consists of a fio job"""
194
195 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
196 fio_pre_success=None, output_format="normal"):
197 """Construct a FioJobTest which is a FioExeTest consisting of a
198 single fio job file with an optional setup step.
199
200 fio_path: location of fio executable
201 fio_job: location of fio job file
202 success: Definition of test success
203 fio_pre_job: fio job for preconditioning
204 fio_pre_success: Definition of test success for fio precon job
205 output_format: normal (default), json, jsonplus, or terse
206 """
207
208 self.fio_job = fio_job
209 self.fio_pre_job = fio_pre_job
210 self.fio_pre_success = fio_pre_success if fio_pre_success else success
211 self.output_format = output_format
212 self.precon_failed = False
213 self.json_data = None
214 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
215 self.fio_args = [
216 "--output-format={0}".format(self.output_format),
217 "--output={0}".format(self.fio_output),
218 self.fio_job,
219 ]
220 FioExeTest.__init__(self, fio_path, self.fio_args, success)
221
222 def setup(self, artifact_root, testnum):
223 super(FioJobTest, self).setup(artifact_root, testnum)
224
225 self.command_file = os.path.join(
226 self.test_dir,
227 "{0}.command".format(os.path.basename(self.fio_job)))
228 self.stdout_file = os.path.join(
229 self.test_dir,
230 "{0}.stdout".format(os.path.basename(self.fio_job)))
231 self.stderr_file = os.path.join(
232 self.test_dir,
233 "{0}.stderr".format(os.path.basename(self.fio_job)))
234 self.exticode_file = os.path.join(
235 self.test_dir,
236 "{0}.exitcode".format(os.path.basename(self.fio_job)))
237
238 def run_pre_job(self):
239 precon = FioJobTest(self.exe_path, self.fio_pre_job,
240 self.fio_pre_success,
241 output_format=self.output_format)
242 precon.setup(self.artifact_root, self.testnum)
243 precon.run()
244 precon.check_result()
245 self.precon_failed = not precon.passed
246 self.failure_reason = precon.failure_reason
247
248 def run(self):
249 if self.fio_pre_job:
250 self.run_pre_job()
251
252 if not self.precon_failed:
253 super(FioJobTest, self).run()
254 else:
255 logging.debug("precondition step failed")
256
257 def check_result(self):
258 if self.precon_failed:
259 self.passed = False
260 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
261 return
262
263 super(FioJobTest, self).check_result()
264
265 if 'json' in self.output_format:
266 output_file = open(os.path.join(self.test_dir, self.fio_output), "r")
267 file_data = output_file.read()
268 output_file.close()
269 try:
270 self.json_data = json.loads(file_data)
271 except json.JSONDecodeError:
272 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
273 self.passed = False
274
275
276class FioJobTest_t0005(FioJobTest):
277 """Test consists of fio test job t0005
278 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
279
280 def check_result(self):
281 super(FioJobTest_t0005, self).check_result()
282
283 if not self.passed:
284 return
285
286 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
287 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
288 self.passed = False
289 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
290 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
291 self.passed = False
292
293
294class FioJobTest_t0006(FioJobTest):
295 """Test consists of fio test job t0006
296 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
297
298 def check_result(self):
299 super(FioJobTest_t0006, self).check_result()
300
301 if not self.passed:
302 return
303
304 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
305 / self.json_data['jobs'][0]['write']['io_kbytes']
306 logging.debug("ratio: %f" % ratio)
307 if ratio < 1.99 or ratio > 2.01:
308 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
309 self.passed = False
310
311
312class FioJobTest_t0007(FioJobTest):
313 """Test consists of fio test job t0007
314 Confirm that read['io_kbytes'] = 87040"""
315
316 def check_result(self):
317 super(FioJobTest_t0007, self).check_result()
318
319 if not self.passed:
320 return
321
322 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
323 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
324 self.passed = False
325
326
327class FioJobTest_t0008(FioJobTest):
328 """Test consists of fio test job t0008
329 Confirm that read['io_kbytes'] = 32768 and that
330 write['io_kbytes'] ~ 16568
331
332 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
333 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
334
335 def check_result(self):
336 super(FioJobTest_t0008, self).check_result()
337
338 if not self.passed:
339 return
340
341 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
342 logging.debug("ratio: %f" % ratio)
343
344 if ratio < 0.99 or ratio > 1.01:
345 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
346 self.passed = False
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
349 self.passed = False
350
351
352class FioJobTest_t0009(FioJobTest):
353 """Test consists of fio test job t0009
354 Confirm that runtime >= 60s"""
355
356 def check_result(self):
357 super(FioJobTest_t0009, self).check_result()
358
359 if not self.passed:
360 return
361
362 logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed'])
363
364 if self.json_data['jobs'][0]['elapsed'] < 60:
365 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
366 self.passed = False
367
368
369class FioJobTest_t0011(FioJobTest):
370 """Test consists of fio test job t0009
371 Confirm that job0 iops == 1000
372 and that job1_iops / job0_iops ~ 8
373 With two runs of fio-3.16 I observed a ratio of 8.3"""
374
375 def check_result(self):
376 super(FioJobTest_t0011, self).check_result()
377
378 if not self.passed:
379 return
380
381 iops1 = self.json_data['jobs'][0]['read']['iops']
382 iops2 = self.json_data['jobs'][1]['read']['iops']
383 ratio = iops2 / iops1
384 logging.debug("ratio: %f" % ratio)
385
386 if iops1 < 999 or iops1 > 1001:
387 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
388 self.passed = False
389
390 if ratio < 7 or ratio > 9:
391 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
392 self.passed = False
393
394
395SUCCESS_DEFAULT = {
396 'zero_return': True,
397 'stderr_empty': True,
398 'timeout': 300,
399 }
400SUCCESS_NONZERO = {
401 'zero_return': False,
402 'stderr_empty': False,
403 'timeout': 300,
404 }
405SUCCESS_STDERR = {
406 'zero_return': True,
407 'stderr_empty': False,
408 'timeout': 300,
409 }
410TEST_LIST = [
411 {
412 'test_id': 1,
413 'test_class': FioJobTest,
414 'job': 't0001-52c58027.fio',
415 'success': SUCCESS_DEFAULT,
416 'pre_job': None,
417 'pre_success': None,
418 },
419 {
420 'test_id': 2,
421 'test_class': FioJobTest,
422 'job': 't0002-13af05ae-post.fio',
423 'success': SUCCESS_DEFAULT,
424 'pre_job': 't0002-13af05ae-pre.fio',
425 'pre_success': None,
426 },
427 {
428 'test_id': 3,
429 'test_class': FioJobTest,
430 'job': 't0003-0ae2c6e1-post.fio',
431 'success': SUCCESS_NONZERO,
432 'pre_job': 't0003-0ae2c6e1-pre.fio',
433 'pre_success': SUCCESS_DEFAULT,
434 },
435 {
436 'test_id': 4,
437 'test_class': FioJobTest,
438 'job': 't0004-8a99fdf6.fio',
439 'success': SUCCESS_DEFAULT,
440 'pre_job': None,
441 'pre_success': None,
442 },
443 {
444 'test_id': 5,
445 'test_class': FioJobTest_t0005,
446 'job': 't0005-f7078f7b.fio',
447 'success': SUCCESS_DEFAULT,
448 'pre_job': None,
449 'pre_success': None,
450 'output_format': 'json',
451 },
452 {
453 'test_id': 6,
454 'test_class': FioJobTest_t0006,
455 'job': 't0006-82af2a7c.fio',
456 'success': SUCCESS_DEFAULT,
457 'pre_job': None,
458 'pre_success': None,
459 'output_format': 'json',
460 },
461 {
462 'test_id': 7,
463 'test_class': FioJobTest_t0007,
464 'job': 't0007-37cf9e3c.fio',
465 'success': SUCCESS_DEFAULT,
466 'pre_job': None,
467 'pre_success': None,
468 'output_format': 'json',
469 },
470 {
471 'test_id': 8,
472 'test_class': FioJobTest_t0008,
473 'job': 't0008-ae2fafc8.fio',
474 'success': SUCCESS_DEFAULT,
475 'pre_job': None,
476 'pre_success': None,
477 'output_format': 'json',
478 },
479 {
480 'test_id': 9,
481 'test_class': FioJobTest_t0009,
482 'job': 't0009-f8b0bd10.fio',
483 'success': SUCCESS_DEFAULT,
484 'pre_job': None,
485 'pre_success': None,
486 'output_format': 'json',
487 },
488 {
489 'test_id': 10,
490 'test_class': FioJobTest,
491 'job': 't0010-b7aae4ba.fio',
492 'success': SUCCESS_DEFAULT,
493 'pre_job': None,
494 'pre_success': None,
495 },
496 {
497 'test_id': 11,
498 'test_class': FioJobTest_t0011,
499 'job': 't0011-5d2788d5.fio',
500 'success': SUCCESS_DEFAULT,
501 'pre_job': None,
502 'pre_success': None,
503 'output_format': 'json',
504 },
505 {
506 'test_id': 1000,
507 'test_class': FioExeTest,
508 'exe': 't/axmap',
509 'parameters': None,
510 'success': SUCCESS_DEFAULT,
511 },
512 {
513 'test_id': 1001,
514 'test_class': FioExeTest,
515 'exe': 't/ieee754',
516 'parameters': None,
517 'success': SUCCESS_DEFAULT,
518 },
519 {
520 'test_id': 1002,
521 'test_class': FioExeTest,
522 'exe': 't/lfsr-test',
523 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
524 'success': SUCCESS_STDERR,
525 },
526 {
527 'test_id': 1003,
528 'test_class': FioExeTest,
529 'exe': 't/readonly.py',
530 'parameters': ['-f', '{fio_path}'],
531 'success': SUCCESS_DEFAULT,
532 },
533 {
534 'test_id': 1004,
535 'test_class': FioExeTest,
536 'exe': 't/steadystate_tests.py',
537 'parameters': ['{fio_path}'],
538 'success': SUCCESS_DEFAULT,
539 },
540 {
541 'test_id': 1005,
542 'test_class': FioExeTest,
543 'exe': 't/stest',
544 'parameters': None,
545 'success': SUCCESS_STDERR,
546 },
547 {
548 'test_id': 1006,
549 'test_class': FioExeTest,
550 'exe': 't/strided.py',
551 'parameters': ['{fio_path}'],
552 'success': SUCCESS_DEFAULT,
553 },
554 {
555 'test_id': 1007,
556 'test_class': FioExeTest,
557 'exe': 't/zbd/run-tests-against-regular-nullb',
558 'parameters': None,
559 'success': SUCCESS_DEFAULT,
560 },
561 {
562 'test_id': 1008,
563 'test_class': FioExeTest,
564 'exe': 't/zbd/run-tests-against-zoned-nullb',
565 'parameters': None,
566 'success': SUCCESS_DEFAULT,
567 },
568 {
569 'test_id': 1009,
570 'test_class': FioExeTest,
571 'exe': 'unittests/unittest',
572 'parameters': None,
573 'success': SUCCESS_DEFAULT,
574 },
575]
576
577
578def parse_args():
579 parser = argparse.ArgumentParser()
580 parser.add_argument('-r', '--fio-root',
581 help='fio root path')
582 parser.add_argument('-f', '--fio',
583 help='path to fio executable (e.g., ./fio)')
584 parser.add_argument('-a', '--artifact-root',
585 help='artifact root directory')
586 parser.add_argument('-s', '--skip', nargs='+', type=int,
587 help='list of test(s) to skip')
588 parser.add_argument('-o', '--run-only', nargs='+', type=int,
589 help='list of test(s) to run, skipping all others')
590 args = parser.parse_args()
591
592 return args
593
594
595def main():
596 logging.basicConfig(level=logging.INFO)
597
598 args = parse_args()
599 if args.fio_root:
600 fio_root = args.fio_root
601 else:
602 fio_root = Path(__file__).absolute().parent.parent
603 logging.debug("fio_root: %s" % fio_root)
604
605 if args.fio:
606 fio_path = args.fio
607 else:
608 fio_path = os.path.join(fio_root, "fio")
609 logging.debug("fio_path: %s" % fio_path)
610
611 artifact_root = args.artifact_root if args.artifact_root else \
612 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
613 os.mkdir(artifact_root)
614 print("Artifact directory is %s" % artifact_root)
615
616 passed = 0
617 failed = 0
618 skipped = 0
619
620 for config in TEST_LIST:
621 if (args.skip and config['test_id'] in args.skip) or \
622 (args.run_only and config['test_id'] not in args.run_only):
623 skipped = skipped + 1
624 print("Test {0} SKIPPED".format(config['test_id']))
625 continue
626
627 if issubclass(config['test_class'], FioJobTest):
628 if config['pre_job']:
629 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
630 config['pre_job'])
631 else:
632 fio_pre_job = None
633 if config['pre_success']:
634 fio_pre_success = config['pre_success']
635 else:
636 fio_pre_success = None
637 if 'output_format' in config:
638 output_format = config['output_format']
639 else:
640 output_format = 'normal'
641 test = config['test_class'](
642 fio_path,
643 os.path.join(fio_root, 't', 'jobs', config['job']),
644 config['success'],
645 fio_pre_job=fio_pre_job,
646 fio_pre_success=fio_pre_success,
647 output_format=output_format)
648 elif issubclass(config['test_class'], FioExeTest):
649 exe_path = os.path.join(fio_root, config['exe'])
650 if config['parameters']:
651 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
652 else:
653 parameters = None
654 test = config['test_class'](exe_path, parameters,
655 config['success'])
656 else:
657 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
658 failed = failed + 1
659 continue
660
661 test.setup(artifact_root, config['test_id'])
662 test.run()
663 test.check_result()
664 if test.passed:
665 result = "PASSED"
666 passed = passed + 1
667 else:
668 result = "FAILED: {0}".format(test.failure_reason)
669 failed = failed + 1
670 print("Test {0} {1}".format(config['test_id'], result))
671
672 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
673
674 sys.exit(failed)
675
676
677if __name__ == '__main__':
678 main()
fio - Flexible IO Tester