projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
t/run-fio-tests: a few small improvements
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
b048455f 17# # git clone git://git.kernel.dk/fio.git
df1eaa36
VF		 18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
24# - Python 3
25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
42# TODO automatically detect dependencies and skip tests accordingly
43#
44
45import os
46import sys
47import json
48import time
6d5470c3 49import shutil
df1eaa36
VF		 50import logging
51import argparse
52import subprocess
53from pathlib import Path
54
55
56class FioTest(object):
57 """Base for all fio tests."""
58
59 def __init__(self, exe_path, parameters, success):
60 self.exe_path = exe_path
61 self.parameters = parameters
62 self.success = success
63 self.output = {}
64 self.artifact_root = None
65 self.testnum = None
66 self.test_dir = None
67 self.passed = True
68 self.failure_reason = ''
69
70 def setup(self, artifact_root, testnum):
71 self.artifact_root = artifact_root
72 self.testnum = testnum
73 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
74 if not os.path.exists(self.test_dir):
75 os.mkdir(self.test_dir)
76
77 self.command_file = os.path.join(
78 self.test_dir,
79 "{0}.command".format(os.path.basename(self.exe_path)))
80 self.stdout_file = os.path.join(
81 self.test_dir,
82 "{0}.stdout".format(os.path.basename(self.exe_path)))
83 self.stderr_file = os.path.join(
84 self.test_dir,
85 "{0}.stderr".format(os.path.basename(self.exe_path)))
86 self.exticode_file = os.path.join(
87 self.test_dir,
88 "{0}.exitcode".format(os.path.basename(self.exe_path)))
89
90 def run(self):
91 raise NotImplementedError()
92
93 def check_result(self):
94 raise NotImplementedError()
95
96
97class FioExeTest(FioTest):
98 """Test consists of an executable binary or script"""
99
100 def __init__(self, exe_path, parameters, success):
101 """Construct a FioExeTest which is a FioTest consisting of an
102 executable binary or script.
103
104 exe_path: location of executable binary or script
105 parameters: list of parameters for executable
106 success: Definition of test success
107 """
108
109 FioTest.__init__(self, exe_path, parameters, success)
110
111 def setup(self, artifact_root, testnum):
112 super(FioExeTest, self).setup(artifact_root, testnum)
113
114 def run(self):
115 if self.parameters:
116 command = [self.exe_path] + self.parameters
117 else:
118 command = [self.exe_path]
119 command_file = open(self.command_file, "w+")
120 command_file.write("%s\n" % command)
121 command_file.close()
122
123 stdout_file = open(self.stdout_file, "w+")
124 stderr_file = open(self.stderr_file, "w+")
125 exticode_file = open(self.exticode_file, "w+")
126 try:
b048455f 127 proc = None
df1eaa36
VF		 128 # Avoid using subprocess.run() here because when a timeout occurs,
129 # fio will be stopped with SIGKILL. This does not give fio a
130 # chance to clean up and means that child processes may continue
131 # running and submitting IO.
132 proc = subprocess.Popen(command,
133 stdout=stdout_file,
134 stderr=stderr_file,
135 cwd=self.test_dir,
136 universal_newlines=True)
137 proc.communicate(timeout=self.success['timeout'])
138 exticode_file.write('{0}\n'.format(proc.returncode))
139 logging.debug("return code: %d" % proc.returncode)
140 self.output['proc'] = proc
141 except subprocess.TimeoutExpired:
142 proc.terminate()
143 proc.communicate()
144 assert proc.poll()
145 self.output['failure'] = 'timeout'
146 except Exception:
b048455f
VF		 147 if proc:
148 if not proc.poll():
149 proc.terminate()
150 proc.communicate()
df1eaa36
VF		 151 self.output['failure'] = 'exception'
152 self.output['exc_info'] = sys.exc_info()
153 finally:
154 stdout_file.close()
155 stderr_file.close()
156 exticode_file.close()
157
158 def check_result(self):
159 if 'proc' not in self.output:
160 if self.output['failure'] == 'timeout':
161 self.failure_reason = "{0} timeout,".format(self.failure_reason)
162 else:
163 assert self.output['failure'] == 'exception'
164 self.failure_reason = '{0} exception: {1}, {2}'.format(
165 self.failure_reason, self.output['exc_info'][0],
166 self.output['exc_info'][1])
167
168 self.passed = False
169 return
170
171 if 'zero_return' in self.success:
172 if self.success['zero_return']:
173 if self.output['proc'].returncode != 0:
174 self.passed = False
175 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
176 else:
177 if self.output['proc'].returncode == 0:
178 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
179 self.passed = False
180
6d5470c3 181 stderr_size = os.path.getsize(self.stderr_file)
df1eaa36 182 if 'stderr_empty' in self.success:
df1eaa36
VF		 183 if self.success['stderr_empty']:
184 if stderr_size != 0:
185 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
186 self.passed = False
187 else:
188 if stderr_size == 0:
189 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
190 self.passed = False
191
192
193class FioJobTest(FioExeTest):
194 """Test consists of a fio job"""
195
196 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
197 fio_pre_success=None, output_format="normal"):
198 """Construct a FioJobTest which is a FioExeTest consisting of a
199 single fio job file with an optional setup step.
200
201 fio_path: location of fio executable
202 fio_job: location of fio job file
203 success: Definition of test success
204 fio_pre_job: fio job for preconditioning
205 fio_pre_success: Definition of test success for fio precon job
206 output_format: normal (default), json, jsonplus, or terse
207 """
208
209 self.fio_job = fio_job
210 self.fio_pre_job = fio_pre_job
211 self.fio_pre_success = fio_pre_success if fio_pre_success else success
212 self.output_format = output_format
213 self.precon_failed = False
214 self.json_data = None
215 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
216 self.fio_args = [
217 "--output-format={0}".format(self.output_format),
218 "--output={0}".format(self.fio_output),
219 self.fio_job,
220 ]
221 FioExeTest.__init__(self, fio_path, self.fio_args, success)
222
223 def setup(self, artifact_root, testnum):
224 super(FioJobTest, self).setup(artifact_root, testnum)
225
226 self.command_file = os.path.join(
227 self.test_dir,
228 "{0}.command".format(os.path.basename(self.fio_job)))
229 self.stdout_file = os.path.join(
230 self.test_dir,
231 "{0}.stdout".format(os.path.basename(self.fio_job)))
232 self.stderr_file = os.path.join(
233 self.test_dir,
234 "{0}.stderr".format(os.path.basename(self.fio_job)))
235 self.exticode_file = os.path.join(
236 self.test_dir,
237 "{0}.exitcode".format(os.path.basename(self.fio_job)))
238
239 def run_pre_job(self):
240 precon = FioJobTest(self.exe_path, self.fio_pre_job,
241 self.fio_pre_success,
242 output_format=self.output_format)
243 precon.setup(self.artifact_root, self.testnum)
244 precon.run()
245 precon.check_result()
246 self.precon_failed = not precon.passed
247 self.failure_reason = precon.failure_reason
248
249 def run(self):
250 if self.fio_pre_job:
251 self.run_pre_job()
252
253 if not self.precon_failed:
254 super(FioJobTest, self).run()
255 else:
256 logging.debug("precondition step failed")
257
258 def check_result(self):
259 if self.precon_failed:
260 self.passed = False
261 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
262 return
263
264 super(FioJobTest, self).check_result()
265
6d5470c3
VF		 266 if not self.passed:
267 return
268
df1eaa36 269 if 'json' in self.output_format:
df1eaa36 270 try:
6d5470c3
VF		 271 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
272 file_data = output_file.read()
273 except EnvironmentError:
274 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
df1eaa36 275 self.passed = False
6d5470c3
VF		 276 else:
277 try:
278 self.json_data = json.loads(file_data)
279 except json.JSONDecodeError:
280 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
281 self.passed = False
df1eaa36
VF		 282
283
284class FioJobTest_t0005(FioJobTest):
285 """Test consists of fio test job t0005
286 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
287
288 def check_result(self):
289 super(FioJobTest_t0005, self).check_result()
290
291 if not self.passed:
292 return
293
294 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
295 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
296 self.passed = False
297 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
298 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
299 self.passed = False
300
301
302class FioJobTest_t0006(FioJobTest):
303 """Test consists of fio test job t0006
304 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
305
306 def check_result(self):
307 super(FioJobTest_t0006, self).check_result()
308
309 if not self.passed:
310 return
311
312 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
313 / self.json_data['jobs'][0]['write']['io_kbytes']
314 logging.debug("ratio: %f" % ratio)
315 if ratio < 1.99 or ratio > 2.01:
316 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
317 self.passed = False
318
319
320class FioJobTest_t0007(FioJobTest):
321 """Test consists of fio test job t0007
322 Confirm that read['io_kbytes'] = 87040"""
323
324 def check_result(self):
325 super(FioJobTest_t0007, self).check_result()
326
327 if not self.passed:
328 return
329
330 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
331 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
332 self.passed = False
333
334
335class FioJobTest_t0008(FioJobTest):
336 """Test consists of fio test job t0008
337 Confirm that read['io_kbytes'] = 32768 and that
338 write['io_kbytes'] ~ 16568
339
340 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
341 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
342
343 def check_result(self):
344 super(FioJobTest_t0008, self).check_result()
345
346 if not self.passed:
347 return
348
349 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
350 logging.debug("ratio: %f" % ratio)
351
352 if ratio < 0.99 or ratio > 1.01:
353 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
354 self.passed = False
355 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
356 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
357 self.passed = False
358
359
360class FioJobTest_t0009(FioJobTest):
361 """Test consists of fio test job t0009
362 Confirm that runtime >= 60s"""
363
364 def check_result(self):
365 super(FioJobTest_t0009, self).check_result()
366
367 if not self.passed:
368 return
369
370 logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed'])
371
372 if self.json_data['jobs'][0]['elapsed'] < 60:
373 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
374 self.passed = False
375
376
377class FioJobTest_t0011(FioJobTest):
378 """Test consists of fio test job t0009
379 Confirm that job0 iops == 1000
380 and that job1_iops / job0_iops ~ 8
381 With two runs of fio-3.16 I observed a ratio of 8.3"""
382
383 def check_result(self):
384 super(FioJobTest_t0011, self).check_result()
385
386 if not self.passed:
387 return
388
389 iops1 = self.json_data['jobs'][0]['read']['iops']
390 iops2 = self.json_data['jobs'][1]['read']['iops']
391 ratio = iops2 / iops1
392 logging.debug("ratio: %f" % ratio)
393
394 if iops1 < 999 or iops1 > 1001:
395 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
396 self.passed = False
397
398 if ratio < 7 or ratio > 9:
399 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
400 self.passed = False
401
402
403SUCCESS_DEFAULT = {
404 'zero_return': True,
405 'stderr_empty': True,
406 'timeout': 300,
407 }
408SUCCESS_NONZERO = {
409 'zero_return': False,
410 'stderr_empty': False,
411 'timeout': 300,
412 }
413SUCCESS_STDERR = {
414 'zero_return': True,
415 'stderr_empty': False,
416 'timeout': 300,
417 }
418TEST_LIST = [
419 {
420 'test_id': 1,
421 'test_class': FioJobTest,
422 'job': 't0001-52c58027.fio',
423 'success': SUCCESS_DEFAULT,
424 'pre_job': None,
425 'pre_success': None,
426 },
427 {
428 'test_id': 2,
429 'test_class': FioJobTest,
430 'job': 't0002-13af05ae-post.fio',
431 'success': SUCCESS_DEFAULT,
432 'pre_job': 't0002-13af05ae-pre.fio',
433 'pre_success': None,
434 },
435 {
436 'test_id': 3,
437 'test_class': FioJobTest,
438 'job': 't0003-0ae2c6e1-post.fio',
439 'success': SUCCESS_NONZERO,
440 'pre_job': 't0003-0ae2c6e1-pre.fio',
441 'pre_success': SUCCESS_DEFAULT,
442 },
443 {
444 'test_id': 4,
445 'test_class': FioJobTest,
446 'job': 't0004-8a99fdf6.fio',
447 'success': SUCCESS_DEFAULT,
448 'pre_job': None,
449 'pre_success': None,
450 },
451 {
452 'test_id': 5,
453 'test_class': FioJobTest_t0005,
454 'job': 't0005-f7078f7b.fio',
455 'success': SUCCESS_DEFAULT,
456 'pre_job': None,
457 'pre_success': None,
458 'output_format': 'json',
459 },
460 {
461 'test_id': 6,
462 'test_class': FioJobTest_t0006,
463 'job': 't0006-82af2a7c.fio',
464 'success': SUCCESS_DEFAULT,
465 'pre_job': None,
466 'pre_success': None,
467 'output_format': 'json',
468 },
469 {
470 'test_id': 7,
471 'test_class': FioJobTest_t0007,
472 'job': 't0007-37cf9e3c.fio',
473 'success': SUCCESS_DEFAULT,
474 'pre_job': None,
475 'pre_success': None,
476 'output_format': 'json',
477 },
478 {
479 'test_id': 8,
480 'test_class': FioJobTest_t0008,
481 'job': 't0008-ae2fafc8.fio',
482 'success': SUCCESS_DEFAULT,
483 'pre_job': None,
484 'pre_success': None,
485 'output_format': 'json',
486 },
487 {
488 'test_id': 9,
489 'test_class': FioJobTest_t0009,
490 'job': 't0009-f8b0bd10.fio',
491 'success': SUCCESS_DEFAULT,
492 'pre_job': None,
493 'pre_success': None,
494 'output_format': 'json',
495 },
496 {
497 'test_id': 10,
498 'test_class': FioJobTest,
499 'job': 't0010-b7aae4ba.fio',
500 'success': SUCCESS_DEFAULT,
501 'pre_job': None,
502 'pre_success': None,
503 },
504 {
505 'test_id': 11,
506 'test_class': FioJobTest_t0011,
507 'job': 't0011-5d2788d5.fio',
508 'success': SUCCESS_DEFAULT,
509 'pre_job': None,
510 'pre_success': None,
511 'output_format': 'json',
512 },
513 {
514 'test_id': 1000,
515 'test_class': FioExeTest,
516 'exe': 't/axmap',
517 'parameters': None,
518 'success': SUCCESS_DEFAULT,
519 },
520 {
521 'test_id': 1001,
522 'test_class': FioExeTest,
523 'exe': 't/ieee754',
524 'parameters': None,
525 'success': SUCCESS_DEFAULT,
526 },
527 {
528 'test_id': 1002,
529 'test_class': FioExeTest,
530 'exe': 't/lfsr-test',
531 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
532 'success': SUCCESS_STDERR,
533 },
534 {
535 'test_id': 1003,
536 'test_class': FioExeTest,
537 'exe': 't/readonly.py',
538 'parameters': ['-f', '{fio_path}'],
539 'success': SUCCESS_DEFAULT,
540 },
541 {
542 'test_id': 1004,
543 'test_class': FioExeTest,
544 'exe': 't/steadystate_tests.py',
545 'parameters': ['{fio_path}'],
546 'success': SUCCESS_DEFAULT,
547 },
548 {
549 'test_id': 1005,
550 'test_class': FioExeTest,
551 'exe': 't/stest',
552 'parameters': None,
553 'success': SUCCESS_STDERR,
554 },
555 {
556 'test_id': 1006,
557 'test_class': FioExeTest,
558 'exe': 't/strided.py',
559 'parameters': ['{fio_path}'],
560 'success': SUCCESS_DEFAULT,
561 },
562 {
563 'test_id': 1007,
564 'test_class': FioExeTest,
565 'exe': 't/zbd/run-tests-against-regular-nullb',
566 'parameters': None,
567 'success': SUCCESS_DEFAULT,
568 },
569 {
570 'test_id': 1008,
571 'test_class': FioExeTest,
572 'exe': 't/zbd/run-tests-against-zoned-nullb',
573 'parameters': None,
574 'success': SUCCESS_DEFAULT,
575 },
576 {
577 'test_id': 1009,
578 'test_class': FioExeTest,
579 'exe': 'unittests/unittest',
580 'parameters': None,
581 'success': SUCCESS_DEFAULT,
582 },
583]
584
585
586def parse_args():
587 parser = argparse.ArgumentParser()
588 parser.add_argument('-r', '--fio-root',
589 help='fio root path')
590 parser.add_argument('-f', '--fio',
591 help='path to fio executable (e.g., ./fio)')
592 parser.add_argument('-a', '--artifact-root',
593 help='artifact root directory')
594 parser.add_argument('-s', '--skip', nargs='+', type=int,
595 help='list of test(s) to skip')
596 parser.add_argument('-o', '--run-only', nargs='+', type=int,
597 help='list of test(s) to run, skipping all others')
6d5470c3
VF		 598 parser.add_argument('-d', '--debug', action='store_true',
599 help='provide debug output')
df1eaa36
VF		 600 args = parser.parse_args()
601
602 return args
603
604
605def main():
df1eaa36 606 args = parse_args()
6d5470c3
VF		 607 if args.debug:
608 logging.basicConfig(level=logging.DEBUG)
609 else:
610 logging.basicConfig(level=logging.INFO)
611
df1eaa36
VF		 612 if args.fio_root:
613 fio_root = args.fio_root
614 else:
6d5470c3
VF		 615 fio_root = str(Path(__file__).absolute().parent.parent)
616 print("fio root is %s" % fio_root)
df1eaa36
VF		 617
618 if args.fio:
619 fio_path = args.fio
620 else:
621 fio_path = os.path.join(fio_root, "fio")
6d5470c3
VF		 622 print("fio path is %s" % fio_path)
623 if not shutil.which(fio_path):
624 print("Warning: fio executable not found")
df1eaa36
VF		 625
626 artifact_root = args.artifact_root if args.artifact_root else \
627 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
628 os.mkdir(artifact_root)
629 print("Artifact directory is %s" % artifact_root)
630
631 passed = 0
632 failed = 0
633 skipped = 0
634
635 for config in TEST_LIST:
636 if (args.skip and config['test_id'] in args.skip) or \
637 (args.run_only and config['test_id'] not in args.run_only):
638 skipped = skipped + 1
639 print("Test {0} SKIPPED".format(config['test_id']))
640 continue
641
642 if issubclass(config['test_class'], FioJobTest):
643 if config['pre_job']:
644 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
645 config['pre_job'])
646 else:
647 fio_pre_job = None
648 if config['pre_success']:
649 fio_pre_success = config['pre_success']
650 else:
651 fio_pre_success = None
652 if 'output_format' in config:
653 output_format = config['output_format']
654 else:
655 output_format = 'normal'
656 test = config['test_class'](
657 fio_path,
658 os.path.join(fio_root, 't', 'jobs', config['job']),
659 config['success'],
660 fio_pre_job=fio_pre_job,
661 fio_pre_success=fio_pre_success,
662 output_format=output_format)
663 elif issubclass(config['test_class'], FioExeTest):
664 exe_path = os.path.join(fio_root, config['exe'])
665 if config['parameters']:
666 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
667 else:
668 parameters = None
669 test = config['test_class'](exe_path, parameters,
670 config['success'])
671 else:
672 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
673 failed = failed + 1
674 continue
675
676 test.setup(artifact_root, config['test_id'])
677 test.run()
678 test.check_result()
679 if test.passed:
680 result = "PASSED"
681 passed = passed + 1
682 else:
683 result = "FAILED: {0}".format(test.failure_reason)
684 failed = failed + 1
6d5470c3
VF		 685 with open(test.stderr_file, "r") as stderr_file:
686 logging.debug("stderr:\n%s" % stderr_file.read())
687 with open(test.stdout_file, "r") as stdout_file:
688 logging.debug("stdout:\n%s" % stdout_file.read())
df1eaa36
VF		 689 print("Test {0} {1}".format(config['test_id'], result))
690
691 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
692
693 sys.exit(failed)
694
695
696if __name__ == '__main__':
697 main()
fio - Flexible IO Tester