projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
t/run-fio-tests: detect requirements and skip tests accordingly
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
b048455f 17# # git clone git://git.kernel.dk/fio.git
df1eaa36
VF		 18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
b1bc705e 24# - Python 3.5 (subprocess.run)
df1eaa36
VF		 25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
df1eaa36
VF		 42#
43
44import os
45import sys
46import json
47import time
6d5470c3 48import shutil
df1eaa36
VF		 49import logging
50import argparse
b1bc705e 51import platform
df1eaa36 52import subprocess
b1bc705e 53import multiprocessing
df1eaa36
VF		 54from pathlib import Path
55
56
57class FioTest(object):
58 """Base for all fio tests."""
59
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
64 self.output = {}
65 self.artifact_root = None
66 self.testnum = None
67 self.test_dir = None
68 self.passed = True
69 self.failure_reason = ''
70
71 def setup(self, artifact_root, testnum):
72 self.artifact_root = artifact_root
73 self.testnum = testnum
74 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
75 if not os.path.exists(self.test_dir):
76 os.mkdir(self.test_dir)
77
78 self.command_file = os.path.join(
79 self.test_dir,
80 "{0}.command".format(os.path.basename(self.exe_path)))
81 self.stdout_file = os.path.join(
82 self.test_dir,
83 "{0}.stdout".format(os.path.basename(self.exe_path)))
84 self.stderr_file = os.path.join(
85 self.test_dir,
86 "{0}.stderr".format(os.path.basename(self.exe_path)))
87 self.exticode_file = os.path.join(
88 self.test_dir,
89 "{0}.exitcode".format(os.path.basename(self.exe_path)))
90
91 def run(self):
92 raise NotImplementedError()
93
94 def check_result(self):
95 raise NotImplementedError()
96
97
98class FioExeTest(FioTest):
99 """Test consists of an executable binary or script"""
100
101 def __init__(self, exe_path, parameters, success):
102 """Construct a FioExeTest which is a FioTest consisting of an
103 executable binary or script.
104
105 exe_path: location of executable binary or script
106 parameters: list of parameters for executable
107 success: Definition of test success
108 """
109
110 FioTest.__init__(self, exe_path, parameters, success)
111
112 def setup(self, artifact_root, testnum):
113 super(FioExeTest, self).setup(artifact_root, testnum)
114
115 def run(self):
116 if self.parameters:
117 command = [self.exe_path] + self.parameters
118 else:
119 command = [self.exe_path]
120 command_file = open(self.command_file, "w+")
121 command_file.write("%s\n" % command)
122 command_file.close()
123
124 stdout_file = open(self.stdout_file, "w+")
125 stderr_file = open(self.stderr_file, "w+")
126 exticode_file = open(self.exticode_file, "w+")
127 try:
b048455f 128 proc = None
df1eaa36
VF		 129 # Avoid using subprocess.run() here because when a timeout occurs,
130 # fio will be stopped with SIGKILL. This does not give fio a
131 # chance to clean up and means that child processes may continue
132 # running and submitting IO.
133 proc = subprocess.Popen(command,
134 stdout=stdout_file,
135 stderr=stderr_file,
136 cwd=self.test_dir,
137 universal_newlines=True)
138 proc.communicate(timeout=self.success['timeout'])
139 exticode_file.write('{0}\n'.format(proc.returncode))
140 logging.debug("return code: %d" % proc.returncode)
141 self.output['proc'] = proc
142 except subprocess.TimeoutExpired:
143 proc.terminate()
144 proc.communicate()
145 assert proc.poll()
146 self.output['failure'] = 'timeout'
147 except Exception:
b048455f
VF		 148 if proc:
149 if not proc.poll():
150 proc.terminate()
151 proc.communicate()
df1eaa36
VF		 152 self.output['failure'] = 'exception'
153 self.output['exc_info'] = sys.exc_info()
154 finally:
155 stdout_file.close()
156 stderr_file.close()
157 exticode_file.close()
158
159 def check_result(self):
160 if 'proc' not in self.output:
161 if self.output['failure'] == 'timeout':
162 self.failure_reason = "{0} timeout,".format(self.failure_reason)
163 else:
164 assert self.output['failure'] == 'exception'
165 self.failure_reason = '{0} exception: {1}, {2}'.format(
166 self.failure_reason, self.output['exc_info'][0],
167 self.output['exc_info'][1])
168
169 self.passed = False
170 return
171
172 if 'zero_return' in self.success:
173 if self.success['zero_return']:
174 if self.output['proc'].returncode != 0:
175 self.passed = False
176 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
177 else:
178 if self.output['proc'].returncode == 0:
179 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
180 self.passed = False
181
6d5470c3 182 stderr_size = os.path.getsize(self.stderr_file)
df1eaa36 183 if 'stderr_empty' in self.success:
df1eaa36
VF		 184 if self.success['stderr_empty']:
185 if stderr_size != 0:
186 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
187 self.passed = False
188 else:
189 if stderr_size == 0:
190 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
191 self.passed = False
192
193
194class FioJobTest(FioExeTest):
195 """Test consists of a fio job"""
196
197 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
198 fio_pre_success=None, output_format="normal"):
199 """Construct a FioJobTest which is a FioExeTest consisting of a
200 single fio job file with an optional setup step.
201
202 fio_path: location of fio executable
203 fio_job: location of fio job file
204 success: Definition of test success
205 fio_pre_job: fio job for preconditioning
206 fio_pre_success: Definition of test success for fio precon job
207 output_format: normal (default), json, jsonplus, or terse
208 """
209
210 self.fio_job = fio_job
211 self.fio_pre_job = fio_pre_job
212 self.fio_pre_success = fio_pre_success if fio_pre_success else success
213 self.output_format = output_format
214 self.precon_failed = False
215 self.json_data = None
216 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
217 self.fio_args = [
218 "--output-format={0}".format(self.output_format),
219 "--output={0}".format(self.fio_output),
220 self.fio_job,
221 ]
222 FioExeTest.__init__(self, fio_path, self.fio_args, success)
223
224 def setup(self, artifact_root, testnum):
225 super(FioJobTest, self).setup(artifact_root, testnum)
226
227 self.command_file = os.path.join(
228 self.test_dir,
229 "{0}.command".format(os.path.basename(self.fio_job)))
230 self.stdout_file = os.path.join(
231 self.test_dir,
232 "{0}.stdout".format(os.path.basename(self.fio_job)))
233 self.stderr_file = os.path.join(
234 self.test_dir,
235 "{0}.stderr".format(os.path.basename(self.fio_job)))
236 self.exticode_file = os.path.join(
237 self.test_dir,
238 "{0}.exitcode".format(os.path.basename(self.fio_job)))
239
240 def run_pre_job(self):
241 precon = FioJobTest(self.exe_path, self.fio_pre_job,
242 self.fio_pre_success,
243 output_format=self.output_format)
244 precon.setup(self.artifact_root, self.testnum)
245 precon.run()
246 precon.check_result()
247 self.precon_failed = not precon.passed
248 self.failure_reason = precon.failure_reason
249
250 def run(self):
251 if self.fio_pre_job:
252 self.run_pre_job()
253
254 if not self.precon_failed:
255 super(FioJobTest, self).run()
256 else:
257 logging.debug("precondition step failed")
258
259 def check_result(self):
260 if self.precon_failed:
261 self.passed = False
262 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
263 return
264
265 super(FioJobTest, self).check_result()
266
6d5470c3
VF		 267 if not self.passed:
268 return
269
df1eaa36 270 if 'json' in self.output_format:
df1eaa36 271 try:
6d5470c3
VF		 272 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
273 file_data = output_file.read()
274 except EnvironmentError:
275 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
df1eaa36 276 self.passed = False
6d5470c3
VF		 277 else:
278 try:
279 self.json_data = json.loads(file_data)
280 except json.JSONDecodeError:
281 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
282 self.passed = False
df1eaa36
VF		 283
284
285class FioJobTest_t0005(FioJobTest):
286 """Test consists of fio test job t0005
287 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
288
289 def check_result(self):
290 super(FioJobTest_t0005, self).check_result()
291
292 if not self.passed:
293 return
294
295 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
296 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
297 self.passed = False
298 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
299 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
300 self.passed = False
301
302
303class FioJobTest_t0006(FioJobTest):
304 """Test consists of fio test job t0006
305 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
306
307 def check_result(self):
308 super(FioJobTest_t0006, self).check_result()
309
310 if not self.passed:
311 return
312
313 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
314 / self.json_data['jobs'][0]['write']['io_kbytes']
315 logging.debug("ratio: %f" % ratio)
316 if ratio < 1.99 or ratio > 2.01:
317 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
318 self.passed = False
319
320
321class FioJobTest_t0007(FioJobTest):
322 """Test consists of fio test job t0007
323 Confirm that read['io_kbytes'] = 87040"""
324
325 def check_result(self):
326 super(FioJobTest_t0007, self).check_result()
327
328 if not self.passed:
329 return
330
331 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
332 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
333 self.passed = False
334
335
336class FioJobTest_t0008(FioJobTest):
337 """Test consists of fio test job t0008
338 Confirm that read['io_kbytes'] = 32768 and that
339 write['io_kbytes'] ~ 16568
340
341 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
342 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
343
344 def check_result(self):
345 super(FioJobTest_t0008, self).check_result()
346
347 if not self.passed:
348 return
349
350 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
351 logging.debug("ratio: %f" % ratio)
352
353 if ratio < 0.99 or ratio > 1.01:
354 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
355 self.passed = False
356 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
357 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
358 self.passed = False
359
360
361class FioJobTest_t0009(FioJobTest):
362 """Test consists of fio test job t0009
363 Confirm that runtime >= 60s"""
364
365 def check_result(self):
366 super(FioJobTest_t0009, self).check_result()
367
368 if not self.passed:
369 return
370
371 logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed'])
372
373 if self.json_data['jobs'][0]['elapsed'] < 60:
374 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
375 self.passed = False
376
377
378class FioJobTest_t0011(FioJobTest):
379 """Test consists of fio test job t0009
380 Confirm that job0 iops == 1000
381 and that job1_iops / job0_iops ~ 8
382 With two runs of fio-3.16 I observed a ratio of 8.3"""
383
384 def check_result(self):
385 super(FioJobTest_t0011, self).check_result()
386
387 if not self.passed:
388 return
389
390 iops1 = self.json_data['jobs'][0]['read']['iops']
391 iops2 = self.json_data['jobs'][1]['read']['iops']
392 ratio = iops2 / iops1
393 logging.debug("ratio: %f" % ratio)
394
395 if iops1 < 999 or iops1 > 1001:
396 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
397 self.passed = False
398
399 if ratio < 7 or ratio > 9:
400 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
401 self.passed = False
402
403
b1bc705e
VF		 404class Requirements(object):
405 """Requirements consists of multiple run environment characteristics.
406 These are to determine if a particular test can be run"""
407
408 _linux = False
409 _libaio = False
410 _zbd = False
411 _root = False
412 _zoned_nullb = False
413 _not_macos = False
414 _unittests = False
415 _cpucount4 = False
416
417 def __init__(self, fio_root):
418 Requirements._not_macos = platform.system() != "Darwin"
419 Requirements._linux = platform.system() == "Linux"
420
421 if Requirements._linux:
422 try:
423 config_file = os.path.join(fio_root, "config-host.h")
424 with open(config_file, "r") as config:
425 contents = config.read()
426 except Exception:
427 print("Unable to open {0} to check requirements".format(config_file))
428 Requirements._zbd = True
429 else:
430 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
431 Requirements._libaio = "CONFIG_LIBAIO" in contents
432
433 Requirements._root = (os.geteuid() == 0)
434 if Requirements._zbd and Requirements._root:
435 subprocess.run(["modprobe", "null_blk"],
436 stdout=subprocess.PIPE,
437 stderr=subprocess.PIPE)
438 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
439 Requirements._zoned_nullb = True
440
441 unittest_path = os.path.join(fio_root, "unittests", "unittest")
442 Requirements._unittests = os.path.exists(unittest_path)
443
444 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
445
446 req_list = [Requirements.linux,
447 Requirements.libaio,
448 Requirements.zbd,
449 Requirements.root,
450 Requirements.zoned_nullb,
451 Requirements.not_macos,
452 Requirements.unittests,
453 Requirements.cpucount4]
454 for req in req_list:
455 value, desc = req()
456 logging.debug("Requirement '%s' met? %s" % (desc, value))
457
458 def linux():
459 return Requirements._linux, "Linux required"
460
461 def libaio():
462 return Requirements._libaio, "libaio required"
463
464 def zbd():
465 return Requirements._zbd, "Zoned block device support required"
466
467 def root():
468 return Requirements._root, "root required"
469
470 def zoned_nullb():
471 return Requirements._zoned_nullb, "Zoned null block device support required"
472
473 def not_macos():
474 return Requirements._not_macos, "platform other than macOS required"
475
476 def unittests():
477 return Requirements._unittests, "Unittests support required"
478
479 def cpucount4():
480 return Requirements._cpucount4, "4+ CPUs required"
481
482
df1eaa36
VF		 483SUCCESS_DEFAULT = {
484 'zero_return': True,
485 'stderr_empty': True,
486 'timeout': 300,
487 }
488SUCCESS_NONZERO = {
489 'zero_return': False,
490 'stderr_empty': False,
491 'timeout': 300,
492 }
493SUCCESS_STDERR = {
494 'zero_return': True,
495 'stderr_empty': False,
496 'timeout': 300,
497 }
498TEST_LIST = [
499 {
500 'test_id': 1,
501 'test_class': FioJobTest,
502 'job': 't0001-52c58027.fio',
503 'success': SUCCESS_DEFAULT,
504 'pre_job': None,
505 'pre_success': None,
b1bc705e 506 'requirements': [],
df1eaa36
VF		 507 },
508 {
509 'test_id': 2,
510 'test_class': FioJobTest,
511 'job': 't0002-13af05ae-post.fio',
512 'success': SUCCESS_DEFAULT,
513 'pre_job': 't0002-13af05ae-pre.fio',
514 'pre_success': None,
b1bc705e 515 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 516 },
517 {
518 'test_id': 3,
519 'test_class': FioJobTest,
520 'job': 't0003-0ae2c6e1-post.fio',
521 'success': SUCCESS_NONZERO,
522 'pre_job': 't0003-0ae2c6e1-pre.fio',
523 'pre_success': SUCCESS_DEFAULT,
b1bc705e 524 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 525 },
526 {
527 'test_id': 4,
528 'test_class': FioJobTest,
529 'job': 't0004-8a99fdf6.fio',
530 'success': SUCCESS_DEFAULT,
531 'pre_job': None,
532 'pre_success': None,
b1bc705e 533 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 534 },
535 {
536 'test_id': 5,
537 'test_class': FioJobTest_t0005,
538 'job': 't0005-f7078f7b.fio',
539 'success': SUCCESS_DEFAULT,
540 'pre_job': None,
541 'pre_success': None,
542 'output_format': 'json',
b1bc705e 543 'requirements': [],
df1eaa36
VF		 544 },
545 {
546 'test_id': 6,
547 'test_class': FioJobTest_t0006,
548 'job': 't0006-82af2a7c.fio',
549 'success': SUCCESS_DEFAULT,
550 'pre_job': None,
551 'pre_success': None,
552 'output_format': 'json',
b1bc705e 553 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 554 },
555 {
556 'test_id': 7,
557 'test_class': FioJobTest_t0007,
558 'job': 't0007-37cf9e3c.fio',
559 'success': SUCCESS_DEFAULT,
560 'pre_job': None,
561 'pre_success': None,
562 'output_format': 'json',
b1bc705e 563 'requirements': [],
df1eaa36
VF		 564 },
565 {
566 'test_id': 8,
567 'test_class': FioJobTest_t0008,
568 'job': 't0008-ae2fafc8.fio',
569 'success': SUCCESS_DEFAULT,
570 'pre_job': None,
571 'pre_success': None,
572 'output_format': 'json',
b1bc705e 573 'requirements': [],
df1eaa36
VF		 574 },
575 {
576 'test_id': 9,
577 'test_class': FioJobTest_t0009,
578 'job': 't0009-f8b0bd10.fio',
579 'success': SUCCESS_DEFAULT,
580 'pre_job': None,
581 'pre_success': None,
582 'output_format': 'json',
b1bc705e
VF		 583 'requirements': [Requirements.not_macos,
584 Requirements.cpucount4],
585 # mac os does not support CPU affinity
df1eaa36
VF		 586 },
587 {
588 'test_id': 10,
589 'test_class': FioJobTest,
590 'job': 't0010-b7aae4ba.fio',
591 'success': SUCCESS_DEFAULT,
592 'pre_job': None,
593 'pre_success': None,
b1bc705e 594 'requirements': [],
df1eaa36
VF		 595 },
596 {
597 'test_id': 11,
598 'test_class': FioJobTest_t0011,
599 'job': 't0011-5d2788d5.fio',
600 'success': SUCCESS_DEFAULT,
601 'pre_job': None,
602 'pre_success': None,
603 'output_format': 'json',
b1bc705e 604 'requirements': [],
df1eaa36
VF		 605 },
606 {
607 'test_id': 1000,
608 'test_class': FioExeTest,
609 'exe': 't/axmap',
610 'parameters': None,
611 'success': SUCCESS_DEFAULT,
b1bc705e 612 'requirements': [],
df1eaa36
VF		 613 },
614 {
615 'test_id': 1001,
616 'test_class': FioExeTest,
617 'exe': 't/ieee754',
618 'parameters': None,
619 'success': SUCCESS_DEFAULT,
b1bc705e 620 'requirements': [],
df1eaa36
VF		 621 },
622 {
623 'test_id': 1002,
624 'test_class': FioExeTest,
625 'exe': 't/lfsr-test',
626 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
627 'success': SUCCESS_STDERR,
b1bc705e 628 'requirements': [],
df1eaa36
VF		 629 },
630 {
631 'test_id': 1003,
632 'test_class': FioExeTest,
633 'exe': 't/readonly.py',
634 'parameters': ['-f', '{fio_path}'],
635 'success': SUCCESS_DEFAULT,
b1bc705e 636 'requirements': [],
df1eaa36
VF		 637 },
638 {
639 'test_id': 1004,
640 'test_class': FioExeTest,
641 'exe': 't/steadystate_tests.py',
642 'parameters': ['{fio_path}'],
643 'success': SUCCESS_DEFAULT,
b1bc705e 644 'requirements': [],
df1eaa36
VF		 645 },
646 {
647 'test_id': 1005,
648 'test_class': FioExeTest,
649 'exe': 't/stest',
650 'parameters': None,
651 'success': SUCCESS_STDERR,
b1bc705e 652 'requirements': [],
df1eaa36
VF		 653 },
654 {
655 'test_id': 1006,
656 'test_class': FioExeTest,
657 'exe': 't/strided.py',
658 'parameters': ['{fio_path}'],
659 'success': SUCCESS_DEFAULT,
b1bc705e 660 'requirements': [],
df1eaa36
VF		 661 },
662 {
663 'test_id': 1007,
664 'test_class': FioExeTest,
665 'exe': 't/zbd/run-tests-against-regular-nullb',
666 'parameters': None,
667 'success': SUCCESS_DEFAULT,
b1bc705e
VF		 668 'requirements': [Requirements.linux, Requirements.zbd,
669 Requirements.root],
df1eaa36
VF		 670 },
671 {
672 'test_id': 1008,
673 'test_class': FioExeTest,
674 'exe': 't/zbd/run-tests-against-zoned-nullb',
675 'parameters': None,
676 'success': SUCCESS_DEFAULT,
b1bc705e
VF		 677 'requirements': [Requirements.linux, Requirements.zbd,
678 Requirements.root, Requirements.zoned_nullb],
df1eaa36
VF		 679 },
680 {
681 'test_id': 1009,
682 'test_class': FioExeTest,
683 'exe': 'unittests/unittest',
684 'parameters': None,
685 'success': SUCCESS_DEFAULT,
b1bc705e 686 'requirements': [Requirements.unittests],
df1eaa36
VF		 687 },
688]
689
690
691def parse_args():
692 parser = argparse.ArgumentParser()
693 parser.add_argument('-r', '--fio-root',
694 help='fio root path')
695 parser.add_argument('-f', '--fio',
696 help='path to fio executable (e.g., ./fio)')
697 parser.add_argument('-a', '--artifact-root',
698 help='artifact root directory')
699 parser.add_argument('-s', '--skip', nargs='+', type=int,
700 help='list of test(s) to skip')
701 parser.add_argument('-o', '--run-only', nargs='+', type=int,
702 help='list of test(s) to run, skipping all others')
6d5470c3
VF		 703 parser.add_argument('-d', '--debug', action='store_true',
704 help='provide debug output')
b1bc705e
VF		 705 parser.add_argument('-k', '--skip-req', action='store_true',
706 help='skip requirements checking')
df1eaa36
VF		 707 args = parser.parse_args()
708
709 return args
710
711
712def main():
df1eaa36 713 args = parse_args()
6d5470c3
VF		 714 if args.debug:
715 logging.basicConfig(level=logging.DEBUG)
716 else:
717 logging.basicConfig(level=logging.INFO)
718
df1eaa36
VF		 719 if args.fio_root:
720 fio_root = args.fio_root
721 else:
6d5470c3
VF		 722 fio_root = str(Path(__file__).absolute().parent.parent)
723 print("fio root is %s" % fio_root)
df1eaa36
VF		 724
725 if args.fio:
726 fio_path = args.fio
727 else:
728 fio_path = os.path.join(fio_root, "fio")
6d5470c3
VF		 729 print("fio path is %s" % fio_path)
730 if not shutil.which(fio_path):
731 print("Warning: fio executable not found")
df1eaa36
VF		 732
733 artifact_root = args.artifact_root if args.artifact_root else \
734 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
735 os.mkdir(artifact_root)
736 print("Artifact directory is %s" % artifact_root)
737
b1bc705e
VF		 738 if not args.skip_req:
739 req = Requirements(fio_root)
740
df1eaa36
VF		 741 passed = 0
742 failed = 0
743 skipped = 0
744
745 for config in TEST_LIST:
746 if (args.skip and config['test_id'] in args.skip) or \
747 (args.run_only and config['test_id'] not in args.run_only):
748 skipped = skipped + 1
b1bc705e 749 print("Test {0} SKIPPED (User request)".format(config['test_id']))
df1eaa36
VF		 750 continue
751
752 if issubclass(config['test_class'], FioJobTest):
753 if config['pre_job']:
754 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
755 config['pre_job'])
756 else:
757 fio_pre_job = None
758 if config['pre_success']:
759 fio_pre_success = config['pre_success']
760 else:
761 fio_pre_success = None
762 if 'output_format' in config:
763 output_format = config['output_format']
764 else:
765 output_format = 'normal'
766 test = config['test_class'](
767 fio_path,
768 os.path.join(fio_root, 't', 'jobs', config['job']),
769 config['success'],
770 fio_pre_job=fio_pre_job,
771 fio_pre_success=fio_pre_success,
772 output_format=output_format)
773 elif issubclass(config['test_class'], FioExeTest):
774 exe_path = os.path.join(fio_root, config['exe'])
775 if config['parameters']:
776 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
777 else:
778 parameters = None
779 test = config['test_class'](exe_path, parameters,
780 config['success'])
781 else:
782 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
783 failed = failed + 1
784 continue
785
b1bc705e
VF		 786 if not args.skip_req:
787 skip = False
788 for req in config['requirements']:
789 ok, reason = req()
790 skip = not ok
791 logging.debug("Requirement '%s' met? %s" % (reason, ok))
792 if skip:
793 break
794 if skip:
795 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
796 skipped = skipped + 1
797 continue
798
df1eaa36
VF		 799 test.setup(artifact_root, config['test_id'])
800 test.run()
801 test.check_result()
802 if test.passed:
803 result = "PASSED"
804 passed = passed + 1
805 else:
806 result = "FAILED: {0}".format(test.failure_reason)
807 failed = failed + 1
6d5470c3
VF		 808 with open(test.stderr_file, "r") as stderr_file:
809 logging.debug("stderr:\n%s" % stderr_file.read())
810 with open(test.stdout_file, "r") as stdout_file:
811 logging.debug("stdout:\n%s" % stdout_file.read())
df1eaa36
VF		 812 print("Test {0} {1}".format(config['test_id'], result))
813
814 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
815
816 sys.exit(failed)
817
818
819if __name__ == '__main__':
820 main()
fio - Flexible IO Tester