projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
.appveyor.yml: run run-fio-tests.py
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
b048455f 17# # git clone git://git.kernel.dk/fio.git
df1eaa36
VF		 18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
b1bc705e 24# - Python 3.5 (subprocess.run)
df1eaa36
VF		 25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
df1eaa36
VF		 42#
43
44import os
45import sys
46import json
47import time
6d5470c3 48import shutil
df1eaa36
VF		 49import logging
50import argparse
b1bc705e 51import platform
df1eaa36 52import subprocess
b1bc705e 53import multiprocessing
df1eaa36
VF		 54from pathlib import Path
55
56
57class FioTest(object):
58 """Base for all fio tests."""
59
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
64 self.output = {}
65 self.artifact_root = None
66 self.testnum = None
67 self.test_dir = None
68 self.passed = True
69 self.failure_reason = ''
70
71 def setup(self, artifact_root, testnum):
72 self.artifact_root = artifact_root
73 self.testnum = testnum
74 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
75 if not os.path.exists(self.test_dir):
76 os.mkdir(self.test_dir)
77
78 self.command_file = os.path.join(
79 self.test_dir,
80 "{0}.command".format(os.path.basename(self.exe_path)))
81 self.stdout_file = os.path.join(
82 self.test_dir,
83 "{0}.stdout".format(os.path.basename(self.exe_path)))
84 self.stderr_file = os.path.join(
85 self.test_dir,
86 "{0}.stderr".format(os.path.basename(self.exe_path)))
87 self.exticode_file = os.path.join(
88 self.test_dir,
89 "{0}.exitcode".format(os.path.basename(self.exe_path)))
90
91 def run(self):
92 raise NotImplementedError()
93
94 def check_result(self):
95 raise NotImplementedError()
96
97
98class FioExeTest(FioTest):
99 """Test consists of an executable binary or script"""
100
101 def __init__(self, exe_path, parameters, success):
102 """Construct a FioExeTest which is a FioTest consisting of an
103 executable binary or script.
104
105 exe_path: location of executable binary or script
106 parameters: list of parameters for executable
107 success: Definition of test success
108 """
109
110 FioTest.__init__(self, exe_path, parameters, success)
111
112 def setup(self, artifact_root, testnum):
113 super(FioExeTest, self).setup(artifact_root, testnum)
114
115 def run(self):
116 if self.parameters:
117 command = [self.exe_path] + self.parameters
118 else:
119 command = [self.exe_path]
120 command_file = open(self.command_file, "w+")
121 command_file.write("%s\n" % command)
122 command_file.close()
123
124 stdout_file = open(self.stdout_file, "w+")
125 stderr_file = open(self.stderr_file, "w+")
126 exticode_file = open(self.exticode_file, "w+")
127 try:
b048455f 128 proc = None
df1eaa36
VF		 129 # Avoid using subprocess.run() here because when a timeout occurs,
130 # fio will be stopped with SIGKILL. This does not give fio a
131 # chance to clean up and means that child processes may continue
132 # running and submitting IO.
133 proc = subprocess.Popen(command,
134 stdout=stdout_file,
135 stderr=stderr_file,
136 cwd=self.test_dir,
137 universal_newlines=True)
138 proc.communicate(timeout=self.success['timeout'])
139 exticode_file.write('{0}\n'.format(proc.returncode))
285622dc 140 logging.debug("Test %d: return code: %d" % (self.testnum, proc.returncode))
df1eaa36
VF		 141 self.output['proc'] = proc
142 except subprocess.TimeoutExpired:
143 proc.terminate()
144 proc.communicate()
145 assert proc.poll()
146 self.output['failure'] = 'timeout'
147 except Exception:
b048455f
VF		 148 if proc:
149 if not proc.poll():
150 proc.terminate()
151 proc.communicate()
df1eaa36
VF		 152 self.output['failure'] = 'exception'
153 self.output['exc_info'] = sys.exc_info()
154 finally:
155 stdout_file.close()
156 stderr_file.close()
157 exticode_file.close()
158
159 def check_result(self):
160 if 'proc' not in self.output:
161 if self.output['failure'] == 'timeout':
162 self.failure_reason = "{0} timeout,".format(self.failure_reason)
163 else:
164 assert self.output['failure'] == 'exception'
165 self.failure_reason = '{0} exception: {1}, {2}'.format(
166 self.failure_reason, self.output['exc_info'][0],
167 self.output['exc_info'][1])
168
169 self.passed = False
170 return
171
172 if 'zero_return' in self.success:
173 if self.success['zero_return']:
174 if self.output['proc'].returncode != 0:
175 self.passed = False
176 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
177 else:
178 if self.output['proc'].returncode == 0:
179 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
180 self.passed = False
181
6d5470c3 182 stderr_size = os.path.getsize(self.stderr_file)
df1eaa36 183 if 'stderr_empty' in self.success:
df1eaa36
VF		 184 if self.success['stderr_empty']:
185 if stderr_size != 0:
186 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
187 self.passed = False
188 else:
189 if stderr_size == 0:
190 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
191 self.passed = False
192
193
194class FioJobTest(FioExeTest):
195 """Test consists of a fio job"""
196
197 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
198 fio_pre_success=None, output_format="normal"):
199 """Construct a FioJobTest which is a FioExeTest consisting of a
200 single fio job file with an optional setup step.
201
202 fio_path: location of fio executable
203 fio_job: location of fio job file
204 success: Definition of test success
205 fio_pre_job: fio job for preconditioning
206 fio_pre_success: Definition of test success for fio precon job
207 output_format: normal (default), json, jsonplus, or terse
208 """
209
210 self.fio_job = fio_job
211 self.fio_pre_job = fio_pre_job
212 self.fio_pre_success = fio_pre_success if fio_pre_success else success
213 self.output_format = output_format
214 self.precon_failed = False
215 self.json_data = None
216 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
217 self.fio_args = [
218 "--output-format={0}".format(self.output_format),
219 "--output={0}".format(self.fio_output),
220 self.fio_job,
221 ]
222 FioExeTest.__init__(self, fio_path, self.fio_args, success)
223
224 def setup(self, artifact_root, testnum):
225 super(FioJobTest, self).setup(artifact_root, testnum)
226
227 self.command_file = os.path.join(
228 self.test_dir,
229 "{0}.command".format(os.path.basename(self.fio_job)))
230 self.stdout_file = os.path.join(
231 self.test_dir,
232 "{0}.stdout".format(os.path.basename(self.fio_job)))
233 self.stderr_file = os.path.join(
234 self.test_dir,
235 "{0}.stderr".format(os.path.basename(self.fio_job)))
236 self.exticode_file = os.path.join(
237 self.test_dir,
238 "{0}.exitcode".format(os.path.basename(self.fio_job)))
239
240 def run_pre_job(self):
241 precon = FioJobTest(self.exe_path, self.fio_pre_job,
242 self.fio_pre_success,
243 output_format=self.output_format)
244 precon.setup(self.artifact_root, self.testnum)
245 precon.run()
246 precon.check_result()
247 self.precon_failed = not precon.passed
248 self.failure_reason = precon.failure_reason
249
250 def run(self):
251 if self.fio_pre_job:
252 self.run_pre_job()
253
254 if not self.precon_failed:
255 super(FioJobTest, self).run()
256 else:
285622dc 257 logging.debug("Test %d: precondition step failed" % self.testnum)
df1eaa36
VF		 258
259 def check_result(self):
260 if self.precon_failed:
261 self.passed = False
262 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
263 return
264
265 super(FioJobTest, self).check_result()
266
6d5470c3
VF		 267 if not self.passed:
268 return
269
742b8799
VF		 270 if not 'json' in self.output_format:
271 return
272
273 try:
274 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
275 file_data = output_file.read()
276 except EnvironmentError:
277 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
278 self.passed = False
279 return
280
281 #
282 # Sometimes fio informational messages are included at the top of the
283 # JSON output, especially under Windows. Try to decode output as JSON
284 # data, lopping off up to the first four lines
285 #
286 lines = file_data.splitlines()
287 for i in range(5):
288 file_data = '\n'.join(lines[i:])
df1eaa36 289 try:
742b8799
VF		 290 self.json_data = json.loads(file_data)
291 except json.JSONDecodeError:
292 continue
6d5470c3 293 else:
285622dc 294 logging.debug("Test %d: skipped %d lines decoding JSON data" % (self.testnum, i))
742b8799
VF		 295 return
296
297 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
298 self.passed = False
df1eaa36
VF		 299
300
301class FioJobTest_t0005(FioJobTest):
302 """Test consists of fio test job t0005
303 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
304
305 def check_result(self):
306 super(FioJobTest_t0005, self).check_result()
307
308 if not self.passed:
309 return
310
311 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
312 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
313 self.passed = False
314 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
315 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
316 self.passed = False
317
318
319class FioJobTest_t0006(FioJobTest):
320 """Test consists of fio test job t0006
321 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
322
323 def check_result(self):
324 super(FioJobTest_t0006, self).check_result()
325
326 if not self.passed:
327 return
328
329 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
330 / self.json_data['jobs'][0]['write']['io_kbytes']
285622dc 331 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
df1eaa36
VF		 332 if ratio < 1.99 or ratio > 2.01:
333 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
334 self.passed = False
335
336
337class FioJobTest_t0007(FioJobTest):
338 """Test consists of fio test job t0007
339 Confirm that read['io_kbytes'] = 87040"""
340
341 def check_result(self):
342 super(FioJobTest_t0007, self).check_result()
343
344 if not self.passed:
345 return
346
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
349 self.passed = False
350
351
352class FioJobTest_t0008(FioJobTest):
353 """Test consists of fio test job t0008
354 Confirm that read['io_kbytes'] = 32768 and that
355 write['io_kbytes'] ~ 16568
356
357 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
358 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
359
360 def check_result(self):
361 super(FioJobTest_t0008, self).check_result()
362
363 if not self.passed:
364 return
365
366 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
285622dc 367 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
df1eaa36
VF		 368
369 if ratio < 0.99 or ratio > 1.01:
370 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
371 self.passed = False
372 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
373 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
374 self.passed = False
375
376
377class FioJobTest_t0009(FioJobTest):
378 """Test consists of fio test job t0009
379 Confirm that runtime >= 60s"""
380
381 def check_result(self):
382 super(FioJobTest_t0009, self).check_result()
383
384 if not self.passed:
385 return
386
285622dc 387 logging.debug('Test %d: elapsed: %d' % (self.testnum, self.json_data['jobs'][0]['elapsed']))
df1eaa36
VF		 388
389 if self.json_data['jobs'][0]['elapsed'] < 60:
390 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
391 self.passed = False
392
393
394class FioJobTest_t0011(FioJobTest):
395 """Test consists of fio test job t0009
396 Confirm that job0 iops == 1000
397 and that job1_iops / job0_iops ~ 8
398 With two runs of fio-3.16 I observed a ratio of 8.3"""
399
400 def check_result(self):
401 super(FioJobTest_t0011, self).check_result()
402
403 if not self.passed:
404 return
405
406 iops1 = self.json_data['jobs'][0]['read']['iops']
407 iops2 = self.json_data['jobs'][1]['read']['iops']
408 ratio = iops2 / iops1
285622dc 409 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
df1eaa36
VF		 410
411 if iops1 < 999 or iops1 > 1001:
412 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
413 self.passed = False
414
415 if ratio < 7 or ratio > 9:
416 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
417 self.passed = False
418
419
b1bc705e
VF		 420class Requirements(object):
421 """Requirements consists of multiple run environment characteristics.
422 These are to determine if a particular test can be run"""
423
424 _linux = False
425 _libaio = False
426 _zbd = False
427 _root = False
428 _zoned_nullb = False
429 _not_macos = False
430 _unittests = False
431 _cpucount4 = False
432
433 def __init__(self, fio_root):
434 Requirements._not_macos = platform.system() != "Darwin"
435 Requirements._linux = platform.system() == "Linux"
436
437 if Requirements._linux:
438 try:
439 config_file = os.path.join(fio_root, "config-host.h")
440 with open(config_file, "r") as config:
441 contents = config.read()
442 except Exception:
443 print("Unable to open {0} to check requirements".format(config_file))
444 Requirements._zbd = True
445 else:
446 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
447 Requirements._libaio = "CONFIG_LIBAIO" in contents
448
449 Requirements._root = (os.geteuid() == 0)
450 if Requirements._zbd and Requirements._root:
451 subprocess.run(["modprobe", "null_blk"],
452 stdout=subprocess.PIPE,
453 stderr=subprocess.PIPE)
454 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
455 Requirements._zoned_nullb = True
456
742b8799
VF		 457 if platform.system() == "Windows":
458 utest_exe = "unittest.exe"
459 else:
460 utest_exe = "unittest"
461 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
b1bc705e
VF		 462 Requirements._unittests = os.path.exists(unittest_path)
463
464 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
465
466 req_list = [Requirements.linux,
467 Requirements.libaio,
468 Requirements.zbd,
469 Requirements.root,
470 Requirements.zoned_nullb,
471 Requirements.not_macos,
472 Requirements.unittests,
473 Requirements.cpucount4]
474 for req in req_list:
475 value, desc = req()
285622dc 476 logging.debug("Requirements: Requirement '%s' met? %s" % (desc, value))
b1bc705e
VF		 477
478 def linux():
479 return Requirements._linux, "Linux required"
480
481 def libaio():
482 return Requirements._libaio, "libaio required"
483
484 def zbd():
485 return Requirements._zbd, "Zoned block device support required"
486
487 def root():
488 return Requirements._root, "root required"
489
490 def zoned_nullb():
491 return Requirements._zoned_nullb, "Zoned null block device support required"
492
493 def not_macos():
494 return Requirements._not_macos, "platform other than macOS required"
495
496 def unittests():
497 return Requirements._unittests, "Unittests support required"
498
499 def cpucount4():
500 return Requirements._cpucount4, "4+ CPUs required"
501
502
df1eaa36
VF		 503SUCCESS_DEFAULT = {
504 'zero_return': True,
505 'stderr_empty': True,
506 'timeout': 300,
507 }
508SUCCESS_NONZERO = {
509 'zero_return': False,
510 'stderr_empty': False,
511 'timeout': 300,
512 }
513SUCCESS_STDERR = {
514 'zero_return': True,
515 'stderr_empty': False,
516 'timeout': 300,
517 }
518TEST_LIST = [
519 {
520 'test_id': 1,
521 'test_class': FioJobTest,
522 'job': 't0001-52c58027.fio',
523 'success': SUCCESS_DEFAULT,
524 'pre_job': None,
525 'pre_success': None,
b1bc705e 526 'requirements': [],
df1eaa36
VF		 527 },
528 {
529 'test_id': 2,
530 'test_class': FioJobTest,
531 'job': 't0002-13af05ae-post.fio',
532 'success': SUCCESS_DEFAULT,
533 'pre_job': 't0002-13af05ae-pre.fio',
534 'pre_success': None,
b1bc705e 535 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 536 },
537 {
538 'test_id': 3,
539 'test_class': FioJobTest,
540 'job': 't0003-0ae2c6e1-post.fio',
541 'success': SUCCESS_NONZERO,
542 'pre_job': 't0003-0ae2c6e1-pre.fio',
543 'pre_success': SUCCESS_DEFAULT,
b1bc705e 544 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 545 },
546 {
547 'test_id': 4,
548 'test_class': FioJobTest,
549 'job': 't0004-8a99fdf6.fio',
550 'success': SUCCESS_DEFAULT,
551 'pre_job': None,
552 'pre_success': None,
b1bc705e 553 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 554 },
555 {
556 'test_id': 5,
557 'test_class': FioJobTest_t0005,
558 'job': 't0005-f7078f7b.fio',
559 'success': SUCCESS_DEFAULT,
560 'pre_job': None,
561 'pre_success': None,
562 'output_format': 'json',
b1bc705e 563 'requirements': [],
df1eaa36
VF		 564 },
565 {
566 'test_id': 6,
567 'test_class': FioJobTest_t0006,
568 'job': 't0006-82af2a7c.fio',
569 'success': SUCCESS_DEFAULT,
570 'pre_job': None,
571 'pre_success': None,
572 'output_format': 'json',
b1bc705e 573 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 574 },
575 {
576 'test_id': 7,
577 'test_class': FioJobTest_t0007,
578 'job': 't0007-37cf9e3c.fio',
579 'success': SUCCESS_DEFAULT,
580 'pre_job': None,
581 'pre_success': None,
582 'output_format': 'json',
b1bc705e 583 'requirements': [],
df1eaa36
VF		 584 },
585 {
586 'test_id': 8,
587 'test_class': FioJobTest_t0008,
588 'job': 't0008-ae2fafc8.fio',
589 'success': SUCCESS_DEFAULT,
590 'pre_job': None,
591 'pre_success': None,
592 'output_format': 'json',
b1bc705e 593 'requirements': [],
df1eaa36
VF		 594 },
595 {
596 'test_id': 9,
597 'test_class': FioJobTest_t0009,
598 'job': 't0009-f8b0bd10.fio',
599 'success': SUCCESS_DEFAULT,
600 'pre_job': None,
601 'pre_success': None,
602 'output_format': 'json',
b1bc705e
VF		 603 'requirements': [Requirements.not_macos,
604 Requirements.cpucount4],
605 # mac os does not support CPU affinity
df1eaa36
VF		 606 },
607 {
608 'test_id': 10,
609 'test_class': FioJobTest,
610 'job': 't0010-b7aae4ba.fio',
611 'success': SUCCESS_DEFAULT,
612 'pre_job': None,
613 'pre_success': None,
b1bc705e 614 'requirements': [],
df1eaa36
VF		 615 },
616 {
617 'test_id': 11,
618 'test_class': FioJobTest_t0011,
619 'job': 't0011-5d2788d5.fio',
620 'success': SUCCESS_DEFAULT,
621 'pre_job': None,
622 'pre_success': None,
623 'output_format': 'json',
b1bc705e 624 'requirements': [],
df1eaa36
VF		 625 },
626 {
627 'test_id': 1000,
628 'test_class': FioExeTest,
629 'exe': 't/axmap',
630 'parameters': None,
631 'success': SUCCESS_DEFAULT,
b1bc705e 632 'requirements': [],
df1eaa36
VF		 633 },
634 {
635 'test_id': 1001,
636 'test_class': FioExeTest,
637 'exe': 't/ieee754',
638 'parameters': None,
639 'success': SUCCESS_DEFAULT,
b1bc705e 640 'requirements': [],
df1eaa36
VF		 641 },
642 {
643 'test_id': 1002,
644 'test_class': FioExeTest,
645 'exe': 't/lfsr-test',
646 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
647 'success': SUCCESS_STDERR,
b1bc705e 648 'requirements': [],
df1eaa36
VF		 649 },
650 {
651 'test_id': 1003,
652 'test_class': FioExeTest,
653 'exe': 't/readonly.py',
654 'parameters': ['-f', '{fio_path}'],
655 'success': SUCCESS_DEFAULT,
b1bc705e 656 'requirements': [],
df1eaa36
VF		 657 },
658 {
659 'test_id': 1004,
660 'test_class': FioExeTest,
661 'exe': 't/steadystate_tests.py',
662 'parameters': ['{fio_path}'],
663 'success': SUCCESS_DEFAULT,
b1bc705e 664 'requirements': [],
df1eaa36
VF		 665 },
666 {
667 'test_id': 1005,
668 'test_class': FioExeTest,
669 'exe': 't/stest',
670 'parameters': None,
671 'success': SUCCESS_STDERR,
b1bc705e 672 'requirements': [],
df1eaa36
VF		 673 },
674 {
675 'test_id': 1006,
676 'test_class': FioExeTest,
677 'exe': 't/strided.py',
678 'parameters': ['{fio_path}'],
679 'success': SUCCESS_DEFAULT,
b1bc705e 680 'requirements': [],
df1eaa36
VF		 681 },
682 {
683 'test_id': 1007,
684 'test_class': FioExeTest,
685 'exe': 't/zbd/run-tests-against-regular-nullb',
686 'parameters': None,
687 'success': SUCCESS_DEFAULT,
b1bc705e
VF		 688 'requirements': [Requirements.linux, Requirements.zbd,
689 Requirements.root],
df1eaa36
VF		 690 },
691 {
692 'test_id': 1008,
693 'test_class': FioExeTest,
694 'exe': 't/zbd/run-tests-against-zoned-nullb',
695 'parameters': None,
696 'success': SUCCESS_DEFAULT,
b1bc705e
VF		 697 'requirements': [Requirements.linux, Requirements.zbd,
698 Requirements.root, Requirements.zoned_nullb],
df1eaa36
VF		 699 },
700 {
701 'test_id': 1009,
702 'test_class': FioExeTest,
703 'exe': 'unittests/unittest',
704 'parameters': None,
705 'success': SUCCESS_DEFAULT,
b1bc705e 706 'requirements': [Requirements.unittests],
df1eaa36
VF		 707 },
708]
709
710
711def parse_args():
712 parser = argparse.ArgumentParser()
713 parser.add_argument('-r', '--fio-root',
714 help='fio root path')
715 parser.add_argument('-f', '--fio',
716 help='path to fio executable (e.g., ./fio)')
717 parser.add_argument('-a', '--artifact-root',
718 help='artifact root directory')
719 parser.add_argument('-s', '--skip', nargs='+', type=int,
720 help='list of test(s) to skip')
721 parser.add_argument('-o', '--run-only', nargs='+', type=int,
722 help='list of test(s) to run, skipping all others')
6d5470c3
VF		 723 parser.add_argument('-d', '--debug', action='store_true',
724 help='provide debug output')
b1bc705e
VF		 725 parser.add_argument('-k', '--skip-req', action='store_true',
726 help='skip requirements checking')
df1eaa36
VF		 727 args = parser.parse_args()
728
729 return args
730
731
732def main():
df1eaa36 733 args = parse_args()
6d5470c3
VF		 734 if args.debug:
735 logging.basicConfig(level=logging.DEBUG)
736 else:
737 logging.basicConfig(level=logging.INFO)
738
df1eaa36
VF		 739 if args.fio_root:
740 fio_root = args.fio_root
741 else:
6d5470c3
VF		 742 fio_root = str(Path(__file__).absolute().parent.parent)
743 print("fio root is %s" % fio_root)
df1eaa36
VF		 744
745 if args.fio:
746 fio_path = args.fio
747 else:
742b8799
VF		 748 if platform.system() == "Windows":
749 fio_exe = "fio.exe"
750 else:
751 fio_exe = "fio"
752 fio_path = os.path.join(fio_root, fio_exe)
6d5470c3
VF		 753 print("fio path is %s" % fio_path)
754 if not shutil.which(fio_path):
755 print("Warning: fio executable not found")
df1eaa36
VF		 756
757 artifact_root = args.artifact_root if args.artifact_root else \
758 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
759 os.mkdir(artifact_root)
760 print("Artifact directory is %s" % artifact_root)
761
b1bc705e
VF		 762 if not args.skip_req:
763 req = Requirements(fio_root)
764
df1eaa36
VF		 765 passed = 0
766 failed = 0
767 skipped = 0
768
769 for config in TEST_LIST:
770 if (args.skip and config['test_id'] in args.skip) or \
771 (args.run_only and config['test_id'] not in args.run_only):
772 skipped = skipped + 1
b1bc705e 773 print("Test {0} SKIPPED (User request)".format(config['test_id']))
df1eaa36
VF		 774 continue
775
776 if issubclass(config['test_class'], FioJobTest):
777 if config['pre_job']:
778 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
779 config['pre_job'])
780 else:
781 fio_pre_job = None
782 if config['pre_success']:
783 fio_pre_success = config['pre_success']
784 else:
785 fio_pre_success = None
786 if 'output_format' in config:
787 output_format = config['output_format']
788 else:
789 output_format = 'normal'
790 test = config['test_class'](
791 fio_path,
792 os.path.join(fio_root, 't', 'jobs', config['job']),
793 config['success'],
794 fio_pre_job=fio_pre_job,
795 fio_pre_success=fio_pre_success,
796 output_format=output_format)
797 elif issubclass(config['test_class'], FioExeTest):
798 exe_path = os.path.join(fio_root, config['exe'])
799 if config['parameters']:
800 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
801 else:
802 parameters = None
742b8799
VF		 803 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
804 if parameters:
805 parameters.insert(0, exe_path)
806 else:
807 parameters = [exe_path]
808 exe_path = "python.exe"
df1eaa36
VF		 809 test = config['test_class'](exe_path, parameters,
810 config['success'])
811 else:
812 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
813 failed = failed + 1
814 continue
815
b1bc705e
VF		 816 if not args.skip_req:
817 skip = False
818 for req in config['requirements']:
819 ok, reason = req()
820 skip = not ok
285622dc 821 logging.debug("Test %d: Requirement '%s' met? %s" % (config['test_id'], reason, ok))
b1bc705e
VF		 822 if skip:
823 break
824 if skip:
825 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
826 skipped = skipped + 1
827 continue
828
df1eaa36
VF		 829 test.setup(artifact_root, config['test_id'])
830 test.run()
831 test.check_result()
832 if test.passed:
833 result = "PASSED"
834 passed = passed + 1
835 else:
836 result = "FAILED: {0}".format(test.failure_reason)
837 failed = failed + 1
6d5470c3 838 with open(test.stderr_file, "r") as stderr_file:
285622dc 839 logging.debug("Test %d: stderr:\n%s" % (config['test_id'], stderr_file.read()))
6d5470c3 840 with open(test.stdout_file, "r") as stdout_file:
285622dc 841 logging.debug("Test %d: stdout:\n%s" % (config['test_id'], stdout_file.read()))
df1eaa36
VF		 842 print("Test {0} {1}".format(config['test_id'], result))
843
844 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
845
846 sys.exit(failed)
847
848
849if __name__ == '__main__':
850 main()
fio - Flexible IO Tester