projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'json1' of https://github.com/kusumi/fio
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
b048455f 17# # git clone git://git.kernel.dk/fio.git
df1eaa36
VF		 18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
b1bc705e 24# - Python 3.5 (subprocess.run)
df1eaa36
VF		 25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
df1eaa36
VF		 42#
43
44import os
45import sys
46import json
47import time
6d5470c3 48import shutil
df1eaa36
VF		 49import logging
50import argparse
b1bc705e 51import platform
df1eaa36 52import subprocess
b1bc705e 53import multiprocessing
df1eaa36
VF		 54from pathlib import Path
55
56
57class FioTest(object):
58 """Base for all fio tests."""
59
60 def __init__(self, exe_path, parameters, success):
61 self.exe_path = exe_path
62 self.parameters = parameters
63 self.success = success
64 self.output = {}
65 self.artifact_root = None
66 self.testnum = None
67 self.test_dir = None
68 self.passed = True
69 self.failure_reason = ''
70
71 def setup(self, artifact_root, testnum):
72 self.artifact_root = artifact_root
73 self.testnum = testnum
74 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
75 if not os.path.exists(self.test_dir):
76 os.mkdir(self.test_dir)
77
78 self.command_file = os.path.join(
79 self.test_dir,
80 "{0}.command".format(os.path.basename(self.exe_path)))
81 self.stdout_file = os.path.join(
82 self.test_dir,
83 "{0}.stdout".format(os.path.basename(self.exe_path)))
84 self.stderr_file = os.path.join(
85 self.test_dir,
86 "{0}.stderr".format(os.path.basename(self.exe_path)))
87 self.exticode_file = os.path.join(
88 self.test_dir,
89 "{0}.exitcode".format(os.path.basename(self.exe_path)))
90
91 def run(self):
92 raise NotImplementedError()
93
94 def check_result(self):
95 raise NotImplementedError()
96
97
98class FioExeTest(FioTest):
99 """Test consists of an executable binary or script"""
100
101 def __init__(self, exe_path, parameters, success):
102 """Construct a FioExeTest which is a FioTest consisting of an
103 executable binary or script.
104
105 exe_path: location of executable binary or script
106 parameters: list of parameters for executable
107 success: Definition of test success
108 """
109
110 FioTest.__init__(self, exe_path, parameters, success)
111
112 def setup(self, artifact_root, testnum):
113 super(FioExeTest, self).setup(artifact_root, testnum)
114
115 def run(self):
116 if self.parameters:
117 command = [self.exe_path] + self.parameters
118 else:
119 command = [self.exe_path]
120 command_file = open(self.command_file, "w+")
121 command_file.write("%s\n" % command)
122 command_file.close()
123
124 stdout_file = open(self.stdout_file, "w+")
125 stderr_file = open(self.stderr_file, "w+")
126 exticode_file = open(self.exticode_file, "w+")
127 try:
b048455f 128 proc = None
df1eaa36
VF		 129 # Avoid using subprocess.run() here because when a timeout occurs,
130 # fio will be stopped with SIGKILL. This does not give fio a
131 # chance to clean up and means that child processes may continue
132 # running and submitting IO.
133 proc = subprocess.Popen(command,
134 stdout=stdout_file,
135 stderr=stderr_file,
136 cwd=self.test_dir,
137 universal_newlines=True)
138 proc.communicate(timeout=self.success['timeout'])
139 exticode_file.write('{0}\n'.format(proc.returncode))
285622dc 140 logging.debug("Test %d: return code: %d" % (self.testnum, proc.returncode))
df1eaa36
VF		 141 self.output['proc'] = proc
142 except subprocess.TimeoutExpired:
143 proc.terminate()
144 proc.communicate()
145 assert proc.poll()
146 self.output['failure'] = 'timeout'
147 except Exception:
b048455f
VF		 148 if proc:
149 if not proc.poll():
150 proc.terminate()
151 proc.communicate()
df1eaa36
VF		 152 self.output['failure'] = 'exception'
153 self.output['exc_info'] = sys.exc_info()
154 finally:
155 stdout_file.close()
156 stderr_file.close()
157 exticode_file.close()
158
159 def check_result(self):
160 if 'proc' not in self.output:
161 if self.output['failure'] == 'timeout':
162 self.failure_reason = "{0} timeout,".format(self.failure_reason)
163 else:
164 assert self.output['failure'] == 'exception'
165 self.failure_reason = '{0} exception: {1}, {2}'.format(
166 self.failure_reason, self.output['exc_info'][0],
167 self.output['exc_info'][1])
168
169 self.passed = False
170 return
171
172 if 'zero_return' in self.success:
173 if self.success['zero_return']:
174 if self.output['proc'].returncode != 0:
175 self.passed = False
176 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
177 else:
178 if self.output['proc'].returncode == 0:
179 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
180 self.passed = False
181
6d5470c3 182 stderr_size = os.path.getsize(self.stderr_file)
df1eaa36 183 if 'stderr_empty' in self.success:
df1eaa36
VF		 184 if self.success['stderr_empty']:
185 if stderr_size != 0:
186 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
187 self.passed = False
188 else:
189 if stderr_size == 0:
190 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
191 self.passed = False
192
193
194class FioJobTest(FioExeTest):
195 """Test consists of a fio job"""
196
197 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
198 fio_pre_success=None, output_format="normal"):
199 """Construct a FioJobTest which is a FioExeTest consisting of a
200 single fio job file with an optional setup step.
201
202 fio_path: location of fio executable
203 fio_job: location of fio job file
204 success: Definition of test success
205 fio_pre_job: fio job for preconditioning
206 fio_pre_success: Definition of test success for fio precon job
207 output_format: normal (default), json, jsonplus, or terse
208 """
209
210 self.fio_job = fio_job
211 self.fio_pre_job = fio_pre_job
212 self.fio_pre_success = fio_pre_success if fio_pre_success else success
213 self.output_format = output_format
214 self.precon_failed = False
215 self.json_data = None
216 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
217 self.fio_args = [
218 "--output-format={0}".format(self.output_format),
219 "--output={0}".format(self.fio_output),
220 self.fio_job,
221 ]
222 FioExeTest.__init__(self, fio_path, self.fio_args, success)
223
224 def setup(self, artifact_root, testnum):
225 super(FioJobTest, self).setup(artifact_root, testnum)
226
227 self.command_file = os.path.join(
228 self.test_dir,
229 "{0}.command".format(os.path.basename(self.fio_job)))
230 self.stdout_file = os.path.join(
231 self.test_dir,
232 "{0}.stdout".format(os.path.basename(self.fio_job)))
233 self.stderr_file = os.path.join(
234 self.test_dir,
235 "{0}.stderr".format(os.path.basename(self.fio_job)))
236 self.exticode_file = os.path.join(
237 self.test_dir,
238 "{0}.exitcode".format(os.path.basename(self.fio_job)))
239
240 def run_pre_job(self):
241 precon = FioJobTest(self.exe_path, self.fio_pre_job,
242 self.fio_pre_success,
243 output_format=self.output_format)
244 precon.setup(self.artifact_root, self.testnum)
245 precon.run()
246 precon.check_result()
247 self.precon_failed = not precon.passed
248 self.failure_reason = precon.failure_reason
249
250 def run(self):
251 if self.fio_pre_job:
252 self.run_pre_job()
253
254 if not self.precon_failed:
255 super(FioJobTest, self).run()
256 else:
285622dc 257 logging.debug("Test %d: precondition step failed" % self.testnum)
df1eaa36
VF		 258
259 def check_result(self):
260 if self.precon_failed:
261 self.passed = False
262 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
263 return
264
265 super(FioJobTest, self).check_result()
266
6d5470c3
VF		 267 if not self.passed:
268 return
269
742b8799
VF		 270 if not 'json' in self.output_format:
271 return
272
273 try:
274 with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file:
275 file_data = output_file.read()
276 except EnvironmentError:
277 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
278 self.passed = False
279 return
280
281 #
282 # Sometimes fio informational messages are included at the top of the
283 # JSON output, especially under Windows. Try to decode output as JSON
284 # data, lopping off up to the first four lines
285 #
286 lines = file_data.splitlines()
287 for i in range(5):
288 file_data = '\n'.join(lines[i:])
df1eaa36 289 try:
742b8799
VF		 290 self.json_data = json.loads(file_data)
291 except json.JSONDecodeError:
292 continue
6d5470c3 293 else:
285622dc 294 logging.debug("Test %d: skipped %d lines decoding JSON data" % (self.testnum, i))
742b8799
VF		 295 return
296
297 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
298 self.passed = False
df1eaa36
VF		 299
300
301class FioJobTest_t0005(FioJobTest):
302 """Test consists of fio test job t0005
303 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
304
305 def check_result(self):
306 super(FioJobTest_t0005, self).check_result()
307
308 if not self.passed:
309 return
310
311 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
312 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
313 self.passed = False
314 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
315 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
316 self.passed = False
317
318
319class FioJobTest_t0006(FioJobTest):
320 """Test consists of fio test job t0006
321 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
322
323 def check_result(self):
324 super(FioJobTest_t0006, self).check_result()
325
326 if not self.passed:
327 return
328
329 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
330 / self.json_data['jobs'][0]['write']['io_kbytes']
285622dc 331 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
df1eaa36
VF		 332 if ratio < 1.99 or ratio > 2.01:
333 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
334 self.passed = False
335
336
337class FioJobTest_t0007(FioJobTest):
338 """Test consists of fio test job t0007
339 Confirm that read['io_kbytes'] = 87040"""
340
341 def check_result(self):
342 super(FioJobTest_t0007, self).check_result()
343
344 if not self.passed:
345 return
346
347 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
348 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
349 self.passed = False
350
351
352class FioJobTest_t0008(FioJobTest):
353 """Test consists of fio test job t0008
354 Confirm that read['io_kbytes'] = 32768 and that
355 write['io_kbytes'] ~ 16568
356
357 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
358 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
359
360 def check_result(self):
361 super(FioJobTest_t0008, self).check_result()
362
363 if not self.passed:
364 return
365
366 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
285622dc 367 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
df1eaa36
VF		 368
369 if ratio < 0.99 or ratio > 1.01:
370 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
371 self.passed = False
372 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
373 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
374 self.passed = False
375
376
377class FioJobTest_t0009(FioJobTest):
378 """Test consists of fio test job t0009
379 Confirm that runtime >= 60s"""
380
381 def check_result(self):
382 super(FioJobTest_t0009, self).check_result()
383
384 if not self.passed:
385 return
386
285622dc 387 logging.debug('Test %d: elapsed: %d' % (self.testnum, self.json_data['jobs'][0]['elapsed']))
df1eaa36
VF		 388
389 if self.json_data['jobs'][0]['elapsed'] < 60:
390 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
391 self.passed = False
392
393
394class FioJobTest_t0011(FioJobTest):
395 """Test consists of fio test job t0009
396 Confirm that job0 iops == 1000
397 and that job1_iops / job0_iops ~ 8
398 With two runs of fio-3.16 I observed a ratio of 8.3"""
399
400 def check_result(self):
401 super(FioJobTest_t0011, self).check_result()
402
403 if not self.passed:
404 return
405
406 iops1 = self.json_data['jobs'][0]['read']['iops']
407 iops2 = self.json_data['jobs'][1]['read']['iops']
408 ratio = iops2 / iops1
41ceb6c7 409 logging.debug("Test %d: iops1: %f" % (self.testnum, iops1))
285622dc 410 logging.debug("Test %d: ratio: %f" % (self.testnum, ratio))
df1eaa36 411
41ceb6c7 412 if iops1 < 998 or iops1 > 1002:
df1eaa36
VF		 413 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
414 self.passed = False
415
416 if ratio < 7 or ratio > 9:
417 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
418 self.passed = False
419
420
b1bc705e
VF		 421class Requirements(object):
422 """Requirements consists of multiple run environment characteristics.
423 These are to determine if a particular test can be run"""
424
425 _linux = False
426 _libaio = False
427 _zbd = False
428 _root = False
429 _zoned_nullb = False
430 _not_macos = False
431 _unittests = False
432 _cpucount4 = False
433
434 def __init__(self, fio_root):
435 Requirements._not_macos = platform.system() != "Darwin"
436 Requirements._linux = platform.system() == "Linux"
437
438 if Requirements._linux:
439 try:
440 config_file = os.path.join(fio_root, "config-host.h")
441 with open(config_file, "r") as config:
442 contents = config.read()
443 except Exception:
444 print("Unable to open {0} to check requirements".format(config_file))
445 Requirements._zbd = True
446 else:
447 Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents
448 Requirements._libaio = "CONFIG_LIBAIO" in contents
449
450 Requirements._root = (os.geteuid() == 0)
451 if Requirements._zbd and Requirements._root:
452 subprocess.run(["modprobe", "null_blk"],
453 stdout=subprocess.PIPE,
454 stderr=subprocess.PIPE)
455 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
456 Requirements._zoned_nullb = True
457
742b8799
VF		 458 if platform.system() == "Windows":
459 utest_exe = "unittest.exe"
460 else:
461 utest_exe = "unittest"
462 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
b1bc705e
VF		 463 Requirements._unittests = os.path.exists(unittest_path)
464
465 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
466
467 req_list = [Requirements.linux,
468 Requirements.libaio,
469 Requirements.zbd,
470 Requirements.root,
471 Requirements.zoned_nullb,
472 Requirements.not_macos,
473 Requirements.unittests,
474 Requirements.cpucount4]
475 for req in req_list:
476 value, desc = req()
285622dc 477 logging.debug("Requirements: Requirement '%s' met? %s" % (desc, value))
b1bc705e
VF		 478
479 def linux():
480 return Requirements._linux, "Linux required"
481
482 def libaio():
483 return Requirements._libaio, "libaio required"
484
485 def zbd():
486 return Requirements._zbd, "Zoned block device support required"
487
488 def root():
489 return Requirements._root, "root required"
490
491 def zoned_nullb():
492 return Requirements._zoned_nullb, "Zoned null block device support required"
493
494 def not_macos():
495 return Requirements._not_macos, "platform other than macOS required"
496
497 def unittests():
498 return Requirements._unittests, "Unittests support required"
499
500 def cpucount4():
501 return Requirements._cpucount4, "4+ CPUs required"
502
503
df1eaa36
VF		 504SUCCESS_DEFAULT = {
505 'zero_return': True,
506 'stderr_empty': True,
507 'timeout': 300,
508 }
509SUCCESS_NONZERO = {
510 'zero_return': False,
511 'stderr_empty': False,
512 'timeout': 300,
513 }
514SUCCESS_STDERR = {
515 'zero_return': True,
516 'stderr_empty': False,
517 'timeout': 300,
518 }
519TEST_LIST = [
520 {
521 'test_id': 1,
522 'test_class': FioJobTest,
523 'job': 't0001-52c58027.fio',
524 'success': SUCCESS_DEFAULT,
525 'pre_job': None,
526 'pre_success': None,
b1bc705e 527 'requirements': [],
df1eaa36
VF		 528 },
529 {
530 'test_id': 2,
531 'test_class': FioJobTest,
532 'job': 't0002-13af05ae-post.fio',
533 'success': SUCCESS_DEFAULT,
534 'pre_job': 't0002-13af05ae-pre.fio',
535 'pre_success': None,
b1bc705e 536 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 537 },
538 {
539 'test_id': 3,
540 'test_class': FioJobTest,
541 'job': 't0003-0ae2c6e1-post.fio',
542 'success': SUCCESS_NONZERO,
543 'pre_job': 't0003-0ae2c6e1-pre.fio',
544 'pre_success': SUCCESS_DEFAULT,
b1bc705e 545 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 546 },
547 {
548 'test_id': 4,
549 'test_class': FioJobTest,
550 'job': 't0004-8a99fdf6.fio',
551 'success': SUCCESS_DEFAULT,
552 'pre_job': None,
553 'pre_success': None,
b1bc705e 554 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 555 },
556 {
557 'test_id': 5,
558 'test_class': FioJobTest_t0005,
559 'job': 't0005-f7078f7b.fio',
560 'success': SUCCESS_DEFAULT,
561 'pre_job': None,
562 'pre_success': None,
563 'output_format': 'json',
b1bc705e 564 'requirements': [],
df1eaa36
VF		 565 },
566 {
567 'test_id': 6,
568 'test_class': FioJobTest_t0006,
569 'job': 't0006-82af2a7c.fio',
570 'success': SUCCESS_DEFAULT,
571 'pre_job': None,
572 'pre_success': None,
573 'output_format': 'json',
b1bc705e 574 'requirements': [Requirements.linux, Requirements.libaio],
df1eaa36
VF		 575 },
576 {
577 'test_id': 7,
578 'test_class': FioJobTest_t0007,
579 'job': 't0007-37cf9e3c.fio',
580 'success': SUCCESS_DEFAULT,
581 'pre_job': None,
582 'pre_success': None,
583 'output_format': 'json',
b1bc705e 584 'requirements': [],
df1eaa36
VF		 585 },
586 {
587 'test_id': 8,
588 'test_class': FioJobTest_t0008,
589 'job': 't0008-ae2fafc8.fio',
590 'success': SUCCESS_DEFAULT,
591 'pre_job': None,
592 'pre_success': None,
593 'output_format': 'json',
b1bc705e 594 'requirements': [],
df1eaa36
VF		 595 },
596 {
597 'test_id': 9,
598 'test_class': FioJobTest_t0009,
599 'job': 't0009-f8b0bd10.fio',
600 'success': SUCCESS_DEFAULT,
601 'pre_job': None,
602 'pre_success': None,
603 'output_format': 'json',
b1bc705e
VF		 604 'requirements': [Requirements.not_macos,
605 Requirements.cpucount4],
606 # mac os does not support CPU affinity
df1eaa36
VF		 607 },
608 {
609 'test_id': 10,
610 'test_class': FioJobTest,
611 'job': 't0010-b7aae4ba.fio',
612 'success': SUCCESS_DEFAULT,
613 'pre_job': None,
614 'pre_success': None,
b1bc705e 615 'requirements': [],
df1eaa36
VF		 616 },
617 {
618 'test_id': 11,
619 'test_class': FioJobTest_t0011,
620 'job': 't0011-5d2788d5.fio',
621 'success': SUCCESS_DEFAULT,
622 'pre_job': None,
623 'pre_success': None,
624 'output_format': 'json',
b1bc705e 625 'requirements': [],
df1eaa36
VF		 626 },
627 {
628 'test_id': 1000,
629 'test_class': FioExeTest,
630 'exe': 't/axmap',
631 'parameters': None,
632 'success': SUCCESS_DEFAULT,
b1bc705e 633 'requirements': [],
df1eaa36
VF		 634 },
635 {
636 'test_id': 1001,
637 'test_class': FioExeTest,
638 'exe': 't/ieee754',
639 'parameters': None,
640 'success': SUCCESS_DEFAULT,
b1bc705e 641 'requirements': [],
df1eaa36
VF		 642 },
643 {
644 'test_id': 1002,
645 'test_class': FioExeTest,
646 'exe': 't/lfsr-test',
647 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
648 'success': SUCCESS_STDERR,
b1bc705e 649 'requirements': [],
df1eaa36
VF		 650 },
651 {
652 'test_id': 1003,
653 'test_class': FioExeTest,
654 'exe': 't/readonly.py',
655 'parameters': ['-f', '{fio_path}'],
656 'success': SUCCESS_DEFAULT,
b1bc705e 657 'requirements': [],
df1eaa36
VF		 658 },
659 {
660 'test_id': 1004,
661 'test_class': FioExeTest,
662 'exe': 't/steadystate_tests.py',
663 'parameters': ['{fio_path}'],
664 'success': SUCCESS_DEFAULT,
b1bc705e 665 'requirements': [],
df1eaa36
VF		 666 },
667 {
668 'test_id': 1005,
669 'test_class': FioExeTest,
670 'exe': 't/stest',
671 'parameters': None,
672 'success': SUCCESS_STDERR,
b1bc705e 673 'requirements': [],
df1eaa36
VF		 674 },
675 {
676 'test_id': 1006,
677 'test_class': FioExeTest,
678 'exe': 't/strided.py',
679 'parameters': ['{fio_path}'],
680 'success': SUCCESS_DEFAULT,
b1bc705e 681 'requirements': [],
df1eaa36
VF		 682 },
683 {
684 'test_id': 1007,
685 'test_class': FioExeTest,
686 'exe': 't/zbd/run-tests-against-regular-nullb',
687 'parameters': None,
688 'success': SUCCESS_DEFAULT,
b1bc705e
VF		 689 'requirements': [Requirements.linux, Requirements.zbd,
690 Requirements.root],
df1eaa36
VF		 691 },
692 {
693 'test_id': 1008,
694 'test_class': FioExeTest,
695 'exe': 't/zbd/run-tests-against-zoned-nullb',
696 'parameters': None,
697 'success': SUCCESS_DEFAULT,
b1bc705e
VF		 698 'requirements': [Requirements.linux, Requirements.zbd,
699 Requirements.root, Requirements.zoned_nullb],
df1eaa36
VF		 700 },
701 {
702 'test_id': 1009,
703 'test_class': FioExeTest,
704 'exe': 'unittests/unittest',
705 'parameters': None,
706 'success': SUCCESS_DEFAULT,
b1bc705e 707 'requirements': [Requirements.unittests],
df1eaa36
VF		 708 },
709]
710
711
712def parse_args():
713 parser = argparse.ArgumentParser()
714 parser.add_argument('-r', '--fio-root',
715 help='fio root path')
716 parser.add_argument('-f', '--fio',
717 help='path to fio executable (e.g., ./fio)')
718 parser.add_argument('-a', '--artifact-root',
719 help='artifact root directory')
720 parser.add_argument('-s', '--skip', nargs='+', type=int,
721 help='list of test(s) to skip')
722 parser.add_argument('-o', '--run-only', nargs='+', type=int,
723 help='list of test(s) to run, skipping all others')
6d5470c3
VF		 724 parser.add_argument('-d', '--debug', action='store_true',
725 help='provide debug output')
b1bc705e
VF		 726 parser.add_argument('-k', '--skip-req', action='store_true',
727 help='skip requirements checking')
df1eaa36
VF		 728 args = parser.parse_args()
729
730 return args
731
732
733def main():
df1eaa36 734 args = parse_args()
6d5470c3
VF		 735 if args.debug:
736 logging.basicConfig(level=logging.DEBUG)
737 else:
738 logging.basicConfig(level=logging.INFO)
739
df1eaa36
VF		 740 if args.fio_root:
741 fio_root = args.fio_root
742 else:
6d5470c3
VF		 743 fio_root = str(Path(__file__).absolute().parent.parent)
744 print("fio root is %s" % fio_root)
df1eaa36
VF		 745
746 if args.fio:
747 fio_path = args.fio
748 else:
742b8799
VF		 749 if platform.system() == "Windows":
750 fio_exe = "fio.exe"
751 else:
752 fio_exe = "fio"
753 fio_path = os.path.join(fio_root, fio_exe)
6d5470c3
VF		 754 print("fio path is %s" % fio_path)
755 if not shutil.which(fio_path):
756 print("Warning: fio executable not found")
df1eaa36
VF		 757
758 artifact_root = args.artifact_root if args.artifact_root else \
759 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
760 os.mkdir(artifact_root)
761 print("Artifact directory is %s" % artifact_root)
762
b1bc705e
VF		 763 if not args.skip_req:
764 req = Requirements(fio_root)
765
df1eaa36
VF		 766 passed = 0
767 failed = 0
768 skipped = 0
769
770 for config in TEST_LIST:
771 if (args.skip and config['test_id'] in args.skip) or \
772 (args.run_only and config['test_id'] not in args.run_only):
773 skipped = skipped + 1
b1bc705e 774 print("Test {0} SKIPPED (User request)".format(config['test_id']))
df1eaa36
VF		 775 continue
776
777 if issubclass(config['test_class'], FioJobTest):
778 if config['pre_job']:
779 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
780 config['pre_job'])
781 else:
782 fio_pre_job = None
783 if config['pre_success']:
784 fio_pre_success = config['pre_success']
785 else:
786 fio_pre_success = None
787 if 'output_format' in config:
788 output_format = config['output_format']
789 else:
790 output_format = 'normal'
791 test = config['test_class'](
792 fio_path,
793 os.path.join(fio_root, 't', 'jobs', config['job']),
794 config['success'],
795 fio_pre_job=fio_pre_job,
796 fio_pre_success=fio_pre_success,
797 output_format=output_format)
798 elif issubclass(config['test_class'], FioExeTest):
799 exe_path = os.path.join(fio_root, config['exe'])
800 if config['parameters']:
801 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
802 else:
803 parameters = None
742b8799
VF		 804 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
805 if parameters:
806 parameters.insert(0, exe_path)
807 else:
808 parameters = [exe_path]
809 exe_path = "python.exe"
df1eaa36
VF		 810 test = config['test_class'](exe_path, parameters,
811 config['success'])
812 else:
813 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
814 failed = failed + 1
815 continue
816
b1bc705e
VF		 817 if not args.skip_req:
818 skip = False
819 for req in config['requirements']:
820 ok, reason = req()
821 skip = not ok
285622dc 822 logging.debug("Test %d: Requirement '%s' met? %s" % (config['test_id'], reason, ok))
b1bc705e
VF		 823 if skip:
824 break
825 if skip:
826 print("Test {0} SKIPPED ({1})".format(config['test_id'], reason))
827 skipped = skipped + 1
828 continue
829
df1eaa36
VF		 830 test.setup(artifact_root, config['test_id'])
831 test.run()
832 test.check_result()
833 if test.passed:
834 result = "PASSED"
835 passed = passed + 1
836 else:
837 result = "FAILED: {0}".format(test.failure_reason)
838 failed = failed + 1
6d5470c3 839 with open(test.stderr_file, "r") as stderr_file:
285622dc 840 logging.debug("Test %d: stderr:\n%s" % (config['test_id'], stderr_file.read()))
6d5470c3 841 with open(test.stdout_file, "r") as stdout_file:
285622dc 842 logging.debug("Test %d: stdout:\n%s" % (config['test_id'], stdout_file.read()))
df1eaa36
VF		 843 print("Test {0} {1}".format(config['test_id'], result))
844
845 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
846
847 sys.exit(failed)
848
849
850if __name__ == '__main__':
851 main()
fio - Flexible IO Tester