Commit | Line | Data |
---|---|---|
df1eaa36 VF |
1 | #!/usr/bin/env python3 |
2 | # SPDX-License-Identifier: GPL-2.0-only | |
3 | # | |
4 | # Copyright (c) 2019 Western Digital Corporation or its affiliates. | |
5 | # | |
6 | """ | |
7 | # run-fio-tests.py | |
8 | # | |
9 | # Automate running of fio tests | |
10 | # | |
11 | # USAGE | |
12 | # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root] | |
13 | # [--skip # # #...] [--run-only # # #...] | |
14 | # | |
15 | # | |
16 | # EXAMPLE | |
b048455f | 17 | # # git clone git://git.kernel.dk/fio.git |
df1eaa36 VF |
18 | # # cd fio |
19 | # # make -j | |
20 | # # python3 t/run-fio-tests.py | |
21 | # | |
22 | # | |
23 | # REQUIREMENTS | |
b1bc705e | 24 | # - Python 3.5 (subprocess.run) |
df1eaa36 VF |
25 | # - Linux (libaio ioengine, zbd tests, etc) |
26 | # - The artifact directory must be on a file system that accepts 512-byte IO | |
27 | # (t0002, t0003, t0004). | |
28 | # - The artifact directory needs to be on an SSD. Otherwise tests that carry | |
29 | # out file-based IO will trigger a timeout (t0006). | |
30 | # - 4 CPUs (t0009) | |
31 | # - SciPy (steadystate_tests.py) | |
32 | # - libzbc (zbd tests) | |
33 | # - root privileges (zbd test) | |
34 | # - kernel 4.19 or later for zoned null block devices (zbd tests) | |
35 | # - CUnit support (unittests) | |
36 | # | |
37 | """ | |
38 | ||
39 | # | |
40 | # TODO run multiple tests simultaneously | |
41 | # TODO Add sgunmap tests (requires SAS SSD) | |
df1eaa36 VF |
42 | # |
43 | ||
44 | import os | |
45 | import sys | |
46 | import json | |
47 | import time | |
6d5470c3 | 48 | import shutil |
df1eaa36 VF |
49 | import logging |
50 | import argparse | |
b1bc705e | 51 | import platform |
df1eaa36 | 52 | import subprocess |
b1bc705e | 53 | import multiprocessing |
df1eaa36 VF |
54 | from pathlib import Path |
55 | ||
56 | ||
57 | class FioTest(object): | |
58 | """Base for all fio tests.""" | |
59 | ||
60 | def __init__(self, exe_path, parameters, success): | |
61 | self.exe_path = exe_path | |
62 | self.parameters = parameters | |
63 | self.success = success | |
64 | self.output = {} | |
65 | self.artifact_root = None | |
66 | self.testnum = None | |
67 | self.test_dir = None | |
68 | self.passed = True | |
69 | self.failure_reason = '' | |
704cc4df VF |
70 | self.command_file = None |
71 | self.stdout_file = None | |
72 | self.stderr_file = None | |
73 | self.exitcode_file = None | |
df1eaa36 VF |
74 | |
75 | def setup(self, artifact_root, testnum): | |
704cc4df VF |
76 | """Setup instance variables for test.""" |
77 | ||
df1eaa36 VF |
78 | self.artifact_root = artifact_root |
79 | self.testnum = testnum | |
80 | self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum)) | |
81 | if not os.path.exists(self.test_dir): | |
82 | os.mkdir(self.test_dir) | |
83 | ||
84 | self.command_file = os.path.join( | |
704cc4df VF |
85 | self.test_dir, |
86 | "{0}.command".format(os.path.basename(self.exe_path))) | |
df1eaa36 | 87 | self.stdout_file = os.path.join( |
704cc4df VF |
88 | self.test_dir, |
89 | "{0}.stdout".format(os.path.basename(self.exe_path))) | |
df1eaa36 | 90 | self.stderr_file = os.path.join( |
704cc4df VF |
91 | self.test_dir, |
92 | "{0}.stderr".format(os.path.basename(self.exe_path))) | |
93 | self.exitcode_file = os.path.join( | |
94 | self.test_dir, | |
95 | "{0}.exitcode".format(os.path.basename(self.exe_path))) | |
df1eaa36 VF |
96 | |
97 | def run(self): | |
704cc4df VF |
98 | """Run the test.""" |
99 | ||
df1eaa36 VF |
100 | raise NotImplementedError() |
101 | ||
102 | def check_result(self): | |
704cc4df VF |
103 | """Check test results.""" |
104 | ||
df1eaa36 VF |
105 | raise NotImplementedError() |
106 | ||
107 | ||
108 | class FioExeTest(FioTest): | |
109 | """Test consists of an executable binary or script""" | |
110 | ||
111 | def __init__(self, exe_path, parameters, success): | |
112 | """Construct a FioExeTest which is a FioTest consisting of an | |
113 | executable binary or script. | |
114 | ||
115 | exe_path: location of executable binary or script | |
116 | parameters: list of parameters for executable | |
117 | success: Definition of test success | |
118 | """ | |
119 | ||
120 | FioTest.__init__(self, exe_path, parameters, success) | |
121 | ||
df1eaa36 | 122 | def run(self): |
704cc4df VF |
123 | """Execute the binary or script described by this instance.""" |
124 | ||
df1eaa36 VF |
125 | if self.parameters: |
126 | command = [self.exe_path] + self.parameters | |
127 | else: | |
128 | command = [self.exe_path] | |
129 | command_file = open(self.command_file, "w+") | |
130 | command_file.write("%s\n" % command) | |
131 | command_file.close() | |
132 | ||
133 | stdout_file = open(self.stdout_file, "w+") | |
134 | stderr_file = open(self.stderr_file, "w+") | |
704cc4df | 135 | exitcode_file = open(self.exitcode_file, "w+") |
df1eaa36 | 136 | try: |
b048455f | 137 | proc = None |
df1eaa36 VF |
138 | # Avoid using subprocess.run() here because when a timeout occurs, |
139 | # fio will be stopped with SIGKILL. This does not give fio a | |
140 | # chance to clean up and means that child processes may continue | |
141 | # running and submitting IO. | |
142 | proc = subprocess.Popen(command, | |
143 | stdout=stdout_file, | |
144 | stderr=stderr_file, | |
145 | cwd=self.test_dir, | |
146 | universal_newlines=True) | |
147 | proc.communicate(timeout=self.success['timeout']) | |
704cc4df VF |
148 | exitcode_file.write('{0}\n'.format(proc.returncode)) |
149 | logging.debug("Test %d: return code: %d", self.testnum, proc.returncode) | |
df1eaa36 VF |
150 | self.output['proc'] = proc |
151 | except subprocess.TimeoutExpired: | |
152 | proc.terminate() | |
153 | proc.communicate() | |
154 | assert proc.poll() | |
155 | self.output['failure'] = 'timeout' | |
156 | except Exception: | |
b048455f VF |
157 | if proc: |
158 | if not proc.poll(): | |
159 | proc.terminate() | |
160 | proc.communicate() | |
df1eaa36 VF |
161 | self.output['failure'] = 'exception' |
162 | self.output['exc_info'] = sys.exc_info() | |
163 | finally: | |
164 | stdout_file.close() | |
165 | stderr_file.close() | |
704cc4df | 166 | exitcode_file.close() |
df1eaa36 VF |
167 | |
168 | def check_result(self): | |
704cc4df VF |
169 | """Check results of test run.""" |
170 | ||
df1eaa36 VF |
171 | if 'proc' not in self.output: |
172 | if self.output['failure'] == 'timeout': | |
173 | self.failure_reason = "{0} timeout,".format(self.failure_reason) | |
174 | else: | |
175 | assert self.output['failure'] == 'exception' | |
176 | self.failure_reason = '{0} exception: {1}, {2}'.format( | |
704cc4df VF |
177 | self.failure_reason, self.output['exc_info'][0], |
178 | self.output['exc_info'][1]) | |
df1eaa36 VF |
179 | |
180 | self.passed = False | |
181 | return | |
182 | ||
183 | if 'zero_return' in self.success: | |
184 | if self.success['zero_return']: | |
185 | if self.output['proc'].returncode != 0: | |
186 | self.passed = False | |
187 | self.failure_reason = "{0} non-zero return code,".format(self.failure_reason) | |
188 | else: | |
189 | if self.output['proc'].returncode == 0: | |
190 | self.failure_reason = "{0} zero return code,".format(self.failure_reason) | |
191 | self.passed = False | |
192 | ||
6d5470c3 | 193 | stderr_size = os.path.getsize(self.stderr_file) |
df1eaa36 | 194 | if 'stderr_empty' in self.success: |
df1eaa36 VF |
195 | if self.success['stderr_empty']: |
196 | if stderr_size != 0: | |
197 | self.failure_reason = "{0} stderr not empty,".format(self.failure_reason) | |
198 | self.passed = False | |
199 | else: | |
200 | if stderr_size == 0: | |
201 | self.failure_reason = "{0} stderr empty,".format(self.failure_reason) | |
202 | self.passed = False | |
203 | ||
204 | ||
205 | class FioJobTest(FioExeTest): | |
206 | """Test consists of a fio job""" | |
207 | ||
208 | def __init__(self, fio_path, fio_job, success, fio_pre_job=None, | |
209 | fio_pre_success=None, output_format="normal"): | |
210 | """Construct a FioJobTest which is a FioExeTest consisting of a | |
211 | single fio job file with an optional setup step. | |
212 | ||
213 | fio_path: location of fio executable | |
214 | fio_job: location of fio job file | |
215 | success: Definition of test success | |
216 | fio_pre_job: fio job for preconditioning | |
217 | fio_pre_success: Definition of test success for fio precon job | |
218 | output_format: normal (default), json, jsonplus, or terse | |
219 | """ | |
220 | ||
221 | self.fio_job = fio_job | |
222 | self.fio_pre_job = fio_pre_job | |
223 | self.fio_pre_success = fio_pre_success if fio_pre_success else success | |
224 | self.output_format = output_format | |
225 | self.precon_failed = False | |
226 | self.json_data = None | |
227 | self.fio_output = "{0}.output".format(os.path.basename(self.fio_job)) | |
228 | self.fio_args = [ | |
229 | "--output-format={0}".format(self.output_format), | |
230 | "--output={0}".format(self.fio_output), | |
231 | self.fio_job, | |
232 | ] | |
233 | FioExeTest.__init__(self, fio_path, self.fio_args, success) | |
234 | ||
235 | def setup(self, artifact_root, testnum): | |
704cc4df VF |
236 | """Setup instance variables for fio job test.""" |
237 | ||
df1eaa36 VF |
238 | super(FioJobTest, self).setup(artifact_root, testnum) |
239 | ||
240 | self.command_file = os.path.join( | |
704cc4df VF |
241 | self.test_dir, |
242 | "{0}.command".format(os.path.basename(self.fio_job))) | |
df1eaa36 | 243 | self.stdout_file = os.path.join( |
704cc4df VF |
244 | self.test_dir, |
245 | "{0}.stdout".format(os.path.basename(self.fio_job))) | |
df1eaa36 | 246 | self.stderr_file = os.path.join( |
704cc4df VF |
247 | self.test_dir, |
248 | "{0}.stderr".format(os.path.basename(self.fio_job))) | |
249 | self.exitcode_file = os.path.join( | |
250 | self.test_dir, | |
251 | "{0}.exitcode".format(os.path.basename(self.fio_job))) | |
df1eaa36 VF |
252 | |
253 | def run_pre_job(self): | |
704cc4df VF |
254 | """Run fio job precondition step.""" |
255 | ||
df1eaa36 VF |
256 | precon = FioJobTest(self.exe_path, self.fio_pre_job, |
257 | self.fio_pre_success, | |
258 | output_format=self.output_format) | |
259 | precon.setup(self.artifact_root, self.testnum) | |
260 | precon.run() | |
261 | precon.check_result() | |
262 | self.precon_failed = not precon.passed | |
263 | self.failure_reason = precon.failure_reason | |
264 | ||
265 | def run(self): | |
704cc4df VF |
266 | """Run fio job test.""" |
267 | ||
df1eaa36 VF |
268 | if self.fio_pre_job: |
269 | self.run_pre_job() | |
270 | ||
271 | if not self.precon_failed: | |
272 | super(FioJobTest, self).run() | |
273 | else: | |
704cc4df | 274 | logging.debug("Test %d: precondition step failed", self.testnum) |
df1eaa36 VF |
275 | |
276 | def check_result(self): | |
704cc4df VF |
277 | """Check fio job results.""" |
278 | ||
df1eaa36 VF |
279 | if self.precon_failed: |
280 | self.passed = False | |
281 | self.failure_reason = "{0} precondition step failed,".format(self.failure_reason) | |
282 | return | |
283 | ||
284 | super(FioJobTest, self).check_result() | |
285 | ||
6d5470c3 VF |
286 | if not self.passed: |
287 | return | |
288 | ||
704cc4df | 289 | if 'json' not in self.output_format: |
742b8799 VF |
290 | return |
291 | ||
292 | try: | |
293 | with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file: | |
294 | file_data = output_file.read() | |
295 | except EnvironmentError: | |
296 | self.failure_reason = "{0} unable to open output file,".format(self.failure_reason) | |
297 | self.passed = False | |
298 | return | |
299 | ||
300 | # | |
301 | # Sometimes fio informational messages are included at the top of the | |
302 | # JSON output, especially under Windows. Try to decode output as JSON | |
303 | # data, lopping off up to the first four lines | |
304 | # | |
305 | lines = file_data.splitlines() | |
306 | for i in range(5): | |
307 | file_data = '\n'.join(lines[i:]) | |
df1eaa36 | 308 | try: |
742b8799 VF |
309 | self.json_data = json.loads(file_data) |
310 | except json.JSONDecodeError: | |
311 | continue | |
6d5470c3 | 312 | else: |
704cc4df | 313 | logging.debug("Test %d: skipped %d lines decoding JSON data", self.testnum, i) |
742b8799 VF |
314 | return |
315 | ||
316 | self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason) | |
317 | self.passed = False | |
df1eaa36 VF |
318 | |
319 | ||
320 | class FioJobTest_t0005(FioJobTest): | |
321 | """Test consists of fio test job t0005 | |
322 | Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400""" | |
323 | ||
324 | def check_result(self): | |
325 | super(FioJobTest_t0005, self).check_result() | |
326 | ||
327 | if not self.passed: | |
328 | return | |
329 | ||
330 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400: | |
331 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
332 | self.passed = False | |
333 | if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400: | |
334 | self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) | |
335 | self.passed = False | |
336 | ||
337 | ||
338 | class FioJobTest_t0006(FioJobTest): | |
339 | """Test consists of fio test job t0006 | |
340 | Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']""" | |
341 | ||
342 | def check_result(self): | |
343 | super(FioJobTest_t0006, self).check_result() | |
344 | ||
345 | if not self.passed: | |
346 | return | |
347 | ||
348 | ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \ | |
349 | / self.json_data['jobs'][0]['write']['io_kbytes'] | |
704cc4df | 350 | logging.debug("Test %d: ratio: %f", self.testnum, ratio) |
df1eaa36 VF |
351 | if ratio < 1.99 or ratio > 2.01: |
352 | self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason) | |
353 | self.passed = False | |
354 | ||
355 | ||
356 | class FioJobTest_t0007(FioJobTest): | |
357 | """Test consists of fio test job t0007 | |
358 | Confirm that read['io_kbytes'] = 87040""" | |
359 | ||
360 | def check_result(self): | |
361 | super(FioJobTest_t0007, self).check_result() | |
362 | ||
363 | if not self.passed: | |
364 | return | |
365 | ||
366 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040: | |
367 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
368 | self.passed = False | |
369 | ||
370 | ||
371 | class FioJobTest_t0008(FioJobTest): | |
372 | """Test consists of fio test job t0008 | |
373 | Confirm that read['io_kbytes'] = 32768 and that | |
374 | write['io_kbytes'] ~ 16568 | |
375 | ||
376 | I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of | |
377 | 16585, 16588. With two runs of fio-3.16 I obtained 16568""" | |
378 | ||
379 | def check_result(self): | |
380 | super(FioJobTest_t0008, self).check_result() | |
381 | ||
382 | if not self.passed: | |
383 | return | |
384 | ||
385 | ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568 | |
704cc4df | 386 | logging.debug("Test %d: ratio: %f", self.testnum, ratio) |
df1eaa36 VF |
387 | |
388 | if ratio < 0.99 or ratio > 1.01: | |
389 | self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) | |
390 | self.passed = False | |
391 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768: | |
392 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
393 | self.passed = False | |
394 | ||
395 | ||
396 | class FioJobTest_t0009(FioJobTest): | |
397 | """Test consists of fio test job t0009 | |
398 | Confirm that runtime >= 60s""" | |
399 | ||
400 | def check_result(self): | |
401 | super(FioJobTest_t0009, self).check_result() | |
402 | ||
403 | if not self.passed: | |
404 | return | |
405 | ||
704cc4df | 406 | logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed']) |
df1eaa36 VF |
407 | |
408 | if self.json_data['jobs'][0]['elapsed'] < 60: | |
409 | self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason) | |
410 | self.passed = False | |
411 | ||
412 | ||
413 | class FioJobTest_t0011(FioJobTest): | |
414 | """Test consists of fio test job t0009 | |
415 | Confirm that job0 iops == 1000 | |
416 | and that job1_iops / job0_iops ~ 8 | |
417 | With two runs of fio-3.16 I observed a ratio of 8.3""" | |
418 | ||
419 | def check_result(self): | |
420 | super(FioJobTest_t0011, self).check_result() | |
421 | ||
422 | if not self.passed: | |
423 | return | |
424 | ||
425 | iops1 = self.json_data['jobs'][0]['read']['iops'] | |
426 | iops2 = self.json_data['jobs'][1]['read']['iops'] | |
427 | ratio = iops2 / iops1 | |
704cc4df VF |
428 | logging.debug("Test %d: iops1: %f", self.testnum, iops1) |
429 | logging.debug("Test %d: ratio: %f", self.testnum, ratio) | |
df1eaa36 | 430 | |
41ceb6c7 | 431 | if iops1 < 998 or iops1 > 1002: |
df1eaa36 VF |
432 | self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason) |
433 | self.passed = False | |
434 | ||
435 | if ratio < 7 or ratio > 9: | |
436 | self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason) | |
437 | self.passed = False | |
438 | ||
439 | ||
b1bc705e VF |
440 | class Requirements(object): |
441 | """Requirements consists of multiple run environment characteristics. | |
442 | These are to determine if a particular test can be run""" | |
443 | ||
444 | _linux = False | |
445 | _libaio = False | |
446 | _zbd = False | |
447 | _root = False | |
448 | _zoned_nullb = False | |
449 | _not_macos = False | |
c58b33b4 | 450 | _not_windows = False |
b1bc705e VF |
451 | _unittests = False |
452 | _cpucount4 = False | |
453 | ||
454 | def __init__(self, fio_root): | |
455 | Requirements._not_macos = platform.system() != "Darwin" | |
c58b33b4 | 456 | Requirements._not_windows = platform.system() != "Windows" |
b1bc705e VF |
457 | Requirements._linux = platform.system() == "Linux" |
458 | ||
459 | if Requirements._linux: | |
460 | try: | |
461 | config_file = os.path.join(fio_root, "config-host.h") | |
462 | with open(config_file, "r") as config: | |
463 | contents = config.read() | |
464 | except Exception: | |
465 | print("Unable to open {0} to check requirements".format(config_file)) | |
466 | Requirements._zbd = True | |
467 | else: | |
468 | Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents | |
469 | Requirements._libaio = "CONFIG_LIBAIO" in contents | |
470 | ||
471 | Requirements._root = (os.geteuid() == 0) | |
472 | if Requirements._zbd and Requirements._root: | |
704cc4df VF |
473 | subprocess.run(["modprobe", "null_blk"], |
474 | stdout=subprocess.PIPE, | |
475 | stderr=subprocess.PIPE) | |
476 | if os.path.exists("/sys/module/null_blk/parameters/zoned"): | |
477 | Requirements._zoned_nullb = True | |
b1bc705e | 478 | |
742b8799 VF |
479 | if platform.system() == "Windows": |
480 | utest_exe = "unittest.exe" | |
481 | else: | |
482 | utest_exe = "unittest" | |
483 | unittest_path = os.path.join(fio_root, "unittests", utest_exe) | |
b1bc705e VF |
484 | Requirements._unittests = os.path.exists(unittest_path) |
485 | ||
486 | Requirements._cpucount4 = multiprocessing.cpu_count() >= 4 | |
487 | ||
488 | req_list = [Requirements.linux, | |
489 | Requirements.libaio, | |
490 | Requirements.zbd, | |
491 | Requirements.root, | |
492 | Requirements.zoned_nullb, | |
493 | Requirements.not_macos, | |
c58b33b4 | 494 | Requirements.not_windows, |
b1bc705e VF |
495 | Requirements.unittests, |
496 | Requirements.cpucount4] | |
497 | for req in req_list: | |
498 | value, desc = req() | |
704cc4df | 499 | logging.debug("Requirements: Requirement '%s' met? %s", desc, value) |
b1bc705e | 500 | |
704cc4df VF |
501 | @classmethod |
502 | def linux(cls): | |
503 | """Are we running on Linux?""" | |
b1bc705e VF |
504 | return Requirements._linux, "Linux required" |
505 | ||
704cc4df VF |
506 | @classmethod |
507 | def libaio(cls): | |
508 | """Is libaio available?""" | |
b1bc705e VF |
509 | return Requirements._libaio, "libaio required" |
510 | ||
704cc4df VF |
511 | @classmethod |
512 | def zbd(cls): | |
513 | """Is ZBD support available?""" | |
b1bc705e VF |
514 | return Requirements._zbd, "Zoned block device support required" |
515 | ||
704cc4df VF |
516 | @classmethod |
517 | def root(cls): | |
518 | """Are we running as root?""" | |
b1bc705e VF |
519 | return Requirements._root, "root required" |
520 | ||
704cc4df VF |
521 | @classmethod |
522 | def zoned_nullb(cls): | |
523 | """Are zoned null block devices available?""" | |
b1bc705e VF |
524 | return Requirements._zoned_nullb, "Zoned null block device support required" |
525 | ||
704cc4df VF |
526 | @classmethod |
527 | def not_macos(cls): | |
528 | """Are we running on a platform other than macOS?""" | |
b1bc705e VF |
529 | return Requirements._not_macos, "platform other than macOS required" |
530 | ||
704cc4df VF |
531 | @classmethod |
532 | def not_windows(cls): | |
533 | """Are we running on a platform other than Windws?""" | |
c58b33b4 VF |
534 | return Requirements._not_windows, "platform other than Windows required" |
535 | ||
704cc4df VF |
536 | @classmethod |
537 | def unittests(cls): | |
538 | """Were unittests built?""" | |
b1bc705e VF |
539 | return Requirements._unittests, "Unittests support required" |
540 | ||
704cc4df VF |
541 | @classmethod |
542 | def cpucount4(cls): | |
543 | """Do we have at least 4 CPUs?""" | |
b1bc705e VF |
544 | return Requirements._cpucount4, "4+ CPUs required" |
545 | ||
546 | ||
df1eaa36 | 547 | SUCCESS_DEFAULT = { |
704cc4df VF |
548 | 'zero_return': True, |
549 | 'stderr_empty': True, | |
550 | 'timeout': 600, | |
551 | } | |
df1eaa36 | 552 | SUCCESS_NONZERO = { |
704cc4df VF |
553 | 'zero_return': False, |
554 | 'stderr_empty': False, | |
555 | 'timeout': 600, | |
556 | } | |
df1eaa36 | 557 | SUCCESS_STDERR = { |
704cc4df VF |
558 | 'zero_return': True, |
559 | 'stderr_empty': False, | |
560 | 'timeout': 600, | |
561 | } | |
df1eaa36 | 562 | TEST_LIST = [ |
704cc4df VF |
563 | { |
564 | 'test_id': 1, | |
565 | 'test_class': FioJobTest, | |
566 | 'job': 't0001-52c58027.fio', | |
567 | 'success': SUCCESS_DEFAULT, | |
568 | 'pre_job': None, | |
569 | 'pre_success': None, | |
570 | 'requirements': [], | |
571 | }, | |
572 | { | |
573 | 'test_id': 2, | |
574 | 'test_class': FioJobTest, | |
575 | 'job': 't0002-13af05ae-post.fio', | |
576 | 'success': SUCCESS_DEFAULT, | |
577 | 'pre_job': 't0002-13af05ae-pre.fio', | |
578 | 'pre_success': None, | |
579 | 'requirements': [Requirements.linux, Requirements.libaio], | |
580 | }, | |
581 | { | |
582 | 'test_id': 3, | |
583 | 'test_class': FioJobTest, | |
584 | 'job': 't0003-0ae2c6e1-post.fio', | |
585 | 'success': SUCCESS_NONZERO, | |
586 | 'pre_job': 't0003-0ae2c6e1-pre.fio', | |
587 | 'pre_success': SUCCESS_DEFAULT, | |
588 | 'requirements': [Requirements.linux, Requirements.libaio], | |
589 | }, | |
590 | { | |
591 | 'test_id': 4, | |
592 | 'test_class': FioJobTest, | |
593 | 'job': 't0004-8a99fdf6.fio', | |
594 | 'success': SUCCESS_DEFAULT, | |
595 | 'pre_job': None, | |
596 | 'pre_success': None, | |
597 | 'requirements': [Requirements.linux, Requirements.libaio], | |
598 | }, | |
599 | { | |
600 | 'test_id': 5, | |
601 | 'test_class': FioJobTest_t0005, | |
602 | 'job': 't0005-f7078f7b.fio', | |
603 | 'success': SUCCESS_DEFAULT, | |
604 | 'pre_job': None, | |
605 | 'pre_success': None, | |
606 | 'output_format': 'json', | |
607 | 'requirements': [Requirements.not_windows], | |
608 | }, | |
609 | { | |
610 | 'test_id': 6, | |
611 | 'test_class': FioJobTest_t0006, | |
612 | 'job': 't0006-82af2a7c.fio', | |
613 | 'success': SUCCESS_DEFAULT, | |
614 | 'pre_job': None, | |
615 | 'pre_success': None, | |
616 | 'output_format': 'json', | |
617 | 'requirements': [Requirements.linux, Requirements.libaio], | |
618 | }, | |
619 | { | |
620 | 'test_id': 7, | |
621 | 'test_class': FioJobTest_t0007, | |
622 | 'job': 't0007-37cf9e3c.fio', | |
623 | 'success': SUCCESS_DEFAULT, | |
624 | 'pre_job': None, | |
625 | 'pre_success': None, | |
626 | 'output_format': 'json', | |
627 | 'requirements': [], | |
628 | }, | |
629 | { | |
630 | 'test_id': 8, | |
631 | 'test_class': FioJobTest_t0008, | |
632 | 'job': 't0008-ae2fafc8.fio', | |
633 | 'success': SUCCESS_DEFAULT, | |
634 | 'pre_job': None, | |
635 | 'pre_success': None, | |
636 | 'output_format': 'json', | |
637 | 'requirements': [], | |
638 | }, | |
639 | { | |
640 | 'test_id': 9, | |
641 | 'test_class': FioJobTest_t0009, | |
642 | 'job': 't0009-f8b0bd10.fio', | |
643 | 'success': SUCCESS_DEFAULT, | |
644 | 'pre_job': None, | |
645 | 'pre_success': None, | |
646 | 'output_format': 'json', | |
647 | 'requirements': [Requirements.not_macos, | |
648 | Requirements.cpucount4], | |
649 | # mac os does not support CPU affinity | |
650 | }, | |
651 | { | |
652 | 'test_id': 10, | |
653 | 'test_class': FioJobTest, | |
654 | 'job': 't0010-b7aae4ba.fio', | |
655 | 'success': SUCCESS_DEFAULT, | |
656 | 'pre_job': None, | |
657 | 'pre_success': None, | |
658 | 'requirements': [], | |
659 | }, | |
660 | { | |
661 | 'test_id': 11, | |
662 | 'test_class': FioJobTest_t0011, | |
663 | 'job': 't0011-5d2788d5.fio', | |
664 | 'success': SUCCESS_DEFAULT, | |
665 | 'pre_job': None, | |
666 | 'pre_success': None, | |
667 | 'output_format': 'json', | |
668 | 'requirements': [], | |
669 | }, | |
670 | { | |
671 | 'test_id': 1000, | |
672 | 'test_class': FioExeTest, | |
673 | 'exe': 't/axmap', | |
674 | 'parameters': None, | |
675 | 'success': SUCCESS_DEFAULT, | |
676 | 'requirements': [], | |
677 | }, | |
678 | { | |
679 | 'test_id': 1001, | |
680 | 'test_class': FioExeTest, | |
681 | 'exe': 't/ieee754', | |
682 | 'parameters': None, | |
683 | 'success': SUCCESS_DEFAULT, | |
684 | 'requirements': [], | |
685 | }, | |
686 | { | |
687 | 'test_id': 1002, | |
688 | 'test_class': FioExeTest, | |
689 | 'exe': 't/lfsr-test', | |
690 | 'parameters': ['0xFFFFFF', '0', '0', 'verify'], | |
691 | 'success': SUCCESS_STDERR, | |
692 | 'requirements': [], | |
693 | }, | |
694 | { | |
695 | 'test_id': 1003, | |
696 | 'test_class': FioExeTest, | |
697 | 'exe': 't/readonly.py', | |
698 | 'parameters': ['-f', '{fio_path}'], | |
699 | 'success': SUCCESS_DEFAULT, | |
700 | 'requirements': [], | |
701 | }, | |
702 | { | |
703 | 'test_id': 1004, | |
704 | 'test_class': FioExeTest, | |
705 | 'exe': 't/steadystate_tests.py', | |
706 | 'parameters': ['{fio_path}'], | |
707 | 'success': SUCCESS_DEFAULT, | |
708 | 'requirements': [], | |
709 | }, | |
710 | { | |
711 | 'test_id': 1005, | |
712 | 'test_class': FioExeTest, | |
713 | 'exe': 't/stest', | |
714 | 'parameters': None, | |
715 | 'success': SUCCESS_STDERR, | |
716 | 'requirements': [], | |
717 | }, | |
718 | { | |
719 | 'test_id': 1006, | |
720 | 'test_class': FioExeTest, | |
721 | 'exe': 't/strided.py', | |
722 | 'parameters': ['{fio_path}'], | |
723 | 'success': SUCCESS_DEFAULT, | |
724 | 'requirements': [], | |
725 | }, | |
726 | { | |
727 | 'test_id': 1007, | |
728 | 'test_class': FioExeTest, | |
729 | 'exe': 't/zbd/run-tests-against-regular-nullb', | |
730 | 'parameters': None, | |
731 | 'success': SUCCESS_DEFAULT, | |
732 | 'requirements': [Requirements.linux, Requirements.zbd, | |
733 | Requirements.root], | |
734 | }, | |
735 | { | |
736 | 'test_id': 1008, | |
737 | 'test_class': FioExeTest, | |
738 | 'exe': 't/zbd/run-tests-against-zoned-nullb', | |
739 | 'parameters': None, | |
740 | 'success': SUCCESS_DEFAULT, | |
741 | 'requirements': [Requirements.linux, Requirements.zbd, | |
742 | Requirements.root, Requirements.zoned_nullb], | |
743 | }, | |
744 | { | |
745 | 'test_id': 1009, | |
746 | 'test_class': FioExeTest, | |
747 | 'exe': 'unittests/unittest', | |
748 | 'parameters': None, | |
749 | 'success': SUCCESS_DEFAULT, | |
750 | 'requirements': [Requirements.unittests], | |
751 | }, | |
752 | { | |
753 | 'test_id': 1010, | |
754 | 'test_class': FioExeTest, | |
755 | 'exe': 't/latency_percentiles.py', | |
756 | 'parameters': ['-f', '{fio_path}'], | |
757 | 'success': SUCCESS_DEFAULT, | |
758 | 'requirements': [], | |
759 | }, | |
df1eaa36 VF |
760 | ] |
761 | ||
762 | ||
763 | def parse_args(): | |
704cc4df VF |
764 | """Parse command-line arguments.""" |
765 | ||
df1eaa36 VF |
766 | parser = argparse.ArgumentParser() |
767 | parser.add_argument('-r', '--fio-root', | |
768 | help='fio root path') | |
769 | parser.add_argument('-f', '--fio', | |
770 | help='path to fio executable (e.g., ./fio)') | |
771 | parser.add_argument('-a', '--artifact-root', | |
772 | help='artifact root directory') | |
773 | parser.add_argument('-s', '--skip', nargs='+', type=int, | |
774 | help='list of test(s) to skip') | |
775 | parser.add_argument('-o', '--run-only', nargs='+', type=int, | |
776 | help='list of test(s) to run, skipping all others') | |
6d5470c3 VF |
777 | parser.add_argument('-d', '--debug', action='store_true', |
778 | help='provide debug output') | |
b1bc705e VF |
779 | parser.add_argument('-k', '--skip-req', action='store_true', |
780 | help='skip requirements checking') | |
df1eaa36 VF |
781 | args = parser.parse_args() |
782 | ||
783 | return args | |
784 | ||
785 | ||
786 | def main(): | |
704cc4df VF |
787 | """Entry point.""" |
788 | ||
df1eaa36 | 789 | args = parse_args() |
6d5470c3 VF |
790 | if args.debug: |
791 | logging.basicConfig(level=logging.DEBUG) | |
792 | else: | |
793 | logging.basicConfig(level=logging.INFO) | |
794 | ||
df1eaa36 VF |
795 | if args.fio_root: |
796 | fio_root = args.fio_root | |
797 | else: | |
6d5470c3 VF |
798 | fio_root = str(Path(__file__).absolute().parent.parent) |
799 | print("fio root is %s" % fio_root) | |
df1eaa36 VF |
800 | |
801 | if args.fio: | |
802 | fio_path = args.fio | |
803 | else: | |
742b8799 VF |
804 | if platform.system() == "Windows": |
805 | fio_exe = "fio.exe" | |
806 | else: | |
807 | fio_exe = "fio" | |
808 | fio_path = os.path.join(fio_root, fio_exe) | |
6d5470c3 VF |
809 | print("fio path is %s" % fio_path) |
810 | if not shutil.which(fio_path): | |
811 | print("Warning: fio executable not found") | |
df1eaa36 VF |
812 | |
813 | artifact_root = args.artifact_root if args.artifact_root else \ | |
814 | "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S")) | |
815 | os.mkdir(artifact_root) | |
816 | print("Artifact directory is %s" % artifact_root) | |
817 | ||
b1bc705e VF |
818 | if not args.skip_req: |
819 | req = Requirements(fio_root) | |
820 | ||
df1eaa36 VF |
821 | passed = 0 |
822 | failed = 0 | |
823 | skipped = 0 | |
824 | ||
825 | for config in TEST_LIST: | |
826 | if (args.skip and config['test_id'] in args.skip) or \ | |
827 | (args.run_only and config['test_id'] not in args.run_only): | |
828 | skipped = skipped + 1 | |
b1bc705e | 829 | print("Test {0} SKIPPED (User request)".format(config['test_id'])) |
df1eaa36 VF |
830 | continue |
831 | ||
832 | if issubclass(config['test_class'], FioJobTest): | |
833 | if config['pre_job']: | |
834 | fio_pre_job = os.path.join(fio_root, 't', 'jobs', | |
835 | config['pre_job']) | |
836 | else: | |
837 | fio_pre_job = None | |
838 | if config['pre_success']: | |
839 | fio_pre_success = config['pre_success'] | |
840 | else: | |
841 | fio_pre_success = None | |
842 | if 'output_format' in config: | |
843 | output_format = config['output_format'] | |
844 | else: | |
845 | output_format = 'normal' | |
846 | test = config['test_class']( | |
847 | fio_path, | |
848 | os.path.join(fio_root, 't', 'jobs', config['job']), | |
849 | config['success'], | |
850 | fio_pre_job=fio_pre_job, | |
851 | fio_pre_success=fio_pre_success, | |
852 | output_format=output_format) | |
853 | elif issubclass(config['test_class'], FioExeTest): | |
854 | exe_path = os.path.join(fio_root, config['exe']) | |
855 | if config['parameters']: | |
856 | parameters = [p.format(fio_path=fio_path) for p in config['parameters']] | |
857 | else: | |
858 | parameters = None | |
742b8799 VF |
859 | if Path(exe_path).suffix == '.py' and platform.system() == "Windows": |
860 | if parameters: | |
861 | parameters.insert(0, exe_path) | |
862 | else: | |
863 | parameters = [exe_path] | |
864 | exe_path = "python.exe" | |
df1eaa36 VF |
865 | test = config['test_class'](exe_path, parameters, |
866 | config['success']) | |
867 | else: | |
868 | print("Test {0} FAILED: unable to process test config".format(config['test_id'])) | |
869 | failed = failed + 1 | |
870 | continue | |
871 | ||
b1bc705e | 872 | if not args.skip_req: |
704cc4df | 873 | reqs_met = True |
b1bc705e | 874 | for req in config['requirements']: |
704cc4df VF |
875 | reqs_met, reason = req() |
876 | logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason, | |
877 | reqs_met) | |
878 | if not reqs_met: | |
b1bc705e | 879 | break |
704cc4df | 880 | if not reqs_met: |
b1bc705e VF |
881 | print("Test {0} SKIPPED ({1})".format(config['test_id'], reason)) |
882 | skipped = skipped + 1 | |
883 | continue | |
884 | ||
df1eaa36 VF |
885 | test.setup(artifact_root, config['test_id']) |
886 | test.run() | |
887 | test.check_result() | |
888 | if test.passed: | |
889 | result = "PASSED" | |
890 | passed = passed + 1 | |
891 | else: | |
892 | result = "FAILED: {0}".format(test.failure_reason) | |
893 | failed = failed + 1 | |
6d5470c3 | 894 | with open(test.stderr_file, "r") as stderr_file: |
704cc4df | 895 | logging.debug("Test %d: stderr:\n%s", config['test_id'], stderr_file.read()) |
6d5470c3 | 896 | with open(test.stdout_file, "r") as stdout_file: |
704cc4df | 897 | logging.debug("Test %d: stdout:\n%s", config['test_id'], stdout_file.read()) |
df1eaa36 VF |
898 | print("Test {0} {1}".format(config['test_id'], result)) |
899 | ||
900 | print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped)) | |
901 | ||
902 | sys.exit(failed) | |
903 | ||
904 | ||
905 | if __name__ == '__main__': | |
906 | main() |