Commit | Line | Data |
---|---|---|
df1eaa36 VF |
1 | #!/usr/bin/env python3 |
2 | # SPDX-License-Identifier: GPL-2.0-only | |
3 | # | |
4 | # Copyright (c) 2019 Western Digital Corporation or its affiliates. | |
5 | # | |
6 | """ | |
7 | # run-fio-tests.py | |
8 | # | |
9 | # Automate running of fio tests | |
10 | # | |
11 | # USAGE | |
12 | # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root] | |
13 | # [--skip # # #...] [--run-only # # #...] | |
14 | # | |
15 | # | |
16 | # EXAMPLE | |
17 | # # git clone [fio-repository] | |
18 | # # cd fio | |
19 | # # make -j | |
20 | # # python3 t/run-fio-tests.py | |
21 | # | |
22 | # | |
23 | # REQUIREMENTS | |
24 | # - Python 3 | |
25 | # - Linux (libaio ioengine, zbd tests, etc) | |
26 | # - The artifact directory must be on a file system that accepts 512-byte IO | |
27 | # (t0002, t0003, t0004). | |
28 | # - The artifact directory needs to be on an SSD. Otherwise tests that carry | |
29 | # out file-based IO will trigger a timeout (t0006). | |
30 | # - 4 CPUs (t0009) | |
31 | # - SciPy (steadystate_tests.py) | |
32 | # - libzbc (zbd tests) | |
33 | # - root privileges (zbd test) | |
34 | # - kernel 4.19 or later for zoned null block devices (zbd tests) | |
35 | # - CUnit support (unittests) | |
36 | # | |
37 | """ | |
38 | ||
39 | # | |
40 | # TODO run multiple tests simultaneously | |
41 | # TODO Add sgunmap tests (requires SAS SSD) | |
42 | # TODO automatically detect dependencies and skip tests accordingly | |
43 | # | |
44 | ||
45 | import os | |
46 | import sys | |
47 | import json | |
48 | import time | |
49 | import logging | |
50 | import argparse | |
51 | import subprocess | |
52 | from pathlib import Path | |
53 | ||
54 | ||
55 | class FioTest(object): | |
56 | """Base for all fio tests.""" | |
57 | ||
58 | def __init__(self, exe_path, parameters, success): | |
59 | self.exe_path = exe_path | |
60 | self.parameters = parameters | |
61 | self.success = success | |
62 | self.output = {} | |
63 | self.artifact_root = None | |
64 | self.testnum = None | |
65 | self.test_dir = None | |
66 | self.passed = True | |
67 | self.failure_reason = '' | |
68 | ||
69 | def setup(self, artifact_root, testnum): | |
70 | self.artifact_root = artifact_root | |
71 | self.testnum = testnum | |
72 | self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum)) | |
73 | if not os.path.exists(self.test_dir): | |
74 | os.mkdir(self.test_dir) | |
75 | ||
76 | self.command_file = os.path.join( | |
77 | self.test_dir, | |
78 | "{0}.command".format(os.path.basename(self.exe_path))) | |
79 | self.stdout_file = os.path.join( | |
80 | self.test_dir, | |
81 | "{0}.stdout".format(os.path.basename(self.exe_path))) | |
82 | self.stderr_file = os.path.join( | |
83 | self.test_dir, | |
84 | "{0}.stderr".format(os.path.basename(self.exe_path))) | |
85 | self.exticode_file = os.path.join( | |
86 | self.test_dir, | |
87 | "{0}.exitcode".format(os.path.basename(self.exe_path))) | |
88 | ||
89 | def run(self): | |
90 | raise NotImplementedError() | |
91 | ||
92 | def check_result(self): | |
93 | raise NotImplementedError() | |
94 | ||
95 | ||
96 | class FioExeTest(FioTest): | |
97 | """Test consists of an executable binary or script""" | |
98 | ||
99 | def __init__(self, exe_path, parameters, success): | |
100 | """Construct a FioExeTest which is a FioTest consisting of an | |
101 | executable binary or script. | |
102 | ||
103 | exe_path: location of executable binary or script | |
104 | parameters: list of parameters for executable | |
105 | success: Definition of test success | |
106 | """ | |
107 | ||
108 | FioTest.__init__(self, exe_path, parameters, success) | |
109 | ||
110 | def setup(self, artifact_root, testnum): | |
111 | super(FioExeTest, self).setup(artifact_root, testnum) | |
112 | ||
113 | def run(self): | |
114 | if self.parameters: | |
115 | command = [self.exe_path] + self.parameters | |
116 | else: | |
117 | command = [self.exe_path] | |
118 | command_file = open(self.command_file, "w+") | |
119 | command_file.write("%s\n" % command) | |
120 | command_file.close() | |
121 | ||
122 | stdout_file = open(self.stdout_file, "w+") | |
123 | stderr_file = open(self.stderr_file, "w+") | |
124 | exticode_file = open(self.exticode_file, "w+") | |
125 | try: | |
126 | # Avoid using subprocess.run() here because when a timeout occurs, | |
127 | # fio will be stopped with SIGKILL. This does not give fio a | |
128 | # chance to clean up and means that child processes may continue | |
129 | # running and submitting IO. | |
130 | proc = subprocess.Popen(command, | |
131 | stdout=stdout_file, | |
132 | stderr=stderr_file, | |
133 | cwd=self.test_dir, | |
134 | universal_newlines=True) | |
135 | proc.communicate(timeout=self.success['timeout']) | |
136 | exticode_file.write('{0}\n'.format(proc.returncode)) | |
137 | logging.debug("return code: %d" % proc.returncode) | |
138 | self.output['proc'] = proc | |
139 | except subprocess.TimeoutExpired: | |
140 | proc.terminate() | |
141 | proc.communicate() | |
142 | assert proc.poll() | |
143 | self.output['failure'] = 'timeout' | |
144 | except Exception: | |
145 | if not proc.poll(): | |
146 | proc.terminate() | |
147 | proc.communicate() | |
148 | self.output['failure'] = 'exception' | |
149 | self.output['exc_info'] = sys.exc_info() | |
150 | finally: | |
151 | stdout_file.close() | |
152 | stderr_file.close() | |
153 | exticode_file.close() | |
154 | ||
155 | def check_result(self): | |
156 | if 'proc' not in self.output: | |
157 | if self.output['failure'] == 'timeout': | |
158 | self.failure_reason = "{0} timeout,".format(self.failure_reason) | |
159 | else: | |
160 | assert self.output['failure'] == 'exception' | |
161 | self.failure_reason = '{0} exception: {1}, {2}'.format( | |
162 | self.failure_reason, self.output['exc_info'][0], | |
163 | self.output['exc_info'][1]) | |
164 | ||
165 | self.passed = False | |
166 | return | |
167 | ||
168 | if 'zero_return' in self.success: | |
169 | if self.success['zero_return']: | |
170 | if self.output['proc'].returncode != 0: | |
171 | self.passed = False | |
172 | self.failure_reason = "{0} non-zero return code,".format(self.failure_reason) | |
173 | else: | |
174 | if self.output['proc'].returncode == 0: | |
175 | self.failure_reason = "{0} zero return code,".format(self.failure_reason) | |
176 | self.passed = False | |
177 | ||
178 | if 'stderr_empty' in self.success: | |
179 | stderr_size = os.path.getsize(self.stderr_file) | |
180 | if self.success['stderr_empty']: | |
181 | if stderr_size != 0: | |
182 | self.failure_reason = "{0} stderr not empty,".format(self.failure_reason) | |
183 | self.passed = False | |
184 | else: | |
185 | if stderr_size == 0: | |
186 | self.failure_reason = "{0} stderr empty,".format(self.failure_reason) | |
187 | self.passed = False | |
188 | ||
189 | ||
190 | class FioJobTest(FioExeTest): | |
191 | """Test consists of a fio job""" | |
192 | ||
193 | def __init__(self, fio_path, fio_job, success, fio_pre_job=None, | |
194 | fio_pre_success=None, output_format="normal"): | |
195 | """Construct a FioJobTest which is a FioExeTest consisting of a | |
196 | single fio job file with an optional setup step. | |
197 | ||
198 | fio_path: location of fio executable | |
199 | fio_job: location of fio job file | |
200 | success: Definition of test success | |
201 | fio_pre_job: fio job for preconditioning | |
202 | fio_pre_success: Definition of test success for fio precon job | |
203 | output_format: normal (default), json, jsonplus, or terse | |
204 | """ | |
205 | ||
206 | self.fio_job = fio_job | |
207 | self.fio_pre_job = fio_pre_job | |
208 | self.fio_pre_success = fio_pre_success if fio_pre_success else success | |
209 | self.output_format = output_format | |
210 | self.precon_failed = False | |
211 | self.json_data = None | |
212 | self.fio_output = "{0}.output".format(os.path.basename(self.fio_job)) | |
213 | self.fio_args = [ | |
214 | "--output-format={0}".format(self.output_format), | |
215 | "--output={0}".format(self.fio_output), | |
216 | self.fio_job, | |
217 | ] | |
218 | FioExeTest.__init__(self, fio_path, self.fio_args, success) | |
219 | ||
220 | def setup(self, artifact_root, testnum): | |
221 | super(FioJobTest, self).setup(artifact_root, testnum) | |
222 | ||
223 | self.command_file = os.path.join( | |
224 | self.test_dir, | |
225 | "{0}.command".format(os.path.basename(self.fio_job))) | |
226 | self.stdout_file = os.path.join( | |
227 | self.test_dir, | |
228 | "{0}.stdout".format(os.path.basename(self.fio_job))) | |
229 | self.stderr_file = os.path.join( | |
230 | self.test_dir, | |
231 | "{0}.stderr".format(os.path.basename(self.fio_job))) | |
232 | self.exticode_file = os.path.join( | |
233 | self.test_dir, | |
234 | "{0}.exitcode".format(os.path.basename(self.fio_job))) | |
235 | ||
236 | def run_pre_job(self): | |
237 | precon = FioJobTest(self.exe_path, self.fio_pre_job, | |
238 | self.fio_pre_success, | |
239 | output_format=self.output_format) | |
240 | precon.setup(self.artifact_root, self.testnum) | |
241 | precon.run() | |
242 | precon.check_result() | |
243 | self.precon_failed = not precon.passed | |
244 | self.failure_reason = precon.failure_reason | |
245 | ||
246 | def run(self): | |
247 | if self.fio_pre_job: | |
248 | self.run_pre_job() | |
249 | ||
250 | if not self.precon_failed: | |
251 | super(FioJobTest, self).run() | |
252 | else: | |
253 | logging.debug("precondition step failed") | |
254 | ||
255 | def check_result(self): | |
256 | if self.precon_failed: | |
257 | self.passed = False | |
258 | self.failure_reason = "{0} precondition step failed,".format(self.failure_reason) | |
259 | return | |
260 | ||
261 | super(FioJobTest, self).check_result() | |
262 | ||
263 | if 'json' in self.output_format: | |
264 | output_file = open(os.path.join(self.test_dir, self.fio_output), "r") | |
265 | file_data = output_file.read() | |
266 | output_file.close() | |
267 | try: | |
268 | self.json_data = json.loads(file_data) | |
269 | except json.JSONDecodeError: | |
270 | self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason) | |
271 | self.passed = False | |
272 | ||
273 | ||
274 | class FioJobTest_t0005(FioJobTest): | |
275 | """Test consists of fio test job t0005 | |
276 | Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400""" | |
277 | ||
278 | def check_result(self): | |
279 | super(FioJobTest_t0005, self).check_result() | |
280 | ||
281 | if not self.passed: | |
282 | return | |
283 | ||
284 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400: | |
285 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
286 | self.passed = False | |
287 | if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400: | |
288 | self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) | |
289 | self.passed = False | |
290 | ||
291 | ||
292 | class FioJobTest_t0006(FioJobTest): | |
293 | """Test consists of fio test job t0006 | |
294 | Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']""" | |
295 | ||
296 | def check_result(self): | |
297 | super(FioJobTest_t0006, self).check_result() | |
298 | ||
299 | if not self.passed: | |
300 | return | |
301 | ||
302 | ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \ | |
303 | / self.json_data['jobs'][0]['write']['io_kbytes'] | |
304 | logging.debug("ratio: %f" % ratio) | |
305 | if ratio < 1.99 or ratio > 2.01: | |
306 | self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason) | |
307 | self.passed = False | |
308 | ||
309 | ||
310 | class FioJobTest_t0007(FioJobTest): | |
311 | """Test consists of fio test job t0007 | |
312 | Confirm that read['io_kbytes'] = 87040""" | |
313 | ||
314 | def check_result(self): | |
315 | super(FioJobTest_t0007, self).check_result() | |
316 | ||
317 | if not self.passed: | |
318 | return | |
319 | ||
320 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040: | |
321 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
322 | self.passed = False | |
323 | ||
324 | ||
325 | class FioJobTest_t0008(FioJobTest): | |
326 | """Test consists of fio test job t0008 | |
327 | Confirm that read['io_kbytes'] = 32768 and that | |
328 | write['io_kbytes'] ~ 16568 | |
329 | ||
330 | I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of | |
331 | 16585, 16588. With two runs of fio-3.16 I obtained 16568""" | |
332 | ||
333 | def check_result(self): | |
334 | super(FioJobTest_t0008, self).check_result() | |
335 | ||
336 | if not self.passed: | |
337 | return | |
338 | ||
339 | ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568 | |
340 | logging.debug("ratio: %f" % ratio) | |
341 | ||
342 | if ratio < 0.99 or ratio > 1.01: | |
343 | self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) | |
344 | self.passed = False | |
345 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768: | |
346 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
347 | self.passed = False | |
348 | ||
349 | ||
350 | class FioJobTest_t0009(FioJobTest): | |
351 | """Test consists of fio test job t0009 | |
352 | Confirm that runtime >= 60s""" | |
353 | ||
354 | def check_result(self): | |
355 | super(FioJobTest_t0009, self).check_result() | |
356 | ||
357 | if not self.passed: | |
358 | return | |
359 | ||
360 | logging.debug('elapsed: %d' % self.json_data['jobs'][0]['elapsed']) | |
361 | ||
362 | if self.json_data['jobs'][0]['elapsed'] < 60: | |
363 | self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason) | |
364 | self.passed = False | |
365 | ||
366 | ||
367 | class FioJobTest_t0011(FioJobTest): | |
368 | """Test consists of fio test job t0009 | |
369 | Confirm that job0 iops == 1000 | |
370 | and that job1_iops / job0_iops ~ 8 | |
371 | With two runs of fio-3.16 I observed a ratio of 8.3""" | |
372 | ||
373 | def check_result(self): | |
374 | super(FioJobTest_t0011, self).check_result() | |
375 | ||
376 | if not self.passed: | |
377 | return | |
378 | ||
379 | iops1 = self.json_data['jobs'][0]['read']['iops'] | |
380 | iops2 = self.json_data['jobs'][1]['read']['iops'] | |
381 | ratio = iops2 / iops1 | |
382 | logging.debug("ratio: %f" % ratio) | |
383 | ||
384 | if iops1 < 999 or iops1 > 1001: | |
385 | self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason) | |
386 | self.passed = False | |
387 | ||
388 | if ratio < 7 or ratio > 9: | |
389 | self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason) | |
390 | self.passed = False | |
391 | ||
392 | ||
393 | SUCCESS_DEFAULT = { | |
394 | 'zero_return': True, | |
395 | 'stderr_empty': True, | |
396 | 'timeout': 300, | |
397 | } | |
398 | SUCCESS_NONZERO = { | |
399 | 'zero_return': False, | |
400 | 'stderr_empty': False, | |
401 | 'timeout': 300, | |
402 | } | |
403 | SUCCESS_STDERR = { | |
404 | 'zero_return': True, | |
405 | 'stderr_empty': False, | |
406 | 'timeout': 300, | |
407 | } | |
408 | TEST_LIST = [ | |
409 | { | |
410 | 'test_id': 1, | |
411 | 'test_class': FioJobTest, | |
412 | 'job': 't0001-52c58027.fio', | |
413 | 'success': SUCCESS_DEFAULT, | |
414 | 'pre_job': None, | |
415 | 'pre_success': None, | |
416 | }, | |
417 | { | |
418 | 'test_id': 2, | |
419 | 'test_class': FioJobTest, | |
420 | 'job': 't0002-13af05ae-post.fio', | |
421 | 'success': SUCCESS_DEFAULT, | |
422 | 'pre_job': 't0002-13af05ae-pre.fio', | |
423 | 'pre_success': None, | |
424 | }, | |
425 | { | |
426 | 'test_id': 3, | |
427 | 'test_class': FioJobTest, | |
428 | 'job': 't0003-0ae2c6e1-post.fio', | |
429 | 'success': SUCCESS_NONZERO, | |
430 | 'pre_job': 't0003-0ae2c6e1-pre.fio', | |
431 | 'pre_success': SUCCESS_DEFAULT, | |
432 | }, | |
433 | { | |
434 | 'test_id': 4, | |
435 | 'test_class': FioJobTest, | |
436 | 'job': 't0004-8a99fdf6.fio', | |
437 | 'success': SUCCESS_DEFAULT, | |
438 | 'pre_job': None, | |
439 | 'pre_success': None, | |
440 | }, | |
441 | { | |
442 | 'test_id': 5, | |
443 | 'test_class': FioJobTest_t0005, | |
444 | 'job': 't0005-f7078f7b.fio', | |
445 | 'success': SUCCESS_DEFAULT, | |
446 | 'pre_job': None, | |
447 | 'pre_success': None, | |
448 | 'output_format': 'json', | |
449 | }, | |
450 | { | |
451 | 'test_id': 6, | |
452 | 'test_class': FioJobTest_t0006, | |
453 | 'job': 't0006-82af2a7c.fio', | |
454 | 'success': SUCCESS_DEFAULT, | |
455 | 'pre_job': None, | |
456 | 'pre_success': None, | |
457 | 'output_format': 'json', | |
458 | }, | |
459 | { | |
460 | 'test_id': 7, | |
461 | 'test_class': FioJobTest_t0007, | |
462 | 'job': 't0007-37cf9e3c.fio', | |
463 | 'success': SUCCESS_DEFAULT, | |
464 | 'pre_job': None, | |
465 | 'pre_success': None, | |
466 | 'output_format': 'json', | |
467 | }, | |
468 | { | |
469 | 'test_id': 8, | |
470 | 'test_class': FioJobTest_t0008, | |
471 | 'job': 't0008-ae2fafc8.fio', | |
472 | 'success': SUCCESS_DEFAULT, | |
473 | 'pre_job': None, | |
474 | 'pre_success': None, | |
475 | 'output_format': 'json', | |
476 | }, | |
477 | { | |
478 | 'test_id': 9, | |
479 | 'test_class': FioJobTest_t0009, | |
480 | 'job': 't0009-f8b0bd10.fio', | |
481 | 'success': SUCCESS_DEFAULT, | |
482 | 'pre_job': None, | |
483 | 'pre_success': None, | |
484 | 'output_format': 'json', | |
485 | }, | |
486 | { | |
487 | 'test_id': 10, | |
488 | 'test_class': FioJobTest, | |
489 | 'job': 't0010-b7aae4ba.fio', | |
490 | 'success': SUCCESS_DEFAULT, | |
491 | 'pre_job': None, | |
492 | 'pre_success': None, | |
493 | }, | |
494 | { | |
495 | 'test_id': 11, | |
496 | 'test_class': FioJobTest_t0011, | |
497 | 'job': 't0011-5d2788d5.fio', | |
498 | 'success': SUCCESS_DEFAULT, | |
499 | 'pre_job': None, | |
500 | 'pre_success': None, | |
501 | 'output_format': 'json', | |
502 | }, | |
503 | { | |
504 | 'test_id': 1000, | |
505 | 'test_class': FioExeTest, | |
506 | 'exe': 't/axmap', | |
507 | 'parameters': None, | |
508 | 'success': SUCCESS_DEFAULT, | |
509 | }, | |
510 | { | |
511 | 'test_id': 1001, | |
512 | 'test_class': FioExeTest, | |
513 | 'exe': 't/ieee754', | |
514 | 'parameters': None, | |
515 | 'success': SUCCESS_DEFAULT, | |
516 | }, | |
517 | { | |
518 | 'test_id': 1002, | |
519 | 'test_class': FioExeTest, | |
520 | 'exe': 't/lfsr-test', | |
521 | 'parameters': ['0xFFFFFF', '0', '0', 'verify'], | |
522 | 'success': SUCCESS_STDERR, | |
523 | }, | |
524 | { | |
525 | 'test_id': 1003, | |
526 | 'test_class': FioExeTest, | |
527 | 'exe': 't/readonly.py', | |
528 | 'parameters': ['-f', '{fio_path}'], | |
529 | 'success': SUCCESS_DEFAULT, | |
530 | }, | |
531 | { | |
532 | 'test_id': 1004, | |
533 | 'test_class': FioExeTest, | |
534 | 'exe': 't/steadystate_tests.py', | |
535 | 'parameters': ['{fio_path}'], | |
536 | 'success': SUCCESS_DEFAULT, | |
537 | }, | |
538 | { | |
539 | 'test_id': 1005, | |
540 | 'test_class': FioExeTest, | |
541 | 'exe': 't/stest', | |
542 | 'parameters': None, | |
543 | 'success': SUCCESS_STDERR, | |
544 | }, | |
545 | { | |
546 | 'test_id': 1006, | |
547 | 'test_class': FioExeTest, | |
548 | 'exe': 't/strided.py', | |
549 | 'parameters': ['{fio_path}'], | |
550 | 'success': SUCCESS_DEFAULT, | |
551 | }, | |
552 | { | |
553 | 'test_id': 1007, | |
554 | 'test_class': FioExeTest, | |
555 | 'exe': 't/zbd/run-tests-against-regular-nullb', | |
556 | 'parameters': None, | |
557 | 'success': SUCCESS_DEFAULT, | |
558 | }, | |
559 | { | |
560 | 'test_id': 1008, | |
561 | 'test_class': FioExeTest, | |
562 | 'exe': 't/zbd/run-tests-against-zoned-nullb', | |
563 | 'parameters': None, | |
564 | 'success': SUCCESS_DEFAULT, | |
565 | }, | |
566 | { | |
567 | 'test_id': 1009, | |
568 | 'test_class': FioExeTest, | |
569 | 'exe': 'unittests/unittest', | |
570 | 'parameters': None, | |
571 | 'success': SUCCESS_DEFAULT, | |
572 | }, | |
573 | ] | |
574 | ||
575 | ||
576 | def parse_args(): | |
577 | parser = argparse.ArgumentParser() | |
578 | parser.add_argument('-r', '--fio-root', | |
579 | help='fio root path') | |
580 | parser.add_argument('-f', '--fio', | |
581 | help='path to fio executable (e.g., ./fio)') | |
582 | parser.add_argument('-a', '--artifact-root', | |
583 | help='artifact root directory') | |
584 | parser.add_argument('-s', '--skip', nargs='+', type=int, | |
585 | help='list of test(s) to skip') | |
586 | parser.add_argument('-o', '--run-only', nargs='+', type=int, | |
587 | help='list of test(s) to run, skipping all others') | |
588 | args = parser.parse_args() | |
589 | ||
590 | return args | |
591 | ||
592 | ||
593 | def main(): | |
594 | logging.basicConfig(level=logging.INFO) | |
595 | ||
596 | args = parse_args() | |
597 | if args.fio_root: | |
598 | fio_root = args.fio_root | |
599 | else: | |
600 | fio_root = Path(__file__).absolute().parent.parent | |
601 | logging.debug("fio_root: %s" % fio_root) | |
602 | ||
603 | if args.fio: | |
604 | fio_path = args.fio | |
605 | else: | |
606 | fio_path = os.path.join(fio_root, "fio") | |
607 | logging.debug("fio_path: %s" % fio_path) | |
608 | ||
609 | artifact_root = args.artifact_root if args.artifact_root else \ | |
610 | "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S")) | |
611 | os.mkdir(artifact_root) | |
612 | print("Artifact directory is %s" % artifact_root) | |
613 | ||
614 | passed = 0 | |
615 | failed = 0 | |
616 | skipped = 0 | |
617 | ||
618 | for config in TEST_LIST: | |
619 | if (args.skip and config['test_id'] in args.skip) or \ | |
620 | (args.run_only and config['test_id'] not in args.run_only): | |
621 | skipped = skipped + 1 | |
622 | print("Test {0} SKIPPED".format(config['test_id'])) | |
623 | continue | |
624 | ||
625 | if issubclass(config['test_class'], FioJobTest): | |
626 | if config['pre_job']: | |
627 | fio_pre_job = os.path.join(fio_root, 't', 'jobs', | |
628 | config['pre_job']) | |
629 | else: | |
630 | fio_pre_job = None | |
631 | if config['pre_success']: | |
632 | fio_pre_success = config['pre_success'] | |
633 | else: | |
634 | fio_pre_success = None | |
635 | if 'output_format' in config: | |
636 | output_format = config['output_format'] | |
637 | else: | |
638 | output_format = 'normal' | |
639 | test = config['test_class']( | |
640 | fio_path, | |
641 | os.path.join(fio_root, 't', 'jobs', config['job']), | |
642 | config['success'], | |
643 | fio_pre_job=fio_pre_job, | |
644 | fio_pre_success=fio_pre_success, | |
645 | output_format=output_format) | |
646 | elif issubclass(config['test_class'], FioExeTest): | |
647 | exe_path = os.path.join(fio_root, config['exe']) | |
648 | if config['parameters']: | |
649 | parameters = [p.format(fio_path=fio_path) for p in config['parameters']] | |
650 | else: | |
651 | parameters = None | |
652 | test = config['test_class'](exe_path, parameters, | |
653 | config['success']) | |
654 | else: | |
655 | print("Test {0} FAILED: unable to process test config".format(config['test_id'])) | |
656 | failed = failed + 1 | |
657 | continue | |
658 | ||
659 | test.setup(artifact_root, config['test_id']) | |
660 | test.run() | |
661 | test.check_result() | |
662 | if test.passed: | |
663 | result = "PASSED" | |
664 | passed = passed + 1 | |
665 | else: | |
666 | result = "FAILED: {0}".format(test.failure_reason) | |
667 | failed = failed + 1 | |
668 | print("Test {0} {1}".format(config['test_id'], result)) | |
669 | ||
670 | print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped)) | |
671 | ||
672 | sys.exit(failed) | |
673 | ||
674 | ||
675 | if __name__ == '__main__': | |
676 | main() |