Commit | Line | Data |
---|---|---|
df1eaa36 VF |
1 | #!/usr/bin/env python3 |
2 | # SPDX-License-Identifier: GPL-2.0-only | |
3 | # | |
4 | # Copyright (c) 2019 Western Digital Corporation or its affiliates. | |
5 | # | |
6 | """ | |
7 | # run-fio-tests.py | |
8 | # | |
9 | # Automate running of fio tests | |
10 | # | |
11 | # USAGE | |
12 | # python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root] | |
13 | # [--skip # # #...] [--run-only # # #...] | |
14 | # | |
15 | # | |
16 | # EXAMPLE | |
b048455f | 17 | # # git clone git://git.kernel.dk/fio.git |
df1eaa36 VF |
18 | # # cd fio |
19 | # # make -j | |
20 | # # python3 t/run-fio-tests.py | |
21 | # | |
22 | # | |
23 | # REQUIREMENTS | |
b1bc705e | 24 | # - Python 3.5 (subprocess.run) |
df1eaa36 VF |
25 | # - Linux (libaio ioengine, zbd tests, etc) |
26 | # - The artifact directory must be on a file system that accepts 512-byte IO | |
27 | # (t0002, t0003, t0004). | |
28 | # - The artifact directory needs to be on an SSD. Otherwise tests that carry | |
29 | # out file-based IO will trigger a timeout (t0006). | |
30 | # - 4 CPUs (t0009) | |
31 | # - SciPy (steadystate_tests.py) | |
32 | # - libzbc (zbd tests) | |
33 | # - root privileges (zbd test) | |
34 | # - kernel 4.19 or later for zoned null block devices (zbd tests) | |
35 | # - CUnit support (unittests) | |
36 | # | |
37 | """ | |
38 | ||
39 | # | |
40 | # TODO run multiple tests simultaneously | |
41 | # TODO Add sgunmap tests (requires SAS SSD) | |
df1eaa36 VF |
42 | # |
43 | ||
44 | import os | |
45 | import sys | |
46 | import json | |
47 | import time | |
6d5470c3 | 48 | import shutil |
df1eaa36 VF |
49 | import logging |
50 | import argparse | |
b1bc705e | 51 | import platform |
aa9f2627 | 52 | import traceback |
df1eaa36 | 53 | import subprocess |
b1bc705e | 54 | import multiprocessing |
df1eaa36 VF |
55 | from pathlib import Path |
56 | ||
57 | ||
58 | class FioTest(object): | |
59 | """Base for all fio tests.""" | |
60 | ||
61 | def __init__(self, exe_path, parameters, success): | |
62 | self.exe_path = exe_path | |
63 | self.parameters = parameters | |
64 | self.success = success | |
65 | self.output = {} | |
66 | self.artifact_root = None | |
67 | self.testnum = None | |
68 | self.test_dir = None | |
69 | self.passed = True | |
70 | self.failure_reason = '' | |
704cc4df VF |
71 | self.command_file = None |
72 | self.stdout_file = None | |
73 | self.stderr_file = None | |
74 | self.exitcode_file = None | |
df1eaa36 VF |
75 | |
76 | def setup(self, artifact_root, testnum): | |
704cc4df VF |
77 | """Setup instance variables for test.""" |
78 | ||
df1eaa36 VF |
79 | self.artifact_root = artifact_root |
80 | self.testnum = testnum | |
81 | self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum)) | |
82 | if not os.path.exists(self.test_dir): | |
83 | os.mkdir(self.test_dir) | |
84 | ||
85 | self.command_file = os.path.join( | |
704cc4df VF |
86 | self.test_dir, |
87 | "{0}.command".format(os.path.basename(self.exe_path))) | |
df1eaa36 | 88 | self.stdout_file = os.path.join( |
704cc4df VF |
89 | self.test_dir, |
90 | "{0}.stdout".format(os.path.basename(self.exe_path))) | |
df1eaa36 | 91 | self.stderr_file = os.path.join( |
704cc4df VF |
92 | self.test_dir, |
93 | "{0}.stderr".format(os.path.basename(self.exe_path))) | |
94 | self.exitcode_file = os.path.join( | |
95 | self.test_dir, | |
96 | "{0}.exitcode".format(os.path.basename(self.exe_path))) | |
df1eaa36 VF |
97 | |
98 | def run(self): | |
704cc4df VF |
99 | """Run the test.""" |
100 | ||
df1eaa36 VF |
101 | raise NotImplementedError() |
102 | ||
103 | def check_result(self): | |
704cc4df VF |
104 | """Check test results.""" |
105 | ||
df1eaa36 VF |
106 | raise NotImplementedError() |
107 | ||
108 | ||
109 | class FioExeTest(FioTest): | |
110 | """Test consists of an executable binary or script""" | |
111 | ||
112 | def __init__(self, exe_path, parameters, success): | |
113 | """Construct a FioExeTest which is a FioTest consisting of an | |
114 | executable binary or script. | |
115 | ||
116 | exe_path: location of executable binary or script | |
117 | parameters: list of parameters for executable | |
118 | success: Definition of test success | |
119 | """ | |
120 | ||
121 | FioTest.__init__(self, exe_path, parameters, success) | |
122 | ||
df1eaa36 | 123 | def run(self): |
704cc4df VF |
124 | """Execute the binary or script described by this instance.""" |
125 | ||
58a77d2a | 126 | command = [self.exe_path] + self.parameters |
df1eaa36 VF |
127 | command_file = open(self.command_file, "w+") |
128 | command_file.write("%s\n" % command) | |
129 | command_file.close() | |
130 | ||
131 | stdout_file = open(self.stdout_file, "w+") | |
132 | stderr_file = open(self.stderr_file, "w+") | |
704cc4df | 133 | exitcode_file = open(self.exitcode_file, "w+") |
df1eaa36 | 134 | try: |
b048455f | 135 | proc = None |
df1eaa36 VF |
136 | # Avoid using subprocess.run() here because when a timeout occurs, |
137 | # fio will be stopped with SIGKILL. This does not give fio a | |
138 | # chance to clean up and means that child processes may continue | |
139 | # running and submitting IO. | |
140 | proc = subprocess.Popen(command, | |
141 | stdout=stdout_file, | |
142 | stderr=stderr_file, | |
143 | cwd=self.test_dir, | |
144 | universal_newlines=True) | |
145 | proc.communicate(timeout=self.success['timeout']) | |
704cc4df VF |
146 | exitcode_file.write('{0}\n'.format(proc.returncode)) |
147 | logging.debug("Test %d: return code: %d", self.testnum, proc.returncode) | |
df1eaa36 VF |
148 | self.output['proc'] = proc |
149 | except subprocess.TimeoutExpired: | |
150 | proc.terminate() | |
151 | proc.communicate() | |
152 | assert proc.poll() | |
153 | self.output['failure'] = 'timeout' | |
154 | except Exception: | |
b048455f VF |
155 | if proc: |
156 | if not proc.poll(): | |
157 | proc.terminate() | |
158 | proc.communicate() | |
df1eaa36 VF |
159 | self.output['failure'] = 'exception' |
160 | self.output['exc_info'] = sys.exc_info() | |
161 | finally: | |
162 | stdout_file.close() | |
163 | stderr_file.close() | |
704cc4df | 164 | exitcode_file.close() |
df1eaa36 VF |
165 | |
166 | def check_result(self): | |
704cc4df VF |
167 | """Check results of test run.""" |
168 | ||
df1eaa36 VF |
169 | if 'proc' not in self.output: |
170 | if self.output['failure'] == 'timeout': | |
171 | self.failure_reason = "{0} timeout,".format(self.failure_reason) | |
172 | else: | |
173 | assert self.output['failure'] == 'exception' | |
174 | self.failure_reason = '{0} exception: {1}, {2}'.format( | |
704cc4df VF |
175 | self.failure_reason, self.output['exc_info'][0], |
176 | self.output['exc_info'][1]) | |
df1eaa36 VF |
177 | |
178 | self.passed = False | |
179 | return | |
180 | ||
181 | if 'zero_return' in self.success: | |
182 | if self.success['zero_return']: | |
183 | if self.output['proc'].returncode != 0: | |
184 | self.passed = False | |
185 | self.failure_reason = "{0} non-zero return code,".format(self.failure_reason) | |
186 | else: | |
187 | if self.output['proc'].returncode == 0: | |
188 | self.failure_reason = "{0} zero return code,".format(self.failure_reason) | |
189 | self.passed = False | |
190 | ||
6d5470c3 | 191 | stderr_size = os.path.getsize(self.stderr_file) |
df1eaa36 | 192 | if 'stderr_empty' in self.success: |
df1eaa36 VF |
193 | if self.success['stderr_empty']: |
194 | if stderr_size != 0: | |
195 | self.failure_reason = "{0} stderr not empty,".format(self.failure_reason) | |
196 | self.passed = False | |
197 | else: | |
198 | if stderr_size == 0: | |
199 | self.failure_reason = "{0} stderr empty,".format(self.failure_reason) | |
200 | self.passed = False | |
201 | ||
202 | ||
203 | class FioJobTest(FioExeTest): | |
204 | """Test consists of a fio job""" | |
205 | ||
206 | def __init__(self, fio_path, fio_job, success, fio_pre_job=None, | |
207 | fio_pre_success=None, output_format="normal"): | |
208 | """Construct a FioJobTest which is a FioExeTest consisting of a | |
209 | single fio job file with an optional setup step. | |
210 | ||
211 | fio_path: location of fio executable | |
212 | fio_job: location of fio job file | |
213 | success: Definition of test success | |
214 | fio_pre_job: fio job for preconditioning | |
215 | fio_pre_success: Definition of test success for fio precon job | |
216 | output_format: normal (default), json, jsonplus, or terse | |
217 | """ | |
218 | ||
219 | self.fio_job = fio_job | |
220 | self.fio_pre_job = fio_pre_job | |
221 | self.fio_pre_success = fio_pre_success if fio_pre_success else success | |
222 | self.output_format = output_format | |
223 | self.precon_failed = False | |
224 | self.json_data = None | |
225 | self.fio_output = "{0}.output".format(os.path.basename(self.fio_job)) | |
226 | self.fio_args = [ | |
771dbb52 | 227 | "--max-jobs=16", |
df1eaa36 VF |
228 | "--output-format={0}".format(self.output_format), |
229 | "--output={0}".format(self.fio_output), | |
230 | self.fio_job, | |
231 | ] | |
232 | FioExeTest.__init__(self, fio_path, self.fio_args, success) | |
233 | ||
234 | def setup(self, artifact_root, testnum): | |
704cc4df VF |
235 | """Setup instance variables for fio job test.""" |
236 | ||
df1eaa36 VF |
237 | super(FioJobTest, self).setup(artifact_root, testnum) |
238 | ||
239 | self.command_file = os.path.join( | |
704cc4df VF |
240 | self.test_dir, |
241 | "{0}.command".format(os.path.basename(self.fio_job))) | |
df1eaa36 | 242 | self.stdout_file = os.path.join( |
704cc4df VF |
243 | self.test_dir, |
244 | "{0}.stdout".format(os.path.basename(self.fio_job))) | |
df1eaa36 | 245 | self.stderr_file = os.path.join( |
704cc4df VF |
246 | self.test_dir, |
247 | "{0}.stderr".format(os.path.basename(self.fio_job))) | |
248 | self.exitcode_file = os.path.join( | |
249 | self.test_dir, | |
250 | "{0}.exitcode".format(os.path.basename(self.fio_job))) | |
df1eaa36 VF |
251 | |
252 | def run_pre_job(self): | |
704cc4df VF |
253 | """Run fio job precondition step.""" |
254 | ||
df1eaa36 VF |
255 | precon = FioJobTest(self.exe_path, self.fio_pre_job, |
256 | self.fio_pre_success, | |
257 | output_format=self.output_format) | |
258 | precon.setup(self.artifact_root, self.testnum) | |
259 | precon.run() | |
260 | precon.check_result() | |
261 | self.precon_failed = not precon.passed | |
262 | self.failure_reason = precon.failure_reason | |
263 | ||
264 | def run(self): | |
704cc4df VF |
265 | """Run fio job test.""" |
266 | ||
df1eaa36 VF |
267 | if self.fio_pre_job: |
268 | self.run_pre_job() | |
269 | ||
270 | if not self.precon_failed: | |
271 | super(FioJobTest, self).run() | |
272 | else: | |
704cc4df | 273 | logging.debug("Test %d: precondition step failed", self.testnum) |
df1eaa36 | 274 | |
15a73987 VF |
275 | @classmethod |
276 | def get_file(cls, filename): | |
277 | """Safely read a file.""" | |
278 | file_data = '' | |
279 | success = True | |
280 | ||
281 | try: | |
282 | with open(filename, "r") as output_file: | |
283 | file_data = output_file.read() | |
284 | except OSError: | |
285 | success = False | |
286 | ||
287 | return file_data, success | |
288 | ||
df1eaa36 | 289 | def check_result(self): |
704cc4df VF |
290 | """Check fio job results.""" |
291 | ||
df1eaa36 VF |
292 | if self.precon_failed: |
293 | self.passed = False | |
294 | self.failure_reason = "{0} precondition step failed,".format(self.failure_reason) | |
295 | return | |
296 | ||
297 | super(FioJobTest, self).check_result() | |
298 | ||
6d5470c3 VF |
299 | if not self.passed: |
300 | return | |
301 | ||
704cc4df | 302 | if 'json' not in self.output_format: |
742b8799 VF |
303 | return |
304 | ||
15a73987 VF |
305 | file_data, success = self.get_file(os.path.join(self.test_dir, self.fio_output)) |
306 | if not success: | |
742b8799 VF |
307 | self.failure_reason = "{0} unable to open output file,".format(self.failure_reason) |
308 | self.passed = False | |
309 | return | |
310 | ||
311 | # | |
312 | # Sometimes fio informational messages are included at the top of the | |
313 | # JSON output, especially under Windows. Try to decode output as JSON | |
db2637ec | 314 | # data, skipping everything until the first { |
742b8799 VF |
315 | # |
316 | lines = file_data.splitlines() | |
db2637ec VF |
317 | file_data = '\n'.join(lines[lines.index("{"):]) |
318 | try: | |
319 | self.json_data = json.loads(file_data) | |
320 | except json.JSONDecodeError: | |
321 | self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason) | |
322 | self.passed = False | |
df1eaa36 VF |
323 | |
324 | ||
325 | class FioJobTest_t0005(FioJobTest): | |
326 | """Test consists of fio test job t0005 | |
327 | Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400""" | |
328 | ||
329 | def check_result(self): | |
330 | super(FioJobTest_t0005, self).check_result() | |
331 | ||
332 | if not self.passed: | |
333 | return | |
334 | ||
335 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400: | |
336 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
337 | self.passed = False | |
338 | if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400: | |
339 | self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) | |
340 | self.passed = False | |
341 | ||
342 | ||
343 | class FioJobTest_t0006(FioJobTest): | |
344 | """Test consists of fio test job t0006 | |
345 | Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']""" | |
346 | ||
347 | def check_result(self): | |
348 | super(FioJobTest_t0006, self).check_result() | |
349 | ||
350 | if not self.passed: | |
351 | return | |
352 | ||
353 | ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \ | |
354 | / self.json_data['jobs'][0]['write']['io_kbytes'] | |
704cc4df | 355 | logging.debug("Test %d: ratio: %f", self.testnum, ratio) |
df1eaa36 VF |
356 | if ratio < 1.99 or ratio > 2.01: |
357 | self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason) | |
358 | self.passed = False | |
359 | ||
360 | ||
361 | class FioJobTest_t0007(FioJobTest): | |
362 | """Test consists of fio test job t0007 | |
363 | Confirm that read['io_kbytes'] = 87040""" | |
364 | ||
365 | def check_result(self): | |
366 | super(FioJobTest_t0007, self).check_result() | |
367 | ||
368 | if not self.passed: | |
369 | return | |
370 | ||
371 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040: | |
372 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
373 | self.passed = False | |
374 | ||
375 | ||
376 | class FioJobTest_t0008(FioJobTest): | |
377 | """Test consists of fio test job t0008 | |
378 | Confirm that read['io_kbytes'] = 32768 and that | |
379 | write['io_kbytes'] ~ 16568 | |
380 | ||
381 | I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of | |
382 | 16585, 16588. With two runs of fio-3.16 I obtained 16568""" | |
383 | ||
384 | def check_result(self): | |
385 | super(FioJobTest_t0008, self).check_result() | |
386 | ||
387 | if not self.passed: | |
388 | return | |
389 | ||
390 | ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568 | |
704cc4df | 391 | logging.debug("Test %d: ratio: %f", self.testnum, ratio) |
df1eaa36 VF |
392 | |
393 | if ratio < 0.99 or ratio > 1.01: | |
394 | self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason) | |
395 | self.passed = False | |
396 | if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768: | |
397 | self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason) | |
398 | self.passed = False | |
399 | ||
400 | ||
401 | class FioJobTest_t0009(FioJobTest): | |
402 | """Test consists of fio test job t0009 | |
403 | Confirm that runtime >= 60s""" | |
404 | ||
405 | def check_result(self): | |
406 | super(FioJobTest_t0009, self).check_result() | |
407 | ||
408 | if not self.passed: | |
409 | return | |
410 | ||
704cc4df | 411 | logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed']) |
df1eaa36 VF |
412 | |
413 | if self.json_data['jobs'][0]['elapsed'] < 60: | |
414 | self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason) | |
415 | self.passed = False | |
416 | ||
417 | ||
d4e74fda DB |
418 | class FioJobTest_t0012(FioJobTest): |
419 | """Test consists of fio test job t0012 | |
420 | Confirm ratios of job iops are 1:5:10 | |
421 | job1,job2,job3 respectively""" | |
422 | ||
423 | def check_result(self): | |
424 | super(FioJobTest_t0012, self).check_result() | |
425 | ||
426 | if not self.passed: | |
427 | return | |
428 | ||
429 | iops_files = [] | |
430 | for i in range(1,4): | |
431 | file_data, success = self.get_file(os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(self.fio_job), i))) | |
432 | ||
433 | if not success: | |
434 | self.failure_reason = "{0} unable to open output file,".format(self.failure_reason) | |
435 | self.passed = False | |
436 | return | |
437 | ||
438 | iops_files.append(file_data.splitlines()) | |
439 | ||
440 | # there are 9 samples for job1 and job2, 4 samples for job3 | |
441 | iops1 = 0.0 | |
442 | iops2 = 0.0 | |
443 | iops3 = 0.0 | |
444 | for i in range(9): | |
445 | iops1 = iops1 + float(iops_files[0][i].split(',')[1]) | |
446 | iops2 = iops2 + float(iops_files[1][i].split(',')[1]) | |
447 | iops3 = iops3 + float(iops_files[2][i].split(',')[1]) | |
448 | ||
449 | ratio1 = iops3/iops2 | |
450 | ratio2 = iops3/iops1 | |
451 | logging.debug( | |
452 | "sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} job3/job2={4:.3f} job3/job1={5:.3f}".format( | |
453 | i, iops1, iops2, iops3, ratio1, ratio2 | |
454 | ) | |
455 | ) | |
456 | ||
457 | # test job1 and job2 succeeded to recalibrate | |
458 | if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13: | |
459 | self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} iops3={3} expected r1~2 r2~10 got r1={4:.3f} r2={5:.3f},".format( | |
460 | self.failure_reason, iops1, iops2, iops3, ratio1, ratio2 | |
461 | ) | |
462 | self.passed = False | |
463 | return | |
464 | ||
465 | ||
466 | class FioJobTest_t0014(FioJobTest): | |
467 | """Test consists of fio test job t0014 | |
468 | Confirm that job1_iops / job2_iops ~ 1:2 for entire duration | |
469 | and that job1_iops / job3_iops ~ 1:3 for first half of duration. | |
470 | ||
471 | The test is about making sure the flow feature can | |
472 | re-calibrate the activity dynamically""" | |
473 | ||
474 | def check_result(self): | |
475 | super(FioJobTest_t0014, self).check_result() | |
476 | ||
477 | if not self.passed: | |
478 | return | |
479 | ||
480 | iops_files = [] | |
481 | for i in range(1,4): | |
482 | file_data, success = self.get_file(os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(self.fio_job), i))) | |
483 | ||
484 | if not success: | |
485 | self.failure_reason = "{0} unable to open output file,".format(self.failure_reason) | |
486 | self.passed = False | |
487 | return | |
488 | ||
489 | iops_files.append(file_data.splitlines()) | |
490 | ||
491 | # there are 9 samples for job1 and job2, 4 samples for job3 | |
492 | iops1 = 0.0 | |
493 | iops2 = 0.0 | |
494 | iops3 = 0.0 | |
495 | for i in range(9): | |
496 | if i < 4: | |
497 | iops3 = iops3 + float(iops_files[2][i].split(',')[1]) | |
498 | elif i == 4: | |
499 | ratio1 = iops1 / iops2 | |
500 | ratio2 = iops1 / iops3 | |
501 | ||
502 | ||
503 | if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45: | |
504 | self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} iops3={3}\ | |
505 | expected r1~0.5 r2~0.33 got r1={4:.3f} r2={5:.3f},".format( | |
506 | self.failure_reason, iops1, iops2, iops3, ratio1, ratio2 | |
507 | ) | |
508 | self.passed = False | |
509 | ||
510 | iops1 = iops1 + float(iops_files[0][i].split(',')[1]) | |
511 | iops2 = iops2 + float(iops_files[1][i].split(',')[1]) | |
512 | ||
513 | ratio1 = iops1/iops2 | |
514 | ratio2 = iops1/iops3 | |
515 | logging.debug( | |
516 | "sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} job1/job2={4:.3f} job1/job3={5:.3f}".format( | |
517 | i, iops1, iops2, iops3, ratio1, ratio2 | |
518 | ) | |
519 | ) | |
520 | ||
521 | # test job1 and job2 succeeded to recalibrate | |
522 | if ratio1 < 0.43 or ratio1 > 0.57: | |
523 | self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} expected ratio~0.5 got ratio={3:.3f},".format( | |
524 | self.failure_reason, iops1, iops2, ratio1 | |
525 | ) | |
526 | self.passed = False | |
527 | return | |
528 | ||
529 | ||
60ebb939 | 530 | class FioJobTest_t0015(FioJobTest): |
de31fe9a | 531 | """Test consists of fio test jobs t0015 and t0016 |
60ebb939 VF |
532 | Confirm that mean(slat) + mean(clat) = mean(tlat)""" |
533 | ||
534 | def check_result(self): | |
535 | super(FioJobTest_t0015, self).check_result() | |
536 | ||
537 | if not self.passed: | |
538 | return | |
539 | ||
540 | slat = self.json_data['jobs'][0]['read']['slat_ns']['mean'] | |
541 | clat = self.json_data['jobs'][0]['read']['clat_ns']['mean'] | |
542 | tlat = self.json_data['jobs'][0]['read']['lat_ns']['mean'] | |
543 | logging.debug('Test %d: slat %f, clat %f, tlat %f', self.testnum, slat, clat, tlat) | |
544 | ||
545 | if abs(slat + clat - tlat) > 1: | |
546 | self.failure_reason = "{0} slat {1} + clat {2} = {3} != tlat {4},".format( | |
547 | self.failure_reason, slat, clat, slat+clat, tlat) | |
548 | self.passed = False | |
549 | ||
550 | ||
ef54f290 VF |
551 | class FioJobTest_t0019(FioJobTest): |
552 | """Test consists of fio test job t0019 | |
553 | Confirm that all offsets were touched sequentially""" | |
554 | ||
555 | def check_result(self): | |
556 | super(FioJobTest_t0019, self).check_result() | |
557 | ||
558 | bw_log_filename = os.path.join(self.test_dir, "test_bw.log") | |
559 | file_data, success = self.get_file(bw_log_filename) | |
560 | log_lines = file_data.split('\n') | |
561 | ||
562 | prev = -4096 | |
563 | for line in log_lines: | |
564 | if len(line.strip()) == 0: | |
565 | continue | |
566 | cur = int(line.split(',')[4]) | |
567 | if cur - prev != 4096: | |
568 | self.passed = False | |
569 | self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur) | |
570 | return | |
571 | prev = cur | |
572 | ||
573 | if cur/4096 != 255: | |
574 | self.passed = False | |
575 | self.failure_reason = "unexpected last offset {0}".format(cur) | |
576 | ||
577 | ||
578 | class FioJobTest_t0020(FioJobTest): | |
eb40b275 | 579 | """Test consists of fio test jobs t0020 and t0021 |
ef54f290 VF |
580 | Confirm that almost all offsets were touched non-sequentially""" |
581 | ||
582 | def check_result(self): | |
583 | super(FioJobTest_t0020, self).check_result() | |
584 | ||
585 | bw_log_filename = os.path.join(self.test_dir, "test_bw.log") | |
586 | file_data, success = self.get_file(bw_log_filename) | |
587 | log_lines = file_data.split('\n') | |
588 | ||
589 | seq_count = 0 | |
590 | offsets = set() | |
591 | ||
592 | prev = int(log_lines[0].split(',')[4]) | |
593 | for line in log_lines[1:]: | |
594 | offsets.add(prev/4096) | |
595 | if len(line.strip()) == 0: | |
596 | continue | |
597 | cur = int(line.split(',')[4]) | |
598 | if cur - prev == 4096: | |
599 | seq_count += 1 | |
600 | prev = cur | |
601 | ||
602 | # 10 is an arbitrary threshold | |
603 | if seq_count > 10: | |
604 | self.passed = False | |
605 | self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count) | |
606 | ||
607 | if len(offsets) != 256: | |
608 | self.passed = False | |
609 | self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets)) | |
610 | ||
611 | for i in range(256): | |
612 | if not i in offsets: | |
613 | self.passed = False | |
614 | self.failure_reason += " missing offset {0}".format(i*4096) | |
615 | ||
616 | ||
eb40b275 VF |
617 | class FioJobTest_t0022(FioJobTest): |
618 | """Test consists of fio test job t0022""" | |
619 | ||
620 | def check_result(self): | |
621 | super(FioJobTest_t0022, self).check_result() | |
622 | ||
623 | bw_log_filename = os.path.join(self.test_dir, "test_bw.log") | |
624 | file_data, success = self.get_file(bw_log_filename) | |
625 | log_lines = file_data.split('\n') | |
626 | ||
627 | filesize = 1024*1024 | |
628 | bs = 4096 | |
629 | seq_count = 0 | |
630 | offsets = set() | |
631 | ||
632 | prev = int(log_lines[0].split(',')[4]) | |
633 | for line in log_lines[1:]: | |
634 | offsets.add(prev/bs) | |
635 | if len(line.strip()) == 0: | |
636 | continue | |
637 | cur = int(line.split(',')[4]) | |
638 | if cur - prev == bs: | |
639 | seq_count += 1 | |
640 | prev = cur | |
641 | ||
642 | # 10 is an arbitrary threshold | |
643 | if seq_count > 10: | |
644 | self.passed = False | |
645 | self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count) | |
646 | ||
647 | if len(offsets) == filesize/bs: | |
648 | self.passed = False | |
649 | self.failure_reason += " no duplicate offsets found with norandommap=1".format(len(offsets)) | |
650 | ||
651 | ||
0a602473 | 652 | class FioJobTest_iops_rate(FioJobTest): |
df1eaa36 VF |
653 | """Test consists of fio test job t0009 |
654 | Confirm that job0 iops == 1000 | |
655 | and that job1_iops / job0_iops ~ 8 | |
656 | With two runs of fio-3.16 I observed a ratio of 8.3""" | |
657 | ||
658 | def check_result(self): | |
0a602473 | 659 | super(FioJobTest_iops_rate, self).check_result() |
df1eaa36 VF |
660 | |
661 | if not self.passed: | |
662 | return | |
663 | ||
664 | iops1 = self.json_data['jobs'][0]['read']['iops'] | |
7ddc4ed1 | 665 | logging.debug("Test %d: iops1: %f", self.testnum, iops1) |
df1eaa36 | 666 | iops2 = self.json_data['jobs'][1]['read']['iops'] |
7ddc4ed1 | 667 | logging.debug("Test %d: iops2: %f", self.testnum, iops2) |
df1eaa36 | 668 | ratio = iops2 / iops1 |
704cc4df | 669 | logging.debug("Test %d: ratio: %f", self.testnum, ratio) |
df1eaa36 | 670 | |
0a9b8988 | 671 | if iops1 < 950 or iops1 > 1050: |
df1eaa36 VF |
672 | self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason) |
673 | self.passed = False | |
674 | ||
d4e74fda | 675 | if ratio < 6 or ratio > 10: |
df1eaa36 VF |
676 | self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason) |
677 | self.passed = False | |
678 | ||
679 | ||
b1bc705e VF |
680 | class Requirements(object): |
681 | """Requirements consists of multiple run environment characteristics. | |
682 | These are to determine if a particular test can be run""" | |
683 | ||
684 | _linux = False | |
685 | _libaio = False | |
a2947c33 | 686 | _io_uring = False |
b1bc705e VF |
687 | _zbd = False |
688 | _root = False | |
689 | _zoned_nullb = False | |
690 | _not_macos = False | |
c58b33b4 | 691 | _not_windows = False |
b1bc705e VF |
692 | _unittests = False |
693 | _cpucount4 = False | |
694 | ||
695 | def __init__(self, fio_root): | |
696 | Requirements._not_macos = platform.system() != "Darwin" | |
c58b33b4 | 697 | Requirements._not_windows = platform.system() != "Windows" |
b1bc705e VF |
698 | Requirements._linux = platform.system() == "Linux" |
699 | ||
700 | if Requirements._linux: | |
15a73987 VF |
701 | config_file = os.path.join(fio_root, "config-host.h") |
702 | contents, success = FioJobTest.get_file(config_file) | |
703 | if not success: | |
b1bc705e VF |
704 | print("Unable to open {0} to check requirements".format(config_file)) |
705 | Requirements._zbd = True | |
706 | else: | |
b7694961 | 707 | Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents |
b1bc705e VF |
708 | Requirements._libaio = "CONFIG_LIBAIO" in contents |
709 | ||
a2947c33 VF |
710 | contents, success = FioJobTest.get_file("/proc/kallsyms") |
711 | if not success: | |
712 | print("Unable to open '/proc/kallsyms' to probe for io_uring support") | |
713 | else: | |
714 | Requirements._io_uring = "io_uring_setup" in contents | |
715 | ||
b1bc705e VF |
716 | Requirements._root = (os.geteuid() == 0) |
717 | if Requirements._zbd and Requirements._root: | |
8854e368 VF |
718 | try: |
719 | subprocess.run(["modprobe", "null_blk"], | |
720 | stdout=subprocess.PIPE, | |
721 | stderr=subprocess.PIPE) | |
722 | if os.path.exists("/sys/module/null_blk/parameters/zoned"): | |
723 | Requirements._zoned_nullb = True | |
724 | except Exception: | |
725 | pass | |
b1bc705e | 726 | |
742b8799 VF |
727 | if platform.system() == "Windows": |
728 | utest_exe = "unittest.exe" | |
729 | else: | |
730 | utest_exe = "unittest" | |
731 | unittest_path = os.path.join(fio_root, "unittests", utest_exe) | |
b1bc705e VF |
732 | Requirements._unittests = os.path.exists(unittest_path) |
733 | ||
734 | Requirements._cpucount4 = multiprocessing.cpu_count() >= 4 | |
735 | ||
736 | req_list = [Requirements.linux, | |
737 | Requirements.libaio, | |
a2947c33 | 738 | Requirements.io_uring, |
b1bc705e VF |
739 | Requirements.zbd, |
740 | Requirements.root, | |
741 | Requirements.zoned_nullb, | |
742 | Requirements.not_macos, | |
c58b33b4 | 743 | Requirements.not_windows, |
b1bc705e VF |
744 | Requirements.unittests, |
745 | Requirements.cpucount4] | |
746 | for req in req_list: | |
747 | value, desc = req() | |
704cc4df | 748 | logging.debug("Requirements: Requirement '%s' met? %s", desc, value) |
b1bc705e | 749 | |
704cc4df VF |
750 | @classmethod |
751 | def linux(cls): | |
752 | """Are we running on Linux?""" | |
b1bc705e VF |
753 | return Requirements._linux, "Linux required" |
754 | ||
704cc4df VF |
755 | @classmethod |
756 | def libaio(cls): | |
757 | """Is libaio available?""" | |
b1bc705e VF |
758 | return Requirements._libaio, "libaio required" |
759 | ||
a2947c33 VF |
760 | @classmethod |
761 | def io_uring(cls): | |
762 | """Is io_uring available?""" | |
763 | return Requirements._io_uring, "io_uring required" | |
764 | ||
704cc4df VF |
765 | @classmethod |
766 | def zbd(cls): | |
767 | """Is ZBD support available?""" | |
b1bc705e VF |
768 | return Requirements._zbd, "Zoned block device support required" |
769 | ||
704cc4df VF |
770 | @classmethod |
771 | def root(cls): | |
772 | """Are we running as root?""" | |
b1bc705e VF |
773 | return Requirements._root, "root required" |
774 | ||
704cc4df VF |
775 | @classmethod |
776 | def zoned_nullb(cls): | |
777 | """Are zoned null block devices available?""" | |
b1bc705e VF |
778 | return Requirements._zoned_nullb, "Zoned null block device support required" |
779 | ||
704cc4df VF |
780 | @classmethod |
781 | def not_macos(cls): | |
782 | """Are we running on a platform other than macOS?""" | |
b1bc705e VF |
783 | return Requirements._not_macos, "platform other than macOS required" |
784 | ||
704cc4df VF |
785 | @classmethod |
786 | def not_windows(cls): | |
787 | """Are we running on a platform other than Windws?""" | |
c58b33b4 VF |
788 | return Requirements._not_windows, "platform other than Windows required" |
789 | ||
704cc4df VF |
790 | @classmethod |
791 | def unittests(cls): | |
792 | """Were unittests built?""" | |
b1bc705e VF |
793 | return Requirements._unittests, "Unittests support required" |
794 | ||
704cc4df VF |
795 | @classmethod |
796 | def cpucount4(cls): | |
797 | """Do we have at least 4 CPUs?""" | |
b1bc705e VF |
798 | return Requirements._cpucount4, "4+ CPUs required" |
799 | ||
800 | ||
df1eaa36 | 801 | SUCCESS_DEFAULT = { |
704cc4df VF |
802 | 'zero_return': True, |
803 | 'stderr_empty': True, | |
804 | 'timeout': 600, | |
805 | } | |
df1eaa36 | 806 | SUCCESS_NONZERO = { |
704cc4df VF |
807 | 'zero_return': False, |
808 | 'stderr_empty': False, | |
809 | 'timeout': 600, | |
810 | } | |
df1eaa36 | 811 | SUCCESS_STDERR = { |
704cc4df VF |
812 | 'zero_return': True, |
813 | 'stderr_empty': False, | |
814 | 'timeout': 600, | |
815 | } | |
df1eaa36 | 816 | TEST_LIST = [ |
704cc4df VF |
817 | { |
818 | 'test_id': 1, | |
819 | 'test_class': FioJobTest, | |
820 | 'job': 't0001-52c58027.fio', | |
821 | 'success': SUCCESS_DEFAULT, | |
822 | 'pre_job': None, | |
823 | 'pre_success': None, | |
824 | 'requirements': [], | |
825 | }, | |
826 | { | |
827 | 'test_id': 2, | |
828 | 'test_class': FioJobTest, | |
829 | 'job': 't0002-13af05ae-post.fio', | |
830 | 'success': SUCCESS_DEFAULT, | |
831 | 'pre_job': 't0002-13af05ae-pre.fio', | |
832 | 'pre_success': None, | |
833 | 'requirements': [Requirements.linux, Requirements.libaio], | |
834 | }, | |
835 | { | |
836 | 'test_id': 3, | |
837 | 'test_class': FioJobTest, | |
838 | 'job': 't0003-0ae2c6e1-post.fio', | |
839 | 'success': SUCCESS_NONZERO, | |
840 | 'pre_job': 't0003-0ae2c6e1-pre.fio', | |
841 | 'pre_success': SUCCESS_DEFAULT, | |
842 | 'requirements': [Requirements.linux, Requirements.libaio], | |
843 | }, | |
844 | { | |
845 | 'test_id': 4, | |
846 | 'test_class': FioJobTest, | |
847 | 'job': 't0004-8a99fdf6.fio', | |
848 | 'success': SUCCESS_DEFAULT, | |
849 | 'pre_job': None, | |
850 | 'pre_success': None, | |
851 | 'requirements': [Requirements.linux, Requirements.libaio], | |
852 | }, | |
853 | { | |
854 | 'test_id': 5, | |
855 | 'test_class': FioJobTest_t0005, | |
856 | 'job': 't0005-f7078f7b.fio', | |
857 | 'success': SUCCESS_DEFAULT, | |
858 | 'pre_job': None, | |
859 | 'pre_success': None, | |
860 | 'output_format': 'json', | |
861 | 'requirements': [Requirements.not_windows], | |
862 | }, | |
863 | { | |
864 | 'test_id': 6, | |
865 | 'test_class': FioJobTest_t0006, | |
866 | 'job': 't0006-82af2a7c.fio', | |
867 | 'success': SUCCESS_DEFAULT, | |
868 | 'pre_job': None, | |
869 | 'pre_success': None, | |
870 | 'output_format': 'json', | |
871 | 'requirements': [Requirements.linux, Requirements.libaio], | |
872 | }, | |
873 | { | |
874 | 'test_id': 7, | |
875 | 'test_class': FioJobTest_t0007, | |
876 | 'job': 't0007-37cf9e3c.fio', | |
877 | 'success': SUCCESS_DEFAULT, | |
878 | 'pre_job': None, | |
879 | 'pre_success': None, | |
880 | 'output_format': 'json', | |
881 | 'requirements': [], | |
882 | }, | |
883 | { | |
884 | 'test_id': 8, | |
885 | 'test_class': FioJobTest_t0008, | |
886 | 'job': 't0008-ae2fafc8.fio', | |
887 | 'success': SUCCESS_DEFAULT, | |
888 | 'pre_job': None, | |
889 | 'pre_success': None, | |
890 | 'output_format': 'json', | |
891 | 'requirements': [], | |
892 | }, | |
893 | { | |
894 | 'test_id': 9, | |
895 | 'test_class': FioJobTest_t0009, | |
896 | 'job': 't0009-f8b0bd10.fio', | |
897 | 'success': SUCCESS_DEFAULT, | |
898 | 'pre_job': None, | |
899 | 'pre_success': None, | |
900 | 'output_format': 'json', | |
901 | 'requirements': [Requirements.not_macos, | |
902 | Requirements.cpucount4], | |
903 | # mac os does not support CPU affinity | |
904 | }, | |
905 | { | |
906 | 'test_id': 10, | |
907 | 'test_class': FioJobTest, | |
908 | 'job': 't0010-b7aae4ba.fio', | |
909 | 'success': SUCCESS_DEFAULT, | |
910 | 'pre_job': None, | |
911 | 'pre_success': None, | |
912 | 'requirements': [], | |
913 | }, | |
914 | { | |
915 | 'test_id': 11, | |
0a602473 | 916 | 'test_class': FioJobTest_iops_rate, |
704cc4df VF |
917 | 'job': 't0011-5d2788d5.fio', |
918 | 'success': SUCCESS_DEFAULT, | |
919 | 'pre_job': None, | |
920 | 'pre_success': None, | |
921 | 'output_format': 'json', | |
922 | 'requirements': [], | |
923 | }, | |
0a602473 BVA |
924 | { |
925 | 'test_id': 12, | |
d4e74fda | 926 | 'test_class': FioJobTest_t0012, |
0a602473 BVA |
927 | 'job': 't0012.fio', |
928 | 'success': SUCCESS_DEFAULT, | |
929 | 'pre_job': None, | |
930 | 'pre_success': None, | |
931 | 'output_format': 'json', | |
d4e74fda | 932 | 'requirements': [], |
0a602473 | 933 | }, |
f0c7ae7a BVA |
934 | { |
935 | 'test_id': 13, | |
061a0773 | 936 | 'test_class': FioJobTest, |
f0c7ae7a BVA |
937 | 'job': 't0013.fio', |
938 | 'success': SUCCESS_DEFAULT, | |
939 | 'pre_job': None, | |
940 | 'pre_success': None, | |
941 | 'output_format': 'json', | |
942 | 'requirements': [], | |
943 | }, | |
d4e74fda DB |
944 | { |
945 | 'test_id': 14, | |
946 | 'test_class': FioJobTest_t0014, | |
947 | 'job': 't0014.fio', | |
948 | 'success': SUCCESS_DEFAULT, | |
949 | 'pre_job': None, | |
950 | 'pre_success': None, | |
951 | 'output_format': 'json', | |
952 | 'requirements': [], | |
953 | }, | |
60ebb939 VF |
954 | { |
955 | 'test_id': 15, | |
956 | 'test_class': FioJobTest_t0015, | |
957 | 'job': 't0015-e78980ff.fio', | |
958 | 'success': SUCCESS_DEFAULT, | |
959 | 'pre_job': None, | |
960 | 'pre_success': None, | |
961 | 'output_format': 'json', | |
962 | 'requirements': [Requirements.linux, Requirements.libaio], | |
963 | }, | |
de31fe9a VF |
964 | { |
965 | 'test_id': 16, | |
966 | 'test_class': FioJobTest_t0015, | |
f045d5f8 | 967 | 'job': 't0016-d54ae22.fio', |
de31fe9a VF |
968 | 'success': SUCCESS_DEFAULT, |
969 | 'pre_job': None, | |
970 | 'pre_success': None, | |
971 | 'output_format': 'json', | |
972 | 'requirements': [], | |
973 | }, | |
31a58cba VF |
974 | { |
975 | 'test_id': 17, | |
976 | 'test_class': FioJobTest_t0015, | |
977 | 'job': 't0017.fio', | |
978 | 'success': SUCCESS_DEFAULT, | |
979 | 'pre_job': None, | |
980 | 'pre_success': None, | |
981 | 'output_format': 'json', | |
982 | 'requirements': [Requirements.not_windows], | |
983 | }, | |
a2947c33 VF |
984 | { |
985 | 'test_id': 18, | |
986 | 'test_class': FioJobTest, | |
987 | 'job': 't0018.fio', | |
988 | 'success': SUCCESS_DEFAULT, | |
989 | 'pre_job': None, | |
990 | 'pre_success': None, | |
991 | 'requirements': [Requirements.linux, Requirements.io_uring], | |
992 | }, | |
ef54f290 VF |
993 | { |
994 | 'test_id': 19, | |
995 | 'test_class': FioJobTest_t0019, | |
996 | 'job': 't0019.fio', | |
997 | 'success': SUCCESS_DEFAULT, | |
998 | 'pre_job': None, | |
999 | 'pre_success': None, | |
1000 | 'requirements': [], | |
1001 | }, | |
1002 | { | |
1003 | 'test_id': 20, | |
1004 | 'test_class': FioJobTest_t0020, | |
1005 | 'job': 't0020.fio', | |
1006 | 'success': SUCCESS_DEFAULT, | |
1007 | 'pre_job': None, | |
1008 | 'pre_success': None, | |
1009 | 'requirements': [], | |
1010 | }, | |
eb40b275 VF |
1011 | { |
1012 | 'test_id': 21, | |
1013 | 'test_class': FioJobTest_t0020, | |
1014 | 'job': 't0021.fio', | |
1015 | 'success': SUCCESS_DEFAULT, | |
1016 | 'pre_job': None, | |
1017 | 'pre_success': None, | |
1018 | 'requirements': [], | |
1019 | }, | |
1020 | { | |
1021 | 'test_id': 22, | |
1022 | 'test_class': FioJobTest_t0022, | |
1023 | 'job': 't0022.fio', | |
1024 | 'success': SUCCESS_DEFAULT, | |
1025 | 'pre_job': None, | |
1026 | 'pre_success': None, | |
1027 | 'requirements': [], | |
1028 | }, | |
704cc4df VF |
1029 | { |
1030 | 'test_id': 1000, | |
1031 | 'test_class': FioExeTest, | |
1032 | 'exe': 't/axmap', | |
1033 | 'parameters': None, | |
1034 | 'success': SUCCESS_DEFAULT, | |
1035 | 'requirements': [], | |
1036 | }, | |
1037 | { | |
1038 | 'test_id': 1001, | |
1039 | 'test_class': FioExeTest, | |
1040 | 'exe': 't/ieee754', | |
1041 | 'parameters': None, | |
1042 | 'success': SUCCESS_DEFAULT, | |
1043 | 'requirements': [], | |
1044 | }, | |
1045 | { | |
1046 | 'test_id': 1002, | |
1047 | 'test_class': FioExeTest, | |
1048 | 'exe': 't/lfsr-test', | |
1049 | 'parameters': ['0xFFFFFF', '0', '0', 'verify'], | |
1050 | 'success': SUCCESS_STDERR, | |
1051 | 'requirements': [], | |
1052 | }, | |
1053 | { | |
1054 | 'test_id': 1003, | |
1055 | 'test_class': FioExeTest, | |
1056 | 'exe': 't/readonly.py', | |
1057 | 'parameters': ['-f', '{fio_path}'], | |
1058 | 'success': SUCCESS_DEFAULT, | |
1059 | 'requirements': [], | |
1060 | }, | |
1061 | { | |
1062 | 'test_id': 1004, | |
1063 | 'test_class': FioExeTest, | |
1064 | 'exe': 't/steadystate_tests.py', | |
1065 | 'parameters': ['{fio_path}'], | |
1066 | 'success': SUCCESS_DEFAULT, | |
1067 | 'requirements': [], | |
1068 | }, | |
1069 | { | |
1070 | 'test_id': 1005, | |
1071 | 'test_class': FioExeTest, | |
1072 | 'exe': 't/stest', | |
1073 | 'parameters': None, | |
1074 | 'success': SUCCESS_STDERR, | |
1075 | 'requirements': [], | |
1076 | }, | |
1077 | { | |
1078 | 'test_id': 1006, | |
1079 | 'test_class': FioExeTest, | |
1080 | 'exe': 't/strided.py', | |
1081 | 'parameters': ['{fio_path}'], | |
1082 | 'success': SUCCESS_DEFAULT, | |
1083 | 'requirements': [], | |
1084 | }, | |
1085 | { | |
1086 | 'test_id': 1007, | |
1087 | 'test_class': FioExeTest, | |
037b2b50 DF |
1088 | 'exe': 't/zbd/run-tests-against-nullb', |
1089 | 'parameters': ['-s', '1'], | |
704cc4df VF |
1090 | 'success': SUCCESS_DEFAULT, |
1091 | 'requirements': [Requirements.linux, Requirements.zbd, | |
1092 | Requirements.root], | |
1093 | }, | |
1094 | { | |
1095 | 'test_id': 1008, | |
1096 | 'test_class': FioExeTest, | |
037b2b50 DF |
1097 | 'exe': 't/zbd/run-tests-against-nullb', |
1098 | 'parameters': ['-s', '2'], | |
704cc4df VF |
1099 | 'success': SUCCESS_DEFAULT, |
1100 | 'requirements': [Requirements.linux, Requirements.zbd, | |
1101 | Requirements.root, Requirements.zoned_nullb], | |
1102 | }, | |
1103 | { | |
1104 | 'test_id': 1009, | |
1105 | 'test_class': FioExeTest, | |
1106 | 'exe': 'unittests/unittest', | |
1107 | 'parameters': None, | |
1108 | 'success': SUCCESS_DEFAULT, | |
1109 | 'requirements': [Requirements.unittests], | |
1110 | }, | |
1111 | { | |
1112 | 'test_id': 1010, | |
1113 | 'test_class': FioExeTest, | |
1114 | 'exe': 't/latency_percentiles.py', | |
1115 | 'parameters': ['-f', '{fio_path}'], | |
1116 | 'success': SUCCESS_DEFAULT, | |
1117 | 'requirements': [], | |
1118 | }, | |
8403eca6 VF |
1119 | { |
1120 | 'test_id': 1011, | |
1121 | 'test_class': FioExeTest, | |
1122 | 'exe': 't/jsonplus2csv_test.py', | |
1123 | 'parameters': ['-f', '{fio_path}'], | |
1124 | 'success': SUCCESS_DEFAULT, | |
1125 | 'requirements': [], | |
1126 | }, | |
df1eaa36 VF |
1127 | ] |
1128 | ||
1129 | ||
1130 | def parse_args(): | |
704cc4df VF |
1131 | """Parse command-line arguments.""" |
1132 | ||
df1eaa36 VF |
1133 | parser = argparse.ArgumentParser() |
1134 | parser.add_argument('-r', '--fio-root', | |
1135 | help='fio root path') | |
1136 | parser.add_argument('-f', '--fio', | |
1137 | help='path to fio executable (e.g., ./fio)') | |
1138 | parser.add_argument('-a', '--artifact-root', | |
1139 | help='artifact root directory') | |
1140 | parser.add_argument('-s', '--skip', nargs='+', type=int, | |
1141 | help='list of test(s) to skip') | |
1142 | parser.add_argument('-o', '--run-only', nargs='+', type=int, | |
1143 | help='list of test(s) to run, skipping all others') | |
6d5470c3 VF |
1144 | parser.add_argument('-d', '--debug', action='store_true', |
1145 | help='provide debug output') | |
b1bc705e VF |
1146 | parser.add_argument('-k', '--skip-req', action='store_true', |
1147 | help='skip requirements checking') | |
58a77d2a VF |
1148 | parser.add_argument('-p', '--pass-through', action='append', |
1149 | help='pass-through an argument to an executable test') | |
df1eaa36 VF |
1150 | args = parser.parse_args() |
1151 | ||
1152 | return args | |
1153 | ||
1154 | ||
1155 | def main(): | |
704cc4df VF |
1156 | """Entry point.""" |
1157 | ||
df1eaa36 | 1158 | args = parse_args() |
6d5470c3 VF |
1159 | if args.debug: |
1160 | logging.basicConfig(level=logging.DEBUG) | |
1161 | else: | |
1162 | logging.basicConfig(level=logging.INFO) | |
1163 | ||
58a77d2a VF |
1164 | pass_through = {} |
1165 | if args.pass_through: | |
1166 | for arg in args.pass_through: | |
1167 | if not ':' in arg: | |
1168 | print("Invalid --pass-through argument '%s'" % arg) | |
1169 | print("Syntax for --pass-through is TESTNUMBER:ARGUMENT") | |
1170 | return | |
061a0773 | 1171 | split = arg.split(":", 1) |
58a77d2a | 1172 | pass_through[int(split[0])] = split[1] |
061a0773 | 1173 | logging.debug("Pass-through arguments: %s", pass_through) |
58a77d2a | 1174 | |
df1eaa36 VF |
1175 | if args.fio_root: |
1176 | fio_root = args.fio_root | |
1177 | else: | |
6d5470c3 VF |
1178 | fio_root = str(Path(__file__).absolute().parent.parent) |
1179 | print("fio root is %s" % fio_root) | |
df1eaa36 VF |
1180 | |
1181 | if args.fio: | |
1182 | fio_path = args.fio | |
1183 | else: | |
742b8799 VF |
1184 | if platform.system() == "Windows": |
1185 | fio_exe = "fio.exe" | |
1186 | else: | |
1187 | fio_exe = "fio" | |
1188 | fio_path = os.path.join(fio_root, fio_exe) | |
6d5470c3 VF |
1189 | print("fio path is %s" % fio_path) |
1190 | if not shutil.which(fio_path): | |
1191 | print("Warning: fio executable not found") | |
df1eaa36 VF |
1192 | |
1193 | artifact_root = args.artifact_root if args.artifact_root else \ | |
1194 | "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S")) | |
1195 | os.mkdir(artifact_root) | |
1196 | print("Artifact directory is %s" % artifact_root) | |
1197 | ||
b1bc705e VF |
1198 | if not args.skip_req: |
1199 | req = Requirements(fio_root) | |
1200 | ||
df1eaa36 VF |
1201 | passed = 0 |
1202 | failed = 0 | |
1203 | skipped = 0 | |
1204 | ||
1205 | for config in TEST_LIST: | |
1206 | if (args.skip and config['test_id'] in args.skip) or \ | |
1207 | (args.run_only and config['test_id'] not in args.run_only): | |
1208 | skipped = skipped + 1 | |
b1bc705e | 1209 | print("Test {0} SKIPPED (User request)".format(config['test_id'])) |
df1eaa36 VF |
1210 | continue |
1211 | ||
1212 | if issubclass(config['test_class'], FioJobTest): | |
1213 | if config['pre_job']: | |
1214 | fio_pre_job = os.path.join(fio_root, 't', 'jobs', | |
1215 | config['pre_job']) | |
1216 | else: | |
1217 | fio_pre_job = None | |
1218 | if config['pre_success']: | |
1219 | fio_pre_success = config['pre_success'] | |
1220 | else: | |
1221 | fio_pre_success = None | |
1222 | if 'output_format' in config: | |
1223 | output_format = config['output_format'] | |
1224 | else: | |
1225 | output_format = 'normal' | |
1226 | test = config['test_class']( | |
1227 | fio_path, | |
1228 | os.path.join(fio_root, 't', 'jobs', config['job']), | |
1229 | config['success'], | |
1230 | fio_pre_job=fio_pre_job, | |
1231 | fio_pre_success=fio_pre_success, | |
1232 | output_format=output_format) | |
9393cdaa | 1233 | desc = config['job'] |
df1eaa36 VF |
1234 | elif issubclass(config['test_class'], FioExeTest): |
1235 | exe_path = os.path.join(fio_root, config['exe']) | |
1236 | if config['parameters']: | |
1237 | parameters = [p.format(fio_path=fio_path) for p in config['parameters']] | |
1238 | else: | |
58a77d2a | 1239 | parameters = [] |
742b8799 | 1240 | if Path(exe_path).suffix == '.py' and platform.system() == "Windows": |
58a77d2a | 1241 | parameters.insert(0, exe_path) |
742b8799 | 1242 | exe_path = "python.exe" |
58a77d2a VF |
1243 | if config['test_id'] in pass_through: |
1244 | parameters += pass_through[config['test_id']].split() | |
df1eaa36 VF |
1245 | test = config['test_class'](exe_path, parameters, |
1246 | config['success']) | |
9393cdaa | 1247 | desc = config['exe'] |
df1eaa36 VF |
1248 | else: |
1249 | print("Test {0} FAILED: unable to process test config".format(config['test_id'])) | |
1250 | failed = failed + 1 | |
1251 | continue | |
1252 | ||
b1bc705e | 1253 | if not args.skip_req: |
704cc4df | 1254 | reqs_met = True |
b1bc705e | 1255 | for req in config['requirements']: |
704cc4df VF |
1256 | reqs_met, reason = req() |
1257 | logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason, | |
1258 | reqs_met) | |
1259 | if not reqs_met: | |
b1bc705e | 1260 | break |
704cc4df | 1261 | if not reqs_met: |
9393cdaa | 1262 | print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc)) |
b1bc705e VF |
1263 | skipped = skipped + 1 |
1264 | continue | |
1265 | ||
aa9f2627 VF |
1266 | try: |
1267 | test.setup(artifact_root, config['test_id']) | |
1268 | test.run() | |
1269 | test.check_result() | |
1270 | except KeyboardInterrupt: | |
1271 | break | |
1272 | except Exception as e: | |
1273 | test.passed = False | |
1274 | test.failure_reason += str(e) | |
1275 | logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc()) | |
df1eaa36 VF |
1276 | if test.passed: |
1277 | result = "PASSED" | |
1278 | passed = passed + 1 | |
1279 | else: | |
1280 | result = "FAILED: {0}".format(test.failure_reason) | |
1281 | failed = failed + 1 | |
15a73987 VF |
1282 | contents, _ = FioJobTest.get_file(test.stderr_file) |
1283 | logging.debug("Test %d: stderr:\n%s", config['test_id'], contents) | |
1284 | contents, _ = FioJobTest.get_file(test.stdout_file) | |
1285 | logging.debug("Test %d: stdout:\n%s", config['test_id'], contents) | |
9393cdaa | 1286 | print("Test {0} {1} {2}".format(config['test_id'], result, desc)) |
df1eaa36 VF |
1287 | |
1288 | print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped)) | |
1289 | ||
1290 | sys.exit(failed) | |
1291 | ||
1292 | ||
1293 | if __name__ == '__main__': | |
1294 | main() |