projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
test: get 32-bit Ubuntu 22.04 build working
[fio.git] / t / run-fio-tests.py
... / ...
CommitLineData
1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
17# # git clone git://git.kernel.dk/fio.git
18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
24# - Python 3.5 (subprocess.run)
25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
42#
43
44import os
45import sys
46import json
47import time
48import shutil
49import logging
50import argparse
51import platform
52import traceback
53import subprocess
54import multiprocessing
55from pathlib import Path
56
57
58class FioTest(object):
59 """Base for all fio tests."""
60
61 def __init__(self, exe_path, parameters, success):
62 self.exe_path = exe_path
63 self.parameters = parameters
64 self.success = success
65 self.output = {}
66 self.artifact_root = None
67 self.testnum = None
68 self.test_dir = None
69 self.passed = True
70 self.failure_reason = ''
71 self.command_file = None
72 self.stdout_file = None
73 self.stderr_file = None
74 self.exitcode_file = None
75
76 def setup(self, artifact_root, testnum):
77 """Setup instance variables for test."""
78
79 self.artifact_root = artifact_root
80 self.testnum = testnum
81 self.test_dir = os.path.join(artifact_root, "{:04d}".format(testnum))
82 if not os.path.exists(self.test_dir):
83 os.mkdir(self.test_dir)
84
85 self.command_file = os.path.join(
86 self.test_dir,
87 "{0}.command".format(os.path.basename(self.exe_path)))
88 self.stdout_file = os.path.join(
89 self.test_dir,
90 "{0}.stdout".format(os.path.basename(self.exe_path)))
91 self.stderr_file = os.path.join(
92 self.test_dir,
93 "{0}.stderr".format(os.path.basename(self.exe_path)))
94 self.exitcode_file = os.path.join(
95 self.test_dir,
96 "{0}.exitcode".format(os.path.basename(self.exe_path)))
97
98 def run(self):
99 """Run the test."""
100
101 raise NotImplementedError()
102
103 def check_result(self):
104 """Check test results."""
105
106 raise NotImplementedError()
107
108
109class FioExeTest(FioTest):
110 """Test consists of an executable binary or script"""
111
112 def __init__(self, exe_path, parameters, success):
113 """Construct a FioExeTest which is a FioTest consisting of an
114 executable binary or script.
115
116 exe_path: location of executable binary or script
117 parameters: list of parameters for executable
118 success: Definition of test success
119 """
120
121 FioTest.__init__(self, exe_path, parameters, success)
122
123 def run(self):
124 """Execute the binary or script described by this instance."""
125
126 command = [self.exe_path] + self.parameters
127 command_file = open(self.command_file, "w+")
128 command_file.write("%s\n" % command)
129 command_file.close()
130
131 stdout_file = open(self.stdout_file, "w+")
132 stderr_file = open(self.stderr_file, "w+")
133 exitcode_file = open(self.exitcode_file, "w+")
134 try:
135 proc = None
136 # Avoid using subprocess.run() here because when a timeout occurs,
137 # fio will be stopped with SIGKILL. This does not give fio a
138 # chance to clean up and means that child processes may continue
139 # running and submitting IO.
140 proc = subprocess.Popen(command,
141 stdout=stdout_file,
142 stderr=stderr_file,
143 cwd=self.test_dir,
144 universal_newlines=True)
145 proc.communicate(timeout=self.success['timeout'])
146 exitcode_file.write('{0}\n'.format(proc.returncode))
147 logging.debug("Test %d: return code: %d", self.testnum, proc.returncode)
148 self.output['proc'] = proc
149 except subprocess.TimeoutExpired:
150 proc.terminate()
151 proc.communicate()
152 assert proc.poll()
153 self.output['failure'] = 'timeout'
154 except Exception:
155 if proc:
156 if not proc.poll():
157 proc.terminate()
158 proc.communicate()
159 self.output['failure'] = 'exception'
160 self.output['exc_info'] = sys.exc_info()
161 finally:
162 stdout_file.close()
163 stderr_file.close()
164 exitcode_file.close()
165
166 def check_result(self):
167 """Check results of test run."""
168
169 if 'proc' not in self.output:
170 if self.output['failure'] == 'timeout':
171 self.failure_reason = "{0} timeout,".format(self.failure_reason)
172 else:
173 assert self.output['failure'] == 'exception'
174 self.failure_reason = '{0} exception: {1}, {2}'.format(
175 self.failure_reason, self.output['exc_info'][0],
176 self.output['exc_info'][1])
177
178 self.passed = False
179 return
180
181 if 'zero_return' in self.success:
182 if self.success['zero_return']:
183 if self.output['proc'].returncode != 0:
184 self.passed = False
185 self.failure_reason = "{0} non-zero return code,".format(self.failure_reason)
186 else:
187 if self.output['proc'].returncode == 0:
188 self.failure_reason = "{0} zero return code,".format(self.failure_reason)
189 self.passed = False
190
191 stderr_size = os.path.getsize(self.stderr_file)
192 if 'stderr_empty' in self.success:
193 if self.success['stderr_empty']:
194 if stderr_size != 0:
195 self.failure_reason = "{0} stderr not empty,".format(self.failure_reason)
196 self.passed = False
197 else:
198 if stderr_size == 0:
199 self.failure_reason = "{0} stderr empty,".format(self.failure_reason)
200 self.passed = False
201
202
203class FioJobTest(FioExeTest):
204 """Test consists of a fio job"""
205
206 def __init__(self, fio_path, fio_job, success, fio_pre_job=None,
207 fio_pre_success=None, output_format="normal"):
208 """Construct a FioJobTest which is a FioExeTest consisting of a
209 single fio job file with an optional setup step.
210
211 fio_path: location of fio executable
212 fio_job: location of fio job file
213 success: Definition of test success
214 fio_pre_job: fio job for preconditioning
215 fio_pre_success: Definition of test success for fio precon job
216 output_format: normal (default), json, jsonplus, or terse
217 """
218
219 self.fio_job = fio_job
220 self.fio_pre_job = fio_pre_job
221 self.fio_pre_success = fio_pre_success if fio_pre_success else success
222 self.output_format = output_format
223 self.precon_failed = False
224 self.json_data = None
225 self.fio_output = "{0}.output".format(os.path.basename(self.fio_job))
226 self.fio_args = [
227 "--max-jobs=16",
228 "--output-format={0}".format(self.output_format),
229 "--output={0}".format(self.fio_output),
230 self.fio_job,
231 ]
232 FioExeTest.__init__(self, fio_path, self.fio_args, success)
233
234 def setup(self, artifact_root, testnum):
235 """Setup instance variables for fio job test."""
236
237 super(FioJobTest, self).setup(artifact_root, testnum)
238
239 self.command_file = os.path.join(
240 self.test_dir,
241 "{0}.command".format(os.path.basename(self.fio_job)))
242 self.stdout_file = os.path.join(
243 self.test_dir,
244 "{0}.stdout".format(os.path.basename(self.fio_job)))
245 self.stderr_file = os.path.join(
246 self.test_dir,
247 "{0}.stderr".format(os.path.basename(self.fio_job)))
248 self.exitcode_file = os.path.join(
249 self.test_dir,
250 "{0}.exitcode".format(os.path.basename(self.fio_job)))
251
252 def run_pre_job(self):
253 """Run fio job precondition step."""
254
255 precon = FioJobTest(self.exe_path, self.fio_pre_job,
256 self.fio_pre_success,
257 output_format=self.output_format)
258 precon.setup(self.artifact_root, self.testnum)
259 precon.run()
260 precon.check_result()
261 self.precon_failed = not precon.passed
262 self.failure_reason = precon.failure_reason
263
264 def run(self):
265 """Run fio job test."""
266
267 if self.fio_pre_job:
268 self.run_pre_job()
269
270 if not self.precon_failed:
271 super(FioJobTest, self).run()
272 else:
273 logging.debug("Test %d: precondition step failed", self.testnum)
274
275 @classmethod
276 def get_file(cls, filename):
277 """Safely read a file."""
278 file_data = ''
279 success = True
280
281 try:
282 with open(filename, "r") as output_file:
283 file_data = output_file.read()
284 except OSError:
285 success = False
286
287 return file_data, success
288
289 def check_result(self):
290 """Check fio job results."""
291
292 if self.precon_failed:
293 self.passed = False
294 self.failure_reason = "{0} precondition step failed,".format(self.failure_reason)
295 return
296
297 super(FioJobTest, self).check_result()
298
299 if not self.passed:
300 return
301
302 if 'json' not in self.output_format:
303 return
304
305 file_data, success = self.get_file(os.path.join(self.test_dir, self.fio_output))
306 if not success:
307 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
308 self.passed = False
309 return
310
311 #
312 # Sometimes fio informational messages are included at the top of the
313 # JSON output, especially under Windows. Try to decode output as JSON
314 # data, skipping everything until the first {
315 #
316 lines = file_data.splitlines()
317 file_data = '\n'.join(lines[lines.index("{"):])
318 try:
319 self.json_data = json.loads(file_data)
320 except json.JSONDecodeError:
321 self.failure_reason = "{0} unable to decode JSON data,".format(self.failure_reason)
322 self.passed = False
323
324
325class FioJobTest_t0005(FioJobTest):
326 """Test consists of fio test job t0005
327 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
328
329 def check_result(self):
330 super(FioJobTest_t0005, self).check_result()
331
332 if not self.passed:
333 return
334
335 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
336 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
337 self.passed = False
338 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
339 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
340 self.passed = False
341
342
343class FioJobTest_t0006(FioJobTest):
344 """Test consists of fio test job t0006
345 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
346
347 def check_result(self):
348 super(FioJobTest_t0006, self).check_result()
349
350 if not self.passed:
351 return
352
353 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
354 / self.json_data['jobs'][0]['write']['io_kbytes']
355 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
356 if ratio < 1.99 or ratio > 2.01:
357 self.failure_reason = "{0} read/write ratio mismatch,".format(self.failure_reason)
358 self.passed = False
359
360
361class FioJobTest_t0007(FioJobTest):
362 """Test consists of fio test job t0007
363 Confirm that read['io_kbytes'] = 87040"""
364
365 def check_result(self):
366 super(FioJobTest_t0007, self).check_result()
367
368 if not self.passed:
369 return
370
371 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
372 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
373 self.passed = False
374
375
376class FioJobTest_t0008(FioJobTest):
377 """Test consists of fio test job t0008
378 Confirm that read['io_kbytes'] = 32768 and that
379 write['io_kbytes'] ~ 16568
380
381 I did runs with fio-ae2fafc8 and saw write['io_kbytes'] values of
382 16585, 16588. With two runs of fio-3.16 I obtained 16568"""
383
384 def check_result(self):
385 super(FioJobTest_t0008, self).check_result()
386
387 if not self.passed:
388 return
389
390 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16568
391 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
392
393 if ratio < 0.99 or ratio > 1.01:
394 self.failure_reason = "{0} bytes written mismatch,".format(self.failure_reason)
395 self.passed = False
396 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
397 self.failure_reason = "{0} bytes read mismatch,".format(self.failure_reason)
398 self.passed = False
399
400
401class FioJobTest_t0009(FioJobTest):
402 """Test consists of fio test job t0009
403 Confirm that runtime >= 60s"""
404
405 def check_result(self):
406 super(FioJobTest_t0009, self).check_result()
407
408 if not self.passed:
409 return
410
411 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
412
413 if self.json_data['jobs'][0]['elapsed'] < 60:
414 self.failure_reason = "{0} elapsed time mismatch,".format(self.failure_reason)
415 self.passed = False
416
417
418class FioJobTest_t0012(FioJobTest):
419 """Test consists of fio test job t0012
420 Confirm ratios of job iops are 1:5:10
421 job1,job2,job3 respectively"""
422
423 def check_result(self):
424 super(FioJobTest_t0012, self).check_result()
425
426 if not self.passed:
427 return
428
429 iops_files = []
430 for i in range(1,4):
431 file_data, success = self.get_file(os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(self.fio_job), i)))
432
433 if not success:
434 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
435 self.passed = False
436 return
437
438 iops_files.append(file_data.splitlines())
439
440 # there are 9 samples for job1 and job2, 4 samples for job3
441 iops1 = 0.0
442 iops2 = 0.0
443 iops3 = 0.0
444 for i in range(9):
445 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
446 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
447 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
448
449 ratio1 = iops3/iops2
450 ratio2 = iops3/iops1
451 logging.debug(
452 "sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} job3/job2={4:.3f} job3/job1={5:.3f}".format(
453 i, iops1, iops2, iops3, ratio1, ratio2
454 )
455 )
456
457 # test job1 and job2 succeeded to recalibrate
458 if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13:
459 self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} iops3={3} expected r1~2 r2~10 got r1={4:.3f} r2={5:.3f},".format(
460 self.failure_reason, iops1, iops2, iops3, ratio1, ratio2
461 )
462 self.passed = False
463 return
464
465
466class FioJobTest_t0014(FioJobTest):
467 """Test consists of fio test job t0014
468 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
469 and that job1_iops / job3_iops ~ 1:3 for first half of duration.
470
471 The test is about making sure the flow feature can
472 re-calibrate the activity dynamically"""
473
474 def check_result(self):
475 super(FioJobTest_t0014, self).check_result()
476
477 if not self.passed:
478 return
479
480 iops_files = []
481 for i in range(1,4):
482 file_data, success = self.get_file(os.path.join(self.test_dir, "{0}_iops.{1}.log".format(os.path.basename(self.fio_job), i)))
483
484 if not success:
485 self.failure_reason = "{0} unable to open output file,".format(self.failure_reason)
486 self.passed = False
487 return
488
489 iops_files.append(file_data.splitlines())
490
491 # there are 9 samples for job1 and job2, 4 samples for job3
492 iops1 = 0.0
493 iops2 = 0.0
494 iops3 = 0.0
495 for i in range(9):
496 if i < 4:
497 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
498 elif i == 4:
499 ratio1 = iops1 / iops2
500 ratio2 = iops1 / iops3
501
502
503 if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45:
504 self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} iops3={3}\
505 expected r1~0.5 r2~0.33 got r1={4:.3f} r2={5:.3f},".format(
506 self.failure_reason, iops1, iops2, iops3, ratio1, ratio2
507 )
508 self.passed = False
509
510 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
511 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
512
513 ratio1 = iops1/iops2
514 ratio2 = iops1/iops3
515 logging.debug(
516 "sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} job1/job2={4:.3f} job1/job3={5:.3f}".format(
517 i, iops1, iops2, iops3, ratio1, ratio2
518 )
519 )
520
521 # test job1 and job2 succeeded to recalibrate
522 if ratio1 < 0.43 or ratio1 > 0.57:
523 self.failure_reason = "{0} iops ratio mismatch iops1={1} iops2={2} expected ratio~0.5 got ratio={3:.3f},".format(
524 self.failure_reason, iops1, iops2, ratio1
525 )
526 self.passed = False
527 return
528
529
530class FioJobTest_t0015(FioJobTest):
531 """Test consists of fio test jobs t0015 and t0016
532 Confirm that mean(slat) + mean(clat) = mean(tlat)"""
533
534 def check_result(self):
535 super(FioJobTest_t0015, self).check_result()
536
537 if not self.passed:
538 return
539
540 slat = self.json_data['jobs'][0]['read']['slat_ns']['mean']
541 clat = self.json_data['jobs'][0]['read']['clat_ns']['mean']
542 tlat = self.json_data['jobs'][0]['read']['lat_ns']['mean']
543 logging.debug('Test %d: slat %f, clat %f, tlat %f', self.testnum, slat, clat, tlat)
544
545 if abs(slat + clat - tlat) > 1:
546 self.failure_reason = "{0} slat {1} + clat {2} = {3} != tlat {4},".format(
547 self.failure_reason, slat, clat, slat+clat, tlat)
548 self.passed = False
549
550
551class FioJobTest_t0019(FioJobTest):
552 """Test consists of fio test job t0019
553 Confirm that all offsets were touched sequentially"""
554
555 def check_result(self):
556 super(FioJobTest_t0019, self).check_result()
557
558 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
559 file_data, success = self.get_file(bw_log_filename)
560 log_lines = file_data.split('\n')
561
562 prev = -4096
563 for line in log_lines:
564 if len(line.strip()) == 0:
565 continue
566 cur = int(line.split(',')[4])
567 if cur - prev != 4096:
568 self.passed = False
569 self.failure_reason = "offsets {0}, {1} not sequential".format(prev, cur)
570 return
571 prev = cur
572
573 if cur/4096 != 255:
574 self.passed = False
575 self.failure_reason = "unexpected last offset {0}".format(cur)
576
577
578class FioJobTest_t0020(FioJobTest):
579 """Test consists of fio test job t0020
580 Confirm that almost all offsets were touched non-sequentially"""
581
582 def check_result(self):
583 super(FioJobTest_t0020, self).check_result()
584
585 bw_log_filename = os.path.join(self.test_dir, "test_bw.log")
586 file_data, success = self.get_file(bw_log_filename)
587 log_lines = file_data.split('\n')
588
589 seq_count = 0
590 offsets = set()
591
592 prev = int(log_lines[0].split(',')[4])
593 for line in log_lines[1:]:
594 offsets.add(prev/4096)
595 if len(line.strip()) == 0:
596 continue
597 cur = int(line.split(',')[4])
598 if cur - prev == 4096:
599 seq_count += 1
600 prev = cur
601
602 # 10 is an arbitrary threshold
603 if seq_count > 10:
604 self.passed = False
605 self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
606
607 if len(offsets) != 256:
608 self.passed = False
609 self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
610
611 for i in range(256):
612 if not i in offsets:
613 self.passed = False
614 self.failure_reason += " missing offset {0}".format(i*4096)
615
616
617class FioJobTest_iops_rate(FioJobTest):
618 """Test consists of fio test job t0009
619 Confirm that job0 iops == 1000
620 and that job1_iops / job0_iops ~ 8
621 With two runs of fio-3.16 I observed a ratio of 8.3"""
622
623 def check_result(self):
624 super(FioJobTest_iops_rate, self).check_result()
625
626 if not self.passed:
627 return
628
629 iops1 = self.json_data['jobs'][0]['read']['iops']
630 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
631 iops2 = self.json_data['jobs'][1]['read']['iops']
632 logging.debug("Test %d: iops2: %f", self.testnum, iops2)
633 ratio = iops2 / iops1
634 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
635
636 if iops1 < 950 or iops1 > 1050:
637 self.failure_reason = "{0} iops value mismatch,".format(self.failure_reason)
638 self.passed = False
639
640 if ratio < 6 or ratio > 10:
641 self.failure_reason = "{0} iops ratio mismatch,".format(self.failure_reason)
642 self.passed = False
643
644
645class Requirements(object):
646 """Requirements consists of multiple run environment characteristics.
647 These are to determine if a particular test can be run"""
648
649 _linux = False
650 _libaio = False
651 _io_uring = False
652 _zbd = False
653 _root = False
654 _zoned_nullb = False
655 _not_macos = False
656 _not_windows = False
657 _unittests = False
658 _cpucount4 = False
659
660 def __init__(self, fio_root):
661 Requirements._not_macos = platform.system() != "Darwin"
662 Requirements._not_windows = platform.system() != "Windows"
663 Requirements._linux = platform.system() == "Linux"
664
665 if Requirements._linux:
666 config_file = os.path.join(fio_root, "config-host.h")
667 contents, success = FioJobTest.get_file(config_file)
668 if not success:
669 print("Unable to open {0} to check requirements".format(config_file))
670 Requirements._zbd = True
671 else:
672 Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents
673 Requirements._libaio = "CONFIG_LIBAIO" in contents
674
675 contents, success = FioJobTest.get_file("/proc/kallsyms")
676 if not success:
677 print("Unable to open '/proc/kallsyms' to probe for io_uring support")
678 else:
679 Requirements._io_uring = "io_uring_setup" in contents
680
681 Requirements._root = (os.geteuid() == 0)
682 if Requirements._zbd and Requirements._root:
683 try:
684 subprocess.run(["modprobe", "null_blk"],
685 stdout=subprocess.PIPE,
686 stderr=subprocess.PIPE)
687 if os.path.exists("/sys/module/null_blk/parameters/zoned"):
688 Requirements._zoned_nullb = True
689 except Exception:
690 pass
691
692 if platform.system() == "Windows":
693 utest_exe = "unittest.exe"
694 else:
695 utest_exe = "unittest"
696 unittest_path = os.path.join(fio_root, "unittests", utest_exe)
697 Requirements._unittests = os.path.exists(unittest_path)
698
699 Requirements._cpucount4 = multiprocessing.cpu_count() >= 4
700
701 req_list = [Requirements.linux,
702 Requirements.libaio,
703 Requirements.io_uring,
704 Requirements.zbd,
705 Requirements.root,
706 Requirements.zoned_nullb,
707 Requirements.not_macos,
708 Requirements.not_windows,
709 Requirements.unittests,
710 Requirements.cpucount4]
711 for req in req_list:
712 value, desc = req()
713 logging.debug("Requirements: Requirement '%s' met? %s", desc, value)
714
715 @classmethod
716 def linux(cls):
717 """Are we running on Linux?"""
718 return Requirements._linux, "Linux required"
719
720 @classmethod
721 def libaio(cls):
722 """Is libaio available?"""
723 return Requirements._libaio, "libaio required"
724
725 @classmethod
726 def io_uring(cls):
727 """Is io_uring available?"""
728 return Requirements._io_uring, "io_uring required"
729
730 @classmethod
731 def zbd(cls):
732 """Is ZBD support available?"""
733 return Requirements._zbd, "Zoned block device support required"
734
735 @classmethod
736 def root(cls):
737 """Are we running as root?"""
738 return Requirements._root, "root required"
739
740 @classmethod
741 def zoned_nullb(cls):
742 """Are zoned null block devices available?"""
743 return Requirements._zoned_nullb, "Zoned null block device support required"
744
745 @classmethod
746 def not_macos(cls):
747 """Are we running on a platform other than macOS?"""
748 return Requirements._not_macos, "platform other than macOS required"
749
750 @classmethod
751 def not_windows(cls):
752 """Are we running on a platform other than Windws?"""
753 return Requirements._not_windows, "platform other than Windows required"
754
755 @classmethod
756 def unittests(cls):
757 """Were unittests built?"""
758 return Requirements._unittests, "Unittests support required"
759
760 @classmethod
761 def cpucount4(cls):
762 """Do we have at least 4 CPUs?"""
763 return Requirements._cpucount4, "4+ CPUs required"
764
765
766SUCCESS_DEFAULT = {
767 'zero_return': True,
768 'stderr_empty': True,
769 'timeout': 600,
770 }
771SUCCESS_NONZERO = {
772 'zero_return': False,
773 'stderr_empty': False,
774 'timeout': 600,
775 }
776SUCCESS_STDERR = {
777 'zero_return': True,
778 'stderr_empty': False,
779 'timeout': 600,
780 }
781TEST_LIST = [
782 {
783 'test_id': 1,
784 'test_class': FioJobTest,
785 'job': 't0001-52c58027.fio',
786 'success': SUCCESS_DEFAULT,
787 'pre_job': None,
788 'pre_success': None,
789 'requirements': [],
790 },
791 {
792 'test_id': 2,
793 'test_class': FioJobTest,
794 'job': 't0002-13af05ae-post.fio',
795 'success': SUCCESS_DEFAULT,
796 'pre_job': 't0002-13af05ae-pre.fio',
797 'pre_success': None,
798 'requirements': [Requirements.linux, Requirements.libaio],
799 },
800 {
801 'test_id': 3,
802 'test_class': FioJobTest,
803 'job': 't0003-0ae2c6e1-post.fio',
804 'success': SUCCESS_NONZERO,
805 'pre_job': 't0003-0ae2c6e1-pre.fio',
806 'pre_success': SUCCESS_DEFAULT,
807 'requirements': [Requirements.linux, Requirements.libaio],
808 },
809 {
810 'test_id': 4,
811 'test_class': FioJobTest,
812 'job': 't0004-8a99fdf6.fio',
813 'success': SUCCESS_DEFAULT,
814 'pre_job': None,
815 'pre_success': None,
816 'requirements': [Requirements.linux, Requirements.libaio],
817 },
818 {
819 'test_id': 5,
820 'test_class': FioJobTest_t0005,
821 'job': 't0005-f7078f7b.fio',
822 'success': SUCCESS_DEFAULT,
823 'pre_job': None,
824 'pre_success': None,
825 'output_format': 'json',
826 'requirements': [Requirements.not_windows],
827 },
828 {
829 'test_id': 6,
830 'test_class': FioJobTest_t0006,
831 'job': 't0006-82af2a7c.fio',
832 'success': SUCCESS_DEFAULT,
833 'pre_job': None,
834 'pre_success': None,
835 'output_format': 'json',
836 'requirements': [Requirements.linux, Requirements.libaio],
837 },
838 {
839 'test_id': 7,
840 'test_class': FioJobTest_t0007,
841 'job': 't0007-37cf9e3c.fio',
842 'success': SUCCESS_DEFAULT,
843 'pre_job': None,
844 'pre_success': None,
845 'output_format': 'json',
846 'requirements': [],
847 },
848 {
849 'test_id': 8,
850 'test_class': FioJobTest_t0008,
851 'job': 't0008-ae2fafc8.fio',
852 'success': SUCCESS_DEFAULT,
853 'pre_job': None,
854 'pre_success': None,
855 'output_format': 'json',
856 'requirements': [],
857 },
858 {
859 'test_id': 9,
860 'test_class': FioJobTest_t0009,
861 'job': 't0009-f8b0bd10.fio',
862 'success': SUCCESS_DEFAULT,
863 'pre_job': None,
864 'pre_success': None,
865 'output_format': 'json',
866 'requirements': [Requirements.not_macos,
867 Requirements.cpucount4],
868 # mac os does not support CPU affinity
869 },
870 {
871 'test_id': 10,
872 'test_class': FioJobTest,
873 'job': 't0010-b7aae4ba.fio',
874 'success': SUCCESS_DEFAULT,
875 'pre_job': None,
876 'pre_success': None,
877 'requirements': [],
878 },
879 {
880 'test_id': 11,
881 'test_class': FioJobTest_iops_rate,
882 'job': 't0011-5d2788d5.fio',
883 'success': SUCCESS_DEFAULT,
884 'pre_job': None,
885 'pre_success': None,
886 'output_format': 'json',
887 'requirements': [],
888 },
889 {
890 'test_id': 12,
891 'test_class': FioJobTest_t0012,
892 'job': 't0012.fio',
893 'success': SUCCESS_DEFAULT,
894 'pre_job': None,
895 'pre_success': None,
896 'output_format': 'json',
897 'requirements': [],
898 },
899 {
900 'test_id': 13,
901 'test_class': FioJobTest,
902 'job': 't0013.fio',
903 'success': SUCCESS_DEFAULT,
904 'pre_job': None,
905 'pre_success': None,
906 'output_format': 'json',
907 'requirements': [],
908 },
909 {
910 'test_id': 14,
911 'test_class': FioJobTest_t0014,
912 'job': 't0014.fio',
913 'success': SUCCESS_DEFAULT,
914 'pre_job': None,
915 'pre_success': None,
916 'output_format': 'json',
917 'requirements': [],
918 },
919 {
920 'test_id': 15,
921 'test_class': FioJobTest_t0015,
922 'job': 't0015-e78980ff.fio',
923 'success': SUCCESS_DEFAULT,
924 'pre_job': None,
925 'pre_success': None,
926 'output_format': 'json',
927 'requirements': [Requirements.linux, Requirements.libaio],
928 },
929 {
930 'test_id': 16,
931 'test_class': FioJobTest_t0015,
932 'job': 't0016-d54ae22.fio',
933 'success': SUCCESS_DEFAULT,
934 'pre_job': None,
935 'pre_success': None,
936 'output_format': 'json',
937 'requirements': [],
938 },
939 {
940 'test_id': 17,
941 'test_class': FioJobTest_t0015,
942 'job': 't0017.fio',
943 'success': SUCCESS_DEFAULT,
944 'pre_job': None,
945 'pre_success': None,
946 'output_format': 'json',
947 'requirements': [Requirements.not_windows],
948 },
949 {
950 'test_id': 18,
951 'test_class': FioJobTest,
952 'job': 't0018.fio',
953 'success': SUCCESS_DEFAULT,
954 'pre_job': None,
955 'pre_success': None,
956 'requirements': [Requirements.linux, Requirements.io_uring],
957 },
958 {
959 'test_id': 19,
960 'test_class': FioJobTest_t0019,
961 'job': 't0019.fio',
962 'success': SUCCESS_DEFAULT,
963 'pre_job': None,
964 'pre_success': None,
965 'requirements': [],
966 },
967 {
968 'test_id': 20,
969 'test_class': FioJobTest_t0020,
970 'job': 't0020.fio',
971 'success': SUCCESS_DEFAULT,
972 'pre_job': None,
973 'pre_success': None,
974 'requirements': [],
975 },
976 {
977 'test_id': 1000,
978 'test_class': FioExeTest,
979 'exe': 't/axmap',
980 'parameters': None,
981 'success': SUCCESS_DEFAULT,
982 'requirements': [],
983 },
984 {
985 'test_id': 1001,
986 'test_class': FioExeTest,
987 'exe': 't/ieee754',
988 'parameters': None,
989 'success': SUCCESS_DEFAULT,
990 'requirements': [],
991 },
992 {
993 'test_id': 1002,
994 'test_class': FioExeTest,
995 'exe': 't/lfsr-test',
996 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
997 'success': SUCCESS_STDERR,
998 'requirements': [],
999 },
1000 {
1001 'test_id': 1003,
1002 'test_class': FioExeTest,
1003 'exe': 't/readonly.py',
1004 'parameters': ['-f', '{fio_path}'],
1005 'success': SUCCESS_DEFAULT,
1006 'requirements': [],
1007 },
1008 {
1009 'test_id': 1004,
1010 'test_class': FioExeTest,
1011 'exe': 't/steadystate_tests.py',
1012 'parameters': ['{fio_path}'],
1013 'success': SUCCESS_DEFAULT,
1014 'requirements': [],
1015 },
1016 {
1017 'test_id': 1005,
1018 'test_class': FioExeTest,
1019 'exe': 't/stest',
1020 'parameters': None,
1021 'success': SUCCESS_STDERR,
1022 'requirements': [],
1023 },
1024 {
1025 'test_id': 1006,
1026 'test_class': FioExeTest,
1027 'exe': 't/strided.py',
1028 'parameters': ['{fio_path}'],
1029 'success': SUCCESS_DEFAULT,
1030 'requirements': [],
1031 },
1032 {
1033 'test_id': 1007,
1034 'test_class': FioExeTest,
1035 'exe': 't/zbd/run-tests-against-nullb',
1036 'parameters': ['-s', '1'],
1037 'success': SUCCESS_DEFAULT,
1038 'requirements': [Requirements.linux, Requirements.zbd,
1039 Requirements.root],
1040 },
1041 {
1042 'test_id': 1008,
1043 'test_class': FioExeTest,
1044 'exe': 't/zbd/run-tests-against-nullb',
1045 'parameters': ['-s', '2'],
1046 'success': SUCCESS_DEFAULT,
1047 'requirements': [Requirements.linux, Requirements.zbd,
1048 Requirements.root, Requirements.zoned_nullb],
1049 },
1050 {
1051 'test_id': 1009,
1052 'test_class': FioExeTest,
1053 'exe': 'unittests/unittest',
1054 'parameters': None,
1055 'success': SUCCESS_DEFAULT,
1056 'requirements': [Requirements.unittests],
1057 },
1058 {
1059 'test_id': 1010,
1060 'test_class': FioExeTest,
1061 'exe': 't/latency_percentiles.py',
1062 'parameters': ['-f', '{fio_path}'],
1063 'success': SUCCESS_DEFAULT,
1064 'requirements': [],
1065 },
1066 {
1067 'test_id': 1011,
1068 'test_class': FioExeTest,
1069 'exe': 't/jsonplus2csv_test.py',
1070 'parameters': ['-f', '{fio_path}'],
1071 'success': SUCCESS_DEFAULT,
1072 'requirements': [],
1073 },
1074]
1075
1076
1077def parse_args():
1078 """Parse command-line arguments."""
1079
1080 parser = argparse.ArgumentParser()
1081 parser.add_argument('-r', '--fio-root',
1082 help='fio root path')
1083 parser.add_argument('-f', '--fio',
1084 help='path to fio executable (e.g., ./fio)')
1085 parser.add_argument('-a', '--artifact-root',
1086 help='artifact root directory')
1087 parser.add_argument('-s', '--skip', nargs='+', type=int,
1088 help='list of test(s) to skip')
1089 parser.add_argument('-o', '--run-only', nargs='+', type=int,
1090 help='list of test(s) to run, skipping all others')
1091 parser.add_argument('-d', '--debug', action='store_true',
1092 help='provide debug output')
1093 parser.add_argument('-k', '--skip-req', action='store_true',
1094 help='skip requirements checking')
1095 parser.add_argument('-p', '--pass-through', action='append',
1096 help='pass-through an argument to an executable test')
1097 args = parser.parse_args()
1098
1099 return args
1100
1101
1102def main():
1103 """Entry point."""
1104
1105 args = parse_args()
1106 if args.debug:
1107 logging.basicConfig(level=logging.DEBUG)
1108 else:
1109 logging.basicConfig(level=logging.INFO)
1110
1111 pass_through = {}
1112 if args.pass_through:
1113 for arg in args.pass_through:
1114 if not ':' in arg:
1115 print("Invalid --pass-through argument '%s'" % arg)
1116 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
1117 return
1118 split = arg.split(":", 1)
1119 pass_through[int(split[0])] = split[1]
1120 logging.debug("Pass-through arguments: %s", pass_through)
1121
1122 if args.fio_root:
1123 fio_root = args.fio_root
1124 else:
1125 fio_root = str(Path(__file__).absolute().parent.parent)
1126 print("fio root is %s" % fio_root)
1127
1128 if args.fio:
1129 fio_path = args.fio
1130 else:
1131 if platform.system() == "Windows":
1132 fio_exe = "fio.exe"
1133 else:
1134 fio_exe = "fio"
1135 fio_path = os.path.join(fio_root, fio_exe)
1136 print("fio path is %s" % fio_path)
1137 if not shutil.which(fio_path):
1138 print("Warning: fio executable not found")
1139
1140 artifact_root = args.artifact_root if args.artifact_root else \
1141 "fio-test-{0}".format(time.strftime("%Y%m%d-%H%M%S"))
1142 os.mkdir(artifact_root)
1143 print("Artifact directory is %s" % artifact_root)
1144
1145 if not args.skip_req:
1146 req = Requirements(fio_root)
1147
1148 passed = 0
1149 failed = 0
1150 skipped = 0
1151
1152 for config in TEST_LIST:
1153 if (args.skip and config['test_id'] in args.skip) or \
1154 (args.run_only and config['test_id'] not in args.run_only):
1155 skipped = skipped + 1
1156 print("Test {0} SKIPPED (User request)".format(config['test_id']))
1157 continue
1158
1159 if issubclass(config['test_class'], FioJobTest):
1160 if config['pre_job']:
1161 fio_pre_job = os.path.join(fio_root, 't', 'jobs',
1162 config['pre_job'])
1163 else:
1164 fio_pre_job = None
1165 if config['pre_success']:
1166 fio_pre_success = config['pre_success']
1167 else:
1168 fio_pre_success = None
1169 if 'output_format' in config:
1170 output_format = config['output_format']
1171 else:
1172 output_format = 'normal'
1173 test = config['test_class'](
1174 fio_path,
1175 os.path.join(fio_root, 't', 'jobs', config['job']),
1176 config['success'],
1177 fio_pre_job=fio_pre_job,
1178 fio_pre_success=fio_pre_success,
1179 output_format=output_format)
1180 desc = config['job']
1181 elif issubclass(config['test_class'], FioExeTest):
1182 exe_path = os.path.join(fio_root, config['exe'])
1183 if config['parameters']:
1184 parameters = [p.format(fio_path=fio_path) for p in config['parameters']]
1185 else:
1186 parameters = []
1187 if Path(exe_path).suffix == '.py' and platform.system() == "Windows":
1188 parameters.insert(0, exe_path)
1189 exe_path = "python.exe"
1190 if config['test_id'] in pass_through:
1191 parameters += pass_through[config['test_id']].split()
1192 test = config['test_class'](exe_path, parameters,
1193 config['success'])
1194 desc = config['exe']
1195 else:
1196 print("Test {0} FAILED: unable to process test config".format(config['test_id']))
1197 failed = failed + 1
1198 continue
1199
1200 if not args.skip_req:
1201 reqs_met = True
1202 for req in config['requirements']:
1203 reqs_met, reason = req()
1204 logging.debug("Test %d: Requirement '%s' met? %s", config['test_id'], reason,
1205 reqs_met)
1206 if not reqs_met:
1207 break
1208 if not reqs_met:
1209 print("Test {0} SKIPPED ({1}) {2}".format(config['test_id'], reason, desc))
1210 skipped = skipped + 1
1211 continue
1212
1213 try:
1214 test.setup(artifact_root, config['test_id'])
1215 test.run()
1216 test.check_result()
1217 except KeyboardInterrupt:
1218 break
1219 except Exception as e:
1220 test.passed = False
1221 test.failure_reason += str(e)
1222 logging.debug("Test %d exception:\n%s\n", config['test_id'], traceback.format_exc())
1223 if test.passed:
1224 result = "PASSED"
1225 passed = passed + 1
1226 else:
1227 result = "FAILED: {0}".format(test.failure_reason)
1228 failed = failed + 1
1229 contents, _ = FioJobTest.get_file(test.stderr_file)
1230 logging.debug("Test %d: stderr:\n%s", config['test_id'], contents)
1231 contents, _ = FioJobTest.get_file(test.stdout_file)
1232 logging.debug("Test %d: stdout:\n%s", config['test_id'], contents)
1233 print("Test {0} {1} {2}".format(config['test_id'], result, desc))
1234
1235 print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped))
1236
1237 sys.exit(failed)
1238
1239
1240if __name__ == '__main__':
1241 main()
fio - Flexible IO Tester