projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'atomic-writes'
[fio.git] / t / run-fio-tests.py
CommitLineData
df1eaa36
VF		 1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright (c) 2019 Western Digital Corporation or its affiliates.
5#
6"""
7# run-fio-tests.py
8#
9# Automate running of fio tests
10#
11# USAGE
12# python3 run-fio-tests.py [-r fio-root] [-f fio-path] [-a artifact-root]
13# [--skip # # #...] [--run-only # # #...]
14#
15#
16# EXAMPLE
b048455f 17# # git clone git://git.kernel.dk/fio.git
df1eaa36
VF		 18# # cd fio
19# # make -j
20# # python3 t/run-fio-tests.py
21#
22#
23# REQUIREMENTS
b1bc705e 24# - Python 3.5 (subprocess.run)
df1eaa36
VF		 25# - Linux (libaio ioengine, zbd tests, etc)
26# - The artifact directory must be on a file system that accepts 512-byte IO
27# (t0002, t0003, t0004).
28# - The artifact directory needs to be on an SSD. Otherwise tests that carry
29# out file-based IO will trigger a timeout (t0006).
30# - 4 CPUs (t0009)
31# - SciPy (steadystate_tests.py)
32# - libzbc (zbd tests)
33# - root privileges (zbd test)
34# - kernel 4.19 or later for zoned null block devices (zbd tests)
35# - CUnit support (unittests)
36#
37"""
38
39#
40# TODO run multiple tests simultaneously
41# TODO Add sgunmap tests (requires SAS SSD)
df1eaa36
VF		 42#
43
44import os
45import sys
df1eaa36 46import time
6d5470c3 47import shutil
df1eaa36
VF		 48import logging
49import argparse
8b0018e9 50import re
df1eaa36 51from pathlib import Path
888dbd62 52from statsmodels.sandbox.stats.runs import runstest_1samp
0dc6e911 53from fiotestlib import FioExeTest, FioJobFileTest, run_fio_tests
fb551941 54from fiotestcommon import *
df1eaa36
VF		 55
56
0dc6e911 57class FioJobFileTest_t0005(FioJobFileTest):
df1eaa36
VF		 58 """Test consists of fio test job t0005
59 Confirm that read['io_kbytes'] == write['io_kbytes'] == 102400"""
60
61 def check_result(self):
1b4ba547 62 super().check_result()
df1eaa36
VF		 63
64 if not self.passed:
65 return
66
67 if self.json_data['jobs'][0]['read']['io_kbytes'] != 102400:
1b4ba547 68 self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
df1eaa36
VF		 69 self.passed = False
70 if self.json_data['jobs'][0]['write']['io_kbytes'] != 102400:
1b4ba547 71 self.failure_reason = f"{self.failure_reason} bytes written mismatch,"
df1eaa36
VF		 72 self.passed = False
73
74
0dc6e911 75class FioJobFileTest_t0006(FioJobFileTest):
df1eaa36
VF		 76 """Test consists of fio test job t0006
77 Confirm that read['io_kbytes'] ~ 2*write['io_kbytes']"""
78
79 def check_result(self):
1b4ba547 80 super().check_result()
df1eaa36
VF		 81
82 if not self.passed:
83 return
84
85 ratio = self.json_data['jobs'][0]['read']['io_kbytes'] \
86 / self.json_data['jobs'][0]['write']['io_kbytes']
704cc4df 87 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
df1eaa36 88 if ratio < 1.99 or ratio > 2.01:
1b4ba547 89 self.failure_reason = f"{self.failure_reason} read/write ratio mismatch,"
df1eaa36
VF		 90 self.passed = False
91
92
0dc6e911 93class FioJobFileTest_t0007(FioJobFileTest):
df1eaa36
VF		 94 """Test consists of fio test job t0007
95 Confirm that read['io_kbytes'] = 87040"""
96
97 def check_result(self):
1b4ba547 98 super().check_result()
df1eaa36
VF		 99
100 if not self.passed:
101 return
102
103 if self.json_data['jobs'][0]['read']['io_kbytes'] != 87040:
1b4ba547 104 self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
df1eaa36
VF		 105 self.passed = False
106
107
0dc6e911 108class FioJobFileTest_t0008(FioJobFileTest):
df1eaa36
VF		 109 """Test consists of fio test job t0008
110 Confirm that read['io_kbytes'] = 32768 and that
77c758db 111 write['io_kbytes'] ~ 16384
df1eaa36 112
77c758db
VF		 113 This is a 50/50 seq read/write workload. Since fio flips a coin to
114 determine whether to issue a read or a write, total bytes written will not
115 be exactly 16384K. But total bytes read will be exactly 32768K because
116 reads will include the initial phase as well as the verify phase where all
117 the blocks originally written will be read."""
df1eaa36
VF		 118
119 def check_result(self):
1b4ba547 120 super().check_result()
df1eaa36
VF		 121
122 if not self.passed:
123 return
124
77c758db 125 ratio = self.json_data['jobs'][0]['write']['io_kbytes'] / 16384
704cc4df 126 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
df1eaa36 127
77c758db 128 if ratio < 0.97 or ratio > 1.03:
1b4ba547 129 self.failure_reason = f"{self.failure_reason} bytes written mismatch,"
df1eaa36
VF		 130 self.passed = False
131 if self.json_data['jobs'][0]['read']['io_kbytes'] != 32768:
1b4ba547 132 self.failure_reason = f"{self.failure_reason} bytes read mismatch,"
df1eaa36
VF		 133 self.passed = False
134
135
0dc6e911 136class FioJobFileTest_t0009(FioJobFileTest):
df1eaa36
VF		 137 """Test consists of fio test job t0009
138 Confirm that runtime >= 60s"""
139
140 def check_result(self):
1b4ba547 141 super().check_result()
df1eaa36
VF		 142
143 if not self.passed:
144 return
145
704cc4df 146 logging.debug('Test %d: elapsed: %d', self.testnum, self.json_data['jobs'][0]['elapsed'])
df1eaa36
VF		 147
148 if self.json_data['jobs'][0]['elapsed'] < 60:
1b4ba547 149 self.failure_reason = f"{self.failure_reason} elapsed time mismatch,"
df1eaa36
VF		 150 self.passed = False
151
152
0dc6e911 153class FioJobFileTest_t0012(FioJobFileTest):
d4e74fda
DB		 154 """Test consists of fio test job t0012
155 Confirm ratios of job iops are 1:5:10
156 job1,job2,job3 respectively"""
157
158 def check_result(self):
1b4ba547 159 super().check_result()
d4e74fda
DB		 160
161 if not self.passed:
162 return
163
164 iops_files = []
114eadb2 165 for i in range(1, 4):
68b3a741 166 filename = os.path.join(self.paths['test_dir'], "{0}_iops.{1}.log".format(os.path.basename(
114eadb2 167 self.fio_job), i))
0f5234b4
VF		 168 file_data = self.get_file_fail(filename)
169 if not file_data:
d4e74fda
DB		 170 return
171
172 iops_files.append(file_data.splitlines())
173
174 # there are 9 samples for job1 and job2, 4 samples for job3
175 iops1 = 0.0
176 iops2 = 0.0
177 iops3 = 0.0
178 for i in range(9):
179 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
180 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
181 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
182
183 ratio1 = iops3/iops2
184 ratio2 = iops3/iops1
114eadb2
VF		 185 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
186 "job3/job2={4:.3f} job3/job1={5:.3f}".format(i, iops1, iops2, iops3, ratio1,
187 ratio2))
d4e74fda
DB		 188
189 # test job1 and job2 succeeded to recalibrate
190 if ratio1 < 1 or ratio1 > 3 or ratio2 < 7 or ratio2 > 13:
114eadb2
VF		 191 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
192 "expected r1~2 r2~10 got r1={3:.3f} r2={4:.3f},".format(iops1, iops2, iops3,
193 ratio1, ratio2)
d4e74fda
DB		 194 self.passed = False
195 return
196
197
0dc6e911 198class FioJobFileTest_t0014(FioJobFileTest):
d4e74fda
DB		 199 """Test consists of fio test job t0014
200 Confirm that job1_iops / job2_iops ~ 1:2 for entire duration
201 and that job1_iops / job3_iops ~ 1:3 for first half of duration.
202
203 The test is about making sure the flow feature can
204 re-calibrate the activity dynamically"""
205
206 def check_result(self):
1b4ba547 207 super().check_result()
d4e74fda
DB		 208
209 if not self.passed:
210 return
211
212 iops_files = []
114eadb2 213 for i in range(1, 4):
68b3a741 214 filename = os.path.join(self.paths['test_dir'], "{0}_iops.{1}.log".format(os.path.basename(
114eadb2 215 self.fio_job), i))
0f5234b4
VF		 216 file_data = self.get_file_fail(filename)
217 if not file_data:
d4e74fda
DB		 218 return
219
220 iops_files.append(file_data.splitlines())
221
222 # there are 9 samples for job1 and job2, 4 samples for job3
223 iops1 = 0.0
224 iops2 = 0.0
225 iops3 = 0.0
226 for i in range(9):
227 if i < 4:
228 iops3 = iops3 + float(iops_files[2][i].split(',')[1])
229 elif i == 4:
230 ratio1 = iops1 / iops2
231 ratio2 = iops1 / iops3
232
233
234 if ratio1 < 0.43 or ratio1 > 0.57 or ratio2 < 0.21 or ratio2 > 0.45:
114eadb2
VF		 235 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} iops3={2} " \
236 "expected r1~0.5 r2~0.33 got r1={3:.3f} r2={4:.3f},".format(
237 iops1, iops2, iops3, ratio1, ratio2)
d4e74fda
DB		 238 self.passed = False
239
240 iops1 = iops1 + float(iops_files[0][i].split(',')[1])
241 iops2 = iops2 + float(iops_files[1][i].split(',')[1])
242
243 ratio1 = iops1/iops2
244 ratio2 = iops1/iops3
114eadb2
VF		 245 logging.debug("sample {0}: job1 iops={1} job2 iops={2} job3 iops={3} " \
246 "job1/job2={4:.3f} job1/job3={5:.3f}".format(i, iops1, iops2, iops3,
247 ratio1, ratio2))
d4e74fda
DB		 248
249 # test job1 and job2 succeeded to recalibrate
250 if ratio1 < 0.43 or ratio1 > 0.57:
114eadb2
VF		 251 self.failure_reason += " iops ratio mismatch iops1={0} iops2={1} expected ratio~0.5 " \
252 "got ratio={2:.3f},".format(iops1, iops2, ratio1)
d4e74fda
DB		 253 self.passed = False
254 return
255
256
0dc6e911 257class FioJobFileTest_t0015(FioJobFileTest):
de31fe9a 258 """Test consists of fio test jobs t0015 and t0016
60ebb939
VF		 259 Confirm that mean(slat) + mean(clat) = mean(tlat)"""
260
261 def check_result(self):
1b4ba547 262 super().check_result()
60ebb939
VF		 263
264 if not self.passed:
265 return
266
267 slat = self.json_data['jobs'][0]['read']['slat_ns']['mean']
268 clat = self.json_data['jobs'][0]['read']['clat_ns']['mean']
269 tlat = self.json_data['jobs'][0]['read']['lat_ns']['mean']
270 logging.debug('Test %d: slat %f, clat %f, tlat %f', self.testnum, slat, clat, tlat)
271
272 if abs(slat + clat - tlat) > 1:
273 self.failure_reason = "{0} slat {1} + clat {2} = {3} != tlat {4},".format(
274 self.failure_reason, slat, clat, slat+clat, tlat)
275 self.passed = False
276
277
0dc6e911 278class FioJobFileTest_t0019(FioJobFileTest):
ef54f290
VF		 279 """Test consists of fio test job t0019
280 Confirm that all offsets were touched sequentially"""
281
282 def check_result(self):
1b4ba547 283 super().check_result()
ef54f290 284
68b3a741 285 bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log")
0f5234b4
VF		 286 file_data = self.get_file_fail(bw_log_filename)
287 if not file_data:
114eadb2
VF		 288 return
289
ef54f290
VF		 290 log_lines = file_data.split('\n')
291
292 prev = -4096
293 for line in log_lines:
294 if len(line.strip()) == 0:
295 continue
296 cur = int(line.split(',')[4])
297 if cur - prev != 4096:
298 self.passed = False
1b4ba547 299 self.failure_reason = f"offsets {prev}, {cur} not sequential"
ef54f290
VF		 300 return
301 prev = cur
302
303 if cur/4096 != 255:
304 self.passed = False
1b4ba547 305 self.failure_reason = f"unexpected last offset {cur}"
ef54f290
VF		 306
307
0dc6e911 308class FioJobFileTest_t0020(FioJobFileTest):
eb40b275 309 """Test consists of fio test jobs t0020 and t0021
ef54f290
VF		 310 Confirm that almost all offsets were touched non-sequentially"""
311
312 def check_result(self):
1b4ba547 313 super().check_result()
ef54f290 314
68b3a741 315 bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log")
0f5234b4
VF		 316 file_data = self.get_file_fail(bw_log_filename)
317 if not file_data:
114eadb2
VF		 318 return
319
ef54f290
VF		 320 log_lines = file_data.split('\n')
321
888dbd62 322 offsets = []
ef54f290
VF		 323
324 prev = int(log_lines[0].split(',')[4])
325 for line in log_lines[1:]:
888dbd62 326 offsets.append(prev/4096)
ef54f290
VF		 327 if len(line.strip()) == 0:
328 continue
329 cur = int(line.split(',')[4])
ef54f290
VF		 330 prev = cur
331
ef54f290
VF		 332 if len(offsets) != 256:
333 self.passed = False
1b4ba547 334 self.failure_reason += f" number of offsets is {len(offsets)} instead of 256"
ef54f290
VF		 335
336 for i in range(256):
337 if not i in offsets:
338 self.passed = False
1b4ba547 339 self.failure_reason += f" missing offset {i * 4096}"
ef54f290 340
1b4ba547 341 (_, p) = runstest_1samp(list(offsets))
888dbd62
VF		 342 if p < 0.05:
343 self.passed = False
344 self.failure_reason += f" runs test failed with p = {p}"
345
ef54f290 346
0dc6e911 347class FioJobFileTest_t0022(FioJobFileTest):
eb40b275
VF		 348 """Test consists of fio test job t0022"""
349
350 def check_result(self):
1b4ba547 351 super().check_result()
eb40b275 352
68b3a741 353 bw_log_filename = os.path.join(self.paths['test_dir'], "test_bw.log")
0f5234b4
VF		 354 file_data = self.get_file_fail(bw_log_filename)
355 if not file_data:
114eadb2
VF		 356 return
357
eb40b275
VF		 358 log_lines = file_data.split('\n')
359
360 filesize = 1024*1024
361 bs = 4096
362 seq_count = 0
363 offsets = set()
364
365 prev = int(log_lines[0].split(',')[4])
366 for line in log_lines[1:]:
367 offsets.add(prev/bs)
368 if len(line.strip()) == 0:
369 continue
370 cur = int(line.split(',')[4])
371 if cur - prev == bs:
372 seq_count += 1
373 prev = cur
374
375 # 10 is an arbitrary threshold
376 if seq_count > 10:
377 self.passed = False
1b4ba547 378 self.failure_reason = f"too many ({seq_count}) consecutive offsets"
eb40b275
VF		 379
380 if len(offsets) == filesize/bs:
381 self.passed = False
114eadb2 382 self.failure_reason += " no duplicate offsets found with norandommap=1"
eb40b275
VF		 383
384
0dc6e911 385class FioJobFileTest_t0023(FioJobFileTest):
21a20229 386 """Test consists of fio test job t0023 randtrimwrite test."""
c37183f8 387
1fb78294
VF		 388 def check_trimwrite(self, filename):
389 """Make sure that trims are followed by writes of the same size at the same offset."""
390
68b3a741 391 bw_log_filename = os.path.join(self.paths['test_dir'], filename)
0f5234b4
VF		 392 file_data = self.get_file_fail(bw_log_filename)
393 if not file_data:
114eadb2
VF		 394 return
395
c37183f8
VF		 396 log_lines = file_data.split('\n')
397
398 prev_ddir = 1
399 for line in log_lines:
400 if len(line.strip()) == 0:
401 continue
402 vals = line.split(',')
403 ddir = int(vals[2])
404 bs = int(vals[3])
405 offset = int(vals[4])
406 if prev_ddir == 1:
407 if ddir != 2:
408 self.passed = False
21a20229
VF		 409 self.failure_reason += " {0}: write not preceeded by trim: {1}".format(
410 bw_log_filename, line)
c37183f8
VF		 411 break
412 else:
1b4ba547 413 if ddir != 1: # pylint: disable=no-else-break
c37183f8 414 self.passed = False
21a20229
VF		 415 self.failure_reason += " {0}: trim not preceeded by write: {1}".format(
416 bw_log_filename, line)
c37183f8
VF		 417 break
418 else:
419 if prev_bs != bs:
420 self.passed = False
21a20229
VF		 421 self.failure_reason += " {0}: block size does not match: {1}".format(
422 bw_log_filename, line)
c37183f8 423 break
1b4ba547 424
c37183f8
VF		 425 if prev_offset != offset:
426 self.passed = False
21a20229
VF		 427 self.failure_reason += " {0}: offset does not match: {1}".format(
428 bw_log_filename, line)
c37183f8 429 break
1b4ba547 430
c37183f8
VF		 431 prev_ddir = ddir
432 prev_bs = bs
433 prev_offset = offset
434
435
9e83aa16 436 def check_all_offsets(self, filename, sectorsize, filesize):
21a20229
VF		 437 """Make sure all offsets were touched."""
438
68b3a741 439 file_data = self.get_file_fail(os.path.join(self.paths['test_dir'], filename))
0f5234b4 440 if not file_data:
9e83aa16
VF		 441 return
442
443 log_lines = file_data.split('\n')
444
445 offsets = set()
446
447 for line in log_lines:
448 if len(line.strip()) == 0:
449 continue
450 vals = line.split(',')
451 bs = int(vals[3])
452 offset = int(vals[4])
453 if offset % sectorsize != 0:
454 self.passed = False
21a20229
VF		 455 self.failure_reason += " {0}: offset {1} not a multiple of sector size {2}".format(
456 filename, offset, sectorsize)
457 break
9e83aa16
VF		 458 if bs % sectorsize != 0:
459 self.passed = False
21a20229
VF		 460 self.failure_reason += " {0}: block size {1} not a multiple of sector size " \
461 "{2}".format(filename, bs, sectorsize)
462 break
9e83aa16
VF		 463 for i in range(int(bs/sectorsize)):
464 offsets.add(offset/sectorsize + i)
465
466 if len(offsets) != filesize/sectorsize:
467 self.passed = False
21a20229
VF		 468 self.failure_reason += " {0}: only {1} offsets touched; expected {2}".format(
469 filename, len(offsets), filesize/sectorsize)
9e83aa16 470 else:
21a20229 471 logging.debug("%s: %d sectors touched", filename, len(offsets))
9e83aa16
VF		 472
473
c37183f8 474 def check_result(self):
1b4ba547 475 super().check_result()
c37183f8 476
9e83aa16
VF		 477 filesize = 1024*1024
478
1fb78294
VF		 479 self.check_trimwrite("basic_bw.log")
480 self.check_trimwrite("bs_bw.log")
481 self.check_trimwrite("bsrange_bw.log")
482 self.check_trimwrite("bssplit_bw.log")
483 self.check_trimwrite("basic_no_rm_bw.log")
484 self.check_trimwrite("bs_no_rm_bw.log")
485 self.check_trimwrite("bsrange_no_rm_bw.log")
486 self.check_trimwrite("bssplit_no_rm_bw.log")
c37183f8 487
9e83aa16
VF		 488 self.check_all_offsets("basic_bw.log", 4096, filesize)
489 self.check_all_offsets("bs_bw.log", 8192, filesize)
490 self.check_all_offsets("bsrange_bw.log", 512, filesize)
491 self.check_all_offsets("bssplit_bw.log", 512, filesize)
c37183f8
VF		 492
493
0dc6e911 494class FioJobFileTest_t0024(FioJobFileTest_t0023):
2d5c4600
VF		 495 """Test consists of fio test job t0024 trimwrite test."""
496
497 def check_result(self):
0dc6e911
VF		 498 # call FioJobFileTest_t0023's parent to skip checks done by t0023
499 super(FioJobFileTest_t0023, self).check_result()
2d5c4600
VF		 500
501 filesize = 1024*1024
502
503 self.check_trimwrite("basic_bw.log")
504 self.check_trimwrite("bs_bw.log")
505 self.check_trimwrite("bsrange_bw.log")
506 self.check_trimwrite("bssplit_bw.log")
507
508 self.check_all_offsets("basic_bw.log", 4096, filesize)
509 self.check_all_offsets("bs_bw.log", 8192, filesize)
510 self.check_all_offsets("bsrange_bw.log", 512, filesize)
511 self.check_all_offsets("bssplit_bw.log", 512, filesize)
512
513
0dc6e911 514class FioJobFileTest_t0025(FioJobFileTest):
d661fb18
SK		 515 """Test experimental verify read backs written data pattern."""
516 def check_result(self):
1b4ba547 517 super().check_result()
d661fb18
SK		 518
519 if not self.passed:
520 return
521
522 if self.json_data['jobs'][0]['read']['io_kbytes'] != 128:
523 self.passed = False
524
0dc6e911 525class FioJobFileTest_t0027(FioJobFileTest):
ede04c27 526 def setup(self, *args, **kws):
1b4ba547 527 super().setup(*args, **kws)
68b3a741
VF		 528 self.pattern_file = os.path.join(self.paths['test_dir'], "t0027.pattern")
529 self.output_file = os.path.join(self.paths['test_dir'], "t0027file")
ede04c27
LG		 530 self.pattern = os.urandom(16 << 10)
531 with open(self.pattern_file, "wb") as f:
532 f.write(self.pattern)
533
534 def check_result(self):
1b4ba547 535 super().check_result()
ede04c27
LG		 536
537 if not self.passed:
538 return
539
540 with open(self.output_file, "rb") as f:
541 data = f.read()
542
543 if data != self.pattern:
544 self.passed = False
d661fb18 545
7aec5ac0
SK		 546class FioJobFileTest_t0029(FioJobFileTest):
547 """Test loops option works with read-verify workload."""
548 def check_result(self):
549 super().check_result()
550
551 if not self.passed:
552 return
553
554 if self.json_data['jobs'][1]['read']['io_kbytes'] != 8:
555 self.passed = False
556
20d0ba84 557class FioJobFileTest_LogFileFormat(FioJobFileTest):
8b0018e9 558 """Test log file format"""
20d0ba84
SK		 559 def setup(self, *args, **kws):
560 super().setup(*args, **kws)
561 self.patterns = {}
562
8b0018e9
SK		 563 def check_result(self):
564 super().check_result()
565
566 if not self.passed:
567 return
568
20d0ba84 569 for logfile in self.patterns.keys():
8b0018e9
SK		 570 file_path = os.path.join(self.paths['test_dir'], logfile)
571 with open(file_path, "r") as f:
572 line = f.readline()
20d0ba84 573 if not re.match(self.patterns[logfile], line):
8b0018e9
SK		 574 self.passed = False
575 self.failure_reason = "wrong log file format: " + logfile
576 return
577
20d0ba84
SK		 578class FioJobFileTest_t0033(FioJobFileTest_LogFileFormat):
579 """Test log file format"""
580 def setup(self, *args, **kws):
581 super().setup(*args, **kws)
582 self.patterns = {
583 'log_bw.1.log': '\\d+, \\d+, \\d+, \\d+, 0x[\\da-f]+\\n',
584 'log_clat.2.log': '\\d+, \\d+, \\d+, \\d+, 0, \\d+\\n',
585 'log_iops.3.log': '\\d+, \\d+, \\d+, \\d+, \\d+, 0x[\\da-f]+\\n',
586 'log_iops.4.log': '\\d+, \\d+, \\d+, \\d+, 0, 0, \\d+\\n',
587 }
588
589class FioJobFileTest_t0034(FioJobFileTest_LogFileFormat):
590 """Test log file format"""
591 def setup(self, *args, **kws):
592 super().setup(*args, **kws)
593 self.patterns = {
594 'log_clat.1.log': '\\d+, \\d+, \\d+, \\d+, \\d+, \\d+, \\d+\\n',
595 'log_slat.1.log': '\\d+, \\d+, \\d+, \\d+, \\d+, \\d+, \\d+\\n',
596 'log_lat.1.log': '\\d+, \\d+, \\d+, \\d+, \\d+, \\d+, 0\\n',
597 'log_clat.2.log': '\\d+, \\d+, \\d+, \\d+, 0, 0, \\d+, 0\\n',
598 'log_bw.3.log': '\\d+, \\d+, \\d+, \\d+, \\d+, \\d+, 0\\n',
599 'log_iops.3.log': '\\d+, \\d+, \\d+, \\d+, \\d+, \\d+, 0\\n',
600 }
601
0dc6e911 602class FioJobFileTest_iops_rate(FioJobFileTest):
12b609b5 603 """Test consists of fio test job t0011
df1eaa36
VF		 604 Confirm that job0 iops == 1000
605 and that job1_iops / job0_iops ~ 8
606 With two runs of fio-3.16 I observed a ratio of 8.3"""
607
608 def check_result(self):
1b4ba547 609 super().check_result()
df1eaa36
VF		 610
611 if not self.passed:
612 return
613
614 iops1 = self.json_data['jobs'][0]['read']['iops']
7ddc4ed1 615 logging.debug("Test %d: iops1: %f", self.testnum, iops1)
df1eaa36 616 iops2 = self.json_data['jobs'][1]['read']['iops']
7ddc4ed1 617 logging.debug("Test %d: iops2: %f", self.testnum, iops2)
df1eaa36 618 ratio = iops2 / iops1
704cc4df 619 logging.debug("Test %d: ratio: %f", self.testnum, ratio)
df1eaa36 620
0a9b8988 621 if iops1 < 950 or iops1 > 1050:
1b4ba547 622 self.failure_reason = f"{self.failure_reason} iops value mismatch,"
df1eaa36
VF		 623 self.passed = False
624
d4e74fda 625 if ratio < 6 or ratio > 10:
1b4ba547 626 self.failure_reason = f"{self.failure_reason} iops ratio mismatch,"
df1eaa36
VF		 627 self.passed = False
628
629
df1eaa36 630TEST_LIST = [
704cc4df
VF		 631 {
632 'test_id': 1,
0dc6e911 633 'test_class': FioJobFileTest,
704cc4df
VF		 634 'job': 't0001-52c58027.fio',
635 'success': SUCCESS_DEFAULT,
636 'pre_job': None,
637 'pre_success': None,
638 'requirements': [],
639 },
640 {
641 'test_id': 2,
0dc6e911 642 'test_class': FioJobFileTest,
704cc4df
VF		 643 'job': 't0002-13af05ae-post.fio',
644 'success': SUCCESS_DEFAULT,
645 'pre_job': 't0002-13af05ae-pre.fio',
646 'pre_success': None,
647 'requirements': [Requirements.linux, Requirements.libaio],
648 },
649 {
650 'test_id': 3,
0dc6e911 651 'test_class': FioJobFileTest,
704cc4df
VF		 652 'job': 't0003-0ae2c6e1-post.fio',
653 'success': SUCCESS_NONZERO,
654 'pre_job': 't0003-0ae2c6e1-pre.fio',
655 'pre_success': SUCCESS_DEFAULT,
656 'requirements': [Requirements.linux, Requirements.libaio],
657 },
658 {
659 'test_id': 4,
0dc6e911 660 'test_class': FioJobFileTest,
704cc4df
VF		 661 'job': 't0004-8a99fdf6.fio',
662 'success': SUCCESS_DEFAULT,
663 'pre_job': None,
664 'pre_success': None,
665 'requirements': [Requirements.linux, Requirements.libaio],
666 },
667 {
668 'test_id': 5,
0dc6e911 669 'test_class': FioJobFileTest_t0005,
704cc4df
VF		 670 'job': 't0005-f7078f7b.fio',
671 'success': SUCCESS_DEFAULT,
672 'pre_job': None,
673 'pre_success': None,
674 'output_format': 'json',
675 'requirements': [Requirements.not_windows],
676 },
677 {
678 'test_id': 6,
0dc6e911 679 'test_class': FioJobFileTest_t0006,
704cc4df
VF		 680 'job': 't0006-82af2a7c.fio',
681 'success': SUCCESS_DEFAULT,
682 'pre_job': None,
683 'pre_success': None,
684 'output_format': 'json',
685 'requirements': [Requirements.linux, Requirements.libaio],
686 },
687 {
688 'test_id': 7,
0dc6e911 689 'test_class': FioJobFileTest_t0007,
704cc4df
VF		 690 'job': 't0007-37cf9e3c.fio',
691 'success': SUCCESS_DEFAULT,
692 'pre_job': None,
693 'pre_success': None,
694 'output_format': 'json',
695 'requirements': [],
696 },
697 {
698 'test_id': 8,
0dc6e911 699 'test_class': FioJobFileTest_t0008,
704cc4df
VF		 700 'job': 't0008-ae2fafc8.fio',
701 'success': SUCCESS_DEFAULT,
702 'pre_job': None,
703 'pre_success': None,
704 'output_format': 'json',
705 'requirements': [],
706 },
707 {
708 'test_id': 9,
0dc6e911 709 'test_class': FioJobFileTest_t0009,
704cc4df
VF		 710 'job': 't0009-f8b0bd10.fio',
711 'success': SUCCESS_DEFAULT,
712 'pre_job': None,
713 'pre_success': None,
714 'output_format': 'json',
715 'requirements': [Requirements.not_macos,
716 Requirements.cpucount4],
717 # mac os does not support CPU affinity
718 },
719 {
720 'test_id': 10,
0dc6e911 721 'test_class': FioJobFileTest,
704cc4df
VF		 722 'job': 't0010-b7aae4ba.fio',
723 'success': SUCCESS_DEFAULT,
724 'pre_job': None,
725 'pre_success': None,
726 'requirements': [],
727 },
728 {
729 'test_id': 11,
0dc6e911 730 'test_class': FioJobFileTest_iops_rate,
704cc4df
VF		 731 'job': 't0011-5d2788d5.fio',
732 'success': SUCCESS_DEFAULT,
733 'pre_job': None,
734 'pre_success': None,
735 'output_format': 'json',
736 'requirements': [],
737 },
0a602473
BVA		 738 {
739 'test_id': 12,
0dc6e911 740 'test_class': FioJobFileTest_t0012,
0a602473
BVA		 741 'job': 't0012.fio',
742 'success': SUCCESS_DEFAULT,
743 'pre_job': None,
744 'pre_success': None,
745 'output_format': 'json',
d4e74fda 746 'requirements': [],
0a602473 747 },
f0c7ae7a
BVA		 748 {
749 'test_id': 13,
0dc6e911 750 'test_class': FioJobFileTest,
f0c7ae7a
BVA		 751 'job': 't0013.fio',
752 'success': SUCCESS_DEFAULT,
753 'pre_job': None,
754 'pre_success': None,
755 'output_format': 'json',
756 'requirements': [],
757 },
d4e74fda
DB		 758 {
759 'test_id': 14,
0dc6e911 760 'test_class': FioJobFileTest_t0014,
d4e74fda
DB		 761 'job': 't0014.fio',
762 'success': SUCCESS_DEFAULT,
763 'pre_job': None,
764 'pre_success': None,
765 'output_format': 'json',
766 'requirements': [],
767 },
60ebb939
VF		 768 {
769 'test_id': 15,
0dc6e911 770 'test_class': FioJobFileTest_t0015,
5c763be5 771 'job': 't0015-4e7e7898.fio',
60ebb939
VF		 772 'success': SUCCESS_DEFAULT,
773 'pre_job': None,
774 'pre_success': None,
775 'output_format': 'json',
776 'requirements': [Requirements.linux, Requirements.libaio],
777 },
de31fe9a
VF		 778 {
779 'test_id': 16,
0dc6e911 780 'test_class': FioJobFileTest_t0015,
f045d5f8 781 'job': 't0016-d54ae22.fio',
de31fe9a
VF		 782 'success': SUCCESS_DEFAULT,
783 'pre_job': None,
784 'pre_success': None,
785 'output_format': 'json',
786 'requirements': [],
787 },
31a58cba
VF		 788 {
789 'test_id': 17,
0dc6e911 790 'test_class': FioJobFileTest_t0015,
31a58cba
VF		 791 'job': 't0017.fio',
792 'success': SUCCESS_DEFAULT,
793 'pre_job': None,
794 'pre_success': None,
795 'output_format': 'json',
796 'requirements': [Requirements.not_windows],
797 },
a2947c33
VF		 798 {
799 'test_id': 18,
0dc6e911 800 'test_class': FioJobFileTest,
a2947c33
VF		 801 'job': 't0018.fio',
802 'success': SUCCESS_DEFAULT,
803 'pre_job': None,
804 'pre_success': None,
805 'requirements': [Requirements.linux, Requirements.io_uring],
806 },
ef54f290
VF		 807 {
808 'test_id': 19,
0dc6e911 809 'test_class': FioJobFileTest_t0019,
ef54f290
VF		 810 'job': 't0019.fio',
811 'success': SUCCESS_DEFAULT,
812 'pre_job': None,
813 'pre_success': None,
814 'requirements': [],
815 },
816 {
817 'test_id': 20,
0dc6e911 818 'test_class': FioJobFileTest_t0020,
ef54f290
VF		 819 'job': 't0020.fio',
820 'success': SUCCESS_DEFAULT,
821 'pre_job': None,
822 'pre_success': None,
823 'requirements': [],
824 },
eb40b275
VF		 825 {
826 'test_id': 21,
0dc6e911 827 'test_class': FioJobFileTest_t0020,
eb40b275
VF		 828 'job': 't0021.fio',
829 'success': SUCCESS_DEFAULT,
830 'pre_job': None,
831 'pre_success': None,
832 'requirements': [],
833 },
834 {
835 'test_id': 22,
0dc6e911 836 'test_class': FioJobFileTest_t0022,
eb40b275
VF		 837 'job': 't0022.fio',
838 'success': SUCCESS_DEFAULT,
839 'pre_job': None,
840 'pre_success': None,
841 'requirements': [],
842 },
c37183f8
VF		 843 {
844 'test_id': 23,
0dc6e911 845 'test_class': FioJobFileTest_t0023,
c37183f8
VF		 846 'job': 't0023.fio',
847 'success': SUCCESS_DEFAULT,
848 'pre_job': None,
849 'pre_success': None,
850 'requirements': [],
851 },
2d5c4600
VF		 852 {
853 'test_id': 24,
0dc6e911 854 'test_class': FioJobFileTest_t0024,
2d5c4600
VF		 855 'job': 't0024.fio',
856 'success': SUCCESS_DEFAULT,
857 'pre_job': None,
858 'pre_success': None,
859 'requirements': [],
860 },
d661fb18
SK		 861 {
862 'test_id': 25,
0dc6e911 863 'test_class': FioJobFileTest_t0025,
d661fb18
SK		 864 'job': 't0025.fio',
865 'success': SUCCESS_DEFAULT,
866 'pre_job': None,
867 'pre_success': None,
868 'output_format': 'json',
869 'requirements': [],
870 },
c4704c08
SK		 871 {
872 'test_id': 26,
0dc6e911 873 'test_class': FioJobFileTest,
c4704c08
SK		 874 'job': 't0026.fio',
875 'success': SUCCESS_DEFAULT,
876 'pre_job': None,
877 'pre_success': None,
878 'requirements': [Requirements.not_windows],
879 },
ede04c27
LG		 880 {
881 'test_id': 27,
0dc6e911 882 'test_class': FioJobFileTest_t0027,
ede04c27
LG		 883 'job': 't0027.fio',
884 'success': SUCCESS_DEFAULT,
885 'pre_job': None,
886 'pre_success': None,
887 'requirements': [],
888 },
23fb1642
VF		 889 {
890 'test_id': 28,
0dc6e911 891 'test_class': FioJobFileTest,
23fb1642
VF		 892 'job': 't0028-c6cade16.fio',
893 'success': SUCCESS_DEFAULT,
894 'pre_job': None,
895 'pre_success': None,
896 'requirements': [],
7aec5ac0
SK		 897 },
898 {
899 'test_id': 29,
900 'test_class': FioJobFileTest_t0029,
901 'job': 't0029.fio',
902 'success': SUCCESS_DEFAULT,
903 'pre_job': None,
904 'pre_success': None,
905 'output_format': 'json',
906 'requirements': [],
23fb1642 907 },
f2262200
VF		 908 {
909 'test_id': 30,
910 'test_class': FioJobFileTest,
911 'job': 't0030.fio',
912 'success': SUCCESS_DEFAULT,
913 'pre_job': None,
914 'pre_success': None,
915 'parameters': ['--bandwidth-log'],
916 'requirements': [],
917 },
140c58be
SK		 918 {
919 'test_id': 31,
920 'test_class': FioJobFileTest,
921 'job': 't0031.fio',
922 'success': SUCCESS_DEFAULT,
923 'pre_job': 't0031-pre.fio',
924 'pre_success': SUCCESS_DEFAULT,
20f42c10 925 'requirements': [Requirements.linux, Requirements.libaio],
140c58be 926 },
8b0018e9
SK		 927 {
928 'test_id': 33,
929 'test_class': FioJobFileTest_t0033,
930 'job': 't0033.fio',
931 'success': SUCCESS_DEFAULT,
932 'pre_job': None,
933 'pre_success': None,
934 'pre_success': SUCCESS_DEFAULT,
935 'requirements': [Requirements.linux, Requirements.libaio],
936 },
20d0ba84
SK		 937 {
938 'test_id': 34,
939 'test_class': FioJobFileTest_t0034,
940 'job': 't0034.fio',
941 'success': SUCCESS_DEFAULT,
942 'pre_job': None,
943 'pre_success': None,
944 'pre_success': SUCCESS_DEFAULT,
945 'requirements': [Requirements.linux, Requirements.libaio],
946 },
a189fc72
SK		 947 {
948 'test_id': 35,
949 'test_class': FioJobFileTest,
950 'job': 't0035.fio',
951 'success': SUCCESS_DEFAULT,
952 'pre_job': None,
953 'pre_success': None,
954 'pre_success': SUCCESS_DEFAULT,
955 'requirements': [],
956 },
704cc4df
VF		 957 {
958 'test_id': 1000,
959 'test_class': FioExeTest,
960 'exe': 't/axmap',
961 'parameters': None,
962 'success': SUCCESS_DEFAULT,
963 'requirements': [],
964 },
965 {
966 'test_id': 1001,
967 'test_class': FioExeTest,
968 'exe': 't/ieee754',
969 'parameters': None,
970 'success': SUCCESS_DEFAULT,
971 'requirements': [],
972 },
973 {
974 'test_id': 1002,
975 'test_class': FioExeTest,
976 'exe': 't/lfsr-test',
977 'parameters': ['0xFFFFFF', '0', '0', 'verify'],
978 'success': SUCCESS_STDERR,
979 'requirements': [],
980 },
981 {
982 'test_id': 1003,
983 'test_class': FioExeTest,
984 'exe': 't/readonly.py',
985 'parameters': ['-f', '{fio_path}'],
986 'success': SUCCESS_DEFAULT,
987 'requirements': [],
988 },
989 {
990 'test_id': 1004,
991 'test_class': FioExeTest,
992 'exe': 't/steadystate_tests.py',
993 'parameters': ['{fio_path}'],
994 'success': SUCCESS_DEFAULT,
995 'requirements': [],
996 },
997 {
998 'test_id': 1005,
999 'test_class': FioExeTest,
1000 'exe': 't/stest',
1001 'parameters': None,
1002 'success': SUCCESS_STDERR,
1003 'requirements': [],
1004 },
1005 {
1006 'test_id': 1006,
1007 'test_class': FioExeTest,
1008 'exe': 't/strided.py',
5a36d0e4 1009 'parameters': ['--fio', '{fio_path}'],
704cc4df
VF		 1010 'success': SUCCESS_DEFAULT,
1011 'requirements': [],
1012 },
1013 {
1014 'test_id': 1007,
1015 'test_class': FioExeTest,
037b2b50
DF		 1016 'exe': 't/zbd/run-tests-against-nullb',
1017 'parameters': ['-s', '1'],
704cc4df
VF		 1018 'success': SUCCESS_DEFAULT,
1019 'requirements': [Requirements.linux, Requirements.zbd,
1020 Requirements.root],
1021 },
1022 {
1023 'test_id': 1008,
1024 'test_class': FioExeTest,
037b2b50
DF		 1025 'exe': 't/zbd/run-tests-against-nullb',
1026 'parameters': ['-s', '2'],
704cc4df
VF		 1027 'success': SUCCESS_DEFAULT,
1028 'requirements': [Requirements.linux, Requirements.zbd,
1029 Requirements.root, Requirements.zoned_nullb],
1030 },
1031 {
1032 'test_id': 1009,
1033 'test_class': FioExeTest,
1034 'exe': 'unittests/unittest',
1035 'parameters': None,
1036 'success': SUCCESS_DEFAULT,
1037 'requirements': [Requirements.unittests],
1038 },
1039 {
1040 'test_id': 1010,
1041 'test_class': FioExeTest,
1042 'exe': 't/latency_percentiles.py',
1043 'parameters': ['-f', '{fio_path}'],
1044 'success': SUCCESS_DEFAULT,
1045 'requirements': [],
1046 },
8403eca6
VF		 1047 {
1048 'test_id': 1011,
1049 'test_class': FioExeTest,
1050 'exe': 't/jsonplus2csv_test.py',
1051 'parameters': ['-f', '{fio_path}'],
1052 'success': SUCCESS_DEFAULT,
1053 'requirements': [],
1054 },
03900b0b 1055 {
1056 'test_id': 1012,
1057 'test_class': FioExeTest,
1058 'exe': 't/log_compression.py',
1059 'parameters': ['-f', '{fio_path}'],
1060 'success': SUCCESS_DEFAULT,
1061 'requirements': [],
1062 },
c994fa62
VF		 1063 {
1064 'test_id': 1013,
1065 'test_class': FioExeTest,
1066 'exe': 't/random_seed.py',
1067 'parameters': ['-f', '{fio_path}'],
1068 'success': SUCCESS_DEFAULT,
1069 'requirements': [],
1070 },
2128f93a
VF		 1071 {
1072 'test_id': 1014,
1073 'test_class': FioExeTest,
1074 'exe': 't/nvmept.py',
1075 'parameters': ['-f', '{fio_path}', '--dut', '{nvmecdev}'],
1076 'success': SUCCESS_DEFAULT,
2d0debb3
VF		 1077 'requirements': [Requirements.linux, Requirements.nvmecdev],
1078 },
1079 {
1080 'test_id': 1015,
1081 'test_class': FioExeTest,
1082 'exe': 't/nvmept_trim.py',
1083 'parameters': ['-f', '{fio_path}', '--dut', '{nvmecdev}'],
1084 'success': SUCCESS_DEFAULT,
2128f93a
VF		 1085 'requirements': [Requirements.linux, Requirements.nvmecdev],
1086 },
df1eaa36
VF		 1087]
1088
1089
1090def parse_args():
704cc4df
VF		 1091 """Parse command-line arguments."""
1092
df1eaa36
VF		 1093 parser = argparse.ArgumentParser()
1094 parser.add_argument('-r', '--fio-root',
1095 help='fio root path')
1096 parser.add_argument('-f', '--fio',
1097 help='path to fio executable (e.g., ./fio)')
1098 parser.add_argument('-a', '--artifact-root',
1099 help='artifact root directory')
1100 parser.add_argument('-s', '--skip', nargs='+', type=int,
1101 help='list of test(s) to skip')
1102 parser.add_argument('-o', '--run-only', nargs='+', type=int,
1103 help='list of test(s) to run, skipping all others')
6d5470c3
VF		 1104 parser.add_argument('-d', '--debug', action='store_true',
1105 help='provide debug output')
b1bc705e
VF		 1106 parser.add_argument('-k', '--skip-req', action='store_true',
1107 help='skip requirements checking')
58a77d2a
VF		 1108 parser.add_argument('-p', '--pass-through', action='append',
1109 help='pass-through an argument to an executable test')
2128f93a
VF		 1110 parser.add_argument('--nvmecdev', action='store', default=None,
1111 help='NVMe character device for **DESTRUCTIVE** testing (e.g., /dev/ng0n1)')
df1eaa36
VF		 1112 args = parser.parse_args()
1113
1114 return args
1115
1116
1117def main():
704cc4df
VF		 1118 """Entry point."""
1119
df1eaa36 1120 args = parse_args()
6d5470c3
VF		 1121 if args.debug:
1122 logging.basicConfig(level=logging.DEBUG)
1123 else:
1124 logging.basicConfig(level=logging.INFO)
1125
58a77d2a
VF		 1126 pass_through = {}
1127 if args.pass_through:
1128 for arg in args.pass_through:
1129 if not ':' in arg:
1b4ba547 1130 print(f"Invalid --pass-through argument '{arg}'")
58a77d2a
VF		 1131 print("Syntax for --pass-through is TESTNUMBER:ARGUMENT")
1132 return
061a0773 1133 split = arg.split(":", 1)
58a77d2a 1134 pass_through[int(split[0])] = split[1]
061a0773 1135 logging.debug("Pass-through arguments: %s", pass_through)
58a77d2a 1136
df1eaa36
VF		 1137 if args.fio_root:
1138 fio_root = args.fio_root
1139 else:
6d5470c3 1140 fio_root = str(Path(__file__).absolute().parent.parent)
1b4ba547 1141 print(f"fio root is {fio_root}")
df1eaa36
VF		 1142
1143 if args.fio:
1144 fio_path = args.fio
1145 else:
742b8799
VF		 1146 if platform.system() == "Windows":
1147 fio_exe = "fio.exe"
1148 else:
1149 fio_exe = "fio"
1150 fio_path = os.path.join(fio_root, fio_exe)
1b4ba547 1151 print(f"fio path is {fio_path}")
6d5470c3
VF		 1152 if not shutil.which(fio_path):
1153 print("Warning: fio executable not found")
df1eaa36
VF		 1154
1155 artifact_root = args.artifact_root if args.artifact_root else \
1b4ba547 1156 f"fio-test-{time.strftime('%Y%m%d-%H%M%S')}"
df1eaa36 1157 os.mkdir(artifact_root)
1b4ba547 1158 print(f"Artifact directory is {artifact_root}")
df1eaa36 1159
b1bc705e 1160 if not args.skip_req:
fb551941
VF		 1161 Requirements(fio_root, args)
1162
1163 test_env = {
1164 'fio_path': fio_path,
1165 'fio_root': fio_root,
1166 'artifact_root': artifact_root,
1167 'pass_through': pass_through,
1168 }
1169 _, failed, _ = run_fio_tests(TEST_LIST, test_env, args)
df1eaa36
VF		 1170 sys.exit(failed)
1171
1172
1173if __name__ == '__main__':
1174 main()
fio - Flexible IO Tester