03900b0b |
1 | #!/usr/bin/env python3 |
2 | # |
3 | # log_compression.py |
4 | # |
5 | # Test log_compression and log_store_compressed. Uses null ioengine. |
6 | # Previous bugs have caused output in per I/O log files to be missing |
7 | # and/or out of order |
8 | # |
9 | # Expected result: 8000 log entries, offset starting at 0 and increasing by bs |
10 | # Buggy result: Log entries out of order (usually without log_store_compressed) |
11 | # and/or missing log entries (usually with log_store_compressed) |
12 | # |
13 | # USAGE |
14 | # python log_compression.py [-f fio-executable] |
15 | # |
16 | # EXAMPLES |
17 | # python t/log_compression.py |
18 | # python t/log_compression.py -f ./fio |
19 | # |
20 | # REQUIREMENTS |
21 | # Python 3.5+ |
22 | # |
23 | # ===TEST MATRIX=== |
24 | # |
25 | # With log_compression=10K |
26 | # With log_store_compressed=1 and log_compression=10K |
27 | |
28 | import os |
29 | import sys |
30 | import platform |
31 | import argparse |
32 | import subprocess |
33 | |
34 | |
35 | def parse_args(): |
36 | """Parse command-line arguments.""" |
37 | parser = argparse.ArgumentParser() |
38 | parser.add_argument('-f', '--fio', |
39 | help='path to fio executable (e.g., ./fio)') |
40 | return parser.parse_args() |
41 | |
42 | |
43 | def run_fio(fio,log_store_compressed): |
44 | fio_args = [ |
45 | '--name=job', |
46 | '--ioengine=null', |
47 | '--filesize=1000M', |
48 | '--bs=128K', |
49 | '--rw=write', |
50 | '--iodepth=1', |
51 | '--write_bw_log=test', |
52 | '--per_job_logs=0', |
53 | '--log_offset=1', |
54 | '--log_compression=10K', |
55 | ] |
56 | if log_store_compressed: |
57 | fio_args.append('--log_store_compressed=1') |
58 | |
59 | subprocess.check_output([fio] + fio_args) |
60 | |
61 | if log_store_compressed: |
62 | fio_inflate_args = [ |
63 | '--inflate-log=test_bw.log.fz' |
64 | ] |
65 | with open('test_bw.from_fz.log','wt') as f: |
66 | subprocess.check_call([fio]+fio_inflate_args,stdout=f) |
67 | |
68 | def check_log_file(log_store_compressed): |
69 | filename = 'test_bw.from_fz.log' if log_store_compressed else 'test_bw.log' |
70 | with open(filename,'rt') as f: |
71 | file_data = f.read() |
72 | log_lines = [x for x in file_data.split('\n') if len(x.strip())!=0] |
73 | log_ios = len(log_lines) |
74 | |
75 | filesize = 1000*1024*1024 |
76 | bs = 128*1024 |
77 | ios = filesize//bs |
78 | if log_ios!=ios: |
79 | print('wrong number of ios ({}) in log; should be {}'.format(log_ios,ios)) |
80 | return False |
81 | |
82 | expected_offset = 0 |
83 | for line_number,line in enumerate(log_lines): |
84 | log_offset = int(line.split(',')[4]) |
85 | if log_offset != expected_offset: |
86 | print('wrong offset ({}) for io number {} in log; should be {}'.format( |
87 | log_offset, line_number, expected_offset)) |
88 | return False |
89 | expected_offset += bs |
90 | return True |
91 | |
92 | def main(): |
93 | """Entry point for this script.""" |
94 | args = parse_args() |
95 | if args.fio: |
96 | fio_path = args.fio |
97 | else: |
98 | fio_path = os.path.join(os.path.dirname(__file__), '../fio') |
99 | if not os.path.exists(fio_path): |
100 | fio_path = 'fio' |
101 | print("fio path is", fio_path) |
102 | |
103 | passed_count = 0 |
104 | failed_count = 0 |
105 | for log_store_compressed in [False, True]: |
106 | run_fio(fio_path, log_store_compressed) |
107 | passed = check_log_file(log_store_compressed) |
108 | print('Test with log_store_compressed={} {}'.format(log_store_compressed, |
109 | 'PASSED' if passed else 'FAILED')) |
110 | if passed: |
111 | passed_count+=1 |
112 | else: |
113 | failed_count+=1 |
114 | |
115 | print('{} tests passed, {} failed'.format(passed_count, failed_count)) |
116 | |
117 | sys.exit(failed_count) |
118 | |
119 | if __name__ == '__main__': |
120 | main() |
121 | |