| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # log_compression.py |
| 4 | # |
| 5 | # Test log_compression and log_store_compressed. Uses null ioengine. |
| 6 | # Previous bugs have caused output in per I/O log files to be missing |
| 7 | # and/or out of order |
| 8 | # |
| 9 | # Expected result: 8000 log entries, offset starting at 0 and increasing by bs |
| 10 | # Buggy result: Log entries out of order (usually without log_store_compressed) |
| 11 | # and/or missing log entries (usually with log_store_compressed) |
| 12 | # |
| 13 | # USAGE |
| 14 | # python log_compression.py [-f fio-executable] |
| 15 | # |
| 16 | # EXAMPLES |
| 17 | # python t/log_compression.py |
| 18 | # python t/log_compression.py -f ./fio |
| 19 | # |
| 20 | # REQUIREMENTS |
| 21 | # Python 3.5+ |
| 22 | # |
| 23 | # ===TEST MATRIX=== |
| 24 | # |
| 25 | # With log_compression=10K |
| 26 | # With log_store_compressed=1 and log_compression=10K |
| 27 | |
| 28 | import os |
| 29 | import sys |
| 30 | import platform |
| 31 | import argparse |
| 32 | import subprocess |
| 33 | |
| 34 | |
| 35 | def parse_args(): |
| 36 | """Parse command-line arguments.""" |
| 37 | parser = argparse.ArgumentParser() |
| 38 | parser.add_argument('-f', '--fio', |
| 39 | help='path to fio executable (e.g., ./fio)') |
| 40 | return parser.parse_args() |
| 41 | |
| 42 | |
| 43 | def run_fio(fio,log_store_compressed): |
| 44 | fio_args = [ |
| 45 | '--name=job', |
| 46 | '--ioengine=null', |
| 47 | '--filesize=1000M', |
| 48 | '--bs=128K', |
| 49 | '--rw=write', |
| 50 | '--iodepth=1', |
| 51 | '--write_bw_log=test', |
| 52 | '--per_job_logs=0', |
| 53 | '--log_offset=1', |
| 54 | '--log_compression=10K', |
| 55 | ] |
| 56 | if log_store_compressed: |
| 57 | fio_args.append('--log_store_compressed=1') |
| 58 | |
| 59 | subprocess.check_output([fio] + fio_args) |
| 60 | |
| 61 | if log_store_compressed: |
| 62 | fio_inflate_args = [ |
| 63 | '--inflate-log=test_bw.log.fz' |
| 64 | ] |
| 65 | with open('test_bw.from_fz.log','wt') as f: |
| 66 | subprocess.check_call([fio]+fio_inflate_args,stdout=f) |
| 67 | |
| 68 | def check_log_file(log_store_compressed): |
| 69 | filename = 'test_bw.from_fz.log' if log_store_compressed else 'test_bw.log' |
| 70 | with open(filename,'rt') as f: |
| 71 | file_data = f.read() |
| 72 | log_lines = [x for x in file_data.split('\n') if len(x.strip())!=0] |
| 73 | log_ios = len(log_lines) |
| 74 | |
| 75 | filesize = 1000*1024*1024 |
| 76 | bs = 128*1024 |
| 77 | ios = filesize//bs |
| 78 | if log_ios!=ios: |
| 79 | print('wrong number of ios ({}) in log; should be {}'.format(log_ios,ios)) |
| 80 | return False |
| 81 | |
| 82 | expected_offset = 0 |
| 83 | for line_number,line in enumerate(log_lines): |
| 84 | log_offset = int(line.split(',')[4]) |
| 85 | if log_offset != expected_offset: |
| 86 | print('wrong offset ({}) for io number {} in log; should be {}'.format( |
| 87 | log_offset, line_number, expected_offset)) |
| 88 | return False |
| 89 | expected_offset += bs |
| 90 | return True |
| 91 | |
| 92 | def main(): |
| 93 | """Entry point for this script.""" |
| 94 | args = parse_args() |
| 95 | if args.fio: |
| 96 | fio_path = args.fio |
| 97 | else: |
| 98 | fio_path = os.path.join(os.path.dirname(__file__), '../fio') |
| 99 | if not os.path.exists(fio_path): |
| 100 | fio_path = 'fio' |
| 101 | print("fio path is", fio_path) |
| 102 | |
| 103 | passed_count = 0 |
| 104 | failed_count = 0 |
| 105 | for log_store_compressed in [False, True]: |
| 106 | run_fio(fio_path, log_store_compressed) |
| 107 | passed = check_log_file(log_store_compressed) |
| 108 | print('Test with log_store_compressed={} {}'.format(log_store_compressed, |
| 109 | 'PASSED' if passed else 'FAILED')) |
| 110 | if passed: |
| 111 | passed_count+=1 |
| 112 | else: |
| 113 | failed_count+=1 |
| 114 | |
| 115 | print('{} tests passed, {} failed'.format(passed_count, failed_count)) |
| 116 | |
| 117 | sys.exit(failed_count) |
| 118 | |
| 119 | if __name__ == '__main__': |
| 120 | main() |
| 121 | |