5 # Test fio's io_uring_cmd ioengine support for DIF/DIX end-to-end data
9 # see python3 nvmept_pi.py --help
11 # EXAMPLES (THIS IS A DESTRUCTIVE TEST!!)
12 # python3 t/nvmept_pi.py --dut /dev/ng0n1 -f ./fio
13 # python3 t/nvmept_pi.py --dut /dev/ng0n1 -f ./fio --lbaf 1
28 from pathlib import Path
29 from fiotestlib import FioJobCmdTest, run_fio_tests
30 from fiotestcommon import SUCCESS_NONZERO
36 class DifDixTest(FioJobCmdTest):
38 NVMe DIF/DIX test class.
41 def setup(self, parameters):
46 "--ioengine=io_uring_cmd",
48 f"--filename={self.fio_opts['filename']}",
49 f"--rw={self.fio_opts['rw']}",
50 f"--bsrange={self.fio_opts['bsrange']}",
51 f"--output={self.filenames['output']}",
52 f"--output-format={self.fio_opts['output-format']}",
53 f"--md_per_io_size={self.fio_opts['md_per_io_size']}",
54 f"--pi_act={self.fio_opts['pi_act']}",
55 f"--pi_chk={self.fio_opts['pi_chk']}",
56 f"--apptag={self.fio_opts['apptag']}",
57 f"--apptag_mask={self.fio_opts['apptag_mask']}",
59 for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles',
60 'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait',
61 'time_based', 'runtime', 'verify', 'io_size', 'offset', 'number_ios']:
62 if opt in self.fio_opts:
63 option = f"--{opt}={self.fio_opts[opt]}"
64 fio_args.append(option)
66 super().setup(fio_args)
71 # Write data with pi_act=1 and then read the data back (with both
75 # Write workload with variable IO sizes
80 "number_ios": NUMBER_IOS,
81 "output-format": "json",
83 "apptag_mask": "0xFFFF",
86 "pi_chk": "APPTAG,GUARD,REFTAG",
89 "test_class": DifDixTest,
92 # Read workload with fixed small IO size
97 "number_ios": NUMBER_IOS,
98 "output-format": "json",
101 "apptag_mask": "0xFFFF",
103 "pi_chk": "APPTAG,GUARD,REFTAG",
106 "test_class": DifDixTest,
109 # Read workload with fixed small IO size
114 "number_ios": NUMBER_IOS,
115 "output-format": "json",
118 "apptag_mask": "0xFFFF",
120 "pi_chk": "APPTAG,GUARD,REFTAG",
123 "test_class": DifDixTest,
126 # Write workload with fixed large IO size
127 # Precondition for read workloads to follow
132 "number_ios": NUMBER_IOS,
133 "output-format": "json",
135 "apptag_mask": "0xFFFF",
138 "pi_chk": "APPTAG,GUARD,REFTAG",
141 "test_class": DifDixTest,
144 # Read workload with variable IO sizes
149 "number_ios": NUMBER_IOS,
150 "output-format": "json",
153 "apptag_mask": "0xFFFF",
155 "pi_chk": "APPTAG,GUARD,REFTAG",
158 "test_class": DifDixTest,
161 # Read workload with variable IO sizes
166 "number_ios": NUMBER_IOS,
167 "output-format": "json",
170 "apptag_mask": "0xFFFF",
172 "pi_chk": "APPTAG,GUARD,REFTAG",
175 "test_class": DifDixTest,
178 # Write data with pi_act=0 and then read the data back (with both
182 # Write workload with variable IO sizes
187 "number_ios": NUMBER_IOS,
188 "output-format": "json",
190 "apptag_mask": "0xFFFF",
193 "pi_chk": "APPTAG,GUARD,REFTAG",
196 "test_class": DifDixTest,
199 # Read workload with fixed small IO size
204 "number_ios": NUMBER_IOS,
205 "output-format": "json",
208 "apptag_mask": "0xFFFF",
210 "pi_chk": "APPTAG,GUARD,REFTAG",
213 "test_class": DifDixTest,
216 # Read workload with fixed small IO size
221 "number_ios": NUMBER_IOS,
222 "output-format": "json",
225 "apptag_mask": "0xFFFF",
227 "pi_chk": "APPTAG,GUARD,REFTAG",
230 "test_class": DifDixTest,
233 # Write workload with fixed large IO sizes
238 "number_ios": NUMBER_IOS,
239 "output-format": "json",
241 "apptag_mask": "0xFFFF",
244 "pi_chk": "APPTAG,GUARD,REFTAG",
247 "test_class": DifDixTest,
250 # Read workload with variable IO sizes
255 "number_ios": NUMBER_IOS,
256 "output-format": "json",
259 "apptag_mask": "0xFFFF",
261 "pi_chk": "APPTAG,GUARD,REFTAG",
264 "test_class": DifDixTest,
267 # Read workload with variable IO sizes
272 "number_ios": NUMBER_IOS,
273 "output-format": "json",
276 "apptag_mask": "0xFFFF",
278 "pi_chk": "APPTAG,GUARD,REFTAG",
281 "test_class": DifDixTest,
284 # Test apptag errors.
287 # Read workload with variable IO sizes
289 # trigger an apptag error
293 "number_ios": NUMBER_IOS,
294 "output-format": "json",
297 "apptag_mask": "0xFFFF",
299 "pi_chk": "APPTAG,GUARD,REFTAG",
302 "success": SUCCESS_NONZERO,
303 "test_class": DifDixTest,
306 # Read workload with variable IO sizes
308 # trigger an apptag error
312 "number_ios": NUMBER_IOS,
313 "output-format": "json",
316 "apptag_mask": "0xFFFF",
318 "pi_chk": "APPTAG,GUARD,REFTAG",
321 "success": SUCCESS_NONZERO,
322 "test_class": DifDixTest,
325 # Read workload with variable IO sizes
327 # trigger an apptag error
328 # same as above but with pi_chk=APPTAG only
332 "number_ios": NUMBER_IOS,
333 "output-format": "json",
336 "apptag_mask": "0xFFFF",
341 "success": SUCCESS_NONZERO,
342 "test_class": DifDixTest,
345 # Read workload with variable IO sizes
347 # trigger an apptag error
348 # same as above but with pi_chk=APPTAG only
352 "number_ios": NUMBER_IOS,
353 "output-format": "json",
356 "apptag_mask": "0xFFFF",
361 "success": SUCCESS_NONZERO,
362 "test_class": DifDixTest,
365 # Read workload with variable IO sizes
367 # this case would trigger an apptag error, but pi_chk says to check
368 # only the Guard PI and reftag, so there should be no error
372 "number_ios": NUMBER_IOS,
373 "output-format": "json",
376 "apptag_mask": "0xFFFF",
378 "pi_chk": "GUARD,REFTAG",
381 "test_class": DifDixTest,
384 # Read workload with variable IO sizes
386 # this case would trigger an apptag error, but pi_chk says to check
387 # only the Guard PI and reftag, so there should be no error
391 "number_ios": NUMBER_IOS,
392 "output-format": "json",
395 "apptag_mask": "0xFFFF",
397 "pi_chk": "GUARD,REFTAG",
400 "test_class": DifDixTest,
403 # Read workload with variable IO sizes
405 # this case would trigger an apptag error, but pi_chk says to check
406 # only the Guard PI, so there should be no error
410 "number_ios": NUMBER_IOS,
411 "output-format": "json",
414 "apptag_mask": "0xFFFF",
419 "test_class": DifDixTest,
422 # Read workload with variable IO sizes
424 # this case would trigger an apptag error, but pi_chk says to check
425 # only the Guard PI, so there should be no error
429 "number_ios": NUMBER_IOS,
430 "output-format": "json",
433 "apptag_mask": "0xFFFF",
438 "test_class": DifDixTest,
441 # Read workload with variable IO sizes
443 # this case would trigger an apptag error, but pi_chk says to check
444 # only the reftag, so there should be no error
445 # This case will be skipped when the device is formatted with Type 3 PI
446 # since Type 3 PI ignores the reftag
450 "number_ios": NUMBER_IOS,
451 "output-format": "json",
454 "apptag_mask": "0xFFFF",
460 "test_class": DifDixTest,
463 # Read workload with variable IO sizes
465 # this case would trigger an apptag error, but pi_chk says to check
466 # only the reftag, so there should be no error
467 # This case will be skipped when the device is formatted with Type 3 PI
468 # since Type 3 PI ignores the reftag
472 "number_ios": NUMBER_IOS,
473 "output-format": "json",
476 "apptag_mask": "0xFFFF",
482 "test_class": DifDixTest,
485 # Read workload with variable IO sizes
487 # use apptag mask to ignore apptag mismatch
491 "number_ios": NUMBER_IOS,
492 "output-format": "json",
495 "apptag_mask": "0x0FFF",
497 "pi_chk": "APPTAG,GUARD,REFTAG",
500 "test_class": DifDixTest,
503 # Read workload with variable IO sizes
505 # use apptag mask to ignore apptag mismatch
509 "number_ios": NUMBER_IOS,
510 "output-format": "json",
513 "apptag_mask": "0x0FFF",
515 "pi_chk": "APPTAG,GUARD,REFTAG",
518 "test_class": DifDixTest,
521 # Read workload with variable IO sizes
523 # use apptag mask to ignore apptag mismatch
527 "number_ios": NUMBER_IOS,
528 "output-format": "json",
531 "apptag_mask": "0x0FFF",
533 "pi_chk": "APPTAG,GUARD,REFTAG",
536 "test_class": DifDixTest,
539 # Read workload with variable IO sizes
541 # use apptag mask to ignore apptag mismatch
545 "number_ios": NUMBER_IOS,
546 "output-format": "json",
549 "apptag_mask": "0x0FFF",
551 "pi_chk": "APPTAG,GUARD,REFTAG",
554 "test_class": DifDixTest,
557 # Write workload with fixed large IO sizes
558 # Set apptag=0xFFFF to disable all checking for Type 1 and 2
563 "number_ios": NUMBER_IOS,
564 "output-format": "json",
566 "apptag_mask": "0xFFFF",
569 "pi_chk": "APPTAG,GUARD,REFTAG",
573 "test_class": DifDixTest,
576 # Read workload with variable IO sizes
578 # Data was written with apptag=0xFFFF
579 # Reading the data back should disable all checking for Type 1 and 2
583 "number_ios": NUMBER_IOS,
584 "output-format": "json",
587 "apptag_mask": "0xFFFF",
589 "pi_chk": "APPTAG,GUARD,REFTAG",
593 "test_class": DifDixTest,
596 # Read workload with variable IO sizes
598 # Data was written with apptag=0xFFFF
599 # Reading the data back should disable all checking for Type 1 and 2
603 "number_ios": NUMBER_IOS,
604 "output-format": "json",
607 "apptag_mask": "0xFFFF",
609 "pi_chk": "APPTAG,GUARD,REFTAG",
613 "test_class": DifDixTest,
616 # Error cases related to block size and metadata size
619 # Use a min block size that is not a multiple of lba/elba size to
624 "number_ios": NUMBER_IOS,
625 "output-format": "json",
628 "apptag_mask": "0x0FFF",
630 "pi_chk": "APPTAG,GUARD,REFTAG",
631 "bs_low": BS_LOW+0.5,
633 "success": SUCCESS_NONZERO,
634 "test_class": DifDixTest,
637 # Use metadata size that is too small
641 "number_ios": NUMBER_IOS,
642 "output-format": "json",
645 "apptag_mask": "0x0FFF",
647 "pi_chk": "APPTAG,GUARD,REFTAG",
650 "mdsize_adjustment": -1,
651 "success": SUCCESS_NONZERO,
653 "test_class": DifDixTest,
656 # Read workload with variable IO sizes
658 # Should still work even if metadata size is too large
662 "number_ios": NUMBER_IOS,
663 "output-format": "json",
666 "apptag_mask": "0x0FFF",
668 "pi_chk": "APPTAG,GUARD,REFTAG",
671 "mdsize_adjustment": 1,
672 "test_class": DifDixTest,
678 """Parse command-line arguments."""
680 parser = argparse.ArgumentParser()
681 parser.add_argument('-d', '--debug', help='Enable debug messages', action='store_true')
682 parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)')
683 parser.add_argument('-a', '--artifact-root', help='artifact root directory')
684 parser.add_argument('-s', '--skip', nargs='+', type=int,
685 help='list of test(s) to skip')
686 parser.add_argument('-o', '--run-only', nargs='+', type=int,
687 help='list of test(s) to run, skipping all others')
688 parser.add_argument('--dut', help='target NVMe character device to test '
689 '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True)
690 parser.add_argument('-l', '--lbaf', nargs='+', type=int,
691 help='list of lba formats to test')
692 args = parser.parse_args()
699 Determine which LBA formats to use. Use either the ones specified on the
700 command line or if none are specified query the device and use all lba
701 formats with metadata.
704 id_ns_cmd = f"sudo nvme id-ns --output-format=json {args.dut}".split(' ')
705 id_ns_output = subprocess.check_output(id_ns_cmd)
706 lbafs = json.loads(id_ns_output)['lbafs']
708 for lbaf in args.lbaf:
709 lbaf_list.append({'lbaf': lbaf, 'ds': 2 ** lbafs[lbaf]['ds'],
710 'ms': lbafs[lbaf]['ms'], })
711 if lbafs[lbaf]['ms'] == 0:
712 print(f'Error: lbaf {lbaf} has metadata size zero')
715 for lbaf_num, lbaf in enumerate(lbafs):
717 lbaf_list.append({'lbaf': lbaf_num, 'ds': 2 ** lbaf['ds'],
723 def get_guard_pi(lbaf_list, args):
725 Find out how many bits of guard protection information are associated with
726 each lbaf to be used. If this is not available assume 16-bit guard pi.
727 Also record the bytes of protection information associated with the number
730 nvm_id_ns_cmd = f"sudo nvme nvm-id-ns --output-format=json {args.dut}".split(' ')
732 nvm_id_ns_output = subprocess.check_output(nvm_id_ns_cmd)
733 except subprocess.CalledProcessError:
734 print(f"Non-zero return code from {' '.join(nvm_id_ns_cmd)}; " \
735 "assuming all lbafs use 16b Guard Protection Information")
736 for lbaf in lbaf_list:
737 lbaf['guard_pi_bits'] = 16
739 elbafs = json.loads(nvm_id_ns_output)['elbafs']
740 for elbaf_num, elbaf in enumerate(elbafs):
741 for lbaf in lbaf_list:
742 if lbaf['lbaf'] == elbaf_num:
743 lbaf['guard_pi_bits'] = 16 << elbaf['pif']
745 # For 16b Guard Protection Information, the PI requires 8 bytes
746 # For 32b and 64b Guard PI, the PI requires 16 bytes
747 for lbaf in lbaf_list:
748 if lbaf['guard_pi_bits'] == 16:
751 lbaf['pi_bytes'] = 16
754 def get_capabilities(args):
756 Determine what end-to-end data protection features the device supports.
758 caps = { 'pil': [], 'pitype': [], 'elba': [] }
759 id_ns_cmd = f"sudo nvme id-ns --output-format=json {args.dut}".split(' ')
760 id_ns_output = subprocess.check_output(id_ns_cmd)
761 id_ns_json = json.loads(id_ns_output)
763 mc = id_ns_json['mc']
765 caps['elba'].append(1)
767 caps['elba'].append(0)
769 dpc = id_ns_json['dpc']
771 caps['pitype'].append(1)
773 caps['pitype'].append(2)
775 caps['pitype'].append(3)
777 caps['pil'].append(1)
779 caps['pil'].append(0)
781 for _, value in caps.items():
783 logging.error("One or more end-to-end data protection features unsupported: %s", caps)
789 def format_device(args, lbaf, pitype, pil, elba):
791 Format device using specified lba format with specified pitype, pil, and
795 format_cmd = f"sudo nvme format {args.dut} --lbaf={lbaf['lbaf']} " \
796 f"--pi={pitype} --pil={pil} --ms={elba} --force"
797 logging.debug("Format command: %s", format_cmd)
798 format_cmd = format_cmd.split(' ')
799 format_cmd_result = subprocess.run(format_cmd, capture_output=True, check=False,
800 encoding=locale.getpreferredencoding())
802 # Sometimes nvme-cli may format the device successfully but fail to
803 # rescan the namespaces after the format. Continue if this happens but
804 # abort if some other error occurs.
805 if format_cmd_result.returncode != 0:
806 if 'failed to rescan namespaces' not in format_cmd_result.stderr \
807 or 'Success formatting namespace' not in format_cmd_result.stdout:
808 logging.error(format_cmd_result.stdout)
809 logging.error(format_cmd_result.stderr)
810 print("Unable to format device; skipping this configuration")
813 logging.debug(format_cmd_result.stdout)
817 def difdix_test(test_env, args, lbaf, pitype, elba):
819 Adjust test arguments based on values of lbaf, pitype, and elba. Then run
822 for test in TEST_LIST:
823 test['force_skip'] = False
825 blocksize = lbaf['ds']
826 # Set fio blocksize parameter at runtime
827 # If we formatted the device in extended LBA mode (e.g., 520-byte
828 # sectors), we usually need to add the lba data size and metadata size
829 # together for fio's bs parameter. However, if pi_act == 1 and the
830 # device is formatted so that the metadata is the same size as the PI,
831 # then the device will take care of everything and the application
832 # should just use regular power of 2 lba data size even when the device
833 # is in extended lba mode.
835 if not test['fio_opts']['pi_act'] or lbaf['ms'] != lbaf['pi_bytes']:
836 blocksize += lbaf['ms']
837 test['fio_opts']['md_per_io_size'] = 0
839 # If we are using a separate buffer for metadata, fio doesn't need to
840 # do anything when pi_act==1 and protection information size is equal to
841 # metadata size since the device is taking care of it all. If either of
842 # the two conditions do not hold, then we do need to allocate a
843 # separate metadata buffer.
844 if test['fio_opts']['pi_act'] and lbaf['ms'] == lbaf['pi_bytes']:
845 test['fio_opts']['md_per_io_size'] = 0
847 test['fio_opts']['md_per_io_size'] = lbaf['ms'] * test['bs_high']
849 test['fio_opts']['bsrange'] = f"{blocksize * test['bs_low']}-{blocksize * test['bs_high']}"
850 if 'mdsize_adjustment' in test:
851 test['fio_opts']['md_per_io_size'] += test['mdsize_adjustment']
853 # Set fio pi_chk parameter at runtime. If the device is formatted
854 # with Type 3 protection information, this means that the reference
855 # tag is not checked and I/O commands may throw an error if they
856 # are submitted with the REFTAG bit set in pi_chk. Make sure fio
857 # does not set pi_chk's REFTAG bit if the device is formatted with
860 if pitype == 3 and 'REFTAG' in test['pi_chk']:
861 test['fio_opts']['pi_chk'] = test['pi_chk'].replace('REFTAG','')
862 logging.debug("Type 3 PI: dropping REFTAG bit")
864 test['fio_opts']['pi_chk'] = test['pi_chk']
867 if pitype == 3 and 'type3' in test['skip']:
868 test['force_skip'] = True
869 logging.debug("Type 3 PI: skipping test case")
870 if elba and 'elba' in test['skip']:
871 test['force_skip'] = True
872 logging.debug("extended lba format: skipping test case")
874 logging.debug("Test %d: pi_act=%d, bsrange=%s, md_per_io_size=%d", test['test_id'],
875 test['fio_opts']['pi_act'], test['fio_opts']['bsrange'],
876 test['fio_opts']['md_per_io_size'])
878 return run_fio_tests(TEST_LIST, test_env, args)
883 Run tests using fio's io_uring_cmd ioengine to exercise end-to-end data
884 protection capabilities.
890 logging.basicConfig(level=logging.DEBUG)
892 logging.basicConfig(level=logging.INFO)
894 artifact_root = args.artifact_root if args.artifact_root else \
895 f"nvmept_pi-test-{time.strftime('%Y%m%d-%H%M%S')}"
896 os.mkdir(artifact_root)
897 print(f"Artifact directory is {artifact_root}")
900 fio_path = str(Path(args.fio).absolute())
903 print(f"fio path is {fio_path}")
905 lbaf_list = get_lbafs(args)
906 get_guard_pi(lbaf_list, args)
907 caps = get_capabilities(args)
908 print("Device capabilities:", caps)
910 for test in TEST_LIST:
911 test['fio_opts']['filename'] = args.dut
914 'fio_path': fio_path,
915 'fio_root': str(Path(__file__).absolute().parent.parent),
916 'artifact_root': artifact_root,
917 'basename': 'nvmept_pi',
920 total = { 'passed': 0, 'failed': 0, 'skipped': 0 }
923 for lbaf, pil, pitype, elba in itertools.product(lbaf_list, caps['pil'], caps['pitype'],
925 print(f"\nlbaf: {lbaf}, pil: {pil}, pitype: {pitype}, elba: {elba}")
927 if not format_device(args, lbaf, pitype, pil, elba):
930 test_env['artifact_root'] = \
931 os.path.join(artifact_root, f"lbaf{lbaf['lbaf']}pil{pil}pitype{pitype}" \
933 os.mkdir(test_env['artifact_root'])
935 passed, failed, skipped = difdix_test(test_env, args, lbaf, pitype, elba)
937 total['passed'] += passed
938 total['failed'] += failed
939 total['skipped'] += skipped
940 except KeyboardInterrupt:
943 print(f"\n\n{total['passed']} test(s) passed, {total['failed']} failed, " \
944 f"{total['skipped']} skipped")
945 sys.exit(total['failed'])
948 if __name__ == '__main__':