Commit | Line | Data |
---|---|---|
0c8c808d VF |
1 | #!/usr/bin/env python3 |
2 | # | |
3 | # Copyright 2024 Samsung Electronics Co., Ltd All Rights Reserved | |
4 | # | |
5 | # For conditions of distribution and use, see the accompanying COPYING file. | |
6 | # | |
7 | """ | |
8 | # nvmept_streams.py | |
9 | # | |
10 | # Test fio's NVMe streams support using the io_uring_cmd ioengine with NVMe | |
11 | # pass-through commands. | |
12 | # | |
13 | # USAGE | |
14 | # see python3 nvmept_streams.py --help | |
15 | # | |
16 | # EXAMPLES | |
17 | # python3 t/nvmept_streams.py --dut /dev/ng0n1 | |
18 | # python3 t/nvmept_streams.py --dut /dev/ng1n1 -f ./fio | |
19 | # | |
20 | # REQUIREMENTS | |
21 | # Python 3.6 | |
22 | # | |
23 | # WARNING | |
24 | # This is a destructive test | |
25 | # | |
26 | # Enable streams with | |
27 | # nvme dir-send -D 0 -O 1 -e 1 -T 1 /dev/nvme0n1 | |
28 | # | |
29 | # See streams directive status with | |
30 | # nvme dir-receive -D 0 -O 1 -H /dev/nvme0n1 | |
31 | """ | |
32 | import os | |
33 | import sys | |
34 | import time | |
35 | import locale | |
36 | import logging | |
37 | import argparse | |
38 | import subprocess | |
39 | from pathlib import Path | |
40 | from fiotestlib import FioJobCmdTest, run_fio_tests | |
41 | from fiotestcommon import SUCCESS_NONZERO | |
42 | ||
43 | ||
44 | class StreamsTest(FioJobCmdTest): | |
45 | """ | |
46 | NVMe pass-through test class for streams. Check to make sure output for | |
47 | selected data direction(s) is non-zero and that zero data appears for other | |
48 | directions. | |
49 | """ | |
50 | ||
51 | def setup(self, parameters): | |
52 | """Setup a test.""" | |
53 | ||
54 | fio_args = [ | |
55 | "--name=nvmept-streams", | |
56 | "--ioengine=io_uring_cmd", | |
57 | "--cmd_type=nvme", | |
58 | "--randrepeat=0", | |
59 | f"--filename={self.fio_opts['filename']}", | |
60 | f"--rw={self.fio_opts['rw']}", | |
61 | f"--output={self.filenames['output']}", | |
62 | f"--output-format={self.fio_opts['output-format']}", | |
63 | ] | |
64 | for opt in ['fixedbufs', 'nonvectored', 'force_async', 'registerfiles', | |
65 | 'sqthread_poll', 'sqthread_poll_cpu', 'hipri', 'nowait', | |
66 | 'time_based', 'runtime', 'verify', 'io_size', 'num_range', | |
67 | 'iodepth', 'iodepth_batch', 'iodepth_batch_complete', | |
68 | 'size', 'rate', 'bs', 'bssplit', 'bsrange', 'randrepeat', | |
69 | 'buffer_pattern', 'verify_pattern', 'offset', 'dataplacement', | |
70 | 'plids', 'plid_select' ]: | |
71 | if opt in self.fio_opts: | |
72 | option = f"--{opt}={self.fio_opts[opt]}" | |
73 | fio_args.append(option) | |
74 | ||
75 | super().setup(fio_args) | |
76 | ||
77 | ||
78 | def check_result(self): | |
79 | try: | |
80 | self._check_result() | |
81 | finally: | |
82 | release_all_streams(self.fio_opts['filename']) | |
83 | ||
84 | ||
85 | def _check_result(self): | |
86 | ||
87 | super().check_result() | |
88 | ||
89 | if 'rw' not in self.fio_opts or \ | |
90 | not self.passed or \ | |
91 | 'json' not in self.fio_opts['output-format']: | |
92 | return | |
93 | ||
94 | job = self.json_data['jobs'][0] | |
95 | ||
96 | if self.fio_opts['rw'] in ['read', 'randread']: | |
97 | self.passed = self.check_all_ddirs(['read'], job) | |
98 | elif self.fio_opts['rw'] in ['write', 'randwrite']: | |
99 | if 'verify' not in self.fio_opts: | |
100 | self.passed = self.check_all_ddirs(['write'], job) | |
101 | else: | |
102 | self.passed = self.check_all_ddirs(['read', 'write'], job) | |
103 | elif self.fio_opts['rw'] in ['trim', 'randtrim']: | |
104 | self.passed = self.check_all_ddirs(['trim'], job) | |
105 | elif self.fio_opts['rw'] in ['readwrite', 'randrw']: | |
106 | self.passed = self.check_all_ddirs(['read', 'write'], job) | |
107 | elif self.fio_opts['rw'] in ['trimwrite', 'randtrimwrite']: | |
108 | self.passed = self.check_all_ddirs(['trim', 'write'], job) | |
109 | else: | |
110 | logging.error("Unhandled rw value %s", self.fio_opts['rw']) | |
111 | self.passed = False | |
112 | ||
113 | if 'iodepth' in self.fio_opts: | |
114 | # We will need to figure something out if any test uses an iodepth | |
115 | # different from 8 | |
116 | if job['iodepth_level']['8'] < 95: | |
117 | logging.error("Did not achieve requested iodepth") | |
118 | self.passed = False | |
119 | else: | |
120 | logging.debug("iodepth 8 target met %s", job['iodepth_level']['8']) | |
121 | ||
122 | stream_ids = [int(stream) for stream in self.fio_opts['plids'].split(',')] | |
123 | if not self.check_streams(self.fio_opts['filename'], stream_ids): | |
124 | self.passed = False | |
125 | logging.error("Streams not as expected") | |
126 | else: | |
127 | logging.debug("Streams created as expected") | |
128 | ||
129 | ||
130 | def check_streams(self, dut, stream_ids): | |
131 | """ | |
132 | Confirm that the specified stream IDs exist on the specified device. | |
133 | """ | |
134 | ||
135 | id_list = get_device_stream_ids(dut) | |
136 | if not id_list: | |
137 | return False | |
138 | ||
139 | for stream in stream_ids: | |
140 | if stream in id_list: | |
141 | logging.debug("Stream ID %d found active on device", stream) | |
142 | id_list.remove(stream) | |
143 | else: | |
144 | if self.__class__.__name__ != "StreamsTestRand": | |
145 | logging.error("Stream ID %d not found on device", stream) | |
146 | else: | |
147 | logging.debug("Stream ID %d not found on device", stream) | |
148 | return False | |
149 | ||
150 | if len(id_list) != 0: | |
151 | logging.error("Extra stream IDs %s found on device", str(id_list)) | |
152 | return False | |
153 | ||
154 | return True | |
155 | ||
156 | ||
157 | class StreamsTestRR(StreamsTest): | |
158 | """ | |
159 | NVMe pass-through test class for streams. Check to make sure output for | |
160 | selected data direction(s) is non-zero and that zero data appears for other | |
161 | directions. Check that Stream IDs are accessed in round robin order. | |
162 | """ | |
163 | ||
164 | def check_streams(self, dut, stream_ids): | |
165 | """ | |
166 | The number of IOs is less than the number of stream IDs provided. Let N | |
167 | be the number of IOs. Make sure that the device only has the first N of | |
168 | the stream IDs provided. | |
169 | ||
170 | This will miss some cases where some other selection algorithm happens | |
171 | to select the first N stream IDs. The solution would be to repeat this | |
172 | test multiple times. Multiple trials passing would be evidence that | |
173 | round robin is working correctly. | |
174 | """ | |
175 | ||
176 | id_list = get_device_stream_ids(dut) | |
177 | if not id_list: | |
178 | return False | |
179 | ||
180 | num_streams = int(self.fio_opts['io_size'] / self.fio_opts['bs']) | |
181 | stream_ids = sorted(stream_ids)[0:num_streams] | |
182 | ||
183 | return super().check_streams(dut, stream_ids) | |
184 | ||
185 | ||
186 | class StreamsTestRand(StreamsTest): | |
187 | """ | |
188 | NVMe pass-through test class for streams. Check to make sure output for | |
189 | selected data direction(s) is non-zero and that zero data appears for other | |
190 | directions. Check that Stream IDs are accessed in random order. | |
191 | """ | |
192 | ||
193 | def check_streams(self, dut, stream_ids): | |
194 | """ | |
195 | The number of IOs is less than the number of stream IDs provided. Let N | |
196 | be the number of IOs. Confirm that the stream IDs on the device are not | |
197 | the first N stream IDs. | |
198 | ||
199 | This will produce false positives because it is possible for the first | |
200 | N stream IDs to be randomly selected. We can reduce the probability of | |
201 | false positives by increasing N and increasing the number of streams | |
202 | IDs to choose from, although fio has a max of 16 placement IDs. | |
203 | """ | |
204 | ||
205 | id_list = get_device_stream_ids(dut) | |
206 | if not id_list: | |
207 | return False | |
208 | ||
209 | num_streams = int(self.fio_opts['io_size'] / self.fio_opts['bs']) | |
210 | stream_ids = sorted(stream_ids)[0:num_streams] | |
211 | ||
212 | return not super().check_streams(dut, stream_ids) | |
213 | ||
214 | ||
215 | def get_device_stream_ids(dut): | |
216 | cmd = f"sudo nvme dir-receive -D 1 -O 2 -H {dut}" | |
217 | logging.debug("check streams command: %s", cmd) | |
218 | cmd = cmd.split(' ') | |
219 | cmd_result = subprocess.run(cmd, capture_output=True, check=False, | |
220 | encoding=locale.getpreferredencoding()) | |
221 | ||
222 | logging.debug(cmd_result.stdout) | |
223 | ||
224 | if cmd_result.returncode != 0: | |
225 | logging.error("Error obtaining device %s stream IDs: %s", dut, cmd_result.stderr) | |
226 | return False | |
227 | ||
228 | id_list = [] | |
229 | for line in cmd_result.stdout.split('\n'): | |
230 | if not 'Stream Identifier' in line: | |
231 | continue | |
232 | tokens = line.split(':') | |
233 | id_list.append(int(tokens[1])) | |
234 | ||
235 | return id_list | |
236 | ||
237 | ||
238 | def release_stream(dut, stream_id): | |
239 | """ | |
240 | Release stream on given device with selected ID. | |
241 | """ | |
242 | cmd = f"nvme dir-send -D 1 -O 1 -S {stream_id} {dut}" | |
243 | logging.debug("release stream command: %s", cmd) | |
244 | cmd = cmd.split(' ') | |
245 | cmd_result = subprocess.run(cmd, capture_output=True, check=False, | |
246 | encoding=locale.getpreferredencoding()) | |
247 | ||
248 | if cmd_result.returncode != 0: | |
249 | logging.error("Error releasing %s stream %d", dut, stream_id) | |
250 | return False | |
251 | ||
252 | return True | |
253 | ||
254 | ||
255 | def release_all_streams(dut): | |
256 | """ | |
257 | Release all streams on specified device. | |
258 | """ | |
259 | ||
260 | id_list = get_device_stream_ids(dut) | |
261 | if not id_list: | |
262 | return False | |
263 | ||
264 | for stream in id_list: | |
265 | if not release_stream(dut, stream): | |
266 | return False | |
267 | ||
268 | return True | |
269 | ||
270 | ||
271 | TEST_LIST = [ | |
272 | # 4k block size | |
273 | # {seq write, rand write} x {single stream, four streams} | |
274 | { | |
275 | "test_id": 1, | |
276 | "fio_opts": { | |
277 | "rw": 'write', | |
278 | "bs": 4096, | |
279 | "io_size": 256*1024*1024, | |
280 | "verify": "crc32c", | |
281 | "plids": "8", | |
282 | "dataplacement": "streams", | |
283 | "output-format": "json", | |
284 | }, | |
285 | "test_class": StreamsTest, | |
286 | }, | |
287 | { | |
288 | "test_id": 2, | |
289 | "fio_opts": { | |
290 | "rw": 'randwrite', | |
291 | "bs": 4096, | |
292 | "io_size": 256*1024*1024, | |
293 | "verify": "crc32c", | |
294 | "plids": "3", | |
295 | "dataplacement": "streams", | |
296 | "output-format": "json", | |
297 | }, | |
298 | "test_class": StreamsTest, | |
299 | }, | |
300 | { | |
301 | "test_id": 3, | |
302 | "fio_opts": { | |
303 | "rw": 'write', | |
304 | "bs": 4096, | |
305 | "io_size": 256*1024*1024, | |
306 | "verify": "crc32c", | |
307 | "plids": "1,2,3,4", | |
308 | "dataplacement": "streams", | |
309 | "output-format": "json", | |
310 | }, | |
311 | "test_class": StreamsTest, | |
312 | }, | |
313 | { | |
314 | "test_id": 4, | |
315 | "fio_opts": { | |
316 | "rw": 'randwrite', | |
317 | "bs": 4096, | |
318 | "io_size": 256*1024*1024, | |
319 | "verify": "crc32c", | |
320 | "plids": "5,6,7,8", | |
321 | "dataplacement": "streams", | |
322 | "output-format": "json", | |
323 | }, | |
324 | "test_class": StreamsTest, | |
325 | }, | |
326 | # 256KiB block size | |
327 | # {seq write, rand write} x {single stream, four streams} | |
328 | { | |
329 | "test_id": 10, | |
330 | "fio_opts": { | |
331 | "rw": 'write', | |
332 | "bs": 256*1024, | |
333 | "io_size": 256*1024*1024, | |
334 | "verify": "crc32c", | |
335 | "plids": "88", | |
336 | "dataplacement": "streams", | |
337 | "output-format": "json", | |
338 | }, | |
339 | "test_class": StreamsTest, | |
340 | }, | |
341 | { | |
342 | "test_id": 11, | |
343 | "fio_opts": { | |
344 | "rw": 'randwrite', | |
345 | "bs": 256*1024, | |
346 | "io_size": 256*1024*1024, | |
347 | "verify": "crc32c", | |
348 | "plids": "20", | |
349 | "dataplacement": "streams", | |
350 | "output-format": "json", | |
351 | }, | |
352 | "test_class": StreamsTest, | |
353 | }, | |
354 | { | |
355 | "test_id": 12, | |
356 | "fio_opts": { | |
357 | "rw": 'write', | |
358 | "bs": 256*1024, | |
359 | "io_size": 256*1024*1024, | |
360 | "verify": "crc32c", | |
361 | "plids": "16,32,64,128", | |
362 | "dataplacement": "streams", | |
363 | "output-format": "json", | |
364 | }, | |
365 | "test_class": StreamsTest, | |
366 | }, | |
367 | { | |
368 | "test_id": 13, | |
369 | "fio_opts": { | |
370 | "rw": 'randwrite', | |
371 | "bs": 256*1024, | |
372 | "io_size": 256*1024*1024, | |
373 | "verify": "crc32c", | |
374 | "plids": "10,20,40,82", | |
375 | "dataplacement": "streams", | |
376 | "output-format": "json", | |
377 | }, | |
378 | "test_class": StreamsTest, | |
379 | }, | |
380 | # Test placement ID selection patterns | |
381 | # default is round robin | |
382 | { | |
383 | "test_id": 20, | |
384 | "fio_opts": { | |
385 | "rw": 'write', | |
386 | "bs": 4096, | |
387 | "io_size": 8192, | |
388 | "plids": '88,99,100,123,124,125,126,127,128,129,130,131,132,133,134,135', | |
389 | "dataplacement": "streams", | |
390 | "output-format": "json", | |
391 | }, | |
392 | "test_class": StreamsTestRR, | |
393 | }, | |
394 | { | |
395 | "test_id": 21, | |
396 | "fio_opts": { | |
397 | "rw": 'write', | |
398 | "bs": 4096, | |
399 | "io_size": 8192, | |
400 | "plids": '12,88,99,100,123,124,125,126,127,128,129,130,131,132,133,11', | |
401 | "dataplacement": "streams", | |
402 | "output-format": "json", | |
403 | }, | |
404 | "test_class": StreamsTestRR, | |
405 | }, | |
406 | # explicitly select round robin | |
407 | { | |
408 | "test_id": 22, | |
409 | "fio_opts": { | |
410 | "rw": 'write', | |
411 | "bs": 4096, | |
412 | "io_size": 8192, | |
413 | "plids": '22,88,99,100,123,124,125,126,127,128,129,130,131,132,133,134', | |
414 | "dataplacement": "streams", | |
415 | "output-format": "json", | |
416 | "plid_select": "roundrobin", | |
417 | }, | |
418 | "test_class": StreamsTestRR, | |
419 | }, | |
420 | # explicitly select random | |
421 | { | |
422 | "test_id": 23, | |
423 | "fio_opts": { | |
424 | "rw": 'write', | |
425 | "bs": 4096, | |
426 | "io_size": 8192, | |
427 | "plids": '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16', | |
428 | "dataplacement": "streams", | |
429 | "output-format": "json", | |
430 | "plid_select": "random", | |
431 | }, | |
432 | "test_class": StreamsTestRand, | |
433 | }, | |
434 | # Error case with placement ID > 0xFFFF | |
435 | { | |
436 | "test_id": 30, | |
437 | "fio_opts": { | |
438 | "rw": 'write', | |
439 | "bs": 4096, | |
440 | "io_size": 8192, | |
441 | "plids": "1,2,3,0x10000", | |
442 | "dataplacement": "streams", | |
443 | "output-format": "normal", | |
444 | "plid_select": "random", | |
445 | }, | |
446 | "test_class": StreamsTestRand, | |
447 | "success": SUCCESS_NONZERO, | |
448 | }, | |
449 | # Error case with no stream IDs provided | |
450 | { | |
451 | "test_id": 31, | |
452 | "fio_opts": { | |
453 | "rw": 'write', | |
454 | "bs": 4096, | |
455 | "io_size": 8192, | |
456 | "dataplacement": "streams", | |
457 | "output-format": "normal", | |
458 | }, | |
459 | "test_class": StreamsTestRand, | |
460 | "success": SUCCESS_NONZERO, | |
461 | }, | |
462 | ||
463 | ] | |
464 | ||
465 | def parse_args(): | |
466 | """Parse command-line arguments.""" | |
467 | ||
468 | parser = argparse.ArgumentParser() | |
469 | parser.add_argument('-d', '--debug', help='Enable debug messages', action='store_true') | |
470 | parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)') | |
471 | parser.add_argument('-a', '--artifact-root', help='artifact root directory') | |
472 | parser.add_argument('-s', '--skip', nargs='+', type=int, | |
473 | help='list of test(s) to skip') | |
474 | parser.add_argument('-o', '--run-only', nargs='+', type=int, | |
475 | help='list of test(s) to run, skipping all others') | |
476 | parser.add_argument('--dut', help='target NVMe character device to test ' | |
477 | '(e.g., /dev/ng0n1). WARNING: THIS IS A DESTRUCTIVE TEST', required=True) | |
478 | args = parser.parse_args() | |
479 | ||
480 | return args | |
481 | ||
482 | ||
483 | def main(): | |
484 | """Run tests using fio's io_uring_cmd ioengine to send NVMe pass through commands.""" | |
485 | ||
486 | args = parse_args() | |
487 | ||
488 | if args.debug: | |
489 | logging.basicConfig(level=logging.DEBUG) | |
490 | else: | |
491 | logging.basicConfig(level=logging.INFO) | |
492 | ||
493 | artifact_root = args.artifact_root if args.artifact_root else \ | |
494 | f"nvmept-streams-test-{time.strftime('%Y%m%d-%H%M%S')}" | |
495 | os.mkdir(artifact_root) | |
496 | print(f"Artifact directory is {artifact_root}") | |
497 | ||
498 | if args.fio: | |
499 | fio_path = str(Path(args.fio).absolute()) | |
500 | else: | |
501 | fio_path = 'fio' | |
502 | print(f"fio path is {fio_path}") | |
503 | ||
504 | for test in TEST_LIST: | |
505 | test['fio_opts']['filename'] = args.dut | |
506 | ||
507 | release_all_streams(args.dut) | |
508 | test_env = { | |
509 | 'fio_path': fio_path, | |
510 | 'fio_root': str(Path(__file__).absolute().parent.parent), | |
511 | 'artifact_root': artifact_root, | |
512 | 'basename': 'nvmept-streams', | |
513 | } | |
514 | ||
515 | _, failed, _ = run_fio_tests(TEST_LIST, test_env, args) | |
516 | sys.exit(failed) | |
517 | ||
518 | ||
519 | if __name__ == '__main__': | |
520 | main() |