projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
When verify fails, dump the good/bad blocks to files
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13#include "trim.h"
14#include "lib/rand.h"
15
16#include "crc/md5.h"
17#include "crc/crc64.h"
18#include "crc/crc32.h"
19#include "crc/crc32c.h"
20#include "crc/crc16.h"
21#include "crc/crc7.h"
22#include "crc/sha256.h"
23#include "crc/sha512.h"
24#include "crc/sha1.h"
25
26static void populate_hdr(struct thread_data *td, struct io_u *io_u,
27 struct verify_header *hdr, unsigned int header_num,
28 unsigned int header_len);
29
30void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u, unsigned long seed, int use_seed)
31{
32 switch (td->o.verify_pattern_bytes) {
33 case 0:
34 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
35 if (use_seed)
36 __fill_random_buf(p, len, seed);
37 else
38 io_u->rand_seed = fill_random_buf(p, len);
39 break;
40 case 1:
41 if (io_u->buf_filled_len >= len) {
42 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
43 return;
44 }
45 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
46 memset(p, td->o.verify_pattern[0], len);
47 io_u->buf_filled_len = len;
48 break;
49 default: {
50 unsigned int i = 0, size = 0;
51 unsigned char *b = p;
52
53 if (io_u->buf_filled_len >= len) {
54 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
55 td->o.verify_pattern_bytes, len);
56 return;
57 }
58 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
59 td->o.verify_pattern_bytes, len);
60
61 while (i < len) {
62 size = td->o.verify_pattern_bytes;
63 if (size > (len - i))
64 size = len - i;
65 memcpy(b+i, td->o.verify_pattern, size);
66 i += size;
67 }
68 io_u->buf_filled_len = len;
69 break;
70 }
71 }
72}
73
74static void memswp(void *buf1, void *buf2, unsigned int len)
75{
76 char swap[200];
77
78 assert(len <= sizeof(swap));
79
80 memcpy(&swap, buf1, len);
81 memcpy(buf1, buf2, len);
82 memcpy(buf2, &swap, len);
83}
84
85static void hexdump(void *buffer, int len)
86{
87 unsigned char *p = buffer;
88 int i;
89
90 for (i = 0; i < len; i++)
91 log_err("%02x", p[i]);
92 log_err("\n");
93}
94
95/*
96 * Prepare for seperation of verify_header and checksum header
97 */
98static inline unsigned int __hdr_size(int verify_type)
99{
100 unsigned int len = 0;
101
102 switch (verify_type) {
103 case VERIFY_NONE:
104 case VERIFY_NULL:
105 len = 0;
106 break;
107 case VERIFY_MD5:
108 len = sizeof(struct vhdr_md5);
109 break;
110 case VERIFY_CRC64:
111 len = sizeof(struct vhdr_crc64);
112 break;
113 case VERIFY_CRC32C:
114 case VERIFY_CRC32:
115 case VERIFY_CRC32C_INTEL:
116 len = sizeof(struct vhdr_crc32);
117 break;
118 case VERIFY_CRC16:
119 len = sizeof(struct vhdr_crc16);
120 break;
121 case VERIFY_CRC7:
122 len = sizeof(struct vhdr_crc7);
123 break;
124 case VERIFY_SHA256:
125 len = sizeof(struct vhdr_sha256);
126 break;
127 case VERIFY_SHA512:
128 len = sizeof(struct vhdr_sha512);
129 break;
130 case VERIFY_META:
131 len = sizeof(struct vhdr_meta);
132 break;
133 case VERIFY_SHA1:
134 len = sizeof(struct vhdr_sha1);
135 break;
136 default:
137 log_err("fio: unknown verify header!\n");
138 assert(0);
139 }
140
141 return len + sizeof(struct verify_header);
142}
143
144static inline unsigned int hdr_size(struct verify_header *hdr)
145{
146 return __hdr_size(hdr->verify_type);
147}
148
149static void *hdr_priv(struct verify_header *hdr)
150{
151 void *priv = hdr;
152
153 return priv + sizeof(struct verify_header);
154}
155
156/*
157 * Verify container, pass info to verify handlers and allow them to
158 * pass info back in case of error
159 */
160struct vcont {
161 /*
162 * Input
163 */
164 struct io_u *io_u;
165 unsigned int hdr_num;
166 struct thread_data *td;
167
168 /*
169 * Output, only valid in case of error
170 */
171 const char *name;
172 void *good_crc;
173 void *bad_crc;
174 unsigned int crc_len;
175};
176
177static void dump_buf(char *buf, unsigned int len, const char *name,
178 unsigned long long offset)
179{
180 char fname[80];
181 int ret, fd;
182
183 sprintf(fname, "%llu.%s", offset, name);
184
185 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
186 if (fd < 0) {
187 perror("open verify buf file");
188 return;
189 }
190
191 while (len) {
192 ret = write(fd, buf, len);
193 if (!ret)
194 break;
195 else if (ret < 0) {
196 perror("write verify buf file");
197 break;
198 }
199 len -= ret;
200 buf += ret;
201 }
202
203 close(fd);
204 log_err(" %s data dumped as %s\n", name, fname);
205}
206
207static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
208{
209 struct thread_data *td = vc->td;
210 struct io_u *io_u = vc->io_u;
211 unsigned long hdr_offset;
212 unsigned int hdr_inc, header_num;
213 struct io_u dummy;
214 void *buf, *p;
215
216 hdr_offset = vc->hdr_num * hdr->len;
217
218 dump_buf(io_u->buf + hdr_offset, hdr->len, "received",
219 io_u->offset + hdr_offset);
220
221 buf = p = malloc(io_u->buflen);
222 dummy = *io_u;
223 fill_pattern(td, p, io_u->buflen, &dummy, hdr->rand_seed, 1);
224
225 hdr_inc = io_u->buflen;
226 if (td->o.verify_interval)
227 hdr_inc = td->o.verify_interval;
228
229 header_num = 0;
230 for (; p < buf + io_u->buflen; p += hdr_inc) {
231 hdr = p;
232 populate_hdr(td, io_u, hdr, header_num, hdr_inc);
233 header_num++;
234 }
235
236 dump_buf(buf + hdr_offset, hdr->len, "expected",
237 io_u->offset + hdr_offset);
238 free(buf);
239}
240
241static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
242{
243 unsigned long long offset;
244
245 offset = vc->io_u->offset;
246 offset += vc->hdr_num * hdr->len;
247 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
248 vc->name, vc->io_u->file->file_name, offset, hdr->len);
249
250 if (vc->good_crc && vc->bad_crc) {
251 log_err(" Expected CRC: ");
252 hexdump(vc->good_crc, vc->crc_len);
253 log_err(" Received CRC: ");
254 hexdump(vc->bad_crc, vc->crc_len);
255 }
256
257 dump_verify_buffers(hdr, vc);
258}
259
260/*
261 * Return data area 'header_num'
262 */
263static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
264{
265 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
266}
267
268static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
269 struct vcont *vc)
270{
271 struct vhdr_meta *vh = hdr_priv(hdr);
272 struct io_u *io_u = vc->io_u;
273
274 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
275
276 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
277 return 0;
278
279 vc->name = "meta";
280 log_verify_failure(hdr, vc);
281 return EILSEQ;
282}
283
284static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
285{
286 void *p = io_u_verify_off(hdr, vc);
287 struct vhdr_sha512 *vh = hdr_priv(hdr);
288 uint8_t sha512[128];
289 struct sha512_ctx sha512_ctx = {
290 .buf = sha512,
291 };
292
293 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
294
295 sha512_init(&sha512_ctx);
296 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
297
298 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
299 return 0;
300
301 vc->name = "sha512";
302 vc->good_crc = vh->sha512;
303 vc->bad_crc = sha512_ctx.buf;
304 vc->crc_len = sizeof(vh->sha512);
305 log_verify_failure(hdr, vc);
306 return EILSEQ;
307}
308
309static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
310{
311 void *p = io_u_verify_off(hdr, vc);
312 struct vhdr_sha256 *vh = hdr_priv(hdr);
313 uint8_t sha256[64];
314 struct sha256_ctx sha256_ctx = {
315 .buf = sha256,
316 };
317
318 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
319
320 sha256_init(&sha256_ctx);
321 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
322
323 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
324 return 0;
325
326 vc->name = "sha256";
327 vc->good_crc = vh->sha256;
328 vc->bad_crc = sha256_ctx.buf;
329 vc->crc_len = sizeof(vh->sha256);
330 log_verify_failure(hdr, vc);
331 return EILSEQ;
332}
333
334static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
335{
336 void *p = io_u_verify_off(hdr, vc);
337 struct vhdr_sha1 *vh = hdr_priv(hdr);
338 uint32_t sha1[5];
339 struct sha1_ctx sha1_ctx = {
340 .H = sha1,
341 };
342
343 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
344
345 sha1_init(&sha1_ctx);
346 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
347
348 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
349 return 0;
350
351 vc->name = "sha1";
352 vc->good_crc = vh->sha1;
353 vc->bad_crc = sha1_ctx.H;
354 vc->crc_len = sizeof(vh->sha1);
355 log_verify_failure(hdr, vc);
356 return EILSEQ;
357}
358
359static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
360{
361 void *p = io_u_verify_off(hdr, vc);
362 struct vhdr_crc7 *vh = hdr_priv(hdr);
363 unsigned char c;
364
365 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
366
367 c = crc7(p, hdr->len - hdr_size(hdr));
368
369 if (c == vh->crc7)
370 return 0;
371
372 vc->name = "crc7";
373 vc->good_crc = &vh->crc7;
374 vc->bad_crc = &c;
375 vc->crc_len = 1;
376 log_verify_failure(hdr, vc);
377 return EILSEQ;
378}
379
380static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
381{
382 void *p = io_u_verify_off(hdr, vc);
383 struct vhdr_crc16 *vh = hdr_priv(hdr);
384 unsigned short c;
385
386 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
387
388 c = crc16(p, hdr->len - hdr_size(hdr));
389
390 if (c == vh->crc16)
391 return 0;
392
393 vc->name = "crc16";
394 vc->good_crc = &vh->crc16;
395 vc->bad_crc = &c;
396 vc->crc_len = 2;
397 log_verify_failure(hdr, vc);
398 return EILSEQ;
399}
400
401static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
402{
403 void *p = io_u_verify_off(hdr, vc);
404 struct vhdr_crc64 *vh = hdr_priv(hdr);
405 unsigned long long c;
406
407 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
408
409 c = crc64(p, hdr->len - hdr_size(hdr));
410
411 if (c == vh->crc64)
412 return 0;
413
414 vc->name = "crc64";
415 vc->good_crc = &vh->crc64;
416 vc->bad_crc = &c;
417 vc->crc_len = 8;
418 log_verify_failure(hdr, vc);
419 return EILSEQ;
420}
421
422static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
423{
424 void *p = io_u_verify_off(hdr, vc);
425 struct vhdr_crc32 *vh = hdr_priv(hdr);
426 uint32_t c;
427
428 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
429
430 c = crc32(p, hdr->len - hdr_size(hdr));
431
432 if (c == vh->crc32)
433 return 0;
434
435 vc->name = "crc32";
436 vc->good_crc = &vh->crc32;
437 vc->bad_crc = &c;
438 vc->crc_len = 4;
439 log_verify_failure(hdr, vc);
440 return EILSEQ;
441}
442
443static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
444{
445 void *p = io_u_verify_off(hdr, vc);
446 struct vhdr_crc32 *vh = hdr_priv(hdr);
447 uint32_t c;
448
449 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
450
451 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
452 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
453 else
454 c = crc32c(p, hdr->len - hdr_size(hdr));
455
456 if (c == vh->crc32)
457 return 0;
458
459 vc->name = "crc32c";
460 vc->good_crc = &vh->crc32;
461 vc->bad_crc = &c;
462 vc->crc_len = 4;
463 log_verify_failure(hdr, vc);
464 return EILSEQ;
465}
466
467static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
468{
469 void *p = io_u_verify_off(hdr, vc);
470 struct vhdr_md5 *vh = hdr_priv(hdr);
471 uint32_t hash[MD5_HASH_WORDS];
472 struct md5_ctx md5_ctx = {
473 .hash = hash,
474 };
475
476 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
477
478 md5_init(&md5_ctx);
479 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
480
481 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
482 return 0;
483
484 vc->name = "md5";
485 vc->good_crc = vh->md5_digest;
486 vc->bad_crc = md5_ctx.hash;
487 vc->crc_len = sizeof(hash);
488 log_verify_failure(hdr, vc);
489 return EILSEQ;
490}
491
492static unsigned int hweight8(unsigned int w)
493{
494 unsigned int res = w - ((w >> 1) & 0x55);
495
496 res = (res & 0x33) + ((res >> 2) & 0x33);
497 return (res + (res >> 4)) & 0x0F;
498}
499
500int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
501 char *buf, unsigned int len, unsigned int mod)
502{
503 unsigned int i;
504
505 for (i = 0; i < len; i++) {
506 if (buf[i] != pattern[mod]) {
507 unsigned int bits;
508
509 bits = hweight8(buf[i] ^ pattern[mod]);
510 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
511 buf[i], pattern[mod], bits);
512 log_err("fio: bad pattern block offset %u\n", i);
513 return EILSEQ;
514 }
515 mod++;
516 if (mod == pattern_size)
517 mod = 0;
518 }
519
520 return 0;
521}
522
523/*
524 * Push IO verification to a separate thread
525 */
526int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
527{
528 if (io_u->file)
529 put_file_log(td, io_u->file);
530
531 io_u->file = NULL;
532
533 pthread_mutex_lock(&td->io_u_lock);
534
535 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
536 td->cur_depth--;
537 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
538 }
539 flist_del(&io_u->list);
540 flist_add_tail(&io_u->list, &td->verify_list);
541 io_u->flags |= IO_U_F_FREE_DEF;
542 pthread_mutex_unlock(&td->io_u_lock);
543
544 pthread_cond_signal(&td->verify_cond);
545 return 0;
546}
547
548static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
549{
550 static char zero_buf[1024];
551 unsigned int this_len, len;
552 int ret = 0;
553 void *p;
554
555 if (!td->o.trim_zero)
556 return 0;
557
558 len = io_u->buflen;
559 p = io_u->buf;
560 do {
561 this_len = sizeof(zero_buf);
562 if (this_len > len)
563 this_len = len;
564 if (memcmp(p, zero_buf, this_len)) {
565 ret = EILSEQ;
566 break;
567 }
568 len -= this_len;
569 p += this_len;
570 } while (len);
571
572 if (!ret)
573 return 0;
574
575 log_err("trim: verify failed at file %s offset %llu, length %lu"
576 ", block offset %lu\n",
577 io_u->file->file_name, io_u->offset, io_u->buflen,
578 (unsigned long) (p - io_u->buf));
579 return ret;
580}
581
582int verify_io_u(struct thread_data *td, struct io_u *io_u)
583{
584 struct verify_header *hdr;
585 unsigned int hdr_size, hdr_inc, hdr_num = 0;
586 void *p;
587 int ret;
588
589 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
590 return 0;
591 if (io_u->flags & IO_U_F_TRIMMED) {
592 ret = verify_trimmed_io_u(td, io_u);
593 goto done;
594 }
595
596 hdr_inc = io_u->buflen;
597 if (td->o.verify_interval)
598 hdr_inc = td->o.verify_interval;
599
600 ret = 0;
601 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
602 p += hdr_inc, hdr_num++) {
603 struct vcont vc = {
604 .io_u = io_u,
605 .hdr_num = hdr_num,
606 .td = td,
607 };
608
609 if (ret && td->o.verify_fatal)
610 break;
611
612 hdr_size = __hdr_size(td->o.verify);
613 if (td->o.verify_offset)
614 memswp(p, p + td->o.verify_offset, hdr_size);
615 hdr = p;
616
617 if (hdr->fio_magic != FIO_HDR_MAGIC) {
618 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
619 hdr->fio_magic, FIO_HDR_MAGIC,
620 io_u->file->file_name,
621 io_u->offset + hdr_num * hdr->len, hdr->len);
622 return EILSEQ;
623 }
624
625 if (td->o.verify_pattern_bytes) {
626 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
627 io_u, hdr->len);
628 ret = verify_io_u_pattern(td->o.verify_pattern,
629 td->o.verify_pattern_bytes,
630 p + hdr_size,
631 hdr_inc - hdr_size,
632 hdr_size % td->o.verify_pattern_bytes);
633
634 if (ret) {
635 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
636 io_u->file->file_name,
637 io_u->offset + hdr_num * hdr->len,
638 hdr->len);
639 }
640
641 /*
642 * Also verify the meta data, if applicable
643 */
644 if (hdr->verify_type == VERIFY_META)
645 ret |= verify_io_u_meta(hdr, td, &vc);
646 continue;
647 }
648
649 switch (hdr->verify_type) {
650 case VERIFY_MD5:
651 ret = verify_io_u_md5(hdr, &vc);
652 break;
653 case VERIFY_CRC64:
654 ret = verify_io_u_crc64(hdr, &vc);
655 break;
656 case VERIFY_CRC32C:
657 case VERIFY_CRC32C_INTEL:
658 ret = verify_io_u_crc32c(hdr, &vc);
659 break;
660 case VERIFY_CRC32:
661 ret = verify_io_u_crc32(hdr, &vc);
662 break;
663 case VERIFY_CRC16:
664 ret = verify_io_u_crc16(hdr, &vc);
665 break;
666 case VERIFY_CRC7:
667 ret = verify_io_u_crc7(hdr, &vc);
668 break;
669 case VERIFY_SHA256:
670 ret = verify_io_u_sha256(hdr, &vc);
671 break;
672 case VERIFY_SHA512:
673 ret = verify_io_u_sha512(hdr, &vc);
674 break;
675 case VERIFY_META:
676 ret = verify_io_u_meta(hdr, td, &vc);
677 break;
678 case VERIFY_SHA1:
679 ret = verify_io_u_sha1(hdr, &vc);
680 break;
681 default:
682 log_err("Bad verify type %u\n", hdr->verify_type);
683 ret = EINVAL;
684 }
685 }
686
687done:
688 if (ret && td->o.verify_fatal)
689 td->terminate = 1;
690
691 return ret;
692}
693
694static void fill_meta(struct verify_header *hdr, struct thread_data *td,
695 struct io_u *io_u, unsigned int header_num)
696{
697 struct vhdr_meta *vh = hdr_priv(hdr);
698
699 vh->thread = td->thread_number;
700
701 vh->time_sec = io_u->start_time.tv_sec;
702 vh->time_usec = io_u->start_time.tv_usec;
703
704 vh->numberio = td->io_issues[DDIR_WRITE];
705
706 vh->offset = io_u->offset + header_num * td->o.verify_interval;
707}
708
709static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
710{
711 struct vhdr_sha512 *vh = hdr_priv(hdr);
712 struct sha512_ctx sha512_ctx = {
713 .buf = vh->sha512,
714 };
715
716 sha512_init(&sha512_ctx);
717 sha512_update(&sha512_ctx, p, len);
718}
719
720static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
721{
722 struct vhdr_sha256 *vh = hdr_priv(hdr);
723 struct sha256_ctx sha256_ctx = {
724 .buf = vh->sha256,
725 };
726
727 sha256_init(&sha256_ctx);
728 sha256_update(&sha256_ctx, p, len);
729}
730
731static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
732{
733 struct vhdr_sha1 *vh = hdr_priv(hdr);
734 struct sha1_ctx sha1_ctx = {
735 .H = vh->sha1,
736 };
737
738 sha1_init(&sha1_ctx);
739 sha1_update(&sha1_ctx, p, len);
740}
741
742static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
743{
744 struct vhdr_crc7 *vh = hdr_priv(hdr);
745
746 vh->crc7 = crc7(p, len);
747}
748
749static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
750{
751 struct vhdr_crc16 *vh = hdr_priv(hdr);
752
753 vh->crc16 = crc16(p, len);
754}
755
756static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
757{
758 struct vhdr_crc32 *vh = hdr_priv(hdr);
759
760 vh->crc32 = crc32(p, len);
761}
762
763static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
764{
765 struct vhdr_crc32 *vh = hdr_priv(hdr);
766
767 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
768 vh->crc32 = crc32c_intel(p, len);
769 else
770 vh->crc32 = crc32c(p, len);
771}
772
773static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
774{
775 struct vhdr_crc64 *vh = hdr_priv(hdr);
776
777 vh->crc64 = crc64(p, len);
778}
779
780static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
781{
782 struct vhdr_md5 *vh = hdr_priv(hdr);
783 struct md5_ctx md5_ctx = {
784 .hash = (uint32_t *) vh->md5_digest,
785 };
786
787 md5_init(&md5_ctx);
788 md5_update(&md5_ctx, p, len);
789}
790
791static void populate_hdr(struct thread_data *td, struct io_u *io_u,
792 struct verify_header *hdr, unsigned int header_num,
793 unsigned int header_len)
794{
795 unsigned int data_len;
796 void *data, *p;
797
798 p = (void *) hdr;
799
800 hdr->fio_magic = FIO_HDR_MAGIC;
801 hdr->len = header_len;
802 hdr->verify_type = td->o.verify;
803 hdr->rand_seed = io_u->rand_seed;
804 data_len = header_len - hdr_size(hdr);
805
806 data = p + hdr_size(hdr);
807 switch (td->o.verify) {
808 case VERIFY_MD5:
809 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
810 io_u, hdr->len);
811 fill_md5(hdr, data, data_len);
812 break;
813 case VERIFY_CRC64:
814 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
815 io_u, hdr->len);
816 fill_crc64(hdr, data, data_len);
817 break;
818 case VERIFY_CRC32C:
819 case VERIFY_CRC32C_INTEL:
820 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
821 io_u, hdr->len);
822 fill_crc32c(hdr, data, data_len);
823 break;
824 case VERIFY_CRC32:
825 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
826 io_u, hdr->len);
827 fill_crc32(hdr, data, data_len);
828 break;
829 case VERIFY_CRC16:
830 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
831 io_u, hdr->len);
832 fill_crc16(hdr, data, data_len);
833 break;
834 case VERIFY_CRC7:
835 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
836 io_u, hdr->len);
837 fill_crc7(hdr, data, data_len);
838 break;
839 case VERIFY_SHA256:
840 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
841 io_u, hdr->len);
842 fill_sha256(hdr, data, data_len);
843 break;
844 case VERIFY_SHA512:
845 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
846 io_u, hdr->len);
847 fill_sha512(hdr, data, data_len);
848 break;
849 case VERIFY_META:
850 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
851 io_u, hdr->len);
852 fill_meta(hdr, td, io_u, header_num);
853 break;
854 case VERIFY_SHA1:
855 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
856 io_u, hdr->len);
857 fill_sha1(hdr, data, data_len);
858 break;
859 default:
860 log_err("fio: bad verify type: %d\n", td->o.verify);
861 assert(0);
862 }
863 if (td->o.verify_offset)
864 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
865}
866
867/*
868 * fill body of io_u->buf with random data and add a header with the
869 * crc32 or md5 sum of that data.
870 */
871void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
872{
873 struct verify_header *hdr;
874 unsigned int hdr_inc, header_num = 0;
875 void *p = io_u->buf;
876
877 if (td->o.verify == VERIFY_NULL)
878 return;
879
880 fill_pattern(td, p, io_u->buflen, io_u, 0, 0);
881
882 hdr_inc = io_u->buflen;
883 if (td->o.verify_interval)
884 hdr_inc = td->o.verify_interval;
885
886 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
887 hdr = p;
888 populate_hdr(td, io_u, hdr, header_num, hdr_inc);
889 header_num++;
890 }
891}
892
893int get_next_verify(struct thread_data *td, struct io_u *io_u)
894{
895 struct io_piece *ipo = NULL;
896
897 /*
898 * this io_u is from a requeue, we already filled the offsets
899 */
900 if (io_u->file)
901 return 0;
902
903 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
904 struct rb_node *n = rb_first(&td->io_hist_tree);
905
906 ipo = rb_entry(n, struct io_piece, rb_node);
907 rb_erase(n, &td->io_hist_tree);
908 assert(ipo->flags & IP_F_ONRB);
909 ipo->flags &= ~IP_F_ONRB;
910 } else if (!flist_empty(&td->io_hist_list)) {
911 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
912 flist_del(&ipo->list);
913 assert(ipo->flags & IP_F_ONLIST);
914 ipo->flags &= ~IP_F_ONLIST;
915 }
916
917 if (ipo) {
918 td->io_hist_len--;
919
920 io_u->offset = ipo->offset;
921 io_u->buflen = ipo->len;
922 io_u->file = ipo->file;
923
924 if (ipo->flags & IP_F_TRIMMED)
925 io_u->flags |= IO_U_F_TRIMMED;
926
927 if (!fio_file_open(io_u->file)) {
928 int r = td_io_open_file(td, io_u->file);
929
930 if (r) {
931 dprint(FD_VERIFY, "failed file %s open\n",
932 io_u->file->file_name);
933 return 1;
934 }
935 }
936
937 get_file(ipo->file);
938 assert(fio_file_open(io_u->file));
939 io_u->ddir = DDIR_READ;
940 io_u->xfer_buf = io_u->buf;
941 io_u->xfer_buflen = io_u->buflen;
942
943 remove_trim_entry(td, ipo);
944 free(ipo);
945 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
946 return 0;
947 }
948
949 dprint(FD_VERIFY, "get_next_verify: empty\n");
950 return 1;
951}
952
953static void *verify_async_thread(void *data)
954{
955 struct thread_data *td = data;
956 struct io_u *io_u;
957 int ret = 0;
958
959 if (td->o.verify_cpumask_set &&
960 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
961 log_err("fio: failed setting verify thread affinity\n");
962 goto done;
963 }
964
965 do {
966 FLIST_HEAD(list);
967
968 read_barrier();
969 if (td->verify_thread_exit)
970 break;
971
972 pthread_mutex_lock(&td->io_u_lock);
973
974 while (flist_empty(&td->verify_list) &&
975 !td->verify_thread_exit) {
976 ret = pthread_cond_wait(&td->verify_cond,
977 &td->io_u_lock);
978 if (ret) {
979 pthread_mutex_unlock(&td->io_u_lock);
980 break;
981 }
982 }
983
984 flist_splice_init(&td->verify_list, &list);
985 pthread_mutex_unlock(&td->io_u_lock);
986
987 if (flist_empty(&list))
988 continue;
989
990 while (!flist_empty(&list)) {
991 io_u = flist_entry(list.next, struct io_u, list);
992 flist_del_init(&io_u->list);
993
994 ret = verify_io_u(td, io_u);
995 put_io_u(td, io_u);
996 if (!ret)
997 continue;
998 if (td->o.continue_on_error &&
999 td_non_fatal_error(ret)) {
1000 update_error_count(td, ret);
1001 td_clear_error(td);
1002 ret = 0;
1003 }
1004 }
1005 } while (!ret);
1006
1007 if (ret) {
1008 td_verror(td, ret, "async_verify");
1009 if (td->o.verify_fatal)
1010 td->terminate = 1;
1011 }
1012
1013done:
1014 pthread_mutex_lock(&td->io_u_lock);
1015 td->nr_verify_threads--;
1016 pthread_mutex_unlock(&td->io_u_lock);
1017
1018 pthread_cond_signal(&td->free_cond);
1019 return NULL;
1020}
1021
1022int verify_async_init(struct thread_data *td)
1023{
1024 int i, ret;
1025 pthread_attr_t attr;
1026
1027 pthread_attr_init(&attr);
1028 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
1029
1030 td->verify_thread_exit = 0;
1031
1032 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
1033 for (i = 0; i < td->o.verify_async; i++) {
1034 ret = pthread_create(&td->verify_threads[i], &attr,
1035 verify_async_thread, td);
1036 if (ret) {
1037 log_err("fio: async verify creation failed: %s\n",
1038 strerror(ret));
1039 break;
1040 }
1041 ret = pthread_detach(td->verify_threads[i]);
1042 if (ret) {
1043 log_err("fio: async verify thread detach failed: %s\n",
1044 strerror(ret));
1045 break;
1046 }
1047 td->nr_verify_threads++;
1048 }
1049
1050 pthread_attr_destroy(&attr);
1051
1052 if (i != td->o.verify_async) {
1053 log_err("fio: only %d verify threads started, exiting\n", i);
1054 td->verify_thread_exit = 1;
1055 write_barrier();
1056 pthread_cond_broadcast(&td->verify_cond);
1057 return 1;
1058 }
1059
1060 return 0;
1061}
1062
1063void verify_async_exit(struct thread_data *td)
1064{
1065 td->verify_thread_exit = 1;
1066 write_barrier();
1067 pthread_cond_broadcast(&td->verify_cond);
1068
1069 pthread_mutex_lock(&td->io_u_lock);
1070
1071 while (td->nr_verify_threads)
1072 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
1073
1074 pthread_mutex_unlock(&td->io_u_lock);
1075 free(td->verify_threads);
1076 td->verify_threads = NULL;
1077}
fio - Flexible IO Tester