projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Pretty up the good/bad block verify dumping
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13#include "trim.h"
14#include "lib/rand.h"
15
16#include "crc/md5.h"
17#include "crc/crc64.h"
18#include "crc/crc32.h"
19#include "crc/crc32c.h"
20#include "crc/crc16.h"
21#include "crc/crc7.h"
22#include "crc/sha256.h"
23#include "crc/sha512.h"
24#include "crc/sha1.h"
25
26static void populate_hdr(struct thread_data *td, struct io_u *io_u,
27 struct verify_header *hdr, unsigned int header_num,
28 unsigned int header_len);
29
30void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u, unsigned long seed, int use_seed)
31{
32 switch (td->o.verify_pattern_bytes) {
33 case 0:
34 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
35 if (use_seed)
36 __fill_random_buf(p, len, seed);
37 else
38 io_u->rand_seed = fill_random_buf(p, len);
39 break;
40 case 1:
41 if (io_u->buf_filled_len >= len) {
42 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
43 return;
44 }
45 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
46 memset(p, td->o.verify_pattern[0], len);
47 io_u->buf_filled_len = len;
48 break;
49 default: {
50 unsigned int i = 0, size = 0;
51 unsigned char *b = p;
52
53 if (io_u->buf_filled_len >= len) {
54 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
55 td->o.verify_pattern_bytes, len);
56 return;
57 }
58 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
59 td->o.verify_pattern_bytes, len);
60
61 while (i < len) {
62 size = td->o.verify_pattern_bytes;
63 if (size > (len - i))
64 size = len - i;
65 memcpy(b+i, td->o.verify_pattern, size);
66 i += size;
67 }
68 io_u->buf_filled_len = len;
69 break;
70 }
71 }
72}
73
74static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u,
75 unsigned long seed, int use_seed)
76{
77 unsigned int hdr_inc, header_num;
78 struct verify_header *hdr;
79 void *p = io_u->buf;
80
81 fill_pattern(td, p, io_u->buflen, io_u, seed, use_seed);
82
83 hdr_inc = io_u->buflen;
84 if (td->o.verify_interval)
85 hdr_inc = td->o.verify_interval;
86
87 header_num = 0;
88 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
89 hdr = p;
90 populate_hdr(td, io_u, hdr, header_num, hdr_inc);
91 header_num++;
92 }
93}
94
95static void memswp(void *buf1, void *buf2, unsigned int len)
96{
97 char swap[200];
98
99 assert(len <= sizeof(swap));
100
101 memcpy(&swap, buf1, len);
102 memcpy(buf1, buf2, len);
103 memcpy(buf2, &swap, len);
104}
105
106static void hexdump(void *buffer, int len)
107{
108 unsigned char *p = buffer;
109 int i;
110
111 for (i = 0; i < len; i++)
112 log_err("%02x", p[i]);
113 log_err("\n");
114}
115
116/*
117 * Prepare for seperation of verify_header and checksum header
118 */
119static inline unsigned int __hdr_size(int verify_type)
120{
121 unsigned int len = 0;
122
123 switch (verify_type) {
124 case VERIFY_NONE:
125 case VERIFY_NULL:
126 len = 0;
127 break;
128 case VERIFY_MD5:
129 len = sizeof(struct vhdr_md5);
130 break;
131 case VERIFY_CRC64:
132 len = sizeof(struct vhdr_crc64);
133 break;
134 case VERIFY_CRC32C:
135 case VERIFY_CRC32:
136 case VERIFY_CRC32C_INTEL:
137 len = sizeof(struct vhdr_crc32);
138 break;
139 case VERIFY_CRC16:
140 len = sizeof(struct vhdr_crc16);
141 break;
142 case VERIFY_CRC7:
143 len = sizeof(struct vhdr_crc7);
144 break;
145 case VERIFY_SHA256:
146 len = sizeof(struct vhdr_sha256);
147 break;
148 case VERIFY_SHA512:
149 len = sizeof(struct vhdr_sha512);
150 break;
151 case VERIFY_META:
152 len = sizeof(struct vhdr_meta);
153 break;
154 case VERIFY_SHA1:
155 len = sizeof(struct vhdr_sha1);
156 break;
157 default:
158 log_err("fio: unknown verify header!\n");
159 assert(0);
160 }
161
162 return len + sizeof(struct verify_header);
163}
164
165static inline unsigned int hdr_size(struct verify_header *hdr)
166{
167 return __hdr_size(hdr->verify_type);
168}
169
170static void *hdr_priv(struct verify_header *hdr)
171{
172 void *priv = hdr;
173
174 return priv + sizeof(struct verify_header);
175}
176
177/*
178 * Verify container, pass info to verify handlers and allow them to
179 * pass info back in case of error
180 */
181struct vcont {
182 /*
183 * Input
184 */
185 struct io_u *io_u;
186 unsigned int hdr_num;
187 struct thread_data *td;
188
189 /*
190 * Output, only valid in case of error
191 */
192 const char *name;
193 void *good_crc;
194 void *bad_crc;
195 unsigned int crc_len;
196};
197
198static void dump_buf(char *buf, unsigned int len, unsigned long long offset,
199 const char *type, struct fio_file *f)
200{
201 char fname[256];
202 int ret, fd;
203
204 strcpy(fname, f->file_name);
205 basename(fname);
206
207 sprintf(fname + strlen(fname), ".%llu.%s", offset, type);
208
209 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
210 if (fd < 0) {
211 perror("open verify buf file");
212 return;
213 }
214
215 while (len) {
216 ret = write(fd, buf, len);
217 if (!ret)
218 break;
219 else if (ret < 0) {
220 perror("write verify buf file");
221 break;
222 }
223 len -= ret;
224 buf += ret;
225 }
226
227 close(fd);
228 log_err(" %s data dumped as %s\n", type, fname);
229}
230
231/*
232 * Dump the contents of the read block and re-generate the correct data
233 * and dump that too.
234 */
235static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
236{
237 struct thread_data *td = vc->td;
238 struct io_u *io_u = vc->io_u;
239 unsigned long hdr_offset;
240 struct io_u dummy;
241 void *buf;
242
243 /*
244 * Dump the contents we just read off disk
245 */
246 hdr_offset = vc->hdr_num * hdr->len;
247
248 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
249 "received", vc->io_u->file);
250
251 /*
252 * Allocate a new buf and re-generate the original data
253 */
254 buf = malloc(io_u->buflen);
255 dummy = *io_u;
256 dummy.buf = buf;
257
258 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1);
259
260 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
261 "expected", vc->io_u->file);
262 free(buf);
263}
264
265static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
266{
267 unsigned long long offset;
268
269 offset = vc->io_u->offset;
270 offset += vc->hdr_num * hdr->len;
271 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
272 vc->name, vc->io_u->file->file_name, offset, hdr->len);
273
274 if (vc->good_crc && vc->bad_crc) {
275 log_err(" Expected CRC: ");
276 hexdump(vc->good_crc, vc->crc_len);
277 log_err(" Received CRC: ");
278 hexdump(vc->bad_crc, vc->crc_len);
279 }
280
281 dump_verify_buffers(hdr, vc);
282}
283
284/*
285 * Return data area 'header_num'
286 */
287static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
288{
289 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
290}
291
292static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
293 struct vcont *vc)
294{
295 struct vhdr_meta *vh = hdr_priv(hdr);
296 struct io_u *io_u = vc->io_u;
297
298 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
299
300 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
301 return 0;
302
303 vc->name = "meta";
304 log_verify_failure(hdr, vc);
305 return EILSEQ;
306}
307
308static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
309{
310 void *p = io_u_verify_off(hdr, vc);
311 struct vhdr_sha512 *vh = hdr_priv(hdr);
312 uint8_t sha512[128];
313 struct sha512_ctx sha512_ctx = {
314 .buf = sha512,
315 };
316
317 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
318
319 sha512_init(&sha512_ctx);
320 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
321
322 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
323 return 0;
324
325 vc->name = "sha512";
326 vc->good_crc = vh->sha512;
327 vc->bad_crc = sha512_ctx.buf;
328 vc->crc_len = sizeof(vh->sha512);
329 log_verify_failure(hdr, vc);
330 return EILSEQ;
331}
332
333static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
334{
335 void *p = io_u_verify_off(hdr, vc);
336 struct vhdr_sha256 *vh = hdr_priv(hdr);
337 uint8_t sha256[64];
338 struct sha256_ctx sha256_ctx = {
339 .buf = sha256,
340 };
341
342 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
343
344 sha256_init(&sha256_ctx);
345 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
346
347 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
348 return 0;
349
350 vc->name = "sha256";
351 vc->good_crc = vh->sha256;
352 vc->bad_crc = sha256_ctx.buf;
353 vc->crc_len = sizeof(vh->sha256);
354 log_verify_failure(hdr, vc);
355 return EILSEQ;
356}
357
358static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
359{
360 void *p = io_u_verify_off(hdr, vc);
361 struct vhdr_sha1 *vh = hdr_priv(hdr);
362 uint32_t sha1[5];
363 struct sha1_ctx sha1_ctx = {
364 .H = sha1,
365 };
366
367 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
368
369 sha1_init(&sha1_ctx);
370 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
371
372 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
373 return 0;
374
375 vc->name = "sha1";
376 vc->good_crc = vh->sha1;
377 vc->bad_crc = sha1_ctx.H;
378 vc->crc_len = sizeof(vh->sha1);
379 log_verify_failure(hdr, vc);
380 return EILSEQ;
381}
382
383static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
384{
385 void *p = io_u_verify_off(hdr, vc);
386 struct vhdr_crc7 *vh = hdr_priv(hdr);
387 unsigned char c;
388
389 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
390
391 c = crc7(p, hdr->len - hdr_size(hdr));
392
393 if (c == vh->crc7)
394 return 0;
395
396 vc->name = "crc7";
397 vc->good_crc = &vh->crc7;
398 vc->bad_crc = &c;
399 vc->crc_len = 1;
400 log_verify_failure(hdr, vc);
401 return EILSEQ;
402}
403
404static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
405{
406 void *p = io_u_verify_off(hdr, vc);
407 struct vhdr_crc16 *vh = hdr_priv(hdr);
408 unsigned short c;
409
410 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
411
412 c = crc16(p, hdr->len - hdr_size(hdr));
413
414 if (c == vh->crc16)
415 return 0;
416
417 vc->name = "crc16";
418 vc->good_crc = &vh->crc16;
419 vc->bad_crc = &c;
420 vc->crc_len = 2;
421 log_verify_failure(hdr, vc);
422 return EILSEQ;
423}
424
425static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
426{
427 void *p = io_u_verify_off(hdr, vc);
428 struct vhdr_crc64 *vh = hdr_priv(hdr);
429 unsigned long long c;
430
431 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
432
433 c = crc64(p, hdr->len - hdr_size(hdr));
434
435 if (c == vh->crc64)
436 return 0;
437
438 vc->name = "crc64";
439 vc->good_crc = &vh->crc64;
440 vc->bad_crc = &c;
441 vc->crc_len = 8;
442 log_verify_failure(hdr, vc);
443 return EILSEQ;
444}
445
446static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
447{
448 void *p = io_u_verify_off(hdr, vc);
449 struct vhdr_crc32 *vh = hdr_priv(hdr);
450 uint32_t c;
451
452 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
453
454 c = crc32(p, hdr->len - hdr_size(hdr));
455
456 if (c == vh->crc32)
457 return 0;
458
459 vc->name = "crc32";
460 vc->good_crc = &vh->crc32;
461 vc->bad_crc = &c;
462 vc->crc_len = 4;
463 log_verify_failure(hdr, vc);
464 return EILSEQ;
465}
466
467static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
468{
469 void *p = io_u_verify_off(hdr, vc);
470 struct vhdr_crc32 *vh = hdr_priv(hdr);
471 uint32_t c;
472
473 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
474
475 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
476 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
477 else
478 c = crc32c(p, hdr->len - hdr_size(hdr));
479
480 if (c == vh->crc32)
481 return 0;
482
483 vc->name = "crc32c";
484 vc->good_crc = &vh->crc32;
485 vc->bad_crc = &c;
486 vc->crc_len = 4;
487 log_verify_failure(hdr, vc);
488 return EILSEQ;
489}
490
491static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
492{
493 void *p = io_u_verify_off(hdr, vc);
494 struct vhdr_md5 *vh = hdr_priv(hdr);
495 uint32_t hash[MD5_HASH_WORDS];
496 struct md5_ctx md5_ctx = {
497 .hash = hash,
498 };
499
500 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
501
502 md5_init(&md5_ctx);
503 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
504
505 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
506 return 0;
507
508 vc->name = "md5";
509 vc->good_crc = vh->md5_digest;
510 vc->bad_crc = md5_ctx.hash;
511 vc->crc_len = sizeof(hash);
512 log_verify_failure(hdr, vc);
513 return EILSEQ;
514}
515
516static unsigned int hweight8(unsigned int w)
517{
518 unsigned int res = w - ((w >> 1) & 0x55);
519
520 res = (res & 0x33) + ((res >> 2) & 0x33);
521 return (res + (res >> 4)) & 0x0F;
522}
523
524int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
525 char *buf, unsigned int len, unsigned int mod)
526{
527 unsigned int i;
528
529 for (i = 0; i < len; i++) {
530 if (buf[i] != pattern[mod]) {
531 unsigned int bits;
532
533 bits = hweight8(buf[i] ^ pattern[mod]);
534 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
535 buf[i], pattern[mod], bits);
536 log_err("fio: bad pattern block offset %u\n", i);
537 return EILSEQ;
538 }
539 mod++;
540 if (mod == pattern_size)
541 mod = 0;
542 }
543
544 return 0;
545}
546
547/*
548 * Push IO verification to a separate thread
549 */
550int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
551{
552 if (io_u->file)
553 put_file_log(td, io_u->file);
554
555 io_u->file = NULL;
556
557 pthread_mutex_lock(&td->io_u_lock);
558
559 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
560 td->cur_depth--;
561 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
562 }
563 flist_del(&io_u->list);
564 flist_add_tail(&io_u->list, &td->verify_list);
565 io_u->flags |= IO_U_F_FREE_DEF;
566 pthread_mutex_unlock(&td->io_u_lock);
567
568 pthread_cond_signal(&td->verify_cond);
569 return 0;
570}
571
572static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
573{
574 static char zero_buf[1024];
575 unsigned int this_len, len;
576 int ret = 0;
577 void *p;
578
579 if (!td->o.trim_zero)
580 return 0;
581
582 len = io_u->buflen;
583 p = io_u->buf;
584 do {
585 this_len = sizeof(zero_buf);
586 if (this_len > len)
587 this_len = len;
588 if (memcmp(p, zero_buf, this_len)) {
589 ret = EILSEQ;
590 break;
591 }
592 len -= this_len;
593 p += this_len;
594 } while (len);
595
596 if (!ret)
597 return 0;
598
599 log_err("trim: verify failed at file %s offset %llu, length %lu"
600 ", block offset %lu\n",
601 io_u->file->file_name, io_u->offset, io_u->buflen,
602 (unsigned long) (p - io_u->buf));
603 return ret;
604}
605
606int verify_io_u(struct thread_data *td, struct io_u *io_u)
607{
608 struct verify_header *hdr;
609 unsigned int hdr_size, hdr_inc, hdr_num = 0;
610 void *p;
611 int ret;
612
613 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
614 return 0;
615 if (io_u->flags & IO_U_F_TRIMMED) {
616 ret = verify_trimmed_io_u(td, io_u);
617 goto done;
618 }
619
620 hdr_inc = io_u->buflen;
621 if (td->o.verify_interval)
622 hdr_inc = td->o.verify_interval;
623
624 ret = 0;
625 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
626 p += hdr_inc, hdr_num++) {
627 struct vcont vc = {
628 .io_u = io_u,
629 .hdr_num = hdr_num,
630 .td = td,
631 };
632
633 if (ret && td->o.verify_fatal)
634 break;
635
636 hdr_size = __hdr_size(td->o.verify);
637 if (td->o.verify_offset)
638 memswp(p, p + td->o.verify_offset, hdr_size);
639 hdr = p;
640
641 if (hdr->fio_magic != FIO_HDR_MAGIC) {
642 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
643 hdr->fio_magic, FIO_HDR_MAGIC,
644 io_u->file->file_name,
645 io_u->offset + hdr_num * hdr->len, hdr->len);
646 return EILSEQ;
647 }
648
649 if (td->o.verify_pattern_bytes) {
650 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
651 io_u, hdr->len);
652 ret = verify_io_u_pattern(td->o.verify_pattern,
653 td->o.verify_pattern_bytes,
654 p + hdr_size,
655 hdr_inc - hdr_size,
656 hdr_size % td->o.verify_pattern_bytes);
657
658 if (ret) {
659 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
660 io_u->file->file_name,
661 io_u->offset + hdr_num * hdr->len,
662 hdr->len);
663 }
664
665 /*
666 * Also verify the meta data, if applicable
667 */
668 if (hdr->verify_type == VERIFY_META)
669 ret |= verify_io_u_meta(hdr, td, &vc);
670 continue;
671 }
672
673 switch (hdr->verify_type) {
674 case VERIFY_MD5:
675 ret = verify_io_u_md5(hdr, &vc);
676 break;
677 case VERIFY_CRC64:
678 ret = verify_io_u_crc64(hdr, &vc);
679 break;
680 case VERIFY_CRC32C:
681 case VERIFY_CRC32C_INTEL:
682 ret = verify_io_u_crc32c(hdr, &vc);
683 break;
684 case VERIFY_CRC32:
685 ret = verify_io_u_crc32(hdr, &vc);
686 break;
687 case VERIFY_CRC16:
688 ret = verify_io_u_crc16(hdr, &vc);
689 break;
690 case VERIFY_CRC7:
691 ret = verify_io_u_crc7(hdr, &vc);
692 break;
693 case VERIFY_SHA256:
694 ret = verify_io_u_sha256(hdr, &vc);
695 break;
696 case VERIFY_SHA512:
697 ret = verify_io_u_sha512(hdr, &vc);
698 break;
699 case VERIFY_META:
700 ret = verify_io_u_meta(hdr, td, &vc);
701 break;
702 case VERIFY_SHA1:
703 ret = verify_io_u_sha1(hdr, &vc);
704 break;
705 default:
706 log_err("Bad verify type %u\n", hdr->verify_type);
707 ret = EINVAL;
708 }
709 }
710
711done:
712 if (ret && td->o.verify_fatal)
713 td->terminate = 1;
714
715 return ret;
716}
717
718static void fill_meta(struct verify_header *hdr, struct thread_data *td,
719 struct io_u *io_u, unsigned int header_num)
720{
721 struct vhdr_meta *vh = hdr_priv(hdr);
722
723 vh->thread = td->thread_number;
724
725 vh->time_sec = io_u->start_time.tv_sec;
726 vh->time_usec = io_u->start_time.tv_usec;
727
728 vh->numberio = td->io_issues[DDIR_WRITE];
729
730 vh->offset = io_u->offset + header_num * td->o.verify_interval;
731}
732
733static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
734{
735 struct vhdr_sha512 *vh = hdr_priv(hdr);
736 struct sha512_ctx sha512_ctx = {
737 .buf = vh->sha512,
738 };
739
740 sha512_init(&sha512_ctx);
741 sha512_update(&sha512_ctx, p, len);
742}
743
744static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
745{
746 struct vhdr_sha256 *vh = hdr_priv(hdr);
747 struct sha256_ctx sha256_ctx = {
748 .buf = vh->sha256,
749 };
750
751 sha256_init(&sha256_ctx);
752 sha256_update(&sha256_ctx, p, len);
753}
754
755static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
756{
757 struct vhdr_sha1 *vh = hdr_priv(hdr);
758 struct sha1_ctx sha1_ctx = {
759 .H = vh->sha1,
760 };
761
762 sha1_init(&sha1_ctx);
763 sha1_update(&sha1_ctx, p, len);
764}
765
766static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
767{
768 struct vhdr_crc7 *vh = hdr_priv(hdr);
769
770 vh->crc7 = crc7(p, len);
771}
772
773static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
774{
775 struct vhdr_crc16 *vh = hdr_priv(hdr);
776
777 vh->crc16 = crc16(p, len);
778}
779
780static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
781{
782 struct vhdr_crc32 *vh = hdr_priv(hdr);
783
784 vh->crc32 = crc32(p, len);
785}
786
787static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
788{
789 struct vhdr_crc32 *vh = hdr_priv(hdr);
790
791 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
792 vh->crc32 = crc32c_intel(p, len);
793 else
794 vh->crc32 = crc32c(p, len);
795}
796
797static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
798{
799 struct vhdr_crc64 *vh = hdr_priv(hdr);
800
801 vh->crc64 = crc64(p, len);
802}
803
804static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
805{
806 struct vhdr_md5 *vh = hdr_priv(hdr);
807 struct md5_ctx md5_ctx = {
808 .hash = (uint32_t *) vh->md5_digest,
809 };
810
811 md5_init(&md5_ctx);
812 md5_update(&md5_ctx, p, len);
813}
814
815static void populate_hdr(struct thread_data *td, struct io_u *io_u,
816 struct verify_header *hdr, unsigned int header_num,
817 unsigned int header_len)
818{
819 unsigned int data_len;
820 void *data, *p;
821
822 p = (void *) hdr;
823
824 hdr->fio_magic = FIO_HDR_MAGIC;
825 hdr->len = header_len;
826 hdr->verify_type = td->o.verify;
827 hdr->rand_seed = io_u->rand_seed;
828 data_len = header_len - hdr_size(hdr);
829
830 data = p + hdr_size(hdr);
831 switch (td->o.verify) {
832 case VERIFY_MD5:
833 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
834 io_u, hdr->len);
835 fill_md5(hdr, data, data_len);
836 break;
837 case VERIFY_CRC64:
838 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
839 io_u, hdr->len);
840 fill_crc64(hdr, data, data_len);
841 break;
842 case VERIFY_CRC32C:
843 case VERIFY_CRC32C_INTEL:
844 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
845 io_u, hdr->len);
846 fill_crc32c(hdr, data, data_len);
847 break;
848 case VERIFY_CRC32:
849 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
850 io_u, hdr->len);
851 fill_crc32(hdr, data, data_len);
852 break;
853 case VERIFY_CRC16:
854 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
855 io_u, hdr->len);
856 fill_crc16(hdr, data, data_len);
857 break;
858 case VERIFY_CRC7:
859 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
860 io_u, hdr->len);
861 fill_crc7(hdr, data, data_len);
862 break;
863 case VERIFY_SHA256:
864 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
865 io_u, hdr->len);
866 fill_sha256(hdr, data, data_len);
867 break;
868 case VERIFY_SHA512:
869 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
870 io_u, hdr->len);
871 fill_sha512(hdr, data, data_len);
872 break;
873 case VERIFY_META:
874 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
875 io_u, hdr->len);
876 fill_meta(hdr, td, io_u, header_num);
877 break;
878 case VERIFY_SHA1:
879 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
880 io_u, hdr->len);
881 fill_sha1(hdr, data, data_len);
882 break;
883 default:
884 log_err("fio: bad verify type: %d\n", td->o.verify);
885 assert(0);
886 }
887 if (td->o.verify_offset)
888 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
889}
890
891/*
892 * fill body of io_u->buf with random data and add a header with the
893 * checksum of choice
894 */
895void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
896{
897 if (td->o.verify == VERIFY_NULL)
898 return;
899
900 fill_pattern_headers(td, io_u, 0, 0);
901}
902
903int get_next_verify(struct thread_data *td, struct io_u *io_u)
904{
905 struct io_piece *ipo = NULL;
906
907 /*
908 * this io_u is from a requeue, we already filled the offsets
909 */
910 if (io_u->file)
911 return 0;
912
913 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
914 struct rb_node *n = rb_first(&td->io_hist_tree);
915
916 ipo = rb_entry(n, struct io_piece, rb_node);
917 rb_erase(n, &td->io_hist_tree);
918 assert(ipo->flags & IP_F_ONRB);
919 ipo->flags &= ~IP_F_ONRB;
920 } else if (!flist_empty(&td->io_hist_list)) {
921 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
922 flist_del(&ipo->list);
923 assert(ipo->flags & IP_F_ONLIST);
924 ipo->flags &= ~IP_F_ONLIST;
925 }
926
927 if (ipo) {
928 td->io_hist_len--;
929
930 io_u->offset = ipo->offset;
931 io_u->buflen = ipo->len;
932 io_u->file = ipo->file;
933
934 if (ipo->flags & IP_F_TRIMMED)
935 io_u->flags |= IO_U_F_TRIMMED;
936
937 if (!fio_file_open(io_u->file)) {
938 int r = td_io_open_file(td, io_u->file);
939
940 if (r) {
941 dprint(FD_VERIFY, "failed file %s open\n",
942 io_u->file->file_name);
943 return 1;
944 }
945 }
946
947 get_file(ipo->file);
948 assert(fio_file_open(io_u->file));
949 io_u->ddir = DDIR_READ;
950 io_u->xfer_buf = io_u->buf;
951 io_u->xfer_buflen = io_u->buflen;
952
953 remove_trim_entry(td, ipo);
954 free(ipo);
955 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
956 return 0;
957 }
958
959 dprint(FD_VERIFY, "get_next_verify: empty\n");
960 return 1;
961}
962
963static void *verify_async_thread(void *data)
964{
965 struct thread_data *td = data;
966 struct io_u *io_u;
967 int ret = 0;
968
969 if (td->o.verify_cpumask_set &&
970 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
971 log_err("fio: failed setting verify thread affinity\n");
972 goto done;
973 }
974
975 do {
976 FLIST_HEAD(list);
977
978 read_barrier();
979 if (td->verify_thread_exit)
980 break;
981
982 pthread_mutex_lock(&td->io_u_lock);
983
984 while (flist_empty(&td->verify_list) &&
985 !td->verify_thread_exit) {
986 ret = pthread_cond_wait(&td->verify_cond,
987 &td->io_u_lock);
988 if (ret) {
989 pthread_mutex_unlock(&td->io_u_lock);
990 break;
991 }
992 }
993
994 flist_splice_init(&td->verify_list, &list);
995 pthread_mutex_unlock(&td->io_u_lock);
996
997 if (flist_empty(&list))
998 continue;
999
1000 while (!flist_empty(&list)) {
1001 io_u = flist_entry(list.next, struct io_u, list);
1002 flist_del_init(&io_u->list);
1003
1004 ret = verify_io_u(td, io_u);
1005 put_io_u(td, io_u);
1006 if (!ret)
1007 continue;
1008 if (td->o.continue_on_error &&
1009 td_non_fatal_error(ret)) {
1010 update_error_count(td, ret);
1011 td_clear_error(td);
1012 ret = 0;
1013 }
1014 }
1015 } while (!ret);
1016
1017 if (ret) {
1018 td_verror(td, ret, "async_verify");
1019 if (td->o.verify_fatal)
1020 td->terminate = 1;
1021 }
1022
1023done:
1024 pthread_mutex_lock(&td->io_u_lock);
1025 td->nr_verify_threads--;
1026 pthread_mutex_unlock(&td->io_u_lock);
1027
1028 pthread_cond_signal(&td->free_cond);
1029 return NULL;
1030}
1031
1032int verify_async_init(struct thread_data *td)
1033{
1034 int i, ret;
1035 pthread_attr_t attr;
1036
1037 pthread_attr_init(&attr);
1038 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
1039
1040 td->verify_thread_exit = 0;
1041
1042 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
1043 for (i = 0; i < td->o.verify_async; i++) {
1044 ret = pthread_create(&td->verify_threads[i], &attr,
1045 verify_async_thread, td);
1046 if (ret) {
1047 log_err("fio: async verify creation failed: %s\n",
1048 strerror(ret));
1049 break;
1050 }
1051 ret = pthread_detach(td->verify_threads[i]);
1052 if (ret) {
1053 log_err("fio: async verify thread detach failed: %s\n",
1054 strerror(ret));
1055 break;
1056 }
1057 td->nr_verify_threads++;
1058 }
1059
1060 pthread_attr_destroy(&attr);
1061
1062 if (i != td->o.verify_async) {
1063 log_err("fio: only %d verify threads started, exiting\n", i);
1064 td->verify_thread_exit = 1;
1065 write_barrier();
1066 pthread_cond_broadcast(&td->verify_cond);
1067 return 1;
1068 }
1069
1070 return 0;
1071}
1072
1073void verify_async_exit(struct thread_data *td)
1074{
1075 td->verify_thread_exit = 1;
1076 write_barrier();
1077 pthread_cond_broadcast(&td->verify_cond);
1078
1079 pthread_mutex_lock(&td->io_u_lock);
1080
1081 while (td->nr_verify_threads)
1082 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
1083
1084 pthread_mutex_unlock(&td->io_u_lock);
1085 free(td->verify_threads);
1086 td->verify_threads = NULL;
1087}
fio - Flexible IO Tester