projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
First snapshot of FIO for Windows
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13#include "trim.h"
14#include "lib/rand.h"
15
16#include "crc/md5.h"
17#include "crc/crc64.h"
18#include "crc/crc32.h"
19#include "crc/crc32c.h"
20#include "crc/crc16.h"
21#include "crc/crc7.h"
22#include "crc/sha256.h"
23#include "crc/sha512.h"
24#include "crc/sha1.h"
25
26void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u)
27{
28 switch (td->o.verify_pattern_bytes) {
29 case 0:
30 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
31 fill_random_buf(p, len);
32 break;
33 case 1:
34 if (io_u->buf_filled_len >= len) {
35 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
36 return;
37 }
38 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
39 memset(p, td->o.verify_pattern[0], len);
40 io_u->buf_filled_len = len;
41 break;
42 default: {
43 unsigned int i = 0, size = 0;
44 unsigned char *b = p;
45
46 if (io_u->buf_filled_len >= len) {
47 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
48 td->o.verify_pattern_bytes, len);
49 return;
50 }
51 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
52 td->o.verify_pattern_bytes, len);
53
54 while (i < len) {
55 size = td->o.verify_pattern_bytes;
56 if (size > (len - i))
57 size = len - i;
58 memcpy(b+i, td->o.verify_pattern, size);
59 i += size;
60 }
61 io_u->buf_filled_len = len;
62 break;
63 }
64 }
65}
66
67static void memswp(void *buf1, void *buf2, unsigned int len)
68{
69 char swap[200];
70
71 assert(len <= sizeof(swap));
72
73 memcpy(&swap, buf1, len);
74 memcpy(buf1, buf2, len);
75 memcpy(buf2, &swap, len);
76}
77
78static void hexdump(void *buffer, int len)
79{
80 unsigned char *p = buffer;
81 int i;
82
83 for (i = 0; i < len; i++)
84 log_err("%02x", p[i]);
85 log_err("\n");
86}
87
88/*
89 * Prepare for seperation of verify_header and checksum header
90 */
91static inline unsigned int __hdr_size(int verify_type)
92{
93 unsigned int len = 0;
94
95 switch (verify_type) {
96 case VERIFY_NONE:
97 case VERIFY_NULL:
98 len = 0;
99 break;
100 case VERIFY_MD5:
101 len = sizeof(struct vhdr_md5);
102 break;
103 case VERIFY_CRC64:
104 len = sizeof(struct vhdr_crc64);
105 break;
106 case VERIFY_CRC32C:
107 case VERIFY_CRC32:
108 case VERIFY_CRC32C_INTEL:
109 len = sizeof(struct vhdr_crc32);
110 break;
111 case VERIFY_CRC16:
112 len = sizeof(struct vhdr_crc16);
113 break;
114 case VERIFY_CRC7:
115 len = sizeof(struct vhdr_crc7);
116 break;
117 case VERIFY_SHA256:
118 len = sizeof(struct vhdr_sha256);
119 break;
120 case VERIFY_SHA512:
121 len = sizeof(struct vhdr_sha512);
122 break;
123 case VERIFY_META:
124 len = sizeof(struct vhdr_meta);
125 break;
126 case VERIFY_SHA1:
127 len = sizeof(struct vhdr_sha1);
128 break;
129 default:
130 log_err("fio: unknown verify header!\n");
131 assert(0);
132 }
133
134 return len + sizeof(struct verify_header);
135}
136
137static inline unsigned int hdr_size(struct verify_header *hdr)
138{
139 return __hdr_size(hdr->verify_type);
140}
141
142static void *hdr_priv(struct verify_header *hdr)
143{
144 void *priv = hdr;
145
146 return priv + sizeof(struct verify_header);
147}
148
149/*
150 * Verify container, pass info to verify handlers and allow them to
151 * pass info back in case of error
152 */
153struct vcont {
154 /*
155 * Input
156 */
157 struct io_u *io_u;
158 unsigned int hdr_num;
159
160 /*
161 * Output, only valid in case of error
162 */
163 const char *name;
164 void *good_crc;
165 void *bad_crc;
166 unsigned int crc_len;
167};
168
169static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
170{
171 unsigned long long offset;
172
173 offset = vc->io_u->offset;
174 offset += vc->hdr_num * hdr->len;
175 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
176 vc->name, vc->io_u->file->file_name, offset, hdr->len);
177
178 if (vc->good_crc && vc->bad_crc) {
179 log_err(" Expected CRC: ");
180 hexdump(vc->good_crc, vc->crc_len);
181 log_err(" Received CRC: ");
182 hexdump(vc->bad_crc, vc->crc_len);
183 }
184}
185
186/*
187 * Return data area 'header_num'
188 */
189static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
190{
191 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
192}
193
194static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
195 struct vcont *vc)
196{
197 struct vhdr_meta *vh = hdr_priv(hdr);
198 struct io_u *io_u = vc->io_u;
199
200 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
201
202 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
203 return 0;
204
205 vc->name = "meta";
206 log_verify_failure(hdr, vc);
207 return EILSEQ;
208}
209
210static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
211{
212 void *p = io_u_verify_off(hdr, vc);
213 struct vhdr_sha512 *vh = hdr_priv(hdr);
214 uint8_t sha512[128];
215 struct sha512_ctx sha512_ctx = {
216 .buf = sha512,
217 };
218
219 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
220
221 sha512_init(&sha512_ctx);
222 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
223
224 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
225 return 0;
226
227 vc->name = "sha512";
228 vc->good_crc = vh->sha512;
229 vc->bad_crc = sha512_ctx.buf;
230 vc->crc_len = sizeof(vh->sha512);
231 log_verify_failure(hdr, vc);
232 return EILSEQ;
233}
234
235static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
236{
237 void *p = io_u_verify_off(hdr, vc);
238 struct vhdr_sha256 *vh = hdr_priv(hdr);
239 uint8_t sha256[64];
240 struct sha256_ctx sha256_ctx = {
241 .buf = sha256,
242 };
243
244 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
245
246 sha256_init(&sha256_ctx);
247 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
248
249 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
250 return 0;
251
252 vc->name = "sha256";
253 vc->good_crc = vh->sha256;
254 vc->bad_crc = sha256_ctx.buf;
255 vc->crc_len = sizeof(vh->sha256);
256 log_verify_failure(hdr, vc);
257 return EILSEQ;
258}
259
260static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
261{
262 void *p = io_u_verify_off(hdr, vc);
263 struct vhdr_sha1 *vh = hdr_priv(hdr);
264 uint32_t sha1[5];
265 struct sha1_ctx sha1_ctx = {
266 .H = sha1,
267 };
268
269 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
270
271 sha1_init(&sha1_ctx);
272 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
273
274 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
275 return 0;
276
277 vc->name = "sha1";
278 vc->good_crc = vh->sha1;
279 vc->bad_crc = sha1_ctx.H;
280 vc->crc_len = sizeof(vh->sha1);
281 log_verify_failure(hdr, vc);
282 return EILSEQ;
283}
284
285static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
286{
287 void *p = io_u_verify_off(hdr, vc);
288 struct vhdr_crc7 *vh = hdr_priv(hdr);
289 unsigned char c;
290
291 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
292
293 c = crc7(p, hdr->len - hdr_size(hdr));
294
295 if (c == vh->crc7)
296 return 0;
297
298 vc->name = "crc7";
299 vc->good_crc = &vh->crc7;
300 vc->bad_crc = &c;
301 vc->crc_len = 1;
302 log_verify_failure(hdr, vc);
303 return EILSEQ;
304}
305
306static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
307{
308 void *p = io_u_verify_off(hdr, vc);
309 struct vhdr_crc16 *vh = hdr_priv(hdr);
310 unsigned short c;
311
312 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
313
314 c = crc16(p, hdr->len - hdr_size(hdr));
315
316 if (c == vh->crc16)
317 return 0;
318
319 vc->name = "crc16";
320 vc->good_crc = &vh->crc16;
321 vc->bad_crc = &c;
322 vc->crc_len = 2;
323 log_verify_failure(hdr, vc);
324 return EILSEQ;
325}
326
327static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
328{
329 void *p = io_u_verify_off(hdr, vc);
330 struct vhdr_crc64 *vh = hdr_priv(hdr);
331 unsigned long long c;
332
333 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
334
335 c = crc64(p, hdr->len - hdr_size(hdr));
336
337 if (c == vh->crc64)
338 return 0;
339
340 vc->name = "crc64";
341 vc->good_crc = &vh->crc64;
342 vc->bad_crc = &c;
343 vc->crc_len = 8;
344 log_verify_failure(hdr, vc);
345 return EILSEQ;
346}
347
348static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
349{
350 void *p = io_u_verify_off(hdr, vc);
351 struct vhdr_crc32 *vh = hdr_priv(hdr);
352 uint32_t c;
353
354 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
355
356 c = crc32(p, hdr->len - hdr_size(hdr));
357
358 if (c == vh->crc32)
359 return 0;
360
361 vc->name = "crc32";
362 vc->good_crc = &vh->crc32;
363 vc->bad_crc = &c;
364 vc->crc_len = 4;
365 log_verify_failure(hdr, vc);
366 return EILSEQ;
367}
368
369static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
370{
371 void *p = io_u_verify_off(hdr, vc);
372 struct vhdr_crc32 *vh = hdr_priv(hdr);
373 uint32_t c;
374
375 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
376
377 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
378 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
379 else
380 c = crc32c(p, hdr->len - hdr_size(hdr));
381
382 if (c == vh->crc32)
383 return 0;
384
385 vc->name = "crc32c";
386 vc->good_crc = &vh->crc32;
387 vc->bad_crc = &c;
388 vc->crc_len = 4;
389 log_verify_failure(hdr, vc);
390 return EILSEQ;
391}
392
393static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
394{
395 void *p = io_u_verify_off(hdr, vc);
396 struct vhdr_md5 *vh = hdr_priv(hdr);
397 uint32_t hash[MD5_HASH_WORDS];
398 struct md5_ctx md5_ctx = {
399 .hash = hash,
400 };
401
402 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
403
404 md5_init(&md5_ctx);
405 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
406
407 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
408 return 0;
409
410 vc->name = "md5";
411 vc->good_crc = vh->md5_digest;
412 vc->bad_crc = md5_ctx.hash;
413 vc->crc_len = sizeof(hash);
414 log_verify_failure(hdr, vc);
415 return EILSEQ;
416}
417
418static unsigned int hweight8(unsigned int w)
419{
420 unsigned int res = w - ((w >> 1) & 0x55);
421
422 res = (res & 0x33) + ((res >> 2) & 0x33);
423 return (res + (res >> 4)) & 0x0F;
424}
425
426int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
427 char *buf, unsigned int len, unsigned int mod)
428{
429 unsigned int i;
430
431 for (i = 0; i < len; i++) {
432 if (buf[i] != pattern[mod]) {
433 unsigned int bits;
434
435 bits = hweight8(buf[i] ^ pattern[mod]);
436 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
437 buf[i], pattern[mod], bits);
438 log_err("fio: bad pattern block offset %u\n", i);
439 return EILSEQ;
440 }
441 mod++;
442 if (mod == pattern_size)
443 mod = 0;
444 }
445
446 return 0;
447}
448
449/*
450 * Push IO verification to a separate thread
451 */
452int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
453{
454 if (io_u->file)
455 put_file_log(td, io_u->file);
456
457 io_u->file = NULL;
458
459 pthread_mutex_lock(&td->io_u_lock);
460
461 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
462 td->cur_depth--;
463 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
464 }
465 flist_del(&io_u->list);
466 flist_add_tail(&io_u->list, &td->verify_list);
467 io_u->flags |= IO_U_F_FREE_DEF;
468 pthread_mutex_unlock(&td->io_u_lock);
469
470 pthread_cond_signal(&td->verify_cond);
471 return 0;
472}
473
474static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
475{
476 static char zero_buf[1024];
477 unsigned int this_len, len;
478 int ret = 0;
479 void *p;
480
481 if (!td->o.trim_zero)
482 return 0;
483
484 len = io_u->buflen;
485 p = io_u->buf;
486 do {
487 this_len = sizeof(zero_buf);
488 if (this_len > len)
489 this_len = len;
490 if (memcmp(p, zero_buf, this_len)) {
491 ret = EILSEQ;
492 break;
493 }
494 len -= this_len;
495 p += this_len;
496 } while (len);
497
498 if (!ret)
499 return 0;
500
501 log_err("trim: verify failed at file %s offset %llu, length %lu"
502 ", block offset %lu\n",
503 io_u->file->file_name, io_u->offset, io_u->buflen,
504 (unsigned long) (p - io_u->buf));
505 return ret;
506}
507
508int verify_io_u(struct thread_data *td, struct io_u *io_u)
509{
510 struct verify_header *hdr;
511 unsigned int hdr_size, hdr_inc, hdr_num = 0;
512 void *p;
513 int ret;
514
515 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
516 return 0;
517 if (io_u->flags & IO_U_F_TRIMMED) {
518 ret = verify_trimmed_io_u(td, io_u);
519 goto done;
520 }
521
522 hdr_inc = io_u->buflen;
523 if (td->o.verify_interval)
524 hdr_inc = td->o.verify_interval;
525
526 ret = 0;
527 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
528 p += hdr_inc, hdr_num++) {
529 struct vcont vc = {
530 .io_u = io_u,
531 .hdr_num = hdr_num,
532 };
533
534 if (ret && td->o.verify_fatal)
535 break;
536
537 hdr_size = __hdr_size(td->o.verify);
538 if (td->o.verify_offset)
539 memswp(p, p + td->o.verify_offset, hdr_size);
540 hdr = p;
541
542 if (hdr->fio_magic != FIO_HDR_MAGIC) {
543 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
544 hdr->fio_magic, FIO_HDR_MAGIC,
545 io_u->file->file_name,
546 io_u->offset + hdr_num * hdr->len, hdr->len);
547 return EILSEQ;
548 }
549
550 if (td->o.verify_pattern_bytes) {
551 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
552 io_u, hdr->len);
553 ret = verify_io_u_pattern(td->o.verify_pattern,
554 td->o.verify_pattern_bytes,
555 p + hdr_size,
556 hdr_inc - hdr_size,
557 hdr_size % td->o.verify_pattern_bytes);
558
559 if (ret) {
560 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
561 io_u->file->file_name,
562 io_u->offset + hdr_num * hdr->len,
563 hdr->len);
564 }
565
566 /*
567 * Also verify the meta data, if applicable
568 */
569 if (hdr->verify_type == VERIFY_META)
570 ret |= verify_io_u_meta(hdr, td, &vc);
571 continue;
572 }
573
574 switch (hdr->verify_type) {
575 case VERIFY_MD5:
576 ret = verify_io_u_md5(hdr, &vc);
577 break;
578 case VERIFY_CRC64:
579 ret = verify_io_u_crc64(hdr, &vc);
580 break;
581 case VERIFY_CRC32C:
582 case VERIFY_CRC32C_INTEL:
583 ret = verify_io_u_crc32c(hdr, &vc);
584 break;
585 case VERIFY_CRC32:
586 ret = verify_io_u_crc32(hdr, &vc);
587 break;
588 case VERIFY_CRC16:
589 ret = verify_io_u_crc16(hdr, &vc);
590 break;
591 case VERIFY_CRC7:
592 ret = verify_io_u_crc7(hdr, &vc);
593 break;
594 case VERIFY_SHA256:
595 ret = verify_io_u_sha256(hdr, &vc);
596 break;
597 case VERIFY_SHA512:
598 ret = verify_io_u_sha512(hdr, &vc);
599 break;
600 case VERIFY_META:
601 ret = verify_io_u_meta(hdr, td, &vc);
602 break;
603 case VERIFY_SHA1:
604 ret = verify_io_u_sha1(hdr, &vc);
605 break;
606 default:
607 log_err("Bad verify type %u\n", hdr->verify_type);
608 ret = EINVAL;
609 }
610 }
611
612done:
613 if (ret && td->o.verify_fatal)
614 td->terminate = 1;
615
616 return ret;
617}
618
619static void fill_meta(struct verify_header *hdr, struct thread_data *td,
620 struct io_u *io_u, unsigned int header_num)
621{
622 struct vhdr_meta *vh = hdr_priv(hdr);
623
624 vh->thread = td->thread_number;
625
626 vh->time_sec = io_u->start_time.tv_sec;
627 vh->time_usec = io_u->start_time.tv_usec;
628
629 vh->numberio = td->io_issues[DDIR_WRITE];
630
631 vh->offset = io_u->offset + header_num * td->o.verify_interval;
632}
633
634static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
635{
636 struct vhdr_sha512 *vh = hdr_priv(hdr);
637 struct sha512_ctx sha512_ctx = {
638 .buf = vh->sha512,
639 };
640
641 sha512_init(&sha512_ctx);
642 sha512_update(&sha512_ctx, p, len);
643}
644
645static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
646{
647 struct vhdr_sha256 *vh = hdr_priv(hdr);
648 struct sha256_ctx sha256_ctx = {
649 .buf = vh->sha256,
650 };
651
652 sha256_init(&sha256_ctx);
653 sha256_update(&sha256_ctx, p, len);
654}
655
656static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
657{
658 struct vhdr_sha1 *vh = hdr_priv(hdr);
659 struct sha1_ctx sha1_ctx = {
660 .H = vh->sha1,
661 };
662
663 sha1_init(&sha1_ctx);
664 sha1_update(&sha1_ctx, p, len);
665}
666
667static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
668{
669 struct vhdr_crc7 *vh = hdr_priv(hdr);
670
671 vh->crc7 = crc7(p, len);
672}
673
674static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
675{
676 struct vhdr_crc16 *vh = hdr_priv(hdr);
677
678 vh->crc16 = crc16(p, len);
679}
680
681static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
682{
683 struct vhdr_crc32 *vh = hdr_priv(hdr);
684
685 vh->crc32 = crc32(p, len);
686}
687
688static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
689{
690 struct vhdr_crc32 *vh = hdr_priv(hdr);
691
692 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
693 vh->crc32 = crc32c_intel(p, len);
694 else
695 vh->crc32 = crc32c(p, len);
696}
697
698static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
699{
700 struct vhdr_crc64 *vh = hdr_priv(hdr);
701
702 vh->crc64 = crc64(p, len);
703}
704
705static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
706{
707 struct vhdr_md5 *vh = hdr_priv(hdr);
708 struct md5_ctx md5_ctx = {
709 .hash = (uint32_t *) vh->md5_digest,
710 };
711
712 md5_init(&md5_ctx);
713 md5_update(&md5_ctx, p, len);
714}
715
716/*
717 * fill body of io_u->buf with random data and add a header with the
718 * crc32 or md5 sum of that data.
719 */
720void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
721{
722 struct verify_header *hdr;
723 void *p = io_u->buf, *data;
724 unsigned int hdr_inc, data_len, header_num = 0;
725
726 if (td->o.verify == VERIFY_NULL)
727 return;
728
729 fill_pattern(td, p, io_u->buflen, io_u);
730
731 hdr_inc = io_u->buflen;
732 if (td->o.verify_interval)
733 hdr_inc = td->o.verify_interval;
734
735 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
736 hdr = p;
737
738 hdr->fio_magic = FIO_HDR_MAGIC;
739 hdr->verify_type = td->o.verify;
740 hdr->len = hdr_inc;
741 data_len = hdr_inc - hdr_size(hdr);
742
743 data = p + hdr_size(hdr);
744 switch (td->o.verify) {
745 case VERIFY_MD5:
746 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
747 io_u, hdr->len);
748 fill_md5(hdr, data, data_len);
749 break;
750 case VERIFY_CRC64:
751 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
752 io_u, hdr->len);
753 fill_crc64(hdr, data, data_len);
754 break;
755 case VERIFY_CRC32C:
756 case VERIFY_CRC32C_INTEL:
757 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
758 io_u, hdr->len);
759 fill_crc32c(hdr, data, data_len);
760 break;
761 case VERIFY_CRC32:
762 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
763 io_u, hdr->len);
764 fill_crc32(hdr, data, data_len);
765 break;
766 case VERIFY_CRC16:
767 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
768 io_u, hdr->len);
769 fill_crc16(hdr, data, data_len);
770 break;
771 case VERIFY_CRC7:
772 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
773 io_u, hdr->len);
774 fill_crc7(hdr, data, data_len);
775 break;
776 case VERIFY_SHA256:
777 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
778 io_u, hdr->len);
779 fill_sha256(hdr, data, data_len);
780 break;
781 case VERIFY_SHA512:
782 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
783 io_u, hdr->len);
784 fill_sha512(hdr, data, data_len);
785 break;
786 case VERIFY_META:
787 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
788 io_u, hdr->len);
789 fill_meta(hdr, td, io_u, header_num);
790 break;
791 case VERIFY_SHA1:
792 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
793 io_u, hdr->len);
794 fill_sha1(hdr, data, data_len);
795 break;
796 default:
797 log_err("fio: bad verify type: %d\n", td->o.verify);
798 assert(0);
799 }
800 if (td->o.verify_offset)
801 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
802 header_num++;
803 }
804}
805
806int get_next_verify(struct thread_data *td, struct io_u *io_u)
807{
808 struct io_piece *ipo = NULL;
809
810 /*
811 * this io_u is from a requeue, we already filled the offsets
812 */
813 if (io_u->file)
814 return 0;
815
816 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
817 struct rb_node *n = rb_first(&td->io_hist_tree);
818
819 ipo = rb_entry(n, struct io_piece, rb_node);
820 rb_erase(n, &td->io_hist_tree);
821 assert(ipo->flags & IP_F_ONRB);
822 ipo->flags &= ~IP_F_ONRB;
823 } else if (!flist_empty(&td->io_hist_list)) {
824 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
825 flist_del(&ipo->list);
826 assert(ipo->flags & IP_F_ONLIST);
827 ipo->flags &= ~IP_F_ONLIST;
828 }
829
830 if (ipo) {
831 td->io_hist_len--;
832
833 io_u->offset = ipo->offset;
834 io_u->buflen = ipo->len;
835 io_u->file = ipo->file;
836
837 if (ipo->flags & IP_F_TRIMMED)
838 io_u->flags |= IO_U_F_TRIMMED;
839
840 if (!fio_file_open(io_u->file)) {
841 int r = td_io_open_file(td, io_u->file);
842
843 if (r) {
844 dprint(FD_VERIFY, "failed file %s open\n",
845 io_u->file->file_name);
846 return 1;
847 }
848 }
849
850 get_file(ipo->file);
851 assert(fio_file_open(io_u->file));
852 io_u->ddir = DDIR_READ;
853 io_u->xfer_buf = io_u->buf;
854 io_u->xfer_buflen = io_u->buflen;
855
856 remove_trim_entry(td, ipo);
857 free(ipo);
858 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
859 return 0;
860 }
861
862 dprint(FD_VERIFY, "get_next_verify: empty\n");
863 return 1;
864}
865
866static void *verify_async_thread(void *data)
867{
868 struct thread_data *td = data;
869 struct io_u *io_u;
870 int ret = 0;
871
872 if (td->o.verify_cpumask_set &&
873 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
874 log_err("fio: failed setting verify thread affinity\n");
875 goto done;
876 }
877
878 do {
879 FLIST_HEAD(list);
880
881 read_barrier();
882 if (td->verify_thread_exit)
883 break;
884
885 pthread_mutex_lock(&td->io_u_lock);
886
887 while (flist_empty(&td->verify_list) &&
888 !td->verify_thread_exit) {
889 ret = pthread_cond_wait(&td->verify_cond,
890 &td->io_u_lock);
891 if (ret) {
892 pthread_mutex_unlock(&td->io_u_lock);
893 break;
894 }
895 }
896
897 flist_splice_init(&td->verify_list, &list);
898 pthread_mutex_unlock(&td->io_u_lock);
899
900 if (flist_empty(&list))
901 continue;
902
903 while (!flist_empty(&list)) {
904 io_u = flist_entry(list.next, struct io_u, list);
905 flist_del_init(&io_u->list);
906
907 ret = verify_io_u(td, io_u);
908 put_io_u(td, io_u);
909 if (!ret)
910 continue;
911 if (td->o.continue_on_error &&
912 td_non_fatal_error(ret)) {
913 update_error_count(td, ret);
914 td_clear_error(td);
915 ret = 0;
916 }
917 }
918 } while (!ret);
919
920 if (ret) {
921 td_verror(td, ret, "async_verify");
922 if (td->o.verify_fatal)
923 td->terminate = 1;
924 }
925
926done:
927 pthread_mutex_lock(&td->io_u_lock);
928 td->nr_verify_threads--;
929 pthread_mutex_unlock(&td->io_u_lock);
930
931 pthread_cond_signal(&td->free_cond);
932 return NULL;
933}
934
935int verify_async_init(struct thread_data *td)
936{
937 int i, ret;
938 pthread_attr_t attr;
939
940 pthread_attr_init(&attr);
941 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
942
943 td->verify_thread_exit = 0;
944
945 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
946 for (i = 0; i < td->o.verify_async; i++) {
947 ret = pthread_create(&td->verify_threads[i], &attr,
948 verify_async_thread, td);
949 if (ret) {
950 log_err("fio: async verify creation failed: %s\n",
951 strerror(ret));
952 break;
953 }
954 ret = pthread_detach(td->verify_threads[i]);
955 if (ret) {
956 log_err("fio: async verify thread detach failed: %s\n",
957 strerror(ret));
958 break;
959 }
960 td->nr_verify_threads++;
961 }
962
963 pthread_attr_destroy(&attr);
964
965 if (i != td->o.verify_async) {
966 log_err("fio: only %d verify threads started, exiting\n", i);
967 td->verify_thread_exit = 1;
968 write_barrier();
969 pthread_cond_broadcast(&td->verify_cond);
970 return 1;
971 }
972
973 return 0;
974}
975
976void verify_async_exit(struct thread_data *td)
977{
978 td->verify_thread_exit = 1;
979 write_barrier();
980 pthread_cond_broadcast(&td->verify_cond);
981
982 pthread_mutex_lock(&td->io_u_lock);
983
984 while (td->nr_verify_threads)
985 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
986
987 pthread_mutex_unlock(&td->io_u_lock);
988 free(td->verify_threads);
989 td->verify_threads = NULL;
990}
fio - Flexible IO Tester