projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
No need to use specific flag for io_u fill length
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13#include "lib/rand.h"
14
15#include "crc/md5.h"
16#include "crc/crc64.h"
17#include "crc/crc32.h"
18#include "crc/crc32c.h"
19#include "crc/crc16.h"
20#include "crc/crc7.h"
21#include "crc/sha256.h"
22#include "crc/sha512.h"
23#include "crc/sha1.h"
24
25void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u)
26{
27 switch (td->o.verify_pattern_bytes) {
28 case 0:
29 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
30 fill_random_buf(p, len);
31 break;
32 case 1:
33 if (io_u->buf_filled_len >= len) {
34 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
35 return;
36 }
37 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
38 memset(p, td->o.verify_pattern[0], len);
39 io_u->buf_filled_len = len;
40 break;
41 default: {
42 unsigned int i = 0, size = 0;
43 unsigned char *b = p;
44
45 if (io_u->buf_filled_len >= len) {
46 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
47 td->o.verify_pattern_bytes, len);
48 return;
49 }
50 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
51 td->o.verify_pattern_bytes, len);
52
53 while (i < len) {
54 size = td->o.verify_pattern_bytes;
55 if (size > (len - i))
56 size = len - i;
57 memcpy(b+i, td->o.verify_pattern, size);
58 i += size;
59 }
60 io_u->buf_filled_len = len;
61 break;
62 }
63 }
64}
65
66static void memswp(void *buf1, void *buf2, unsigned int len)
67{
68 char swap[200];
69
70 assert(len <= sizeof(swap));
71
72 memcpy(&swap, buf1, len);
73 memcpy(buf1, buf2, len);
74 memcpy(buf2, &swap, len);
75}
76
77static void hexdump(void *buffer, int len)
78{
79 unsigned char *p = buffer;
80 int i;
81
82 for (i = 0; i < len; i++)
83 log_err("%02x", p[i]);
84 log_err("\n");
85}
86
87/*
88 * Prepare for seperation of verify_header and checksum header
89 */
90static inline unsigned int __hdr_size(int verify_type)
91{
92 unsigned int len = len;
93
94 switch (verify_type) {
95 case VERIFY_NONE:
96 case VERIFY_NULL:
97 len = 0;
98 break;
99 case VERIFY_MD5:
100 len = sizeof(struct vhdr_md5);
101 break;
102 case VERIFY_CRC64:
103 len = sizeof(struct vhdr_crc64);
104 break;
105 case VERIFY_CRC32C:
106 case VERIFY_CRC32:
107 case VERIFY_CRC32C_INTEL:
108 len = sizeof(struct vhdr_crc32);
109 break;
110 case VERIFY_CRC16:
111 len = sizeof(struct vhdr_crc16);
112 break;
113 case VERIFY_CRC7:
114 len = sizeof(struct vhdr_crc7);
115 break;
116 case VERIFY_SHA256:
117 len = sizeof(struct vhdr_sha256);
118 break;
119 case VERIFY_SHA512:
120 len = sizeof(struct vhdr_sha512);
121 break;
122 case VERIFY_META:
123 len = sizeof(struct vhdr_meta);
124 break;
125 case VERIFY_SHA1:
126 len = sizeof(struct vhdr_sha1);
127 break;
128 default:
129 log_err("fio: unknown verify header!\n");
130 assert(0);
131 }
132
133 return len + sizeof(struct verify_header);
134}
135
136static inline unsigned int hdr_size(struct verify_header *hdr)
137{
138 return __hdr_size(hdr->verify_type);
139}
140
141static void *hdr_priv(struct verify_header *hdr)
142{
143 void *priv = hdr;
144
145 return priv + sizeof(struct verify_header);
146}
147
148/*
149 * Verify container, pass info to verify handlers and allow them to
150 * pass info back in case of error
151 */
152struct vcont {
153 /*
154 * Input
155 */
156 struct io_u *io_u;
157 unsigned int hdr_num;
158
159 /*
160 * Output, only valid in case of error
161 */
162 const char *name;
163 void *good_crc;
164 void *bad_crc;
165 unsigned int crc_len;
166};
167
168static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
169{
170 unsigned long long offset;
171
172 offset = vc->io_u->offset;
173 offset += vc->hdr_num * hdr->len;
174 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
175 vc->name, vc->io_u->file->file_name, offset, hdr->len);
176
177 if (vc->good_crc && vc->bad_crc) {
178 log_err(" Expected CRC: ");
179 hexdump(vc->good_crc, vc->crc_len);
180 log_err(" Received CRC: ");
181 hexdump(vc->bad_crc, vc->crc_len);
182 }
183}
184
185/*
186 * Return data area 'header_num'
187 */
188static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
189{
190 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
191}
192
193static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
194 struct vcont *vc)
195{
196 struct vhdr_meta *vh = hdr_priv(hdr);
197 struct io_u *io_u = vc->io_u;
198
199 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
200
201 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
202 return 0;
203
204 vc->name = "meta";
205 log_verify_failure(hdr, vc);
206 return EILSEQ;
207}
208
209static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
210{
211 void *p = io_u_verify_off(hdr, vc);
212 struct vhdr_sha512 *vh = hdr_priv(hdr);
213 uint8_t sha512[128];
214 struct sha512_ctx sha512_ctx = {
215 .buf = sha512,
216 };
217
218 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
219
220 sha512_init(&sha512_ctx);
221 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
222
223 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
224 return 0;
225
226 vc->name = "sha512";
227 vc->good_crc = vh->sha512;
228 vc->bad_crc = sha512_ctx.buf;
229 vc->crc_len = sizeof(vh->sha512);
230 log_verify_failure(hdr, vc);
231 return EILSEQ;
232}
233
234static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
235{
236 void *p = io_u_verify_off(hdr, vc);
237 struct vhdr_sha256 *vh = hdr_priv(hdr);
238 uint8_t sha256[64];
239 struct sha256_ctx sha256_ctx = {
240 .buf = sha256,
241 };
242
243 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
244
245 sha256_init(&sha256_ctx);
246 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
247
248 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
249 return 0;
250
251 vc->name = "sha256";
252 vc->good_crc = vh->sha256;
253 vc->bad_crc = sha256_ctx.buf;
254 vc->crc_len = sizeof(vh->sha256);
255 log_verify_failure(hdr, vc);
256 return EILSEQ;
257}
258
259static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
260{
261 void *p = io_u_verify_off(hdr, vc);
262 struct vhdr_sha1 *vh = hdr_priv(hdr);
263 uint32_t sha1[5];
264 struct sha1_ctx sha1_ctx = {
265 .H = sha1,
266 };
267
268 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
269
270 sha1_init(&sha1_ctx);
271 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
272
273 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
274 return 0;
275
276 vc->name = "sha1";
277 vc->good_crc = vh->sha1;
278 vc->bad_crc = sha1_ctx.H;
279 vc->crc_len = sizeof(vh->sha1);
280 log_verify_failure(hdr, vc);
281 return EILSEQ;
282}
283
284static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
285{
286 void *p = io_u_verify_off(hdr, vc);
287 struct vhdr_crc7 *vh = hdr_priv(hdr);
288 unsigned char c;
289
290 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
291
292 c = crc7(p, hdr->len - hdr_size(hdr));
293
294 if (c == vh->crc7)
295 return 0;
296
297 vc->name = "crc7";
298 vc->good_crc = &vh->crc7;
299 vc->bad_crc = &c;
300 vc->crc_len = 1;
301 log_verify_failure(hdr, vc);
302 return EILSEQ;
303}
304
305static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
306{
307 void *p = io_u_verify_off(hdr, vc);
308 struct vhdr_crc16 *vh = hdr_priv(hdr);
309 unsigned short c;
310
311 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
312
313 c = crc16(p, hdr->len - hdr_size(hdr));
314
315 if (c == vh->crc16)
316 return 0;
317
318 vc->name = "crc16";
319 vc->good_crc = &vh->crc16;
320 vc->bad_crc = &c;
321 vc->crc_len = 2;
322 log_verify_failure(hdr, vc);
323 return EILSEQ;
324}
325
326static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
327{
328 void *p = io_u_verify_off(hdr, vc);
329 struct vhdr_crc64 *vh = hdr_priv(hdr);
330 unsigned long long c;
331
332 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
333
334 c = crc64(p, hdr->len - hdr_size(hdr));
335
336 if (c == vh->crc64)
337 return 0;
338
339 vc->name = "crc64";
340 vc->good_crc = &vh->crc64;
341 vc->bad_crc = &c;
342 vc->crc_len = 8;
343 log_verify_failure(hdr, vc);
344 return EILSEQ;
345}
346
347static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
348{
349 void *p = io_u_verify_off(hdr, vc);
350 struct vhdr_crc32 *vh = hdr_priv(hdr);
351 uint32_t c;
352
353 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
354
355 c = crc32(p, hdr->len - hdr_size(hdr));
356
357 if (c == vh->crc32)
358 return 0;
359
360 vc->name = "crc32";
361 vc->good_crc = &vh->crc32;
362 vc->bad_crc = &c;
363 vc->crc_len = 4;
364 log_verify_failure(hdr, vc);
365 return EILSEQ;
366}
367
368static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
369{
370 void *p = io_u_verify_off(hdr, vc);
371 struct vhdr_crc32 *vh = hdr_priv(hdr);
372 uint32_t c;
373
374 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
375
376 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
377 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
378 else
379 c = crc32c(p, hdr->len - hdr_size(hdr));
380
381 if (c == vh->crc32)
382 return 0;
383
384 vc->name = "crc32c";
385 vc->good_crc = &vh->crc32;
386 vc->bad_crc = &c;
387 vc->crc_len = 4;
388 log_verify_failure(hdr, vc);
389 return EILSEQ;
390}
391
392static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
393{
394 void *p = io_u_verify_off(hdr, vc);
395 struct vhdr_md5 *vh = hdr_priv(hdr);
396 uint32_t hash[MD5_HASH_WORDS];
397 struct md5_ctx md5_ctx = {
398 .hash = hash,
399 };
400
401 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
402
403 md5_init(&md5_ctx);
404 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
405
406 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
407 return 0;
408
409 vc->name = "md5";
410 vc->good_crc = vh->md5_digest;
411 vc->bad_crc = md5_ctx.hash;
412 vc->crc_len = sizeof(hash);
413 log_verify_failure(hdr, vc);
414 return EILSEQ;
415}
416
417static unsigned int hweight8(unsigned int w)
418{
419 unsigned int res = w - ((w >> 1) & 0x55);
420
421 res = (res & 0x33) + ((res >> 2) & 0x33);
422 return (res + (res >> 4)) & 0x0F;
423}
424
425int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
426 char *buf, unsigned int len, unsigned int mod)
427{
428 unsigned int i;
429
430 for (i = 0; i < len; i++) {
431 if (buf[i] != pattern[mod]) {
432 unsigned int bits;
433
434 bits = hweight8(buf[i] ^ pattern[mod]);
435 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
436 buf[i], pattern[mod], bits);
437 log_err("fio: bad pattern block offset %u\n", i);
438 return EILSEQ;
439 }
440 mod++;
441 if (mod == pattern_size)
442 mod = 0;
443 }
444
445 return 0;
446}
447
448/*
449 * Push IO verification to a separate thread
450 */
451int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
452{
453 if (io_u->file)
454 put_file_log(td, io_u->file);
455
456 io_u->file = NULL;
457
458 pthread_mutex_lock(&td->io_u_lock);
459
460 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
461 td->cur_depth--;
462 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
463 }
464 flist_del(&io_u->list);
465 flist_add_tail(&io_u->list, &td->verify_list);
466 io_u->flags |= IO_U_F_FREE_DEF;
467 pthread_mutex_unlock(&td->io_u_lock);
468
469 pthread_cond_signal(&td->verify_cond);
470 return 0;
471}
472
473int verify_io_u(struct thread_data *td, struct io_u *io_u)
474{
475 struct verify_header *hdr;
476 unsigned int hdr_size, hdr_inc, hdr_num = 0;
477 void *p;
478 int ret;
479
480 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
481 return 0;
482
483 hdr_inc = io_u->buflen;
484 if (td->o.verify_interval)
485 hdr_inc = td->o.verify_interval;
486
487 ret = 0;
488 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
489 p += hdr_inc, hdr_num++) {
490 struct vcont vc = {
491 .io_u = io_u,
492 .hdr_num = hdr_num,
493 };
494
495 if (ret && td->o.verify_fatal)
496 break;
497
498 hdr_size = __hdr_size(td->o.verify);
499 if (td->o.verify_offset)
500 memswp(p, p + td->o.verify_offset, hdr_size);
501 hdr = p;
502
503 if (hdr->fio_magic != FIO_HDR_MAGIC) {
504 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
505 hdr->fio_magic, FIO_HDR_MAGIC,
506 io_u->file->file_name,
507 io_u->offset + hdr_num * hdr->len, hdr->len);
508 return EILSEQ;
509 }
510
511 if (td->o.verify_pattern_bytes) {
512 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
513 io_u, hdr->len);
514 ret = verify_io_u_pattern(td->o.verify_pattern,
515 td->o.verify_pattern_bytes,
516 p + hdr_size,
517 hdr_inc - hdr_size,
518 hdr_size % td->o.verify_pattern_bytes);
519
520 if (ret) {
521 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
522 io_u->file->file_name,
523 io_u->offset + hdr_num * hdr->len,
524 hdr->len);
525 }
526
527 /*
528 * Also verify the meta data, if applicable
529 */
530 if (hdr->verify_type == VERIFY_META)
531 ret |= verify_io_u_meta(hdr, td, &vc);
532 continue;
533 }
534
535 switch (hdr->verify_type) {
536 case VERIFY_MD5:
537 ret = verify_io_u_md5(hdr, &vc);
538 break;
539 case VERIFY_CRC64:
540 ret = verify_io_u_crc64(hdr, &vc);
541 break;
542 case VERIFY_CRC32C:
543 case VERIFY_CRC32C_INTEL:
544 ret = verify_io_u_crc32c(hdr, &vc);
545 break;
546 case VERIFY_CRC32:
547 ret = verify_io_u_crc32(hdr, &vc);
548 break;
549 case VERIFY_CRC16:
550 ret = verify_io_u_crc16(hdr, &vc);
551 break;
552 case VERIFY_CRC7:
553 ret = verify_io_u_crc7(hdr, &vc);
554 break;
555 case VERIFY_SHA256:
556 ret = verify_io_u_sha256(hdr, &vc);
557 break;
558 case VERIFY_SHA512:
559 ret = verify_io_u_sha512(hdr, &vc);
560 break;
561 case VERIFY_META:
562 ret = verify_io_u_meta(hdr, td, &vc);
563 break;
564 case VERIFY_SHA1:
565 ret = verify_io_u_sha1(hdr, &vc);
566 break;
567 default:
568 log_err("Bad verify type %u\n", hdr->verify_type);
569 ret = EINVAL;
570 }
571 }
572
573 if (ret && td->o.verify_fatal)
574 td->terminate = 1;
575
576 return ret;
577}
578
579static void fill_meta(struct verify_header *hdr, struct thread_data *td,
580 struct io_u *io_u, unsigned int header_num)
581{
582 struct vhdr_meta *vh = hdr_priv(hdr);
583
584 vh->thread = td->thread_number;
585
586 vh->time_sec = io_u->start_time.tv_sec;
587 vh->time_usec = io_u->start_time.tv_usec;
588
589 vh->numberio = td->io_issues[DDIR_WRITE];
590
591 vh->offset = io_u->offset + header_num * td->o.verify_interval;
592}
593
594static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
595{
596 struct vhdr_sha512 *vh = hdr_priv(hdr);
597 struct sha512_ctx sha512_ctx = {
598 .buf = vh->sha512,
599 };
600
601 sha512_init(&sha512_ctx);
602 sha512_update(&sha512_ctx, p, len);
603}
604
605static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
606{
607 struct vhdr_sha256 *vh = hdr_priv(hdr);
608 struct sha256_ctx sha256_ctx = {
609 .buf = vh->sha256,
610 };
611
612 sha256_init(&sha256_ctx);
613 sha256_update(&sha256_ctx, p, len);
614}
615
616static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
617{
618 struct vhdr_sha1 *vh = hdr_priv(hdr);
619 struct sha1_ctx sha1_ctx = {
620 .H = vh->sha1,
621 };
622
623 sha1_init(&sha1_ctx);
624 sha1_update(&sha1_ctx, p, len);
625}
626
627static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
628{
629 struct vhdr_crc7 *vh = hdr_priv(hdr);
630
631 vh->crc7 = crc7(p, len);
632}
633
634static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
635{
636 struct vhdr_crc16 *vh = hdr_priv(hdr);
637
638 vh->crc16 = crc16(p, len);
639}
640
641static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
642{
643 struct vhdr_crc32 *vh = hdr_priv(hdr);
644
645 vh->crc32 = crc32(p, len);
646}
647
648static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
649{
650 struct vhdr_crc32 *vh = hdr_priv(hdr);
651
652 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
653 vh->crc32 = crc32c_intel(p, len);
654 else
655 vh->crc32 = crc32c(p, len);
656}
657
658static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
659{
660 struct vhdr_crc64 *vh = hdr_priv(hdr);
661
662 vh->crc64 = crc64(p, len);
663}
664
665static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
666{
667 struct vhdr_md5 *vh = hdr_priv(hdr);
668 struct md5_ctx md5_ctx = {
669 .hash = (uint32_t *) vh->md5_digest,
670 };
671
672 md5_init(&md5_ctx);
673 md5_update(&md5_ctx, p, len);
674}
675
676/*
677 * fill body of io_u->buf with random data and add a header with the
678 * crc32 or md5 sum of that data.
679 */
680void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
681{
682 struct verify_header *hdr;
683 void *p = io_u->buf, *data;
684 unsigned int hdr_inc, data_len, header_num = 0;
685
686 if (td->o.verify == VERIFY_NULL)
687 return;
688
689 fill_pattern(td, p, io_u->buflen, io_u);
690
691 hdr_inc = io_u->buflen;
692 if (td->o.verify_interval)
693 hdr_inc = td->o.verify_interval;
694
695 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
696 hdr = p;
697
698 hdr->fio_magic = FIO_HDR_MAGIC;
699 hdr->verify_type = td->o.verify;
700 hdr->len = hdr_inc;
701 data_len = hdr_inc - hdr_size(hdr);
702
703 data = p + hdr_size(hdr);
704 switch (td->o.verify) {
705 case VERIFY_MD5:
706 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
707 io_u, hdr->len);
708 fill_md5(hdr, data, data_len);
709 break;
710 case VERIFY_CRC64:
711 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
712 io_u, hdr->len);
713 fill_crc64(hdr, data, data_len);
714 break;
715 case VERIFY_CRC32C:
716 case VERIFY_CRC32C_INTEL:
717 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
718 io_u, hdr->len);
719 fill_crc32c(hdr, data, data_len);
720 break;
721 case VERIFY_CRC32:
722 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
723 io_u, hdr->len);
724 fill_crc32(hdr, data, data_len);
725 break;
726 case VERIFY_CRC16:
727 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
728 io_u, hdr->len);
729 fill_crc16(hdr, data, data_len);
730 break;
731 case VERIFY_CRC7:
732 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
733 io_u, hdr->len);
734 fill_crc7(hdr, data, data_len);
735 break;
736 case VERIFY_SHA256:
737 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
738 io_u, hdr->len);
739 fill_sha256(hdr, data, data_len);
740 break;
741 case VERIFY_SHA512:
742 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
743 io_u, hdr->len);
744 fill_sha512(hdr, data, data_len);
745 break;
746 case VERIFY_META:
747 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
748 io_u, hdr->len);
749 fill_meta(hdr, td, io_u, header_num);
750 break;
751 case VERIFY_SHA1:
752 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
753 io_u, hdr->len);
754 fill_sha1(hdr, data, data_len);
755 break;
756 default:
757 log_err("fio: bad verify type: %d\n", td->o.verify);
758 assert(0);
759 }
760 if (td->o.verify_offset)
761 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
762 header_num++;
763 }
764}
765
766int get_next_verify(struct thread_data *td, struct io_u *io_u)
767{
768 struct io_piece *ipo = NULL;
769
770 /*
771 * this io_u is from a requeue, we already filled the offsets
772 */
773 if (io_u->file)
774 return 0;
775
776 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
777 struct rb_node *n = rb_first(&td->io_hist_tree);
778
779 ipo = rb_entry(n, struct io_piece, rb_node);
780 rb_erase(n, &td->io_hist_tree);
781 td->io_hist_len--;
782 } else if (!flist_empty(&td->io_hist_list)) {
783 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
784 td->io_hist_len--;
785 flist_del(&ipo->list);
786 }
787
788 if (ipo) {
789 io_u->offset = ipo->offset;
790 io_u->buflen = ipo->len;
791 io_u->file = ipo->file;
792
793 if (!fio_file_open(io_u->file)) {
794 int r = td_io_open_file(td, io_u->file);
795
796 if (r) {
797 dprint(FD_VERIFY, "failed file %s open\n",
798 io_u->file->file_name);
799 return 1;
800 }
801 }
802
803 get_file(ipo->file);
804 assert(fio_file_open(io_u->file));
805 io_u->ddir = DDIR_READ;
806 io_u->xfer_buf = io_u->buf;
807 io_u->xfer_buflen = io_u->buflen;
808 free(ipo);
809 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
810 return 0;
811 }
812
813 dprint(FD_VERIFY, "get_next_verify: empty\n");
814 return 1;
815}
816
817static void *verify_async_thread(void *data)
818{
819 struct thread_data *td = data;
820 struct io_u *io_u;
821 int ret = 0;
822
823 if (td->o.verify_cpumask_set &&
824 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
825 log_err("fio: failed setting verify thread affinity\n");
826 goto done;
827 }
828
829 do {
830 FLIST_HEAD(list);
831
832 read_barrier();
833 if (td->verify_thread_exit)
834 break;
835
836 pthread_mutex_lock(&td->io_u_lock);
837
838 while (flist_empty(&td->verify_list) &&
839 !td->verify_thread_exit) {
840 ret = pthread_cond_wait(&td->verify_cond,
841 &td->io_u_lock);
842 if (ret) {
843 pthread_mutex_unlock(&td->io_u_lock);
844 break;
845 }
846 }
847
848 flist_splice_init(&td->verify_list, &list);
849 pthread_mutex_unlock(&td->io_u_lock);
850
851 if (flist_empty(&list))
852 continue;
853
854 while (!flist_empty(&list)) {
855 io_u = flist_entry(list.next, struct io_u, list);
856 flist_del_init(&io_u->list);
857
858 ret = verify_io_u(td, io_u);
859 put_io_u(td, io_u);
860 if (!ret)
861 continue;
862 if (td->o.continue_on_error &&
863 td_non_fatal_error(ret)) {
864 update_error_count(td, ret);
865 td_clear_error(td);
866 ret = 0;
867 }
868 }
869 } while (!ret);
870
871 if (ret) {
872 td_verror(td, ret, "async_verify");
873 if (td->o.verify_fatal)
874 td->terminate = 1;
875 }
876
877done:
878 pthread_mutex_lock(&td->io_u_lock);
879 td->nr_verify_threads--;
880 pthread_mutex_unlock(&td->io_u_lock);
881
882 pthread_cond_signal(&td->free_cond);
883 return NULL;
884}
885
886int verify_async_init(struct thread_data *td)
887{
888 int i, ret;
889
890 td->verify_thread_exit = 0;
891
892 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
893 for (i = 0; i < td->o.verify_async; i++) {
894 ret = pthread_create(&td->verify_threads[i], NULL,
895 verify_async_thread, td);
896 if (ret) {
897 log_err("fio: async verify creation failed: %s\n",
898 strerror(ret));
899 break;
900 }
901 ret = pthread_detach(td->verify_threads[i]);
902 if (ret) {
903 log_err("fio: async verify thread detach failed: %s\n",
904 strerror(ret));
905 break;
906 }
907 td->nr_verify_threads++;
908 }
909
910 if (i != td->o.verify_async) {
911 log_err("fio: only %d verify threads started, exiting\n", i);
912 td->verify_thread_exit = 1;
913 write_barrier();
914 pthread_cond_broadcast(&td->verify_cond);
915 return 1;
916 }
917
918 return 0;
919}
920
921void verify_async_exit(struct thread_data *td)
922{
923 td->verify_thread_exit = 1;
924 write_barrier();
925 pthread_cond_broadcast(&td->verify_cond);
926
927 pthread_mutex_lock(&td->io_u_lock);
928
929 while (td->nr_verify_threads)
930 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
931
932 pthread_mutex_unlock(&td->io_u_lock);
933 free(td->verify_threads);
934 td->verify_threads = NULL;
935}
fio - Flexible IO Tester