projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Really fix broken option length check
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13#include "lib/rand.h"
14
15#include "crc/md5.h"
16#include "crc/crc64.h"
17#include "crc/crc32.h"
18#include "crc/crc32c.h"
19#include "crc/crc16.h"
20#include "crc/crc7.h"
21#include "crc/sha256.h"
22#include "crc/sha512.h"
23#include "crc/sha1.h"
24
25void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u)
26{
27 switch (td->o.verify_pattern_bytes) {
28 case 0:
29 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
30 fill_random_buf(p, len);
31 break;
32 case 1:
33 if ((io_u->flags & IO_U_F_FILLED) &&
34 io_u->buf_filled_len >= len) {
35 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
36 return;
37 }
38 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
39 memset(p, td->o.verify_pattern[0], len);
40 io_u->flags |= IO_U_F_FILLED;
41 io_u->buf_filled_len = len;
42 break;
43 default: {
44 unsigned int i = 0, size = 0;
45 unsigned char *b = p;
46
47 if ((io_u->flags & IO_U_F_FILLED) &&
48 io_u->buf_filled_len >= len) {
49 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
50 td->o.verify_pattern_bytes, len);
51 return;
52 }
53 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
54 td->o.verify_pattern_bytes, len);
55
56 while (i < len) {
57 size = td->o.verify_pattern_bytes;
58 if (size > (len - i))
59 size = len - i;
60 memcpy(b+i, td->o.verify_pattern, size);
61 i += size;
62 }
63 io_u->flags |= IO_U_F_FILLED;
64 io_u->buf_filled_len = len;
65 break;
66 }
67 }
68}
69
70static void memswp(void *buf1, void *buf2, unsigned int len)
71{
72 char swap[200];
73
74 assert(len <= sizeof(swap));
75
76 memcpy(&swap, buf1, len);
77 memcpy(buf1, buf2, len);
78 memcpy(buf2, &swap, len);
79}
80
81static void hexdump(void *buffer, int len)
82{
83 unsigned char *p = buffer;
84 int i;
85
86 for (i = 0; i < len; i++)
87 log_err("%02x", p[i]);
88 log_err("\n");
89}
90
91/*
92 * Prepare for seperation of verify_header and checksum header
93 */
94static inline unsigned int __hdr_size(int verify_type)
95{
96 unsigned int len = len;
97
98 switch (verify_type) {
99 case VERIFY_NONE:
100 case VERIFY_NULL:
101 len = 0;
102 break;
103 case VERIFY_MD5:
104 len = sizeof(struct vhdr_md5);
105 break;
106 case VERIFY_CRC64:
107 len = sizeof(struct vhdr_crc64);
108 break;
109 case VERIFY_CRC32C:
110 case VERIFY_CRC32:
111 case VERIFY_CRC32C_INTEL:
112 len = sizeof(struct vhdr_crc32);
113 break;
114 case VERIFY_CRC16:
115 len = sizeof(struct vhdr_crc16);
116 break;
117 case VERIFY_CRC7:
118 len = sizeof(struct vhdr_crc7);
119 break;
120 case VERIFY_SHA256:
121 len = sizeof(struct vhdr_sha256);
122 break;
123 case VERIFY_SHA512:
124 len = sizeof(struct vhdr_sha512);
125 break;
126 case VERIFY_META:
127 len = sizeof(struct vhdr_meta);
128 break;
129 case VERIFY_SHA1:
130 len = sizeof(struct vhdr_sha1);
131 break;
132 default:
133 log_err("fio: unknown verify header!\n");
134 assert(0);
135 }
136
137 return len + sizeof(struct verify_header);
138}
139
140static inline unsigned int hdr_size(struct verify_header *hdr)
141{
142 return __hdr_size(hdr->verify_type);
143}
144
145static void *hdr_priv(struct verify_header *hdr)
146{
147 void *priv = hdr;
148
149 return priv + sizeof(struct verify_header);
150}
151
152/*
153 * Verify container, pass info to verify handlers and allow them to
154 * pass info back in case of error
155 */
156struct vcont {
157 /*
158 * Input
159 */
160 struct io_u *io_u;
161 unsigned int hdr_num;
162
163 /*
164 * Output, only valid in case of error
165 */
166 const char *name;
167 void *good_crc;
168 void *bad_crc;
169 unsigned int crc_len;
170};
171
172static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
173{
174 unsigned long long offset;
175
176 offset = vc->io_u->offset;
177 offset += vc->hdr_num * hdr->len;
178 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
179 vc->name, vc->io_u->file->file_name, offset, hdr->len);
180
181 if (vc->good_crc && vc->bad_crc) {
182 log_err(" Expected CRC: ");
183 hexdump(vc->good_crc, vc->crc_len);
184 log_err(" Received CRC: ");
185 hexdump(vc->bad_crc, vc->crc_len);
186 }
187}
188
189/*
190 * Return data area 'header_num'
191 */
192static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
193{
194 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
195}
196
197static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
198 struct vcont *vc)
199{
200 struct vhdr_meta *vh = hdr_priv(hdr);
201 struct io_u *io_u = vc->io_u;
202
203 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
204
205 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
206 return 0;
207
208 vc->name = "meta";
209 log_verify_failure(hdr, vc);
210 return EILSEQ;
211}
212
213static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
214{
215 void *p = io_u_verify_off(hdr, vc);
216 struct vhdr_sha512 *vh = hdr_priv(hdr);
217 uint8_t sha512[128];
218 struct sha512_ctx sha512_ctx = {
219 .buf = sha512,
220 };
221
222 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
223
224 sha512_init(&sha512_ctx);
225 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
226
227 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
228 return 0;
229
230 vc->name = "sha512";
231 vc->good_crc = vh->sha512;
232 vc->bad_crc = sha512_ctx.buf;
233 vc->crc_len = sizeof(vh->sha512);
234 log_verify_failure(hdr, vc);
235 return EILSEQ;
236}
237
238static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
239{
240 void *p = io_u_verify_off(hdr, vc);
241 struct vhdr_sha256 *vh = hdr_priv(hdr);
242 uint8_t sha256[64];
243 struct sha256_ctx sha256_ctx = {
244 .buf = sha256,
245 };
246
247 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
248
249 sha256_init(&sha256_ctx);
250 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
251
252 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
253 return 0;
254
255 vc->name = "sha256";
256 vc->good_crc = vh->sha256;
257 vc->bad_crc = sha256_ctx.buf;
258 vc->crc_len = sizeof(vh->sha256);
259 log_verify_failure(hdr, vc);
260 return EILSEQ;
261}
262
263static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
264{
265 void *p = io_u_verify_off(hdr, vc);
266 struct vhdr_sha1 *vh = hdr_priv(hdr);
267 uint32_t sha1[5];
268 struct sha1_ctx sha1_ctx = {
269 .H = sha1,
270 };
271
272 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
273
274 sha1_init(&sha1_ctx);
275 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
276
277 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
278 return 0;
279
280 vc->name = "sha1";
281 vc->good_crc = vh->sha1;
282 vc->bad_crc = sha1_ctx.H;
283 vc->crc_len = sizeof(vh->sha1);
284 log_verify_failure(hdr, vc);
285 return EILSEQ;
286}
287
288static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
289{
290 void *p = io_u_verify_off(hdr, vc);
291 struct vhdr_crc7 *vh = hdr_priv(hdr);
292 unsigned char c;
293
294 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
295
296 c = crc7(p, hdr->len - hdr_size(hdr));
297
298 if (c == vh->crc7)
299 return 0;
300
301 vc->name = "crc7";
302 vc->good_crc = &vh->crc7;
303 vc->bad_crc = &c;
304 vc->crc_len = 1;
305 log_verify_failure(hdr, vc);
306 return EILSEQ;
307}
308
309static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
310{
311 void *p = io_u_verify_off(hdr, vc);
312 struct vhdr_crc16 *vh = hdr_priv(hdr);
313 unsigned short c;
314
315 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
316
317 c = crc16(p, hdr->len - hdr_size(hdr));
318
319 if (c == vh->crc16)
320 return 0;
321
322 vc->name = "crc16";
323 vc->good_crc = &vh->crc16;
324 vc->bad_crc = &c;
325 vc->crc_len = 2;
326 log_verify_failure(hdr, vc);
327 return EILSEQ;
328}
329
330static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
331{
332 void *p = io_u_verify_off(hdr, vc);
333 struct vhdr_crc64 *vh = hdr_priv(hdr);
334 unsigned long long c;
335
336 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
337
338 c = crc64(p, hdr->len - hdr_size(hdr));
339
340 if (c == vh->crc64)
341 return 0;
342
343 vc->name = "crc64";
344 vc->good_crc = &vh->crc64;
345 vc->bad_crc = &c;
346 vc->crc_len = 8;
347 log_verify_failure(hdr, vc);
348 return EILSEQ;
349}
350
351static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
352{
353 void *p = io_u_verify_off(hdr, vc);
354 struct vhdr_crc32 *vh = hdr_priv(hdr);
355 uint32_t c;
356
357 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
358
359 c = crc32(p, hdr->len - hdr_size(hdr));
360
361 if (c == vh->crc32)
362 return 0;
363
364 vc->name = "crc32";
365 vc->good_crc = &vh->crc32;
366 vc->bad_crc = &c;
367 vc->crc_len = 4;
368 log_verify_failure(hdr, vc);
369 return EILSEQ;
370}
371
372static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
373{
374 void *p = io_u_verify_off(hdr, vc);
375 struct vhdr_crc32 *vh = hdr_priv(hdr);
376 uint32_t c;
377
378 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
379
380 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
381 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
382 else
383 c = crc32c(p, hdr->len - hdr_size(hdr));
384
385 if (c == vh->crc32)
386 return 0;
387
388 vc->name = "crc32c";
389 vc->good_crc = &vh->crc32;
390 vc->bad_crc = &c;
391 vc->crc_len = 4;
392 log_verify_failure(hdr, vc);
393 return EILSEQ;
394}
395
396static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
397{
398 void *p = io_u_verify_off(hdr, vc);
399 struct vhdr_md5 *vh = hdr_priv(hdr);
400 uint32_t hash[MD5_HASH_WORDS];
401 struct md5_ctx md5_ctx = {
402 .hash = hash,
403 };
404
405 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
406
407 md5_init(&md5_ctx);
408 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
409
410 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
411 return 0;
412
413 vc->name = "md5";
414 vc->good_crc = vh->md5_digest;
415 vc->bad_crc = md5_ctx.hash;
416 vc->crc_len = sizeof(hash);
417 log_verify_failure(hdr, vc);
418 return EILSEQ;
419}
420
421static unsigned int hweight8(unsigned int w)
422{
423 unsigned int res = w - ((w >> 1) & 0x55);
424
425 res = (res & 0x33) + ((res >> 2) & 0x33);
426 return (res + (res >> 4)) & 0x0F;
427}
428
429int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
430 char *buf, unsigned int len, unsigned int mod)
431{
432 unsigned int i;
433
434 for (i = 0; i < len; i++) {
435 if (buf[i] != pattern[mod]) {
436 unsigned int bits;
437
438 bits = hweight8(buf[i] ^ pattern[mod]);
439 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
440 buf[i], pattern[mod], bits);
441 log_err("fio: bad pattern block offset %u\n", i);
442 return EILSEQ;
443 }
444 mod++;
445 if (mod == pattern_size)
446 mod = 0;
447 }
448
449 return 0;
450}
451
452/*
453 * Push IO verification to a separate thread
454 */
455int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
456{
457 if (io_u->file)
458 put_file_log(td, io_u->file);
459
460 io_u->file = NULL;
461
462 pthread_mutex_lock(&td->io_u_lock);
463
464 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
465 td->cur_depth--;
466 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
467 }
468 flist_del(&io_u->list);
469 flist_add_tail(&io_u->list, &td->verify_list);
470 io_u->flags |= IO_U_F_FREE_DEF;
471 pthread_mutex_unlock(&td->io_u_lock);
472
473 pthread_cond_signal(&td->verify_cond);
474 return 0;
475}
476
477int verify_io_u(struct thread_data *td, struct io_u *io_u)
478{
479 struct verify_header *hdr;
480 unsigned int hdr_size, hdr_inc, hdr_num = 0;
481 void *p;
482 int ret;
483
484 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
485 return 0;
486
487 hdr_inc = io_u->buflen;
488 if (td->o.verify_interval)
489 hdr_inc = td->o.verify_interval;
490
491 ret = 0;
492 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
493 p += hdr_inc, hdr_num++) {
494 struct vcont vc = {
495 .io_u = io_u,
496 .hdr_num = hdr_num,
497 };
498
499 if (ret && td->o.verify_fatal)
500 break;
501
502 hdr_size = __hdr_size(td->o.verify);
503 if (td->o.verify_offset)
504 memswp(p, p + td->o.verify_offset, hdr_size);
505 hdr = p;
506
507 if (hdr->fio_magic != FIO_HDR_MAGIC) {
508 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
509 hdr->fio_magic, FIO_HDR_MAGIC,
510 io_u->file->file_name,
511 io_u->offset + hdr_num * hdr->len, hdr->len);
512 return EILSEQ;
513 }
514
515 if (td->o.verify_pattern_bytes) {
516 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
517 io_u, hdr->len);
518 ret = verify_io_u_pattern(td->o.verify_pattern,
519 td->o.verify_pattern_bytes,
520 p + hdr_size,
521 hdr_inc - hdr_size,
522 hdr_size % td->o.verify_pattern_bytes);
523
524 if (ret) {
525 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
526 io_u->file->file_name,
527 io_u->offset + hdr_num * hdr->len,
528 hdr->len);
529 }
530
531 /*
532 * Also verify the meta data, if applicable
533 */
534 if (hdr->verify_type == VERIFY_META)
535 ret |= verify_io_u_meta(hdr, td, &vc);
536 continue;
537 }
538
539 switch (hdr->verify_type) {
540 case VERIFY_MD5:
541 ret = verify_io_u_md5(hdr, &vc);
542 break;
543 case VERIFY_CRC64:
544 ret = verify_io_u_crc64(hdr, &vc);
545 break;
546 case VERIFY_CRC32C:
547 case VERIFY_CRC32C_INTEL:
548 ret = verify_io_u_crc32c(hdr, &vc);
549 break;
550 case VERIFY_CRC32:
551 ret = verify_io_u_crc32(hdr, &vc);
552 break;
553 case VERIFY_CRC16:
554 ret = verify_io_u_crc16(hdr, &vc);
555 break;
556 case VERIFY_CRC7:
557 ret = verify_io_u_crc7(hdr, &vc);
558 break;
559 case VERIFY_SHA256:
560 ret = verify_io_u_sha256(hdr, &vc);
561 break;
562 case VERIFY_SHA512:
563 ret = verify_io_u_sha512(hdr, &vc);
564 break;
565 case VERIFY_META:
566 ret = verify_io_u_meta(hdr, td, &vc);
567 break;
568 case VERIFY_SHA1:
569 ret = verify_io_u_sha1(hdr, &vc);
570 break;
571 default:
572 log_err("Bad verify type %u\n", hdr->verify_type);
573 ret = EINVAL;
574 }
575 }
576
577 if (ret && td->o.verify_fatal)
578 td->terminate = 1;
579
580 return ret;
581}
582
583static void fill_meta(struct verify_header *hdr, struct thread_data *td,
584 struct io_u *io_u, unsigned int header_num)
585{
586 struct vhdr_meta *vh = hdr_priv(hdr);
587
588 vh->thread = td->thread_number;
589
590 vh->time_sec = io_u->start_time.tv_sec;
591 vh->time_usec = io_u->start_time.tv_usec;
592
593 vh->numberio = td->io_issues[DDIR_WRITE];
594
595 vh->offset = io_u->offset + header_num * td->o.verify_interval;
596}
597
598static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
599{
600 struct vhdr_sha512 *vh = hdr_priv(hdr);
601 struct sha512_ctx sha512_ctx = {
602 .buf = vh->sha512,
603 };
604
605 sha512_init(&sha512_ctx);
606 sha512_update(&sha512_ctx, p, len);
607}
608
609static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
610{
611 struct vhdr_sha256 *vh = hdr_priv(hdr);
612 struct sha256_ctx sha256_ctx = {
613 .buf = vh->sha256,
614 };
615
616 sha256_init(&sha256_ctx);
617 sha256_update(&sha256_ctx, p, len);
618}
619
620static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
621{
622 struct vhdr_sha1 *vh = hdr_priv(hdr);
623 struct sha1_ctx sha1_ctx = {
624 .H = vh->sha1,
625 };
626
627 sha1_init(&sha1_ctx);
628 sha1_update(&sha1_ctx, p, len);
629}
630
631static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
632{
633 struct vhdr_crc7 *vh = hdr_priv(hdr);
634
635 vh->crc7 = crc7(p, len);
636}
637
638static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
639{
640 struct vhdr_crc16 *vh = hdr_priv(hdr);
641
642 vh->crc16 = crc16(p, len);
643}
644
645static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
646{
647 struct vhdr_crc32 *vh = hdr_priv(hdr);
648
649 vh->crc32 = crc32(p, len);
650}
651
652static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
653{
654 struct vhdr_crc32 *vh = hdr_priv(hdr);
655
656 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
657 vh->crc32 = crc32c_intel(p, len);
658 else
659 vh->crc32 = crc32c(p, len);
660}
661
662static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
663{
664 struct vhdr_crc64 *vh = hdr_priv(hdr);
665
666 vh->crc64 = crc64(p, len);
667}
668
669static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
670{
671 struct vhdr_md5 *vh = hdr_priv(hdr);
672 struct md5_ctx md5_ctx = {
673 .hash = (uint32_t *) vh->md5_digest,
674 };
675
676 md5_init(&md5_ctx);
677 md5_update(&md5_ctx, p, len);
678}
679
680/*
681 * fill body of io_u->buf with random data and add a header with the
682 * crc32 or md5 sum of that data.
683 */
684void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
685{
686 struct verify_header *hdr;
687 void *p = io_u->buf, *data;
688 unsigned int hdr_inc, data_len, header_num = 0;
689
690 if (td->o.verify == VERIFY_NULL)
691 return;
692
693 fill_pattern(td, p, io_u->buflen, io_u);
694
695 hdr_inc = io_u->buflen;
696 if (td->o.verify_interval)
697 hdr_inc = td->o.verify_interval;
698
699 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
700 hdr = p;
701
702 hdr->fio_magic = FIO_HDR_MAGIC;
703 hdr->verify_type = td->o.verify;
704 hdr->len = hdr_inc;
705 data_len = hdr_inc - hdr_size(hdr);
706
707 data = p + hdr_size(hdr);
708 switch (td->o.verify) {
709 case VERIFY_MD5:
710 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
711 io_u, hdr->len);
712 fill_md5(hdr, data, data_len);
713 break;
714 case VERIFY_CRC64:
715 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
716 io_u, hdr->len);
717 fill_crc64(hdr, data, data_len);
718 break;
719 case VERIFY_CRC32C:
720 case VERIFY_CRC32C_INTEL:
721 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
722 io_u, hdr->len);
723 fill_crc32c(hdr, data, data_len);
724 break;
725 case VERIFY_CRC32:
726 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
727 io_u, hdr->len);
728 fill_crc32(hdr, data, data_len);
729 break;
730 case VERIFY_CRC16:
731 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
732 io_u, hdr->len);
733 fill_crc16(hdr, data, data_len);
734 break;
735 case VERIFY_CRC7:
736 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
737 io_u, hdr->len);
738 fill_crc7(hdr, data, data_len);
739 break;
740 case VERIFY_SHA256:
741 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
742 io_u, hdr->len);
743 fill_sha256(hdr, data, data_len);
744 break;
745 case VERIFY_SHA512:
746 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
747 io_u, hdr->len);
748 fill_sha512(hdr, data, data_len);
749 break;
750 case VERIFY_META:
751 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
752 io_u, hdr->len);
753 fill_meta(hdr, td, io_u, header_num);
754 break;
755 case VERIFY_SHA1:
756 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
757 io_u, hdr->len);
758 fill_sha1(hdr, data, data_len);
759 break;
760 default:
761 log_err("fio: bad verify type: %d\n", td->o.verify);
762 assert(0);
763 }
764 if (td->o.verify_offset)
765 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
766 header_num++;
767 }
768}
769
770int get_next_verify(struct thread_data *td, struct io_u *io_u)
771{
772 struct io_piece *ipo = NULL;
773
774 /*
775 * this io_u is from a requeue, we already filled the offsets
776 */
777 if (io_u->file)
778 return 0;
779
780 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
781 struct rb_node *n = rb_first(&td->io_hist_tree);
782
783 ipo = rb_entry(n, struct io_piece, rb_node);
784 rb_erase(n, &td->io_hist_tree);
785 td->io_hist_len--;
786 } else if (!flist_empty(&td->io_hist_list)) {
787 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
788 td->io_hist_len--;
789 flist_del(&ipo->list);
790 }
791
792 if (ipo) {
793 io_u->offset = ipo->offset;
794 io_u->buflen = ipo->len;
795 io_u->file = ipo->file;
796
797 if (!fio_file_open(io_u->file)) {
798 int r = td_io_open_file(td, io_u->file);
799
800 if (r) {
801 dprint(FD_VERIFY, "failed file %s open\n",
802 io_u->file->file_name);
803 return 1;
804 }
805 }
806
807 get_file(ipo->file);
808 assert(fio_file_open(io_u->file));
809 io_u->ddir = DDIR_READ;
810 io_u->xfer_buf = io_u->buf;
811 io_u->xfer_buflen = io_u->buflen;
812 free(ipo);
813 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
814 return 0;
815 }
816
817 dprint(FD_VERIFY, "get_next_verify: empty\n");
818 return 1;
819}
820
821static void *verify_async_thread(void *data)
822{
823 struct thread_data *td = data;
824 struct io_u *io_u;
825 int ret = 0;
826
827 if (td->o.verify_cpumask_set &&
828 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
829 log_err("fio: failed setting verify thread affinity\n");
830 goto done;
831 }
832
833 do {
834 FLIST_HEAD(list);
835
836 read_barrier();
837 if (td->verify_thread_exit)
838 break;
839
840 pthread_mutex_lock(&td->io_u_lock);
841
842 while (flist_empty(&td->verify_list) &&
843 !td->verify_thread_exit) {
844 ret = pthread_cond_wait(&td->verify_cond,
845 &td->io_u_lock);
846 if (ret) {
847 pthread_mutex_unlock(&td->io_u_lock);
848 break;
849 }
850 }
851
852 flist_splice_init(&td->verify_list, &list);
853 pthread_mutex_unlock(&td->io_u_lock);
854
855 if (flist_empty(&list))
856 continue;
857
858 while (!flist_empty(&list)) {
859 io_u = flist_entry(list.next, struct io_u, list);
860 flist_del_init(&io_u->list);
861
862 ret = verify_io_u(td, io_u);
863 put_io_u(td, io_u);
864 if (!ret)
865 continue;
866 if (td->o.continue_on_error &&
867 td_non_fatal_error(ret)) {
868 update_error_count(td, ret);
869 td_clear_error(td);
870 ret = 0;
871 }
872 }
873 } while (!ret);
874
875 if (ret) {
876 td_verror(td, ret, "async_verify");
877 if (td->o.verify_fatal)
878 td->terminate = 1;
879 }
880
881done:
882 pthread_mutex_lock(&td->io_u_lock);
883 td->nr_verify_threads--;
884 pthread_mutex_unlock(&td->io_u_lock);
885
886 pthread_cond_signal(&td->free_cond);
887 return NULL;
888}
889
890int verify_async_init(struct thread_data *td)
891{
892 int i, ret;
893
894 td->verify_thread_exit = 0;
895
896 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
897 for (i = 0; i < td->o.verify_async; i++) {
898 ret = pthread_create(&td->verify_threads[i], NULL,
899 verify_async_thread, td);
900 if (ret) {
901 log_err("fio: async verify creation failed: %s\n",
902 strerror(ret));
903 break;
904 }
905 ret = pthread_detach(td->verify_threads[i]);
906 if (ret) {
907 log_err("fio: async verify thread detach failed: %s\n",
908 strerror(ret));
909 break;
910 }
911 td->nr_verify_threads++;
912 }
913
914 if (i != td->o.verify_async) {
915 log_err("fio: only %d verify threads started, exiting\n", i);
916 td->verify_thread_exit = 1;
917 write_barrier();
918 pthread_cond_broadcast(&td->verify_cond);
919 return 1;
920 }
921
922 return 0;
923}
924
925void verify_async_exit(struct thread_data *td)
926{
927 td->verify_thread_exit = 1;
928 write_barrier();
929 pthread_cond_broadcast(&td->verify_cond);
930
931 pthread_mutex_lock(&td->io_u_lock);
932
933 while (td->nr_verify_threads)
934 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
935
936 pthread_mutex_unlock(&td->io_u_lock);
937 free(td->verify_threads);
938 td->verify_threads = NULL;
939}
fio - Flexible IO Tester