projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Async verify HOWTO/man update
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13
14#include "crc/md5.h"
15#include "crc/crc64.h"
16#include "crc/crc32.h"
17#include "crc/crc32c.h"
18#include "crc/crc16.h"
19#include "crc/crc7.h"
20#include "crc/sha256.h"
21#include "crc/sha512.h"
22
23static void fill_random_bytes(struct thread_data *td, void *p, unsigned int len)
24{
25 unsigned int todo;
26 int r;
27
28 while (len) {
29 r = os_random_long(&td->verify_state);
30
31 /*
32 * lrand48_r seems to be broken and only fill the bottom
33 * 32-bits, even on 64-bit archs with 64-bit longs
34 */
35 todo = sizeof(r);
36 if (todo > len)
37 todo = len;
38
39 memcpy(p, &r, todo);
40
41 len -= todo;
42 p += todo;
43 }
44}
45
46static void fill_pattern(struct thread_data *td, void *p, unsigned int len)
47{
48 switch (td->o.verify_pattern_bytes) {
49 case 0:
50 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
51 fill_random_bytes(td, p, len);
52 break;
53 case 1:
54 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
55 memset(p, td->o.verify_pattern, len);
56 break;
57 case 2:
58 case 3:
59 case 4: {
60 unsigned int pattern = td->o.verify_pattern;
61 unsigned int i = 0;
62 unsigned char c1, c2, c3, c4;
63 unsigned char *b = p;
64
65 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
66 td->o.verify_pattern_bytes, len);
67
68 c1 = pattern & 0xff;
69 pattern >>= 8;
70 c2 = pattern & 0xff;
71 pattern >>= 8;
72 c3 = pattern & 0xff;
73 pattern >>= 8;
74 c4 = pattern & 0xff;
75
76 while (i < len) {
77 b[i++] = c1;
78 if (i == len)
79 break;
80 b[i++] = c2;
81 if (td->o.verify_pattern_bytes == 2 || i == len)
82 continue;
83 b[i++] = c3;
84 if (td->o.verify_pattern_bytes == 3 || i == len)
85 continue;
86 b[i++] = c4;
87 }
88 break;
89 }
90 }
91}
92
93static void memswp(void *buf1, void *buf2, unsigned int len)
94{
95 char swap[200];
96
97 assert(len <= sizeof(swap));
98
99 memcpy(&swap, buf1, len);
100 memcpy(buf1, buf2, len);
101 memcpy(buf2, &swap, len);
102}
103
104static void hexdump(void *buffer, int len)
105{
106 unsigned char *p = buffer;
107 int i;
108
109 for (i = 0; i < len; i++)
110 log_info("%02x", p[i]);
111 log_info("\n");
112}
113
114/*
115 * Prepare for seperation of verify_header and checksum header
116 */
117static inline unsigned int __hdr_size(int verify_type)
118{
119 unsigned int len = len;
120
121 switch (verify_type) {
122 case VERIFY_NONE:
123 case VERIFY_NULL:
124 len = 0;
125 break;
126 case VERIFY_MD5:
127 len = sizeof(struct vhdr_md5);
128 break;
129 case VERIFY_CRC64:
130 len = sizeof(struct vhdr_crc64);
131 break;
132 case VERIFY_CRC32C:
133 case VERIFY_CRC32:
134 case VERIFY_CRC32C_INTEL:
135 len = sizeof(struct vhdr_crc32);
136 break;
137 case VERIFY_CRC16:
138 len = sizeof(struct vhdr_crc16);
139 break;
140 case VERIFY_CRC7:
141 len = sizeof(struct vhdr_crc7);
142 break;
143 case VERIFY_SHA256:
144 len = sizeof(struct vhdr_sha256);
145 break;
146 case VERIFY_SHA512:
147 len = sizeof(struct vhdr_sha512);
148 break;
149 case VERIFY_META:
150 len = sizeof(struct vhdr_meta);
151 break;
152 default:
153 log_err("fio: unknown verify header!\n");
154 assert(0);
155 }
156
157 return len + sizeof(struct verify_header);
158}
159
160static inline unsigned int hdr_size(struct verify_header *hdr)
161{
162 return __hdr_size(hdr->verify_type);
163}
164
165static void *hdr_priv(struct verify_header *hdr)
166{
167 void *priv = hdr;
168
169 return priv + sizeof(struct verify_header);
170}
171
172/*
173 * Return data area 'header_num'
174 */
175static inline void *io_u_verify_off(struct verify_header *hdr,
176 struct io_u *io_u, unsigned char header_num)
177{
178 return io_u->buf + header_num * hdr->len + hdr_size(hdr);
179}
180
181static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
182 struct io_u *io_u, unsigned int header_num)
183{
184 struct vhdr_meta *vh = hdr_priv(hdr);
185
186 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
187
188 if (vh->offset != io_u->offset + header_num * td->o.verify_interval) {
189 log_err("meta: verify failed at %llu/%u\n",
190 io_u->offset + header_num * hdr->len, hdr->len);
191 return EILSEQ;
192 }
193
194 return 0;
195}
196
197static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u,
198 unsigned int header_num)
199{
200 void *p = io_u_verify_off(hdr, io_u, header_num);
201 struct vhdr_sha512 *vh = hdr_priv(hdr);
202 uint8_t sha512[128];
203 struct sha512_ctx sha512_ctx = {
204 .buf = sha512,
205 };
206
207 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", io_u, hdr->len);
208
209 sha512_init(&sha512_ctx);
210 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
211
212 if (memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) {
213 log_err("sha512: verify failed at %llu/%u\n",
214 io_u->offset + header_num * hdr->len, hdr->len);
215 hexdump(vh->sha512, sizeof(vh->sha512));
216 hexdump(sha512_ctx.buf, sizeof(sha512));
217 return EILSEQ;
218 }
219
220 return 0;
221}
222
223static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u,
224 unsigned int header_num)
225{
226 void *p = io_u_verify_off(hdr, io_u, header_num);
227 struct vhdr_sha256 *vh = hdr_priv(hdr);
228 uint8_t sha256[128];
229 struct sha256_ctx sha256_ctx = {
230 .buf = sha256,
231 };
232
233 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", io_u, hdr->len);
234
235 sha256_init(&sha256_ctx);
236 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
237
238 if (memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) {
239 log_err("sha256: verify failed at %llu/%u\n",
240 io_u->offset + header_num * hdr->len, hdr->len);
241 hexdump(vh->sha256, sizeof(vh->sha256));
242 hexdump(sha256_ctx.buf, sizeof(sha256));
243 return EILSEQ;
244 }
245
246 return 0;
247}
248
249static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u,
250 unsigned char header_num)
251{
252 void *p = io_u_verify_off(hdr, io_u, header_num);
253 struct vhdr_crc7 *vh = hdr_priv(hdr);
254 unsigned char c;
255
256 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", io_u, hdr->len);
257
258 c = crc7(p, hdr->len - hdr_size(hdr));
259
260 if (c != vh->crc7) {
261 log_err("crc7: verify failed at %llu/%u\n",
262 io_u->offset + header_num * hdr->len, hdr->len);
263 log_err("crc7: wanted %x, got %x\n", vh->crc7, c);
264 return EILSEQ;
265 }
266
267 return 0;
268}
269
270static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u,
271 unsigned int header_num)
272{
273 void *p = io_u_verify_off(hdr, io_u, header_num);
274 struct vhdr_crc16 *vh = hdr_priv(hdr);
275 unsigned short c;
276
277 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", io_u, hdr->len);
278
279 c = crc16(p, hdr->len - hdr_size(hdr));
280
281 if (c != vh->crc16) {
282 log_err("crc16: verify failed at %llu/%u\n",
283 io_u->offset + header_num * hdr->len, hdr->len);
284 log_err("crc16: wanted %x, got %x\n", vh->crc16, c);
285 return EILSEQ;
286 }
287
288 return 0;
289}
290
291static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u,
292 unsigned int header_num)
293{
294 void *p = io_u_verify_off(hdr, io_u, header_num);
295 struct vhdr_crc64 *vh = hdr_priv(hdr);
296 unsigned long long c;
297
298 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", io_u, hdr->len);
299
300 c = crc64(p, hdr->len - hdr_size(hdr));
301
302 if (c != vh->crc64) {
303 log_err("crc64: verify failed at %llu/%u\n",
304 io_u->offset + header_num * hdr->len,
305 hdr->len);
306 log_err("crc64: wanted %llx, got %llx\n",
307 (unsigned long long) vh->crc64, c);
308 return EILSEQ;
309 }
310
311 return 0;
312}
313
314static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u,
315 unsigned int header_num)
316{
317 void *p = io_u_verify_off(hdr, io_u, header_num);
318 struct vhdr_crc32 *vh = hdr_priv(hdr);
319 uint32_t c;
320
321 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", io_u, hdr->len);
322
323 c = crc32(p, hdr->len - hdr_size(hdr));
324
325 if (c != vh->crc32) {
326 log_err("crc32: verify failed at %llu/%u\n",
327 io_u->offset + header_num * hdr->len, hdr->len);
328 log_err("crc32: wanted %x, got %x\n", vh->crc32, c);
329 return EILSEQ;
330 }
331
332 return 0;
333}
334
335static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u,
336 unsigned int header_num)
337{
338 void *p = io_u_verify_off(hdr, io_u, header_num);
339 struct vhdr_crc32 *vh = hdr_priv(hdr);
340 uint32_t c;
341
342 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len);
343
344 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
345 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
346 else
347 c = crc32c(p, hdr->len - hdr_size(hdr));
348
349 if (c != vh->crc32) {
350 log_err("crc32c: verify failed at %llu/%u\n",
351 io_u->offset + header_num * hdr->len, hdr->len);
352 log_err("crc32c: wanted %x, got %x\n", vh->crc32, c);
353 return EILSEQ;
354 }
355
356 return 0;
357}
358
359static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u,
360 unsigned int header_num)
361{
362 void *p = io_u_verify_off(hdr, io_u, header_num);
363 struct vhdr_md5 *vh = hdr_priv(hdr);
364 uint32_t hash[MD5_HASH_WORDS];
365 struct md5_ctx md5_ctx = {
366 .hash = hash,
367 };
368
369 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", io_u, hdr->len);
370
371 md5_init(&md5_ctx);
372 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
373
374 if (memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) {
375 log_err("md5: verify failed at %llu/%u\n",
376 io_u->offset + header_num * hdr->len, hdr->len);
377 hexdump(vh->md5_digest, sizeof(vh->md5_digest));
378 hexdump(md5_ctx.hash, sizeof(hash));
379 return EILSEQ;
380 }
381
382 return 0;
383}
384
385static unsigned int hweight8(unsigned int w)
386{
387 unsigned int res = w - ((w >> 1) & 0x55);
388
389 res = (res & 0x33) + ((res >> 2) & 0x33);
390 return (res + (res >> 4)) & 0x0F;
391}
392
393int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size,
394 char *buf, unsigned int len, unsigned int mod)
395{
396 unsigned int i;
397 char split_pattern[4];
398
399 for (i = 0; i < 4; i++) {
400 split_pattern[i] = pattern & 0xff;
401 pattern >>= 8;
402 }
403
404 for (i = 0; i < len; i++) {
405 if (buf[i] != split_pattern[mod]) {
406 unsigned int bits;
407
408 bits = hweight8(buf[i] ^ split_pattern[mod]);
409 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
410 buf[i], split_pattern[mod], bits);
411 log_err("fio: bad pattern block offset %u\n", i);
412 return EILSEQ;
413 }
414 mod++;
415 if (mod == pattern_size)
416 mod = 0;
417 }
418
419 return 0;
420}
421
422/*
423 * Push IO verification to a separate thread
424 */
425int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
426{
427 if (io_u->file)
428 put_file_log(td, io_u->file);
429
430 io_u->file = NULL;
431
432 pthread_mutex_lock(&td->io_u_lock);
433 flist_del(&io_u->list);
434 flist_add_tail(&io_u->list, &td->verify_list);
435 pthread_mutex_unlock(&td->io_u_lock);
436
437 pthread_cond_signal(&td->verify_cond);
438 io_u->flags |= IO_U_F_FREE_DEF;
439 return 0;
440}
441
442int verify_io_u(struct thread_data *td, struct io_u *io_u)
443{
444 struct verify_header *hdr;
445 unsigned int hdr_size, hdr_inc, hdr_num = 0;
446 void *p;
447 int ret;
448
449 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
450 return 0;
451
452 hdr_inc = io_u->buflen;
453 if (td->o.verify_interval)
454 hdr_inc = td->o.verify_interval;
455
456 ret = 0;
457 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
458 p += hdr_inc, hdr_num++) {
459 if (ret && td->o.verify_fatal) {
460 td->terminate = 1;
461 break;
462 }
463 hdr_size = __hdr_size(td->o.verify);
464 if (td->o.verify_offset)
465 memswp(p, p + td->o.verify_offset, hdr_size);
466 hdr = p;
467
468 if (hdr->fio_magic != FIO_HDR_MAGIC) {
469 log_err("Bad verify header %x\n", hdr->fio_magic);
470 return EILSEQ;
471 }
472
473 if (td->o.verify_pattern_bytes) {
474 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
475 io_u, hdr->len);
476 ret = verify_io_u_pattern(td->o.verify_pattern,
477 td->o.verify_pattern_bytes,
478 p + hdr_size,
479 hdr_inc - hdr_size,
480 hdr_size % 4);
481 if (ret)
482 log_err("fio: verify failed at %llu/%u\n",
483 io_u->offset + hdr_num * hdr->len,
484 hdr->len);
485 continue;
486 }
487
488 switch (hdr->verify_type) {
489 case VERIFY_MD5:
490 ret = verify_io_u_md5(hdr, io_u, hdr_num);
491 break;
492 case VERIFY_CRC64:
493 ret = verify_io_u_crc64(hdr, io_u, hdr_num);
494 break;
495 case VERIFY_CRC32C:
496 case VERIFY_CRC32C_INTEL:
497 ret = verify_io_u_crc32c(hdr, io_u, hdr_num);
498 break;
499 case VERIFY_CRC32:
500 ret = verify_io_u_crc32(hdr, io_u, hdr_num);
501 break;
502 case VERIFY_CRC16:
503 ret = verify_io_u_crc16(hdr, io_u, hdr_num);
504 break;
505 case VERIFY_CRC7:
506 ret = verify_io_u_crc7(hdr, io_u, hdr_num);
507 break;
508 case VERIFY_SHA256:
509 ret = verify_io_u_sha256(hdr, io_u, hdr_num);
510 break;
511 case VERIFY_SHA512:
512 ret = verify_io_u_sha512(hdr, io_u, hdr_num);
513 break;
514 case VERIFY_META:
515 ret = verify_io_u_meta(hdr, td, io_u, hdr_num);
516 break;
517 default:
518 log_err("Bad verify type %u\n", hdr->verify_type);
519 ret = EINVAL;
520 }
521 }
522
523 return ret;
524}
525
526static void fill_meta(struct verify_header *hdr, struct thread_data *td,
527 struct io_u *io_u, unsigned int header_num)
528{
529 struct vhdr_meta *vh = hdr_priv(hdr);
530
531 vh->thread = td->thread_number;
532
533 vh->time_sec = io_u->start_time.tv_sec;
534 vh->time_usec = io_u->start_time.tv_usec;
535
536 vh->numberio = td->io_issues[DDIR_WRITE];
537
538 vh->offset = io_u->offset + header_num * td->o.verify_interval;
539}
540
541static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
542{
543 struct vhdr_sha512 *vh = hdr_priv(hdr);
544 struct sha512_ctx sha512_ctx = {
545 .buf = vh->sha512,
546 };
547
548 sha512_init(&sha512_ctx);
549 sha512_update(&sha512_ctx, p, len);
550}
551
552static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
553{
554 struct vhdr_sha256 *vh = hdr_priv(hdr);
555 struct sha256_ctx sha256_ctx = {
556 .buf = vh->sha256,
557 };
558
559 sha256_init(&sha256_ctx);
560 sha256_update(&sha256_ctx, p, len);
561}
562
563static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
564{
565 struct vhdr_crc7 *vh = hdr_priv(hdr);
566
567 vh->crc7 = crc7(p, len);
568}
569
570static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
571{
572 struct vhdr_crc16 *vh = hdr_priv(hdr);
573
574 vh->crc16 = crc16(p, len);
575}
576
577static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
578{
579 struct vhdr_crc32 *vh = hdr_priv(hdr);
580
581 vh->crc32 = crc32(p, len);
582}
583
584static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
585{
586 struct vhdr_crc32 *vh = hdr_priv(hdr);
587
588 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
589 vh->crc32 = crc32c_intel(p, len);
590 else
591 vh->crc32 = crc32c(p, len);
592}
593
594static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
595{
596 struct vhdr_crc64 *vh = hdr_priv(hdr);
597
598 vh->crc64 = crc64(p, len);
599}
600
601static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
602{
603 struct vhdr_md5 *vh = hdr_priv(hdr);
604 struct md5_ctx md5_ctx = {
605 .hash = (uint32_t *) vh->md5_digest,
606 };
607
608 md5_init(&md5_ctx);
609 md5_update(&md5_ctx, p, len);
610}
611
612/*
613 * fill body of io_u->buf with random data and add a header with the
614 * crc32 or md5 sum of that data.
615 */
616void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
617{
618 struct verify_header *hdr;
619 void *p = io_u->buf, *data;
620 unsigned int hdr_inc, data_len, header_num = 0;
621
622 if (td->o.verify == VERIFY_NULL)
623 return;
624
625 fill_pattern(td, p, io_u->buflen);
626
627 hdr_inc = io_u->buflen;
628 if (td->o.verify_interval)
629 hdr_inc = td->o.verify_interval;
630
631 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
632 hdr = p;
633
634 hdr->fio_magic = FIO_HDR_MAGIC;
635 hdr->verify_type = td->o.verify;
636 hdr->len = hdr_inc;
637 data_len = hdr_inc - hdr_size(hdr);
638
639 data = p + hdr_size(hdr);
640 switch (td->o.verify) {
641 case VERIFY_MD5:
642 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
643 io_u, hdr->len);
644 fill_md5(hdr, data, data_len);
645 break;
646 case VERIFY_CRC64:
647 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
648 io_u, hdr->len);
649 fill_crc64(hdr, data, data_len);
650 break;
651 case VERIFY_CRC32C:
652 case VERIFY_CRC32C_INTEL:
653 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
654 io_u, hdr->len);
655 fill_crc32c(hdr, data, data_len);
656 break;
657 case VERIFY_CRC32:
658 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
659 io_u, hdr->len);
660 fill_crc32(hdr, data, data_len);
661 break;
662 case VERIFY_CRC16:
663 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
664 io_u, hdr->len);
665 fill_crc16(hdr, data, data_len);
666 break;
667 case VERIFY_CRC7:
668 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
669 io_u, hdr->len);
670 fill_crc7(hdr, data, data_len);
671 break;
672 case VERIFY_SHA256:
673 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
674 io_u, hdr->len);
675 fill_sha256(hdr, data, data_len);
676 break;
677 case VERIFY_SHA512:
678 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
679 io_u, hdr->len);
680 fill_sha512(hdr, data, data_len);
681 break;
682 case VERIFY_META:
683 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
684 io_u, hdr->len);
685 fill_meta(hdr, td, io_u, header_num);
686 break;
687 default:
688 log_err("fio: bad verify type: %d\n", td->o.verify);
689 assert(0);
690 }
691 if (td->o.verify_offset)
692 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
693 header_num++;
694 }
695}
696
697int get_next_verify(struct thread_data *td, struct io_u *io_u)
698{
699 struct io_piece *ipo = NULL;
700
701 /*
702 * this io_u is from a requeue, we already filled the offsets
703 */
704 if (io_u->file)
705 return 0;
706
707 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
708 struct rb_node *n = rb_first(&td->io_hist_tree);
709
710 ipo = rb_entry(n, struct io_piece, rb_node);
711 rb_erase(n, &td->io_hist_tree);
712 } else if (!flist_empty(&td->io_hist_list)) {
713 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
714 flist_del(&ipo->list);
715 }
716
717 if (ipo) {
718 io_u->offset = ipo->offset;
719 io_u->buflen = ipo->len;
720 io_u->file = ipo->file;
721
722 if (!fio_file_open(io_u->file)) {
723 int r = td_io_open_file(td, io_u->file);
724
725 if (r) {
726 dprint(FD_VERIFY, "failed file %s open\n",
727 io_u->file->file_name);
728 return 1;
729 }
730 }
731
732 get_file(ipo->file);
733 assert(fio_file_open(io_u->file));
734 io_u->ddir = DDIR_READ;
735 io_u->xfer_buf = io_u->buf;
736 io_u->xfer_buflen = io_u->buflen;
737 free(ipo);
738 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
739 return 0;
740 }
741
742 dprint(FD_VERIFY, "get_next_verify: empty\n");
743 return 1;
744}
745
746static void *verify_async_thread(void *data)
747{
748 struct thread_data *td = data;
749 struct io_u *io_u;
750 int ret = 0;
751
752 if (td->o.verify_cpumask_set &&
753 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
754 log_err("fio: failed setting verify thread affinity\n");
755 goto done;
756 }
757
758 do {
759 FLIST_HEAD(list);
760
761 read_barrier();
762 if (td->verify_thread_exit)
763 break;
764
765 pthread_mutex_lock(&td->io_u_lock);
766
767 while (flist_empty(&td->verify_list) &&
768 !td->verify_thread_exit) {
769 ret = pthread_cond_wait(&td->verify_cond,
770 &td->io_u_lock);
771 if (ret) {
772 pthread_mutex_unlock(&td->io_u_lock);
773 break;
774 }
775 }
776
777 flist_splice_init(&td->verify_list, &list);
778 pthread_mutex_unlock(&td->io_u_lock);
779
780 if (flist_empty(&list))
781 continue;
782
783 while (!flist_empty(&list)) {
784 io_u = flist_entry(list.next, struct io_u, list);
785 flist_del_init(&io_u->list);
786
787 ret = verify_io_u(td, io_u);
788 put_io_u(td, io_u);
789 if (!ret)
790 continue;
791 if (td->o.continue_on_error &&
792 td_non_fatal_error(ret)) {
793 update_error_count(td, ret);
794 td_clear_error(td);
795 ret = 0;
796 }
797 }
798 } while (!ret);
799
800 if (ret) {
801 td_verror(td, ret, "async_verify");
802 td->terminate = 1;
803 }
804
805done:
806 pthread_mutex_lock(&td->io_u_lock);
807 td->nr_verify_threads--;
808 pthread_mutex_unlock(&td->io_u_lock);
809
810 pthread_cond_signal(&td->free_cond);
811 return NULL;
812}
813
814int verify_async_init(struct thread_data *td)
815{
816 int i, ret;
817
818 td->verify_thread_exit = 0;
819
820 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
821 for (i = 0; i < td->o.verify_async; i++) {
822 ret = pthread_create(&td->verify_threads[i], NULL,
823 verify_async_thread, td);
824 if (ret) {
825 log_err("fio: async verify creation failed: %s\n",
826 strerror(ret));
827 break;
828 }
829 ret = pthread_detach(td->verify_threads[i]);
830 if (ret) {
831 log_err("fio: async verify thread detach failed: %s\n",
832 strerror(ret));
833 break;
834 }
835 td->nr_verify_threads++;
836 }
837
838 if (i != td->o.verify_async) {
839 td->verify_thread_exit = 1;
840 write_barrier();
841 pthread_cond_broadcast(&td->verify_cond);
842 return 1;
843 }
844
845 return 0;
846}
847
848void verify_async_exit(struct thread_data *td)
849{
850 td->verify_thread_exit = 1;
851 write_barrier();
852 pthread_cond_broadcast(&td->verify_cond);
853
854 pthread_mutex_lock(&td->io_u_lock);
855
856 while (td->nr_verify_threads)
857 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
858
859 pthread_mutex_unlock(&td->io_u_lock);
860 free(td->verify_threads);
861 td->verify_threads = NULL;
862}
fio - Flexible IO Tester