projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Add strong madvise() hint for cache pruning
[fio.git] / verify.c
... / ...
CommitLineData
1/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
7#include <assert.h>
8#include <pthread.h>
9
10#include "fio.h"
11#include "verify.h"
12#include "smalloc.h"
13
14#include "crc/md5.h"
15#include "crc/crc64.h"
16#include "crc/crc32.h"
17#include "crc/crc32c.h"
18#include "crc/crc16.h"
19#include "crc/crc7.h"
20#include "crc/sha256.h"
21#include "crc/sha512.h"
22
23static void fill_random_bytes(struct thread_data *td, void *p, unsigned int len)
24{
25 unsigned int todo;
26 int r;
27
28 while (len) {
29 r = os_random_long(&td->verify_state);
30
31 /*
32 * lrand48_r seems to be broken and only fill the bottom
33 * 32-bits, even on 64-bit archs with 64-bit longs
34 */
35 todo = sizeof(r);
36 if (todo > len)
37 todo = len;
38
39 memcpy(p, &r, todo);
40
41 len -= todo;
42 p += todo;
43 }
44}
45
46static void fill_pattern(struct thread_data *td, void *p, unsigned int len)
47{
48 switch (td->o.verify_pattern_bytes) {
49 case 0:
50 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
51 fill_random_bytes(td, p, len);
52 break;
53 case 1:
54 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
55 memset(p, td->o.verify_pattern, len);
56 break;
57 case 2:
58 case 3:
59 case 4: {
60 unsigned int pattern = td->o.verify_pattern;
61 unsigned int i = 0;
62 unsigned char c1, c2, c3, c4;
63 unsigned char *b = p;
64
65 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
66 td->o.verify_pattern_bytes, len);
67
68 c1 = pattern & 0xff;
69 pattern >>= 8;
70 c2 = pattern & 0xff;
71 pattern >>= 8;
72 c3 = pattern & 0xff;
73 pattern >>= 8;
74 c4 = pattern & 0xff;
75
76 while (i < len) {
77 b[i++] = c1;
78 if (i == len)
79 break;
80 b[i++] = c2;
81 if (td->o.verify_pattern_bytes == 2 || i == len)
82 continue;
83 b[i++] = c3;
84 if (td->o.verify_pattern_bytes == 3 || i == len)
85 continue;
86 b[i++] = c4;
87 }
88 break;
89 }
90 }
91}
92
93static void memswp(void *buf1, void *buf2, unsigned int len)
94{
95 char swap[200];
96
97 assert(len <= sizeof(swap));
98
99 memcpy(&swap, buf1, len);
100 memcpy(buf1, buf2, len);
101 memcpy(buf2, &swap, len);
102}
103
104static void hexdump(void *buffer, int len)
105{
106 unsigned char *p = buffer;
107 int i;
108
109 for (i = 0; i < len; i++)
110 log_info("%02x", p[i]);
111 log_info("\n");
112}
113
114/*
115 * Prepare for seperation of verify_header and checksum header
116 */
117static inline unsigned int __hdr_size(int verify_type)
118{
119 unsigned int len = len;
120
121 switch (verify_type) {
122 case VERIFY_NONE:
123 case VERIFY_NULL:
124 len = 0;
125 break;
126 case VERIFY_MD5:
127 len = sizeof(struct vhdr_md5);
128 break;
129 case VERIFY_CRC64:
130 len = sizeof(struct vhdr_crc64);
131 break;
132 case VERIFY_CRC32C:
133 case VERIFY_CRC32:
134 case VERIFY_CRC32C_INTEL:
135 len = sizeof(struct vhdr_crc32);
136 break;
137 case VERIFY_CRC16:
138 len = sizeof(struct vhdr_crc16);
139 break;
140 case VERIFY_CRC7:
141 len = sizeof(struct vhdr_crc7);
142 break;
143 case VERIFY_SHA256:
144 len = sizeof(struct vhdr_sha256);
145 break;
146 case VERIFY_SHA512:
147 len = sizeof(struct vhdr_sha512);
148 break;
149 case VERIFY_META:
150 len = sizeof(struct vhdr_meta);
151 break;
152 default:
153 log_err("fio: unknown verify header!\n");
154 assert(0);
155 }
156
157 return len + sizeof(struct verify_header);
158}
159
160static inline unsigned int hdr_size(struct verify_header *hdr)
161{
162 return __hdr_size(hdr->verify_type);
163}
164
165static void *hdr_priv(struct verify_header *hdr)
166{
167 void *priv = hdr;
168
169 return priv + sizeof(struct verify_header);
170}
171
172/*
173 * Return data area 'header_num'
174 */
175static inline void *io_u_verify_off(struct verify_header *hdr,
176 struct io_u *io_u, unsigned char header_num)
177{
178 return io_u->buf + header_num * hdr->len + hdr_size(hdr);
179}
180
181static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
182 struct io_u *io_u, unsigned int header_num)
183{
184 struct vhdr_meta *vh = hdr_priv(hdr);
185
186 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
187
188 if (vh->offset != io_u->offset + header_num * td->o.verify_interval) {
189 log_err("meta: verify failed at %llu/%u\n",
190 io_u->offset + header_num * hdr->len, hdr->len);
191 return EILSEQ;
192 }
193
194 return 0;
195}
196
197static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u,
198 unsigned int header_num)
199{
200 void *p = io_u_verify_off(hdr, io_u, header_num);
201 struct vhdr_sha512 *vh = hdr_priv(hdr);
202 uint8_t sha512[128];
203 struct sha512_ctx sha512_ctx = {
204 .buf = sha512,
205 };
206
207 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", io_u, hdr->len);
208
209 sha512_init(&sha512_ctx);
210 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
211
212 if (memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) {
213 log_err("sha512: verify failed at %llu/%u\n",
214 io_u->offset + header_num * hdr->len, hdr->len);
215 hexdump(vh->sha512, sizeof(vh->sha512));
216 hexdump(sha512_ctx.buf, sizeof(sha512));
217 return EILSEQ;
218 }
219
220 return 0;
221}
222
223static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u,
224 unsigned int header_num)
225{
226 void *p = io_u_verify_off(hdr, io_u, header_num);
227 struct vhdr_sha256 *vh = hdr_priv(hdr);
228 uint8_t sha256[128];
229 struct sha256_ctx sha256_ctx = {
230 .buf = sha256,
231 };
232
233 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", io_u, hdr->len);
234
235 sha256_init(&sha256_ctx);
236 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
237
238 if (memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) {
239 log_err("sha256: verify failed at %llu/%u\n",
240 io_u->offset + header_num * hdr->len, hdr->len);
241 hexdump(vh->sha256, sizeof(vh->sha256));
242 hexdump(sha256_ctx.buf, sizeof(sha256));
243 return EILSEQ;
244 }
245
246 return 0;
247}
248
249static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u,
250 unsigned char header_num)
251{
252 void *p = io_u_verify_off(hdr, io_u, header_num);
253 struct vhdr_crc7 *vh = hdr_priv(hdr);
254 unsigned char c;
255
256 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", io_u, hdr->len);
257
258 c = crc7(p, hdr->len - hdr_size(hdr));
259
260 if (c != vh->crc7) {
261 log_err("crc7: verify failed at %llu/%u\n",
262 io_u->offset + header_num * hdr->len, hdr->len);
263 log_err("crc7: wanted %x, got %x\n", vh->crc7, c);
264 return EILSEQ;
265 }
266
267 return 0;
268}
269
270static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u,
271 unsigned int header_num)
272{
273 void *p = io_u_verify_off(hdr, io_u, header_num);
274 struct vhdr_crc16 *vh = hdr_priv(hdr);
275 unsigned short c;
276
277 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", io_u, hdr->len);
278
279 c = crc16(p, hdr->len - hdr_size(hdr));
280
281 if (c != vh->crc16) {
282 log_err("crc16: verify failed at %llu/%u\n",
283 io_u->offset + header_num * hdr->len, hdr->len);
284 log_err("crc16: wanted %x, got %x\n", vh->crc16, c);
285 return EILSEQ;
286 }
287
288 return 0;
289}
290
291static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u,
292 unsigned int header_num)
293{
294 void *p = io_u_verify_off(hdr, io_u, header_num);
295 struct vhdr_crc64 *vh = hdr_priv(hdr);
296 unsigned long long c;
297
298 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", io_u, hdr->len);
299
300 c = crc64(p, hdr->len - hdr_size(hdr));
301
302 if (c != vh->crc64) {
303 log_err("crc64: verify failed at %llu/%u\n",
304 io_u->offset + header_num * hdr->len,
305 hdr->len);
306 log_err("crc64: wanted %llx, got %llx\n",
307 (unsigned long long) vh->crc64, c);
308 return EILSEQ;
309 }
310
311 return 0;
312}
313
314static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u,
315 unsigned int header_num)
316{
317 void *p = io_u_verify_off(hdr, io_u, header_num);
318 struct vhdr_crc32 *vh = hdr_priv(hdr);
319 uint32_t c;
320
321 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", io_u, hdr->len);
322
323 c = crc32(p, hdr->len - hdr_size(hdr));
324
325 if (c != vh->crc32) {
326 log_err("crc32: verify failed at %llu/%u\n",
327 io_u->offset + header_num * hdr->len, hdr->len);
328 log_err("crc32: wanted %x, got %x\n", vh->crc32, c);
329 return EILSEQ;
330 }
331
332 return 0;
333}
334
335static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u,
336 unsigned int header_num)
337{
338 void *p = io_u_verify_off(hdr, io_u, header_num);
339 struct vhdr_crc32 *vh = hdr_priv(hdr);
340 uint32_t c;
341
342 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len);
343
344 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
345 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
346 else
347 c = crc32c(p, hdr->len - hdr_size(hdr));
348
349 if (c != vh->crc32) {
350 log_err("crc32c: verify failed at %llu/%u\n",
351 io_u->offset + header_num * hdr->len, hdr->len);
352 log_err("crc32c: wanted %x, got %x\n", vh->crc32, c);
353 return EILSEQ;
354 }
355
356 return 0;
357}
358
359static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u,
360 unsigned int header_num)
361{
362 void *p = io_u_verify_off(hdr, io_u, header_num);
363 struct vhdr_md5 *vh = hdr_priv(hdr);
364 uint32_t hash[MD5_HASH_WORDS];
365 struct md5_ctx md5_ctx = {
366 .hash = hash,
367 };
368
369 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", io_u, hdr->len);
370
371 md5_init(&md5_ctx);
372 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
373
374 if (memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) {
375 log_err("md5: verify failed at %llu/%u\n",
376 io_u->offset + header_num * hdr->len, hdr->len);
377 hexdump(vh->md5_digest, sizeof(vh->md5_digest));
378 hexdump(md5_ctx.hash, sizeof(hash));
379 return EILSEQ;
380 }
381
382 return 0;
383}
384
385static unsigned int hweight8(unsigned int w)
386{
387 unsigned int res = w - ((w >> 1) & 0x55);
388
389 res = (res & 0x33) + ((res >> 2) & 0x33);
390 return (res + (res >> 4)) & 0x0F;
391}
392
393int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size,
394 char *buf, unsigned int len, unsigned int mod)
395{
396 unsigned int i;
397 char split_pattern[4];
398
399 for (i = 0; i < 4; i++) {
400 split_pattern[i] = pattern & 0xff;
401 pattern >>= 8;
402 }
403
404 for (i = 0; i < len; i++) {
405 if (buf[i] != split_pattern[mod]) {
406 unsigned int bits;
407
408 bits = hweight8(buf[i] ^ split_pattern[mod]);
409 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
410 buf[i], split_pattern[mod], bits);
411 log_err("fio: bad pattern block offset %u\n", i);
412 return EILSEQ;
413 }
414 mod++;
415 if (mod == pattern_size)
416 mod = 0;
417 }
418
419 return 0;
420}
421
422/*
423 * Push IO verification to a separate thread
424 */
425int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
426{
427 if (io_u->file)
428 put_file_log(td, io_u->file);
429
430 io_u->file = NULL;
431
432 pthread_mutex_lock(&td->io_u_lock);
433 flist_del(&io_u->list);
434 flist_add_tail(&io_u->list, &td->verify_list);
435 pthread_mutex_unlock(&td->io_u_lock);
436
437 pthread_cond_signal(&td->verify_cond);
438 io_u->flags |= IO_U_F_FREE_DEF;
439 return 0;
440}
441
442int verify_io_u(struct thread_data *td, struct io_u *io_u)
443{
444 struct verify_header *hdr;
445 unsigned int hdr_size, hdr_inc, hdr_num = 0;
446 void *p;
447 int ret;
448
449 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
450 return 0;
451
452 hdr_inc = io_u->buflen;
453 if (td->o.verify_interval)
454 hdr_inc = td->o.verify_interval;
455
456 ret = 0;
457 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
458 p += hdr_inc, hdr_num++) {
459 if (ret && td->o.verify_fatal) {
460 td->terminate = 1;
461 break;
462 }
463 hdr_size = __hdr_size(td->o.verify);
464 if (td->o.verify_offset)
465 memswp(p, p + td->o.verify_offset, hdr_size);
466 hdr = p;
467
468 if (hdr->fio_magic != FIO_HDR_MAGIC) {
469 log_err("Bad verify header %x\n", hdr->fio_magic);
470 return EILSEQ;
471 }
472
473 if (td->o.verify_pattern_bytes) {
474 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
475 io_u, hdr->len);
476 ret = verify_io_u_pattern(td->o.verify_pattern,
477 td->o.verify_pattern_bytes,
478 p + hdr_size,
479 hdr_inc - hdr_size,
480 hdr_size % 4);
481 /*
482 * Also verify the meta data, if applicable
483 */
484 if (hdr->verify_type == VERIFY_META)
485 ret |= verify_io_u_meta(hdr, td, io_u, hdr_num);
486
487 if (ret)
488 log_err("fio: verify failed at %llu/%u\n",
489 io_u->offset + hdr_num * hdr->len,
490 hdr->len);
491 continue;
492 }
493
494 switch (hdr->verify_type) {
495 case VERIFY_MD5:
496 ret = verify_io_u_md5(hdr, io_u, hdr_num);
497 break;
498 case VERIFY_CRC64:
499 ret = verify_io_u_crc64(hdr, io_u, hdr_num);
500 break;
501 case VERIFY_CRC32C:
502 case VERIFY_CRC32C_INTEL:
503 ret = verify_io_u_crc32c(hdr, io_u, hdr_num);
504 break;
505 case VERIFY_CRC32:
506 ret = verify_io_u_crc32(hdr, io_u, hdr_num);
507 break;
508 case VERIFY_CRC16:
509 ret = verify_io_u_crc16(hdr, io_u, hdr_num);
510 break;
511 case VERIFY_CRC7:
512 ret = verify_io_u_crc7(hdr, io_u, hdr_num);
513 break;
514 case VERIFY_SHA256:
515 ret = verify_io_u_sha256(hdr, io_u, hdr_num);
516 break;
517 case VERIFY_SHA512:
518 ret = verify_io_u_sha512(hdr, io_u, hdr_num);
519 break;
520 case VERIFY_META:
521 ret = verify_io_u_meta(hdr, td, io_u, hdr_num);
522 break;
523 default:
524 log_err("Bad verify type %u\n", hdr->verify_type);
525 ret = EINVAL;
526 }
527 }
528
529 return ret;
530}
531
532static void fill_meta(struct verify_header *hdr, struct thread_data *td,
533 struct io_u *io_u, unsigned int header_num)
534{
535 struct vhdr_meta *vh = hdr_priv(hdr);
536
537 vh->thread = td->thread_number;
538
539 vh->time_sec = io_u->start_time.tv_sec;
540 vh->time_usec = io_u->start_time.tv_usec;
541
542 vh->numberio = td->io_issues[DDIR_WRITE];
543
544 vh->offset = io_u->offset + header_num * td->o.verify_interval;
545}
546
547static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
548{
549 struct vhdr_sha512 *vh = hdr_priv(hdr);
550 struct sha512_ctx sha512_ctx = {
551 .buf = vh->sha512,
552 };
553
554 sha512_init(&sha512_ctx);
555 sha512_update(&sha512_ctx, p, len);
556}
557
558static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
559{
560 struct vhdr_sha256 *vh = hdr_priv(hdr);
561 struct sha256_ctx sha256_ctx = {
562 .buf = vh->sha256,
563 };
564
565 sha256_init(&sha256_ctx);
566 sha256_update(&sha256_ctx, p, len);
567}
568
569static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
570{
571 struct vhdr_crc7 *vh = hdr_priv(hdr);
572
573 vh->crc7 = crc7(p, len);
574}
575
576static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
577{
578 struct vhdr_crc16 *vh = hdr_priv(hdr);
579
580 vh->crc16 = crc16(p, len);
581}
582
583static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
584{
585 struct vhdr_crc32 *vh = hdr_priv(hdr);
586
587 vh->crc32 = crc32(p, len);
588}
589
590static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
591{
592 struct vhdr_crc32 *vh = hdr_priv(hdr);
593
594 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
595 vh->crc32 = crc32c_intel(p, len);
596 else
597 vh->crc32 = crc32c(p, len);
598}
599
600static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
601{
602 struct vhdr_crc64 *vh = hdr_priv(hdr);
603
604 vh->crc64 = crc64(p, len);
605}
606
607static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
608{
609 struct vhdr_md5 *vh = hdr_priv(hdr);
610 struct md5_ctx md5_ctx = {
611 .hash = (uint32_t *) vh->md5_digest,
612 };
613
614 md5_init(&md5_ctx);
615 md5_update(&md5_ctx, p, len);
616}
617
618/*
619 * fill body of io_u->buf with random data and add a header with the
620 * crc32 or md5 sum of that data.
621 */
622void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
623{
624 struct verify_header *hdr;
625 void *p = io_u->buf, *data;
626 unsigned int hdr_inc, data_len, header_num = 0;
627
628 if (td->o.verify == VERIFY_NULL)
629 return;
630
631 fill_pattern(td, p, io_u->buflen);
632
633 hdr_inc = io_u->buflen;
634 if (td->o.verify_interval)
635 hdr_inc = td->o.verify_interval;
636
637 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
638 hdr = p;
639
640 hdr->fio_magic = FIO_HDR_MAGIC;
641 hdr->verify_type = td->o.verify;
642 hdr->len = hdr_inc;
643 data_len = hdr_inc - hdr_size(hdr);
644
645 data = p + hdr_size(hdr);
646 switch (td->o.verify) {
647 case VERIFY_MD5:
648 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
649 io_u, hdr->len);
650 fill_md5(hdr, data, data_len);
651 break;
652 case VERIFY_CRC64:
653 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
654 io_u, hdr->len);
655 fill_crc64(hdr, data, data_len);
656 break;
657 case VERIFY_CRC32C:
658 case VERIFY_CRC32C_INTEL:
659 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
660 io_u, hdr->len);
661 fill_crc32c(hdr, data, data_len);
662 break;
663 case VERIFY_CRC32:
664 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
665 io_u, hdr->len);
666 fill_crc32(hdr, data, data_len);
667 break;
668 case VERIFY_CRC16:
669 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
670 io_u, hdr->len);
671 fill_crc16(hdr, data, data_len);
672 break;
673 case VERIFY_CRC7:
674 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
675 io_u, hdr->len);
676 fill_crc7(hdr, data, data_len);
677 break;
678 case VERIFY_SHA256:
679 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
680 io_u, hdr->len);
681 fill_sha256(hdr, data, data_len);
682 break;
683 case VERIFY_SHA512:
684 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
685 io_u, hdr->len);
686 fill_sha512(hdr, data, data_len);
687 break;
688 case VERIFY_META:
689 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
690 io_u, hdr->len);
691 fill_meta(hdr, td, io_u, header_num);
692 break;
693 default:
694 log_err("fio: bad verify type: %d\n", td->o.verify);
695 assert(0);
696 }
697 if (td->o.verify_offset)
698 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
699 header_num++;
700 }
701}
702
703int get_next_verify(struct thread_data *td, struct io_u *io_u)
704{
705 struct io_piece *ipo = NULL;
706
707 /*
708 * this io_u is from a requeue, we already filled the offsets
709 */
710 if (io_u->file)
711 return 0;
712
713 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
714 struct rb_node *n = rb_first(&td->io_hist_tree);
715
716 ipo = rb_entry(n, struct io_piece, rb_node);
717 rb_erase(n, &td->io_hist_tree);
718 } else if (!flist_empty(&td->io_hist_list)) {
719 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
720 flist_del(&ipo->list);
721 }
722
723 if (ipo) {
724 io_u->offset = ipo->offset;
725 io_u->buflen = ipo->len;
726 io_u->file = ipo->file;
727
728 if (!fio_file_open(io_u->file)) {
729 int r = td_io_open_file(td, io_u->file);
730
731 if (r) {
732 dprint(FD_VERIFY, "failed file %s open\n",
733 io_u->file->file_name);
734 return 1;
735 }
736 }
737
738 get_file(ipo->file);
739 assert(fio_file_open(io_u->file));
740 io_u->ddir = DDIR_READ;
741 io_u->xfer_buf = io_u->buf;
742 io_u->xfer_buflen = io_u->buflen;
743 free(ipo);
744 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
745 return 0;
746 }
747
748 dprint(FD_VERIFY, "get_next_verify: empty\n");
749 return 1;
750}
751
752static void *verify_async_thread(void *data)
753{
754 struct thread_data *td = data;
755 struct io_u *io_u;
756 int ret = 0;
757
758 if (td->o.verify_cpumask_set &&
759 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
760 log_err("fio: failed setting verify thread affinity\n");
761 goto done;
762 }
763
764 do {
765 FLIST_HEAD(list);
766
767 read_barrier();
768 if (td->verify_thread_exit)
769 break;
770
771 pthread_mutex_lock(&td->io_u_lock);
772
773 while (flist_empty(&td->verify_list) &&
774 !td->verify_thread_exit) {
775 ret = pthread_cond_wait(&td->verify_cond,
776 &td->io_u_lock);
777 if (ret) {
778 pthread_mutex_unlock(&td->io_u_lock);
779 break;
780 }
781 }
782
783 flist_splice_init(&td->verify_list, &list);
784 pthread_mutex_unlock(&td->io_u_lock);
785
786 if (flist_empty(&list))
787 continue;
788
789 while (!flist_empty(&list)) {
790 io_u = flist_entry(list.next, struct io_u, list);
791 flist_del_init(&io_u->list);
792
793 ret = verify_io_u(td, io_u);
794 put_io_u(td, io_u);
795 if (!ret)
796 continue;
797 if (td->o.continue_on_error &&
798 td_non_fatal_error(ret)) {
799 update_error_count(td, ret);
800 td_clear_error(td);
801 ret = 0;
802 }
803 }
804 } while (!ret);
805
806 if (ret) {
807 td_verror(td, ret, "async_verify");
808 td->terminate = 1;
809 }
810
811done:
812 pthread_mutex_lock(&td->io_u_lock);
813 td->nr_verify_threads--;
814 pthread_mutex_unlock(&td->io_u_lock);
815
816 pthread_cond_signal(&td->free_cond);
817 return NULL;
818}
819
820int verify_async_init(struct thread_data *td)
821{
822 int i, ret;
823
824 td->verify_thread_exit = 0;
825
826 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
827 for (i = 0; i < td->o.verify_async; i++) {
828 ret = pthread_create(&td->verify_threads[i], NULL,
829 verify_async_thread, td);
830 if (ret) {
831 log_err("fio: async verify creation failed: %s\n",
832 strerror(ret));
833 break;
834 }
835 ret = pthread_detach(td->verify_threads[i]);
836 if (ret) {
837 log_err("fio: async verify thread detach failed: %s\n",
838 strerror(ret));
839 break;
840 }
841 td->nr_verify_threads++;
842 }
843
844 if (i != td->o.verify_async) {
845 log_err("fio: only %d verify threads started, exiting\n", i);
846 td->verify_thread_exit = 1;
847 write_barrier();
848 pthread_cond_broadcast(&td->verify_cond);
849 return 1;
850 }
851
852 return 0;
853}
854
855void verify_async_exit(struct thread_data *td)
856{
857 td->verify_thread_exit = 1;
858 write_barrier();
859 pthread_cond_broadcast(&td->verify_cond);
860
861 pthread_mutex_lock(&td->io_u_lock);
862
863 while (td->nr_verify_threads)
864 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
865
866 pthread_mutex_unlock(&td->io_u_lock);
867 free(td->verify_threads);
868 td->verify_threads = NULL;
869}
fio - Flexible IO Tester