2 * IO verification helpers
16 #include "crc/crc64.h"
17 #include "crc/crc32.h"
18 #include "crc/crc32c.h"
19 #include "crc/crc16.h"
21 #include "crc/sha256.h"
22 #include "crc/sha512.h"
25 void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u)
27 switch (td->o.verify_pattern_bytes) {
29 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
30 fill_random_buf(p, len);
33 if (io_u->buf_filled_len >= len) {
34 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
37 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
38 memset(p, td->o.verify_pattern[0], len);
39 io_u->buf_filled_len = len;
42 unsigned int i = 0, size = 0;
45 if (io_u->buf_filled_len >= len) {
46 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
47 td->o.verify_pattern_bytes, len);
50 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
51 td->o.verify_pattern_bytes, len);
54 size = td->o.verify_pattern_bytes;
57 memcpy(b+i, td->o.verify_pattern, size);
60 io_u->buf_filled_len = len;
66 static void memswp(void *buf1, void *buf2, unsigned int len)
70 assert(len <= sizeof(swap));
72 memcpy(&swap, buf1, len);
73 memcpy(buf1, buf2, len);
74 memcpy(buf2, &swap, len);
77 static void hexdump(void *buffer, int len)
79 unsigned char *p = buffer;
82 for (i = 0; i < len; i++)
83 log_err("%02x", p[i]);
88 * Prepare for seperation of verify_header and checksum header
90 static inline unsigned int __hdr_size(int verify_type)
92 unsigned int len = len;
94 switch (verify_type) {
100 len = sizeof(struct vhdr_md5);
103 len = sizeof(struct vhdr_crc64);
107 case VERIFY_CRC32C_INTEL:
108 len = sizeof(struct vhdr_crc32);
111 len = sizeof(struct vhdr_crc16);
114 len = sizeof(struct vhdr_crc7);
117 len = sizeof(struct vhdr_sha256);
120 len = sizeof(struct vhdr_sha512);
123 len = sizeof(struct vhdr_meta);
126 len = sizeof(struct vhdr_sha1);
129 log_err("fio: unknown verify header!\n");
133 return len + sizeof(struct verify_header);
136 static inline unsigned int hdr_size(struct verify_header *hdr)
138 return __hdr_size(hdr->verify_type);
141 static void *hdr_priv(struct verify_header *hdr)
145 return priv + sizeof(struct verify_header);
149 * Verify container, pass info to verify handlers and allow them to
150 * pass info back in case of error
157 unsigned int hdr_num;
160 * Output, only valid in case of error
165 unsigned int crc_len;
168 static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
170 unsigned long long offset;
172 offset = vc->io_u->offset;
173 offset += vc->hdr_num * hdr->len;
174 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
175 vc->name, vc->io_u->file->file_name, offset, hdr->len);
177 if (vc->good_crc && vc->bad_crc) {
178 log_err(" Expected CRC: ");
179 hexdump(vc->good_crc, vc->crc_len);
180 log_err(" Received CRC: ");
181 hexdump(vc->bad_crc, vc->crc_len);
186 * Return data area 'header_num'
188 static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
190 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
193 static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
196 struct vhdr_meta *vh = hdr_priv(hdr);
197 struct io_u *io_u = vc->io_u;
199 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
201 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
205 log_verify_failure(hdr, vc);
209 static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
211 void *p = io_u_verify_off(hdr, vc);
212 struct vhdr_sha512 *vh = hdr_priv(hdr);
214 struct sha512_ctx sha512_ctx = {
218 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
220 sha512_init(&sha512_ctx);
221 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
223 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
227 vc->good_crc = vh->sha512;
228 vc->bad_crc = sha512_ctx.buf;
229 vc->crc_len = sizeof(vh->sha512);
230 log_verify_failure(hdr, vc);
234 static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
236 void *p = io_u_verify_off(hdr, vc);
237 struct vhdr_sha256 *vh = hdr_priv(hdr);
239 struct sha256_ctx sha256_ctx = {
243 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
245 sha256_init(&sha256_ctx);
246 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
248 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
252 vc->good_crc = vh->sha256;
253 vc->bad_crc = sha256_ctx.buf;
254 vc->crc_len = sizeof(vh->sha256);
255 log_verify_failure(hdr, vc);
259 static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
261 void *p = io_u_verify_off(hdr, vc);
262 struct vhdr_sha1 *vh = hdr_priv(hdr);
264 struct sha1_ctx sha1_ctx = {
268 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
270 sha1_init(&sha1_ctx);
271 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
273 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
277 vc->good_crc = vh->sha1;
278 vc->bad_crc = sha1_ctx.H;
279 vc->crc_len = sizeof(vh->sha1);
280 log_verify_failure(hdr, vc);
284 static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
286 void *p = io_u_verify_off(hdr, vc);
287 struct vhdr_crc7 *vh = hdr_priv(hdr);
290 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
292 c = crc7(p, hdr->len - hdr_size(hdr));
298 vc->good_crc = &vh->crc7;
301 log_verify_failure(hdr, vc);
305 static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
307 void *p = io_u_verify_off(hdr, vc);
308 struct vhdr_crc16 *vh = hdr_priv(hdr);
311 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
313 c = crc16(p, hdr->len - hdr_size(hdr));
319 vc->good_crc = &vh->crc16;
322 log_verify_failure(hdr, vc);
326 static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
328 void *p = io_u_verify_off(hdr, vc);
329 struct vhdr_crc64 *vh = hdr_priv(hdr);
330 unsigned long long c;
332 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
334 c = crc64(p, hdr->len - hdr_size(hdr));
340 vc->good_crc = &vh->crc64;
343 log_verify_failure(hdr, vc);
347 static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
349 void *p = io_u_verify_off(hdr, vc);
350 struct vhdr_crc32 *vh = hdr_priv(hdr);
353 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
355 c = crc32(p, hdr->len - hdr_size(hdr));
361 vc->good_crc = &vh->crc32;
364 log_verify_failure(hdr, vc);
368 static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
370 void *p = io_u_verify_off(hdr, vc);
371 struct vhdr_crc32 *vh = hdr_priv(hdr);
374 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
376 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
377 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
379 c = crc32c(p, hdr->len - hdr_size(hdr));
385 vc->good_crc = &vh->crc32;
388 log_verify_failure(hdr, vc);
392 static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
394 void *p = io_u_verify_off(hdr, vc);
395 struct vhdr_md5 *vh = hdr_priv(hdr);
396 uint32_t hash[MD5_HASH_WORDS];
397 struct md5_ctx md5_ctx = {
401 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
404 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
406 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
410 vc->good_crc = vh->md5_digest;
411 vc->bad_crc = md5_ctx.hash;
412 vc->crc_len = sizeof(hash);
413 log_verify_failure(hdr, vc);
417 static unsigned int hweight8(unsigned int w)
419 unsigned int res = w - ((w >> 1) & 0x55);
421 res = (res & 0x33) + ((res >> 2) & 0x33);
422 return (res + (res >> 4)) & 0x0F;
425 int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
426 char *buf, unsigned int len, unsigned int mod)
430 for (i = 0; i < len; i++) {
431 if (buf[i] != pattern[mod]) {
434 bits = hweight8(buf[i] ^ pattern[mod]);
435 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
436 buf[i], pattern[mod], bits);
437 log_err("fio: bad pattern block offset %u\n", i);
441 if (mod == pattern_size)
449 * Push IO verification to a separate thread
451 int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
454 put_file_log(td, io_u->file);
458 pthread_mutex_lock(&td->io_u_lock);
460 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
462 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
464 flist_del(&io_u->list);
465 flist_add_tail(&io_u->list, &td->verify_list);
466 io_u->flags |= IO_U_F_FREE_DEF;
467 pthread_mutex_unlock(&td->io_u_lock);
469 pthread_cond_signal(&td->verify_cond);
473 int verify_io_u(struct thread_data *td, struct io_u *io_u)
475 struct verify_header *hdr;
476 unsigned int hdr_size, hdr_inc, hdr_num = 0;
480 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
483 hdr_inc = io_u->buflen;
484 if (td->o.verify_interval)
485 hdr_inc = td->o.verify_interval;
488 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
489 p += hdr_inc, hdr_num++) {
495 if (ret && td->o.verify_fatal)
498 hdr_size = __hdr_size(td->o.verify);
499 if (td->o.verify_offset)
500 memswp(p, p + td->o.verify_offset, hdr_size);
503 if (hdr->fio_magic != FIO_HDR_MAGIC) {
504 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
505 hdr->fio_magic, FIO_HDR_MAGIC,
506 io_u->file->file_name,
507 io_u->offset + hdr_num * hdr->len, hdr->len);
511 if (td->o.verify_pattern_bytes) {
512 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
514 ret = verify_io_u_pattern(td->o.verify_pattern,
515 td->o.verify_pattern_bytes,
518 hdr_size % td->o.verify_pattern_bytes);
521 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
522 io_u->file->file_name,
523 io_u->offset + hdr_num * hdr->len,
528 * Also verify the meta data, if applicable
530 if (hdr->verify_type == VERIFY_META)
531 ret |= verify_io_u_meta(hdr, td, &vc);
535 switch (hdr->verify_type) {
537 ret = verify_io_u_md5(hdr, &vc);
540 ret = verify_io_u_crc64(hdr, &vc);
543 case VERIFY_CRC32C_INTEL:
544 ret = verify_io_u_crc32c(hdr, &vc);
547 ret = verify_io_u_crc32(hdr, &vc);
550 ret = verify_io_u_crc16(hdr, &vc);
553 ret = verify_io_u_crc7(hdr, &vc);
556 ret = verify_io_u_sha256(hdr, &vc);
559 ret = verify_io_u_sha512(hdr, &vc);
562 ret = verify_io_u_meta(hdr, td, &vc);
565 ret = verify_io_u_sha1(hdr, &vc);
568 log_err("Bad verify type %u\n", hdr->verify_type);
573 if (ret && td->o.verify_fatal)
579 static void fill_meta(struct verify_header *hdr, struct thread_data *td,
580 struct io_u *io_u, unsigned int header_num)
582 struct vhdr_meta *vh = hdr_priv(hdr);
584 vh->thread = td->thread_number;
586 vh->time_sec = io_u->start_time.tv_sec;
587 vh->time_usec = io_u->start_time.tv_usec;
589 vh->numberio = td->io_issues[DDIR_WRITE];
591 vh->offset = io_u->offset + header_num * td->o.verify_interval;
594 static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
596 struct vhdr_sha512 *vh = hdr_priv(hdr);
597 struct sha512_ctx sha512_ctx = {
601 sha512_init(&sha512_ctx);
602 sha512_update(&sha512_ctx, p, len);
605 static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
607 struct vhdr_sha256 *vh = hdr_priv(hdr);
608 struct sha256_ctx sha256_ctx = {
612 sha256_init(&sha256_ctx);
613 sha256_update(&sha256_ctx, p, len);
616 static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
618 struct vhdr_sha1 *vh = hdr_priv(hdr);
619 struct sha1_ctx sha1_ctx = {
623 sha1_init(&sha1_ctx);
624 sha1_update(&sha1_ctx, p, len);
627 static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
629 struct vhdr_crc7 *vh = hdr_priv(hdr);
631 vh->crc7 = crc7(p, len);
634 static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
636 struct vhdr_crc16 *vh = hdr_priv(hdr);
638 vh->crc16 = crc16(p, len);
641 static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
643 struct vhdr_crc32 *vh = hdr_priv(hdr);
645 vh->crc32 = crc32(p, len);
648 static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
650 struct vhdr_crc32 *vh = hdr_priv(hdr);
652 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
653 vh->crc32 = crc32c_intel(p, len);
655 vh->crc32 = crc32c(p, len);
658 static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
660 struct vhdr_crc64 *vh = hdr_priv(hdr);
662 vh->crc64 = crc64(p, len);
665 static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
667 struct vhdr_md5 *vh = hdr_priv(hdr);
668 struct md5_ctx md5_ctx = {
669 .hash = (uint32_t *) vh->md5_digest,
673 md5_update(&md5_ctx, p, len);
677 * fill body of io_u->buf with random data and add a header with the
678 * crc32 or md5 sum of that data.
680 void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
682 struct verify_header *hdr;
683 void *p = io_u->buf, *data;
684 unsigned int hdr_inc, data_len, header_num = 0;
686 if (td->o.verify == VERIFY_NULL)
689 fill_pattern(td, p, io_u->buflen, io_u);
691 hdr_inc = io_u->buflen;
692 if (td->o.verify_interval)
693 hdr_inc = td->o.verify_interval;
695 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
698 hdr->fio_magic = FIO_HDR_MAGIC;
699 hdr->verify_type = td->o.verify;
701 data_len = hdr_inc - hdr_size(hdr);
703 data = p + hdr_size(hdr);
704 switch (td->o.verify) {
706 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
708 fill_md5(hdr, data, data_len);
711 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
713 fill_crc64(hdr, data, data_len);
716 case VERIFY_CRC32C_INTEL:
717 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
719 fill_crc32c(hdr, data, data_len);
722 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
724 fill_crc32(hdr, data, data_len);
727 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
729 fill_crc16(hdr, data, data_len);
732 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
734 fill_crc7(hdr, data, data_len);
737 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
739 fill_sha256(hdr, data, data_len);
742 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
744 fill_sha512(hdr, data, data_len);
747 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
749 fill_meta(hdr, td, io_u, header_num);
752 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
754 fill_sha1(hdr, data, data_len);
757 log_err("fio: bad verify type: %d\n", td->o.verify);
760 if (td->o.verify_offset)
761 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
766 int get_next_verify(struct thread_data *td, struct io_u *io_u)
768 struct io_piece *ipo = NULL;
771 * this io_u is from a requeue, we already filled the offsets
776 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
777 struct rb_node *n = rb_first(&td->io_hist_tree);
779 ipo = rb_entry(n, struct io_piece, rb_node);
780 rb_erase(n, &td->io_hist_tree);
782 } else if (!flist_empty(&td->io_hist_list)) {
783 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
785 flist_del(&ipo->list);
789 io_u->offset = ipo->offset;
790 io_u->buflen = ipo->len;
791 io_u->file = ipo->file;
793 if (!fio_file_open(io_u->file)) {
794 int r = td_io_open_file(td, io_u->file);
797 dprint(FD_VERIFY, "failed file %s open\n",
798 io_u->file->file_name);
804 assert(fio_file_open(io_u->file));
805 io_u->ddir = DDIR_READ;
806 io_u->xfer_buf = io_u->buf;
807 io_u->xfer_buflen = io_u->buflen;
809 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
813 dprint(FD_VERIFY, "get_next_verify: empty\n");
817 static void *verify_async_thread(void *data)
819 struct thread_data *td = data;
823 if (td->o.verify_cpumask_set &&
824 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
825 log_err("fio: failed setting verify thread affinity\n");
833 if (td->verify_thread_exit)
836 pthread_mutex_lock(&td->io_u_lock);
838 while (flist_empty(&td->verify_list) &&
839 !td->verify_thread_exit) {
840 ret = pthread_cond_wait(&td->verify_cond,
843 pthread_mutex_unlock(&td->io_u_lock);
848 flist_splice_init(&td->verify_list, &list);
849 pthread_mutex_unlock(&td->io_u_lock);
851 if (flist_empty(&list))
854 while (!flist_empty(&list)) {
855 io_u = flist_entry(list.next, struct io_u, list);
856 flist_del_init(&io_u->list);
858 ret = verify_io_u(td, io_u);
862 if (td->o.continue_on_error &&
863 td_non_fatal_error(ret)) {
864 update_error_count(td, ret);
872 td_verror(td, ret, "async_verify");
873 if (td->o.verify_fatal)
878 pthread_mutex_lock(&td->io_u_lock);
879 td->nr_verify_threads--;
880 pthread_mutex_unlock(&td->io_u_lock);
882 pthread_cond_signal(&td->free_cond);
886 int verify_async_init(struct thread_data *td)
890 td->verify_thread_exit = 0;
892 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
893 for (i = 0; i < td->o.verify_async; i++) {
894 ret = pthread_create(&td->verify_threads[i], NULL,
895 verify_async_thread, td);
897 log_err("fio: async verify creation failed: %s\n",
901 ret = pthread_detach(td->verify_threads[i]);
903 log_err("fio: async verify thread detach failed: %s\n",
907 td->nr_verify_threads++;
910 if (i != td->o.verify_async) {
911 log_err("fio: only %d verify threads started, exiting\n", i);
912 td->verify_thread_exit = 1;
914 pthread_cond_broadcast(&td->verify_cond);
921 void verify_async_exit(struct thread_data *td)
923 td->verify_thread_exit = 1;
925 pthread_cond_broadcast(&td->verify_cond);
927 pthread_mutex_lock(&td->io_u_lock);
929 while (td->nr_verify_threads)
930 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
932 pthread_mutex_unlock(&td->io_u_lock);
933 free(td->verify_threads);
934 td->verify_threads = NULL;