+ return len + sizeof(struct verify_header);
+}
+
+static inline unsigned int hdr_size(struct thread_data *td,
+ struct verify_header *hdr)
+{
+ if (td->o.verify == VERIFY_PATTERN_NO_HDR)
+ return 0;
+
+ return __hdr_size(hdr->verify_type);
+}
+
+static void *hdr_priv(struct verify_header *hdr)
+{
+ void *priv = hdr;
+
+ return priv + sizeof(struct verify_header);
+}
+
+/*
+ * Verify container, pass info to verify handlers and allow them to
+ * pass info back in case of error
+ */
+struct vcont {
+ /*
+ * Input
+ */
+ struct io_u *io_u;
+ unsigned int hdr_num;
+ struct thread_data *td;
+
+ /*
+ * Output, only valid in case of error
+ */
+ const char *name;
+ void *good_crc;
+ void *bad_crc;
+ unsigned int crc_len;
+};
+
+#define DUMP_BUF_SZ 255
+static int dump_buf_warned;
+
+static void dump_buf(char *buf, unsigned int len, unsigned long long offset,
+ const char *type, struct fio_file *f)
+{
+ char *ptr, fname[DUMP_BUF_SZ];
+ size_t buf_left = DUMP_BUF_SZ;
+ int ret, fd;
+
+ ptr = strdup(f->file_name);
+
+ memset(fname, 0, sizeof(fname));
+ if (aux_path)
+ sprintf(fname, "%s%s", aux_path, FIO_OS_PATH_SEPARATOR);
+
+ strncpy(fname + strlen(fname), basename(ptr), buf_left - 1);
+
+ buf_left -= strlen(fname);
+ if (buf_left <= 0) {
+ if (!dump_buf_warned) {
+ log_err("fio: verify failure dump buffer too small\n");
+ dump_buf_warned = 1;
+ }
+ free(ptr);
+ return;
+ }
+
+ snprintf(fname + strlen(fname), buf_left, ".%llu.%s", offset, type);
+
+ fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
+ if (fd < 0) {
+ perror("open verify buf file");
+ return;
+ }
+
+ while (len) {
+ ret = write(fd, buf, len);
+ if (!ret)
+ break;
+ else if (ret < 0) {
+ perror("write verify buf file");
+ break;
+ }
+ len -= ret;
+ buf += ret;
+ }
+
+ close(fd);
+ log_err(" %s data dumped as %s\n", type, fname);
+ free(ptr);
+}
+
+/*
+ * Dump the contents of the read block and re-generate the correct data
+ * and dump that too.
+ */
+static void __dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
+{
+ struct thread_data *td = vc->td;
+ struct io_u *io_u = vc->io_u;
+ unsigned long hdr_offset;
+ struct io_u dummy;
+ void *buf;
+
+ if (!td->o.verify_dump)
+ return;
+
+ /*
+ * Dump the contents we just read off disk
+ */
+ hdr_offset = vc->hdr_num * hdr->len;
+
+ dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
+ "received", vc->io_u->file);
+
+ /*
+ * Allocate a new buf and re-generate the original data
+ */
+ buf = malloc(io_u->buflen);
+ dummy = *io_u;
+ dummy.buf = buf;
+ dummy.rand_seed = hdr->rand_seed;
+ dummy.buf_filled_len = 0;
+ dummy.buflen = io_u->buflen;
+
+ fill_pattern_headers(td, &dummy, hdr->rand_seed, 1);
+
+ dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
+ "expected", vc->io_u->file);
+ free(buf);
+}
+
+static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
+{
+ struct thread_data *td = vc->td;
+ struct verify_header shdr;
+
+ if (td->o.verify == VERIFY_PATTERN_NO_HDR) {
+ __fill_hdr(td, vc->io_u, &shdr, 0, vc->io_u->buflen, 0);
+ hdr = &shdr;
+ }
+
+ __dump_verify_buffers(hdr, vc);
+}
+
+static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
+{
+ unsigned long long offset;
+
+ offset = vc->io_u->offset;
+ offset += vc->hdr_num * hdr->len;
+ log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
+ vc->name, vc->io_u->file->file_name, offset, hdr->len);
+
+ if (vc->good_crc && vc->bad_crc) {
+ log_err(" Expected CRC: ");
+ hexdump(vc->good_crc, vc->crc_len);
+ log_err(" Received CRC: ");
+ hexdump(vc->bad_crc, vc->crc_len);
+ }
+
+ dump_verify_buffers(hdr, vc);
+}
+
+/*
+ * Return data area 'header_num'
+ */
+static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
+{
+ return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(vc->td, hdr);
+}
+
+static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc)
+{
+ struct thread_data *td = vc->td;
+ struct io_u *io_u = vc->io_u;
+ char *buf, *pattern;
+ unsigned int header_size = __hdr_size(td->o.verify);
+ unsigned int len, mod, i, pattern_size;
+ int rc;
+
+ pattern = td->o.verify_pattern;
+ pattern_size = td->o.verify_pattern_bytes;
+ assert(pattern_size != 0);
+
+ (void)paste_format_inplace(pattern, pattern_size,
+ td->o.verify_fmt, td->o.verify_fmt_sz, io_u);
+
+ buf = (void *) hdr + header_size;
+ len = get_hdr_inc(td, io_u) - header_size;
+ mod = (get_hdr_inc(td, io_u) * vc->hdr_num + header_size) % pattern_size;
+
+ rc = cmp_pattern(pattern, pattern_size, mod, buf, len);
+ if (!rc)
+ return 0;
+
+ /* Slow path, compare each byte */
+ for (i = 0; i < len; i++) {
+ if (buf[i] != pattern[mod]) {
+ unsigned int bits;
+
+ bits = hweight8(buf[i] ^ pattern[mod]);
+ log_err("fio: got pattern '%02x', wanted '%02x'. Bad bits %d\n",
+ (unsigned char)buf[i],
+ (unsigned char)pattern[mod],
+ bits);
+ log_err("fio: bad pattern block offset %u\n", i);
+ dump_verify_buffers(hdr, vc);
+ return EILSEQ;
+ }
+ mod++;
+ if (mod == td->o.verify_pattern_bytes)
+ mod = 0;
+ }
+
+ /* Unreachable line */
+ assert(0);
+ return EILSEQ;
+}
+
+static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_xxhash *vh = hdr_priv(hdr);
+ uint32_t hash;
+ void *state;
+
+ dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ state = XXH32_init(1);
+ XXH32_update(state, p, hdr->len - hdr_size(vc->td, hdr));
+ hash = XXH32_digest(state);
+
+ if (vh->hash == hash)
+ return 0;
+
+ vc->name = "xxhash";
+ vc->good_crc = &vh->hash;
+ vc->bad_crc = &hash;
+ vc->crc_len = sizeof(hash);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_sha512 *vh = hdr_priv(hdr);
+ uint8_t sha512[128];
+ struct fio_sha512_ctx sha512_ctx = {
+ .buf = sha512,
+ };
+
+ dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ fio_sha512_init(&sha512_ctx);
+ fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(vc->td, hdr));
+
+ if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
+ return 0;
+
+ vc->name = "sha512";
+ vc->good_crc = vh->sha512;
+ vc->bad_crc = sha512_ctx.buf;
+ vc->crc_len = sizeof(vh->sha512);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_sha256 *vh = hdr_priv(hdr);
+ uint8_t sha256[64];
+ struct fio_sha256_ctx sha256_ctx = {
+ .buf = sha256,
+ };
+
+ dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ fio_sha256_init(&sha256_ctx);
+ fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(vc->td, hdr));
+ fio_sha256_final(&sha256_ctx);
+
+ if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
+ return 0;
+
+ vc->name = "sha256";
+ vc->good_crc = vh->sha256;
+ vc->bad_crc = sha256_ctx.buf;
+ vc->crc_len = sizeof(vh->sha256);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_sha1 *vh = hdr_priv(hdr);
+ uint32_t sha1[5];
+ struct fio_sha1_ctx sha1_ctx = {
+ .H = sha1,
+ };
+
+ dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ fio_sha1_init(&sha1_ctx);
+ fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(vc->td, hdr));
+ fio_sha1_final(&sha1_ctx);
+
+ if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
+ return 0;
+
+ vc->name = "sha1";
+ vc->good_crc = vh->sha1;
+ vc->bad_crc = sha1_ctx.H;
+ vc->crc_len = sizeof(vh->sha1);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc7 *vh = hdr_priv(hdr);
+ unsigned char c;
+
+ dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = fio_crc7(p, hdr->len - hdr_size(vc->td, hdr));
+
+ if (c == vh->crc7)
+ return 0;
+
+ vc->name = "crc7";
+ vc->good_crc = &vh->crc7;
+ vc->bad_crc = &c;
+ vc->crc_len = 1;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc16 *vh = hdr_priv(hdr);
+ unsigned short c;
+
+ dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = fio_crc16(p, hdr->len - hdr_size(vc->td, hdr));
+
+ if (c == vh->crc16)
+ return 0;
+
+ vc->name = "crc16";
+ vc->good_crc = &vh->crc16;
+ vc->bad_crc = &c;
+ vc->crc_len = 2;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc64 *vh = hdr_priv(hdr);
+ unsigned long long c;
+
+ dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = fio_crc64(p, hdr->len - hdr_size(vc->td, hdr));
+
+ if (c == vh->crc64)
+ return 0;
+
+ vc->name = "crc64";
+ vc->good_crc = &vh->crc64;
+ vc->bad_crc = &c;
+ vc->crc_len = 8;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc32 *vh = hdr_priv(hdr);
+ uint32_t c;
+
+ dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = fio_crc32(p, hdr->len - hdr_size(vc->td, hdr));
+
+ if (c == vh->crc32)
+ return 0;
+
+ vc->name = "crc32";
+ vc->good_crc = &vh->crc32;
+ vc->bad_crc = &c;
+ vc->crc_len = 4;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc32 *vh = hdr_priv(hdr);
+ uint32_t c;
+
+ dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = fio_crc32c(p, hdr->len - hdr_size(vc->td, hdr));
+
+ if (c == vh->crc32)
+ return 0;
+
+ vc->name = "crc32c";
+ vc->good_crc = &vh->crc32;
+ vc->bad_crc = &c;
+ vc->crc_len = 4;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_md5 *vh = hdr_priv(hdr);
+ uint32_t hash[MD5_HASH_WORDS];
+ struct fio_md5_ctx md5_ctx = {
+ .hash = hash,
+ };
+
+ dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ fio_md5_init(&md5_ctx);
+ fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(vc->td, hdr));
+ fio_md5_final(&md5_ctx);
+
+ if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
+ return 0;
+
+ vc->name = "md5";
+ vc->good_crc = vh->md5_digest;
+ vc->bad_crc = md5_ctx.hash;
+ vc->crc_len = sizeof(hash);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+/*
+ * Push IO verification to a separate thread
+ */
+int verify_io_u_async(struct thread_data *td, struct io_u **io_u_ptr)
+{
+ struct io_u *io_u = *io_u_ptr;
+
+ pthread_mutex_lock(&td->io_u_lock);
+
+ if (io_u->file)
+ put_file_log(td, io_u->file);
+
+ if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
+ td->cur_depth--;
+ io_u_clear(td, io_u, IO_U_F_IN_CUR_DEPTH);