+static inline unsigned int hdr_size(struct verify_header *hdr)
+{
+ return __hdr_size(hdr->verify_type);
+}
+
+static void *hdr_priv(struct verify_header *hdr)
+{
+ void *priv = hdr;
+
+ return priv + sizeof(struct verify_header);
+}
+
+/*
+ * Verify container, pass info to verify handlers and allow them to
+ * pass info back in case of error
+ */
+struct vcont {
+ /*
+ * Input
+ */
+ struct io_u *io_u;
+ unsigned int hdr_num;
+
+ /*
+ * Output, only valid in case of error
+ */
+ const char *name;
+ void *good_crc;
+ void *bad_crc;
+ unsigned int crc_len;
+};
+
+static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
+{
+ unsigned long long offset;
+
+ offset = vc->io_u->offset;
+ offset += vc->hdr_num * hdr->len;
+ log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
+ vc->name, vc->io_u->file->file_name, offset, hdr->len);
+
+ if (vc->good_crc && vc->bad_crc) {
+ log_err(" Expected CRC: ");
+ hexdump(vc->good_crc, vc->crc_len);
+ log_err(" Received CRC: ");
+ hexdump(vc->bad_crc, vc->crc_len);
+ }
+}
+
+/*
+ * Return data area 'header_num'
+ */
+static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
+{
+ return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
+}
+
+static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
+ struct vcont *vc)
+{
+ struct vhdr_meta *vh = hdr_priv(hdr);
+ struct io_u *io_u = vc->io_u;
+
+ dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
+
+ if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
+ return 0;
+
+ vc->name = "meta";
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_sha512 *vh = hdr_priv(hdr);
+ uint8_t sha512[128];
+ struct sha512_ctx sha512_ctx = {
+ .buf = sha512,
+ };
+
+ dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ sha512_init(&sha512_ctx);
+ sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
+
+ if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
+ return 0;
+
+ vc->name = "sha512";
+ vc->good_crc = vh->sha512;
+ vc->bad_crc = sha512_ctx.buf;
+ vc->crc_len = sizeof(vh->sha512);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_sha256 *vh = hdr_priv(hdr);
+ uint8_t sha256[64];
+ struct sha256_ctx sha256_ctx = {
+ .buf = sha256,
+ };
+
+ dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ sha256_init(&sha256_ctx);
+ sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
+
+ if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
+ return 0;
+
+ vc->name = "sha256";
+ vc->good_crc = vh->sha256;
+ vc->bad_crc = sha256_ctx.buf;
+ vc->crc_len = sizeof(vh->sha256);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_sha1 *vh = hdr_priv(hdr);
+ uint32_t sha1[5];
+ struct sha1_ctx sha1_ctx = {
+ .H = sha1,
+ };
+
+ dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ sha1_init(&sha1_ctx);
+ sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
+
+ if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
+ return 0;
+
+ vc->name = "sha1";
+ vc->good_crc = vh->sha1;
+ vc->bad_crc = sha1_ctx.H;
+ vc->crc_len = sizeof(vh->sha1);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc7 *vh = hdr_priv(hdr);
+ unsigned char c;
+
+ dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = crc7(p, hdr->len - hdr_size(hdr));
+
+ if (c == vh->crc7)
+ return 0;
+
+ vc->name = "crc7";
+ vc->good_crc = &vh->crc7;
+ vc->bad_crc = &c;
+ vc->crc_len = 1;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc16 *vh = hdr_priv(hdr);
+ unsigned short c;
+
+ dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = crc16(p, hdr->len - hdr_size(hdr));
+
+ if (c == vh->crc16)
+ return 0;
+
+ vc->name = "crc16";
+ vc->good_crc = &vh->crc16;
+ vc->bad_crc = &c;
+ vc->crc_len = 2;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc64 *vh = hdr_priv(hdr);
+ unsigned long long c;
+
+ dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = crc64(p, hdr->len - hdr_size(hdr));
+
+ if (c == vh->crc64)
+ return 0;
+
+ vc->name = "crc64";
+ vc->good_crc = &vh->crc64;
+ vc->bad_crc = &c;
+ vc->crc_len = 8;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc32 *vh = hdr_priv(hdr);
+ uint32_t c;
+
+ dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ c = crc32(p, hdr->len - hdr_size(hdr));
+
+ if (c == vh->crc32)
+ return 0;
+
+ vc->name = "crc32";
+ vc->good_crc = &vh->crc32;
+ vc->bad_crc = &c;
+ vc->crc_len = 4;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_crc32 *vh = hdr_priv(hdr);
+ uint32_t c;
+
+ dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ if (hdr->verify_type == VERIFY_CRC32C_INTEL)
+ c = crc32c_intel(p, hdr->len - hdr_size(hdr));
+ else
+ c = crc32c(p, hdr->len - hdr_size(hdr));
+
+ if (c == vh->crc32)
+ return 0;
+
+ vc->name = "crc32c";
+ vc->good_crc = &vh->crc32;
+ vc->bad_crc = &c;
+ vc->crc_len = 4;
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
+{
+ void *p = io_u_verify_off(hdr, vc);
+ struct vhdr_md5 *vh = hdr_priv(hdr);
+ uint32_t hash[MD5_HASH_WORDS];
+ struct md5_ctx md5_ctx = {
+ .hash = hash,
+ };
+
+ dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
+
+ md5_init(&md5_ctx);
+ md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
+
+ if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
+ return 0;
+
+ vc->name = "md5";
+ vc->good_crc = vh->md5_digest;
+ vc->bad_crc = md5_ctx.hash;
+ vc->crc_len = sizeof(hash);
+ log_verify_failure(hdr, vc);
+ return EILSEQ;
+}
+
+static unsigned int hweight8(unsigned int w)
+{
+ unsigned int res = w - ((w >> 1) & 0x55);
+
+ res = (res & 0x33) + ((res >> 2) & 0x33);
+ return (res + (res >> 4)) & 0x0F;
+}
+
+int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
+ char *buf, unsigned int len, unsigned int mod)
+{
+ unsigned int i;
+
+ for (i = 0; i < len; i++) {
+ if (buf[i] != pattern[mod]) {
+ unsigned int bits;
+
+ bits = hweight8(buf[i] ^ pattern[mod]);
+ log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
+ buf[i], pattern[mod], bits);
+ log_err("fio: bad pattern block offset %u\n", i);
+ return EILSEQ;
+ }
+ mod++;
+ if (mod == pattern_size)
+ mod = 0;