verify.c

   1 /*
   2  * IO verification helpers
   3  */
   4 #include <unistd.h>
   5 #include <fcntl.h>
   6 #include <string.h>
   7 #include <assert.h>
   8 #include <pthread.h>
   9 #include <libgen.h>
  10
  11 #include "fio.h"
  12 #include "verify.h"
  13 #include "trim.h"
  14 #include "lib/rand.h"
  15 #include "lib/hweight.h"
  16 #include "lib/pattern.h"
  17 #include "oslib/asprintf.h"
  18
  19 #include "crc/md5.h"
  20 #include "crc/crc64.h"
  21 #include "crc/crc32.h"
  22 #include "crc/crc32c.h"
  23 #include "crc/crc16.h"
  24 #include "crc/crc7.h"
  25 #include "crc/sha256.h"
  26 #include "crc/sha512.h"
  27 #include "crc/sha1.h"
  28 #include "crc/xxhash.h"
  29 #include "crc/sha3.h"
  30
  31 static void populate_hdr(struct thread_data *td, struct io_u *io_u,
  32                          struct verify_header *hdr, unsigned int header_num,
  33                          unsigned int header_len);
  34 static void __fill_hdr(struct thread_data *td, struct io_u *io_u,
  35                        struct verify_header *hdr, unsigned int header_num,
  36                        unsigned int header_len, uint64_t rand_seed);
  37
  38 void fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len)
  39 {
  40         (void)cpy_pattern(td->o.buffer_pattern, td->o.buffer_pattern_bytes, p, len);
  41 }
  42
  43 static void __fill_buffer(struct thread_options *o, uint64_t seed, void *p,
  44                           unsigned int len)
  45 {
  46         __fill_random_buf_percentage(seed, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes);
  47 }
  48
  49 void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len,
  50                          struct io_u *io_u, uint64_t seed, int use_seed)
  51 {
  52         struct thread_options *o = &td->o;
  53
  54         if (!o->verify_pattern_bytes) {
  55                 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
  56
  57                 if (!use_seed) {
  58                         seed = __rand(&td->verify_state);
  59                         if (sizeof(int) != sizeof(long *))
  60                                 seed *= (unsigned long)__rand(&td->verify_state);
  61                 }
  62                 io_u->rand_seed = seed;
  63                 __fill_buffer(o, seed, p, len);
  64                 return;
  65         }
  66
  67         /* Skip if we were here and we do not need to patch pattern
  68          * with format */
  69         if (!td->o.verify_fmt_sz && io_u->buf_filled_len >= len) {
  70                 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
  71                         o->verify_pattern_bytes, len);
  72                 return;
  73         }
  74
  75         (void)paste_format(td->o.verify_pattern, td->o.verify_pattern_bytes,
  76                            td->o.verify_fmt, td->o.verify_fmt_sz,
  77                            p, len, io_u);
  78         io_u->buf_filled_len = len;
  79 }
  80
  81 static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u)
  82 {
  83         unsigned int hdr_inc;
  84
  85         /*
  86          * If we use bs_unaligned, buflen can be larger than the verify
  87          * interval (which just defaults to the smallest blocksize possible).
  88          */
  89         hdr_inc = io_u->buflen;
  90         if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen &&
  91             !td->o.bs_unaligned)
  92                 hdr_inc = td->o.verify_interval;
  93
  94         return hdr_inc;
  95 }
  96
  97 static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u,
  98                                  uint64_t seed, int use_seed)
  99 {
 100         unsigned int hdr_inc, header_num;
 101         struct verify_header *hdr;
 102         void *p = io_u->buf;
 103
 104         fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed);
 105
 106         hdr_inc = get_hdr_inc(td, io_u);
 107         header_num = 0;
 108         for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
 109                 hdr = p;
 110                 populate_hdr(td, io_u, hdr, header_num, hdr_inc);
 111                 header_num++;
 112         }
 113 }
 114
 115 static void memswp(void *buf1, void *buf2, unsigned int len)
 116 {
 117         char swap[200];
 118
 119         assert(len <= sizeof(swap));
 120
 121         memcpy(&swap, buf1, len);
 122         memcpy(buf1, buf2, len);
 123         memcpy(buf2, &swap, len);
 124 }
 125
 126 static void hexdump(void *buffer, int len)
 127 {
 128         unsigned char *p = buffer;
 129         int i;
 130
 131         for (i = 0; i < len; i++)
 132                 log_err("%02x", p[i]);
 133         log_err("\n");
 134 }
 135
 136 /*
 137  * Prepare for separation of verify_header and checksum header
 138  */
 139 static inline unsigned int __hdr_size(int verify_type)
 140 {
 141         unsigned int len = 0;
 142
 143         switch (verify_type) {
 144         case VERIFY_NONE:
 145         case VERIFY_HDR_ONLY:
 146         case VERIFY_NULL:
 147         case VERIFY_PATTERN:
 148                 len = 0;
 149                 break;
 150         case VERIFY_MD5:
 151                 len = sizeof(struct vhdr_md5);
 152                 break;
 153         case VERIFY_CRC64:
 154                 len = sizeof(struct vhdr_crc64);
 155                 break;
 156         case VERIFY_CRC32C:
 157         case VERIFY_CRC32:
 158         case VERIFY_CRC32C_INTEL:
 159                 len = sizeof(struct vhdr_crc32);
 160                 break;
 161         case VERIFY_CRC16:
 162                 len = sizeof(struct vhdr_crc16);
 163                 break;
 164         case VERIFY_CRC7:
 165                 len = sizeof(struct vhdr_crc7);
 166                 break;
 167         case VERIFY_SHA256:
 168                 len = sizeof(struct vhdr_sha256);
 169                 break;
 170         case VERIFY_SHA512:
 171                 len = sizeof(struct vhdr_sha512);
 172                 break;
 173         case VERIFY_SHA3_224:
 174                 len = sizeof(struct vhdr_sha3_224);
 175                 break;
 176         case VERIFY_SHA3_256:
 177                 len = sizeof(struct vhdr_sha3_256);
 178                 break;
 179         case VERIFY_SHA3_384:
 180                 len = sizeof(struct vhdr_sha3_384);
 181                 break;
 182         case VERIFY_SHA3_512:
 183                 len = sizeof(struct vhdr_sha3_512);
 184                 break;
 185         case VERIFY_XXHASH:
 186                 len = sizeof(struct vhdr_xxhash);
 187                 break;
 188         case VERIFY_SHA1:
 189                 len = sizeof(struct vhdr_sha1);
 190                 break;
 191         case VERIFY_PATTERN_NO_HDR:
 192                 return 0;
 193         default:
 194                 log_err("fio: unknown verify header!\n");
 195                 assert(0);
 196         }
 197
 198         return len + sizeof(struct verify_header);
 199 }
 200
 201 static inline unsigned int hdr_size(struct thread_data *td,
 202                                     struct verify_header *hdr)
 203 {
 204         if (td->o.verify == VERIFY_PATTERN_NO_HDR)
 205                 return 0;
 206
 207         return __hdr_size(hdr->verify_type);
 208 }
 209
 210 static void *hdr_priv(struct verify_header *hdr)
 211 {
 212         void *priv = hdr;
 213
 214         return priv + sizeof(struct verify_header);
 215 }
 216
 217 /*
 218  * Verify container, pass info to verify handlers and allow them to
 219  * pass info back in case of error
 220  */
 221 struct vcont {
 222         /*
 223          * Input
 224          */
 225         struct io_u *io_u;
 226         unsigned int hdr_num;
 227         struct thread_data *td;
 228
 229         /*
 230          * Output, only valid in case of error
 231          */
 232         const char *name;
 233         void *good_crc;
 234         void *bad_crc;
 235         unsigned int crc_len;
 236 };
 237
 238 #define DUMP_BUF_SZ     255
 239
 240 static void dump_buf(char *buf, unsigned int len, unsigned long long offset,
 241                      const char *type, struct fio_file *f)
 242 {
 243         char *ptr, *fname;
 244         char sep[2] = { FIO_OS_PATH_SEPARATOR, 0 };
 245         int ret, fd;
 246
 247         ptr = strdup(f->file_name);
 248
 249         if (asprintf(&fname, "%s%s%s.%llu.%s", aux_path ? : "",
 250                      aux_path ? sep : "", basename(ptr), offset, type) < 0) {
 251                 if (!fio_did_warn(FIO_WARN_VERIFY_BUF))
 252                         log_err("fio: not enough memory for dump buffer filename\n");
 253                 goto free_ptr;
 254         }
 255
 256         fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
 257         if (fd < 0) {
 258                 perror("open verify buf file");
 259                 goto free_fname;
 260         }
 261
 262         while (len) {
 263                 ret = write(fd, buf, len);
 264                 if (!ret)
 265                         break;
 266                 else if (ret < 0) {
 267                         perror("write verify buf file");
 268                         break;
 269                 }
 270                 len -= ret;
 271                 buf += ret;
 272         }
 273
 274         close(fd);
 275         log_err("       %s data dumped as %s\n", type, fname);
 276
 277 free_fname:
 278         free(fname);
 279
 280 free_ptr:
 281         free(ptr);
 282 }
 283
 284 /*
 285  * Dump the contents of the read block and re-generate the correct data
 286  * and dump that too.
 287  */
 288 static void __dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
 289 {
 290         struct thread_data *td = vc->td;
 291         struct io_u *io_u = vc->io_u;
 292         unsigned long hdr_offset;
 293         struct io_u dummy;
 294         void *buf;
 295
 296         if (!td->o.verify_dump)
 297                 return;
 298
 299         /*
 300          * Dump the contents we just read off disk
 301          */
 302         hdr_offset = vc->hdr_num * hdr->len;
 303
 304         dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
 305                         "received", vc->io_u->file);
 306
 307         /*
 308          * Allocate a new buf and re-generate the original data
 309          */
 310         buf = malloc(io_u->buflen);
 311         dummy = *io_u;
 312         dummy.buf = buf;
 313         dummy.rand_seed = hdr->rand_seed;
 314         dummy.buf_filled_len = 0;
 315         dummy.buflen = io_u->buflen;
 316
 317         fill_pattern_headers(td, &dummy, hdr->rand_seed, 1);
 318
 319         dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
 320                         "expected", vc->io_u->file);
 321         free(buf);
 322 }
 323
 324 static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
 325 {
 326         struct thread_data *td = vc->td;
 327         struct verify_header shdr;
 328
 329         if (td->o.verify == VERIFY_PATTERN_NO_HDR) {
 330                 __fill_hdr(td, vc->io_u, &shdr, 0, vc->io_u->buflen, 0);
 331                 hdr = &shdr;
 332         }
 333
 334         __dump_verify_buffers(hdr, vc);
 335 }
 336
 337 static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
 338 {
 339         unsigned long long offset;
 340
 341         offset = vc->io_u->offset;
 342         offset += vc->hdr_num * hdr->len;
 343         log_err("%.8s: verify failed at file %s offset %llu, length %u"
 344                         " (requested block: offset=%llu, length=%llu)\n",
 345                         vc->name, vc->io_u->file->file_name, offset, hdr->len,
 346                         vc->io_u->offset, vc->io_u->buflen);
 347
 348         if (vc->good_crc && vc->bad_crc) {
 349                 log_err("       Expected CRC: ");
 350                 hexdump(vc->good_crc, vc->crc_len);
 351                 log_err("       Received CRC: ");
 352                 hexdump(vc->bad_crc, vc->crc_len);
 353         }
 354
 355         dump_verify_buffers(hdr, vc);
 356 }
 357
 358 /*
 359  * Return data area 'header_num'
 360  */
 361 static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
 362 {
 363         return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(vc->td, hdr);
 364 }
 365
 366 static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc)
 367 {
 368         struct thread_data *td = vc->td;
 369         struct io_u *io_u = vc->io_u;
 370         char *buf, *pattern;
 371         unsigned int header_size = __hdr_size(td->o.verify);
 372         unsigned int len, mod, i, pattern_size;
 373         int rc;
 374
 375         pattern = td->o.verify_pattern;
 376         pattern_size = td->o.verify_pattern_bytes;
 377         assert(pattern_size != 0);
 378
 379         (void)paste_format_inplace(pattern, pattern_size,
 380                                    td->o.verify_fmt, td->o.verify_fmt_sz, io_u);
 381
 382         buf = (char *) hdr + header_size;
 383         len = get_hdr_inc(td, io_u) - header_size;
 384         mod = (get_hdr_inc(td, io_u) * vc->hdr_num + header_size) % pattern_size;
 385
 386         rc = cmp_pattern(pattern, pattern_size, mod, buf, len);
 387         if (!rc)
 388                 return 0;
 389
 390         /* Slow path, compare each byte */
 391         for (i = 0; i < len; i++) {
 392                 if (buf[i] != pattern[mod]) {
 393                         unsigned int bits;
 394
 395                         bits = hweight8(buf[i] ^ pattern[mod]);
 396                         log_err("fio: got pattern '%02x', wanted '%02x'. Bad bits %d\n",
 397                                 (unsigned char)buf[i],
 398                                 (unsigned char)pattern[mod],
 399                                 bits);
 400                         log_err("fio: bad pattern block offset %u\n", i);
 401                         vc->name = "pattern";
 402                         log_verify_failure(hdr, vc);
 403                         return EILSEQ;
 404                 }
 405                 mod++;
 406                 if (mod == td->o.verify_pattern_bytes)
 407                         mod = 0;
 408         }
 409
 410         /* Unreachable line */
 411         assert(0);
 412         return EILSEQ;
 413 }
 414
 415 static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc)
 416 {
 417         void *p = io_u_verify_off(hdr, vc);
 418         struct vhdr_xxhash *vh = hdr_priv(hdr);
 419         uint32_t hash;
 420         void *state;
 421
 422         dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len);
 423
 424         state = XXH32_init(1);
 425         XXH32_update(state, p, hdr->len - hdr_size(vc->td, hdr));
 426         hash = XXH32_digest(state);
 427
 428         if (vh->hash == hash)
 429                 return 0;
 430
 431         vc->name = "xxhash";
 432         vc->good_crc = &vh->hash;
 433         vc->bad_crc = &hash;
 434         vc->crc_len = sizeof(hash);
 435         log_verify_failure(hdr, vc);
 436         return EILSEQ;
 437 }
 438
 439 static int verify_io_u_sha3(struct verify_header *hdr, struct vcont *vc,
 440                             struct fio_sha3_ctx *sha3_ctx, uint8_t *sha,
 441                             unsigned int sha_size, const char *name)
 442 {
 443         void *p = io_u_verify_off(hdr, vc);
 444
 445         dprint(FD_VERIFY, "%s verify io_u %p, len %u\n", name, vc->io_u, hdr->len);
 446
 447         fio_sha3_update(sha3_ctx, p, hdr->len - hdr_size(vc->td, hdr));
 448         fio_sha3_final(sha3_ctx);
 449
 450         if (!memcmp(sha, sha3_ctx->sha, sha_size))
 451                 return 0;
 452
 453         vc->name = name;
 454         vc->good_crc = sha;
 455         vc->bad_crc = sha3_ctx->sha;
 456         vc->crc_len = sha_size;
 457         log_verify_failure(hdr, vc);
 458         return EILSEQ;
 459 }
 460
 461 static int verify_io_u_sha3_224(struct verify_header *hdr, struct vcont *vc)
 462 {
 463         struct vhdr_sha3_224 *vh = hdr_priv(hdr);
 464         uint8_t sha[SHA3_224_DIGEST_SIZE];
 465         struct fio_sha3_ctx sha3_ctx = {
 466                 .sha = sha,
 467         };
 468
 469         fio_sha3_224_init(&sha3_ctx);
 470
 471         return verify_io_u_sha3(hdr, vc, &sha3_ctx, vh->sha,
 472                                 SHA3_224_DIGEST_SIZE, "sha3-224");
 473 }
 474
 475 static int verify_io_u_sha3_256(struct verify_header *hdr, struct vcont *vc)
 476 {
 477         struct vhdr_sha3_256 *vh = hdr_priv(hdr);
 478         uint8_t sha[SHA3_256_DIGEST_SIZE];
 479         struct fio_sha3_ctx sha3_ctx = {
 480                 .sha = sha,
 481         };
 482
 483         fio_sha3_256_init(&sha3_ctx);
 484
 485         return verify_io_u_sha3(hdr, vc, &sha3_ctx, vh->sha,
 486                                 SHA3_256_DIGEST_SIZE, "sha3-256");
 487 }
 488
 489 static int verify_io_u_sha3_384(struct verify_header *hdr, struct vcont *vc)
 490 {
 491         struct vhdr_sha3_384 *vh = hdr_priv(hdr);
 492         uint8_t sha[SHA3_384_DIGEST_SIZE];
 493         struct fio_sha3_ctx sha3_ctx = {
 494                 .sha = sha,
 495         };
 496
 497         fio_sha3_384_init(&sha3_ctx);
 498
 499         return verify_io_u_sha3(hdr, vc, &sha3_ctx, vh->sha,
 500                                 SHA3_384_DIGEST_SIZE, "sha3-384");
 501 }
 502
 503 static int verify_io_u_sha3_512(struct verify_header *hdr, struct vcont *vc)
 504 {
 505         struct vhdr_sha3_512 *vh = hdr_priv(hdr);
 506         uint8_t sha[SHA3_512_DIGEST_SIZE];
 507         struct fio_sha3_ctx sha3_ctx = {
 508                 .sha = sha,
 509         };
 510
 511         fio_sha3_512_init(&sha3_ctx);
 512
 513         return verify_io_u_sha3(hdr, vc, &sha3_ctx, vh->sha,
 514                                 SHA3_512_DIGEST_SIZE, "sha3-512");
 515 }
 516
 517 static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
 518 {
 519         void *p = io_u_verify_off(hdr, vc);
 520         struct vhdr_sha512 *vh = hdr_priv(hdr);
 521         uint8_t sha512[128];
 522         struct fio_sha512_ctx sha512_ctx = {
 523                 .buf = sha512,
 524         };
 525
 526         dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 527
 528         fio_sha512_init(&sha512_ctx);
 529         fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(vc->td, hdr));
 530
 531         if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
 532                 return 0;
 533
 534         vc->name = "sha512";
 535         vc->good_crc = vh->sha512;
 536         vc->bad_crc = sha512_ctx.buf;
 537         vc->crc_len = sizeof(vh->sha512);
 538         log_verify_failure(hdr, vc);
 539         return EILSEQ;
 540 }
 541
 542 static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
 543 {
 544         void *p = io_u_verify_off(hdr, vc);
 545         struct vhdr_sha256 *vh = hdr_priv(hdr);
 546         uint8_t sha256[64];
 547         struct fio_sha256_ctx sha256_ctx = {
 548                 .buf = sha256,
 549         };
 550
 551         dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 552
 553         fio_sha256_init(&sha256_ctx);
 554         fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(vc->td, hdr));
 555         fio_sha256_final(&sha256_ctx);
 556
 557         if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
 558                 return 0;
 559
 560         vc->name = "sha256";
 561         vc->good_crc = vh->sha256;
 562         vc->bad_crc = sha256_ctx.buf;
 563         vc->crc_len = sizeof(vh->sha256);
 564         log_verify_failure(hdr, vc);
 565         return EILSEQ;
 566 }
 567
 568 static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
 569 {
 570         void *p = io_u_verify_off(hdr, vc);
 571         struct vhdr_sha1 *vh = hdr_priv(hdr);
 572         uint32_t sha1[5];
 573         struct fio_sha1_ctx sha1_ctx = {
 574                 .H = sha1,
 575         };
 576
 577         dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 578
 579         fio_sha1_init(&sha1_ctx);
 580         fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(vc->td, hdr));
 581         fio_sha1_final(&sha1_ctx);
 582
 583         if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
 584                 return 0;
 585
 586         vc->name = "sha1";
 587         vc->good_crc = vh->sha1;
 588         vc->bad_crc = sha1_ctx.H;
 589         vc->crc_len = sizeof(vh->sha1);
 590         log_verify_failure(hdr, vc);
 591         return EILSEQ;
 592 }
 593
 594 static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
 595 {
 596         void *p = io_u_verify_off(hdr, vc);
 597         struct vhdr_crc7 *vh = hdr_priv(hdr);
 598         unsigned char c;
 599
 600         dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 601
 602         c = fio_crc7(p, hdr->len - hdr_size(vc->td, hdr));
 603
 604         if (c == vh->crc7)
 605                 return 0;
 606
 607         vc->name = "crc7";
 608         vc->good_crc = &vh->crc7;
 609         vc->bad_crc = &c;
 610         vc->crc_len = 1;
 611         log_verify_failure(hdr, vc);
 612         return EILSEQ;
 613 }
 614
 615 static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
 616 {
 617         void *p = io_u_verify_off(hdr, vc);
 618         struct vhdr_crc16 *vh = hdr_priv(hdr);
 619         unsigned short c;
 620
 621         dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 622
 623         c = fio_crc16(p, hdr->len - hdr_size(vc->td, hdr));
 624
 625         if (c == vh->crc16)
 626                 return 0;
 627
 628         vc->name = "crc16";
 629         vc->good_crc = &vh->crc16;
 630         vc->bad_crc = &c;
 631         vc->crc_len = 2;
 632         log_verify_failure(hdr, vc);
 633         return EILSEQ;
 634 }
 635
 636 static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
 637 {
 638         void *p = io_u_verify_off(hdr, vc);
 639         struct vhdr_crc64 *vh = hdr_priv(hdr);
 640         unsigned long long c;
 641
 642         dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 643
 644         c = fio_crc64(p, hdr->len - hdr_size(vc->td, hdr));
 645
 646         if (c == vh->crc64)
 647                 return 0;
 648
 649         vc->name = "crc64";
 650         vc->good_crc = &vh->crc64;
 651         vc->bad_crc = &c;
 652         vc->crc_len = 8;
 653         log_verify_failure(hdr, vc);
 654         return EILSEQ;
 655 }
 656
 657 static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
 658 {
 659         void *p = io_u_verify_off(hdr, vc);
 660         struct vhdr_crc32 *vh = hdr_priv(hdr);
 661         uint32_t c;
 662
 663         dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 664
 665         c = fio_crc32(p, hdr->len - hdr_size(vc->td, hdr));
 666
 667         if (c == vh->crc32)
 668                 return 0;
 669
 670         vc->name = "crc32";
 671         vc->good_crc = &vh->crc32;
 672         vc->bad_crc = &c;
 673         vc->crc_len = 4;
 674         log_verify_failure(hdr, vc);
 675         return EILSEQ;
 676 }
 677
 678 static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
 679 {
 680         void *p = io_u_verify_off(hdr, vc);
 681         struct vhdr_crc32 *vh = hdr_priv(hdr);
 682         uint32_t c;
 683
 684         dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
 685
 686         c = fio_crc32c(p, hdr->len - hdr_size(vc->td, hdr));
 687
 688         if (c == vh->crc32)
 689                 return 0;
 690
 691         vc->name = "crc32c";
 692         vc->good_crc = &vh->crc32;
 693         vc->bad_crc = &c;
 694         vc->crc_len = 4;
 695         log_verify_failure(hdr, vc);
 696         return EILSEQ;
 697 }
 698
 699 static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
 700 {
 701         void *p = io_u_verify_off(hdr, vc);
 702         struct vhdr_md5 *vh = hdr_priv(hdr);
 703         uint32_t hash[MD5_HASH_WORDS];
 704         struct fio_md5_ctx md5_ctx = {
 705                 .hash = hash,
 706         };
 707
 708         dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
 709
 710         fio_md5_init(&md5_ctx);
 711         fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(vc->td, hdr));
 712         fio_md5_final(&md5_ctx);
 713
 714         if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
 715                 return 0;
 716
 717         vc->name = "md5";
 718         vc->good_crc = vh->md5_digest;
 719         vc->bad_crc = md5_ctx.hash;
 720         vc->crc_len = sizeof(hash);
 721         log_verify_failure(hdr, vc);
 722         return EILSEQ;
 723 }
 724
 725 /*
 726  * Push IO verification to a separate thread
 727  */
 728 int verify_io_u_async(struct thread_data *td, struct io_u **io_u_ptr)
 729 {
 730         struct io_u *io_u = *io_u_ptr;
 731
 732         pthread_mutex_lock(&td->io_u_lock);
 733
 734         if (io_u->file)
 735                 put_file_log(td, io_u->file);
 736
 737         if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
 738                 td->cur_depth--;
 739                 io_u_clear(td, io_u, IO_U_F_IN_CUR_DEPTH);
 740         }
 741         flist_add_tail(&io_u->verify_list, &td->verify_list);
 742         *io_u_ptr = NULL;
 743
 744         pthread_cond_signal(&td->verify_cond);
 745         pthread_mutex_unlock(&td->io_u_lock);
 746         return 0;
 747 }
 748
 749 /*
 750  * Thanks Rusty, for spending the time so I don't have to.
 751  *
 752  * http://rusty.ozlabs.org/?p=560
 753  */
 754 static int mem_is_zero(const void *data, size_t length)
 755 {
 756         const unsigned char *p = data;
 757         size_t len;
 758
 759         /* Check first 16 bytes manually */
 760         for (len = 0; len < 16; len++) {
 761                 if (!length)
 762                         return 1;
 763                 if (*p)
 764                         return 0;
 765                 p++;
 766                 length--;
 767         }
 768
 769         /* Now we know that's zero, memcmp with self. */
 770         return memcmp(data, p, length) == 0;
 771 }
 772
 773 static int mem_is_zero_slow(const void *data, size_t length, size_t *offset)
 774 {
 775         const unsigned char *p = data;
 776
 777         *offset = 0;
 778         while (length) {
 779                 if (*p)
 780                         break;
 781                 (*offset)++;
 782                 length--;
 783                 p++;
 784         }
 785
 786         return !length;
 787 }
 788
 789 static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
 790 {
 791         size_t offset;
 792
 793         if (!td->o.trim_zero)
 794                 return 0;
 795
 796         if (mem_is_zero(io_u->buf, io_u->buflen))
 797                 return 0;
 798
 799         mem_is_zero_slow(io_u->buf, io_u->buflen, &offset);
 800
 801         log_err("trim: verify failed at file %s offset %llu, length %llu"
 802                 ", block offset %lu\n",
 803                         io_u->file->file_name, io_u->offset, io_u->buflen,
 804                         (unsigned long) offset);
 805         return EILSEQ;
 806 }
 807
 808 static int verify_header(struct io_u *io_u, struct thread_data *td,
 809                          struct verify_header *hdr, unsigned int hdr_num,
 810                          unsigned int hdr_len)
 811 {
 812         void *p = hdr;
 813         uint32_t crc;
 814
 815         if (hdr->magic != FIO_HDR_MAGIC) {
 816                 log_err("verify: bad magic header %x, wanted %x",
 817                         hdr->magic, FIO_HDR_MAGIC);
 818                 goto err;
 819         }
 820         if (hdr->len != hdr_len) {
 821                 log_err("verify: bad header length %u, wanted %u",
 822                         hdr->len, hdr_len);
 823                 goto err;
 824         }
 825         if (hdr->rand_seed != io_u->rand_seed) {
 826                 log_err("verify: bad header rand_seed %"PRIu64
 827                         ", wanted %"PRIu64,
 828                         hdr->rand_seed, io_u->rand_seed);
 829                 goto err;
 830         }
 831         if (hdr->offset != io_u->offset + hdr_num * td->o.verify_interval) {
 832                 log_err("verify: bad header offset %"PRIu64
 833                         ", wanted %llu",
 834                         hdr->offset, io_u->offset);
 835                 goto err;
 836         }
 837
 838         /*
 839          * For read-only workloads, the program cannot be certain of the
 840          * last numberio written to a block. Checking of numberio will be
 841          * done only for workloads that write data.  For verify_only,
 842          * numberio check is skipped.
 843          */
 844         if (td_write(td) && (td_min_bs(td) == td_max_bs(td)) &&
 845             !td->o.time_based)
 846                 if (!td->o.verify_only)
 847                         if (hdr->numberio != io_u->numberio) {
 848                                 log_err("verify: bad header numberio %"PRIu16
 849                                         ", wanted %"PRIu16,
 850                                         hdr->numberio, io_u->numberio);
 851                                 goto err;
 852                         }
 853
 854         crc = fio_crc32c(p, offsetof(struct verify_header, crc32));
 855         if (crc != hdr->crc32) {
 856                 log_err("verify: bad header crc %x, calculated %x",
 857                         hdr->crc32, crc);
 858                 goto err;
 859         }
 860         return 0;
 861
 862 err:
 863         log_err(" at file %s offset %llu, length %u"
 864                 " (requested block: offset=%llu, length=%llu)\n",
 865                 io_u->file->file_name,
 866                 io_u->offset + hdr_num * hdr_len, hdr_len,
 867                 io_u->offset, io_u->buflen);
 868
 869         if (td->o.verify_dump)
 870                 dump_buf(p, hdr_len, io_u->offset + hdr_num * hdr_len,
 871                                 "hdr_fail", io_u->file);
 872
 873         return EILSEQ;
 874 }
 875
 876 int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)
 877 {
 878         struct verify_header *hdr;
 879         struct io_u *io_u = *io_u_ptr;
 880         unsigned int header_size, hdr_inc, hdr_num = 0;
 881         void *p;
 882         int ret;
 883
 884         if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
 885                 return 0;
 886         /*
 887          * If the IO engine is faking IO (like null), then just pretend
 888          * we verified everything.
 889          */
 890         if (td_ioengine_flagged(td, FIO_FAKEIO))
 891                 return 0;
 892
 893         if (io_u->flags & IO_U_F_TRIMMED) {
 894                 ret = verify_trimmed_io_u(td, io_u);
 895                 goto done;
 896         }
 897
 898         hdr_inc = get_hdr_inc(td, io_u);
 899
 900         ret = 0;
 901         for (p = io_u->buf; p < io_u->buf + io_u->buflen;
 902              p += hdr_inc, hdr_num++) {
 903                 struct vcont vc = {
 904                         .io_u           = io_u,
 905                         .hdr_num        = hdr_num,
 906                         .td             = td,
 907                 };
 908                 unsigned int verify_type;
 909
 910                 if (ret && td->o.verify_fatal)
 911                         break;
 912
 913                 header_size = __hdr_size(td->o.verify);
 914                 if (td->o.verify_offset)
 915                         memswp(p, p + td->o.verify_offset, header_size);
 916                 hdr = p;
 917
 918                 /*
 919                  * Make rand_seed check pass when have verify_backlog.
 920                  */
 921                 if (!td_rw(td) || (td->flags & TD_F_VER_BACKLOG))
 922                         io_u->rand_seed = hdr->rand_seed;
 923
 924                 if (td->o.verify != VERIFY_PATTERN_NO_HDR) {
 925                         ret = verify_header(io_u, td, hdr, hdr_num, hdr_inc);
 926                         if (ret)
 927                                 return ret;
 928                 }
 929
 930                 if (td->o.verify != VERIFY_NONE)
 931                         verify_type = td->o.verify;
 932                 else
 933                         verify_type = hdr->verify_type;
 934
 935                 switch (verify_type) {
 936                 case VERIFY_HDR_ONLY:
 937                         /* Header is always verified, check if pattern is left
 938                          * for verification. */
 939                         if (td->o.verify_pattern_bytes)
 940                                 ret = verify_io_u_pattern(hdr, &vc);
 941                         break;
 942                 case VERIFY_MD5:
 943                         ret = verify_io_u_md5(hdr, &vc);
 944                         break;
 945                 case VERIFY_CRC64:
 946                         ret = verify_io_u_crc64(hdr, &vc);
 947                         break;
 948                 case VERIFY_CRC32C:
 949                 case VERIFY_CRC32C_INTEL:
 950                         ret = verify_io_u_crc32c(hdr, &vc);
 951                         break;
 952                 case VERIFY_CRC32:
 953                         ret = verify_io_u_crc32(hdr, &vc);
 954                         break;
 955                 case VERIFY_CRC16:
 956                         ret = verify_io_u_crc16(hdr, &vc);
 957                         break;
 958                 case VERIFY_CRC7:
 959                         ret = verify_io_u_crc7(hdr, &vc);
 960                         break;
 961                 case VERIFY_SHA256:
 962                         ret = verify_io_u_sha256(hdr, &vc);
 963                         break;
 964                 case VERIFY_SHA512:
 965                         ret = verify_io_u_sha512(hdr, &vc);
 966                         break;
 967                 case VERIFY_SHA3_224:
 968                         ret = verify_io_u_sha3_224(hdr, &vc);
 969                         break;
 970                 case VERIFY_SHA3_256:
 971                         ret = verify_io_u_sha3_256(hdr, &vc);
 972                         break;
 973                 case VERIFY_SHA3_384:
 974                         ret = verify_io_u_sha3_384(hdr, &vc);
 975                         break;
 976                 case VERIFY_SHA3_512:
 977                         ret = verify_io_u_sha3_512(hdr, &vc);
 978                         break;
 979                 case VERIFY_XXHASH:
 980                         ret = verify_io_u_xxhash(hdr, &vc);
 981                         break;
 982                 case VERIFY_SHA1:
 983                         ret = verify_io_u_sha1(hdr, &vc);
 984                         break;
 985                 case VERIFY_PATTERN:
 986                 case VERIFY_PATTERN_NO_HDR:
 987                         ret = verify_io_u_pattern(hdr, &vc);
 988                         break;
 989                 default:
 990                         log_err("Bad verify type %u\n", hdr->verify_type);
 991                         ret = EINVAL;
 992                 }
 993
 994                 if (ret && verify_type != hdr->verify_type)
 995                         log_err("fio: verify type mismatch (%u media, %u given)\n",
 996                                         hdr->verify_type, verify_type);
 997         }
 998
 999 done:
1000         if (ret && td->o.verify_fatal)
1001                 fio_mark_td_terminate(td);
1002
1003         return ret;
1004 }
1005
1006 static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len)
1007 {
1008         struct vhdr_xxhash *vh = hdr_priv(hdr);
1009         void *state;
1010
1011         state = XXH32_init(1);
1012         XXH32_update(state, p, len);
1013         vh->hash = XXH32_digest(state);
1014 }
1015
1016 static void fill_sha3(struct fio_sha3_ctx *sha3_ctx, void *p, unsigned int len)
1017 {
1018         fio_sha3_update(sha3_ctx, p, len);
1019         fio_sha3_final(sha3_ctx);
1020 }
1021
1022 static void fill_sha3_224(struct verify_header *hdr, void *p, unsigned int len)
1023 {
1024         struct vhdr_sha3_224 *vh = hdr_priv(hdr);
1025         struct fio_sha3_ctx sha3_ctx = {
1026                 .sha = vh->sha,
1027         };
1028
1029         fio_sha3_224_init(&sha3_ctx);
1030         fill_sha3(&sha3_ctx, p, len);
1031 }
1032
1033 static void fill_sha3_256(struct verify_header *hdr, void *p, unsigned int len)
1034 {
1035         struct vhdr_sha3_256 *vh = hdr_priv(hdr);
1036         struct fio_sha3_ctx sha3_ctx = {
1037                 .sha = vh->sha,
1038         };
1039
1040         fio_sha3_256_init(&sha3_ctx);
1041         fill_sha3(&sha3_ctx, p, len);
1042 }
1043
1044 static void fill_sha3_384(struct verify_header *hdr, void *p, unsigned int len)
1045 {
1046         struct vhdr_sha3_384 *vh = hdr_priv(hdr);
1047         struct fio_sha3_ctx sha3_ctx = {
1048                 .sha = vh->sha,
1049         };
1050
1051         fio_sha3_384_init(&sha3_ctx);
1052         fill_sha3(&sha3_ctx, p, len);
1053 }
1054
1055 static void fill_sha3_512(struct verify_header *hdr, void *p, unsigned int len)
1056 {
1057         struct vhdr_sha3_512 *vh = hdr_priv(hdr);
1058         struct fio_sha3_ctx sha3_ctx = {
1059                 .sha = vh->sha,
1060         };
1061
1062         fio_sha3_512_init(&sha3_ctx);
1063         fill_sha3(&sha3_ctx, p, len);
1064 }
1065
1066 static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
1067 {
1068         struct vhdr_sha512 *vh = hdr_priv(hdr);
1069         struct fio_sha512_ctx sha512_ctx = {
1070                 .buf = vh->sha512,
1071         };
1072
1073         fio_sha512_init(&sha512_ctx);
1074         fio_sha512_update(&sha512_ctx, p, len);
1075 }
1076
1077 static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
1078 {
1079         struct vhdr_sha256 *vh = hdr_priv(hdr);
1080         struct fio_sha256_ctx sha256_ctx = {
1081                 .buf = vh->sha256,
1082         };
1083
1084         fio_sha256_init(&sha256_ctx);
1085         fio_sha256_update(&sha256_ctx, p, len);
1086         fio_sha256_final(&sha256_ctx);
1087 }
1088
1089 static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
1090 {
1091         struct vhdr_sha1 *vh = hdr_priv(hdr);
1092         struct fio_sha1_ctx sha1_ctx = {
1093                 .H = vh->sha1,
1094         };
1095
1096         fio_sha1_init(&sha1_ctx);
1097         fio_sha1_update(&sha1_ctx, p, len);
1098         fio_sha1_final(&sha1_ctx);
1099 }
1100
1101 static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
1102 {
1103         struct vhdr_crc7 *vh = hdr_priv(hdr);
1104
1105         vh->crc7 = fio_crc7(p, len);
1106 }
1107
1108 static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
1109 {
1110         struct vhdr_crc16 *vh = hdr_priv(hdr);
1111
1112         vh->crc16 = fio_crc16(p, len);
1113 }
1114
1115 static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
1116 {
1117         struct vhdr_crc32 *vh = hdr_priv(hdr);
1118
1119         vh->crc32 = fio_crc32(p, len);
1120 }
1121
1122 static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
1123 {
1124         struct vhdr_crc32 *vh = hdr_priv(hdr);
1125
1126         vh->crc32 = fio_crc32c(p, len);
1127 }
1128
1129 static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
1130 {
1131         struct vhdr_crc64 *vh = hdr_priv(hdr);
1132
1133         vh->crc64 = fio_crc64(p, len);
1134 }
1135
1136 static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
1137 {
1138         struct vhdr_md5 *vh = hdr_priv(hdr);
1139         struct fio_md5_ctx md5_ctx = {
1140                 .hash = (uint32_t *) vh->md5_digest,
1141         };
1142
1143         fio_md5_init(&md5_ctx);
1144         fio_md5_update(&md5_ctx, p, len);
1145         fio_md5_final(&md5_ctx);
1146 }
1147
1148 static void __fill_hdr(struct thread_data *td, struct io_u *io_u,
1149                        struct verify_header *hdr, unsigned int header_num,
1150                        unsigned int header_len, uint64_t rand_seed)
1151 {
1152         void *p = hdr;
1153
1154         hdr->magic = FIO_HDR_MAGIC;
1155         hdr->verify_type = td->o.verify;
1156         hdr->len = header_len;
1157         hdr->rand_seed = rand_seed;
1158         hdr->offset = io_u->offset + header_num * td->o.verify_interval;
1159         hdr->time_sec = io_u->start_time.tv_sec;
1160         hdr->time_nsec = io_u->start_time.tv_nsec;
1161         hdr->thread = td->thread_number;
1162         hdr->numberio = io_u->numberio;
1163         hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
1164 }
1165
1166
1167 static void fill_hdr(struct thread_data *td, struct io_u *io_u,
1168                      struct verify_header *hdr, unsigned int header_num,
1169                      unsigned int header_len, uint64_t rand_seed)
1170 {
1171         if (td->o.verify != VERIFY_PATTERN_NO_HDR)
1172                 __fill_hdr(td, io_u, hdr, header_num, header_len, rand_seed);
1173 }
1174
1175 static void populate_hdr(struct thread_data *td, struct io_u *io_u,
1176                          struct verify_header *hdr, unsigned int header_num,
1177                          unsigned int header_len)
1178 {
1179         unsigned int data_len;
1180         void *data;
1181         char *p;
1182
1183         p = (char *) hdr;
1184
1185         fill_hdr(td, io_u, hdr, header_num, header_len, io_u->rand_seed);
1186
1187         if (header_len <= hdr_size(td, hdr)) {
1188                 td_verror(td, EINVAL, "Blocksize too small");
1189                 return;
1190         }
1191         data_len = header_len - hdr_size(td, hdr);
1192
1193         data = p + hdr_size(td, hdr);
1194         switch (td->o.verify) {
1195         case VERIFY_MD5:
1196                 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
1197                                                 io_u, hdr->len);
1198                 fill_md5(hdr, data, data_len);
1199                 break;
1200         case VERIFY_CRC64:
1201                 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
1202                                                 io_u, hdr->len);
1203                 fill_crc64(hdr, data, data_len);
1204                 break;
1205         case VERIFY_CRC32C:
1206         case VERIFY_CRC32C_INTEL:
1207                 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
1208                                                 io_u, hdr->len);
1209                 fill_crc32c(hdr, data, data_len);
1210                 break;
1211         case VERIFY_CRC32:
1212                 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
1213                                                 io_u, hdr->len);
1214                 fill_crc32(hdr, data, data_len);
1215                 break;
1216         case VERIFY_CRC16:
1217                 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
1218                                                 io_u, hdr->len);
1219                 fill_crc16(hdr, data, data_len);
1220                 break;
1221         case VERIFY_CRC7:
1222                 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
1223                                                 io_u, hdr->len);
1224                 fill_crc7(hdr, data, data_len);
1225                 break;
1226         case VERIFY_SHA256:
1227                 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
1228                                                 io_u, hdr->len);
1229                 fill_sha256(hdr, data, data_len);
1230                 break;
1231         case VERIFY_SHA512:
1232                 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
1233                                                 io_u, hdr->len);
1234                 fill_sha512(hdr, data, data_len);
1235                 break;
1236         case VERIFY_SHA3_224:
1237                 dprint(FD_VERIFY, "fill sha3-224 io_u %p, len %u\n",
1238                                                 io_u, hdr->len);
1239                 fill_sha3_224(hdr, data, data_len);
1240                 break;
1241         case VERIFY_SHA3_256:
1242                 dprint(FD_VERIFY, "fill sha3-256 io_u %p, len %u\n",
1243                                                 io_u, hdr->len);
1244                 fill_sha3_256(hdr, data, data_len);
1245                 break;
1246         case VERIFY_SHA3_384:
1247                 dprint(FD_VERIFY, "fill sha3-384 io_u %p, len %u\n",
1248                                                 io_u, hdr->len);
1249                 fill_sha3_384(hdr, data, data_len);
1250                 break;
1251         case VERIFY_SHA3_512:
1252                 dprint(FD_VERIFY, "fill sha3-512 io_u %p, len %u\n",
1253                                                 io_u, hdr->len);
1254                 fill_sha3_512(hdr, data, data_len);
1255                 break;
1256         case VERIFY_XXHASH:
1257                 dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n",
1258                                                 io_u, hdr->len);
1259                 fill_xxhash(hdr, data, data_len);
1260                 break;
1261         case VERIFY_SHA1:
1262                 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
1263                                                 io_u, hdr->len);
1264                 fill_sha1(hdr, data, data_len);
1265                 break;
1266         case VERIFY_HDR_ONLY:
1267         case VERIFY_PATTERN:
1268         case VERIFY_PATTERN_NO_HDR:
1269                 /* nothing to do here */
1270                 break;
1271         default:
1272                 log_err("fio: bad verify type: %d\n", td->o.verify);
1273                 assert(0);
1274         }
1275
1276         if (td->o.verify_offset && hdr_size(td, hdr))
1277                 memswp(p, p + td->o.verify_offset, hdr_size(td, hdr));
1278 }
1279
1280 /*
1281  * fill body of io_u->buf with random data and add a header with the
1282  * checksum of choice
1283  */
1284 void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
1285 {
1286         if (td->o.verify == VERIFY_NULL)
1287                 return;
1288
1289         io_u->numberio = td->io_issues[io_u->ddir];
1290
1291         fill_pattern_headers(td, io_u, 0, 0);
1292 }
1293
1294 int get_next_verify(struct thread_data *td, struct io_u *io_u)
1295 {
1296         struct io_piece *ipo = NULL;
1297
1298         /*
1299          * this io_u is from a requeue, we already filled the offsets
1300          */
1301         if (io_u->file)
1302                 return 0;
1303
1304         if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
1305                 struct fio_rb_node *n = rb_first(&td->io_hist_tree);
1306
1307                 ipo = rb_entry(n, struct io_piece, rb_node);
1308
1309                 /*
1310                  * Ensure that the associated IO has completed
1311                  */
1312                 read_barrier();
1313                 if (ipo->flags & IP_F_IN_FLIGHT)
1314                         goto nothing;
1315
1316                 rb_erase(n, &td->io_hist_tree);
1317                 assert(ipo->flags & IP_F_ONRB);
1318                 ipo->flags &= ~IP_F_ONRB;
1319         } else if (!flist_empty(&td->io_hist_list)) {
1320                 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
1321
1322                 /*
1323                  * Ensure that the associated IO has completed
1324                  */
1325                 read_barrier();
1326                 if (ipo->flags & IP_F_IN_FLIGHT)
1327                         goto nothing;
1328
1329                 flist_del(&ipo->list);
1330                 assert(ipo->flags & IP_F_ONLIST);
1331                 ipo->flags &= ~IP_F_ONLIST;
1332         }
1333
1334         if (ipo) {
1335                 td->io_hist_len--;
1336
1337                 io_u->offset = ipo->offset;
1338                 io_u->buflen = ipo->len;
1339                 io_u->numberio = ipo->numberio;
1340                 io_u->file = ipo->file;
1341                 io_u_set(td, io_u, IO_U_F_VER_LIST);
1342
1343                 if (ipo->flags & IP_F_TRIMMED)
1344                         io_u_set(td, io_u, IO_U_F_TRIMMED);
1345
1346                 if (!fio_file_open(io_u->file)) {
1347                         int r = td_io_open_file(td, io_u->file);
1348
1349                         if (r) {
1350                                 dprint(FD_VERIFY, "failed file %s open\n",
1351                                                 io_u->file->file_name);
1352                                 return 1;
1353                         }
1354                 }
1355
1356                 get_file(ipo->file);
1357                 assert(fio_file_open(io_u->file));
1358                 io_u->ddir = DDIR_READ;
1359                 io_u->xfer_buf = io_u->buf;
1360                 io_u->xfer_buflen = io_u->buflen;
1361
1362                 remove_trim_entry(td, ipo);
1363                 free(ipo);
1364                 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
1365
1366                 if (!td->o.verify_pattern_bytes) {
1367                         io_u->rand_seed = __rand(&td->verify_state);
1368                         if (sizeof(int) != sizeof(long *))
1369                                 io_u->rand_seed *= __rand(&td->verify_state);
1370                 }
1371                 return 0;
1372         }
1373
1374 nothing:
1375         dprint(FD_VERIFY, "get_next_verify: empty\n");
1376         return 1;
1377 }
1378
1379 void fio_verify_init(struct thread_data *td)
1380 {
1381         if (td->o.verify == VERIFY_CRC32C_INTEL ||
1382             td->o.verify == VERIFY_CRC32C) {
1383                 crc32c_arm64_probe();
1384                 crc32c_intel_probe();
1385         }
1386 }
1387
1388 static void *verify_async_thread(void *data)
1389 {
1390         struct thread_data *td = data;
1391         struct io_u *io_u;
1392         int ret = 0;
1393
1394         if (fio_option_is_set(&td->o, verify_cpumask) &&
1395             fio_setaffinity(td->pid, td->o.verify_cpumask)) {
1396                 log_err("fio: failed setting verify thread affinity\n");
1397                 goto done;
1398         }
1399
1400         do {
1401                 FLIST_HEAD(list);
1402
1403                 read_barrier();
1404                 if (td->verify_thread_exit)
1405                         break;
1406
1407                 pthread_mutex_lock(&td->io_u_lock);
1408
1409                 while (flist_empty(&td->verify_list) &&
1410                        !td->verify_thread_exit) {
1411                         ret = pthread_cond_wait(&td->verify_cond,
1412                                                         &td->io_u_lock);
1413                         if (ret) {
1414                                 pthread_mutex_unlock(&td->io_u_lock);
1415                                 break;
1416                         }
1417                 }
1418
1419                 flist_splice_init(&td->verify_list, &list);
1420                 pthread_mutex_unlock(&td->io_u_lock);
1421
1422                 if (flist_empty(&list))
1423                         continue;
1424
1425                 while (!flist_empty(&list)) {
1426                         io_u = flist_first_entry(&list, struct io_u, verify_list);
1427                         flist_del_init(&io_u->verify_list);
1428
1429                         io_u_set(td, io_u, IO_U_F_NO_FILE_PUT);
1430                         ret = verify_io_u(td, &io_u);
1431
1432                         put_io_u(td, io_u);
1433                         if (!ret)
1434                                 continue;
1435                         if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) {
1436                                 update_error_count(td, ret);
1437                                 td_clear_error(td);
1438                                 ret = 0;
1439                         }
1440                 }
1441         } while (!ret);
1442
1443         if (ret) {
1444                 td_verror(td, ret, "async_verify");
1445                 if (td->o.verify_fatal)
1446                         fio_mark_td_terminate(td);
1447         }
1448
1449 done:
1450         pthread_mutex_lock(&td->io_u_lock);
1451         td->nr_verify_threads--;
1452         pthread_cond_signal(&td->free_cond);
1453         pthread_mutex_unlock(&td->io_u_lock);
1454
1455         return NULL;
1456 }
1457
1458 int verify_async_init(struct thread_data *td)
1459 {
1460         int i, ret;
1461         pthread_attr_t attr;
1462
1463         pthread_attr_init(&attr);
1464         pthread_attr_setstacksize(&attr, 2 * PTHREAD_STACK_MIN);
1465
1466         td->verify_thread_exit = 0;
1467
1468         td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
1469         for (i = 0; i < td->o.verify_async; i++) {
1470                 ret = pthread_create(&td->verify_threads[i], &attr,
1471                                         verify_async_thread, td);
1472                 if (ret) {
1473                         log_err("fio: async verify creation failed: %s\n",
1474                                         strerror(ret));
1475                         break;
1476                 }
1477                 ret = pthread_detach(td->verify_threads[i]);
1478                 if (ret) {
1479                         log_err("fio: async verify thread detach failed: %s\n",
1480                                         strerror(ret));
1481                         break;
1482                 }
1483                 td->nr_verify_threads++;
1484         }
1485
1486         pthread_attr_destroy(&attr);
1487
1488         if (i != td->o.verify_async) {
1489                 log_err("fio: only %d verify threads started, exiting\n", i);
1490
1491                 pthread_mutex_lock(&td->io_u_lock);
1492                 td->verify_thread_exit = 1;
1493                 pthread_cond_broadcast(&td->verify_cond);
1494                 pthread_mutex_unlock(&td->io_u_lock);
1495
1496                 return 1;
1497         }
1498
1499         return 0;
1500 }
1501
1502 void verify_async_exit(struct thread_data *td)
1503 {
1504         pthread_mutex_lock(&td->io_u_lock);
1505         td->verify_thread_exit = 1;
1506         pthread_cond_broadcast(&td->verify_cond);
1507
1508         while (td->nr_verify_threads)
1509                 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
1510
1511         pthread_mutex_unlock(&td->io_u_lock);
1512         free(td->verify_threads);
1513         td->verify_threads = NULL;
1514 }
1515
1516 int paste_blockoff(char *buf, unsigned int len, void *priv)
1517 {
1518         struct io_u *io = priv;
1519         unsigned long long off;
1520
1521         typecheck(__typeof__(off), io->offset);
1522         off = cpu_to_le64((uint64_t)io->offset);
1523         len = min(len, (unsigned int)sizeof(off));
1524         memcpy(buf, &off, len);
1525         return 0;
1526 }
1527
1528 static int __fill_file_completions(struct thread_data *td,
1529                                    struct thread_io_list *s,
1530                                    struct fio_file *f, unsigned int *index)
1531 {
1532         unsigned int comps;
1533         int i, j;
1534
1535         if (!f->last_write_comp)
1536                 return 0;
1537
1538         if (td->io_blocks[DDIR_WRITE] < td->o.iodepth)
1539                 comps = td->io_blocks[DDIR_WRITE];
1540         else
1541                 comps = td->o.iodepth;
1542
1543         j = f->last_write_idx - 1;
1544         for (i = 0; i < comps; i++) {
1545                 if (j == -1)
1546                         j = td->o.iodepth - 1;
1547                 s->comps[*index].fileno = __cpu_to_le64(f->fileno);
1548                 s->comps[*index].offset = cpu_to_le64(f->last_write_comp[j]);
1549                 (*index)++;
1550                 j--;
1551         }
1552
1553         return comps;
1554 }
1555
1556 static int fill_file_completions(struct thread_data *td,
1557                                  struct thread_io_list *s, unsigned int *index)
1558 {
1559         struct fio_file *f;
1560         unsigned int i;
1561         int comps = 0;
1562
1563         for_each_file(td, f, i)
1564                 comps += __fill_file_completions(td, s, f, index);
1565
1566         return comps;
1567 }
1568
1569 struct all_io_list *get_all_io_list(int save_mask, size_t *sz)
1570 {
1571         struct all_io_list *rep;
1572         struct thread_data *td;
1573         size_t depth;
1574         void *next;
1575         int i, nr;
1576
1577         compiletime_assert(sizeof(struct all_io_list) == 8, "all_io_list");
1578
1579         /*
1580          * Calculate reply space needed. We need one 'io_state' per thread,
1581          * and the size will vary depending on depth.
1582          */
1583         depth = 0;
1584         nr = 0;
1585         for_each_td(td, i) {
1586                 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask)
1587                         continue;
1588                 td->stop_io = 1;
1589                 td->flags |= TD_F_VSTATE_SAVED;
1590                 depth += (td->o.iodepth * td->o.nr_files);
1591                 nr++;
1592         }
1593
1594         if (!nr)
1595                 return NULL;
1596
1597         *sz = sizeof(*rep);
1598         *sz += nr * sizeof(struct thread_io_list);
1599         *sz += depth * sizeof(struct file_comp);
1600         rep = malloc(*sz);
1601         memset(rep, 0, *sz);
1602
1603         rep->threads = cpu_to_le64((uint64_t) nr);
1604
1605         next = &rep->state[0];
1606         for_each_td(td, i) {
1607                 struct thread_io_list *s = next;
1608                 unsigned int comps, index = 0;
1609
1610                 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask)
1611                         continue;
1612
1613                 comps = fill_file_completions(td, s, &index);
1614
1615                 s->no_comps = cpu_to_le64((uint64_t) comps);
1616                 s->depth = cpu_to_le64((uint64_t) td->o.iodepth);
1617                 s->nofiles = cpu_to_le64((uint64_t) td->o.nr_files);
1618                 s->numberio = cpu_to_le64((uint64_t) td->io_issues[DDIR_WRITE]);
1619                 s->index = cpu_to_le64((uint64_t) i);
1620                 if (td->random_state.use64) {
1621                         s->rand.state64.s[0] = cpu_to_le64(td->random_state.state64.s1);
1622                         s->rand.state64.s[1] = cpu_to_le64(td->random_state.state64.s2);
1623                         s->rand.state64.s[2] = cpu_to_le64(td->random_state.state64.s3);
1624                         s->rand.state64.s[3] = cpu_to_le64(td->random_state.state64.s4);
1625                         s->rand.state64.s[4] = cpu_to_le64(td->random_state.state64.s5);
1626                         s->rand.state64.s[5] = 0;
1627                         s->rand.use64 = cpu_to_le64((uint64_t)1);
1628                 } else {
1629                         s->rand.state32.s[0] = cpu_to_le32(td->random_state.state32.s1);
1630                         s->rand.state32.s[1] = cpu_to_le32(td->random_state.state32.s2);
1631                         s->rand.state32.s[2] = cpu_to_le32(td->random_state.state32.s3);
1632                         s->rand.state32.s[3] = 0;
1633                         s->rand.use64 = 0;
1634                 }
1635                 snprintf((char *) s->name, sizeof(s->name), "%s", td->o.name);
1636                 next = io_list_next(s);
1637         }
1638
1639         return rep;
1640 }
1641
1642 static int open_state_file(const char *name, const char *prefix, int num,
1643                            int for_write)
1644 {
1645         char out[PATH_MAX];
1646         int flags;
1647         int fd;
1648
1649         if (for_write)
1650                 flags = O_CREAT | O_TRUNC | O_WRONLY | O_SYNC;
1651         else
1652                 flags = O_RDONLY;
1653
1654         verify_state_gen_name(out, sizeof(out), name, prefix, num);
1655
1656         fd = open(out, flags, 0644);
1657         if (fd == -1) {
1658                 perror("fio: open state file");
1659                 log_err("fio: state file: %s (for_write=%d)\n", out, for_write);
1660                 return -1;
1661         }
1662
1663         return fd;
1664 }
1665
1666 static int write_thread_list_state(struct thread_io_list *s,
1667                                    const char *prefix)
1668 {
1669         struct verify_state_hdr hdr;
1670         uint64_t crc;
1671         ssize_t ret;
1672         int fd;
1673
1674         fd = open_state_file((const char *) s->name, prefix, s->index, 1);
1675         if (fd == -1)
1676                 return 1;
1677
1678         crc = fio_crc32c((void *)s, thread_io_list_sz(s));
1679
1680         hdr.version = cpu_to_le64((uint64_t) VSTATE_HDR_VERSION);
1681         hdr.size = cpu_to_le64((uint64_t) thread_io_list_sz(s));
1682         hdr.crc = cpu_to_le64(crc);
1683         ret = write(fd, &hdr, sizeof(hdr));
1684         if (ret != sizeof(hdr))
1685                 goto write_fail;
1686
1687         ret = write(fd, s, thread_io_list_sz(s));
1688         if (ret != thread_io_list_sz(s)) {
1689 write_fail:
1690                 if (ret < 0)
1691                         perror("fio: write state file");
1692                 log_err("fio: failed to write state file\n");
1693                 ret = 1;
1694         } else
1695                 ret = 0;
1696
1697         close(fd);
1698         return ret;
1699 }
1700
1701 void __verify_save_state(struct all_io_list *state, const char *prefix)
1702 {
1703         struct thread_io_list *s = &state->state[0];
1704         unsigned int i;
1705
1706         for (i = 0; i < le64_to_cpu(state->threads); i++) {
1707                 write_thread_list_state(s,  prefix);
1708                 s = io_list_next(s);
1709         }
1710 }
1711
1712 void verify_save_state(int mask)
1713 {
1714         struct all_io_list *state;
1715         size_t sz;
1716
1717         state = get_all_io_list(mask, &sz);
1718         if (state) {
1719                 char prefix[PATH_MAX];
1720
1721                 if (aux_path)
1722                         sprintf(prefix, "%s%clocal", aux_path, FIO_OS_PATH_SEPARATOR);
1723                 else
1724                         strcpy(prefix, "local");
1725
1726                 __verify_save_state(state, prefix);
1727                 free(state);
1728         }
1729 }
1730
1731 void verify_free_state(struct thread_data *td)
1732 {
1733         if (td->vstate)
1734                 free(td->vstate);
1735 }
1736
1737 void verify_assign_state(struct thread_data *td, void *p)
1738 {
1739         struct thread_io_list *s = p;
1740         int i;
1741
1742         s->no_comps = le64_to_cpu(s->no_comps);
1743         s->depth = le32_to_cpu(s->depth);
1744         s->nofiles = le32_to_cpu(s->nofiles);
1745         s->numberio = le64_to_cpu(s->numberio);
1746         s->rand.use64 = le64_to_cpu(s->rand.use64);
1747
1748         if (s->rand.use64) {
1749                 for (i = 0; i < 6; i++)
1750                         s->rand.state64.s[i] = le64_to_cpu(s->rand.state64.s[i]);
1751         } else {
1752                 for (i = 0; i < 4; i++)
1753                         s->rand.state32.s[i] = le32_to_cpu(s->rand.state32.s[i]);
1754         }
1755
1756         for (i = 0; i < s->no_comps; i++) {
1757                 s->comps[i].fileno = le64_to_cpu(s->comps[i].fileno);
1758                 s->comps[i].offset = le64_to_cpu(s->comps[i].offset);
1759         }
1760
1761         td->vstate = p;
1762 }
1763
1764 int verify_state_hdr(struct verify_state_hdr *hdr, struct thread_io_list *s)
1765 {
1766         uint64_t crc;
1767
1768         hdr->version = le64_to_cpu(hdr->version);
1769         hdr->size = le64_to_cpu(hdr->size);
1770         hdr->crc = le64_to_cpu(hdr->crc);
1771
1772         if (hdr->version != VSTATE_HDR_VERSION)
1773                 return 1;
1774
1775         crc = fio_crc32c((void *)s, hdr->size);
1776         if (crc != hdr->crc)
1777                 return 1;
1778
1779         return 0;
1780 }
1781
1782 int verify_load_state(struct thread_data *td, const char *prefix)
1783 {
1784         struct verify_state_hdr hdr;
1785         void *s = NULL;
1786         uint64_t crc;
1787         ssize_t ret;
1788         int fd;
1789
1790         if (!td->o.verify_state)
1791                 return 0;
1792
1793         fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0);
1794         if (fd == -1)
1795                 return 1;
1796
1797         ret = read(fd, &hdr, sizeof(hdr));
1798         if (ret != sizeof(hdr)) {
1799                 if (ret < 0)
1800                         td_verror(td, errno, "read verify state hdr");
1801                 log_err("fio: failed reading verify state header\n");
1802                 goto err;
1803         }
1804
1805         hdr.version = le64_to_cpu(hdr.version);
1806         hdr.size = le64_to_cpu(hdr.size);
1807         hdr.crc = le64_to_cpu(hdr.crc);
1808
1809         if (hdr.version != VSTATE_HDR_VERSION) {
1810                 log_err("fio: unsupported (%d) version in verify state header\n",
1811                                 (unsigned int) hdr.version);
1812                 goto err;
1813         }
1814
1815         s = malloc(hdr.size);
1816         ret = read(fd, s, hdr.size);
1817         if (ret != hdr.size) {
1818                 if (ret < 0)
1819                         td_verror(td, errno, "read verify state");
1820                 log_err("fio: failed reading verity state\n");
1821                 goto err;
1822         }
1823
1824         crc = fio_crc32c(s, hdr.size);
1825         if (crc != hdr.crc) {
1826                 log_err("fio: verify state is corrupt\n");
1827                 goto err;
1828         }
1829
1830         close(fd);
1831
1832         verify_assign_state(td, s);
1833         return 0;
1834 err:
1835         if (s)
1836                 free(s);
1837         close(fd);
1838         return 1;
1839 }
1840
1841 /*
1842  * Use the loaded verify state to know when to stop doing verification
1843  */
1844 int verify_state_should_stop(struct thread_data *td, struct io_u *io_u)
1845 {
1846         struct thread_io_list *s = td->vstate;
1847         struct fio_file *f = io_u->file;
1848         int i;
1849
1850         if (!s || !f)
1851                 return 0;
1852
1853         /*
1854          * If we're not into the window of issues - depth yet, continue. If
1855          * issue is shorter than depth, do check.
1856          */
1857         if ((td->io_blocks[DDIR_READ] < s->depth ||
1858             s->numberio - td->io_blocks[DDIR_READ] > s->depth) &&
1859             s->numberio > s->depth)
1860                 return 0;
1861
1862         /*
1863          * We're in the window of having to check if this io was
1864          * completed or not. If the IO was seen as completed, then
1865          * lets verify it.
1866          */
1867         for (i = 0; i < s->no_comps; i++) {
1868                 if (s->comps[i].fileno != f->fileno)
1869                         continue;
1870                 if (io_u->offset == s->comps[i].offset)
1871                         return 0;
1872         }
1873
1874         /*
1875          * Not found, we have to stop
1876          */
1877         return 1;
1878 }