From 3f9f4e2631c4f5d33f125dd71db20c1773d22c91 Mon Sep 17 00:00:00 2001 From: Shawn Lewis Date: Sat, 28 Jul 2007 21:10:37 +0200 Subject: [PATCH 1/1] [PATCH] Allow verification to be done at finer granularity Allow verification to be done at a finer granularity than blocksize. Signed-off-by: Jens Axboe --- HOWTO | 5 ++ fio.h | 1 + options.c | 7 +++ verify.c | 170 ++++++++++++++++++++++++++++++++---------------------- 4 files changed, 114 insertions(+), 69 deletions(-) diff --git a/HOWTO b/HOWTO index b862d74a..2797a58c 100644 --- a/HOWTO +++ b/HOWTO @@ -602,6 +602,11 @@ verifysort=bool If set, fio will sort written verify blocks when it deems can ignore this option unless doing huge amounts of really fast IO where the red-black tree sorting CPU time becomes significant. + +header_interval=siint Write the verification header at a finer granularity + than the blocksize. It will be written for chunks the + size of header_interval. blocksize should divide this + evenly. stonewall Wait for preceeding jobs in the job file to exit, before starting this one. Can be used to insert serialization diff --git a/fio.h b/fio.h index 1173bde5..653f5022 100644 --- a/fio.h +++ b/fio.h @@ -395,6 +395,7 @@ struct thread_options { unsigned int sync_io; unsigned int verify; unsigned int verifysort; + unsigned int header_interval; unsigned int use_thread; unsigned int unlink; unsigned int do_disk_util; diff --git a/options.c b/options.c index 3bf0d247..829b808c 100644 --- a/options.c +++ b/options.c @@ -603,6 +603,13 @@ static struct fio_option options[] = { .help = "Sort written verify blocks for read back", .def = "1", }, + { + .name = "header_interval", + .type = FIO_OPT_STR_VAL_INT, + .off1 = td_var_offset(header_interval), + .help = "Store buffer header every N bytes", + .def = "0", + }, { .name = "write_iolog", .type = FIO_OPT_STR_STORE, diff --git a/verify.c b/verify.c index 156f0686..a362933d 100644 --- a/verify.c +++ b/verify.c @@ -42,16 +42,19 @@ static void hexdump(void *buffer, int len) log_info("\n"); } -static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u) +static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u, + unsigned char header_num) { unsigned char *p = io_u->buf; unsigned char c; - p += sizeof(*hdr); + p += header_num * hdr->len + sizeof(*hdr); c = crc7(p, hdr->len - sizeof(*hdr)); if (c != hdr->crc7) { - log_err("crc7: verify failed at %llu/%lu\n", io_u->offset, io_u->buflen); + log_err("crc7: verify failed at %llu/%u\n", + io_u->offset + header_num * hdr->len, + hdr->len); log_err("crc7: wanted %x, got %x\n", hdr->crc7, c); return 1; } @@ -59,16 +62,19 @@ static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u) return 0; } -static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u) +static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u, + unsigned int header_num) { unsigned char *p = io_u->buf; unsigned short c; - p += sizeof(*hdr); + p += header_num * hdr->len + sizeof(*hdr); c = crc16(p, hdr->len - sizeof(*hdr)); if (c != hdr->crc16) { - log_err("crc16: verify failed at %llu/%lu\n", io_u->offset, io_u->buflen); + log_err("crc16: verify failed at %llu/%u\n", + io_u->offset + header_num * hdr->len, + hdr->len); log_err("crc16: wanted %x, got %x\n", hdr->crc16, c); return 1; } @@ -76,15 +82,19 @@ static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u) return 0; } -static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u) +static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u, + unsigned int header_num) { - unsigned char *p = io_u->buf + sizeof(*hdr); + unsigned char *p = io_u->buf; unsigned long long c; + p += header_num * hdr->len + sizeof(*hdr); c = crc64(p, hdr->len - sizeof(*hdr)); if (c != hdr->crc64) { - log_err("crc64: verify failed at %llu/%lu\n", io_u->offset, io_u->buflen); + log_err("crc64: verify failed at %llu/%u\n", + io_u->offset + header_num * hdr->len, + hdr->len); log_err("crc64: wanted %llx, got %llx\n", hdr->crc64, c); return 1; } @@ -92,16 +102,19 @@ static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u) return 0; } -static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u) +static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u, + unsigned int header_num) { unsigned char *p = io_u->buf; unsigned long c; - p += sizeof(*hdr); + p += header_num * hdr->len + sizeof(*hdr); c = crc32(p, hdr->len - sizeof(*hdr)); if (c != hdr->crc32) { - log_err("crc32: verify failed at %llu/%lu\n", io_u->offset, io_u->buflen); + log_err("crc32: verify failed at %llu/%u\n", + io_u->offset + header_num * hdr->len, + hdr->len); log_err("crc32: wanted %lx, got %lx\n", hdr->crc32, c); return 1; } @@ -109,18 +122,22 @@ static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u) return 0; } -static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u) +static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u, + unsigned int header_num) { - unsigned char *p = io_u->buf + sizeof(*hdr); + unsigned char *p = io_u->buf; uint32_t hash[MD5_HASH_WORDS]; struct md5_ctx md5_ctx = { .hash = hash, }; + p += header_num * hdr->len + sizeof(*hdr); md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr)); - if (memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(hash))) { - log_err("md5: verify failed at %llu/%lu\n", io_u->offset, io_u->buflen); + if (memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash))) { + log_err("md5: verify failed at %llu/%u\n", + io_u->offset + header_num * hdr->len, + hdr->len); hexdump(hdr->md5_digest, sizeof(hdr->md5_digest)); hexdump(md5_ctx.hash, sizeof(hash)); return 1; @@ -131,40 +148,48 @@ static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u) int verify_io_u(struct thread_data *td, struct io_u *io_u) { - struct verify_header *hdr = (struct verify_header *) io_u->buf; + unsigned char *p = (unsigned char*) io_u->buf; + struct verify_header *hdr; + unsigned int hdr_inc, hdr_num = 0; int ret; if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) return 0; - if (hdr->fio_magic != FIO_HDR_MAGIC) { - log_err("Bad verify header %x\n", hdr->fio_magic); - return EIO; - } + hdr_inc = io_u->buflen; + if (td->o.header_interval) + hdr_inc = td->o.header_interval; - switch (hdr->verify_type) { - case VERIFY_MD5: - ret = verify_io_u_md5(hdr, io_u); - break; - case VERIFY_CRC64: - ret = verify_io_u_crc64(hdr, io_u); - break; - case VERIFY_CRC32: - ret = verify_io_u_crc32(hdr, io_u); - break; - case VERIFY_CRC16: - ret = verify_io_u_crc16(hdr, io_u); - break; - case VERIFY_CRC7: - ret = verify_io_u_crc7(hdr, io_u); - break; - default: - log_err("Bad verify type %u\n", hdr->verify_type); - ret = 1; - } + for (; p < (unsigned char*) io_u->buf + io_u->buflen; p += hdr_inc) { + hdr = (struct verify_header*) p; - if (ret) - return EIO; + if (hdr->fio_magic != FIO_HDR_MAGIC) { + log_err("Bad verify header %x\n", hdr->fio_magic); + return EIO; + } + + switch (hdr->verify_type) { + case VERIFY_MD5: + ret = verify_io_u_md5(hdr, io_u, hdr_num); + break; + case VERIFY_CRC64: + ret = verify_io_u_crc64(hdr, io_u, hdr_num); + break; + case VERIFY_CRC32: + ret = verify_io_u_crc32(hdr, io_u, hdr_num); + break; + case VERIFY_CRC16: + ret = verify_io_u_crc16(hdr, io_u, hdr_num); + break; + case VERIFY_CRC7: + ret = verify_io_u_crc7(hdr, io_u, hdr_num); + break; + default: + log_err("Bad verify type %u\n", hdr->verify_type); + ret = 1; + } + hdr_num++; + } return 0; } @@ -206,38 +231,45 @@ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) { const unsigned int len = io_u->buflen - sizeof(struct verify_header); struct verify_header *hdr; - unsigned char *p; + unsigned char *p = io_u->buf, *data; + unsigned int data_len; if (td->o.verify == VERIFY_NULL) return; - hdr = (struct verify_header *) io_u->buf; - hdr->fio_magic = FIO_HDR_MAGIC; - hdr->len = io_u->buflen; - hdr->verify_type = td->o.verify; - - p = io_u->buf + sizeof(*hdr); fill_random_bytes(td, p, len); - switch (td->o.verify) { - case VERIFY_MD5: - fill_md5(hdr, p, len); - break; - case VERIFY_CRC64: - fill_crc64(hdr, p, len); - break; - case VERIFY_CRC32: - fill_crc32(hdr, p, len); - break; - case VERIFY_CRC16: - fill_crc16(hdr, p, len); - break; - case VERIFY_CRC7: - fill_crc7(hdr, p, len); - break; - default: - log_err("fio: bad verify type: %d\n", td->o.verify); - assert(0); + for (;p < (unsigned char*) io_u->buf + io_u->buflen; p += hdr->len) { + hdr = (struct verify_header*) p; + + hdr->fio_magic = FIO_HDR_MAGIC; + hdr->verify_type = td->o.verify; + hdr->len = io_u->buflen; + if (td->o.header_interval) + hdr->len = td->o.header_interval; + + data = p + sizeof(*hdr); + data_len = hdr->len - sizeof(*hdr); + switch (td->o.verify) { + case VERIFY_MD5: + fill_md5(hdr, data, data_len); + break; + case VERIFY_CRC64: + fill_crc64(hdr, data, data_len); + break; + case VERIFY_CRC32: + fill_crc32(hdr, data, data_len); + break; + case VERIFY_CRC16: + fill_crc16(hdr, data, data_len); + break; + case VERIFY_CRC7: + fill_crc7(hdr, data, data_len); + break; + default: + log_err("fio: bad verify type: %d\n", td->o.verify); + assert(0); + } } } -- 2.25.1