X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=verify.c;h=fc154ab8cbde68854fb584e36fdf71e2756c4a9c;hb=b36e298b5bdfc31ffc9a4c14e94a9c5c748888b0;hp=c9f2374391d6c74eb6278406bc50e63270149c67;hpb=bd6f78b2f0b6375383948f126368804612ec0035;p=fio.git diff --git a/verify.c b/verify.c index c9f23743..fc154ab8 100644 --- a/verify.c +++ b/verify.c @@ -5,12 +5,16 @@ #include #include #include +#include #include "fio.h" +#include "verify.h" +#include "smalloc.h" #include "crc/md5.h" #include "crc/crc64.h" #include "crc/crc32.h" +#include "crc/crc32c.h" #include "crc/crc16.h" #include "crc/crc7.h" #include "crc/sha256.h" @@ -112,7 +116,7 @@ static void hexdump(void *buffer, int len) */ static inline unsigned int __hdr_size(int verify_type) { - unsigned int len; + unsigned int len = len; switch (verify_type) { case VERIFY_NONE: @@ -125,7 +129,9 @@ static inline unsigned int __hdr_size(int verify_type) case VERIFY_CRC64: len = sizeof(struct vhdr_crc64); break; + case VERIFY_CRC32C: case VERIFY_CRC32: + case VERIFY_CRC32C_INTEL: len = sizeof(struct vhdr_crc32); break; case VERIFY_CRC16: @@ -167,14 +173,13 @@ static void *hdr_priv(struct verify_header *hdr) * Return data area 'header_num' */ static inline void *io_u_verify_off(struct verify_header *hdr, - struct io_u *io_u, - unsigned char header_num) + struct io_u *io_u, unsigned char header_num) { return io_u->buf + header_num * hdr->len + hdr_size(hdr); } static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, - struct io_u *io_u, unsigned int header_num) + struct io_u *io_u, unsigned int header_num) { struct vhdr_meta *vh = hdr_priv(hdr); @@ -182,9 +187,8 @@ static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td, if (vh->offset != io_u->offset + header_num * td->o.verify_interval) { log_err("meta: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); - return EIO; + io_u->offset + header_num * hdr->len, hdr->len); + return EILSEQ; } return 0; @@ -207,11 +211,10 @@ static int verify_io_u_sha512(struct verify_header *hdr, struct io_u *io_u, if (memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) { log_err("sha512: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); + io_u->offset + header_num * hdr->len, hdr->len); hexdump(vh->sha512, sizeof(vh->sha512)); hexdump(sha512_ctx.buf, sizeof(sha512)); - return EIO; + return EILSEQ; } return 0; @@ -234,18 +237,17 @@ static int verify_io_u_sha256(struct verify_header *hdr, struct io_u *io_u, if (memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) { log_err("sha256: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); + io_u->offset + header_num * hdr->len, hdr->len); hexdump(vh->sha256, sizeof(vh->sha256)); hexdump(sha256_ctx.buf, sizeof(sha256)); - return EIO; + return EILSEQ; } return 0; } static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u, - unsigned char header_num) + unsigned char header_num) { void *p = io_u_verify_off(hdr, io_u, header_num); struct vhdr_crc7 *vh = hdr_priv(hdr); @@ -257,17 +259,16 @@ static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u, if (c != vh->crc7) { log_err("crc7: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); + io_u->offset + header_num * hdr->len, hdr->len); log_err("crc7: wanted %x, got %x\n", vh->crc7, c); - return EIO; + return EILSEQ; } return 0; } static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u, - unsigned int header_num) + unsigned int header_num) { void *p = io_u_verify_off(hdr, io_u, header_num); struct vhdr_crc16 *vh = hdr_priv(hdr); @@ -279,17 +280,16 @@ static int verify_io_u_crc16(struct verify_header *hdr, struct io_u *io_u, if (c != vh->crc16) { log_err("crc16: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); + io_u->offset + header_num * hdr->len, hdr->len); log_err("crc16: wanted %x, got %x\n", vh->crc16, c); - return EIO; + return EILSEQ; } return 0; } static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u, - unsigned int header_num) + unsigned int header_num) { void *p = io_u_verify_off(hdr, io_u, header_num); struct vhdr_crc64 *vh = hdr_priv(hdr); @@ -303,15 +303,16 @@ static int verify_io_u_crc64(struct verify_header *hdr, struct io_u *io_u, log_err("crc64: verify failed at %llu/%u\n", io_u->offset + header_num * hdr->len, hdr->len); - log_err("crc64: wanted %llx, got %llx\n", (unsigned long long) vh->crc64, c); - return EIO; + log_err("crc64: wanted %llx, got %llx\n", + (unsigned long long) vh->crc64, c); + return EILSEQ; } return 0; } static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u, - unsigned int header_num) + unsigned int header_num) { void *p = io_u_verify_off(hdr, io_u, header_num); struct vhdr_crc32 *vh = hdr_priv(hdr); @@ -323,17 +324,40 @@ static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u, if (c != vh->crc32) { log_err("crc32: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); + io_u->offset + header_num * hdr->len, hdr->len); log_err("crc32: wanted %x, got %x\n", vh->crc32, c); - return EIO; + return EILSEQ; + } + + return 0; +} + +static int verify_io_u_crc32c(struct verify_header *hdr, struct io_u *io_u, + unsigned int header_num) +{ + void *p = io_u_verify_off(hdr, io_u, header_num); + struct vhdr_crc32 *vh = hdr_priv(hdr); + uint32_t c; + + dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", io_u, hdr->len); + + if (hdr->verify_type == VERIFY_CRC32C_INTEL) + c = crc32c_intel(p, hdr->len - hdr_size(hdr)); + else + c = crc32c(p, hdr->len - hdr_size(hdr)); + + if (c != vh->crc32) { + log_err("crc32c: verify failed at %llu/%u\n", + io_u->offset + header_num * hdr->len, hdr->len); + log_err("crc32c: wanted %x, got %x\n", vh->crc32, c); + return EILSEQ; } return 0; } static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u, - unsigned int header_num) + unsigned int header_num) { void *p = io_u_verify_off(hdr, io_u, header_num); struct vhdr_md5 *vh = hdr_priv(hdr); @@ -349,11 +373,10 @@ static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u, if (memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) { log_err("md5: verify failed at %llu/%u\n", - io_u->offset + header_num * hdr->len, - hdr->len); + io_u->offset + header_num * hdr->len, hdr->len); hexdump(vh->md5_digest, sizeof(vh->md5_digest)); hexdump(md5_ctx.hash, sizeof(hash)); - return EIO; + return EILSEQ; } return 0; @@ -368,7 +391,7 @@ static unsigned int hweight8(unsigned int w) } int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size, - char *buf, unsigned int len, unsigned int mod) + char *buf, unsigned int len, unsigned int mod) { unsigned int i; char split_pattern[4]; @@ -386,7 +409,7 @@ int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size, log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", buf[i], split_pattern[mod], bits); log_err("fio: bad pattern block offset %u\n", i); - return EIO; + return EILSEQ; } mod++; if (mod == pattern_size) @@ -396,6 +419,26 @@ int verify_io_u_pattern(unsigned long pattern, unsigned long pattern_size, return 0; } +/* + * Push IO verification to a separate thread + */ +int verify_io_u_async(struct thread_data *td, struct io_u *io_u) +{ + if (io_u->file) + put_file_log(td, io_u->file); + + io_u->file = NULL; + + pthread_mutex_lock(&td->io_u_lock); + flist_del(&io_u->list); + flist_add_tail(&io_u->list, &td->verify_list); + pthread_mutex_unlock(&td->io_u_lock); + + pthread_cond_signal(&td->verify_cond); + io_u->flags |= IO_U_F_FREE_DEF; + return 0; +} + int verify_io_u(struct thread_data *td, struct io_u *io_u) { struct verify_header *hdr; @@ -411,7 +454,8 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u) hdr_inc = td->o.verify_interval; ret = 0; - for (p = io_u->buf; p < io_u->buf + io_u->buflen; p += hdr_inc, hdr_num++) { + for (p = io_u->buf; p < io_u->buf + io_u->buflen; + p += hdr_inc, hdr_num++) { if (ret && td->o.verify_fatal) { td->terminate = 1; break; @@ -423,17 +467,17 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u) if (hdr->fio_magic != FIO_HDR_MAGIC) { log_err("Bad verify header %x\n", hdr->fio_magic); - return EIO; + return EILSEQ; } if (td->o.verify_pattern_bytes) { dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n", io_u, hdr->len); ret = verify_io_u_pattern(td->o.verify_pattern, - td->o.verify_pattern_bytes, - p + hdr_size, - hdr_inc - hdr_size, - hdr_size % 4); + td->o.verify_pattern_bytes, + p + hdr_size, + hdr_inc - hdr_size, + hdr_size % 4); if (ret) log_err("fio: verify failed at %llu/%u\n", io_u->offset + hdr_num * hdr->len, @@ -448,6 +492,10 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u) case VERIFY_CRC64: ret = verify_io_u_crc64(hdr, io_u, hdr_num); break; + case VERIFY_CRC32C: + case VERIFY_CRC32C_INTEL: + ret = verify_io_u_crc32c(hdr, io_u, hdr_num); + break; case VERIFY_CRC32: ret = verify_io_u_crc32(hdr, io_u, hdr_num); break; @@ -476,7 +524,7 @@ int verify_io_u(struct thread_data *td, struct io_u *io_u) } static void fill_meta(struct verify_header *hdr, struct thread_data *td, - struct io_u *io_u, unsigned int header_num) + struct io_u *io_u, unsigned int header_num) { struct vhdr_meta *vh = hdr_priv(hdr); @@ -533,6 +581,16 @@ static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) vh->crc32 = crc32(p, len); } +static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) +{ + struct vhdr_crc32 *vh = hdr_priv(hdr); + + if (hdr->verify_type == VERIFY_CRC32C_INTEL) + vh->crc32 = crc32c_intel(p, len); + else + vh->crc32 = crc32c(p, len); +} + static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) { struct vhdr_crc64 *vh = hdr_priv(hdr); @@ -570,7 +628,7 @@ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) if (td->o.verify_interval) hdr_inc = td->o.verify_interval; - for (;p < io_u->buf + io_u->buflen; p += hdr_inc) { + for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { hdr = p; hdr->fio_magic = FIO_HDR_MAGIC; @@ -590,6 +648,12 @@ void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) io_u, hdr->len); fill_crc64(hdr, data, data_len); break; + case VERIFY_CRC32C: + case VERIFY_CRC32C_INTEL: + dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", + io_u, hdr->len); + fill_crc32c(hdr, data, data_len); + break; case VERIFY_CRC32: dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", io_u, hdr->len); @@ -645,9 +709,9 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u) ipo = rb_entry(n, struct io_piece, rb_node); rb_erase(n, &td->io_hist_tree); - } else if (!list_empty(&td->io_hist_list)) { - ipo = list_entry(td->io_hist_list.next, struct io_piece, list); - list_del(&ipo->list); + } else if (!flist_empty(&td->io_hist_list)) { + ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); + flist_del(&ipo->list); } if (ipo) { @@ -655,7 +719,7 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u) io_u->buflen = ipo->len; io_u->file = ipo->file; - if ((io_u->file->flags & FIO_FILE_OPEN) == 0) { + if (!fio_file_open(io_u->file)) { int r = td_io_open_file(td, io_u->file); if (r) { @@ -666,7 +730,7 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u) } get_file(ipo->file); - assert(io_u->file->flags & FIO_FILE_OPEN); + assert(fio_file_open(io_u->file)); io_u->ddir = DDIR_READ; io_u->xfer_buf = io_u->buf; io_u->xfer_buflen = io_u->buflen; @@ -678,3 +742,121 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u) dprint(FD_VERIFY, "get_next_verify: empty\n"); return 1; } + +static void *verify_async_thread(void *data) +{ + struct thread_data *td = data; + struct io_u *io_u; + int ret = 0; + + if (td->o.verify_cpumask_set && + fio_setaffinity(td->pid, td->o.verify_cpumask)) { + log_err("fio: failed setting verify thread affinity\n"); + goto done; + } + + do { + FLIST_HEAD(list); + + read_barrier(); + if (td->verify_thread_exit) + break; + + pthread_mutex_lock(&td->io_u_lock); + + while (flist_empty(&td->verify_list) && + !td->verify_thread_exit) { + ret = pthread_cond_wait(&td->verify_cond, + &td->io_u_lock); + if (ret) { + pthread_mutex_unlock(&td->io_u_lock); + break; + } + } + + flist_splice_init(&td->verify_list, &list); + pthread_mutex_unlock(&td->io_u_lock); + + if (flist_empty(&list)) + continue; + + while (!flist_empty(&list)) { + io_u = flist_entry(list.next, struct io_u, list); + flist_del_init(&io_u->list); + + ret = verify_io_u(td, io_u); + put_io_u(td, io_u); + if (!ret) + continue; + if (td->o.continue_on_error && + td_non_fatal_error(ret)) { + update_error_count(td, ret); + td_clear_error(td); + ret = 0; + } + } + } while (!ret); + + if (ret) { + td_verror(td, ret, "async_verify"); + td->terminate = 1; + } + +done: + pthread_mutex_lock(&td->io_u_lock); + td->nr_verify_threads--; + pthread_mutex_unlock(&td->io_u_lock); + + pthread_cond_signal(&td->free_cond); + return NULL; +} + +int verify_async_init(struct thread_data *td) +{ + int i, ret; + + td->verify_thread_exit = 0; + + td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); + for (i = 0; i < td->o.verify_async; i++) { + ret = pthread_create(&td->verify_threads[i], NULL, + verify_async_thread, td); + if (ret) { + log_err("fio: async verify creation failed: %s\n", + strerror(ret)); + break; + } + ret = pthread_detach(td->verify_threads[i]); + if (ret) { + log_err("fio: async verify thread detach failed: %s\n", + strerror(ret)); + break; + } + td->nr_verify_threads++; + } + + if (i != td->o.verify_async) { + td->verify_thread_exit = 1; + write_barrier(); + pthread_cond_broadcast(&td->verify_cond); + return 1; + } + + return 0; +} + +void verify_async_exit(struct thread_data *td) +{ + td->verify_thread_exit = 1; + write_barrier(); + pthread_cond_broadcast(&td->verify_cond); + + pthread_mutex_lock(&td->io_u_lock); + + while (td->nr_verify_threads) + pthread_cond_wait(&td->free_cond, &td->io_u_lock); + + pthread_mutex_unlock(&td->io_u_lock); + free(td->verify_threads); + td->verify_threads = NULL; +}