X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=t%2Fdedupe.c;h=0a033a4c91e48e77b6a0790b0cf5c45bee4d6134;hp=37437c2c4e970825ae61c7f0378503a6f7040879;hb=d08a6886fa57f819682f05548d284a079a99e77d;hpb=9b36c122ab218c278651d0565d69339eb9948808 diff --git a/t/dedupe.c b/t/dedupe.c index 37437c2c..0a033a4c 100644 --- a/t/dedupe.c +++ b/t/dedupe.c @@ -78,17 +78,20 @@ static uint64_t total_size; static uint64_t cur_offset; static struct fio_mutex *size_lock; -static int dev_fd; +static struct fio_file file; -static uint64_t get_size(int fd, struct stat *sb) +static uint64_t get_size(struct fio_file *f, struct stat *sb) { uint64_t ret; if (S_ISBLK(sb->st_mode)) { - if (ioctl(fd, BLKGETSIZE64, &ret) < 0) { - perror("ioctl"); + unsigned long long bytes; + + if (blockdev_size(f, &bytes)) { + log_err("dedupe: failed getting bdev size\n"); return 0; } + ret = bytes; } else ret = sb->st_size; @@ -114,17 +117,17 @@ static int get_work(uint64_t *offset, uint64_t *size) return ret; } -static int read_block(int fd, void *buf, off_t offset) +static int __read_block(int fd, void *buf, off_t offset, size_t count) { ssize_t ret; - ret = pread(fd, buf, blocksize, offset); + ret = pread(fd, buf, count, offset); if (ret < 0) { perror("pread"); return 1; } else if (!ret) return 1; - else if (ret != blocksize) { + else if (ret != count) { log_err("dedupe: short read on block\n"); return 1; } @@ -132,6 +135,11 @@ static int read_block(int fd, void *buf, off_t offset) return 0; } +static int read_block(int fd, void *buf, off_t offset) +{ + return __read_block(fd, buf, offset, blocksize); +} + static void add_item(struct chunk *c, struct item *i) { /* @@ -159,10 +167,10 @@ static int col_check(struct chunk *c, struct item *i) ibuf = fio_memalign(blocksize, blocksize); e = flist_entry(c->extent_list[0].next, struct extent, list); - if (read_block(dev_fd, cbuf, e->offset)) + if (read_block(file.fd, cbuf, e->offset)) goto out; - if (read_block(dev_fd, ibuf, i->offset)) + if (read_block(file.fd, ibuf, i->offset)) goto out; ret = memcmp(ibuf, cbuf, blocksize); @@ -260,24 +268,36 @@ static void crc_buf(void *buf, uint32_t *hash) fio_md5_final(&ctx); } +static unsigned int read_blocks(int fd, void *buf, off_t offset, size_t size) +{ + if (__read_block(fd, buf, offset, size)) + return 0; + + return size / blocksize; +} + static int do_work(struct worker_thread *thread, void *buf) { unsigned int nblocks, i; off_t offset; - int err = 0, nitems = 0; + int nitems = 0; uint64_t ndupes = 0; struct item *items; - nblocks = thread->size / blocksize; offset = thread->cur_offset; + + nblocks = read_blocks(thread->fd, buf, offset, min(thread->size, (uint64_t)chunk_size)); + if (!nblocks) + return 1; + items = malloc(sizeof(*items) * nblocks); for (i = 0; i < nblocks; i++) { - if (read_block(thread->fd, buf, offset)) - break; + void *thisptr = buf + (i * blocksize); + if (items) items[i].offset = offset; - crc_buf(buf, items[i].hash); + crc_buf(thisptr, items[i].hash); offset += blocksize; nitems++; } @@ -287,7 +307,7 @@ static int do_work(struct worker_thread *thread, void *buf) free(items); thread->items += nitems; thread->dupes += ndupes; - return err; + return 0; } static void *thread_fn(void *data) @@ -295,7 +315,7 @@ static void *thread_fn(void *data) struct worker_thread *thread = data; void *buf; - buf = fio_memalign(blocksize, blocksize); + buf = fio_memalign(blocksize, chunk_size); do { if (get_work(&thread->cur_offset, &thread->size)) { @@ -309,7 +329,7 @@ static void *thread_fn(void *data) } while (1); thread->done = 1; - fio_memfree(buf, blocksize); + fio_memfree(buf, chunk_size); return NULL; } @@ -325,7 +345,7 @@ static void show_progress(struct worker_thread *threads, unsigned long total) unsigned long nitems = 0; uint64_t tdiff; float perc; - int some_done; + int some_done = 0; int i; for (i = 0; i < num_threads; i++) { @@ -344,7 +364,7 @@ static void show_progress(struct worker_thread *threads, unsigned long total) this_items *= blocksize; tdiff = mtime_since_now(&last_tv); if (tdiff) { - this_items /= tdiff; + this_items = (this_items * 1000) / (tdiff * 1024); printf("%3.2f%% done (%luKB/sec)\r", perc, this_items); last_nitems = nitems; fio_gettime(&last_tv, NULL); @@ -355,8 +375,8 @@ static void show_progress(struct worker_thread *threads, unsigned long total) }; } -static int run_dedupe_threads(int fd, uint64_t dev_size, uint64_t *nextents, - uint64_t *nchunks) +static int run_dedupe_threads(struct fio_file *f, uint64_t dev_size, + uint64_t *nextents, uint64_t *nchunks) { struct worker_thread *threads; unsigned long nitems, total_items; @@ -369,7 +389,7 @@ static int run_dedupe_threads(int fd, uint64_t dev_size, uint64_t *nextents, threads = malloc(num_threads * sizeof(struct worker_thread)); for (i = 0; i < num_threads; i++) { - threads[i].fd = fd; + threads[i].fd = f->fd; threads[i].items = 0; threads[i].err = 0; threads[i].done = 0; @@ -414,34 +434,39 @@ static int dedupe_check(const char *filename, uint64_t *nextents, if (odirect) flags |= O_DIRECT; - dev_fd = open(filename, flags); - if (dev_fd == -1) { + memset(&file, 0, sizeof(file)); + file.file_name = strdup(filename); + + file.fd = open(filename, flags); + if (file.fd == -1) { perror("open"); - return 1; + goto err; } - if (fstat(dev_fd, &sb) < 0) { + if (fstat(file.fd, &sb) < 0) { perror("fstat"); - close(dev_fd); - return 1; + goto err; } - dev_size = get_size(dev_fd, &sb); - if (!dev_size) { - close(dev_fd); - return 1; - } + dev_size = get_size(&file, &sb); + if (!dev_size) + goto err; if (use_bloom) { uint64_t bloom_entries; - bloom_entries = (3 * dev_size ) / (blocksize * 2); + bloom_entries = 8 * (dev_size / blocksize); bloom = bloom_new(bloom_entries); } printf("Will check <%s>, size <%llu>, using %u threads\n", filename, (unsigned long long) dev_size, num_threads); - return run_dedupe_threads(dev_fd, dev_size, nextents, nchunks); + return run_dedupe_threads(&file, dev_size, nextents, nchunks); +err: + if (file.fd != -1) + close(file.fd); + free(file.file_name); + return 1; } static void show_chunk(struct chunk *c) @@ -458,10 +483,15 @@ static void show_chunk(struct chunk *c) static void show_stat(uint64_t nextents, uint64_t nchunks) { - double perc; + double perc, ratio; printf("Extents=%lu, Unique extents=%lu\n", (unsigned long) nextents, (unsigned long) nchunks); - printf("De-dupe factor: %3.2f\n", (double) nextents / (double) nchunks); + + if (nchunks) { + ratio = (double) nextents / (double) nchunks; + printf("De-dupe ratio: 1:%3.2f\n", ratio - 1.0); + } else + printf("De-dupe ratio: 1:infinite\n"); perc = 1.00 - ((double) nchunks / (double) nextents); perc *= 100.0; @@ -558,10 +588,12 @@ int main(int argc, char *argv[]) ret = dedupe_check(argv[optind], &nextents, &nchunks); - if (!bloom) - iter_rb_tree(&nextents, &nchunks); + if (!ret) { + if (!bloom) + iter_rb_tree(&nextents, &nchunks); - show_stat(nextents, nchunks); + show_stat(nextents, nchunks); + } fio_mutex_remove(rb_lock); if (bloom)