projects
/
fio.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
t/dedupe: fix init of all thread variables
[fio.git]
/
t
/
dedupe.c
diff --git
a/t/dedupe.c
b/t/dedupe.c
index ac5d7f2d4b3a42975f43e1eaabd143f661480b74..5b88fcb8961ad6904b096c29082534ec59228cc3 100644
(file)
--- a/
t/dedupe.c
+++ b/
t/dedupe.c
@@
-11,7
+11,6
@@
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
-#include <linux/fs.h>
#include <fcntl.h>
#include <string.h>
#include <fcntl.h>
#include <string.h>
@@
-28,14
+27,7
@@
#include "../fio_time.h"
#include "../lib/bloom.h"
#include "../fio_time.h"
#include "../lib/bloom.h"
-
-FILE *f_err;
-struct timeval *fio_tv = NULL;
-unsigned int fio_debug = 0;
-
-void __dprint(int type, const char *str, ...)
-{
-}
+#include "debug.h"
struct worker_thread {
pthread_t thread;
struct worker_thread {
pthread_t thread;
@@
-85,17
+77,20
@@
static uint64_t total_size;
static uint64_t cur_offset;
static struct fio_mutex *size_lock;
static uint64_t cur_offset;
static struct fio_mutex *size_lock;
-static
int dev_fd
;
+static
struct fio_file file
;
-static uint64_t get_size(
int fd
, struct stat *sb)
+static uint64_t get_size(
struct fio_file *f
, struct stat *sb)
{
uint64_t ret;
if (S_ISBLK(sb->st_mode)) {
{
uint64_t ret;
if (S_ISBLK(sb->st_mode)) {
- if (ioctl(fd, BLKGETSIZE64, &ret) < 0) {
- perror("ioctl");
+ unsigned long long bytes;
+
+ if (blockdev_size(f, &bytes)) {
+ log_err("dedupe: failed getting bdev size\n");
return 0;
}
return 0;
}
+ ret = bytes;
} else
ret = sb->st_size;
} else
ret = sb->st_size;
@@
-121,17
+116,17
@@
static int get_work(uint64_t *offset, uint64_t *size)
return ret;
}
return ret;
}
-static int
read_block(int fd, void *buf, off_t offse
t)
+static int
__read_block(int fd, void *buf, off_t offset, size_t coun
t)
{
ssize_t ret;
{
ssize_t ret;
- ret = pread(fd, buf,
blocksize
, offset);
+ ret = pread(fd, buf,
count
, offset);
if (ret < 0) {
perror("pread");
return 1;
} else if (!ret)
return 1;
if (ret < 0) {
perror("pread");
return 1;
} else if (!ret)
return 1;
- else if (ret !=
blocksize
) {
+ else if (ret !=
count
) {
log_err("dedupe: short read on block\n");
return 1;
}
log_err("dedupe: short read on block\n");
return 1;
}
@@
-139,6
+134,11
@@
static int read_block(int fd, void *buf, off_t offset)
return 0;
}
return 0;
}
+static int read_block(int fd, void *buf, off_t offset)
+{
+ return __read_block(fd, buf, offset, blocksize);
+}
+
static void add_item(struct chunk *c, struct item *i)
{
/*
static void add_item(struct chunk *c, struct item *i)
{
/*
@@
-166,10
+166,10
@@
static int col_check(struct chunk *c, struct item *i)
ibuf = fio_memalign(blocksize, blocksize);
e = flist_entry(c->extent_list[0].next, struct extent, list);
ibuf = fio_memalign(blocksize, blocksize);
e = flist_entry(c->extent_list[0].next, struct extent, list);
- if (read_block(
dev_
fd, cbuf, e->offset))
+ if (read_block(
file.
fd, cbuf, e->offset))
goto out;
goto out;
- if (read_block(
dev_
fd, ibuf, i->offset))
+ if (read_block(
file.
fd, ibuf, i->offset))
goto out;
ret = memcmp(ibuf, cbuf, blocksize);
goto out;
ret = memcmp(ibuf, cbuf, blocksize);
@@
-267,24
+267,35
@@
static void crc_buf(void *buf, uint32_t *hash)
fio_md5_final(&ctx);
}
fio_md5_final(&ctx);
}
+static unsigned int read_blocks(int fd, void *buf, off_t offset, size_t size)
+{
+ if (__read_block(fd, buf, offset, size))
+ return 0;
+
+ return size / blocksize;
+}
+
static int do_work(struct worker_thread *thread, void *buf)
{
unsigned int nblocks, i;
off_t offset;
static int do_work(struct worker_thread *thread, void *buf)
{
unsigned int nblocks, i;
off_t offset;
- int
err = 0,
nitems = 0;
+ int nitems = 0;
uint64_t ndupes = 0;
struct item *items;
uint64_t ndupes = 0;
struct item *items;
- nblocks = thread->size / blocksize;
offset = thread->cur_offset;
offset = thread->cur_offset;
+
+ nblocks = read_blocks(thread->fd, buf, offset, min(thread->size, (uint64_t)chunk_size));
+ if (!nblocks)
+ return 1;
+
items = malloc(sizeof(*items) * nblocks);
for (i = 0; i < nblocks; i++) {
items = malloc(sizeof(*items) * nblocks);
for (i = 0; i < nblocks; i++) {
- if (read_block(thread->fd, buf, offset))
- break;
- if (items)
- items[i].offset = offset;
- crc_buf(buf, items[i].hash);
+ void *thisptr = buf + (i * blocksize);
+
+ items[i].offset = offset;
+ crc_buf(thisptr, items[i].hash);
offset += blocksize;
nitems++;
}
offset += blocksize;
nitems++;
}
@@
-294,7
+305,7
@@
static int do_work(struct worker_thread *thread, void *buf)
free(items);
thread->items += nitems;
thread->dupes += ndupes;
free(items);
thread->items += nitems;
thread->dupes += ndupes;
- return
err
;
+ return
0
;
}
static void *thread_fn(void *data)
}
static void *thread_fn(void *data)
@@
-302,7
+313,7
@@
static void *thread_fn(void *data)
struct worker_thread *thread = data;
void *buf;
struct worker_thread *thread = data;
void *buf;
- buf = fio_memalign(blocksize,
block
size);
+ buf = fio_memalign(blocksize,
chunk_
size);
do {
if (get_work(&thread->cur_offset, &thread->size)) {
do {
if (get_work(&thread->cur_offset, &thread->size)) {
@@
-316,7
+327,7
@@
static void *thread_fn(void *data)
} while (1);
thread->done = 1;
} while (1);
thread->done = 1;
- fio_memfree(buf,
block
size);
+ fio_memfree(buf,
chunk_
size);
return NULL;
}
return NULL;
}
@@
-332,7
+343,7
@@
static void show_progress(struct worker_thread *threads, unsigned long total)
unsigned long nitems = 0;
uint64_t tdiff;
float perc;
unsigned long nitems = 0;
uint64_t tdiff;
float perc;
- int some_done;
+ int some_done
= 0
;
int i;
for (i = 0; i < num_threads; i++) {
int i;
for (i = 0; i < num_threads; i++) {
@@
-351,7
+362,7
@@
static void show_progress(struct worker_thread *threads, unsigned long total)
this_items *= blocksize;
tdiff = mtime_since_now(&last_tv);
if (tdiff) {
this_items *= blocksize;
tdiff = mtime_since_now(&last_tv);
if (tdiff) {
- this_items
/= tdiff
;
+ this_items
= (this_items * 1000) / (tdiff * 1024)
;
printf("%3.2f%% done (%luKB/sec)\r", perc, this_items);
last_nitems = nitems;
fio_gettime(&last_tv, NULL);
printf("%3.2f%% done (%luKB/sec)\r", perc, this_items);
last_nitems = nitems;
fio_gettime(&last_tv, NULL);
@@
-362,8
+373,8
@@
static void show_progress(struct worker_thread *threads, unsigned long total)
};
}
};
}
-static int run_dedupe_threads(
int fd, uint64_t dev_size, uint64_t *nextents
,
-
uint64_t *nchunks)
+static int run_dedupe_threads(
struct fio_file *f, uint64_t dev_size
,
+
uint64_t *nextents,
uint64_t *nchunks)
{
struct worker_thread *threads;
unsigned long nitems, total_items;
{
struct worker_thread *threads;
unsigned long nitems, total_items;
@@
-376,10
+387,8
@@
static int run_dedupe_threads(int fd, uint64_t dev_size, uint64_t *nextents,
threads = malloc(num_threads * sizeof(struct worker_thread));
for (i = 0; i < num_threads; i++) {
threads = malloc(num_threads * sizeof(struct worker_thread));
for (i = 0; i < num_threads; i++) {
- threads[i].fd = fd;
- threads[i].items = 0;
- threads[i].err = 0;
- threads[i].done = 0;
+ memset(&threads[i], 0, sizeof(struct worker_thread));
+ threads[i].fd = f->fd;
err = pthread_create(&threads[i].thread, NULL, thread_fn, &threads[i]);
if (err) {
err = pthread_create(&threads[i].thread, NULL, thread_fn, &threads[i]);
if (err) {
@@
-419,36
+428,41
@@
static int dedupe_check(const char *filename, uint64_t *nextents,
flags = O_RDONLY;
if (odirect)
flags = O_RDONLY;
if (odirect)
- flags |= O_DIRECT;
+ flags |= OS_O_DIRECT;
+
+ memset(&file, 0, sizeof(file));
+ file.file_name = strdup(filename);
-
dev_
fd = open(filename, flags);
- if (
dev_
fd == -1) {
+
file.
fd = open(filename, flags);
+ if (
file.
fd == -1) {
perror("open");
perror("open");
-
return 1
;
+
goto err
;
}
}
- if (fstat(
dev_
fd, &sb) < 0) {
+ if (fstat(
file.
fd, &sb) < 0) {
perror("fstat");
perror("fstat");
- close(dev_fd);
- return 1;
+ goto err;
}
}
- dev_size = get_size(dev_fd, &sb);
- if (!dev_size) {
- close(dev_fd);
- return 1;
- }
+ dev_size = get_size(&file, &sb);
+ if (!dev_size)
+ goto err;
if (use_bloom) {
uint64_t bloom_entries;
if (use_bloom) {
uint64_t bloom_entries;
- bloom_entries =
(3 * dev_size ) / (blocksize * 2
);
+ bloom_entries =
8 * (dev_size / blocksize
);
bloom = bloom_new(bloom_entries);
}
printf("Will check <%s>, size <%llu>, using %u threads\n", filename, (unsigned long long) dev_size, num_threads);
bloom = bloom_new(bloom_entries);
}
printf("Will check <%s>, size <%llu>, using %u threads\n", filename, (unsigned long long) dev_size, num_threads);
- return run_dedupe_threads(dev_fd, dev_size, nextents, nchunks);
+ return run_dedupe_threads(&file, dev_size, nextents, nchunks);
+err:
+ if (file.fd != -1)
+ close(file.fd);
+ free(file.file_name);
+ return 1;
}
static void show_chunk(struct chunk *c)
}
static void show_chunk(struct chunk *c)
@@
-465,10
+479,15
@@
static void show_chunk(struct chunk *c)
static void show_stat(uint64_t nextents, uint64_t nchunks)
{
static void show_stat(uint64_t nextents, uint64_t nchunks)
{
- double perc;
+ double perc
, ratio
;
printf("Extents=%lu, Unique extents=%lu\n", (unsigned long) nextents, (unsigned long) nchunks);
printf("Extents=%lu, Unique extents=%lu\n", (unsigned long) nextents, (unsigned long) nchunks);
- printf("De-dupe factor: %3.2f\n", (double) nextents / (double) nchunks);
+
+ if (nchunks) {
+ ratio = (double) nextents / (double) nchunks;
+ printf("De-dupe ratio: 1:%3.2f\n", ratio - 1.0);
+ } else
+ printf("De-dupe ratio: 1:infinite\n");
perc = 1.00 - ((double) nchunks / (double) nextents);
perc *= 100.0;
perc = 1.00 - ((double) nchunks / (double) nextents);
perc *= 100.0;
@@
-515,9
+534,11
@@
static int usage(char *argv[])
int main(int argc, char *argv[])
{
int main(int argc, char *argv[])
{
- uint64_t nextents
, nchunks
;
+ uint64_t nextents
= 0, nchunks = 0
;
int c, ret;
int c, ret;
+ debug_init();
+
while ((c = getopt(argc, argv, "b:t:d:o:c:p:B:")) != -1) {
switch (c) {
case 'b':
while ((c = getopt(argc, argv, "b:t:d:o:c:p:B:")) != -1) {
switch (c) {
case 'b':
@@
-563,13
+584,16
@@
int main(int argc, char *argv[])
ret = dedupe_check(argv[optind], &nextents, &nchunks);
ret = dedupe_check(argv[optind], &nextents, &nchunks);
- if (!bloom)
- iter_rb_tree(&nextents, &nchunks);
+ if (!ret) {
+ if (!bloom)
+ iter_rb_tree(&nextents, &nchunks);
- show_stat(nextents, nchunks);
+ show_stat(nextents, nchunks);
+ }
fio_mutex_remove(rb_lock);
fio_mutex_remove(rb_lock);
- bloom_free(bloom);
+ if (bloom)
+ bloom_free(bloom);
scleanup();
return ret;
}
scleanup();
return ret;
}