projects
/
fio.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
glusterfs: update for new API
[fio.git]
/
t
/
dedupe.c
diff --git
a/t/dedupe.c
b/t/dedupe.c
index 1577a691eb0dfc575913bd40e18afa54741615a6..2ef8dc539809884cc0372597ef1a260385ee9165 100644
(file)
--- a/
t/dedupe.c
+++ b/
t/dedupe.c
@@
-3,29
+3,24
@@
* just scans the filename for extents of the given size, checksums them,
* and orders them up.
*/
* just scans the filename for extents of the given size, checksums them,
* and orders them up.
*/
+#include <fcntl.h>
+#include <inttypes.h>
#include <stdio.h>
#include <stdio.h>
-#include <st
dio
.h>
+#include <st
ring
.h>
#include <unistd.h>
#include <unistd.h>
-#include <inttypes.h>
-#include <assert.h>
-#include <sys/types.h>
#include <sys/stat.h>
#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <linux/fs.h>
-#include <fcntl.h>
-#include <string.h>
-#include "../lib/rbtree.h"
#include "../flist.h"
#include "../log.h"
#include "../flist.h"
#include "../log.h"
-#include "../
mutex
.h"
+#include "../
fio_sem
.h"
#include "../smalloc.h"
#include "../minmax.h"
#include "../crc/md5.h"
#include "../smalloc.h"
#include "../minmax.h"
#include "../crc/md5.h"
-#include "../memalign.h"
+#include "../
lib/
memalign.h"
#include "../os/os.h"
#include "../gettime.h"
#include "../fio_time.h"
#include "../os/os.h"
#include "../gettime.h"
#include "../fio_time.h"
+#include "../lib/rbtree.h"
#include "../lib/bloom.h"
#include "debug.h"
#include "../lib/bloom.h"
#include "debug.h"
@@
-50,7
+45,7
@@
struct extent {
};
struct chunk {
};
struct chunk {
- struct rb_node rb_node;
+ struct
fio_
rb_node rb_node;
uint64_t count;
uint32_t hash[MD5_HASH_WORDS];
struct flist_head extent_list[0];
uint64_t count;
uint32_t hash[MD5_HASH_WORDS];
struct flist_head extent_list[0];
@@
-63,7
+58,7
@@
struct item {
static struct rb_root rb_root;
static struct bloom *bloom;
static struct rb_root rb_root;
static struct bloom *bloom;
-static struct fio_
mutex
*rb_lock;
+static struct fio_
sem
*rb_lock;
static unsigned int blocksize = 4096;
static unsigned int num_threads;
static unsigned int blocksize = 4096;
static unsigned int num_threads;
@@
-76,19
+71,22
@@
static unsigned int use_bloom = 1;
static uint64_t total_size;
static uint64_t cur_offset;
static uint64_t total_size;
static uint64_t cur_offset;
-static struct fio_
mutex
*size_lock;
+static struct fio_
sem
*size_lock;
-static
int dev_fd
;
+static
struct fio_file file
;
-static uint64_t get_size(
int fd
, struct stat *sb)
+static uint64_t get_size(
struct fio_file *f
, struct stat *sb)
{
uint64_t ret;
if (S_ISBLK(sb->st_mode)) {
{
uint64_t ret;
if (S_ISBLK(sb->st_mode)) {
- if (ioctl(fd, BLKGETSIZE64, &ret) < 0) {
- perror("ioctl");
+ unsigned long long bytes = 0;
+
+ if (blockdev_size(f, &bytes)) {
+ log_err("dedupe: failed getting bdev size\n");
return 0;
}
return 0;
}
+ ret = bytes;
} else
ret = sb->st_size;
} else
ret = sb->st_size;
@@
-100,7
+98,7
@@
static int get_work(uint64_t *offset, uint64_t *size)
uint64_t this_chunk;
int ret = 1;
uint64_t this_chunk;
int ret = 1;
- fio_
mutex
_down(size_lock);
+ fio_
sem
_down(size_lock);
if (cur_offset < total_size) {
*offset = cur_offset;
if (cur_offset < total_size) {
*offset = cur_offset;
@@
-110,7
+108,7
@@
static int get_work(uint64_t *offset, uint64_t *size)
ret = 0;
}
ret = 0;
}
- fio_
mutex
_up(size_lock);
+ fio_
sem
_up(size_lock);
return ret;
}
return ret;
}
@@
-160,20
+158,20
@@
static int col_check(struct chunk *c, struct item *i)
char *cbuf, *ibuf;
int ret = 1;
char *cbuf, *ibuf;
int ret = 1;
- cbuf = fio_memalign(blocksize, blocksize);
- ibuf = fio_memalign(blocksize, blocksize);
+ cbuf = fio_memalign(blocksize, blocksize
, false
);
+ ibuf = fio_memalign(blocksize, blocksize
, false
);
e = flist_entry(c->extent_list[0].next, struct extent, list);
e = flist_entry(c->extent_list[0].next, struct extent, list);
- if (read_block(
dev_
fd, cbuf, e->offset))
+ if (read_block(
file.
fd, cbuf, e->offset))
goto out;
goto out;
- if (read_block(
dev_
fd, ibuf, i->offset))
+ if (read_block(
file.
fd, ibuf, i->offset))
goto out;
ret = memcmp(ibuf, cbuf, blocksize);
out:
goto out;
ret = memcmp(ibuf, cbuf, blocksize);
out:
- fio_memfree(cbuf, blocksize);
- fio_memfree(ibuf, blocksize);
+ fio_memfree(cbuf, blocksize
, false
);
+ fio_memfree(ibuf, blocksize
, false
);
return ret;
}
return ret;
}
@@
-192,7
+190,7
@@
static struct chunk *alloc_chunk(void)
static void insert_chunk(struct item *i)
{
static void insert_chunk(struct item *i)
{
- struct rb_node **p, *parent;
+ struct
fio_
rb_node **p, *parent;
struct chunk *c;
int diff;
struct chunk *c;
int diff;
@@
-213,9
+211,9
@@
static void insert_chunk(struct item *i)
if (!collision_check)
goto add;
if (!collision_check)
goto add;
- fio_
mutex
_up(rb_lock);
+ fio_
sem
_up(rb_lock);
ret = col_check(c, i);
ret = col_check(c, i);
- fio_
mutex
_down(rb_lock);
+ fio_
sem
_down(rb_lock);
if (!ret)
goto add;
if (!ret)
goto add;
@@
-239,7
+237,7
@@
static void insert_chunks(struct item *items, unsigned int nitems,
{
int i;
{
int i;
- fio_
mutex
_down(rb_lock);
+ fio_
sem
_down(rb_lock);
for (i = 0; i < nitems; i++) {
if (bloom) {
for (i = 0; i < nitems; i++) {
if (bloom) {
@@
-253,7
+251,7
@@
static void insert_chunks(struct item *items, unsigned int nitems,
insert_chunk(&items[i]);
}
insert_chunk(&items[i]);
}
- fio_
mutex
_up(rb_lock);
+ fio_
sem
_up(rb_lock);
}
static void crc_buf(void *buf, uint32_t *hash)
}
static void crc_buf(void *buf, uint32_t *hash)
@@
-292,8
+290,7
@@
static int do_work(struct worker_thread *thread, void *buf)
for (i = 0; i < nblocks; i++) {
void *thisptr = buf + (i * blocksize);
for (i = 0; i < nblocks; i++) {
void *thisptr = buf + (i * blocksize);
- if (items)
- items[i].offset = offset;
+ items[i].offset = offset;
crc_buf(thisptr, items[i].hash);
offset += blocksize;
nitems++;
crc_buf(thisptr, items[i].hash);
offset += blocksize;
nitems++;
@@
-312,7
+309,7
@@
static void *thread_fn(void *data)
struct worker_thread *thread = data;
void *buf;
struct worker_thread *thread = data;
void *buf;
- buf = fio_memalign(blocksize, chunk_size);
+ buf = fio_memalign(blocksize, chunk_size
, false
);
do {
if (get_work(&thread->cur_offset, &thread->size)) {
do {
if (get_work(&thread->cur_offset, &thread->size)) {
@@
-326,14
+323,14
@@
static void *thread_fn(void *data)
} while (1);
thread->done = 1;
} while (1);
thread->done = 1;
- fio_memfree(buf, chunk_size);
+ fio_memfree(buf, chunk_size
, false
);
return NULL;
}
static void show_progress(struct worker_thread *threads, unsigned long total)
{
unsigned long last_nitems = 0;
return NULL;
}
static void show_progress(struct worker_thread *threads, unsigned long total)
{
unsigned long last_nitems = 0;
- struct time
val
last_tv;
+ struct time
spec
last_tv;
fio_gettime(&last_tv, NULL);
fio_gettime(&last_tv, NULL);
@@
-362,7
+359,7
@@
static void show_progress(struct worker_thread *threads, unsigned long total)
tdiff = mtime_since_now(&last_tv);
if (tdiff) {
this_items = (this_items * 1000) / (tdiff * 1024);
tdiff = mtime_since_now(&last_tv);
if (tdiff) {
this_items = (this_items * 1000) / (tdiff * 1024);
- printf("%3.2f%% done (%luKB/sec)\r", perc, this_items);
+ printf("%3.2f%% done (%luK
i
B/sec)\r", perc, this_items);
last_nitems = nitems;
fio_gettime(&last_tv, NULL);
} else
last_nitems = nitems;
fio_gettime(&last_tv, NULL);
} else
@@
-372,8
+369,8
@@
static void show_progress(struct worker_thread *threads, unsigned long total)
};
}
};
}
-static int run_dedupe_threads(
int fd, uint64_t dev_size, uint64_t *nextents
,
-
uint64_t *nchunks)
+static int run_dedupe_threads(
struct fio_file *f, uint64_t dev_size
,
+
uint64_t *nextents,
uint64_t *nchunks)
{
struct worker_thread *threads;
unsigned long nitems, total_items;
{
struct worker_thread *threads;
unsigned long nitems, total_items;
@@
-382,14
+379,12
@@
static int run_dedupe_threads(int fd, uint64_t dev_size, uint64_t *nextents,
total_size = dev_size;
total_items = dev_size / blocksize;
cur_offset = 0;
total_size = dev_size;
total_items = dev_size / blocksize;
cur_offset = 0;
- size_lock = fio_
mutex_init(FIO_MUTEX
_UNLOCKED);
+ size_lock = fio_
sem_init(FIO_SEM
_UNLOCKED);
threads = malloc(num_threads * sizeof(struct worker_thread));
for (i = 0; i < num_threads; i++) {
threads = malloc(num_threads * sizeof(struct worker_thread));
for (i = 0; i < num_threads; i++) {
- threads[i].fd = fd;
- threads[i].items = 0;
- threads[i].err = 0;
- threads[i].done = 0;
+ memset(&threads[i], 0, sizeof(struct worker_thread));
+ threads[i].fd = f->fd;
err = pthread_create(&threads[i].thread, NULL, thread_fn, &threads[i]);
if (err) {
err = pthread_create(&threads[i].thread, NULL, thread_fn, &threads[i]);
if (err) {
@@
-415,7
+410,7
@@
static int run_dedupe_threads(int fd, uint64_t dev_size, uint64_t *nextents,
*nextents = nitems;
*nchunks = nitems - *nchunks;
*nextents = nitems;
*nchunks = nitems - *nchunks;
- fio_
mutex
_remove(size_lock);
+ fio_
sem
_remove(size_lock);
free(threads);
return err;
}
free(threads);
return err;
}
@@
-429,25
+424,25
@@
static int dedupe_check(const char *filename, uint64_t *nextents,
flags = O_RDONLY;
if (odirect)
flags = O_RDONLY;
if (odirect)
- flags |= O_DIRECT;
+ flags |= O
S_O
_DIRECT;
- dev_fd = open(filename, flags);
- if (dev_fd == -1) {
+ memset(&file, 0, sizeof(file));
+ file.file_name = strdup(filename);
+
+ file.fd = open(filename, flags);
+ if (file.fd == -1) {
perror("open");
perror("open");
-
return 1
;
+
goto err
;
}
}
- if (fstat(
dev_
fd, &sb) < 0) {
+ if (fstat(
file.
fd, &sb) < 0) {
perror("fstat");
perror("fstat");
- close(dev_fd);
- return 1;
+ goto err;
}
}
- dev_size = get_size(dev_fd, &sb);
- if (!dev_size) {
- close(dev_fd);
- return 1;
- }
+ dev_size = get_size(&file, &sb);
+ if (!dev_size)
+ goto err;
if (use_bloom) {
uint64_t bloom_entries;
if (use_bloom) {
uint64_t bloom_entries;
@@
-458,7
+453,12
@@
static int dedupe_check(const char *filename, uint64_t *nextents,
printf("Will check <%s>, size <%llu>, using %u threads\n", filename, (unsigned long long) dev_size, num_threads);
printf("Will check <%s>, size <%llu>, using %u threads\n", filename, (unsigned long long) dev_size, num_threads);
- return run_dedupe_threads(dev_fd, dev_size, nextents, nchunks);
+ return run_dedupe_threads(&file, dev_size, nextents, nchunks);
+err:
+ if (file.fd != -1)
+ close(file.fd);
+ free(file.file_name);
+ return 1;
}
static void show_chunk(struct chunk *c)
}
static void show_chunk(struct chunk *c)
@@
-493,7
+493,7
@@
static void show_stat(uint64_t nextents, uint64_t nchunks)
static void iter_rb_tree(uint64_t *nextents, uint64_t *nchunks)
{
static void iter_rb_tree(uint64_t *nextents, uint64_t *nchunks)
{
- struct rb_node *n;
+ struct
fio_
rb_node *n;
*nchunks = *nextents = 0;
*nchunks = *nextents = 0;
@@
-533,6
+533,7
@@
int main(int argc, char *argv[])
uint64_t nextents = 0, nchunks = 0;
int c, ret;
uint64_t nextents = 0, nchunks = 0;
int c, ret;
+ arch_init(argv);
debug_init();
while ((c = getopt(argc, argv, "b:t:d:o:c:p:B:")) != -1) {
debug_init();
while ((c = getopt(argc, argv, "b:t:d:o:c:p:B:")) != -1) {
@@
-576,7
+577,7
@@
int main(int argc, char *argv[])
sinit();
rb_root = RB_ROOT;
sinit();
rb_root = RB_ROOT;
- rb_lock = fio_
mutex_init(FIO_MUTEX
_UNLOCKED);
+ rb_lock = fio_
sem_init(FIO_SEM
_UNLOCKED);
ret = dedupe_check(argv[optind], &nextents, &nchunks);
ret = dedupe_check(argv[optind], &nextents, &nchunks);
@@
-587,7
+588,7
@@
int main(int argc, char *argv[])
show_stat(nextents, nchunks);
}
show_stat(nextents, nchunks);
}
- fio_
mutex
_remove(rb_lock);
+ fio_
sem
_remove(rb_lock);
if (bloom)
bloom_free(bloom);
scleanup();
if (bloom)
bloom_free(bloom);
scleanup();