If we have a lot of jobs, with a lot of subjobs, and they each
have a lot of files, we get pathetic startup performance
because each file add will check all the previously added
files.
Fixes:
bcbfeefa7bce ("fio: add multi directory support")
Signed-off-by: Jens Axboe <axboe@fb.com>
#include "flist.h"
#include "hash.h"
#include "filehash.h"
+#include "lib/bloom.h"
#define HASH_BUCKETS 512
#define HASH_MASK (HASH_BUCKETS - 1)
+#define BLOOM_SIZE 16*1024*1024
+
unsigned int file_hash_size = HASH_BUCKETS * sizeof(struct flist_head);
static struct flist_head *file_hash;
static struct fio_mutex *hash_lock;
+static struct bloom *file_bloom;
static unsigned short hash(const char *name)
{
return alias;
}
+bool file_bloom_exists(const char *fname, bool set)
+{
+ return bloom_string(file_bloom, fname, strlen(fname), set);
+}
+
void file_hash_exit(void)
{
unsigned int i, has_entries = 0;
file_hash = NULL;
fio_mutex_remove(hash_lock);
hash_lock = NULL;
+ bloom_free(file_bloom);
+ file_bloom = NULL;
}
void file_hash_init(void *ptr)
INIT_FLIST_HEAD(&file_hash[i]);
hash_lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ file_bloom = bloom_new(BLOOM_SIZE);
}
#ifndef FIO_FILE_HASH_H
#define FIO_FILE_HASH_H
+#include "lib/types.h"
+
extern unsigned int file_hash_size;
extern void file_hash_init(void *);
extern void remove_file_hash(struct fio_file *);
extern void fio_file_hash_lock(void);
extern void fio_file_hash_unlock(void);
+extern bool file_bloom_exists(const char *, bool);
#endif
}
}
-static bool __is_already_allocated(const char *fname)
+static bool __is_already_allocated(const char *fname, bool set)
{
struct flist_head *entry;
+ bool ret;
- if (flist_empty(&filename_list))
- return false;
+ ret = file_bloom_exists(fname, set);
+ if (!ret)
+ return ret;
flist_for_each(entry, &filename_list) {
struct file_name *fn;
bool ret;
fio_file_hash_lock();
- ret = __is_already_allocated(fname);
+ ret = __is_already_allocated(fname, false);
fio_file_hash_unlock();
return ret;
fn->filename = strdup(fname);
fio_file_hash_lock();
- if (!__is_already_allocated(fname)) {
+ if (!__is_already_allocated(fname, true)) {
flist_add_tail(&fn->list, &filename_list);
fn = NULL;
}