SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
$(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/lib/*.c))) \
gettime.c ioengines.c init.c stat.c log.c time.c filesetup.c \
- eta.c verify.c memory.c io_u.c parse.c mutex.c options.c \
+ eta.c verify.c memory.c io_u.c parse.c fio_sem.c rwlock.c \
+ pshared.c options.c \
smalloc.c filehash.c profile.c debug.c engines/cpu.c \
engines/mmap.c engines/sync.c engines/null.c engines/net.c \
engines/ftruncate.c engines/filecreate.c \
-include $(OBJS:.o=.d)
T_SMALLOC_OBJS = t/stest.o
-T_SMALLOC_OBJS += gettime.o mutex.o smalloc.o t/log.o t/debug.o t/arch.o
+T_SMALLOC_OBJS += gettime.o fio_sem.o pshared.o smalloc.o t/log.o t/debug.o \
+ t/arch.o
T_SMALLOC_PROGS = t/stest
T_IEEE_OBJS = t/ieee754.o
T_AXMAP_PROGS = t/axmap
T_LFSR_TEST_OBJS = t/lfsr-test.o
-T_LFSR_TEST_OBJS += lib/lfsr.o gettime.o t/log.o t/debug.o t/arch.o
+T_LFSR_TEST_OBJS += lib/lfsr.o gettime.o fio_sem.o pshared.o \
+ t/log.o t/debug.o t/arch.o
T_LFSR_TEST_PROGS = t/lfsr-test
T_GEN_RAND_OBJS = t/gen-rand.o
endif
T_DEDUPE_OBJS = t/dedupe.o
-T_DEDUPE_OBJS += lib/rbtree.o t/log.o mutex.o smalloc.o gettime.o crc/md5.o \
- lib/memalign.o lib/bloom.o t/debug.o crc/xxhash.o t/arch.o \
- crc/murmur3.o crc/crc32c.o crc/crc32c-intel.o crc/crc32c-arm64.o crc/fnv.o
+T_DEDUPE_OBJS += lib/rbtree.o t/log.o fio_sem.o pshared.o smalloc.o gettime.o \
+ crc/md5.o lib/memalign.o lib/bloom.o t/debug.o crc/xxhash.o \
+ t/arch.o crc/murmur3.o crc/crc32c.o crc/crc32c-intel.o \
+ crc/crc32c-arm64.o crc/fnv.o
T_DEDUPE_PROGS = t/fio-dedupe
T_VS_OBJS = t/verify-state.o t/log.o crc/crc32c.o crc/crc32c-intel.o crc/crc32c-arm64.o t/debug.o
#include "lib/mountcheck.h"
#include "rate-submit.h"
#include "helper_thread.h"
+#include "pshared.h"
-static struct fio_mutex *startup_mutex;
+static struct fio_sem *startup_sem;
static struct flist_head *cgroup_list;
static char *cgroup_mnt;
static int exit_value;
if (td->update_rusage) {
td->update_rusage = 0;
update_rusage_stat(td);
- fio_mutex_up(td->rusage_sem);
+ fio_sem_up(td->rusage_sem);
}
}
}
td_set_runstate(td, TD_INITIALIZED);
- dprint(FD_MUTEX, "up startup_mutex\n");
- fio_mutex_up(startup_mutex);
- dprint(FD_MUTEX, "wait on td->mutex\n");
- fio_mutex_down(td->mutex);
- dprint(FD_MUTEX, "done waiting on td->mutex\n");
+ dprint(FD_MUTEX, "up startup_sem\n");
+ fio_sem_up(startup_sem);
+ dprint(FD_MUTEX, "wait on td->sem\n");
+ fio_sem_down(td->sem);
+ dprint(FD_MUTEX, "done waiting on td->sem\n");
/*
* A new gid requires privilege, so we need to do this before setting
deadlock_loop_cnt = 0;
do {
check_update_rusage(td);
- if (!fio_mutex_down_trylock(stat_mutex))
+ if (!fio_sem_down_trylock(stat_sem))
break;
usleep(1000);
if (deadlock_loop_cnt++ > 5000) {
- log_err("fio seems to be stuck grabbing stat_mutex, forcibly exiting\n");
+ log_err("fio seems to be stuck grabbing stat_sem, forcibly exiting\n");
td->error = EDEADLK;
goto err;
}
if (td_trim(td) && td->io_bytes[DDIR_TRIM])
update_runtime(td, elapsed_us, DDIR_TRIM);
fio_gettime(&td->start, NULL);
- fio_mutex_up(stat_mutex);
+ fio_sem_up(stat_sem);
if (td->error || td->terminate)
break;
*/
check_update_rusage(td);
- fio_mutex_down(stat_mutex);
+ fio_sem_down(stat_sem);
update_runtime(td, elapsed_us, DDIR_READ);
fio_gettime(&td->start, NULL);
- fio_mutex_up(stat_mutex);
+ fio_sem_up(stat_sem);
if (td->error || td->terminate)
break;
init_disk_util(td);
- td->rusage_sem = fio_mutex_init(FIO_MUTEX_LOCKED);
+ td->rusage_sem = fio_sem_init(FIO_SEM_LOCKED);
td->update_rusage = 0;
/*
} else if (i == fio_debug_jobno)
*fio_debug_jobp = pid;
}
- dprint(FD_MUTEX, "wait on startup_mutex\n");
- if (fio_mutex_down_timeout(startup_mutex, 10000)) {
+ dprint(FD_MUTEX, "wait on startup_sem\n");
+ if (fio_sem_down_timeout(startup_sem, 10000)) {
log_err("fio: job startup hung? exiting.\n");
fio_terminate_threads(TERMINATE_ALL);
fio_abort = 1;
free(fd);
break;
}
- dprint(FD_MUTEX, "done waiting on startup_mutex\n");
+ dprint(FD_MUTEX, "done waiting on startup_sem\n");
}
/*
m_rate += ddir_rw_sum(td->o.ratemin);
t_rate += ddir_rw_sum(td->o.rate);
todo--;
- fio_mutex_up(td->mutex);
+ fio_sem_up(td->sem);
}
reap_threads(&nr_running, &t_rate, &m_rate);
setup_log(&agg_io_log[DDIR_TRIM], &p, "agg-trim_bw.log");
}
- startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
- if (startup_mutex == NULL)
+ startup_sem = fio_sem_init(FIO_SEM_LOCKED);
+ if (startup_sem == NULL)
return 1;
set_genesis_time();
stat_init();
- helper_thread_create(startup_mutex, sk_out);
+ helper_thread_create(startup_sem, sk_out);
cgroup_list = smalloc(sizeof(*cgroup_list));
INIT_FLIST_HEAD(cgroup_list);
steadystate_free(td);
fio_options_free(td);
if (td->rusage_sem) {
- fio_mutex_remove(td->rusage_sem);
+ fio_sem_remove(td->rusage_sem);
td->rusage_sem = NULL;
}
- fio_mutex_remove(td->mutex);
- td->mutex = NULL;
+ fio_sem_remove(td->sem);
+ td->sem = NULL;
}
free_disk_util();
sfree(cgroup_list);
sfree(cgroup_mnt);
- fio_mutex_remove(startup_mutex);
+ fio_sem_remove(startup_sem);
stat_exit();
return exit_value;
}
#include "cgroup.h"
#include "smalloc.h"
-static struct fio_mutex *lock;
+static struct fio_sem *lock;
struct cgroup_member {
struct flist_head list;
}
if (td->o.cgroup_nodelete)
cm->cgroup_nodelete = 1;
- fio_mutex_down(lock);
+ fio_sem_down(lock);
flist_add_tail(&cm->list, clist);
- fio_mutex_up(lock);
+ fio_sem_up(lock);
}
void cgroup_kill(struct flist_head *clist)
if (!lock)
return;
- fio_mutex_down(lock);
+ fio_sem_down(lock);
flist_for_each_safe(n, tmp, clist) {
cm = flist_entry(n, struct cgroup_member, list);
sfree(cm);
}
- fio_mutex_up(lock);
+ fio_sem_up(lock);
}
static char *get_cgroup_root(struct thread_data *td, char *mnt)
static void fio_init cgroup_init(void)
{
- lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ lock = fio_sem_init(FIO_SEM_UNLOCKED);
if (!lock)
log_err("fio: failed to allocate cgroup lock\n");
}
static void fio_exit cgroup_exit(void)
{
- fio_mutex_remove(lock);
+ fio_sem_remove(lock);
}
print_config "strndup" "$strndup"
##########################################
+# <valgrind/drd.h> probe
+# Note: presence of <valgrind/drd.h> implies that <valgrind/valgrind.h> is
+# also available but not the other way around.
+if test "$valgrind_dev" != "yes" ; then
+ valgrind_dev="no"
+fi
+cat > $TMPC << EOF
+#include <valgrind/drd.h>
+int main(int argc, char **argv)
+{
+ return 0;
+}
+EOF
+if compile_prog "" "" "valgrind_dev"; then
+ valgrind_dev="yes"
+fi
+print_config "Valgrind headers" "$valgrind_dev"
+
# check march=armv8-a+crc+crypto
if test "$march_armv8_a_crc_crypto" != "yes" ; then
march_armv8_a_crc_crypto="no"
if test "$disable_opt" = "yes" ; then
output_sym "CONFIG_DISABLE_OPTIMIZATIONS"
fi
+if test "$valgrind_dev" = "yes"; then
+ output_sym "CONFIG_VALGRIND_DEV"
+fi
if test "$zlib" = "no" ; then
echo "Consider installing zlib-dev (zlib-devel, some fio features depend on it."
if test "$build_static" = "yes"; then
#include <libgen.h>
#include <math.h>
#include <assert.h>
+#ifdef CONFIG_VALGRIND_DEV
+#include <valgrind/drd.h>
+#else
+#define DRD_IGNORE_VAR(x) do { } while (0)
+#endif
#include "fio.h"
#include "smalloc.h"
static int last_majdev, last_mindev;
static struct disk_util *last_du;
-static struct fio_mutex *disk_util_mutex;
+static struct fio_sem *disk_util_sem;
static struct disk_util *__init_per_file_disk_util(struct thread_data *td,
int majdev, int mindev, char *path);
slave->users--;
}
- fio_mutex_remove(du->lock);
+ fio_sem_remove(du->lock);
free(du->sysfs_root);
sfree(du);
}
dprint(FD_DISKUTIL, "update io ticks\n");
- fio_mutex_down(disk_util_mutex);
+ fio_sem_down(disk_util_sem);
if (!helper_should_exit()) {
flist_for_each(entry, &disk_list) {
} else
ret = 1;
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
return ret;
}
struct flist_head *entry;
struct disk_util *du;
- fio_mutex_down(disk_util_mutex);
+ fio_sem_down(disk_util_sem);
flist_for_each(entry, &disk_list) {
du = flist_entry(entry, struct disk_util, list);
if (major == du->major && minor == du->minor) {
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
return du;
}
}
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
return NULL;
}
if (!du)
return NULL;
+ DRD_IGNORE_VAR(du->users);
memset(du, 0, sizeof(*du));
INIT_FLIST_HEAD(&du->list);
l = snprintf(du->path, sizeof(du->path), "%s/stat", path);
du->minor = mindev;
INIT_FLIST_HEAD(&du->slavelist);
INIT_FLIST_HEAD(&du->slaves);
- du->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ du->lock = fio_sem_init(FIO_SEM_UNLOCKED);
du->users = 0;
- fio_mutex_down(disk_util_mutex);
+ fio_sem_down(disk_util_sem);
flist_for_each(entry, &disk_list) {
__du = flist_entry(entry, struct disk_util, list);
if (!strcmp((char *) du->dus.name, (char *) __du->dus.name)) {
disk_util_free(du);
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
return __du;
}
}
get_io_ticks(du, &du->last_dus);
flist_add_tail(&du->list, &disk_list);
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
find_add_disk_slaves(td, path, du);
return du;
void disk_util_prune_entries(void)
{
- fio_mutex_down(disk_util_mutex);
+ fio_sem_down(disk_util_sem);
while (!flist_empty(&disk_list)) {
struct disk_util *du;
}
last_majdev = last_mindev = -1;
- fio_mutex_up(disk_util_mutex);
- fio_mutex_remove(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
+ fio_sem_remove(disk_util_sem);
}
void print_disk_util(struct disk_util_stat *dus, struct disk_util_agg *agg,
struct disk_util *du;
bool do_json;
- if (!disk_util_mutex)
+ if (!disk_util_sem)
return;
- fio_mutex_down(disk_util_mutex);
+ fio_sem_down(disk_util_sem);
if (flist_empty(&disk_list)) {
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
return;
}
}
}
- fio_mutex_up(disk_util_mutex);
+ fio_sem_up(disk_util_sem);
}
void setup_disk_util(void)
{
- disk_util_mutex = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ disk_util_sem = fio_sem_init(FIO_SEM_UNLOCKED);
}
#include "lib/output_buffer.h"
#include "helper_thread.h"
+#include "fio_sem.h"
struct disk_util_stats {
uint64_t ios[2];
struct timespec time;
- struct fio_mutex *lock;
+ struct fio_sem *lock;
unsigned long users;
};
if (du) {
struct flist_head *n;
- fio_mutex_down(du->lock);
+ fio_sem_down(du->lock);
du->users += val;
flist_for_each(n, &du->slavelist) {
slave = flist_entry(n, struct disk_util, slavelist);
slave->users += val;
}
- fio_mutex_up(du->lock);
+ fio_sem_up(du->lock);
}
}
static inline void disk_util_inc(struct disk_util *du)
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
+#ifdef CONFIG_VALGRIND_DEV
+#include <valgrind/drd.h>
+#else
+#define DRD_IGNORE_VAR(x) do { } while (0)
+#endif
#include "fio.h"
#include "lib/pow2.h"
void print_status_init(int thr_number)
{
+ DRD_IGNORE_VAR(__run_str);
__run_str[thr_number] = 'P';
update_condensed_str(__run_str, run_str);
}
* if io is protected by a semaphore, this is set
*/
union {
- struct fio_mutex *lock;
+ struct fio_sem *lock;
struct fio_rwlock *rwlock;
};
static unsigned int file_hash_size = HASH_BUCKETS * sizeof(struct flist_head);
static struct flist_head *file_hash;
-static struct fio_mutex *hash_lock;
+static struct fio_sem *hash_lock;
static struct bloom *file_bloom;
static unsigned short hash(const char *name)
void fio_file_hash_lock(void)
{
if (hash_lock)
- fio_mutex_down(hash_lock);
+ fio_sem_down(hash_lock);
}
void fio_file_hash_unlock(void)
{
if (hash_lock)
- fio_mutex_up(hash_lock);
+ fio_sem_up(hash_lock);
}
void remove_file_hash(struct fio_file *f)
{
- fio_mutex_down(hash_lock);
+ fio_sem_down(hash_lock);
if (fio_file_hashed(f)) {
assert(!flist_empty(&f->hash_list));
fio_file_clear_hashed(f);
}
- fio_mutex_up(hash_lock);
+ fio_sem_up(hash_lock);
}
static struct fio_file *__lookup_file_hash(const char *name)
{
struct fio_file *f;
- fio_mutex_down(hash_lock);
+ fio_sem_down(hash_lock);
f = __lookup_file_hash(name);
- fio_mutex_up(hash_lock);
+ fio_sem_up(hash_lock);
return f;
}
INIT_FLIST_HEAD(&f->hash_list);
- fio_mutex_down(hash_lock);
+ fio_sem_down(hash_lock);
alias = __lookup_file_hash(f->file_name);
if (!alias) {
flist_add_tail(&f->hash_list, &file_hash[hash(f->file_name)]);
}
- fio_mutex_up(hash_lock);
+ fio_sem_up(hash_lock);
return alias;
}
{
unsigned int i, has_entries = 0;
- fio_mutex_down(hash_lock);
+ fio_sem_down(hash_lock);
for (i = 0; i < HASH_BUCKETS; i++)
has_entries += !flist_empty(&file_hash[i]);
- fio_mutex_up(hash_lock);
+ fio_sem_up(hash_lock);
if (has_entries)
log_err("fio: file hash not empty on exit\n");
sfree(file_hash);
file_hash = NULL;
- fio_mutex_remove(hash_lock);
+ fio_sem_remove(hash_lock);
hash_lock = NULL;
bloom_free(file_bloom);
file_bloom = NULL;
for (i = 0; i < HASH_BUCKETS; i++)
INIT_FLIST_HEAD(&file_hash[i]);
- hash_lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ hash_lock = fio_sem_init(FIO_SEM_UNLOCKED);
file_bloom = bloom_new(BLOOM_SIZE);
}
#include "flist.h"
#include "filelock.h"
#include "smalloc.h"
-#include "mutex.h"
+#include "fio_sem.h"
#include "hash.h"
#include "log.h"
struct fio_filelock {
uint32_t hash;
- struct fio_mutex lock;
+ struct fio_sem lock;
struct flist_head list;
unsigned int references;
};
static struct filelock_data {
struct flist_head list;
- struct fio_mutex lock;
+ struct fio_sem lock;
struct flist_head free_list;
struct fio_filelock ffs[MAX_FILELOCKS];
if (ff || trylock)
break;
- fio_mutex_up(&fld->lock);
+ fio_sem_up(&fld->lock);
usleep(1000);
- fio_mutex_down(&fld->lock);
+ fio_sem_down(&fld->lock);
*retry = 1;
} while (1);
INIT_FLIST_HEAD(&fld->list);
INIT_FLIST_HEAD(&fld->free_list);
- if (__fio_mutex_init(&fld->lock, FIO_MUTEX_UNLOCKED))
+ if (__fio_sem_init(&fld->lock, FIO_SEM_UNLOCKED))
goto err;
for (i = 0; i < MAX_FILELOCKS; i++) {
struct fio_filelock *ff = &fld->ffs[i];
- if (__fio_mutex_init(&ff->lock, FIO_MUTEX_UNLOCKED))
+ if (__fio_sem_init(&ff->lock, FIO_SEM_UNLOCKED))
goto err;
flist_add_tail(&ff->list, &fld->free_list);
}
return;
assert(flist_empty(&fld->list));
- __fio_mutex_remove(&fld->lock);
+ __fio_sem_remove(&fld->lock);
while (!flist_empty(&fld->free_list)) {
struct fio_filelock *ff;
ff = flist_first_entry(&fld->free_list, struct fio_filelock, list);
flist_del_init(&ff->list);
- __fio_mutex_remove(&ff->lock);
+ __fio_sem_remove(&ff->lock);
}
sfree(fld);
hash = jhash(fname, strlen(fname), 0);
- fio_mutex_down(&fld->lock);
+ fio_sem_down(&fld->lock);
ff = fio_hash_get(hash, trylock);
if (ff)
ff->references++;
- fio_mutex_up(&fld->lock);
+ fio_sem_up(&fld->lock);
if (!ff) {
assert(!trylock);
}
if (!trylock) {
- fio_mutex_down(&ff->lock);
+ fio_sem_down(&ff->lock);
return false;
}
- if (!fio_mutex_down_trylock(&ff->lock))
+ if (!fio_sem_down_trylock(&ff->lock))
return false;
- fio_mutex_down(&fld->lock);
+ fio_sem_down(&fld->lock);
/*
* If we raced and the only reference to the lock is us, we can
ff = NULL;
}
- fio_mutex_up(&fld->lock);
+ fio_sem_up(&fld->lock);
if (ff) {
- fio_mutex_down(&ff->lock);
+ fio_sem_down(&ff->lock);
return false;
}
hash = jhash(fname, strlen(fname), 0);
- fio_mutex_down(&fld->lock);
+ fio_sem_down(&fld->lock);
ff = fio_hash_find(hash);
if (ff) {
int refs = --ff->references;
- fio_mutex_up(&ff->lock);
+ fio_sem_up(&ff->lock);
if (!refs) {
flist_del_init(&ff->list);
put_filelock(ff);
} else
log_err("fio: file not found for unlocking\n");
- fio_mutex_up(&fld->lock);
+ fio_sem_up(&fld->lock);
}
#include "os/os.h"
#include "hash.h"
#include "lib/axmap.h"
+#include "rwlock.h"
#ifdef CONFIG_LINUX_FALLOCATE
#include <linux/falloc.h>
f->rwlock = fio_rwlock_init();
break;
case FILE_LOCK_EXCLUSIVE:
- f->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ f->lock = fio_sem_init(FIO_SEM_UNLOCKED);
break;
default:
log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode);
else
fio_rwlock_write(f->rwlock);
} else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
- fio_mutex_down(f->lock);
+ fio_sem_down(f->lock);
td->file_locks[f->fileno] = td->o.file_lock_mode;
}
if (td->o.file_lock_mode == FILE_LOCK_READWRITE)
fio_rwlock_unlock(f->rwlock);
else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
- fio_mutex_up(f->lock);
+ fio_sem_up(f->lock);
td->file_locks[f->fileno] = FILE_LOCK_NONE;
}
#include "fifo.h"
#include "arch/arch.h"
#include "os/os.h"
-#include "mutex.h"
#include "log.h"
#include "debug.h"
#include "file.h"
#include <cuda.h>
#endif
+struct fio_sem;
+
/*
* offset generator types
*/
struct timespec iops_sample_time;
volatile int update_rusage;
- struct fio_mutex *rusage_sem;
+ struct fio_sem *rusage_sem;
struct rusage ru_start;
struct rusage ru_end;
uint64_t this_io_bytes[DDIR_RWDIR_CNT];
uint64_t io_skip_bytes;
uint64_t zone_bytes;
- struct fio_mutex *mutex;
+ struct fio_sem *sem;
uint64_t bytes_done[DDIR_RWDIR_CNT];
/*
--- /dev/null
+#include <string.h>
+#include <sys/mman.h>
+#include <assert.h>
+#ifdef CONFIG_VALGRIND_DEV
+#include <valgrind/valgrind.h>
+#else
+#define RUNNING_ON_VALGRIND 0
+#endif
+
+#include "log.h"
+#include "fio_sem.h"
+#include "pshared.h"
+#include "os/os.h"
+#include "fio_time.h"
+#include "gettime.h"
+
+void __fio_sem_remove(struct fio_sem *sem)
+{
+ assert(sem->magic == FIO_SEM_MAGIC);
+ pthread_mutex_destroy(&sem->lock);
+ pthread_cond_destroy(&sem->cond);
+
+ /*
+ * When not running on Valgrind, ensure any subsequent attempt to grab
+ * this semaphore will fail with an assert, instead of just silently
+ * hanging. When running on Valgrind, let Valgrind detect
+ * use-after-free.
+ */
+ if (!RUNNING_ON_VALGRIND)
+ memset(sem, 0, sizeof(*sem));
+}
+
+void fio_sem_remove(struct fio_sem *sem)
+{
+ __fio_sem_remove(sem);
+ munmap((void *) sem, sizeof(*sem));
+}
+
+int __fio_sem_init(struct fio_sem *sem, int value)
+{
+ int ret;
+
+ sem->value = value;
+ /* Initialize .waiters explicitly for Valgrind. */
+ sem->waiters = 0;
+ sem->magic = FIO_SEM_MAGIC;
+
+ ret = mutex_cond_init_pshared(&sem->lock, &sem->cond);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+struct fio_sem *fio_sem_init(int value)
+{
+ struct fio_sem *sem = NULL;
+
+ sem = (void *) mmap(NULL, sizeof(struct fio_sem),
+ PROT_READ | PROT_WRITE,
+ OS_MAP_ANON | MAP_SHARED, -1, 0);
+ if (sem == MAP_FAILED) {
+ perror("mmap semaphore");
+ return NULL;
+ }
+
+ if (!__fio_sem_init(sem, value))
+ return sem;
+
+ fio_sem_remove(sem);
+ return NULL;
+}
+
+static bool sem_timed_out(struct timespec *t, unsigned int msecs)
+{
+ struct timeval tv;
+ struct timespec now;
+
+ gettimeofday(&tv, NULL);
+ now.tv_sec = tv.tv_sec;
+ now.tv_nsec = tv.tv_usec * 1000;
+
+ return mtime_since(t, &now) >= msecs;
+}
+
+int fio_sem_down_timeout(struct fio_sem *sem, unsigned int msecs)
+{
+ struct timeval tv_s;
+ struct timespec base;
+ struct timespec t;
+ int ret = 0;
+
+ assert(sem->magic == FIO_SEM_MAGIC);
+
+ gettimeofday(&tv_s, NULL);
+ base.tv_sec = t.tv_sec = tv_s.tv_sec;
+ base.tv_nsec = t.tv_nsec = tv_s.tv_usec * 1000;
+
+ t.tv_sec += msecs / 1000;
+ t.tv_nsec += ((msecs * 1000000ULL) % 1000000000);
+ if (t.tv_nsec >= 1000000000) {
+ t.tv_nsec -= 1000000000;
+ t.tv_sec++;
+ }
+
+ pthread_mutex_lock(&sem->lock);
+
+ sem->waiters++;
+ while (!sem->value && !ret) {
+ /*
+ * Some platforms (FreeBSD 9?) seems to return timed out
+ * way too early, double check.
+ */
+ ret = pthread_cond_timedwait(&sem->cond, &sem->lock, &t);
+ if (ret == ETIMEDOUT && !sem_timed_out(&base, msecs))
+ ret = 0;
+ }
+ sem->waiters--;
+
+ if (!ret) {
+ sem->value--;
+ pthread_mutex_unlock(&sem->lock);
+ return 0;
+ }
+
+ pthread_mutex_unlock(&sem->lock);
+ return ret;
+}
+
+bool fio_sem_down_trylock(struct fio_sem *sem)
+{
+ bool ret = true;
+
+ assert(sem->magic == FIO_SEM_MAGIC);
+
+ pthread_mutex_lock(&sem->lock);
+ if (sem->value) {
+ sem->value--;
+ ret = false;
+ }
+ pthread_mutex_unlock(&sem->lock);
+
+ return ret;
+}
+
+void fio_sem_down(struct fio_sem *sem)
+{
+ assert(sem->magic == FIO_SEM_MAGIC);
+
+ pthread_mutex_lock(&sem->lock);
+
+ while (!sem->value) {
+ sem->waiters++;
+ pthread_cond_wait(&sem->cond, &sem->lock);
+ sem->waiters--;
+ }
+
+ sem->value--;
+ pthread_mutex_unlock(&sem->lock);
+}
+
+void fio_sem_up(struct fio_sem *sem)
+{
+ int do_wake = 0;
+
+ assert(sem->magic == FIO_SEM_MAGIC);
+
+ pthread_mutex_lock(&sem->lock);
+ read_barrier();
+ if (!sem->value && sem->waiters)
+ do_wake = 1;
+ sem->value++;
+
+ if (do_wake)
+ pthread_cond_signal(&sem->cond);
+
+ pthread_mutex_unlock(&sem->lock);
+}
--- /dev/null
+#ifndef FIO_SEM_H
+#define FIO_SEM_H
+
+#include <pthread.h>
+#include "lib/types.h"
+
+#define FIO_SEM_MAGIC 0x4d555445U
+
+struct fio_sem {
+ pthread_mutex_t lock;
+ pthread_cond_t cond;
+ int value;
+ int waiters;
+ int magic;
+};
+
+enum {
+ FIO_SEM_LOCKED = 0,
+ FIO_SEM_UNLOCKED = 1,
+};
+
+extern int __fio_sem_init(struct fio_sem *, int);
+extern struct fio_sem *fio_sem_init(int);
+extern void __fio_sem_remove(struct fio_sem *);
+extern void fio_sem_remove(struct fio_sem *);
+extern void fio_sem_up(struct fio_sem *);
+extern void fio_sem_down(struct fio_sem *);
+extern bool fio_sem_down_trylock(struct fio_sem *);
+extern int fio_sem_down_timeout(struct fio_sem *, unsigned int);
+
+#endif
#include "fio.h"
-#include "mutex.h"
+#include "fio_sem.h"
#include "smalloc.h"
#include "flist.h"
};
static struct flist_head *flow_list;
-static struct fio_mutex *flow_lock;
+static struct fio_sem *flow_lock;
int flow_threshold_exceeded(struct thread_data *td)
{
if (!flow_lock)
return NULL;
- fio_mutex_down(flow_lock);
+ fio_sem_down(flow_lock);
flist_for_each(n, flow_list) {
flow = flist_entry(n, struct fio_flow, list);
if (!flow) {
flow = smalloc(sizeof(*flow));
if (!flow) {
- fio_mutex_up(flow_lock);
+ fio_sem_up(flow_lock);
return NULL;
}
flow->refs = 0;
}
flow->refs++;
- fio_mutex_up(flow_lock);
+ fio_sem_up(flow_lock);
return flow;
}
if (!flow_lock)
return;
- fio_mutex_down(flow_lock);
+ fio_sem_down(flow_lock);
if (!--flow->refs) {
flist_del(&flow->list);
sfree(flow);
}
- fio_mutex_up(flow_lock);
+ fio_sem_up(flow_lock);
}
void flow_init_job(struct thread_data *td)
return;
}
- flow_lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ flow_lock = fio_sem_init(FIO_SEM_UNLOCKED);
if (!flow_lock) {
log_err("fio: failed to allocate flow lock\n");
sfree(flow_list);
void flow_exit(void)
{
if (flow_lock)
- fio_mutex_remove(flow_lock);
+ fio_sem_remove(flow_lock);
if (flow_list)
sfree(flow_list);
}
}
struct gtod_cpu_data {
- struct fio_mutex *mutex;
+ struct fio_sem *sem;
unsigned int cpu;
};
static void *gtod_thread_main(void *data)
{
- struct fio_mutex *mutex = data;
+ struct fio_sem *sem = data;
int ret;
ret = fio_setaffinity(gettid(), fio_gtod_cpumask);
- fio_mutex_up(mutex);
+ fio_sem_up(sem);
if (ret == -1) {
log_err("gtod: setaffinity failed\n");
int fio_start_gtod_thread(void)
{
- struct fio_mutex *mutex;
+ struct fio_sem *sem;
pthread_attr_t attr;
int ret;
- mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
- if (!mutex)
+ sem = fio_sem_init(FIO_SEM_LOCKED);
+ if (!sem)
return 1;
pthread_attr_init(&attr);
pthread_attr_setstacksize(&attr, 2 * PTHREAD_STACK_MIN);
- ret = pthread_create(>od_thread, &attr, gtod_thread_main, mutex);
+ ret = pthread_create(>od_thread, &attr, gtod_thread_main, sem);
pthread_attr_destroy(&attr);
if (ret) {
log_err("Can't create gtod thread: %s\n", strerror(ret));
goto err;
}
- dprint(FD_MUTEX, "wait on startup_mutex\n");
- fio_mutex_down(mutex);
- dprint(FD_MUTEX, "done waiting on startup_mutex\n");
+ dprint(FD_MUTEX, "wait on startup_sem\n");
+ fio_sem_down(sem);
+ dprint(FD_MUTEX, "done waiting on startup_sem\n");
err:
- fio_mutex_remove(mutex);
+ fio_sem_remove(sem);
return ret;
}
#include <time.h>
#include "fio.h"
+#include "fio_sem.h"
#include "smalloc.h"
#include "hash.h"
pthread_t thread;
int cpu;
int debug;
- pthread_mutex_t lock;
- pthread_mutex_t started;
+ struct fio_sem lock;
unsigned long nr_entries;
uint32_t *seq;
struct clock_entry *entries;
goto err;
}
- pthread_mutex_lock(&t->lock);
- pthread_mutex_unlock(&t->started);
+ fio_sem_down(&t->lock);
first = get_cpu_clock();
c = &t->entries[0];
t->seq = &seq;
t->nr_entries = nr_entries;
t->entries = &entries[i * nr_entries];
- pthread_mutex_init(&t->lock, NULL);
- pthread_mutex_init(&t->started, NULL);
- pthread_mutex_lock(&t->lock);
+ __fio_sem_init(&t->lock, FIO_SEM_LOCKED);
if (pthread_create(&t->thread, NULL, clock_thread_fn, t)) {
failed++;
nr_cpus = i;
for (i = 0; i < nr_cpus; i++) {
struct clock_thread *t = &cthreads[i];
- pthread_mutex_lock(&t->started);
- }
-
- for (i = 0; i < nr_cpus; i++) {
- struct clock_thread *t = &cthreads[i];
-
- pthread_mutex_unlock(&t->lock);
+ fio_sem_up(&t->lock);
}
for (i = 0; i < nr_cpus; i++) {
pthread_join(t->thread, &ret);
if (ret)
failed++;
+ __fio_sem_remove(&t->lock);
}
free(cthreads);
+#ifdef CONFIG_VALGRIND_DEV
+#include <valgrind/drd.h>
+#else
+#define DRD_IGNORE_VAR(x) do { } while (0)
+#endif
+
#include "fio.h"
#include "smalloc.h"
#include "helper_thread.h"
#include "steadystate.h"
+#include "pshared.h"
static struct helper_data {
volatile int exit;
pthread_t thread;
pthread_mutex_t lock;
pthread_cond_t cond;
- struct fio_mutex *startup_mutex;
+ struct fio_sem *startup_sem;
} *helper_data;
void helper_thread_destroy(void)
memcpy(&last_du, &ts, sizeof(ts));
memcpy(&last_ss, &ts, sizeof(ts));
- fio_mutex_up(hd->startup_mutex);
+ fio_sem_up(hd->startup_sem);
msec_to_next_event = DISK_UTIL_MSEC;
while (!ret && !hd->exit) {
return NULL;
}
-int helper_thread_create(struct fio_mutex *startup_mutex, struct sk_out *sk_out)
+int helper_thread_create(struct fio_sem *startup_sem, struct sk_out *sk_out)
{
struct helper_data *hd;
int ret;
if (ret)
return 1;
- hd->startup_mutex = startup_mutex;
+ hd->startup_sem = startup_sem;
+
+ DRD_IGNORE_VAR(helper_data);
ret = pthread_create(&hd->thread, NULL, helper_thread_main, hd);
if (ret) {
helper_data = hd;
- dprint(FD_MUTEX, "wait on startup_mutex\n");
- fio_mutex_down(startup_mutex);
- dprint(FD_MUTEX, "done waiting on startup_mutex\n");
+ dprint(FD_MUTEX, "wait on startup_sem\n");
+ fio_sem_down(startup_sem);
+ dprint(FD_MUTEX, "done waiting on startup_sem\n");
return 0;
}
extern bool helper_should_exit(void);
extern void helper_thread_destroy(void);
extern void helper_thread_exit(void);
-extern int helper_thread_create(struct fio_mutex *, struct sk_out *);
+extern int helper_thread_create(struct fio_sem *, struct sk_out *);
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <dlfcn.h>
+#ifdef CONFIG_VALGRIND_DEV
+#include <valgrind/drd.h>
+#else
+#define DRD_IGNORE_VAR(x) do { } while (0)
+#endif
#include "fio.h"
#ifndef FIO_NO_HAVE_SHM_H
*/
static int setup_thread_area(void)
{
+ int i;
+
if (threads)
return 0;
#endif
memset(threads, 0, max_jobs * sizeof(struct thread_data));
+ for (i = 0; i < max_jobs; i++)
+ DRD_IGNORE_VAR(threads[i]);
fio_debug_jobp = (unsigned int *)(threads + max_jobs);
*fio_debug_jobp = -1;
fio_warned = fio_debug_jobp + 1;
f->real_file_size = -1ULL;
}
- td->mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
+ td->sem = fio_sem_init(FIO_SEM_LOCKED);
td->ts.clat_percentiles = o->clat_percentiles;
td->ts.lat_percentiles = o->lat_percentiles;
#include "filelock.h"
#include "smalloc.h"
#include "blktrace.h"
+#include "pshared.h"
static int iolog_flush(struct io_log *log);
+++ /dev/null
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <fcntl.h>
-#include <time.h>
-#include <errno.h>
-#include <pthread.h>
-#include <sys/mman.h>
-#include <assert.h>
-
-#include "fio.h"
-#include "log.h"
-#include "mutex.h"
-#include "arch/arch.h"
-#include "os/os.h"
-#include "helpers.h"
-#include "fio_time.h"
-#include "gettime.h"
-
-void __fio_mutex_remove(struct fio_mutex *mutex)
-{
- assert(mutex->magic == FIO_MUTEX_MAGIC);
- pthread_cond_destroy(&mutex->cond);
-
- /*
- * Ensure any subsequent attempt to grab this mutex will fail
- * with an assert, instead of just silently hanging.
- */
- memset(mutex, 0, sizeof(*mutex));
-}
-
-void fio_mutex_remove(struct fio_mutex *mutex)
-{
- __fio_mutex_remove(mutex);
- munmap((void *) mutex, sizeof(*mutex));
-}
-
-int cond_init_pshared(pthread_cond_t *cond)
-{
- pthread_condattr_t cattr;
- int ret;
-
- ret = pthread_condattr_init(&cattr);
- if (ret) {
- log_err("pthread_condattr_init: %s\n", strerror(ret));
- return ret;
- }
-
-#ifdef CONFIG_PSHARED
- ret = pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED);
- if (ret) {
- log_err("pthread_condattr_setpshared: %s\n", strerror(ret));
- return ret;
- }
-#endif
- ret = pthread_cond_init(cond, &cattr);
- if (ret) {
- log_err("pthread_cond_init: %s\n", strerror(ret));
- return ret;
- }
-
- return 0;
-}
-
-int mutex_init_pshared(pthread_mutex_t *mutex)
-{
- pthread_mutexattr_t mattr;
- int ret;
-
- ret = pthread_mutexattr_init(&mattr);
- if (ret) {
- log_err("pthread_mutexattr_init: %s\n", strerror(ret));
- return ret;
- }
-
- /*
- * Not all platforms support process shared mutexes (FreeBSD)
- */
-#ifdef CONFIG_PSHARED
- ret = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
- if (ret) {
- log_err("pthread_mutexattr_setpshared: %s\n", strerror(ret));
- return ret;
- }
-#endif
- ret = pthread_mutex_init(mutex, &mattr);
- if (ret) {
- log_err("pthread_mutex_init: %s\n", strerror(ret));
- return ret;
- }
-
- return 0;
-}
-
-int mutex_cond_init_pshared(pthread_mutex_t *mutex, pthread_cond_t *cond)
-{
- int ret;
-
- ret = mutex_init_pshared(mutex);
- if (ret)
- return ret;
-
- ret = cond_init_pshared(cond);
- if (ret)
- return ret;
-
- return 0;
-}
-
-int __fio_mutex_init(struct fio_mutex *mutex, int value)
-{
- int ret;
-
- mutex->value = value;
- mutex->magic = FIO_MUTEX_MAGIC;
-
- ret = mutex_cond_init_pshared(&mutex->lock, &mutex->cond);
- if (ret)
- return ret;
-
- return 0;
-}
-
-struct fio_mutex *fio_mutex_init(int value)
-{
- struct fio_mutex *mutex = NULL;
-
- mutex = (void *) mmap(NULL, sizeof(struct fio_mutex),
- PROT_READ | PROT_WRITE,
- OS_MAP_ANON | MAP_SHARED, -1, 0);
- if (mutex == MAP_FAILED) {
- perror("mmap mutex");
- return NULL;
- }
-
- if (!__fio_mutex_init(mutex, value))
- return mutex;
-
- fio_mutex_remove(mutex);
- return NULL;
-}
-
-static bool mutex_timed_out(struct timespec *t, unsigned int msecs)
-{
- struct timeval tv;
- struct timespec now;
-
- gettimeofday(&tv, NULL);
- now.tv_sec = tv.tv_sec;
- now.tv_nsec = tv.tv_usec * 1000;
-
- return mtime_since(t, &now) >= msecs;
-}
-
-int fio_mutex_down_timeout(struct fio_mutex *mutex, unsigned int msecs)
-{
- struct timeval tv_s;
- struct timespec base;
- struct timespec t;
- int ret = 0;
-
- assert(mutex->magic == FIO_MUTEX_MAGIC);
-
- gettimeofday(&tv_s, NULL);
- base.tv_sec = t.tv_sec = tv_s.tv_sec;
- base.tv_nsec = t.tv_nsec = tv_s.tv_usec * 1000;
-
- t.tv_sec += msecs / 1000;
- t.tv_nsec += ((msecs * 1000000ULL) % 1000000000);
- if (t.tv_nsec >= 1000000000) {
- t.tv_nsec -= 1000000000;
- t.tv_sec++;
- }
-
- pthread_mutex_lock(&mutex->lock);
-
- mutex->waiters++;
- while (!mutex->value && !ret) {
- /*
- * Some platforms (FreeBSD 9?) seems to return timed out
- * way too early, double check.
- */
- ret = pthread_cond_timedwait(&mutex->cond, &mutex->lock, &t);
- if (ret == ETIMEDOUT && !mutex_timed_out(&base, msecs))
- ret = 0;
- }
- mutex->waiters--;
-
- if (!ret) {
- mutex->value--;
- pthread_mutex_unlock(&mutex->lock);
- return 0;
- }
-
- pthread_mutex_unlock(&mutex->lock);
- return ret;
-}
-
-bool fio_mutex_down_trylock(struct fio_mutex *mutex)
-{
- bool ret = true;
-
- assert(mutex->magic == FIO_MUTEX_MAGIC);
-
- pthread_mutex_lock(&mutex->lock);
- if (mutex->value) {
- mutex->value--;
- ret = false;
- }
- pthread_mutex_unlock(&mutex->lock);
-
- return ret;
-}
-
-void fio_mutex_down(struct fio_mutex *mutex)
-{
- assert(mutex->magic == FIO_MUTEX_MAGIC);
-
- pthread_mutex_lock(&mutex->lock);
-
- while (!mutex->value) {
- mutex->waiters++;
- pthread_cond_wait(&mutex->cond, &mutex->lock);
- mutex->waiters--;
- }
-
- mutex->value--;
- pthread_mutex_unlock(&mutex->lock);
-}
-
-void fio_mutex_up(struct fio_mutex *mutex)
-{
- int do_wake = 0;
-
- assert(mutex->magic == FIO_MUTEX_MAGIC);
-
- pthread_mutex_lock(&mutex->lock);
- read_barrier();
- if (!mutex->value && mutex->waiters)
- do_wake = 1;
- mutex->value++;
-
- if (do_wake)
- pthread_cond_signal(&mutex->cond);
-
- pthread_mutex_unlock(&mutex->lock);
-}
-
-void fio_rwlock_write(struct fio_rwlock *lock)
-{
- assert(lock->magic == FIO_RWLOCK_MAGIC);
- pthread_rwlock_wrlock(&lock->lock);
-}
-
-void fio_rwlock_read(struct fio_rwlock *lock)
-{
- assert(lock->magic == FIO_RWLOCK_MAGIC);
- pthread_rwlock_rdlock(&lock->lock);
-}
-
-void fio_rwlock_unlock(struct fio_rwlock *lock)
-{
- assert(lock->magic == FIO_RWLOCK_MAGIC);
- pthread_rwlock_unlock(&lock->lock);
-}
-
-void fio_rwlock_remove(struct fio_rwlock *lock)
-{
- assert(lock->magic == FIO_RWLOCK_MAGIC);
- munmap((void *) lock, sizeof(*lock));
-}
-
-struct fio_rwlock *fio_rwlock_init(void)
-{
- struct fio_rwlock *lock;
- pthread_rwlockattr_t attr;
- int ret;
-
- lock = (void *) mmap(NULL, sizeof(struct fio_rwlock),
- PROT_READ | PROT_WRITE,
- OS_MAP_ANON | MAP_SHARED, -1, 0);
- if (lock == MAP_FAILED) {
- perror("mmap rwlock");
- lock = NULL;
- goto err;
- }
-
- lock->magic = FIO_RWLOCK_MAGIC;
-
- ret = pthread_rwlockattr_init(&attr);
- if (ret) {
- log_err("pthread_rwlockattr_init: %s\n", strerror(ret));
- goto err;
- }
-#ifdef CONFIG_PSHARED
- ret = pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
- if (ret) {
- log_err("pthread_rwlockattr_setpshared: %s\n", strerror(ret));
- goto destroy_attr;
- }
-
- ret = pthread_rwlock_init(&lock->lock, &attr);
-#else
- ret = pthread_rwlock_init(&lock->lock, NULL);
-#endif
-
- if (ret) {
- log_err("pthread_rwlock_init: %s\n", strerror(ret));
- goto destroy_attr;
- }
-
- pthread_rwlockattr_destroy(&attr);
-
- return lock;
-destroy_attr:
- pthread_rwlockattr_destroy(&attr);
-err:
- if (lock)
- fio_rwlock_remove(lock);
- return NULL;
-}
+++ /dev/null
-#ifndef FIO_MUTEX_H
-#define FIO_MUTEX_H
-
-#include <pthread.h>
-#include "lib/types.h"
-
-#define FIO_MUTEX_MAGIC 0x4d555445U
-#define FIO_RWLOCK_MAGIC 0x52574c4fU
-
-struct fio_mutex {
- pthread_mutex_t lock;
- pthread_cond_t cond;
- int value;
- int waiters;
- int magic;
-};
-
-struct fio_rwlock {
- pthread_rwlock_t lock;
- int magic;
-};
-
-enum {
- FIO_MUTEX_LOCKED = 0,
- FIO_MUTEX_UNLOCKED = 1,
-};
-
-extern int __fio_mutex_init(struct fio_mutex *, int);
-extern struct fio_mutex *fio_mutex_init(int);
-extern void __fio_mutex_remove(struct fio_mutex *);
-extern void fio_mutex_remove(struct fio_mutex *);
-extern void fio_mutex_up(struct fio_mutex *);
-extern void fio_mutex_down(struct fio_mutex *);
-extern bool fio_mutex_down_trylock(struct fio_mutex *);
-extern int fio_mutex_down_timeout(struct fio_mutex *, unsigned int);
-
-extern void fio_rwlock_read(struct fio_rwlock *);
-extern void fio_rwlock_write(struct fio_rwlock *);
-extern void fio_rwlock_unlock(struct fio_rwlock *);
-extern struct fio_rwlock *fio_rwlock_init(void);
-extern void fio_rwlock_remove(struct fio_rwlock *);
-
-extern int mutex_init_pshared(pthread_mutex_t *);
-extern int cond_init_pshared(pthread_cond_t *);
-extern int mutex_cond_init_pshared(pthread_mutex_t *, pthread_cond_t *);
-
-#endif
};
struct act_run_data {
- struct fio_mutex *mutex;
+ struct fio_sem *sem;
unsigned int pending;
struct act_slice *slices;
static void get_act_ref(void)
{
- fio_mutex_down(act_run_data->mutex);
+ fio_sem_down(act_run_data->sem);
act_run_data->pending++;
- fio_mutex_up(act_run_data->mutex);
+ fio_sem_up(act_run_data->sem);
}
static int show_slice(struct act_slice *slice, unsigned int slice_num)
struct act_prof_data *apd = td->prof_data;
unsigned int i, slice;
- fio_mutex_down(act_run_data->mutex);
+ fio_sem_down(act_run_data->sem);
if (!act_run_data->slices) {
act_run_data->slices = calloc(apd->nr_slices, sizeof(struct act_slice));
if (!--act_run_data->pending)
act_show_all_stats();
- fio_mutex_up(act_run_data->mutex);
+ fio_sem_up(act_run_data->sem);
}
static int act_td_init(struct thread_data *td)
static void fio_init act_register(void)
{
act_run_data = calloc(1, sizeof(*act_run_data));
- act_run_data->mutex = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ act_run_data->sem = fio_sem_init(FIO_SEM_UNLOCKED);
if (register_profile(&act_profile))
log_err("fio: failed to register profile 'act'\n");
free((void *) act_opts[++org_idx]);
unregister_profile(&act_profile);
- fio_mutex_remove(act_run_data->mutex);
+ fio_sem_remove(act_run_data->sem);
free(act_run_data->slices);
free(act_run_data);
act_run_data = NULL;
--- /dev/null
+#include <string.h>
+
+#include "log.h"
+#include "pshared.h"
+
+int cond_init_pshared(pthread_cond_t *cond)
+{
+ pthread_condattr_t cattr;
+ int ret;
+
+ ret = pthread_condattr_init(&cattr);
+ if (ret) {
+ log_err("pthread_condattr_init: %s\n", strerror(ret));
+ return ret;
+ }
+
+#ifdef CONFIG_PSHARED
+ ret = pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED);
+ if (ret) {
+ log_err("pthread_condattr_setpshared: %s\n", strerror(ret));
+ return ret;
+ }
+#endif
+ ret = pthread_cond_init(cond, &cattr);
+ if (ret) {
+ log_err("pthread_cond_init: %s\n", strerror(ret));
+ return ret;
+ }
+
+ return 0;
+}
+
+int mutex_init_pshared(pthread_mutex_t *mutex)
+{
+ pthread_mutexattr_t mattr;
+ int ret;
+
+ ret = pthread_mutexattr_init(&mattr);
+ if (ret) {
+ log_err("pthread_mutexattr_init: %s\n", strerror(ret));
+ return ret;
+ }
+
+ /*
+ * Not all platforms support process shared mutexes (FreeBSD)
+ */
+#ifdef CONFIG_PSHARED
+ ret = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
+ if (ret) {
+ log_err("pthread_mutexattr_setpshared: %s\n", strerror(ret));
+ return ret;
+ }
+#endif
+ ret = pthread_mutex_init(mutex, &mattr);
+ if (ret) {
+ log_err("pthread_mutex_init: %s\n", strerror(ret));
+ return ret;
+ }
+
+ return 0;
+}
+
+int mutex_cond_init_pshared(pthread_mutex_t *mutex, pthread_cond_t *cond)
+{
+ int ret;
+
+ ret = mutex_init_pshared(mutex);
+ if (ret)
+ return ret;
+
+ ret = cond_init_pshared(cond);
+ if (ret)
+ return ret;
+
+ return 0;
+}
--- /dev/null
+#ifndef FIO_PSHARED_H
+#define FIO_PSHARED_H
+
+#include <pthread.h>
+
+extern int mutex_init_pshared(pthread_mutex_t *);
+extern int cond_init_pshared(pthread_cond_t *);
+extern int mutex_cond_init_pshared(pthread_mutex_t *, pthread_cond_t *);
+
+#endif
--- /dev/null
+#include <stdio.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <assert.h>
+
+#include "log.h"
+#include "rwlock.h"
+#include "os/os.h"
+
+void fio_rwlock_write(struct fio_rwlock *lock)
+{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
+ pthread_rwlock_wrlock(&lock->lock);
+}
+
+void fio_rwlock_read(struct fio_rwlock *lock)
+{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
+ pthread_rwlock_rdlock(&lock->lock);
+}
+
+void fio_rwlock_unlock(struct fio_rwlock *lock)
+{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
+ pthread_rwlock_unlock(&lock->lock);
+}
+
+void fio_rwlock_remove(struct fio_rwlock *lock)
+{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
+ pthread_rwlock_destroy(&lock->lock);
+ munmap((void *) lock, sizeof(*lock));
+}
+
+struct fio_rwlock *fio_rwlock_init(void)
+{
+ struct fio_rwlock *lock;
+ pthread_rwlockattr_t attr;
+ int ret;
+
+ lock = (void *) mmap(NULL, sizeof(struct fio_rwlock),
+ PROT_READ | PROT_WRITE,
+ OS_MAP_ANON | MAP_SHARED, -1, 0);
+ if (lock == MAP_FAILED) {
+ perror("mmap rwlock");
+ lock = NULL;
+ goto err;
+ }
+
+ lock->magic = FIO_RWLOCK_MAGIC;
+
+ ret = pthread_rwlockattr_init(&attr);
+ if (ret) {
+ log_err("pthread_rwlockattr_init: %s\n", strerror(ret));
+ goto err;
+ }
+#ifdef CONFIG_PSHARED
+ ret = pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
+ if (ret) {
+ log_err("pthread_rwlockattr_setpshared: %s\n", strerror(ret));
+ goto destroy_attr;
+ }
+
+ ret = pthread_rwlock_init(&lock->lock, &attr);
+#else
+ ret = pthread_rwlock_init(&lock->lock, NULL);
+#endif
+
+ if (ret) {
+ log_err("pthread_rwlock_init: %s\n", strerror(ret));
+ goto destroy_attr;
+ }
+
+ pthread_rwlockattr_destroy(&attr);
+
+ return lock;
+destroy_attr:
+ pthread_rwlockattr_destroy(&attr);
+err:
+ if (lock)
+ fio_rwlock_remove(lock);
+ return NULL;
+}
--- /dev/null
+#ifndef FIO_RWLOCK_H
+#define FIO_RWLOCK_H
+
+#include <pthread.h>
+
+#define FIO_RWLOCK_MAGIC 0x52574c4fU
+
+struct fio_rwlock {
+ pthread_rwlock_t lock;
+ int magic;
+};
+
+extern void fio_rwlock_read(struct fio_rwlock *);
+extern void fio_rwlock_write(struct fio_rwlock *);
+extern void fio_rwlock_unlock(struct fio_rwlock *);
+extern struct fio_rwlock *fio_rwlock_init(void);
+extern void fio_rwlock_remove(struct fio_rwlock *);
+
+#endif
};
struct cmd_reply {
- struct fio_mutex lock;
+ struct fio_sem lock;
void *data;
size_t size;
int error;
static void sk_lock(struct sk_out *sk_out)
{
- fio_mutex_down(&sk_out->lock);
+ fio_sem_down(&sk_out->lock);
}
static void sk_unlock(struct sk_out *sk_out)
{
- fio_mutex_up(&sk_out->lock);
+ fio_sem_up(&sk_out->lock);
}
void sk_out_assign(struct sk_out *sk_out)
static void sk_out_free(struct sk_out *sk_out)
{
- __fio_mutex_remove(&sk_out->lock);
- __fio_mutex_remove(&sk_out->wait);
- __fio_mutex_remove(&sk_out->xmit);
+ __fio_sem_remove(&sk_out->lock);
+ __fio_sem_remove(&sk_out->wait);
+ __fio_sem_remove(&sk_out->xmit);
sfree(sk_out);
}
flist_add_tail(&entry->list, &sk_out->list);
sk_unlock(sk_out);
- fio_mutex_up(&sk_out->wait);
+ fio_sem_up(&sk_out->wait);
}
}
memcpy(rep->data, in->data, in->size);
}
}
- fio_mutex_up(&rep->lock);
+ fio_sem_up(&rep->lock);
break;
}
default:
{
int ret;
- fio_mutex_down(&sk_out->xmit);
+ fio_sem_down(&sk_out->xmit);
if (entry->flags & SK_F_VEC)
ret = send_vec_entry(sk_out, entry);
entry->size, &entry->tag, NULL);
}
- fio_mutex_up(&sk_out->xmit);
+ fio_sem_up(&sk_out->xmit);
if (ret)
log_err("fio: failed handling cmd %s\n", fio_server_op(entry->opcode));
break;
} else if (!ret) {
fio_server_check_jobs(&job_list);
- fio_mutex_down_timeout(&sk_out->wait, timeout);
+ fio_sem_down_timeout(&sk_out->wait, timeout);
continue;
}
sk_out = smalloc(sizeof(*sk_out));
sk_out->sk = sk;
INIT_FLIST_HEAD(&sk_out->list);
- __fio_mutex_init(&sk_out->lock, FIO_MUTEX_UNLOCKED);
- __fio_mutex_init(&sk_out->wait, FIO_MUTEX_LOCKED);
- __fio_mutex_init(&sk_out->xmit, FIO_MUTEX_UNLOCKED);
+ __fio_sem_init(&sk_out->lock, FIO_SEM_UNLOCKED);
+ __fio_sem_init(&sk_out->wait, FIO_SEM_LOCKED);
+ __fio_sem_init(&sk_out->xmit, FIO_SEM_UNLOCKED);
pid = fork();
if (pid) {
if (!rep)
return ENOMEM;
- __fio_mutex_init(&rep->lock, FIO_MUTEX_LOCKED);
+ __fio_sem_init(&rep->lock, FIO_SEM_LOCKED);
rep->data = NULL;
rep->error = 0;
/*
* Wait for the backend to receive the reply
*/
- if (fio_mutex_down_timeout(&rep->lock, 10000)) {
+ if (fio_sem_down_timeout(&rep->lock, 10000)) {
log_err("fio: timed out waiting for reply\n");
ret = ETIMEDOUT;
goto fail;
*datap = data;
sfree(rep->data);
- __fio_mutex_remove(&rep->lock);
+ __fio_sem_remove(&rep->lock);
sfree(rep);
return ret;
}
* protected by below ->lock */
int sk; /* socket fd to talk to client */
- struct fio_mutex lock; /* protects ref and below list */
+ struct fio_sem lock; /* protects ref and below list */
struct flist_head list; /* list of pending transmit work */
- struct fio_mutex wait; /* wake backend when items added to list */
- struct fio_mutex xmit; /* held while sending data */
+ struct fio_sem wait; /* wake backend when items added to list */
+ struct fio_sem xmit; /* held while sending data */
};
/*
#include <sys/types.h>
#include <limits.h>
#include <fcntl.h>
+#ifdef CONFIG_VALGRIND_DEV
+#include <valgrind/valgrind.h>
+#else
+#define RUNNING_ON_VALGRIND 0
+#define VALGRIND_MALLOCLIKE_BLOCK(addr, size, rzB, is_zeroed) do { } while (0)
+#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do { } while (0)
+#endif
#include "fio.h"
-#include "mutex.h"
+#include "fio_sem.h"
#include "arch/arch.h"
#include "os/os.h"
#include "smalloc.h"
#endif
struct pool {
- struct fio_mutex *lock; /* protects this pool */
+ struct fio_sem *lock; /* protects this pool */
void *map; /* map of blocks */
unsigned int *bitmap; /* blocks free/busy map */
size_t free_blocks; /* free blocks */
size_t mmap_size;
};
+#ifdef SMALLOC_REDZONE
+#define REDZONE_SIZE sizeof(unsigned int)
+#else
+#define REDZONE_SIZE 0
+#endif
+
struct block_hdr {
size_t size;
#ifdef SMALLOC_REDZONE
pool->bitmap = (unsigned int *)((char *) ptr + (pool->nr_blocks * SMALLOC_BPL));
memset(pool->bitmap, 0, bitmap_blocks * sizeof(unsigned int));
- pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ pool->lock = fio_sem_init(FIO_SEM_UNLOCKED);
if (!pool->lock)
goto out_fail;
munmap(pool->map, pool->mmap_size);
if (pool->lock)
- fio_mutex_remove(pool->lock);
+ fio_sem_remove(pool->lock);
}
void scleanup(void)
{
unsigned int *postred = postred_ptr(hdr);
+ /* Let Valgrind fill the red zones. */
+ if (RUNNING_ON_VALGRIND)
+ return;
+
hdr->prered = SMALLOC_PRE_RED;
*postred = SMALLOC_POST_RED;
}
{
unsigned int *postred = postred_ptr(hdr);
+ /* Let Valgrind check the red zones. */
+ if (RUNNING_ON_VALGRIND)
+ return;
+
if (hdr->prered != SMALLOC_PRE_RED) {
log_err("smalloc pre redzone destroyed!\n"
" ptr=%p, prered=%x, expected %x\n",
i = offset / SMALLOC_BPL;
idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
- fio_mutex_down(pool->lock);
+ fio_sem_down(pool->lock);
clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
if (i < pool->next_non_full)
pool->next_non_full = i;
pool->free_blocks += size_to_blocks(hdr->size);
- fio_mutex_up(pool->lock);
+ fio_sem_up(pool->lock);
}
void sfree(void *ptr)
}
if (pool) {
+ VALGRIND_FREELIKE_BLOCK(ptr, REDZONE_SIZE);
sfree_pool(pool, ptr);
return;
}
unsigned int last_idx;
void *ret = NULL;
- fio_mutex_down(pool->lock);
+ fio_sem_down(pool->lock);
nr_blocks = size_to_blocks(size);
if (nr_blocks > pool->free_blocks)
ret = pool->map + offset;
}
fail:
- fio_mutex_up(pool->lock);
+ fio_sem_up(pool->lock);
return ret;
}
return ptr;
}
-void *smalloc(size_t size)
+static void *__smalloc(size_t size, bool is_zeroed)
{
unsigned int i, end_pool;
if (ptr) {
last_pool = i;
+ VALGRIND_MALLOCLIKE_BLOCK(ptr, size,
+ REDZONE_SIZE,
+ is_zeroed);
return ptr;
}
}
return NULL;
}
+void *smalloc(size_t size)
+{
+ return __smalloc(size, false);
+}
+
void *scalloc(size_t nmemb, size_t size)
{
- return smalloc(nmemb * size);
+ return __smalloc(nmemb * size, true);
}
char *smalloc_strdup(const char *str)
#define LOG_MSEC_SLACK 1
-struct fio_mutex *stat_mutex;
+struct fio_sem *stat_sem;
void clear_rusage_stat(struct thread_data *td)
{
void show_run_stats(void)
{
- fio_mutex_down(stat_mutex);
+ fio_sem_down(stat_sem);
__show_run_stats();
- fio_mutex_up(stat_mutex);
+ fio_sem_up(stat_sem);
}
void __show_running_run_stats(void)
struct timespec ts;
int i;
- fio_mutex_down(stat_mutex);
+ fio_sem_down(stat_sem);
rt = malloc(thread_number * sizeof(unsigned long long));
fio_gettime(&ts, NULL);
continue;
if (td->rusage_sem) {
td->update_rusage = 1;
- fio_mutex_down(td->rusage_sem);
+ fio_sem_down(td->rusage_sem);
}
td->update_rusage = 0;
}
}
free(rt);
- fio_mutex_up(stat_mutex);
+ fio_sem_up(stat_sem);
}
static bool status_interval_init;
void stat_init(void)
{
- stat_mutex = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ stat_sem = fio_sem_init(FIO_SEM_UNLOCKED);
}
void stat_exit(void)
* When we have the mutex, we know out-of-band access to it
* have ended.
*/
- fio_mutex_down(stat_mutex);
- fio_mutex_remove(stat_mutex);
+ fio_sem_down(stat_sem);
+ fio_sem_remove(stat_sem);
}
/*
uint64_t io_u_plat[FIO_IO_U_PLAT_NR];
};
-extern struct fio_mutex *stat_mutex;
+extern struct fio_sem *stat_sem;
extern struct jobs_eta *get_jobs_eta(bool force, size_t *size);
#include "../flist.h"
#include "../log.h"
-#include "../mutex.h"
+#include "../fio_sem.h"
#include "../smalloc.h"
#include "../minmax.h"
#include "../crc/md5.h"
static struct rb_root rb_root;
static struct bloom *bloom;
-static struct fio_mutex *rb_lock;
+static struct fio_sem *rb_lock;
static unsigned int blocksize = 4096;
static unsigned int num_threads;
static uint64_t total_size;
static uint64_t cur_offset;
-static struct fio_mutex *size_lock;
+static struct fio_sem *size_lock;
static struct fio_file file;
uint64_t this_chunk;
int ret = 1;
- fio_mutex_down(size_lock);
+ fio_sem_down(size_lock);
if (cur_offset < total_size) {
*offset = cur_offset;
ret = 0;
}
- fio_mutex_up(size_lock);
+ fio_sem_up(size_lock);
return ret;
}
if (!collision_check)
goto add;
- fio_mutex_up(rb_lock);
+ fio_sem_up(rb_lock);
ret = col_check(c, i);
- fio_mutex_down(rb_lock);
+ fio_sem_down(rb_lock);
if (!ret)
goto add;
{
int i;
- fio_mutex_down(rb_lock);
+ fio_sem_down(rb_lock);
for (i = 0; i < nitems; i++) {
if (bloom) {
insert_chunk(&items[i]);
}
- fio_mutex_up(rb_lock);
+ fio_sem_up(rb_lock);
}
static void crc_buf(void *buf, uint32_t *hash)
total_size = dev_size;
total_items = dev_size / blocksize;
cur_offset = 0;
- size_lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ size_lock = fio_sem_init(FIO_SEM_UNLOCKED);
threads = malloc(num_threads * sizeof(struct worker_thread));
for (i = 0; i < num_threads; i++) {
*nextents = nitems;
*nchunks = nitems - *nchunks;
- fio_mutex_remove(size_lock);
+ fio_sem_remove(size_lock);
free(threads);
return err;
}
sinit();
rb_root = RB_ROOT;
- rb_lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
+ rb_lock = fio_sem_init(FIO_SEM_UNLOCKED);
ret = dedupe_check(argv[optind], &nextents, &nchunks);
show_stat(nextents, nchunks);
}
- fio_mutex_remove(rb_lock);
+ fio_sem_remove(rb_lock);
if (bloom)
bloom_free(bloom);
scleanup();
#include "flist.h"
#include "workqueue.h"
#include "smalloc.h"
+#include "pshared.h"
enum {
SW_F_IDLE = 1 << 0,
projects / fio.git / commitdiff