/*
* glusterfs engine
*
- * IO engine using Glusterfs's gfapi interface
+ * common Glusterfs's gfapi interface
*
*/
-#include <glusterfs/api/glfs.h>
-#include <glusterfs/api/glfs-handles.h>
-#include "../fio.h"
-
-struct gf_options {
- struct thread_data *td;
- char *gf_vol;
- char *gf_brick;
+#include "gfapi.h"
+#include "../optgroup.h"
+
+struct fio_option gfapi_options[] = {
+ {
+ .name = "volume",
+ .lname = "Glusterfs volume",
+ .type = FIO_OPT_STR_STORE,
+ .help = "Name of the Glusterfs volume",
+ .off1 = offsetof(struct gf_options, gf_vol),
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_GFAPI,
+ },
+ {
+ .name = "brick",
+ .lname = "Glusterfs brick name",
+ .type = FIO_OPT_STR_STORE,
+ .help = "Name of the Glusterfs brick to connect",
+ .off1 = offsetof(struct gf_options, gf_brick),
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_GFAPI,
+ },
+ {
+ .name = "single-instance",
+ .lname = "Single glusterfs instance",
+ .type = FIO_OPT_BOOL,
+ .help = "Only one glusterfs instance",
+ .off1 = offsetof(struct gf_options, gf_single_instance),
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_GFAPI,
+ },
+ {
+ .name = NULL,
+ },
};
-struct gf_data {
- glfs_t *fs;
- glfs_fd_t *fd;
+struct glfs_info {
+ struct flist_head list;
+ char *volume;
+ char *brick;
+ glfs_t *fs;
+ int refcount;
};
-static struct fio_option options[] = {
- {
- .name = "volume",
- .lname = "Glusterfs volume",
- .type = FIO_OPT_STR_STORE,
- .help = "Name of the Glusterfs volume",
- .off1 = offsetof(struct gf_options, gf_vol),
- .category = FIO_OPT_C_ENGINE,
- .group = FIO_OPT_G_GFAPI,
- },
- {
- .name = "brick",
- .lname = "Glusterfs brick name",
- .type = FIO_OPT_STR_STORE,
- .help = "Name of the Glusterfs brick to connect",
- .off1 = offsetof(struct gf_options, gf_brick),
- .category = FIO_OPT_C_ENGINE,
- .group = FIO_OPT_G_GFAPI,
- },
- {
- .name = NULL,
- },
-};
+static pthread_mutex_t glfs_lock = PTHREAD_MUTEX_INITIALIZER;
+static FLIST_HEAD(glfs_list_head);
-static int fio_gf_setup(struct thread_data *td)
+static glfs_t *fio_gf_new_fs(char *volume, char *brick)
{
int r = 0;
- struct gf_data *g = NULL;
- struct gf_options *opt = td->eo;
- struct stat sb = {0, };
-
- dprint(FD_IO, "fio setup\n");
+ glfs_t *fs;
+ struct stat sb = { 0, };
- if (td->io_ops->data)
- return 0;
-
- g = malloc(sizeof(struct gf_data));
- if (!g){
- log_err("malloc failed.\n");
- return -ENOMEM;
+ fs = glfs_new(volume);
+ if (!fs) {
+ log_err("glfs_new failed.\n");
+ goto out;
}
- g->fs = NULL; g->fd = NULL;
-
- g->fs = glfs_new (opt->gf_vol);
- if (!g->fs){
- log_err("glfs_new failed.\n");
- goto cleanup;
- }
- glfs_set_logging (g->fs, "/tmp/fio_gfapi.log", 7);
+ glfs_set_logging(fs, "/tmp/fio_gfapi.log", 7);
/* default to tcp */
- r = glfs_set_volfile_server(g->fs, "tcp", opt->gf_brick, 0);
- if (r){
- log_err("glfs_set_volfile_server failed.\n");
- goto cleanup;
+ r = glfs_set_volfile_server(fs, "tcp", brick, 0);
+ if (r) {
+ log_err("glfs_set_volfile_server failed.\n");
+ goto out;
}
- r = glfs_init(g->fs);
- if (r){
- log_err("glfs_init failed. Is glusterd running on brick?\n");
- goto cleanup;
+ r = glfs_init(fs);
+ if (r) {
+ log_err("glfs_init failed. Is glusterd running on brick?\n");
+ goto out;
}
sleep(2);
- r = glfs_lstat (g->fs, ".", &sb);
- if (r){
- log_err("glfs_lstat failed.\n");
- goto cleanup;
+ r = glfs_lstat(fs, ".", &sb);
+ if (r) {
+ log_err("glfs_lstat failed.\n");
+ goto out;
}
- dprint(FD_FILE, "fio setup %p\n", g->fs);
- td->io_ops->data = g;
-cleanup:
- if (r){
- if (g){
- if (g->fs){
- glfs_fini(g->fs);
- }
- free(g);
- }
+
+out:
+ if (r) {
+ glfs_fini(fs);
+ fs = NULL;
}
- return r;
+ return fs;
}
-static void fio_gf_cleanup(struct thread_data *td)
+static glfs_t *fio_gf_get_glfs(struct gf_options *opt,
+ char *volume, char *brick)
{
-}
+ struct glfs_info *glfs = NULL;
+ struct glfs_info *tmp;
+ struct flist_head *entry;
-static int fio_gf_get_file_size(struct thread_data *td, struct fio_file *f)
-{
- struct stat buf;
- int ret;
- struct gf_data *g = td->io_ops->data;
+ if (!opt->gf_single_instance)
+ return fio_gf_new_fs(volume, brick);
- dprint(FD_FILE, "get file size %s\n", f->file_name);
+ pthread_mutex_lock (&glfs_lock);
- if (!g || !g->fs)
- {
- f->real_file_size = 0;
- fio_file_set_size_known(f);
- }
- if (fio_file_size_known(f))
- return 0;
+ flist_for_each(entry, &glfs_list_head) {
+ tmp = flist_entry(entry, struct glfs_info, list);
+ if (!strcmp(volume, tmp->volume) &&
+ !strcmp(brick, tmp->brick)) {
+ glfs = tmp;
+ break;
+ }
+ }
- ret = glfs_lstat (g->fs, f->file_name, &buf);
- if (ret < 0){
- log_err("glfs_lstat failed.\n");
- return ret;
- }
+ if (glfs) {
+ glfs->refcount++;
+ } else {
+ glfs = malloc(sizeof(*glfs));
+ if (!glfs)
+ goto out;
+ INIT_FLIST_HEAD(&glfs->list);
+ glfs->refcount = 0;
+ glfs->volume = strdup(volume);
+ glfs->brick = strdup(brick);
+ glfs->fs = fio_gf_new_fs(volume, brick);
+ if (!glfs->fs) {
+ free(glfs);
+ glfs = NULL;
+ goto out;
+ }
- f->real_file_size = buf.st_size;
- fio_file_set_size_known(f);
+ flist_add_tail(&glfs->list, &glfs_list_head);
+ glfs->refcount = 1;
+ }
- return 0;
+out:
+ pthread_mutex_unlock (&glfs_lock);
+ if (glfs)
+ return glfs->fs;
+ return NULL;
}
-static int fio_gf_open_file(struct thread_data *td, struct fio_file *f)
+static void fio_gf_put_glfs(struct gf_options *opt, glfs_t *fs)
{
+ struct glfs_info *glfs = NULL;
+ struct glfs_info *tmp;
+ struct flist_head *entry;
+
+ if (!opt->gf_single_instance) {
+ glfs_fini(fs);
+ return;
+ }
+
+ pthread_mutex_lock (&glfs_lock);
+
+ flist_for_each(entry, &glfs_list_head) {
+ tmp = flist_entry(entry, struct glfs_info, list);
+ if (tmp->fs == fs) {
+ glfs = tmp;
+ break;
+ }
+ }
+
+ if (!glfs) {
+ log_err("glfs not found to fini.\n");
+ } else {
+ glfs->refcount--;
+
+ if (glfs->refcount == 0) {
+ glfs_fini(glfs->fs);
+ free(glfs->volume);
+ free(glfs->brick);
+ flist_del(&glfs->list);
+ }
+ }
- int flags = 0;
- int ret = 0;
- struct gf_data *g = td->io_ops->data;
-
- dprint(FD_FILE, "fio open %s\n", f->file_name);
- if (td_write(td)) {
- if (!read_only)
- flags = O_RDWR;
- } else if (td_read(td)) {
- if (!read_only)
- flags = O_RDWR;
- else
- flags = O_RDONLY;
- }
- g->fd = glfs_creat(g->fs, f->file_name, flags, 0644);
- if (!g->fd){
- log_err("glfs_creat failed.\n");
- ret = errno;
- }
- dprint(FD_FILE, "fio %p created %s\n", g->fs, f->file_name);
- f->fd = -1;
- f->shadow_fd = -1;
-
- return ret;
+ pthread_mutex_unlock (&glfs_lock);
}
-static int fio_gf_close_file(struct thread_data *td, struct fio_file *f)
+int fio_gf_setup(struct thread_data *td)
{
- int ret = 0;
- struct gf_data *g = td->io_ops->data;
-
- dprint(FD_FILE, "fd close %s\n", f->file_name);
+ struct gf_data *g = NULL;
+ struct gf_options *opt = td->eo;
- if (g->fd && glfs_close(g->fd) < 0)
- ret = errno;
+ dprint(FD_IO, "fio setup\n");
- if (g->fs)
- glfs_fini(g->fs);
+ if (td->io_ops_data)
+ return 0;
+ g = malloc(sizeof(struct gf_data));
+ if (!g) {
+ log_err("malloc failed.\n");
+ return -ENOMEM;
+ }
g->fd = NULL;
+ g->aio_events = NULL;
+
+ g->fs = fio_gf_get_glfs(opt, opt->gf_vol, opt->gf_brick);
+ if (!g->fs)
+ goto cleanup;
+
+ dprint(FD_FILE, "fio setup %p\n", g->fs);
+ td->io_ops_data = g;
+ return 0;
+cleanup:
free(g);
- td->io_ops->data = NULL;
- f->engine_data = 0;
+ td->io_ops_data = NULL;
+ return -EIO;
+}
- return ret;
+void fio_gf_cleanup(struct thread_data *td)
+{
+ struct gf_data *g = td->io_ops_data;
+
+ if (g) {
+ if (g->aio_events)
+ free(g->aio_events);
+ if (g->fd)
+ glfs_close(g->fd);
+ if (g->fs)
+ fio_gf_put_glfs(td->eo, g->fs);
+ free(g);
+ td->io_ops_data = NULL;
+ }
}
-#define LAST_POS(f) ((f)->engine_data)
-static int fio_gf_prep(struct thread_data *td, struct io_u *io_u)
+int fio_gf_get_file_size(struct thread_data *td, struct fio_file *f)
{
- struct fio_file *f = io_u->file;
- struct gf_data *g = td->io_ops->data;
+ struct stat buf;
+ int ret;
+ struct gf_data *g = td->io_ops_data;
- dprint(FD_FILE, "fio prep\n");
+ dprint(FD_FILE, "get file size %s\n", f->file_name);
- if (!ddir_rw(io_u->ddir))
+ if (!g || !g->fs) {
return 0;
-
- if (LAST_POS(f) != -1ULL && LAST_POS(f) == io_u->offset)
+ }
+ if (fio_file_size_known(f))
return 0;
- if (glfs_lseek(g->fd, io_u->offset, SEEK_SET) < 0) {
- td_verror(td, errno, "lseek");
- return 1;
+ ret = glfs_lstat(g->fs, f->file_name, &buf);
+ if (ret < 0) {
+ log_err("glfs_lstat failed.\n");
+ return ret;
}
+ f->real_file_size = buf.st_size;
+ fio_file_set_size_known(f);
+
return 0;
+
}
-static int fio_gf_queue(struct thread_data *td, struct io_u *io_u)
+int fio_gf_open_file(struct thread_data *td, struct fio_file *f)
{
- struct gf_data *g = td->io_ops->data;
- int ret = 0;
-
- dprint(FD_FILE, "fio queue len %lu\n", io_u->xfer_buflen);
- fio_ro_check(td, io_u);
-
- if (io_u->ddir == DDIR_READ)
- ret = glfs_read(g->fd, io_u->xfer_buf, io_u->xfer_buflen, 0);
- else if (io_u->ddir == DDIR_WRITE)
- ret = glfs_write(g->fd, io_u->xfer_buf, io_u->xfer_buflen, 0);
- else {
- log_err("unsupported operation.\n");
- return -EINVAL;
- }
-
- if (io_u->file && ret >= 0 && ddir_rw(io_u->ddir))
- LAST_POS(io_u->file) = io_u->offset + ret;
-
- if (ret != (int) io_u->xfer_buflen) {
- if (ret >= 0) {
- io_u->resid = io_u->xfer_buflen - ret;
- io_u->error = 0;
- return FIO_Q_COMPLETED;
- } else
- io_u->error = errno;
- }
-
- if (io_u->error){
- log_err("IO failed.\n");
- td_verror(td, io_u->error, "xfer");
- }
-
- return FIO_Q_COMPLETED;
-}
+ int flags = 0;
+ int ret = 0;
+ struct gf_data *g = td->io_ops_data;
+ struct stat sb = { 0, };
+
+ if (td_write(td)) {
+ if (!read_only)
+ flags = O_RDWR;
+ } else if (td_read(td)) {
+ if (!read_only)
+ flags = O_RDWR;
+ else
+ flags = O_RDONLY;
+ }
-static struct ioengine_ops ioengine = {
- .name = "gfapi",
- .version = FIO_IOOPS_VERSION,
- .init = fio_gf_setup,
- .cleanup = fio_gf_cleanup,
- .prep = fio_gf_prep,
- .queue = fio_gf_queue,
- .open_file = fio_gf_open_file,
- .close_file = fio_gf_close_file,
- .get_file_size = fio_gf_get_file_size,
- .options = options,
- .option_struct_size = sizeof(struct gf_options),
- .flags = FIO_SYNCIO | FIO_DISKLESSIO,
-};
+ if (td->o.odirect)
+ flags |= OS_O_DIRECT;
+ if (td->o.sync_io)
+ flags |= O_SYNC;
+
+ dprint(FD_FILE, "fio file %s open mode %s td rw %s\n", f->file_name,
+ flags & O_RDONLY ? "ro" : "rw", td_read(td) ? "read" : "write");
+ g->fd = glfs_creat(g->fs, f->file_name, flags, 0644);
+ if (!g->fd) {
+ ret = errno;
+ log_err("glfs_creat failed.\n");
+ return ret;
+ }
+ /* file for read doesn't exist or shorter than required, create/extend it */
+ if (td_read(td)) {
+ if (glfs_lstat(g->fs, f->file_name, &sb)
+ || sb.st_size < f->real_file_size) {
+ dprint(FD_FILE, "fio extend file %s from %jd to %" PRIu64 "\n",
+ f->file_name, (intmax_t) sb.st_size, f->real_file_size);
+#if defined(CONFIG_GF_NEW_API)
+ ret = glfs_ftruncate(g->fd, f->real_file_size, NULL, NULL);
+#else
+ ret = glfs_ftruncate(g->fd, f->real_file_size);
+#endif
+ if (ret) {
+ log_err("failed fio extend file %s to %" PRIu64 "\n",
+ f->file_name, f->real_file_size);
+ } else {
+ unsigned long long left;
+ unsigned int bs;
+ char *b;
+ int r;
+
+ /* fill the file, copied from extend_file */
+ b = malloc(td->o.max_bs[DDIR_WRITE]);
+
+ left = f->real_file_size;
+ while (left && !td->terminate) {
+ bs = td->o.max_bs[DDIR_WRITE];
+ if (bs > left)
+ bs = left;
+
+ fill_io_buffer(td, b, bs, bs);
+
+ r = glfs_write(g->fd, b, bs, 0);
+ dprint(FD_IO,
+ "fio write %d of %" PRIu64 " file %s\n",
+ r, f->real_file_size,
+ f->file_name);
+
+ if (r > 0) {
+ left -= r;
+ continue;
+ } else {
+ if (r < 0) {
+ int __e = errno;
+
+ if (__e == ENOSPC) {
+ if (td->o.
+ fill_device)
+ break;
+ log_info
+ ("fio: ENOSPC on laying out "
+ "file, stopping\n");
+ break;
+ }
+ td_verror(td, errno,
+ "write");
+ } else
+ td_verror(td, EIO,
+ "write");
+
+ break;
+ }
+ }
+
+ if (b)
+ free(b);
+ glfs_lseek(g->fd, 0, SEEK_SET);
+
+ if (td->terminate && td->o.unlink) {
+ dprint(FD_FILE, "terminate unlink %s\n",
+ f->file_name);
+ glfs_unlink(g->fs, f->file_name);
+ } else if (td->o.create_fsync) {
+#if defined(CONFIG_GF_NEW_API)
+ if (glfs_fsync(g->fd, NULL, NULL) < 0) {
+#else
+ if (glfs_fsync(g->fd) < 0) {
+#endif
+ dprint(FD_FILE,
+ "failed to sync, close %s\n",
+ f->file_name);
+ td_verror(td, errno, "fsync");
+ glfs_close(g->fd);
+ g->fd = NULL;
+ return 1;
+ }
+ }
+ }
+ }
+ }
+#if defined(GFAPI_USE_FADVISE)
+ {
+ int r = 0;
+ if (td_random(td)) {
+ r = glfs_fadvise(g->fd, 0, f->real_file_size,
+ POSIX_FADV_RANDOM);
+ } else {
+ r = glfs_fadvise(g->fd, 0, f->real_file_size,
+ POSIX_FADV_SEQUENTIAL);
+ }
+ if (r) {
+ dprint(FD_FILE, "fio %p fadvise %s status %d\n", g->fs,
+ f->file_name, r);
+ }
+ }
+#endif
+ dprint(FD_FILE, "fio %p created %s\n", g->fs, f->file_name);
+ f->fd = -1;
+ f->shadow_fd = -1;
+ td->o.open_files ++;
+ return ret;
+}
-static void fio_init fio_gf_register(void)
+int fio_gf_close_file(struct thread_data *td, struct fio_file *f)
{
- register_ioengine(&ioengine);
+ int ret = 0;
+ struct gf_data *g = td->io_ops_data;
+
+ dprint(FD_FILE, "fd close %s\n", f->file_name);
+
+ if (g) {
+ if (g->fd && glfs_close(g->fd) < 0)
+ ret = errno;
+ g->fd = NULL;
+ }
+
+ return ret;
}
-static void fio_exit fio_gf_unregister(void)
+int fio_gf_unlink_file(struct thread_data *td, struct fio_file *f)
{
- unregister_ioengine(&ioengine);
+ int ret = 0;
+ struct gf_data *g = td->io_ops_data;
+
+ dprint(FD_FILE, "fd unlink %s\n", f->file_name);
+
+ if (g) {
+ if (g->fd && glfs_close(g->fd) < 0)
+ ret = errno;
+
+ glfs_unlink(g->fs, f->file_name);
+
+ if (g->fs)
+ glfs_fini(g->fs);
+
+ g->fd = NULL;
+ free(g);
+ }
+ td->io_ops_data = NULL;
+
+ return ret;
}
projects / fio.git / blobdiff