engines/glusterfs.c

   1 /*
   2  * glusterfs engine
   3  *
   4  * common Glusterfs's gfapi interface
   5  *
   6  */
   7
   8 #include "gfapi.h"
   9 #include "../optgroup.h"
  10
  11 struct fio_option gfapi_options[] = {
  12         {
  13          .name = "volume",
  14          .lname = "Glusterfs volume",
  15          .type = FIO_OPT_STR_STORE,
  16          .help = "Name of the Glusterfs volume",
  17          .off1 = offsetof(struct gf_options, gf_vol),
  18          .category = FIO_OPT_C_ENGINE,
  19          .group = FIO_OPT_G_GFAPI,
  20          },
  21         {
  22          .name = "brick",
  23          .lname = "Glusterfs brick name",
  24          .type = FIO_OPT_STR_STORE,
  25          .help = "Name of the Glusterfs brick to connect",
  26          .off1 = offsetof(struct gf_options, gf_brick),
  27          .category = FIO_OPT_C_ENGINE,
  28          .group = FIO_OPT_G_GFAPI,
  29          },
  30         {
  31          .name = "single-instance",
  32          .lname = "Single glusterfs instance",
  33          .type = FIO_OPT_BOOL,
  34          .help = "Only one glusterfs instance",
  35          .off1 = offsetof(struct gf_options, gf_single_instance),
  36          .category = FIO_OPT_C_ENGINE,
  37          .group = FIO_OPT_G_GFAPI,
  38          },
  39         {
  40          .name = NULL,
  41          },
  42 };
  43
  44 struct glfs_info {
  45         struct flist_head       list;
  46         char                    *volume;
  47         char                    *brick;
  48         glfs_t                  *fs;
  49         int                     refcount;
  50 };
  51
  52 static pthread_mutex_t glfs_lock = PTHREAD_MUTEX_INITIALIZER;
  53 static FLIST_HEAD(glfs_list_head);
  54
  55 static glfs_t *fio_gf_new_fs(char *volume, char *brick)
  56 {
  57         int r = 0;
  58         glfs_t *fs;
  59         struct stat sb = { 0, };
  60
  61         fs = glfs_new(volume);
  62         if (!fs) {
  63                 log_err("glfs_new failed.\n");
  64                 goto out;
  65         }
  66         glfs_set_logging(fs, "/tmp/fio_gfapi.log", 7);
  67         /* default to tcp */
  68         r = glfs_set_volfile_server(fs, "tcp", brick, 0);
  69         if (r) {
  70                 log_err("glfs_set_volfile_server failed.\n");
  71                 goto out;
  72         }
  73         r = glfs_init(fs);
  74         if (r) {
  75                 log_err("glfs_init failed. Is glusterd running on brick?\n");
  76                 goto out;
  77         }
  78         sleep(2);
  79         r = glfs_lstat(fs, ".", &sb);
  80         if (r) {
  81                 log_err("glfs_lstat failed.\n");
  82                 goto out;
  83         }
  84
  85 out:
  86         if (r) {
  87                 glfs_fini(fs);
  88                 fs = NULL;
  89         }
  90         return fs;
  91 }
  92
  93 static glfs_t *fio_gf_get_glfs(struct gf_options *opt,
  94                                char *volume, char *brick)
  95 {
  96         struct glfs_info *glfs = NULL;
  97         struct glfs_info *tmp;
  98         struct flist_head *entry;
  99
 100         if (!opt->gf_single_instance)
 101                 return fio_gf_new_fs(volume, brick);
 102
 103         pthread_mutex_lock (&glfs_lock);
 104
 105         flist_for_each(entry, &glfs_list_head) {
 106                 tmp = flist_entry(entry, struct glfs_info, list);
 107                 if (!strcmp(volume, tmp->volume) &&
 108                     !strcmp(brick, tmp->brick)) {
 109                         glfs = tmp;
 110                         break;
 111                 }
 112         }
 113
 114         if (glfs) {
 115                 glfs->refcount++;
 116         } else {
 117                 glfs = malloc(sizeof(*glfs));
 118                 if (!glfs)
 119                         goto out;
 120                 INIT_FLIST_HEAD(&glfs->list);
 121                 glfs->refcount = 0;
 122                 glfs->volume = strdup(volume);
 123                 glfs->brick = strdup(brick);
 124                 glfs->fs = fio_gf_new_fs(volume, brick);
 125                 if (!glfs->fs) {
 126                         free(glfs);
 127                         glfs = NULL;
 128                         goto out;
 129                 }
 130
 131                 flist_add_tail(&glfs->list, &glfs_list_head);
 132                 glfs->refcount = 1;
 133         }
 134
 135 out:
 136         pthread_mutex_unlock (&glfs_lock);
 137
 138         if (glfs)
 139                 return glfs->fs;
 140         return NULL;
 141 }
 142
 143 static void fio_gf_put_glfs(struct gf_options *opt, glfs_t *fs)
 144 {
 145         struct glfs_info *glfs = NULL;
 146         struct glfs_info *tmp;
 147         struct flist_head *entry;
 148
 149         if (!opt->gf_single_instance) {
 150                 glfs_fini(fs);
 151                 return;
 152         }
 153
 154         pthread_mutex_lock (&glfs_lock);
 155
 156         flist_for_each(entry, &glfs_list_head) {
 157                 tmp = flist_entry(entry, struct glfs_info, list);
 158                 if (tmp->fs == fs) {
 159                         glfs = tmp;
 160                         break;
 161                 }
 162         }
 163
 164         if (!glfs) {
 165                 log_err("glfs not found to fini.\n");
 166         } else {
 167                 glfs->refcount--;
 168
 169                 if (glfs->refcount == 0) {
 170                         glfs_fini(glfs->fs);
 171                         free(glfs->volume);
 172                         free(glfs->brick);
 173                         flist_del(&glfs->list);
 174                 }
 175         }
 176
 177         pthread_mutex_unlock (&glfs_lock);
 178 }
 179
 180 int fio_gf_setup(struct thread_data *td)
 181 {
 182         struct gf_data *g = NULL;
 183         struct gf_options *opt = td->eo;
 184
 185         dprint(FD_IO, "fio setup\n");
 186
 187         if (td->io_ops_data)
 188                 return 0;
 189
 190         g = malloc(sizeof(struct gf_data));
 191         if (!g) {
 192                 log_err("malloc failed.\n");
 193                 return -ENOMEM;
 194         }
 195         g->fd = NULL;
 196         g->aio_events = NULL;
 197
 198         g->fs = fio_gf_get_glfs(opt, opt->gf_vol, opt->gf_brick);
 199         if (!g->fs)
 200                 goto cleanup;
 201
 202         dprint(FD_FILE, "fio setup %p\n", g->fs);
 203         td->io_ops_data = g;
 204         return 0;
 205 cleanup:
 206         free(g);
 207         td->io_ops_data = NULL;
 208         return -EIO;
 209 }
 210
 211 void fio_gf_cleanup(struct thread_data *td)
 212 {
 213         struct gf_data *g = td->io_ops_data;
 214
 215         if (g) {
 216                 if (g->aio_events)
 217                         free(g->aio_events);
 218                 if (g->fd)
 219                         glfs_close(g->fd);
 220                 if (g->fs)
 221                         fio_gf_put_glfs(td->eo, g->fs);
 222                 free(g);
 223                 td->io_ops_data = NULL;
 224         }
 225 }
 226
 227 int fio_gf_get_file_size(struct thread_data *td, struct fio_file *f)
 228 {
 229         struct stat buf;
 230         int ret;
 231         struct gf_data *g = td->io_ops_data;
 232
 233         dprint(FD_FILE, "get file size %s\n", f->file_name);
 234
 235         if (!g || !g->fs) {
 236                 return 0;
 237         }
 238         if (fio_file_size_known(f))
 239                 return 0;
 240
 241         ret = glfs_lstat(g->fs, f->file_name, &buf);
 242         if (ret < 0) {
 243                 log_err("glfs_lstat failed.\n");
 244                 return ret;
 245         }
 246
 247         f->real_file_size = buf.st_size;
 248         fio_file_set_size_known(f);
 249
 250         return 0;
 251
 252 }
 253
 254 int fio_gf_open_file(struct thread_data *td, struct fio_file *f)
 255 {
 256
 257         int flags = 0;
 258         int ret = 0;
 259         struct gf_data *g = td->io_ops_data;
 260         struct stat sb = { 0, };
 261
 262         if (td_write(td)) {
 263                 if (!read_only)
 264                         flags = O_RDWR;
 265         } else if (td_read(td)) {
 266                 if (!read_only)
 267                         flags = O_RDWR;
 268                 else
 269                         flags = O_RDONLY;
 270         }
 271
 272         if (td->o.odirect)
 273                 flags |= OS_O_DIRECT;
 274         if (td->o.sync_io)
 275                 flags |= O_SYNC;
 276
 277         dprint(FD_FILE, "fio file %s open mode %s td rw %s\n", f->file_name,
 278                flags & O_RDONLY ? "ro" : "rw", td_read(td) ? "read" : "write");
 279         g->fd = glfs_creat(g->fs, f->file_name, flags, 0644);
 280         if (!g->fd) {
 281                 ret = errno;
 282                 log_err("glfs_creat failed.\n");
 283                 return ret;
 284         }
 285         /* file for read doesn't exist or shorter than required, create/extend it */
 286         if (td_read(td)) {
 287                 if (glfs_lstat(g->fs, f->file_name, &sb)
 288                     || sb.st_size < f->real_file_size) {
 289                         dprint(FD_FILE, "fio extend file %s from %jd to %" PRIu64 "\n",
 290                                f->file_name, (intmax_t) sb.st_size, f->real_file_size);
 291                         ret = glfs_ftruncate(g->fd, f->real_file_size);
 292                         if (ret) {
 293                                 log_err("failed fio extend file %s to %" PRIu64 "\n",
 294                                         f->file_name, f->real_file_size);
 295                         } else {
 296                                 unsigned long long left;
 297                                 unsigned int bs;
 298                                 char *b;
 299                                 int r;
 300
 301                                 /* fill the file, copied from extend_file */
 302                                 b = malloc(td->o.max_bs[DDIR_WRITE]);
 303
 304                                 left = f->real_file_size;
 305                                 while (left && !td->terminate) {
 306                                         bs = td->o.max_bs[DDIR_WRITE];
 307                                         if (bs > left)
 308                                                 bs = left;
 309
 310                                         fill_io_buffer(td, b, bs, bs);
 311
 312                                         r = glfs_write(g->fd, b, bs, 0);
 313                                         dprint(FD_IO,
 314                                                "fio write %d of %" PRIu64 " file %s\n",
 315                                                r, f->real_file_size,
 316                                                f->file_name);
 317
 318                                         if (r > 0) {
 319                                                 left -= r;
 320                                                 continue;
 321                                         } else {
 322                                                 if (r < 0) {
 323                                                         int __e = errno;
 324
 325                                                         if (__e == ENOSPC) {
 326                                                                 if (td->o.
 327                                                                     fill_device)
 328                                                                         break;
 329                                                                 log_info
 330                                                                     ("fio: ENOSPC on laying out "
 331                                                                      "file, stopping\n");
 332                                                                 break;
 333                                                         }
 334                                                         td_verror(td, errno,
 335                                                                   "write");
 336                                                 } else
 337                                                         td_verror(td, EIO,
 338                                                                   "write");
 339
 340                                                 break;
 341                                         }
 342                                 }
 343
 344                                 if (b)
 345                                         free(b);
 346                                 glfs_lseek(g->fd, 0, SEEK_SET);
 347
 348                                 if (td->terminate && td->o.unlink) {
 349                                         dprint(FD_FILE, "terminate unlink %s\n",
 350                                                f->file_name);
 351                                         glfs_unlink(g->fs, f->file_name);
 352                                 } else if (td->o.create_fsync) {
 353                                         if (glfs_fsync(g->fd) < 0) {
 354                                                 dprint(FD_FILE,
 355                                                        "failed to sync, close %s\n",
 356                                                        f->file_name);
 357                                                 td_verror(td, errno, "fsync");
 358                                                 glfs_close(g->fd);
 359                                                 g->fd = NULL;
 360                                                 return 1;
 361                                         }
 362                                 }
 363                         }
 364                 }
 365         }
 366 #if defined(GFAPI_USE_FADVISE)
 367         {
 368                 int r = 0;
 369                 if (td_random(td)) {
 370                         r = glfs_fadvise(g->fd, 0, f->real_file_size,
 371                                          POSIX_FADV_RANDOM);
 372                 } else {
 373                         r = glfs_fadvise(g->fd, 0, f->real_file_size,
 374                                          POSIX_FADV_SEQUENTIAL);
 375                 }
 376                 if (r) {
 377                         dprint(FD_FILE, "fio %p fadvise %s status %d\n", g->fs,
 378                                f->file_name, r);
 379                 }
 380         }
 381 #endif
 382         dprint(FD_FILE, "fio %p created %s\n", g->fs, f->file_name);
 383         f->fd = -1;
 384         f->shadow_fd = -1;
 385         td->o.open_files ++;
 386         return ret;
 387 }
 388
 389 int fio_gf_close_file(struct thread_data *td, struct fio_file *f)
 390 {
 391         int ret = 0;
 392         struct gf_data *g = td->io_ops_data;
 393
 394         dprint(FD_FILE, "fd close %s\n", f->file_name);
 395
 396         if (g) {
 397                 if (g->fd && glfs_close(g->fd) < 0)
 398                         ret = errno;
 399                 g->fd = NULL;
 400         }
 401
 402         return ret;
 403 }
 404
 405 int fio_gf_unlink_file(struct thread_data *td, struct fio_file *f)
 406 {
 407         int ret = 0;
 408         struct gf_data *g = td->io_ops_data;
 409
 410         dprint(FD_FILE, "fd unlink %s\n", f->file_name);
 411
 412         if (g) {
 413                 if (g->fd && glfs_close(g->fd) < 0)
 414                         ret = errno;
 415
 416                 glfs_unlink(g->fs, f->file_name);
 417
 418                 if (g->fs)
 419                         glfs_fini(g->fs);
 420
 421                 g->fd = NULL;
 422                 free(g);
 423         }
 424         td->io_ops_data = NULL;
 425
 426         return ret;
 427 }