filesetup.c

   1 #include <unistd.h>
   2 #include <fcntl.h>
   3 #include <string.h>
   4 #include <assert.h>
   5 #include <dirent.h>
   6 #include <libgen.h>
   7 #include <sys/stat.h>
   8 #include <sys/mman.h>
   9 #include <sys/types.h>
  10
  11 #include "fio.h"
  12 #include "smalloc.h"
  13 #include "filehash.h"
  14 #include "options.h"
  15 #include "os/os.h"
  16 #include "hash.h"
  17 #include "lib/axmap.h"
  18
  19 #ifdef CONFIG_LINUX_FALLOCATE
  20 #include <linux/falloc.h>
  21 #endif
  22
  23 static int root_warn;
  24
  25 static FLIST_HEAD(filename_list);
  26
  27 /*
  28  * List entry for filename_list
  29  */
  30 struct file_name {
  31         struct flist_head list;
  32         char *filename;
  33 };
  34
  35 static inline void clear_error(struct thread_data *td)
  36 {
  37         td->error = 0;
  38         td->verror[0] = '\0';
  39 }
  40
  41 /*
  42  * Leaves f->fd open on success, caller must close
  43  */
  44 static int extend_file(struct thread_data *td, struct fio_file *f)
  45 {
  46         int r, new_layout = 0, unlink_file = 0, flags;
  47         unsigned long long left;
  48         unsigned int bs;
  49         char *b = NULL;
  50
  51         if (read_only) {
  52                 log_err("fio: refusing extend of file due to read-only\n");
  53                 return 0;
  54         }
  55
  56         /*
  57          * check if we need to lay the file out complete again. fio
  58          * does that for operations involving reads, or for writes
  59          * where overwrite is set
  60          */
  61         if (td_read(td) ||
  62            (td_write(td) && td->o.overwrite && !td->o.file_append) ||
  63             (td_write(td) && td_ioengine_flagged(td, FIO_NOEXTEND)))
  64                 new_layout = 1;
  65         if (td_write(td) && !td->o.overwrite && !td->o.file_append)
  66                 unlink_file = 1;
  67
  68         if (unlink_file || new_layout) {
  69                 int ret;
  70
  71                 dprint(FD_FILE, "layout unlink %s\n", f->file_name);
  72
  73                 ret = td_io_unlink_file(td, f);
  74                 if (ret != 0 && ret != ENOENT) {
  75                         td_verror(td, errno, "unlink");
  76                         return 1;
  77                 }
  78         }
  79
  80         flags = O_WRONLY;
  81         if (td->o.allow_create)
  82                 flags |= O_CREAT;
  83         if (new_layout)
  84                 flags |= O_TRUNC;
  85
  86 #ifdef WIN32
  87         flags |= _O_BINARY;
  88 #endif
  89
  90         dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags);
  91         f->fd = open(f->file_name, flags, 0644);
  92         if (f->fd < 0) {
  93                 int err = errno;
  94
  95                 if (err == ENOENT && !td->o.allow_create)
  96                         log_err("fio: file creation disallowed by "
  97                                         "allow_file_create=0\n");
  98                 else
  99                         td_verror(td, err, "open");
 100                 return 1;
 101         }
 102
 103 #ifdef CONFIG_POSIX_FALLOCATE
 104         if (!td->o.fill_device) {
 105                 switch (td->o.fallocate_mode) {
 106                 case FIO_FALLOCATE_NONE:
 107                         break;
 108                 case FIO_FALLOCATE_POSIX:
 109                         dprint(FD_FILE, "posix_fallocate file %s size %llu\n",
 110                                  f->file_name,
 111                                  (unsigned long long) f->real_file_size);
 112
 113                         r = posix_fallocate(f->fd, 0, f->real_file_size);
 114                         if (r > 0) {
 115                                 log_err("fio: posix_fallocate fails: %s\n",
 116                                                 strerror(r));
 117                         }
 118                         break;
 119 #ifdef CONFIG_LINUX_FALLOCATE
 120                 case FIO_FALLOCATE_KEEP_SIZE:
 121                         dprint(FD_FILE,
 122                                 "fallocate(FALLOC_FL_KEEP_SIZE) "
 123                                 "file %s size %llu\n", f->file_name,
 124                                 (unsigned long long) f->real_file_size);
 125
 126                         r = fallocate(f->fd, FALLOC_FL_KEEP_SIZE, 0,
 127                                         f->real_file_size);
 128                         if (r != 0)
 129                                 td_verror(td, errno, "fallocate");
 130
 131                         break;
 132 #endif /* CONFIG_LINUX_FALLOCATE */
 133                 default:
 134                         log_err("fio: unknown fallocate mode: %d\n",
 135                                 td->o.fallocate_mode);
 136                         assert(0);
 137                 }
 138         }
 139 #endif /* CONFIG_POSIX_FALLOCATE */
 140
 141         /*
 142          * If our jobs don't require regular files initially, we're done.
 143          */
 144         if (!new_layout)
 145                 goto done;
 146
 147         /*
 148          * The size will be -1ULL when fill_device is used, so don't truncate
 149          * or fallocate this file, just write it
 150          */
 151         if (!td->o.fill_device) {
 152                 dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name,
 153                                         (unsigned long long) f->real_file_size);
 154                 if (ftruncate(f->fd, f->real_file_size) == -1) {
 155                         if (errno != EFBIG) {
 156                                 td_verror(td, errno, "ftruncate");
 157                                 goto err;
 158                         }
 159                 }
 160         }
 161
 162         b = malloc(td->o.max_bs[DDIR_WRITE]);
 163
 164         left = f->real_file_size;
 165         while (left && !td->terminate) {
 166                 bs = td->o.max_bs[DDIR_WRITE];
 167                 if (bs > left)
 168                         bs = left;
 169
 170                 fill_io_buffer(td, b, bs, bs);
 171
 172                 r = write(f->fd, b, bs);
 173
 174                 if (r > 0) {
 175                         left -= r;
 176                         continue;
 177                 } else {
 178                         if (r < 0) {
 179                                 int __e = errno;
 180
 181                                 if (__e == ENOSPC) {
 182                                         if (td->o.fill_device)
 183                                                 break;
 184                                         log_info("fio: ENOSPC on laying out "
 185                                                  "file, stopping\n");
 186                                         break;
 187                                 }
 188                                 td_verror(td, errno, "write");
 189                         } else
 190                                 td_verror(td, EIO, "write");
 191
 192                         break;
 193                 }
 194         }
 195
 196         if (td->terminate) {
 197                 dprint(FD_FILE, "terminate unlink %s\n", f->file_name);
 198                 td_io_unlink_file(td, f);
 199         } else if (td->o.create_fsync) {
 200                 if (fsync(f->fd) < 0) {
 201                         td_verror(td, errno, "fsync");
 202                         goto err;
 203                 }
 204         }
 205         if (td->o.fill_device && !td_write(td)) {
 206                 fio_file_clear_size_known(f);
 207                 if (td_io_get_file_size(td, f))
 208                         goto err;
 209                 if (f->io_size > f->real_file_size)
 210                         f->io_size = f->real_file_size;
 211         }
 212
 213         free(b);
 214 done:
 215         return 0;
 216 err:
 217         close(f->fd);
 218         f->fd = -1;
 219         if (b)
 220                 free(b);
 221         return 1;
 222 }
 223
 224 static int pre_read_file(struct thread_data *td, struct fio_file *f)
 225 {
 226         int ret = 0, r, did_open = 0, old_runstate;
 227         unsigned long long left;
 228         unsigned int bs;
 229         char *b;
 230
 231         if (td_ioengine_flagged(td, FIO_PIPEIO))
 232                 return 0;
 233
 234         if (!fio_file_open(f)) {
 235                 if (td->io_ops->open_file(td, f)) {
 236                         log_err("fio: cannot pre-read, failed to open file\n");
 237                         return 1;
 238                 }
 239                 did_open = 1;
 240         }
 241
 242         old_runstate = td_bump_runstate(td, TD_PRE_READING);
 243
 244         bs = td->o.max_bs[DDIR_READ];
 245         b = malloc(bs);
 246         memset(b, 0, bs);
 247
 248         if (lseek(f->fd, f->file_offset, SEEK_SET) < 0) {
 249                 td_verror(td, errno, "lseek");
 250                 log_err("fio: failed to lseek pre-read file\n");
 251                 ret = 1;
 252                 goto error;
 253         }
 254
 255         left = f->io_size;
 256
 257         while (left && !td->terminate) {
 258                 if (bs > left)
 259                         bs = left;
 260
 261                 r = read(f->fd, b, bs);
 262
 263                 if (r == (int) bs) {
 264                         left -= bs;
 265                         continue;
 266                 } else {
 267                         td_verror(td, EIO, "pre_read");
 268                         break;
 269                 }
 270         }
 271
 272 error:
 273         td_restore_runstate(td, old_runstate);
 274
 275         if (did_open)
 276                 td->io_ops->close_file(td, f);
 277
 278         free(b);
 279         return ret;
 280 }
 281
 282 unsigned long long get_rand_file_size(struct thread_data *td)
 283 {
 284         unsigned long long ret, sized;
 285         uint64_t frand_max;
 286         unsigned long r;
 287
 288         frand_max = rand_max(&td->file_size_state);
 289         r = __rand(&td->file_size_state);
 290         sized = td->o.file_size_high - td->o.file_size_low;
 291         ret = (unsigned long long) ((double) sized * (r / (frand_max + 1.0)));
 292         ret += td->o.file_size_low;
 293         ret -= (ret % td->o.rw_min_bs);
 294         return ret;
 295 }
 296
 297 static int file_size(struct thread_data *td, struct fio_file *f)
 298 {
 299         struct stat st;
 300
 301         if (stat(f->file_name, &st) == -1) {
 302                 td_verror(td, errno, "fstat");
 303                 return 1;
 304         }
 305
 306         f->real_file_size = st.st_size;
 307         return 0;
 308 }
 309
 310 static int bdev_size(struct thread_data *td, struct fio_file *f)
 311 {
 312         unsigned long long bytes = 0;
 313         int r;
 314
 315         if (td->io_ops->open_file(td, f)) {
 316                 log_err("fio: failed opening blockdev %s for size check\n",
 317                         f->file_name);
 318                 return 1;
 319         }
 320
 321         r = blockdev_size(f, &bytes);
 322         if (r) {
 323                 td_verror(td, r, "blockdev_size");
 324                 goto err;
 325         }
 326
 327         if (!bytes) {
 328                 log_err("%s: zero sized block device?\n", f->file_name);
 329                 goto err;
 330         }
 331
 332         f->real_file_size = bytes;
 333         td->io_ops->close_file(td, f);
 334         return 0;
 335 err:
 336         td->io_ops->close_file(td, f);
 337         return 1;
 338 }
 339
 340 static int char_size(struct thread_data *td, struct fio_file *f)
 341 {
 342 #ifdef FIO_HAVE_CHARDEV_SIZE
 343         unsigned long long bytes = 0;
 344         int r;
 345
 346         if (td->io_ops->open_file(td, f)) {
 347                 log_err("fio: failed opening chardev %s for size check\n",
 348                         f->file_name);
 349                 return 1;
 350         }
 351
 352         r = chardev_size(f, &bytes);
 353         if (r) {
 354                 td_verror(td, r, "chardev_size");
 355                 goto err;
 356         }
 357
 358         if (!bytes) {
 359                 log_err("%s: zero sized char device?\n", f->file_name);
 360                 goto err;
 361         }
 362
 363         f->real_file_size = bytes;
 364         td->io_ops->close_file(td, f);
 365         return 0;
 366 err:
 367         td->io_ops->close_file(td, f);
 368         return 1;
 369 #else
 370         f->real_file_size = -1ULL;
 371         return 0;
 372 #endif
 373 }
 374
 375 static int get_file_size(struct thread_data *td, struct fio_file *f)
 376 {
 377         int ret = 0;
 378
 379         if (fio_file_size_known(f))
 380                 return 0;
 381
 382         if (f->filetype == FIO_TYPE_FILE)
 383                 ret = file_size(td, f);
 384         else if (f->filetype == FIO_TYPE_BLOCK)
 385                 ret = bdev_size(td, f);
 386         else if (f->filetype == FIO_TYPE_CHAR)
 387                 ret = char_size(td, f);
 388         else
 389                 f->real_file_size = -1ULL;
 390
 391         /*
 392          * Leave ->real_file_size with 0 since it could be expectation
 393          * of initial setup for regular files.
 394          */
 395         if (ret)
 396                 return ret;
 397
 398         /*
 399          * If ->real_file_size is -1, a conditional for the message
 400          * "offset extends end" is always true, but it makes no sense,
 401          * so just return the same value here.
 402          */
 403         if (f->real_file_size == -1ULL) {
 404                 log_info("%s: failed to get file size of %s\n", td->o.name,
 405                                         f->file_name);
 406                 return 1;
 407         }
 408
 409         if (td->o.start_offset && f->file_offset == 0)
 410                 dprint(FD_FILE, "offset of file %s not initialized yet\n",
 411                                         f->file_name);
 412         /*
 413          * ->file_offset normally hasn't been initialized yet, so this
 414          * is basically always false.
 415          */
 416         if (f->file_offset > f->real_file_size) {
 417                 log_err("%s: offset extends end (%llu > %llu)\n", td->o.name,
 418                                         (unsigned long long) f->file_offset,
 419                                         (unsigned long long) f->real_file_size);
 420                 return 1;
 421         }
 422
 423         fio_file_set_size_known(f);
 424         return 0;
 425 }
 426
 427 static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f,
 428                                    unsigned long long off,
 429                                    unsigned long long len)
 430 {
 431         int errval = 0, ret = 0;
 432
 433 #ifdef CONFIG_ESX
 434         return 0;
 435 #endif
 436
 437         if (len == -1ULL)
 438                 len = f->io_size;
 439         if (off == -1ULL)
 440                 off = f->file_offset;
 441
 442         if (len == -1ULL || off == -1ULL)
 443                 return 0;
 444
 445         dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off,
 446                                                                 len);
 447
 448         if (td->io_ops->invalidate) {
 449                 ret = td->io_ops->invalidate(td, f);
 450                 if (ret < 0)
 451                         errval = ret;
 452         } else if (f->filetype == FIO_TYPE_FILE) {
 453                 ret = posix_fadvise(f->fd, off, len, POSIX_FADV_DONTNEED);
 454                 if (ret)
 455                         errval = ret;
 456         } else if (f->filetype == FIO_TYPE_BLOCK) {
 457                 int retry_count = 0;
 458
 459                 ret = blockdev_invalidate_cache(f);
 460                 while (ret < 0 && errno == EAGAIN && retry_count++ < 25) {
 461                         /*
 462                          * Linux multipath devices reject ioctl while
 463                          * the maps are being updated. That window can
 464                          * last tens of milliseconds; we'll try up to
 465                          * a quarter of a second.
 466                          */
 467                         usleep(10000);
 468                         ret = blockdev_invalidate_cache(f);
 469                 }
 470                 if (ret < 0 && errno == EACCES && geteuid()) {
 471                         if (!root_warn) {
 472                                 log_err("fio: only root may flush block "
 473                                         "devices. Cache flush bypassed!\n");
 474                                 root_warn = 1;
 475                         }
 476                         ret = 0;
 477                 }
 478                 if (ret < 0)
 479                         errval = errno;
 480         } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE)
 481                 ret = 0;
 482
 483         /*
 484          * Cache flushing isn't a fatal condition, and we know it will
 485          * happen on some platforms where we don't have the proper
 486          * function to flush eg block device caches. So just warn and
 487          * continue on our way.
 488          */
 489         if (errval)
 490                 log_info("fio: cache invalidation of %s failed: %s\n", f->file_name, strerror(errval));
 491
 492         return 0;
 493
 494 }
 495
 496 int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
 497 {
 498         if (!fio_file_open(f))
 499                 return 0;
 500
 501         return __file_invalidate_cache(td, f, -1ULL, -1ULL);
 502 }
 503
 504 int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
 505 {
 506         int ret = 0;
 507
 508         dprint(FD_FILE, "fd close %s\n", f->file_name);
 509
 510         remove_file_hash(f);
 511
 512         if (close(f->fd) < 0)
 513                 ret = errno;
 514
 515         f->fd = -1;
 516
 517         if (f->shadow_fd != -1) {
 518                 close(f->shadow_fd);
 519                 f->shadow_fd = -1;
 520         }
 521
 522         f->engine_pos = 0;
 523         return ret;
 524 }
 525
 526 int file_lookup_open(struct fio_file *f, int flags)
 527 {
 528         struct fio_file *__f;
 529         int from_hash;
 530
 531         __f = lookup_file_hash(f->file_name);
 532         if (__f) {
 533                 dprint(FD_FILE, "found file in hash %s\n", f->file_name);
 534                 f->lock = __f->lock;
 535                 from_hash = 1;
 536         } else {
 537                 dprint(FD_FILE, "file not found in hash %s\n", f->file_name);
 538                 from_hash = 0;
 539         }
 540
 541 #ifdef WIN32
 542         flags |= _O_BINARY;
 543 #endif
 544
 545         f->fd = open(f->file_name, flags, 0600);
 546         return from_hash;
 547 }
 548
 549 static int file_close_shadow_fds(struct thread_data *td)
 550 {
 551         struct fio_file *f;
 552         int num_closed = 0;
 553         unsigned int i;
 554
 555         for_each_file(td, f, i) {
 556                 if (f->shadow_fd == -1)
 557                         continue;
 558
 559                 close(f->shadow_fd);
 560                 f->shadow_fd = -1;
 561                 num_closed++;
 562         }
 563
 564         return num_closed;
 565 }
 566
 567 int generic_open_file(struct thread_data *td, struct fio_file *f)
 568 {
 569         int is_std = 0;
 570         int flags = 0;
 571         int from_hash = 0;
 572
 573         dprint(FD_FILE, "fd open %s\n", f->file_name);
 574
 575         if (!strcmp(f->file_name, "-")) {
 576                 if (td_rw(td)) {
 577                         log_err("fio: can't read/write to stdin/out\n");
 578                         return 1;
 579                 }
 580                 is_std = 1;
 581
 582                 /*
 583                  * move output logging to stderr, if we are writing to stdout
 584                  */
 585                 if (td_write(td))
 586                         f_out = stderr;
 587         }
 588
 589         if (td_trim(td))
 590                 goto skip_flags;
 591         if (td->o.odirect)
 592                 flags |= OS_O_DIRECT;
 593         if (td->o.oatomic) {
 594                 if (!FIO_O_ATOMIC) {
 595                         td_verror(td, EINVAL, "OS does not support atomic IO");
 596                         return 1;
 597                 }
 598                 flags |= OS_O_DIRECT | FIO_O_ATOMIC;
 599         }
 600         if (td->o.sync_io)
 601                 flags |= O_SYNC;
 602         if (td->o.create_on_open && td->o.allow_create)
 603                 flags |= O_CREAT;
 604 skip_flags:
 605         if (f->filetype != FIO_TYPE_FILE)
 606                 flags |= FIO_O_NOATIME;
 607
 608 open_again:
 609         if (td_write(td)) {
 610                 if (!read_only)
 611                         flags |= O_RDWR;
 612
 613                 if (f->filetype == FIO_TYPE_FILE && td->o.allow_create)
 614                         flags |= O_CREAT;
 615
 616                 if (is_std)
 617                         f->fd = dup(STDOUT_FILENO);
 618                 else
 619                         from_hash = file_lookup_open(f, flags);
 620         } else if (td_read(td)) {
 621                 if (f->filetype == FIO_TYPE_CHAR && !read_only)
 622                         flags |= O_RDWR;
 623                 else
 624                         flags |= O_RDONLY;
 625
 626                 if (is_std)
 627                         f->fd = dup(STDIN_FILENO);
 628                 else
 629                         from_hash = file_lookup_open(f, flags);
 630         } else if (td_trim(td)) {
 631                 assert(!td_rw(td)); /* should have matched above */
 632                 flags |= O_RDWR;
 633                 from_hash = file_lookup_open(f, flags);
 634         }
 635
 636         if (f->fd == -1) {
 637                 char buf[FIO_VERROR_SIZE];
 638                 int __e = errno;
 639
 640                 if (__e == EPERM && (flags & FIO_O_NOATIME)) {
 641                         flags &= ~FIO_O_NOATIME;
 642                         goto open_again;
 643                 }
 644                 if (__e == EMFILE && file_close_shadow_fds(td))
 645                         goto open_again;
 646
 647                 snprintf(buf, sizeof(buf), "open(%s)", f->file_name);
 648
 649                 if (__e == EINVAL && (flags & OS_O_DIRECT)) {
 650                         log_err("fio: looks like your file system does not " \
 651                                 "support direct=1/buffered=0\n");
 652                 }
 653
 654                 td_verror(td, __e, buf);
 655                 return 1;
 656         }
 657
 658         if (!from_hash && f->fd != -1) {
 659                 if (add_file_hash(f)) {
 660                         int fio_unused ret;
 661
 662                         /*
 663                          * Stash away descriptor for later close. This is to
 664                          * work-around a "feature" on Linux, where a close of
 665                          * an fd that has been opened for write will trigger
 666                          * udev to call blkid to check partitions, fs id, etc.
 667                          * That pollutes the device cache, which can slow down
 668                          * unbuffered accesses.
 669                          */
 670                         if (f->shadow_fd == -1)
 671                                 f->shadow_fd = f->fd;
 672                         else {
 673                                 /*
 674                                  * OK to ignore, we haven't done anything
 675                                  * with it
 676                                  */
 677                                 ret = generic_close_file(td, f);
 678                         }
 679                         goto open_again;
 680                 }
 681         }
 682
 683         return 0;
 684 }
 685
 686 /*
 687  * This function i.e. get_file_size() is the default .get_file_size
 688  * implementation of majority of I/O engines.
 689  */
 690 int generic_get_file_size(struct thread_data *td, struct fio_file *f)
 691 {
 692         return get_file_size(td, f);
 693 }
 694
 695 /*
 696  * open/close all files, so that ->real_file_size gets set
 697  */
 698 static int get_file_sizes(struct thread_data *td)
 699 {
 700         struct fio_file *f;
 701         unsigned int i;
 702         int err = 0;
 703
 704         for_each_file(td, f, i) {
 705                 dprint(FD_FILE, "get file size for %p/%d/%s\n", f, i,
 706                                                                 f->file_name);
 707
 708                 if (td_io_get_file_size(td, f)) {
 709                         if (td->error != ENOENT) {
 710                                 log_err("%s\n", td->verror);
 711                                 err = 1;
 712                                 break;
 713                         }
 714                         clear_error(td);
 715                 }
 716
 717                 /*
 718                  * There are corner cases where we end up with -1 for
 719                  * ->real_file_size due to unsupported file type, etc.
 720                  * We then just set to size option value divided by number
 721                  * of files, similar to the way file ->io_size is set.
 722                  * stat(2) failure doesn't set ->real_file_size to -1.
 723                  */
 724                 if (f->real_file_size == -1ULL && td->o.size)
 725                         f->real_file_size = td->o.size / td->o.nr_files;
 726         }
 727
 728         return err;
 729 }
 730
 731 struct fio_mount {
 732         struct flist_head list;
 733         const char *base;
 734         char __base[256];
 735         unsigned int key;
 736 };
 737
 738 /*
 739  * Get free number of bytes for each file on each unique mount.
 740  */
 741 static unsigned long long get_fs_free_counts(struct thread_data *td)
 742 {
 743         struct flist_head *n, *tmp;
 744         unsigned long long ret = 0;
 745         struct fio_mount *fm;
 746         FLIST_HEAD(list);
 747         struct fio_file *f;
 748         unsigned int i;
 749
 750         for_each_file(td, f, i) {
 751                 struct stat sb;
 752                 char buf[256];
 753
 754                 if (f->filetype == FIO_TYPE_BLOCK || f->filetype == FIO_TYPE_CHAR) {
 755                         if (f->real_file_size != -1ULL)
 756                                 ret += f->real_file_size;
 757                         continue;
 758                 } else if (f->filetype != FIO_TYPE_FILE)
 759                         continue;
 760
 761                 buf[255] = '\0';
 762                 strncpy(buf, f->file_name, 255);
 763
 764                 if (stat(buf, &sb) < 0) {
 765                         if (errno != ENOENT)
 766                                 break;
 767                         strcpy(buf, ".");
 768                         if (stat(buf, &sb) < 0)
 769                                 break;
 770                 }
 771
 772                 fm = NULL;
 773                 flist_for_each(n, &list) {
 774                         fm = flist_entry(n, struct fio_mount, list);
 775                         if (fm->key == sb.st_dev)
 776                                 break;
 777
 778                         fm = NULL;
 779                 }
 780
 781                 if (fm)
 782                         continue;
 783
 784                 fm = calloc(1, sizeof(*fm));
 785                 strncpy(fm->__base, buf, sizeof(fm->__base) - 1);
 786                 fm->base = basename(fm->__base);
 787                 fm->key = sb.st_dev;
 788                 flist_add(&fm->list, &list);
 789         }
 790
 791         flist_for_each_safe(n, tmp, &list) {
 792                 unsigned long long sz;
 793
 794                 fm = flist_entry(n, struct fio_mount, list);
 795                 flist_del(&fm->list);
 796
 797                 sz = get_fs_free_size(fm->base);
 798                 if (sz && sz != -1ULL)
 799                         ret += sz;
 800
 801                 free(fm);
 802         }
 803
 804         return ret;
 805 }
 806
 807 uint64_t get_start_offset(struct thread_data *td, struct fio_file *f)
 808 {
 809         struct thread_options *o = &td->o;
 810
 811         if (o->file_append && f->filetype == FIO_TYPE_FILE)
 812                 return f->real_file_size;
 813
 814         return td->o.start_offset +
 815                 td->subjob_number * td->o.offset_increment;
 816 }
 817
 818 /*
 819  * Open the files and setup files sizes, creating files if necessary.
 820  */
 821 int setup_files(struct thread_data *td)
 822 {
 823         unsigned long long total_size, extend_size;
 824         struct thread_options *o = &td->o;
 825         struct fio_file *f;
 826         unsigned int i, nr_fs_extra = 0;
 827         int err = 0, need_extend;
 828         int old_state;
 829         const unsigned int bs = td_min_bs(td);
 830         uint64_t fs = 0;
 831
 832         dprint(FD_FILE, "setup files\n");
 833
 834         old_state = td_bump_runstate(td, TD_SETTING_UP);
 835
 836         if (o->read_iolog_file)
 837                 goto done;
 838
 839         /*
 840          * Find out physical size of files or devices for this thread,
 841          * before we determine I/O size and range of our targets.
 842          * If ioengine defines a setup() method, it's responsible for
 843          * opening the files and setting f->real_file_size to indicate
 844          * the valid range for that file.
 845          */
 846         if (td->io_ops->setup)
 847                 err = td->io_ops->setup(td);
 848         else
 849                 err = get_file_sizes(td);
 850
 851         if (err)
 852                 goto err_out;
 853
 854         /*
 855          * check sizes. if the files/devices do not exist and the size
 856          * isn't passed to fio, abort.
 857          */
 858         total_size = 0;
 859         for_each_file(td, f, i) {
 860                 f->fileno = i;
 861                 if (f->real_file_size == -1ULL)
 862                         total_size = -1ULL;
 863                 else
 864                         total_size += f->real_file_size;
 865         }
 866
 867         if (o->fill_device)
 868                 td->fill_device_size = get_fs_free_counts(td);
 869
 870         /*
 871          * device/file sizes are zero and no size given, punt
 872          */
 873         if ((!total_size || total_size == -1ULL) && !o->size &&
 874             !td_ioengine_flagged(td, FIO_NOIO) && !o->fill_device &&
 875             !(o->nr_files && (o->file_size_low || o->file_size_high))) {
 876                 log_err("%s: you need to specify size=\n", o->name);
 877                 td_verror(td, EINVAL, "total_file_size");
 878                 goto err_out;
 879         }
 880
 881         /*
 882          * Calculate per-file size and potential extra size for the
 883          * first files, if needed (i.e. if we don't have a fixed size).
 884          */
 885         if (!o->file_size_low && o->nr_files) {
 886                 uint64_t all_fs;
 887
 888                 fs = o->size / o->nr_files;
 889                 all_fs = fs * o->nr_files;
 890
 891                 if (all_fs < o->size)
 892                         nr_fs_extra = (o->size - all_fs) / bs;
 893         }
 894
 895         /*
 896          * now file sizes are known, so we can set ->io_size. if size= is
 897          * not given, ->io_size is just equal to ->real_file_size. if size
 898          * is given, ->io_size is size / nr_files.
 899          */
 900         extend_size = total_size = 0;
 901         need_extend = 0;
 902         for_each_file(td, f, i) {
 903                 f->file_offset = get_start_offset(td, f);
 904
 905                 /*
 906                  * Update ->io_size depending on options specified.
 907                  * ->file_size_low being 0 means filesize option isn't set.
 908                  * Non zero ->file_size_low equals ->file_size_high means
 909                  * filesize option is set in a fixed size format.
 910                  * Non zero ->file_size_low not equals ->file_size_high means
 911                  * filesize option is set in a range format.
 912                  */
 913                 if (!o->file_size_low) {
 914                         /*
 915                          * no file size or range given, file size is equal to
 916                          * total size divided by number of files. If the size
 917                          * doesn't divide nicely with the min blocksize,
 918                          * make the first files bigger.
 919                          */
 920                         f->io_size = fs;
 921                         if (nr_fs_extra) {
 922                                 nr_fs_extra--;
 923                                 f->io_size += bs;
 924                         }
 925
 926                         /*
 927                          * We normally don't come here, but if the result is 0,
 928                          * set it to the real file size. This could be size of
 929                          * the existing one if it already exists, but otherwise
 930                          * will be set to 0. A new file won't be created because
 931                          * ->io_size + ->file_offset equals ->real_file_size.
 932                          */
 933                         if (!f->io_size) {
 934                                 if (f->file_offset > f->real_file_size)
 935                                         goto err_offset;
 936                                 f->io_size = f->real_file_size - f->file_offset;
 937                                 log_info("fio: forcing file %s size to %llu\n",
 938                                         f->file_name,
 939                                         (unsigned long long)f->io_size);
 940                                 if (!f->io_size)
 941                                         log_info("fio: file %s may be ignored\n",
 942                                                 f->file_name);
 943                         }
 944                 } else if (f->real_file_size < o->file_size_low ||
 945                            f->real_file_size > o->file_size_high) {
 946                         if (f->file_offset > o->file_size_low)
 947                                 goto err_offset;
 948                         /*
 949                          * file size given. if it's fixed, use that. if it's a
 950                          * range, generate a random size in-between.
 951                          */
 952                         if (o->file_size_low == o->file_size_high)
 953                                 f->io_size = o->file_size_low - f->file_offset;
 954                         else {
 955                                 f->io_size = get_rand_file_size(td)
 956                                                 - f->file_offset;
 957                         }
 958                 } else
 959                         f->io_size = f->real_file_size - f->file_offset;
 960
 961                 if (f->io_size == -1ULL)
 962                         total_size = -1ULL;
 963                 else {
 964                         if (o->size_percent) {
 965                                 f->io_size = (f->io_size * o->size_percent) / 100;
 966                                 f->io_size -= (f->io_size % td_min_bs(td));
 967                         }
 968                         total_size += f->io_size;
 969                 }
 970
 971                 if (f->filetype == FIO_TYPE_FILE &&
 972                     (f->io_size + f->file_offset) > f->real_file_size &&
 973                     !td_ioengine_flagged(td, FIO_DISKLESSIO)) {
 974                         if (!o->create_on_open) {
 975                                 need_extend++;
 976                                 extend_size += (f->io_size + f->file_offset);
 977                                 fio_file_set_extend(f);
 978                         } else
 979                                 f->real_file_size = f->io_size + f->file_offset;
 980                 }
 981         }
 982
 983         if (td->o.block_error_hist) {
 984                 int len;
 985
 986                 assert(td->o.nr_files == 1);    /* checked in fixup_options */
 987                 f = td->files[0];
 988                 len = f->io_size / td->o.bs[DDIR_TRIM];
 989                 if (len > MAX_NR_BLOCK_INFOS || len <= 0) {
 990                         log_err("fio: cannot calculate block histogram with "
 991                                 "%d trim blocks, maximum %d\n",
 992                                 len, MAX_NR_BLOCK_INFOS);
 993                         td_verror(td, EINVAL, "block_error_hist");
 994                         goto err_out;
 995                 }
 996
 997                 td->ts.nr_block_infos = len;
 998                 for (i = 0; i < len; i++)
 999                         td->ts.block_infos[i] =
1000                                 BLOCK_INFO(0, BLOCK_STATE_UNINIT);
1001         } else
1002                 td->ts.nr_block_infos = 0;
1003
1004         if (!o->size || (total_size && o->size > total_size))
1005                 o->size = total_size;
1006
1007         if (o->size < td_min_bs(td)) {
1008                 log_err("fio: blocksize too large for data set\n");
1009                 goto err_out;
1010         }
1011
1012         /*
1013          * See if we need to extend some files, typically needed when our
1014          * target regular files don't exist yet, but our jobs require them
1015          * initially due to read I/Os.
1016          */
1017         if (need_extend) {
1018                 temp_stall_ts = 1;
1019                 if (output_format & FIO_OUTPUT_NORMAL) {
1020                         log_info("%s: Laying out IO file%s (%u file%s / %s%lluMiB)\n",
1021                                  o->name,
1022                                  need_extend > 1 ? "s" : "",
1023                                  need_extend,
1024                                  need_extend > 1 ? "s" : "",
1025                                  need_extend > 1 ? "total " : "",
1026                                  extend_size >> 20);
1027                 }
1028
1029                 for_each_file(td, f, i) {
1030                         unsigned long long old_len = -1ULL, extend_len = -1ULL;
1031
1032                         if (!fio_file_extend(f))
1033                                 continue;
1034
1035                         assert(f->filetype == FIO_TYPE_FILE);
1036                         fio_file_clear_extend(f);
1037                         if (!o->fill_device) {
1038                                 old_len = f->real_file_size;
1039                                 extend_len = f->io_size + f->file_offset -
1040                                                 old_len;
1041                         }
1042                         f->real_file_size = (f->io_size + f->file_offset);
1043                         err = extend_file(td, f);
1044                         if (err)
1045                                 break;
1046
1047                         err = __file_invalidate_cache(td, f, old_len,
1048                                                                 extend_len);
1049
1050                         /*
1051                          * Shut up static checker
1052                          */
1053                         if (f->fd != -1)
1054                                 close(f->fd);
1055
1056                         f->fd = -1;
1057                         if (err)
1058                                 break;
1059                 }
1060                 temp_stall_ts = 0;
1061         }
1062
1063         if (err)
1064                 goto err_out;
1065
1066         if (!o->zone_size)
1067                 o->zone_size = o->size;
1068
1069         /*
1070          * iolog already set the total io size, if we read back
1071          * stored entries.
1072          */
1073         if (!o->read_iolog_file) {
1074                 if (o->io_size)
1075                         td->total_io_size = o->io_size * o->loops;
1076                 else
1077                         td->total_io_size = o->size * o->loops;
1078         }
1079
1080 done:
1081         if (o->create_only)
1082                 td->done = 1;
1083
1084         td_restore_runstate(td, old_state);
1085         return 0;
1086 err_offset:
1087         log_err("%s: you need to specify valid offset=\n", o->name);
1088 err_out:
1089         td_restore_runstate(td, old_state);
1090         return 1;
1091 }
1092
1093 int pre_read_files(struct thread_data *td)
1094 {
1095         struct fio_file *f;
1096         unsigned int i;
1097
1098         dprint(FD_FILE, "pre_read files\n");
1099
1100         for_each_file(td, f, i) {
1101                 pre_read_file(td, f);
1102         }
1103
1104         return 1;
1105 }
1106
1107 static int __init_rand_distribution(struct thread_data *td, struct fio_file *f)
1108 {
1109         unsigned int range_size, seed;
1110         unsigned long nranges;
1111         uint64_t fsize;
1112
1113         range_size = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]);
1114         fsize = min(f->real_file_size, f->io_size);
1115
1116         nranges = (fsize + range_size - 1) / range_size;
1117
1118         seed = jhash(f->file_name, strlen(f->file_name), 0) * td->thread_number;
1119         if (!td->o.rand_repeatable)
1120                 seed = td->rand_seeds[4];
1121
1122         if (td->o.random_distribution == FIO_RAND_DIST_ZIPF)
1123                 zipf_init(&f->zipf, nranges, td->o.zipf_theta.u.f, seed);
1124         else if (td->o.random_distribution == FIO_RAND_DIST_PARETO)
1125                 pareto_init(&f->zipf, nranges, td->o.pareto_h.u.f, seed);
1126         else if (td->o.random_distribution == FIO_RAND_DIST_GAUSS)
1127                 gauss_init(&f->gauss, nranges, td->o.gauss_dev.u.f, seed);
1128
1129         return 1;
1130 }
1131
1132 static int init_rand_distribution(struct thread_data *td)
1133 {
1134         struct fio_file *f;
1135         unsigned int i;
1136         int state;
1137
1138         if (td->o.random_distribution == FIO_RAND_DIST_RANDOM)
1139                 return 0;
1140
1141         state = td_bump_runstate(td, TD_SETTING_UP);
1142
1143         for_each_file(td, f, i)
1144                 __init_rand_distribution(td, f);
1145
1146         td_restore_runstate(td, state);
1147
1148         return 1;
1149 }
1150
1151 /*
1152  * Check if the number of blocks exceeds the randomness capability of
1153  * the selected generator. Tausworthe is 32-bit, the others are fullly
1154  * 64-bit capable.
1155  */
1156 static int check_rand_gen_limits(struct thread_data *td, struct fio_file *f,
1157                                  uint64_t blocks)
1158 {
1159         if (blocks <= FRAND32_MAX)
1160                 return 0;
1161         if (td->o.random_generator != FIO_RAND_GEN_TAUSWORTHE)
1162                 return 0;
1163
1164         /*
1165          * If the user hasn't specified a random generator, switch
1166          * to tausworthe64 with informational warning. If the user did
1167          * specify one, just warn.
1168          */
1169         log_info("fio: file %s exceeds 32-bit tausworthe random generator.\n",
1170                         f->file_name);
1171
1172         if (!fio_option_is_set(&td->o, random_generator)) {
1173                 log_info("fio: Switching to tausworthe64. Use the "
1174                          "random_generator= option to get rid of this "
1175                          "warning.\n");
1176                 td->o.random_generator = FIO_RAND_GEN_TAUSWORTHE64;
1177                 return 0;
1178         }
1179
1180         /*
1181          * Just make this information to avoid breaking scripts.
1182          */
1183         log_info("fio: Use the random_generator= option to switch to lfsr or "
1184                          "tausworthe64.\n");
1185         return 0;
1186 }
1187
1188 int init_random_map(struct thread_data *td)
1189 {
1190         unsigned long long blocks;
1191         struct fio_file *f;
1192         unsigned int i;
1193
1194         if (init_rand_distribution(td))
1195                 return 0;
1196         if (!td_random(td))
1197                 return 0;
1198
1199         for_each_file(td, f, i) {
1200                 uint64_t fsize = min(f->real_file_size, f->io_size);
1201
1202                 blocks = fsize / (unsigned long long) td->o.rw_min_bs;
1203
1204                 if (check_rand_gen_limits(td, f, blocks))
1205                         return 1;
1206
1207                 if (td->o.random_generator == FIO_RAND_GEN_LFSR) {
1208                         unsigned long seed;
1209
1210                         seed = td->rand_seeds[FIO_RAND_BLOCK_OFF];
1211
1212                         if (!lfsr_init(&f->lfsr, blocks, seed, 0)) {
1213                                 fio_file_set_lfsr(f);
1214                                 continue;
1215                         }
1216                 } else if (!td->o.norandommap) {
1217                         f->io_axmap = axmap_new(blocks);
1218                         if (f->io_axmap) {
1219                                 fio_file_set_axmap(f);
1220                                 continue;
1221                         }
1222                 } else if (td->o.norandommap)
1223                         continue;
1224
1225                 if (!td->o.softrandommap) {
1226                         log_err("fio: failed allocating random map. If running"
1227                                 " a large number of jobs, try the 'norandommap'"
1228                                 " option or set 'softrandommap'. Or give"
1229                                 " a larger --alloc-size to fio.\n");
1230                         return 1;
1231                 }
1232
1233                 log_info("fio: file %s failed allocating random map. Running "
1234                          "job without.\n", f->file_name);
1235         }
1236
1237         return 0;
1238 }
1239
1240 void close_files(struct thread_data *td)
1241 {
1242         struct fio_file *f;
1243         unsigned int i;
1244
1245         for_each_file(td, f, i) {
1246                 if (fio_file_open(f))
1247                         td_io_close_file(td, f);
1248         }
1249 }
1250
1251 void close_and_free_files(struct thread_data *td)
1252 {
1253         struct fio_file *f;
1254         unsigned int i;
1255
1256         dprint(FD_FILE, "close files\n");
1257
1258         for_each_file(td, f, i) {
1259                 if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {
1260                         dprint(FD_FILE, "free unlink %s\n", f->file_name);
1261                         td_io_unlink_file(td, f);
1262                 }
1263
1264                 if (fio_file_open(f))
1265                         td_io_close_file(td, f);
1266
1267                 remove_file_hash(f);
1268
1269                 if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {
1270                         dprint(FD_FILE, "free unlink %s\n", f->file_name);
1271                         td_io_unlink_file(td, f);
1272                 }
1273
1274                 sfree(f->file_name);
1275                 f->file_name = NULL;
1276                 if (fio_file_axmap(f)) {
1277                         axmap_free(f->io_axmap);
1278                         f->io_axmap = NULL;
1279                 }
1280                 sfree(f);
1281         }
1282
1283         td->o.filename = NULL;
1284         free(td->files);
1285         free(td->file_locks);
1286         td->files_index = 0;
1287         td->files = NULL;
1288         td->file_locks = NULL;
1289         td->o.file_lock_mode = FILE_LOCK_NONE;
1290         td->o.nr_files = 0;
1291 }
1292
1293 static void get_file_type(struct fio_file *f)
1294 {
1295         struct stat sb;
1296
1297         if (!strcmp(f->file_name, "-"))
1298                 f->filetype = FIO_TYPE_PIPE;
1299         else
1300                 f->filetype = FIO_TYPE_FILE;
1301
1302 #ifdef WIN32
1303         /* \\.\ is the device namespace in Windows, where every file is
1304          * a block device */
1305         if (strncmp(f->file_name, "\\\\.\\", 4) == 0)
1306                 f->filetype = FIO_TYPE_BLOCK;
1307 #endif
1308
1309         if (!stat(f->file_name, &sb)) {
1310                 if (S_ISBLK(sb.st_mode))
1311                         f->filetype = FIO_TYPE_BLOCK;
1312                 else if (S_ISCHR(sb.st_mode))
1313                         f->filetype = FIO_TYPE_CHAR;
1314                 else if (S_ISFIFO(sb.st_mode))
1315                         f->filetype = FIO_TYPE_PIPE;
1316         }
1317 }
1318
1319 static bool __is_already_allocated(const char *fname, bool set)
1320 {
1321         struct flist_head *entry;
1322         bool ret;
1323
1324         ret = file_bloom_exists(fname, set);
1325         if (!ret)
1326                 return ret;
1327
1328         flist_for_each(entry, &filename_list) {
1329                 struct file_name *fn;
1330
1331                 fn = flist_entry(entry, struct file_name, list);
1332
1333                 if (!strcmp(fn->filename, fname))
1334                         return true;
1335         }
1336
1337         return false;
1338 }
1339
1340 static bool is_already_allocated(const char *fname)
1341 {
1342         bool ret;
1343
1344         fio_file_hash_lock();
1345         ret = __is_already_allocated(fname, false);
1346         fio_file_hash_unlock();
1347
1348         return ret;
1349 }
1350
1351 static void set_already_allocated(const char *fname)
1352 {
1353         struct file_name *fn;
1354
1355         fn = malloc(sizeof(struct file_name));
1356         fn->filename = strdup(fname);
1357
1358         fio_file_hash_lock();
1359         if (!__is_already_allocated(fname, true)) {
1360                 flist_add_tail(&fn->list, &filename_list);
1361                 fn = NULL;
1362         }
1363         fio_file_hash_unlock();
1364
1365         if (fn) {
1366                 free(fn->filename);
1367                 free(fn);
1368         }
1369 }
1370
1371 static void free_already_allocated(void)
1372 {
1373         struct flist_head *entry, *tmp;
1374         struct file_name *fn;
1375
1376         if (flist_empty(&filename_list))
1377                 return;
1378
1379         fio_file_hash_lock();
1380         flist_for_each_safe(entry, tmp, &filename_list) {
1381                 fn = flist_entry(entry, struct file_name, list);
1382                 free(fn->filename);
1383                 flist_del(&fn->list);
1384                 free(fn);
1385         }
1386
1387         fio_file_hash_unlock();
1388 }
1389
1390 static struct fio_file *alloc_new_file(struct thread_data *td)
1391 {
1392         struct fio_file *f;
1393
1394         f = smalloc(sizeof(*f));
1395         if (!f) {
1396                 assert(0);
1397                 return NULL;
1398         }
1399
1400         f->fd = -1;
1401         f->shadow_fd = -1;
1402         fio_file_reset(td, f);
1403         return f;
1404 }
1405
1406 bool exists_and_not_regfile(const char *filename)
1407 {
1408         struct stat sb;
1409
1410         if (lstat(filename, &sb) == -1)
1411                 return false;
1412
1413 #ifndef WIN32 /* NOT Windows */
1414         if (S_ISREG(sb.st_mode))
1415                 return false;
1416 #else
1417         /* \\.\ is the device namespace in Windows, where every file
1418          * is a device node */
1419         if (S_ISREG(sb.st_mode) && strncmp(filename, "\\\\.\\", 4) != 0)
1420                 return false;
1421 #endif
1422
1423         return true;
1424 }
1425
1426 int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
1427 {
1428         int cur_files = td->files_index;
1429         char file_name[PATH_MAX];
1430         struct fio_file *f;
1431         int len = 0;
1432
1433         dprint(FD_FILE, "add file %s\n", fname);
1434
1435         if (td->o.directory)
1436                 len = set_name_idx(file_name, PATH_MAX, td->o.directory, numjob,
1437                                         td->o.unique_filename);
1438
1439         sprintf(file_name + len, "%s", fname);
1440
1441         /* clean cloned siblings using existing files */
1442         if (numjob && is_already_allocated(file_name) &&
1443             !exists_and_not_regfile(fname))
1444                 return 0;
1445
1446         f = alloc_new_file(td);
1447
1448         if (td->files_size <= td->files_index) {
1449                 unsigned int new_size = td->o.nr_files + 1;
1450
1451                 dprint(FD_FILE, "resize file array to %d files\n", new_size);
1452
1453                 td->files = realloc(td->files, new_size * sizeof(f));
1454                 if (td->files == NULL) {
1455                         log_err("fio: realloc OOM\n");
1456                         assert(0);
1457                 }
1458                 if (td->o.file_lock_mode != FILE_LOCK_NONE) {
1459                         td->file_locks = realloc(td->file_locks, new_size);
1460                         if (!td->file_locks) {
1461                                 log_err("fio: realloc OOM\n");
1462                                 assert(0);
1463                         }
1464                         td->file_locks[cur_files] = FILE_LOCK_NONE;
1465                 }
1466                 td->files_size = new_size;
1467         }
1468         td->files[cur_files] = f;
1469         f->fileno = cur_files;
1470
1471         /*
1472          * init function, io engine may not be loaded yet
1473          */
1474         if (td->io_ops && td_ioengine_flagged(td, FIO_DISKLESSIO))
1475                 f->real_file_size = -1ULL;
1476
1477         f->file_name = smalloc_strdup(file_name);
1478         if (!f->file_name)
1479                 assert(0);
1480
1481         get_file_type(f);
1482
1483         switch (td->o.file_lock_mode) {
1484         case FILE_LOCK_NONE:
1485                 break;
1486         case FILE_LOCK_READWRITE:
1487                 f->rwlock = fio_rwlock_init();
1488                 break;
1489         case FILE_LOCK_EXCLUSIVE:
1490                 f->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
1491                 break;
1492         default:
1493                 log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode);
1494                 assert(0);
1495         }
1496
1497         td->files_index++;
1498         if (f->filetype == FIO_TYPE_FILE)
1499                 td->nr_normal_files++;
1500
1501         set_already_allocated(file_name);
1502
1503         if (inc)
1504                 td->o.nr_files++;
1505
1506         dprint(FD_FILE, "file %p \"%s\" added at %d\n", f, f->file_name,
1507                                                         cur_files);
1508
1509         return cur_files;
1510 }
1511
1512 int add_file_exclusive(struct thread_data *td, const char *fname)
1513 {
1514         struct fio_file *f;
1515         unsigned int i;
1516
1517         for_each_file(td, f, i) {
1518                 if (!strcmp(f->file_name, fname))
1519                         return i;
1520         }
1521
1522         return add_file(td, fname, 0, 1);
1523 }
1524
1525 void get_file(struct fio_file *f)
1526 {
1527         dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references);
1528         assert(fio_file_open(f));
1529         f->references++;
1530 }
1531
1532 int put_file(struct thread_data *td, struct fio_file *f)
1533 {
1534         int f_ret = 0, ret = 0;
1535
1536         dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references);
1537
1538         if (!fio_file_open(f)) {
1539                 assert(f->fd == -1);
1540                 return 0;
1541         }
1542
1543         assert(f->references);
1544         if (--f->references)
1545                 return 0;
1546
1547         if (should_fsync(td) && td->o.fsync_on_close) {
1548                 f_ret = fsync(f->fd);
1549                 if (f_ret < 0)
1550                         f_ret = errno;
1551         }
1552
1553         if (td->io_ops->close_file)
1554                 ret = td->io_ops->close_file(td, f);
1555
1556         if (!ret)
1557                 ret = f_ret;
1558
1559         td->nr_open_files--;
1560         fio_file_clear_open(f);
1561         assert(f->fd == -1);
1562         return ret;
1563 }
1564
1565 void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir)
1566 {
1567         if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)
1568                 return;
1569
1570         if (td->o.file_lock_mode == FILE_LOCK_READWRITE) {
1571                 if (ddir == DDIR_READ)
1572                         fio_rwlock_read(f->rwlock);
1573                 else
1574                         fio_rwlock_write(f->rwlock);
1575         } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
1576                 fio_mutex_down(f->lock);
1577
1578         td->file_locks[f->fileno] = td->o.file_lock_mode;
1579 }
1580
1581 void unlock_file(struct thread_data *td, struct fio_file *f)
1582 {
1583         if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)
1584                 return;
1585
1586         if (td->o.file_lock_mode == FILE_LOCK_READWRITE)
1587                 fio_rwlock_unlock(f->rwlock);
1588         else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
1589                 fio_mutex_up(f->lock);
1590
1591         td->file_locks[f->fileno] = FILE_LOCK_NONE;
1592 }
1593
1594 void unlock_file_all(struct thread_data *td, struct fio_file *f)
1595 {
1596         if (td->o.file_lock_mode == FILE_LOCK_NONE || !td->file_locks)
1597                 return;
1598         if (td->file_locks[f->fileno] != FILE_LOCK_NONE)
1599                 unlock_file(td, f);
1600 }
1601
1602 static int recurse_dir(struct thread_data *td, const char *dirname)
1603 {
1604         struct dirent *dir;
1605         int ret = 0;
1606         DIR *D;
1607
1608         D = opendir(dirname);
1609         if (!D) {
1610                 char buf[FIO_VERROR_SIZE];
1611
1612                 snprintf(buf, FIO_VERROR_SIZE, "opendir(%s)", dirname);
1613                 td_verror(td, errno, buf);
1614                 return 1;
1615         }
1616
1617         while ((dir = readdir(D)) != NULL) {
1618                 char full_path[PATH_MAX];
1619                 struct stat sb;
1620
1621                 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
1622                         continue;
1623
1624                 sprintf(full_path, "%s%s%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name);
1625
1626                 if (lstat(full_path, &sb) == -1) {
1627                         if (errno != ENOENT) {
1628                                 td_verror(td, errno, "stat");
1629                                 ret = 1;
1630                                 break;
1631                         }
1632                 }
1633
1634                 if (S_ISREG(sb.st_mode)) {
1635                         add_file(td, full_path, 0, 1);
1636                         continue;
1637                 }
1638                 if (!S_ISDIR(sb.st_mode))
1639                         continue;
1640
1641                 ret = recurse_dir(td, full_path);
1642                 if (ret)
1643                         break;
1644         }
1645
1646         closedir(D);
1647         return ret;
1648 }
1649
1650 int add_dir_files(struct thread_data *td, const char *path)
1651 {
1652         int ret = recurse_dir(td, path);
1653
1654         if (!ret)
1655                 log_info("fio: opendir added %d files\n", td->o.nr_files);
1656
1657         return ret;
1658 }
1659
1660 void dup_files(struct thread_data *td, struct thread_data *org)
1661 {
1662         struct fio_file *f;
1663         unsigned int i;
1664
1665         dprint(FD_FILE, "dup files: %d\n", org->files_index);
1666
1667         if (!org->files)
1668                 return;
1669
1670         td->files = malloc(org->files_index * sizeof(f));
1671
1672         if (td->o.file_lock_mode != FILE_LOCK_NONE)
1673                 td->file_locks = malloc(org->files_index);
1674
1675         for_each_file(org, f, i) {
1676                 struct fio_file *__f;
1677
1678                 __f = alloc_new_file(td);
1679
1680                 if (f->file_name) {
1681                         __f->file_name = smalloc_strdup(f->file_name);
1682                         if (!__f->file_name)
1683                                 assert(0);
1684
1685                         __f->filetype = f->filetype;
1686                 }
1687
1688                 if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
1689                         __f->lock = f->lock;
1690                 else if (td->o.file_lock_mode == FILE_LOCK_READWRITE)
1691                         __f->rwlock = f->rwlock;
1692
1693                 td->files[i] = __f;
1694         }
1695 }
1696
1697 /*
1698  * Returns the index that matches the filename, or -1 if not there
1699  */
1700 int get_fileno(struct thread_data *td, const char *fname)
1701 {
1702         struct fio_file *f;
1703         unsigned int i;
1704
1705         for_each_file(td, f, i)
1706                 if (!strcmp(f->file_name, fname))
1707                         return i;
1708
1709         return -1;
1710 }
1711
1712 /*
1713  * For log usage, where we add/open/close files automatically
1714  */
1715 void free_release_files(struct thread_data *td)
1716 {
1717         close_files(td);
1718         td->o.nr_files = 0;
1719         td->o.open_files = 0;
1720         td->files_index = 0;
1721         td->nr_normal_files = 0;
1722 }
1723
1724 void fio_file_reset(struct thread_data *td, struct fio_file *f)
1725 {
1726         int i;
1727
1728         for (i = 0; i < DDIR_RWDIR_CNT; i++) {
1729                 f->last_pos[i] = f->file_offset;
1730                 f->last_start[i] = -1ULL;
1731         }
1732
1733         if (fio_file_axmap(f))
1734                 axmap_reset(f->io_axmap);
1735         else if (fio_file_lfsr(f))
1736                 lfsr_reset(&f->lfsr, td->rand_seeds[FIO_RAND_BLOCK_OFF]);
1737 }
1738
1739 bool fio_files_done(struct thread_data *td)
1740 {
1741         struct fio_file *f;
1742         unsigned int i;
1743
1744         for_each_file(td, f, i)
1745                 if (!fio_file_done(f))
1746                         return false;
1747
1748         return true;
1749 }
1750
1751 /* free memory used in initialization phase only */
1752 void filesetup_mem_free(void)
1753 {
1754         free_already_allocated();
1755 }