X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=filesetup.c;h=88d6565afca65335ea88627f810862385a15da94;hp=28580188c1e98fc56aeb73d3e465c6d7d63e1cc2;hb=142c7f2d081337362afd85c1613627d8ee9ddeaf;hpb=afad68f778a764cbe57d4a5d54cbea32444aaa45 diff --git a/filesetup.c b/filesetup.c index 28580188..88d6565a 100644 --- a/filesetup.c +++ b/filesetup.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -10,9 +11,22 @@ #include "fio.h" #include "smalloc.h" #include "filehash.h" +#include "os/os.h" +#include "hash.h" +#include "lib/axmap.h" + +#ifdef CONFIG_LINUX_FALLOCATE +#include +#endif static int root_warn; +static inline void clear_error(struct thread_data *td) +{ + td->error = 0; + td->verror[0] = '\0'; +} + /* * Leaves f->fd open on success, caller must close */ @@ -58,24 +72,60 @@ static int extend_file(struct thread_data *td, struct fio_file *f) return 1; } +#ifdef CONFIG_POSIX_FALLOCATE + if (!td->o.fill_device) { + switch (td->o.fallocate_mode) { + case FIO_FALLOCATE_NONE: + break; + case FIO_FALLOCATE_POSIX: + dprint(FD_FILE, "posix_fallocate file %s size %llu\n", + f->file_name, + (unsigned long long) f->real_file_size); + + r = posix_fallocate(f->fd, 0, f->real_file_size); + if (r > 0) { + log_err("fio: posix_fallocate fails: %s\n", + strerror(r)); + } + break; +#ifdef CONFIG_LINUX_FALLOCATE + case FIO_FALLOCATE_KEEP_SIZE: + dprint(FD_FILE, + "fallocate(FALLOC_FL_KEEP_SIZE) " + "file %s size %llu\n", f->file_name, + (unsigned long long) f->real_file_size); + + r = fallocate(f->fd, FALLOC_FL_KEEP_SIZE, 0, + f->real_file_size); + if (r != 0) { + td_verror(td, errno, "fallocate"); + } + break; +#endif /* CONFIG_LINUX_FALLOCATE */ + default: + log_err("fio: unknown fallocate mode: %d\n", + td->o.fallocate_mode); + assert(0); + } + } +#endif /* CONFIG_POSIX_FALLOCATE */ + if (!new_layout) goto done; - dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name, - f->real_file_size); - if (ftruncate(f->fd, f->real_file_size) == -1) { - td_verror(td, errno, "ftruncate"); - goto err; + /* + * The size will be -1ULL when fill_device is used, so don't truncate + * or fallocate this file, just write it + */ + if (!td->o.fill_device) { + dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name, + (unsigned long long) f->real_file_size); + if (ftruncate(f->fd, f->real_file_size) == -1) { + td_verror(td, errno, "ftruncate"); + goto err; + } } -#ifdef FIO_HAVE_FALLOCATE - dprint(FD_FILE, "fallocate file %s, size %llu\n", f->file_name, - f->real_file_size); - r = posix_fallocate(f->fd, 0, f->real_file_size); - if (r < 0) - log_err("fio: posix_fallocate fails: %s\n", strerror(-r)); -#endif - b = malloc(td->o.max_bs[DDIR_WRITE]); memset(b, 0, td->o.max_bs[DDIR_WRITE]); @@ -87,13 +137,22 @@ static int extend_file(struct thread_data *td, struct fio_file *f) r = write(f->fd, b, bs); - if (r == (int) bs) { - left -= bs; + if (r > 0) { + left -= r; continue; } else { - if (r < 0) + if (r < 0) { + int __e = errno; + + if (__e == ENOSPC) { + if (td->o.fill_device) + break; + log_info("fio: ENOSPC on laying out " + "file, stopping\n"); + break; + } td_verror(td, errno, "write"); - else + } else td_verror(td, EIO, "write"); break; @@ -109,6 +168,13 @@ static int extend_file(struct thread_data *td, struct fio_file *f) goto err; } } + if (td->o.fill_device && !td_write(td)) { + fio_file_clear_size_known(f); + if (td_io_get_file_size(td, f)) + goto err; + if (f->io_size > f->real_file_size) + f->io_size = f->real_file_size; + } free(b); done: @@ -121,11 +187,25 @@ err: static int pre_read_file(struct thread_data *td, struct fio_file *f) { - int r; + int r, did_open = 0, old_runstate; unsigned long long left; unsigned int bs; char *b; + if (td->io_ops->flags & FIO_PIPEIO) + return 0; + + if (!fio_file_open(f)) { + if (td->io_ops->open_file(td, f)) { + log_err("fio: cannot pre-read, failed to open file\n"); + return 1; + } + did_open = 1; + } + + old_runstate = td->runstate; + td_set_runstate(td, TD_PRE_READING); + bs = td->o.max_bs[DDIR_READ]; b = malloc(bs); memset(b, 0, bs); @@ -148,6 +228,10 @@ static int pre_read_file(struct thread_data *td, struct fio_file *f) } } + td_set_runstate(td, old_runstate); + + if (did_open) + td->io_ops->close_file(td, f); free(b); return 0; } @@ -155,11 +239,18 @@ static int pre_read_file(struct thread_data *td, struct fio_file *f) static unsigned long long get_rand_file_size(struct thread_data *td) { unsigned long long ret, sized; - long r; + unsigned long r; + + if (td->o.use_os_rand) { + r = os_random_long(&td->file_size_state); + sized = td->o.file_size_high - td->o.file_size_low; + ret = (unsigned long long) ((double) sized * (r / (OS_RAND_MAX + 1.0))); + } else { + r = __rand(&td->__file_size_state); + sized = td->o.file_size_high - td->o.file_size_low; + ret = (unsigned long long) ((double) sized * (r / (FRAND_MAX + 1.0))); + } - r = os_random_long(&td->file_size_state); - sized = td->o.file_size_high - td->o.file_size_low; - ret = (unsigned long long) ((double) sized * (r / (OS_RAND_MAX + 1.0))); ret += td->o.file_size_low; ret -= (ret % td->o.rw_min_bs); return ret; @@ -180,7 +271,7 @@ static int file_size(struct thread_data *td, struct fio_file *f) static int bdev_size(struct thread_data *td, struct fio_file *f) { - unsigned long long bytes; + unsigned long long bytes = 0; int r; if (td->io_ops->open_file(td, f)) { @@ -189,7 +280,7 @@ static int bdev_size(struct thread_data *td, struct fio_file *f) return 1; } - r = blockdev_size(f->fd, &bytes); + r = blockdev_size(f, &bytes); if (r) { td_verror(td, r, "blockdev_size"); goto err; @@ -201,23 +292,61 @@ static int bdev_size(struct thread_data *td, struct fio_file *f) } f->real_file_size = bytes; + td->io_ops->close_file(td, f); + return 0; +err: + td->io_ops->close_file(td, f); + return 1; +} + +static int char_size(struct thread_data *td, struct fio_file *f) +{ +#ifdef FIO_HAVE_CHARDEV_SIZE + unsigned long long bytes = 0; + int r; + + if (td->io_ops->open_file(td, f)) { + log_err("fio: failed opening blockdev %s for size check\n", + f->file_name); + return 1; + } + + r = chardev_size(f, &bytes); + if (r) { + td_verror(td, r, "chardev_size"); + goto err; + } + + if (!bytes) { + log_err("%s: zero sized char device?\n", f->file_name); + goto err; + } + + f->real_file_size = bytes; + td->io_ops->close_file(td, f); return 0; err: td->io_ops->close_file(td, f); return 1; +#else + f->real_file_size = -1ULL; + return 0; +#endif } static int get_file_size(struct thread_data *td, struct fio_file *f) { int ret = 0; - if (f->flags & FIO_SIZE_KNOWN) + if (fio_file_size_known(f)) return 0; if (f->filetype == FIO_TYPE_FILE) ret = file_size(td, f); else if (f->filetype == FIO_TYPE_BD) ret = bdev_size(td, f); + else if (f->filetype == FIO_TYPE_CHAR) + ret = char_size(td, f); else f->real_file_size = -1; @@ -225,12 +354,12 @@ static int get_file_size(struct thread_data *td, struct fio_file *f) return ret; if (f->file_offset > f->real_file_size) { - log_err("%s: offset extends end (%Lu > %Lu)\n", td->o.name, + log_err("%s: offset extends end (%llu > %llu)\n", td->o.name, f->file_offset, f->real_file_size); return 1; } - f->flags |= FIO_SIZE_KNOWN; + fio_file_set_size_known(f); return 0; } @@ -245,18 +374,24 @@ static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f, if (off == -1ULL) off = f->file_offset; + if (len == -1ULL || off == -1ULL) + return 0; + dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off, len); /* * FIXME: add blockdev flushing too */ - if (f->mmap) - ret = madvise(f->mmap, len, MADV_DONTNEED); - else if (f->filetype == FIO_TYPE_FILE) { - ret = fadvise(f->fd, off, len, POSIX_FADV_DONTNEED); + if (f->mmap_ptr) { + ret = posix_madvise(f->mmap_ptr, f->mmap_sz, POSIX_MADV_DONTNEED); +#ifdef FIO_MADV_FREE + (void) posix_madvise(f->mmap_ptr, f->mmap_sz, FIO_MADV_FREE); +#endif + } else if (f->filetype == FIO_TYPE_FILE) { + ret = posix_fadvise(f->fd, off, len, POSIX_FADV_DONTNEED); } else if (f->filetype == FIO_TYPE_BD) { - ret = blockdev_invalidate_cache(f->fd); + ret = blockdev_invalidate_cache(f); if (ret < 0 && errno == EACCES && geteuid()) { if (!root_warn) { log_err("fio: only root may flush block " @@ -282,10 +417,10 @@ static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f, int file_invalidate_cache(struct thread_data *td, struct fio_file *f) { - if (!(f->flags & FIO_FILE_OPEN)) + if (!fio_file_open(f)) return 0; - return __file_invalidate_cache(td, f, -1, -1); + return __file_invalidate_cache(td, f, -1ULL, -1ULL); } int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f) @@ -300,10 +435,16 @@ int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f) ret = errno; f->fd = -1; + + if (f->shadow_fd != -1) { + close(f->shadow_fd); + f->shadow_fd = -1; + } + return ret; } -static int file_lookup_open(struct fio_file *f, int flags) +int file_lookup_open(struct fio_file *f, int flags) { struct fio_file *__f; int from_hash; @@ -315,9 +456,6 @@ static int file_lookup_open(struct fio_file *f, int flags) * racy, need the __f->lock locked */ f->lock = __f->lock; - f->lock_owner = __f->lock_owner; - f->lock_batch = __f->lock_batch; - f->lock_ddir = __f->lock_ddir; from_hash = 1; } else { dprint(FD_FILE, "file not found in hash %s\n", f->file_name); @@ -328,6 +466,24 @@ static int file_lookup_open(struct fio_file *f, int flags) return from_hash; } +static int file_close_shadow_fds(struct thread_data *td) +{ + struct fio_file *f; + int num_closed = 0; + unsigned int i; + + for_each_file(td, f, i) { + if (f->shadow_fd == -1) + continue; + + close(f->shadow_fd); + f->shadow_fd = -1; + num_closed++; + } + + return num_closed; +} + int generic_open_file(struct thread_data *td, struct fio_file *f) { int is_std = 0; @@ -336,6 +492,11 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) dprint(FD_FILE, "fd open %s\n", f->file_name); + if (td_trim(td) && f->filetype != FIO_TYPE_BD) { + log_err("fio: trim only applies to block device\n"); + return 1; + } + if (!strcmp(f->file_name, "-")) { if (td_rw(td)) { log_err("fio: can't read/write to stdin/out\n"); @@ -350,14 +511,17 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) f_out = stderr; } + if (td_trim(td)) + goto skip_flags; if (td->o.odirect) flags |= OS_O_DIRECT; if (td->o.sync_io) flags |= O_SYNC; - if (f->filetype != FIO_TYPE_FILE) - flags |= FIO_O_NOATIME; if (td->o.create_on_open) flags |= O_CREAT; +skip_flags: + if (f->filetype != FIO_TYPE_FILE) + flags |= FIO_O_NOATIME; open_again: if (td_write(td)) { @@ -371,7 +535,7 @@ open_again: f->fd = dup(STDOUT_FILENO); else from_hash = file_lookup_open(f, flags); - } else { + } else if (td_read(td)) { if (f->filetype == FIO_TYPE_CHAR && !read_only) flags |= O_RDWR; else @@ -381,30 +545,53 @@ open_again: f->fd = dup(STDIN_FILENO); else from_hash = file_lookup_open(f, flags); + } else { //td trim + flags |= O_RDWR; + from_hash = file_lookup_open(f, flags); } if (f->fd == -1) { char buf[FIO_VERROR_SIZE]; int __e = errno; - if (errno == EPERM && (flags & FIO_O_NOATIME)) { + if (__e == EPERM && (flags & FIO_O_NOATIME)) { flags &= ~FIO_O_NOATIME; goto open_again; } + if (__e == EMFILE && file_close_shadow_fds(td)) + goto open_again; - snprintf(buf, sizeof(buf) - 1, "open(%s)", f->file_name); + snprintf(buf, sizeof(buf), "open(%s)", f->file_name); + + if (__e == EINVAL && (flags & OS_O_DIRECT)) { + log_err("fio: looks like your file system does not " \ + "support direct=1/buffered=0\n"); + } td_verror(td, __e, buf); } if (!from_hash && f->fd != -1) { if (add_file_hash(f)) { - int ret; + int fio_unused ret; /* - * OK to ignore, we haven't done anything with it + * Stash away descriptor for later close. This is to + * work-around a "feature" on Linux, where a close of + * an fd that has been opened for write will trigger + * udev to call blkid to check partitions, fs id, etc. + * That polutes the device cache, which can slow down + * unbuffered accesses. */ - ret = generic_close_file(td, f); + if (f->shadow_fd == -1) + f->shadow_fd = f->fd; + else { + /* + * OK to ignore, we haven't done anything + * with it + */ + ret = generic_close_file(td, f); + } goto open_again; } } @@ -445,20 +632,102 @@ static int get_file_sizes(struct thread_data *td) return err; } +struct fio_mount { + struct flist_head list; + const char *base; + char __base[256]; + unsigned int key; +}; + +/* + * Get free number of bytes for each file on each unique mount. + */ +static unsigned long long get_fs_free_counts(struct thread_data *td) +{ + struct flist_head *n, *tmp; + unsigned long long ret = 0; + struct fio_mount *fm; + FLIST_HEAD(list); + struct fio_file *f; + unsigned int i; + + for_each_file(td, f, i) { + struct stat sb; + char buf[256]; + + if (f->filetype == FIO_TYPE_BD || f->filetype == FIO_TYPE_CHAR) { + if (f->real_file_size != -1ULL) + ret += f->real_file_size; + continue; + } else if (f->filetype != FIO_TYPE_FILE) + continue; + + strcpy(buf, f->file_name); + + if (stat(buf, &sb) < 0) { + if (errno != ENOENT) + break; + strcpy(buf, "."); + if (stat(buf, &sb) < 0) + break; + } + + fm = NULL; + flist_for_each(n, &list) { + fm = flist_entry(n, struct fio_mount, list); + if (fm->key == sb.st_dev) + break; + + fm = NULL; + } + + if (fm) + continue; + + fm = malloc(sizeof(*fm)); + strcpy(fm->__base, buf); + fm->base = basename(fm->__base); + fm->key = sb.st_dev; + flist_add(&fm->list, &list); + } + + flist_for_each_safe(n, tmp, &list) { + unsigned long long sz; + + fm = flist_entry(n, struct fio_mount, list); + flist_del(&fm->list); + + sz = get_fs_size(fm->base); + if (sz && sz != -1ULL) + ret += sz; + + free(fm); + } + + return ret; +} + +uint64_t get_start_offset(struct thread_data *td) +{ + return td->o.start_offset + + (td->thread_number - 1) * td->o.offset_increment; +} + /* * Open the files and setup files sizes, creating files if necessary. */ int setup_files(struct thread_data *td) { unsigned long long total_size, extend_size; + struct thread_options *o = &td->o; struct fio_file *f; unsigned int i; int err = 0, need_extend; dprint(FD_FILE, "setup files\n"); - if (td->o.read_iolog_file) - return 0; + if (o->read_iolog_file) + goto done; /* * if ioengine defines a setup() method, it's responsible for @@ -485,12 +754,16 @@ int setup_files(struct thread_data *td) total_size += f->real_file_size; } + if (o->fill_device) + td->fill_device_size = get_fs_free_counts(td); + /* * device/file sizes are zero and no size given, punt */ - if ((!total_size || total_size == -1ULL) && !td->o.size && - !(td->io_ops->flags & FIO_NOIO) && !td->o.fill_device) { - log_err("%s: you need to specify size=\n", td->o.name); + if ((!total_size || total_size == -1ULL) && !o->size && + !(td->io_ops->flags & FIO_NOIO) && !o->fill_device && + !(o->nr_files && (o->file_size_low || o->file_size_high))) { + log_err("%s: you need to specify size=\n", o->name); td_verror(td, EINVAL, "total_file_size"); return 1; } @@ -503,29 +776,28 @@ int setup_files(struct thread_data *td) extend_size = total_size = 0; need_extend = 0; for_each_file(td, f, i) { - f->file_offset = td->o.start_offset; + f->file_offset = get_start_offset(td); - if (!td->o.file_size_low) { + if (!o->file_size_low) { /* * no file size range given, file size is equal to * total size divided by number of files. if that is * zero, set it to the real file size. */ - f->io_size = td->o.size / td->o.nr_files; + f->io_size = o->size / o->nr_files; if (!f->io_size) f->io_size = f->real_file_size - f->file_offset; - } else if (f->real_file_size < td->o.file_size_low || - f->real_file_size > td->o.file_size_high) { - if (f->file_offset > td->o.file_size_low) + } else if (f->real_file_size < o->file_size_low || + f->real_file_size > o->file_size_high) { + if (f->file_offset > o->file_size_low) goto err_offset; /* * file size given. if it's fixed, use that. if it's a * range, generate a random size in-between. */ - if (td->o.file_size_low == td->o.file_size_high) { - f->io_size = td->o.file_size_low - - f->file_offset; - } else { + if (o->file_size_low == o->file_size_high) + f->io_size = o->file_size_low - f->file_offset; + else { f->io_size = get_rand_file_size(td) - f->file_offset; } @@ -534,49 +806,55 @@ int setup_files(struct thread_data *td) if (f->io_size == -1ULL) total_size = -1ULL; - else + else { + if (o->size_percent) + f->io_size = (f->io_size * o->size_percent) / 100; total_size += f->io_size; + } if (f->filetype == FIO_TYPE_FILE && (f->io_size + f->file_offset) > f->real_file_size && !(td->io_ops->flags & FIO_DISKLESSIO)) { - if (!td->o.create_on_open) { + if (!o->create_on_open) { need_extend++; extend_size += (f->io_size + f->file_offset); } else f->real_file_size = f->io_size + f->file_offset; - f->flags |= FIO_FILE_EXTEND; + fio_file_set_extend(f); } } - if (!td->o.size || td->o.size > total_size) - td->o.size = total_size; + if (!o->size || o->size > total_size) + o->size = total_size; /* * See if we need to extend some files */ if (need_extend) { temp_stall_ts = 1; - if (!terse_output) + if (output_format == FIO_OUTPUT_NORMAL) log_info("%s: Laying out IO file(s) (%u file(s) /" - " %LuMiB)\n", td->o.name, need_extend, + " %lluMB)\n", o->name, need_extend, extend_size >> 20); for_each_file(td, f, i) { - unsigned long long old_len, extend_len; + unsigned long long old_len = -1ULL, extend_len = -1ULL; - if (!(f->flags & FIO_FILE_EXTEND)) + if (!fio_file_extend(f)) continue; assert(f->filetype == FIO_TYPE_FILE); - f->flags &= ~FIO_FILE_EXTEND; - old_len = f->real_file_size; - extend_len = f->io_size + f->file_offset - old_len; + fio_file_clear_extend(f); + if (!o->fill_device) { + old_len = f->real_file_size; + extend_len = f->io_size + f->file_offset - + old_len; + } f->real_file_size = (f->io_size + f->file_offset); err = extend_file(td, f); if (err) break; - + err = __file_invalidate_cache(td, f, old_len, extend_len); close(f->fd); @@ -590,18 +868,23 @@ int setup_files(struct thread_data *td) if (err) return err; - if (!td->o.zone_size) - td->o.zone_size = td->o.size; + if (!o->zone_size) + o->zone_size = o->size; /* * iolog already set the total io size, if we read back * stored entries. */ - if (!td->o.read_iolog_file) - td->total_io_size = td->o.size * td->o.loops; + if (!o->read_iolog_file) + td->total_io_size = o->size * o->loops; + +done: + if (o->create_only) + td->done = 1; + return 0; err_offset: - log_err("%s: you need to specify valid offset=\n", td->o.name); + log_err("%s: you need to specify valid offset=\n", o->name); return 1; } @@ -619,25 +902,77 @@ int pre_read_files(struct thread_data *td) return 1; } +static int __init_rand_distribution(struct thread_data *td, struct fio_file *f) +{ + unsigned int range_size, seed; + unsigned long nranges; + uint64_t file_size; + + range_size = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]); + file_size = min(f->real_file_size, f->io_size); + + nranges = (file_size + range_size - 1) / range_size; + + seed = jhash(f->file_name, strlen(f->file_name), 0) * td->thread_number; + if (!td->o.rand_repeatable) + seed = td->rand_seeds[4]; + + if (td->o.random_distribution == FIO_RAND_DIST_ZIPF) + zipf_init(&f->zipf, nranges, td->o.zipf_theta, seed); + else + pareto_init(&f->zipf, nranges, td->o.pareto_h, seed); + + return 1; +} + +static int init_rand_distribution(struct thread_data *td) +{ + struct fio_file *f; + unsigned int i; + int state; + + if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) + return 0; + + state = td->runstate; + td_set_runstate(td, TD_SETTING_UP); + for_each_file(td, f, i) + __init_rand_distribution(td, f); + td_set_runstate(td, state); + + return 1; +} + int init_random_map(struct thread_data *td) { - unsigned long long blocks, num_maps; + unsigned long long blocks; struct fio_file *f; unsigned int i; - if (td->o.norandommap || !td_random(td)) + if (init_rand_distribution(td)) + return 0; + if (!td_random(td)) return 0; for_each_file(td, f, i) { - blocks = (f->real_file_size + td->o.rw_min_bs - 1) / - (unsigned long long) td->o.rw_min_bs; - num_maps = (blocks + BLOCKS_PER_MAP - 1) / - (unsigned long long) BLOCKS_PER_MAP; - f->file_map = smalloc(num_maps * sizeof(int)); - if (f->file_map) { - f->num_maps = num_maps; + uint64_t file_size = min(f->real_file_size, f->io_size); + + blocks = file_size / (unsigned long long) td->o.rw_min_bs; + + if (td->o.random_generator == FIO_RAND_GEN_LFSR) { + unsigned long seed; + + seed = td->rand_seeds[FIO_RAND_BLOCK_OFF]; + + if (!lfsr_init(&f->lfsr, blocks, seed, seed & 0xF)) + continue; + } else if (!td->o.norandommap) { + f->io_axmap = axmap_new(blocks); + if (f->io_axmap) + continue; + } else if (td->o.norandommap) continue; - } + if (!td->o.softrandommap) { log_err("fio: failed allocating random map. If running" " a large number of jobs, try the 'norandommap'" @@ -648,7 +983,6 @@ int init_random_map(struct thread_data *td) log_info("fio: file %s failed allocating random map. Running " "job without.\n", f->file_name); - f->num_maps = 0; } return 0; @@ -659,8 +993,10 @@ void close_files(struct thread_data *td) struct fio_file *f; unsigned int i; - for_each_file(td, f, i) - td_io_close_file(td, f); + for_each_file(td, f, i) { + if (fio_file_open(f)) + td_io_close_file(td, f); + } } void close_and_free_files(struct thread_data *td) @@ -676,23 +1012,24 @@ void close_and_free_files(struct thread_data *td) unlink(f->file_name); } - td_io_close_file(td, f); + if (fio_file_open(f)) + td_io_close_file(td, f); + remove_file_hash(f); sfree(f->file_name); f->file_name = NULL; - - if (f->file_map) { - sfree(f->file_map); - f->file_map = NULL; - } + axmap_free(f->io_axmap); + f->io_axmap = NULL; sfree(f); } td->o.filename = NULL; free(td->files); + free(td->file_locks); td->files_index = 0; td->files = NULL; + td->file_locks = NULL; td->o.nr_files = 0; } @@ -705,7 +1042,12 @@ static void get_file_type(struct fio_file *f) else f->filetype = FIO_TYPE_FILE; - if (!lstat(f->file_name, &sb)) { + /* \\.\ is the device namespace in Windows, where every file is + * a block device */ + if (strncmp(f->file_name, "\\\\.\\", 4) == 0) + f->filetype = FIO_TYPE_BD; + + if (!stat(f->file_name, &sb)) { if (S_ISBLK(sb.st_mode)) f->filetype = FIO_TYPE_BD; else if (S_ISCHR(sb.st_mode)) @@ -729,18 +1071,33 @@ int add_file(struct thread_data *td, const char *fname) log_err("fio: smalloc OOM\n"); assert(0); } - + f->fd = -1; + f->shadow_fd = -1; + fio_file_reset(td, f); if (td->files_size <= td->files_index) { - int new_size = td->o.nr_files + 1; + unsigned int new_size = td->o.nr_files + 1; dprint(FD_FILE, "resize file array to %d files\n", new_size); td->files = realloc(td->files, new_size * sizeof(f)); + if (td->files == NULL) { + log_err("fio: realloc OOM\n"); + assert(0); + } + if (td->o.file_lock_mode != FILE_LOCK_NONE) { + td->file_locks = realloc(td->file_locks, new_size); + if (!td->file_locks) { + log_err("fio: realloc OOM\n"); + assert(0); + } + td->file_locks[cur_files] = FILE_LOCK_NONE; + } td->files_size = new_size; } td->files[cur_files] = f; + f->fileno = cur_files; /* * init function, io engine may not be loaded yet @@ -757,17 +1114,17 @@ int add_file(struct thread_data *td, const char *fname) log_err("fio: smalloc OOM\n"); assert(0); } - + get_file_type(f); switch (td->o.file_lock_mode) { case FILE_LOCK_NONE: break; case FILE_LOCK_READWRITE: - f->lock = fio_mutex_rw_init(); + f->rwlock = fio_rwlock_init(); break; case FILE_LOCK_EXCLUSIVE: - f->lock = fio_mutex_init(1); + f->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); break; default: log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode); @@ -784,10 +1141,23 @@ int add_file(struct thread_data *td, const char *fname) return cur_files; } +int add_file_exclusive(struct thread_data *td, const char *fname) +{ + struct fio_file *f; + unsigned int i; + + for_each_file(td, f, i) { + if (!strcmp(f->file_name, fname)) + return i; + } + + return add_file(td, fname); +} + void get_file(struct fio_file *f) { dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references); - assert(f->flags & FIO_FILE_OPEN); + assert(fio_file_open(f)); f->references++; } @@ -797,8 +1167,10 @@ int put_file(struct thread_data *td, struct fio_file *f) dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references); - if (!(f->flags & FIO_FILE_OPEN)) + if (!fio_file_open(f)) { + assert(f->fd == -1); return 0; + } assert(f->references); if (--f->references) @@ -814,7 +1186,8 @@ int put_file(struct thread_data *td, struct fio_file *f) ret = f_ret; td->nr_open_files--; - f->flags &= ~FIO_FILE_OPEN; + fio_file_clear_open(f); + assert(f->fd == -1); return ret; } @@ -823,57 +1196,34 @@ void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir) if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE) return; - if (f->lock_owner == td && f->lock_batch--) - return; - if (td->o.file_lock_mode == FILE_LOCK_READWRITE) { if (ddir == DDIR_READ) - fio_mutex_down_read(f->lock); + fio_rwlock_read(f->rwlock); else - fio_mutex_down_write(f->lock); + fio_rwlock_write(f->rwlock); } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) fio_mutex_down(f->lock); - f->lock_owner = td; - f->lock_batch = td->o.lockfile_batch; - f->lock_ddir = ddir; + td->file_locks[f->fileno] = td->o.file_lock_mode; } void unlock_file(struct thread_data *td, struct fio_file *f) { if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE) return; - if (f->lock_batch) - return; - - if (td->o.file_lock_mode == FILE_LOCK_READWRITE) { - const int is_read = f->lock_ddir == DDIR_READ; - int val = fio_mutex_getval(f->lock); - - if ((is_read && val == 1) || (!is_read && val == -1)) - f->lock_owner = NULL; - - if (is_read) - fio_mutex_up_read(f->lock); - else - fio_mutex_up_write(f->lock); - } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) { - int val = fio_mutex_getval(f->lock); - - if (val == 0) - f->lock_owner = NULL; + if (td->o.file_lock_mode == FILE_LOCK_READWRITE) + fio_rwlock_unlock(f->rwlock); + else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) fio_mutex_up(f->lock); - } + + td->file_locks[f->fileno] = FILE_LOCK_NONE; } void unlock_file_all(struct thread_data *td, struct fio_file *f) { - if (f->lock_owner != td) - return; - - f->lock_batch = 0; - unlock_file(td, f); + if (td->file_locks[f->fileno] != FILE_LOCK_NONE) + unlock_file(td, f); } static int recurse_dir(struct thread_data *td, const char *dirname) @@ -886,7 +1236,7 @@ static int recurse_dir(struct thread_data *td, const char *dirname) if (!D) { char buf[FIO_VERROR_SIZE]; - snprintf(buf, FIO_VERROR_SIZE - 1, "opendir(%s)", dirname); + snprintf(buf, FIO_VERROR_SIZE, "opendir(%s)", dirname); td_verror(td, errno, buf); return 1; } @@ -898,7 +1248,7 @@ static int recurse_dir(struct thread_data *td, const char *dirname) if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) continue; - sprintf(full_path, "%s/%s", dirname, dir->d_name); + sprintf(full_path, "%s%s%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name); if (lstat(full_path, &sb) == -1) { if (errno != ENOENT) { @@ -946,6 +1296,9 @@ void dup_files(struct thread_data *td, struct thread_data *org) td->files = malloc(org->files_index * sizeof(f)); + if (td->o.file_lock_mode != FILE_LOCK_NONE) + td->file_locks = malloc(org->files_index); + for_each_file(org, f, i) { struct fio_file *__f; @@ -954,14 +1307,16 @@ void dup_files(struct thread_data *td, struct thread_data *org) log_err("fio: smalloc OOM\n"); assert(0); } - + __f->fd = -1; + fio_file_reset(td, __f); + if (f->file_name) { __f->file_name = smalloc_strdup(f->file_name); if (!__f->file_name) { log_err("fio: smalloc OOM\n"); assert(0); } - + __f->filetype = f->filetype; } @@ -993,3 +1348,13 @@ void free_release_files(struct thread_data *td) td->files_index = 0; td->nr_normal_files = 0; } + +void fio_file_reset(struct thread_data *td, struct fio_file *f) +{ + f->last_pos = f->file_offset; + f->last_start = -1ULL; + if (f->io_axmap) + axmap_reset(f->io_axmap); + if (td->o.random_generator == FIO_RAND_GEN_LFSR) + lfsr_reset(&f->lfsr, td->rand_seeds[FIO_RAND_BLOCK_OFF]); +}