X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=filesetup.c;h=ac2bed74d4c3e19705caefbbf76c689888adb291;hp=6ccd357ec4f5d0b9dde6cb92d2691139eeb2aa34;hb=f17c43928c202e557f2f325272c60d9050ee528e;hpb=ce98fa663bb7c157b0db45791731d4795d6208a5 diff --git a/filesetup.c b/filesetup.c index 6ccd357e..ac2bed74 100644 --- a/filesetup.c +++ b/filesetup.c @@ -8,6 +8,9 @@ #include #include "fio.h" +#include "smalloc.h" + +static int root_warn; static int extend_file(struct thread_data *td, struct fio_file *f) { @@ -16,6 +19,11 @@ static int extend_file(struct thread_data *td, struct fio_file *f) unsigned int bs; char *b; + if (read_only) { + log_err("fio: refusing extend of file due to read-only\n"); + return 0; + } + /* * check if we need to lay the file out complete again. fio * does that for operations involving reads, or for writes @@ -26,8 +34,8 @@ static int extend_file(struct thread_data *td, struct fio_file *f) if (td_write(td) && !td->o.overwrite) unlink_file = 1; - if ((unlink_file || new_layout) && (f->flags & FIO_FILE_EXISTS)) { - if (unlink(f->file_name) < 0) { + if (unlink_file || new_layout) { + if ((unlink(f->file_name) < 0) && (errno != ENOENT)) { td_verror(td, errno, "unlink"); return 1; } @@ -37,20 +45,25 @@ static int extend_file(struct thread_data *td, struct fio_file *f) if (new_layout) flags |= O_TRUNC; + dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags); f->fd = open(f->file_name, flags, 0644); if (f->fd < 0) { td_verror(td, errno, "open"); return 1; } + if (!new_layout) + goto done; + + dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name, + f->real_file_size); if (ftruncate(f->fd, f->real_file_size) == -1) { td_verror(td, errno, "ftruncate"); goto err; } - if (!new_layout) - goto done; - + dprint(FD_FILE, "fallocate file %s, size %llu\n", f->file_name, + f->real_file_size); if (posix_fallocate(f->fd, 0, f->real_file_size) < 0) { td_verror(td, errno, "posix_fallocate"); goto err; @@ -102,7 +115,7 @@ static unsigned long long get_rand_file_size(struct thread_data *td) long r; r = os_random_long(&td->file_size_state); - ret = td->o.file_size_low + (unsigned long long) ((double) td->o.file_size_high * (r / (RAND_MAX + 1.0))); + ret = td->o.file_size_low + (unsigned long long) ((double) (td->o.file_size_high - td->o.file_size_low) * (r / (RAND_MAX + 1.0))); ret -= (ret % td->o.rw_min_bs); return ret; } @@ -165,6 +178,8 @@ int file_invalidate_cache(struct thread_data *td, struct fio_file *f) { int ret = 0; + dprint(FD_IO, "invalidate cache (%d)\n", td->o.odirect); + if (td->o.odirect) return 0; @@ -178,7 +193,10 @@ int file_invalidate_cache(struct thread_data *td, struct fio_file *f) else if (f->filetype == FIO_TYPE_BD) { ret = blockdev_invalidate_cache(f->fd); if (ret < 0 && errno == EACCES && geteuid()) { - log_err("fio: only root may flush block devices. Cache flush bypassed!\n"); + if (!root_warn) { + log_err("fio: only root may flush block devices. Cache flush bypassed!\n"); + root_warn = 1; + } ret = 0; } } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE) @@ -192,10 +210,16 @@ int file_invalidate_cache(struct thread_data *td, struct fio_file *f) return ret; } -void generic_close_file(struct thread_data fio_unused *td, struct fio_file *f) +int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f) { - close(f->fd); + int ret = 0; + + dprint(FD_FILE, "fd close %s\n", f->file_name); + if (close(f->fd) < 0) + ret = errno; + f->fd = -1; + return ret; } int generic_open_file(struct thread_data *td, struct fio_file *f) @@ -203,6 +227,8 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) int is_std = 0; int flags = 0; + dprint(FD_FILE, "fd open %s\n", f->file_name); + if (!strcmp(f->file_name, "-")) { if (td_rw(td)) { log_err("fio: can't read/write to stdin/out\n"); @@ -221,8 +247,13 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) flags |= OS_O_DIRECT; if (td->o.sync_io) flags |= O_SYNC; + if (f->filetype != FIO_TYPE_FILE) + flags |= O_NOATIME; + +open_again: + if (td_write(td)) { + assert(!read_only); - if (td_write(td) || td_rw(td)) { flags |= O_RDWR; if (f->filetype == FIO_TYPE_FILE) @@ -233,7 +264,7 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) else f->fd = open(f->file_name, flags, 0600); } else { - if (f->filetype == FIO_TYPE_CHAR) + if (f->filetype == FIO_TYPE_CHAR && !read_only) flags |= O_RDWR; else flags |= O_RDONLY; @@ -248,6 +279,11 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) char buf[FIO_VERROR_SIZE]; int __e = errno; + if (errno == EPERM && (flags & O_NOATIME)) { + flags &= ~O_NOATIME; + goto open_again; + } + snprintf(buf, sizeof(buf) - 1, "open(%s)", f->file_name); td_verror(td, __e, buf); @@ -268,6 +304,8 @@ int open_files(struct thread_data *td) unsigned int i; int err = 0; + dprint(FD_FILE, "open files\n"); + for_each_file(td, f, i) { err = td_io_open_file(td, f); if (err) { @@ -331,6 +369,8 @@ int setup_files(struct thread_data *td) unsigned int i; int err = 0, need_extend; + dprint(FD_FILE, "setup files\n"); + /* * if ioengine defines a setup() method, it's responsible for * opening the files and setting f->real_file_size to indicate @@ -359,7 +399,8 @@ int setup_files(struct thread_data *td) /* * device/file sizes are zero and no size given, punt */ - if ((!total_size || total_size == -1ULL) && !td->o.size) { + if ((!total_size || total_size == -1ULL) && !td->o.size && + !(td->io_ops->flags & FIO_NOIO) && !td->o.fill_device) { log_err("%s: you need to specify size=\n", td->o.name); td_verror(td, EINVAL, "total_file_size"); return 1; @@ -373,6 +414,8 @@ int setup_files(struct thread_data *td) extend_size = total_size = 0; need_extend = 0; for_each_file(td, f, i) { + f->file_offset = td->o.start_offset; + if (!td->o.file_size_low) { /* * no file size range given, file size is equal to @@ -381,19 +424,21 @@ int setup_files(struct thread_data *td) */ f->io_size = td->o.size / td->o.nr_files; if (!f->io_size) - f->io_size = f->real_file_size; + f->io_size = f->real_file_size - f->file_offset; } else if (f->real_file_size < td->o.file_size_low || f->real_file_size > td->o.file_size_high) { + if (f->file_offset > td->o.file_size_low) + goto err_offset; /* * file size given. if it's fixed, use that. if it's a * range, generate a random size in-between. */ if (td->o.file_size_low == td->o.file_size_high) - f->io_size = td->o.file_size_low; + f->io_size = td->o.file_size_low - f->file_offset; else - f->io_size = get_rand_file_size(td); + f->io_size = get_rand_file_size(td) - f->file_offset; } else - f->io_size = f->real_file_size; + f->io_size = f->real_file_size - f->file_offset; if (f->io_size == -1ULL) total_size = -1ULL; @@ -401,15 +446,15 @@ int setup_files(struct thread_data *td) total_size += f->io_size; if (f->filetype == FIO_TYPE_FILE && - f->io_size > f->real_file_size && + (f->io_size + f->file_offset) > f->real_file_size && !(td->io_ops->flags & FIO_DISKLESSIO)) { need_extend++; - extend_size += f->io_size; + extend_size += (f->io_size + f->file_offset); f->flags |= FIO_FILE_EXTEND; - } + } } - if (!td->o.size) + if (!td->o.size || td->o.size > total_size) td->o.size = total_size; /* @@ -417,7 +462,7 @@ int setup_files(struct thread_data *td) */ if (need_extend) { temp_stall_ts = 1; - log_info("%s: Laying out IO file(s) (%u files / %LuMiB)\n", + log_info("%s: Laying out IO file(s) (%u file(s) / %LuMiB)\n", td->o.name, need_extend, extend_size >> 20); for_each_file(td, f, i) { @@ -426,7 +471,7 @@ int setup_files(struct thread_data *td) assert(f->filetype == FIO_TYPE_FILE); f->flags &= ~FIO_FILE_EXTEND; - f->real_file_size = f->io_size; + f->real_file_size = (f->io_size + f->file_offset); err = extend_file(td, f); if (err) break; @@ -440,29 +485,36 @@ int setup_files(struct thread_data *td) if (!td->o.zone_size) td->o.zone_size = td->o.size; - td->total_io_size = td->o.size * td->o.loops; + /* + * iolog already set the total io size, if we read back + * stored entries. + */ + if (!td->o.read_iolog_file) + td->total_io_size = td->o.size * td->o.loops; return 0; +err_offset: + log_err("%s: you need to specify valid offset=\n", td->o.name); + return 1; } int init_random_map(struct thread_data *td) { - int num_maps, blocks; + unsigned long long blocks, num_maps; struct fio_file *f; unsigned int i; - if (td->o.norandommap) + if (td->o.norandommap || !td_random(td)) return 0; for_each_file(td, f, i) { - blocks = (f->real_file_size + td->o.rw_min_bs - 1) / td->o.rw_min_bs; - num_maps = (blocks + BLOCKS_PER_MAP-1)/ BLOCKS_PER_MAP; - f->file_map = malloc(num_maps * sizeof(long)); + blocks = (f->real_file_size + td->o.rw_min_bs - 1) / (unsigned long long) td->o.rw_min_bs; + num_maps = (blocks + BLOCKS_PER_MAP-1)/ (unsigned long long) BLOCKS_PER_MAP; + f->file_map = smalloc(num_maps * sizeof(long)); if (!f->file_map) { log_err("fio: failed allocating random map. If running a large number of jobs, try the 'norandommap' option\n"); return 1; } f->num_maps = num_maps; - memset(f->file_map, 0, num_maps * sizeof(long)); } return 0; @@ -473,18 +525,28 @@ void close_files(struct thread_data *td) struct fio_file *f; unsigned int i; + for_each_file(td, f, i) + td_io_close_file(td, f); +} + +void close_and_free_files(struct thread_data *td) +{ + struct fio_file *f; + unsigned int i; + + dprint(FD_FILE, "close files\n"); + for_each_file(td, f, i) { - if ((f->flags & FIO_FILE_UNLINK) && - f->filetype == FIO_TYPE_FILE) + if (td->o.unlink && f->filetype == FIO_TYPE_FILE) unlink(f->file_name); td_io_close_file(td, f); - free(f->file_name); + sfree(f->file_name); f->file_name = NULL; if (f->file_map) { - free(f->file_map); + sfree(f->file_map); f->file_map = NULL; } } @@ -514,19 +576,22 @@ static void get_file_type(struct fio_file *f) } } -void add_file(struct thread_data *td, const char *fname) +int add_file(struct thread_data *td, const char *fname) { int cur_files = td->files_index; char file_name[PATH_MAX]; struct fio_file *f; int len = 0; - td->files = realloc(td->files, (cur_files + 1) * sizeof(*f)); + dprint(FD_FILE, "add file %s\n", fname); - f = &td->files[cur_files]; - memset(f, 0, sizeof(*f)); + f = smalloc(sizeof(*f)); f->fd = -1; + td->files = realloc(td->files, (cur_files + 1) * sizeof(f)); + + td->files[cur_files] = f; + /* * init function, io engine may not be loaded yet */ @@ -537,37 +602,54 @@ void add_file(struct thread_data *td, const char *fname) len = sprintf(file_name, "%s/", td->o.directory); sprintf(file_name + len, "%s", fname); - f->file_name = strdup(file_name); + f->file_name = smalloc_strdup(file_name); get_file_type(f); td->files_index++; if (f->filetype == FIO_TYPE_FILE) td->nr_normal_files++; + + return cur_files; } void get_file(struct fio_file *f) { + dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references); + assert(f->flags & FIO_FILE_OPEN); f->references++; } -void put_file(struct thread_data *td, struct fio_file *f) +int put_file(struct thread_data *td, struct fio_file *f) { + int ret = 0; + + dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references); + if (!(f->flags & FIO_FILE_OPEN)) - return; + return 0; assert(f->references); if (--f->references) - return; + return 0; if (should_fsync(td) && td->o.fsync_on_close) fsync(f->fd); if (td->io_ops->close_file) - td->io_ops->close_file(td, f); + ret = td->io_ops->close_file(td, f); td->nr_open_files--; f->flags &= ~FIO_FILE_OPEN; + return ret; +} + +void lock_file(struct thread_data *td, struct fio_file *f) +{ +} + +void unlock_file(struct fio_file *f) +{ } static int recurse_dir(struct thread_data *td, const char *dirname) @@ -636,12 +718,43 @@ void dup_files(struct thread_data *td, struct thread_data *org) if (!org->files) return; - bytes = org->files_index * sizeof(*f); + bytes = org->files_index * sizeof(f); td->files = malloc(bytes); memcpy(td->files, org->files, bytes); for_each_file(td, f, i) { + struct fio_file *__f; + + __f = smalloc(sizeof(*__f)); + if (f->file_name) - f->file_name = strdup(f->file_name); + __f->file_name = smalloc_strdup(f->file_name); + + td->files[i] = __f; } } + +/* + * Returns the index that matches the filename, or -1 if not there + */ +int get_fileno(struct thread_data *td, const char *fname) +{ + struct fio_file *f; + unsigned int i; + + for_each_file(td, f, i) + if (!strcmp(f->file_name, fname)) + return i; + + return -1; +} + +/* + * For log usage, where we add/open/close files automatically + */ +void free_release_files(struct thread_data *td) +{ + close_files(td); + td->files_index = 0; + td->nr_normal_files = 0; +}