X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=filesetup.c;h=abcdc1d05da17051073124457d7be62ab461e124;hp=948cecda7fb56e70de64c0c00ef9723e7564be74;hb=e407bec29fa8252f7809b2e9f88b95e619cd468b;hpb=af52b3455ad892322aab2791282b6bd4efdfdbf3 diff --git a/filesetup.c b/filesetup.c index 948cecda..abcdc1d0 100644 --- a/filesetup.c +++ b/filesetup.c @@ -2,8 +2,10 @@ #include #include #include +#include #include #include +#include #include "fio.h" #include "os.h" @@ -99,10 +101,28 @@ static int create_files(struct thread_data *td) { struct fio_file *f; int err, need_create, can_extend; - unsigned int i; + unsigned long long total_file_size; + unsigned int i, new_files; - for_each_file(td, f, i) - f->file_size = td->total_file_size / td->nr_files; + new_files = 0; + total_file_size = td->total_file_size; + for_each_file(td, f, i) { + unsigned long long s; + + f->file_offset = td->start_offset; + + if (f->filetype != FIO_TYPE_FILE) + continue; + + if (f->flags & FIO_FILE_EXISTS) { + s = f->file_size; + if (s > total_file_size) + s = total_file_size; + + total_file_size -= s; + } else + new_files++; + } /* * unless specifically asked for overwrite, let normal io extend it @@ -117,6 +137,10 @@ static int create_files(struct thread_data *td) if (f->filetype != FIO_TYPE_FILE) continue; + if (f->flags & FIO_FILE_EXISTS) + continue; + + f->file_size = total_file_size / new_files; file_there = !file_ok(td, f); @@ -139,18 +163,20 @@ static int create_files(struct thread_data *td) temp_stall_ts = 1; fprintf(f_out, "%s: Laying out IO file(s) (%u x %LuMiB == %LuMiB)\n", - td->name, td->nr_uniq_files, - (td->total_file_size >> 20) / td->nr_uniq_files, - td->total_file_size >> 20); + td->name, new_files, + (total_file_size >> 20) / new_files, + total_file_size >> 20); err = 0; for_each_file(td, f, i) { /* * Only unlink files that we created. */ - f->unlink = 0; + f->flags &= ~FIO_FILE_UNLINK; if (file_ok(td, f)) { - f->unlink = td->unlink; + if (td->unlink) + f->flags |= FIO_FILE_UNLINK; + err = create_file(td, f); if (err) break; @@ -208,9 +234,10 @@ static int get_file_size(struct thread_data *td, struct fio_file *f) { int ret = 0; - if (f->filetype == FIO_TYPE_FILE) - ret = file_size(td, f); - else if (f->filetype == FIO_TYPE_BD) + if (f->filetype == FIO_TYPE_FILE) { + if (!(f->flags & FIO_FILE_EXISTS)) + ret = file_size(td, f); + } else if (f->filetype == FIO_TYPE_BD) ret = bdev_size(td, f); else f->real_file_size = -1; @@ -290,6 +317,8 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) td_verror(td, __e, "open"); if (__e == EINVAL && td->odirect) log_err("fio: destination does not support O_DIRECT\n"); + if (__e == EMFILE) + log_err("fio: try reducing/setting openfiles (failed at %u of %u)\n", td->nr_open_files, td->nr_files); return 1; } @@ -368,8 +397,6 @@ int setup_files(struct thread_data *td) for_each_file(td, f, i) td->total_file_size += f->file_size; - td->total_file_size = (td->total_file_size * td->nr_files) / td->open_files; - td->io_size = td->total_file_size; if (td->io_size == 0) { log_err("%s: no io blocks\n", td->name); @@ -394,9 +421,10 @@ void close_files(struct thread_data *td) unsigned int i; for_each_file(td, f, i) { - if (!td->filename && f->unlink && + if (!f->file_name && (f->flags & FIO_FILE_UNLINK) && f->filetype == FIO_TYPE_FILE) { unlink(f->file_name); + free(f->file_name); f->file_name = NULL; } @@ -412,23 +440,32 @@ void close_files(struct thread_data *td) td->nr_files = 0; } -static void get_file_type(struct thread_data *td, struct fio_file *f) +static void get_file_type(struct fio_file *f) { struct stat sb; f->filetype = FIO_TYPE_FILE; - if (!lstat(td->filename, &sb)) { + if (!lstat(f->file_name, &sb)) { + f->flags |= FIO_FILE_EXISTS; + if (S_ISBLK(sb.st_mode)) f->filetype = FIO_TYPE_BD; else if (S_ISCHR(sb.st_mode)) f->filetype = FIO_TYPE_CHAR; + else { + /* + * might as well do this here, and save a stat later on + */ + f->real_file_size = sb.st_size; + f->file_size = f->real_file_size; + } } } void add_file(struct thread_data *td, const char *fname) { - int cur_files = td->open_files; + int cur_files = td->files_index; struct fio_file *f; td->files = realloc(td->files, (cur_files + 1) * sizeof(*f)); @@ -436,10 +473,81 @@ void add_file(struct thread_data *td, const char *fname) f = &td->files[cur_files]; memset(f, 0, sizeof(*f)); f->fd = -1; - f->file_name = fname; + f->file_name = strdup(fname); + + get_file_type(f); + + td->files_index++; + if (f->filetype == FIO_TYPE_FILE) + td->nr_normal_files++; +} + +void get_file(struct fio_file *f) +{ + f->references++; +} + +void put_file(struct thread_data *td, struct fio_file *f) +{ + if (!(f->flags & FIO_FILE_OPEN)) + return; - get_file_type(td, f); + assert(f->references); + if (--f->references) + return; - td->open_files++; - td->nr_uniq_files = td->open_files; + if (should_fsync(td) && td->fsync_on_close) + fsync(f->fd); + + if (td->io_ops->close_file) + td->io_ops->close_file(td, f); + td->nr_open_files--; + f->flags &= ~FIO_FILE_OPEN; +} + +static int recurse_dir(struct thread_data *td, const char *dirname) +{ + struct dirent *dir; + int ret = 0; + DIR *D; + + D = opendir(dirname); + if (!D) { + td_verror(td, errno, "opendir"); + return 1; + } + + while ((dir = readdir(D)) != NULL) { + char full_path[PATH_MAX]; + struct stat sb; + + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) + continue; + + sprintf(full_path, "%s/%s", dirname, dir->d_name); + + if (lstat(full_path, &sb) == -1) { + if (errno != ENOENT) { + td_verror(td, errno, "stat"); + return 1; + } + } + + if (S_ISREG(sb.st_mode)) { + add_file(td, full_path); + td->nr_files++; + continue; + } + + if ((ret = recurse_dir(td, full_path)) != 0) + break; + } + + closedir(D); + return ret; +} + +int add_dir_files(struct thread_data *td, const char *path) +{ + return recurse_dir(td, path); }