X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=filesetup.c;h=faef907aeefd44429896aa156f2fa4944593cac4;hp=a5b84c215cf740a50611ca20e16dfb0eedf1a49e;hb=6d86144dd10b05e8b82e9b895c35dd778e5e71ab;hpb=e3bab463d4397e3378d8c7437e31de8aaf861a4f diff --git a/filesetup.c b/filesetup.c index a5b84c21..faef907a 100644 --- a/filesetup.c +++ b/filesetup.c @@ -2,8 +2,10 @@ #include #include #include +#include #include #include +#include #include "fio.h" #include "os.h" @@ -95,14 +97,54 @@ err: return 1; } +static unsigned long long set_rand_file_size(struct thread_data *td, + unsigned long long total_size) +{ + unsigned long long upper = total_size; + unsigned long long ret; + long r; + + if (upper > td->file_size_high) + upper = td->file_size_high; + else if (upper < td->file_size_low) + return 0; + else if (!upper) + return 0; + + r = os_random_long(&td->file_size_state); + ret = td->file_size_low + (unsigned long long) ((double) upper * (r / (RAND_MAX + 1.0))); + ret -= (ret % td->rw_min_bs); + if (ret > upper) + ret = upper; + return ret; +} + static int create_files(struct thread_data *td) { struct fio_file *f; int err, need_create, can_extend; - unsigned int i; + unsigned long long total_file_size, local_file_size; + unsigned int i, new_files; - for_each_file(td, f, i) - f->file_size = td->total_file_size / td->nr_files; + new_files = 0; + total_file_size = td->total_file_size; + for_each_file(td, f, i) { + unsigned long long s; + + f->file_offset = td->start_offset; + + if (f->filetype != FIO_TYPE_FILE) + continue; + + if (f->flags & FIO_FILE_EXISTS) { + s = f->file_size; + if (s > total_file_size) + s = total_file_size; + + total_file_size -= s; + } else + new_files++; + } /* * unless specifically asked for overwrite, let normal io extend it @@ -112,11 +154,35 @@ static int create_files(struct thread_data *td) return 0; need_create = 0; + local_file_size = total_file_size; + if (!local_file_size) + local_file_size = -1; + for_each_file(td, f, i) { int file_there; if (f->filetype != FIO_TYPE_FILE) continue; + if (f->flags & FIO_FILE_EXISTS) + continue; + + if (!td->file_size_low) + f->file_size = total_file_size / new_files; + else { + /* + * If we don't have enough space left for a file + * of the minimum size, bail. + */ + if (local_file_size < td->file_size_low) { + log_info("fio: limited to %d files\n", i); + new_files -= (td->nr_files - i); + td->nr_files = i; + break; + } + + f->file_size = set_rand_file_size(td, local_file_size); + local_file_size -= f->file_size; + } file_there = !file_ok(td, f); @@ -138,10 +204,8 @@ static int create_files(struct thread_data *td) } temp_stall_ts = 1; - fprintf(f_out, "%s: Laying out IO file(s) (%u x %LuMiB == %LuMiB)\n", - td->name, td->nr_uniq_files, - (td->total_file_size >> 20) / td->nr_uniq_files, - td->total_file_size >> 20); + log_info("%s: Laying out IO file(s) (%u files / %LuMiB)\n", + td->name, new_files, total_file_size >> 20); err = 0; for_each_file(td, f, i) { @@ -210,9 +274,10 @@ static int get_file_size(struct thread_data *td, struct fio_file *f) { int ret = 0; - if (f->filetype == FIO_TYPE_FILE) - ret = file_size(td, f); - else if (f->filetype == FIO_TYPE_BD) + if (f->filetype == FIO_TYPE_FILE) { + if (!(f->flags & FIO_FILE_EXISTS)) + ret = file_size(td, f); + } else if (f->filetype == FIO_TYPE_BD) ret = bdev_size(td, f); else f->real_file_size = -1; @@ -292,6 +357,8 @@ int generic_open_file(struct thread_data *td, struct fio_file *f) td_verror(td, __e, "open"); if (__e == EINVAL && td->odirect) log_err("fio: destination does not support O_DIRECT\n"); + if (__e == EMFILE) + log_err("fio: try reducing/setting openfiles (failed at %u of %u)\n", td->nr_open_files, td->nr_files); return 1; } @@ -370,8 +437,6 @@ int setup_files(struct thread_data *td) for_each_file(td, f, i) td->total_file_size += f->file_size; - td->total_file_size = (td->total_file_size * td->nr_files) / td->open_files; - td->io_size = td->total_file_size; if (td->io_size == 0) { log_err("%s: no io blocks\n", td->name); @@ -396,9 +461,10 @@ void close_files(struct thread_data *td) unsigned int i; for_each_file(td, f, i) { - if (!td->filename && (f->flags & FIO_FILE_UNLINK) && + if (!f->file_name && (f->flags & FIO_FILE_UNLINK) && f->filetype == FIO_TYPE_FILE) { unlink(f->file_name); + free(f->file_name); f->file_name = NULL; } @@ -409,7 +475,6 @@ void close_files(struct thread_data *td) } td->filename = NULL; - free(td->files); td->files = NULL; td->nr_files = 0; } @@ -421,16 +486,25 @@ static void get_file_type(struct fio_file *f) f->filetype = FIO_TYPE_FILE; if (!lstat(f->file_name, &sb)) { + f->flags |= FIO_FILE_EXISTS; + if (S_ISBLK(sb.st_mode)) f->filetype = FIO_TYPE_BD; else if (S_ISCHR(sb.st_mode)) f->filetype = FIO_TYPE_CHAR; + else { + /* + * might as well do this here, and save a stat later on + */ + f->real_file_size = sb.st_size; + f->file_size = f->real_file_size; + } } } void add_file(struct thread_data *td, const char *fname) { - int cur_files = td->open_files; + int cur_files = td->files_index; struct fio_file *f; td->files = realloc(td->files, (cur_files + 1) * sizeof(*f)); @@ -442,8 +516,9 @@ void add_file(struct thread_data *td, const char *fname) get_file_type(f); - td->open_files++; - td->nr_uniq_files = td->open_files; + td->files_index++; + if (f->filetype == FIO_TYPE_FILE) + td->nr_normal_files++; } void get_file(struct fio_file *f) @@ -460,8 +535,58 @@ void put_file(struct thread_data *td, struct fio_file *f) if (--f->references) return; + if (should_fsync(td) && td->fsync_on_close) + fsync(f->fd); + if (td->io_ops->close_file) td->io_ops->close_file(td, f); td->nr_open_files--; f->flags &= ~FIO_FILE_OPEN; } + +static int recurse_dir(struct thread_data *td, const char *dirname) +{ + struct dirent *dir; + int ret = 0; + DIR *D; + + D = opendir(dirname); + if (!D) { + td_verror(td, errno, "opendir"); + return 1; + } + + while ((dir = readdir(D)) != NULL) { + char full_path[PATH_MAX]; + struct stat sb; + + if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) + continue; + + sprintf(full_path, "%s/%s", dirname, dir->d_name); + + if (lstat(full_path, &sb) == -1) { + if (errno != ENOENT) { + td_verror(td, errno, "stat"); + return 1; + } + } + + if (S_ISREG(sb.st_mode)) { + add_file(td, full_path); + td->nr_files++; + continue; + } + + if ((ret = recurse_dir(td, full_path)) != 0) + break; + } + + closedir(D); + return ret; +} + +int add_dir_files(struct thread_data *td, const char *path) +{ + return recurse_dir(td, path); +}