X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=init.c;h=e794b3720c3dc1813c7018bd86e586bfa8dc4697;hp=62080f74949f09a9abbdb1ce2fa40ee2b9a7731d;hb=9c60ce649d3f976d196709d4399bb7c540ce97b5;hpb=80be24f4671164d597463e4b5c794d4cbd0a4537

diff --git a/init.c b/init.c
index 62080f74..e794b372 100644
--- a/init.c
+++ b/init.c
@@ -30,6 +30,10 @@ static int str_prioclass_cb(void *, unsigned int *);
 #endif
 static int str_exitall_cb(void);
 static int str_cpumask_cb(void *, unsigned int *);
+static int str_fst_cb(void *, const char *);
+static int str_filename_cb(void *, const char *);
+static int str_directory_cb(void *, const char *);
+static int str_opendir_cb(void *, const char *);
 
 #define __stringify_1(x)	#x
 #define __stringify(x)		__stringify_1(x)
@@ -54,13 +58,22 @@ static struct fio_option options[] = {
 		.name	= "directory",
 		.type	= FIO_OPT_STR_STORE,
 		.off1	= td_var_offset(directory),
+		.cb	= str_directory_cb,
 		.help	= "Directory to store files in",
 	},
 	{
 		.name	= "filename",
 		.type	= FIO_OPT_STR_STORE,
 		.off1	= td_var_offset(filename),
-		.help	= "Force the use of a specific file",
+		.cb	= str_filename_cb,
+		.help	= "File(s) to use for the workload",
+	},
+	{
+		.name	= "opendir",
+		.type	= FIO_OPT_STR_STORE,
+		.off1	= td_var_offset(opendir),
+		.cb	= str_opendir_cb,
+		.help	= "Recursively add files from this directory and down",
 	},
 	{
 		.name	= "rw",
@@ -69,13 +82,31 @@ static struct fio_option options[] = {
 		.help	= "IO direction",
 		.def	= "read",
 		.posval = {
-			  { .ival = "read", .oval = TD_DDIR_READ },
-			  { .ival = "write", .oval = TD_DDIR_WRITE },
-			  { .ival = "randread", .oval = TD_DDIR_RANDREAD },
-			  { .ival = "randwrite", .oval = TD_DDIR_RANDWRITE },
-			  { .ival = "rw", .oval = TD_DDIR_RW },
-			  { .ival = "randrw", .oval = TD_DDIR_RANDRW },
+			  { .ival = "read",
+			    .oval = TD_DDIR_READ,
+			    .help = "Sequential read",
+			  },
+			  { .ival = "write",
+			    .oval = TD_DDIR_WRITE,
+			    .help = "Sequential write",
+			  },
+			  { .ival = "randread",
+			    .oval = TD_DDIR_RANDREAD,
+			    .help = "Random read",
+			  },
+			  { .ival = "randwrite",
+			    .oval = TD_DDIR_RANDWRITE,
+			    .help = "Random write",
+			  },
+			  { .ival = "rw",
+			    .oval = TD_DDIR_RW,
+			    .help = "Sequential read and write mix",
 			  },
+			  { .ival = "randrw",
+			    .oval = TD_DDIR_RANDRW,
+			    .help = "Random read and write mix"
+			  },
+		},
 	},
 	{
 		.name	= "ioengine",
@@ -84,27 +115,50 @@ static struct fio_option options[] = {
 		.help	= "IO engine to use",
 		.def	= "sync",
 		.posval	= {
-			  { .ival = "sync", },
+			  { .ival = "sync",
+			    .help = "Use read/write",
+			  },
 #ifdef FIO_HAVE_LIBAIO
-			  { .ival = "libaio", },
+			  { .ival = "libaio",
+			    .help = "Linux native asynchronous IO",
+			  },
 #endif
 #ifdef FIO_HAVE_POSIXAIO
-			  { .ival = "posixaio", },
+			  { .ival = "posixaio",
+			    .help = "POSIX asynchronous IO",
+			  },
 #endif
-			  { .ival = "mmap", },
+			  { .ival = "mmap",
+			    .help = "Memory mapped IO",
+			  },
 #ifdef FIO_HAVE_SPLICE
-			  { .ival = "splice", },
+			  { .ival = "splice",
+			    .help = "splice/vmsplice based IO",
+			  },
 #endif
 #ifdef FIO_HAVE_SGIO
-			  { .ival = "sg", },
+			  { .ival = "sg",
+			    .help = "SCSI generic v3 IO",
+			  },
 #endif
-			  { .ival = "null", }, { .ival = "net", },
+			  { .ival = "null",
+			    .help = "Testing engine (no data transfer)",
+			  },
+			  { .ival = "net",
+			    .help = "Network IO",
+			  },
 #ifdef FIO_HAVE_SYSLET
-			  { .ival = "syslet-rw", },
+			  { .ival = "syslet-rw",
+			    .help = "syslet enabled async pread/pwrite IO",
+			  },
 #endif
-			  { .ival = "cpuio", },
-			  { .ival = "external", },
+			  { .ival = "cpuio",
+			    .help = "CPU cycler burner engine",
+			  },
+			  { .ival = "external",
+			    .help = "Load external engine (append name)",
 			  },
+		},
 	},
 	{
 		.name	= "iodepth",
@@ -129,7 +183,14 @@ static struct fio_option options[] = {
 		.name	= "size",
 		.type	= FIO_OPT_STR_VAL,
 		.off1	= td_var_offset(total_file_size),
-		.help	= "Size of device or file",
+		.help	= "Total size of device or files",
+	},
+	{
+		.name	= "filesize",
+		.type	= FIO_OPT_STR_VAL,
+		.off1	= td_var_offset(file_size_low),
+		.off2	= td_var_offset(file_size_high),
+		.help	= "Size of individual files",
 	},
 	{
 		.name	= "bs",
@@ -190,13 +251,20 @@ static struct fio_option options[] = {
 	{
 		.name	= "file_service_type",
 		.type	= FIO_OPT_STR,
+		.cb	= str_fst_cb,
 		.off1	= td_var_offset(file_service_type),
 		.help	= "How to select which file to service next",
 		.def	= "roundrobin",
 		.posval	= {
-			  { .ival = "random", .oval = FIO_FSERVICE_RANDOM },
-			  { .ival = "roundrobin", .oval = FIO_FSERVICE_RR },
+			  { .ival = "random",
+			    .oval = FIO_FSERVICE_RANDOM,
+			    .help = "Choose a file at random",
 			  },
+			  { .ival = "roundrobin",
+			    .oval = FIO_FSERVICE_RR,
+			    .help = "Round robin select files",
+			  },
+		},
 	},
 	{
 		.name	= "fsync",
@@ -264,28 +332,52 @@ static struct fio_option options[] = {
 		.help	= "Backing type for IO buffers",
 		.def	= "malloc",
 		.posval	= {
-			  { .ival = "malloc", .oval = MEM_MALLOC },
-			  { .ival = "shm", .oval = MEM_SHM },
+			  { .ival = "malloc",
+			    .oval = MEM_MALLOC,
+			    .help = "Use malloc(3) for IO buffers",
+			  },
+			  { .ival = "shm",
+			    .oval = MEM_SHM,
+			    .help = "Use shared memory segments for IO buffers",
+			  },
 #ifdef FIO_HAVE_HUGETLB
-			  { .ival = "shmhuge", .oval = MEM_SHMHUGE },
+			  { .ival = "shmhuge",
+			    .oval = MEM_SHMHUGE,
+			    .help = "Like shm, but use huge pages",
+			  },
 #endif
-			  { .ival = "mmap", .oval = MEM_MMAP },
+			  { .ival = "mmap",
+			    .oval = MEM_MMAP,
+			    .help = "Use mmap(2) (file or anon) for IO buffers",
+			  },
 #ifdef FIO_HAVE_HUGETLB
-			  { .ival = "mmaphuge", .oval = MEM_MMAPHUGE },
-#endif
+			  { .ival = "mmaphuge",
+			    .oval = MEM_MMAPHUGE,
+			    .help = "Like mmap, but use huge pages",
 			  },
+#endif
+		  },
 	},
 	{
 		.name	= "verify",
 		.type	= FIO_OPT_STR,
 		.off1	= td_var_offset(verify),
-		.help	= "Verify sum function",
+		.help	= "Verify data written",
 		.def	= "0",
 		.posval = {
-			  { .ival = "0", .oval = VERIFY_NONE },
-			  { .ival = "crc32", .oval = VERIFY_CRC32 },
-			  { .ival = "md5", .oval = VERIFY_MD5 },
+			  { .ival = "0",
+			    .oval = VERIFY_NONE,
+			    .help = "Don't do IO verification",
 			  },
+			  { .ival = "crc32",
+			    .oval = VERIFY_CRC32,
+			    .help = "Use crc32 checksums for verification",
+			  },
+			  { .ival = "md5",
+			    .oval = VERIFY_MD5,
+			    .help = "Use md5 checksums for verification",
+			  },
+		},
 	},
 	{
 		.name	= "write_iolog",
@@ -493,6 +585,13 @@ static struct fio_option options[] = {
 		.help	= "Include fsync at the end of job",
 		.def	= "0",
 	},
+	{
+		.name	= "fsync_on_close",
+		.type	= FIO_OPT_BOOL,
+		.off1	= td_var_offset(fsync_on_close),
+		.help	= "fsync files on close",
+		.def	= "0",
+	},
 	{
 		.name	= "unlink",
 		.type	= FIO_OPT_BOOL,
@@ -604,7 +703,7 @@ static struct option long_options[FIO_JOB_OPTS + FIO_CMD_OPTS] = {
 
 static int def_timeout = 0;
 
-static char fio_version_string[] = "fio 1.13";
+static char fio_version_string[] = "fio 1.14a";
 
 static char **ini_file;
 static int max_jobs = MAX_JOBS;
@@ -621,6 +720,8 @@ FILE *f_err = NULL;
 static int write_lat_log = 0;
 int write_bw_log = 0;
 
+static int prev_group_jobs;
+
 FILE *get_f_out()
 {
 	return f_out;
@@ -707,6 +808,9 @@ static void fixup_options(struct thread_data *td)
 
 	td->rw_min_bs = min(td->min_bs[DDIR_READ], td->min_bs[DDIR_WRITE]);
 
+	if (!td->file_size_high)
+		td->file_size_high = td->file_size_low;
+
 	if (td_read(td) && !td_rw(td))
 		td->verify = 0;
 
@@ -717,12 +821,6 @@ static void fixup_options(struct thread_data *td)
 	if (td->bs_unaligned && (td->odirect || td->io_ops->flags & FIO_RAWIO))
 		log_err("fio: bs_unaligned may not work with raw io\n");
 
-	/*
-	 * O_DIRECT and char doesn't mix, clear that flag if necessary.
-	 */
-	if (td->filetype == FIO_TYPE_CHAR && td->odirect)
-		td->odirect = 0;
-
 	/*
 	 * thinktime_spin must be less than thinktime
 	 */
@@ -750,6 +848,9 @@ static void fixup_options(struct thread_data *td)
 	if (td->iodepth_batch > td->iodepth || !td->iodepth_batch)
 		td->iodepth_batch = td->iodepth;
 
+	if (td->nr_files > td->files_index)
+		td->nr_files = td->files_index;
+
 	if (td->open_files > td->nr_files || !td->open_files)
 		td->open_files = td->nr_files;
 }
@@ -789,6 +890,78 @@ static const char *get_engine_name(const char *str)
 	return p;
 }
 
+static int exists_and_not_file(const char *filename)
+{
+	struct stat sb;
+
+	if (lstat(filename, &sb) == -1)
+		return 0;
+
+	if (S_ISREG(sb.st_mode))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * Initialize the various random states we need (random io, block size ranges,
+ * read/write mix, etc).
+ */
+static int init_random_state(struct thread_data *td)
+{
+	unsigned long seeds[6];
+	int fd, num_maps, blocks;
+	struct fio_file *f;
+	unsigned int i;
+
+	fd = open("/dev/urandom", O_RDONLY);
+	if (fd == -1) {
+		td_verror(td, errno, "open");
+		return 1;
+	}
+
+	if (read(fd, seeds, sizeof(seeds)) < (int) sizeof(seeds)) {
+		td_verror(td, EIO, "read");
+		close(fd);
+		return 1;
+	}
+
+	close(fd);
+
+	os_random_seed(seeds[0], &td->bsrange_state);
+	os_random_seed(seeds[1], &td->verify_state);
+	os_random_seed(seeds[2], &td->rwmix_state);
+
+	if (td->file_service_type == FIO_FSERVICE_RANDOM)
+		os_random_seed(seeds[3], &td->next_file_state);
+
+	os_random_seed(seeds[5], &td->file_size_state);
+
+	if (!td_random(td))
+		return 0;
+
+	if (td->rand_repeatable)
+		seeds[4] = FIO_RANDSEED * td->thread_number;
+
+	if (!td->norandommap) {
+		for_each_file(td, f, i) {
+			blocks = (f->real_file_size + td->rw_min_bs - 1) / td->rw_min_bs;
+			num_maps = (blocks + BLOCKS_PER_MAP-1)/ BLOCKS_PER_MAP;
+			f->file_map = malloc(num_maps * sizeof(long));
+			if (!f->file_map) {
+				log_err("fio: failed allocating random map. If running a large number of jobs, try the 'norandommap' option\n");
+				return 1;
+			}
+			f->num_maps = num_maps;
+			memset(f->file_map, 0, num_maps * sizeof(long));
+		}
+	}
+
+	os_random_seed(seeds[4], &td->random_state);
+	return 0;
+}
+
+
 /*
  * Adds a job to the list of things todo. Sanitizes the various options
  * to make sure we don't have conflicts, and initializes various
@@ -798,10 +971,11 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
 {
 	const char *ddir_str[] = { NULL, "read", "write", "rw", NULL,
 				   "randread", "randwrite", "randrw" };
-	struct stat sb;
-	int numjobs, i;
+	unsigned int i;
 	struct fio_file *f;
 	const char *engine;
+	char fname[PATH_MAX];
+	int numjobs, file_alloced;
 
 	/*
 	 * the def_thread is just for options, it's not a real job
@@ -816,69 +990,35 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
 		return 1;
 	}
 
+	if (td->use_thread)
+		nr_thread++;
+	else
+		nr_process++;
+
 	if (td->odirect)
 		td->io_ops->flags |= FIO_RAWIO;
 
-	if (!td->filename)
-		td->filename = strdup(jobname);
-
-	td->filetype = FIO_TYPE_FILE;
-	if (!lstat(td->filename, &sb)) {
-		if (S_ISBLK(sb.st_mode))
-			td->filetype = FIO_TYPE_BD;
-		else if (S_ISCHR(sb.st_mode))
-			td->filetype = FIO_TYPE_CHAR;
-	}
-
-	fixup_options(td);
-
-	if (td->filename)
-		td->nr_uniq_files = 1;
-	else
-		td->nr_uniq_files = td->open_files;
-
-	if (td->filetype == FIO_TYPE_FILE || td->filename) {
-		char tmp[PATH_MAX];
-		int len = 0;
+	file_alloced = 0;
+	if (!td->filename && !td->files_index) {
+		file_alloced = 1;
 
-		if (td->directory && td->directory[0] != '\0') {
-			if (lstat(td->directory, &sb) < 0) {
-				log_err("fio: %s is not a directory\n", td->directory);
-				td_verror(td, errno, "lstat");
-				return 1;
-			}
-			if (!S_ISDIR(sb.st_mode)) {
-				log_err("fio: %s is not a directory\n", td->directory);
-				return 1;
+		if (td->nr_files == 1 && exists_and_not_file(jobname))
+			add_file(td, jobname);
+		else {
+			for (i = 0; i < td->nr_files; i++) {
+				sprintf(fname, "%s.%d.%d", jobname, td->thread_number, i);
+				add_file(td, fname);
 			}
-			len = sprintf(tmp, "%s/", td->directory);
 		}
-
-		td->files = malloc(sizeof(struct fio_file) * td->open_files);
-
-		for_each_file(td, f, i) {
-			memset(f, 0, sizeof(*f));
-			f->fd = -1;
-
-			if (td->filename)
-				sprintf(tmp + len, "%s", td->filename);
-			else
-				sprintf(tmp + len, "%s.%d.%d", jobname, td->thread_number, i);
-			f->file_name = strdup(tmp);
-		}
-	} else {
-		td->open_files = td->nr_files = 1;
-		td->files = malloc(sizeof(struct fio_file));
-		f = &td->files[0];
-
-		memset(f, 0, sizeof(*f));
-		f->fd = -1;
-		f->file_name = strdup(jobname);
 	}
 
+	fixup_options(td);
+
 	for_each_file(td, f, i) {
-		f->file_size = td->total_file_size / td->nr_files;
-		f->file_offset = td->start_offset;
+		if (td->directory && f->filetype == FIO_TYPE_FILE) {
+			sprintf(fname, "%s/%s", td->directory, f->file_name);
+			f->file_name = strdup(fname);
+		}
 	}
 		
 	td->mutex = fio_sem_init(0);
@@ -887,10 +1027,16 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
 	td->ts.slat_stat[0].min_val = td->ts.slat_stat[1].min_val = ULONG_MAX;
 	td->ts.bw_stat[0].min_val = td->ts.bw_stat[1].min_val = ULONG_MAX;
 
-	if (td->stonewall && td->thread_number > 1)
+	if ((td->stonewall || td->numjobs > 1) && prev_group_jobs) {
+		prev_group_jobs = 0;
 		groupid++;
+	}
 
 	td->groupid = groupid;
+	prev_group_jobs++;
+
+	if (init_random_state(td))
+		goto err;
 
 	if (setup_rate(td))
 		goto err;
@@ -941,69 +1087,28 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
 
 		td_new->numjobs = 1;
 		td_new->stonewall = 0;
+
+		if (file_alloced) {
+			td_new->filename = NULL;
+			td_new->files_index = 0;
+			td_new->files = NULL;
+		}
+
 		job_add_num = numjobs - 1;
 
 		if (add_job(td_new, jobname, job_add_num))
 			goto err;
 	}
-	return 0;
-err:
-	put_job(td);
-	return -1;
-}
-
-/*
- * Initialize the various random states we need (random io, block size ranges,
- * read/write mix, etc).
- */
-int init_random_state(struct thread_data *td)
-{
-	unsigned long seeds[5];
-	int fd, num_maps, blocks, i;
-	struct fio_file *f;
-
-	if (td->io_ops->flags & FIO_DISKLESSIO)
-		return 0;
-
-	fd = open("/dev/urandom", O_RDONLY);
-	if (fd == -1) {
-		td_verror(td, errno, "open");
-		return 1;
-	}
-
-	if (read(fd, seeds, sizeof(seeds)) < (int) sizeof(seeds)) {
-		td_verror(td, EIO, "read");
-		close(fd);
-		return 1;
-	}
-
-	close(fd);
-
-	os_random_seed(seeds[0], &td->bsrange_state);
-	os_random_seed(seeds[1], &td->verify_state);
-	os_random_seed(seeds[2], &td->rwmix_state);
 
-	if (td->file_service_type == FIO_FSERVICE_RANDOM)
-		os_random_seed(seeds[3], &td->next_file_state);
-
-	if (!td_random(td))
-		return 0;
-
-	if (td->rand_repeatable)
-		seeds[4] = FIO_RANDSEED * td->thread_number;
-
-	if (!td->norandommap) {
-		for_each_file(td, f, i) {
-			blocks = (f->real_file_size + td->rw_min_bs - 1) / td->rw_min_bs;
-			num_maps = (blocks + BLOCKS_PER_MAP-1)/ BLOCKS_PER_MAP;
-			f->file_map = malloc(num_maps * sizeof(long));
-			f->num_maps = num_maps;
-			memset(f->file_map, 0, num_maps * sizeof(long));
-		}
+	if (td->numjobs > 1) {
+		groupid++;
+		prev_group_jobs = 0;
 	}
 
-	os_random_seed(seeds[4], &td->random_state);
 	return 0;
+err:
+	put_job(td);
+	return -1;
 }
 
 static void fill_cpu_mask(os_cpu_mask_t cpumask, int cpu)
@@ -1039,7 +1144,7 @@ static int is_empty_or_comment(char *line)
 /*
  * Check if mmap/mmaphuge has a :/foo/bar/file at the end. If so, return that.
  */
-static char *get_mmap_file(const char *str)
+static char *get_opt_postfix(const char *str)
 {
 	char *p = strstr(str, ":");
 
@@ -1057,7 +1162,7 @@ static int str_mem_cb(void *data, const char *mem)
 	struct thread_data *td = data;
 
 	if (td->mem_type == MEM_MMAPHUGE || td->mem_type == MEM_MMAP) {
-		td->mmapfile = get_mmap_file(mem);
+		td->mmapfile = get_opt_postfix(mem);
 		if (td->mem_type == MEM_MMAPHUGE && !td->mmapfile) {
 			log_err("fio: mmaphuge:/path/to/file\n");
 			return 1;
@@ -1105,6 +1210,70 @@ static int str_cpumask_cb(void *data, unsigned int *val)
 	return 0;
 }
 
+static int str_fst_cb(void *data, const char *str)
+{
+	struct thread_data *td = data;
+	char *nr = get_opt_postfix(str);
+
+	td->file_service_nr = 1;
+	if (nr)
+		td->file_service_nr = atoi(nr);
+
+	return 0;
+}
+
+static int str_filename_cb(void *data, const char *input)
+{
+	struct thread_data *td = data;
+	char *fname, *str, *p;
+
+	p = str = strdup(input);
+
+	strip_blank_front(&str);
+	strip_blank_end(str);
+
+	if (!td->files_index)
+		td->nr_files = 0;
+
+	while ((fname = strsep(&str, ":")) != NULL) {
+		if (!strlen(fname))
+			break;
+		add_file(td, fname);
+		td->nr_files++;
+	}
+
+	free(p);
+	return 0;
+}
+
+static int str_directory_cb(void *data, const char fio_unused *str)
+{
+	struct thread_data *td = data;
+	struct stat sb;
+
+	if (lstat(td->directory, &sb) < 0) {
+		log_err("fio: %s is not a directory\n", td->directory);
+		td_verror(td, errno, "lstat");
+		return 1;
+	}
+	if (!S_ISDIR(sb.st_mode)) {
+		log_err("fio: %s is not a directory\n", td->directory);
+		return 1;
+	}
+
+	return 0;
+}
+
+static int str_opendir_cb(void *data, const char fio_unused *str)
+{
+	struct thread_data *td = data;
+
+	if (!td->files_index)
+		td->nr_files = 0;
+
+	return add_dir_files(td, td->opendir);
+}
+
 /*
  * This is our [ini] type file parser.
  */