X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=backend.c;h=5ac365925fdd469a948643bd9ed14bc9759b3f94;hp=c2fa34ccc7731fb3c11c280d7f887c00cd6e36d0;hb=e413c2ccf8af0a0bce29530568ded346a894cf3d;hpb=8d271b66d2bc1025f760d3c5a976dac42262a178

diff --git a/backend.c b/backend.c
index c2fa34cc..5ac36592 100644
--- a/backend.c
+++ b/backend.c
@@ -52,11 +52,11 @@
 #include "server.h"
 #include "lib/getrusage.h"
 #include "idletime.h"
+#include "err.h"
 
 static pthread_t disk_util_thread;
 static struct fio_mutex *disk_thread_mutex;
 static struct fio_mutex *startup_mutex;
-static struct fio_mutex *writeout_mutex;
 static struct flist_head *cgroup_list;
 static char *cgroup_mnt;
 static int exit_value;
@@ -345,7 +345,7 @@ static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
 		return 0;
 	if (!td->o.timeout)
 		return 0;
-	if (mtime_since(&td->epoch, t) >= td->o.timeout * 1000)
+	if (utime_since(&td->epoch, t) >= td->o.timeout)
 		return 1;
 
 	return 0;
@@ -478,6 +478,12 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 				break;
 
 			while ((io_u = get_io_u(td)) != NULL) {
+				if (IS_ERR(io_u)) {
+					io_u = NULL;
+					ret = FIO_Q_BUSY;
+					goto reap;
+				}
+
 				/*
 				 * We are only interested in the places where
 				 * we wrote or trimmed IOs. Turn those into
@@ -574,6 +580,7 @@ sync_done:
 		 * completed io_u's first. Note that we can get BUSY even
 		 * without IO queued, if the system is resource starved.
 		 */
+reap:
 		full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
 		if (full || !td->o.iodepth_batch_complete) {
 			min_events = min(td->o.iodepth_batch_complete,
@@ -615,6 +622,19 @@ sync_done:
 	dprint(FD_VERIFY, "exiting loop\n");
 }
 
+static unsigned int exceeds_number_ios(struct thread_data *td)
+{
+	unsigned long long number_ios;
+
+	if (!td->o.number_ios)
+		return 0;
+
+	number_ios = ddir_rw_sum(td->this_io_blocks);
+	number_ios += td->io_u_queued + td->io_u_in_flight;
+
+	return number_ios >= td->o.number_ios;
+}
+
 static int io_bytes_exceeded(struct thread_data *td)
 {
 	unsigned long long bytes;
@@ -628,7 +648,7 @@ static int io_bytes_exceeded(struct thread_data *td)
 	else
 		bytes = td->this_io_bytes[DDIR_TRIM];
 
-	return bytes >= td->o.size;
+	return bytes >= td->o.size || exceeds_number_ios(td);
 }
 
 /*
@@ -642,12 +662,25 @@ static uint64_t do_io(struct thread_data *td)
 	uint64_t bytes_done[DDIR_RWDIR_CNT] = { 0, 0, 0 };
 	unsigned int i;
 	int ret = 0;
+	uint64_t total_bytes, bytes_issued = 0;
 
 	if (in_ramp_time(td))
 		td_set_runstate(td, TD_RAMP);
 	else
 		td_set_runstate(td, TD_RUNNING);
 
+	lat_target_init(td);
+
+	/*
+	 * If verify_backlog is enabled, we'll run the verify in this
+	 * handler as well. For that case, we may need up to twice the
+	 * amount of bytes.
+	 */
+	total_bytes = td->o.size;
+	if (td->o.verify != VERIFY_NONE &&
+	   (td_write(td) && td->o.verify_backlog))
+		total_bytes += td->o.size;
+
 	while ((td->o.read_iolog_file && !flist_empty(&td->io_log_list)) ||
 		(!flist_empty(&td->trim_list)) || !io_bytes_exceeded(td) ||
 		td->o.time_based) {
@@ -675,9 +708,22 @@ static uint64_t do_io(struct thread_data *td)
 		if (flow_threshold_exceeded(td))
 			continue;
 
+		if (bytes_issued >= total_bytes)
+			break;
+
 		io_u = get_io_u(td);
-		if (!io_u)
+		if (IS_ERR_OR_NULL(io_u)) {
+			int err = PTR_ERR(io_u);
+
+			io_u = NULL;
+			if (err == -EBUSY) {
+				ret = FIO_Q_BUSY;
+				goto reap;
+			}
+			if (td->o.latency_target)
+				goto reap;
 			break;
+		}
 
 		ddir = io_u->ddir;
 
@@ -688,6 +734,13 @@ static uint64_t do_io(struct thread_data *td)
 		 */
 		if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_READ &&
 		    ((io_u->flags & IO_U_F_VER_LIST) || !td_rw(td))) {
+
+			if (!td->o.verify_pattern_bytes) {
+				io_u->rand_seed = __rand(&td->__verify_state);
+				if (sizeof(int) != sizeof(long *))
+					io_u->rand_seed *= __rand(&td->__verify_state);
+			}
+
 			if (td->o.verify_async)
 				io_u->end_io = verify_io_u_async;
 			else
@@ -698,6 +751,17 @@ static uint64_t do_io(struct thread_data *td)
 		else
 			td_set_runstate(td, TD_RUNNING);
 
+		/*
+		 * Always log IO before it's issued, so we know the specific
+		 * order of it. The logged unit will track when the IO has
+		 * completed.
+		 */
+		if (td_write(td) && io_u->ddir == DDIR_WRITE &&
+		    td->o.do_verify &&
+		    td->o.verify != VERIFY_NONE &&
+		    !td->o.experimental_verify)
+			log_io_piece(td, io_u);
+
 		ret = td_io_queue(td, io_u);
 		switch (ret) {
 		case FIO_Q_COMPLETED:
@@ -708,6 +772,7 @@ static uint64_t do_io(struct thread_data *td)
 				int bytes = io_u->xfer_buflen - io_u->resid;
 				struct fio_file *f = io_u->file;
 
+				bytes_issued += bytes;
 				/*
 				 * zero read, fail
 				 */
@@ -738,6 +803,7 @@ sync_done:
 				ret = io_u_sync_complete(td, io_u, bytes_done);
 				if (ret < 0)
 					break;
+				bytes_issued += io_u->xfer_buflen;
 			}
 			break;
 		case FIO_Q_QUEUED:
@@ -748,6 +814,7 @@ sync_done:
 			 */
 			if (td->io_ops->commit == NULL)
 				io_u_queued(td, io_u);
+			bytes_issued += io_u->xfer_buflen;
 			break;
 		case FIO_Q_BUSY:
 			requeue_io_u(td, &io_u);
@@ -769,6 +836,7 @@ sync_done:
 		 * can get BUSY even without IO queued, if the system is
 		 * resource starved.
 		 */
+reap:
 		full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
 		if (full || !td->o.iodepth_batch_complete) {
 			min_evts = min(td->o.iodepth_batch_complete,
@@ -805,6 +873,8 @@ sync_done:
 				break;
 			}
 		}
+		if (!in_ramp_time(td) && td->o.latency_target)
+			lat_target_check(td);
 
 		if (td->o.thinktime) {
 			unsigned long long b;
@@ -919,7 +989,8 @@ static int init_io_u(struct thread_data *td)
 	 * overflow later. this adjustment may be too much if we get
 	 * lucky and the allocator gives us an aligned address.
 	 */
-	if (td->o.odirect || td->o.mem_align || (td->io_ops->flags & FIO_RAWIO))
+	if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
+	    (td->io_ops->flags & FIO_RAWIO))
 		td->orig_buffer_size += page_mask + td->o.mem_align;
 
 	if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) {
@@ -937,7 +1008,7 @@ static int init_io_u(struct thread_data *td)
 	if (data_xfer && allocate_io_mem(td))
 		return 1;
 
-	if (td->o.odirect || td->o.mem_align ||
+	if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
 	    (td->io_ops->flags & FIO_RAWIO))
 		p = PAGE_ALIGN(td->orig_buffer) + td->o.mem_align;
 	else
@@ -973,7 +1044,7 @@ static int init_io_u(struct thread_data *td)
 				 * Fill the buffer with the pattern if we are
 				 * going to be doing writes.
 				 */
-				fill_pattern(td, io_u->buf, max_bs, io_u, 0, 0);
+				fill_verify_pattern(td, io_u->buf, max_bs, io_u, 0, 0);
 			}
 		}
 
@@ -1068,6 +1139,8 @@ static int keep_running(struct thread_data *td)
 		td->o.loops--;
 		return 1;
 	}
+	if (exceeds_number_ios(td))
+		return 0;
 
 	if (td->o.size != -1ULL && ddir_rw_sum(td->io_bytes) < td->o.size) {
 		uint64_t diff;
@@ -1080,20 +1153,24 @@ static int keep_running(struct thread_data *td)
 		if (diff < td_max_bs(td))
 			return 0;
 
+		if (fio_files_done(td))
+			return 0;
+
 		return 1;
 	}
 
 	return 0;
 }
 
-static int exec_string(const char *string)
+static int exec_string(struct thread_options *o, const char *string, const char *mode)
 {
-	int ret, newlen = strlen(string) + 1 + 2;
+	int ret, newlen = strlen(string) + strlen(o->name) + strlen(mode) + 9 + 1;
 	char *str;
 
 	str = malloc(newlen);
-	sprintf(str, "%s", string);
+	sprintf(str, "%s &> %s.%s.txt", string, o->name, mode);
 
+	log_info("%s : Saving output of %s in %s.%s.txt\n",o->name, mode, o->name, mode);
 	ret = system(str);
 	if (ret == -1)
 		log_err("fio: exec of cmd <%s> failed\n", str);
@@ -1102,6 +1179,50 @@ static int exec_string(const char *string)
 	return ret;
 }
 
+/*
+ * Dry run to compute correct state of numberio for verification.
+ */
+static uint64_t do_dry_run(struct thread_data *td)
+{
+	uint64_t bytes_done[DDIR_RWDIR_CNT] = { 0, 0, 0 };
+
+	td_set_runstate(td, TD_RUNNING);
+
+	while ((td->o.read_iolog_file && !flist_empty(&td->io_log_list)) ||
+		(!flist_empty(&td->trim_list)) || !io_bytes_exceeded(td)) {
+		struct io_u *io_u;
+		int ret;
+
+		if (td->terminate || td->done)
+			break;
+
+		io_u = get_io_u(td);
+		if (!io_u)
+			break;
+
+		io_u->flags |= IO_U_F_FLIGHT;
+		io_u->error = 0;
+		io_u->resid = 0;
+		if (ddir_rw(acct_ddir(io_u)))
+			td->io_issues[acct_ddir(io_u)]++;
+		if (ddir_rw(io_u->ddir)) {
+			io_u_mark_depth(td, 1);
+			td->ts.total_io_u[io_u->ddir]++;
+		}
+
+		if (td_write(td) && io_u->ddir == DDIR_WRITE &&
+		    td->o.do_verify &&
+		    td->o.verify != VERIFY_NONE &&
+		    !td->o.experimental_verify)
+			log_io_piece(td, io_u);
+
+		ret = io_u_sync_complete(td, io_u, bytes_done);
+		(void) ret;
+	}
+
+	return bytes_done[DDIR_WRITE] + bytes_done[DDIR_TRIM];
+}
+
 /*
  * Entry point for the thread based jobs. The process based jobs end up
  * here as well, after a little setup.
@@ -1121,6 +1242,11 @@ static void *thread_main(void *data)
 	} else
 		td->pid = gettid();
 
+	/*
+	 * fio_time_init() may not have been called yet if running as a server
+	 */
+	fio_time_init();
+
 	fio_local_clock_init(o->use_thread);
 
 	dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid);
@@ -1147,13 +1273,6 @@ static void *thread_main(void *data)
 	fio_mutex_down(td->mutex);
 	dprint(FD_MUTEX, "done waiting on td->mutex\n");
 
-	/*
-	 * the ->mutex mutex is now no longer used, close it to avoid
-	 * eating a file descriptor
-	 */
-	fio_mutex_remove(td->mutex);
-	td->mutex = NULL;
-
 	/*
 	 * A new gid requires privilege, so we need to do this before setting
 	 * the uid.
@@ -1179,6 +1298,15 @@ static void *thread_main(void *data)
 	 * allocations.
 	 */
 	if (o->cpumask_set) {
+		if (o->cpus_allowed_policy == FIO_CPUS_SPLIT) {
+			ret = fio_cpus_split(&o->cpumask, td->thread_number - 1);
+			if (!ret) {
+				log_err("fio: no CPUs set\n");
+				log_err("fio: Try increasing number of available CPUs\n");
+				td_verror(td, EINVAL, "cpus_split");
+				goto err;
+			}
+		}
 		ret = fio_setaffinity(td->pid, o->cpumask);
 		if (ret == -1) {
 			td_verror(td, errno, "cpu_set_affinity");
@@ -1274,7 +1402,7 @@ static void *thread_main(void *data)
 	if (init_random_map(td))
 		goto err;
 
-	if (o->exec_prerun && exec_string(o->exec_prerun))
+	if (o->exec_prerun && exec_string(o, o->exec_prerun, (const char *)"prerun"))
 		goto err;
 
 	if (o->pre_read) {
@@ -1310,7 +1438,10 @@ static void *thread_main(void *data)
 
 		prune_io_piece_log(td);
 
-		verify_bytes = do_io(td);
+		if (td->o.verify_only && (td_write(td) || td_rw(td)))
+			verify_bytes = do_dry_run(td);
+		else
+			verify_bytes = do_io(td);
 
 		clear_state = 1;
 
@@ -1358,46 +1489,10 @@ static void *thread_main(void *data)
 
 	fio_unpin_memory(td);
 
-	fio_mutex_down(writeout_mutex);
-	if (td->bw_log) {
-		if (o->bw_log_file) {
-			finish_log_named(td, td->bw_log,
-						o->bw_log_file, "bw");
-		} else
-			finish_log(td, td->bw_log, "bw");
-	}
-	if (td->lat_log) {
-		if (o->lat_log_file) {
-			finish_log_named(td, td->lat_log,
-						o->lat_log_file, "lat");
-		} else
-			finish_log(td, td->lat_log, "lat");
-	}
-	if (td->slat_log) {
-		if (o->lat_log_file) {
-			finish_log_named(td, td->slat_log,
-						o->lat_log_file, "slat");
-		} else
-			finish_log(td, td->slat_log, "slat");
-	}
-	if (td->clat_log) {
-		if (o->lat_log_file) {
-			finish_log_named(td, td->clat_log,
-						o->lat_log_file, "clat");
-		} else
-			finish_log(td, td->clat_log, "clat");
-	}
-	if (td->iops_log) {
-		if (o->iops_log_file) {
-			finish_log_named(td, td->iops_log,
-						o->iops_log_file, "iops");
-		} else
-			finish_log(td, td->iops_log, "iops");
-	}
+	fio_writeout_logs(td);
 
-	fio_mutex_up(writeout_mutex);
 	if (o->exec_postrun)
-		exec_string(o->exec_postrun);
+		exec_string(o, o->exec_postrun, (const char *)"postrun");
 
 	if (exitall_on_terminate)
 		fio_terminate_threads(td->groupid);
@@ -1430,6 +1525,9 @@ err:
 	fio_mutex_remove(td->rusage_sem);
 	td->rusage_sem = NULL;
 
+	fio_mutex_remove(td->mutex);
+	td->mutex = NULL;
+
 	td_set_runstate(td, TD_EXITED);
 	return (void *) (uintptr_t) td->error;
 }
@@ -1577,12 +1675,12 @@ static void do_usleep(unsigned int usecs)
 static void run_threads(void)
 {
 	struct thread_data *td;
-	unsigned long spent;
 	unsigned int i, todo, nr_running, m_rate, t_rate, nr_started;
+	uint64_t spent;
 
 	if (fio_gtod_offload && fio_start_gtod_thread())
 		return;
-	
+
 	fio_idle_prof_init();
 
 	set_sig_handlers();
@@ -1676,9 +1774,9 @@ static void run_threads(void)
 			}
 
 			if (td->o.start_delay) {
-				spent = mtime_since_genesis();
+				spent = utime_since_genesis();
 
-				if (td->o.start_delay * 1000 > spent)
+				if (td->o.start_delay > spent)
 					continue;
 			}
 
@@ -1897,9 +1995,6 @@ int fio_backend(void)
 	startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
 	if (startup_mutex == NULL)
 		return 1;
-	writeout_mutex = fio_mutex_init(FIO_MUTEX_UNLOCKED);
-	if (writeout_mutex == NULL)
-		return 1;
 
 	set_genesis_time();
 	stat_init();
@@ -1930,7 +2025,6 @@ int fio_backend(void)
 	sfree(cgroup_mnt);
 
 	fio_mutex_remove(startup_mutex);
-	fio_mutex_remove(writeout_mutex);
 	fio_mutex_remove(disk_thread_mutex);
 	stat_exit();
 	return exit_value;