From: Jens Axboe <axboe@fb.com>
Date: Sun, 7 Aug 2016 21:07:53 +0000 (-0600)
Subject: Merge branch 'histograms-PR' of https://github.com/cronburg/fio
X-Git-Tag: fio-2.14~57
X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=3c746856fd63f3f7624a34ea70e226a290d9cc01;hp=-c

Merge branch 'histograms-PR' of https://github.com/cronburg/fio
---

3c746856fd63f3f7624a34ea70e226a290d9cc01
diff --combined HOWTO
index d18d59b4,d4b23c89..0085b747
--- a/HOWTO
+++ b/HOWTO
@@@ -1175,7 -1175,7 +1175,7 @@@ cpus_allowed_policy=str Set the policy 
  		one cpu per job. If not enough CPUs are given for the jobs
  		listed, then fio will roundrobin the CPUs in the set.
  
 -numa_cpu_nodes=str Set this job running on spcified NUMA nodes' CPUs. The
 +numa_cpu_nodes=str Set this job running on specified NUMA nodes' CPUs. The
  		arguments allow comma delimited list of cpu numbers,
  		A-B ranges, or 'all'. Note, to enable numa options support,
  		fio must be built on a system with libnuma-dev(el) installed.
@@@ -1606,10 -1606,18 +1606,18 @@@ write_lat_log=str Same as write_bw_log
  		The actual log names will be foo_slat.x.log, foo_clat.x.log,
  		and foo_lat.x.log, where x is the index of the job (1..N,
  		where N is the number of jobs). This helps fio_generate_plot
 -		fine the logs automatically. If 'per_job_logs' is false, then
 +		find the logs automatically. If 'per_job_logs' is false, then
  		the filename will not include the job index. See 'Log File
  		Formats'.
  
+ write_hist_log=str Same as write_lat_log, but writes I/O completion
+ 		latency histograms. If no filename is given with this option, the
+ 		default filename of "jobname_clat_hist.x.log" is used, where x is
+ 		the index of the job (1..N, where N is the number of jobs). Even
+ 		if the filename is given, fio will still append the type of log.
+ 		If per_job_logs is false, then the filename will not include the
+ 		job index. See 'Log File Formats'.
+ 
  write_iops_log=str Same as write_bw_log, but writes IOPS. If no filename is
  		given with this option, the default filename of
  		"jobname_type.x.log" is used,where x is the index of the job
@@@ -1625,6 -1633,20 +1633,20 @@@ log_avg_msec=int By default, fio will l
  		specified period of time, reducing the resolution of the log.
  		See log_max_value as well. Defaults to 0, logging all entries.
  
+ log_hist_msec=int Same as log_avg_msec, but logs entries for completion
+ 		latency histograms. Computing latency percentiles from averages of
+ 		intervals using log_avg_msec is innacurate. Setting this option makes
+ 		fio log histogram entries over the specified period of time, reducing
+ 		log sizes for high IOPS devices while retaining percentile accuracy.
+ 		See log_hist_coarseness as well. Defaults to 0, meaning histogram
+ 		logging is disabled.
+ 
+ log_hist_coarseness=int Integer ranging from 0 to 6, defining the coarseness
+ 		of the resolution of the histogram logs enabled with log_hist_msec. For
+ 		each increment in coarseness, fio outputs half as many bins. Defaults to
+ 		0, for which histogram logs contain 1216 latency bins. See
+ 		'Log File Formats'.
+ 
  log_max_value=bool	If log_avg_msec is set, fio logs the average over that
  		window. If you instead want to log the maximum value, set this
  		option to 1. Defaults to 0, meaning that averaged values are
diff --combined stat.c
index d6787b7f,965ff9fc..96af94ba
--- a/stat.c
+++ b/stat.c
@@@ -1965,6 -1965,7 +1965,7 @@@ void regrow_logs(struct thread_data *td
  {
  	regrow_log(td->slat_log);
  	regrow_log(td->clat_log);
+ 	regrow_log(td->clat_hist_log);
  	regrow_log(td->lat_log);
  	regrow_log(td->bw_log);
  	regrow_log(td->iops_log);
@@@ -2139,9 -2140,7 +2140,9 @@@ static long add_log_sample(struct threa
  	 * need to do.
  	 */
  	this_window = elapsed - iolog->avg_last;
 -	if (this_window < iolog->avg_msec) {
 +	if (elapsed < iolog->avg_last)
 +		return iolog->avg_last - elapsed;
 +	else if (this_window < iolog->avg_msec) {
  		int diff = iolog->avg_msec - this_window;
  
  		if (inline_log(iolog) || diff > LOG_MSEC_SLACK)
@@@ -2195,7 -2194,9 +2196,9 @@@ static void add_clat_percentile_sample(
  void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
  		     unsigned long usec, unsigned int bs, uint64_t offset)
  {
+ 	unsigned long elapsed, this_window;
  	struct thread_stat *ts = &td->ts;
+ 	struct io_log *iolog = td->clat_hist_log;
  
  	td_io_u_lock(td);
  
@@@ -2207,6 -2208,35 +2210,35 @@@
  	if (ts->clat_percentiles)
  		add_clat_percentile_sample(ts, usec, ddir);
  
+ 	if (iolog && iolog->hist_msec) {
+ 		struct io_hist *hw = &(iolog->hist_window[ddir]);
+ 		(hw->samples)++;
+ 		elapsed = mtime_since_now(&td->epoch);
+ 		if (! hw->hist_last)
+ 			hw->hist_last = elapsed;
+ 		this_window = elapsed - hw->hist_last;
+ 		
+ 		if (this_window >= iolog->hist_msec) {
+ 			/*
+ 			 * Make a byte-for-byte copy of the latency histogram stored in
+ 			 * td->ts.io_u_plat[ddir], recording it in a log sample. Note that
+ 			 * the matching call to free() is located in iolog.c after printing
+ 			 * this sample to the log file.
+ 			 */
+ 			unsigned int *io_u_plat = (unsigned int *)(td->ts.io_u_plat[ddir]);
+ 			unsigned int *dst = malloc(FIO_IO_U_PLAT_NR * sizeof(unsigned int));
+ 			memcpy(dst, io_u_plat, FIO_IO_U_PLAT_NR * sizeof(unsigned int));
+ 			__add_log_sample(iolog, (uint64_t)dst, ddir, bs, elapsed, offset);
+ 
+ 			/*
+ 			 * Update the last time we recorded as being now, minus any drift
+ 			 * in time we encountered before actually making the record.
+ 			 */
+ 			hw->hist_last = elapsed - (this_window - iolog->hist_msec);
+ 			hw->samples = 0;
+ 		}
+ 	}
+ 
  	td_io_u_unlock(td);
  }