[fio.git] / target.c

#include <unistd.h>

#include "fio.h"
#include "target.h"
#include "smalloc.h"
#include "stat.h"

void lat_fatal(struct thread_data *td, unsigned long long tnsec,
	       unsigned long long max_nsec)
{
	if (!td->error)
		log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec)\n", tnsec, max_nsec);
	td_verror(td, ETIMEDOUT, "max latency exceeded");
}

static void lat_ios_note(struct thread_data *td)
{
	int i;

	for (i = 0; i < DDIR_RWDIR_CNT; i++)
		td->latency_ios[i] = td->io_blocks[i];
}

static void lat_new_cycle(struct thread_data *td)
{
	fio_gettime(&td->latency_ts, NULL);
	lat_ios_note(td);
	td->latency_failed = 0;
}

/*
 * We had an IO outside the latency target. Reduce the queue depth. If we
 * are at QD=1, then it's time to give up.
 */
static bool __lat_target_failed(struct thread_data *td)
{
	if (td->latency_qd == 1)
		return true;

	td->latency_qd_high = td->latency_qd;

	if (td->latency_qd == td->latency_qd_low)
		td->latency_qd_low--;

	td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2;

	dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);

	/*
	 * When we ramp QD down, quiesce existing IO to prevent
	 * a storm of ramp downs due to pending higher depth.
	 */
	io_u_quiesce(td);
	lat_new_cycle(td);
	return false;
}

bool lat_target_failed(struct thread_data *td)
{
	if (td->o.latency_percentile.u.f == 100.0)
		return __lat_target_failed(td);

	td->latency_failed++;
	return false;
}

static void lat_step_init(struct thread_data *td)
{
	struct thread_options *o = &td->o;

	fio_gettime(&td->latency_ts, NULL);
	td->latency_state = IOD_STATE_PROBE_RAMP;
	td->latency_step = 0;
	td->latency_qd = td->o.iodepth;
	dprint(FD_RATE, "Stepped: %d-%d/%d,%d/%d\n", o->lat_step_low,
				o->lat_step_high, o->lat_step_inc,
				o->lat_step_ramp, o->lat_step_run);
}

void lat_target_init(struct thread_data *td)
{
	td->latency_end_run = 0;

	if (td->o.latency_target) {
		dprint(FD_RATE, "Latency target=%llu\n", td->o.latency_target);
		fio_gettime(&td->latency_ts, NULL);
		td->latency_qd = 1;
		td->latency_qd_high = td->o.iodepth;
		td->latency_qd_low = 1;
		lat_ios_note(td);
	} else if (td->o.iodepth_mode == IOD_STEPPED)
		lat_step_init(td);
	else
		td->latency_qd = td->o.iodepth;
}

void lat_target_reset(struct thread_data *td)
{
	if (td->o.latency_target && !td->latency_end_run)
		lat_target_init(td);
}

static void lat_target_success(struct thread_data *td)
{
	const unsigned int qd = td->latency_qd;
	struct thread_options *o = &td->o;

	td->latency_qd_low = td->latency_qd;

	/*
	 * If we haven't failed yet, we double up to a failing value instead
	 * of bisecting from highest possible queue depth. If we have set
	 * a limit other than td->o.iodepth, bisect between that.
	 */
	if (td->latency_qd_high != o->iodepth)
		td->latency_qd = (td->latency_qd + td->latency_qd_high) / 2;
	else
		td->latency_qd *= 2;

	if (td->latency_qd > o->iodepth)
		td->latency_qd = o->iodepth;

	dprint(FD_RATE, "Ramped up: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);

	/*
	 * Same as last one, we are done. Let it run a latency cycle, so
	 * we get only the results from the targeted depth.
	 */
	if (td->latency_qd == qd) {
		if (td->latency_end_run) {
			dprint(FD_RATE, "We are done\n");
			td->done = 1;
		} else {
			dprint(FD_RATE, "Quiesce and final run\n");
			io_u_quiesce(td);
			td->latency_end_run = 1;
			reset_all_stats(td);
			reset_io_stats(td);
		}
	}

	lat_new_cycle(td);
}

void __lat_target_check(struct thread_data *td)
{
	uint64_t usec_window;
	uint64_t ios;
	double success_ios;

	usec_window = utime_since_now(&td->latency_ts);
	if (usec_window < td->o.latency_window)
		return;

	ios = ddir_rw_sum(td->io_blocks) - ddir_rw_sum(td->latency_ios);
	success_ios = (double) (ios - td->latency_failed) / (double) ios;
	success_ios *= 100.0;

	dprint(FD_RATE, "Success rate: %.2f%% (target %.2f%%)\n", success_ios, td->o.latency_percentile.u.f);

	if (success_ios >= td->o.latency_percentile.u.f)
		lat_target_success(td);
	else
		__lat_target_failed(td);
}

static void lat_clear_rate(struct thread_data *td)
{
	int i;

	td->flags &= ~TD_F_CHECK_RATE;
	for (i = 0; i < DDIR_RWDIR_CNT; i++)
		td->o.rate_iops[i] = 0;
}

/*
 * Returns true if we're done stepping
 */
static bool lat_step_recalc(struct thread_data *td)
{
	struct thread_options *o = &td->o;
	unsigned int cur, perc;

	cur = td->latency_step * o->lat_step_inc;
	if (cur >= o->lat_step_high)
		return true;

	perc = (td->latency_step + 1) * o->lat_step_inc;
	if (perc < 100) {
		int i;

		for (i = 0; i < DDIR_RWDIR_CNT; i++) {
			unsigned int this_iops;

			this_iops = (perc * td->latency_iops[i]) / 100;
			td->o.rate_iops[i] = this_iops;
		}
		setup_rate(td);
		td->flags |= TD_F_CHECK_RATE;
		td->latency_qd = td->o.iodepth * 100 / o->lat_step_high;
	} else {
		td->latency_qd = td->o.iodepth * perc / o->lat_step_high;
		lat_clear_rate(td);
	}
		
	dprint(FD_RATE, "Stepped: step=%d, perc=%d, qd=%d\n", td->latency_step,
						perc, td->latency_qd);
	return false;
}

static void lat_step_reset(struct thread_data *td)
{
	struct thread_stat *ts = &td->ts;
	struct io_stat *ios = &ts->clat_stat[DDIR_RWDIR_CNT];

	ios->max_val = ios->min_val = ios->samples = 0;
	ios->mean.u.f = ios->S.u.f = 0;

	lat_clear_rate(td);
	reset_all_stats(td);
	reset_io_stats(td);
}

static uint64_t lat_iops_since(struct thread_data *td, uint64_t msec,
			       enum fio_ddir ddir)
{
	if (msec) {
		uint64_t ios;

		ios = td->io_blocks[ddir] - td->latency_ios[ddir];
		return (ios * 1000) / msec;
	}

	return 0;
}

static void lat_step_add_sample(struct thread_data *td, uint64_t msec)
{
	struct thread_stat *ts = &td->ts;
	unsigned long long min, max;
	struct lat_step_stats *ls;
	double mean[DDIR_RWDIR_CNT], dev;
	int i;

	if (td->nr_lat_stats == ARRAY_SIZE(td->ts.step_stats)) {
		log_err("fio: ts->step_stats too small, dropping entries\n");
		return;
	}

	for (i = 0; i < DDIR_RWDIR_CNT; i++)
		calc_lat(&ts->clat_stat[i], &min, &max, &mean[i], &dev);

	for (i = 0; i < DDIR_RWDIR_CNT; i++) {
		ls = &td->ts.step_stats[td->nr_lat_stats];

		ls->iops[i] = lat_iops_since(td, msec, i);
		ls->avg[i].u.f = mean[i];
	}

	td->nr_lat_stats++;
}

bool __lat_ts_has_stats(struct thread_stat *ts, enum fio_ddir ddir)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) {
		struct lat_step_stats *ls = &ts->step_stats[i];

		if (ls->iops[ddir])
			return true;
	}

	return false;
}

bool lat_ts_has_stats(struct thread_stat *ts)
{
	int i;

	for (i = 0; i < DDIR_RWDIR_CNT; i++)
		if (__lat_ts_has_stats(ts, i))
			return true;

	return false;
}

void lat_step_report(struct thread_stat *ts, struct buf_output *out)
{
	int i, j;

	for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) {
		struct lat_step_stats *ls = &ts->step_stats[i];

		for (j = 0; j < DDIR_RWDIR_CNT; j++) {
			if (!ls->iops[j])
				continue;

			if (!j)
				__log_buf(out, "    [%2d] ", i);
			else
				__log_buf(out, "         ");

			__log_buf(out, "%5s: iops=%llu, lat=%.1f nsec\n",
					io_ddir_name(j),
					(unsigned long long) ls->iops[j],
					ls->avg[j].u.f);
		}
	}
}

static void lat_next_state(struct thread_data *td, int new_state)
{
	td->latency_state = new_state;
	fio_gettime(&td->latency_ts, NULL);
}

bool lat_step_check(struct thread_data *td)
{
	struct thread_options *o = &td->o;
	uint64_t msec;

	msec = mtime_since_now(&td->latency_ts);

	switch (td->latency_state) {
	case IOD_STATE_PROBE_RAMP:
		if (msec < o->lat_step_ramp)
			break;

		lat_step_reset(td);
		lat_ios_note(td);

		lat_next_state(td, IOD_STATE_PROBE_RUN);
		break;
	case IOD_STATE_PROBE_RUN: {
		int i;

		if (msec < o->lat_step_run)
			break;

		io_u_quiesce(td);

		for (i = 0; i < DDIR_RWDIR_CNT; i++)
			td->latency_iops[i] = lat_iops_since(td, msec, i);

		lat_step_reset(td);
		lat_step_recalc(td);

		io_u_quiesce(td);
		lat_next_state(td, IOD_STATE_RAMP);
		break;
		}
	case IOD_STATE_RAMP:
		if (msec < o->lat_step_ramp)
			break;

		lat_ios_note(td);
		lat_next_state(td, IOD_STATE_RUN);
		break;
	case IOD_STATE_RUN:
		if (msec < o->lat_step_run)
			break;

		io_u_quiesce(td);
		fio_gettime(&td->latency_ts, NULL);
		td->latency_step++;

		lat_step_add_sample(td, msec);
		lat_step_reset(td);

		if (!lat_step_recalc(td))
			break;

		td->done = 1;
		lat_next_state(td, IOD_STATE_DONE);
		break;
	};

	return td->latency_state == IOD_STATE_DONE;
}
Commit	Line	Data
1a7081c7 JA	1	#include <unistd.h>
	2
	3	#include "fio.h"
	4	#include "target.h"
	5	#include "smalloc.h"
	6	#include "stat.h"
	7
	8	void lat_fatal(struct thread_data *td, unsigned long long tnsec,
	9	unsigned long long max_nsec)
	10	{
	11	if (!td->error)
	12	log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec)\n", tnsec, max_nsec);
	13	td_verror(td, ETIMEDOUT, "max latency exceeded");
	14	}
	15
	16	static void lat_ios_note(struct thread_data *td)
	17	{
	18	int i;
	19
	20	for (i = 0; i < DDIR_RWDIR_CNT; i++)
	21	td->latency_ios[i] = td->io_blocks[i];
	22	}
	23
	24	static void lat_new_cycle(struct thread_data *td)
	25	{
	26	fio_gettime(&td->latency_ts, NULL);
	27	lat_ios_note(td);
	28	td->latency_failed = 0;
	29	}
	30
	31	/*
	32	* We had an IO outside the latency target. Reduce the queue depth. If we
	33	* are at QD=1, then it's time to give up.
	34	*/
	35	static bool __lat_target_failed(struct thread_data *td)
	36	{
	37	if (td->latency_qd == 1)
	38	return true;
	39
	40	td->latency_qd_high = td->latency_qd;
	41
	42	if (td->latency_qd == td->latency_qd_low)
	43	td->latency_qd_low--;
	44
	45	td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2;
	46
	47	dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
	48
	49	/*
	50	* When we ramp QD down, quiesce existing IO to prevent
	51	* a storm of ramp downs due to pending higher depth.
	52	*/
	53	io_u_quiesce(td);
	54	lat_new_cycle(td);
	55	return false;
	56	}
	57
	58	bool lat_target_failed(struct thread_data *td)
	59	{
	60	if (td->o.latency_percentile.u.f == 100.0)
	61	return __lat_target_failed(td);
	62
	63	td->latency_failed++;
	64	return false;
65	}
66
67	static void lat_step_init(struct thread_data *td)
68	{
69	struct thread_options *o = &td->o;
70
71	fio_gettime(&td->latency_ts, NULL);
72	td->latency_state = IOD_STATE_PROBE_RAMP;
73	td->latency_step = 0;
74	td->latency_qd = td->o.iodepth;
75	dprint(FD_RATE, "Stepped: %d-%d/%d,%d/%d\n", o->lat_step_low,
76	o->lat_step_high, o->lat_step_inc,
77	o->lat_step_ramp, o->lat_step_run);
78	}
79
80	void lat_target_init(struct thread_data *td)
81	{
82	td->latency_end_run = 0;
83
84	if (td->o.latency_target) {
85	dprint(FD_RATE, "Latency target=%llu\n", td->o.latency_target);
86	fio_gettime(&td->latency_ts, NULL);
87	td->latency_qd = 1;
88	td->latency_qd_high = td->o.iodepth;
89	td->latency_qd_low = 1;
90	lat_ios_note(td);
91	} else if (td->o.iodepth_mode == IOD_STEPPED)
92	lat_step_init(td);
93	else
94	td->latency_qd = td->o.iodepth;
95	}
96
97	void lat_target_reset(struct thread_data *td)
98	{
99	if (td->o.latency_target && !td->latency_end_run)
100	lat_target_init(td);
101	}
102
103	static void lat_target_success(struct thread_data *td)
104	{
105	const unsigned int qd = td->latency_qd;
106	struct thread_options *o = &td->o;
107
108	td->latency_qd_low = td->latency_qd;
109
110	/*
111	* If we haven't failed yet, we double up to a failing value instead
112	* of bisecting from highest possible queue depth. If we have set
113	* a limit other than td->o.iodepth, bisect between that.
114	*/
115	if (td->latency_qd_high != o->iodepth)
116	td->latency_qd = (td->latency_qd + td->latency_qd_high) / 2;
117	else
118	td->latency_qd *= 2;
119
120	if (td->latency_qd > o->iodepth)
121	td->latency_qd = o->iodepth;
122
123	dprint(FD_RATE, "Ramped up: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
124
125	/*
126	* Same as last one, we are done. Let it run a latency cycle, so
127	* we get only the results from the targeted depth.
128	*/
129	if (td->latency_qd == qd) {
130	if (td->latency_end_run) {
131	dprint(FD_RATE, "We are done\n");
132	td->done = 1;
133	} else {
134	dprint(FD_RATE, "Quiesce and final run\n");
135	io_u_quiesce(td);
136	td->latency_end_run = 1;
137	reset_all_stats(td);
138	reset_io_stats(td);
139	}
140	}
141
142	lat_new_cycle(td);
143	}
144
145	void __lat_target_check(struct thread_data *td)
146	{
147	uint64_t usec_window;
148	uint64_t ios;
149	double success_ios;
150
151	usec_window = utime_since_now(&td->latency_ts);
152	if (usec_window < td->o.latency_window)
153	return;
154
155	ios = ddir_rw_sum(td->io_blocks) - ddir_rw_sum(td->latency_ios);
156	success_ios = (double) (ios - td->latency_failed) / (double) ios;
157	success_ios *= 100.0;
158
159	dprint(FD_RATE, "Success rate: %.2f%% (target %.2f%%)\n", success_ios, td->o.latency_percentile.u.f);
160
161	if (success_ios >= td->o.latency_percentile.u.f)
162	lat_target_success(td);
163	else
164	__lat_target_failed(td);
165	}
166
167	static void lat_clear_rate(struct thread_data *td)
168	{
169	int i;
170
171	td->flags &= ~TD_F_CHECK_RATE;
172	for (i = 0; i < DDIR_RWDIR_CNT; i++)
173	td->o.rate_iops[i] = 0;
174	}
175
176	/*
177	* Returns true if we're done stepping
178	*/
179	static bool lat_step_recalc(struct thread_data *td)
180	{
181	struct thread_options *o = &td->o;
182	unsigned int cur, perc;
183
184	cur = td->latency_step * o->lat_step_inc;
185	if (cur >= o->lat_step_high)
186	return true;
187
188	perc = (td->latency_step + 1) * o->lat_step_inc;
189	if (perc < 100) {
190	int i;
191
192	for (i = 0; i < DDIR_RWDIR_CNT; i++) {
193	unsigned int this_iops;
194
195	this_iops = (perc * td->latency_iops[i]) / 100;
196	td->o.rate_iops[i] = this_iops;
197	}
198	setup_rate(td);
199	td->flags \|= TD_F_CHECK_RATE;
200	td->latency_qd = td->o.iodepth * 100 / o->lat_step_high;
201	} else {
202	td->latency_qd = td->o.iodepth * perc / o->lat_step_high;
203	lat_clear_rate(td);
204	}
205
206	dprint(FD_RATE, "Stepped: step=%d, perc=%d, qd=%d\n", td->latency_step,
207	perc, td->latency_qd);
208	return false;
209	}
210
211	static void lat_step_reset(struct thread_data *td)
212	{
213	struct thread_stat *ts = &td->ts;
214	struct io_stat *ios = &ts->clat_stat[DDIR_RWDIR_CNT];
215
216	ios->max_val = ios->min_val = ios->samples = 0;
217	ios->mean.u.f = ios->S.u.f = 0;
218
219	lat_clear_rate(td);
220	reset_all_stats(td);
221	reset_io_stats(td);
222	}
223
224	static uint64_t lat_iops_since(struct thread_data *td, uint64_t msec,
225	enum fio_ddir ddir)
226	{
227	if (msec) {
228	uint64_t ios;
229
230	ios = td->io_blocks[ddir] - td->latency_ios[ddir];
231	return (ios * 1000) / msec;
232	}
233
234	return 0;
235	}
236
237	static void lat_step_add_sample(struct thread_data *td, uint64_t msec)
238	{
239	struct thread_stat *ts = &td->ts;
240	unsigned long long min, max;
241	struct lat_step_stats *ls;
242	double mean[DDIR_RWDIR_CNT], dev;
243	int i;
244
245	if (td->nr_lat_stats == ARRAY_SIZE(td->ts.step_stats)) {
246	log_err("fio: ts->step_stats too small, dropping entries\n");
247	return;
248	}
249
250	for (i = 0; i < DDIR_RWDIR_CNT; i++)
251	calc_lat(&ts->clat_stat[i], &min, &max, &mean[i], &dev);
252
253	for (i = 0; i < DDIR_RWDIR_CNT; i++) {
254	ls = &td->ts.step_stats[td->nr_lat_stats];
255
256	ls->iops[i] = lat_iops_since(td, msec, i);
257	ls->avg[i].u.f = mean[i];
258	}
259
260	td->nr_lat_stats++;
261	}
262
263	bool __lat_ts_has_stats(struct thread_stat *ts, enum fio_ddir ddir)
264	{
265	int i;
266
267	for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) {
268	struct lat_step_stats *ls = &ts->step_stats[i];
269
270	if (ls->iops[ddir])
271	return true;
272	}
273
274	return false;
275	}
276
277	bool lat_ts_has_stats(struct thread_stat *ts)
278	{
279	int i;
280
281	for (i = 0; i < DDIR_RWDIR_CNT; i++)
282	if (__lat_ts_has_stats(ts, i))
283	return true;
284
285	return false;
286	}
287
288	void lat_step_report(struct thread_stat ts, struct buf_output out)
289	{
290	int i, j;
291
292	for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) {
293	struct lat_step_stats *ls = &ts->step_stats[i];
294
295	for (j = 0; j < DDIR_RWDIR_CNT; j++) {
296	if (!ls->iops[j])
297	continue;
298
fcd4e744 JA	299	if (!j)
	300	__log_buf(out, " [%2d] ", i);
	301	else
	302	__log_buf(out, " ");
	303
	304	__log_buf(out, "%5s: iops=%llu, lat=%.1f nsec\n",
1a7081c7 JA	305	io_ddir_name(j),
	306	(unsigned long long) ls->iops[j],
	307	ls->avg[j].u.f);
	308	}
	309	}
	310	}
	311
	312	static void lat_next_state(struct thread_data *td, int new_state)
	313	{
	314	td->latency_state = new_state;
	315	fio_gettime(&td->latency_ts, NULL);
	316	}
	317
	318	bool lat_step_check(struct thread_data *td)
	319	{
	320	struct thread_options *o = &td->o;
	321	uint64_t msec;
	322
	323	msec = mtime_since_now(&td->latency_ts);
	324
	325	switch (td->latency_state) {
	326	case IOD_STATE_PROBE_RAMP:
	327	if (msec < o->lat_step_ramp)
	328	break;
	329
	330	lat_step_reset(td);
	331	lat_ios_note(td);
	332
	333	lat_next_state(td, IOD_STATE_PROBE_RUN);
	334	break;
	335	case IOD_STATE_PROBE_RUN: {
	336	int i;
	337
	338	if (msec < o->lat_step_run)
	339	break;
	340
	341	io_u_quiesce(td);
	342
	343	for (i = 0; i < DDIR_RWDIR_CNT; i++)
	344	td->latency_iops[i] = lat_iops_since(td, msec, i);
	345
	346	lat_step_reset(td);
	347	lat_step_recalc(td);
	348
	349	io_u_quiesce(td);
	350	lat_next_state(td, IOD_STATE_RAMP);
	351	break;
	352	}
	353	case IOD_STATE_RAMP:
	354	if (msec < o->lat_step_ramp)
	355	break;
	356
	357	lat_ios_note(td);
	358	lat_next_state(td, IOD_STATE_RUN);
	359	break;
	360	case IOD_STATE_RUN:
	361	if (msec < o->lat_step_run)
	362	break;
	363
	364	io_u_quiesce(td);
	365	fio_gettime(&td->latency_ts, NULL);
	366	td->latency_step++;
	367
	368	lat_step_add_sample(td, msec);
369	lat_step_reset(td);
370
371	if (!lat_step_recalc(td))
372	break;
373
374	td->done = 1;
375	lat_next_state(td, IOD_STATE_DONE);
376	break;
377	};
378
379	return td->latency_state == IOD_STATE_DONE;
380	}