8 void lat_fatal(struct thread_data *td, unsigned long long tnsec,
9 unsigned long long max_nsec)
12 log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec)\n", tnsec, max_nsec);
13 td_verror(td, ETIMEDOUT, "max latency exceeded");
16 static void lat_ios_note(struct thread_data *td)
20 for (i = 0; i < DDIR_RWDIR_CNT; i++)
21 td->latency_ios[i] = td->io_blocks[i];
24 static void lat_new_cycle(struct thread_data *td)
26 fio_gettime(&td->latency_ts, NULL);
28 td->latency_failed = 0;
32 * We had an IO outside the latency target. Reduce the queue depth. If we
33 * are at QD=1, then it's time to give up.
35 static bool __lat_target_failed(struct thread_data *td)
37 if (td->latency_qd == 1)
40 td->latency_qd_high = td->latency_qd;
42 if (td->latency_qd == td->latency_qd_low)
45 td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2;
47 dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
50 * When we ramp QD down, quiesce existing IO to prevent
51 * a storm of ramp downs due to pending higher depth.
58 bool lat_target_failed(struct thread_data *td)
60 if (td->o.latency_percentile.u.f == 100.0)
61 return __lat_target_failed(td);
67 static void lat_step_init(struct thread_data *td)
69 struct thread_options *o = &td->o;
71 fio_gettime(&td->latency_ts, NULL);
72 td->latency_state = IOD_STATE_PROBE_RAMP;
74 td->latency_qd = td->o.iodepth;
75 dprint(FD_RATE, "Stepped: %d-%d/%d,%d/%d\n", o->lat_step_low,
76 o->lat_step_high, o->lat_step_inc,
77 o->lat_step_ramp, o->lat_step_run);
80 void lat_target_init(struct thread_data *td)
82 td->latency_end_run = 0;
84 if (td->o.latency_target) {
85 dprint(FD_RATE, "Latency target=%llu\n", td->o.latency_target);
86 fio_gettime(&td->latency_ts, NULL);
88 td->latency_qd_high = td->o.iodepth;
89 td->latency_qd_low = 1;
91 } else if (td->o.iodepth_mode == IOD_STEPPED)
94 td->latency_qd = td->o.iodepth;
97 void lat_target_reset(struct thread_data *td)
99 if (td->o.latency_target && !td->latency_end_run)
103 static void lat_target_success(struct thread_data *td)
105 const unsigned int qd = td->latency_qd;
106 struct thread_options *o = &td->o;
108 td->latency_qd_low = td->latency_qd;
111 * If we haven't failed yet, we double up to a failing value instead
112 * of bisecting from highest possible queue depth. If we have set
113 * a limit other than td->o.iodepth, bisect between that.
115 if (td->latency_qd_high != o->iodepth)
116 td->latency_qd = (td->latency_qd + td->latency_qd_high) / 2;
120 if (td->latency_qd > o->iodepth)
121 td->latency_qd = o->iodepth;
123 dprint(FD_RATE, "Ramped up: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
126 * Same as last one, we are done. Let it run a latency cycle, so
127 * we get only the results from the targeted depth.
129 if (td->latency_qd == qd) {
130 if (td->latency_end_run) {
131 dprint(FD_RATE, "We are done\n");
134 dprint(FD_RATE, "Quiesce and final run\n");
136 td->latency_end_run = 1;
145 void __lat_target_check(struct thread_data *td)
147 uint64_t usec_window;
151 usec_window = utime_since_now(&td->latency_ts);
152 if (usec_window < td->o.latency_window)
155 ios = ddir_rw_sum(td->io_blocks) - ddir_rw_sum(td->latency_ios);
156 success_ios = (double) (ios - td->latency_failed) / (double) ios;
157 success_ios *= 100.0;
159 dprint(FD_RATE, "Success rate: %.2f%% (target %.2f%%)\n", success_ios, td->o.latency_percentile.u.f);
161 if (success_ios >= td->o.latency_percentile.u.f)
162 lat_target_success(td);
164 __lat_target_failed(td);
167 static void lat_clear_rate(struct thread_data *td)
171 td->flags &= ~TD_F_CHECK_RATE;
172 for (i = 0; i < DDIR_RWDIR_CNT; i++)
173 td->o.rate_iops[i] = 0;
177 * Returns true if we're done stepping
179 static bool lat_step_recalc(struct thread_data *td)
181 struct thread_options *o = &td->o;
182 unsigned int cur, perc;
184 cur = td->latency_step * o->lat_step_inc;
185 if (cur >= o->lat_step_high)
188 perc = (td->latency_step + 1) * o->lat_step_inc;
192 for (i = 0; i < DDIR_RWDIR_CNT; i++) {
193 unsigned int this_iops;
195 this_iops = (perc * td->latency_iops[i]) / 100;
196 td->o.rate_iops[i] = this_iops;
199 td->flags |= TD_F_CHECK_RATE;
200 td->latency_qd = td->o.iodepth * 100 / o->lat_step_high;
202 td->latency_qd = td->o.iodepth * perc / o->lat_step_high;
206 dprint(FD_RATE, "Stepped: step=%d, perc=%d, qd=%d\n", td->latency_step,
207 perc, td->latency_qd);
211 static void lat_step_reset(struct thread_data *td)
213 struct thread_stat *ts = &td->ts;
214 struct io_stat *ios = &ts->clat_stat[DDIR_RWDIR_CNT];
216 ios->max_val = ios->min_val = ios->samples = 0;
217 ios->mean.u.f = ios->S.u.f = 0;
224 static uint64_t lat_iops_since(struct thread_data *td, uint64_t msec,
230 ios = td->io_blocks[ddir] - td->latency_ios[ddir];
231 return (ios * 1000) / msec;
237 static void lat_step_add_sample(struct thread_data *td, uint64_t msec)
239 struct thread_stat *ts = &td->ts;
240 unsigned long long min, max;
241 struct lat_step_stats *ls;
242 double mean[DDIR_RWDIR_CNT], dev;
245 if (td->nr_lat_stats == ARRAY_SIZE(td->ts.step_stats)) {
246 log_err("fio: ts->step_stats too small, dropping entries\n");
250 for (i = 0; i < DDIR_RWDIR_CNT; i++)
251 calc_lat(&ts->clat_stat[i], &min, &max, &mean[i], &dev);
253 for (i = 0; i < DDIR_RWDIR_CNT; i++) {
254 ls = &td->ts.step_stats[td->nr_lat_stats];
256 ls->iops[i] = lat_iops_since(td, msec, i);
257 ls->avg[i].u.f = mean[i];
263 bool __lat_ts_has_stats(struct thread_stat *ts, enum fio_ddir ddir)
267 for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) {
268 struct lat_step_stats *ls = &ts->step_stats[i];
277 bool lat_ts_has_stats(struct thread_stat *ts)
281 for (i = 0; i < DDIR_RWDIR_CNT; i++)
282 if (__lat_ts_has_stats(ts, i))
288 void lat_step_report(struct thread_stat *ts, struct buf_output *out)
292 for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) {
293 struct lat_step_stats *ls = &ts->step_stats[i];
295 for (j = 0; j < DDIR_RWDIR_CNT; j++) {
299 __log_buf(out, " %s: iops=%llu, lat=%.1f nsec\n",
301 (unsigned long long) ls->iops[j],
307 static void lat_next_state(struct thread_data *td, int new_state)
309 td->latency_state = new_state;
310 fio_gettime(&td->latency_ts, NULL);
313 bool lat_step_check(struct thread_data *td)
315 struct thread_options *o = &td->o;
318 msec = mtime_since_now(&td->latency_ts);
320 switch (td->latency_state) {
321 case IOD_STATE_PROBE_RAMP:
322 if (msec < o->lat_step_ramp)
328 lat_next_state(td, IOD_STATE_PROBE_RUN);
330 case IOD_STATE_PROBE_RUN: {
333 if (msec < o->lat_step_run)
338 for (i = 0; i < DDIR_RWDIR_CNT; i++)
339 td->latency_iops[i] = lat_iops_since(td, msec, i);
345 lat_next_state(td, IOD_STATE_RAMP);
349 if (msec < o->lat_step_ramp)
353 lat_next_state(td, IOD_STATE_RUN);
356 if (msec < o->lat_step_run)
360 fio_gettime(&td->latency_ts, NULL);
363 lat_step_add_sample(td, msec);
366 if (!lat_step_recalc(td))
370 lat_next_state(td, IOD_STATE_DONE);
374 return td->latency_state == IOD_STATE_DONE;