Commit | Line | Data |
---|---|---|
1a7081c7 JA |
1 | #include <unistd.h> |
2 | ||
3 | #include "fio.h" | |
4 | #include "target.h" | |
5 | #include "smalloc.h" | |
6 | #include "stat.h" | |
7 | ||
8 | void lat_fatal(struct thread_data *td, unsigned long long tnsec, | |
9 | unsigned long long max_nsec) | |
10 | { | |
11 | if (!td->error) | |
12 | log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec)\n", tnsec, max_nsec); | |
13 | td_verror(td, ETIMEDOUT, "max latency exceeded"); | |
14 | } | |
15 | ||
16 | static void lat_ios_note(struct thread_data *td) | |
17 | { | |
18 | int i; | |
19 | ||
20 | for (i = 0; i < DDIR_RWDIR_CNT; i++) | |
21 | td->latency_ios[i] = td->io_blocks[i]; | |
22 | } | |
23 | ||
24 | static void lat_new_cycle(struct thread_data *td) | |
25 | { | |
26 | fio_gettime(&td->latency_ts, NULL); | |
27 | lat_ios_note(td); | |
28 | td->latency_failed = 0; | |
29 | } | |
30 | ||
31 | /* | |
32 | * We had an IO outside the latency target. Reduce the queue depth. If we | |
33 | * are at QD=1, then it's time to give up. | |
34 | */ | |
35 | static bool __lat_target_failed(struct thread_data *td) | |
36 | { | |
37 | if (td->latency_qd == 1) | |
38 | return true; | |
39 | ||
40 | td->latency_qd_high = td->latency_qd; | |
41 | ||
42 | if (td->latency_qd == td->latency_qd_low) | |
43 | td->latency_qd_low--; | |
44 | ||
45 | td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2; | |
46 | ||
47 | dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high); | |
48 | ||
49 | /* | |
50 | * When we ramp QD down, quiesce existing IO to prevent | |
51 | * a storm of ramp downs due to pending higher depth. | |
52 | */ | |
53 | io_u_quiesce(td); | |
54 | lat_new_cycle(td); | |
55 | return false; | |
56 | } | |
57 | ||
58 | bool lat_target_failed(struct thread_data *td) | |
59 | { | |
60 | if (td->o.latency_percentile.u.f == 100.0) | |
61 | return __lat_target_failed(td); | |
62 | ||
63 | td->latency_failed++; | |
64 | return false; | |
65 | } | |
66 | ||
67 | static void lat_step_init(struct thread_data *td) | |
68 | { | |
69 | struct thread_options *o = &td->o; | |
70 | ||
71 | fio_gettime(&td->latency_ts, NULL); | |
72 | td->latency_state = IOD_STATE_PROBE_RAMP; | |
73 | td->latency_step = 0; | |
74 | td->latency_qd = td->o.iodepth; | |
75 | dprint(FD_RATE, "Stepped: %d-%d/%d,%d/%d\n", o->lat_step_low, | |
76 | o->lat_step_high, o->lat_step_inc, | |
77 | o->lat_step_ramp, o->lat_step_run); | |
78 | } | |
79 | ||
80 | void lat_target_init(struct thread_data *td) | |
81 | { | |
82 | td->latency_end_run = 0; | |
83 | ||
84 | if (td->o.latency_target) { | |
85 | dprint(FD_RATE, "Latency target=%llu\n", td->o.latency_target); | |
86 | fio_gettime(&td->latency_ts, NULL); | |
87 | td->latency_qd = 1; | |
88 | td->latency_qd_high = td->o.iodepth; | |
89 | td->latency_qd_low = 1; | |
90 | lat_ios_note(td); | |
91 | } else if (td->o.iodepth_mode == IOD_STEPPED) | |
92 | lat_step_init(td); | |
93 | else | |
94 | td->latency_qd = td->o.iodepth; | |
95 | } | |
96 | ||
97 | void lat_target_reset(struct thread_data *td) | |
98 | { | |
99 | if (td->o.latency_target && !td->latency_end_run) | |
100 | lat_target_init(td); | |
101 | } | |
102 | ||
103 | static void lat_target_success(struct thread_data *td) | |
104 | { | |
105 | const unsigned int qd = td->latency_qd; | |
106 | struct thread_options *o = &td->o; | |
107 | ||
108 | td->latency_qd_low = td->latency_qd; | |
109 | ||
110 | /* | |
111 | * If we haven't failed yet, we double up to a failing value instead | |
112 | * of bisecting from highest possible queue depth. If we have set | |
113 | * a limit other than td->o.iodepth, bisect between that. | |
114 | */ | |
115 | if (td->latency_qd_high != o->iodepth) | |
116 | td->latency_qd = (td->latency_qd + td->latency_qd_high) / 2; | |
117 | else | |
118 | td->latency_qd *= 2; | |
119 | ||
120 | if (td->latency_qd > o->iodepth) | |
121 | td->latency_qd = o->iodepth; | |
122 | ||
123 | dprint(FD_RATE, "Ramped up: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high); | |
124 | ||
125 | /* | |
126 | * Same as last one, we are done. Let it run a latency cycle, so | |
127 | * we get only the results from the targeted depth. | |
128 | */ | |
129 | if (td->latency_qd == qd) { | |
130 | if (td->latency_end_run) { | |
131 | dprint(FD_RATE, "We are done\n"); | |
132 | td->done = 1; | |
133 | } else { | |
134 | dprint(FD_RATE, "Quiesce and final run\n"); | |
135 | io_u_quiesce(td); | |
136 | td->latency_end_run = 1; | |
137 | reset_all_stats(td); | |
138 | reset_io_stats(td); | |
139 | } | |
140 | } | |
141 | ||
142 | lat_new_cycle(td); | |
143 | } | |
144 | ||
145 | void __lat_target_check(struct thread_data *td) | |
146 | { | |
147 | uint64_t usec_window; | |
148 | uint64_t ios; | |
149 | double success_ios; | |
150 | ||
151 | usec_window = utime_since_now(&td->latency_ts); | |
152 | if (usec_window < td->o.latency_window) | |
153 | return; | |
154 | ||
155 | ios = ddir_rw_sum(td->io_blocks) - ddir_rw_sum(td->latency_ios); | |
156 | success_ios = (double) (ios - td->latency_failed) / (double) ios; | |
157 | success_ios *= 100.0; | |
158 | ||
159 | dprint(FD_RATE, "Success rate: %.2f%% (target %.2f%%)\n", success_ios, td->o.latency_percentile.u.f); | |
160 | ||
161 | if (success_ios >= td->o.latency_percentile.u.f) | |
162 | lat_target_success(td); | |
163 | else | |
164 | __lat_target_failed(td); | |
165 | } | |
166 | ||
167 | static void lat_clear_rate(struct thread_data *td) | |
168 | { | |
169 | int i; | |
170 | ||
171 | td->flags &= ~TD_F_CHECK_RATE; | |
172 | for (i = 0; i < DDIR_RWDIR_CNT; i++) | |
173 | td->o.rate_iops[i] = 0; | |
174 | } | |
175 | ||
176 | /* | |
177 | * Returns true if we're done stepping | |
178 | */ | |
179 | static bool lat_step_recalc(struct thread_data *td) | |
180 | { | |
181 | struct thread_options *o = &td->o; | |
182 | unsigned int cur, perc; | |
183 | ||
184 | cur = td->latency_step * o->lat_step_inc; | |
185 | if (cur >= o->lat_step_high) | |
186 | return true; | |
187 | ||
188 | perc = (td->latency_step + 1) * o->lat_step_inc; | |
189 | if (perc < 100) { | |
190 | int i; | |
191 | ||
192 | for (i = 0; i < DDIR_RWDIR_CNT; i++) { | |
193 | unsigned int this_iops; | |
194 | ||
195 | this_iops = (perc * td->latency_iops[i]) / 100; | |
196 | td->o.rate_iops[i] = this_iops; | |
197 | } | |
198 | setup_rate(td); | |
199 | td->flags |= TD_F_CHECK_RATE; | |
200 | td->latency_qd = td->o.iodepth * 100 / o->lat_step_high; | |
201 | } else { | |
202 | td->latency_qd = td->o.iodepth * perc / o->lat_step_high; | |
203 | lat_clear_rate(td); | |
204 | } | |
205 | ||
206 | dprint(FD_RATE, "Stepped: step=%d, perc=%d, qd=%d\n", td->latency_step, | |
207 | perc, td->latency_qd); | |
208 | return false; | |
209 | } | |
210 | ||
211 | static void lat_step_reset(struct thread_data *td) | |
212 | { | |
213 | struct thread_stat *ts = &td->ts; | |
214 | struct io_stat *ios = &ts->clat_stat[DDIR_RWDIR_CNT]; | |
215 | ||
216 | ios->max_val = ios->min_val = ios->samples = 0; | |
217 | ios->mean.u.f = ios->S.u.f = 0; | |
218 | ||
219 | lat_clear_rate(td); | |
220 | reset_all_stats(td); | |
221 | reset_io_stats(td); | |
222 | } | |
223 | ||
224 | static uint64_t lat_iops_since(struct thread_data *td, uint64_t msec, | |
225 | enum fio_ddir ddir) | |
226 | { | |
227 | if (msec) { | |
228 | uint64_t ios; | |
229 | ||
230 | ios = td->io_blocks[ddir] - td->latency_ios[ddir]; | |
231 | return (ios * 1000) / msec; | |
232 | } | |
233 | ||
234 | return 0; | |
235 | } | |
236 | ||
237 | static void lat_step_add_sample(struct thread_data *td, uint64_t msec) | |
238 | { | |
239 | struct thread_stat *ts = &td->ts; | |
240 | unsigned long long min, max; | |
241 | struct lat_step_stats *ls; | |
242 | double mean[DDIR_RWDIR_CNT], dev; | |
243 | int i; | |
244 | ||
245 | if (td->nr_lat_stats == ARRAY_SIZE(td->ts.step_stats)) { | |
246 | log_err("fio: ts->step_stats too small, dropping entries\n"); | |
247 | return; | |
248 | } | |
249 | ||
250 | for (i = 0; i < DDIR_RWDIR_CNT; i++) | |
251 | calc_lat(&ts->clat_stat[i], &min, &max, &mean[i], &dev); | |
252 | ||
253 | for (i = 0; i < DDIR_RWDIR_CNT; i++) { | |
254 | ls = &td->ts.step_stats[td->nr_lat_stats]; | |
255 | ||
256 | ls->iops[i] = lat_iops_since(td, msec, i); | |
257 | ls->avg[i].u.f = mean[i]; | |
258 | } | |
259 | ||
260 | td->nr_lat_stats++; | |
261 | } | |
262 | ||
263 | bool __lat_ts_has_stats(struct thread_stat *ts, enum fio_ddir ddir) | |
264 | { | |
265 | int i; | |
266 | ||
267 | for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) { | |
268 | struct lat_step_stats *ls = &ts->step_stats[i]; | |
269 | ||
270 | if (ls->iops[ddir]) | |
271 | return true; | |
272 | } | |
273 | ||
274 | return false; | |
275 | } | |
276 | ||
277 | bool lat_ts_has_stats(struct thread_stat *ts) | |
278 | { | |
279 | int i; | |
280 | ||
281 | for (i = 0; i < DDIR_RWDIR_CNT; i++) | |
282 | if (__lat_ts_has_stats(ts, i)) | |
283 | return true; | |
284 | ||
285 | return false; | |
286 | } | |
287 | ||
288 | void lat_step_report(struct thread_stat *ts, struct buf_output *out) | |
289 | { | |
290 | int i, j; | |
291 | ||
292 | for (i = 0; i < ARRAY_SIZE(ts->step_stats); i++) { | |
293 | struct lat_step_stats *ls = &ts->step_stats[i]; | |
294 | ||
295 | for (j = 0; j < DDIR_RWDIR_CNT; j++) { | |
296 | if (!ls->iops[j]) | |
297 | continue; | |
298 | ||
fcd4e744 JA |
299 | if (!j) |
300 | __log_buf(out, " [%2d] ", i); | |
301 | else | |
302 | __log_buf(out, " "); | |
303 | ||
304 | __log_buf(out, "%5s: iops=%llu, lat=%.1f nsec\n", | |
1a7081c7 JA |
305 | io_ddir_name(j), |
306 | (unsigned long long) ls->iops[j], | |
307 | ls->avg[j].u.f); | |
308 | } | |
309 | } | |
310 | } | |
311 | ||
312 | static void lat_next_state(struct thread_data *td, int new_state) | |
313 | { | |
314 | td->latency_state = new_state; | |
315 | fio_gettime(&td->latency_ts, NULL); | |
316 | } | |
317 | ||
318 | bool lat_step_check(struct thread_data *td) | |
319 | { | |
320 | struct thread_options *o = &td->o; | |
321 | uint64_t msec; | |
322 | ||
323 | msec = mtime_since_now(&td->latency_ts); | |
324 | ||
325 | switch (td->latency_state) { | |
326 | case IOD_STATE_PROBE_RAMP: | |
327 | if (msec < o->lat_step_ramp) | |
328 | break; | |
329 | ||
330 | lat_step_reset(td); | |
331 | lat_ios_note(td); | |
332 | ||
333 | lat_next_state(td, IOD_STATE_PROBE_RUN); | |
334 | break; | |
335 | case IOD_STATE_PROBE_RUN: { | |
336 | int i; | |
337 | ||
338 | if (msec < o->lat_step_run) | |
339 | break; | |
340 | ||
341 | io_u_quiesce(td); | |
342 | ||
343 | for (i = 0; i < DDIR_RWDIR_CNT; i++) | |
344 | td->latency_iops[i] = lat_iops_since(td, msec, i); | |
345 | ||
346 | lat_step_reset(td); | |
347 | lat_step_recalc(td); | |
348 | ||
349 | io_u_quiesce(td); | |
350 | lat_next_state(td, IOD_STATE_RAMP); | |
351 | break; | |
352 | } | |
353 | case IOD_STATE_RAMP: | |
354 | if (msec < o->lat_step_ramp) | |
355 | break; | |
356 | ||
357 | lat_ios_note(td); | |
358 | lat_next_state(td, IOD_STATE_RUN); | |
359 | break; | |
360 | case IOD_STATE_RUN: | |
361 | if (msec < o->lat_step_run) | |
362 | break; | |
363 | ||
364 | io_u_quiesce(td); | |
365 | fio_gettime(&td->latency_ts, NULL); | |
366 | td->latency_step++; | |
367 | ||
368 | lat_step_add_sample(td, msec); | |
369 | lat_step_reset(td); | |
370 | ||
371 | if (!lat_step_recalc(td)) | |
372 | break; | |
373 | ||
374 | td->done = 1; | |
375 | lat_next_state(td, IOD_STATE_DONE); | |
376 | break; | |
377 | }; | |
378 | ||
379 | return td->latency_state == IOD_STATE_DONE; | |
380 | } |