HOWTO/manpage: update percentage explanation using '%'
[fio.git] / iolog.c
CommitLineData
ac9b9101
JA
1/*
2 * Code related to writing an iolog of what a thread is doing, and to
3 * later read that back and replay
4 */
5#include <stdio.h>
6#include <stdlib.h>
7#include <libgen.h>
8#include <assert.h>
b26317c9
JA
9#include <sys/types.h>
10#include <sys/stat.h>
11#include <unistd.h>
aee2ab67
JA
12#ifdef CONFIG_ZLIB
13#include <zlib.h>
14#endif
15
ac9b9101
JA
16#include "flist.h"
17#include "fio.h"
18#include "verify.h"
19#include "trim.h"
243bfe19 20#include "filelock.h"
a47591e4 21#include "smalloc.h"
a3e59412 22#include "blktrace.h"
ac9b9101 23
7e419452
JA
24static int iolog_flush(struct io_log *log);
25
ac9b9101
JA
26static const char iolog_ver2[] = "fio version 2 iolog";
27
28void queue_io_piece(struct thread_data *td, struct io_piece *ipo)
29{
30 flist_add_tail(&ipo->list, &td->io_log_list);
31 td->total_io_size += ipo->len;
32}
33
1f440ece 34void log_io_u(const struct thread_data *td, const struct io_u *io_u)
ac9b9101 35{
ac9b9101
JA
36 if (!td->o.write_iolog_file)
37 return;
38
39 fprintf(td->iolog_f, "%s %s %llu %lu\n", io_u->file->file_name,
173375c2
JA
40 io_ddir_name(io_u->ddir),
41 io_u->offset, io_u->buflen);
ac9b9101
JA
42}
43
44void log_file(struct thread_data *td, struct fio_file *f,
45 enum file_log_act what)
46{
47 const char *act[] = { "add", "open", "close" };
48
49 assert(what < 3);
50
51 if (!td->o.write_iolog_file)
52 return;
53
54
55 /*
56 * this happens on the pre-open/close done before the job starts
57 */
58 if (!td->iolog_f)
59 return;
60
61 fprintf(td->iolog_f, "%s %s\n", f->file_name, act[what]);
62}
63
64static void iolog_delay(struct thread_data *td, unsigned long delay)
65{
df8fb609
JA
66 uint64_t usec = utime_since_now(&td->last_issue);
67 uint64_t this_delay;
8b6a404c 68 struct timespec ts;
ac9b9101 69
df8fb609
JA
70 if (delay < td->time_offset) {
71 td->time_offset = 0;
72 return;
73 }
74
75 delay -= td->time_offset;
ac9b9101
JA
76 if (delay < usec)
77 return;
78
79 delay -= usec;
80
8b6a404c 81 fio_gettime(&ts, NULL);
30b18672
JA
82 while (delay && !td->terminate) {
83 this_delay = delay;
84 if (this_delay > 500000)
85 this_delay = 500000;
86
87 usec_sleep(td, this_delay);
88 delay -= this_delay;
89 }
df8fb609 90
8b6a404c 91 usec = utime_since_now(&ts);
df8fb609
JA
92 if (usec > delay)
93 td->time_offset = usec - delay;
94 else
95 td->time_offset = 0;
ac9b9101
JA
96}
97
98static int ipo_special(struct thread_data *td, struct io_piece *ipo)
99{
100 struct fio_file *f;
101 int ret;
102
103 /*
104 * Not a special ipo
105 */
106 if (ipo->ddir != DDIR_INVAL)
107 return 0;
108
109 f = td->files[ipo->fileno];
110
111 switch (ipo->file_action) {
112 case FIO_LOG_OPEN_FILE:
96d663ad
SW
113 if (td->o.replay_redirect && fio_file_open(f)) {
114 dprint(FD_FILE, "iolog: ignoring re-open of file %s\n",
115 f->file_name);
116 break;
117 }
ac9b9101
JA
118 ret = td_io_open_file(td, f);
119 if (!ret)
120 break;
121 td_verror(td, ret, "iolog open file");
122 return -1;
123 case FIO_LOG_CLOSE_FILE:
124 td_io_close_file(td, f);
125 break;
126 case FIO_LOG_UNLINK_FILE:
38ef9c90 127 td_io_unlink_file(td, f);
ac9b9101
JA
128 break;
129 default:
130 log_err("fio: bad file action %d\n", ipo->file_action);
131 break;
132 }
133
134 return 1;
135}
136
137int read_iolog_get(struct thread_data *td, struct io_u *io_u)
138{
139 struct io_piece *ipo;
140 unsigned long elapsed;
3c3ed070 141
ac9b9101
JA
142 while (!flist_empty(&td->io_log_list)) {
143 int ret;
144
9342d5f8 145 ipo = flist_first_entry(&td->io_log_list, struct io_piece, list);
ac9b9101
JA
146 flist_del(&ipo->list);
147 remove_trim_entry(td, ipo);
148
149 ret = ipo_special(td, ipo);
150 if (ret < 0) {
151 free(ipo);
152 break;
153 } else if (ret > 0) {
154 free(ipo);
155 continue;
156 }
157
158 io_u->ddir = ipo->ddir;
159 if (ipo->ddir != DDIR_WAIT) {
160 io_u->offset = ipo->offset;
161 io_u->buflen = ipo->len;
162 io_u->file = td->files[ipo->fileno];
163 get_file(io_u->file);
164 dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset,
165 io_u->buflen, io_u->file->file_name);
166 if (ipo->delay)
167 iolog_delay(td, ipo->delay);
168 } else {
169 elapsed = mtime_since_genesis();
170 if (ipo->delay > elapsed)
171 usec_sleep(td, (ipo->delay - elapsed) * 1000);
ac9b9101
JA
172 }
173
174 free(ipo);
3c3ed070 175
ac9b9101
JA
176 if (io_u->ddir != DDIR_WAIT)
177 return 0;
178 }
179
180 td->done = 1;
181 return 1;
182}
183
184void prune_io_piece_log(struct thread_data *td)
185{
186 struct io_piece *ipo;
187 struct rb_node *n;
188
189 while ((n = rb_first(&td->io_hist_tree)) != NULL) {
190 ipo = rb_entry(n, struct io_piece, rb_node);
191 rb_erase(n, &td->io_hist_tree);
192 remove_trim_entry(td, ipo);
193 td->io_hist_len--;
194 free(ipo);
195 }
196
197 while (!flist_empty(&td->io_hist_list)) {
4e1020f8 198 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
ac9b9101
JA
199 flist_del(&ipo->list);
200 remove_trim_entry(td, ipo);
201 td->io_hist_len--;
202 free(ipo);
203 }
204}
205
206/*
207 * log a successful write, so we can unwind the log for verify
208 */
209void log_io_piece(struct thread_data *td, struct io_u *io_u)
210{
211 struct rb_node **p, *parent;
212 struct io_piece *ipo, *__ipo;
213
214 ipo = malloc(sizeof(struct io_piece));
215 init_ipo(ipo);
216 ipo->file = io_u->file;
217 ipo->offset = io_u->offset;
218 ipo->len = io_u->buflen;
da0a7bd2 219 ipo->numberio = io_u->numberio;
f9401285
JA
220 ipo->flags = IP_F_IN_FLIGHT;
221
222 io_u->ipo = ipo;
ac9b9101
JA
223
224 if (io_u_should_trim(td, io_u)) {
225 flist_add_tail(&ipo->trim_list, &td->trim_list);
226 td->trim_entries++;
227 }
228
229 /*
230 * We don't need to sort the entries, if:
231 *
232 * Sequential writes, or
233 * Random writes that lay out the file as it goes along
234 *
235 * For both these cases, just reading back data in the order we
236 * wrote it out is the fastest.
237 *
238 * One exception is if we don't have a random map AND we are doing
239 * verifies, in that case we need to check for duplicate blocks and
240 * drop the old one, which we rely on the rb insert/lookup for
241 * handling.
242 */
c4b6117b 243 if (((!td->o.verifysort) || !td_random(td) || !td->o.overwrite) &&
ac9b9101
JA
244 (file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
245 INIT_FLIST_HEAD(&ipo->list);
246 flist_add_tail(&ipo->list, &td->io_hist_list);
247 ipo->flags |= IP_F_ONLIST;
248 td->io_hist_len++;
249 return;
250 }
251
252 RB_CLEAR_NODE(&ipo->rb_node);
253
254 /*
255 * Sort the entry into the verification list
256 */
257restart:
258 p = &td->io_hist_tree.rb_node;
259 parent = NULL;
260 while (*p) {
3b1ff869 261 int overlap = 0;
ac9b9101
JA
262 parent = *p;
263
264 __ipo = rb_entry(parent, struct io_piece, rb_node);
265 if (ipo->file < __ipo->file)
266 p = &(*p)->rb_left;
267 else if (ipo->file > __ipo->file)
268 p = &(*p)->rb_right;
3b1ff869 269 else if (ipo->offset < __ipo->offset) {
ac9b9101 270 p = &(*p)->rb_left;
3b1ff869
JE
271 overlap = ipo->offset + ipo->len > __ipo->offset;
272 }
273 else if (ipo->offset > __ipo->offset) {
ac9b9101 274 p = &(*p)->rb_right;
3b1ff869
JE
275 overlap = __ipo->offset + __ipo->len > ipo->offset;
276 }
277 else
278 overlap = 1;
279
280 if (overlap) {
7abe6a27 281 dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu\n",
885ac623
JA
282 __ipo->offset, __ipo->len,
283 ipo->offset, ipo->len);
ac9b9101
JA
284 td->io_hist_len--;
285 rb_erase(parent, &td->io_hist_tree);
286 remove_trim_entry(td, __ipo);
287 free(__ipo);
288 goto restart;
289 }
290 }
291
292 rb_link_node(&ipo->rb_node, parent, p);
293 rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
294 ipo->flags |= IP_F_ONRB;
295 td->io_hist_len++;
296}
297
890b6656
JA
298void unlog_io_piece(struct thread_data *td, struct io_u *io_u)
299{
300 struct io_piece *ipo = io_u->ipo;
301
66347cfa
DE
302 if (td->ts.nr_block_infos) {
303 uint32_t *info = io_u_block_info(td, io_u);
304 if (BLOCK_INFO_STATE(*info) < BLOCK_STATE_TRIM_FAILURE) {
305 if (io_u->ddir == DDIR_TRIM)
306 *info = BLOCK_INFO_SET_STATE(*info,
307 BLOCK_STATE_TRIM_FAILURE);
308 else if (io_u->ddir == DDIR_WRITE)
309 *info = BLOCK_INFO_SET_STATE(*info,
310 BLOCK_STATE_WRITE_FAILURE);
311 }
312 }
313
890b6656
JA
314 if (!ipo)
315 return;
316
317 if (ipo->flags & IP_F_ONRB)
318 rb_erase(&ipo->rb_node, &td->io_hist_tree);
319 else if (ipo->flags & IP_F_ONLIST)
320 flist_del(&ipo->list);
321
322 free(ipo);
323 io_u->ipo = NULL;
324 td->io_hist_len--;
325}
326
1f440ece 327void trim_io_piece(struct thread_data *td, const struct io_u *io_u)
890b6656
JA
328{
329 struct io_piece *ipo = io_u->ipo;
330
331 if (!ipo)
332 return;
333
334 ipo->len = io_u->xfer_buflen - io_u->resid;
335}
336
ac9b9101
JA
337void write_iolog_close(struct thread_data *td)
338{
339 fflush(td->iolog_f);
340 fclose(td->iolog_f);
341 free(td->iolog_buf);
342 td->iolog_f = NULL;
343 td->iolog_buf = NULL;
344}
345
346/*
347 * Read version 2 iolog data. It is enhanced to include per-file logging,
348 * syncs, etc.
349 */
350static int read_iolog2(struct thread_data *td, FILE *f)
351{
352 unsigned long long offset;
353 unsigned int bytes;
354 int reads, writes, waits, fileno = 0, file_action = 0; /* stupid gcc */
dac34079 355 char *rfname, *fname, *act;
ac9b9101
JA
356 char *str, *p;
357 enum fio_ddir rw;
358
359 free_release_files(td);
360
361 /*
362 * Read in the read iolog and store it, reuse the infrastructure
363 * for doing verifications.
364 */
365 str = malloc(4096);
dac34079 366 rfname = fname = malloc(256+16);
ac9b9101
JA
367 act = malloc(256+16);
368
369 reads = writes = waits = 0;
370 while ((p = fgets(str, 4096, f)) != NULL) {
371 struct io_piece *ipo;
372 int r;
373
dac34079 374 r = sscanf(p, "%256s %256s %llu %u", rfname, act, &offset,
ac9b9101 375 &bytes);
dac34079
JA
376
377 if (td->o.replay_redirect)
378 fname = td->o.replay_redirect;
379
ac9b9101
JA
380 if (r == 4) {
381 /*
382 * Check action first
383 */
384 if (!strcmp(act, "wait"))
385 rw = DDIR_WAIT;
386 else if (!strcmp(act, "read"))
387 rw = DDIR_READ;
388 else if (!strcmp(act, "write"))
389 rw = DDIR_WRITE;
390 else if (!strcmp(act, "sync"))
391 rw = DDIR_SYNC;
392 else if (!strcmp(act, "datasync"))
393 rw = DDIR_DATASYNC;
394 else if (!strcmp(act, "trim"))
395 rw = DDIR_TRIM;
396 else {
397 log_err("fio: bad iolog file action: %s\n",
398 act);
399 continue;
400 }
033ace1e 401 fileno = get_fileno(td, fname);
ac9b9101
JA
402 } else if (r == 2) {
403 rw = DDIR_INVAL;
404 if (!strcmp(act, "add")) {
96d663ad
SW
405 if (td->o.replay_redirect &&
406 get_fileno(td, fname) != -1) {
407 dprint(FD_FILE, "iolog: ignoring"
408 " re-add of file %s\n", fname);
409 } else {
410 fileno = add_file(td, fname, 0, 1);
411 file_action = FIO_LOG_ADD_FILE;
412 }
ac9b9101
JA
413 continue;
414 } else if (!strcmp(act, "open")) {
415 fileno = get_fileno(td, fname);
416 file_action = FIO_LOG_OPEN_FILE;
417 } else if (!strcmp(act, "close")) {
418 fileno = get_fileno(td, fname);
419 file_action = FIO_LOG_CLOSE_FILE;
420 } else {
421 log_err("fio: bad iolog file action: %s\n",
422 act);
423 continue;
424 }
425 } else {
c5839011 426 log_err("bad iolog2: %s\n", p);
ac9b9101
JA
427 continue;
428 }
429
430 if (rw == DDIR_READ)
431 reads++;
432 else if (rw == DDIR_WRITE) {
433 /*
434 * Don't add a write for ro mode
435 */
436 if (read_only)
437 continue;
438 writes++;
439 } else if (rw == DDIR_WAIT) {
74dff443
JA
440 if (td->o.no_stall)
441 continue;
ac9b9101
JA
442 waits++;
443 } else if (rw == DDIR_INVAL) {
444 } else if (!ddir_sync(rw)) {
445 log_err("bad ddir: %d\n", rw);
446 continue;
447 }
448
449 /*
450 * Make note of file
451 */
452 ipo = malloc(sizeof(*ipo));
453 init_ipo(ipo);
454 ipo->ddir = rw;
455 if (rw == DDIR_WAIT) {
456 ipo->delay = offset;
457 } else {
a79f17bf
SW
458 if (td->o.replay_scale)
459 ipo->offset = offset / td->o.replay_scale;
460 else
461 ipo->offset = offset;
462 ipo_bytes_align(td->o.replay_align, ipo);
463
ac9b9101 464 ipo->len = bytes;
42793d94 465 if (rw != DDIR_INVAL && bytes > td->o.max_bs[rw])
ac9b9101
JA
466 td->o.max_bs[rw] = bytes;
467 ipo->fileno = fileno;
468 ipo->file_action = file_action;
70afff59 469 td->o.size += bytes;
ac9b9101 470 }
3c3ed070 471
ac9b9101
JA
472 queue_io_piece(td, ipo);
473 }
474
475 free(str);
476 free(act);
dac34079 477 free(rfname);
ac9b9101
JA
478
479 if (writes && read_only) {
480 log_err("fio: <%s> skips replay of %d writes due to"
481 " read-only\n", td->o.name, writes);
482 writes = 0;
483 }
484
485 if (!reads && !writes && !waits)
486 return 1;
487 else if (reads && !writes)
488 td->o.td_ddir = TD_DDIR_READ;
489 else if (!reads && writes)
490 td->o.td_ddir = TD_DDIR_WRITE;
491 else
492 td->o.td_ddir = TD_DDIR_RW;
493
494 return 0;
495}
496
497/*
498 * open iolog, check version, and call appropriate parser
499 */
500static int init_iolog_read(struct thread_data *td)
501{
502 char buffer[256], *p;
503 FILE *f;
504 int ret;
505
506 f = fopen(td->o.read_iolog_file, "r");
507 if (!f) {
508 perror("fopen read iolog");
509 return 1;
510 }
511
512 p = fgets(buffer, sizeof(buffer), f);
513 if (!p) {
514 td_verror(td, errno, "iolog read");
515 log_err("fio: unable to read iolog\n");
516 fclose(f);
517 return 1;
518 }
519
520 /*
521 * version 2 of the iolog stores a specific string as the
522 * first line, check for that
523 */
524 if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2)))
525 ret = read_iolog2(td, f);
526 else {
527 log_err("fio: iolog version 1 is no longer supported\n");
528 ret = 1;
529 }
530
531 fclose(f);
532 return ret;
533}
534
535/*
536 * Set up a log for storing io patterns.
537 */
538static int init_iolog_write(struct thread_data *td)
539{
540 struct fio_file *ff;
541 FILE *f;
542 unsigned int i;
543
544 f = fopen(td->o.write_iolog_file, "a");
545 if (!f) {
546 perror("fopen write iolog");
547 return 1;
548 }
549
550 /*
551 * That's it for writing, setup a log buffer and we're done.
552 */
553 td->iolog_f = f;
554 td->iolog_buf = malloc(8192);
555 setvbuf(f, td->iolog_buf, _IOFBF, 8192);
556
557 /*
558 * write our version line
559 */
560 if (fprintf(f, "%s\n", iolog_ver2) < 0) {
561 perror("iolog init\n");
562 return 1;
563 }
564
565 /*
566 * add all known files
567 */
568 for_each_file(td, ff, i)
569 log_file(td, ff, FIO_LOG_ADD_FILE);
570
571 return 0;
572}
573
574int init_iolog(struct thread_data *td)
575{
576 int ret = 0;
577
578 if (td->o.read_iolog_file) {
d95b34a6
JA
579 int need_swap;
580
ac9b9101
JA
581 /*
582 * Check if it's a blktrace file and load that if possible.
583 * Otherwise assume it's a normal log file and load that.
584 */
d95b34a6
JA
585 if (is_blktrace(td->o.read_iolog_file, &need_swap))
586 ret = load_blktrace(td, td->o.read_iolog_file, need_swap);
ac9b9101
JA
587 else
588 ret = init_iolog_read(td);
589 } else if (td->o.write_iolog_file)
590 ret = init_iolog_write(td);
591
f01b34ae
JA
592 if (ret)
593 td_verror(td, EINVAL, "failed initializing iolog");
594
ac9b9101
JA
595 return ret;
596}
597
aee2ab67
JA
598void setup_log(struct io_log **log, struct log_params *p,
599 const char *filename)
ac9b9101 600{
e75cc8f3 601 struct io_log *l;
65a4d15c
KC
602 int i;
603 struct io_u_plat_entry *entry;
604 struct flist_head *list;
ac9b9101 605
7e419452
JA
606 l = scalloc(1, sizeof(*l));
607 INIT_FLIST_HEAD(&l->io_logs);
aee2ab67
JA
608 l->log_type = p->log_type;
609 l->log_offset = p->log_offset;
610 l->log_gz = p->log_gz;
b26317c9 611 l->log_gz_store = p->log_gz_store;
aee2ab67 612 l->avg_msec = p->avg_msec;
1e613c9c
KC
613 l->hist_msec = p->hist_msec;
614 l->hist_coarseness = p->hist_coarseness;
cb7e0ace 615 l->filename = strdup(filename);
aee2ab67
JA
616 l->td = p->td;
617
65a4d15c
KC
618 /* Initialize histogram lists for each r/w direction,
619 * with initial io_u_plat of all zeros:
620 */
621 for (i = 0; i < DDIR_RWDIR_CNT; i++) {
d730bc58 622 list = &l->hist_window[i].list;
65a4d15c
KC
623 INIT_FLIST_HEAD(list);
624 entry = calloc(1, sizeof(struct io_u_plat_entry));
625 flist_add(&entry->list, list);
626 }
627
1fed2080
JA
628 if (l->td && l->td->o.io_submit_mode != IO_MODE_OFFLOAD) {
629 struct io_logs *p;
630
631 p = calloc(1, sizeof(*l->pending));
a9afa45a 632 p->max_samples = DEF_LOG_ENTRIES;
1fed2080
JA
633 p->log = calloc(p->max_samples, log_entry_sz(l));
634 l->pending = p;
635 }
636
b26317c9 637 if (l->log_offset)
49e98daa 638 l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
b26317c9 639
aee2ab67
JA
640 INIT_FLIST_HEAD(&l->chunk_list);
641
642 if (l->log_gz && !p->td)
643 l->log_gz = 0;
19417eda 644 else if (l->log_gz || l->log_gz_store) {
34febb23 645 mutex_init_pshared(&l->chunk_lock);
858bce70 646 mutex_init_pshared(&l->deferred_free_lock);
aee2ab67
JA
647 p->td->flags |= TD_F_COMPRESS_LOG;
648 }
649
ac9b9101
JA
650 *log = l;
651}
652
2e802282
JA
653#ifdef CONFIG_SETVBUF
654static void *set_file_buffer(FILE *f)
655{
656 size_t size = 1048576;
657 void *buf;
658
659 buf = malloc(size);
660 setvbuf(f, buf, _IOFBF, size);
661 return buf;
662}
663
664static void clear_file_buffer(void *buf)
665{
666 free(buf);
667}
668#else
669static void *set_file_buffer(FILE *f)
670{
671 return NULL;
672}
673
674static void clear_file_buffer(void *buf)
675{
676}
677#endif
678
518dac09 679void free_log(struct io_log *log)
cb7e0ace 680{
7e419452
JA
681 while (!flist_empty(&log->io_logs)) {
682 struct io_logs *cur_log;
683
684 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
685 flist_del_init(&cur_log->list);
686 free(cur_log->log);
2b7625e2 687 sfree(cur_log);
7e419452
JA
688 }
689
1fed2080
JA
690 if (log->pending) {
691 free(log->pending->log);
692 free(log->pending);
693 log->pending = NULL;
694 }
695
696 free(log->pending);
cb7e0ace 697 free(log->filename);
a47591e4 698 sfree(log);
cb7e0ace
JA
699}
700
0225849f 701unsigned long hist_sum(int j, int stride, unsigned int *io_u_plat,
65a4d15c 702 unsigned int *io_u_plat_last)
548df930 703{
f2c972cc
JA
704 unsigned long sum;
705 int k;
548df930 706
868f6f03
KC
707 if (io_u_plat_last) {
708 for (k = sum = 0; k < stride; k++)
709 sum += io_u_plat[j + k] - io_u_plat_last[j + k];
710 } else {
711 for (k = sum = 0; k < stride; k++)
712 sum += io_u_plat[j + k];
713 }
548df930 714
1e613c9c
KC
715 return sum;
716}
717
a89ba4b1
JA
718static void flush_hist_samples(FILE *f, int hist_coarseness, void *samples,
719 uint64_t sample_size)
1e613c9c
KC
720{
721 struct io_sample *s;
722 int log_offset;
723 uint64_t i, j, nr_samples;
65a4d15c 724 struct io_u_plat_entry *entry, *entry_before;
1e613c9c 725 unsigned int *io_u_plat;
65a4d15c 726 unsigned int *io_u_plat_before;
1e613c9c
KC
727
728 int stride = 1 << hist_coarseness;
729
730 if (!sample_size)
731 return;
732
733 s = __get_sample(samples, 0, 0);
734 log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
735
736 nr_samples = sample_size / __log_entry_sz(log_offset);
737
738 for (i = 0; i < nr_samples; i++) {
739 s = __get_sample(samples, log_offset, i);
d730bc58 740
af2fde19 741 entry = s->data.plat_entry;
65a4d15c 742 io_u_plat = entry->io_u_plat;
d730bc58 743
65a4d15c
KC
744 entry_before = flist_first_entry(&entry->list, struct io_u_plat_entry, list);
745 io_u_plat_before = entry_before->io_u_plat;
d730bc58
JA
746
747 fprintf(f, "%lu, %u, %u, ", (unsigned long) s->time,
748 io_sample_ddir(s), s->bs);
1e613c9c 749 for (j = 0; j < FIO_IO_U_PLAT_NR - stride; j += stride) {
d730bc58
JA
750 fprintf(f, "%lu, ", hist_sum(j, stride, io_u_plat,
751 io_u_plat_before));
1e613c9c 752 }
d730bc58 753 fprintf(f, "%lu\n", (unsigned long)
65a4d15c 754 hist_sum(FIO_IO_U_PLAT_NR - stride, stride, io_u_plat,
d730bc58
JA
755 io_u_plat_before));
756
65a4d15c
KC
757 flist_del(&entry_before->list);
758 free(entry_before);
1e613c9c
KC
759 }
760}
761
bead01d7 762void flush_samples(FILE *f, void *samples, uint64_t sample_size)
ac9b9101 763{
b26317c9
JA
764 struct io_sample *s;
765 int log_offset;
766 uint64_t i, nr_samples;
767
768 if (!sample_size)
769 return;
770
771 s = __get_sample(samples, 0, 0);
49e98daa 772 log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
b26317c9
JA
773
774 nr_samples = sample_size / __log_entry_sz(log_offset);
ac9b9101 775
aee2ab67 776 for (i = 0; i < nr_samples; i++) {
b26317c9 777 s = __get_sample(samples, log_offset, i);
ac9b9101 778
aee2ab67 779 if (!log_offset) {
af2fde19 780 fprintf(f, "%lu, %" PRId64 ", %u, %u\n",
ae588852 781 (unsigned long) s->time,
af2fde19 782 s->data.val,
b26317c9 783 io_sample_ddir(s), s->bs);
ae588852
JA
784 } else {
785 struct io_sample_offset *so = (void *) s;
786
af2fde19 787 fprintf(f, "%lu, %" PRId64 ", %u, %u, %llu\n",
ae588852 788 (unsigned long) s->time,
af2fde19 789 s->data.val,
b26317c9 790 io_sample_ddir(s), s->bs,
ae588852
JA
791 (unsigned long long) so->offset);
792 }
ac9b9101 793 }
aee2ab67
JA
794}
795
796#ifdef CONFIG_ZLIB
97eabb2a
JA
797
798struct iolog_flush_data {
155f2f02 799 struct workqueue_work work;
97eabb2a
JA
800 struct io_log *log;
801 void *samples;
7e419452
JA
802 uint32_t nr_samples;
803 bool free;
97eabb2a
JA
804};
805
97eabb2a
JA
806#define GZ_CHUNK 131072
807
808static struct iolog_compress *get_new_chunk(unsigned int seq)
809{
810 struct iolog_compress *c;
811
812 c = malloc(sizeof(*c));
813 INIT_FLIST_HEAD(&c->list);
814 c->buf = malloc(GZ_CHUNK);
815 c->len = 0;
816 c->seq = seq;
97eabb2a
JA
817 return c;
818}
819
820static void free_chunk(struct iolog_compress *ic)
821{
c07826cd
JA
822 free(ic->buf);
823 free(ic);
97eabb2a
JA
824}
825
b26317c9 826static int z_stream_init(z_stream *stream, int gz_hdr)
aee2ab67 827{
b26317c9
JA
828 int wbits = 15;
829
acb2a69b 830 memset(stream, 0, sizeof(*stream));
aee2ab67
JA
831 stream->zalloc = Z_NULL;
832 stream->zfree = Z_NULL;
833 stream->opaque = Z_NULL;
834 stream->next_in = Z_NULL;
835
1a8e7458
JA
836 /*
837 * zlib magic - add 32 for auto-detection of gz header or not,
838 * if we decide to store files in a gzip friendly format.
839 */
b26317c9
JA
840 if (gz_hdr)
841 wbits += 32;
842
843 if (inflateInit2(stream, wbits) != Z_OK)
aee2ab67
JA
844 return 1;
845
846 return 0;
847}
848
1a8e7458 849struct inflate_chunk_iter {
aee2ab67 850 unsigned int seq;
dad95da1 851 int err;
aee2ab67
JA
852 void *buf;
853 size_t buf_size;
854 size_t buf_used;
855 size_t chunk_sz;
856};
857
b26317c9 858static void finish_chunk(z_stream *stream, FILE *f,
1a8e7458 859 struct inflate_chunk_iter *iter)
aee2ab67 860{
aee2ab67
JA
861 int ret;
862
863 ret = inflateEnd(stream);
864 if (ret != Z_OK)
50f940f1
JA
865 log_err("fio: failed to end log inflation seq %d (%d)\n",
866 iter->seq, ret);
aee2ab67 867
b26317c9 868 flush_samples(f, iter->buf, iter->buf_used);
aee2ab67
JA
869 free(iter->buf);
870 iter->buf = NULL;
871 iter->buf_size = iter->buf_used = 0;
872}
873
1a8e7458
JA
874/*
875 * Iterative chunk inflation. Handles cases where we cross into a new
876 * sequence, doing flush finish of previous chunk if needed.
877 */
878static size_t inflate_chunk(struct iolog_compress *ic, int gz_hdr, FILE *f,
879 z_stream *stream, struct inflate_chunk_iter *iter)
aee2ab67 880{
0c56718d
JA
881 size_t ret;
882
0f989d2e 883 dprint(FD_COMPRESS, "inflate chunk size=%lu, seq=%u\n",
0c56718d
JA
884 (unsigned long) ic->len, ic->seq);
885
aee2ab67
JA
886 if (ic->seq != iter->seq) {
887 if (iter->seq)
b26317c9 888 finish_chunk(stream, f, iter);
aee2ab67 889
b26317c9 890 z_stream_init(stream, gz_hdr);
aee2ab67
JA
891 iter->seq = ic->seq;
892 }
893
894 stream->avail_in = ic->len;
895 stream->next_in = ic->buf;
896
897 if (!iter->buf_size) {
898 iter->buf_size = iter->chunk_sz;
899 iter->buf = malloc(iter->buf_size);
900 }
901
902 while (stream->avail_in) {
b26317c9 903 size_t this_out = iter->buf_size - iter->buf_used;
aee2ab67
JA
904 int err;
905
b26317c9 906 stream->avail_out = this_out;
aee2ab67
JA
907 stream->next_out = iter->buf + iter->buf_used;
908
909 err = inflate(stream, Z_NO_FLUSH);
910 if (err < 0) {
911 log_err("fio: failed inflating log: %d\n", err);
dad95da1 912 iter->err = err;
aee2ab67
JA
913 break;
914 }
915
b26317c9
JA
916 iter->buf_used += this_out - stream->avail_out;
917
918 if (!stream->avail_out) {
919 iter->buf_size += iter->chunk_sz;
920 iter->buf = realloc(iter->buf, iter->buf_size);
921 continue;
922 }
923
924 if (err == Z_STREAM_END)
925 break;
aee2ab67
JA
926 }
927
0c56718d
JA
928 ret = (void *) stream->next_in - ic->buf;
929
d0d0cf0a 930 dprint(FD_COMPRESS, "inflated to size=%lu\n", (unsigned long) iter->buf_size);
0c56718d
JA
931
932 return ret;
aee2ab67
JA
933}
934
1a8e7458
JA
935/*
936 * Inflate stored compressed chunks, or write them directly to the log
937 * file if so instructed.
938 */
dad95da1 939static int inflate_gz_chunks(struct io_log *log, FILE *f)
aee2ab67 940{
1a8e7458 941 struct inflate_chunk_iter iter = { .chunk_sz = log->log_gz, };
aee2ab67
JA
942 z_stream stream;
943
944 while (!flist_empty(&log->chunk_list)) {
945 struct iolog_compress *ic;
946
9342d5f8 947 ic = flist_first_entry(&log->chunk_list, struct iolog_compress, list);
aee2ab67 948 flist_del(&ic->list);
b26317c9 949
1a8e7458
JA
950 if (log->log_gz_store) {
951 size_t ret;
952
0c56718d
JA
953 dprint(FD_COMPRESS, "log write chunk size=%lu, "
954 "seq=%u\n", (unsigned long) ic->len, ic->seq);
955
1a8e7458 956 ret = fwrite(ic->buf, ic->len, 1, f);
dad95da1
JA
957 if (ret != 1 || ferror(f)) {
958 iter.err = errno;
1a8e7458 959 log_err("fio: error writing compressed log\n");
dad95da1 960 }
1a8e7458
JA
961 } else
962 inflate_chunk(ic, log->log_gz_store, f, &stream, &iter);
c07826cd
JA
963
964 free_chunk(ic);
aee2ab67
JA
965 }
966
967 if (iter.seq) {
b26317c9 968 finish_chunk(&stream, f, &iter);
aee2ab67
JA
969 free(iter.buf);
970 }
dad95da1
JA
971
972 return iter.err;
aee2ab67
JA
973}
974
1a8e7458
JA
975/*
976 * Open compressed log file and decompress the stored chunks and
977 * write them to stdout. The chunks are stored sequentially in the
978 * file, so we iterate over them and do them one-by-one.
979 */
b26317c9
JA
980int iolog_file_inflate(const char *file)
981{
1a8e7458 982 struct inflate_chunk_iter iter = { .chunk_sz = 64 * 1024 * 1024, };
b26317c9
JA
983 struct iolog_compress ic;
984 z_stream stream;
985 struct stat sb;
fd771971 986 ssize_t ret;
f302710c
JA
987 size_t total;
988 void *buf;
b26317c9
JA
989 FILE *f;
990
b26317c9
JA
991 f = fopen(file, "r");
992 if (!f) {
993 perror("fopen");
994 return 1;
995 }
996
fd771971
JA
997 if (stat(file, &sb) < 0) {
998 fclose(f);
999 perror("stat");
1000 return 1;
1001 }
1002
f302710c 1003 ic.buf = buf = malloc(sb.st_size);
b26317c9 1004 ic.len = sb.st_size;
b26317c9
JA
1005 ic.seq = 1;
1006
1007 ret = fread(ic.buf, ic.len, 1, f);
1008 if (ret < 0) {
1009 perror("fread");
1010 fclose(f);
bb1116fe 1011 free(buf);
b26317c9
JA
1012 return 1;
1013 } else if (ret != 1) {
1014 log_err("fio: short read on reading log\n");
1015 fclose(f);
bb1116fe 1016 free(buf);
b26317c9
JA
1017 return 1;
1018 }
1019
1020 fclose(f);
1021
1a8e7458
JA
1022 /*
1023 * Each chunk will return Z_STREAM_END. We don't know how many
1024 * chunks are in the file, so we just keep looping and incrementing
1025 * the sequence number until we have consumed the whole compressed
1026 * file.
1027 */
f302710c
JA
1028 total = ic.len;
1029 do {
8a1db9a1 1030 size_t iret;
f302710c 1031
8a1db9a1
JA
1032 iret = inflate_chunk(&ic, 1, stdout, &stream, &iter);
1033 total -= iret;
f302710c
JA
1034 if (!total)
1035 break;
dad95da1
JA
1036 if (iter.err)
1037 break;
f302710c
JA
1038
1039 ic.seq++;
8a1db9a1
JA
1040 ic.len -= iret;
1041 ic.buf += iret;
f302710c 1042 } while (1);
b26317c9
JA
1043
1044 if (iter.seq) {
1045 finish_chunk(&stream, stdout, &iter);
1046 free(iter.buf);
1047 }
1048
f302710c 1049 free(buf);
dad95da1 1050 return iter.err;
b26317c9
JA
1051}
1052
aee2ab67
JA
1053#else
1054
dad95da1 1055static int inflate_gz_chunks(struct io_log *log, FILE *f)
aee2ab67 1056{
b7364e21 1057 return 0;
aee2ab67
JA
1058}
1059
a68c66b2
JA
1060int iolog_file_inflate(const char *file)
1061{
1062 log_err("fio: log inflation not possible without zlib\n");
1063 return 1;
1064}
1065
aee2ab67
JA
1066#endif
1067
60a25727 1068void flush_log(struct io_log *log, bool do_append)
aee2ab67
JA
1069{
1070 void *buf;
1071 FILE *f;
1072
3a5db920
JA
1073 if (!do_append)
1074 f = fopen(log->filename, "w");
1075 else
1076 f = fopen(log->filename, "a");
aee2ab67
JA
1077 if (!f) {
1078 perror("fopen log");
1079 return;
1080 }
1081
1082 buf = set_file_buffer(f);
1083
1a8e7458 1084 inflate_gz_chunks(log, f);
aee2ab67 1085
7e419452
JA
1086 while (!flist_empty(&log->io_logs)) {
1087 struct io_logs *cur_log;
1088
1089 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
1090 flist_del_init(&cur_log->list);
1e613c9c 1091
a9179eeb 1092 if (log->td && log == log->td->clat_hist_log)
1e613c9c 1093 flush_hist_samples(f, log->hist_coarseness, cur_log->log,
868f6f03 1094 log_sample_sz(log, cur_log));
1e613c9c 1095 else
868f6f03 1096 flush_samples(f, cur_log->log, log_sample_sz(log, cur_log));
1e613c9c 1097
90d2d53f 1098 sfree(cur_log);
7e419452 1099 }
ac9b9101
JA
1100
1101 fclose(f);
2e802282 1102 clear_file_buffer(buf);
ac9b9101
JA
1103}
1104
cb7e0ace 1105static int finish_log(struct thread_data *td, struct io_log *log, int trylock)
ac9b9101 1106{
155f2f02 1107 if (td->flags & TD_F_COMPRESS_LOG)
7e419452 1108 iolog_flush(log);
aee2ab67 1109
243bfe19 1110 if (trylock) {
cb7e0ace 1111 if (fio_trylock_file(log->filename))
243bfe19
JA
1112 return 1;
1113 } else
cb7e0ace 1114 fio_lock_file(log->filename);
243bfe19 1115
0cba0f91 1116 if (td->client_type == FIO_CLIENT_TYPE_GUI || is_backend)
cb7e0ace 1117 fio_send_iolog(td, log, log->filename);
aee2ab67 1118 else
3a5db920 1119 flush_log(log, !td->o.per_job_logs);
243bfe19 1120
cb7e0ace 1121 fio_unlock_file(log->filename);
518dac09 1122 free_log(log);
243bfe19 1123 return 0;
ac9b9101
JA
1124}
1125
0cba0f91
JA
1126size_t log_chunk_sizes(struct io_log *log)
1127{
1128 struct flist_head *entry;
1129 size_t ret;
1130
1131 if (flist_empty(&log->chunk_list))
1132 return 0;
1133
1134 ret = 0;
1135 pthread_mutex_lock(&log->chunk_lock);
1136 flist_for_each(entry, &log->chunk_list) {
1137 struct iolog_compress *c;
1138
1139 c = flist_entry(entry, struct iolog_compress, list);
1140 ret += c->len;
1141 }
1142 pthread_mutex_unlock(&log->chunk_lock);
1143 return ret;
1144}
1145
aee2ab67
JA
1146#ifdef CONFIG_ZLIB
1147
858bce70
JA
1148static bool warned_on_drop;
1149
1150static void iolog_put_deferred(struct io_log *log, void *ptr)
1151{
1152 if (!ptr)
1153 return;
1154
1155 pthread_mutex_lock(&log->deferred_free_lock);
1156 if (log->deferred < IOLOG_MAX_DEFER) {
1157 log->deferred_items[log->deferred] = ptr;
1158 log->deferred++;
1159 } else if (!warned_on_drop) {
1160 log_err("fio: had to drop log entry free\n");
1161 warned_on_drop = true;
1162 }
1163 pthread_mutex_unlock(&log->deferred_free_lock);
1164}
1165
1166static void iolog_free_deferred(struct io_log *log)
1167{
1168 int i;
1169
1170 if (!log->deferred)
1171 return;
1172
1173 pthread_mutex_lock(&log->deferred_free_lock);
1174
1175 for (i = 0; i < log->deferred; i++) {
1176 free(log->deferred_items[i]);
1177 log->deferred_items[i] = NULL;
1178 }
1179
1180 log->deferred = 0;
1181 pthread_mutex_unlock(&log->deferred_free_lock);
1182}
1183
78e93aa6 1184static int gz_work(struct iolog_flush_data *data)
c143980a 1185{
040238ec 1186 struct iolog_compress *c = NULL;
aee2ab67
JA
1187 struct flist_head list;
1188 unsigned int seq;
1189 z_stream stream;
1190 size_t total = 0;
d73ac887 1191 int ret;
aee2ab67
JA
1192
1193 INIT_FLIST_HEAD(&list);
1194
7e419452 1195 memset(&stream, 0, sizeof(stream));
aee2ab67
JA
1196 stream.zalloc = Z_NULL;
1197 stream.zfree = Z_NULL;
1198 stream.opaque = Z_NULL;
1199
b26317c9 1200 ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
b26317c9 1201 if (ret != Z_OK) {
aee2ab67 1202 log_err("fio: failed to init gz stream\n");
152ea035 1203 goto err;
aee2ab67
JA
1204 }
1205
1206 seq = ++data->log->chunk_seq;
b26317c9 1207
aee2ab67
JA
1208 stream.next_in = (void *) data->samples;
1209 stream.avail_in = data->nr_samples * log_entry_sz(data->log);
1210
d0d0cf0a
JA
1211 dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u, log=%s\n",
1212 (unsigned long) stream.avail_in, seq,
1213 data->log->filename);
aee2ab67 1214 do {
040238ec 1215 if (c)
0caccfa7
JA
1216 dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq,
1217 (unsigned long) c->len);
aee2ab67
JA
1218 c = get_new_chunk(seq);
1219 stream.avail_out = GZ_CHUNK;
1220 stream.next_out = c->buf;
1221 ret = deflate(&stream, Z_NO_FLUSH);
1222 if (ret < 0) {
1223 log_err("fio: deflate log (%d)\n", ret);
dad95da1
JA
1224 free_chunk(c);
1225 goto err;
aee2ab67
JA
1226 }
1227
1228 c->len = GZ_CHUNK - stream.avail_out;
1229 flist_add_tail(&c->list, &list);
1230 total += c->len;
1231 } while (stream.avail_in);
1232
1233 stream.next_out = c->buf + c->len;
1234 stream.avail_out = GZ_CHUNK - c->len;
1235
1236 ret = deflate(&stream, Z_FINISH);
6fa3ad51
JA
1237 if (ret < 0) {
1238 /*
1239 * Z_BUF_ERROR is special, it just means we need more
1240 * output space. We'll handle that below. Treat any other
1241 * error as fatal.
1242 */
1243 if (ret != Z_BUF_ERROR) {
1244 log_err("fio: deflate log (%d)\n", ret);
1245 flist_del(&c->list);
1246 free_chunk(c);
1247 goto err;
1248 }
1249 }
1250
1251 total -= c->len;
1252 c->len = GZ_CHUNK - stream.avail_out;
1253 total += c->len;
0caccfa7 1254 dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq, (unsigned long) c->len);
040238ec 1255
6fa3ad51 1256 if (ret != Z_STREAM_END) {
aee2ab67
JA
1257 do {
1258 c = get_new_chunk(seq);
1259 stream.avail_out = GZ_CHUNK;
1260 stream.next_out = c->buf;
1261 ret = deflate(&stream, Z_FINISH);
1262 c->len = GZ_CHUNK - stream.avail_out;
0c56718d 1263 total += c->len;
aee2ab67 1264 flist_add_tail(&c->list, &list);
0caccfa7
JA
1265 dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq,
1266 (unsigned long) c->len);
aee2ab67
JA
1267 } while (ret != Z_STREAM_END);
1268 }
1269
0c56718d
JA
1270 dprint(FD_COMPRESS, "deflated to size=%lu\n", (unsigned long) total);
1271
aee2ab67
JA
1272 ret = deflateEnd(&stream);
1273 if (ret != Z_OK)
1274 log_err("fio: deflateEnd %d\n", ret);
1275
858bce70 1276 iolog_put_deferred(data->log, data->samples);
aee2ab67
JA
1277
1278 if (!flist_empty(&list)) {
1279 pthread_mutex_lock(&data->log->chunk_lock);
1280 flist_splice_tail(&list, &data->log->chunk_list);
1281 pthread_mutex_unlock(&data->log->chunk_lock);
1282 }
1283
dad95da1
JA
1284 ret = 0;
1285done:
7e419452 1286 if (data->free)
dab41b1c 1287 sfree(data);
dad95da1
JA
1288 return ret;
1289err:
1290 while (!flist_empty(&list)) {
1291 c = flist_first_entry(list.next, struct iolog_compress, list);
1292 flist_del(&c->list);
1293 free_chunk(c);
1294 }
1295 ret = 1;
1296 goto done;
aee2ab67
JA
1297}
1298
78e93aa6
JA
1299/*
1300 * Invoked from our compress helper thread, when logging would have exceeded
1301 * the specified memory limitation. Compresses the previously stored
1302 * entries.
1303 */
1304static int gz_work_async(struct submit_worker *sw, struct workqueue_work *work)
1305{
1306 return gz_work(container_of(work, struct iolog_flush_data, work));
1307}
1308
c08f9fe2
JA
1309static int gz_init_worker(struct submit_worker *sw)
1310{
1311 struct thread_data *td = sw->wq->td;
1312
1313 if (!fio_option_is_set(&td->o, log_gz_cpumask))
1314 return 0;
1315
1316 if (fio_setaffinity(gettid(), td->o.log_gz_cpumask) == -1) {
1317 log_err("gz: failed to set CPU affinity\n");
1318 return 1;
1319 }
1320
1321 return 0;
1322}
1323
155f2f02 1324static struct workqueue_ops log_compress_wq_ops = {
78e93aa6 1325 .fn = gz_work_async,
c08f9fe2 1326 .init_worker_fn = gz_init_worker,
24660963 1327 .nice = 1,
155f2f02
JA
1328};
1329
24660963 1330int iolog_compress_init(struct thread_data *td, struct sk_out *sk_out)
155f2f02
JA
1331{
1332 if (!(td->flags & TD_F_COMPRESS_LOG))
1333 return 0;
1334
24660963 1335 workqueue_init(td, &td->log_compress_wq, &log_compress_wq_ops, 1, sk_out);
155f2f02
JA
1336 return 0;
1337}
1338
1339void iolog_compress_exit(struct thread_data *td)
1340{
1341 if (!(td->flags & TD_F_COMPRESS_LOG))
1342 return;
1343
1344 workqueue_exit(&td->log_compress_wq);
1345}
1346
1a8e7458 1347/*
949ae6dc
JA
1348 * Queue work item to compress the existing log entries. We reset the
1349 * current log to a small size, and reference the existing log in the
1350 * data that we queue for compression. Once compression has been done,
7e419452
JA
1351 * this old log is freed. If called with finish == true, will not return
1352 * until the log compression has completed, and will flush all previous
1353 * logs too
1a8e7458 1354 */
7e419452 1355static int iolog_flush(struct io_log *log)
aee2ab67 1356{
aee2ab67 1357 struct iolog_flush_data *data;
aee2ab67
JA
1358
1359 data = malloc(sizeof(*data));
1360 if (!data)
1361 return 1;
1362
1363 data->log = log;
7e419452 1364 data->free = false;
aee2ab67 1365
7e419452
JA
1366 while (!flist_empty(&log->io_logs)) {
1367 struct io_logs *cur_log;
949ae6dc 1368
7e419452
JA
1369 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
1370 flist_del_init(&cur_log->list);
1371
1372 data->samples = cur_log->log;
1373 data->nr_samples = cur_log->nr_samples;
1374
2b7625e2 1375 sfree(cur_log);
aee2ab67 1376
78e93aa6 1377 gz_work(data);
7e419452 1378 }
aee2ab67 1379
7e419452 1380 free(data);
aee2ab67
JA
1381 return 0;
1382}
1383
7e419452
JA
1384int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log)
1385{
1386 struct iolog_flush_data *data;
1387
dab41b1c 1388 data = smalloc(sizeof(*data));
7e419452
JA
1389 if (!data)
1390 return 1;
1391
1392 data->log = log;
1393
1394 data->samples = cur_log->log;
1395 data->nr_samples = cur_log->nr_samples;
1396 data->free = true;
1397
1398 cur_log->nr_samples = cur_log->max_samples = 0;
1399 cur_log->log = NULL;
1400
1401 workqueue_enqueue(&log->td->log_compress_wq, &data->work);
858bce70
JA
1402
1403 iolog_free_deferred(log);
1404
7e419452
JA
1405 return 0;
1406}
aee2ab67
JA
1407#else
1408
7e419452
JA
1409static int iolog_flush(struct io_log *log)
1410{
1411 return 1;
1412}
1413
1414int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log)
aee2ab67
JA
1415{
1416 return 1;
1417}
1418
24660963 1419int iolog_compress_init(struct thread_data *td, struct sk_out *sk_out)
52618a2a
JA
1420{
1421 return 0;
1422}
1423
1424void iolog_compress_exit(struct thread_data *td)
1425{
1426}
1427
aee2ab67
JA
1428#endif
1429
7e419452
JA
1430struct io_logs *iolog_cur_log(struct io_log *log)
1431{
1432 if (flist_empty(&log->io_logs))
1433 return NULL;
1434
1435 return flist_last_entry(&log->io_logs, struct io_logs, list);
1436}
1437
1438uint64_t iolog_nr_samples(struct io_log *iolog)
1439{
1440 struct flist_head *entry;
1441 uint64_t ret = 0;
1442
1443 flist_for_each(entry, &iolog->io_logs) {
1444 struct io_logs *cur_log;
1445
1446 cur_log = flist_entry(entry, struct io_logs, list);
1447 ret += cur_log->nr_samples;
1448 }
1449
1450 return ret;
1451}
1452
1c725e70 1453static int __write_log(struct thread_data *td, struct io_log *log, int try)
905e3d4f 1454{
1c725e70
JA
1455 if (log)
1456 return finish_log(td, log, try);
905e3d4f 1457
1c725e70
JA
1458 return 0;
1459}
905e3d4f 1460
a47591e4 1461static int write_iops_log(struct thread_data *td, int try, bool unit_log)
1c725e70 1462{
a47591e4
JA
1463 int ret;
1464
1465 if (per_unit_log(td->iops_log) != unit_log)
1466 return 0;
1467
1468 ret = __write_log(td, td->iops_log, try);
1469 if (!ret)
1470 td->iops_log = NULL;
1471
1472 return ret;
905e3d4f
JA
1473}
1474
a47591e4 1475static int write_slat_log(struct thread_data *td, int try, bool unit_log)
905e3d4f 1476{
a47591e4
JA
1477 int ret;
1478
1479 if (!unit_log)
1480 return 0;
1481
1482 ret = __write_log(td, td->slat_log, try);
1483 if (!ret)
1484 td->slat_log = NULL;
1485
1486 return ret;
905e3d4f
JA
1487}
1488
a47591e4 1489static int write_clat_log(struct thread_data *td, int try, bool unit_log)
905e3d4f 1490{
a47591e4
JA
1491 int ret;
1492
1493 if (!unit_log)
1494 return 0;
1495
1496 ret = __write_log(td, td->clat_log, try);
1497 if (!ret)
1498 td->clat_log = NULL;
1499
1500 return ret;
905e3d4f
JA
1501}
1502
1e613c9c
KC
1503static int write_clat_hist_log(struct thread_data *td, int try, bool unit_log)
1504{
1505 int ret;
1506
1507 if (!unit_log)
1508 return 0;
1509
1510 ret = __write_log(td, td->clat_hist_log, try);
1511 if (!ret)
1512 td->clat_hist_log = NULL;
1513
1514 return ret;
1515}
1516
a47591e4 1517static int write_lat_log(struct thread_data *td, int try, bool unit_log)
905e3d4f 1518{
a47591e4
JA
1519 int ret;
1520
1521 if (!unit_log)
1522 return 0;
1523
1524 ret = __write_log(td, td->lat_log, try);
1525 if (!ret)
1526 td->lat_log = NULL;
1527
1528 return ret;
905e3d4f
JA
1529}
1530
a47591e4 1531static int write_bandw_log(struct thread_data *td, int try, bool unit_log)
905e3d4f 1532{
a47591e4
JA
1533 int ret;
1534
1535 if (per_unit_log(td->bw_log) != unit_log)
1536 return 0;
1537
1538 ret = __write_log(td, td->bw_log, try);
1539 if (!ret)
1540 td->bw_log = NULL;
1541
1542 return ret;
905e3d4f
JA
1543}
1544
1545enum {
1546 BW_LOG_MASK = 1,
1547 LAT_LOG_MASK = 2,
1548 SLAT_LOG_MASK = 4,
1549 CLAT_LOG_MASK = 8,
1550 IOPS_LOG_MASK = 16,
1e613c9c 1551 CLAT_HIST_LOG_MASK = 32,
905e3d4f 1552
1e613c9c 1553 ALL_LOG_NR = 6,
905e3d4f
JA
1554};
1555
1556struct log_type {
1557 unsigned int mask;
a47591e4 1558 int (*fn)(struct thread_data *, int, bool);
905e3d4f
JA
1559};
1560
1561static struct log_type log_types[] = {
1562 {
1563 .mask = BW_LOG_MASK,
1564 .fn = write_bandw_log,
1565 },
1566 {
1567 .mask = LAT_LOG_MASK,
1568 .fn = write_lat_log,
1569 },
1570 {
1571 .mask = SLAT_LOG_MASK,
1572 .fn = write_slat_log,
1573 },
1574 {
1575 .mask = CLAT_LOG_MASK,
1576 .fn = write_clat_log,
1577 },
1578 {
1579 .mask = IOPS_LOG_MASK,
1580 .fn = write_iops_log,
1581 },
1e613c9c
KC
1582 {
1583 .mask = CLAT_HIST_LOG_MASK,
1584 .fn = write_clat_hist_log,
1585 }
905e3d4f
JA
1586};
1587
a47591e4 1588void td_writeout_logs(struct thread_data *td, bool unit_logs)
905e3d4f 1589{
ea5409f9 1590 unsigned int log_mask = 0;
905e3d4f
JA
1591 unsigned int log_left = ALL_LOG_NR;
1592 int old_state, i;
1593
1594 old_state = td_bump_runstate(td, TD_FINISHING);
1595
a47591e4 1596 finalize_logs(td, unit_logs);
905e3d4f
JA
1597
1598 while (log_left) {
1599 int prev_log_left = log_left;
1600
1601 for (i = 0; i < ALL_LOG_NR && log_left; i++) {
1602 struct log_type *lt = &log_types[i];
1603 int ret;
1604
ea5409f9 1605 if (!(log_mask & lt->mask)) {
a47591e4 1606 ret = lt->fn(td, log_left != 1, unit_logs);
905e3d4f
JA
1607 if (!ret) {
1608 log_left--;
ea5409f9 1609 log_mask |= lt->mask;
905e3d4f
JA
1610 }
1611 }
1612 }
1613
1614 if (prev_log_left == log_left)
1615 usleep(5000);
1616 }
1617
1618 td_restore_runstate(td, old_state);
1619}
a47591e4
JA
1620
1621void fio_writeout_logs(bool unit_logs)
1622{
1623 struct thread_data *td;
1624 int i;
1625
1626 for_each_td(td, i)
1627 td_writeout_logs(td, unit_logs);
1628}