projects / fio.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
iolog: add iolog_write for version 3
[fio.git] / iolog.c
... / ...
CommitLineData
1/*
2 * Code related to writing an iolog of what a thread is doing, and to
3 * later read that back and replay
4 */
5#include <stdio.h>
6#include <stdlib.h>
7#include <assert.h>
8#include <sys/types.h>
9#include <sys/stat.h>
10#include <unistd.h>
11#ifdef CONFIG_ZLIB
12#include <zlib.h>
13#endif
14
15#include "flist.h"
16#include "fio.h"
17#include "trim.h"
18#include "filelock.h"
19#include "smalloc.h"
20#include "blktrace.h"
21#include "pshared.h"
22#include "lib/roundup.h"
23
24#include <netinet/in.h>
25#include <netinet/tcp.h>
26#include <arpa/inet.h>
27#include <sys/stat.h>
28#include <sys/socket.h>
29#include <sys/un.h>
30
31static int iolog_flush(struct io_log *log);
32
33static const char iolog_ver2[] = "fio version 2 iolog";
34static const char iolog_ver3[] = "fio version 3 iolog";
35
36void queue_io_piece(struct thread_data *td, struct io_piece *ipo)
37{
38 flist_add_tail(&ipo->list, &td->io_log_list);
39 td->total_io_size += ipo->len;
40}
41
42void log_io_u(const struct thread_data *td, const struct io_u *io_u)
43{
44 struct timespec now;
45
46 if (!td->o.write_iolog_file)
47 return;
48
49 fio_gettime(&now, NULL);
50 fprintf(td->iolog_f, "%lu %s %s %llu %llu\n", utime_since_now(&td->io_log_start_time),
51 io_u->file->file_name,
52 io_ddir_name(io_u->ddir),
53 io_u->offset, io_u->buflen);
54
55}
56
57void log_file(struct thread_data *td, struct fio_file *f,
58 enum file_log_act what)
59{
60 const char *act[] = { "add", "open", "close" };
61 struct timespec now;
62
63 assert(what < 3);
64
65 if (!td->o.write_iolog_file)
66 return;
67
68
69 /*
70 * this happens on the pre-open/close done before the job starts
71 */
72 if (!td->iolog_f)
73 return;
74
75 fio_gettime(&now, NULL);
76 fprintf(td->iolog_f, "%lu %s %s\n", utime_since_now(&td->io_log_start_time),
77 f->file_name, act[what]);
78}
79
80static void iolog_delay(struct thread_data *td, unsigned long delay)
81{
82 uint64_t usec = utime_since_now(&td->last_issue);
83 unsigned long orig_delay = delay;
84 uint64_t this_delay;
85 struct timespec ts;
86
87 if (delay < td->time_offset) {
88 td->time_offset = 0;
89 return;
90 }
91
92 delay -= td->time_offset;
93 if (delay < usec)
94 return;
95
96 delay -= usec;
97
98 fio_gettime(&ts, NULL);
99 while (delay && !td->terminate) {
100 this_delay = delay;
101 if (this_delay > 500000)
102 this_delay = 500000;
103
104 usec_sleep(td, this_delay);
105 delay -= this_delay;
106 }
107
108 usec = utime_since_now(&ts);
109 if (usec > orig_delay)
110 td->time_offset = usec - orig_delay;
111 else
112 td->time_offset = 0;
113}
114
115static int ipo_special(struct thread_data *td, struct io_piece *ipo)
116{
117 struct fio_file *f;
118 int ret;
119
120 /*
121 * Not a special ipo
122 */
123 if (ipo->ddir != DDIR_INVAL)
124 return 0;
125
126 f = td->files[ipo->fileno];
127
128 if (ipo->delay)
129 iolog_delay(td, ipo->delay);
130 if (fio_fill_issue_time(td))
131 fio_gettime(&td->last_issue, NULL);
132 switch (ipo->file_action) {
133 case FIO_LOG_OPEN_FILE:
134 if (td->o.replay_redirect && fio_file_open(f)) {
135 dprint(FD_FILE, "iolog: ignoring re-open of file %s\n",
136 f->file_name);
137 break;
138 }
139 ret = td_io_open_file(td, f);
140 if (!ret)
141 break;
142 td_verror(td, ret, "iolog open file");
143 return -1;
144 case FIO_LOG_CLOSE_FILE:
145 td_io_close_file(td, f);
146 break;
147 case FIO_LOG_UNLINK_FILE:
148 td_io_unlink_file(td, f);
149 break;
150 case FIO_LOG_ADD_FILE:
151 /*
152 * Nothing to do
153 */
154 break;
155 default:
156 log_err("fio: bad file action %d\n", ipo->file_action);
157 break;
158 }
159
160 return 1;
161}
162
163static bool read_iolog(struct thread_data *td);
164
165unsigned long long delay_since_ttime(const struct thread_data *td,
166 unsigned long long time)
167{
168 double tmp;
169 double scale;
170 const unsigned long long *last_ttime = &td->io_log_last_ttime;
171
172 if (!*last_ttime || td->o.no_stall || time < *last_ttime)
173 return 0;
174 else if (td->o.replay_time_scale == 100)
175 return time - *last_ttime;
176
177
178 scale = (double) 100.0 / (double) td->o.replay_time_scale;
179 tmp = time - *last_ttime;
180 return tmp * scale;
181}
182
183int read_iolog_get(struct thread_data *td, struct io_u *io_u)
184{
185 struct io_piece *ipo;
186 unsigned long elapsed;
187
188 while (!flist_empty(&td->io_log_list)) {
189 int ret;
190
191 if (td->o.read_iolog_chunked) {
192 if (td->io_log_checkmark == td->io_log_current) {
193 if (td->io_log_blktrace) {
194 if (!read_blktrace(td))
195 return 1;
196 } else {
197 if (!read_iolog(td))
198 return 1;
199 }
200 }
201 td->io_log_current--;
202 }
203 ipo = flist_first_entry(&td->io_log_list, struct io_piece, list);
204 flist_del(&ipo->list);
205 remove_trim_entry(td, ipo);
206
207 ret = ipo_special(td, ipo);
208 if (ret < 0) {
209 free(ipo);
210 break;
211 } else if (ret > 0) {
212 free(ipo);
213 continue;
214 }
215
216 io_u->ddir = ipo->ddir;
217 if (ipo->ddir != DDIR_WAIT) {
218 io_u->offset = ipo->offset;
219 io_u->verify_offset = ipo->offset;
220 io_u->buflen = ipo->len;
221 io_u->file = td->files[ipo->fileno];
222 get_file(io_u->file);
223 dprint(FD_IO, "iolog: get %llu/%llu/%s\n", io_u->offset,
224 io_u->buflen, io_u->file->file_name);
225 if (ipo->delay)
226 iolog_delay(td, ipo->delay);
227 } else {
228 elapsed = mtime_since_genesis();
229 if (ipo->delay > elapsed)
230 usec_sleep(td, (ipo->delay - elapsed) * 1000);
231 }
232
233 free(ipo);
234
235 if (io_u->ddir != DDIR_WAIT)
236 return 0;
237 }
238
239 td->done = 1;
240 return 1;
241}
242
243void prune_io_piece_log(struct thread_data *td)
244{
245 struct io_piece *ipo;
246 struct fio_rb_node *n;
247
248 while ((n = rb_first(&td->io_hist_tree)) != NULL) {
249 ipo = rb_entry(n, struct io_piece, rb_node);
250 rb_erase(n, &td->io_hist_tree);
251 remove_trim_entry(td, ipo);
252 td->io_hist_len--;
253 free(ipo);
254 }
255
256 while (!flist_empty(&td->io_hist_list)) {
257 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
258 flist_del(&ipo->list);
259 remove_trim_entry(td, ipo);
260 td->io_hist_len--;
261 free(ipo);
262 }
263}
264
265/*
266 * log a successful write, so we can unwind the log for verify
267 */
268void log_io_piece(struct thread_data *td, struct io_u *io_u)
269{
270 struct fio_rb_node **p, *parent;
271 struct io_piece *ipo, *__ipo;
272
273 ipo = calloc(1, sizeof(struct io_piece));
274 init_ipo(ipo);
275 ipo->file = io_u->file;
276 ipo->offset = io_u->offset;
277 ipo->len = io_u->buflen;
278 ipo->numberio = io_u->numberio;
279 ipo->flags = IP_F_IN_FLIGHT;
280
281 io_u->ipo = ipo;
282
283 if (io_u_should_trim(td, io_u)) {
284 flist_add_tail(&ipo->trim_list, &td->trim_list);
285 td->trim_entries++;
286 }
287
288 /*
289 * Only sort writes if we don't have a random map in which case we need
290 * to check for duplicate blocks and drop the old one, which we rely on
291 * the rb insert/lookup for handling.
292 */
293 if (file_randommap(td, ipo->file)) {
294 INIT_FLIST_HEAD(&ipo->list);
295 flist_add_tail(&ipo->list, &td->io_hist_list);
296 ipo->flags |= IP_F_ONLIST;
297 td->io_hist_len++;
298 return;
299 }
300
301 RB_CLEAR_NODE(&ipo->rb_node);
302
303 /*
304 * Sort the entry into the verification list
305 */
306restart:
307 p = &td->io_hist_tree.rb_node;
308 parent = NULL;
309 while (*p) {
310 int overlap = 0;
311 parent = *p;
312
313 __ipo = rb_entry(parent, struct io_piece, rb_node);
314 if (ipo->file < __ipo->file)
315 p = &(*p)->rb_left;
316 else if (ipo->file > __ipo->file)
317 p = &(*p)->rb_right;
318 else if (ipo->offset < __ipo->offset) {
319 p = &(*p)->rb_left;
320 overlap = ipo->offset + ipo->len > __ipo->offset;
321 }
322 else if (ipo->offset > __ipo->offset) {
323 p = &(*p)->rb_right;
324 overlap = __ipo->offset + __ipo->len > ipo->offset;
325 }
326 else
327 overlap = 1;
328
329 if (overlap) {
330 dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu\n",
331 __ipo->offset, __ipo->len,
332 ipo->offset, ipo->len);
333 td->io_hist_len--;
334 rb_erase(parent, &td->io_hist_tree);
335 remove_trim_entry(td, __ipo);
336 if (!(__ipo->flags & IP_F_IN_FLIGHT))
337 free(__ipo);
338 goto restart;
339 }
340 }
341
342 rb_link_node(&ipo->rb_node, parent, p);
343 rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
344 ipo->flags |= IP_F_ONRB;
345 td->io_hist_len++;
346}
347
348void unlog_io_piece(struct thread_data *td, struct io_u *io_u)
349{
350 struct io_piece *ipo = io_u->ipo;
351
352 if (td->ts.nr_block_infos) {
353 uint32_t *info = io_u_block_info(td, io_u);
354 if (BLOCK_INFO_STATE(*info) < BLOCK_STATE_TRIM_FAILURE) {
355 if (io_u->ddir == DDIR_TRIM)
356 *info = BLOCK_INFO_SET_STATE(*info,
357 BLOCK_STATE_TRIM_FAILURE);
358 else if (io_u->ddir == DDIR_WRITE)
359 *info = BLOCK_INFO_SET_STATE(*info,
360 BLOCK_STATE_WRITE_FAILURE);
361 }
362 }
363
364 if (!ipo)
365 return;
366
367 if (ipo->flags & IP_F_ONRB)
368 rb_erase(&ipo->rb_node, &td->io_hist_tree);
369 else if (ipo->flags & IP_F_ONLIST)
370 flist_del(&ipo->list);
371
372 free(ipo);
373 io_u->ipo = NULL;
374 td->io_hist_len--;
375}
376
377void trim_io_piece(const struct io_u *io_u)
378{
379 struct io_piece *ipo = io_u->ipo;
380
381 if (!ipo)
382 return;
383
384 ipo->len = io_u->xfer_buflen - io_u->resid;
385}
386
387void write_iolog_close(struct thread_data *td)
388{
389 if (!td->iolog_f)
390 return;
391
392 fflush(td->iolog_f);
393 fclose(td->iolog_f);
394 free(td->iolog_buf);
395 td->iolog_f = NULL;
396 td->iolog_buf = NULL;
397}
398
399int64_t iolog_items_to_fetch(struct thread_data *td)
400{
401 struct timespec now;
402 uint64_t elapsed;
403 uint64_t for_1s;
404 int64_t items_to_fetch;
405
406 if (!td->io_log_highmark)
407 return 10;
408
409
410 fio_gettime(&now, NULL);
411 elapsed = ntime_since(&td->io_log_highmark_time, &now);
412 if (elapsed) {
413 for_1s = (td->io_log_highmark - td->io_log_current) * 1000000000 / elapsed;
414 items_to_fetch = for_1s - td->io_log_current;
415 if (items_to_fetch < 0)
416 items_to_fetch = 0;
417 } else
418 items_to_fetch = 0;
419
420 td->io_log_highmark = td->io_log_current + items_to_fetch;
421 td->io_log_checkmark = (td->io_log_highmark + 1) / 2;
422 fio_gettime(&td->io_log_highmark_time, NULL);
423
424 return items_to_fetch;
425}
426
427#define io_act(_td, _r) (((_td)->io_log_version == 3 && (r) == 5) || \
428 ((_td)->io_log_version == 2 && (r) == 4))
429#define file_act(_td, _r) (((_td)->io_log_version == 3 && (r) == 3) || \
430 ((_td)->io_log_version == 2 && (r) == 2))
431
432/*
433 * Read version 2 and 3 iolog data. It is enhanced to include per-file logging,
434 * syncs, etc.
435 */
436static bool read_iolog(struct thread_data *td)
437{
438 unsigned long long offset;
439 unsigned int bytes;
440 unsigned long long delay = 0;
441 int reads, writes, waits, fileno = 0, file_action = 0; /* stupid gcc */
442 char *rfname, *fname, *act;
443 char *str, *p;
444 enum fio_ddir rw;
445 bool realloc = false;
446 int64_t items_to_fetch = 0;
447 int syncs;
448
449 if (td->o.read_iolog_chunked) {
450 items_to_fetch = iolog_items_to_fetch(td);
451 if (!items_to_fetch)
452 return true;
453 }
454
455 /*
456 * Read in the read iolog and store it, reuse the infrastructure
457 * for doing verifications.
458 */
459 str = malloc(4096);
460 rfname = fname = malloc(256+16);
461 act = malloc(256+16);
462
463 syncs = reads = writes = waits = 0;
464 while ((p = fgets(str, 4096, td->io_log_rfile)) != NULL) {
465 struct io_piece *ipo;
466 int r;
467 unsigned long long ttime;
468
469 if (td->io_log_version == 3) {
470 r = sscanf(p, "%llu %256s %256s %llu %u", &ttime, rfname, act,
471 &offset, &bytes);
472 delay = delay_since_ttime(td, ttime);
473 td->io_log_last_ttime = ttime;
474 /*
475 * "wait" is not allowed with version 3
476 */
477 if (!strcmp(act, "wait")) {
478 log_err("iolog: ignoring wait command with"
479 " version 3 for file %s\n", fname);
480 continue;
481 }
482 } else /* version 2 */
483 r = sscanf(p, "%256s %256s %llu %u", rfname, act, &offset, &bytes);
484
485 if (td->o.replay_redirect)
486 fname = td->o.replay_redirect;
487
488 if (io_act(td, r)) {
489 /*
490 * Check action first
491 */
492 if (!strcmp(act, "wait"))
493 rw = DDIR_WAIT;
494 else if (!strcmp(act, "read"))
495 rw = DDIR_READ;
496 else if (!strcmp(act, "write"))
497 rw = DDIR_WRITE;
498 else if (!strcmp(act, "sync"))
499 rw = DDIR_SYNC;
500 else if (!strcmp(act, "datasync"))
501 rw = DDIR_DATASYNC;
502 else if (!strcmp(act, "trim"))
503 rw = DDIR_TRIM;
504 else {
505 log_err("fio: bad iolog file action: %s\n",
506 act);
507 continue;
508 }
509 fileno = get_fileno(td, fname);
510 } else if (file_act(td, r)) {
511 rw = DDIR_INVAL;
512 if (!strcmp(act, "add")) {
513 if (td->o.replay_redirect &&
514 get_fileno(td, fname) != -1) {
515 dprint(FD_FILE, "iolog: ignoring"
516 " re-add of file %s\n", fname);
517 } else {
518 fileno = add_file(td, fname, td->subjob_number, 1);
519 file_action = FIO_LOG_ADD_FILE;
520 }
521 } else if (!strcmp(act, "open")) {
522 fileno = get_fileno(td, fname);
523 file_action = FIO_LOG_OPEN_FILE;
524 } else if (!strcmp(act, "close")) {
525 fileno = get_fileno(td, fname);
526 file_action = FIO_LOG_CLOSE_FILE;
527 } else {
528 log_err("fio: bad iolog file action: %s\n",
529 act);
530 continue;
531 }
532 } else {
533 log_err("bad iolog%d: %s\n", td->io_log_version, p);
534 continue;
535 }
536
537 if (rw == DDIR_READ)
538 reads++;
539 else if (rw == DDIR_WRITE) {
540 /*
541 * Don't add a write for ro mode
542 */
543 if (read_only)
544 continue;
545 writes++;
546 } else if (rw == DDIR_WAIT) {
547 if (td->o.no_stall)
548 continue;
549 waits++;
550 } else if (rw == DDIR_INVAL) {
551 } else if (ddir_sync(rw)) {
552 syncs++;
553 } else {
554 log_err("bad ddir: %d\n", rw);
555 continue;
556 }
557
558 /*
559 * Make note of file
560 */
561 ipo = calloc(1, sizeof(*ipo));
562 init_ipo(ipo);
563 ipo->ddir = rw;
564 if (td->io_log_version == 3)
565 ipo->delay = delay;
566 if (rw == DDIR_WAIT) {
567 ipo->delay = offset;
568 } else {
569 if (td->o.replay_scale)
570 ipo->offset = offset / td->o.replay_scale;
571 else
572 ipo->offset = offset;
573 ipo_bytes_align(td->o.replay_align, ipo);
574
575 ipo->len = bytes;
576 if (rw != DDIR_INVAL && bytes > td->o.max_bs[rw]) {
577 realloc = true;
578 td->o.max_bs[rw] = bytes;
579 }
580 ipo->fileno = fileno;
581 ipo->file_action = file_action;
582 td->o.size += bytes;
583 }
584
585 queue_io_piece(td, ipo);
586
587 if (td->o.read_iolog_chunked) {
588 td->io_log_current++;
589 items_to_fetch--;
590 if (items_to_fetch == 0)
591 break;
592 }
593 }
594
595 free(str);
596 free(act);
597 free(rfname);
598
599 if (td->o.read_iolog_chunked) {
600 td->io_log_highmark = td->io_log_current;
601 td->io_log_checkmark = (td->io_log_highmark + 1) / 2;
602 fio_gettime(&td->io_log_highmark_time, NULL);
603 }
604
605 if (writes && read_only) {
606 log_err("fio: <%s> skips replay of %d writes due to"
607 " read-only\n", td->o.name, writes);
608 writes = 0;
609 }
610 if (syncs)
611 td->flags |= TD_F_SYNCS;
612
613 if (td->o.read_iolog_chunked) {
614 if (td->io_log_current == 0) {
615 return false;
616 }
617 td->o.td_ddir = TD_DDIR_RW;
618 if (realloc && td->orig_buffer)
619 {
620 io_u_quiesce(td);
621 free_io_mem(td);
622 init_io_u_buffers(td);
623 }
624 return true;
625 }
626
627 if (!reads && !writes && !waits)
628 return false;
629 else if (reads && !writes)
630 td->o.td_ddir = TD_DDIR_READ;
631 else if (!reads && writes)
632 td->o.td_ddir = TD_DDIR_WRITE;
633 else
634 td->o.td_ddir = TD_DDIR_RW;
635
636 return true;
637}
638
639static bool is_socket(const char *path)
640{
641 struct stat buf;
642 int r;
643
644 r = stat(path, &buf);
645 if (r == -1)
646 return false;
647
648 return S_ISSOCK(buf.st_mode);
649}
650
651static int open_socket(const char *path)
652{
653 struct sockaddr_un addr;
654 int ret, fd;
655
656 fd = socket(AF_UNIX, SOCK_STREAM, 0);
657 if (fd < 0)
658 return fd;
659
660 addr.sun_family = AF_UNIX;
661 if (snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path) >=
662 sizeof(addr.sun_path)) {
663 log_err("%s: path name %s is too long for a Unix socket\n",
664 __func__, path);
665 }
666
667 ret = connect(fd, (const struct sockaddr *)&addr, strlen(path) + sizeof(addr.sun_family));
668 if (!ret)
669 return fd;
670
671 close(fd);
672 return -1;
673}
674
675/*
676 * open iolog, check version, and call appropriate parser
677 */
678static bool init_iolog_read(struct thread_data *td, char *fname)
679{
680 char buffer[256], *p;
681 FILE *f = NULL;
682
683 dprint(FD_IO, "iolog: name=%s\n", fname);
684
685 if (is_socket(fname)) {
686 int fd;
687
688 fd = open_socket(fname);
689 if (fd >= 0)
690 f = fdopen(fd, "r");
691 } else if (!strcmp(fname, "-")) {
692 f = stdin;
693 } else
694 f = fopen(fname, "r");
695
696 if (!f) {
697 perror("fopen read iolog");
698 return false;
699 }
700
701 p = fgets(buffer, sizeof(buffer), f);
702 if (!p) {
703 td_verror(td, errno, "iolog read");
704 log_err("fio: unable to read iolog\n");
705 fclose(f);
706 return false;
707 }
708
709 /*
710 * versions 2 and 3 of the iolog store a specific string as the
711 * first line, check for that
712 */
713 if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2)))
714 td->io_log_version = 2;
715 else if (!strncmp(iolog_ver3, buffer, strlen(iolog_ver3)))
716 td->io_log_version = 3;
717 else {
718 log_err("fio: iolog version 1 is no longer supported\n");
719 fclose(f);
720 return false;
721 }
722
723 free_release_files(td);
724 td->io_log_rfile = f;
725 return read_iolog(td);
726}
727
728/*
729 * Set up a log for storing io patterns.
730 */
731static bool init_iolog_write(struct thread_data *td)
732{
733 struct fio_file *ff;
734 FILE *f;
735 unsigned int i;
736
737 f = fopen(td->o.write_iolog_file, "a");
738 if (!f) {
739 perror("fopen write iolog");
740 return false;
741 }
742
743 /*
744 * That's it for writing, setup a log buffer and we're done.
745 */
746 td->iolog_f = f;
747 td->iolog_buf = malloc(8192);
748 setvbuf(f, td->iolog_buf, _IOFBF, 8192);
749 fio_gettime(&td->io_log_start_time, NULL);
750
751 /*
752 * write our version line
753 */
754 if (fprintf(f, "%s\n", iolog_ver3) < 0) {
755 perror("iolog init\n");
756 return false;
757 }
758
759 /*
760 * add all known files
761 */
762 for_each_file(td, ff, i)
763 log_file(td, ff, FIO_LOG_ADD_FILE);
764
765 return true;
766}
767
768bool init_iolog(struct thread_data *td)
769{
770 bool ret;
771
772 if (td->o.read_iolog_file) {
773 int need_swap;
774 char * fname = get_name_by_idx(td->o.read_iolog_file, td->subjob_number);
775
776 /*
777 * Check if it's a blktrace file and load that if possible.
778 * Otherwise assume it's a normal log file and load that.
779 */
780 if (is_blktrace(fname, &need_swap)) {
781 td->io_log_blktrace = 1;
782 ret = init_blktrace_read(td, fname, need_swap);
783 } else {
784 td->io_log_blktrace = 0;
785 ret = init_iolog_read(td, fname);
786 }
787 free(fname);
788 } else if (td->o.write_iolog_file)
789 ret = init_iolog_write(td);
790 else
791 ret = true;
792
793 if (!ret)
794 td_verror(td, EINVAL, "failed initializing iolog");
795
796 return ret;
797}
798
799void setup_log(struct io_log **log, struct log_params *p,
800 const char *filename)
801{
802 struct io_log *l;
803 int i;
804 struct io_u_plat_entry *entry;
805 struct flist_head *list;
806
807 l = scalloc(1, sizeof(*l));
808 INIT_FLIST_HEAD(&l->io_logs);
809 l->log_type = p->log_type;
810 l->log_offset = p->log_offset;
811 l->log_prio = p->log_prio;
812 l->log_gz = p->log_gz;
813 l->log_gz_store = p->log_gz_store;
814 l->avg_msec = p->avg_msec;
815 l->hist_msec = p->hist_msec;
816 l->hist_coarseness = p->hist_coarseness;
817 l->filename = strdup(filename);
818 l->td = p->td;
819
820 /* Initialize histogram lists for each r/w direction,
821 * with initial io_u_plat of all zeros:
822 */
823 for (i = 0; i < DDIR_RWDIR_CNT; i++) {
824 list = &l->hist_window[i].list;
825 INIT_FLIST_HEAD(list);
826 entry = calloc(1, sizeof(struct io_u_plat_entry));
827 flist_add(&entry->list, list);
828 }
829
830 if (l->td && l->td->o.io_submit_mode != IO_MODE_OFFLOAD) {
831 unsigned int def_samples = DEF_LOG_ENTRIES;
832 struct io_logs *__p;
833
834 __p = calloc(1, sizeof(*l->pending));
835 if (l->td->o.iodepth > DEF_LOG_ENTRIES)
836 def_samples = roundup_pow2(l->td->o.iodepth);
837 __p->max_samples = def_samples;
838 __p->log = calloc(__p->max_samples, log_entry_sz(l));
839 l->pending = __p;
840 }
841
842 if (l->log_offset)
843 l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
844 if (l->log_prio)
845 l->log_ddir_mask |= LOG_PRIO_SAMPLE_BIT;
846
847 INIT_FLIST_HEAD(&l->chunk_list);
848
849 if (l->log_gz && !p->td)
850 l->log_gz = 0;
851 else if (l->log_gz || l->log_gz_store) {
852 mutex_init_pshared(&l->chunk_lock);
853 mutex_init_pshared(&l->deferred_free_lock);
854 p->td->flags |= TD_F_COMPRESS_LOG;
855 }
856
857 *log = l;
858}
859
860#ifdef CONFIG_SETVBUF
861static void *set_file_buffer(FILE *f)
862{
863 size_t size = 1048576;
864 void *buf;
865
866 buf = malloc(size);
867 setvbuf(f, buf, _IOFBF, size);
868 return buf;
869}
870
871static void clear_file_buffer(void *buf)
872{
873 free(buf);
874}
875#else
876static void *set_file_buffer(FILE *f)
877{
878 return NULL;
879}
880
881static void clear_file_buffer(void *buf)
882{
883}
884#endif
885
886void free_log(struct io_log *log)
887{
888 while (!flist_empty(&log->io_logs)) {
889 struct io_logs *cur_log;
890
891 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
892 flist_del_init(&cur_log->list);
893 free(cur_log->log);
894 sfree(cur_log);
895 }
896
897 if (log->pending) {
898 free(log->pending->log);
899 free(log->pending);
900 log->pending = NULL;
901 }
902
903 free(log->pending);
904 free(log->filename);
905 sfree(log);
906}
907
908uint64_t hist_sum(int j, int stride, uint64_t *io_u_plat,
909 uint64_t *io_u_plat_last)
910{
911 uint64_t sum;
912 int k;
913
914 if (io_u_plat_last) {
915 for (k = sum = 0; k < stride; k++)
916 sum += io_u_plat[j + k] - io_u_plat_last[j + k];
917 } else {
918 for (k = sum = 0; k < stride; k++)
919 sum += io_u_plat[j + k];
920 }
921
922 return sum;
923}
924
925static void flush_hist_samples(FILE *f, int hist_coarseness, void *samples,
926 uint64_t sample_size)
927{
928 struct io_sample *s;
929 int log_offset;
930 uint64_t i, j, nr_samples;
931 struct io_u_plat_entry *entry, *entry_before;
932 uint64_t *io_u_plat;
933 uint64_t *io_u_plat_before;
934
935 int stride = 1 << hist_coarseness;
936
937 if (!sample_size)
938 return;
939
940 s = __get_sample(samples, 0, 0);
941 log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
942
943 nr_samples = sample_size / __log_entry_sz(log_offset);
944
945 for (i = 0; i < nr_samples; i++) {
946 s = __get_sample(samples, log_offset, i);
947
948 entry = s->data.plat_entry;
949 io_u_plat = entry->io_u_plat;
950
951 entry_before = flist_first_entry(&entry->list, struct io_u_plat_entry, list);
952 io_u_plat_before = entry_before->io_u_plat;
953
954 fprintf(f, "%lu, %u, %llu, ", (unsigned long) s->time,
955 io_sample_ddir(s), (unsigned long long) s->bs);
956 for (j = 0; j < FIO_IO_U_PLAT_NR - stride; j += stride) {
957 fprintf(f, "%llu, ", (unsigned long long)
958 hist_sum(j, stride, io_u_plat, io_u_plat_before));
959 }
960 fprintf(f, "%llu\n", (unsigned long long)
961 hist_sum(FIO_IO_U_PLAT_NR - stride, stride, io_u_plat,
962 io_u_plat_before));
963
964 flist_del(&entry_before->list);
965 free(entry_before);
966 }
967}
968
969void flush_samples(FILE *f, void *samples, uint64_t sample_size)
970{
971 struct io_sample *s;
972 int log_offset, log_prio;
973 uint64_t i, nr_samples;
974 unsigned int prio_val;
975 const char *fmt;
976
977 if (!sample_size)
978 return;
979
980 s = __get_sample(samples, 0, 0);
981 log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
982 log_prio = (s->__ddir & LOG_PRIO_SAMPLE_BIT) != 0;
983
984 if (log_offset) {
985 if (log_prio)
986 fmt = "%lu, %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
987 else
988 fmt = "%lu, %" PRId64 ", %u, %llu, %llu, %u\n";
989 } else {
990 if (log_prio)
991 fmt = "%lu, %" PRId64 ", %u, %llu, 0x%04x\n";
992 else
993 fmt = "%lu, %" PRId64 ", %u, %llu, %u\n";
994 }
995
996 nr_samples = sample_size / __log_entry_sz(log_offset);
997
998 for (i = 0; i < nr_samples; i++) {
999 s = __get_sample(samples, log_offset, i);
1000
1001 if (log_prio)
1002 prio_val = s->priority;
1003 else
1004 prio_val = ioprio_value_is_class_rt(s->priority);
1005
1006 if (!log_offset) {
1007 fprintf(f, fmt,
1008 (unsigned long) s->time,
1009 s->data.val,
1010 io_sample_ddir(s), (unsigned long long) s->bs,
1011 prio_val);
1012 } else {
1013 struct io_sample_offset *so = (void *) s;
1014
1015 fprintf(f, fmt,
1016 (unsigned long) s->time,
1017 s->data.val,
1018 io_sample_ddir(s), (unsigned long long) s->bs,
1019 (unsigned long long) so->offset,
1020 prio_val);
1021 }
1022 }
1023}
1024
1025#ifdef CONFIG_ZLIB
1026
1027struct iolog_flush_data {
1028 struct workqueue_work work;
1029 struct io_log *log;
1030 void *samples;
1031 uint32_t nr_samples;
1032 bool free;
1033};
1034
1035#define GZ_CHUNK 131072
1036
1037static struct iolog_compress *get_new_chunk(unsigned int seq)
1038{
1039 struct iolog_compress *c;
1040
1041 c = malloc(sizeof(*c));
1042 INIT_FLIST_HEAD(&c->list);
1043 c->buf = malloc(GZ_CHUNK);
1044 c->len = 0;
1045 c->seq = seq;
1046 return c;
1047}
1048
1049static void free_chunk(struct iolog_compress *ic)
1050{
1051 free(ic->buf);
1052 free(ic);
1053}
1054
1055static int z_stream_init(z_stream *stream, int gz_hdr)
1056{
1057 int wbits = 15;
1058
1059 memset(stream, 0, sizeof(*stream));
1060 stream->zalloc = Z_NULL;
1061 stream->zfree = Z_NULL;
1062 stream->opaque = Z_NULL;
1063 stream->next_in = Z_NULL;
1064
1065 /*
1066 * zlib magic - add 32 for auto-detection of gz header or not,
1067 * if we decide to store files in a gzip friendly format.
1068 */
1069 if (gz_hdr)
1070 wbits += 32;
1071
1072 if (inflateInit2(stream, wbits) != Z_OK)
1073 return 1;
1074
1075 return 0;
1076}
1077
1078struct inflate_chunk_iter {
1079 unsigned int seq;
1080 int err;
1081 void *buf;
1082 size_t buf_size;
1083 size_t buf_used;
1084 size_t chunk_sz;
1085};
1086
1087static void finish_chunk(z_stream *stream, FILE *f,
1088 struct inflate_chunk_iter *iter)
1089{
1090 int ret;
1091
1092 ret = inflateEnd(stream);
1093 if (ret != Z_OK)
1094 log_err("fio: failed to end log inflation seq %d (%d)\n",
1095 iter->seq, ret);
1096
1097 flush_samples(f, iter->buf, iter->buf_used);
1098 free(iter->buf);
1099 iter->buf = NULL;
1100 iter->buf_size = iter->buf_used = 0;
1101}
1102
1103/*
1104 * Iterative chunk inflation. Handles cases where we cross into a new
1105 * sequence, doing flush finish of previous chunk if needed.
1106 */
1107static size_t inflate_chunk(struct iolog_compress *ic, int gz_hdr, FILE *f,
1108 z_stream *stream, struct inflate_chunk_iter *iter)
1109{
1110 size_t ret;
1111
1112 dprint(FD_COMPRESS, "inflate chunk size=%lu, seq=%u\n",
1113 (unsigned long) ic->len, ic->seq);
1114
1115 if (ic->seq != iter->seq) {
1116 if (iter->seq)
1117 finish_chunk(stream, f, iter);
1118
1119 z_stream_init(stream, gz_hdr);
1120 iter->seq = ic->seq;
1121 }
1122
1123 stream->avail_in = ic->len;
1124 stream->next_in = ic->buf;
1125
1126 if (!iter->buf_size) {
1127 iter->buf_size = iter->chunk_sz;
1128 iter->buf = malloc(iter->buf_size);
1129 }
1130
1131 while (stream->avail_in) {
1132 size_t this_out = iter->buf_size - iter->buf_used;
1133 int err;
1134
1135 stream->avail_out = this_out;
1136 stream->next_out = iter->buf + iter->buf_used;
1137
1138 err = inflate(stream, Z_NO_FLUSH);
1139 if (err < 0) {
1140 log_err("fio: failed inflating log: %d\n", err);
1141 iter->err = err;
1142 break;
1143 }
1144
1145 iter->buf_used += this_out - stream->avail_out;
1146
1147 if (!stream->avail_out) {
1148 iter->buf_size += iter->chunk_sz;
1149 iter->buf = realloc(iter->buf, iter->buf_size);
1150 continue;
1151 }
1152
1153 if (err == Z_STREAM_END)
1154 break;
1155 }
1156
1157 ret = (void *) stream->next_in - ic->buf;
1158
1159 dprint(FD_COMPRESS, "inflated to size=%lu\n", (unsigned long) iter->buf_size);
1160
1161 return ret;
1162}
1163
1164/*
1165 * Inflate stored compressed chunks, or write them directly to the log
1166 * file if so instructed.
1167 */
1168static int inflate_gz_chunks(struct io_log *log, FILE *f)
1169{
1170 struct inflate_chunk_iter iter = { .chunk_sz = log->log_gz, };
1171 z_stream stream;
1172
1173 while (!flist_empty(&log->chunk_list)) {
1174 struct iolog_compress *ic;
1175
1176 ic = flist_first_entry(&log->chunk_list, struct iolog_compress, list);
1177 flist_del(&ic->list);
1178
1179 if (log->log_gz_store) {
1180 size_t ret;
1181
1182 dprint(FD_COMPRESS, "log write chunk size=%lu, "
1183 "seq=%u\n", (unsigned long) ic->len, ic->seq);
1184
1185 ret = fwrite(ic->buf, ic->len, 1, f);
1186 if (ret != 1 || ferror(f)) {
1187 iter.err = errno;
1188 log_err("fio: error writing compressed log\n");
1189 }
1190 } else
1191 inflate_chunk(ic, log->log_gz_store, f, &stream, &iter);
1192
1193 free_chunk(ic);
1194 }
1195
1196 if (iter.seq) {
1197 finish_chunk(&stream, f, &iter);
1198 free(iter.buf);
1199 }
1200
1201 return iter.err;
1202}
1203
1204/*
1205 * Open compressed log file and decompress the stored chunks and
1206 * write them to stdout. The chunks are stored sequentially in the
1207 * file, so we iterate over them and do them one-by-one.
1208 */
1209int iolog_file_inflate(const char *file)
1210{
1211 struct inflate_chunk_iter iter = { .chunk_sz = 64 * 1024 * 1024, };
1212 struct iolog_compress ic;
1213 z_stream stream;
1214 struct stat sb;
1215 size_t ret;
1216 size_t total;
1217 void *buf;
1218 FILE *f;
1219
1220 f = fopen(file, "r");
1221 if (!f) {
1222 perror("fopen");
1223 return 1;
1224 }
1225
1226 if (stat(file, &sb) < 0) {
1227 fclose(f);
1228 perror("stat");
1229 return 1;
1230 }
1231
1232 ic.buf = buf = malloc(sb.st_size);
1233 ic.len = sb.st_size;
1234 ic.seq = 1;
1235
1236 ret = fread(ic.buf, ic.len, 1, f);
1237 if (ret == 0 && ferror(f)) {
1238 perror("fread");
1239 fclose(f);
1240 free(buf);
1241 return 1;
1242 } else if (ferror(f) || (!feof(f) && ret != 1)) {
1243 log_err("fio: short read on reading log\n");
1244 fclose(f);
1245 free(buf);
1246 return 1;
1247 }
1248
1249 fclose(f);
1250
1251 /*
1252 * Each chunk will return Z_STREAM_END. We don't know how many
1253 * chunks are in the file, so we just keep looping and incrementing
1254 * the sequence number until we have consumed the whole compressed
1255 * file.
1256 */
1257 total = ic.len;
1258 do {
1259 size_t iret;
1260
1261 iret = inflate_chunk(&ic, 1, stdout, &stream, &iter);
1262 total -= iret;
1263 if (!total)
1264 break;
1265 if (iter.err)
1266 break;
1267
1268 ic.seq++;
1269 ic.len -= iret;
1270 ic.buf += iret;
1271 } while (1);
1272
1273 if (iter.seq) {
1274 finish_chunk(&stream, stdout, &iter);
1275 free(iter.buf);
1276 }
1277
1278 free(buf);
1279 return iter.err;
1280}
1281
1282#else
1283
1284static int inflate_gz_chunks(struct io_log *log, FILE *f)
1285{
1286 return 0;
1287}
1288
1289int iolog_file_inflate(const char *file)
1290{
1291 log_err("fio: log inflation not possible without zlib\n");
1292 return 1;
1293}
1294
1295#endif
1296
1297void flush_log(struct io_log *log, bool do_append)
1298{
1299 void *buf;
1300 FILE *f;
1301
1302 if (!do_append)
1303 f = fopen(log->filename, "w");
1304 else
1305 f = fopen(log->filename, "a");
1306 if (!f) {
1307 perror("fopen log");
1308 return;
1309 }
1310
1311 buf = set_file_buffer(f);
1312
1313 inflate_gz_chunks(log, f);
1314
1315 while (!flist_empty(&log->io_logs)) {
1316 struct io_logs *cur_log;
1317
1318 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
1319 flist_del_init(&cur_log->list);
1320
1321 if (log->td && log == log->td->clat_hist_log)
1322 flush_hist_samples(f, log->hist_coarseness, cur_log->log,
1323 log_sample_sz(log, cur_log));
1324 else
1325 flush_samples(f, cur_log->log, log_sample_sz(log, cur_log));
1326
1327 sfree(cur_log);
1328 }
1329
1330 fclose(f);
1331 clear_file_buffer(buf);
1332}
1333
1334static int finish_log(struct thread_data *td, struct io_log *log, int trylock)
1335{
1336 if (td->flags & TD_F_COMPRESS_LOG)
1337 iolog_flush(log);
1338
1339 if (trylock) {
1340 if (fio_trylock_file(log->filename))
1341 return 1;
1342 } else
1343 fio_lock_file(log->filename);
1344
1345 if (td->client_type == FIO_CLIENT_TYPE_GUI || is_backend)
1346 fio_send_iolog(td, log, log->filename);
1347 else
1348 flush_log(log, !td->o.per_job_logs);
1349
1350 fio_unlock_file(log->filename);
1351 free_log(log);
1352 return 0;
1353}
1354
1355size_t log_chunk_sizes(struct io_log *log)
1356{
1357 struct flist_head *entry;
1358 size_t ret;
1359
1360 if (flist_empty(&log->chunk_list))
1361 return 0;
1362
1363 ret = 0;
1364 pthread_mutex_lock(&log->chunk_lock);
1365 flist_for_each(entry, &log->chunk_list) {
1366 struct iolog_compress *c;
1367
1368 c = flist_entry(entry, struct iolog_compress, list);
1369 ret += c->len;
1370 }
1371 pthread_mutex_unlock(&log->chunk_lock);
1372 return ret;
1373}
1374
1375#ifdef CONFIG_ZLIB
1376
1377static void iolog_put_deferred(struct io_log *log, void *ptr)
1378{
1379 if (!ptr)
1380 return;
1381
1382 pthread_mutex_lock(&log->deferred_free_lock);
1383 if (log->deferred < IOLOG_MAX_DEFER) {
1384 log->deferred_items[log->deferred] = ptr;
1385 log->deferred++;
1386 } else if (!fio_did_warn(FIO_WARN_IOLOG_DROP))
1387 log_err("fio: had to drop log entry free\n");
1388 pthread_mutex_unlock(&log->deferred_free_lock);
1389}
1390
1391static void iolog_free_deferred(struct io_log *log)
1392{
1393 int i;
1394
1395 if (!log->deferred)
1396 return;
1397
1398 pthread_mutex_lock(&log->deferred_free_lock);
1399
1400 for (i = 0; i < log->deferred; i++) {
1401 free(log->deferred_items[i]);
1402 log->deferred_items[i] = NULL;
1403 }
1404
1405 log->deferred = 0;
1406 pthread_mutex_unlock(&log->deferred_free_lock);
1407}
1408
1409static int gz_work(struct iolog_flush_data *data)
1410{
1411 struct iolog_compress *c = NULL;
1412 struct flist_head list;
1413 unsigned int seq;
1414 z_stream stream;
1415 size_t total = 0;
1416 int ret;
1417
1418 INIT_FLIST_HEAD(&list);
1419
1420 memset(&stream, 0, sizeof(stream));
1421 stream.zalloc = Z_NULL;
1422 stream.zfree = Z_NULL;
1423 stream.opaque = Z_NULL;
1424
1425 ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
1426 if (ret != Z_OK) {
1427 log_err("fio: failed to init gz stream\n");
1428 goto err;
1429 }
1430
1431 seq = ++data->log->chunk_seq;
1432
1433 stream.next_in = (void *) data->samples;
1434 stream.avail_in = data->nr_samples * log_entry_sz(data->log);
1435
1436 dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u, log=%s\n",
1437 (unsigned long) stream.avail_in, seq,
1438 data->log->filename);
1439 do {
1440 if (c)
1441 dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq,
1442 (unsigned long) c->len);
1443 c = get_new_chunk(seq);
1444 stream.avail_out = GZ_CHUNK;
1445 stream.next_out = c->buf;
1446 ret = deflate(&stream, Z_NO_FLUSH);
1447 if (ret < 0) {
1448 log_err("fio: deflate log (%d)\n", ret);
1449 free_chunk(c);
1450 goto err;
1451 }
1452
1453 c->len = GZ_CHUNK - stream.avail_out;
1454 flist_add_tail(&c->list, &list);
1455 total += c->len;
1456 } while (stream.avail_in);
1457
1458 stream.next_out = c->buf + c->len;
1459 stream.avail_out = GZ_CHUNK - c->len;
1460
1461 ret = deflate(&stream, Z_FINISH);
1462 if (ret < 0) {
1463 /*
1464 * Z_BUF_ERROR is special, it just means we need more
1465 * output space. We'll handle that below. Treat any other
1466 * error as fatal.
1467 */
1468 if (ret != Z_BUF_ERROR) {
1469 log_err("fio: deflate log (%d)\n", ret);
1470 flist_del(&c->list);
1471 free_chunk(c);
1472 goto err;
1473 }
1474 }
1475
1476 total -= c->len;
1477 c->len = GZ_CHUNK - stream.avail_out;
1478 total += c->len;
1479 dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq, (unsigned long) c->len);
1480
1481 if (ret != Z_STREAM_END) {
1482 do {
1483 c = get_new_chunk(seq);
1484 stream.avail_out = GZ_CHUNK;
1485 stream.next_out = c->buf;
1486 ret = deflate(&stream, Z_FINISH);
1487 c->len = GZ_CHUNK - stream.avail_out;
1488 total += c->len;
1489 flist_add_tail(&c->list, &list);
1490 dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq,
1491 (unsigned long) c->len);
1492 } while (ret != Z_STREAM_END);
1493 }
1494
1495 dprint(FD_COMPRESS, "deflated to size=%lu\n", (unsigned long) total);
1496
1497 ret = deflateEnd(&stream);
1498 if (ret != Z_OK)
1499 log_err("fio: deflateEnd %d\n", ret);
1500
1501 iolog_put_deferred(data->log, data->samples);
1502
1503 if (!flist_empty(&list)) {
1504 pthread_mutex_lock(&data->log->chunk_lock);
1505 flist_splice_tail(&list, &data->log->chunk_list);
1506 pthread_mutex_unlock(&data->log->chunk_lock);
1507 }
1508
1509 ret = 0;
1510done:
1511 if (data->free)
1512 sfree(data);
1513 return ret;
1514err:
1515 while (!flist_empty(&list)) {
1516 c = flist_first_entry(list.next, struct iolog_compress, list);
1517 flist_del(&c->list);
1518 free_chunk(c);
1519 }
1520 ret = 1;
1521 goto done;
1522}
1523
1524/*
1525 * Invoked from our compress helper thread, when logging would have exceeded
1526 * the specified memory limitation. Compresses the previously stored
1527 * entries.
1528 */
1529static int gz_work_async(struct submit_worker *sw, struct workqueue_work *work)
1530{
1531 return gz_work(container_of(work, struct iolog_flush_data, work));
1532}
1533
1534static int gz_init_worker(struct submit_worker *sw)
1535{
1536 struct thread_data *td = sw->wq->td;
1537
1538 if (!fio_option_is_set(&td->o, log_gz_cpumask))
1539 return 0;
1540
1541 if (fio_setaffinity(gettid(), td->o.log_gz_cpumask) == -1) {
1542 log_err("gz: failed to set CPU affinity\n");
1543 return 1;
1544 }
1545
1546 return 0;
1547}
1548
1549static struct workqueue_ops log_compress_wq_ops = {
1550 .fn = gz_work_async,
1551 .init_worker_fn = gz_init_worker,
1552 .nice = 1,
1553};
1554
1555int iolog_compress_init(struct thread_data *td, struct sk_out *sk_out)
1556{
1557 if (!(td->flags & TD_F_COMPRESS_LOG))
1558 return 0;
1559
1560 workqueue_init(td, &td->log_compress_wq, &log_compress_wq_ops, 1, sk_out);
1561 return 0;
1562}
1563
1564void iolog_compress_exit(struct thread_data *td)
1565{
1566 if (!(td->flags & TD_F_COMPRESS_LOG))
1567 return;
1568
1569 workqueue_exit(&td->log_compress_wq);
1570}
1571
1572/*
1573 * Queue work item to compress the existing log entries. We reset the
1574 * current log to a small size, and reference the existing log in the
1575 * data that we queue for compression. Once compression has been done,
1576 * this old log is freed. If called with finish == true, will not return
1577 * until the log compression has completed, and will flush all previous
1578 * logs too
1579 */
1580static int iolog_flush(struct io_log *log)
1581{
1582 struct iolog_flush_data *data;
1583
1584 data = malloc(sizeof(*data));
1585 if (!data)
1586 return 1;
1587
1588 data->log = log;
1589 data->free = false;
1590
1591 while (!flist_empty(&log->io_logs)) {
1592 struct io_logs *cur_log;
1593
1594 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
1595 flist_del_init(&cur_log->list);
1596
1597 data->samples = cur_log->log;
1598 data->nr_samples = cur_log->nr_samples;
1599
1600 sfree(cur_log);
1601
1602 gz_work(data);
1603 }
1604
1605 free(data);
1606 return 0;
1607}
1608
1609int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log)
1610{
1611 struct iolog_flush_data *data;
1612
1613 data = smalloc(sizeof(*data));
1614 if (!data)
1615 return 1;
1616
1617 data->log = log;
1618
1619 data->samples = cur_log->log;
1620 data->nr_samples = cur_log->nr_samples;
1621 data->free = true;
1622
1623 cur_log->nr_samples = cur_log->max_samples = 0;
1624 cur_log->log = NULL;
1625
1626 workqueue_enqueue(&log->td->log_compress_wq, &data->work);
1627
1628 iolog_free_deferred(log);
1629
1630 return 0;
1631}
1632#else
1633
1634static int iolog_flush(struct io_log *log)
1635{
1636 return 1;
1637}
1638
1639int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log)
1640{
1641 return 1;
1642}
1643
1644int iolog_compress_init(struct thread_data *td, struct sk_out *sk_out)
1645{
1646 return 0;
1647}
1648
1649void iolog_compress_exit(struct thread_data *td)
1650{
1651}
1652
1653#endif
1654
1655struct io_logs *iolog_cur_log(struct io_log *log)
1656{
1657 if (flist_empty(&log->io_logs))
1658 return NULL;
1659
1660 return flist_last_entry(&log->io_logs, struct io_logs, list);
1661}
1662
1663uint64_t iolog_nr_samples(struct io_log *iolog)
1664{
1665 struct flist_head *entry;
1666 uint64_t ret = 0;
1667
1668 flist_for_each(entry, &iolog->io_logs) {
1669 struct io_logs *cur_log;
1670
1671 cur_log = flist_entry(entry, struct io_logs, list);
1672 ret += cur_log->nr_samples;
1673 }
1674
1675 return ret;
1676}
1677
1678static int __write_log(struct thread_data *td, struct io_log *log, int try)
1679{
1680 if (log)
1681 return finish_log(td, log, try);
1682
1683 return 0;
1684}
1685
1686static int write_iops_log(struct thread_data *td, int try, bool unit_log)
1687{
1688 int ret;
1689
1690 if (per_unit_log(td->iops_log) != unit_log)
1691 return 0;
1692
1693 ret = __write_log(td, td->iops_log, try);
1694 if (!ret)
1695 td->iops_log = NULL;
1696
1697 return ret;
1698}
1699
1700static int write_slat_log(struct thread_data *td, int try, bool unit_log)
1701{
1702 int ret;
1703
1704 if (!unit_log)
1705 return 0;
1706
1707 ret = __write_log(td, td->slat_log, try);
1708 if (!ret)
1709 td->slat_log = NULL;
1710
1711 return ret;
1712}
1713
1714static int write_clat_log(struct thread_data *td, int try, bool unit_log)
1715{
1716 int ret;
1717
1718 if (!unit_log)
1719 return 0;
1720
1721 ret = __write_log(td, td->clat_log, try);
1722 if (!ret)
1723 td->clat_log = NULL;
1724
1725 return ret;
1726}
1727
1728static int write_clat_hist_log(struct thread_data *td, int try, bool unit_log)
1729{
1730 int ret;
1731
1732 if (!unit_log)
1733 return 0;
1734
1735 ret = __write_log(td, td->clat_hist_log, try);
1736 if (!ret)
1737 td->clat_hist_log = NULL;
1738
1739 return ret;
1740}
1741
1742static int write_lat_log(struct thread_data *td, int try, bool unit_log)
1743{
1744 int ret;
1745
1746 if (!unit_log)
1747 return 0;
1748
1749 ret = __write_log(td, td->lat_log, try);
1750 if (!ret)
1751 td->lat_log = NULL;
1752
1753 return ret;
1754}
1755
1756static int write_bandw_log(struct thread_data *td, int try, bool unit_log)
1757{
1758 int ret;
1759
1760 if (per_unit_log(td->bw_log) != unit_log)
1761 return 0;
1762
1763 ret = __write_log(td, td->bw_log, try);
1764 if (!ret)
1765 td->bw_log = NULL;
1766
1767 return ret;
1768}
1769
1770enum {
1771 BW_LOG_MASK = 1,
1772 LAT_LOG_MASK = 2,
1773 SLAT_LOG_MASK = 4,
1774 CLAT_LOG_MASK = 8,
1775 IOPS_LOG_MASK = 16,
1776 CLAT_HIST_LOG_MASK = 32,
1777
1778 ALL_LOG_NR = 6,
1779};
1780
1781struct log_type {
1782 unsigned int mask;
1783 int (*fn)(struct thread_data *, int, bool);
1784};
1785
1786static struct log_type log_types[] = {
1787 {
1788 .mask = BW_LOG_MASK,
1789 .fn = write_bandw_log,
1790 },
1791 {
1792 .mask = LAT_LOG_MASK,
1793 .fn = write_lat_log,
1794 },
1795 {
1796 .mask = SLAT_LOG_MASK,
1797 .fn = write_slat_log,
1798 },
1799 {
1800 .mask = CLAT_LOG_MASK,
1801 .fn = write_clat_log,
1802 },
1803 {
1804 .mask = IOPS_LOG_MASK,
1805 .fn = write_iops_log,
1806 },
1807 {
1808 .mask = CLAT_HIST_LOG_MASK,
1809 .fn = write_clat_hist_log,
1810 }
1811};
1812
1813void td_writeout_logs(struct thread_data *td, bool unit_logs)
1814{
1815 unsigned int log_mask = 0;
1816 unsigned int log_left = ALL_LOG_NR;
1817 int old_state, i;
1818
1819 old_state = td_bump_runstate(td, TD_FINISHING);
1820
1821 finalize_logs(td, unit_logs);
1822
1823 while (log_left) {
1824 int prev_log_left = log_left;
1825
1826 for (i = 0; i < ALL_LOG_NR && log_left; i++) {
1827 struct log_type *lt = &log_types[i];
1828 int ret;
1829
1830 if (!(log_mask & lt->mask)) {
1831 ret = lt->fn(td, log_left != 1, unit_logs);
1832 if (!ret) {
1833 log_left--;
1834 log_mask |= lt->mask;
1835 }
1836 }
1837 }
1838
1839 if (prev_log_left == log_left)
1840 usleep(5000);
1841 }
1842
1843 td_restore_runstate(td, old_state);
1844}
1845
1846void fio_writeout_logs(bool unit_logs)
1847{
1848 struct thread_data *td;
1849 int i;
1850
1851 for_each_td(td, i)
1852 td_writeout_logs(td, unit_logs);
1853}
fio - Flexible IO Tester