iolog.c

   1 /*
   2  * Code related to writing an iolog of what a thread is doing, and to
   3  * later read that back and replay
   4  */
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <libgen.h>
   8 #include <assert.h>
   9 #include <sys/types.h>
  10 #include <sys/stat.h>
  11 #include <unistd.h>
  12 #ifdef CONFIG_ZLIB
  13 #include <zlib.h>
  14 #endif
  15
  16 #include "flist.h"
  17 #include "fio.h"
  18 #include "verify.h"
  19 #include "trim.h"
  20 #include "filelock.h"
  21 #include "smalloc.h"
  22
  23 static int iolog_flush(struct io_log *log);
  24
  25 static const char iolog_ver2[] = "fio version 2 iolog";
  26
  27 void queue_io_piece(struct thread_data *td, struct io_piece *ipo)
  28 {
  29         flist_add_tail(&ipo->list, &td->io_log_list);
  30         td->total_io_size += ipo->len;
  31 }
  32
  33 void log_io_u(const struct thread_data *td, const struct io_u *io_u)
  34 {
  35         if (!td->o.write_iolog_file)
  36                 return;
  37
  38         fprintf(td->iolog_f, "%s %s %llu %lu\n", io_u->file->file_name,
  39                                                 io_ddir_name(io_u->ddir),
  40                                                 io_u->offset, io_u->buflen);
  41 }
  42
  43 void log_file(struct thread_data *td, struct fio_file *f,
  44               enum file_log_act what)
  45 {
  46         const char *act[] = { "add", "open", "close" };
  47
  48         assert(what < 3);
  49
  50         if (!td->o.write_iolog_file)
  51                 return;
  52
  53
  54         /*
  55          * this happens on the pre-open/close done before the job starts
  56          */
  57         if (!td->iolog_f)
  58                 return;
  59
  60         fprintf(td->iolog_f, "%s %s\n", f->file_name, act[what]);
  61 }
  62
  63 static void iolog_delay(struct thread_data *td, unsigned long delay)
  64 {
  65         uint64_t usec = utime_since_now(&td->last_issue);
  66         uint64_t this_delay;
  67         struct timeval tv;
  68
  69         if (delay < td->time_offset) {
  70                 td->time_offset = 0;
  71                 return;
  72         }
  73
  74         delay -= td->time_offset;
  75         if (delay < usec)
  76                 return;
  77
  78         delay -= usec;
  79
  80         fio_gettime(&tv, NULL);
  81         while (delay && !td->terminate) {
  82                 this_delay = delay;
  83                 if (this_delay > 500000)
  84                         this_delay = 500000;
  85
  86                 usec_sleep(td, this_delay);
  87                 delay -= this_delay;
  88         }
  89
  90         usec = utime_since_now(&tv);
  91         if (usec > delay)
  92                 td->time_offset = usec - delay;
  93         else
  94                 td->time_offset = 0;
  95 }
  96
  97 static int ipo_special(struct thread_data *td, struct io_piece *ipo)
  98 {
  99         struct fio_file *f;
 100         int ret;
 101
 102         /*
 103          * Not a special ipo
 104          */
 105         if (ipo->ddir != DDIR_INVAL)
 106                 return 0;
 107
 108         f = td->files[ipo->fileno];
 109
 110         switch (ipo->file_action) {
 111         case FIO_LOG_OPEN_FILE:
 112                 ret = td_io_open_file(td, f);
 113                 if (!ret)
 114                         break;
 115                 td_verror(td, ret, "iolog open file");
 116                 return -1;
 117         case FIO_LOG_CLOSE_FILE:
 118                 td_io_close_file(td, f);
 119                 break;
 120         case FIO_LOG_UNLINK_FILE:
 121                 td_io_unlink_file(td, f);
 122                 break;
 123         default:
 124                 log_err("fio: bad file action %d\n", ipo->file_action);
 125                 break;
 126         }
 127
 128         return 1;
 129 }
 130
 131 int read_iolog_get(struct thread_data *td, struct io_u *io_u)
 132 {
 133         struct io_piece *ipo;
 134         unsigned long elapsed;
 135
 136         while (!flist_empty(&td->io_log_list)) {
 137                 int ret;
 138
 139                 ipo = flist_first_entry(&td->io_log_list, struct io_piece, list);
 140                 flist_del(&ipo->list);
 141                 remove_trim_entry(td, ipo);
 142
 143                 ret = ipo_special(td, ipo);
 144                 if (ret < 0) {
 145                         free(ipo);
 146                         break;
 147                 } else if (ret > 0) {
 148                         free(ipo);
 149                         continue;
 150                 }
 151
 152                 io_u->ddir = ipo->ddir;
 153                 if (ipo->ddir != DDIR_WAIT) {
 154                         io_u->offset = ipo->offset;
 155                         io_u->buflen = ipo->len;
 156                         io_u->file = td->files[ipo->fileno];
 157                         get_file(io_u->file);
 158                         dprint(FD_IO, "iolog: get %llu/%lu/%s\n", io_u->offset,
 159                                                 io_u->buflen, io_u->file->file_name);
 160                         if (ipo->delay)
 161                                 iolog_delay(td, ipo->delay);
 162                 } else {
 163                         elapsed = mtime_since_genesis();
 164                         if (ipo->delay > elapsed)
 165                                 usec_sleep(td, (ipo->delay - elapsed) * 1000);
 166                 }
 167
 168                 free(ipo);
 169
 170                 if (io_u->ddir != DDIR_WAIT)
 171                         return 0;
 172         }
 173
 174         td->done = 1;
 175         return 1;
 176 }
 177
 178 void prune_io_piece_log(struct thread_data *td)
 179 {
 180         struct io_piece *ipo;
 181         struct rb_node *n;
 182
 183         while ((n = rb_first(&td->io_hist_tree)) != NULL) {
 184                 ipo = rb_entry(n, struct io_piece, rb_node);
 185                 rb_erase(n, &td->io_hist_tree);
 186                 remove_trim_entry(td, ipo);
 187                 td->io_hist_len--;
 188                 free(ipo);
 189         }
 190
 191         while (!flist_empty(&td->io_hist_list)) {
 192                 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
 193                 flist_del(&ipo->list);
 194                 remove_trim_entry(td, ipo);
 195                 td->io_hist_len--;
 196                 free(ipo);
 197         }
 198 }
 199
 200 /*
 201  * log a successful write, so we can unwind the log for verify
 202  */
 203 void log_io_piece(struct thread_data *td, struct io_u *io_u)
 204 {
 205         struct rb_node **p, *parent;
 206         struct io_piece *ipo, *__ipo;
 207
 208         ipo = malloc(sizeof(struct io_piece));
 209         init_ipo(ipo);
 210         ipo->file = io_u->file;
 211         ipo->offset = io_u->offset;
 212         ipo->len = io_u->buflen;
 213         ipo->numberio = io_u->numberio;
 214         ipo->flags = IP_F_IN_FLIGHT;
 215
 216         io_u->ipo = ipo;
 217
 218         if (io_u_should_trim(td, io_u)) {
 219                 flist_add_tail(&ipo->trim_list, &td->trim_list);
 220                 td->trim_entries++;
 221         }
 222
 223         /*
 224          * We don't need to sort the entries, if:
 225          *
 226          *      Sequential writes, or
 227          *      Random writes that lay out the file as it goes along
 228          *
 229          * For both these cases, just reading back data in the order we
 230          * wrote it out is the fastest.
 231          *
 232          * One exception is if we don't have a random map AND we are doing
 233          * verifies, in that case we need to check for duplicate blocks and
 234          * drop the old one, which we rely on the rb insert/lookup for
 235          * handling.
 236          */
 237         if (((!td->o.verifysort) || !td_random(td) || !td->o.overwrite) &&
 238               (file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
 239                 INIT_FLIST_HEAD(&ipo->list);
 240                 flist_add_tail(&ipo->list, &td->io_hist_list);
 241                 ipo->flags |= IP_F_ONLIST;
 242                 td->io_hist_len++;
 243                 return;
 244         }
 245
 246         RB_CLEAR_NODE(&ipo->rb_node);
 247
 248         /*
 249          * Sort the entry into the verification list
 250          */
 251 restart:
 252         p = &td->io_hist_tree.rb_node;
 253         parent = NULL;
 254         while (*p) {
 255                 int overlap = 0;
 256                 parent = *p;
 257
 258                 __ipo = rb_entry(parent, struct io_piece, rb_node);
 259                 if (ipo->file < __ipo->file)
 260                         p = &(*p)->rb_left;
 261                 else if (ipo->file > __ipo->file)
 262                         p = &(*p)->rb_right;
 263                 else if (ipo->offset < __ipo->offset) {
 264                         p = &(*p)->rb_left;
 265                         overlap = ipo->offset + ipo->len > __ipo->offset;
 266                 }
 267                 else if (ipo->offset > __ipo->offset) {
 268                         p = &(*p)->rb_right;
 269                         overlap = __ipo->offset + __ipo->len > ipo->offset;
 270                 }
 271                 else
 272                         overlap = 1;
 273
 274                 if (overlap) {
 275                         dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu",
 276                                 __ipo->offset, __ipo->len,
 277                                 ipo->offset, ipo->len);
 278                         td->io_hist_len--;
 279                         rb_erase(parent, &td->io_hist_tree);
 280                         remove_trim_entry(td, __ipo);
 281                         free(__ipo);
 282                         goto restart;
 283                 }
 284         }
 285
 286         rb_link_node(&ipo->rb_node, parent, p);
 287         rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
 288         ipo->flags |= IP_F_ONRB;
 289         td->io_hist_len++;
 290 }
 291
 292 void unlog_io_piece(struct thread_data *td, struct io_u *io_u)
 293 {
 294         struct io_piece *ipo = io_u->ipo;
 295
 296         if (td->ts.nr_block_infos) {
 297                 uint32_t *info = io_u_block_info(td, io_u);
 298                 if (BLOCK_INFO_STATE(*info) < BLOCK_STATE_TRIM_FAILURE) {
 299                         if (io_u->ddir == DDIR_TRIM)
 300                                 *info = BLOCK_INFO_SET_STATE(*info,
 301                                                 BLOCK_STATE_TRIM_FAILURE);
 302                         else if (io_u->ddir == DDIR_WRITE)
 303                                 *info = BLOCK_INFO_SET_STATE(*info,
 304                                                 BLOCK_STATE_WRITE_FAILURE);
 305                 }
 306         }
 307
 308         if (!ipo)
 309                 return;
 310
 311         if (ipo->flags & IP_F_ONRB)
 312                 rb_erase(&ipo->rb_node, &td->io_hist_tree);
 313         else if (ipo->flags & IP_F_ONLIST)
 314                 flist_del(&ipo->list);
 315
 316         free(ipo);
 317         io_u->ipo = NULL;
 318         td->io_hist_len--;
 319 }
 320
 321 void trim_io_piece(struct thread_data *td, const struct io_u *io_u)
 322 {
 323         struct io_piece *ipo = io_u->ipo;
 324
 325         if (!ipo)
 326                 return;
 327
 328         ipo->len = io_u->xfer_buflen - io_u->resid;
 329 }
 330
 331 void write_iolog_close(struct thread_data *td)
 332 {
 333         fflush(td->iolog_f);
 334         fclose(td->iolog_f);
 335         free(td->iolog_buf);
 336         td->iolog_f = NULL;
 337         td->iolog_buf = NULL;
 338 }
 339
 340 /*
 341  * Read version 2 iolog data. It is enhanced to include per-file logging,
 342  * syncs, etc.
 343  */
 344 static int read_iolog2(struct thread_data *td, FILE *f)
 345 {
 346         unsigned long long offset;
 347         unsigned int bytes;
 348         int reads, writes, waits, fileno = 0, file_action = 0; /* stupid gcc */
 349         char *rfname, *fname, *act;
 350         char *str, *p;
 351         enum fio_ddir rw;
 352
 353         free_release_files(td);
 354
 355         /*
 356          * Read in the read iolog and store it, reuse the infrastructure
 357          * for doing verifications.
 358          */
 359         str = malloc(4096);
 360         rfname = fname = malloc(256+16);
 361         act = malloc(256+16);
 362
 363         reads = writes = waits = 0;
 364         while ((p = fgets(str, 4096, f)) != NULL) {
 365                 struct io_piece *ipo;
 366                 int r;
 367
 368                 r = sscanf(p, "%256s %256s %llu %u", rfname, act, &offset,
 369                                                                         &bytes);
 370
 371                 if (td->o.replay_redirect)
 372                         fname = td->o.replay_redirect;
 373
 374                 if (r == 4) {
 375                         /*
 376                          * Check action first
 377                          */
 378                         if (!strcmp(act, "wait"))
 379                                 rw = DDIR_WAIT;
 380                         else if (!strcmp(act, "read"))
 381                                 rw = DDIR_READ;
 382                         else if (!strcmp(act, "write"))
 383                                 rw = DDIR_WRITE;
 384                         else if (!strcmp(act, "sync"))
 385                                 rw = DDIR_SYNC;
 386                         else if (!strcmp(act, "datasync"))
 387                                 rw = DDIR_DATASYNC;
 388                         else if (!strcmp(act, "trim"))
 389                                 rw = DDIR_TRIM;
 390                         else {
 391                                 log_err("fio: bad iolog file action: %s\n",
 392                                                                         act);
 393                                 continue;
 394                         }
 395                         fileno = get_fileno(td, fname);
 396                 } else if (r == 2) {
 397                         rw = DDIR_INVAL;
 398                         if (!strcmp(act, "add")) {
 399                                 fileno = add_file(td, fname, 0, 1);
 400                                 file_action = FIO_LOG_ADD_FILE;
 401                                 continue;
 402                         } else if (!strcmp(act, "open")) {
 403                                 fileno = get_fileno(td, fname);
 404                                 file_action = FIO_LOG_OPEN_FILE;
 405                         } else if (!strcmp(act, "close")) {
 406                                 fileno = get_fileno(td, fname);
 407                                 file_action = FIO_LOG_CLOSE_FILE;
 408                         } else {
 409                                 log_err("fio: bad iolog file action: %s\n",
 410                                                                         act);
 411                                 continue;
 412                         }
 413                 } else {
 414                         log_err("bad iolog2: %s", p);
 415                         continue;
 416                 }
 417
 418                 if (rw == DDIR_READ)
 419                         reads++;
 420                 else if (rw == DDIR_WRITE) {
 421                         /*
 422                          * Don't add a write for ro mode
 423                          */
 424                         if (read_only)
 425                                 continue;
 426                         writes++;
 427                 } else if (rw == DDIR_WAIT) {
 428                         if (td->o.no_stall)
 429                                 continue;
 430                         waits++;
 431                 } else if (rw == DDIR_INVAL) {
 432                 } else if (!ddir_sync(rw)) {
 433                         log_err("bad ddir: %d\n", rw);
 434                         continue;
 435                 }
 436
 437                 /*
 438                  * Make note of file
 439                  */
 440                 ipo = malloc(sizeof(*ipo));
 441                 init_ipo(ipo);
 442                 ipo->ddir = rw;
 443                 if (rw == DDIR_WAIT) {
 444                         ipo->delay = offset;
 445                 } else {
 446                         ipo->offset = offset;
 447                         ipo->len = bytes;
 448                         if (rw != DDIR_INVAL && bytes > td->o.max_bs[rw])
 449                                 td->o.max_bs[rw] = bytes;
 450                         ipo->fileno = fileno;
 451                         ipo->file_action = file_action;
 452                         td->o.size += bytes;
 453                 }
 454
 455                 queue_io_piece(td, ipo);
 456         }
 457
 458         free(str);
 459         free(act);
 460         free(rfname);
 461
 462         if (writes && read_only) {
 463                 log_err("fio: <%s> skips replay of %d writes due to"
 464                         " read-only\n", td->o.name, writes);
 465                 writes = 0;
 466         }
 467
 468         if (!reads && !writes && !waits)
 469                 return 1;
 470         else if (reads && !writes)
 471                 td->o.td_ddir = TD_DDIR_READ;
 472         else if (!reads && writes)
 473                 td->o.td_ddir = TD_DDIR_WRITE;
 474         else
 475                 td->o.td_ddir = TD_DDIR_RW;
 476
 477         return 0;
 478 }
 479
 480 /*
 481  * open iolog, check version, and call appropriate parser
 482  */
 483 static int init_iolog_read(struct thread_data *td)
 484 {
 485         char buffer[256], *p;
 486         FILE *f;
 487         int ret;
 488
 489         f = fopen(td->o.read_iolog_file, "r");
 490         if (!f) {
 491                 perror("fopen read iolog");
 492                 return 1;
 493         }
 494
 495         p = fgets(buffer, sizeof(buffer), f);
 496         if (!p) {
 497                 td_verror(td, errno, "iolog read");
 498                 log_err("fio: unable to read iolog\n");
 499                 fclose(f);
 500                 return 1;
 501         }
 502
 503         /*
 504          * version 2 of the iolog stores a specific string as the
 505          * first line, check for that
 506          */
 507         if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2)))
 508                 ret = read_iolog2(td, f);
 509         else {
 510                 log_err("fio: iolog version 1 is no longer supported\n");
 511                 ret = 1;
 512         }
 513
 514         fclose(f);
 515         return ret;
 516 }
 517
 518 /*
 519  * Set up a log for storing io patterns.
 520  */
 521 static int init_iolog_write(struct thread_data *td)
 522 {
 523         struct fio_file *ff;
 524         FILE *f;
 525         unsigned int i;
 526
 527         f = fopen(td->o.write_iolog_file, "a");
 528         if (!f) {
 529                 perror("fopen write iolog");
 530                 return 1;
 531         }
 532
 533         /*
 534          * That's it for writing, setup a log buffer and we're done.
 535           */
 536         td->iolog_f = f;
 537         td->iolog_buf = malloc(8192);
 538         setvbuf(f, td->iolog_buf, _IOFBF, 8192);
 539
 540         /*
 541          * write our version line
 542          */
 543         if (fprintf(f, "%s\n", iolog_ver2) < 0) {
 544                 perror("iolog init\n");
 545                 return 1;
 546         }
 547
 548         /*
 549          * add all known files
 550          */
 551         for_each_file(td, ff, i)
 552                 log_file(td, ff, FIO_LOG_ADD_FILE);
 553
 554         return 0;
 555 }
 556
 557 int init_iolog(struct thread_data *td)
 558 {
 559         int ret = 0;
 560
 561         if (td->o.read_iolog_file) {
 562                 int need_swap;
 563
 564                 /*
 565                  * Check if it's a blktrace file and load that if possible.
 566                  * Otherwise assume it's a normal log file and load that.
 567                  */
 568                 if (is_blktrace(td->o.read_iolog_file, &need_swap))
 569                         ret = load_blktrace(td, td->o.read_iolog_file, need_swap);
 570                 else
 571                         ret = init_iolog_read(td);
 572         } else if (td->o.write_iolog_file)
 573                 ret = init_iolog_write(td);
 574
 575         if (ret)
 576                 td_verror(td, EINVAL, "failed initializing iolog");
 577
 578         return ret;
 579 }
 580
 581 void setup_log(struct io_log **log, struct log_params *p,
 582                const char *filename)
 583 {
 584         struct io_log *l;
 585         int i;
 586         struct io_u_plat_entry *entry;
 587         struct flist_head *list;
 588
 589         l = scalloc(1, sizeof(*l));
 590         INIT_FLIST_HEAD(&l->io_logs);
 591         l->log_type = p->log_type;
 592         l->log_offset = p->log_offset;
 593         l->log_gz = p->log_gz;
 594         l->log_gz_store = p->log_gz_store;
 595         l->avg_msec = p->avg_msec;
 596         l->hist_msec = p->hist_msec;
 597         l->hist_coarseness = p->hist_coarseness;
 598         l->filename = strdup(filename);
 599         l->td = p->td;
 600
 601         /* Initialize histogram lists for each r/w direction,
 602          * with initial io_u_plat of all zeros:
 603          */
 604         for (i = 0; i < DDIR_RWDIR_CNT; i++) {
 605                 list = &l->hist_window[i].list;
 606                 INIT_FLIST_HEAD(list);
 607                 entry = calloc(1, sizeof(struct io_u_plat_entry));
 608                 flist_add(&entry->list, list);
 609         }
 610
 611         if (l->td && l->td->o.io_submit_mode != IO_MODE_OFFLOAD) {
 612                 struct io_logs *p;
 613
 614                 p = calloc(1, sizeof(*l->pending));
 615                 p->max_samples = DEF_LOG_ENTRIES;
 616                 p->log = calloc(p->max_samples, log_entry_sz(l));
 617                 l->pending = p;
 618         }
 619
 620         if (l->log_offset)
 621                 l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
 622
 623         INIT_FLIST_HEAD(&l->chunk_list);
 624
 625         if (l->log_gz && !p->td)
 626                 l->log_gz = 0;
 627         else if (l->log_gz || l->log_gz_store) {
 628                 mutex_init_pshared(&l->chunk_lock);
 629                 p->td->flags |= TD_F_COMPRESS_LOG;
 630         }
 631
 632         *log = l;
 633 }
 634
 635 #ifdef CONFIG_SETVBUF
 636 static void *set_file_buffer(FILE *f)
 637 {
 638         size_t size = 1048576;
 639         void *buf;
 640
 641         buf = malloc(size);
 642         setvbuf(f, buf, _IOFBF, size);
 643         return buf;
 644 }
 645
 646 static void clear_file_buffer(void *buf)
 647 {
 648         free(buf);
 649 }
 650 #else
 651 static void *set_file_buffer(FILE *f)
 652 {
 653         return NULL;
 654 }
 655
 656 static void clear_file_buffer(void *buf)
 657 {
 658 }
 659 #endif
 660
 661 void free_log(struct io_log *log)
 662 {
 663         while (!flist_empty(&log->io_logs)) {
 664                 struct io_logs *cur_log;
 665
 666                 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
 667                 flist_del_init(&cur_log->list);
 668                 free(cur_log->log);
 669                 sfree(cur_log);
 670         }
 671
 672         if (log->pending) {
 673                 free(log->pending->log);
 674                 free(log->pending);
 675                 log->pending = NULL;
 676         }
 677
 678         free(log->pending);
 679         free(log->filename);
 680         sfree(log);
 681 }
 682
 683 inline unsigned long hist_sum(int j, int stride, unsigned int *io_u_plat,
 684                 unsigned int *io_u_plat_last)
 685 {
 686         unsigned long sum;
 687         int k;
 688
 689         if (io_u_plat_last) {
 690                 for (k = sum = 0; k < stride; k++)
 691                         sum += io_u_plat[j + k] - io_u_plat_last[j + k];
 692         } else {
 693                 for (k = sum = 0; k < stride; k++)
 694                         sum += io_u_plat[j + k];
 695         }
 696
 697         return sum;
 698 }
 699
 700 static void flush_hist_samples(FILE *f, int hist_coarseness, void *samples,
 701                                uint64_t sample_size)
 702 {
 703         struct io_sample *s;
 704         int log_offset;
 705         uint64_t i, j, nr_samples;
 706         struct io_u_plat_entry *entry, *entry_before;
 707         unsigned int *io_u_plat;
 708         unsigned int *io_u_plat_before;
 709
 710         int stride = 1 << hist_coarseness;
 711
 712         if (!sample_size)
 713                 return;
 714
 715         s = __get_sample(samples, 0, 0);
 716         log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
 717
 718         nr_samples = sample_size / __log_entry_sz(log_offset);
 719
 720         for (i = 0; i < nr_samples; i++) {
 721                 s = __get_sample(samples, log_offset, i);
 722
 723                 entry = (struct io_u_plat_entry *) (uintptr_t) s->val;
 724                 io_u_plat = entry->io_u_plat;
 725
 726                 entry_before = flist_first_entry(&entry->list, struct io_u_plat_entry, list);
 727                 io_u_plat_before = entry_before->io_u_plat;
 728
 729                 fprintf(f, "%lu, %u, %u, ", (unsigned long) s->time,
 730                                                 io_sample_ddir(s), s->bs);
 731                 for (j = 0; j < FIO_IO_U_PLAT_NR - stride; j += stride) {
 732                         fprintf(f, "%lu, ", hist_sum(j, stride, io_u_plat,
 733                                                 io_u_plat_before));
 734                 }
 735                 fprintf(f, "%lu\n", (unsigned long)
 736                         hist_sum(FIO_IO_U_PLAT_NR - stride, stride, io_u_plat,
 737                                         io_u_plat_before));
 738
 739                 flist_del(&entry_before->list);
 740                 free(entry_before);
 741         }
 742 }
 743
 744 void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 745 {
 746         struct io_sample *s;
 747         int log_offset;
 748         uint64_t i, nr_samples;
 749
 750         if (!sample_size)
 751                 return;
 752
 753         s = __get_sample(samples, 0, 0);
 754         log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
 755
 756         nr_samples = sample_size / __log_entry_sz(log_offset);
 757
 758         for (i = 0; i < nr_samples; i++) {
 759                 s = __get_sample(samples, log_offset, i);
 760
 761                 if (!log_offset) {
 762                         fprintf(f, "%lu, %lu, %u, %u\n",
 763                                         (unsigned long) s->time,
 764                                         (unsigned long) s->val,
 765                                         io_sample_ddir(s), s->bs);
 766                 } else {
 767                         struct io_sample_offset *so = (void *) s;
 768
 769                         fprintf(f, "%lu, %lu, %u, %u, %llu\n",
 770                                         (unsigned long) s->time,
 771                                         (unsigned long) s->val,
 772                                         io_sample_ddir(s), s->bs,
 773                                         (unsigned long long) so->offset);
 774                 }
 775         }
 776 }
 777
 778 #ifdef CONFIG_ZLIB
 779
 780 struct iolog_flush_data {
 781         struct workqueue_work work;
 782         struct io_log *log;
 783         void *samples;
 784         uint32_t nr_samples;
 785         bool free;
 786 };
 787
 788 #define GZ_CHUNK        131072
 789
 790 static struct iolog_compress *get_new_chunk(unsigned int seq)
 791 {
 792         struct iolog_compress *c;
 793
 794         c = malloc(sizeof(*c));
 795         INIT_FLIST_HEAD(&c->list);
 796         c->buf = malloc(GZ_CHUNK);
 797         c->len = 0;
 798         c->seq = seq;
 799         return c;
 800 }
 801
 802 static void free_chunk(struct iolog_compress *ic)
 803 {
 804         free(ic->buf);
 805         free(ic);
 806 }
 807
 808 static int z_stream_init(z_stream *stream, int gz_hdr)
 809 {
 810         int wbits = 15;
 811
 812         memset(stream, 0, sizeof(*stream));
 813         stream->zalloc = Z_NULL;
 814         stream->zfree = Z_NULL;
 815         stream->opaque = Z_NULL;
 816         stream->next_in = Z_NULL;
 817
 818         /*
 819          * zlib magic - add 32 for auto-detection of gz header or not,
 820          * if we decide to store files in a gzip friendly format.
 821          */
 822         if (gz_hdr)
 823                 wbits += 32;
 824
 825         if (inflateInit2(stream, wbits) != Z_OK)
 826                 return 1;
 827
 828         return 0;
 829 }
 830
 831 struct inflate_chunk_iter {
 832         unsigned int seq;
 833         int err;
 834         void *buf;
 835         size_t buf_size;
 836         size_t buf_used;
 837         size_t chunk_sz;
 838 };
 839
 840 static void finish_chunk(z_stream *stream, FILE *f,
 841                          struct inflate_chunk_iter *iter)
 842 {
 843         int ret;
 844
 845         ret = inflateEnd(stream);
 846         if (ret != Z_OK)
 847                 log_err("fio: failed to end log inflation seq %d (%d)\n",
 848                                 iter->seq, ret);
 849
 850         flush_samples(f, iter->buf, iter->buf_used);
 851         free(iter->buf);
 852         iter->buf = NULL;
 853         iter->buf_size = iter->buf_used = 0;
 854 }
 855
 856 /*
 857  * Iterative chunk inflation. Handles cases where we cross into a new
 858  * sequence, doing flush finish of previous chunk if needed.
 859  */
 860 static size_t inflate_chunk(struct iolog_compress *ic, int gz_hdr, FILE *f,
 861                             z_stream *stream, struct inflate_chunk_iter *iter)
 862 {
 863         size_t ret;
 864
 865         dprint(FD_COMPRESS, "inflate chunk size=%lu, seq=%u\n",
 866                                 (unsigned long) ic->len, ic->seq);
 867
 868         if (ic->seq != iter->seq) {
 869                 if (iter->seq)
 870                         finish_chunk(stream, f, iter);
 871
 872                 z_stream_init(stream, gz_hdr);
 873                 iter->seq = ic->seq;
 874         }
 875
 876         stream->avail_in = ic->len;
 877         stream->next_in = ic->buf;
 878
 879         if (!iter->buf_size) {
 880                 iter->buf_size = iter->chunk_sz;
 881                 iter->buf = malloc(iter->buf_size);
 882         }
 883
 884         while (stream->avail_in) {
 885                 size_t this_out = iter->buf_size - iter->buf_used;
 886                 int err;
 887
 888                 stream->avail_out = this_out;
 889                 stream->next_out = iter->buf + iter->buf_used;
 890
 891                 err = inflate(stream, Z_NO_FLUSH);
 892                 if (err < 0) {
 893                         log_err("fio: failed inflating log: %d\n", err);
 894                         iter->err = err;
 895                         break;
 896                 }
 897
 898                 iter->buf_used += this_out - stream->avail_out;
 899
 900                 if (!stream->avail_out) {
 901                         iter->buf_size += iter->chunk_sz;
 902                         iter->buf = realloc(iter->buf, iter->buf_size);
 903                         continue;
 904                 }
 905
 906                 if (err == Z_STREAM_END)
 907                         break;
 908         }
 909
 910         ret = (void *) stream->next_in - ic->buf;
 911
 912         dprint(FD_COMPRESS, "inflated to size=%lu\n", (unsigned long) iter->buf_size);
 913
 914         return ret;
 915 }
 916
 917 /*
 918  * Inflate stored compressed chunks, or write them directly to the log
 919  * file if so instructed.
 920  */
 921 static int inflate_gz_chunks(struct io_log *log, FILE *f)
 922 {
 923         struct inflate_chunk_iter iter = { .chunk_sz = log->log_gz, };
 924         z_stream stream;
 925
 926         while (!flist_empty(&log->chunk_list)) {
 927                 struct iolog_compress *ic;
 928
 929                 ic = flist_first_entry(&log->chunk_list, struct iolog_compress, list);
 930                 flist_del(&ic->list);
 931
 932                 if (log->log_gz_store) {
 933                         size_t ret;
 934
 935                         dprint(FD_COMPRESS, "log write chunk size=%lu, "
 936                                 "seq=%u\n", (unsigned long) ic->len, ic->seq);
 937
 938                         ret = fwrite(ic->buf, ic->len, 1, f);
 939                         if (ret != 1 || ferror(f)) {
 940                                 iter.err = errno;
 941                                 log_err("fio: error writing compressed log\n");
 942                         }
 943                 } else
 944                         inflate_chunk(ic, log->log_gz_store, f, &stream, &iter);
 945
 946                 free_chunk(ic);
 947         }
 948
 949         if (iter.seq) {
 950                 finish_chunk(&stream, f, &iter);
 951                 free(iter.buf);
 952         }
 953
 954         return iter.err;
 955 }
 956
 957 /*
 958  * Open compressed log file and decompress the stored chunks and
 959  * write them to stdout. The chunks are stored sequentially in the
 960  * file, so we iterate over them and do them one-by-one.
 961  */
 962 int iolog_file_inflate(const char *file)
 963 {
 964         struct inflate_chunk_iter iter = { .chunk_sz = 64 * 1024 * 1024, };
 965         struct iolog_compress ic;
 966         z_stream stream;
 967         struct stat sb;
 968         ssize_t ret;
 969         size_t total;
 970         void *buf;
 971         FILE *f;
 972
 973         f = fopen(file, "r");
 974         if (!f) {
 975                 perror("fopen");
 976                 return 1;
 977         }
 978
 979         if (stat(file, &sb) < 0) {
 980                 fclose(f);
 981                 perror("stat");
 982                 return 1;
 983         }
 984
 985         ic.buf = buf = malloc(sb.st_size);
 986         ic.len = sb.st_size;
 987         ic.seq = 1;
 988
 989         ret = fread(ic.buf, ic.len, 1, f);
 990         if (ret < 0) {
 991                 perror("fread");
 992                 fclose(f);
 993                 free(buf);
 994                 return 1;
 995         } else if (ret != 1) {
 996                 log_err("fio: short read on reading log\n");
 997                 fclose(f);
 998                 free(buf);
 999                 return 1;
1000         }
1001
1002         fclose(f);
1003
1004         /*
1005          * Each chunk will return Z_STREAM_END. We don't know how many
1006          * chunks are in the file, so we just keep looping and incrementing
1007          * the sequence number until we have consumed the whole compressed
1008          * file.
1009          */
1010         total = ic.len;
1011         do {
1012                 size_t iret;
1013
1014                 iret = inflate_chunk(&ic,  1, stdout, &stream, &iter);
1015                 total -= iret;
1016                 if (!total)
1017                         break;
1018                 if (iter.err)
1019                         break;
1020
1021                 ic.seq++;
1022                 ic.len -= iret;
1023                 ic.buf += iret;
1024         } while (1);
1025
1026         if (iter.seq) {
1027                 finish_chunk(&stream, stdout, &iter);
1028                 free(iter.buf);
1029         }
1030
1031         free(buf);
1032         return iter.err;
1033 }
1034
1035 #else
1036
1037 static int inflate_gz_chunks(struct io_log *log, FILE *f)
1038 {
1039         return 0;
1040 }
1041
1042 int iolog_file_inflate(const char *file)
1043 {
1044         log_err("fio: log inflation not possible without zlib\n");
1045         return 1;
1046 }
1047
1048 #endif
1049
1050 void flush_log(struct io_log *log, bool do_append)
1051 {
1052         void *buf;
1053         FILE *f;
1054
1055         if (!do_append)
1056                 f = fopen(log->filename, "w");
1057         else
1058                 f = fopen(log->filename, "a");
1059         if (!f) {
1060                 perror("fopen log");
1061                 return;
1062         }
1063
1064         buf = set_file_buffer(f);
1065
1066         inflate_gz_chunks(log, f);
1067
1068         while (!flist_empty(&log->io_logs)) {
1069                 struct io_logs *cur_log;
1070
1071                 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
1072                 flist_del_init(&cur_log->list);
1073
1074                 if (log->td && log == log->td->clat_hist_log)
1075                         flush_hist_samples(f, log->hist_coarseness, cur_log->log,
1076                                            log_sample_sz(log, cur_log));
1077                 else
1078                         flush_samples(f, cur_log->log, log_sample_sz(log, cur_log));
1079
1080                 sfree(cur_log);
1081         }
1082
1083         fclose(f);
1084         clear_file_buffer(buf);
1085 }
1086
1087 static int finish_log(struct thread_data *td, struct io_log *log, int trylock)
1088 {
1089         if (td->flags & TD_F_COMPRESS_LOG)
1090                 iolog_flush(log);
1091
1092         if (trylock) {
1093                 if (fio_trylock_file(log->filename))
1094                         return 1;
1095         } else
1096                 fio_lock_file(log->filename);
1097
1098         if (td->client_type == FIO_CLIENT_TYPE_GUI || is_backend)
1099                 fio_send_iolog(td, log, log->filename);
1100         else
1101                 flush_log(log, !td->o.per_job_logs);
1102
1103         fio_unlock_file(log->filename);
1104         free_log(log);
1105         return 0;
1106 }
1107
1108 size_t log_chunk_sizes(struct io_log *log)
1109 {
1110         struct flist_head *entry;
1111         size_t ret;
1112
1113         if (flist_empty(&log->chunk_list))
1114                 return 0;
1115
1116         ret = 0;
1117         pthread_mutex_lock(&log->chunk_lock);
1118         flist_for_each(entry, &log->chunk_list) {
1119                 struct iolog_compress *c;
1120
1121                 c = flist_entry(entry, struct iolog_compress, list);
1122                 ret += c->len;
1123         }
1124         pthread_mutex_unlock(&log->chunk_lock);
1125         return ret;
1126 }
1127
1128 #ifdef CONFIG_ZLIB
1129
1130 static int gz_work(struct iolog_flush_data *data)
1131 {
1132         struct iolog_compress *c = NULL;
1133         struct flist_head list;
1134         unsigned int seq;
1135         z_stream stream;
1136         size_t total = 0;
1137         int ret;
1138
1139         INIT_FLIST_HEAD(&list);
1140
1141         memset(&stream, 0, sizeof(stream));
1142         stream.zalloc = Z_NULL;
1143         stream.zfree = Z_NULL;
1144         stream.opaque = Z_NULL;
1145
1146         ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
1147         if (ret != Z_OK) {
1148                 log_err("fio: failed to init gz stream\n");
1149                 goto err;
1150         }
1151
1152         seq = ++data->log->chunk_seq;
1153
1154         stream.next_in = (void *) data->samples;
1155         stream.avail_in = data->nr_samples * log_entry_sz(data->log);
1156
1157         dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u, log=%s\n",
1158                                 (unsigned long) stream.avail_in, seq,
1159                                 data->log->filename);
1160         do {
1161                 if (c)
1162                         dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq,
1163                                 (unsigned long) c->len);
1164                 c = get_new_chunk(seq);
1165                 stream.avail_out = GZ_CHUNK;
1166                 stream.next_out = c->buf;
1167                 ret = deflate(&stream, Z_NO_FLUSH);
1168                 if (ret < 0) {
1169                         log_err("fio: deflate log (%d)\n", ret);
1170                         free_chunk(c);
1171                         goto err;
1172                 }
1173
1174                 c->len = GZ_CHUNK - stream.avail_out;
1175                 flist_add_tail(&c->list, &list);
1176                 total += c->len;
1177         } while (stream.avail_in);
1178
1179         stream.next_out = c->buf + c->len;
1180         stream.avail_out = GZ_CHUNK - c->len;
1181
1182         ret = deflate(&stream, Z_FINISH);
1183         if (ret < 0) {
1184                 /*
1185                  * Z_BUF_ERROR is special, it just means we need more
1186                  * output space. We'll handle that below. Treat any other
1187                  * error as fatal.
1188                  */
1189                 if (ret != Z_BUF_ERROR) {
1190                         log_err("fio: deflate log (%d)\n", ret);
1191                         flist_del(&c->list);
1192                         free_chunk(c);
1193                         goto err;
1194                 }
1195         }
1196
1197         total -= c->len;
1198         c->len = GZ_CHUNK - stream.avail_out;
1199         total += c->len;
1200         dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq, (unsigned long) c->len);
1201
1202         if (ret != Z_STREAM_END) {
1203                 do {
1204                         c = get_new_chunk(seq);
1205                         stream.avail_out = GZ_CHUNK;
1206                         stream.next_out = c->buf;
1207                         ret = deflate(&stream, Z_FINISH);
1208                         c->len = GZ_CHUNK - stream.avail_out;
1209                         total += c->len;
1210                         flist_add_tail(&c->list, &list);
1211                         dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq,
1212                                 (unsigned long) c->len);
1213                 } while (ret != Z_STREAM_END);
1214         }
1215
1216         dprint(FD_COMPRESS, "deflated to size=%lu\n", (unsigned long) total);
1217
1218         ret = deflateEnd(&stream);
1219         if (ret != Z_OK)
1220                 log_err("fio: deflateEnd %d\n", ret);
1221
1222         free(data->samples);
1223
1224         if (!flist_empty(&list)) {
1225                 pthread_mutex_lock(&data->log->chunk_lock);
1226                 flist_splice_tail(&list, &data->log->chunk_list);
1227                 pthread_mutex_unlock(&data->log->chunk_lock);
1228         }
1229
1230         ret = 0;
1231 done:
1232         if (data->free)
1233                 free(data);
1234         return ret;
1235 err:
1236         while (!flist_empty(&list)) {
1237                 c = flist_first_entry(list.next, struct iolog_compress, list);
1238                 flist_del(&c->list);
1239                 free_chunk(c);
1240         }
1241         ret = 1;
1242         goto done;
1243 }
1244
1245 /*
1246  * Invoked from our compress helper thread, when logging would have exceeded
1247  * the specified memory limitation. Compresses the previously stored
1248  * entries.
1249  */
1250 static int gz_work_async(struct submit_worker *sw, struct workqueue_work *work)
1251 {
1252         return gz_work(container_of(work, struct iolog_flush_data, work));
1253 }
1254
1255 static int gz_init_worker(struct submit_worker *sw)
1256 {
1257         struct thread_data *td = sw->wq->td;
1258
1259         if (!fio_option_is_set(&td->o, log_gz_cpumask))
1260                 return 0;
1261
1262         if (fio_setaffinity(gettid(), td->o.log_gz_cpumask) == -1) {
1263                 log_err("gz: failed to set CPU affinity\n");
1264                 return 1;
1265         }
1266
1267         return 0;
1268 }
1269
1270 static struct workqueue_ops log_compress_wq_ops = {
1271         .fn             = gz_work_async,
1272         .init_worker_fn = gz_init_worker,
1273         .nice           = 1,
1274 };
1275
1276 int iolog_compress_init(struct thread_data *td, struct sk_out *sk_out)
1277 {
1278         if (!(td->flags & TD_F_COMPRESS_LOG))
1279                 return 0;
1280
1281         workqueue_init(td, &td->log_compress_wq, &log_compress_wq_ops, 1, sk_out);
1282         return 0;
1283 }
1284
1285 void iolog_compress_exit(struct thread_data *td)
1286 {
1287         if (!(td->flags & TD_F_COMPRESS_LOG))
1288                 return;
1289
1290         workqueue_exit(&td->log_compress_wq);
1291 }
1292
1293 /*
1294  * Queue work item to compress the existing log entries. We reset the
1295  * current log to a small size, and reference the existing log in the
1296  * data that we queue for compression. Once compression has been done,
1297  * this old log is freed. If called with finish == true, will not return
1298  * until the log compression has completed, and will flush all previous
1299  * logs too
1300  */
1301 static int iolog_flush(struct io_log *log)
1302 {
1303         struct iolog_flush_data *data;
1304
1305         data = malloc(sizeof(*data));
1306         if (!data)
1307                 return 1;
1308
1309         data->log = log;
1310         data->free = false;
1311
1312         while (!flist_empty(&log->io_logs)) {
1313                 struct io_logs *cur_log;
1314
1315                 cur_log = flist_first_entry(&log->io_logs, struct io_logs, list);
1316                 flist_del_init(&cur_log->list);
1317
1318                 data->samples = cur_log->log;
1319                 data->nr_samples = cur_log->nr_samples;
1320
1321                 sfree(cur_log);
1322
1323                 gz_work(data);
1324         }
1325
1326         free(data);
1327         return 0;
1328 }
1329
1330 int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log)
1331 {
1332         struct iolog_flush_data *data;
1333
1334         data = malloc(sizeof(*data));
1335         if (!data)
1336                 return 1;
1337
1338         data->log = log;
1339
1340         data->samples = cur_log->log;
1341         data->nr_samples = cur_log->nr_samples;
1342         data->free = true;
1343
1344         cur_log->nr_samples = cur_log->max_samples = 0;
1345         cur_log->log = NULL;
1346
1347         workqueue_enqueue(&log->td->log_compress_wq, &data->work);
1348         return 0;
1349 }
1350 #else
1351
1352 static int iolog_flush(struct io_log *log)
1353 {
1354         return 1;
1355 }
1356
1357 int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log)
1358 {
1359         return 1;
1360 }
1361
1362 int iolog_compress_init(struct thread_data *td, struct sk_out *sk_out)
1363 {
1364         return 0;
1365 }
1366
1367 void iolog_compress_exit(struct thread_data *td)
1368 {
1369 }
1370
1371 #endif
1372
1373 struct io_logs *iolog_cur_log(struct io_log *log)
1374 {
1375         if (flist_empty(&log->io_logs))
1376                 return NULL;
1377
1378         return flist_last_entry(&log->io_logs, struct io_logs, list);
1379 }
1380
1381 uint64_t iolog_nr_samples(struct io_log *iolog)
1382 {
1383         struct flist_head *entry;
1384         uint64_t ret = 0;
1385
1386         flist_for_each(entry, &iolog->io_logs) {
1387                 struct io_logs *cur_log;
1388
1389                 cur_log = flist_entry(entry, struct io_logs, list);
1390                 ret += cur_log->nr_samples;
1391         }
1392
1393         return ret;
1394 }
1395
1396 static int __write_log(struct thread_data *td, struct io_log *log, int try)
1397 {
1398         if (log)
1399                 return finish_log(td, log, try);
1400
1401         return 0;
1402 }
1403
1404 static int write_iops_log(struct thread_data *td, int try, bool unit_log)
1405 {
1406         int ret;
1407
1408         if (per_unit_log(td->iops_log) != unit_log)
1409                 return 0;
1410
1411         ret = __write_log(td, td->iops_log, try);
1412         if (!ret)
1413                 td->iops_log = NULL;
1414
1415         return ret;
1416 }
1417
1418 static int write_slat_log(struct thread_data *td, int try, bool unit_log)
1419 {
1420         int ret;
1421
1422         if (!unit_log)
1423                 return 0;
1424
1425         ret = __write_log(td, td->slat_log, try);
1426         if (!ret)
1427                 td->slat_log = NULL;
1428
1429         return ret;
1430 }
1431
1432 static int write_clat_log(struct thread_data *td, int try, bool unit_log)
1433 {
1434         int ret;
1435
1436         if (!unit_log)
1437                 return 0;
1438
1439         ret = __write_log(td, td->clat_log, try);
1440         if (!ret)
1441                 td->clat_log = NULL;
1442
1443         return ret;
1444 }
1445
1446 static int write_clat_hist_log(struct thread_data *td, int try, bool unit_log)
1447 {
1448         int ret;
1449
1450         if (!unit_log)
1451                 return 0;
1452
1453         ret = __write_log(td, td->clat_hist_log, try);
1454         if (!ret)
1455                 td->clat_hist_log = NULL;
1456
1457         return ret;
1458 }
1459
1460 static int write_lat_log(struct thread_data *td, int try, bool unit_log)
1461 {
1462         int ret;
1463
1464         if (!unit_log)
1465                 return 0;
1466
1467         ret = __write_log(td, td->lat_log, try);
1468         if (!ret)
1469                 td->lat_log = NULL;
1470
1471         return ret;
1472 }
1473
1474 static int write_bandw_log(struct thread_data *td, int try, bool unit_log)
1475 {
1476         int ret;
1477
1478         if (per_unit_log(td->bw_log) != unit_log)
1479                 return 0;
1480
1481         ret = __write_log(td, td->bw_log, try);
1482         if (!ret)
1483                 td->bw_log = NULL;
1484
1485         return ret;
1486 }
1487
1488 enum {
1489         BW_LOG_MASK     = 1,
1490         LAT_LOG_MASK    = 2,
1491         SLAT_LOG_MASK   = 4,
1492         CLAT_LOG_MASK   = 8,
1493         IOPS_LOG_MASK   = 16,
1494         CLAT_HIST_LOG_MASK = 32,
1495
1496         ALL_LOG_NR      = 6,
1497 };
1498
1499 struct log_type {
1500         unsigned int mask;
1501         int (*fn)(struct thread_data *, int, bool);
1502 };
1503
1504 static struct log_type log_types[] = {
1505         {
1506                 .mask   = BW_LOG_MASK,
1507                 .fn     = write_bandw_log,
1508         },
1509         {
1510                 .mask   = LAT_LOG_MASK,
1511                 .fn     = write_lat_log,
1512         },
1513         {
1514                 .mask   = SLAT_LOG_MASK,
1515                 .fn     = write_slat_log,
1516         },
1517         {
1518                 .mask   = CLAT_LOG_MASK,
1519                 .fn     = write_clat_log,
1520         },
1521         {
1522                 .mask   = IOPS_LOG_MASK,
1523                 .fn     = write_iops_log,
1524         },
1525         {
1526                 .mask   = CLAT_HIST_LOG_MASK,
1527                 .fn     = write_clat_hist_log,
1528         }
1529 };
1530
1531 void td_writeout_logs(struct thread_data *td, bool unit_logs)
1532 {
1533         unsigned int log_mask = 0;
1534         unsigned int log_left = ALL_LOG_NR;
1535         int old_state, i;
1536
1537         old_state = td_bump_runstate(td, TD_FINISHING);
1538
1539         finalize_logs(td, unit_logs);
1540
1541         while (log_left) {
1542                 int prev_log_left = log_left;
1543
1544                 for (i = 0; i < ALL_LOG_NR && log_left; i++) {
1545                         struct log_type *lt = &log_types[i];
1546                         int ret;
1547
1548                         if (!(log_mask & lt->mask)) {
1549                                 ret = lt->fn(td, log_left != 1, unit_logs);
1550                                 if (!ret) {
1551                                         log_left--;
1552                                         log_mask |= lt->mask;
1553                                 }
1554                         }
1555                 }
1556
1557                 if (prev_log_left == log_left)
1558                         usleep(5000);
1559         }
1560
1561         td_restore_runstate(td, old_state);
1562 }
1563
1564 void fio_writeout_logs(bool unit_logs)
1565 {
1566         struct thread_data *td;
1567         int i;
1568
1569         for_each_td(td, i)
1570                 td_writeout_logs(td, unit_logs);
1571 }