Merge branch 'master' into gfio
authorJens Axboe <axboe@kernel.dk>
Wed, 20 Feb 2013 10:52:28 +0000 (11:52 +0100)
committerJens Axboe <axboe@kernel.dk>
Wed, 20 Feb 2013 10:52:28 +0000 (11:52 +0100)
Conflicts:
configure
fio.c
init.c

Signed-off-by: Jens Axboe <axboe@kernel.dk>
1  2 
Makefile
backend.c
configure
init.c
iolog.c

diff --combined Makefile
index aff4fba2251b2a2a387a7ef6557dd86e4e2d704e,bb345ed9899bf7ca44b8ffa646e3e026ffe213ff..8eb2f45d005c4fd56b8189b0698dca40f91d6a56
+++ b/Makefile
@@@ -2,7 -2,7 +2,7 @@@ DEBUGFLAGS = -D_FORTIFY_SOURCE=2 -DFIO_
  CPPFLAGS= -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 $(DEBUGFLAGS)
  OPTFLAGS= -O3 -g -ffast-math $(EXTFLAGS)
  CFLAGS        = -std=gnu99 -Wwrite-strings -Wall $(OPTFLAGS)
 -LIBS  = -lm $(EXTLIBS)
 +LIBS  = -lm -lz $(EXTLIBS)
  PROGS = fio
  SCRIPTS = fio_generate_plots
  UNAME  := $(shell uname)
@@@ -21,19 -21,15 +21,19 @@@ all
  include config-host.mak
  endif
  
 -SOURCE := gettime.c fio.c ioengines.c init.c stat.c log.c time.c filesetup.c \
 +ifdef CONFIG_GFIO
 +  PROGS += gfio
 +endif
 +
 +SOURCE := gettime.c ioengines.c init.c stat.c log.c time.c filesetup.c \
                eta.c verify.c memory.c io_u.c parse.c mutex.c options.c \
 -              rbtree.c smalloc.c filehash.c profile.c debug.c lib/rand.c \
 +              lib/rbtree.c smalloc.c filehash.c profile.c debug.c lib/rand.c \
                lib/num2str.c lib/ieee754.c $(wildcard crc/*.c) engines/cpu.c \
                engines/mmap.c engines/sync.c engines/null.c engines/net.c \
                memalign.c server.c client.c iolog.c backend.c libfio.c flow.c \
 -              json.c lib/zipf.c lib/axmap.c lib/lfsr.c gettime-thread.c \
 -              helpers.c lib/flist_sort.c lib/hweight.c lib/getrusage.c \
 -              idletime.c
 +              cconv.c lib/prio_tree.c json.c lib/zipf.c lib/axmap.c \
 +              lib/lfsr.c gettime-thread.c helpers.c lib/flist_sort.c \
 +              lib/hweight.c lib/getrusage.c idletime.c
  
  ifdef CONFIG_64BIT_LLP64
    CFLAGS += -DBITS_PER_LONG=32
@@@ -94,7 -90,6 +94,6 @@@ ifeq ($(UNAME), Android
    SOURCE += diskutil.c fifo.c blktrace.c trim.c profiles/tiobench.c
    LIBS += -ldl
    LDFLAGS += -rdynamic
-   CPPFLAGS += -DFIO_NO_HAVE_SHM_H
  endif
  ifeq ($(UNAME), SunOS)
    LIBS         += -lpthread -ldl -laio -lrt -lnsl -lsocket
@@@ -128,11 -123,6 +127,11 @@@ ifneq (,$(findstring CYGWIN,$(UNAME))
  endif
  
  OBJS = $(SOURCE:.c=.o)
 +
 +FIO_OBJS = $(OBJS) fio.o
 +GFIO_OBJS = $(OBJS) gfio.o graph.o tickmarks.o ghelpers.o goptions.o gerror.o \
 +                      gclient.o gcompat.o cairo_text_helpers.o printing.o
 +
  -include $(OBJS:.o=.d)
  
  T_SMALLOC_OBJS = t/stest.o
@@@ -202,53 -192,23 +201,53 @@@ override CFLAGS += -DFIO_VERSION='"$(FI
  init.o: FIO-VERSION-FILE init.c
        $(QUIET_CC)$(CC) -o init.o $(CFLAGS) $(CPPFLAGS) -c init.c
  
 +gcompat.o: gcompat.c gcompat.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c gcompat.c
 +
 +goptions.o: goptions.c goptions.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c goptions.c
 +
 +ghelpers.o: ghelpers.c ghelpers.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c ghelpers.c
 +
 +gerror.o: gerror.c gerror.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c gerror.c
 +
 +gclient.o: gclient.c gclient.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c gclient.c
 +
 +gfio.o: gfio.c ghelpers.c
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c gfio.c
 +
 +graph.o: graph.c graph.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c graph.c
 +
 +cairo_text_helpers.o: cairo_text_helpers.c cairo_text_helpers.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c cairo_text_helpers.c
 +
 +printing.o: printing.c printing.h
 +      $(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c printing.c
 +
  t/stest: $(T_SMALLOC_OBJS)
        $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_SMALLOC_OBJS) $(LIBS) $(LDFLAGS)
  
  t/ieee754: $(T_IEEE_OBJS)
        $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_IEEE_OBJS) $(LIBS) $(LDFLAGS)
  
 +fio: $(FIO_OBJS)
 +      $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(FIO_OBJS) $(LIBS) $(LDFLAGS)
 +
 +gfio: $(GFIO_OBJS)
 +      $(QUIET_LINK)$(CC) $(LIBS) -o gfio $(GFIO_OBJS) $(LIBS) $(GTK_LDFLAGS)
 +
  t/genzipf: $(T_ZIPF_OBJS)
        $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_ZIPF_OBJS) $(LIBS) $(LDFLAGS)
  
  t/axmap: $(T_AXMAP_OBJS)
        $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_AXMAP_OBJS) $(LIBS) $(LDFLAGS)
  
 -fio: $(OBJS)
 -      $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(OBJS) $(LIBS) $(LDFLAGS)
 -
  clean: FORCE
 -      -rm -f .depend $(OBJS) $(T_OBJS) $(PROGS) $(T_PROGS) core.* core FIO-VERSION-FILE config-host.mak cscope.out *.d
 +      -rm -f .depend $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(PROGS) $(T_PROGS) core.* core gfio FIO-VERSION-FILE config-host.mak cscope.out *.d
  
  cscope:
        @cscope -b -R
diff --combined backend.c
index 592ea8bfaa05cc17b4a8a3e23c1cd54b5aebea91,d1fe9641d88ebaa47489fc10a8f039e9443bbc10..4e8a4389a642d3c5b269fb235264f7c27bd367e1
+++ b/backend.c
@@@ -61,14 -61,14 +61,14 @@@ static struct flist_head *cgroup_list
  static char *cgroup_mnt;
  static int exit_value;
  static volatile int fio_abort;
 +static unsigned int nr_process = 0;
 +static unsigned int nr_thread = 0;
  
  struct io_log *agg_io_log[DDIR_RWDIR_CNT];
  
  int groupid = 0;
  unsigned int thread_number = 0;
  unsigned int stat_number = 0;
 -unsigned int nr_process = 0;
 -unsigned int nr_thread = 0;
  int shm_id = 0;
  int temp_stall_ts;
  unsigned long done_secs = 0;
@@@ -322,6 -322,21 +322,21 @@@ requeue
        return 0;
  }
  
+ static int fio_file_fsync(struct thread_data *td, struct fio_file *f)
+ {
+       int ret;
+       if (fio_file_open(f))
+               return fio_io_sync(td, f);
+       if (td_io_open_file(td, f))
+               return 1;
+       ret = fio_io_sync(td, f);
+       td_io_close_file(td, f);
+       return ret;
+ }
  static inline void __update_tv_cache(struct thread_data *td)
  {
        fio_gettime(&td->tv_cache, NULL);
@@@ -822,9 -837,11 +837,11 @@@ sync_done
                        td_set_runstate(td, TD_FSYNCING);
  
                        for_each_file(td, f, i) {
-                               if (!fio_file_open(f))
+                               if (!fio_file_fsync(td, f))
                                        continue;
-                               fio_io_sync(td, f);
+                               log_err("fio: end_fsync failed for file %s\n",
+                                                               f->file_name);
                        }
                }
        } else
@@@ -1058,12 -1075,10 +1075,12 @@@ static void *thread_main(void *data
  {
        unsigned long long elapsed;
        struct thread_data *td = data;
 +      struct thread_options *o = &td->o;
        pthread_condattr_t attr;
        int clear_state;
 +      int ret;
  
 -      if (!td->o.use_thread) {
 +      if (!o->use_thread) {
                setsid();
                td->pid = getpid();
        } else
  
        dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid);
  
 +      if (is_backend)
 +              fio_server_send_start(td);
 +
        INIT_FLIST_HEAD(&td->io_u_freelist);
        INIT_FLIST_HEAD(&td->io_u_busylist);
        INIT_FLIST_HEAD(&td->io_u_requeues);
         * eating a file descriptor
         */
        fio_mutex_remove(td->mutex);
 +      td->mutex = NULL;
  
        /*
         * A new gid requires privilege, so we need to do this before setting
         * the uid.
         */
 -      if (td->o.gid != -1U && setgid(td->o.gid)) {
 +      if (o->gid != -1U && setgid(o->gid)) {
                td_verror(td, errno, "setgid");
                goto err;
        }
 -      if (td->o.uid != -1U && setuid(td->o.uid)) {
 +      if (o->uid != -1U && setuid(o->uid)) {
                td_verror(td, errno, "setuid");
                goto err;
        }
         * If we have a gettimeofday() thread, make sure we exclude that
         * thread from this job
         */
 -      if (td->o.gtod_cpu)
 -              fio_cpu_clear(&td->o.cpumask, td->o.gtod_cpu);
 +      if (o->gtod_cpu)
 +              fio_cpu_clear(&o->cpumask, o->gtod_cpu);
  
        /*
         * Set affinity first, in case it has an impact on the memory
         * allocations.
         */
 -      if (td->o.cpumask_set && fio_setaffinity(td->pid, td->o.cpumask) == -1) {
 -              td_verror(td, errno, "cpu_set_affinity");
 -              goto err;
 +      if (o->cpumask_set) {
 +              ret = fio_setaffinity(td->pid, o->cpumask);
 +              if (ret == -1) {
 +                      td_verror(td, errno, "cpu_set_affinity");
 +                      goto err;
 +              }
        }
  
  #ifdef CONFIG_LIBNUMA
        if (init_io_u(td))
                goto err;
  
 -      if (td->o.verify_async && verify_async_init(td))
 +      if (o->verify_async && verify_async_init(td))
                goto err;
  
 -      if (td->ioprio_set) {
 -              if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
 +      if (o->ioprio) {
 +              ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class, o->ioprio);
 +              if (ret == -1) {
                        td_verror(td, errno, "ioprio_set");
                        goto err;
                }
                goto err;
  
        errno = 0;
 -      if (nice(td->o.nice) == -1 && errno != 0) {
 +      if (nice(o->nice) == -1 && errno != 0) {
                td_verror(td, errno, "nice");
                goto err;
        }
  
 -      if (td->o.ioscheduler && switch_ioscheduler(td))
 +      if (o->ioscheduler && switch_ioscheduler(td))
                goto err;
  
 -      if (!td->o.create_serialize && setup_files(td))
 +      if (!o->create_serialize && setup_files(td))
                goto err;
  
        if (td_io_init(td))
        if (init_random_map(td))
                goto err;
  
 -      if (td->o.exec_prerun) {
 -              if (exec_string(td->o.exec_prerun))
 -                      goto err;
 -      }
 +      if (o->exec_prerun && exec_string(o->exec_prerun))
 +              goto err;
  
 -      if (td->o.pre_read) {
 +      if (o->pre_read) {
                if (pre_read_files(td) < 0)
                        goto err;
        }
  
 +      fio_verify_init(td);
 +
        fio_gettime(&td->epoch, NULL);
        fio_getrusage(&td->ru_start);
        clear_state = 0;
        td->ts.io_bytes[DDIR_WRITE] = td->io_bytes[DDIR_WRITE];
        td->ts.io_bytes[DDIR_TRIM] = td->io_bytes[DDIR_TRIM];
  
 +      fio_unpin_memory(td);
 +
        fio_mutex_down(writeout_mutex);
        if (td->bw_log) {
                if (td->o.bw_log_file) {
@@@ -1363,8 -1368,8 +1380,8 @@@ err
        close_ioengine(td);
        cgroup_shutdown(td, &cgroup_mnt);
  
 -      if (td->o.cpumask_set) {
 -              int ret = fio_cpuset_exit(&td->o.cpumask);
 +      if (o->cpumask_set) {
 +              int ret = fio_cpuset_exit(&o->cpumask);
  
                td_verror(td, ret, "fio_cpuset_exit");
        }
@@@ -1518,6 -1523,9 +1535,6 @@@ static void run_threads(void
        unsigned long spent;
        unsigned int i, todo, nr_running, m_rate, t_rate, nr_started;
  
 -      if (fio_pin_memory())
 -              return;
 -
        if (fio_gtod_offload && fio_start_gtod_thread())
                return;
        
  
        set_sig_handlers();
  
 +      nr_thread = nr_process = 0;
 +      for_each_td(td, i) {
 +              if (td->o.use_thread)
 +                      nr_thread++;
 +              else
 +                      nr_process++;
 +      }
 +
        if (output_format == FIO_OUTPUT_NORMAL) {
                log_info("Starting ");
                if (nr_thread)
  
                reap_threads(&nr_running, &t_rate, &m_rate);
  
 -              if (todo) {
 -                      if (is_backend)
 -                              fio_server_idle_loop();
 -                      else
 -                              usleep(100000);
 -              }
 +              if (todo)
 +                      usleep(100000);
        }
  
        while (nr_running) {
                reap_threads(&nr_running, &t_rate, &m_rate);
 -
 -              if (is_backend)
 -                      fio_server_idle_loop();
 -              else
 -                      usleep(10000);
 +              usleep(10000);
        }
  
        fio_idle_prof_stop();
  
        update_io_ticks();
 -      fio_unpin_memory();
  }
  
  void wait_for_disk_thread_exit(void)
@@@ -1824,9 -1833,9 +1841,9 @@@ int fio_backend(void
                return 0;
  
        if (write_bw_log) {
 -              setup_log(&agg_io_log[DDIR_READ], 0);
 -              setup_log(&agg_io_log[DDIR_WRITE], 0);
 -              setup_log(&agg_io_log[DDIR_TRIM], 0);
 +              setup_log(&agg_io_log[DDIR_READ], 0, IO_LOG_TYPE_BW);
 +              setup_log(&agg_io_log[DDIR_WRITE], 0, IO_LOG_TYPE_BW);
 +              setup_log(&agg_io_log[DDIR_TRIM], 0, IO_LOG_TYPE_BW);
        }
  
        startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
diff --combined configure
index 1c8b2bb0d78821979e0c79430e36cf6292a8e8dc,caaf30e05608adfe1e48a2b872ed816ac36a175c..71faf9184a9e9fb57a386f79ebd55868facf2403
+++ b/configure
@@@ -29,6 -29,14 +29,14 @@@ config_host_h="config-host.h
  rm -rf $config_host_mak
  rm -rf $config_host_h
  
+ fatal() {
+   echo $@
+   echo "Configure failed, check config.log and/or the above output"
+   rm -rf $config_host_mak
+   rm -rf $config_host_h
+   exit 1
+ }
  # Default CFLAGS
  CFLAGS="-D_GNU_SOURCE"
  EXTFLAGS="-include config-host.h"
@@@ -70,8 -78,7 +78,7 @@@ do_cc() 
      echo "ERROR: configure test passed without -Werror but failed with -Werror."
      echo "This is probably a bug in the configure script. The failing command"
      echo "will be at the bottom of config.log."
-     echo "You can run configure with --disable-werror to bypass this check."
-     exit 1
+     fatal "You can run configure with --disable-werror to bypass this check."
  }
  
  compile_object() {
@@@ -91,8 -98,7 +98,7 @@@ feature_not_found() 
    echo "ERROR"
    echo "ERROR: User requested feature $feature"
    echo "ERROR: configure was not able to find it"
-   echo "ERROR"
-   exit 1;
+   fatal "ERROR"
  }
  
  has() {
@@@ -122,29 -128,24 +128,29 @@@ cpu="
  
  cc="${CC-${cross_prefix}gcc}"
  
 +# default options
  show_help="no"
  exit_val=0
 +gfio="no"
  
  # parse options
  for opt do
    optarg=`expr "x$opt" : 'x[^=]*=\(.*\)'`
    case "$opt" in
-   --cc=*)
-     CC="$optarg"
-     ;;
-   --extra-cflags=*)
-   CFLAGS="$CFLAGS $optarg"
+   --cpu=*) cpu="$optarg"
+   ;;
+   --cc=*) CC="$optarg"
+   ;;
+   --extra-cflags=*) CFLAGS="$CFLAGS $optarg"
    ;;
    --build-32bit-win=*) build_32bit_win="$optarg"
    ;;
 -  --help)
 -  show_help="yes"
 +  --enable-gfio)
 +  gfio="yes"
    ;;
 +  --help)
 +    show_help="yes"
 +    ;;
    *)
    echo "Bad option $opt"
    show_help="yes"
  done
  
  if test "$show_help" = "yes" ; then
+   echo "--cpu=                 Specify target CPU if auto-detect fails"
    echo "--cc=                  Specify compiler to use"
    echo "--extra-cflags=        Specify extra CFLAGS to pass to compiler"
    echo "--build-32bit-win=     Specify yes for a 32-bit build on Windows"
 +  echo "--enable-gfio          Enable building of gtk gfio"
    exit $exit_val
  fi
  
- if check_define __linux__ ; then
+ if check_define __ANDROID__ ; then
+   targetos="Android"
+ elif check_define __linux__ ; then
    targetos="Linux"
  elif check_define __OpenBSD__ ; then
    targetos='OpenBSD'
@@@ -221,6 -224,17 +230,17 @@@ CYGWIN*
    echo "CC=$CC" >> $config_host_mak
    echo "EXTFLAGS=$CFLAGS -include config-host.h -D_GNU_SOURCE" >> $config_host_mak
    exit 0
+   ;;
+ Android)
+   output_sym "CONFIG_32BIT"
+   output_sym "CONFIG_LITTLE_ENDIAN"
+   output_sym "CONFIG_SOCKLEN_T"
+   output_sym "CONFIG_GETTIMEOFDAY"
+   output_sym "CONFIG_CLOCK_GETTIME"
+   output_sym "CONFIG_CLOCK_MONOTONIC"
+   echo "CC=$cc" >> $config_host_mak
+   echo "EXTFLAGS=$CFLAGS -include config-host.h -DFIO_NO_HAVE_SHM_H -D_GNU_SOURCE" >> $config_host_mak
+   exit 0
  esac
  
  if test ! -z "$cpu" ; then
@@@ -284,8 -298,7 +304,7 @@@ case "$cpu" i
      cpu="sparc"
    ;;
    *)
-     echo "Unknown CPU"
-     exit 1;
+   echo "Unknown CPU"
    ;;
  esac
  
  echo "__thread                      $tls_thread"
  
  ##########################################
 +# Whether or not __thread is supported for TLS
 +if test "$gfio" = "yes" ; then
 +  cat > $TMPC << EOF
 +#include <glib.h>
 +#include <cairo.h>
 +#include <gtk/gtk.h>
 +int main(void)
 +{
 +  gdk_threads_enter();
 +  gdk_threads_leave();
 +
 +  printf("%d", GTK_CHECK_VERSION(2, 18, 0));
 +}
 +EOF
 +GTK_CFLAGS=$(pkg-config --cflags gtk+-2.0 gthread-2.0)
 +if test "$?" != "0" ; then
 +  echo "configure: gtk and gthread not found"
 +  exit 1
 +fi
 +GTK_LIBS=$(pkg-config --libs gtk+-2.0 gthread-2.0)
 +if test "$?" != "0" ; then
 +  echo "configure: gtk and gthread not found"
 +  exit 1
 +fi
 +if compile_prog "$GTK_CFLAGS" "$GTK_LIBS" "gfio" ; then
 +  r=$($TMPE)
 +  if test "$r" != "0" ; then
 +    gfio="yes"
 +    LIBS="$LIBS $GTK_LIBS"
 +    CFLAGS="$CFLAGS $GTK_CFLAGS"
 +  else
 +    echo "GTK found, but need version 2.18 or higher"
 +    gfio="no"
 +  fi
 +else
 +  echo "Please install gtk and gdk libraries"
 +  gfio="no"
 +fi
 +fi
 +
 +echo "gfio                          $gfio"
 +
  # Check whether we have getrusage(RUSAGE_THREAD)
  rusage_thread="no"
  cat > $TMPC << EOF
@@@ -950,8 -921,7 +969,7 @@@ if test "$wordsize" = "64" ; the
  elif test "$wordsize" = "32" ; then
    output_sym "CONFIG_32BIT"
  else
-   echo "Unknown wordsize!"
-   exit 1
+   fatal "Unknown wordsize!"
  fi
  if test "$bigendian" = "yes" ; then
    output_sym "CONFIG_BIG_ENDIAN"
  if test "$rusage_thread" = "yes" ; then
    output_sym "CONFIG_RUSAGE_THREAD"
  fi
 +if test "$gfio" = "yes" ; then
 +  echo "CONFIG_GFIO=y" >> $config_host_mak
 +fi
  if test "$sched_idle" = "yes" ; then
    output_sym "CONFIG_SCHED_IDLE"
  fi
@@@ -1052,6 -1019,5 +1070,6 @@@ if test "$tcp_nodelay" = "yes" ; the
  fi
  
  echo "LIBS+=$LIBS" >> $config_host_mak
 +echo "CFLAGS+=$CFLAGS" >> $config_host_mak
  echo "CC=$cc" >> $config_host_mak
  echo "EXTFLAGS=$EXTFLAGS $CFLAGS" >> $config_host_mak
diff --combined init.c
index ce699dfcd6ca07caf7748664ef502eec2dd97423,4709ff29146292a41b6994b450b68730ce8a3032..8ec1eb978e90910620452c34015ec1a42923fa55
--- 1/init.c
--- 2/init.c
+++ b/init.c
@@@ -40,7 -40,8 +40,7 @@@ struct thread_data *threads = NULL
  
  int exitall_on_terminate = 0;
  int output_format = FIO_OUTPUT_NORMAL;
- int eta_print;
+ int eta_print = FIO_ETA_AUTO;
 -unsigned long long mlock_size = 0;
  FILE *f_out = NULL;
  FILE *f_err = NULL;
  char **job_sections = NULL;
@@@ -202,7 -203,7 +202,7 @@@ static struct option l_opts[FIO_NR_OPTI
        },
  };
  
 -static void free_shm(void)
 +void free_threads_shm(void)
  {
        struct shmid_ds sbuf;
  
                void *tp = threads;
  
                threads = NULL;
 +              shmdt(tp);
 +              shmctl(shm_id, IPC_RMID, &sbuf);
 +              shm_id = -1;
 +      }
 +}
 +
 +void free_shm(void)
 +{
 +      if (threads) {
                file_hash_exit();
                flow_exit();
                fio_debug_jobp = NULL;
 -              shmdt(tp);
 -              shmctl(shm_id, IPC_RMID, &sbuf);
 +              free_threads_shm();
        }
  
        scleanup();
@@@ -600,7 -593,7 +600,7 @@@ static int fixup_options(struct thread_
  /*
   * This function leaks the buffer
   */
 -static char *to_kmg(unsigned int val)
 +char *fio_uint_to_kmg(unsigned int val)
  {
        char *buf = malloc(32);
        char post[] = { 0, 'K', 'M', 'G', 'P', 'E', 0 };
@@@ -798,9 -791,11 +798,9 @@@ static int setup_random_seeds(struct th
   * to make sure we don't have conflicts, and initializes various
   * members of td.
   */
 -static int add_job(struct thread_data *td, const char *jobname, int job_add_num)
 +static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
 +                 int recursed, int client_type)
  {
 -      const char *ddir_str[] = { NULL, "read", "write", "rw", NULL,
 -                                 "randread", "randwrite", "randrw",
 -                                 "trim", NULL, NULL, NULL, "randtrim" };
        unsigned int i;
        char fname[PATH_MAX];
        int numjobs, file_alloced;
                return 0;
        }
  
 +      td->client_type = client_type;
 +
        if (profile_td_init(td))
                goto err;
  
        if (ioengine_load(td))
                goto err;
  
 -      if (td->o.use_thread)
 -              nr_thread++;
 -      else
 -              nr_process++;
 -
        if (td->o.odirect)
                td->io_ops->flags |= FIO_RAWIO;
  
        if (setup_rate(td))
                goto err;
  
 -      if (td->o.write_lat_log) {
 -              setup_log(&td->lat_log, td->o.log_avg_msec);
 -              setup_log(&td->slat_log, td->o.log_avg_msec);
 -              setup_log(&td->clat_log, td->o.log_avg_msec);
 +      if (td->o.lat_log_file) {
 +              setup_log(&td->lat_log, td->o.log_avg_msec, IO_LOG_TYPE_LAT);
 +              setup_log(&td->slat_log, td->o.log_avg_msec, IO_LOG_TYPE_SLAT);
 +              setup_log(&td->clat_log, td->o.log_avg_msec, IO_LOG_TYPE_CLAT);
        }
 -      if (td->o.write_bw_log)
 -              setup_log(&td->bw_log, td->o.log_avg_msec);
 -      if (td->o.write_iops_log)
 -              setup_log(&td->iops_log, td->o.log_avg_msec);
 +      if (td->o.bw_log_file)
 +              setup_log(&td->bw_log, td->o.log_avg_msec, IO_LOG_TYPE_BW);
 +      if (td->o.iops_log_file)
 +              setup_log(&td->iops_log, td->o.log_avg_msec, IO_LOG_TYPE_IOPS);
  
        if (!td->o.name)
                td->o.name = strdup(jobname);
  
        if (output_format == FIO_OUTPUT_NORMAL) {
                if (!job_add_num) {
 -                      if (!strcmp(td->io_ops->name, "cpuio")) {
 -                              log_info("%s: ioengine=cpu, cpuload=%u,"
 -                                       " cpucycle=%u\n", td->o.name,
 -                                                      td->o.cpuload,
 -                                                      td->o.cpucycle);
 -                      } else {
 +                      if (is_backend && !recursed)
 +                              fio_server_send_add_job(td);
 +
 +                      if (!(td->io_ops->flags & FIO_NOIO)) {
                                char *c1, *c2, *c3, *c4, *c5, *c6;
  
 -                              c1 = to_kmg(td->o.min_bs[DDIR_READ]);
 -                              c2 = to_kmg(td->o.max_bs[DDIR_READ]);
 -                              c3 = to_kmg(td->o.min_bs[DDIR_WRITE]);
 -                              c4 = to_kmg(td->o.max_bs[DDIR_WRITE]);
 -                              c5 = to_kmg(td->o.min_bs[DDIR_TRIM]);
 -                              c6 = to_kmg(td->o.max_bs[DDIR_TRIM]);
 +                              c1 = fio_uint_to_kmg(td->o.min_bs[DDIR_READ]);
 +                              c2 = fio_uint_to_kmg(td->o.max_bs[DDIR_READ]);
 +                              c3 = fio_uint_to_kmg(td->o.min_bs[DDIR_WRITE]);
 +                              c4 = fio_uint_to_kmg(td->o.max_bs[DDIR_WRITE]);
 +                              c5 = fio_uint_to_kmg(td->o.min_bs[DDIR_TRIM]);
 +                              c6 = fio_uint_to_kmg(td->o.max_bs[DDIR_TRIM]);
  
                                log_info("%s: (g=%d): rw=%s, bs=%s-%s/%s-%s/%s-%s,"
                                         " ioengine=%s, iodepth=%u\n",
                                                td->o.name, td->groupid,
 -                                              ddir_str[td->o.td_ddir],
 +                                              ddir_str(td->o.td_ddir),
                                                c1, c2, c3, c4, c5, c6,
                                                td->io_ops->name,
                                                td->o.iodepth);
  
                job_add_num = numjobs - 1;
  
 -              if (add_job(td_new, jobname, job_add_num))
 +              if (add_job(td_new, jobname, job_add_num, 1, client_type))
                        goto err;
        }
  
@@@ -980,7 -980,7 +980,7 @@@ err
  /*
   * Parse as if 'o' was a command line
   */
 -void add_job_opts(const char **o)
 +void add_job_opts(const char **o, int client_type)
  {
        struct thread_data *td, *td_parent;
        int i, in_global = 1;
                if (!strncmp(o[i], "name", 4)) {
                        in_global = 0;
                        if (td)
 -                              add_job(td, jobname, 0);
 +                              add_job(td, jobname, 0, 0, client_type);
                        td = NULL;
                        sprintf(jobname, "%s", o[i] + 5);
                }
        }
  
        if (td)
 -              add_job(td, jobname, 0);
 +              add_job(td, jobname, 0, 0, client_type);
  }
  
  static int skip_this_section(const char *name)
@@@ -1049,7 -1049,7 +1049,7 @@@ static int is_empty_or_comment(char *li
  /*
   * This is our [ini] type file parser.
   */
 -int parse_jobs_ini(char *file, int is_buf, int stonewall_flag)
 +int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type)
  {
        unsigned int global;
        struct thread_data *td;
                                for (i = 0; i < num_opts; i++)
                                        log_info("--%s ", opts[i]);
  
 -                      ret = add_job(td, name, 0);
 +                      ret = add_job(td, name, 0, 0, type);
                } else {
                        log_err("fio: job %s dropped\n", name);
                        put_job(td);
@@@ -1283,62 -1283,20 +1283,62 @@@ static void usage(const char *name
  
  #ifdef FIO_INC_DEBUG
  struct debug_level debug_levels[] = {
 -      { .name = "process",    .shift = FD_PROCESS, },
 -      { .name = "file",       .shift = FD_FILE, },
 -      { .name = "io",         .shift = FD_IO, },
 -      { .name = "mem",        .shift = FD_MEM, },
 -      { .name = "blktrace",   .shift = FD_BLKTRACE },
 -      { .name = "verify",     .shift = FD_VERIFY },
 -      { .name = "random",     .shift = FD_RANDOM },
 -      { .name = "parse",      .shift = FD_PARSE },
 -      { .name = "diskutil",   .shift = FD_DISKUTIL },
 -      { .name = "job",        .shift = FD_JOB },
 -      { .name = "mutex",      .shift = FD_MUTEX },
 -      { .name = "profile",    .shift = FD_PROFILE },
 -      { .name = "time",       .shift = FD_TIME },
 -      { .name = "net",        .shift = FD_NET },
 +      { .name = "process",
 +        .help = "Process creation/exit logging",
 +        .shift = FD_PROCESS,
 +      },
 +      { .name = "file",
 +        .help = "File related action logging",
 +        .shift = FD_FILE,
 +      },
 +      { .name = "io",
 +        .help = "IO and IO engine action logging (offsets, queue, completions, etc)",
 +        .shift = FD_IO,
 +      },
 +      { .name = "mem",
 +        .help = "Memory allocation/freeing logging",
 +        .shift = FD_MEM,
 +      },
 +      { .name = "blktrace",
 +        .help = "blktrace action logging",
 +        .shift = FD_BLKTRACE,
 +      },
 +      { .name = "verify",
 +        .help = "IO verification action logging",
 +        .shift = FD_VERIFY,
 +      },
 +      { .name = "random",
 +        .help = "Random generation logging",
 +        .shift = FD_RANDOM,
 +      },
 +      { .name = "parse",
 +        .help = "Parser logging",
 +        .shift = FD_PARSE,
 +      },
 +      { .name = "diskutil",
 +        .help = "Disk utility logging actions",
 +        .shift = FD_DISKUTIL,
 +      },
 +      { .name = "job",
 +        .help = "Logging related to creating/destroying jobs",
 +        .shift = FD_JOB,
 +      },
 +      { .name = "mutex",
 +        .help = "Mutex logging",
 +        .shift = FD_MUTEX
 +      },
 +      { .name = "profile",
 +        .help = "Logging related to profiles",
 +        .shift = FD_PROFILE,
 +      },
 +      { .name = "time",
 +        .help = "Logging related to time keeping functions",
 +        .shift = FD_TIME,
 +      },
 +      { .name = "net",
 +        .help = "Network logging",
 +        .shift = FD_NET,
 +      },
        { .name = NULL, },
  };
  
@@@ -1445,7 -1403,7 +1445,7 @@@ void parse_cmd_client(void *client, cha
        fio_client_add_cmd_option(client, opt);
  }
  
 -int parse_cmd_line(int argc, char *argv[])
 +int parse_cmd_line(int argc, char *argv[], int client_type)
  {
        struct thread_data *td = NULL;
        int c, ini_idx = 0, lidx, ret = 0, do_exit = 0, exit_val = 0;
                        char *val = optarg;
  
                        if (!strncmp(opt, "name", 4) && td) {
 -                              ret = add_job(td, td->o.name ?: "fio", 0);
 +                              ret = add_job(td, td->o.name ?: "fio", 0, 0, client_type);
                                if (ret)
                                        return 0;
                                td = NULL;
                                exit_val = 1;
                                break;
                        }
 -                      if (fio_client_add(optarg, &cur_client)) {
 +                      if (fio_client_add(&fio_client_ops, optarg, &cur_client)) {
                                log_err("fio: failed adding client %s\n", optarg);
                                do_exit++;
                                exit_val = 1;
  
        if (td) {
                if (!ret)
 -                      ret = add_job(td, td->o.name ?: "fio", 0);
 +                      ret = add_job(td, td->o.name ?: "fio", 0, 0, client_type);
        }
  
        while (!ret && optind < argc) {
        return ini_idx;
  }
  
 -int parse_options(int argc, char *argv[])
 +int fio_init_options(void)
  {
 -      int job_files, i;
 -
        f_out = stdout;
        f_err = stderr;
  
        if (fill_def_thread())
                return 1;
  
 -      job_files = parse_cmd_line(argc, argv);
 +      return 0;
 +}
 +
 +extern int fio_check_options(struct thread_options *);
 +
 +int parse_options(int argc, char *argv[])
 +{
 +      const int type = FIO_CLIENT_TYPE_CLI;
 +      int job_files, i;
 +
 +      if (fio_init_options())
 +              return 1;
 +      if (fio_test_cconv(&def_thread.o))
 +              log_err("fio: failed internal cconv test\n");
 +
 +      job_files = parse_cmd_line(argc, argv, type);
  
        if (job_files > 0) {
                for (i = 0; i < job_files; i++) {
                                        return 1;
                                free(ini_file[i]);
                        } else if (!is_backend) {
 -                              if (parse_jobs_ini(ini_file[i], 0, i))
 +                              if (parse_jobs_ini(ini_file[i], 0, i, type))
                                        return 1;
                                free(ini_file[i]);
                        }
  
        return 0;
  }
 +
 +void options_default_fill(struct thread_options *o)
 +{
 +      memcpy(o, &def_thread.o, sizeof(*o));
 +}
diff --combined iolog.c
index e54016d6e6c1027b0e462043e914203a9b506ae9,b73c208e468e5e948c799d4a2d8d9beb72c60f49..9bcf0d8e2982e37ce5d48fbd64e9653872e54ea0
+++ b/iolog.c
@@@ -110,7 -110,7 +110,7 @@@ int read_iolog_get(struct thread_data *
  {
        struct io_piece *ipo;
        unsigned long elapsed;
 -      
 +
        while (!flist_empty(&td->io_log_list)) {
                int ret;
  
                        elapsed = mtime_since_genesis();
                        if (ipo->delay > elapsed)
                                usec_sleep(td, (ipo->delay - elapsed) * 1000);
 -                              
                }
  
                free(ipo);
 -              
 +
                if (io_u->ddir != DDIR_WAIT)
                        return 0;
        }
@@@ -315,6 -316,7 +315,7 @@@ static int read_iolog2(struct thread_da
                                                                        act);
                                continue;
                        }
+                       fileno = get_fileno(td, fname);
                } else if (r == 2) {
                        rw = DDIR_INVAL;
                        if (!strcmp(act, "add")) {
                        ipo->fileno = fileno;
                        ipo->file_action = file_action;
                }
 -                      
 +
                queue_io_piece(td, ipo);
        }
  
@@@ -493,14 -495,13 +494,14 @@@ int init_iolog(struct thread_data *td
        return ret;
  }
  
 -void setup_log(struct io_log **log, unsigned long avg_msec)
 +void setup_log(struct io_log **log, unsigned long avg_msec, int log_type)
  {
        struct io_log *l = malloc(sizeof(*l));
  
        memset(l, 0, sizeof(*l));
        l->nr_samples = 0;
        l->max_samples = 1024;
 +      l->log_type = log_type;
        l->log = malloc(l->max_samples * sizeof(struct io_sample));
        l->avg_msec = avg_msec;
        *log = l;
@@@ -518,10 -519,10 +519,10 @@@ void __finish_log(struct io_log *log, c
        }
  
        for (i = 0; i < log->nr_samples; i++) {
 -              fprintf(f, "%lu, %lu, %u, %u\n", log->log[i].time,
 -                                              log->log[i].val,
 -                                              log->log[i].ddir,
 -                                              log->log[i].bs);
 +              fprintf(f, "%lu, %lu, %u, %u\n",
 +                              (unsigned long) log->log[i].time,
 +                              (unsigned long) log->log[i].val,
 +                              log->log[i].ddir, log->log[i].bs);
        }
  
        fclose(f);
@@@ -536,13 -537,7 +537,13 @@@ void finish_log_named(struct thread_dat
  
        snprintf(file_name, sizeof(file_name), "%s_%s.log", prefix, postfix);
        p = basename(file_name);
 -      __finish_log(log, p);
 +
 +      if (td->client_type == FIO_CLIENT_TYPE_GUI) {
 +              fio_send_iolog(td, log, p);
 +              free(log->log);
 +              free(log);
 +      } else
 +              __finish_log(log, p);
  }
  
  void finish_log(struct thread_data *td, struct io_log *log, const char *name)