Merge branch 'master' into gfio

diff --combined backend.c
index 3600319b233fb923fd214bea4179d436ae0f0f0e,88623bea3d88c97e34c19029849fa31dc81529ef..6f40038799831a4f521af7ac25942b660e56978b
--- 1/backend.c
--- 2/backend.c
+++ b/backend.c
@@@ -57,13 -57,13 +57,13 @@@ static struct flist_head *cgroup_list
  static char *cgroup_mnt;
  static int exit_value;
  static volatile int fio_abort;
 +static unsigned int nr_process = 0;
 +static unsigned int nr_thread = 0;
  
  struct io_log *agg_io_log[DDIR_RWDIR_CNT];
  
  int groupid = 0;
  unsigned int thread_number = 0;
 -unsigned int nr_process = 0;
 -unsigned int nr_thread = 0;
  int shm_id = 0;
  int temp_stall_ts;
  unsigned long done_secs = 0;
@@@ -802,6 -802,7 +802,7 @@@ static int init_io_u(struct thread_dat
        struct io_u *io_u;
        unsigned int max_bs, min_write;
        int cl_align, i, max_units;
+       int data_xfer = 1;
        char *p;
  
        max_units = td->o.iodepth;
@@@ -811,6 -812,9 +812,9 @@@
        td->orig_buffer_size = (unsigned long long) max_bs
                                        * (unsigned long long) max_units;
  
+       if ((td->io_ops->flags & FIO_NOIO) || !td_rw(td))
+               data_xfer = 0;
+ 
        if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) {
                unsigned long bs;
  
@@@ -823,7 -827,7 +827,7 @@@
                return 1;
        }
  
-       if (allocate_io_mem(td))
+       if (data_xfer && allocate_io_mem(td))
                return 1;
  
        if (td->o.odirect || td->o.mem_align ||
@@@ -851,7 -855,7 +855,7 @@@
                INIT_FLIST_HEAD(&io_u->list);
                dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i);
  
-               if (!(td->io_ops->flags & FIO_NOIO)) {
+               if (data_xfer) {
                        io_u->buf = p;
                        dprint(FD_MEM, "io_u %p, mem %p\n", io_u, io_u->buf);
  
@@@ -972,12 -976,10 +976,12 @@@ static void *thread_main(void *data
  {
        unsigned long long elapsed;
        struct thread_data *td = data;
 +      struct thread_options *o = &td->o;
        pthread_condattr_t attr;
        int clear_state;
 +      int ret;
  
 -      if (!td->o.use_thread) {
 +      if (!o->use_thread) {
                setsid();
                td->pid = getpid();
        } else
@@@ -985,9 -987,6 +989,9 @@@
  
        dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid);
  
 +      if (is_backend)
 +              fio_server_send_start(td);
 +
        INIT_FLIST_HEAD(&td->io_u_freelist);
        INIT_FLIST_HEAD(&td->io_u_busylist);
        INIT_FLIST_HEAD(&td->io_u_requeues);
@@@ -1014,17 -1013,16 +1018,17 @@@
         * eating a file descriptor
         */
        fio_mutex_remove(td->mutex);
 +      td->mutex = NULL;
  
        /*
         * A new gid requires privilege, so we need to do this before setting
         * the uid.
         */
 -      if (td->o.gid != -1U && setgid(td->o.gid)) {
 +      if (o->gid != -1U && setgid(o->gid)) {
                td_verror(td, errno, "setgid");
                goto err;
        }
 -      if (td->o.uid != -1U && setuid(td->o.uid)) {
 +      if (o->uid != -1U && setuid(o->uid)) {
                td_verror(td, errno, "setuid");
                goto err;
        }
@@@ -1033,24 -1031,18 +1037,24 @@@
         * If we have a gettimeofday() thread, make sure we exclude that
         * thread from this job
         */
 -      if (td->o.gtod_cpu)
 -              fio_cpu_clear(&td->o.cpumask, td->o.gtod_cpu);
 +      if (o->gtod_cpu)
 +              fio_cpu_clear(&o->cpumask, o->gtod_cpu);
  
        /*
         * Set affinity first, in case it has an impact on the memory
         * allocations.
         */
 -      if (td->o.cpumask_set && fio_setaffinity(td->pid, td->o.cpumask) == -1) {
 -              td_verror(td, errno, "cpu_set_affinity");
 -              goto err;
 +      if (o->cpumask_set) {
 +              ret = fio_setaffinity(td->pid, o->cpumask);
 +              if (ret == -1) {
 +                      td_verror(td, errno, "cpu_set_affinity");
 +                      goto err;
 +              }
        }
  
 +      if (fio_pin_memory(td))
 +              goto err;
 +
        /*
         * May alter parameters that init_io_u() will use, so we need to
         * do this first.
@@@ -1061,12 -1053,11 +1065,12 @@@
        if (init_io_u(td))
                goto err;
  
 -      if (td->o.verify_async && verify_async_init(td))
 +      if (o->verify_async && verify_async_init(td))
                goto err;
  
 -      if (td->ioprio_set) {
 -              if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
 +      if (o->ioprio) {
 +              ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class, o->ioprio);
 +              if (ret == -1) {
                        td_verror(td, errno, "ioprio_set");
                        goto err;
                }
@@@ -1076,15 -1067,15 +1080,15 @@@
                goto err;
  
        errno = 0;
 -      if (nice(td->o.nice) == -1 && errno != 0) {
 +      if (nice(o->nice) == -1 && errno != 0) {
                td_verror(td, errno, "nice");
                goto err;
        }
  
 -      if (td->o.ioscheduler && switch_ioscheduler(td))
 +      if (o->ioscheduler && switch_ioscheduler(td))
                goto err;
  
 -      if (!td->o.create_serialize && setup_files(td))
 +      if (!o->create_serialize && setup_files(td))
                goto err;
  
        if (td_io_init(td))
@@@ -1093,16 -1084,16 +1097,16 @@@
        if (init_random_map(td))
                goto err;
  
 -      if (td->o.exec_prerun) {
 -              if (exec_string(td->o.exec_prerun))
 -                      goto err;
 -      }
 +      if (o->exec_prerun && exec_string(o->exec_prerun))
 +              goto err;
  
 -      if (td->o.pre_read) {
 +      if (o->pre_read) {
                if (pre_read_files(td) < 0)
                        goto err;
        }
  
 +      fio_verify_init(td);
 +
        fio_gettime(&td->epoch, NULL);
        getrusage(RUSAGE_SELF, &td->ru_start);
  
@@@ -1174,8 -1165,6 +1178,8 @@@
        td->ts.io_bytes[DDIR_WRITE] = td->io_bytes[DDIR_WRITE];
        td->ts.io_bytes[DDIR_TRIM] = td->io_bytes[DDIR_TRIM];
  
 +      fio_unpin_memory(td);
 +
        fio_mutex_down(writeout_mutex);
        if (td->bw_log) {
                if (td->o.bw_log_file) {
@@@ -1233,8 -1222,8 +1237,8 @@@ err
        cleanup_io_u(td);
        cgroup_shutdown(td, &cgroup_mnt);
  
 -      if (td->o.cpumask_set) {
 -              int ret = fio_cpuset_exit(&td->o.cpumask);
 +      if (o->cpumask_set) {
 +              int ret = fio_cpuset_exit(&o->cpumask);
  
                td_verror(td, ret, "fio_cpuset_exit");
        }
@@@ -1388,19 -1377,14 +1392,19 @@@ static void run_threads(void
        unsigned long spent;
        unsigned int i, todo, nr_running, m_rate, t_rate, nr_started;
  
 -      if (fio_pin_memory())
 -              return;
 -
        if (fio_gtod_offload && fio_start_gtod_thread())
                return;
  
        set_sig_handlers();
  
 +      nr_thread = nr_process = 0;
 +      for_each_td(td, i) {
 +              if (td->o.use_thread)
 +                      nr_thread++;
 +              else
 +                      nr_process++;
 +      }
 +
        if (output_format == FIO_OUTPUT_NORMAL) {
                log_info("Starting ");
                if (nr_thread)
@@@ -1600,16 -1584,25 +1604,16 @@@
  
                reap_threads(&nr_running, &t_rate, &m_rate);
  
 -              if (todo) {
 -                      if (is_backend)
 -                              fio_server_idle_loop();
 -                      else
 -                              usleep(100000);
 -              }
 +              if (todo)
 +                      usleep(100000);
        }
  
        while (nr_running) {
                reap_threads(&nr_running, &t_rate, &m_rate);
 -
 -              if (is_backend)
 -                      fio_server_idle_loop();
 -              else
 -                      usleep(10000);
 +              usleep(10000);
        }
  
        update_io_ticks();
 -      fio_unpin_memory();
  }
  
  void wait_for_disk_thread_exit(void)
@@@ -1680,9 -1673,9 +1684,9 @@@ int fio_backend(void
                return 0;
  
        if (write_bw_log) {
 -              setup_log(&agg_io_log[DDIR_READ], 0);
 -              setup_log(&agg_io_log[DDIR_WRITE], 0);
 -              setup_log(&agg_io_log[DDIR_TRIM], 0);
 +              setup_log(&agg_io_log[DDIR_READ], 0, IO_LOG_TYPE_BW);
 +              setup_log(&agg_io_log[DDIR_WRITE], 0, IO_LOG_TYPE_BW);
 +              setup_log(&agg_io_log[DDIR_TRIM], 0, IO_LOG_TYPE_BW);
        }
  
        startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED);

diff --combined parse.c
index 419e80f886e56e9b9ef463d38ebea66673f35316,1a686e4cceb89a5ee8446d10a4472556d3445764..9a6494f32db811746ba46303b5f300a83e1ddfc6
--- 1/parse.c
--- 2/parse.c
+++ b/parse.c
@@@ -15,7 -15,8 +15,7 @@@
  #include "debug.h"
  #include "options.h"
  
 -static struct fio_option *fio_options;
 -extern unsigned int fio_get_kb_base(void *);
 +static struct fio_option *__fio_options;
  
  static int vp_cmp(const void *p1, const void *p2)
  {
@@@ -46,7 -47,7 +46,7 @@@ static void posval_sort(struct fio_opti
  static void show_option_range(struct fio_option *o,
                                int (*logger)(const char *format, ...))
  {
 -      if (o->type == FIO_OPT_FLOAT_LIST){
 +      if (o->type == FIO_OPT_FLOAT_LIST) {
                if (isnan(o->minfp) && isnan(o->maxfp))
                        return;
  
@@@ -152,32 -153,32 +152,32 @@@ static unsigned long long __get_mult_by
        for (i = 0; i < strlen(c); i++)
                c[i] = tolower(c[i]);
  
 -      if (!strcmp("pib", c)) {
 +      if (!strncmp("pib", c, 3)) {
                pow = 5;
                mult = 1000;
 -      } else if (!strcmp("tib", c)) {
 +      } else if (!strncmp("tib", c, 3)) {
                pow = 4;
                mult = 1000;
 -      } else if (!strcmp("gib", c)) {
 +      } else if (!strncmp("gib", c, 3)) {
                pow = 3;
                mult = 1000;
 -      } else if (!strcmp("mib", c)) {
 +      } else if (!strncmp("mib", c, 3)) {
                pow = 2;
                mult = 1000;
 -      } else if (!strcmp("kib", c)) {
 +      } else if (!strncmp("kib", c, 3)) {
                pow = 1;
                mult = 1000;
 -      } else if (!strcmp("p", c) || !strcmp("pb", c))
 +      } else if (!strncmp("p", c, 1) || !strncmp("pb", c, 2))
                pow = 5;
 -      else if (!strcmp("t", c) || !strcmp("tb", c))
 +      else if (!strncmp("t", c, 1) || !strncmp("tb", c, 2))
                pow = 4;
 -      else if (!strcmp("g", c) || !strcmp("gb", c))
 +      else if (!strncmp("g", c, 1) || !strncmp("gb", c, 2))
                pow = 3;
 -      else if (!strcmp("m", c) || !strcmp("mb", c))
 +      else if (!strncmp("m", c, 1) || !strncmp("mb", c, 2))
                pow = 2;
 -      else if (!strcmp("k", c) || !strcmp("kb", c))
 +      else if (!strncmp("k", c, 1) || !strncmp("kb", c, 2))
                pow = 1;
 -      else if (!strcmp("%", c)) {
 +      else if (!strncmp("%", c, 1)) {
                *percent = 1;
                free(c);
                return ret;
@@@ -259,7 -260,7 +259,7 @@@ int str_to_decimal(const char *str, lon
        return 0;
  }
  
 -static int check_str_bytes(const char *p, long long *val, void *data)
 +int check_str_bytes(const char *p, long long *val, void *data)
  {
        return str_to_decimal(p, val, 1, data);
  }
@@@ -355,7 -356,7 +355,7 @@@ static int __handle_option(struct fio_o
                           int first, int more, int curr)
  {
        int il, *ilp;
 -      double* flp;
 +      double *flp;
        long long ull, *ullp;
        long ul1, ul2;
        double uf;
@@@ -429,13 -430,13 +429,13 @@@
                        break;
  
                if (o->maxval && ull > o->maxval) {
-                       log_err("max value out of range: %lld"
-                                       " (%d max)\n", ull, o->maxval);
+                       log_err("max value out of range: %llu"
+                                       " (%u max)\n", ull, o->maxval);
                        return 1;
                }
                if (o->minval && ull < o->minval) {
-                       log_err("min value out of range: %lld"
-                                       " (%d min)\n", ull, o->minval);
+                       log_err("min value out of range: %llu"
+                                       " (%u min)\n", ull, o->minval);
                        return 1;
                }
  
@@@ -504,7 -505,7 +504,7 @@@
                                        o->maxlen);
                        return 1;
                }
 -              if(!str_to_float(ptr, &uf)){
 +              if (!str_to_float(ptr, &uf)) {
                        log_err("not a floating point value: %s\n", ptr);
                        return 1;
                }
@@@ -826,14 -827,14 +826,14 @@@ static int opt_cmp(const void *p1, cons
  
        if (*(char **)p1) {
                s = strdup(*((char **) p1));
 -              o = get_option(s, fio_options, &foo);
 +              o = get_option(s, __fio_options, &foo);
                if (o)
                        prio1 = o->prio;
                free(s);
        }
        if (*(char **)p2) {
                s = strdup(*((char **) p2));
 -              o = get_option(s, fio_options, &foo);
 +              o = get_option(s, __fio_options, &foo);
                if (o)
                        prio2 = o->prio;
                free(s);
@@@ -844,9 -845,9 +844,9 @@@
  
  void sort_options(char **opts, struct fio_option *options, int num_opts)
  {
 -      fio_options = options;
 +      __fio_options = options;
        qsort(opts, num_opts, sizeof(char *), opt_cmp);
 -      fio_options = NULL;
 +      __fio_options = NULL;
  }
  
  int parse_cmd_option(const char *opt, const char *val,
@@@ -889,8 -890,9 +889,8 @@@ int parse_option(char *opt, const char 
                return 1;
        }
  
 -      if (!handle_option(*o, post, data)) {
 +      if (!handle_option(*o, post, data))
                return 0;
 -      }
  
        log_err("fio: failed parsing %s\n", input);
        return 1;
@@@ -1085,6 -1087,10 +1085,10 @@@ void option_init(struct fio_option *o
                o->minval = 0;
                o->maxval = 1;
        }
+       if (o->type == FIO_OPT_INT) {
+               if (!o->maxval)
+                       o->maxval = UINT_MAX;
+       }
        if (o->type == FIO_OPT_FLOAT_LIST) {
                o->minfp = NAN;
                o->maxfp = NAN;
@@@ -1102,10 -1108,6 +1106,10 @@@
                      (o->roff1 || o->roff2 || o->roff3 || o->roff4))) {
                log_err("Option %s: both cb and offset given\n", o->name);
        }
 +      if (!o->category) {
 +              log_info("Options %s: no category defined. Setting to misc\n", o->name);
 +              o->category = FIO_OPT_C_GENERAL;
 +      }
  }
  
  /*
@@@ -1118,11 -1120,8 +1122,11 @@@ void options_init(struct fio_option *op
  
        dprint(FD_PARSE, "init options\n");
  
 -      for (o = &options[0]; o->name; o++)
 +      for (o = &options[0]; o->name; o++) {
                option_init(o);
 +              if (o->inverse)
 +                      o->inv_opt = find_option(options, o->inverse);
 +      }
  }
  
  void options_free(struct fio_option *options, void *data)