From: Jens Axboe Date: Wed, 11 Oct 2017 14:23:41 +0000 (-0600) Subject: Merge branch 'ci_and_configure' of https://github.com/sitsofe/fio X-Git-Tag: fio-3.2~45 X-Git-Url: https://git.kernel.dk/?a=commitdiff_plain;h=d25f5ae81c4a5dedf4f5c5a2323a20fe77e5b86f;hp=b73af7389a47bb04d841d749eb8a4e996eb5a271;p=fio.git Merge branch 'ci_and_configure' of https://github.com/sitsofe/fio --- diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN index 31acf1c2..8c075cbd 100755 --- a/FIO-VERSION-GEN +++ b/FIO-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=FIO-VERSION-FILE -DEF_VER=fio-3.0 +DEF_VER=fio-3.1 LF=' ' diff --git a/HOWTO b/HOWTO index 8fad2ce6..d3f957bf 100644 --- a/HOWTO +++ b/HOWTO @@ -217,6 +217,9 @@ Command line options .. option:: --max-jobs=nr Set the maximum number of threads/processes to support to `nr`. + NOTE: On Linux, it may be necessary to increase the shared-memory + limit ('/proc/sys/kernel/shmmax') if fio runs into errors while + creating jobs. .. option:: --server=args @@ -1797,6 +1800,10 @@ I/O engine absolute or relative. See :file:`engines/skeleton_external.c` for details of writing an external I/O engine. + **filecreate** + Simply create the files and do no IO to them. You still need to + set `filesize` so that all the accounting still occurs, but no + actual IO will be done other than creating the file. I/O engine specific parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Makefile b/Makefile index 3764da55..76243ffb 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ eta.c verify.c memory.c io_u.c parse.c mutex.c options.c \ smalloc.c filehash.c profile.c debug.c engines/cpu.c \ engines/mmap.c engines/sync.c engines/null.c engines/net.c \ - engines/ftruncate.c \ + engines/ftruncate.c engines/filecreate.c \ server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ diff --git a/backend.c b/backend.c index 6198c3d9..d98e5fe4 100644 --- a/backend.c +++ b/backend.c @@ -499,7 +499,6 @@ int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret, if (ddir_rw(io_u->ddir)) td->ts.short_io_u[io_u->ddir]++; - f = io_u->file; if (io_u->offset == f->real_file_size) goto sync_done; @@ -1505,7 +1504,7 @@ static void *thread_main(void *data) struct sk_out *sk_out = fd->sk_out; uint64_t bytes_done[DDIR_RWDIR_CNT]; int deadlock_loop_cnt; - int clear_state; + bool clear_state, did_some_io; int ret; sk_out_assign(sk_out); @@ -1726,7 +1725,8 @@ static void *thread_main(void *data) } memset(bytes_done, 0, sizeof(bytes_done)); - clear_state = 0; + clear_state = false; + did_some_io = false; while (keep_running(td)) { uint64_t verify_bytes; @@ -1765,7 +1765,7 @@ static void *thread_main(void *data) if (td->runstate >= TD_EXITED) break; - clear_state = 1; + clear_state = true; /* * Make sure we've successfully updated the rusage stats @@ -1804,6 +1804,9 @@ static void *thread_main(void *data) td_ioengine_flagged(td, FIO_UNIDIR)) continue; + if (ddir_rw_sum(bytes_done)) + did_some_io = true; + clear_io_state(td, 0); fio_gettime(&td->start, NULL); @@ -1830,6 +1833,7 @@ static void *thread_main(void *data) * (Are we not missing other flags that can be ignored ?) */ if ((td->o.size || td->o.io_size) && !ddir_rw_sum(bytes_done) && + !did_some_io && !td->o.create_only && !(td_ioengine_flagged(td, FIO_NOIO) || td_ioengine_flagged(td, FIO_DISKLESSIO))) log_err("%s: No I/O performed by %s, " @@ -1925,11 +1929,7 @@ static void reap_threads(unsigned int *nr_running, uint64_t *t_rate, for_each_td(td, i) { int flags = 0; - /* - * ->io_ops is NULL for a thread that has closed its - * io engine - */ - if (td->io_ops && !strcmp(td->io_ops->name, "cpuio")) + if (!strcmp(td->o.ioengine, "cpuio")) cputhreads++; else realthreads++; @@ -2342,6 +2342,7 @@ reap: fio_terminate_threads(TERMINATE_ALL); fio_abort = 1; nr_started--; + free(fd); break; } dprint(FD_MUTEX, "done waiting on startup_mutex\n"); diff --git a/blktrace.c b/blktrace.c index 65b600f5..4b791d7e 100644 --- a/blktrace.c +++ b/blktrace.c @@ -500,10 +500,8 @@ int load_blktrace(struct thread_data *td, const char *filename, int need_swap) handle_trace(td, &t, ios, rw_bs); } while (1); - for (i = 0; i < td->files_index; i++) { - f = td->files[i]; + for_each_file(td, f, i) trace_add_open_close_event(td, f->fileno, FIO_LOG_CLOSE_FILE); - } fifo_free(fifo); close(fd); diff --git a/client.c b/client.c index 09e810af..779fb9d7 100644 --- a/client.c +++ b/client.c @@ -1312,14 +1312,16 @@ static void client_flush_hist_samples(FILE *f, int hist_coarseness, void *sample static int fio_client_handle_iolog(struct fio_client *client, struct fio_net_cmd *cmd) { - struct cmd_iolog_pdu *pdu; + struct cmd_iolog_pdu *pdu = NULL; bool store_direct; - char *log_pathname; + char *log_pathname = NULL; + int ret = 0; pdu = convert_iolog(cmd, &store_direct); if (!pdu) { log_err("fio: failed converting IO log\n"); - return 1; + ret = 1; + goto out; } /* allocate buffer big enough for next sprintf() call */ @@ -1327,7 +1329,8 @@ static int fio_client_handle_iolog(struct fio_client *client, strlen(client->hostname)); if (!log_pathname) { log_err("fio: memory allocation of unique pathname failed\n"); - return -1; + ret = -1; + goto out; } /* generate a unique pathname for the log file using hostname */ sprintf(log_pathname, "%s.%s", pdu->name, client->hostname); @@ -1342,7 +1345,8 @@ static int fio_client_handle_iolog(struct fio_client *client, if (fd < 0) { log_err("fio: open log %s: %s\n", log_pathname, strerror(errno)); - return 1; + ret = 1; + goto out; } sz = cmd->pdu_len - sizeof(*pdu); @@ -1351,17 +1355,19 @@ static int fio_client_handle_iolog(struct fio_client *client, if (ret != sz) { log_err("fio: short write on compressed log\n"); - return 1; + ret = 1; + goto out; } - return 0; + ret = 0; } else { FILE *f; f = fopen((const char *) log_pathname, "w"); if (!f) { log_err("fio: fopen log %s : %s\n", log_pathname, strerror(errno)); - return 1; + ret = 1; + goto out; } if (pdu->log_type == IO_LOG_TYPE_HIST) { @@ -1372,8 +1378,17 @@ static int fio_client_handle_iolog(struct fio_client *client, pdu->nr_samples * sizeof(struct io_sample)); } fclose(f); - return 0; + ret = 0; } + +out: + if (pdu && pdu != (void *) cmd->payload) + free(pdu); + + if (log_pathname) + free(log_pathname); + + return ret; } static void handle_probe(struct fio_client *client, struct fio_net_cmd *cmd) @@ -1849,10 +1864,12 @@ static void request_client_etas(struct client_ops *ops) static int handle_cmd_timeout(struct fio_client *client, struct fio_net_cmd_reply *reply) { + uint16_t reply_opcode = reply->opcode; + flist_del(&reply->list); free(reply); - if (reply->opcode != FIO_NET_CMD_SEND_ETA) + if (reply_opcode != FIO_NET_CMD_SEND_ETA) return 1; log_info("client <%s>: timeout on SEND_ETA\n", client->hostname); diff --git a/engines/filecreate.c b/engines/filecreate.c new file mode 100644 index 00000000..0c3bcdd6 --- /dev/null +++ b/engines/filecreate.c @@ -0,0 +1,119 @@ +/* + * filecreate engine + * + * IO engine that doesn't do any IO, just creates files and tracks the latency + * of the file creation. + */ +#include +#include +#include +#include + +#include "../fio.h" +#include "../filehash.h" + +struct fc_data { + enum fio_ddir stat_ddir; +}; + +static int open_file(struct thread_data *td, struct fio_file *f) +{ + struct timespec start; + int do_lat = !td->o.disable_lat; + + dprint(FD_FILE, "fd open %s\n", f->file_name); + + if (f->filetype != FIO_TYPE_FILE) { + log_err("fio: only files are supported fallocate \n"); + return 1; + } + if (!strcmp(f->file_name, "-")) { + log_err("fio: can't read/write to stdin/out\n"); + return 1; + } + + if (do_lat) + fio_gettime(&start, NULL); + + f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600); + + if (f->fd == -1) { + char buf[FIO_VERROR_SIZE]; + int e = errno; + + snprintf(buf, sizeof(buf), "open(%s)", f->file_name); + td_verror(td, e, buf); + return 1; + } + + if (do_lat) { + struct fc_data *data = td->io_ops_data; + uint64_t nsec; + + nsec = ntime_since_now(&start); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0); + } + + return 0; +} + +static int queue_io(struct thread_data *td, struct io_u fio_unused *io_u) +{ + return FIO_Q_COMPLETED; +} + +/* + * Ensure that we at least have a block size worth of IO to do for each + * file. If the job file has td->o.size < nr_files * block_size, then + * fio won't do anything. + */ +static int get_file_size(struct thread_data *td, struct fio_file *f) +{ + f->real_file_size = td_min_bs(td); + return 0; +} + +static int init(struct thread_data *td) +{ + struct fc_data *data; + + data = calloc(1, sizeof(*data)); + + if (td_read(td)) + data->stat_ddir = DDIR_READ; + else if (td_write(td)) + data->stat_ddir = DDIR_WRITE; + + td->io_ops_data = data; + return 0; +} + +static void cleanup(struct thread_data *td) +{ + struct fc_data *data = td->io_ops_data; + + free(data); +} + +static struct ioengine_ops ioengine = { + .name = "filecreate", + .version = FIO_IOOPS_VERSION, + .init = init, + .cleanup = cleanup, + .queue = queue_io, + .get_file_size = get_file_size, + .open_file = open_file, + .close_file = generic_close_file, + .flags = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO | + FIO_NOSTATS | FIO_NOFILEHASH, +}; + +static void fio_init fio_filecreate_register(void) +{ + register_ioengine(&ioengine); +} + +static void fio_exit fio_filecreate_unregister(void) +{ + unregister_ioengine(&ioengine); +} diff --git a/engines/null.c b/engines/null.c index 812cadfe..8a4d106b 100644 --- a/engines/null.c +++ b/engines/null.c @@ -6,7 +6,11 @@ * * It also can act as external C++ engine - compiled with: * - * g++ -O2 -g -shared -rdynamic -fPIC -o null.so null.c -DFIO_EXTERNAL_ENGINE + * g++ -O2 -g -shared -rdynamic -fPIC -o cpp_null null.c -DFIO_EXTERNAL_ENGINE + * + * to test it execute: + * + * LD_LIBRARY_PATH=./engines ./fio examples/cpp_null.fio * */ #include @@ -23,20 +27,17 @@ struct null_data { int events; }; -static struct io_u *fio_null_event(struct thread_data *td, int event) +static struct io_u *null_event(struct null_data *nd, int event) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - return nd->io_us[event]; } -static int fio_null_getevents(struct thread_data *td, unsigned int min_events, - unsigned int fio_unused max, - const struct timespec fio_unused *t) +static int null_getevents(struct null_data *nd, unsigned int min_events, + unsigned int fio_unused max, + const struct timespec fio_unused *t) { - struct null_data *nd = (struct null_data *) td->io_ops_data; int ret = 0; - + if (min_events) { ret = nd->events; nd->events = 0; @@ -45,10 +46,8 @@ static int fio_null_getevents(struct thread_data *td, unsigned int min_events, return ret; } -static int fio_null_commit(struct thread_data *td) +static int null_commit(struct thread_data *td, struct null_data *nd) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - if (!nd->events) { #ifndef FIO_EXTERNAL_ENGINE io_u_mark_submit(td, nd->queued); @@ -60,10 +59,9 @@ static int fio_null_commit(struct thread_data *td) return 0; } -static int fio_null_queue(struct thread_data *td, struct io_u *io_u) +static int null_queue(struct thread_data *td, struct null_data *nd, + struct io_u *io_u) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - fio_ro_check(td, io_u); if (td->io_ops->flags & FIO_SYNCIO) @@ -75,25 +73,23 @@ static int fio_null_queue(struct thread_data *td, struct io_u *io_u) return FIO_Q_QUEUED; } -static int fio_null_open(struct thread_data fio_unused *td, - struct fio_file fio_unused *f) +static int null_open(struct null_data fio_unused *nd, + struct fio_file fio_unused *f) { return 0; } -static void fio_null_cleanup(struct thread_data *td) +static void null_cleanup(struct null_data *nd) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - if (nd) { free(nd->io_us); free(nd); } } -static int fio_null_init(struct thread_data *td) +static int null_init(struct thread_data *td, struct null_data **nd_ptr) { - struct null_data *nd = (struct null_data *) malloc(sizeof(*nd)); + struct null_data *nd = (struct null_data *) malloc(sizeof(**nd_ptr)); memset(nd, 0, sizeof(*nd)); @@ -103,11 +99,49 @@ static int fio_null_init(struct thread_data *td) } else td->io_ops->flags |= FIO_SYNCIO; - td->io_ops_data = nd; + *nd_ptr = nd; return 0; } #ifndef __cplusplus + +static struct io_u *fio_null_event(struct thread_data *td, int event) +{ + return null_event((struct null_data *)td->io_ops_data, event); +} + +static int fio_null_getevents(struct thread_data *td, unsigned int min_events, + unsigned int max, const struct timespec *t) +{ + struct null_data *nd = (struct null_data *)td->io_ops_data; + return null_getevents(nd, min_events, max, t); +} + +static int fio_null_commit(struct thread_data *td) +{ + return null_commit(td, (struct null_data *)td->io_ops_data); +} + +static int fio_null_queue(struct thread_data *td, struct io_u *io_u) +{ + return null_queue(td, (struct null_data *)td->io_ops_data, io_u); +} + +static int fio_null_open(struct thread_data *td, struct fio_file *f) +{ + return null_open((struct null_data *)td->io_ops_data, f); +} + +static void fio_null_cleanup(struct thread_data *td) +{ + null_cleanup((struct null_data *)td->io_ops_data); +} + +static int fio_null_init(struct thread_data *td) +{ + return null_init(td, (struct null_data **)&td->io_ops_data); +} + static struct ioengine_ops ioengine = { .name = "null", .version = FIO_IOOPS_VERSION, @@ -134,7 +168,91 @@ static void fio_exit fio_null_unregister(void) #else #ifdef FIO_EXTERNAL_ENGINE + +struct NullData { + NullData(struct thread_data *td) + { + null_init(td, &impl_); + } + + ~NullData() + { + null_cleanup(impl_); + } + + static NullData *get(struct thread_data *td) + { + return reinterpret_cast(td->io_ops_data); + } + + io_u *fio_null_event(struct thread_data *, int event) + { + return null_event(impl_, event); + } + + int fio_null_getevents(struct thread_data *, unsigned int min_events, + unsigned int max, const struct timespec *t) + { + return null_getevents(impl_, min_events, max, t); + } + + int fio_null_commit(struct thread_data *td) + { + return null_commit(td, impl_); + } + + int fio_null_queue(struct thread_data *td, struct io_u *io_u) + { + return null_queue(td, impl_, io_u); + } + + int fio_null_open(struct thread_data *, struct fio_file *f) + { + return null_open(impl_, f); + } + + struct null_data *impl_; +}; + extern "C" { + +static struct io_u *fio_null_event(struct thread_data *td, int event) +{ + return NullData::get(td)->fio_null_event(td, event); +} + +static int fio_null_getevents(struct thread_data *td, unsigned int min_events, + unsigned int max, const struct timespec *t) +{ + return NullData::get(td)->fio_null_getevents(td, min_events, max, t); +} + +static int fio_null_commit(struct thread_data *td) +{ + return NullData::get(td)->fio_null_commit(td); +} + +static int fio_null_queue(struct thread_data *td, struct io_u *io_u) +{ + return NullData::get(td)->fio_null_queue(td, io_u); +} + +static int fio_null_open(struct thread_data *td, struct fio_file *f) +{ + return NullData::get(td)->fio_null_open(td, f); +} + +static int fio_null_init(struct thread_data *td) +{ + td->io_ops_data = new NullData(td); + return 0; +} + +static void fio_null_cleanup(struct thread_data *td) +{ + delete NullData::get(td); +} + static struct ioengine_ops ioengine; void get_ioengine(struct ioengine_ops **ioengine_ptr) { diff --git a/examples/cpp_null.fio b/examples/cpp_null.fio new file mode 100644 index 00000000..436ed90a --- /dev/null +++ b/examples/cpp_null.fio @@ -0,0 +1,10 @@ +[global] +bs=4k +gtod_reduce=1 + +[null] +ioengine=cpp_null +size=100g +rw=randread +norandommap +time_based=0 diff --git a/examples/filecreate-ioengine.fio b/examples/filecreate-ioengine.fio new file mode 100644 index 00000000..ec7caad5 --- /dev/null +++ b/examples/filecreate-ioengine.fio @@ -0,0 +1,35 @@ +# Example filecreate job +# +# create_on_open is needed so that the open happens during the run and not the +# setup. +# +# openfiles needs to be set so that you do not exceed the maximum allowed open +# files. +# +# filesize needs to be set to a non zero value so fio will actually run, but the +# IO will not really be done and the write latency numbers will only reflect the +# open times. +[global] +create_on_open=1 +nrfiles=31250 +ioengine=filecreate +fallocate=none +filesize=4k +openfiles=1 + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] +[t6] +[t7] +[t8] +[t9] +[t10] +[t11] +[t12] +[t13] +[t14] +[t15] diff --git a/exp/README.md b/exp/README.md deleted file mode 100644 index 48c11c98..00000000 --- a/exp/README.md +++ /dev/null @@ -1,7 +0,0 @@ -simple-expression-parser -======================== - -A simple expression parser for arithmetic expressions made with bison + flex - -To use, see the example test-expression-parser.c - diff --git a/file.h b/file.h index ad8802d3..e3864ee5 100644 --- a/file.h +++ b/file.h @@ -188,9 +188,15 @@ extern void close_and_free_files(struct thread_data *); extern uint64_t get_start_offset(struct thread_data *, struct fio_file *); extern int __must_check setup_files(struct thread_data *); extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *); +#ifdef __cplusplus +extern "C" { +#endif extern int __must_check generic_open_file(struct thread_data *, struct fio_file *); extern int __must_check generic_close_file(struct thread_data *, struct fio_file *); extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *); +#ifdef __cplusplus +} +#endif extern int __must_check file_lookup_open(struct fio_file *f, int flags); extern int __must_check pre_read_files(struct thread_data *); extern unsigned long long get_rand_file_size(struct thread_data *td); diff --git a/filesetup.c b/filesetup.c index 891a55a1..0631a01f 100644 --- a/filesetup.c +++ b/filesetup.c @@ -1342,6 +1342,7 @@ void close_and_free_files(struct thread_data *td) { struct fio_file *f; unsigned int i; + bool use_free = td_ioengine_flagged(td, FIO_NOFILEHASH); dprint(FD_FILE, "close files\n"); @@ -1361,13 +1362,19 @@ void close_and_free_files(struct thread_data *td) td_io_unlink_file(td, f); } - sfree(f->file_name); + if (use_free) + free(f->file_name); + else + sfree(f->file_name); f->file_name = NULL; if (fio_file_axmap(f)) { axmap_free(f->io_axmap); f->io_axmap = NULL; } - sfree(f); + if (use_free) + free(f); + else + sfree(f); } td->o.filename = NULL; @@ -1481,7 +1488,10 @@ static struct fio_file *alloc_new_file(struct thread_data *td) { struct fio_file *f; - f = smalloc(sizeof(*f)); + if (td_ioengine_flagged(td, FIO_NOFILEHASH)) + f = calloc(1, sizeof(*f)); + else + f = smalloc(sizeof(*f)); if (!f) { assert(0); return NULL; @@ -1564,7 +1574,10 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc) if (td->io_ops && td_ioengine_flagged(td, FIO_DISKLESSIO)) f->real_file_size = -1ULL; - f->file_name = smalloc_strdup(file_name); + if (td_ioengine_flagged(td, FIO_NOFILEHASH)) + f->file_name = strdup(file_name); + else + f->file_name = smalloc_strdup(file_name); if (!f->file_name) assert(0); @@ -1588,7 +1601,8 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc) if (f->filetype == FIO_TYPE_FILE) td->nr_normal_files++; - set_already_allocated(file_name); + if (td->o.numjobs > 1) + set_already_allocated(file_name); if (inc) td->o.nr_files++; @@ -1768,7 +1782,10 @@ void dup_files(struct thread_data *td, struct thread_data *org) __f = alloc_new_file(td); if (f->file_name) { - __f->file_name = smalloc_strdup(f->file_name); + if (td_ioengine_flagged(td, FIO_NOFILEHASH)) + __f->file_name = strdup(f->file_name); + else + __f->file_name = smalloc_strdup(f->file_name); if (!__f->file_name) assert(0); diff --git a/fio.1 b/fio.1 index b943db22..6e7d1f8b 100644 --- a/fio.1 +++ b/fio.1 @@ -113,6 +113,8 @@ All fio parser warnings are fatal, causing fio to exit with an error. .TP .BI \-\-max\-jobs \fR=\fPnr Set the maximum number of threads/processes to support to \fInr\fR. +NOTE: On Linux, it may be necessary to increase the shared-memory limit +(`/proc/sys/kernel/shmmax') if fio runs into errors while creating jobs. .TP .BI \-\-server \fR=\fPargs Start a backend server, with \fIargs\fR specifying what to listen to. @@ -1577,6 +1579,10 @@ the engine filename, e.g. `ioengine=external:/tmp/foo.o' to load ioengine `foo.o' in `/tmp'. The path can be either absolute or relative. See `engines/skeleton_external.c' in the fio source for details of writing an external I/O engine. +.TP +.B filecreate +Create empty files only. \fBfilesize\fR still needs to be specified so that fio +will run and grab latency results, but no IO will actually be done on the files. .SS "I/O engine specific parameters" In addition, there are some parameters which are only valid when a specific \fBioengine\fR is in use. These are used identically to normal parameters, diff --git a/fio_time.h b/fio_time.h index f4eac793..c7c3dbba 100644 --- a/fio_time.h +++ b/fio_time.h @@ -5,6 +5,7 @@ struct thread_data; extern uint64_t ntime_since(const struct timespec *, const struct timespec *); +extern uint64_t ntime_since_now(const struct timespec *); extern uint64_t utime_since(const struct timespec *, const struct timespec *); extern uint64_t utime_since_now(const struct timespec *); extern uint64_t mtime_since(const struct timespec *, const struct timespec *); diff --git a/gettime.c b/gettime.c index 3dcaaf68..79455284 100644 --- a/gettime.c +++ b/gettime.c @@ -448,6 +448,14 @@ uint64_t ntime_since(const struct timespec *s, const struct timespec *e) return nsec + (sec * 1000000000LL); } +uint64_t ntime_since_now(const struct timespec *s) +{ + struct timespec now; + + fio_gettime(&now, NULL); + return ntime_since(s, &now); +} + uint64_t utime_since(const struct timespec *s, const struct timespec *e) { int64_t sec, usec; diff --git a/io_u.c b/io_u.c index e98cd318..fb4180a3 100644 --- a/io_u.c +++ b/io_u.c @@ -662,7 +662,7 @@ int io_u_quiesce(struct thread_data *td) static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) { enum fio_ddir odir = ddir ^ 1; - long usec; + uint64_t usec; uint64_t now; assert(ddir_rw(ddir)); @@ -1779,7 +1779,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, if (td->parent) td = td->parent; - if (!td->o.stats) + if (!td->o.stats || td_ioengine_flagged(td, FIO_NOSTATS)) return; if (no_reduce) diff --git a/ioengines.c b/ioengines.c index 9638d804..1bfc06f9 100644 --- a/ioengines.c +++ b/ioengines.c @@ -133,8 +133,10 @@ static struct ioengine_ops *__load_ioengine(const char *name) /* * linux libaio has alias names, so convert to what we want */ - if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) + if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) { + dprint(FD_IO, "converting ioengine name: %s -> libaio\n", name); strcpy(engine, "libaio"); + } dprint(FD_IO, "load ioengine %s\n", engine); return find_ioengine(engine); @@ -436,6 +438,7 @@ int td_io_open_file(struct thread_data *td, struct fio_file *f) { assert(!fio_file_open(f)); assert(f->fd == -1); + assert(td->io_ops->open_file); if (td->io_ops->open_file(td, f)) { if (td->error == EINVAL && td->o.odirect) diff --git a/ioengines.h b/ioengines.h index 177cbc05..32b18eda 100644 --- a/ioengines.h +++ b/ioengines.h @@ -59,6 +59,8 @@ enum fio_ioengine_flags { FIO_MEMALIGN = 1 << 9, /* engine wants aligned memory */ FIO_BIT_BASED = 1 << 10, /* engine uses a bit base (e.g. uses Kbit as opposed to KB) */ FIO_FAKEIO = 1 << 11, /* engine pretends to do IO */ + FIO_NOSTATS = 1 << 12, /* don't do IO stats */ + FIO_NOFILEHASH = 1 << 13, /* doesn't hash the files for lookup later. */ }; /* diff --git a/lib/types.h b/lib/types.h index 287a3b42..bb245068 100644 --- a/lib/types.h +++ b/lib/types.h @@ -1,7 +1,7 @@ #ifndef FIO_TYPES_H #define FIO_TYPES_H -#ifndef CONFIG_HAVE_BOOL +#if !defined(CONFIG_HAVE_BOOL) && !defined(__cplusplus) typedef int bool; #ifndef false #define false 0 diff --git a/options.c b/options.c index 5c1abe91..ddcc4e5a 100644 --- a/options.c +++ b/options.c @@ -1843,6 +1843,10 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .help = "DAX Device based IO engine", }, #endif + { + .ival = "filecreate", + .help = "File creation engine", + }, { .ival = "external", .help = "Load external engine (append name)", .cb = str_ioengine_external_cb, diff --git a/os/windows/install.wxs b/os/windows/install.wxs index edfefa8c..58244c56 100755 --- a/os/windows/install.wxs +++ b/os/windows/install.wxs @@ -10,7 +10,7 @@ + UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="3.1"> clat_percentiles) + json_object_add_value_object(tmp_object, "bins", clat_bins_object); + for(i = 0; i < FIO_IO_U_PLAT_NR; i++) { if (ts->io_u_plat[ddir][i]) { snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); @@ -1055,6 +1057,9 @@ static void add_ddir_status_json(struct thread_stat *ts, json_object_add_value_int(tmp_object, "max", max); json_object_add_value_float(tmp_object, "mean", mean); json_object_add_value_float(tmp_object, "stddev", dev); + if (output_format & FIO_OUTPUT_JSON_PLUS && ts->lat_percentiles) + json_object_add_value_object(tmp_object, "bins", clat_bins_object); + if (ovals) free(ovals); diff --git a/stat.h b/stat.h index 848331bb..3fda0841 100644 --- a/stat.h +++ b/stat.h @@ -23,6 +23,16 @@ struct group_run_stats { #define FIO_IO_U_LAT_U_NR 10 #define FIO_IO_U_LAT_M_NR 12 +/* + * Constants for clat percentiles + */ +#define FIO_IO_U_PLAT_BITS 6 +#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS) +#define FIO_IO_U_PLAT_GROUP_NR 29 +#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL) +#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified + list of percentiles */ + /* * Aggregate clat samples to report percentile(s) of them. * @@ -34,7 +44,7 @@ struct group_run_stats { * * FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the maximum * range being tracked for latency samples. The maximum value tracked - * accurately will be 2^(GROUP_NR + PLAT_BITS -1) microseconds. + * accurately will be 2^(GROUP_NR + PLAT_BITS - 1) nanoseconds. * * FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the memory * requirement of storing those aggregate counts. The memory used will @@ -98,22 +108,15 @@ struct group_run_stats { * 3 8 2 [256,511] 64 * 4 9 3 [512,1023] 64 * ... ... ... [...,...] ... - * 18 23 17 [8838608,+inf]** 64 + * 28 33 27 [8589934592,+inf]** 64 * * * Special cases: when n < (M-1) or when n == (M-1), in both cases, * the value cannot be rounded off. Use all bits of the sample as * index. * - * ** If a sample's MSB is greater than 23, it will be counted as 23. + * ** If a sample's MSB is greater than 33, it will be counted as 33. */ -#define FIO_IO_U_PLAT_BITS 6 -#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS) -#define FIO_IO_U_PLAT_GROUP_NR 29 -#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL) -#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified - list of percentiles */ - /* * Trim cycle count measurements */ diff --git a/t/gen-rand.c b/t/gen-rand.c index 6c31f925..4e9d39c6 100644 --- a/t/gen-rand.c +++ b/t/gen-rand.c @@ -63,6 +63,6 @@ int main(int argc, char *argv[]) } printf("Passes=%lu, Fail=%lu\n", pass, fail); - + free(buckets); return 0; } diff --git a/tools/fio_jsonplus_clat2csv b/tools/fio_jsonplus_clat2csv index d4ac16e4..64fdc9f3 100755 --- a/tools/fio_jsonplus_clat2csv +++ b/tools/fio_jsonplus_clat2csv @@ -107,8 +107,16 @@ def main(): prev_ddir = None for ddir in ddir_set: + if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']: + bins_loc = 'clat_ns' + elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']: + bins_loc = 'lat_ns' + else: + raise RuntimeError("Latency bins not found. " + "Are you sure you are using json+ output?") + bins[ddir] = [[int(key), value] for key, value in - jsondata['jobs'][jobnum][ddir]['clat_ns'] + jsondata['jobs'][jobnum][ddir][bins_loc] ['bins'].iteritems()] bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0]) @@ -123,7 +131,7 @@ def main(): outfile = stub + '_job' + str(jobnum) + ext with open(outfile, 'w') as output: - output.write("clat_nsec, ") + output.write("{0}ec, ".format(bins_loc)) ddir_list = list(ddir_set) for ddir in ddir_list: output.write("{0}_count, {0}_cumulative, {0}_percentile, ".