.. option:: --max-jobs=nr
Set the maximum number of threads/processes to support to `nr`.
+ NOTE: On Linux, it may be necessary to increase the shared-memory
+ limit ('/proc/sys/kernel/shmmax') if fio runs into errors while
+ creating jobs.
.. option:: --server=args
named :file:`testfiles.4`. The default of :file:`$jobname.$jobnum.$filenum`
will be used if no other format specifier is given.
+ If you specify a path then the directories will be created up to the
+ main directory for the file. So for example if you specify
+ ``filename_format=a/b/c/$jobnum`` then the directories a/b/c will be
+ created before the file setup part of the job. If you specify
+ :option:`directory` then the path will be relative that directory,
+ otherwise it is treated as the absolute path.
+
.. option:: unique_filename=bool
To avoid collisions between networked clients, fio defaults to prefixing any
absolute or relative. See :file:`engines/skeleton_external.c` for
details of writing an external I/O engine.
+ **filecreate**
+ Simply create the files and do no IO to them. You still need to
+ set `filesize` so that all the accounting still occurs, but no
+ actual IO will be done other than creating the file.
I/O engine specific parameters
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
eta.c verify.c memory.c io_u.c parse.c mutex.c options.c \
smalloc.c filehash.c profile.c debug.c engines/cpu.c \
engines/mmap.c engines/sync.c engines/null.c engines/net.c \
- engines/ftruncate.c \
+ engines/ftruncate.c engines/filecreate.c \
server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \
gettime-thread.c helpers.c json.c idletime.c td_error.c \
profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \
-clone_depth: 50
+clone_depth: 1
environment:
+ CYG_MIRROR: http://cygwin.mirror.constant.com
+ CYG_ROOT: C:\cygwin64
MAKEFLAGS: -j 2
matrix:
- platform: x86_64
BUILD_ARCH: x64
- CYG_ROOT: C:\cygwin64
+ PACKAGE_ARCH: x86_64
CONFIGURE_OPTIONS:
- platform: x86
BUILD_ARCH: x86
- CYG_ROOT: C:\cygwin
+ PACKAGE_ARCH: i686
CONFIGURE_OPTIONS: --build-32bit-win
+install:
+ - '%CYG_ROOT%\setup-x86_64.exe --quiet-mode --no-shortcuts --only-site --site "%CYG_MIRROR%" --packages "mingw64-%PACKAGE_ARCH%-zlib" > NULL'
+
build_script:
- SET PATH=%CYG_ROOT%\bin;%PATH%
- 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe'
for_each_td(td, i) {
int flags = 0;
- /*
- * ->io_ops is NULL for a thread that has closed its
- * io engine
- */
- if (td->io_ops && !strcmp(td->io_ops->name, "cpuio"))
+ if (!strcmp(td->o.ioengine, "cpuio"))
cputhreads++;
else
realthreads++;
handle_trace(td, &t, ios, rw_bs);
} while (1);
- for (i = 0; i < td->files_index; i++) {
- f = td->files[i];
+ for_each_file(td, f, i)
trace_add_open_close_event(td, f->fileno, FIO_LOG_CLOSE_FILE);
- }
fifo_free(fifo);
close(fd);
fi
cross_prefix=${cross_prefix-${CROSS_COMPILE}}
-cc="${CC-${cross_prefix}gcc}"
+# Preferred compiler (can be overriden later after we know the platform):
+# ${CC} (if set)
+# ${cross_prefix}gcc (if cross-prefix specified)
+# gcc if available
+# clang if available
+if test -z "${CC}${cross_prefix}"; then
+ if has gcc; then
+ cc=gcc
+ elif has clang; then
+ cc=clang
+ fi
+else
+ cc="${CC-${cross_prefix}gcc}"
+fi
if check_define __ANDROID__ ; then
targetos="Android"
CYGWIN*)
# We still force some options, so keep this message here.
echo "Forcing some known good options on Windows"
- if test -z "$CC" ; then
+ if test -z "${CC}${cross_prefix}"; then
if test ! -z "$build_32bit_win" && test "$build_32bit_win" = "yes"; then
- CC="i686-w64-mingw32-gcc"
+ cc="i686-w64-mingw32-gcc"
if test -e "../zlib/contrib/vstudio/vc14/x86/ZlibStatReleaseWithoutAsm/zlibstat.lib"; then
echo "Building with zlib support"
output_sym "CONFIG_ZLIB"
echo "LIBS=../zlib/contrib/vstudio/vc14/x86/ZlibStatReleaseWithoutAsm/zlibstat.lib" >> $config_host_mak
fi
else
- CC="x86_64-w64-mingw32-gcc"
+ cc="x86_64-w64-mingw32-gcc"
if test -e "../zlib/contrib/vstudio/vc14/x64/ZlibStatReleaseWithoutAsm/zlibstat.lib"; then
echo "Building with zlib support"
output_sym "CONFIG_ZLIB"
tls_thread="yes"
static_assert="yes"
ipv6="yes"
- echo "CC=$CC" >> $config_host_mak
+ mkdir_two="no"
echo "BUILD_CFLAGS=$CFLAGS -I../zlib -include config-host.h -D_GNU_SOURCE" >> $config_host_mak
;;
esac
+# Now we know the target platform we can have another guess at the preferred
+# compiler when it wasn't explictly set
+if test -z "${CC}${cross_prefix}"; then
+ if test "$targetos" = "FreeBSD" || test "$targetos" = "Darwin"; then
+ if has clang; then
+ cc=clang
+ fi
+ fi
+fi
+if test -z "$cc"; then
+ echo "configure: failed to find compiler"
+ exit 1
+fi
+
if test ! -z "$cpu" ; then
# command line argument
:
;;
esac
-if test -z "$CC" ; then
- if test "$targetos" = "FreeBSD"; then
- if has clang; then
- CC=clang
- else
- CC=gcc
- fi
- fi
-fi
-
-cc="${CC-${cross_prefix}gcc}"
-
##########################################
# check cross compile
fi
print_config "cuda" "$cuda"
+##########################################
+# mkdir() probe. mingw apparently has a one-argument mkdir :/
+mkdir_two="no"
+cat > $TMPC << EOF
+#include <sys/stat.h>
+#include <sys/types.h>
+int main(int argc, char **argv)
+{
+ return mkdir("/tmp/bla", 0600);
+}
+EOF
+if compile_prog "" "" "mkdir(a, b)"; then
+ mkdir_two="yes"
+fi
+print_config "mkdir(a, b)" "$mkdir_two"
+
#############################################################################
if test "$wordsize" = "64" ; then
if test "$cuda" = "yes" ; then
output_sym "CONFIG_CUDA"
fi
+if test "$mkdir_two" = "yes" ; then
+ output_sym "CONFIG_HAVE_MKDIR_TWO"
+fi
echo "LIBS+=$LIBS" >> $config_host_mak
echo "GFIO_LIBS+=$GFIO_LIBS" >> $config_host_mak
--- /dev/null
+/*
+ * filecreate engine
+ *
+ * IO engine that doesn't do any IO, just creates files and tracks the latency
+ * of the file creation.
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "../fio.h"
+#include "../filehash.h"
+
+struct fc_data {
+ enum fio_ddir stat_ddir;
+};
+
+static int open_file(struct thread_data *td, struct fio_file *f)
+{
+ struct timespec start;
+ int do_lat = !td->o.disable_lat;
+
+ dprint(FD_FILE, "fd open %s\n", f->file_name);
+
+ if (f->filetype != FIO_TYPE_FILE) {
+ log_err("fio: only files are supported fallocate \n");
+ return 1;
+ }
+ if (!strcmp(f->file_name, "-")) {
+ log_err("fio: can't read/write to stdin/out\n");
+ return 1;
+ }
+
+ if (do_lat)
+ fio_gettime(&start, NULL);
+
+ f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600);
+
+ if (f->fd == -1) {
+ char buf[FIO_VERROR_SIZE];
+ int e = errno;
+
+ snprintf(buf, sizeof(buf), "open(%s)", f->file_name);
+ td_verror(td, e, buf);
+ return 1;
+ }
+
+ if (do_lat) {
+ struct fc_data *data = td->io_ops_data;
+ uint64_t nsec;
+
+ nsec = ntime_since_now(&start);
+ add_clat_sample(td, data->stat_ddir, nsec, 0, 0);
+ }
+
+ return 0;
+}
+
+static int queue_io(struct thread_data *td, struct io_u fio_unused *io_u)
+{
+ return FIO_Q_COMPLETED;
+}
+
+/*
+ * Ensure that we at least have a block size worth of IO to do for each
+ * file. If the job file has td->o.size < nr_files * block_size, then
+ * fio won't do anything.
+ */
+static int get_file_size(struct thread_data *td, struct fio_file *f)
+{
+ f->real_file_size = td_min_bs(td);
+ return 0;
+}
+
+static int init(struct thread_data *td)
+{
+ struct fc_data *data;
+
+ data = calloc(1, sizeof(*data));
+
+ if (td_read(td))
+ data->stat_ddir = DDIR_READ;
+ else if (td_write(td))
+ data->stat_ddir = DDIR_WRITE;
+
+ td->io_ops_data = data;
+ return 0;
+}
+
+static void cleanup(struct thread_data *td)
+{
+ struct fc_data *data = td->io_ops_data;
+
+ free(data);
+}
+
+static struct ioengine_ops ioengine = {
+ .name = "filecreate",
+ .version = FIO_IOOPS_VERSION,
+ .init = init,
+ .cleanup = cleanup,
+ .queue = queue_io,
+ .get_file_size = get_file_size,
+ .open_file = open_file,
+ .close_file = generic_close_file,
+ .flags = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO |
+ FIO_NOSTATS | FIO_NOFILEHASH,
+};
+
+static void fio_init fio_filecreate_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_filecreate_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
}
}
+static int windowsaio_invalidate_cache(struct fio_file *f)
+{
+ DWORD error;
+ DWORD isharemode = (FILE_SHARE_DELETE | FILE_SHARE_READ |
+ FILE_SHARE_WRITE);
+ HANDLE ihFile;
+ int rc = 0;
+
+ /*
+ * Encourage Windows to drop cached parts of a file by temporarily
+ * opening it for non-buffered access. Note: this will only work when
+ * the following is the only thing with the file open on the whole
+ * system.
+ */
+ dprint(FD_IO, "windowaio: attempt invalidate cache for %s\n",
+ f->file_name);
+ ihFile = CreateFile(f->file_name, 0, isharemode, NULL, OPEN_EXISTING,
+ FILE_FLAG_NO_BUFFERING, NULL);
+
+ if (ihFile != INVALID_HANDLE_VALUE) {
+ if (!CloseHandle(ihFile)) {
+ error = GetLastError();
+ log_info("windowsaio: invalidation fd close %s "
+ "failed: error %d\n", f->file_name, error);
+ rc = 1;
+ }
+ } else {
+ error = GetLastError();
+ if (error != ERROR_FILE_NOT_FOUND) {
+ log_info("windowsaio: cache invalidation of %s failed: "
+ "error %d\n", f->file_name, error);
+ rc = 1;
+ }
+ }
+
+ return rc;
+}
+
static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
{
int rc = 0;
else
openmode = OPEN_EXISTING;
+ /* If we're going to use direct I/O, Windows will try and invalidate
+ * its cache at that point so there's no need to do it here */
+ if (td->o.invalidate_cache && !td->o.odirect)
+ windowsaio_invalidate_cache(f);
+
f->hFile = CreateFile(f->file_name, access, sharemode,
NULL, openmode, flags, NULL);
--- /dev/null
+# Example filecreate job
+#
+# create_on_open is needed so that the open happens during the run and not the
+# setup.
+#
+# openfiles needs to be set so that you do not exceed the maximum allowed open
+# files.
+#
+# filesize needs to be set to a non zero value so fio will actually run, but the
+# IO will not really be done and the write latency numbers will only reflect the
+# open times.
+[global]
+create_on_open=1
+nrfiles=31250
+ioengine=filecreate
+fallocate=none
+filesize=4k
+openfiles=1
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
+[t6]
+[t7]
+[t8]
+[t9]
+[t10]
+[t11]
+[t12]
+[t13]
+[t14]
+[t15]
{
struct fio_file *f;
unsigned int i;
+ bool use_free = td_ioengine_flagged(td, FIO_NOFILEHASH);
dprint(FD_FILE, "close files\n");
td_io_unlink_file(td, f);
}
- sfree(f->file_name);
+ if (use_free)
+ free(f->file_name);
+ else
+ sfree(f->file_name);
f->file_name = NULL;
if (fio_file_axmap(f)) {
axmap_free(f->io_axmap);
f->io_axmap = NULL;
}
- sfree(f);
+ if (use_free)
+ free(f);
+ else
+ sfree(f);
}
td->o.filename = NULL;
{
struct fio_file *f;
- f = smalloc(sizeof(*f));
+ if (td_ioengine_flagged(td, FIO_NOFILEHASH))
+ f = calloc(1, sizeof(*f));
+ else
+ f = smalloc(sizeof(*f));
if (!f) {
assert(0);
return NULL;
return true;
}
+static int create_work_dirs(struct thread_data *td, const char *fname)
+{
+ char path[PATH_MAX];
+ char *start, *end;
+
+ if (td->o.directory) {
+ snprintf(path, PATH_MAX, "%s%c%s", td->o.directory,
+ FIO_OS_PATH_SEPARATOR, fname);
+ start = strstr(path, fname);
+ } else {
+ snprintf(path, PATH_MAX, "%s", fname);
+ start = path;
+ }
+
+ end = start;
+ while ((end = strchr(end, FIO_OS_PATH_SEPARATOR)) != NULL) {
+ if (end == start)
+ break;
+ *end = '\0';
+ errno = 0;
+#ifdef CONFIG_HAVE_MKDIR_TWO
+ if (mkdir(path, 0600) && errno != EEXIST) {
+#else
+ if (mkdir(path) && errno != EEXIST) {
+#endif
+ log_err("fio: failed to create dir (%s): %d\n",
+ start, errno);
+ return 1;
+ }
+ *end = FIO_OS_PATH_SEPARATOR;
+ end++;
+ }
+ td->flags |= TD_F_DIRS_CREATED;
+ return 0;
+}
+
int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
{
int cur_files = td->files_index;
sprintf(file_name + len, "%s", fname);
+ if (strchr(fname, FIO_OS_PATH_SEPARATOR) &&
+ !(td->flags & TD_F_DIRS_CREATED) &&
+ create_work_dirs(td, fname))
+ return 1;
+
/* clean cloned siblings using existing files */
if (numjob && is_already_allocated(file_name) &&
!exists_and_not_regfile(fname))
if (td->io_ops && td_ioengine_flagged(td, FIO_DISKLESSIO))
f->real_file_size = -1ULL;
- f->file_name = smalloc_strdup(file_name);
+ if (td_ioengine_flagged(td, FIO_NOFILEHASH))
+ f->file_name = strdup(file_name);
+ else
+ f->file_name = smalloc_strdup(file_name);
if (!f->file_name)
assert(0);
if (f->filetype == FIO_TYPE_FILE)
td->nr_normal_files++;
- set_already_allocated(file_name);
+ if (td->o.numjobs > 1)
+ set_already_allocated(file_name);
if (inc)
td->o.nr_files++;
if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
continue;
- sprintf(full_path, "%s%s%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name);
+ sprintf(full_path, "%s%c%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name);
if (lstat(full_path, &sb) == -1) {
if (errno != ENOENT) {
__f = alloc_new_file(td);
if (f->file_name) {
- __f->file_name = smalloc_strdup(f->file_name);
+ if (td_ioengine_flagged(td, FIO_NOFILEHASH))
+ __f->file_name = strdup(f->file_name);
+ else
+ __f->file_name = smalloc_strdup(f->file_name);
if (!__f->file_name)
assert(0);
.TP
.BI \-\-max\-jobs \fR=\fPnr
Set the maximum number of threads/processes to support to \fInr\fR.
+NOTE: On Linux, it may be necessary to increase the shared-memory limit
+(`/proc/sys/kernel/shmmax') if fio runs into errors while creating jobs.
.TP
.BI \-\-server \fR=\fPargs
Start a backend server, with \fIargs\fR specifying what to listen to.
`testfiles.$filenum' is specified, file number 4 for any job will be
named `testfiles.4'. The default of `$jobname.$jobnum.$filenum'
will be used if no other format specifier is given.
+.P
+If you specify a path then the directories will be created up to the main
+directory for the file. So for example if you specify `a/b/c/$jobnum` then the
+directories a/b/c will be created before the file setup part of the job. If you
+specify \fBdirectory\fR then the path will be relative that directory, otherwise
+it is treated as the absolute path.
.RE
.TP
.BI unique_filename \fR=\fPbool
ioengine `foo.o' in `/tmp'. The path can be either
absolute or relative. See `engines/skeleton_external.c' in the fio source for
details of writing an external I/O engine.
+.TP
+.B filecreate
+Create empty files only. \fBfilesize\fR still needs to be specified so that fio
+will run and grab latency results, but no IO will actually be done on the files.
.SS "I/O engine specific parameters"
In addition, there are some parameters which are only valid when a specific
\fBioengine\fR is in use. These are used identically to normal parameters,
};
enum {
- TD_F_VER_BACKLOG = 1U << 0,
- TD_F_TRIM_BACKLOG = 1U << 1,
- TD_F_READ_IOLOG = 1U << 2,
- TD_F_REFILL_BUFFERS = 1U << 3,
- TD_F_SCRAMBLE_BUFFERS = 1U << 4,
- TD_F_VER_NONE = 1U << 5,
- TD_F_PROFILE_OPS = 1U << 6,
- TD_F_COMPRESS = 1U << 7,
- TD_F_RESERVED = 1U << 8, /* not used */
- TD_F_COMPRESS_LOG = 1U << 9,
- TD_F_VSTATE_SAVED = 1U << 10,
- TD_F_NEED_LOCK = 1U << 11,
- TD_F_CHILD = 1U << 12,
- TD_F_NO_PROGRESS = 1U << 13,
- TD_F_REGROW_LOGS = 1U << 14,
- TD_F_MMAP_KEEP = 1U << 15,
+ __TD_F_VER_BACKLOG = 0,
+ __TD_F_TRIM_BACKLOG,
+ __TD_F_READ_IOLOG,
+ __TD_F_REFILL_BUFFERS,
+ __TD_F_SCRAMBLE_BUFFERS,
+ __TD_F_VER_NONE,
+ __TD_F_PROFILE_OPS,
+ __TD_F_COMPRESS,
+ __TD_F_COMPRESS_LOG,
+ __TD_F_VSTATE_SAVED,
+ __TD_F_NEED_LOCK,
+ __TD_F_CHILD,
+ __TD_F_NO_PROGRESS,
+ __TD_F_REGROW_LOGS,
+ __TD_F_MMAP_KEEP,
+ __TD_F_DIRS_CREATED,
+ __TD_F_LAST, /* not a real bit, keep last */
+};
+
+enum {
+ TD_F_VER_BACKLOG = 1U << __TD_F_VER_BACKLOG,
+ TD_F_TRIM_BACKLOG = 1U << __TD_F_TRIM_BACKLOG,
+ TD_F_READ_IOLOG = 1U << __TD_F_READ_IOLOG,
+ TD_F_REFILL_BUFFERS = 1U << __TD_F_REFILL_BUFFERS,
+ TD_F_SCRAMBLE_BUFFERS = 1U << __TD_F_SCRAMBLE_BUFFERS,
+ TD_F_VER_NONE = 1U << __TD_F_VER_NONE,
+ TD_F_PROFILE_OPS = 1U << __TD_F_PROFILE_OPS,
+ TD_F_COMPRESS = 1U << __TD_F_COMPRESS,
+ TD_F_COMPRESS_LOG = 1U << __TD_F_COMPRESS_LOG,
+ TD_F_VSTATE_SAVED = 1U << __TD_F_VSTATE_SAVED,
+ TD_F_NEED_LOCK = 1U << __TD_F_NEED_LOCK,
+ TD_F_CHILD = 1U << __TD_F_CHILD,
+ TD_F_NO_PROGRESS = 1U << __TD_F_NO_PROGRESS,
+ TD_F_REGROW_LOGS = 1U << __TD_F_REGROW_LOGS,
+ TD_F_MMAP_KEEP = 1U << __TD_F_MMAP_KEEP,
+ TD_F_DIRS_CREATED = 1U << __TD_F_DIRS_CREATED,
};
enum {
#define TD_ENG_FLAG_SHIFT 16
#define TD_ENG_FLAG_MASK ((1U << 16) - 1)
-static inline enum fio_ioengine_flags td_ioengine_flags(struct thread_data *td)
-{
- return (enum fio_ioengine_flags)
- ((td->flags >> TD_ENG_FLAG_SHIFT) & TD_ENG_FLAG_MASK);
-}
-
static inline void td_set_ioengine_flags(struct thread_data *td)
{
td->flags = (~(TD_ENG_FLAG_MASK << TD_ENG_FLAG_SHIFT) & td->flags) |
struct thread_data;
extern uint64_t ntime_since(const struct timespec *, const struct timespec *);
+extern uint64_t ntime_since_now(const struct timespec *);
extern uint64_t utime_since(const struct timespec *, const struct timespec *);
extern uint64_t utime_since_now(const struct timespec *);
extern uint64_t mtime_since(const struct timespec *, const struct timespec *);
int ddir)
{
unsigned int *io_u_plat = ts->io_u_plat[ddir];
- unsigned long nr = ts->clat_stat[ddir].samples;
+ unsigned long long nr = ts->clat_stat[ddir].samples;
fio_fp64_t *plist = ts->percentile_list;
unsigned int len, scale_down;
unsigned long long *ovals, minv, maxv;
#if defined(ARCH_HAVE_CPU_CLOCK)
#ifndef ARCH_CPU_CLOCK_CYCLES_PER_USEC
-static unsigned long cycles_per_msec;
+static unsigned long long cycles_per_msec;
static unsigned long long cycles_start;
static unsigned long long clock_mult;
static unsigned long long max_cycles_mask;
return nsec + (sec * 1000000000LL);
}
+uint64_t ntime_since_now(const struct timespec *s)
+{
+ struct timespec now;
+
+ fio_gettime(&now, NULL);
+ return ntime_since(s, &now);
+}
+
uint64_t utime_since(const struct timespec *s, const struct timespec *e)
{
int64_t sec, usec;
if (td->parent)
td = td->parent;
- if (!td->o.stats)
+ if (!td->o.stats || td_ioengine_flagged(td, FIO_NOSTATS))
return;
if (no_reduce)
FIO_MEMALIGN = 1 << 9, /* engine wants aligned memory */
FIO_BIT_BASED = 1 << 10, /* engine uses a bit base (e.g. uses Kbit as opposed to KB) */
FIO_FAKEIO = 1 << 11, /* engine pretends to do IO */
+ FIO_NOSTATS = 1 << 12, /* don't do IO stats */
+ FIO_NOFILEHASH = 1 << 13, /* doesn't hash the files for lookup later. */
};
/*
compiletime_assert((offsetof(struct thread_options_pack, latency_percentile) % 8) == 0, "latency_percentile");
compiletime_assert((offsetof(struct jobs_eta, m_rate) % 8) == 0, "m_rate");
+ compiletime_assert(__TD_F_LAST <= TD_ENG_FLAG_SHIFT, "TD_ENG_FLAG_SHIFT");
+
err = endian_check();
if (err) {
log_err("fio: endianness settings appear wrong.\n");
.help = "DAX Device based IO engine",
},
#endif
+ {
+ .ival = "filecreate",
+ .help = "File creation engine",
+ },
{ .ival = "external",
.help = "Load external engine (append name)",
.cb = str_ioengine_external_cb,
#define FIO_PREFERRED_ENGINE "windowsaio"
#define FIO_PREFERRED_CLOCK_SOURCE CS_CGETTIME
-#define FIO_OS_PATH_SEPARATOR "\\"
+#define FIO_OS_PATH_SEPARATOR '\\'
#define FIO_MAX_CPUS MAXIMUM_PROCESSORS
#endif
#ifndef FIO_OS_PATH_SEPARATOR
-#define FIO_OS_PATH_SEPARATOR "/"
+#define FIO_OS_PATH_SEPARATOR '/'
#endif
#ifndef FIO_PREFERRED_CLOCK_SOURCE
return cmp;
}
-unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
+unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr,
fio_fp64_t *plist, unsigned long long **output,
unsigned long long *maxv, unsigned long long *minv)
{
- unsigned long sum = 0;
+ unsigned long long sum = 0;
unsigned int len, i, j = 0;
unsigned int oval_len = 0;
unsigned long long *ovals = NULL;
/*
* Find and display the p-th percentile of clat
*/
-static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
+static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr,
fio_fp64_t *plist, unsigned int precision,
bool is_clat, struct buf_output *out)
{
unsigned int len;
int i;
const char *ddirname[] = {"read", "write", "trim"};
- struct json_object *dir_object, *tmp_object, *percentile_object, *clat_bins_object;
+ struct json_object *dir_object, *tmp_object, *percentile_object, *clat_bins_object = NULL;
char buf[120];
double p_of_agg = 100.0;
if (output_format & FIO_OUTPUT_JSON_PLUS) {
clat_bins_object = json_create_object();
- json_object_add_value_object(tmp_object, "bins", clat_bins_object);
+ if (ts->clat_percentiles)
+ json_object_add_value_object(tmp_object, "bins", clat_bins_object);
+
for(i = 0; i < FIO_IO_U_PLAT_NR; i++) {
if (ts->io_u_plat[ddir][i]) {
snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i));
json_object_add_value_int(tmp_object, "max", max);
json_object_add_value_float(tmp_object, "mean", mean);
json_object_add_value_float(tmp_object, "stddev", dev);
+ if (output_format & FIO_OUTPUT_JSON_PLUS && ts->lat_percentiles)
+ json_object_add_value_object(tmp_object, "bins", clat_bins_object);
+
if (ovals)
free(ovals);
#define FIO_IO_U_LAT_U_NR 10
#define FIO_IO_U_LAT_M_NR 12
+/*
+ * Constants for clat percentiles
+ */
+#define FIO_IO_U_PLAT_BITS 6
+#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS)
+#define FIO_IO_U_PLAT_GROUP_NR 29
+#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL)
+#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified
+ list of percentiles */
+
/*
* Aggregate clat samples to report percentile(s) of them.
*
*
* FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the maximum
* range being tracked for latency samples. The maximum value tracked
- * accurately will be 2^(GROUP_NR + PLAT_BITS -1) microseconds.
+ * accurately will be 2^(GROUP_NR + PLAT_BITS - 1) nanoseconds.
*
* FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the memory
* requirement of storing those aggregate counts. The memory used will
* 3 8 2 [256,511] 64
* 4 9 3 [512,1023] 64
* ... ... ... [...,...] ...
- * 18 23 17 [8838608,+inf]** 64
+ * 28 33 27 [8589934592,+inf]** 64
*
* * Special cases: when n < (M-1) or when n == (M-1), in both cases,
* the value cannot be rounded off. Use all bits of the sample as
* index.
*
- * ** If a sample's MSB is greater than 23, it will be counted as 23.
+ * ** If a sample's MSB is greater than 33, it will be counted as 33.
*/
-#define FIO_IO_U_PLAT_BITS 6
-#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS)
-#define FIO_IO_U_PLAT_GROUP_NR 29
-#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL)
-#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified
- list of percentiles */
-
/*
* Trim cycle count measurements
*/
extern void init_group_run_stat(struct group_run_stats *gs);
extern void eta_to_str(char *str, unsigned long eta_sec);
extern bool calc_lat(struct io_stat *is, unsigned long long *min, unsigned long long *max, double *mean, double *dev);
-extern unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr, fio_fp64_t *plist, unsigned long long **output, unsigned long long *maxv, unsigned long long *minv);
+extern unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr, fio_fp64_t *plist, unsigned long long **output, unsigned long long *maxv, unsigned long long *minv);
extern void stat_calc_lat_n(struct thread_stat *ts, double *io_u_lat);
extern void stat_calc_lat_m(struct thread_stat *ts, double *io_u_lat);
extern void stat_calc_lat_u(struct thread_stat *ts, double *io_u_lat);
i=0
- for x in *_"$FILETYPE".log
+ for x in *_"$FILETYPE".log *_"$FILETYPE".*.log
do
- i=$((i+1))
- PT=$(echo $x | sed s/_"$FILETYPE".log//g)
- if [ ! -z "$PLOT_LINE" ]
- then
- PLOT_LINE=$PLOT_LINE", "
+ if [ -e "$x" ]; then
+ i=$((i+1))
+ PT=$(echo $x | sed 's/\(.*\)_'$FILETYPE'\(.*\).log$/\1\2/')
+ if [ ! -z "$PLOT_LINE" ]
+ then
+ PLOT_LINE=$PLOT_LINE", "
+ fi
+
+ DEPTH=$(echo $PT | cut -d "-" -f 4)
+ PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i"
fi
-
- DEPTH=$(echo $PT | cut -d "-" -f 4)
- PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i"
-
done
+ if [ $i -eq 0 ]; then
+ echo "No log files found"
+ exit 1
+ fi
+
OUTPUT="set output \"$TITLE-$FILETYPE.svg\" "
echo " $PLOT_TITLE ; $YAXIS ; $DEFAULT_OPTS ; show style lines ; $OUTPUT ; plot " $PLOT_LINE | $GNUPLOT -
prev_ddir = None
for ddir in ddir_set:
+ if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']:
+ bins_loc = 'clat_ns'
+ elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']:
+ bins_loc = 'lat_ns'
+ else:
+ raise RuntimeError("Latency bins not found. "
+ "Are you sure you are using json+ output?")
+
bins[ddir] = [[int(key), value] for key, value in
- jsondata['jobs'][jobnum][ddir]['clat_ns']
+ jsondata['jobs'][jobnum][ddir][bins_loc]
['bins'].iteritems()]
bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0])
outfile = stub + '_job' + str(jobnum) + ext
with open(outfile, 'w') as output:
- output.write("clat_nsec, ")
+ output.write("{0}ec, ".format(bins_loc))
ddir_list = list(ddir_set)
for ddir in ddir_list:
output.write("{0}_count, {0}_cumulative, {0}_percentile, ".
memset(fname, 0, sizeof(fname));
if (aux_path)
- sprintf(fname, "%s%s", aux_path, FIO_OS_PATH_SEPARATOR);
+ sprintf(fname, "%s%c", aux_path, FIO_OS_PATH_SEPARATOR);
strncpy(fname + strlen(fname), basename(ptr), buf_left - 1);
char prefix[PATH_MAX];
if (aux_path)
- sprintf(prefix, "%s%slocal", aux_path, FIO_OS_PATH_SEPARATOR);
+ sprintf(prefix, "%s%clocal", aux_path, FIO_OS_PATH_SEPARATOR);
else
strcpy(prefix, "local");
projects / fio.git / commitdiff