From 2b386d2569c9078ca9790c4e6d318ec3835b8739 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 26 Mar 2008 10:32:57 +0100 Subject: [PATCH] Add softrandommap and --alloc-size options The softrandommap job option allows fio to continue if it cannot allocate from the smalloc backing for the random map. Or the user can specify --alloc-size with a size larger than the default of 1024k to increase the size of the shared allocation pool. Signed-off-by: Jens Axboe --- HOWTO | 6 ++++++ README | 7 +++++++ filesetup.c | 7 +++++++ fio.h | 1 + init.c | 10 ++++++++++ options.c | 8 ++++++++ smalloc.c | 6 ++++-- smalloc.h | 2 ++ 8 files changed, 45 insertions(+), 2 deletions(-) diff --git a/HOWTO b/HOWTO index 3d922933..66ebebe8 100644 --- a/HOWTO +++ b/HOWTO @@ -482,6 +482,12 @@ norandommap Normally fio will cover every block of the file when doing fio doesn't track potential block rewrites which may alter the calculated checksum for that block. +softrandommap See norandommap. If fio runs with the random block map enabled + and it fails to allocate the map, if this option is set it + will continue without a random block map. As coverage will + not be as complete as with random maps, this option is + disabled by default. + nice=int Run the job with the given nice value. See man nice(2). prio=int Set the io priority value of this job. Linux limits us to diff --git a/README b/README index 7d6262ec..db8e6724 100644 --- a/README +++ b/README @@ -76,6 +76,7 @@ $ fio --eta=when When ETA estimate should be printed May be "always", "never" or "auto" --section=name Only run specified section in job file + --alloc-size=kb Set smalloc pool to this size in kb (def 1024) Any parameters following the options will be assumed to be job files, @@ -114,6 +115,12 @@ only by giving it a --section=heavy command line option. The section option only applies to job sections, the reserved 'global' section is always parsed and taken into account. +Fio has an internal allocator for shared memory called smalloc. It +allocates shared structures from this pool. The pool defaults to 1024k +in size, and can grow to 32 pools. If running large jobs with randommap +enabled it can run out of memory, in which case the --alloc-size switch +is handy for starting with a larger pool size. + Job file -------- diff --git a/filesetup.c b/filesetup.c index 6c5770b4..b356b447 100644 --- a/filesetup.c +++ b/filesetup.c @@ -579,6 +579,13 @@ int init_random_map(struct thread_data *td) f->num_maps = num_maps; continue; } + if (!td->o.softrandommap) { + log_err("fio: failed allocating random map. If running" + " a large number of jobs, try the 'norandommap'" + " option or set 'softrandommap'. Or give" + " a larger --alloc-size to fio.\n"); + return 1; + } log_info("fio: file %s failed allocating random map. Running " "job without.\n", f->file_name); diff --git a/fio.h b/fio.h index 55717c79..b2930f11 100644 --- a/fio.h +++ b/fio.h @@ -446,6 +446,7 @@ struct thread_options { unsigned int write_lat_log; unsigned int write_bw_log; unsigned int norandommap; + unsigned int softrandommap; unsigned int bs_unaligned; unsigned int fsync_on_close; diff --git a/init.c b/init.c index effe63c5..1dfaa1f0 100644 --- a/init.c +++ b/init.c @@ -118,6 +118,11 @@ static struct option l_opts[FIO_NR_OPTIONS] = { .has_arg = required_argument, .val = 'x', }, + { + .name = "alloc-size", + .has_arg = required_argument, + .val = 'a', + }, { .name = NULL, }, @@ -826,6 +831,8 @@ static void usage(const char *name) printf("\t--readonly\tTurn on safety read-only checks, preventing" " writes\n"); printf("\t--section=name\tOnly run specified section in job file\n"); + printf("\t--alloc-size=kb\tSet smalloc pool to this size in kb" + " (def 1024)\n"); } #ifdef FIO_INC_DEBUG @@ -897,6 +904,9 @@ static int parse_cmd_line(int argc, char *argv[]) while ((c = getopt_long_only(argc, argv, "", l_opts, &lidx)) != -1) { switch (c) { + case 'a': + smalloc_pool_size = atoi(optarg); + break; case 't': def_timeout = atoi(optarg); break; diff --git a/options.c b/options.c index 5626da79..ead31ace 100644 --- a/options.c +++ b/options.c @@ -668,6 +668,14 @@ static struct fio_option options[] = { .help = "Accept potential duplicate random blocks", .parent = "rw", }, + { + .name = "softrandommap", + .type = FIO_OPT_BOOL, + .off1 = td_var_offset(softrandommap), + .help = "Allow randommap to fail and continue witout", + .parent = "norandommap", + .def = "0", + }, { .name = "nrfiles", .type = FIO_OPT_INT, diff --git a/smalloc.c b/smalloc.c index fd0e9bd0..5baf5a43 100644 --- a/smalloc.c +++ b/smalloc.c @@ -19,8 +19,10 @@ #define INITIAL_SIZE 1048576 /* new pool size */ #define MAX_POOLS 32 /* maximum number of pools to setup */ +unsigned int smalloc_pool_size = INITIAL_SIZE; + #ifdef ENABLE_RESIZE -#define MAX_SIZE 8 * INITIAL_SIZE +#define MAX_SIZE 8 * smalloc_pool_size static unsigned int resize_error; #endif @@ -229,7 +231,7 @@ static int add_pool(struct pool *pool) if (fd < 0) goto out_close; - pool->size = INITIAL_SIZE; + pool->size = smalloc_pool_size; if (ftruncate(fd, pool->size) < 0) goto out_unlink; diff --git a/smalloc.h b/smalloc.h index d5d4557e..6905c6a8 100644 --- a/smalloc.h +++ b/smalloc.h @@ -7,4 +7,6 @@ extern char *smalloc_strdup(const char *); extern void sinit(void); extern void scleanup(void); +extern unsigned int smalloc_pool_size; + #endif -- 2.25.1