X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=init.c;h=63ba32481b9ba87943d320e0b6daa7f2ae31965b;hp=bb53a1e2bc097a256172db7e46075bd5658704da;hb=f9cafb12834f84aeff4f255e8c9e5c5d6b5e3bf5;hpb=f9cbae9ea572353c6aebf3390a23de1dd10ee8c7 diff --git a/init.c b/init.c index bb53a1e2..63ba3248 100644 --- a/init.c +++ b/init.c @@ -27,7 +27,7 @@ #include "filelock.h" #include "lib/getopt.h" -#include "lib/strcasestr.h" +#include "oslib/strcasestr.h" #include "crc/test.h" @@ -47,8 +47,8 @@ static char **job_sections; static int nr_job_sections; int exitall_on_terminate = 0; +int exitall_on_terminate_error = 0; int output_format = FIO_OUTPUT_NORMAL; -int append_terse_output = 0; int eta_print = FIO_ETA_AUTO; int eta_new_line = 0; FILE *f_out = NULL; @@ -69,6 +69,8 @@ long long trigger_timeout = 0; char *trigger_cmd = NULL; char *trigger_remote_cmd = NULL; +char *aux_path = NULL; + static int prev_group_jobs; unsigned long fio_debug = 0; @@ -266,6 +268,11 @@ static struct option l_opts[FIO_NR_OPTIONS] = { .has_arg = required_argument, .val = 'J', }, + { + .name = (char *) "aux-path", + .has_arg = required_argument, + .val = 'K', + }, { .name = NULL, }, @@ -458,14 +465,16 @@ static int __setup_rate(struct thread_data *td, enum fio_ddir ddir) if (td->o.rate[ddir]) td->rate_bps[ddir] = td->o.rate[ddir]; else - td->rate_bps[ddir] = td->o.rate_iops[ddir] * bs; + td->rate_bps[ddir] = (uint64_t) td->o.rate_iops[ddir] * bs; if (!td->rate_bps[ddir]) { log_err("rate lower than supported\n"); return -1; } - td->rate_pending_usleep[ddir] = 0; + td->rate_next_io_time[ddir] = 0; + td->rate_io_issue_bytes[ddir] = 0; + td->last_usec = 0; return 0; } @@ -496,12 +505,14 @@ static int fixed_block_size(struct thread_options *o) static unsigned long long get_rand_start_delay(struct thread_data *td) { unsigned long long delayrange; + uint64_t frand_max; unsigned long r; delayrange = td->o.start_delay_high - td->o.start_delay; + frand_max = rand_max(&td->delay_state); r = __rand(&td->delay_state); - delayrange = (unsigned long long) ((double) delayrange * (r / (FRAND_MAX + 1.0))); + delayrange = (unsigned long long) ((double) delayrange * (r / (frand_max + 1.0))); delayrange += td->o.start_delay; return delayrange; @@ -596,8 +607,7 @@ static int fixup_options(struct thread_data *td) if (o->norandommap && o->verify != VERIFY_NONE && !fixed_block_size(o)) { log_err("fio: norandommap given for variable block sizes, " - "verify disabled\n"); - o->verify = VERIFY_NONE; + "verify limited\n"); ret = warnings_fatal; } if (o->bs_unaligned && (o->odirect || td->io_ops->flags & FIO_RAWIO)) @@ -621,6 +631,13 @@ static int fixup_options(struct thread_data *td) if (o->iodepth_batch > o->iodepth || !o->iodepth_batch) o->iodepth_batch = o->iodepth; + /* + * If max batch complete number isn't set or set incorrectly, + * default to the same as iodepth_batch_complete_min + */ + if (o->iodepth_batch_complete_min > o->iodepth_batch_complete_max) + o->iodepth_batch_complete_max = o->iodepth_batch_complete_min; + if (o->nr_files > td->files_index) o->nr_files = td->files_index; @@ -634,12 +651,12 @@ static int fixup_options(struct thread_data *td) log_err("fio: rate and rate_iops are mutually exclusive\n"); ret = 1; } - if ((o->rate[DDIR_READ] < o->ratemin[DDIR_READ]) || - (o->rate[DDIR_WRITE] < o->ratemin[DDIR_WRITE]) || - (o->rate[DDIR_TRIM] < o->ratemin[DDIR_TRIM]) || - (o->rate_iops[DDIR_READ] < o->rate_iops_min[DDIR_READ]) || - (o->rate_iops[DDIR_WRITE] < o->rate_iops_min[DDIR_WRITE]) || - (o->rate_iops[DDIR_TRIM] < o->rate_iops_min[DDIR_TRIM])) { + if ((o->rate[DDIR_READ] && (o->rate[DDIR_READ] < o->ratemin[DDIR_READ])) || + (o->rate[DDIR_WRITE] && (o->rate[DDIR_WRITE] < o->ratemin[DDIR_WRITE])) || + (o->rate[DDIR_TRIM] && (o->rate[DDIR_TRIM] < o->ratemin[DDIR_TRIM])) || + (o->rate_iops[DDIR_READ] && (o->rate_iops[DDIR_READ] < o->rate_iops_min[DDIR_READ])) || + (o->rate_iops[DDIR_WRITE] && (o->rate_iops[DDIR_WRITE] < o->rate_iops_min[DDIR_WRITE])) || + (o->rate_iops[DDIR_TRIM] && (o->rate_iops[DDIR_TRIM] < o->rate_iops_min[DDIR_TRIM]))) { log_err("fio: minimum rate exceeds rate\n"); ret = 1; } @@ -661,7 +678,9 @@ static int fixup_options(struct thread_data *td) ret = warnings_fatal; } - o->refill_buffers = 1; + if (!fio_option_is_set(o, refill_buffers)) + o->refill_buffers = 1; + if (o->max_bs[DDIR_WRITE] != o->min_bs[DDIR_WRITE] && !o->verify_interval) o->verify_interval = o->min_bs[DDIR_WRITE]; @@ -719,11 +738,16 @@ static int fixup_options(struct thread_data *td) /* * For fully compressible data, just zero them at init time. - * It's faster than repeatedly filling it. + * It's faster than repeatedly filling it. For non-zero + * compression, we should have refill_buffers set. Set it, unless + * the job file already changed it. */ - if (td->o.compress_percentage == 100) { - td->o.zero_buffers = 1; - td->o.compress_percentage = 0; + if (o->compress_percentage) { + if (o->compress_percentage == 100) { + o->zero_buffers = 1; + o->compress_percentage = 0; + } else if (!fio_option_is_set(o, refill_buffers)) + o->refill_buffers = 1; } /* @@ -750,7 +774,7 @@ static int fixup_options(struct thread_data *td) /* * If randseed is set, that overrides randrepeat */ - if (td->o.rand_seed) + if (fio_option_is_set(&td->o, rand_seed)) td->o.rand_repeatable = 0; if ((td->io_ops->flags & FIO_NOEXTEND) && td->o.file_append) { @@ -764,6 +788,17 @@ static int fixup_options(struct thread_data *td) fio_gtod_offload = 1; } + td->loops = o->loops; + if (!td->loops) + td->loops = 1; + + if (td->o.block_error_hist && td->o.nr_files != 1) { + log_err("fio: block error histogram only available with " + "with a single file per job, but %d files " + "provided\n", td->o.nr_files); + ret = 1; + } + return ret; } @@ -817,18 +852,19 @@ static int exists_and_not_file(const char *filename) return 1; } -static void td_fill_rand_seeds_internal(struct thread_data *td) +static void td_fill_rand_seeds_internal(struct thread_data *td, int use64) { - init_rand_seed(&td->bsrange_state, td->rand_seeds[FIO_RAND_BS_OFF]); - init_rand_seed(&td->verify_state, td->rand_seeds[FIO_RAND_VER_OFF]); - init_rand_seed(&td->rwmix_state, td->rand_seeds[FIO_RAND_MIX_OFF]); + init_rand_seed(&td->bsrange_state, td->rand_seeds[FIO_RAND_BS_OFF], use64); + init_rand_seed(&td->verify_state, td->rand_seeds[FIO_RAND_VER_OFF], use64); + init_rand_seed(&td->rwmix_state, td->rand_seeds[FIO_RAND_MIX_OFF], use64); if (td->o.file_service_type == FIO_FSERVICE_RANDOM) - init_rand_seed(&td->next_file_state, td->rand_seeds[FIO_RAND_FILE_OFF]); + init_rand_seed(&td->next_file_state, td->rand_seeds[FIO_RAND_FILE_OFF], use64); - init_rand_seed(&td->file_size_state, td->rand_seeds[FIO_RAND_FILE_SIZE_OFF]); - init_rand_seed(&td->trim_state, td->rand_seeds[FIO_RAND_TRIM_OFF]); - init_rand_seed(&td->delay_state, td->rand_seeds[FIO_RAND_START_DELAY]); + init_rand_seed(&td->file_size_state, td->rand_seeds[FIO_RAND_FILE_SIZE_OFF], use64); + init_rand_seed(&td->trim_state, td->rand_seeds[FIO_RAND_TRIM_OFF], use64); + init_rand_seed(&td->delay_state, td->rand_seeds[FIO_RAND_START_DELAY], use64); + init_rand_seed(&td->poisson_state, td->rand_seeds[FIO_RAND_POISSON_OFF], 0); if (!td_random(td)) return; @@ -836,14 +872,16 @@ static void td_fill_rand_seeds_internal(struct thread_data *td) if (td->o.rand_repeatable) td->rand_seeds[FIO_RAND_BLOCK_OFF] = FIO_RANDSEED * td->thread_number; - init_rand_seed(&td->random_state, td->rand_seeds[FIO_RAND_BLOCK_OFF]); - init_rand_seed(&td->seq_rand_state[DDIR_READ], td->rand_seeds[FIO_RAND_SEQ_RAND_READ_OFF]); - init_rand_seed(&td->seq_rand_state[DDIR_WRITE], td->rand_seeds[FIO_RAND_SEQ_RAND_WRITE_OFF]); - init_rand_seed(&td->seq_rand_state[DDIR_TRIM], td->rand_seeds[FIO_RAND_SEQ_RAND_TRIM_OFF]); + init_rand_seed(&td->random_state, td->rand_seeds[FIO_RAND_BLOCK_OFF], use64); + init_rand_seed(&td->seq_rand_state[DDIR_READ], td->rand_seeds[FIO_RAND_SEQ_RAND_READ_OFF], use64); + init_rand_seed(&td->seq_rand_state[DDIR_WRITE], td->rand_seeds[FIO_RAND_SEQ_RAND_WRITE_OFF], use64); + init_rand_seed(&td->seq_rand_state[DDIR_TRIM], td->rand_seeds[FIO_RAND_SEQ_RAND_TRIM_OFF], use64); } void td_fill_rand_seeds(struct thread_data *td) { + int use64; + if (td->o.allrand_repeatable) { unsigned int i; @@ -852,12 +890,17 @@ void td_fill_rand_seeds(struct thread_data *td) + i; } - td_fill_rand_seeds_internal(td); + if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64) + use64 = 1; + else + use64 = 0; + + td_fill_rand_seeds_internal(td, use64); - init_rand_seed(&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF]); + init_rand_seed(&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF], use64); frand_copy(&td->buf_state_prev, &td->buf_state); - init_rand_seed(&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF]); + init_rand_seed(&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF], use64); } /* @@ -946,6 +989,9 @@ static void init_flags(struct thread_data *td) td->flags |= TD_F_SCRAMBLE_BUFFERS; if (o->verify != VERIFY_NONE) td->flags |= TD_F_VER_NONE; + + if (o->verify_async || o->io_submit_mode == IO_MODE_OFFLOAD) + td->flags |= TD_F_NEED_LOCK; } static int setup_random_seeds(struct thread_data *td) @@ -953,14 +999,10 @@ static int setup_random_seeds(struct thread_data *td) unsigned long seed; unsigned int i; - if (!td->o.rand_repeatable && !td->o.rand_seed) + if (!td->o.rand_repeatable && !fio_option_is_set(&td->o, rand_seed)) return init_random_state(td, td->rand_seeds, sizeof(td->rand_seeds)); - if (!td->o.rand_seed) - seed = 0x89; - else - seed = td->o.rand_seed; - + seed = td->o.rand_seed; for (i = 0; i < 4; i++) seed *= 0x9e370001UL; @@ -1098,6 +1140,16 @@ int parse_dryrun(void) return dump_cmdline || parse_only; } +static void gen_log_name(char *name, size_t size, const char *logtype, + const char *logname, unsigned int num, + const char *suf, int per_job) +{ + if (per_job) + snprintf(name, size, "%s_%s.%d.%s", logname, logtype, num, suf); + else + snprintf(name, size, "%s_%s.%s", logname, logtype, suf); +} + /* * Adds a job to the list of things todo. Sanitizes the various options * to make sure we don't have conflicts, and initializes various @@ -1187,6 +1239,10 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if ((o->stonewall || o->new_group) && prev_group_jobs) { prev_group_jobs = 0; groupid++; + if (groupid == INT_MAX) { + log_err("fio: too many groups defined\n"); + goto err; + } } td->groupid = groupid; @@ -1216,14 +1272,16 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, else suf = "log"; - snprintf(logname, sizeof(logname), "%s_lat.%d.%s", - o->lat_log_file, td->thread_number, suf); + gen_log_name(logname, sizeof(logname), "lat", o->lat_log_file, + td->thread_number, suf, o->per_job_logs); setup_log(&td->lat_log, &p, logname); - snprintf(logname, sizeof(logname), "%s_slat.%d.%s", - o->lat_log_file, td->thread_number, suf); + + gen_log_name(logname, sizeof(logname), "slat", o->lat_log_file, + td->thread_number, suf, o->per_job_logs); setup_log(&td->slat_log, &p, logname); - snprintf(logname, sizeof(logname), "%s_clat.%d.%s", - o->lat_log_file, td->thread_number, suf); + + gen_log_name(logname, sizeof(logname), "clat", o->lat_log_file, + td->thread_number, suf, o->per_job_logs); setup_log(&td->clat_log, &p, logname); } if (o->bw_log_file) { @@ -1242,8 +1300,8 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, else suf = "log"; - snprintf(logname, sizeof(logname), "%s_bw.%d.%s", - o->bw_log_file, td->thread_number, suf); + gen_log_name(logname, sizeof(logname), "bw", o->bw_log_file, + td->thread_number, suf, o->per_job_logs); setup_log(&td->bw_log, &p, logname); } if (o->iops_log_file) { @@ -1262,15 +1320,15 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, else suf = "log"; - snprintf(logname, sizeof(logname), "%s_iops.%d.%s", - o->iops_log_file, td->thread_number, suf); + gen_log_name(logname, sizeof(logname), "iops", o->iops_log_file, + td->thread_number, suf, o->per_job_logs); setup_log(&td->iops_log, &p, logname); } if (!o->name) o->name = strdup(jobname); - if (output_format == FIO_OUTPUT_NORMAL) { + if (output_format & FIO_OUTPUT_NORMAL) { if (!job_add_num) { if (is_backend && !recursed) fio_server_send_add_job(td); @@ -1680,19 +1738,47 @@ static int fill_def_thread(void) return 0; } +static void show_debug_categories(void) +{ + struct debug_level *dl = &debug_levels[0]; + int curlen, first = 1; + + curlen = 0; + while (dl->name) { + int has_next = (dl + 1)->name != NULL; + + if (first || curlen + strlen(dl->name) >= 80) { + if (!first) { + printf("\n"); + curlen = 0; + } + curlen += printf("\t\t\t%s", dl->name); + curlen += 3 * (8 - 1); + if (has_next) + curlen += printf(","); + } else { + curlen += printf("%s", dl->name); + if (has_next) + curlen += printf(","); + } + dl++; + first = 0; + } + printf("\n"); +} + static void usage(const char *name) { printf("%s\n", fio_version_string); printf("%s [options] [job options] \n", name); - printf(" --debug=options\tEnable debug logging. May be one/more of:\n" - "\t\t\tprocess,file,io,mem,blktrace,verify,random,parse,\n" - "\t\t\tdiskutil,job,mutex,profile,time,net,rate,compress\n"); + printf(" --debug=options\tEnable debug logging. May be one/more of:\n"); + show_debug_categories(); printf(" --parse-only\t\tParse options only, don't start any IO\n"); printf(" --output\t\tWrite output to file\n"); printf(" --runtime\t\tRuntime in seconds\n"); printf(" --bandwidth-log\tGenerate per-job bandwidth logs\n"); printf(" --minimal\t\tMinimal (terse) output\n"); - printf(" --output-format=x\tOutput format (terse,json,normal)\n"); + printf(" --output-format=x\tOutput format (terse,json,json+,normal)\n"); printf(" --terse-version=x\tSet terse version output format to 'x'\n"); printf(" --version\t\tPrint version info and exit\n"); printf(" --help\t\tPrint this page\n"); @@ -1732,6 +1818,7 @@ static void usage(const char *name) printf(" --trigger-timeout=t\tExecute trigger af this time\n"); printf(" --trigger=cmd\t\tSet this command as local trigger\n"); printf(" --trigger-remote=cmd\tSet this command as remote trigger\n"); + printf(" --aux-path=path\tUse this path for fio state generated files\n"); printf("\nFio was written by Jens Axboe "); printf("\n Jens Axboe "); printf("\n Jens Axboe \n"); @@ -1813,6 +1900,9 @@ static int set_debug(const char *string) char *opt; int i; + if (!string) + return 0; + if (!strcmp(string, "?") || !strcmp(string, "help")) { log_info("fio: dumping debug options:"); for (i = 0; debug_levels[i].name; i++) { @@ -1929,10 +2019,41 @@ static void show_closest_option(const char *name) i++; } - if (best_option != -1) + if (best_option != -1 && string_distance_ok(name, best_distance)) log_err("Did you mean %s?\n", l_opts[best_option].name); } +static int parse_output_format(const char *optarg) +{ + char *p, *orig, *opt; + int ret = 0; + + p = orig = strdup(optarg); + + output_format = 0; + + while ((opt = strsep(&p, ",")) != NULL) { + if (!strcmp(opt, "minimal") || + !strcmp(opt, "terse") || + !strcmp(opt, "csv")) + output_format |= FIO_OUTPUT_TERSE; + else if (!strcmp(opt, "json")) + output_format |= FIO_OUTPUT_JSON; + else if (!strcmp(opt, "json+")) + output_format |= (FIO_OUTPUT_JSON | FIO_OUTPUT_JSON_PLUS); + else if (!strcmp(opt, "normal")) + output_format |= FIO_OUTPUT_NORMAL; + else { + log_err("fio: invalid output format %s\n", opt); + ret = 1; + break; + } + } + + free(orig); + return ret; +} + int parse_cmd_line(int argc, char *argv[], int client_type) { struct thread_data *td = NULL; @@ -1994,17 +2115,15 @@ int parse_cmd_line(int argc, char *argv[], int client_type) do_exit++; break; } - if (!strcmp(optarg, "minimal") || - !strcmp(optarg, "terse") || - !strcmp(optarg, "csv")) - output_format = FIO_OUTPUT_TERSE; - else if (!strcmp(optarg, "json")) - output_format = FIO_OUTPUT_JSON; - else - output_format = FIO_OUTPUT_NORMAL; + if (parse_output_format(optarg)) { + log_err("fio: failed parsing output-format\n"); + exit_val = 1; + do_exit++; + break; + } break; case 'f': - append_terse_output = 1; + output_format |= FIO_OUTPUT_TERSE; break; case 'h': did_arg = 1; @@ -2132,6 +2251,7 @@ int parse_cmd_line(int argc, char *argv[], int client_type) td = NULL; } do_exit++; + exit_val = 1; break; } fio_options_set_ioengine_opts(l_opts, td); @@ -2150,6 +2270,7 @@ int parse_cmd_line(int argc, char *argv[], int client_type) td = NULL; } do_exit++; + exit_val = 1; } if (!ret && !strcmp(opt, "ioengine")) { @@ -2158,6 +2279,7 @@ int parse_cmd_line(int argc, char *argv[], int client_type) put_job(td); td = NULL; do_exit++; + exit_val = 1; break; } fio_options_set_ioengine_opts(l_opts, td); @@ -2226,6 +2348,35 @@ int parse_cmd_line(int argc, char *argv[], int client_type) exit_val = 1; break; } + /* if --client parameter contains a pathname */ + if (0 == access(optarg, R_OK)) { + /* file contains a list of host addrs or names */ + char hostaddr[PATH_MAX] = {0}; + char formatstr[8]; + FILE * hostf = fopen(optarg, "r"); + if (!hostf) { + log_err("fio: could not open client list file %s for read\n", optarg); + do_exit++; + exit_val = 1; + break; + } + sprintf(formatstr, "%%%ds", PATH_MAX - 1); + /* + * read at most PATH_MAX-1 chars from each + * record in this file + */ + while (fscanf(hostf, formatstr, hostaddr) == 1) { + /* expect EVERY host in file to be valid */ + if (fio_client_add(&fio_client_ops, hostaddr, &cur_client)) { + log_err("fio: failed adding client %s from file %s\n", hostaddr, optarg); + do_exit++; + exit_val = 1; + break; + } + } + fclose(hostf); + break; /* no possibility of job file for "this client only" */ + } if (fio_client_add(&fio_client_ops, optarg, &cur_client)) { log_err("fio: failed adding client %s\n", optarg); do_exit++; @@ -2290,6 +2441,11 @@ int parse_cmd_line(int argc, char *argv[], int client_type) free(trigger_remote_cmd); trigger_remote_cmd = strdup(optarg); break; + case 'K': + if (aux_path) + free(aux_path); + aux_path = strdup(optarg); + break; case 'B': if (check_str_time(optarg, &trigger_timeout, 1)) { log_err("fio: failed parsing time %s\n", optarg); @@ -2419,7 +2575,7 @@ int parse_options(int argc, char *argv[]) return 0; } - if (output_format == FIO_OUTPUT_NORMAL) + if (output_format & FIO_OUTPUT_NORMAL) log_info("%s\n", fio_version_string); return 0;