X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=init.c;h=fabc887b06c75c737e8eb49632f8bebe11e28c47;hp=1b8f08275503fac07b6d45bfea3f116735b1c8b0;hb=8983ce71bc3aa4076cb0c9f2b5c3b73ab7c7de93;hpb=ba8fb6f6f12800ee3a0e637dfdac4e9aaa6b2123 diff --git a/init.c b/init.c index 1b8f0827..fabc887b 100644 --- a/init.c +++ b/init.c @@ -31,6 +31,7 @@ #include "oslib/strcasestr.h" #include "crc/test.h" +#include "lib/pow2.h" const char fio_version_string[] = FIO_VERSION; @@ -39,7 +40,6 @@ const char fio_version_string[] = FIO_VERSION; static char **ini_file; static int max_jobs = FIO_MAX_JOBS; static int dump_cmdline; -static long long def_timeout; static int parse_only; static struct thread_data def_thread; @@ -48,7 +48,6 @@ static char **job_sections; static int nr_job_sections; int exitall_on_terminate = 0; -int exitall_on_terminate_error = 0; int output_format = FIO_OUTPUT_NORMAL; int eta_print = FIO_ETA_AUTO; int eta_new_line = 0; @@ -81,8 +80,6 @@ unsigned int *fio_debug_jobp = NULL; static char cmd_optstr[256]; static int did_arg; -bool steadystate = false; - #define FIO_CLIENT_FLAG (1 << 16) /* @@ -95,11 +92,6 @@ static struct option l_opts[FIO_NR_OPTIONS] = { .has_arg = required_argument, .val = 'o' | FIO_CLIENT_FLAG, }, - { - .name = (char *) "timeout", - .has_arg = required_argument, - .val = 't' | FIO_CLIENT_FLAG, - }, { .name = (char *) "latency-log", .has_arg = required_argument, @@ -107,7 +99,7 @@ static struct option l_opts[FIO_NR_OPTIONS] = { }, { .name = (char *) "bandwidth-log", - .has_arg = required_argument, + .has_arg = no_argument, .val = 'b' | FIO_CLIENT_FLAG, }, { @@ -302,7 +294,6 @@ void free_threads_shm(void) static void free_shm(void) { if (threads) { - file_hash_exit(); flow_exit(); fio_debug_jobp = NULL; free_threads_shm(); @@ -313,8 +304,9 @@ static void free_shm(void) free(trigger_remote_cmd); trigger_file = trigger_cmd = trigger_remote_cmd = NULL; - options_free(fio_options, &def_thread); + options_free(fio_options, &def_thread.o); fio_filelock_exit(); + file_hash_exit(); scleanup(); } @@ -326,8 +318,6 @@ static void free_shm(void) */ static int setup_thread_area(void) { - void *hash; - if (threads) return 0; @@ -338,7 +328,6 @@ static int setup_thread_area(void) do { size_t size = max_jobs * sizeof(struct thread_data); - size += file_hash_size; size += sizeof(unsigned int); #ifndef CONFIG_NO_SHM @@ -367,27 +356,20 @@ static int setup_thread_area(void) perror("shmat"); return 1; } +#ifdef FIO_HAVE_SHM_ATTACH_REMOVED + shmctl(shm_id, IPC_RMID, NULL); +#endif #endif memset(threads, 0, max_jobs * sizeof(struct thread_data)); - hash = (void *) threads + max_jobs * sizeof(struct thread_data); - fio_debug_jobp = (void *) hash + file_hash_size; + fio_debug_jobp = (void *) threads + max_jobs * sizeof(struct thread_data); *fio_debug_jobp = -1; - file_hash_init(hash); flow_init(); return 0; } -static void set_cmd_options(struct thread_data *td) -{ - struct thread_options *o = &td->o; - - if (!o->timeout) - o->timeout = def_timeout; -} - static void dump_print_option(struct print_option *p) { const char *delim; @@ -453,15 +435,13 @@ static void copy_opt_list(struct thread_data *dst, struct thread_data *src) /* * Return a free job structure. */ -static struct thread_data *get_new_job(int global, struct thread_data *parent, - int preserve_eo, const char *jobname) +static struct thread_data *get_new_job(bool global, struct thread_data *parent, + bool preserve_eo, const char *jobname) { struct thread_data *td; - if (global) { - set_cmd_options(&def_thread); + if (global) return &def_thread; - } if (setup_thread_area()) { log_err("error: failed to setup shm segment\n"); return NULL; @@ -499,7 +479,6 @@ static struct thread_data *get_new_job(int global, struct thread_data *parent, if (!parent->o.group_reporting || parent == &def_thread) stat_number++; - set_cmd_options(td); return td; } @@ -588,6 +567,17 @@ static unsigned long long get_rand_start_delay(struct thread_data *td) return delayrange; } +/* + * <3 Johannes + */ +static unsigned int gcd(unsigned int m, unsigned int n) +{ + if (!n) + return m; + + return gcd(n, m % n); +} + /* * Lazy way of fixing up options that depend on each other. We could also * define option callback handlers, but this is easier. @@ -763,6 +753,15 @@ static int fixup_options(struct thread_data *td) o->verify_interval = o->min_bs[DDIR_WRITE]; else if (td_read(td) && o->verify_interval > o->min_bs[DDIR_READ]) o->verify_interval = o->min_bs[DDIR_READ]; + + /* + * Verify interval must be a factor or both min and max + * write size + */ + if (o->verify_interval % o->min_bs[DDIR_WRITE] || + o->verify_interval % o->max_bs[DDIR_WRITE]) + o->verify_interval = gcd(o->min_bs[DDIR_WRITE], + o->max_bs[DDIR_WRITE]); } if (o->pre_read) { @@ -831,7 +830,8 @@ static int fixup_options(struct thread_data *td) * If size is set but less than the min block size, complain */ if (o->size && o->size < td_min_bs(td)) { - log_err("fio: size too small, must be larger than the IO size: %llu\n", (unsigned long long) o->size); + log_err("fio: size too small, must not be less than minimum block size: %llu < %u\n", + (unsigned long long) o->size, td_min_bs(td)); ret = 1; } @@ -872,27 +872,6 @@ static int fixup_options(struct thread_data *td) return ret; } -/* - * This function leaks the buffer - */ -char *fio_uint_to_kmg(unsigned int val) -{ - char *buf = malloc(32); - char post[] = { 0, 'K', 'M', 'G', 'P', 'E', 0 }; - char *p = post; - - do { - if (val & 1023) - break; - - val >>= 10; - p++; - } while (*p); - - snprintf(buf, 32, "%u%c", val, *p); - return buf; -} - /* External engines are specified by "external:name.o") */ static const char *get_engine_name(const char *str) { @@ -907,26 +886,6 @@ static const char *get_engine_name(const char *str) return p; } -static int exists_and_not_regfile(const char *filename) -{ - struct stat sb; - - if (lstat(filename, &sb) == -1) - return 0; - -#ifndef WIN32 /* NOT Windows */ - if (S_ISREG(sb.st_mode)) - return 0; -#else - /* \\.\ is the device namespace in Windows, where every file - * is a device node */ - if (S_ISREG(sb.st_mode) && strncmp(filename, "\\\\.\\", 4) != 0) - return 0; -#endif - - return 1; -} - static void init_rand_file_service(struct thread_data *td) { unsigned long nranges = td->o.nr_files << FIO_FSERVICE_SHIFT; @@ -1064,7 +1023,7 @@ int ioengine_load(struct thread_data *td) */ if (origeo) { memcpy(td->eo, origeo, td->io_ops->option_struct_size); - options_mem_dupe(td->eo, td->io_ops->options); + options_mem_dupe(td->io_ops->options, td->eo); } else { memset(td->eo, 0, td->io_ops->option_struct_size); fill_default_options(td->eo, td->io_ops->options); @@ -1251,7 +1210,7 @@ static char *make_filename(char *buf, size_t buf_size,struct thread_options *o, return buf; } -int parse_dryrun(void) +bool parse_dryrun(void) { return dump_cmdline || parse_only; } @@ -1418,7 +1377,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if (setup_rate(td)) goto err; - if (o->lat_log_file) { + if (o->write_lat_log) { struct log_params p = { .td = td, .avg_msec = o->log_avg_msec, @@ -1429,6 +1388,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, .log_gz = o->log_gz, .log_gz_store = o->log_gz_store, }; + const char *pre = o->lat_log_file ? o->lat_log_file : o->name; const char *suf; if (p.log_gz_store) @@ -1436,20 +1396,20 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, else suf = "log"; - gen_log_name(logname, sizeof(logname), "lat", o->lat_log_file, + gen_log_name(logname, sizeof(logname), "lat", pre, td->thread_number, suf, o->per_job_logs); setup_log(&td->lat_log, &p, logname); - gen_log_name(logname, sizeof(logname), "slat", o->lat_log_file, + gen_log_name(logname, sizeof(logname), "slat", pre, td->thread_number, suf, o->per_job_logs); setup_log(&td->slat_log, &p, logname); - gen_log_name(logname, sizeof(logname), "clat", o->lat_log_file, + gen_log_name(logname, sizeof(logname), "clat", pre, td->thread_number, suf, o->per_job_logs); setup_log(&td->clat_log, &p, logname); } - if (o->hist_log_file) { + if (o->write_hist_log) { struct log_params p = { .td = td, .avg_msec = o->log_avg_msec, @@ -1460,19 +1420,27 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, .log_gz = o->log_gz, .log_gz_store = o->log_gz_store, }; + const char *pre = o->hist_log_file ? o->hist_log_file : o->name; const char *suf; +#ifndef CONFIG_ZLIB + if (td->client_type) { + log_err("fio: --write_hist_log requires zlib in client/server mode\n"); + goto err; + } +#endif + if (p.log_gz_store) suf = "log.fz"; else suf = "log"; - gen_log_name(logname, sizeof(logname), "clat_hist", o->hist_log_file, + gen_log_name(logname, sizeof(logname), "clat_hist", pre, td->thread_number, suf, o->per_job_logs); setup_log(&td->clat_hist_log, &p, logname); } - if (o->bw_log_file) { + if (o->write_bw_log) { struct log_params p = { .td = td, .avg_msec = o->log_avg_msec, @@ -1483,6 +1451,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, .log_gz = o->log_gz, .log_gz_store = o->log_gz_store, }; + const char *pre = o->bw_log_file ? o->bw_log_file : o->name; const char *suf; if (fio_option_is_set(o, bw_avg_time)) @@ -1498,11 +1467,11 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, else suf = "log"; - gen_log_name(logname, sizeof(logname), "bw", o->bw_log_file, + gen_log_name(logname, sizeof(logname), "bw", pre, td->thread_number, suf, o->per_job_logs); setup_log(&td->bw_log, &p, logname); } - if (o->iops_log_file) { + if (o->write_iops_log) { struct log_params p = { .td = td, .avg_msec = o->log_avg_msec, @@ -1513,6 +1482,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, .log_gz = o->log_gz, .log_gz_store = o->log_gz_store, }; + const char *pre = o->iops_log_file ? o->iops_log_file : o->name; const char *suf; if (fio_option_is_set(o, iops_avg_time)) @@ -1528,7 +1498,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, else suf = "log"; - gen_log_name(logname, sizeof(logname), "iops", o->iops_log_file, + gen_log_name(logname, sizeof(logname), "iops", pre, td->thread_number, suf, o->per_job_logs); setup_log(&td->iops_log, &p, logname); } @@ -1544,15 +1514,16 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if (!td_ioengine_flagged(td, FIO_NOIO)) { char *c1, *c2, *c3, *c4; char *c5 = NULL, *c6 = NULL; + int i2p = is_power_of_2(o->kb_base); - c1 = fio_uint_to_kmg(o->min_bs[DDIR_READ]); - c2 = fio_uint_to_kmg(o->max_bs[DDIR_READ]); - c3 = fio_uint_to_kmg(o->min_bs[DDIR_WRITE]); - c4 = fio_uint_to_kmg(o->max_bs[DDIR_WRITE]); + c1 = num2str(o->min_bs[DDIR_READ], 4, 1, i2p, N2S_BYTE); + c2 = num2str(o->max_bs[DDIR_READ], 4, 1, i2p, N2S_BYTE); + c3 = num2str(o->min_bs[DDIR_WRITE], 4, 1, i2p, N2S_BYTE); + c4 = num2str(o->max_bs[DDIR_WRITE], 4, 1, i2p, N2S_BYTE); if (!o->bs_is_seq_rand) { - c5 = fio_uint_to_kmg(o->min_bs[DDIR_TRIM]); - c6 = fio_uint_to_kmg(o->max_bs[DDIR_TRIM]); + c5 = num2str(o->min_bs[DDIR_TRIM], 4, 1, i2p, N2S_BYTE); + c6 = num2str(o->max_bs[DDIR_TRIM], 4, 1, i2p, N2S_BYTE); } log_info("%s: (g=%d): rw=%s, ", td->o.name, @@ -1560,10 +1531,10 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, ddir_str(o->td_ddir)); if (o->bs_is_seq_rand) - log_info("bs(seq/rand)=%s-%s/%s-%s, ", + log_info("bs=%s-%s,%s-%s, bs_is_seq_rand, ", c1, c2, c3, c4); else - log_info("bs=%s-%s/%s-%s/%s-%s, ", + log_info("bs=%s-%s,%s-%s,%s-%s, ", c1, c2, c3, c4, c5, c6); log_info("ioengine=%s, iodepth=%u\n", @@ -1580,49 +1551,8 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, log_info("...\n"); } - if (o->ss_dur) { - steadystate = true; - o->ss_dur /= 1000000L; - - /* put all steady state info in one place */ - td->ss.dur = o->ss_dur; - td->ss.limit = o->ss_limit.u.f; - td->ss.ramp_time = o->ss_ramp_time; - td->ss.pct = o->ss_pct; - - if (o->ss == FIO_STEADYSTATE_IOPS_SLOPE || o->ss == FIO_STEADYSTATE_BW_SLOPE) { - td->ss.check_slope = true; - td->ss.evaluate = &steadystate_slope; - } else { - td->ss.check_slope = false; - td->ss.evaluate = &steadystate_deviation; - } - - if (o->ss == FIO_STEADYSTATE_IOPS || o->ss == FIO_STEADYSTATE_IOPS_SLOPE) - td->ss.check_iops = true; - else - td->ss.check_iops = false; - - td->ss.bw_data = NULL; - td->ss.iops_data = NULL; - td->ss.ramp_time_over = (td->ss.ramp_time == 0); - td->ss.attained = 0; - td->ss.last_in_group = 0; - td->ss.head = 0; - td->ss.tail = 0; - td->ss.sum_x = o->ss_dur * (o->ss_dur - 1) / 2; - td->ss.sum_x_sq = (o->ss_dur - 1) * (o->ss_dur) * (2*o->ss_dur - 1) / 6; - td->ss.prev_bytes = 0; - td->ss.prev_iops = 0; - td->ss.sum_y = 0; - td->ss.oldest_y = 0; - td->ss.criterion = 0.0; - td->ss.slope = 0.0; - td->ss.deviation = 0.0; - td->ts.ss = &td->ss; - } - else - td->ts.ss = NULL; + if (td_steadystate_init(td)) + goto err; /* * recurse add identical jobs, clear numjobs and stonewall options @@ -1630,7 +1560,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, */ numjobs = o->numjobs; while (--numjobs) { - struct thread_data *td_new = get_new_job(0, td, 1, jobname); + struct thread_data *td_new = get_new_job(false, td, true, jobname); if (!td_new) goto err; @@ -1691,11 +1621,11 @@ void add_job_opts(const char **o, int client_type) sprintf(jobname, "%s", o[i] + 5); } if (in_global && !td_parent) - td_parent = get_new_job(1, &def_thread, 0, jobname); + td_parent = get_new_job(true, &def_thread, false, jobname); else if (!in_global && !td) { if (!td_parent) td_parent = &def_thread; - td = get_new_job(0, td_parent, 0, jobname); + td = get_new_job(false, td_parent, false, jobname); } if (in_global) fio_options_parse(td_parent, (char **) &o[i], 1); @@ -1743,11 +1673,11 @@ static int is_empty_or_comment(char *line) /* * This is our [ini] type file parser. */ -int __parse_jobs_ini(struct thread_data *td, +static int __parse_jobs_ini(struct thread_data *td, char *file, int is_buf, int stonewall_flag, int type, int nested, char *name, char ***popts, int *aopts, int *nopts) { - unsigned int global = 0; + bool global = false; char *string; FILE *f; char *p; @@ -1856,7 +1786,7 @@ int __parse_jobs_ini(struct thread_data *td, first_sect = 0; } - td = get_new_job(global, &def_thread, 0, name); + td = get_new_job(global, &def_thread, false, name); if (!td) { ret = 1; break; @@ -2060,6 +1990,11 @@ static void show_debug_categories(void) #endif } +/* + * Following options aren't printed by usage(). + * --append-terse - Equivalent to --output-format=terse, see f6a7df53. + * --latency-log - Deprecated option. + */ static void usage(const char *name) { printf("%s\n", fio_version_string); @@ -2068,15 +2003,15 @@ static void usage(const char *name) show_debug_categories(); printf(" --parse-only\t\tParse options only, don't start any IO\n"); printf(" --output\t\tWrite output to file\n"); - printf(" --runtime\t\tRuntime in seconds\n"); - printf(" --bandwidth-log\tGenerate per-job bandwidth logs\n"); + printf(" --bandwidth-log\tGenerate aggregate bandwidth logs\n"); printf(" --minimal\t\tMinimal (terse) output\n"); - printf(" --output-format=x\tOutput format (terse,json,json+,normal)\n"); - printf(" --terse-version=x\tSet terse version output format to 'x'\n"); + printf(" --output-format=type\tOutput format (terse,json,json+,normal)\n"); + printf(" --terse-version=type\tSet terse version output format" + " (default 3, or 2 or 4)\n"); printf(" --version\t\tPrint version info and exit\n"); printf(" --help\t\tPrint this page\n"); printf(" --cpuclock-test\tPerform test/validation of CPU clock\n"); - printf(" --crctest\t\tTest speed of checksum functions\n"); + printf(" --crctest=type\tTest speed of checksum functions\n"); printf(" --cmdhelp=cmd\t\tPrint command help, \"all\" for all of" " them\n"); printf(" --enghelp=engine\tPrint ioengine help, or list" @@ -2092,14 +2027,15 @@ static void usage(const char *name) printf(" 't' period passed\n"); printf(" --readonly\t\tTurn on safety read-only checks, preventing" " writes\n"); - printf(" --section=name\tOnly run specified section in job file\n"); + printf(" --section=name\tOnly run specified section in job file," + " multiple sections can be specified\n"); printf(" --alloc-size=kb\tSet smalloc pool to this size in kb" - " (def 1024)\n"); + " (def 16384)\n"); printf(" --warnings-fatal\tFio parser warnings are fatal\n"); printf(" --max-jobs=nr\t\tMaximum number of threads/processes to support\n"); printf(" --server=args\t\tStart a backend fio server\n"); printf(" --daemonize=pidfile\tBackground fio server, write pid to file\n"); - printf(" --client=hostname\tTalk to remote backend fio server at hostname\n"); + printf(" --client=hostname\tTalk to remote backend(s) fio server at hostname\n"); printf(" --remote-config=file\tTell fio server to load this local job file\n"); printf(" --idle-prof=option\tReport cpu idleness on a system or percpu basis\n" "\t\t\t(option=system,percpu) or run unit work\n" @@ -2187,6 +2123,10 @@ struct debug_level debug_levels[] = { .help = "Steady state detection logging", .shift = FD_STEADYSTATE, }, + { .name = "helperthread", + .help = "Helper thread logging", + .shift = FD_HELPERTHREAD, + }, { .name = NULL, }, }; @@ -2375,13 +2315,8 @@ int parse_cmd_line(int argc, char *argv[], int client_type) switch (c) { case 'a': smalloc_pool_size = atoi(optarg); - break; - case 't': - if (check_str_time(optarg, &def_timeout, 1)) { - log_err("fio: failed parsing time %s\n", optarg); - do_exit++; - exit_val = 1; - } + smalloc_pool_size <<= 10; + sinit(); break; case 'l': log_err("fio: --latency-log is deprecated. Use per-job latency log options.\n"); @@ -2391,17 +2326,22 @@ int parse_cmd_line(int argc, char *argv[], int client_type) case 'b': write_bw_log = 1; break; - case 'o': + case 'o': { + FILE *tmp; + if (f_out && f_out != stdout) fclose(f_out); - f_out = fopen(optarg, "w+"); - if (!f_out) { - perror("fopen output"); - exit(1); + tmp = fopen(optarg, "w+"); + if (!tmp) { + log_err("fio: output file open error: %s\n", strerror(errno)); + exit_val = 1; + do_exit++; + break; } - f_err = f_out; + f_err = f_out = tmp; break; + } case 'm': output_format = FIO_OUTPUT_TERSE; break; @@ -2535,7 +2475,7 @@ int parse_cmd_line(int argc, char *argv[], int client_type) if (is_section && skip_this_section(val)) continue; - td = get_new_job(global, &def_thread, 1, NULL); + td = get_new_job(global, &def_thread, true, NULL); if (!td || ioengine_load(td)) { if (td) { put_job(td); @@ -2773,7 +2713,7 @@ int parse_cmd_line(int argc, char *argv[], int client_type) if (!ret) { ret = add_job(td, td->o.name ?: "fio", 0, 0, client_type); if (ret) - did_arg = 1; + exit(1); } } @@ -2785,9 +2725,6 @@ int parse_cmd_line(int argc, char *argv[], int client_type) } out_free: - if (pid_file) - free(pid_file); - return ini_idx; } @@ -2856,7 +2793,7 @@ int parse_options(int argc, char *argv[]) if (did_arg) return 0; - log_err("No jobs(s) defined\n\n"); + log_err("No job(s) defined\n\n"); if (!did_arg) { usage(argv[0]);