X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=init.c;h=861b1f5fd0f466d35dfd9a12ed606b80f98e0b1d;hp=8448586847c8db10a74143610a1c233d92b72278;hb=0e9c21a29431e43cd73c83c13271ee6e94fe1328;hpb=3660ceae229f08b4086279be7c82e86926f0304b diff --git a/init.c b/init.c index 84485868..861b1f5f 100644 --- a/init.c +++ b/init.c @@ -64,8 +64,6 @@ int write_bw_log = 0; int read_only = 0; int status_interval = 0; -static int write_lat_log; - static int prev_group_jobs; unsigned long fio_debug = 0; @@ -172,6 +170,13 @@ static struct option l_opts[FIO_NR_OPTIONS] = { .has_arg = required_argument, .val = 'x' | FIO_CLIENT_FLAG, }, +#ifdef CONFIG_ZLIB + { + .name = (char *) "inflate-log", + .has_arg = required_argument, + .val = 'X' | FIO_CLIENT_FLAG, + }, +#endif { .name = (char *) "alloc-size", .has_arg = required_argument, @@ -238,15 +243,19 @@ static struct option l_opts[FIO_NR_OPTIONS] = { void free_threads_shm(void) { - struct shmid_ds sbuf; - if (threads) { void *tp = threads; +#ifndef CONFIG_NO_SHM + struct shmid_ds sbuf; threads = NULL; shmdt(tp); shmctl(shm_id, IPC_RMID, &sbuf); shm_id = -1; +#else + threads = NULL; + free(tp); +#endif } } @@ -287,6 +296,7 @@ static int setup_thread_area(void) size += file_hash_size; size += sizeof(unsigned int); +#ifndef CONFIG_NO_SHM shm_id = shmget(0, size, IPC_CREAT | 0600); if (shm_id != -1) break; @@ -294,10 +304,16 @@ static int setup_thread_area(void) perror("shmget"); break; } +#else + threads = malloc(size); + if (threads) + break; +#endif max_jobs >>= 1; } while (max_jobs); +#ifndef CONFIG_NO_SHM if (shm_id == -1) return 1; @@ -306,6 +322,7 @@ static int setup_thread_area(void) perror("shmat"); return 1; } +#endif memset(threads, 0, max_jobs * sizeof(struct thread_data)); hash = (void *) threads + max_jobs * sizeof(struct thread_data); @@ -330,7 +347,7 @@ static void set_cmd_options(struct thread_data *td) * Return a free job structure. */ static struct thread_data *get_new_job(int global, struct thread_data *parent, - int preserve_eo) + int preserve_eo, const char *jobname) { struct thread_data *td; @@ -363,6 +380,10 @@ static struct thread_data *get_new_job(int global, struct thread_data *parent, profile_add_hooks(td); td->thread_number = thread_number; + td->subjob_number = 0; + + if (jobname) + td->o.name = strdup(jobname); if (!parent->o.group_reporting) stat_number++; @@ -386,6 +407,9 @@ static void put_job(struct thread_data *td) if (td->io_ops) free_ioengine(td); + if (td->o.name) + free(td->o.name); + memset(&threads[td->thread_number - 1], 0, sizeof(*td)); thread_number--; } @@ -612,6 +636,15 @@ static int fixup_options(struct thread_data *td) if (o->max_bs[DDIR_WRITE] != o->min_bs[DDIR_WRITE] && !o->verify_interval) o->verify_interval = o->min_bs[DDIR_WRITE]; + + /* + * Verify interval must be smaller or equal to the + * write size. + */ + if (o->verify_interval > o->min_bs[DDIR_WRITE]) + o->verify_interval = o->min_bs[DDIR_WRITE]; + else if (td_read(td) && o->verify_interval > o->min_bs[DDIR_READ]) + o->verify_interval = o->min_bs[DDIR_READ]; } if (o->pre_read) { @@ -803,7 +836,9 @@ static void td_fill_rand_seeds_internal(struct thread_data *td) void td_fill_rand_seeds(struct thread_data *td) { if (td->o.allrand_repeatable) { - for (int i = 0; i < FIO_RAND_NR_OFFS; i++) + unsigned int i; + + for (i = 0; i < FIO_RAND_NR_OFFS; i++) td->rand_seeds[i] = FIO_RANDSEED * td->thread_number + i; } @@ -814,6 +849,9 @@ void td_fill_rand_seeds(struct thread_data *td) td_fill_rand_seeds_internal(td); init_rand_seed(&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF]); + frand_copy(&td->buf_state_prev, &td->buf_state); + + init_rand_seed(&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF]); } /* @@ -1038,6 +1076,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, char fname[PATH_MAX]; int numjobs, file_alloced; struct thread_options *o = &td->o; + char logname[PATH_MAX + 32]; /* * the def_thread is just for options, it's not a real job @@ -1127,15 +1166,72 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if (setup_rate(td)) goto err; - if (o->lat_log_file || write_lat_log) { - setup_log(&td->lat_log, o->log_avg_msec, IO_LOG_TYPE_LAT); - setup_log(&td->slat_log, o->log_avg_msec, IO_LOG_TYPE_SLAT); - setup_log(&td->clat_log, o->log_avg_msec, IO_LOG_TYPE_CLAT); + if (o->lat_log_file) { + struct log_params p = { + .td = td, + .avg_msec = o->log_avg_msec, + .log_type = IO_LOG_TYPE_LAT, + .log_offset = o->log_offset, + .log_gz = o->log_gz, + .log_gz_store = o->log_gz_store, + }; + const char *suf; + + if (p.log_gz_store) + suf = "log.fz"; + else + suf = "log"; + + snprintf(logname, sizeof(logname), "%s_lat.%d.%s", + o->lat_log_file, td->thread_number, suf); + setup_log(&td->lat_log, &p, logname); + snprintf(logname, sizeof(logname), "%s_slat.%d.%s", + o->lat_log_file, td->thread_number, suf); + setup_log(&td->slat_log, &p, logname); + snprintf(logname, sizeof(logname), "%s_clat.%d.%s", + o->lat_log_file, td->thread_number, suf); + setup_log(&td->clat_log, &p, logname); + } + if (o->bw_log_file) { + struct log_params p = { + .td = td, + .avg_msec = o->log_avg_msec, + .log_type = IO_LOG_TYPE_BW, + .log_offset = o->log_offset, + .log_gz = o->log_gz, + .log_gz_store = o->log_gz_store, + }; + const char *suf; + + if (p.log_gz_store) + suf = "log.fz"; + else + suf = "log"; + + snprintf(logname, sizeof(logname), "%s_bw.%d.%s", + o->bw_log_file, td->thread_number, suf); + setup_log(&td->bw_log, &p, logname); + } + if (o->iops_log_file) { + struct log_params p = { + .td = td, + .avg_msec = o->log_avg_msec, + .log_type = IO_LOG_TYPE_IOPS, + .log_offset = o->log_offset, + .log_gz = o->log_gz, + .log_gz_store = o->log_gz_store, + }; + const char *suf; + + if (p.log_gz_store) + suf = "log.fz"; + else + suf = "log"; + + snprintf(logname, sizeof(logname), "%s_iops.%d.%s", + o->iops_log_file, td->thread_number, suf); + setup_log(&td->iops_log, &p, logname); } - if (o->bw_log_file || write_bw_log) - setup_log(&td->bw_log, o->log_avg_msec, IO_LOG_TYPE_BW); - if (o->iops_log_file) - setup_log(&td->iops_log, o->log_avg_msec, IO_LOG_TYPE_IOPS); if (!o->name) o->name = strdup(jobname); @@ -1190,7 +1286,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, */ numjobs = o->numjobs; while (--numjobs) { - struct thread_data *td_new = get_new_job(0, td, 1); + struct thread_data *td_new = get_new_job(0, td, 1, jobname); if (!td_new) goto err; @@ -1198,6 +1294,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, td_new->o.numjobs = 1; td_new->o.stonewall = 0; td_new->o.new_group = 0; + td_new->subjob_number = numjobs; if (file_alloced) { if (td_new->files) { @@ -1248,11 +1345,11 @@ void add_job_opts(const char **o, int client_type) sprintf(jobname, "%s", o[i] + 5); } if (in_global && !td_parent) - td_parent = get_new_job(1, &def_thread, 0); + td_parent = get_new_job(1, &def_thread, 0, jobname); else if (!in_global && !td) { if (!td_parent) td_parent = &def_thread; - td = get_new_job(0, td_parent, 0); + td = get_new_job(0, td_parent, 0, jobname); } if (in_global) fio_options_parse(td_parent, (char **) &o[i], 1, 0); @@ -1300,11 +1397,12 @@ static int is_empty_or_comment(char *line) /* * This is our [ini] type file parser. */ -int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) +int __parse_jobs_ini(struct thread_data *td, + char *file, int is_buf, int stonewall_flag, int type, + int nested, char *name, char ***popts, int *aopts, int *nopts) { - unsigned int global; - struct thread_data *td; - char *string, *name; + unsigned int global = 0; + char *string; FILE *f; char *p; int ret = 0, stonewall; @@ -1314,6 +1412,9 @@ int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) char **opts; int i, alloc_opts, num_opts; + dprint(FD_PARSE, "Parsing ini file %s\n", file); + assert(td || !nested); + if (is_buf) f = NULL; else { @@ -1333,12 +1434,23 @@ int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) /* * it's really 256 + small bit, 280 should suffice */ - name = malloc(280); - memset(name, 0, 280); + if (!nested) { + name = malloc(280); + memset(name, 0, 280); + } - alloc_opts = 8; - opts = malloc(sizeof(char *) * alloc_opts); - num_opts = 0; + opts = NULL; + if (nested && popts) { + opts = *popts; + alloc_opts = *aopts; + num_opts = *nopts; + } + + if (!opts) { + alloc_opts = 8; + opts = malloc(sizeof(char *) * alloc_opts); + num_opts = 0; + } stonewall = stonewall_flag; do { @@ -1359,58 +1471,73 @@ int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) strip_blank_front(&p); strip_blank_end(p); + dprint(FD_PARSE, "%s\n", p); if (is_empty_or_comment(p)) continue; - if (sscanf(p, "[%255[^\n]]", name) != 1) { - if (inside_skip) + + if (!nested) { + if (sscanf(p, "[%255[^\n]]", name) != 1) { + if (inside_skip) + continue; + + log_err("fio: option <%s> outside of " + "[] job section\n", p); + ret = 1; + break; + } + + name[strlen(name) - 1] = '\0'; + + if (skip_this_section(name)) { + inside_skip = 1; continue; - log_err("fio: option <%s> outside of [] job section\n", - p); - break; - } + } else + inside_skip = 0; - name[strlen(name) - 1] = '\0'; + dprint(FD_PARSE, "Parsing section [%s]\n", name); - if (skip_this_section(name)) { - inside_skip = 1; - continue; - } else - inside_skip = 0; + global = !strncmp(name, "global", 6); - global = !strncmp(name, "global", 6); + if (dump_cmdline) { + if (first_sect) + log_info("fio "); + if (!global) + log_info("--name=%s ", name); + first_sect = 0; + } - if (dump_cmdline) { - if (first_sect) - log_info("fio "); - if (!global) - log_info("--name=%s ", name); - first_sect = 0; - } + td = get_new_job(global, &def_thread, 0, name); + if (!td) { + ret = 1; + break; + } - td = get_new_job(global, &def_thread, 0); - if (!td) { - ret = 1; - break; - } + /* + * Separate multiple job files by a stonewall + */ + if (!global && stonewall) { + td->o.stonewall = stonewall; + stonewall = 0; + } - /* - * Separate multiple job files by a stonewall - */ - if (!global && stonewall) { - td->o.stonewall = stonewall; - stonewall = 0; + num_opts = 0; + memset(opts, 0, alloc_opts * sizeof(char *)); } - - num_opts = 0; - memset(opts, 0, alloc_opts * sizeof(char *)); + else + skip_fgets = 1; while (1) { - if (is_buf) - p = strsep(&file, "\n"); + if (!skip_fgets) { + if (is_buf) + p = strsep(&file, "\n"); + else + p = fgets(string, 4096, f); + if (!p) + break; + dprint(FD_PARSE, "%s", p); + } else - p = fgets(string, 4096, f); - if (!p) - break; + skip_fgets = 0; if (is_empty_or_comment(p)) continue; @@ -1422,12 +1549,30 @@ int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) * fgets() a new line at the top. */ if (p[0] == '[') { + if (nested) { + log_err("No new sections in included files\n"); + return 1; + } + skip_fgets = 1; break; } strip_blank_end(p); + if (!strncmp(p, "include", strlen("include"))) { + char *filename = p + strlen("include") + 1; + + if ((ret = __parse_jobs_ini(td, filename, + is_buf, stonewall_flag, type, 1, + name, &opts, &alloc_opts, &num_opts))) { + log_err("Error %d while parsing include file %s\n", + ret, filename); + break; + } + continue; + } + if (num_opts == alloc_opts) { alloc_opts <<= 1; opts = realloc(opts, @@ -1438,6 +1583,13 @@ int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) num_opts++; } + if (nested) { + *popts = opts; + *aopts = alloc_opts; + *nopts = num_opts; + goto out; + } + ret = fio_options_parse(td, opts, num_opts, dump_cmdline); if (!ret) ret = add_job(td, name, 0, 0, type); @@ -1460,14 +1612,22 @@ int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) i++; } - free(string); - free(name); free(opts); +out: + free(string); + if (!nested) + free(name); if (!is_buf && f != stdin) fclose(f); return ret; } +int parse_jobs_ini(char *file, int is_buf, int stonewall_flag, int type) +{ + return __parse_jobs_ini(NULL, file, is_buf, stonewall_flag, type, + 0, NULL, NULL, NULL, NULL); +} + static int fill_def_thread(void) { memset(&def_thread, 0, sizeof(def_thread)); @@ -1488,11 +1648,10 @@ static void usage(const char *name) printf("%s [options] [job options] \n", name); printf(" --debug=options\tEnable debug logging. May be one/more of:\n" "\t\t\tprocess,file,io,mem,blktrace,verify,random,parse,\n" - "\t\t\tdiskutil,job,mutex,profile,time,net,rate\n"); + "\t\t\tdiskutil,job,mutex,profile,time,net,rate,compress\n"); printf(" --parse-only\t\tParse options only, don't start any IO\n"); printf(" --output\t\tWrite output to file\n"); printf(" --runtime\t\tRuntime in seconds\n"); - printf(" --latency-log\t\tGenerate per-job latency logs\n"); printf(" --bandwidth-log\tGenerate per-job bandwidth logs\n"); printf(" --minimal\t\tMinimal (terse) output\n"); printf(" --output-format=x\tOutput format (terse,json,normal)\n"); @@ -1527,6 +1686,9 @@ static void usage(const char *name) printf(" --idle-prof=option\tReport cpu idleness on a system or percpu basis\n" "\t\t\t(option=system,percpu) or run unit work\n" "\t\t\tcalibration only (option=calibrate)\n"); +#ifdef CONFIG_ZLIB + printf(" --inflate-log=log\tInflate and output compressed log\n"); +#endif printf("\nFio was written by Jens Axboe "); printf("\n Jens Axboe "); printf("\n Jens Axboe \n"); @@ -1594,6 +1756,10 @@ struct debug_level debug_levels[] = { .help = "Rate logging", .shift = FD_RATE, }, + { .name = "compress", + .help = "Log compression logging", + .shift = FD_COMPRESS, + }, { .name = NULL, }, }; @@ -1733,13 +1899,15 @@ int parse_cmd_line(int argc, char *argv[], int client_type) } break; case 'l': - write_lat_log = 1; + log_err("fio: --latency-log is deprecated. Use per-job latency log options.\n"); + do_exit++; + exit_val = 1; break; case 'b': write_bw_log = 1; break; case 'o': - if (f_out) + if (f_out && f_out != stdout) fclose(f_out); f_out = fopen(optarg, "w+"); @@ -1856,6 +2024,13 @@ int parse_cmd_line(int argc, char *argv[], int client_type) nr_job_sections++; break; } +#ifdef CONFIG_ZLIB + case 'X': + exit_val = iolog_file_inflate(optarg); + did_arg++; + do_exit++; + break; +#endif case 'p': did_arg = 1; if (exec_profile) @@ -1883,9 +2058,15 @@ int parse_cmd_line(int argc, char *argv[], int client_type) if (is_section && skip_this_section(val)) continue; - td = get_new_job(global, &def_thread, 1); - if (!td || ioengine_load(td)) - goto out_free; + td = get_new_job(global, &def_thread, 1, NULL); + if (!td || ioengine_load(td)) { + if (td) { + put_job(td); + td = NULL; + } + do_exit++; + break; + } fio_options_set_ioengine_opts(l_opts, td); } @@ -1906,8 +2087,12 @@ int parse_cmd_line(int argc, char *argv[], int client_type) if (!ret && !strcmp(opt, "ioengine")) { free_ioengine(td); - if (ioengine_load(td)) - goto out_free; + if (ioengine_load(td)) { + put_job(td); + td = NULL; + do_exit++; + break; + } fio_options_set_ioengine_opts(l_opts, td); } break; @@ -1920,7 +2105,6 @@ int parse_cmd_line(int argc, char *argv[], int client_type) break; ret = fio_cmd_ioengine_option_parse(td, opt, val); - did_arg = 1; break; } case 'w': @@ -1936,6 +2120,7 @@ int parse_cmd_line(int argc, char *argv[], int client_type) break; case 'S': did_arg = 1; +#ifndef CONFIG_NO_SHM if (nr_clients) { log_err("fio: can't be both client and server\n"); do_exit++; @@ -1946,6 +2131,11 @@ int parse_cmd_line(int argc, char *argv[], int client_type) fio_server_set_arg(optarg); is_backend = 1; backend = 1; +#else + log_err("fio: client/server requires SHM support\n"); + do_exit++; + exit_val = 1; +#endif break; case 'D': if (pid_file)