From: Jens Axboe Date: Fri, 24 May 2013 18:11:02 +0000 (+0200) Subject: Merge branch 'master' of ssh://git.kernel.dk/data/git/fio X-Git-Tag: fio-2.1.1~9 X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=7b9013750b3923ec923924de118a9ccc5964b9ea;hp=774a99b533369dc8d35c6cd09223255760cde9ba Merge branch 'master' of ssh://git.kernel.dk/data/git/fio --- diff --git a/fio_generate_plots b/fio_generate_plots index 5e2febda..f65a326c 100755 --- a/fio_generate_plots +++ b/fio_generate_plots @@ -1,115 +1,126 @@ -#! /bin/sh - -# Use gnuplot to generate plots from fio run with -l and/or -w - -if [ "$1"x = "x" ]; then - echo "Usage: fio_generate_plots title [xres yres]" +#!/bin/sh +# +# This script is an almost total rewrite by Louwrentius +# of the original fio_generate_plots script provided as part of the FIO storage +# benchmark utiliy. I only retained how GNUplot is used to generate graphs, as +# that is something I know nothing about. +# +# The script uses the files generated by FIO to create nice graphs in the +# SVG format. This output format is supported by most modern browsers and +# allows resolution independant graphs to be generated. +# +# This script supports GNUPLOT 4.4 and higher. +# +# Version 1.0 @ 20121231 +# +# +# + +if [ -z "$1" ]; then + echo "Usage: fio_generate_plots subtitle [xres yres]" exit 1 fi GNUPLOT=$(which gnuplot) -if [ ! -x "$GNUPLOT" ]; then +if [ ! -x $GNUPLOT ] +then echo You need gnuplot installed to generate graphs exit 1 fi -TITLE=$1 +TITLE="$1" # set resolution -if [ "$2"x != "x" -a "$3"x != "x" ]; then +if [ ! -z "$2" ] && [ ! -z "$3" ] +then XRES="$2" YRES="$3" else - XRES=1024 + XRES=1280 YRES=768 fi -PLOT_LINE="" -for i in *bw.log; do - if [ ! -r $i ]; then - continue - fi - PT=$(echo $i | sed s/_bw.log//g) - if [ "$PLOT_LINE"x != "x" ]; then - PLOT_LINE=$PLOT_LINE", " - fi - - PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines" -done - -if [ "$PLOT_LINE"x != "x" ]; then - echo Making bw logs - echo "set title 'Bandwidth - $TITLE'; set xlabel 'time (msec)'; set ylabel 'KB/sec'; set terminal png size $XRES,$YRES; set output '$TITLE-bw.png'; plot " $PLOT_LINE | $GNUPLOT - +if [ -z "$SAMPLE_DURATION" ] +then + SAMPLE_DURATION="*" fi -PLOT_LINE="" -for i in *iops.log; do - if [ ! -r $i ]; then - continue - fi - PT=$(echo $i | sed s/_iops.log//g) - if [ "$PLOT_LINE"x != "x" ]; then - PLOT_LINE=$PLOT_LINE", " - fi - - PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines" -done - -if [ "$PLOT_LINE"x != "x" ]; then - echo Making bw logs - echo "set title 'IOPS - $TITLE'; set xlabel 'time (msec)'; set ylabel 'IOPS'; set terminal png size $XRES,$YRES; set output '$TITLE-IOPS.png'; plot " $PLOT_LINE | $GNUPLOT - -fi +DEFAULT_GRID_LINE_TYPE=3 +DEFAULT_LINE_WIDTH=2 +DEFAULT_LINE_COLORS=" +set object 1 rectangle from screen 0,0 to screen 1,1 fillcolor rgb\"#ffffff\" behind +set style line 1 lc rgb \"#E41A1C\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 2 lc rgb \"#377EB8\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 3 lc rgb \"#4DAF4A\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 4 lc rgb \"#984EA3\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 5 lc rgb \"#FF7F00\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 6 lc rgb \"#DADA33\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 7 lc rgb \"#A65628\" lw $DEFAULT_LINE_WIDTH lt 1; +set style line 20 lc rgb \"#000000\" lt $DEFAULT_GRID_LINE_TYPE lw $DEFAULT_LINE_WIDTH; +" + +DEFAULT_TERMINAL="set terminal svg enhanced dashed size $XRES,$YRES dynamic" +DEFAULT_TITLE_FONT="\"Helvetica,28\"" +DEFAULT_AXIS_FONT="\"Helvetica,14\"" +DEFAULT_AXIS_LABEL_FONT="\"Helvetica,16\"" +DEFAULT_XLABEL="set xlabel \"Time (sec)\" font $DEFAULT_AXIS_LABEL_FONT" +DEFAULT_XTIC="set xtics font $DEFAULT_AXIS_FONT" +DEFAULT_YTIC="set ytics font $DEFAULT_AXIS_FONT" +DEFAULT_MXTIC="set mxtics 0" +DEFAULT_MYTIC="set mytics 2" +DEFAULT_XRANGE="set xrange [0:$SAMPLE_DURATION]" +DEFAULT_YRANGE="set yrange [0:*]" +DEFAULT_GRID="set grid ls 20" +DEFAULT_KEY="set key outside bottom center ; set key box enhanced spacing 2.0 samplen 3 horizontal width 4 height 1.2 " +DEFAULT_SOURCE="set label 30 \"Data source: http://example.com\" font $DEFAULT_AXIS_FONT tc rgb \"#00000f\" at screen 0.976,0.175 right" +DEFAULT_OPTS="$DEFAULT_LINE_COLORS ; $DEFAULT_GRID_LINE ; $DEFAULT_GRID ; $DEFAULT_GRID_MINOR ; $DEFAULT_XLABEL ; $DEFAULT_XRANGE ; $DEFAULT_YRANGE ; $DEFAULT_XTIC ; $DEFAULT_YTIC ; $DEFAULT_MXTIC ; $DEFAULT_MYTIC ; $DEFAULT_KEY ; $DEFAULT_TERMINAL ; $DEFAULT_SOURCE" + +plot () { + + if [ -z "$TITLE" ] + then + PLOT_TITLE=" set title \"$1\" font $DEFAULT_TITLE_FONT" + else + PLOT_TITLE=" set title \"$TITLE\\\n\\\n{/*0.6 "$1"}\" font $DEFAULT_TITLE_FONT" + fi + FILETYPE="$2" + YAXIS="set ylabel \"$3\" font $DEFAULT_AXIS_LABEL_FONT" + SCALE=$4 + + echo "Title: $PLOT_TITLE" + echo "File type: $FILETYPE" + echo "yaxis: $YAXIS" + + i=0 + + for x in *_"$FILETYPE".log + do + i=$((i+1)) + PT=$(echo $x | sed s/_"$FILETYPE".log//g) + if [ ! -z "$PLOT_LINE" ] + then + PLOT_LINE=$PLOT_LINE", " + fi + + DEPTH=$(echo $PT | cut -d "-" -f 4) + PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i" + + done + + OUTPUT="set output \"$TITLE-$FILETYPE.svg\" " + + echo " $PLOT_TITLE ; $YAXIS ; $DEFAULT_OPTS ; show style lines ; $OUTPUT ; plot " $PLOT_LINE | $GNUPLOT - + unset PLOT_LINE +} + +# +# plot +# + +plot "I/O Latency" lat "Time (msec)" 1000 +plot "I/O Operations Per Second" iops "IOPS" 1 +plot "I/O Submission Latency" slat "Time (μsec)" 1 +plot "I/O Completion Latency" clat "Time (msec)" 1000 +plot "I/O Bandwidth" bw "Throughput (KB/s)" 1 -PLOT_LINE="" -for i in *slat.log; do - if [ ! -r $i ]; then - continue - fi - PT=$(echo $i | sed s/_slat.log//g) - if [ "$PLOT_LINE"x != "x" ]; then - PLOT_LINE=$PLOT_LINE", " - fi - - PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines" -done - -if [ "$PLOT_LINE"x != "x" ]; then - echo Making slat logs $PLOT_LINE - echo "set title 'Submission latency - $TITLE'; set xlabel 'time (msec)'; set ylabel 'latency (msec)'; set terminal png size $XRES,$YRES; set output '$TITLE-slat.png'; plot " $PLOT_LINE | $GNUPLOT - -fi -PLOT_LINE="" -for i in *clat.log; do - if [ ! -r $i ]; then - continue - fi - PT=$(echo $i | sed s/_clat.log//g) - if [ "$PLOT_LINE"x != "x" ]; then - PLOT_LINE=$PLOT_LINE", " - fi - - PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines" -done - -if [ "$PLOT_LINE"x != "x" ]; then - echo Making clat logs $PLOT_LINE - echo "set title 'Completion latency - $TITLE'; set xlabel 'time (msec)'; set ylabel 'latency (msec)'; set terminal png size $XRES,$YRES; set output '$TITLE-clat.png'; plot " $PLOT_LINE | $GNUPLOT - -fi - -PLOT_LINE="" -for i in *_lat.log; do - if [ ! -r $i ]; then - continue - fi - PT=$(echo $i | sed s/_lat.log//g) - if [ "$PLOT_LINE"x != "x" ]; then - PLOT_LINE=$PLOT_LINE", " - fi - - PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines" -done - -if [ "$PLOT_LINE"x != "x" ]; then - echo Making lat logs $PLOT_LINE - echo "set title 'Latency - $TITLE'; set xlabel 'time (msec)'; set ylabel 'latency (msec)'; set terminal png size $XRES,$YRES; set output '$TITLE-lat.png'; plot " $PLOT_LINE | $GNUPLOT - -fi diff --git a/profile.c b/profile.c index 92f2fc46..5d0b8667 100644 --- a/profile.c +++ b/profile.c @@ -31,7 +31,7 @@ int load_profile(const char *profile) ops = find_profile(profile); if (ops) { if (ops->prep_cmd()) { - log_err("fio: profile prep failed\n"); + log_err("fio: profile %s prep failed\n", profile); return 1; } add_job_opts(ops->cmdline, FIO_CLIENT_TYPE_CLI); diff --git a/profiles/act.c b/profiles/act.c index 2080602a..73cde856 100644 --- a/profiles/act.c +++ b/profiles/act.c @@ -31,29 +31,34 @@ static struct act_pass_criteria act_pass[ACT_MAX_CRIT] = { }, }; +struct act_slice { + uint64_t lat_buckets[ACT_MAX_CRIT]; + uint64_t total_ios; +}; + struct act_run_data { struct fio_mutex *mutex; unsigned int pending; - uint64_t lat_buckets[ACT_MAX_CRIT]; - uint64_t total_ios; + struct act_slice *slices; + unsigned int nr_slices; }; static struct act_run_data *act_run_data; struct act_prof_data { struct timeval sample_tv; - uint64_t lat_buckets[ACT_MAX_CRIT]; - uint64_t total_ios; - uint64_t cum_lat_buckets[ACT_MAX_CRIT]; - uint64_t cum_total_ios; + struct act_slice *slices; + unsigned int cur_slice; + unsigned int nr_slices; }; static char *device_names; -static unsigned int load = 1; +static unsigned int load; static unsigned int prep; static unsigned int threads_per_queue; static unsigned int num_read_blocks; static unsigned int write_size; +static unsigned long long test_duration; #define ACT_MAX_OPTS 128 static const char *act_opts[ACT_MAX_OPTS] = { @@ -89,6 +94,16 @@ static struct fio_option options[] = { .category = FIO_OPT_C_PROFILE, .group = FIO_OPT_G_ACT, }, + { + .name = "test-duration", + .lname = "Test duration", + .type = FIO_OPT_STR_VAL_TIME, + .roff1 = &test_duration, + .help = "How long the entire test takes to run", + .def = "24h", + .category = FIO_OPT_C_PROFILE, + .group = FIO_OPT_G_ACT, + }, { .name = "threads-per-queue", .lname = "Number of read IO threads per device", @@ -222,7 +237,7 @@ static int act_add_dev(const char *dev) if (prep) return act_add_dev_prep(dev); - if (act_add_opt("runtime=24h")) + if (act_add_opt("runtime=%llus", test_duration)) return 1; if (act_add_opt("time_based=1")) return 1; @@ -241,7 +256,8 @@ static int act_add_dev(const char *dev) static int act_prep_cmdline(void) { if (!device_names) { - log_err("act: need device-names\n"); + log_err("act: you need to set IO target(s) with the " + "device-names option.\n"); return 1; } @@ -266,17 +282,26 @@ static int act_prep_cmdline(void) static int act_io_u_lat(struct thread_data *td, uint64_t usec) { struct act_prof_data *apd = td->prof_data; + struct act_slice *slice; int i, ret = 0; double perm; if (prep) return 0; - apd->total_ios++; + /* + * Really should not happen, but lets not let jitter at the end + * ruin our day. + */ + if (apd->cur_slice >= apd->nr_slices) + return 0; + + slice = &apd->slices[apd->cur_slice]; + slice->total_ios++; for (i = ACT_MAX_CRIT - 1; i >= 0; i--) { if (usec > act_pass[i].max_usec) { - apd->lat_buckets[i]++; + slice->lat_buckets[i]++; break; } } @@ -286,7 +311,7 @@ static int act_io_u_lat(struct thread_data *td, uint64_t usec) /* SAMPLE_SEC has passed, check criteria for pass */ for (i = 0; i < ACT_MAX_CRIT; i++) { - perm = (1000.0 * apd->lat_buckets[i]) / apd->total_ios; + perm = (1000.0 * slice->lat_buckets[i]) / slice->total_ios; if (perm < act_pass[i].max_perm) continue; @@ -295,15 +320,8 @@ static int act_io_u_lat(struct thread_data *td, uint64_t usec) break; } - for (i = 0; i < ACT_MAX_CRIT; i++) { - apd->cum_lat_buckets[i] += apd->lat_buckets[i]; - apd->lat_buckets[i] = 0; - } - - apd->cum_total_ios += apd->total_ios; - apd->total_ios = 0; - fio_gettime(&apd->sample_tv, NULL); + apd->cur_slice++; return ret; } @@ -314,51 +332,76 @@ static void get_act_ref(void) fio_mutex_up(act_run_data->mutex); } -static void act_show_all_stats(void) +static int show_slice(struct act_slice *slice, unsigned int slice_num) { - unsigned int i; - - log_info(" trans device\n"); - log_info(" %%>(ms) %%>(ms)\n"); - log_info(" slice"); + unsigned int i, failed = 0; - for (i = 0; i < ACT_MAX_CRIT; i++) - log_info("\t%2u", act_pass[i].max_usec / 1000); - for (i = 0; i < ACT_MAX_CRIT; i++) - log_info("\t%2u", act_pass[i].max_usec / 1000); - - log_info("\n"); - log_info(" ----- ------ ------ ------ ------ ------ ------\n"); - log_info(" 1"); + log_info(" %2u", slice_num); for (i = 0; i < ACT_MAX_CRIT; i++) { - double perc; + double perc = 0.0; - perc = 100.0 * (double) act_run_data->lat_buckets[i] / (double) act_run_data->total_ios; + if (slice->total_ios) + perc = 100.0 * (double) slice->lat_buckets[i] / (double) slice->total_ios; + if ((perc * 10.0) >= act_pass[i].max_perm) + failed++; log_info("\t%2.2f", perc); } for (i = 0; i < ACT_MAX_CRIT; i++) { - double perc; + double perc = 0.0; - perc = 100.0 * (double) act_run_data->lat_buckets[i] / (double) act_run_data->total_ios; + if (slice->total_ios) + perc = 100.0 * (double) slice->lat_buckets[i] / (double) slice->total_ios; log_info("\t%2.2f", perc); } + log_info("\n"); + + return failed; +} + +static void act_show_all_stats(void) +{ + unsigned int i, fails = 0; + + log_info(" trans device\n"); + log_info(" %%>(ms) %%>(ms)\n"); + log_info(" slice"); + + for (i = 0; i < ACT_MAX_CRIT; i++) + log_info("\t %2u", act_pass[i].max_usec / 1000); + for (i = 0; i < ACT_MAX_CRIT; i++) + log_info("\t %2u", act_pass[i].max_usec / 1000); + + log_info("\n"); + log_info(" ----- ----- ----- ------ ----- ----- ------\n"); + + for (i = 0; i < act_run_data->nr_slices; i++) + fails += show_slice(&act_run_data->slices[i], i + 1); + log_info("\nact: test complete, device(s): %s\n", fails ? "FAILED" : "PASSED"); } static void put_act_ref(struct thread_data *td) { struct act_prof_data *apd = td->prof_data; - unsigned int i; + unsigned int i, slice; fio_mutex_down(act_run_data->mutex); - for (i = 0; i < ACT_MAX_CRIT; i++) { - act_run_data->lat_buckets[i] += apd->cum_lat_buckets[i]; - act_run_data->lat_buckets[i] += apd->lat_buckets[i]; + if (!act_run_data->slices) { + act_run_data->slices = calloc(sizeof(struct act_slice), apd->nr_slices); + act_run_data->nr_slices = apd->nr_slices; } - act_run_data->total_ios += apd->cum_total_ios + apd->total_ios; + for (slice = 0; slice < apd->nr_slices; slice++) { + struct act_slice *dst = &act_run_data->slices[slice]; + struct act_slice *src = &apd->slices[slice]; + + dst->total_ios += src->total_ios; + + for (i = 0; i < ACT_MAX_CRIT; i++) + dst->lat_buckets[i] += src->lat_buckets[i]; + } if (!--act_run_data->pending) act_show_all_stats(); @@ -369,10 +412,14 @@ static void put_act_ref(struct thread_data *td) static int act_td_init(struct thread_data *td) { struct act_prof_data *apd; + unsigned int nr_slices; get_act_ref(); apd = calloc(sizeof(*apd), 1); + nr_slices = (test_duration + SAMPLE_SEC - 1) / SAMPLE_SEC; + apd->slices = calloc(sizeof(struct act_slice), nr_slices); + apd->nr_slices = nr_slices; fio_gettime(&apd->sample_tv, NULL); td->prof_data = apd; return 0; @@ -380,8 +427,11 @@ static int act_td_init(struct thread_data *td) static void act_td_exit(struct thread_data *td) { + struct act_prof_data *apd = td->prof_data; + put_act_ref(td); - free(td->prof_data); + free(apd->slices); + free(apd); td->prof_data = NULL; } @@ -416,6 +466,7 @@ static void fio_exit act_unregister(void) unregister_profile(&act_profile); fio_mutex_remove(act_run_data->mutex); + free(act_run_data->slices); free(act_run_data); act_run_data = NULL; }