From: Stephen M. Cameron Date: Mon, 29 Sep 2014 18:10:49 +0000 (-0600) Subject: fio: allow arithmetic expressions to be used in job files X-Git-Tag: fio-2.1.14~101^2~24 X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=b470a02cade02049509d22442addfbc88d10116e fio: allow arithmetic expressions to be used in job files However, the arithmetic expressions must be enclosed in parentheses. That is not a hard requirement, I just did it that way to enable the old code to be used for the existing job files, to reduce risk of breaking something that previously worked. Known issues: 1) if overflow or underflow occurs during the evaluation of arithmetic expressions, it is not detected. Likewise, detection of floating point divide by zero is a bit iffy. Calculations are carried out both as long longs and as integers. If at any point, a double precision floating point value is used, floating point values are used, and for the final long long calculation the double is cast to a long long. There may be other numeric subtleties lurking as well. These kind of things are why I require the parentheses to invoke the arithmetic processing. 2) I made no effort to work this code into the autoconf system. Have no idea how to do that, esp. with lex and yacc involved in this. 3) Suffixes (k, M, GiB, etc.) do not work with expressions. It is probably not very difficult to fix this. Signed-off-by: Stephen M. Cameron Modified by me to make configure auto-detect presence of yacc/lex and enable the arithmetic only if those conditions are met. Also got rid of the Makefile in exp/ and added the targets to the general Makefile, since this makes it easier to do properly. Signed-off-by: Jens Axboe --- diff --git a/Makefile b/Makefile index 8c6c0561..e3941ca7 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,11 @@ endif OBJS = $(SOURCE:.c=.o) FIO_OBJS = $(OBJS) fio.o + +ifdef CONFIG_ARITHMETIC +FIO_OBJS += lex.yy.o y.tab.o +endif + GFIO_OBJS = $(OBJS) gfio.o graph.o tickmarks.o ghelpers.o goptions.o gerror.o \ gclient.o gcompat.o cairo_text_helpers.o printing.o @@ -259,6 +264,25 @@ override CFLAGS += -DFIO_VERSION='"$(FIO_VERSION)"' sed -e 's/^ *//' -e 's/$$/:/' >> $*.d @rm -f $*.d.tmp +ifdef CONFIG_ARITHMETIC +lex.yy.o: lex.yy.c y.tab.h + $(QUIET_CC)$(CC) -o $@ $(CFLAGS) $(CPPFLAGS) -c $< + +y.tab.o: y.tab.c y.tab.h + $(QUIET_CC)$(CC) -o $@ $(CFLAGS) $(CPPFLAGS) -c $< + +y.tab.c: exp/expression-parser.y + $(QUIET_CC)$(YACC) -d exp/expression-parser.y + +y.tab.h: y.tab.c exp/fixup-buggy-yacc-output + exp/fixup-buggy-yacc-output $@ + +lex.yy.c: exp/expression-parser.l + $(QUIET_CC)$(LEX) exp/expression-parser.l + +parse.o: lex.yy.o y.tab.o +endif + init.o: FIO-VERSION-FILE init.c $(QUIET_CC)$(CC) -o init.o $(CFLAGS) $(CPPFLAGS) -c init.c @@ -296,10 +320,10 @@ t/ieee754: $(T_IEEE_OBJS) $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_IEEE_OBJS) $(LIBS) fio: $(FIO_OBJS) - $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(FIO_OBJS) $(LIBS) $(HDFSLIB) + $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(FIO_OBJS) $(PARSER_OBJS) $(LIBS) $(HDFSLIB) gfio: $(GFIO_OBJS) - $(QUIET_LINK)$(CC) $(LDFLAGS) -o gfio $(GFIO_OBJS) $(LIBS) $(GTK_LDFLAGS) + $(QUIET_LINK)$(CC) $(LDFLAGS) -o gfio $(GFIO_OBJS) $(PARSER_OBJS) $(LIBS) $(GTK_LDFLAGS) t/genzipf: $(T_ZIPF_OBJS) $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_ZIPF_OBJS) $(LIBS) @@ -319,7 +343,7 @@ t/dedupe: $(T_DEDUPE_OBJS) $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_DEDUPE_OBJS) $(LIBS) clean: FORCE - -rm -f .depend $(FIO_OBJS) $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(PROGS) $(T_PROGS) core.* core gfio FIO-VERSION-FILE *.d lib/*.d crc/*.d engines/*.d profiles/*.d t/*.d config-host.mak config-host.h + -rm -f .depend $(FIO_OBJS) $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(PROGS) $(T_PROGS) core.* core gfio FIO-VERSION-FILE *.d lib/*.d crc/*.d engines/*.d profiles/*.d t/*.d config-host.mak config-host.h exp/fixup-buggy-yacc-output y.tab.[ch] lex.y.c exp/*.[do] distclean: clean FORCE @rm -f cscope.out fio.pdf fio_generate_plots.pdf fio2gnuplot.pdf diff --git a/configure b/configure index 33d1327e..7ce874d7 100755 --- a/configure +++ b/configure @@ -1270,6 +1270,42 @@ if test "$libhdfs" = "yes" ; then fi echo "HDFS engine $libhdfs" +# Check if we have lex/yacc available +yacc="no" +lex="no" +arith="no" +LEX=$(which lex) +if test -x "$LEX" ; then + lex="yes" +fi +YACC=$(which yacc) +if test -x "$YACC" ; then + yacc="yes" +fi +if test "$yacc" = "yes" && test "$lex" = "yes" ; then + arith="yes" +fi + +if test "$arith" = "yes" ; then +cat > $TMPC << EOF +extern int yywrap(void); + +int main(int argc, char **argv) +{ + yywrap(); + return 0; +} +EOF + +if compile_prog "" "-ll -ly" "lex"; then + LIBS="-ll -ly $LIBS" +else + arith="no" +fi +fi + +echo "lex/yacc for arithmetic $arith" + ############################################################################# if test "$wordsize" = "64" ; then @@ -1414,6 +1450,9 @@ fi if test "$libhdfs" = "yes" ; then output_sym "CONFIG_LIBHDFS" fi +if test "$arith" = "yes" ; then + output_sym "CONFIG_ARITHMETIC" +fi if test "$zlib" = "no" ; then echo "Consider installing zlib-dev (zlib-devel), some fio features depend on it." diff --git a/exp/README.md b/exp/README.md new file mode 100644 index 00000000..48c11c98 --- /dev/null +++ b/exp/README.md @@ -0,0 +1,7 @@ +simple-expression-parser +======================== + +A simple expression parser for arithmetic expressions made with bison + flex + +To use, see the example test-expression-parser.c + diff --git a/exp/expression-parser.l b/exp/expression-parser.l new file mode 100644 index 00000000..388515e2 --- /dev/null +++ b/exp/expression-parser.l @@ -0,0 +1,101 @@ +%{ + +/* + * (C) Copyright 2014, Stephen M. Cameron. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include +#include "y.tab.h" + +#define YYSTYPE PARSER_VALUE_TYPE + +extern int lexer_input(char* buffer, int *nbytes, int buffersize); + +#undef YY_INPUT +#define YY_INPUT(buffer, bytes_read, bytes_requested) \ + lexer_input((buffer), &(bytes_read), (bytes_requested)) + +extern int yyerror(long long *result, double *dresult, + int *has_error, int *bye, const char *msg); + +static void __attribute__((unused)) yyunput(int c,char *buf_ptr); +static int __attribute__((unused)) input(void); + +%} + +%% + + +bye return BYE; +[ \t] ; /* ignore whitespace */ +#.+ ; /* ignore comments */ +[0-9]*[.][0-9]+ { + int rc; + double dval; + + rc = sscanf(yytext, "%lf", &dval); + if (rc == 1) { + yylval.v.dval = dval; + yylval.v.ival = (long long) dval; + yylval.v.has_dval = 1; + yylval.v.has_error = 0; + return NUMBER; + } else { + yyerror(0, 0, 0, 0, "bad number\n"); + yylval.v.has_error = 1; + return NUMBER; + } + } +0x[0-9a-fA-F]+ { + int rc, intval; + rc = sscanf(yytext, "%x", &intval); + if (rc == 1) { + yylval.v.ival = intval; + yylval.v.dval = (double) intval; + yylval.v.has_dval = 0; + yylval.v.has_error = 0; + return NUMBER; + } else { + yyerror(0, 0, 0, 0, "bad number\n"); + yylval.v.has_error = 1; + return NUMBER; + } + } +[0-9]+ { + int rc, intval; + rc = sscanf(yytext, "%d", &intval); + if (rc == 1) { + yylval.v.ival = intval; + yylval.v.dval = (double) intval; + yylval.v.has_dval = 0; + yylval.v.has_error = 0; + return NUMBER; + } else { + yyerror(0, 0, 0, 0, "bad number\n"); + yylval.v.has_error = 1; + return NUMBER; + } + } +\n return 0; +[+-/*()] return yytext[0]; +. { + yylval.v.has_error = 1; + return NUMBER; + } +%% + diff --git a/exp/expression-parser.y b/exp/expression-parser.y new file mode 100644 index 00000000..dbd8b6c2 --- /dev/null +++ b/exp/expression-parser.y @@ -0,0 +1,180 @@ +%{ + +/* + * (C) Copyright 2014, Stephen M. Cameron. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include + +struct parser_value_type { + double dval; + long long ival; + int has_dval; + int has_error; +}; + +typedef union valtype { + struct parser_value_type v; +} PARSER_VALUE_TYPE; + +#define YYSTYPE PARSER_VALUE_TYPE + +int yyerror(__attribute__((unused)) long long *result, + __attribute__((unused)) double *dresult, + __attribute__((unused)) int *has_error, + __attribute__((unused)) int *bye, const char *msg); + +extern int yylex(void); +extern void yyrestart(FILE *file); + +%} + +%union valtype { + struct parser_value_type { + double dval; + long long ival; + int has_dval; + int has_error; + } v; +}; + +%token NUMBER +%token BYE +%left '-' '+' +%left '*' '/' +%nonassoc UMINUS +%parse-param { long long *result } +%parse-param { double *dresult } +%parse-param { int *has_error } +%parse-param { int *bye } + +%type expression +%% + +top_level: expression { + *result = $1.ival; + *dresult = $1.dval; + *has_error = $1.has_error; + } + | expression error { + *result = $1.ival; + *dresult = $1.dval; + *has_error = 1; + } +expression: expression '+' expression { + if (!$1.has_dval && !$3.has_dval) + $$.ival = $1.ival + $3.ival; + else + $$.ival = (long long) ($1.dval + $3.dval); + $$.dval = $1.dval + $3.dval; + $$.has_error = $1.has_error || $3.has_error; + } + | expression '-' expression { + if (!$1.has_dval && !$3.has_dval) + $$.ival = $1.ival - $3.ival; + else + $$.ival = (long long) ($1.dval - $3.dval); + $$.dval = $1.dval - $3.dval; + $$.has_error = $1.has_error || $3.has_error; + } + | expression '*' expression { + if (!$1.has_dval && !$3.has_dval) + $$.ival = $1.ival * $3.ival; + else + $$.ival = (long long) ($1.dval * $3.dval); + $$.dval = $1.dval * $3.dval; + $$.has_error = $1.has_error || $3.has_error; + } + | expression '/' expression { + if ($3.ival == 0) + yyerror(0, 0, 0, 0, "divide by zero"); + else + $$.ival = $1.ival / $3.ival; + if ($3.dval < 1e-20 && $3.dval > -1e-20) + yyerror(0, 0, 0, 0, "divide by zero"); + else + $$.dval = $1.dval / $3.dval; + if ($3.has_dval || $1.has_dval) + $$.ival = (long long) $$.dval; + $$.has_error = $1.has_error || $3.has_error; + } + | '-' expression %prec UMINUS { + $$.ival = -$2.ival; + $$.dval = -$2.dval; + $$.has_error = $2.has_error; + } + | '(' expression ')' { $$ = $2; } + | NUMBER { $$ = $1; } + | BYE { $$ = $1; *bye = 1; }; +%% +#include + +/* Urgh. yacc and lex are kind of horrible. This is not thread safe, obviously. */ +static int lexer_read_offset = 0; +static char lexer_input_buffer[1000]; + +int lexer_input(char* buffer, int *bytes_read, int bytes_requested) +{ + int bytes_left = strlen(lexer_input_buffer) - lexer_read_offset; + + if (bytes_requested > bytes_left ) + bytes_requested = bytes_left; + memcpy(buffer, &lexer_input_buffer[lexer_read_offset], bytes_requested); + *bytes_read = bytes_requested; + lexer_read_offset += bytes_requested; + return 0; +} + +static void setup_to_parse_string(const char *string) +{ + unsigned int len; + + len = strlen(string); + if (len > sizeof(lexer_input_buffer) - 3) + len = sizeof(lexer_input_buffer) - 3; + + strncpy(lexer_input_buffer, string, len); + lexer_input_buffer[len] = '\0'; + lexer_input_buffer[len + 1] = '\0'; /* lex/yacc want string double null terminated! */ + lexer_read_offset = 0; +} + +int evaluate_arithmetic_expression(const char *buffer, long long *ival, double *dval) +{ + int rc, bye = 0, has_error = 0; + + setup_to_parse_string(buffer); + rc = yyparse(ival, dval, &has_error, &bye); + yyrestart(NULL); + if (rc || bye || has_error) { + *ival = 0; + *dval = 0; + has_error = 1; + } + return has_error; +} + +int yyerror(__attribute__((unused)) long long *result, + __attribute__((unused)) double *dresult, + __attribute__((unused)) int *has_error, + __attribute__((unused)) int *bye, const char *msg) +{ + fprintf(stderr, "%s\n", msg); + return 0; +} + diff --git a/exp/fixup-buggy-yacc-output.c b/exp/fixup-buggy-yacc-output.c new file mode 100644 index 00000000..ab9bd8e4 --- /dev/null +++ b/exp/fixup-buggy-yacc-output.c @@ -0,0 +1,170 @@ +/* + * (C) Copyright 2014, Stephen M. Cameron. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static char *programname; + +static char *slurp_file(char *f, off_t *textsize) +{ + int fd; + struct stat statbuf; + off_t bytesleft, bytesread; + char *fileptr = NULL; + char *slurped_file = NULL; + + fd = open(f, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "%s: Cannot open '%s': %s\n", + programname, f, strerror(errno)); + return NULL; + } + if (fstat(fd, &statbuf) != 0) { + fprintf(stderr, "%s: Cannot stat '%s': %s\n", + programname, f, strerror(errno)); + close(fd); + return NULL; + } + bytesleft = statbuf.st_size; + slurped_file = malloc(bytesleft + 1); + fileptr = slurped_file; + if (!slurped_file) { + fprintf(stderr, "%s: malloc returned NULL, out of memory\n", + programname); + goto bail_out; + } + memset(slurped_file, 0, bytesleft + 1); + do { + bytesread = read(fd, fileptr, bytesleft); + if (bytesread < 0 && errno == EAGAIN) + continue; + if (bytesread < 0) { + fprintf(stderr, "%s: error reading '%s: %s'\n", + programname, f, strerror(errno)); + goto bail_out; + } + if (bytesread == 0) { + fprintf(stderr, "%s: unexpected EOF in %s\n", + programname, f); + goto bail_out; + } + fileptr += bytesread; + bytesleft -= bytesread; + } while (bytesleft > 0); + + *textsize = statbuf.st_size; + + close(fd); + return slurped_file; + +bail_out: + if (slurped_file) + free(slurped_file); + close(fd); + return NULL; +} + +static int detect_buggy_yacc(char *text) +{ + char *x; + + x = strstr(text, " #line "); + if (!x) + return 0; + return 1; +} + +static void fixup_buggy_yacc_file(char *f) +{ + char *slurped_file, *x; + off_t textsize; + char *newname; + int fd; + off_t bytesleft, byteswritten; + + newname = alloca(strlen(f) + 10); + strcpy(newname, "broken-"); + strcat(newname, f); + + slurped_file = slurp_file(f, &textsize); + if (!slurped_file) + return; + if (!detect_buggy_yacc(slurped_file)) + return; + + x = slurped_file; + + + /* + * Fixup the '#line' directives which yacc botched. + * Note: this is vulnerable to false positives, but + * since this program is just a hack to make this particular + * program work, it is sufficient for our purposes. + * regexp could make this better, but the real fix needs + * to be made in yacc/bison. + */ + while ((x = strstr(x, " #line ")) != NULL) { + *x = '\n'; + } + + if (rename(f, newname) != 0) { + fprintf(stderr, "%s: Failed to rename '%s' to '%s': %s\n", + programname, f, newname, strerror(errno)); + return; + } + fd = open(f, O_CREAT | O_TRUNC | O_RDWR, 0644); + if (fd < 0) { + fprintf(stderr, "%s: failed to create '%s': %s\n", + programname, f, strerror(errno)); + return; + } + + bytesleft = textsize; + x = slurped_file; + do { + byteswritten = write(fd, x, bytesleft); + if (byteswritten < 0) { + fprintf(stderr, "%s: Error writing '%s': %s\n", + programname, f, strerror(errno)); + return; + } + if (byteswritten == 0 && errno == EINTR) + continue; + x += byteswritten; + bytesleft -= byteswritten; + } while (bytesleft > 0); + close(fd); +} + +int main(int argc, char *argv[]) +{ + int i; + + programname = argv[0]; + + for (i = 1; i < argc; i++) + fixup_buggy_yacc_file(argv[i]); + return 0; +} diff --git a/exp/test-expression-parser.c b/exp/test-expression-parser.c new file mode 100644 index 00000000..6b4ab5dd --- /dev/null +++ b/exp/test-expression-parser.c @@ -0,0 +1,50 @@ +/* + * (C) Copyright 2014, Stephen M. Cameron. + * + * The license below covers all files distributed with fio unless otherwise + * noted in the file itself. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include + +#include "y.tab.h" + +int main(int argc, char *argv[]) +{ + int rc, has_error, bye = 0; + long long result; + double dresult; + char buffer[100]; + + do { + if (fgets(buffer, 90, stdin) == NULL) + break; + rc = strlen(buffer); + if (rc > 0 && buffer[rc - 1] == '\n') + buffer[rc - 1] = '\0'; + rc = evaluate_arithmetic_expression(buffer, &result, &dresult); + if (!rc) { + printf("%lld (%lf)\n", result, dresult); + } else { + result = 0; + dresult = 0; + } + } while (!bye); + return 0; +} + diff --git a/fio.1 b/fio.1 index d64fbb7a..2fbdb00d 100644 --- a/fio.1 +++ b/fio.1 @@ -115,7 +115,12 @@ and there may be any number of global sections. Specific job definitions may override any parameter set in global sections. .SH "JOB PARAMETERS" .SS Types -Some parameters may take arguments of a specific type. The types used are: +Some parameters may take arguments of a specific type. +Anywhere a numeric value is required, an arithmetic expression may be used, +provided it is surrounded by parentheses. Suffixes currently do not +work with arithmetic expressions. Supported operators are +addition, subtraction, multiplication and division. +The types used are: .TP .I str String: a sequence of alphanumeric characters. diff --git a/parse.c b/parse.c index 40cd4658..c2d1cc81 100644 --- a/parse.c +++ b/parse.c @@ -18,6 +18,10 @@ #include "minmax.h" #include "lib/ieee754.h" +#ifdef CONFIG_ARITHMETIC +#include "y.tab.h" +#endif + static struct fio_option *__fio_options; static int vp_cmp(const void *p1, const void *p2) @@ -264,12 +268,28 @@ static unsigned long long get_mult_bytes(const char *str, int len, void *data, return __get_mult_bytes(p, data, percent); } +extern int evaluate_arithmetic_expression(const char *buffer, long long *ival, + double *dval); + /* * Convert string into a floating number. Return 1 for success and 0 otherwise. */ int str_to_float(const char *str, double *val) { - return (1 == sscanf(str, "%lf", val)); +#ifdef CONFIG_ARITHMETIC + int rc; + long long ival; + double dval; + + if (str[0] == '(') { + rc = evaluate_arithmetic_expression(str, &ival, &dval); + if (!rc) { + *val = dval; + return 1; + } + } +#endif + return 1 == sscanf(str, "%lf", val); } /* @@ -279,19 +299,33 @@ int str_to_decimal(const char *str, long long *val, int kilo, void *data, int is_seconds) { int len, base; + int rc = 1; +#ifdef CONFIG_ARITHMETIC + long long ival; + double dval; +#endif len = strlen(str); if (!len) return 1; - if (strstr(str, "0x") || strstr(str, "0X")) - base = 16; - else - base = 10; +#ifdef CONFIG_ARITHMETIC + if (str[0] == '(') + rc = evaluate_arithmetic_expression(str, &ival, &dval); + if (str[0] == '(' && !rc) + *val = ival; +#endif - *val = strtoll(str, NULL, base); - if (*val == LONG_MAX && errno == ERANGE) - return 1; + if (rc == 1) { + if (strstr(str, "0x") || strstr(str, "0X")) + base = 16; + else + base = 10; + + *val = strtoll(str, NULL, base); + if (*val == LONG_MAX && errno == ERANGE) + return 1; + } if (kilo) { unsigned long long mult;