backend: Add configurable non fatal error list

Sometimes it is reasonable to perform test nearly system limits where
errors are possible and expected, in that case one may provide non
fatal error list which will be ignored during execution.
This patch add two options:
    ignore_error: List of non fatal error
    error_dump:   Whenever ignored list should be dumped to log.

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/HOWTO b/HOWTO
index ec7005fe5b44e1c70c2ff6e2195849599a24e0ca..1362aeaa0319c5338d6befd4825fb06b45c2ddda 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -1248,6 +1248,19 @@ continue_on_error=str    Normally fio will exit the job on the first observed
 
                        1               Backward-compatible alias for 'all'.
 
 
                        1               Backward-compatible alias for 'all'.
 

+ignore_error=str Sometimes you want to ignore some errors during test
+                in that case you can specify error list for each error type.
+                ignore_error=READ_ERR_LIST,WRITE_ERR_LIST,VERIFY_ERR_LIST
+                errors for given error type is separated with ':'. Error
+                may be symbol ('ENOSPC', 'ENOMEM') or integer.
+                Example:
+                       ignore_error=EAGAIN,ENOSPC:122
+                This option will ignore EAGAIN from READ, and ENOSPC and 
+                122(EDQUOT) from WRITE. 
+
+error_dump=bool If set dump every error even if it is non fatal, true
+               by default. If disabled only fatal error will be dumped
+                                

 cgroup=str     Add job to this control group. If it doesn't exist, it will
                be created. The system must have a mounted cgroup blkio
                mount point for this to work. If your system doesn't have it
 cgroup=str     Add job to this control group. If it doesn't exist, it will
                be created. The system must have a mounted cgroup blkio
                mount point for this to work. If your system doesn't have it

diff --git a/backend.c b/backend.c
index ce0a0098299294279ea4bf16dfd88b7079d9466e..39d13a330581e91974c4f04b36f187b149ff180b 100644 (file)
--- a/backend.c
+++ b/backend.c
@@ -337,17 +337,17 @@ static int break_on_this_error(struct thread_data *td, enum fio_ddir ddir,
        int ret = *retptr;
 
        if (ret < 0 || td->error) {
        int ret = *retptr;
 
        if (ret < 0 || td->error) {

-               int err;
+               int err = td->error;
+               enum error_type_bit eb;

 
                if (ret < 0)
                        err = -ret;
 
                if (ret < 0)
                        err = -ret;

-               else
-                       err = td->error;

 
 

-               if (!(td->o.continue_on_error & td_error_type(ddir, err)))
+               eb = td_error_type(ddir, err);
+               if (!(td->o.continue_on_error & (1 << eb)))

                        return 1;
 
                        return 1;
 

-               if (td_non_fatal_error(err)) {
+               if (td_non_fatal_error(td, eb, err)) {

                        /*
                         * Continue with the I/Os in case of
                         * a non fatal error.
                        /*
                         * Continue with the I/Os in case of
                         * a non fatal error.

diff --git a/examples/enospc-pressure b/examples/enospc-pressure
new file mode 100644 (file)
index 0000000..ca9d8f7
--- /dev/null
+++ b/examples/enospc-pressure
@@ -0,0 +1,51 @@
+#
+# Test for race-condition DIO-write vs punch_hole
+# If race exist dio may rewrite punched block after
+# it was allocated to another file, we will catch that
+# by verifying blocks content
+#
+[global]
+ioengine=libaio 
+directory=/scratch
+# File size is reasonably huge to provoke ENOSPC
+filesize=128G
+size=999G
+iodepth=128
+
+# Expect write failure due to ENOSPC, skip error dump
+continue_on_error=write
+ignore_error=,ENOSPC
+error_dump=0
+fallocate=none
+exitall
+
+# Two threads (dio and punch_hole) operate on single file:'raicer',
+# We do not care about data content here
+[dio-raicer]
+bs=128k 
+direct=1
+buffered=0 
+rw=randwrite
+runtime=100
+filename=raicer
+time_based
+
+[punch_hole-raicer]
+bs=4k
+rw=randtrim
+filename=raicer
+
+# Verifier thread continiously write to newly allcated blocks
+# and veryfy written content
+[aio-dio-verifier]
+create_on_open=1
+verify=crc32c-intel
+verify_fatal=1
+verify_dump=1
+verify_backlog=1024
+verify_async=4
+direct=1
+# block size should be equals to fs block size to prevent short writes
+bs=4k
+rw=randrw
+filename=aio-dio-verifier

diff --git a/fio.1 b/fio.1
index c22d8b28490e6f694ed051f095c2ff6670bf5ff4..145b547ea5404a84af510f8653a8816dc35db2c4 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -971,6 +971,23 @@ entering the kernel with a gettimeofday() call. The CPU set aside for doing
 these time calls will be excluded from other uses. Fio will manually clear it
 from the CPU mask of other jobs.
 .TP
 these time calls will be excluded from other uses. Fio will manually clear it
 from the CPU mask of other jobs.
 .TP

+.BI ignore_error \fR=\fPstr
+Sometimes you want to ignore some errors during test in that case you can specify
+error list for each error type.
+.br
+ignore_error=READ_ERR_LIST,WRITE_ERR_LIST,VERIFY_ERR_LIST
+.br
+errors for given error type is separated with ':'.
+Error may be symbol ('ENOSPC', 'ENOMEM') or an integer.
+.br
+Example: ignore_error=EAGAIN,ENOSPC:122 .
+.br    
+This option will ignore EAGAIN from READ, and ENOSPC and 122(EDQUOT) from WRITE. 
+.TP
+.BI error_dump \fR=\fPbool
+If set dump every error even if it is non fatal, true by default. If disabled
+only fatal error will be dumped
+.TP

 .BI cgroup \fR=\fPstr
 Add job to this control group. If it doesn't exist, it will be created.
 The system must have a mounted cgroup blkio mount point for this to work. If
 .BI cgroup \fR=\fPstr
 Add job to this control group. If it doesn't exist, it will be created.
 The system must have a mounted cgroup blkio mount point for this to work. If

diff --git a/fio.h b/fio.h
index b2bbe93f5bf5b3c7f8d1463ecd00f08359034baf..8bb5b0346eddde2f13aa17b87f8408d4ae281d8c 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -70,11 +70,18 @@ enum {
 /*
  * What type of errors to continue on when continue_on_error is used
  */
 /*
  * What type of errors to continue on when continue_on_error is used
  */

+enum error_type_bit {
+       ERROR_TYPE_READ_BIT = 0,
+       ERROR_TYPE_WRITE_BIT = 1,
+       ERROR_TYPE_VERIFY_BIT = 2,
+       ERROR_TYPE_CNT = 3,
+};
+

 enum error_type {
         ERROR_TYPE_NONE = 0,
 enum error_type {
         ERROR_TYPE_NONE = 0,

-        ERROR_TYPE_READ = 1 << 0,
-        ERROR_TYPE_WRITE = 1 << 1,
-        ERROR_TYPE_VERIFY = 1 << 2,
+        ERROR_TYPE_READ = 1 << ERROR_TYPE_READ_BIT,
+        ERROR_TYPE_WRITE = 1 << ERROR_TYPE_WRITE_BIT,
+        ERROR_TYPE_VERIFY = 1 << ERROR_TYPE_VERIFY_BIT,

         ERROR_TYPE_ANY = 0xffff,
 };
 
         ERROR_TYPE_ANY = 0xffff,
 };
 


@@ -115,6 +122,10 @@ struct thread_options {
        struct bssplit *bssplit[DDIR_RWDIR_CNT];
        unsigned int bssplit_nr[DDIR_RWDIR_CNT];
 
        struct bssplit *bssplit[DDIR_RWDIR_CNT];
        unsigned int bssplit_nr[DDIR_RWDIR_CNT];
 

+       int *ignore_error[ERROR_TYPE_CNT];
+       unsigned int ignore_error_nr[ERROR_TYPE_CNT];
+       unsigned int error_dump;
+

        unsigned int nr_files;
        unsigned int open_files;
        enum file_lock_mode file_lock_mode;
        unsigned int nr_files;
        unsigned int open_files;
        enum file_lock_mode file_lock_mode;


@@ -559,15 +570,32 @@ static inline void fio_ro_check(struct thread_data *td, struct io_u *io_u)
 
 #define REAL_MAX_JOBS          2048
 
 
 #define REAL_MAX_JOBS          2048
 

-#define td_non_fatal_error(e)  ((e) == EIO || (e) == EILSEQ)
-

 static inline enum error_type td_error_type(enum fio_ddir ddir, int err)
 {
        if (err == EILSEQ)
 static inline enum error_type td_error_type(enum fio_ddir ddir, int err)
 {
        if (err == EILSEQ)

-               return ERROR_TYPE_VERIFY;
+               return ERROR_TYPE_VERIFY_BIT;

        if (ddir == DDIR_READ)
        if (ddir == DDIR_READ)

-               return ERROR_TYPE_READ;
-       return ERROR_TYPE_WRITE;
+               return ERROR_TYPE_READ_BIT;
+       return ERROR_TYPE_WRITE_BIT;
+}
+
+static int __NON_FATAL_ERR[] = {EIO, EILSEQ};
+static inline int td_non_fatal_error(struct thread_data *td,
+                                    enum error_type_bit etype, int err)
+{
+       int i;
+       if (!td->o.ignore_error[etype]) {
+               td->o.ignore_error[etype] = __NON_FATAL_ERR;
+               td->o.ignore_error_nr[etype] = sizeof(__NON_FATAL_ERR)
+                       / sizeof(int);
+       }
+
+       if (!(td->o.continue_on_error & (1 << etype)))
+               return 0;
+       for (i = 0; i < td->o.ignore_error_nr[etype]; i++)
+               if (td->o.ignore_error[etype][i] == err)
+                       return 1;
+       return 0;

 }
 
 static inline void update_error_count(struct thread_data *td, int err)
 }
 
 static inline void update_error_count(struct thread_data *td, int err)

diff --git a/init.c b/init.c
index b3215f52fca3ebb2306c5e02e8437aac0a6d57fa..2ad039b8780d9ae788c8a9fab598b3724ae3cc5d 100644 (file)
--- a/init.c
+++ b/init.c
@@ -1198,7 +1198,7 @@ static int fill_def_thread(void)
 
        fio_getaffinity(getpid(), &def_thread.o.cpumask);
        def_thread.o.timeout = def_timeout;
 
        fio_getaffinity(getpid(), &def_thread.o.cpumask);
        def_thread.o.timeout = def_timeout;

-
+       def_thread.o.error_dump = 1;

        /*
         * fill default options
         */
        /*
         * fill default options
         */

diff --git a/io_u.c b/io_u.c
index db0a6dc50d2d509c5445e5cde26899628ea03e1d..a2c583df26bb43943d7bda1ebf05e6999d9432b6 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -1290,10 +1290,12 @@ err_put:
 
 void io_u_log_error(struct thread_data *td, struct io_u *io_u)
 {
 
 void io_u_log_error(struct thread_data *td, struct io_u *io_u)
 {

+       enum error_type_bit eb = td_error_type(io_u->ddir, io_u->error);

        const char *msg[] = { "read", "write", "sync", "datasync",
                                "sync_file_range", "wait", "trim" };
 
        const char *msg[] = { "read", "write", "sync", "datasync",
                                "sync_file_range", "wait", "trim" };
 

-
+       if (td_non_fatal_error(td, eb, io_u->error) && !td->o.error_dump)
+               return;

 
        log_err("fio: io_u error");
 
 
        log_err("fio: io_u error");
 


@@ -1432,8 +1434,10 @@ static void io_completed(struct thread_data *td, struct io_u *io_u,
                icd->error = io_u->error;
                io_u_log_error(td, io_u);
        }
                icd->error = io_u->error;
                io_u_log_error(td, io_u);
        }

-       if (icd->error && td_non_fatal_error(icd->error) &&
-           (td->o.continue_on_error & td_error_type(io_u->ddir, icd->error))) {
+       if (icd->error) {
+               enum error_type_bit eb = td_error_type(io_u->ddir, icd->error);
+               if (!td_non_fatal_error(td, eb, icd->error))
+                       return;

                /*
                 * If there is a non_fatal error, then add to the error count
                 * and clear all the errors.
                /*
                 * If there is a non_fatal error, then add to the error count
                 * and clear all the errors.

diff --git a/options.c b/options.c
index d1cf7e8a70ba6480c9dc1b1021b5a739c5dec9f9..dd71f1e47747d550a0cb5f20228ab99cffca3504 100644 (file)
--- a/options.c
+++ b/options.c
@@ -214,6 +214,101 @@ static int str_bssplit_cb(void *data, const char *input)
        return ret;
 }
 
        return ret;
 }
 

+static int str2error(char *str)
+{
+       const char * err[] = {"EPERM", "ENOENT", "ESRCH", "EINTR", "EIO",
+                           "ENXIO", "E2BIG", "ENOEXEC", "EBADF",
+                           "ECHILD", "EAGAIN", "ENOMEM", "EACCES",
+                           "EFAULT", "ENOTBLK", "EBUSY", "EEXIST",
+                           "EXDEV", "ENODEV", "ENOTDIR", "EISDIR",
+                           "EINVAL", "ENFILE", "EMFILE", "ENOTTY",
+                           "ETXTBSY","EFBIG", "ENOSPC", "ESPIPE",
+                           "EROFS","EMLINK", "EPIPE", "EDOM", "ERANGE"};
+       int i = 0, num = sizeof(err) / sizeof(void *);
+
+       while( i < num) {
+               if (!strcmp(err[i], str))
+                       return i + 1;
+               i++;
+       }
+       return 0;
+}
+
+static int ignore_error_type(struct thread_data *td, int etype, char *str)
+{
+       unsigned int i;
+       int *error;
+       char *fname;
+
+       if (etype >= ERROR_TYPE_CNT) {
+               log_err("Illegal error type\n");
+               return 1;
+       }
+
+       td->o.ignore_error_nr[etype] = 4;
+       error = malloc(4 * sizeof(struct bssplit));
+
+       i = 0;
+       while ((fname = strsep(&str, ":")) != NULL) {
+
+               if (!strlen(fname))
+                       break;
+
+               /*
+                * grow struct buffer, if needed
+                */
+               if (i == td->o.ignore_error_nr[etype]) {
+                       td->o.ignore_error_nr[etype] <<= 1;
+                       error = realloc(error, td->o.ignore_error_nr[etype]
+                                                 * sizeof(int));
+               }
+               if (fname[0] == 'E') {
+                       error[i] = str2error(fname);
+               } else {
+                       error[i] = atoi(fname);
+                       if (error[i] < 0)
+                               error[i] = error[i];
+               }
+               if (!error[i]) {
+                       log_err("Unknown error %s, please use number value \n",
+                                 fname);
+                       return 1;
+               }
+               i++;
+       }
+       if (i) {
+               td->o.continue_on_error |= 1 << etype;
+               td->o.ignore_error_nr[etype] = i;
+               td->o.ignore_error[etype] = error;
+       }
+       return 0;
+
+}
+
+static int str_ignore_error_cb(void *data, const char *input)
+{
+       struct thread_data *td = data;
+       char *str, *p, *n;
+       int type = 0, ret = 1;
+       p = str = strdup(input);
+
+       strip_blank_front(&str);
+       strip_blank_end(str);
+
+       while (p) {
+               n = strchr(p, ',');
+               if (n)
+                       *n++ = '\0';
+               ret = ignore_error_type(td, type, p);
+               if (ret)
+                       break;
+               p = n;
+               type++;
+       }
+       free(str);
+       return ret;
+}
+

 static int str_rw_cb(void *data, const char *str)
 {
        struct thread_data *td = data;
 static int str_rw_cb(void *data, const char *str)
 {
        struct thread_data *td = data;


@@ -2209,6 +2304,21 @@ static struct fio_option options[FIO_MAX_OPTS] = {
                          },
                },
        },
                          },
                },
        },

+       {
+               .name   = "ignore_error",
+               .type   = FIO_OPT_STR,
+               .cb     = str_ignore_error_cb,
+               .help   = "Set a specific list of errors to ignore",
+               .parent = "rw",
+       },
+       {
+               .name   = "error_dump",
+               .type   = FIO_OPT_BOOL,
+               .off1   = td_var_offset(error_dump),
+               .def    = "0",
+               .help   = "Dump info on each error",
+       },
+

        {
                .name   = "profile",
                .type   = FIO_OPT_STR_STORE,
        {
                .name   = "profile",
                .type   = FIO_OPT_STR_STORE,

diff --git a/verify.c b/verify.c
index f25eab921fc379691466b9e8c3866efd3a2133eb..f246dc8f9ba0cfe3f4a9e2b43316f398b75cb4b1 100644 (file)
--- a/verify.c
+++ b/verify.c
@@ -1049,8 +1049,7 @@ static void *verify_async_thread(void *data)
                        put_io_u(td, io_u);
                        if (!ret)
                                continue;
                        put_io_u(td, io_u);
                        if (!ret)
                                continue;

-                       if (td->o.continue_on_error & ERROR_TYPE_VERIFY &&
-                           td_non_fatal_error(ret)) {
+                       if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) {

                                update_error_count(td, ret);
                                td_clear_error(td);
                                ret = 0;
                                update_error_count(td, ret);
                                td_clear_error(td);
                                ret = 0;