Merge branch 'master' of ssh://brick.kernel.dk/data/git/fio

[fio.git] / io_u.c

diff --git a/io_u.c b/io_u.c
index 21a801f82a0ee77497cbf66fb829cd75ff7fca18..8cc348a835fed67a2310c04f84b4608281db1315 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -8,6 +8,7 @@
 #include "fio.h"
 #include "hash.h"
 #include "verify.h"
+#include "trim.h"
 #include "lib/rand.h"
 
 struct io_completion_data {
@@ -125,16 +126,20 @@ static unsigned long long last_block(struct thread_data *td, struct fio_file *f,
 static int get_next_free_block(struct thread_data *td, struct fio_file *f,
                               enum fio_ddir ddir, unsigned long long *b)
 {
-       unsigned long long min_bs = td->o.rw_min_bs;
+       unsigned long long min_bs = td->o.rw_min_bs, lastb;
        int i;
 
+       lastb = last_block(td, f, ddir);
+       if (!lastb)
+               return 1;
+
        i = f->last_free_lookup;
        *b = (i * BLOCKS_PER_MAP);
        while ((*b) * min_bs < f->real_file_size &&
                (*b) * min_bs < f->io_size) {
                if (f->file_map[i] != (unsigned int) -1) {
                        *b += ffz(f->file_map[i]);
-                       if (*b > last_block(td, f, ddir))
+                       if (*b > lastb)
                                break;
                        f->last_free_lookup = i;
                        return 0;
@@ -151,14 +156,17 @@ static int get_next_free_block(struct thread_data *td, struct fio_file *f,
 static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
                                enum fio_ddir ddir, unsigned long long *b)
 {
-       unsigned long long r;
+       unsigned long long r, lastb;
        int loops = 5;
 
+       lastb = last_block(td, f, ddir);
+       if (!lastb)
+               return 1;
+
        do {
                r = os_random_long(&td->random_state);
                dprint(FD_RANDOM, "off rand %llu\n", r);
-               *b = (last_block(td, f, ddir) - 1)
-                       * (r / ((unsigned long long) OS_RAND_MAX + 1.0));
+               *b = (lastb - 1) * (r / ((unsigned long long) OS_RAND_MAX + 1.0));
 
                /*
                 * if we are not maintaining a random map, we are done.
@@ -245,7 +253,8 @@ static int get_next_block(struct thread_data *td, struct io_u *io_u,
                                ret = get_next_rand_block(td, f, ddir, b);
                } else if (td->o.rw_seq == RW_SEQ_IDENT) {
                        if (f->last_start != -1ULL)
-                               *b = (f->last_start - f->file_offset) / td->o.min_bs[ddir];
+                               *b = (f->last_start - f->file_offset)
+                                       / td->o.min_bs[ddir];
                        else
                                *b = 0;
                        ret = 0;
@@ -277,10 +286,8 @@ static int __get_next_offset(struct thread_data *td, struct io_u *io_u)
                td->ddir_seq_nr = td->o.ddir_seq_nr;
        }
 
-       if (get_next_block(td, io_u, ddir, rw_seq_hit, &b)) {
-               printf("fail\n");
+       if (get_next_block(td, io_u, ddir, rw_seq_hit, &b))
                return 1;
-       }
 
        io_u->offset = b * td->o.ba[ddir];
        if (io_u->offset >= f->io_size) {
@@ -501,6 +508,17 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td)
        return td->rwmix_ddir;
 }
 
+static void set_rw_ddir(struct thread_data *td, struct io_u *io_u)
+{
+       io_u->ddir = get_rw_ddir(td);
+
+       if (io_u->ddir == DDIR_WRITE && (td->io_ops->flags & FIO_BARRIER) &&
+           td->o.barrier_blocks &&
+          !(td->io_issues[DDIR_WRITE] % td->o.barrier_blocks) &&
+            td->io_issues[DDIR_WRITE])
+               io_u->flags |= IO_U_F_BARRIER;
+}
+
 void put_file_log(struct thread_data *td, struct fio_file *f)
 {
        int ret = put_file(td, f);
@@ -560,7 +578,7 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u)
        if (td->io_ops->flags & FIO_NOIO)
                goto out;
 
-       io_u->ddir = get_rw_ddir(td);
+       set_rw_ddir(td, io_u);
 
        /*
         * fsync() or fdatasync() or trim etc, we are done
@@ -963,6 +981,7 @@ again:
        if (io_u) {
                assert(io_u->flags & IO_U_F_FREE);
                io_u->flags &= ~(IO_U_F_FREE | IO_U_F_FREE_DEF);
+               io_u->flags &= ~(IO_U_F_TRIMMED | IO_U_F_BARRIER);
 
                io_u->error = 0;
                flist_del(&io_u->list);
@@ -982,21 +1001,31 @@ again:
        return io_u;
 }
 
-/*
- * Return an io_u to be processed. Gets a buflen and offset, sets direction,
- * etc. The returned io_u is fully ready to be prepped and submitted.
- */
-struct io_u *get_io_u(struct thread_data *td)
+static int check_get_trim(struct thread_data *td, struct io_u *io_u)
 {
-       struct fio_file *f;
-       struct io_u *io_u;
+       if (td->o.trim_backlog && td->trim_entries) {
+               int get_trim = 0;
 
-       io_u = __get_io_u(td);
-       if (!io_u) {
-               dprint(FD_IO, "__get_io_u failed\n");
-               return NULL;
+               if (td->trim_batch) {
+                       td->trim_batch--;
+                       get_trim = 1;
+               } else if (!(td->io_hist_len % td->o.trim_backlog) &&
+                        td->last_ddir != DDIR_READ) {
+                       td->trim_batch = td->o.trim_batch;
+                       if (!td->trim_batch)
+                               td->trim_batch = td->o.trim_backlog;
+                       get_trim = 1;
+               }
+
+               if (get_trim && !get_next_trim(td, io_u))
+                       return 1;
        }
 
+       return 0;
+}
+
+static int check_get_verify(struct thread_data *td, struct io_u *io_u)
+{
        if (td->o.verify_backlog && td->io_hist_len) {
                int get_verify = 0;
 
@@ -1012,9 +1041,32 @@ struct io_u *get_io_u(struct thread_data *td)
                }
 
                if (get_verify && !get_next_verify(td, io_u))
-                       goto out;
+                       return 1;
+       }
+
+       return 0;
+}
+
+/*
+ * Return an io_u to be processed. Gets a buflen and offset, sets direction,
+ * etc. The returned io_u is fully ready to be prepped and submitted.
+ */
+struct io_u *get_io_u(struct thread_data *td)
+{
+       struct fio_file *f;
+       struct io_u *io_u;
+
+       io_u = __get_io_u(td);
+       if (!io_u) {
+               dprint(FD_IO, "__get_io_u failed\n");
+               return NULL;
        }
 
+       if (check_get_verify(td, io_u))
+               goto out;
+       if (check_get_trim(td, io_u))
+               goto out;
+
        /*
         * from a requeue, io_u already setup
         */
@@ -1064,6 +1116,7 @@ struct io_u *get_io_u(struct thread_data *td)
        io_u->xfer_buflen = io_u->buflen;
 
 out:
+       assert(io_u->file);
        if (!td_io_prep(td, io_u)) {
                if (!td->o.disable_slat)
                        fio_gettime(&io_u->start_time, NULL);