Fix disk utilization for md/dm slaves

[fio.git] / io_u.c

diff --git a/io_u.c b/io_u.c
index 41b5cdbb04894192cb06e2d86a49bdf28d48be9d..da9d95054bcf58b8a42985d327439e001d26221c 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -231,11 +231,6 @@ static int get_next_offset(struct thread_data *td, struct io_u *io_u)
        return 0;
 }
 
-static inline int is_power_of_2(unsigned int val)
-{
-       return (val != 0 && ((val & (val - 1)) == 0));
-}
-
 static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u)
 {
        const int ddir = io_u->ddir;
@@ -350,7 +345,7 @@ static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)
        odir = ddir ^ 1;
        if (td_rw(td) && __should_check_rate(td, odir))
                td->rate_pending_usleep[odir] -= usec;
-       
+
        return ddir;
 }
 
@@ -406,7 +401,7 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td)
        return td->rwmix_ddir;
 }
 
-static void put_file_log(struct thread_data *td, struct fio_file *f)
+void put_file_log(struct thread_data *td, struct fio_file *f)
 {
        int ret = put_file(td, f);
 
@@ -416,16 +411,21 @@ static void put_file_log(struct thread_data *td, struct fio_file *f)
 
 void put_io_u(struct thread_data *td, struct io_u *io_u)
 {
+       td_io_u_lock(td);
+
        assert((io_u->flags & IO_U_F_FREE) == 0);
        io_u->flags |= IO_U_F_FREE;
+       io_u->flags &= ~IO_U_F_FREE_DEF;
 
        if (io_u->file)
                put_file_log(td, io_u->file);
 
        io_u->file = NULL;
-       flist_del(&io_u->list);
+       flist_del_init(&io_u->list);
        flist_add(&io_u->list, &td->io_u_freelist);
        td->cur_depth--;
+       td_io_u_unlock(td);
+       td_io_u_free_notify(td);
 }
 
 void clear_io_u(struct thread_data *td, struct io_u *io_u)
@@ -440,6 +440,8 @@ void requeue_io_u(struct thread_data *td, struct io_u **io_u)
 
        dprint(FD_IO, "requeue %p\n", __io_u);
 
+       td_io_u_lock(td);
+
        __io_u->flags |= IO_U_F_FREE;
        if ((__io_u->flags & IO_U_F_FLIGHT) && !ddir_sync(__io_u->ddir))
                td->io_issues[__io_u->ddir]--;
@@ -449,6 +451,7 @@ void requeue_io_u(struct thread_data *td, struct io_u **io_u)
        flist_del(&__io_u->list);
        flist_add_tail(&__io_u->list, &td->io_u_requeues);
        td->cur_depth--;
+       td_io_u_unlock(td);
        *io_u = NULL;
 }
 
@@ -751,7 +754,8 @@ static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
                        opened = 1;
                }
 
-               dprint(FD_FILE, "goodf=%x, badf=%x, ff=%x\n", goodf, badf, f->flags);
+               dprint(FD_FILE, "goodf=%x, badf=%x, ff=%x\n", goodf, badf,
+                                                               f->flags);
                if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
                        break;
 
@@ -820,7 +824,8 @@ static int set_io_u_file(struct thread_data *td, struct io_u *io_u)
                io_u->file = NULL;
                fio_file_set_done(f);
                td->nr_done_files++;
-               dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name, td->nr_done_files, td->o.nr_files);
+               dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name,
+                                       td->nr_done_files, td->o.nr_files);
        } while (1);
 
        return 0;
@@ -831,6 +836,9 @@ struct io_u *__get_io_u(struct thread_data *td)
 {
        struct io_u *io_u = NULL;
 
+       td_io_u_lock(td);
+
+again:
        if (!flist_empty(&td->io_u_requeues))
                io_u = flist_entry(td->io_u_requeues.next, struct io_u, list);
        else if (!queue_full(td)) {
@@ -842,9 +850,18 @@ struct io_u *__get_io_u(struct thread_data *td)
                io_u->end_io = NULL;
        }
 
+       /*
+        * We ran out, wait for async verify threads to finish and return one
+        */
+       if (!io_u && td->o.verify_async) {
+               pthread_cond_wait(&td->free_cond, &td->io_u_lock);
+               goto again;
+       }
+
        if (io_u) {
                assert(io_u->flags & IO_U_F_FREE);
                io_u->flags &= ~IO_U_F_FREE;
+               io_u->flags &= ~IO_U_F_FREE_DEF;
 
                io_u->error = 0;
                flist_del(&io_u->list);
@@ -852,6 +869,7 @@ struct io_u *__get_io_u(struct thread_data *td)
                td->cur_depth++;
        }
 
+       td_io_u_unlock(td);
        return io_u;
 }
 
@@ -983,7 +1001,7 @@ static void io_completed(struct thread_data *td, struct io_u *io_u,
                                                        &icd->time);
 
                        if (!td->o.disable_clat) {
-                               add_clat_sample(td, idx, usec, bytes);
+                               add_clat_sample(td, idx, lusec, bytes);
                                io_u_mark_latency(td, lusec);
                        }
                        if (!td->o.disable_bw)
@@ -1047,7 +1065,9 @@ static void ios_completed(struct thread_data *td,
                io_u = td->io_ops->event(td, i);
 
                io_completed(td, io_u, icd);
-               put_io_u(td, io_u);
+
+               if (!(io_u->flags & IO_U_F_FREE_DEF))
+                       put_io_u(td, io_u);
        }
 }
 
@@ -1061,7 +1081,9 @@ int io_u_sync_complete(struct thread_data *td, struct io_u *io_u,
 
        init_icd(td, &icd, 1);
        io_completed(td, io_u, &icd);
-       put_io_u(td, io_u);
+
+       if (!(io_u->flags & IO_U_F_FREE_DEF))
+               put_io_u(td, io_u);
 
        if (icd.error) {
                td_verror(td, icd.error, "io_u_sync_complete");