Fix end-of-job slowdown for random IO with a random map

[fio.git] / io_u.c

diff --git a/io_u.c b/io_u.c
index 185bba03d9829052877b118f03257156c2eb2bc0..5a3ca7444946751f4e208a32affdad5fb98051b9 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -30,7 +30,7 @@ static int random_map_free(struct fio_file *f, const unsigned long long block)
 
        dprint(FD_RANDOM, "free: b=%llu, idx=%u, bit=%u\n", block, idx, bit);
 
-       return (f->file_map[idx] & (1 << bit)) == 0;
+       return (f->file_map[idx] & (1UL << bit)) == 0;
 }
 
 /*
@@ -50,8 +50,8 @@ static void mark_random_map(struct thread_data *td, struct io_u *io_u)
        busy_check = !(io_u->flags & IO_U_F_BUSY_OK);
 
        while (nr_blocks) {
-               unsigned int this_blocks, mask;
                unsigned int idx, bit;
+               unsigned long mask, this_blocks;
 
                /*
                 * If we have a mixed random workload, we may
@@ -75,9 +75,9 @@ static void mark_random_map(struct thread_data *td, struct io_u *io_u)
 
                do {
                        if (this_blocks == BLOCKS_PER_MAP)
-                               mask = -1U;
+                               mask = -1UL;
                        else
-                               mask = ((1U << this_blocks) - 1) << bit;
+                               mask = ((1UL << this_blocks) - 1) << bit;
        
                        if (!(f->file_map[idx] & mask))
                                break;
@@ -126,7 +126,7 @@ static unsigned long long last_block(struct thread_data *td, struct fio_file *f,
 static int get_next_free_block(struct thread_data *td, struct fio_file *f,
                               enum fio_ddir ddir, unsigned long long *b)
 {
-       unsigned long long min_bs = td->o.rw_min_bs, lastb;
+       unsigned long long block, min_bs = td->o.rw_min_bs, lastb;
        int i;
 
        lastb = last_block(td, f, ddir);
@@ -134,18 +134,19 @@ static int get_next_free_block(struct thread_data *td, struct fio_file *f,
                return 1;
 
        i = f->last_free_lookup;
-       *b = (i * BLOCKS_PER_MAP);
-       while ((*b) * min_bs < f->real_file_size &&
-               (*b) * min_bs < f->io_size) {
-               if (f->file_map[i] != (unsigned int) -1) {
-                       *b += ffz(f->file_map[i]);
-                       if (*b > lastb)
+       block = i * BLOCKS_PER_MAP;
+       while (block * min_bs < f->real_file_size &&
+               block * min_bs < f->io_size) {
+               if (f->file_map[i] != -1UL) {
+                       block += ffz(f->file_map[i]);
+                       if (block > lastb)
                                break;
                        f->last_free_lookup = i;
+                       *b = block;
                        return 0;
                }
 
-               *b += BLOCKS_PER_MAP;
+               block += BLOCKS_PER_MAP;
                i++;
        }
 
@@ -163,6 +164,9 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
        if (!lastb)
                return 1;
 
+       if (f->failed_rands >= 200)
+               goto ffz;
+
        do {
                r = os_random_long(&td->random_state);
                dprint(FD_RANDOM, "off rand %llu\n", r);
@@ -172,18 +176,21 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
                 * if we are not maintaining a random map, we are done.
                 */
                if (!file_randommap(td, f))
-                       return 0;
+                       goto ret_good;
 
                /*
                 * calculate map offset and check if it's free
                 */
                if (random_map_free(f, *b))
-                       return 0;
+                       goto ret_good;
 
                dprint(FD_RANDOM, "get_next_rand_offset: offset %llu busy\n",
                                                                        *b);
        } while (--loops);
 
+       if (!f->failed_rands++)
+               f->last_free_lookup = 0;
+
        /*
         * we get here, if we didn't suceed in looking up a block. generate
         * a random start offset into the filemap, and find the first free
@@ -194,7 +201,7 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
                f->last_free_lookup = (f->num_maps - 1) *
                                        (r / (OS_RAND_MAX + 1.0));
                if (!get_next_free_block(td, f, ddir, b))
-                       return 0;
+                       goto ret;
 
                r = os_random_long(&td->random_state);
        } while (--loops);
@@ -203,7 +210,15 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
         * that didn't work either, try exhaustive search from the start
         */
        f->last_free_lookup = 0;
+ffz:
+       if (!get_next_free_block(td, f, ddir, b))
+               return 0;
+       f->last_free_lookup = 0;
        return get_next_free_block(td, f, ddir, b);
+ret_good:
+       f->failed_rands = 0;
+ret:
+       return 0;
 }
 
 static int get_next_rand_block(struct thread_data *td, struct fio_file *f,
@@ -635,31 +650,31 @@ out:
 
 static void __io_u_mark_map(unsigned int *map, unsigned int nr)
 {
-       int index = 0;
+       int idx = 0;
 
        switch (nr) {
        default:
-               index = 6;
+               idx = 6;
                break;
        case 33 ... 64:
-               index = 5;
+               idx = 5;
                break;
        case 17 ... 32:
-               index = 4;
+               idx = 4;
                break;
        case 9 ... 16:
-               index = 3;
+               idx = 3;
                break;
        case 5 ... 8:
-               index = 2;
+               idx = 2;
                break;
        case 1 ... 4:
-               index = 1;
+               idx = 1;
        case 0:
                break;
        }
 
-       map[index]++;
+       map[idx]++;
 }
 
 void io_u_mark_submit(struct thread_data *td, unsigned int nr)
@@ -676,117 +691,117 @@ void io_u_mark_complete(struct thread_data *td, unsigned int nr)
 
 void io_u_mark_depth(struct thread_data *td, unsigned int nr)
 {
-       int index = 0;
+       int idx = 0;
 
        switch (td->cur_depth) {
        default:
-               index = 6;
+               idx = 6;
                break;
        case 32 ... 63:
-               index = 5;
+               idx = 5;
                break;
        case 16 ... 31:
-               index = 4;
+               idx = 4;
                break;
        case 8 ... 15:
-               index = 3;
+               idx = 3;
                break;
        case 4 ... 7:
-               index = 2;
+               idx = 2;
                break;
        case 2 ... 3:
-               index = 1;
+               idx = 1;
        case 1:
                break;
        }
 
-       td->ts.io_u_map[index] += nr;
+       td->ts.io_u_map[idx] += nr;
 }
 
 static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec)
 {
-       int index = 0;
+       int idx = 0;
 
        assert(usec < 1000);
 
        switch (usec) {
        case 750 ... 999:
-               index = 9;
+               idx = 9;
                break;
        case 500 ... 749:
-               index = 8;
+               idx = 8;
                break;
        case 250 ... 499:
-               index = 7;
+               idx = 7;
                break;
        case 100 ... 249:
-               index = 6;
+               idx = 6;
                break;
        case 50 ... 99:
-               index = 5;
+               idx = 5;
                break;
        case 20 ... 49:
-               index = 4;
+               idx = 4;
                break;
        case 10 ... 19:
-               index = 3;
+               idx = 3;
                break;
        case 4 ... 9:
-               index = 2;
+               idx = 2;
                break;
        case 2 ... 3:
-               index = 1;
+               idx = 1;
        case 0 ... 1:
                break;
        }
 
-       assert(index < FIO_IO_U_LAT_U_NR);
-       td->ts.io_u_lat_u[index]++;
+       assert(idx < FIO_IO_U_LAT_U_NR);
+       td->ts.io_u_lat_u[idx]++;
 }
 
 static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec)
 {
-       int index = 0;
+       int idx = 0;
 
        switch (msec) {
        default:
-               index = 11;
+               idx = 11;
                break;
        case 1000 ... 1999:
-               index = 10;
+               idx = 10;
                break;
        case 750 ... 999:
-               index = 9;
+               idx = 9;
                break;
        case 500 ... 749:
-               index = 8;
+               idx = 8;
                break;
        case 250 ... 499:
-               index = 7;
+               idx = 7;
                break;
        case 100 ... 249:
-               index = 6;
+               idx = 6;
                break;
        case 50 ... 99:
-               index = 5;
+               idx = 5;
                break;
        case 20 ... 49:
-               index = 4;
+               idx = 4;
                break;
        case 10 ... 19:
-               index = 3;
+               idx = 3;
                break;
        case 4 ... 9:
-               index = 2;
+               idx = 2;
                break;
        case 2 ... 3:
-               index = 1;
+               idx = 1;
        case 0 ... 1:
                break;
        }
 
-       assert(index < FIO_IO_U_LAT_M_NR);
-       td->ts.io_u_lat_m[index]++;
+       assert(idx < FIO_IO_U_LAT_M_NR);
+       td->ts.io_u_lat_m[idx]++;
 }
 
 static void io_u_mark_latency(struct thread_data *td, unsigned long usec)