Revert "smalloc: convert to spinlocks"

[fio.git] / smalloc.c

diff --git a/smalloc.c b/smalloc.c
index e97ba011319bd83e33549b51880d63fa2031806a..0a86c43b0f17d58a5aa69587f92eff1d4f39c279 100644 (file)
--- a/smalloc.c
+++ b/smalloc.c
@@ -12,6 +12,7 @@
 #include <limits.h>
 
 #include "mutex.h"
+#include "arch/arch.h"
 
 #define MP_SAFE                        /* define to make thread safe */
 #define SMALLOC_REDZONE                /* define to detect memory corruption */
@@ -90,7 +91,7 @@ static inline void global_write_unlock(void)
 
 static inline int ptr_valid(struct pool *pool, void *ptr)
 {
-       unsigned int pool_size = (pool->nr_blocks + 1) * SMALLOC_BPL;
+       unsigned int pool_size = pool->nr_blocks * SMALLOC_BPL;
 
        return (ptr >= pool->map) && (ptr < pool->map + pool_size);
 }
@@ -100,12 +101,19 @@ static inline unsigned int size_to_blocks(unsigned int size)
        return (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
 }
 
-static int blocks_iter(unsigned int *map, unsigned int idx,
-                      unsigned int nr_blocks,
+static int blocks_iter(struct pool *pool, unsigned int pool_idx,
+                      unsigned int idx, unsigned int nr_blocks,
                       int (*func)(unsigned int *map, unsigned int mask))
 {
+
        while (nr_blocks) {
                unsigned int this_blocks, mask;
+               unsigned int *map;
+
+               if (pool_idx >= pool->nr_blocks)
+                       return 0;
+
+               map = &pool->bitmap[pool_idx];
 
                this_blocks = nr_blocks;
                if (this_blocks + idx > SMALLOC_BPI) {
@@ -123,7 +131,7 @@ static int blocks_iter(unsigned int *map, unsigned int idx,
 
                nr_blocks -= this_blocks;
                idx = 0;
-               map++;
+               pool_idx++;
        }
 
        return 1;
@@ -136,67 +144,41 @@ static int mask_cmp(unsigned int *map, unsigned int mask)
 
 static int mask_clear(unsigned int *map, unsigned int mask)
 {
+       assert((*map & mask) == mask);
        *map &= ~mask;
        return 1;
 }
 
 static int mask_set(unsigned int *map, unsigned int mask)
 {
+       assert(!(*map & mask));
        *map |= mask;
        return 1;
 }
 
-static int blocks_free(unsigned int *map, unsigned int idx,
-                      unsigned int nr_blocks)
+static int blocks_free(struct pool *pool, unsigned int pool_idx,
+                      unsigned int idx, unsigned int nr_blocks)
 {
-       return blocks_iter(map, idx, nr_blocks, mask_cmp);
+       return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);
 }
 
-static void set_blocks(unsigned int *map, unsigned int idx,
-                      unsigned int nr_blocks)
+static void set_blocks(struct pool *pool, unsigned int pool_idx,
+                      unsigned int idx, unsigned int nr_blocks)
 {
-       blocks_iter(map, idx, nr_blocks, mask_set);
+       blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);
 }
 
-static void clear_blocks(unsigned int *map, unsigned int idx,
-                        unsigned int nr_blocks)
+static void clear_blocks(struct pool *pool, unsigned int pool_idx,
+                        unsigned int idx, unsigned int nr_blocks)
 {
-       blocks_iter(map, idx, nr_blocks, mask_clear);
-}
-
-static inline int __ffs(int word)
-{
-       int r = 0;
-
-       if (!(word & 0xffff)) {
-               word >>= 16;
-               r += 16;
-       }
-       if (!(word & 0xff)) {
-               word >>= 8;
-               r += 8;
-       }
-       if (!(word & 0xf)) {
-               word >>= 4;
-               r += 4;
-       }
-       if (!(word & 3)) {
-               word >>= 2;
-               r += 2;
-       }
-       if (!(word & 1)) {
-               word >>= 1;
-               r += 1;
-       }
-
-       return r;
+       blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);
 }
 
 static int find_next_zero(int word, int start)
 {
        assert(word != -1U);
        word >>= (start + 1);
-       return __ffs(~word) + start + 1;
+       return ffz(word) + start + 1;
 }
 
 static int add_pool(struct pool *pool, unsigned int alloc_size)
@@ -291,19 +273,17 @@ void scleanup(void)
                fio_mutex_remove(lock);
 }
 
+#ifdef SMALLOC_REDZONE
 static void fill_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
        unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
 
        hdr->prered = SMALLOC_PRE_RED;
        *postred = SMALLOC_POST_RED;
-#endif
 }
 
 static void sfree_check_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
        unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
 
        if (hdr->prered != SMALLOC_PRE_RED) {
@@ -318,9 +298,17 @@ static void sfree_check_redzone(struct block_hdr *hdr)
                                hdr, *postred, SMALLOC_POST_RED);
                assert(0);
        }
-#endif
+}
+#else
+static void fill_redzone(struct block_hdr *hdr)
+{
 }
 
+static void sfree_check_redzone(struct block_hdr *hdr)
+{
+}
+#endif
+
 static void sfree_pool(struct pool *pool, void *ptr)
 {
        struct block_hdr *hdr;
@@ -342,7 +330,7 @@ static void sfree_pool(struct pool *pool, void *ptr)
        idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
 
        pool_lock(pool);
-       clear_blocks(&pool->bitmap[i], idx, size_to_blocks(hdr->size));
+       clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
        if (i < pool->next_non_full)
                pool->next_non_full = i;
        pool->free_blocks += size_to_blocks(hdr->size);
@@ -400,7 +388,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
                }
 
                idx = find_next_zero(pool->bitmap[i], last_idx);
-               if (!blocks_free(&pool->bitmap[i], idx, nr_blocks)) {
+               if (!blocks_free(pool, i, idx, nr_blocks)) {
                        idx += nr_blocks;
                        if (idx < SMALLOC_BPI)
                                last_idx = idx;
@@ -413,7 +401,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
                        }
                        continue;
                }
-               set_blocks(&pool->bitmap[i], idx, nr_blocks);
+               set_blocks(pool, i, idx, nr_blocks);
                offset = i * SMALLOC_BPL + idx * SMALLOC_BPB;
                break;
        }
@@ -429,26 +417,24 @@ fail:
 
 static void *smalloc_pool(struct pool *pool, unsigned int size)
 {
-       struct block_hdr *hdr;
-       unsigned int alloc_size;
+       unsigned int alloc_size = size + sizeof(struct block_hdr);
        void *ptr;
 
-       alloc_size = size + sizeof(*hdr);
 #ifdef SMALLOC_REDZONE
        alloc_size += sizeof(unsigned int);
 #endif
 
        ptr = __smalloc_pool(pool, alloc_size);
-       if (!ptr)
-               return NULL;
+       if (ptr) {
+               struct block_hdr *hdr = ptr;
 
-       hdr = ptr;
-       hdr->size = alloc_size;
-       ptr += sizeof(*hdr);
+               hdr->size = alloc_size;
+               fill_redzone(hdr);
 
-       fill_redzone(hdr);
+               ptr += sizeof(*hdr);
+               memset(ptr, 0, size);
+       }
 
-       memset(ptr, 0, size);
        return ptr;
 }