smalloc: enlarge pool size

[fio.git] / smalloc.c

diff --git a/smalloc.c b/smalloc.c
index 6cbf59cf320a0eb0fcf1bf9166f6b581bfe0c983..3dd1b4960dd9f15186399432800412fa1c45255d 100644 (file)
--- a/smalloc.c
+++ b/smalloc.c
@@ -12,22 +12,22 @@
 #include <limits.h>
 
 #include "mutex.h"
-#include "lib/ffz.h"
+#include "arch/arch.h"
 
-#define MP_SAFE                        /* define to make thread safe */
 #define SMALLOC_REDZONE                /* define to detect memory corruption */
 
 #define SMALLOC_BPB    32      /* block size, bytes-per-bit in bitmap */
 #define SMALLOC_BPI    (sizeof(unsigned int) * 8)
 #define SMALLOC_BPL    (SMALLOC_BPB * SMALLOC_BPI)
 
-#define INITIAL_SIZE   1024*1024       /* new pool size */
-#define MAX_POOLS      4               /* maximum number of pools to setup */
+#define INITIAL_SIZE   8192*1024       /* new pool size */
+#define MAX_POOLS      128             /* maximum number of pools to setup */
 
 #define SMALLOC_PRE_RED                0xdeadbeefU
 #define SMALLOC_POST_RED       0x5aa55aa5U
 
 unsigned int smalloc_pool_size = INITIAL_SIZE;
+const int int_mask = sizeof(int) - 1;
 
 struct pool {
        struct fio_mutex *lock;                 /* protects this pool */
@@ -37,7 +37,6 @@ struct pool {
        unsigned int nr_blocks;                 /* total blocks */
        unsigned int next_non_full;
        int fd;                                 /* memory backing fd */
-       char file[PATH_MAX];                    /* filename for fd */
        unsigned int mmap_size;
 };
 
@@ -55,38 +54,32 @@ static struct fio_mutex *lock;
 
 static inline void pool_lock(struct pool *pool)
 {
-       if (pool->lock)
-               fio_mutex_down(pool->lock);
+       fio_mutex_down(pool->lock);
 }
 
 static inline void pool_unlock(struct pool *pool)
 {
-       if (pool->lock)
-               fio_mutex_up(pool->lock);
+       fio_mutex_up(pool->lock);
 }
 
 static inline void global_read_lock(void)
 {
-       if (lock)
-               fio_mutex_down_read(lock);
+       fio_mutex_down_read(lock);
 }
 
 static inline void global_read_unlock(void)
 {
-       if (lock)
-               fio_mutex_up_read(lock);
+       fio_mutex_up_read(lock);
 }
 
 static inline void global_write_lock(void)
 {
-       if (lock)
-               fio_mutex_down_write(lock);
+       fio_mutex_down_write(lock);
 }
 
 static inline void global_write_unlock(void)
 {
-       if (lock)
-               fio_mutex_up_write(lock);
+       fio_mutex_up_write(lock);
 }
 
 static inline int ptr_valid(struct pool *pool, void *ptr)
@@ -178,16 +171,16 @@ static int find_next_zero(int word, int start)
 {
        assert(word != -1U);
        word >>= (start + 1);
-       return __ffs(~word) + start + 1;
+       return ffz(word) + start + 1;
 }
 
 static int add_pool(struct pool *pool, unsigned int alloc_size)
 {
-       void *ptr;
        int fd, bitmap_blocks;
+       char file[] = "/tmp/.fio_smalloc.XXXXXX";
+       void *ptr;
 
-       strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
-       fd = mkstemp(pool->file);
+       fd = mkstemp(file);
        if (fd < 0)
                goto out_close;
 
@@ -218,26 +211,27 @@ static int add_pool(struct pool *pool, unsigned int alloc_size)
        pool->map = ptr;
        pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);
 
-#ifdef MP_SAFE
        pool->lock = fio_mutex_init(1);
        if (!pool->lock)
                goto out_unlink;
-#endif
 
+       /*
+        * Unlink pool file now. It wont get deleted until the fd is closed,
+        * which happens both for cleanup or unexpected quit. This way we
+        * don't leave temp files around in case of a crash.
+        */
+       unlink(file);
        pool->fd = fd;
 
-       global_write_lock();
        nr_pools++;
-       global_write_unlock();
        return 0;
 out_unlink:
        fprintf(stderr, "smalloc: failed adding pool\n");
        if (pool->map)
                munmap(pool->map, pool->mmap_size);
-       unlink(pool->file);
+       unlink(file);
 out_close:
-       if (fd >= 0)
-               close(fd);
+       close(fd);
        return 1;
 }
 
@@ -245,16 +239,17 @@ void sinit(void)
 {
        int ret;
 
-#ifdef MP_SAFE
        lock = fio_mutex_rw_init();
-#endif
        ret = add_pool(&mp[0], INITIAL_SIZE);
        assert(!ret);
 }
 
 static void cleanup_pool(struct pool *pool)
 {
-       unlink(pool->file);
+       /*
+        * This will also remove the temporary file we used as a backing
+        * store, it was already unlinked
+        */
        close(pool->fd);
        munmap(pool->map, pool->mmap_size);
 
@@ -274,9 +269,19 @@ void scleanup(void)
 }
 
 #ifdef SMALLOC_REDZONE
+static void *postred_ptr(struct block_hdr *hdr)
+{
+       unsigned long ptr;
+
+       ptr = (unsigned long) hdr + hdr->size - sizeof(unsigned int);
+       ptr = (ptr + int_mask) & ~int_mask;
+
+       return (void *) ptr;
+}
+
 static void fill_redzone(struct block_hdr *hdr)
 {
-       unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+       unsigned int *postred = postred_ptr(hdr);
 
        hdr->prered = SMALLOC_PRE_RED;
        *postred = SMALLOC_POST_RED;
@@ -284,7 +289,7 @@ static void fill_redzone(struct block_hdr *hdr)
 
 static void sfree_check_redzone(struct block_hdr *hdr)
 {
-       unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+       unsigned int *postred = postred_ptr(hdr);
 
        if (hdr->prered != SMALLOC_PRE_RED) {
                fprintf(stderr, "smalloc pre redzone destroyed!\n");
@@ -420,8 +425,13 @@ static void *smalloc_pool(struct pool *pool, unsigned int size)
        unsigned int alloc_size = size + sizeof(struct block_hdr);
        void *ptr;
 
+       /*
+        * Round to int alignment, so that the postred pointer will
+        * be naturally aligned as well.
+        */
 #ifdef SMALLOC_REDZONE
        alloc_size += sizeof(unsigned int);
+       alloc_size = (alloc_size + int_mask) & ~int_mask;
 #endif
 
        ptr = __smalloc_pool(pool, alloc_size);
@@ -442,7 +452,7 @@ void *smalloc(unsigned int size)
 {
        unsigned int i;
 
-       global_read_lock();
+       global_write_lock();
        i = last_pool;
 
        do {
@@ -451,7 +461,7 @@ void *smalloc(unsigned int size)
 
                        if (ptr) {
                                last_pool = i;
-                               global_read_unlock();
+                               global_write_unlock();
                                return ptr;
                        }
                }
@@ -464,15 +474,13 @@ void *smalloc(unsigned int size)
                        break;
                else {
                        i = nr_pools;
-                       global_read_unlock();
                        if (add_pool(&mp[nr_pools], size))
                                goto out;
-                       global_read_lock();
                }
        } while (1);
 
-       global_read_unlock();
 out:
+       global_write_unlock();
        return NULL;
 }