X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=smalloc.c;h=d0b6f1e1f0059fb266aac3334d078f36715fbacd;hp=8eb0f2a03740faa63c8e3b0b5681bd6ecaa3ec87;hb=f681d0bac9c42e5d6bcb3601d2f3cfaa1c2cefb4;hpb=3f1959101ed6a935ff3e2aec0fe7f5828f23fbcb diff --git a/smalloc.c b/smalloc.c index 8eb0f2a0..d0b6f1e1 100644 --- a/smalloc.c +++ b/smalloc.c @@ -8,11 +8,14 @@ #include #include #include +#include #include #include +#include +#include "mutex.h" #include "arch/arch.h" -#include "spinlock.h" +#include "os/os.h" #define SMALLOC_REDZONE /* define to detect memory corruption */ @@ -20,23 +23,22 @@ #define SMALLOC_BPI (sizeof(unsigned int) * 8) #define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) -#define INITIAL_SIZE 1024*1024 /* new pool size */ -#define MAX_POOLS 4 /* maximum number of pools to setup */ +#define INITIAL_SIZE 8192*1024 /* new pool size */ +#define MAX_POOLS 128 /* maximum number of pools to setup */ #define SMALLOC_PRE_RED 0xdeadbeefU #define SMALLOC_POST_RED 0x5aa55aa5U unsigned int smalloc_pool_size = INITIAL_SIZE; +const int int_mask = sizeof(int) - 1; struct pool { - struct fio_spinlock *lock; + struct fio_mutex *lock; /* protects this pool */ void *map; /* map of blocks */ unsigned int *bitmap; /* blocks free/busy map */ unsigned int free_blocks; /* free blocks */ unsigned int nr_blocks; /* total blocks */ unsigned int next_non_full; - int fd; /* memory backing fd */ - char file[PATH_MAX]; /* filename for fd */ unsigned int mmap_size; }; @@ -50,36 +52,36 @@ struct block_hdr { static struct pool mp[MAX_POOLS]; static unsigned int nr_pools; static unsigned int last_pool; -static struct fio_spinlock *slock; +static struct fio_mutex *lock; static inline void pool_lock(struct pool *pool) { - fio_spin_lock(pool->lock); + fio_mutex_down(pool->lock); } static inline void pool_unlock(struct pool *pool) { - fio_spin_unlock(pool->lock); + fio_mutex_up(pool->lock); } static inline void global_read_lock(void) { - fio_spin_lock(slock); + fio_mutex_down_read(lock); } static inline void global_read_unlock(void) { - fio_spin_unlock(slock); + fio_mutex_up_read(lock); } static inline void global_write_lock(void) { - fio_spin_lock(slock); + fio_mutex_down_write(lock); } static inline void global_write_unlock(void) { - fio_spin_unlock(slock); + fio_mutex_up_write(lock); } static inline int ptr_valid(struct pool *pool, void *ptr) @@ -170,19 +172,14 @@ static void clear_blocks(struct pool *pool, unsigned int pool_idx, static int find_next_zero(int word, int start) { assert(word != -1U); - word >>= (start + 1); - return ffz(word) + start + 1; + word >>= start; + return ffz(word) + start; } static int add_pool(struct pool *pool, unsigned int alloc_size) { + int bitmap_blocks; void *ptr; - int fd, bitmap_blocks; - - strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX"); - fd = mkstemp(pool->file); - if (fd < 0) - goto out_close; #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); @@ -196,39 +193,29 @@ static int add_pool(struct pool *pool, unsigned int alloc_size) bitmap_blocks = alloc_size / SMALLOC_BPL; alloc_size += bitmap_blocks * sizeof(unsigned int); pool->mmap_size = alloc_size; - + pool->nr_blocks = bitmap_blocks; pool->free_blocks = bitmap_blocks * SMALLOC_BPB; - if (ftruncate(fd, alloc_size) < 0) - goto out_unlink; - - ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, + MAP_SHARED | OS_MAP_ANON, -1, 0); if (ptr == MAP_FAILED) - goto out_unlink; + goto out_fail; memset(ptr, 0, alloc_size); pool->map = ptr; pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL); - pool->lock = fio_spinlock_init(); + pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); if (!pool->lock) - goto out_unlink; - - pool->fd = fd; + goto out_fail; - global_write_lock(); nr_pools++; - global_write_unlock(); return 0; -out_unlink: +out_fail: fprintf(stderr, "smalloc: failed adding pool\n"); if (pool->map) munmap(pool->map, pool->mmap_size); - unlink(pool->file); -out_close: - if (fd >= 0) - close(fd); return 1; } @@ -236,18 +223,21 @@ void sinit(void) { int ret; - slock = fio_spinlock_init(); - assert(slock); + lock = fio_mutex_rw_init(); ret = add_pool(&mp[0], INITIAL_SIZE); assert(!ret); } static void cleanup_pool(struct pool *pool) { - unlink(pool->file); - close(pool->fd); + /* + * This will also remove the temporary file we used as a backing + * store, it was already unlinked + */ munmap(pool->map, pool->mmap_size); - fio_spinlock_remove(pool->lock); + + if (pool->lock) + fio_mutex_remove(pool->lock); } void scleanup(void) @@ -257,13 +247,24 @@ void scleanup(void) for (i = 0; i < nr_pools; i++) cleanup_pool(&mp[i]); - fio_spinlock_remove(slock); + if (lock) + fio_mutex_remove(lock); } #ifdef SMALLOC_REDZONE +static void *postred_ptr(struct block_hdr *hdr) +{ + uintptr_t ptr; + + ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int); + ptr = (ptr + int_mask) & ~int_mask; + + return (void *) ptr; +} + static void fill_redzone(struct block_hdr *hdr) { - unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int); + unsigned int *postred = postred_ptr(hdr); hdr->prered = SMALLOC_PRE_RED; *postred = SMALLOC_POST_RED; @@ -271,7 +272,7 @@ static void fill_redzone(struct block_hdr *hdr) static void sfree_check_redzone(struct block_hdr *hdr) { - unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int); + unsigned int *postred = postred_ptr(hdr); if (hdr->prered != SMALLOC_PRE_RED) { fprintf(stderr, "smalloc pre redzone destroyed!\n"); @@ -407,8 +408,13 @@ static void *smalloc_pool(struct pool *pool, unsigned int size) unsigned int alloc_size = size + sizeof(struct block_hdr); void *ptr; + /* + * Round to int alignment, so that the postred pointer will + * be naturally aligned as well. + */ #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); + alloc_size = (alloc_size + int_mask) & ~int_mask; #endif ptr = __smalloc_pool(pool, alloc_size); @@ -429,7 +435,7 @@ void *smalloc(unsigned int size) { unsigned int i; - global_read_lock(); + global_write_lock(); i = last_pool; do { @@ -438,7 +444,7 @@ void *smalloc(unsigned int size) if (ptr) { last_pool = i; - global_read_unlock(); + global_write_unlock(); return ptr; } } @@ -451,15 +457,13 @@ void *smalloc(unsigned int size) break; else { i = nr_pools; - global_read_unlock(); if (add_pool(&mp[nr_pools], size)) goto out; - global_read_lock(); } } while (1); - global_read_unlock(); out: + global_write_unlock(); return NULL; }