X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=smalloc.c;h=13995acce07f56c3b396427bb058676889148988;hp=c8f1642eb54cfc7863182286be03d03599dd4dae;hb=0ffccc21fcd67d1e1d2a360e90f3fe8efc0d6b52;hpb=25f488581e510c1b440f2e76842ec23f3dad7b57 diff --git a/smalloc.c b/smalloc.c index c8f1642e..13995acc 100644 --- a/smalloc.c +++ b/smalloc.c @@ -12,11 +12,20 @@ #include #include #include +#ifdef CONFIG_VALGRIND_DEV +#include +#else +#define RUNNING_ON_VALGRIND 0 +#define VALGRIND_MALLOCLIKE_BLOCK(addr, size, rzB, is_zeroed) do { } while (0) +#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do { } while (0) +#endif -#include "mutex.h" +#include "fio.h" +#include "fio_sem.h" #include "arch/arch.h" #include "os/os.h" #include "smalloc.h" +#include "log.h" #define SMALLOC_REDZONE /* define to detect memory corruption */ @@ -24,17 +33,21 @@ #define SMALLOC_BPI (sizeof(unsigned int) * 8) #define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) -#define INITIAL_SIZE 8192*1024 /* new pool size */ -#define MAX_POOLS 128 /* maximum number of pools to setup */ +#define INITIAL_SIZE 16*1024*1024 /* new pool size */ +#define INITIAL_POOLS 8 /* maximum number of pools to setup */ + +#define MAX_POOLS 16 #define SMALLOC_PRE_RED 0xdeadbeefU #define SMALLOC_POST_RED 0x5aa55aa5U unsigned int smalloc_pool_size = INITIAL_SIZE; +#ifdef SMALLOC_REDZONE static const int int_mask = sizeof(int) - 1; +#endif struct pool { - struct fio_mutex *lock; /* protects this pool */ + struct fio_sem *lock; /* protects this pool */ void *map; /* map of blocks */ unsigned int *bitmap; /* blocks free/busy map */ size_t free_blocks; /* free blocks */ @@ -43,6 +56,12 @@ struct pool { size_t mmap_size; }; +#ifdef SMALLOC_REDZONE +#define REDZONE_SIZE sizeof(unsigned int) +#else +#define REDZONE_SIZE 0 +#endif + struct block_hdr { size_t size; #ifdef SMALLOC_REDZONE @@ -53,37 +72,6 @@ struct block_hdr { static struct pool mp[MAX_POOLS]; static unsigned int nr_pools; static unsigned int last_pool; -static struct fio_rwlock *lock; - -static inline void pool_lock(struct pool *pool) -{ - fio_mutex_down(pool->lock); -} - -static inline void pool_unlock(struct pool *pool) -{ - fio_mutex_up(pool->lock); -} - -static inline void global_read_lock(void) -{ - fio_rwlock_read(lock); -} - -static inline void global_read_unlock(void) -{ - fio_rwlock_unlock(lock); -} - -static inline void global_write_lock(void) -{ - fio_rwlock_write(lock); -} - -static inline void global_write_unlock(void) -{ - fio_rwlock_unlock(lock); -} static inline int ptr_valid(struct pool *pool, void *ptr) { @@ -177,11 +165,15 @@ static int find_next_zero(int word, int start) return ffz(word) + start; } -static int add_pool(struct pool *pool, unsigned int alloc_size) +static bool add_pool(struct pool *pool, unsigned int alloc_size) { int bitmap_blocks; + int mmap_flags; void *ptr; + if (nr_pools == MAX_POOLS) + return false; + #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); #endif @@ -198,35 +190,50 @@ static int add_pool(struct pool *pool, unsigned int alloc_size) pool->nr_blocks = bitmap_blocks; pool->free_blocks = bitmap_blocks * SMALLOC_BPB; - ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, - MAP_SHARED | OS_MAP_ANON, -1, 0); + mmap_flags = OS_MAP_ANON; +#ifdef CONFIG_ESX + mmap_flags |= MAP_PRIVATE; +#else + mmap_flags |= MAP_SHARED; +#endif + ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, mmap_flags, -1, 0); + if (ptr == MAP_FAILED) goto out_fail; - memset(ptr, 0, alloc_size); pool->map = ptr; - pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL); + pool->bitmap = (unsigned int *)((char *) ptr + (pool->nr_blocks * SMALLOC_BPL)); + memset(pool->bitmap, 0, bitmap_blocks * sizeof(unsigned int)); - pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); + pool->lock = fio_sem_init(FIO_SEM_UNLOCKED); if (!pool->lock) goto out_fail; nr_pools++; - return 0; + return true; out_fail: - fprintf(stderr, "smalloc: failed adding pool\n"); + log_err("smalloc: failed adding pool\n"); if (pool->map) munmap(pool->map, pool->mmap_size); - return 1; + return false; } void sinit(void) { - int ret; + bool ret; + int i; - lock = fio_rwlock_init(); - ret = add_pool(&mp[0], INITIAL_SIZE); - assert(!ret); + for (i = 0; i < INITIAL_POOLS; i++) { + ret = add_pool(&mp[nr_pools], smalloc_pool_size); + if (!ret) + break; + } + + /* + * If we added at least one pool, we should be OK for most + * cases. + */ + assert(i); } static void cleanup_pool(struct pool *pool) @@ -238,7 +245,7 @@ static void cleanup_pool(struct pool *pool) munmap(pool->map, pool->mmap_size); if (pool->lock) - fio_mutex_remove(pool->lock); + fio_sem_remove(pool->lock); } void scleanup(void) @@ -247,9 +254,6 @@ void scleanup(void) for (i = 0; i < nr_pools; i++) cleanup_pool(&mp[i]); - - if (lock) - fio_rwlock_remove(lock); } #ifdef SMALLOC_REDZONE @@ -258,7 +262,7 @@ static void *postred_ptr(struct block_hdr *hdr) uintptr_t ptr; ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int); - ptr = (ptr + int_mask) & ~int_mask; + ptr = (uintptr_t) PTR_ALIGN(ptr, int_mask); return (void *) ptr; } @@ -267,6 +271,10 @@ static void fill_redzone(struct block_hdr *hdr) { unsigned int *postred = postred_ptr(hdr); + /* Let Valgrind fill the red zones. */ + if (RUNNING_ON_VALGRIND) + return; + hdr->prered = SMALLOC_PRE_RED; *postred = SMALLOC_POST_RED; } @@ -275,15 +283,19 @@ static void sfree_check_redzone(struct block_hdr *hdr) { unsigned int *postred = postred_ptr(hdr); + /* Let Valgrind check the red zones. */ + if (RUNNING_ON_VALGRIND) + return; + if (hdr->prered != SMALLOC_PRE_RED) { - fprintf(stderr, "smalloc pre redzone destroyed!\n"); - fprintf(stderr, " ptr=%p, prered=%x, expected %x\n", + log_err("smalloc pre redzone destroyed!\n" + " ptr=%p, prered=%x, expected %x\n", hdr, hdr->prered, SMALLOC_PRE_RED); assert(0); } if (*postred != SMALLOC_POST_RED) { - fprintf(stderr, "smalloc post redzone destroyed!\n"); - fprintf(stderr, " ptr=%p, postred=%x, expected %x\n", + log_err("smalloc post redzone destroyed!\n" + " ptr=%p, postred=%x, expected %x\n", hdr, *postred, SMALLOC_POST_RED); assert(0); } @@ -318,12 +330,12 @@ static void sfree_pool(struct pool *pool, void *ptr) i = offset / SMALLOC_BPL; idx = (offset % SMALLOC_BPL) / SMALLOC_BPB; - pool_lock(pool); + fio_sem_down(pool->lock); clear_blocks(pool, i, idx, size_to_blocks(hdr->size)); if (i < pool->next_non_full) pool->next_non_full = i; pool->free_blocks += size_to_blocks(hdr->size); - pool_unlock(pool); + fio_sem_up(pool->lock); } void sfree(void *ptr) @@ -334,8 +346,6 @@ void sfree(void *ptr) if (!ptr) return; - global_read_lock(); - for (i = 0; i < nr_pools; i++) { if (ptr_valid(&mp[i], ptr)) { pool = &mp[i]; @@ -343,10 +353,13 @@ void sfree(void *ptr) } } - global_read_unlock(); + if (pool) { + VALGRIND_FREELIKE_BLOCK(ptr, REDZONE_SIZE); + sfree_pool(pool, ptr); + return; + } - assert(pool); - sfree_pool(pool, ptr); + log_err("smalloc: ptr %p not from smalloc pool\n", ptr); } static void *__smalloc_pool(struct pool *pool, size_t size) @@ -357,7 +370,7 @@ static void *__smalloc_pool(struct pool *pool, size_t size) unsigned int last_idx; void *ret = NULL; - pool_lock(pool); + fio_sem_down(pool->lock); nr_blocks = size_to_blocks(size); if (nr_blocks > pool->free_blocks) @@ -400,7 +413,7 @@ static void *__smalloc_pool(struct pool *pool, size_t size) ret = pool->map + offset; } fail: - pool_unlock(pool); + fio_sem_up(pool->lock); return ret; } @@ -432,50 +445,58 @@ static void *smalloc_pool(struct pool *pool, size_t size) return ptr; } -void *smalloc(size_t size) +static void *__smalloc(size_t size, bool is_zeroed) { - unsigned int i; + unsigned int i, end_pool; if (size != (unsigned int) size) return NULL; - global_write_lock(); i = last_pool; + end_pool = nr_pools; do { - for (; i < nr_pools; i++) { + for (; i < end_pool; i++) { void *ptr = smalloc_pool(&mp[i], size); if (ptr) { last_pool = i; - global_write_unlock(); + VALGRIND_MALLOCLIKE_BLOCK(ptr, size, + REDZONE_SIZE, + is_zeroed); return ptr; } } if (last_pool) { - last_pool = 0; + end_pool = last_pool; + last_pool = i = 0; continue; } - if (nr_pools + 1 > MAX_POOLS) - break; - else { - i = nr_pools; - if (add_pool(&mp[nr_pools], size)) - goto out; - } + break; } while (1); -out: - global_write_unlock(); + log_err("smalloc: OOM. Consider using --alloc-size to increase the " + "shared memory available.\n"); return NULL; } +void *smalloc(size_t size) +{ + return __smalloc(size, false); +} + +void *scalloc(size_t nmemb, size_t size) +{ + return __smalloc(nmemb * size, true); +} + char *smalloc_strdup(const char *str) { - char *ptr; + char *ptr = NULL; ptr = smalloc(strlen(str) + 1); - strcpy(ptr, str); + if (ptr) + strcpy(ptr, str); return ptr; }