X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=smalloc.c;h=fa00f0ee3325b25fa6bdc5abe6c3bf405026a743;hp=1e72068541bac82449eb9f13dc3b2cd790247768;hb=ab02be41807ec9451c47b17129cf61457ef21db6;hpb=b8a6582e89999f88c574b905b89743762d8080df diff --git a/smalloc.c b/smalloc.c index 1e720685..fa00f0ee 100644 --- a/smalloc.c +++ b/smalloc.c @@ -3,90 +3,60 @@ * that can be shared across processes and threads */ #include -#include -#include #include #include -#include -#include -#include -#include "mutex.h" -#include "arch/arch.h" +#include "fio.h" +#include "fio_sem.h" +#include "os/os.h" +#include "smalloc.h" +#include "log.h" -#define MP_SAFE /* define to make thread safe */ #define SMALLOC_REDZONE /* define to detect memory corruption */ #define SMALLOC_BPB 32 /* block size, bytes-per-bit in bitmap */ #define SMALLOC_BPI (sizeof(unsigned int) * 8) #define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) -#define INITIAL_SIZE 1024*1024 /* new pool size */ -#define MAX_POOLS 128 /* maximum number of pools to setup */ +#define INITIAL_SIZE 16*1024*1024 /* new pool size */ +#define INITIAL_POOLS 8 /* maximum number of pools to setup */ + +#define MAX_POOLS 16 #define SMALLOC_PRE_RED 0xdeadbeefU #define SMALLOC_POST_RED 0x5aa55aa5U unsigned int smalloc_pool_size = INITIAL_SIZE; +#ifdef SMALLOC_REDZONE +static const int int_mask = sizeof(int) - 1; +#endif struct pool { - struct fio_mutex *lock; /* protects this pool */ + struct fio_sem *lock; /* protects this pool */ void *map; /* map of blocks */ unsigned int *bitmap; /* blocks free/busy map */ - unsigned int free_blocks; /* free blocks */ - unsigned int nr_blocks; /* total blocks */ - unsigned int next_non_full; - int fd; /* memory backing fd */ - unsigned int mmap_size; + size_t free_blocks; /* free blocks */ + size_t nr_blocks; /* total blocks */ + size_t next_non_full; + size_t mmap_size; }; struct block_hdr { - unsigned int size; + size_t size; #ifdef SMALLOC_REDZONE unsigned int prered; #endif }; -static struct pool mp[MAX_POOLS]; +/* + * This suppresses the voluminous potential bitmap printout when + * smalloc encounters an OOM error + */ +static const bool enable_smalloc_debug = false; + +static struct pool *mp; static unsigned int nr_pools; static unsigned int last_pool; -static struct fio_mutex *lock; - -static inline void pool_lock(struct pool *pool) -{ - if (pool->lock) - fio_mutex_down(pool->lock); -} - -static inline void pool_unlock(struct pool *pool) -{ - if (pool->lock) - fio_mutex_up(pool->lock); -} - -static inline void global_read_lock(void) -{ - if (lock) - fio_mutex_down_read(lock); -} - -static inline void global_read_unlock(void) -{ - if (lock) - fio_mutex_up_read(lock); -} - -static inline void global_write_lock(void) -{ - if (lock) - fio_mutex_down_write(lock); -} - -static inline void global_write_unlock(void) -{ - if (lock) - fio_mutex_up_write(lock); -} static inline int ptr_valid(struct pool *pool, void *ptr) { @@ -95,13 +65,13 @@ static inline int ptr_valid(struct pool *pool, void *ptr) return (ptr >= pool->map) && (ptr < pool->map + pool_size); } -static inline unsigned int size_to_blocks(unsigned int size) +static inline size_t size_to_blocks(size_t size) { return (size + SMALLOC_BPB - 1) / SMALLOC_BPB; } static int blocks_iter(struct pool *pool, unsigned int pool_idx, - unsigned int idx, unsigned int nr_blocks, + unsigned int idx, size_t nr_blocks, int (*func)(unsigned int *map, unsigned int mask)) { @@ -156,19 +126,19 @@ static int mask_set(unsigned int *map, unsigned int mask) } static int blocks_free(struct pool *pool, unsigned int pool_idx, - unsigned int idx, unsigned int nr_blocks) + unsigned int idx, size_t nr_blocks) { return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp); } static void set_blocks(struct pool *pool, unsigned int pool_idx, - unsigned int idx, unsigned int nr_blocks) + unsigned int idx, size_t nr_blocks) { blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set); } static void clear_blocks(struct pool *pool, unsigned int pool_idx, - unsigned int idx, unsigned int nr_blocks) + unsigned int idx, size_t nr_blocks) { blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear); } @@ -176,19 +146,18 @@ static void clear_blocks(struct pool *pool, unsigned int pool_idx, static int find_next_zero(int word, int start) { assert(word != -1U); - word >>= (start + 1); - return ffz(word) + start + 1; + word >>= start; + return ffz(word) + start; } -static int add_pool(struct pool *pool, unsigned int alloc_size) +static bool add_pool(struct pool *pool, unsigned int alloc_size) { - int fd, bitmap_blocks; - char file[] = "/tmp/.fio_smalloc.XXXXXX"; + int bitmap_blocks; + int mmap_flags; void *ptr; - fd = mkstemp(file); - if (fd < 0) - goto out_close; + if (nr_pools == MAX_POOLS) + return false; #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); @@ -202,56 +171,68 @@ static int add_pool(struct pool *pool, unsigned int alloc_size) bitmap_blocks = alloc_size / SMALLOC_BPL; alloc_size += bitmap_blocks * sizeof(unsigned int); pool->mmap_size = alloc_size; - + pool->nr_blocks = bitmap_blocks; - pool->free_blocks = bitmap_blocks * SMALLOC_BPB; + pool->free_blocks = bitmap_blocks * SMALLOC_BPI; - if (ftruncate(fd, alloc_size) < 0) - goto out_unlink; + mmap_flags = OS_MAP_ANON; +#ifdef CONFIG_ESX + mmap_flags |= MAP_PRIVATE; +#else + mmap_flags |= MAP_SHARED; +#endif + ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, mmap_flags, -1, 0); - ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); if (ptr == MAP_FAILED) - goto out_unlink; + goto out_fail; - memset(ptr, 0, alloc_size); pool->map = ptr; - pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL); + pool->bitmap = (unsigned int *)((char *) ptr + (pool->nr_blocks * SMALLOC_BPL)); + memset(pool->bitmap, 0, bitmap_blocks * sizeof(unsigned int)); -#ifdef MP_SAFE - pool->lock = fio_mutex_init(1); + pool->lock = fio_sem_init(FIO_SEM_UNLOCKED); if (!pool->lock) - goto out_unlink; -#endif - - /* - * Unlink pool file now. It wont get deleted until the fd is closed, - * which happens both for cleanup or unexpected quit. This way we - * don't leave temp files around in case of a crash. - */ - unlink(file); - pool->fd = fd; + goto out_fail; nr_pools++; - return 0; -out_unlink: - fprintf(stderr, "smalloc: failed adding pool\n"); + return true; +out_fail: + log_err("smalloc: failed adding pool\n"); if (pool->map) munmap(pool->map, pool->mmap_size); - unlink(file); -out_close: - close(fd); - return 1; + return false; } void sinit(void) { - int ret; + bool ret; + int i; -#ifdef MP_SAFE - lock = fio_mutex_rw_init(); -#endif - ret = add_pool(&mp[0], INITIAL_SIZE); - assert(!ret); + /* + * sinit() can be called more than once if alloc-size is + * set. But we want to allocate space for the struct pool + * instances only once. + */ + if (!mp) { + mp = (struct pool *) mmap(NULL, + MAX_POOLS * sizeof(struct pool), + PROT_READ | PROT_WRITE, + OS_MAP_ANON | MAP_SHARED, -1, 0); + + assert(mp != MAP_FAILED); + } + + for (i = 0; i < INITIAL_POOLS; i++) { + ret = add_pool(&mp[nr_pools], smalloc_pool_size); + if (!ret) + break; + } + + /* + * If we added at least one pool, we should be OK for most + * cases. + */ + assert(i); } static void cleanup_pool(struct pool *pool) @@ -260,11 +241,10 @@ static void cleanup_pool(struct pool *pool) * This will also remove the temporary file we used as a backing * store, it was already unlinked */ - close(pool->fd); munmap(pool->map, pool->mmap_size); if (pool->lock) - fio_mutex_remove(pool->lock); + fio_sem_remove(pool->lock); } void scleanup(void) @@ -274,14 +254,23 @@ void scleanup(void) for (i = 0; i < nr_pools; i++) cleanup_pool(&mp[i]); - if (lock) - fio_mutex_remove(lock); + munmap(mp, MAX_POOLS * sizeof(struct pool)); } #ifdef SMALLOC_REDZONE +static void *postred_ptr(struct block_hdr *hdr) +{ + uintptr_t ptr; + + ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int); + ptr = (uintptr_t) PTR_ALIGN(ptr, int_mask); + + return (void *) ptr; +} + static void fill_redzone(struct block_hdr *hdr) { - unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int); + unsigned int *postred = postred_ptr(hdr); hdr->prered = SMALLOC_PRE_RED; *postred = SMALLOC_POST_RED; @@ -289,17 +278,17 @@ static void fill_redzone(struct block_hdr *hdr) static void sfree_check_redzone(struct block_hdr *hdr) { - unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int); + unsigned int *postred = postred_ptr(hdr); if (hdr->prered != SMALLOC_PRE_RED) { - fprintf(stderr, "smalloc pre redzone destroyed!\n"); - fprintf(stderr, " ptr=%p, prered=%x, expected %x\n", + log_err("smalloc pre redzone destroyed!\n" + " ptr=%p, prered=%x, expected %x\n", hdr, hdr->prered, SMALLOC_PRE_RED); assert(0); } if (*postred != SMALLOC_POST_RED) { - fprintf(stderr, "smalloc post redzone destroyed!\n"); - fprintf(stderr, " ptr=%p, postred=%x, expected %x\n", + log_err("smalloc post redzone destroyed!\n" + " ptr=%p, postred=%x, expected %x\n", hdr, *postred, SMALLOC_POST_RED); assert(0); } @@ -334,12 +323,12 @@ static void sfree_pool(struct pool *pool, void *ptr) i = offset / SMALLOC_BPL; idx = (offset % SMALLOC_BPL) / SMALLOC_BPB; - pool_lock(pool); + fio_sem_down(pool->lock); clear_blocks(pool, i, idx, size_to_blocks(hdr->size)); if (i < pool->next_non_full) pool->next_non_full = i; pool->free_blocks += size_to_blocks(hdr->size); - pool_unlock(pool); + fio_sem_up(pool->lock); } void sfree(void *ptr) @@ -350,8 +339,6 @@ void sfree(void *ptr) if (!ptr) return; - global_read_lock(); - for (i = 0; i < nr_pools; i++) { if (ptr_valid(&mp[i], ptr)) { pool = &mp[i]; @@ -359,35 +346,57 @@ void sfree(void *ptr) } } - global_read_unlock(); + if (pool) { + sfree_pool(pool, ptr); + return; + } - assert(pool); - sfree_pool(pool, ptr); + log_err("smalloc: ptr %p not from smalloc pool\n", ptr); } -static void *__smalloc_pool(struct pool *pool, unsigned int size) +static unsigned int find_best_index(struct pool *pool) { - unsigned int nr_blocks; + unsigned int i; + + assert(pool->free_blocks); + + for (i = pool->next_non_full; pool->bitmap[i] == -1U; i++) { + if (i == pool->nr_blocks - 1) { + unsigned int j; + + for (j = 0; j < pool->nr_blocks; j++) + if (pool->bitmap[j] != -1U) + return j; + } + } + + return i; +} + +static void *__smalloc_pool(struct pool *pool, size_t size) +{ + size_t nr_blocks; unsigned int i; unsigned int offset; unsigned int last_idx; void *ret = NULL; - pool_lock(pool); + fio_sem_down(pool->lock); nr_blocks = size_to_blocks(size); if (nr_blocks > pool->free_blocks) goto fail; - i = pool->next_non_full; + pool->next_non_full = find_best_index(pool); + last_idx = 0; offset = -1U; + i = pool->next_non_full; while (i < pool->nr_blocks) { unsigned int idx; if (pool->bitmap[i] == -1U) { i++; - pool->next_non_full = i; last_idx = 0; continue; } @@ -416,19 +425,31 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size) ret = pool->map + offset; } fail: - pool_unlock(pool); + fio_sem_up(pool->lock); return ret; } -static void *smalloc_pool(struct pool *pool, unsigned int size) +static size_t size_to_alloc_size(size_t size) { - unsigned int alloc_size = size + sizeof(struct block_hdr); - void *ptr; + size_t alloc_size = size + sizeof(struct block_hdr); + /* + * Round to int alignment, so that the postred pointer will + * be naturally aligned as well. + */ #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); + alloc_size = (alloc_size + int_mask) & ~int_mask; #endif + return alloc_size; +} + +static void *smalloc_pool(struct pool *pool, size_t size) +{ + size_t alloc_size = size_to_alloc_size(size); + void *ptr; + ptr = __smalloc_pool(pool, alloc_size); if (ptr) { struct block_hdr *hdr = ptr; @@ -443,47 +464,117 @@ static void *smalloc_pool(struct pool *pool, unsigned int size) return ptr; } -void *smalloc(unsigned int size) +static void smalloc_print_bitmap(struct pool *pool) +{ + size_t nr_blocks = pool->nr_blocks; + unsigned int *bitmap = pool->bitmap; + unsigned int i, j; + char *buffer; + + if (!enable_smalloc_debug) + return; + + buffer = malloc(SMALLOC_BPI + 1); + if (!buffer) + return; + buffer[SMALLOC_BPI] = '\0'; + + for (i = 0; i < nr_blocks; i++) { + unsigned int line = bitmap[i]; + + /* skip completely full lines */ + if (line == -1U) + continue; + + for (j = 0; j < SMALLOC_BPI; j++) + if ((1 << j) & line) + buffer[SMALLOC_BPI-1-j] = '1'; + else + buffer[SMALLOC_BPI-1-j] = '0'; + + log_err("smalloc: bitmap %5u, %s\n", i, buffer); + } + + free(buffer); +} + +void smalloc_debug(size_t size) { unsigned int i; + size_t alloc_size = size_to_alloc_size(size); + size_t alloc_blocks; + + alloc_blocks = size_to_blocks(alloc_size); + + if (size) + log_err("smalloc: size = %lu, alloc_size = %lu, blocks = %lu\n", + (unsigned long) size, (unsigned long) alloc_size, + (unsigned long) alloc_blocks); + for (i = 0; i < nr_pools; i++) { + log_err("smalloc: pool %u, free/total blocks %u/%u\n", i, + (unsigned int) (mp[i].free_blocks), + (unsigned int) (mp[i].nr_blocks*sizeof(unsigned int)*8)); + if (size && mp[i].free_blocks >= alloc_blocks) { + void *ptr = smalloc_pool(&mp[i], size); + if (ptr) { + sfree(ptr); + last_pool = i; + log_err("smalloc: smalloc_pool %u succeeded\n", i); + } else { + log_err("smalloc: smalloc_pool %u failed\n", i); + log_err("smalloc: next_non_full=%u, nr_blocks=%u\n", + (unsigned int) mp[i].next_non_full, (unsigned int) mp[i].nr_blocks); + smalloc_print_bitmap(&mp[i]); + } + } + } +} + +void *smalloc(size_t size) +{ + unsigned int i, end_pool; + + if (size != (unsigned int) size) + return NULL; - global_write_lock(); i = last_pool; + end_pool = nr_pools; do { - for (; i < nr_pools; i++) { + for (; i < end_pool; i++) { void *ptr = smalloc_pool(&mp[i], size); if (ptr) { last_pool = i; - global_write_unlock(); return ptr; } } if (last_pool) { - last_pool = 0; + end_pool = last_pool; + last_pool = i = 0; continue; } - if (nr_pools + 1 > MAX_POOLS) - break; - else { - i = nr_pools; - if (add_pool(&mp[nr_pools], size)) - goto out; - } + break; } while (1); -out: - global_write_unlock(); + log_err("smalloc: OOM. Consider using --alloc-size to increase the " + "shared memory available.\n"); + smalloc_debug(size); return NULL; } +void *scalloc(size_t nmemb, size_t size) +{ + return smalloc(nmemb * size); +} + char *smalloc_strdup(const char *str) { - char *ptr; + char *ptr = NULL; ptr = smalloc(strlen(str) + 1); - strcpy(ptr, str); + if (ptr) + strcpy(ptr, str); return ptr; }