X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=smalloc.c;h=7f3c10b137498d01f812e4807298b2f2b06e6870;hb=182ec6ee4e4d79fc5a705cd2e2811fa003add040;hp=c03c4040b847358ff827ddf044f13f7349ce7409;hpb=076b17cea9c9d6c86b926a166ef58f02334c9bdc;p=fio.git diff --git a/smalloc.c b/smalloc.c index c03c4040..7f3c10b1 100644 --- a/smalloc.c +++ b/smalloc.c @@ -12,6 +12,7 @@ #include #include "mutex.h" +#include "arch/arch.h" #define MP_SAFE /* define to make thread safe */ #define SMALLOC_REDZONE /* define to detect memory corruption */ @@ -21,7 +22,7 @@ #define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) #define INITIAL_SIZE 1024*1024 /* new pool size */ -#define MAX_POOLS 4 /* maximum number of pools to setup */ +#define MAX_POOLS 128 /* maximum number of pools to setup */ #define SMALLOC_PRE_RED 0xdeadbeefU #define SMALLOC_POST_RED 0x5aa55aa5U @@ -100,12 +101,19 @@ static inline unsigned int size_to_blocks(unsigned int size) return (size + SMALLOC_BPB - 1) / SMALLOC_BPB; } -static int blocks_iter(unsigned int *map, unsigned int idx, - unsigned int nr_blocks, +static int blocks_iter(struct pool *pool, unsigned int pool_idx, + unsigned int idx, unsigned int nr_blocks, int (*func)(unsigned int *map, unsigned int mask)) { + while (nr_blocks) { unsigned int this_blocks, mask; + unsigned int *map; + + if (pool_idx >= pool->nr_blocks) + return 0; + + map = &pool->bitmap[pool_idx]; this_blocks = nr_blocks; if (this_blocks + idx > SMALLOC_BPI) { @@ -123,7 +131,7 @@ static int blocks_iter(unsigned int *map, unsigned int idx, nr_blocks -= this_blocks; idx = 0; - map++; + pool_idx++; } return 1; @@ -136,67 +144,41 @@ static int mask_cmp(unsigned int *map, unsigned int mask) static int mask_clear(unsigned int *map, unsigned int mask) { + assert((*map & mask) == mask); *map &= ~mask; return 1; } static int mask_set(unsigned int *map, unsigned int mask) { + assert(!(*map & mask)); *map |= mask; return 1; } -static int blocks_free(unsigned int *map, unsigned int idx, - unsigned int nr_blocks) +static int blocks_free(struct pool *pool, unsigned int pool_idx, + unsigned int idx, unsigned int nr_blocks) { - return blocks_iter(map, idx, nr_blocks, mask_cmp); + return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp); } -static void set_blocks(unsigned int *map, unsigned int idx, - unsigned int nr_blocks) +static void set_blocks(struct pool *pool, unsigned int pool_idx, + unsigned int idx, unsigned int nr_blocks) { - blocks_iter(map, idx, nr_blocks, mask_set); + blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set); } -static void clear_blocks(unsigned int *map, unsigned int idx, - unsigned int nr_blocks) +static void clear_blocks(struct pool *pool, unsigned int pool_idx, + unsigned int idx, unsigned int nr_blocks) { - blocks_iter(map, idx, nr_blocks, mask_clear); -} - -static inline int __ffs(int word) -{ - int r = 0; - - if (!(word & 0xffff)) { - word >>= 16; - r += 16; - } - if (!(word & 0xff)) { - word >>= 8; - r += 8; - } - if (!(word & 0xf)) { - word >>= 4; - r += 4; - } - if (!(word & 3)) { - word >>= 2; - r += 2; - } - if (!(word & 1)) { - word >>= 1; - r += 1; - } - - return r; + blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear); } static int find_next_zero(int word, int start) { assert(word != -1U); word >>= (start + 1); - return __ffs(~word) + start + 1; + return ffz(word) + start + 1; } static int add_pool(struct pool *pool, unsigned int alloc_size) @@ -244,9 +226,7 @@ static int add_pool(struct pool *pool, unsigned int alloc_size) pool->fd = fd; - global_write_lock(); nr_pools++; - global_write_unlock(); return 0; out_unlink: fprintf(stderr, "smalloc: failed adding pool\n"); @@ -291,19 +271,17 @@ void scleanup(void) fio_mutex_remove(lock); } +#ifdef SMALLOC_REDZONE static void fill_redzone(struct block_hdr *hdr) { -#ifdef SMALLOC_REDZONE unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int); hdr->prered = SMALLOC_PRE_RED; *postred = SMALLOC_POST_RED; -#endif } static void sfree_check_redzone(struct block_hdr *hdr) { -#ifdef SMALLOC_REDZONE unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int); if (hdr->prered != SMALLOC_PRE_RED) { @@ -318,8 +296,16 @@ static void sfree_check_redzone(struct block_hdr *hdr) hdr, *postred, SMALLOC_POST_RED); assert(0); } -#endif } +#else +static void fill_redzone(struct block_hdr *hdr) +{ +} + +static void sfree_check_redzone(struct block_hdr *hdr) +{ +} +#endif static void sfree_pool(struct pool *pool, void *ptr) { @@ -342,7 +328,7 @@ static void sfree_pool(struct pool *pool, void *ptr) idx = (offset % SMALLOC_BPL) / SMALLOC_BPB; pool_lock(pool); - clear_blocks(&pool->bitmap[i], idx, size_to_blocks(hdr->size)); + clear_blocks(pool, i, idx, size_to_blocks(hdr->size)); if (i < pool->next_non_full) pool->next_non_full = i; pool->free_blocks += size_to_blocks(hdr->size); @@ -400,7 +386,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size) } idx = find_next_zero(pool->bitmap[i], last_idx); - if (!blocks_free(&pool->bitmap[i], idx, nr_blocks)) { + if (!blocks_free(pool, i, idx, nr_blocks)) { idx += nr_blocks; if (idx < SMALLOC_BPI) last_idx = idx; @@ -413,7 +399,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size) } continue; } - set_blocks(&pool->bitmap[i], idx, nr_blocks); + set_blocks(pool, i, idx, nr_blocks); offset = i * SMALLOC_BPL + idx * SMALLOC_BPB; break; } @@ -429,26 +415,24 @@ fail: static void *smalloc_pool(struct pool *pool, unsigned int size) { - struct block_hdr *hdr; - unsigned int alloc_size; + unsigned int alloc_size = size + sizeof(struct block_hdr); void *ptr; - alloc_size = size + sizeof(*hdr); #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); #endif ptr = __smalloc_pool(pool, alloc_size); - if (!ptr) - return NULL; + if (ptr) { + struct block_hdr *hdr = ptr; - hdr = ptr; - hdr->size = alloc_size; - ptr += sizeof(*hdr); + hdr->size = alloc_size; + fill_redzone(hdr); - fill_redzone(hdr); + ptr += sizeof(*hdr); + memset(ptr, 0, size); + } - memset(ptr, 0, size); return ptr; } @@ -456,7 +440,7 @@ void *smalloc(unsigned int size) { unsigned int i; - global_read_lock(); + global_write_lock(); i = last_pool; do { @@ -465,7 +449,7 @@ void *smalloc(unsigned int size) if (ptr) { last_pool = i; - global_read_unlock(); + global_write_unlock(); return ptr; } } @@ -478,15 +462,13 @@ void *smalloc(unsigned int size) break; else { i = nr_pools; - global_read_unlock(); if (add_pool(&mp[nr_pools], size)) goto out; - global_read_lock(); } } while (1); - global_read_unlock(); out: + global_write_unlock(); return NULL; }