X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=smalloc.c;h=0a86c43b0f17d58a5aa69587f92eff1d4f39c279;hb=bac39e0e8807a1d52863ad8304e67221df2bc63b;hp=3af39f3636c59b79a32c6c9ff40deb7fe0b18253;hpb=ec996e9c8f3baacababec55b3be746e035099c14;p=fio.git

diff --git a/smalloc.c b/smalloc.c
index 3af39f36..0a86c43b 100644
--- a/smalloc.c
+++ b/smalloc.c
@@ -12,8 +12,9 @@
 #include <limits.h>
 
 #include "mutex.h"
+#include "arch/arch.h"
 
-#undef MP_SAFE			/* define to make thread safe */
+#define MP_SAFE			/* define to make thread safe */
 #define SMALLOC_REDZONE		/* define to detect memory corruption */
 
 #define SMALLOC_BPB	32	/* block size, bytes-per-bit in bitmap */
@@ -95,12 +96,24 @@ static inline int ptr_valid(struct pool *pool, void *ptr)
 	return (ptr >= pool->map) && (ptr < pool->map + pool_size);
 }
 
-static int blocks_iter(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks,
+static inline unsigned int size_to_blocks(unsigned int size)
+{
+	return (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
+}
+
+static int blocks_iter(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, unsigned int nr_blocks,
 		       int (*func)(unsigned int *map, unsigned int mask))
 {
+
 	while (nr_blocks) {
 		unsigned int this_blocks, mask;
+		unsigned int *map;
+
+		if (pool_idx >= pool->nr_blocks)
+			return 0;
+
+		map = &pool->bitmap[pool_idx];
 
 		this_blocks = nr_blocks;
 		if (this_blocks + idx > SMALLOC_BPI) {
@@ -118,11 +131,10 @@ static int blocks_iter(unsigned int *map, unsigned int idx,
 
 		nr_blocks -= this_blocks;
 		idx = 0;
-		map++;
+		pool_idx++;
 	}
 
 	return 1;
-
 }
 
 static int mask_cmp(unsigned int *map, unsigned int mask)
@@ -132,67 +144,41 @@ static int mask_cmp(unsigned int *map, unsigned int mask)
 
 static int mask_clear(unsigned int *map, unsigned int mask)
 {
+	assert((*map & mask) == mask);
 	*map &= ~mask;
 	return 1;
 }
 
 static int mask_set(unsigned int *map, unsigned int mask)
 {
+	assert(!(*map & mask));
 	*map |= mask;
 	return 1;
 }
 
-static int blocks_free(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static int blocks_free(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, unsigned int nr_blocks)
 {
-	return blocks_iter(map, idx, nr_blocks, mask_cmp);
+	return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);
 }
 
-static void set_blocks(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static void set_blocks(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, unsigned int nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_set);
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);
 }
 
-static void clear_blocks(unsigned int *map, unsigned int idx,
-			 unsigned int nr_blocks)
+static void clear_blocks(struct pool *pool, unsigned int pool_idx,
+			 unsigned int idx, unsigned int nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_clear);
-}
-
-static inline int __ffs(int word)
-{
-	int r = 0;
-
-	if (!(word & 0xffff)) {
-		word >>= 16;
-		r += 16;
-	}
-	if (!(word & 0xff)) {
-		word >>= 8;
-		r += 8;
-	}
-	if (!(word & 0xf)) {
-		word >>= 4;
-		r += 4;
-	}
-	if (!(word & 3)) {
-		word >>= 2;
-		r += 2;
-	}
-	if (!(word & 1)) {
-		word >>= 1;
-		r += 1;
-	}
-
-	return r;
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);
 }
 
 static int find_next_zero(int word, int start)
 {
 	assert(word != -1U);
 	word >>= (start + 1);
-	return __ffs(~word) + start + 1;
+	return ffz(word) + start + 1;
 }
 
 static int add_pool(struct pool *pool, unsigned int alloc_size)
@@ -200,8 +186,6 @@ static int add_pool(struct pool *pool, unsigned int alloc_size)
 	void *ptr;
 	int fd, bitmap_blocks;
 
-	printf("add pool %u\n", alloc_size);
-
 	strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
 	fd = mkstemp(pool->file);
 	if (fd < 0)
@@ -289,19 +273,17 @@ void scleanup(void)
 		fio_mutex_remove(lock);
 }
 
+#ifdef SMALLOC_REDZONE
 static void fill_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
 	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
 
 	hdr->prered = SMALLOC_PRE_RED;
 	*postred = SMALLOC_POST_RED;
-#endif
 }
 
 static void sfree_check_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
 	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
 
 	if (hdr->prered != SMALLOC_PRE_RED) {
@@ -316,13 +298,21 @@ static void sfree_check_redzone(struct block_hdr *hdr)
 				hdr, *postred, SMALLOC_POST_RED);
 		assert(0);
 	}
-#endif
+}
+#else
+static void fill_redzone(struct block_hdr *hdr)
+{
 }
 
+static void sfree_check_redzone(struct block_hdr *hdr)
+{
+}
+#endif
+
 static void sfree_pool(struct pool *pool, void *ptr)
 {
 	struct block_hdr *hdr;
-	unsigned int nr_blocks, i, idx;
+	unsigned int i, idx;
 	unsigned long offset;
 
 	if (!ptr)
@@ -333,7 +323,6 @@ static void sfree_pool(struct pool *pool, void *ptr)
 
 	assert(ptr_valid(pool, ptr));
 
-	nr_blocks = (hdr->size + SMALLOC_BPB - 1) / SMALLOC_BPB;
 	sfree_check_redzone(hdr);
 
 	offset = ptr - pool->map;
@@ -341,10 +330,10 @@ static void sfree_pool(struct pool *pool, void *ptr)
 	idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
 
 	pool_lock(pool);
-	clear_blocks(&pool->bitmap[i], idx, nr_blocks);
+	clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
 	if (i < pool->next_non_full)
 		pool->next_non_full = i;
-	pool->free_blocks += nr_blocks;
+	pool->free_blocks += size_to_blocks(hdr->size);
 	pool_unlock(pool);
 }
 
@@ -379,9 +368,9 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 	unsigned int last_idx;
 	void *ret = NULL;
 
-	nr_blocks = (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
-
 	pool_lock(pool);
+
+	nr_blocks = size_to_blocks(size);
 	if (nr_blocks > pool->free_blocks)
 		goto fail;
 
@@ -399,7 +388,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 		}
 
 		idx = find_next_zero(pool->bitmap[i], last_idx);
-		if (!blocks_free(&pool->bitmap[i], idx, nr_blocks)) {
+		if (!blocks_free(pool, i, idx, nr_blocks)) {
 			idx += nr_blocks;
 			if (idx < SMALLOC_BPI)
 				last_idx = idx;
@@ -412,7 +401,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 			}
 			continue;
 		}
-		set_blocks(&pool->bitmap[i], idx, nr_blocks);
+		set_blocks(pool, i, idx, nr_blocks);
 		offset = i * SMALLOC_BPL + idx * SMALLOC_BPB;
 		break;
 	}
@@ -428,28 +417,24 @@ fail:
 
 static void *smalloc_pool(struct pool *pool, unsigned int size)
 {
-	struct block_hdr *hdr;
-	unsigned int alloc_size;
+	unsigned int alloc_size = size + sizeof(struct block_hdr);
 	void *ptr;
 
-	alloc_size = size + sizeof(*hdr);
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
 #endif
 
 	ptr = __smalloc_pool(pool, alloc_size);
-	if (!ptr) {
-		printf("failed allocating %u\n", alloc_size);
-		return NULL;
-	}
+	if (ptr) {
+		struct block_hdr *hdr = ptr;
 
-	hdr = ptr;
-	hdr->size = alloc_size;
-	ptr += sizeof(*hdr);
+		hdr->size = alloc_size;
+		fill_redzone(hdr);
 
-	fill_redzone(hdr);
+		ptr += sizeof(*hdr);
+		memset(ptr, 0, size);
+	}
 
-	memset(ptr, 0, size);
 	return ptr;
 }