X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=smalloc.c;h=d0b6f1e1f0059fb266aac3334d078f36715fbacd;hp=3af39f3636c59b79a32c6c9ff40deb7fe0b18253;hb=f681d0bac9c42e5d6bcb3601d2f3cfaa1c2cefb4;hpb=ec996e9c8f3baacababec55b3be746e035099c14

diff --git a/smalloc.c b/smalloc.c
index 3af39f36..d0b6f1e1 100644
--- a/smalloc.c
+++ b/smalloc.c
@@ -8,25 +8,29 @@
 #include <assert.h>
 #include <string.h>
 #include <unistd.h>
+#include <inttypes.h>
 #include <sys/types.h>
 #include <limits.h>
+#include <fcntl.h>
 
 #include "mutex.h"
+#include "arch/arch.h"
+#include "os/os.h"
 
-#undef MP_SAFE			/* define to make thread safe */
 #define SMALLOC_REDZONE		/* define to detect memory corruption */
 
 #define SMALLOC_BPB	32	/* block size, bytes-per-bit in bitmap */
 #define SMALLOC_BPI	(sizeof(unsigned int) * 8)
 #define SMALLOC_BPL	(SMALLOC_BPB * SMALLOC_BPI)
 
-#define INITIAL_SIZE	1024*1024	/* new pool size */
-#define MAX_POOLS	4		/* maximum number of pools to setup */
+#define INITIAL_SIZE	8192*1024	/* new pool size */
+#define MAX_POOLS	128		/* maximum number of pools to setup */
 
 #define SMALLOC_PRE_RED		0xdeadbeefU
 #define SMALLOC_POST_RED	0x5aa55aa5U
 
 unsigned int smalloc_pool_size = INITIAL_SIZE;
+const int int_mask = sizeof(int) - 1;
 
 struct pool {
 	struct fio_mutex *lock;			/* protects this pool */
@@ -35,8 +39,6 @@ struct pool {
 	unsigned int free_blocks;		/* free blocks */
 	unsigned int nr_blocks;			/* total blocks */
 	unsigned int next_non_full;
-	int fd;					/* memory backing fd */
-	char file[PATH_MAX];			/* filename for fd */
 	unsigned int mmap_size;
 };
 
@@ -54,38 +56,32 @@ static struct fio_mutex *lock;
 
 static inline void pool_lock(struct pool *pool)
 {
-	if (pool->lock)
-		fio_mutex_down(pool->lock);
+	fio_mutex_down(pool->lock);
 }
 
 static inline void pool_unlock(struct pool *pool)
 {
-	if (pool->lock)
-		fio_mutex_up(pool->lock);
+	fio_mutex_up(pool->lock);
 }
 
 static inline void global_read_lock(void)
 {
-	if (lock)
-		fio_mutex_down_read(lock);
+	fio_mutex_down_read(lock);
 }
 
 static inline void global_read_unlock(void)
 {
-	if (lock)
-		fio_mutex_up_read(lock);
+	fio_mutex_up_read(lock);
 }
 
 static inline void global_write_lock(void)
 {
-	if (lock)
-		fio_mutex_down_write(lock);
+	fio_mutex_down_write(lock);
 }
 
 static inline void global_write_unlock(void)
 {
-	if (lock)
-		fio_mutex_up_write(lock);
+	fio_mutex_up_write(lock);
 }
 
 static inline int ptr_valid(struct pool *pool, void *ptr)
@@ -95,12 +91,24 @@ static inline int ptr_valid(struct pool *pool, void *ptr)
 	return (ptr >= pool->map) && (ptr < pool->map + pool_size);
 }
 
-static int blocks_iter(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks,
+static inline unsigned int size_to_blocks(unsigned int size)
+{
+	return (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
+}
+
+static int blocks_iter(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, unsigned int nr_blocks,
 		       int (*func)(unsigned int *map, unsigned int mask))
 {
+
 	while (nr_blocks) {
 		unsigned int this_blocks, mask;
+		unsigned int *map;
+
+		if (pool_idx >= pool->nr_blocks)
+			return 0;
+
+		map = &pool->bitmap[pool_idx];
 
 		this_blocks = nr_blocks;
 		if (this_blocks + idx > SMALLOC_BPI) {
@@ -118,11 +126,10 @@ static int blocks_iter(unsigned int *map, unsigned int idx,
 
 		nr_blocks -= this_blocks;
 		idx = 0;
-		map++;
+		pool_idx++;
 	}
 
 	return 1;
-
 }
 
 static int mask_cmp(unsigned int *map, unsigned int mask)
@@ -132,80 +139,47 @@ static int mask_cmp(unsigned int *map, unsigned int mask)
 
 static int mask_clear(unsigned int *map, unsigned int mask)
 {
+	assert((*map & mask) == mask);
 	*map &= ~mask;
 	return 1;
 }
 
 static int mask_set(unsigned int *map, unsigned int mask)
 {
+	assert(!(*map & mask));
 	*map |= mask;
 	return 1;
 }
 
-static int blocks_free(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static int blocks_free(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, unsigned int nr_blocks)
 {
-	return blocks_iter(map, idx, nr_blocks, mask_cmp);
+	return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);
 }
 
-static void set_blocks(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static void set_blocks(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, unsigned int nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_set);
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);
 }
 
-static void clear_blocks(unsigned int *map, unsigned int idx,
-			 unsigned int nr_blocks)
+static void clear_blocks(struct pool *pool, unsigned int pool_idx,
+			 unsigned int idx, unsigned int nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_clear);
-}
-
-static inline int __ffs(int word)
-{
-	int r = 0;
-
-	if (!(word & 0xffff)) {
-		word >>= 16;
-		r += 16;
-	}
-	if (!(word & 0xff)) {
-		word >>= 8;
-		r += 8;
-	}
-	if (!(word & 0xf)) {
-		word >>= 4;
-		r += 4;
-	}
-	if (!(word & 3)) {
-		word >>= 2;
-		r += 2;
-	}
-	if (!(word & 1)) {
-		word >>= 1;
-		r += 1;
-	}
-
-	return r;
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);
 }
 
 static int find_next_zero(int word, int start)
 {
 	assert(word != -1U);
-	word >>= (start + 1);
-	return __ffs(~word) + start + 1;
+	word >>= start;
+	return ffz(word) + start;
 }
 
 static int add_pool(struct pool *pool, unsigned int alloc_size)
 {
+	int bitmap_blocks;
 	void *ptr;
-	int fd, bitmap_blocks;
-
-	printf("add pool %u\n", alloc_size);
-
-	strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
-	fd = mkstemp(pool->file);
-	if (fd < 0)
-		goto out_close;
 
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
@@ -219,41 +193,29 @@ static int add_pool(struct pool *pool, unsigned int alloc_size)
 	bitmap_blocks = alloc_size / SMALLOC_BPL;
 	alloc_size += bitmap_blocks * sizeof(unsigned int);
 	pool->mmap_size = alloc_size;
-	
+
 	pool->nr_blocks = bitmap_blocks;
 	pool->free_blocks = bitmap_blocks * SMALLOC_BPB;
 
-	if (ftruncate(fd, alloc_size) < 0)
-		goto out_unlink;
-
-	ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+	ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE,
+			MAP_SHARED | OS_MAP_ANON, -1, 0);
 	if (ptr == MAP_FAILED)
-		goto out_unlink;
+		goto out_fail;
 
 	memset(ptr, 0, alloc_size);
 	pool->map = ptr;
 	pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);
 
-#ifdef MP_SAFE
-	pool->lock = fio_mutex_init(1);
+	pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
 	if (!pool->lock)
-		goto out_unlink;
-#endif
-
-	pool->fd = fd;
+		goto out_fail;
 
-	global_write_lock();
 	nr_pools++;
-	global_write_unlock();
 	return 0;
-out_unlink:
+out_fail:
 	fprintf(stderr, "smalloc: failed adding pool\n");
 	if (pool->map)
 		munmap(pool->map, pool->mmap_size);
-	unlink(pool->file);
-out_close:
-	if (fd >= 0)
-		close(fd);
 	return 1;
 }
 
@@ -261,17 +223,17 @@ void sinit(void)
 {
 	int ret;
 
-#ifdef MP_SAFE
 	lock = fio_mutex_rw_init();
-#endif
 	ret = add_pool(&mp[0], INITIAL_SIZE);
 	assert(!ret);
 }
 
 static void cleanup_pool(struct pool *pool)
 {
-	unlink(pool->file);
-	close(pool->fd);
+	/*
+	 * This will also remove the temporary file we used as a backing
+	 * store, it was already unlinked
+	 */
 	munmap(pool->map, pool->mmap_size);
 
 	if (pool->lock)
@@ -289,20 +251,28 @@ void scleanup(void)
 		fio_mutex_remove(lock);
 }
 
+#ifdef SMALLOC_REDZONE
+static void *postred_ptr(struct block_hdr *hdr)
+{
+	uintptr_t ptr;
+
+	ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int);
+	ptr = (ptr + int_mask) & ~int_mask;
+
+	return (void *) ptr;
+}
+
 static void fill_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
-	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+	unsigned int *postred = postred_ptr(hdr);
 
 	hdr->prered = SMALLOC_PRE_RED;
 	*postred = SMALLOC_POST_RED;
-#endif
 }
 
 static void sfree_check_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
-	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+	unsigned int *postred = postred_ptr(hdr);
 
 	if (hdr->prered != SMALLOC_PRE_RED) {
 		fprintf(stderr, "smalloc pre redzone destroyed!\n");
@@ -316,13 +286,21 @@ static void sfree_check_redzone(struct block_hdr *hdr)
 				hdr, *postred, SMALLOC_POST_RED);
 		assert(0);
 	}
-#endif
+}
+#else
+static void fill_redzone(struct block_hdr *hdr)
+{
 }
 
+static void sfree_check_redzone(struct block_hdr *hdr)
+{
+}
+#endif
+
 static void sfree_pool(struct pool *pool, void *ptr)
 {
 	struct block_hdr *hdr;
-	unsigned int nr_blocks, i, idx;
+	unsigned int i, idx;
 	unsigned long offset;
 
 	if (!ptr)
@@ -333,7 +311,6 @@ static void sfree_pool(struct pool *pool, void *ptr)
 
 	assert(ptr_valid(pool, ptr));
 
-	nr_blocks = (hdr->size + SMALLOC_BPB - 1) / SMALLOC_BPB;
 	sfree_check_redzone(hdr);
 
 	offset = ptr - pool->map;
@@ -341,10 +318,10 @@ static void sfree_pool(struct pool *pool, void *ptr)
 	idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
 
 	pool_lock(pool);
-	clear_blocks(&pool->bitmap[i], idx, nr_blocks);
+	clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
 	if (i < pool->next_non_full)
 		pool->next_non_full = i;
-	pool->free_blocks += nr_blocks;
+	pool->free_blocks += size_to_blocks(hdr->size);
 	pool_unlock(pool);
 }
 
@@ -379,9 +356,9 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 	unsigned int last_idx;
 	void *ret = NULL;
 
-	nr_blocks = (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
-
 	pool_lock(pool);
+
+	nr_blocks = size_to_blocks(size);
 	if (nr_blocks > pool->free_blocks)
 		goto fail;
 
@@ -399,7 +376,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 		}
 
 		idx = find_next_zero(pool->bitmap[i], last_idx);
-		if (!blocks_free(&pool->bitmap[i], idx, nr_blocks)) {
+		if (!blocks_free(pool, i, idx, nr_blocks)) {
 			idx += nr_blocks;
 			if (idx < SMALLOC_BPI)
 				last_idx = idx;
@@ -412,7 +389,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 			}
 			continue;
 		}
-		set_blocks(&pool->bitmap[i], idx, nr_blocks);
+		set_blocks(pool, i, idx, nr_blocks);
 		offset = i * SMALLOC_BPL + idx * SMALLOC_BPB;
 		break;
 	}
@@ -428,28 +405,29 @@ fail:
 
 static void *smalloc_pool(struct pool *pool, unsigned int size)
 {
-	struct block_hdr *hdr;
-	unsigned int alloc_size;
+	unsigned int alloc_size = size + sizeof(struct block_hdr);
 	void *ptr;
 
-	alloc_size = size + sizeof(*hdr);
+	/*
+	 * Round to int alignment, so that the postred pointer will
+	 * be naturally aligned as well.
+	 */
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
+	alloc_size = (alloc_size + int_mask) & ~int_mask;
 #endif
 
 	ptr = __smalloc_pool(pool, alloc_size);
-	if (!ptr) {
-		printf("failed allocating %u\n", alloc_size);
-		return NULL;
-	}
+	if (ptr) {
+		struct block_hdr *hdr = ptr;
 
-	hdr = ptr;
-	hdr->size = alloc_size;
-	ptr += sizeof(*hdr);
+		hdr->size = alloc_size;
+		fill_redzone(hdr);
 
-	fill_redzone(hdr);
+		ptr += sizeof(*hdr);
+		memset(ptr, 0, size);
+	}
 
-	memset(ptr, 0, size);
 	return ptr;
 }
 
@@ -457,7 +435,7 @@ void *smalloc(unsigned int size)
 {
 	unsigned int i;
 
-	global_read_lock();
+	global_write_lock();
 	i = last_pool;
 
 	do {
@@ -466,7 +444,7 @@ void *smalloc(unsigned int size)
 
 			if (ptr) {
 				last_pool = i;
-				global_read_unlock();
+				global_write_unlock();
 				return ptr;
 			}
 		}
@@ -479,15 +457,13 @@ void *smalloc(unsigned int size)
 			break;
 		else {
 			i = nr_pools;
-			global_read_unlock();
 			if (add_pool(&mp[nr_pools], size))
 				goto out;
-			global_read_lock();
 		}
 	} while (1);
 
-	global_read_unlock();
 out:
+	global_write_unlock();
 	return NULL;
 }