X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=smalloc.c;h=5dae7e7c9b79928c2ef69aa792ac71277536fea4;hp=3d3986b84cb2aeef3a89465c7dbb1a6239c88f82;hb=b7c1279471861606b4a03fa676ca589986d1c678;hpb=808e9ea8dd3dc3d631a7982fa0c4b01547cb811a

diff --git a/smalloc.c b/smalloc.c
index 3d3986b8..5dae7e7c 100644
--- a/smalloc.c
+++ b/smalloc.c
@@ -8,40 +8,42 @@
 #include <assert.h>
 #include <string.h>
 #include <unistd.h>
+#include <inttypes.h>
 #include <sys/types.h>
 #include <limits.h>
+#include <fcntl.h>
 
 #include "mutex.h"
+#include "arch/arch.h"
+#include "os/os.h"
 
-#define MP_SAFE			/* define to make thread safe */
 #define SMALLOC_REDZONE		/* define to detect memory corruption */
 
 #define SMALLOC_BPB	32	/* block size, bytes-per-bit in bitmap */
 #define SMALLOC_BPI	(sizeof(unsigned int) * 8)
 #define SMALLOC_BPL	(SMALLOC_BPB * SMALLOC_BPI)
 
-#define INITIAL_SIZE	1024*1024	/* new pool size */
-#define MAX_POOLS	4		/* maximum number of pools to setup */
+#define INITIAL_SIZE	8192*1024	/* new pool size */
+#define MAX_POOLS	128		/* maximum number of pools to setup */
 
 #define SMALLOC_PRE_RED		0xdeadbeefU
 #define SMALLOC_POST_RED	0x5aa55aa5U
 
 unsigned int smalloc_pool_size = INITIAL_SIZE;
+const int int_mask = sizeof(int) - 1;
 
 struct pool {
 	struct fio_mutex *lock;			/* protects this pool */
 	void *map;				/* map of blocks */
 	unsigned int *bitmap;			/* blocks free/busy map */
-	unsigned int free_blocks;		/* free blocks */
-	unsigned int nr_blocks;			/* total blocks */
-	unsigned int next_non_full;
-	int fd;					/* memory backing fd */
-	char file[PATH_MAX];			/* filename for fd */
-	unsigned int mmap_size;
+	size_t free_blocks;		/* free blocks */
+	size_t nr_blocks;			/* total blocks */
+	size_t next_non_full;
+	size_t mmap_size;
 };
 
 struct block_hdr {
-	unsigned int size;
+	size_t size;
 #ifdef SMALLOC_REDZONE
 	unsigned int prered;
 #endif
@@ -50,42 +52,36 @@ struct block_hdr {
 static struct pool mp[MAX_POOLS];
 static unsigned int nr_pools;
 static unsigned int last_pool;
-static struct fio_mutex *lock;
+static struct fio_rwlock *lock;
 
 static inline void pool_lock(struct pool *pool)
 {
-	if (pool->lock)
-		fio_mutex_down(pool->lock);
+	fio_mutex_down(pool->lock);
 }
 
 static inline void pool_unlock(struct pool *pool)
 {
-	if (pool->lock)
-		fio_mutex_up(pool->lock);
+	fio_mutex_up(pool->lock);
 }
 
 static inline void global_read_lock(void)
 {
-	if (lock)
-		fio_mutex_down_read(lock);
+	fio_rwlock_read(lock);
 }
 
 static inline void global_read_unlock(void)
 {
-	if (lock)
-		fio_mutex_up_read(lock);
+	fio_rwlock_unlock(lock);
 }
 
 static inline void global_write_lock(void)
 {
-	if (lock)
-		fio_mutex_down_write(lock);
+	fio_rwlock_write(lock);
 }
 
 static inline void global_write_unlock(void)
 {
-	if (lock)
-		fio_mutex_up_write(lock);
+	fio_rwlock_unlock(lock);
 }
 
 static inline int ptr_valid(struct pool *pool, void *ptr)
@@ -95,17 +91,24 @@ static inline int ptr_valid(struct pool *pool, void *ptr)
 	return (ptr >= pool->map) && (ptr < pool->map + pool_size);
 }
 
-static inline unsigned int size_to_blocks(unsigned int size)
+static inline size_t size_to_blocks(size_t size)
 {
 	return (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
 }
 
-static int blocks_iter(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks,
+static int blocks_iter(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, size_t nr_blocks,
 		       int (*func)(unsigned int *map, unsigned int mask))
 {
+
 	while (nr_blocks) {
 		unsigned int this_blocks, mask;
+		unsigned int *map;
+
+		if (pool_idx >= pool->nr_blocks)
+			return 0;
+
+		map = &pool->bitmap[pool_idx];
 
 		this_blocks = nr_blocks;
 		if (this_blocks + idx > SMALLOC_BPI) {
@@ -123,7 +126,7 @@ static int blocks_iter(unsigned int *map, unsigned int idx,
 
 		nr_blocks -= this_blocks;
 		idx = 0;
-		map++;
+		pool_idx++;
 	}
 
 	return 1;
@@ -136,78 +139,47 @@ static int mask_cmp(unsigned int *map, unsigned int mask)
 
 static int mask_clear(unsigned int *map, unsigned int mask)
 {
+	assert((*map & mask) == mask);
 	*map &= ~mask;
 	return 1;
 }
 
 static int mask_set(unsigned int *map, unsigned int mask)
 {
+	assert(!(*map & mask));
 	*map |= mask;
 	return 1;
 }
 
-static int blocks_free(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static int blocks_free(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, size_t nr_blocks)
 {
-	return blocks_iter(map, idx, nr_blocks, mask_cmp);
+	return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);
 }
 
-static void set_blocks(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static void set_blocks(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, size_t nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_set);
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);
 }
 
-static void clear_blocks(unsigned int *map, unsigned int idx,
-			 unsigned int nr_blocks)
+static void clear_blocks(struct pool *pool, unsigned int pool_idx,
+			 unsigned int idx, size_t nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_clear);
-}
-
-static inline int __ffs(int word)
-{
-	int r = 0;
-
-	if (!(word & 0xffff)) {
-		word >>= 16;
-		r += 16;
-	}
-	if (!(word & 0xff)) {
-		word >>= 8;
-		r += 8;
-	}
-	if (!(word & 0xf)) {
-		word >>= 4;
-		r += 4;
-	}
-	if (!(word & 3)) {
-		word >>= 2;
-		r += 2;
-	}
-	if (!(word & 1)) {
-		word >>= 1;
-		r += 1;
-	}
-
-	return r;
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);
 }
 
 static int find_next_zero(int word, int start)
 {
 	assert(word != -1U);
-	word >>= (start + 1);
-	return __ffs(~word) + start + 1;
+	word >>= start;
+	return ffz(word) + start;
 }
 
 static int add_pool(struct pool *pool, unsigned int alloc_size)
 {
+	int bitmap_blocks;
 	void *ptr;
-	int fd, bitmap_blocks;
-
-	strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
-	fd = mkstemp(pool->file);
-	if (fd < 0)
-		goto out_close;
 
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
@@ -221,41 +193,29 @@ static int add_pool(struct pool *pool, unsigned int alloc_size)
 	bitmap_blocks = alloc_size / SMALLOC_BPL;
 	alloc_size += bitmap_blocks * sizeof(unsigned int);
 	pool->mmap_size = alloc_size;
-	
+
 	pool->nr_blocks = bitmap_blocks;
 	pool->free_blocks = bitmap_blocks * SMALLOC_BPB;
 
-	if (ftruncate(fd, alloc_size) < 0)
-		goto out_unlink;
-
-	ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+	ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE,
+			MAP_SHARED | OS_MAP_ANON, -1, 0);
 	if (ptr == MAP_FAILED)
-		goto out_unlink;
+		goto out_fail;
 
 	memset(ptr, 0, alloc_size);
 	pool->map = ptr;
 	pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);
 
-#ifdef MP_SAFE
-	pool->lock = fio_mutex_init(1);
+	pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
 	if (!pool->lock)
-		goto out_unlink;
-#endif
+		goto out_fail;
 
-	pool->fd = fd;
-
-	global_write_lock();
 	nr_pools++;
-	global_write_unlock();
 	return 0;
-out_unlink:
+out_fail:
 	fprintf(stderr, "smalloc: failed adding pool\n");
 	if (pool->map)
 		munmap(pool->map, pool->mmap_size);
-	unlink(pool->file);
-out_close:
-	if (fd >= 0)
-		close(fd);
 	return 1;
 }
 
@@ -263,17 +223,17 @@ void sinit(void)
 {
 	int ret;
 
-#ifdef MP_SAFE
-	lock = fio_mutex_rw_init();
-#endif
+	lock = fio_rwlock_init();
 	ret = add_pool(&mp[0], INITIAL_SIZE);
 	assert(!ret);
 }
 
 static void cleanup_pool(struct pool *pool)
 {
-	unlink(pool->file);
-	close(pool->fd);
+	/*
+	 * This will also remove the temporary file we used as a backing
+	 * store, it was already unlinked
+	 */
 	munmap(pool->map, pool->mmap_size);
 
 	if (pool->lock)
@@ -288,23 +248,31 @@ void scleanup(void)
 		cleanup_pool(&mp[i]);
 
 	if (lock)
-		fio_mutex_remove(lock);
+		fio_rwlock_remove(lock);
+}
+
+#ifdef SMALLOC_REDZONE
+static void *postred_ptr(struct block_hdr *hdr)
+{
+	uintptr_t ptr;
+
+	ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int);
+	ptr = (ptr + int_mask) & ~int_mask;
+
+	return (void *) ptr;
 }
 
 static void fill_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
-	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+	unsigned int *postred = postred_ptr(hdr);
 
 	hdr->prered = SMALLOC_PRE_RED;
 	*postred = SMALLOC_POST_RED;
-#endif
 }
 
 static void sfree_check_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
-	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+	unsigned int *postred = postred_ptr(hdr);
 
 	if (hdr->prered != SMALLOC_PRE_RED) {
 		fprintf(stderr, "smalloc pre redzone destroyed!\n");
@@ -318,8 +286,16 @@ static void sfree_check_redzone(struct block_hdr *hdr)
 				hdr, *postred, SMALLOC_POST_RED);
 		assert(0);
 	}
-#endif
 }
+#else
+static void fill_redzone(struct block_hdr *hdr)
+{
+}
+
+static void sfree_check_redzone(struct block_hdr *hdr)
+{
+}
+#endif
 
 static void sfree_pool(struct pool *pool, void *ptr)
 {
@@ -342,7 +318,7 @@ static void sfree_pool(struct pool *pool, void *ptr)
 	idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
 
 	pool_lock(pool);
-	clear_blocks(&pool->bitmap[i], idx, size_to_blocks(hdr->size));
+	clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
 	if (i < pool->next_non_full)
 		pool->next_non_full = i;
 	pool->free_blocks += size_to_blocks(hdr->size);
@@ -372,9 +348,9 @@ void sfree(void *ptr)
 	sfree_pool(pool, ptr);
 }
 
-static void *__smalloc_pool(struct pool *pool, unsigned int size)
+static void *__smalloc_pool(struct pool *pool, size_t size)
 {
-	unsigned int nr_blocks;
+	size_t nr_blocks;
 	unsigned int i;
 	unsigned int offset;
 	unsigned int last_idx;
@@ -400,7 +376,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 		}
 
 		idx = find_next_zero(pool->bitmap[i], last_idx);
-		if (!blocks_free(&pool->bitmap[i], idx, nr_blocks)) {
+		if (!blocks_free(pool, i, idx, nr_blocks)) {
 			idx += nr_blocks;
 			if (idx < SMALLOC_BPI)
 				last_idx = idx;
@@ -413,7 +389,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 			}
 			continue;
 		}
-		set_blocks(&pool->bitmap[i], idx, nr_blocks);
+		set_blocks(pool, i, idx, nr_blocks);
 		offset = i * SMALLOC_BPL + idx * SMALLOC_BPB;
 		break;
 	}
@@ -427,38 +403,42 @@ fail:
 	return ret;
 }
 
-static void *smalloc_pool(struct pool *pool, unsigned int size)
+static void *smalloc_pool(struct pool *pool, size_t size)
 {
-	struct block_hdr *hdr;
-	unsigned int alloc_size;
+	size_t alloc_size = size + sizeof(struct block_hdr);
 	void *ptr;
 
-	alloc_size = size + sizeof(*hdr);
+	/*
+	 * Round to int alignment, so that the postred pointer will
+	 * be naturally aligned as well.
+	 */
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
+	alloc_size = (alloc_size + int_mask) & ~int_mask;
 #endif
 
 	ptr = __smalloc_pool(pool, alloc_size);
-	if (!ptr) {
-		printf("failed allocating %u\n", alloc_size);
-		return NULL;
-	}
+	if (ptr) {
+		struct block_hdr *hdr = ptr;
 
-	hdr = ptr;
-	hdr->size = alloc_size;
-	ptr += sizeof(*hdr);
+		hdr->size = alloc_size;
+		fill_redzone(hdr);
 
-	fill_redzone(hdr);
+		ptr += sizeof(*hdr);
+		memset(ptr, 0, size);
+	}
 
-	memset(ptr, 0, size);
 	return ptr;
 }
 
-void *smalloc(unsigned int size)
+void *smalloc(size_t size)
 {
 	unsigned int i;
 
-	global_read_lock();
+	if (size != (unsigned int) size)
+		return NULL;
+
+	global_write_lock();
 	i = last_pool;
 
 	do {
@@ -467,7 +447,7 @@ void *smalloc(unsigned int size)
 
 			if (ptr) {
 				last_pool = i;
-				global_read_unlock();
+				global_write_unlock();
 				return ptr;
 			}
 		}
@@ -480,15 +460,13 @@ void *smalloc(unsigned int size)
 			break;
 		else {
 			i = nr_pools;
-			global_read_unlock();
 			if (add_pool(&mp[nr_pools], size))
 				goto out;
-			global_read_lock();
 		}
 	} while (1);
 
-	global_read_unlock();
 out:
+	global_write_unlock();
 	return NULL;
 }