X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=smalloc.c;h=a2ad25a0a0d03bf5010c743325ed17e594e0fb83;hp=3d3986b84cb2aeef3a89465c7dbb1a6239c88f82;hb=2eca48926723d3ebe8f43d4999302fb826f4a250;hpb=808e9ea8dd3dc3d631a7982fa0c4b01547cb811a

diff --git a/smalloc.c b/smalloc.c
index 3d3986b8..a2ad25a0 100644
--- a/smalloc.c
+++ b/smalloc.c
@@ -3,45 +3,46 @@
  * that can be shared across processes and threads
  */
 #include <sys/mman.h>
-#include <stdio.h>
-#include <stdlib.h>
 #include <assert.h>
 #include <string.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <limits.h>
 
-#include "mutex.h"
+#include "fio.h"
+#include "fio_sem.h"
+#include "os/os.h"
+#include "smalloc.h"
+#include "log.h"
 
-#define MP_SAFE			/* define to make thread safe */
 #define SMALLOC_REDZONE		/* define to detect memory corruption */
 
 #define SMALLOC_BPB	32	/* block size, bytes-per-bit in bitmap */
 #define SMALLOC_BPI	(sizeof(unsigned int) * 8)
 #define SMALLOC_BPL	(SMALLOC_BPB * SMALLOC_BPI)
 
-#define INITIAL_SIZE	1024*1024	/* new pool size */
-#define MAX_POOLS	4		/* maximum number of pools to setup */
+#define INITIAL_SIZE	16*1024*1024	/* new pool size */
+#define INITIAL_POOLS	8		/* maximum number of pools to setup */
+
+#define MAX_POOLS	16
 
 #define SMALLOC_PRE_RED		0xdeadbeefU
 #define SMALLOC_POST_RED	0x5aa55aa5U
 
 unsigned int smalloc_pool_size = INITIAL_SIZE;
+#ifdef SMALLOC_REDZONE
+static const int int_mask = sizeof(int) - 1;
+#endif
 
 struct pool {
-	struct fio_mutex *lock;			/* protects this pool */
+	struct fio_sem *lock;			/* protects this pool */
 	void *map;				/* map of blocks */
 	unsigned int *bitmap;			/* blocks free/busy map */
-	unsigned int free_blocks;		/* free blocks */
-	unsigned int nr_blocks;			/* total blocks */
-	unsigned int next_non_full;
-	int fd;					/* memory backing fd */
-	char file[PATH_MAX];			/* filename for fd */
-	unsigned int mmap_size;
+	size_t free_blocks;		/* free blocks */
+	size_t nr_blocks;			/* total blocks */
+	size_t next_non_full;
+	size_t mmap_size;
 };
 
 struct block_hdr {
-	unsigned int size;
+	size_t size;
 #ifdef SMALLOC_REDZONE
 	unsigned int prered;
 #endif
@@ -50,43 +51,6 @@ struct block_hdr {
 static struct pool mp[MAX_POOLS];
 static unsigned int nr_pools;
 static unsigned int last_pool;
-static struct fio_mutex *lock;
-
-static inline void pool_lock(struct pool *pool)
-{
-	if (pool->lock)
-		fio_mutex_down(pool->lock);
-}
-
-static inline void pool_unlock(struct pool *pool)
-{
-	if (pool->lock)
-		fio_mutex_up(pool->lock);
-}
-
-static inline void global_read_lock(void)
-{
-	if (lock)
-		fio_mutex_down_read(lock);
-}
-
-static inline void global_read_unlock(void)
-{
-	if (lock)
-		fio_mutex_up_read(lock);
-}
-
-static inline void global_write_lock(void)
-{
-	if (lock)
-		fio_mutex_down_write(lock);
-}
-
-static inline void global_write_unlock(void)
-{
-	if (lock)
-		fio_mutex_up_write(lock);
-}
 
 static inline int ptr_valid(struct pool *pool, void *ptr)
 {
@@ -95,17 +59,24 @@ static inline int ptr_valid(struct pool *pool, void *ptr)
 	return (ptr >= pool->map) && (ptr < pool->map + pool_size);
 }
 
-static inline unsigned int size_to_blocks(unsigned int size)
+static inline size_t size_to_blocks(size_t size)
 {
 	return (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
 }
 
-static int blocks_iter(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks,
+static int blocks_iter(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, size_t nr_blocks,
 		       int (*func)(unsigned int *map, unsigned int mask))
 {
+
 	while (nr_blocks) {
 		unsigned int this_blocks, mask;
+		unsigned int *map;
+
+		if (pool_idx >= pool->nr_blocks)
+			return 0;
+
+		map = &pool->bitmap[pool_idx];
 
 		this_blocks = nr_blocks;
 		if (this_blocks + idx > SMALLOC_BPI) {
@@ -123,7 +94,7 @@ static int blocks_iter(unsigned int *map, unsigned int idx,
 
 		nr_blocks -= this_blocks;
 		idx = 0;
-		map++;
+		pool_idx++;
 	}
 
 	return 1;
@@ -136,78 +107,51 @@ static int mask_cmp(unsigned int *map, unsigned int mask)
 
 static int mask_clear(unsigned int *map, unsigned int mask)
 {
+	assert((*map & mask) == mask);
 	*map &= ~mask;
 	return 1;
 }
 
 static int mask_set(unsigned int *map, unsigned int mask)
 {
+	assert(!(*map & mask));
 	*map |= mask;
 	return 1;
 }
 
-static int blocks_free(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
-{
-	return blocks_iter(map, idx, nr_blocks, mask_cmp);
-}
-
-static void set_blocks(unsigned int *map, unsigned int idx,
-		       unsigned int nr_blocks)
+static int blocks_free(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, size_t nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_set);
+	return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);
 }
 
-static void clear_blocks(unsigned int *map, unsigned int idx,
-			 unsigned int nr_blocks)
+static void set_blocks(struct pool *pool, unsigned int pool_idx,
+		       unsigned int idx, size_t nr_blocks)
 {
-	blocks_iter(map, idx, nr_blocks, mask_clear);
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);
 }
 
-static inline int __ffs(int word)
+static void clear_blocks(struct pool *pool, unsigned int pool_idx,
+			 unsigned int idx, size_t nr_blocks)
 {
-	int r = 0;
-
-	if (!(word & 0xffff)) {
-		word >>= 16;
-		r += 16;
-	}
-	if (!(word & 0xff)) {
-		word >>= 8;
-		r += 8;
-	}
-	if (!(word & 0xf)) {
-		word >>= 4;
-		r += 4;
-	}
-	if (!(word & 3)) {
-		word >>= 2;
-		r += 2;
-	}
-	if (!(word & 1)) {
-		word >>= 1;
-		r += 1;
-	}
-
-	return r;
+	blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);
 }
 
 static int find_next_zero(int word, int start)
 {
 	assert(word != -1U);
-	word >>= (start + 1);
-	return __ffs(~word) + start + 1;
+	word >>= start;
+	return ffz(word) + start;
 }
 
-static int add_pool(struct pool *pool, unsigned int alloc_size)
+static bool add_pool(struct pool *pool, unsigned int alloc_size)
 {
+	int bitmap_blocks;
+	int mmap_flags;
 	void *ptr;
-	int fd, bitmap_blocks;
 
-	strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
-	fd = mkstemp(pool->file);
-	if (fd < 0)
-		goto out_close;
+	if (nr_pools == MAX_POOLS)
+		return false;
 
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
@@ -221,63 +165,66 @@ static int add_pool(struct pool *pool, unsigned int alloc_size)
 	bitmap_blocks = alloc_size / SMALLOC_BPL;
 	alloc_size += bitmap_blocks * sizeof(unsigned int);
 	pool->mmap_size = alloc_size;
-	
+
 	pool->nr_blocks = bitmap_blocks;
 	pool->free_blocks = bitmap_blocks * SMALLOC_BPB;
 
-	if (ftruncate(fd, alloc_size) < 0)
-		goto out_unlink;
+	mmap_flags = OS_MAP_ANON;
+#ifdef CONFIG_ESX
+	mmap_flags |= MAP_PRIVATE;
+#else
+	mmap_flags |= MAP_SHARED;
+#endif
+	ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, mmap_flags, -1, 0);
 
-	ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
 	if (ptr == MAP_FAILED)
-		goto out_unlink;
+		goto out_fail;
 
-	memset(ptr, 0, alloc_size);
 	pool->map = ptr;
-	pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);
+	pool->bitmap = (unsigned int *)((char *) ptr + (pool->nr_blocks * SMALLOC_BPL));
+	memset(pool->bitmap, 0, bitmap_blocks * sizeof(unsigned int));
 
-#ifdef MP_SAFE
-	pool->lock = fio_mutex_init(1);
+	pool->lock = fio_sem_init(FIO_SEM_UNLOCKED);
 	if (!pool->lock)
-		goto out_unlink;
-#endif
+		goto out_fail;
 
-	pool->fd = fd;
-
-	global_write_lock();
 	nr_pools++;
-	global_write_unlock();
-	return 0;
-out_unlink:
-	fprintf(stderr, "smalloc: failed adding pool\n");
+	return true;
+out_fail:
+	log_err("smalloc: failed adding pool\n");
 	if (pool->map)
 		munmap(pool->map, pool->mmap_size);
-	unlink(pool->file);
-out_close:
-	if (fd >= 0)
-		close(fd);
-	return 1;
+	return false;
 }
 
 void sinit(void)
 {
-	int ret;
+	bool ret;
+	int i;
 
-#ifdef MP_SAFE
-	lock = fio_mutex_rw_init();
-#endif
-	ret = add_pool(&mp[0], INITIAL_SIZE);
-	assert(!ret);
+	for (i = 0; i < INITIAL_POOLS; i++) {
+		ret = add_pool(&mp[nr_pools], smalloc_pool_size);
+		if (!ret)
+			break;
+	}
+
+	/*
+	 * If we added at least one pool, we should be OK for most
+	 * cases.
+	 */
+	assert(i);
 }
 
 static void cleanup_pool(struct pool *pool)
 {
-	unlink(pool->file);
-	close(pool->fd);
+	/*
+	 * This will also remove the temporary file we used as a backing
+	 * store, it was already unlinked
+	 */
 	munmap(pool->map, pool->mmap_size);
 
 	if (pool->lock)
-		fio_mutex_remove(pool->lock);
+		fio_sem_remove(pool->lock);
 }
 
 void scleanup(void)
@@ -286,40 +233,53 @@ void scleanup(void)
 
 	for (i = 0; i < nr_pools; i++)
 		cleanup_pool(&mp[i]);
+}
+
+#ifdef SMALLOC_REDZONE
+static void *postred_ptr(struct block_hdr *hdr)
+{
+	uintptr_t ptr;
 
-	if (lock)
-		fio_mutex_remove(lock);
+	ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int);
+	ptr = (uintptr_t) PTR_ALIGN(ptr, int_mask);
+
+	return (void *) ptr;
 }
 
 static void fill_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
-	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+	unsigned int *postred = postred_ptr(hdr);
 
 	hdr->prered = SMALLOC_PRE_RED;
 	*postred = SMALLOC_POST_RED;
-#endif
 }
 
 static void sfree_check_redzone(struct block_hdr *hdr)
 {
-#ifdef SMALLOC_REDZONE
-	unsigned int *postred = (void *) hdr + hdr->size - sizeof(unsigned int);
+	unsigned int *postred = postred_ptr(hdr);
 
 	if (hdr->prered != SMALLOC_PRE_RED) {
-		fprintf(stderr, "smalloc pre redzone destroyed!\n");
-		fprintf(stderr, "  ptr=%p, prered=%x, expected %x\n",
+		log_err("smalloc pre redzone destroyed!\n"
+			" ptr=%p, prered=%x, expected %x\n",
 				hdr, hdr->prered, SMALLOC_PRE_RED);
 		assert(0);
 	}
 	if (*postred != SMALLOC_POST_RED) {
-		fprintf(stderr, "smalloc post redzone destroyed!\n");
-		fprintf(stderr, "  ptr=%p, postred=%x, expected %x\n",
+		log_err("smalloc post redzone destroyed!\n"
+			"  ptr=%p, postred=%x, expected %x\n",
 				hdr, *postred, SMALLOC_POST_RED);
 		assert(0);
 	}
-#endif
 }
+#else
+static void fill_redzone(struct block_hdr *hdr)
+{
+}
+
+static void sfree_check_redzone(struct block_hdr *hdr)
+{
+}
+#endif
 
 static void sfree_pool(struct pool *pool, void *ptr)
 {
@@ -341,12 +301,12 @@ static void sfree_pool(struct pool *pool, void *ptr)
 	i = offset / SMALLOC_BPL;
 	idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
 
-	pool_lock(pool);
-	clear_blocks(&pool->bitmap[i], idx, size_to_blocks(hdr->size));
+	fio_sem_down(pool->lock);
+	clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
 	if (i < pool->next_non_full)
 		pool->next_non_full = i;
 	pool->free_blocks += size_to_blocks(hdr->size);
-	pool_unlock(pool);
+	fio_sem_up(pool->lock);
 }
 
 void sfree(void *ptr)
@@ -357,8 +317,6 @@ void sfree(void *ptr)
 	if (!ptr)
 		return;
 
-	global_read_lock();
-
 	for (i = 0; i < nr_pools; i++) {
 		if (ptr_valid(&mp[i], ptr)) {
 			pool = &mp[i];
@@ -366,21 +324,23 @@ void sfree(void *ptr)
 		}
 	}
 
-	global_read_unlock();
+	if (pool) {
+		sfree_pool(pool, ptr);
+		return;
+	}
 
-	assert(pool);
-	sfree_pool(pool, ptr);
+	log_err("smalloc: ptr %p not from smalloc pool\n", ptr);
 }
 
-static void *__smalloc_pool(struct pool *pool, unsigned int size)
+static void *__smalloc_pool(struct pool *pool, size_t size)
 {
-	unsigned int nr_blocks;
+	size_t nr_blocks;
 	unsigned int i;
 	unsigned int offset;
 	unsigned int last_idx;
 	void *ret = NULL;
 
-	pool_lock(pool);
+	fio_sem_down(pool->lock);
 
 	nr_blocks = size_to_blocks(size);
 	if (nr_blocks > pool->free_blocks)
@@ -400,7 +360,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 		}
 
 		idx = find_next_zero(pool->bitmap[i], last_idx);
-		if (!blocks_free(&pool->bitmap[i], idx, nr_blocks)) {
+		if (!blocks_free(pool, i, idx, nr_blocks)) {
 			idx += nr_blocks;
 			if (idx < SMALLOC_BPI)
 				last_idx = idx;
@@ -413,7 +373,7 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 			}
 			continue;
 		}
-		set_blocks(&pool->bitmap[i], idx, nr_blocks);
+		set_blocks(pool, i, idx, nr_blocks);
 		offset = i * SMALLOC_BPL + idx * SMALLOC_BPB;
 		break;
 	}
@@ -423,80 +383,82 @@ static void *__smalloc_pool(struct pool *pool, unsigned int size)
 		ret = pool->map + offset;
 	}
 fail:
-	pool_unlock(pool);
+	fio_sem_up(pool->lock);
 	return ret;
 }
 
-static void *smalloc_pool(struct pool *pool, unsigned int size)
+static void *smalloc_pool(struct pool *pool, size_t size)
 {
-	struct block_hdr *hdr;
-	unsigned int alloc_size;
+	size_t alloc_size = size + sizeof(struct block_hdr);
 	void *ptr;
 
-	alloc_size = size + sizeof(*hdr);
+	/*
+	 * Round to int alignment, so that the postred pointer will
+	 * be naturally aligned as well.
+	 */
 #ifdef SMALLOC_REDZONE
 	alloc_size += sizeof(unsigned int);
+	alloc_size = (alloc_size + int_mask) & ~int_mask;
 #endif
 
 	ptr = __smalloc_pool(pool, alloc_size);
-	if (!ptr) {
-		printf("failed allocating %u\n", alloc_size);
-		return NULL;
-	}
+	if (ptr) {
+		struct block_hdr *hdr = ptr;
 
-	hdr = ptr;
-	hdr->size = alloc_size;
-	ptr += sizeof(*hdr);
+		hdr->size = alloc_size;
+		fill_redzone(hdr);
 
-	fill_redzone(hdr);
+		ptr += sizeof(*hdr);
+		memset(ptr, 0, size);
+	}
 
-	memset(ptr, 0, size);
 	return ptr;
 }
 
-void *smalloc(unsigned int size)
+void *smalloc(size_t size)
 {
-	unsigned int i;
+	unsigned int i, end_pool;
+
+	if (size != (unsigned int) size)
+		return NULL;
 
-	global_read_lock();
 	i = last_pool;
+	end_pool = nr_pools;
 
 	do {
-		for (; i < nr_pools; i++) {
+		for (; i < end_pool; i++) {
 			void *ptr = smalloc_pool(&mp[i], size);
 
 			if (ptr) {
 				last_pool = i;
-				global_read_unlock();
 				return ptr;
 			}
 		}
 		if (last_pool) {
-			last_pool = 0;
+			end_pool = last_pool;
+			last_pool = i = 0;
 			continue;
 		}
 
-		if (nr_pools + 1 > MAX_POOLS)
-			break;
-		else {
-			i = nr_pools;
-			global_read_unlock();
-			if (add_pool(&mp[nr_pools], size))
-				goto out;
-			global_read_lock();
-		}
+		break;
 	} while (1);
 
-	global_read_unlock();
-out:
+	log_err("smalloc: OOM. Consider using --alloc-size to increase the "
+		"shared memory available.\n");
 	return NULL;
 }
 
+void *scalloc(size_t nmemb, size_t size)
+{
+	return smalloc(nmemb * size);
+}
+
 char *smalloc_strdup(const char *str)
 {
-	char *ptr;
+	char *ptr = NULL;
 
 	ptr = smalloc(strlen(str) + 1);
-	strcpy(ptr, str);
+	if (ptr)
+		strcpy(ptr, str);
 	return ptr;
 }