#include "mutex.h"
-#undef ENABLE_RESIZE /* define to enable pool resizing */
#define MP_SAFE /* define to made allocator thread safe */
-#define INITIAL_SIZE 65536 /* new pool size */
+#define INITIAL_SIZE 1048576 /* new pool size */
#define MAX_POOLS 32 /* maximum number of pools to setup */
-#ifdef ENABLE_RESIZE
-#define MAX_SIZE 8 * INITIAL_SIZE
-static unsigned int resize_error;
-#endif
+unsigned int smalloc_pool_size = INITIAL_SIZE;
struct pool {
struct fio_mutex *lock; /* protects this pool */
fio_mutex_up(pool->lock);
}
-static inline void global_lock(void)
+static inline void global_read_lock(void)
+{
+ if (lock)
+ fio_mutex_down_read(lock);
+}
+
+static inline void global_read_unlock(void)
{
if (lock)
- fio_mutex_down(lock);
+ fio_mutex_up_read(lock);
}
-static inline void global_unlock(void)
+static inline void global_write_lock(void)
{
if (lock)
- fio_mutex_up(lock);
+ fio_mutex_down_write(lock);
+}
+
+static inline void global_write_unlock(void)
+{
+ if (lock)
+ fio_mutex_up_write(lock);
}
#define hdr_free(hdr) ((hdr)->size & 0x80000000)
return !!compacted;
}
-static int resize_pool(struct pool *pool)
-{
-#ifdef ENABLE_RESIZE
- unsigned int new_size = pool->size << 1;
- struct mem_hdr *hdr, *last_hdr;
- void *ptr;
-
- if (new_size >= MAX_SIZE || resize_error)
- return 1;
-
- if (ftruncate(pool->fd, new_size) < 0)
- goto fail;
-
- ptr = mremap(pool->map, pool->size, new_size, 0);
- if (ptr == MAP_FAILED)
- goto fail;
-
- pool->map = ptr;
- hdr = pool;
- do {
- last_hdr = hdr;
- } while ((hdr = hdr_nxt(hdr)) != NULL);
-
- if (hdr_free(last_hdr)) {
- last_hdr->size = hdr_size(last_hdr) + new_size - pool_size;
- hdr_mark_free(last_hdr);
- } else {
- struct mem_hdr *nxt;
-
- nxt = (void *) last_hdr + hdr_size(last_hdr) + sizeof(*hdr);
- nxt->size = new_size - pool_size - sizeof(*hdr);
- hdr_mark_free(nxt);
- }
-
- pool_room += new_size - pool_size;
- pool_size = new_size;
- return 0;
-fail:
- perror("resize");
- resize_error = 1;
-#else
- return 1;
-#endif
-}
-
-static int add_pool(struct pool *pool)
+static int add_pool(struct pool *pool, unsigned int alloc_size)
{
struct mem_hdr *hdr;
void *ptr;
if (fd < 0)
goto out_close;
- pool->size = INITIAL_SIZE;
+ alloc_size += sizeof(*hdr);
+ if (alloc_size > smalloc_pool_size)
+ pool->size = alloc_size;
+ else
+ pool->size = smalloc_pool_size;
+
if (ftruncate(fd, pool->size) < 0)
goto out_unlink;
pool->room = hdr->size = pool->size - sizeof(*hdr);
pool->largest_block = pool->room;
hdr_mark_free(hdr);
+ global_write_lock();
nr_pools++;
+ global_write_unlock();
return 0;
out_unlink:
if (pool->map)
void sinit(void)
{
- int ret = add_pool(&mp[0]);
+ int ret;
#ifdef MP_SAFE
- lock = fio_mutex_init(1);
+ lock = fio_mutex_rw_init();
#endif
+ ret = add_pool(&mp[0], INITIAL_SIZE);
assert(!ret);
}
struct pool *pool = NULL;
unsigned int i;
- global_lock();
+ if (!ptr)
+ return;
+
+ global_read_lock();
for (i = 0; i < nr_pools; i++) {
if (ptr_valid(&mp[i], ptr)) {
}
}
- global_unlock();
+ global_read_unlock();
assert(pool);
sfree_pool(pool, ptr);
int did_restart = 0;
void *ret;
- /*
- * slight chance of race with sfree() here, but acceptable
- */
- if (!size || size > pool->room + sizeof(*hdr) ||
- ((size > pool->largest_block) && pool->largest_block))
+ if (!size)
return NULL;
pool_lock(pool);
+ if (size > pool->room + sizeof(*hdr))
+ goto fail;
+ if ((size > pool->largest_block) && pool->largest_block)
+ goto fail;
restart:
hdr = pool->last;
prv = NULL;
do {
if (combine(pool, prv, hdr))
hdr = prv;
-
+
if (hdr_free(hdr) && hdr_size(hdr) >= size)
break;
* if we fail to allocate, first compact the entries that we missed.
* if that also fails, increase the size of the pool
*/
- ++did_restart;
- if (did_restart <= 1) {
+ if (++did_restart <= 1) {
if (!compact_pool(pool)) {
pool->last = pool->map;
goto restart;
}
}
- ++did_restart;
- if (did_restart <= 2) {
- if (!resize_pool(pool)) {
- pool->last = pool->map;
- goto restart;
- }
- }
pool_unlock(pool);
return NULL;
}
{
unsigned int i;
- global_lock();
+ global_read_lock();
i = last_pool;
do {
if (ptr) {
last_pool = i;
- global_unlock();
+ global_read_unlock();
return ptr;
}
}
break;
else {
i = nr_pools;
- if (add_pool(&mp[nr_pools]))
- break;
+ global_read_unlock();
+ if (add_pool(&mp[nr_pools], size))
+ goto out;
+ global_read_lock();
}
} while (1);
- global_unlock();
+ global_read_unlock();
+out:
return NULL;
}