2 * simple memory allocator, backed by mmap() so that it hands out memory
3 * that can be shared across processes and threads
11 #include <sys/types.h>
16 #define MP_SAFE /* define to made allocator thread safe */
18 #define INITIAL_SIZE 1048576 /* new pool size */
19 #define MAX_POOLS 32 /* maximum number of pools to setup */
21 unsigned int smalloc_pool_size = INITIAL_SIZE;
24 struct fio_mutex *lock; /* protects this pool */
25 void *map; /* map of blocks */
26 void *last; /* next free block hint */
27 unsigned int size; /* size of pool */
28 unsigned int room; /* size left in pool */
29 unsigned int largest_block; /* largest block free */
30 unsigned int free_since_compact; /* sfree() since compact() */
31 int fd; /* memory backing fd */
32 char file[PATH_MAX]; /* filename for fd */
35 static struct pool mp[MAX_POOLS];
36 static unsigned int nr_pools;
37 static unsigned int last_pool;
38 static struct fio_mutex *lock;
44 static inline void pool_lock(struct pool *pool)
47 fio_mutex_down(pool->lock);
50 static inline void pool_unlock(struct pool *pool)
53 fio_mutex_up(pool->lock);
56 static inline void global_read_lock(void)
59 fio_mutex_down_read(lock);
62 static inline void global_read_unlock(void)
65 fio_mutex_up_read(lock);
68 static inline void global_write_lock(void)
71 fio_mutex_down_write(lock);
74 static inline void global_write_unlock(void)
77 fio_mutex_up_write(lock);
80 #define hdr_free(hdr) ((hdr)->size & 0x80000000)
81 #define hdr_size(hdr) ((hdr)->size & ~0x80000000)
82 #define hdr_mark_free(hdr) ((hdr)->size |= 0x80000000)
84 static inline int ptr_valid(struct pool *pool, void *ptr)
86 return (ptr >= pool->map) && (ptr < pool->map + pool->size);
89 static inline int __hdr_valid(struct pool *pool, struct mem_hdr *hdr,
92 return ptr_valid(pool, hdr) && ptr_valid(pool, (void *) hdr + size - 1);
95 static inline int hdr_valid(struct pool *pool, struct mem_hdr *hdr)
97 return __hdr_valid(pool, hdr, hdr_size(hdr));
100 static inline int region_free(struct mem_hdr *hdr)
102 return hdr_free(hdr) || (!hdr_free(hdr) && !hdr_size(hdr));
105 static inline struct mem_hdr *__hdr_nxt(struct pool *pool, struct mem_hdr *hdr,
108 struct mem_hdr *nxt = (void *) hdr + size + sizeof(*hdr);
110 if (__hdr_valid(pool, nxt, size))
116 static inline struct mem_hdr *hdr_nxt(struct pool *pool, struct mem_hdr *hdr)
118 return __hdr_nxt(pool, hdr, hdr_size(hdr));
121 static void merge(struct pool *pool, struct mem_hdr *hdr, struct mem_hdr *nxt)
123 unsigned int hfree = hdr_free(hdr);
124 unsigned int nfree = hdr_free(nxt);
126 hdr->size = hdr_size(hdr) + hdr_size(nxt) + sizeof(*nxt);
134 if (pool->last == nxt)
138 static int combine(struct pool *pool, struct mem_hdr *prv, struct mem_hdr *hdr)
140 if (prv && hdr_free(prv) && hdr_free(hdr)) {
141 merge(pool, prv, hdr);
148 static int compact_pool(struct pool *pool)
150 struct mem_hdr *hdr = pool->map, *nxt;
151 unsigned int compacted = 0;
153 if (pool->free_since_compact < 50)
157 nxt = hdr_nxt(pool, hdr);
160 if (hdr_free(nxt) && hdr_free(hdr)) {
161 merge(pool, hdr, nxt);
165 hdr = hdr_nxt(pool, hdr);
168 pool->free_since_compact = 0;
172 static int add_pool(struct pool *pool, unsigned int alloc_size)
178 strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
179 fd = mkstemp(pool->file);
183 alloc_size += sizeof(*hdr);
184 if (alloc_size > smalloc_pool_size)
185 pool->size = alloc_size;
187 pool->size = smalloc_pool_size;
189 if (ftruncate(fd, pool->size) < 0)
192 ptr = mmap(NULL, pool->size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
193 if (ptr == MAP_FAILED)
196 memset(ptr, 0, pool->size);
197 pool->map = pool->last = ptr;
200 pool->lock = fio_mutex_init(1);
208 pool->room = hdr->size = pool->size - sizeof(*hdr);
209 pool->largest_block = pool->room;
213 global_write_unlock();
217 munmap(pool->map, pool->size);
230 lock = fio_mutex_rw_init();
232 ret = add_pool(&mp[0], INITIAL_SIZE);
236 static void cleanup_pool(struct pool *pool)
240 munmap(pool->map, pool->size);
243 fio_mutex_remove(pool->lock);
250 for (i = 0; i < nr_pools; i++)
251 cleanup_pool(&mp[i]);
254 fio_mutex_remove(lock);
257 static void sfree_pool(struct pool *pool, void *ptr)
259 struct mem_hdr *hdr, *nxt;
264 assert(ptr_valid(pool, ptr));
267 hdr = ptr - sizeof(*hdr);
268 assert(!hdr_free(hdr));
270 pool->room -= hdr_size(hdr);
272 nxt = hdr_nxt(pool, hdr);
273 if (nxt && hdr_free(nxt))
274 merge(pool, hdr, nxt);
276 if (hdr_size(hdr) > pool->largest_block)
277 pool->largest_block = hdr_size(hdr);
279 pool->free_since_compact++;
283 void sfree(void *ptr)
285 struct pool *pool = NULL;
293 for (i = 0; i < nr_pools; i++) {
294 if (ptr_valid(&mp[i], ptr)) {
300 global_read_unlock();
303 sfree_pool(pool, ptr);
306 static void *smalloc_pool(struct pool *pool, unsigned int size)
308 struct mem_hdr *hdr, *prv;
316 if (size > pool->room + sizeof(*hdr))
318 if ((size > pool->largest_block) && pool->largest_block)
324 if (combine(pool, prv, hdr))
327 if (hdr_free(hdr) && hdr_size(hdr) >= size)
331 } while ((hdr = hdr_nxt(pool, hdr)) != NULL);
337 * more room, adjust next header if any
339 if (hdr_size(hdr) - size >= 2 * sizeof(*hdr)) {
340 struct mem_hdr *nxt = __hdr_nxt(pool, hdr, size);
343 nxt->size = hdr_size(hdr) - size - sizeof(*hdr);
344 if (hdr_size(hdr) == pool->largest_block)
345 pool->largest_block = hdr_size(nxt);
348 size = hdr_size(hdr);
350 size = hdr_size(hdr);
352 if (size == hdr_size(hdr) && size == pool->largest_block)
353 pool->largest_block = 0;
356 * also clears free bit
359 pool->last = hdr_nxt(pool, hdr);
361 pool->last = pool->map;
365 ret = (void *) hdr + sizeof(*hdr);
366 memset(ret, 0, size);
370 * if we fail to allocate, first compact the entries that we missed.
371 * if that also fails, increase the size of the pool
373 if (++did_restart <= 1) {
374 if (!compact_pool(pool)) {
375 pool->last = pool->map;
383 void *smalloc(unsigned int size)
391 for (; i < nr_pools; i++) {
392 void *ptr = smalloc_pool(&mp[i], size);
396 global_read_unlock();
405 if (nr_pools + 1 >= MAX_POOLS)
409 global_read_unlock();
410 if (add_pool(&mp[nr_pools], size))
416 global_read_unlock();
421 char *smalloc_strdup(const char *str)
425 ptr = smalloc(strlen(str) + 1);