2 * simple memory allocator, backed by mmap() so that it hands out memory
3 * that can be shared across processes and threads
11 #include <sys/types.h>
16 #define MP_SAFE /* define to made allocator thread safe */
17 #define SMALLOC_REDZONE /* define to detect memory corruption */
19 #define INITIAL_SIZE 32768 /* new pool size */
20 #define MAX_POOLS 4 /* maximum number of pools to setup */
22 #define SMALLOC_PRE_RED 0xdeadbeefU
23 #define SMALLOC_POST_RED 0x5aa55aa5U
24 #define SMALLOC_REDZONE_SZ (2 * sizeof(unsigned int))
26 unsigned int smalloc_pool_size = INITIAL_SIZE;
29 struct fio_mutex *lock; /* protects this pool */
30 void *map; /* map of blocks */
31 void *last; /* next free block hint */
32 unsigned int size; /* size of pool */
33 unsigned int room; /* size left in pool */
34 unsigned int largest_block; /* largest block free */
35 unsigned int free_since_compact; /* sfree() since compact() */
36 int fd; /* memory backing fd */
37 char file[PATH_MAX]; /* filename for fd */
40 static struct pool mp[MAX_POOLS];
41 static unsigned int nr_pools;
42 static unsigned int last_pool;
43 static struct fio_mutex *lock;
49 static inline void pool_lock(struct pool *pool)
52 fio_mutex_down(pool->lock);
55 static inline void pool_unlock(struct pool *pool)
58 fio_mutex_up(pool->lock);
61 static inline void global_read_lock(void)
64 fio_mutex_down_read(lock);
67 static inline void global_read_unlock(void)
70 fio_mutex_up_read(lock);
73 static inline void global_write_lock(void)
76 fio_mutex_down_write(lock);
79 static inline void global_write_unlock(void)
82 fio_mutex_up_write(lock);
85 #define hdr_free(hdr) ((hdr)->size & 0x80000000)
86 #define hdr_size(hdr) ((hdr)->size & ~0x80000000)
87 #define hdr_mark_free(hdr) ((hdr)->size |= 0x80000000)
89 static inline int ptr_valid(struct pool *pool, void *ptr)
91 return (ptr >= pool->map) && (ptr < pool->map + pool->size);
94 static inline int __hdr_valid(struct pool *pool, struct mem_hdr *hdr,
97 return ptr_valid(pool, hdr) && ptr_valid(pool, (void *) hdr + size - 1);
100 static inline int hdr_valid(struct pool *pool, struct mem_hdr *hdr)
102 return __hdr_valid(pool, hdr, hdr_size(hdr));
105 static inline int region_free(struct mem_hdr *hdr)
107 return hdr_free(hdr) || (!hdr_free(hdr) && !hdr_size(hdr));
110 static inline struct mem_hdr *__hdr_nxt(struct pool *pool, struct mem_hdr *hdr,
113 struct mem_hdr *nxt = (void *) hdr + size + sizeof(*hdr);
115 if (__hdr_valid(pool, nxt, size))
121 static inline struct mem_hdr *hdr_nxt(struct pool *pool, struct mem_hdr *hdr)
123 return __hdr_nxt(pool, hdr, hdr_size(hdr));
126 static void merge(struct pool *pool, struct mem_hdr *hdr, struct mem_hdr *nxt)
128 unsigned int hfree = hdr_free(hdr);
129 unsigned int nfree = hdr_free(nxt);
131 hdr->size = hdr_size(hdr) + hdr_size(nxt) + sizeof(*nxt);
139 if (pool->last == nxt)
143 static int combine(struct pool *pool, struct mem_hdr *prv, struct mem_hdr *hdr)
145 if (prv && hdr_free(prv) && hdr_free(hdr)) {
146 merge(pool, prv, hdr);
153 static int compact_pool(struct pool *pool)
155 struct mem_hdr *hdr = pool->map, *nxt;
156 unsigned int compacted = 0;
158 if (pool->free_since_compact < 50)
162 nxt = hdr_nxt(pool, hdr);
165 if (hdr_free(nxt) && hdr_free(hdr)) {
166 merge(pool, hdr, nxt);
170 hdr = hdr_nxt(pool, hdr);
173 pool->free_since_compact = 0;
177 static int add_pool(struct pool *pool, unsigned int alloc_size)
183 strcpy(pool->file, "/tmp/.fio_smalloc.XXXXXX");
184 fd = mkstemp(pool->file);
188 alloc_size += sizeof(*hdr);
189 #ifdef SMALLOC_REDZONE
190 alloc_size += SMALLOC_REDZONE_SZ;
193 if (alloc_size > smalloc_pool_size)
194 pool->size = alloc_size;
196 pool->size = smalloc_pool_size;
198 if (ftruncate(fd, pool->size) < 0)
201 ptr = mmap(NULL, pool->size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
202 if (ptr == MAP_FAILED)
205 memset(ptr, 0, pool->size);
206 pool->map = pool->last = ptr;
209 pool->lock = fio_mutex_init(1);
217 pool->room = hdr->size = pool->size - sizeof(*hdr);
218 pool->largest_block = pool->room;
222 global_write_unlock();
226 munmap(pool->map, pool->size);
239 lock = fio_mutex_rw_init();
241 ret = add_pool(&mp[0], INITIAL_SIZE);
245 static void cleanup_pool(struct pool *pool)
249 munmap(pool->map, pool->size);
252 fio_mutex_remove(pool->lock);
259 for (i = 0; i < nr_pools; i++)
260 cleanup_pool(&mp[i]);
263 fio_mutex_remove(lock);
266 static void sfree_check_redzone(struct mem_hdr *hdr, void *ptr)
268 #ifdef SMALLOC_REDZONE
269 unsigned int *prered, *postred;
271 prered = (unsigned int *) ptr;
272 postred = (unsigned int *) (ptr + hdr_size(hdr) - sizeof(unsigned int));
274 if (*prered != SMALLOC_PRE_RED) {
275 fprintf(stderr, "smalloc pre redzone destroyed!\n");
276 fprintf(stderr, " ptr=%p, prered=%x, expected %x\n",
277 ptr, *prered, SMALLOC_PRE_RED);
280 if (*postred != SMALLOC_POST_RED) {
281 fprintf(stderr, "smalloc post redzone destroyed!\n");
282 fprintf(stderr, " ptr=%p, postred=%x, expected %x\n",
283 ptr, *postred, SMALLOC_POST_RED);
289 static void sfree_pool(struct pool *pool, void *ptr)
291 struct mem_hdr *hdr, *nxt;
296 #ifdef SMALLOC_REDZONE
297 ptr -= sizeof(unsigned int);
300 assert(ptr_valid(pool, ptr));
303 hdr = ptr - sizeof(*hdr);
304 sfree_check_redzone(hdr, ptr);
305 assert(!hdr_free(hdr));
307 pool->room -= hdr_size(hdr);
309 nxt = hdr_nxt(pool, hdr);
310 if (nxt && hdr_free(nxt))
311 merge(pool, hdr, nxt);
313 if (hdr_size(hdr) > pool->largest_block)
314 pool->largest_block = hdr_size(hdr);
316 pool->free_since_compact++;
320 void sfree(void *ptr)
322 struct pool *pool = NULL;
330 for (i = 0; i < nr_pools; i++) {
331 if (ptr_valid(&mp[i], ptr)) {
337 global_read_unlock();
340 sfree_pool(pool, ptr);
343 static void *__smalloc_pool(struct pool *pool, unsigned int size)
345 struct mem_hdr *hdr, *prv;
353 if (size > pool->room + sizeof(*hdr))
355 if ((size > pool->largest_block) && pool->largest_block)
361 if (combine(pool, prv, hdr))
364 if (hdr_free(hdr) && hdr_size(hdr) >= size)
368 } while ((hdr = hdr_nxt(pool, hdr)) != NULL);
374 * more room, adjust next header if any
376 if (hdr_size(hdr) - size >= 2 * sizeof(*hdr)) {
377 struct mem_hdr *nxt = __hdr_nxt(pool, hdr, size);
380 nxt->size = hdr_size(hdr) - size - sizeof(*hdr);
381 if (hdr_size(hdr) == pool->largest_block)
382 pool->largest_block = hdr_size(nxt);
385 size = hdr_size(hdr);
387 size = hdr_size(hdr);
389 if (size == hdr_size(hdr) && size == pool->largest_block)
390 pool->largest_block = 0;
393 * also clears free bit
396 pool->last = hdr_nxt(pool, hdr);
398 pool->last = pool->map;
402 ret = (void *) hdr + sizeof(*hdr);
403 memset(ret, 0, size);
407 * if we fail to allocate, first compact the entries that we missed.
408 * if that also fails, increase the size of the pool
410 if (++did_restart <= 1) {
411 if (!compact_pool(pool)) {
412 pool->last = pool->map;
420 static void *smalloc_pool(struct pool *pool, unsigned int size)
422 #ifdef SMALLOC_REDZONE
423 unsigned int *prered, *postred;
426 ptr = __smalloc_pool(pool, size + 2 * sizeof(unsigned int));
431 *prered = SMALLOC_PRE_RED;
432 ptr += sizeof(unsigned int);
433 postred = ptr + size;
434 *postred = SMALLOC_POST_RED;
438 return __smalloc_pool(pool, size);
442 void *smalloc(unsigned int size)
450 for (; i < nr_pools; i++) {
451 void *ptr = smalloc_pool(&mp[i], size);
455 global_read_unlock();
464 if (nr_pools + 1 >= MAX_POOLS)
468 global_read_unlock();
469 if (add_pool(&mp[nr_pools], size))
475 global_read_unlock();
480 char *smalloc_strdup(const char *str)
484 ptr = smalloc(strlen(str) + 1);