[linux-2.6-block.git] / mm / slob.c

/*
 * SLOB Allocator: Simple List Of Blocks
 *
 * Matt Mackall <mpm@selenic.com> 12/30/03
 *
 * How SLOB works:
 *
 * The core of SLOB is a traditional K&R style heap allocator, with
 * support for returning aligned objects. The granularity of this
 * allocator is 8 bytes on x86, though it's perhaps possible to reduce
 * this to 4 if it's deemed worth the effort. The slob heap is a
 * singly-linked list of pages from __get_free_page, grown on demand
 * and allocation from the heap is currently first-fit.
 *
 * Above this is an implementation of kmalloc/kfree. Blocks returned
 * from kmalloc are 8-byte aligned and prepended with a 8-byte header.
 * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls
 * __get_free_pages directly so that it can return page-aligned blocks
 * and keeps a linked list of such pages and their orders. These
 * objects are detected in kfree() by their page alignment.
 *
 * SLAB is emulated on top of SLOB by simply calling constructors and
 * destructors for every SLAB allocation. Objects are returned with
 * the 8-byte alignment unless the SLAB_MUST_HWCACHE_ALIGN flag is
 * set, in which case the low-level allocator will fragment blocks to
 * create the proper alignment. Again, objects of page-size or greater
 * are allocated by calling __get_free_pages. As SLAB objects know
 * their size, no separate size bookkeeping is necessary and there is
 * essentially no allocation space overhead.
 */

#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/timer.h>

struct slob_block {
	int units;
	struct slob_block *next;
};
typedef struct slob_block slob_t;

#define SLOB_UNIT sizeof(slob_t)
#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
#define SLOB_ALIGN L1_CACHE_BYTES

struct bigblock {
	int order;
	void *pages;
	struct bigblock *next;
};
typedef struct bigblock bigblock_t;

static slob_t arena = { .next = &arena, .units = 1 };
static slob_t *slobfree = &arena;
static bigblock_t *bigblocks;
static DEFINE_SPINLOCK(slob_lock);
static DEFINE_SPINLOCK(block_lock);

static void slob_free(void *b, int size);
static void slob_timer_cbk(void);


static void *slob_alloc(size_t size, gfp_t gfp, int align)
{
	slob_t *prev, *cur, *aligned = 0;
	int delta = 0, units = SLOB_UNITS(size);
	unsigned long flags;

	spin_lock_irqsave(&slob_lock, flags);
	prev = slobfree;
	for (cur = prev->next; ; prev = cur, cur = cur->next) {
		if (align) {
			aligned = (slob_t *)ALIGN((unsigned long)cur, align);
			delta = aligned - cur;
		}
		if (cur->units >= units + delta) { /* room enough? */
			if (delta) { /* need to fragment head to align? */
				aligned->units = cur->units - delta;
				aligned->next = cur->next;
				cur->next = aligned;
				cur->units = delta;
				prev = cur;
				cur = aligned;
			}

			if (cur->units == units) /* exact fit? */
				prev->next = cur->next; /* unlink */
			else { /* fragment */
				prev->next = cur + units;
				prev->next->units = cur->units - units;
				prev->next->next = cur->next;
				cur->units = units;
			}

			slobfree = prev;
			spin_unlock_irqrestore(&slob_lock, flags);
			return cur;
		}
		if (cur == slobfree) {
			spin_unlock_irqrestore(&slob_lock, flags);

			if (size == PAGE_SIZE) /* trying to shrink arena? */
				return 0;

			cur = (slob_t *)__get_free_page(gfp);
			if (!cur)
				return 0;

			slob_free(cur, PAGE_SIZE);
			spin_lock_irqsave(&slob_lock, flags);
			cur = slobfree;
		}
	}
}

static void slob_free(void *block, int size)
{
	slob_t *cur, *b = (slob_t *)block;
	unsigned long flags;

	if (!block)
		return;

	if (size)
		b->units = SLOB_UNITS(size);

	/* Find reinsertion point */
	spin_lock_irqsave(&slob_lock, flags);
	for (cur = slobfree; !(b > cur && b < cur->next); cur = cur->next)
		if (cur >= cur->next && (b > cur || b < cur->next))
			break;

	if (b + b->units == cur->next) {
		b->units += cur->next->units;
		b->next = cur->next->next;
	} else
		b->next = cur->next;

	if (cur + cur->units == b) {
		cur->units += b->units;
		cur->next = b->next;
	} else
		cur->next = b;

	slobfree = cur;

	spin_unlock_irqrestore(&slob_lock, flags);
}

static int FASTCALL(find_order(int size));
static int fastcall find_order(int size)
{
	int order = 0;
	for ( ; size > 4096 ; size >>=1)
		order++;
	return order;
}

void *__kmalloc(size_t size, gfp_t gfp)
{
	slob_t *m;
	bigblock_t *bb;
	unsigned long flags;

	if (size < PAGE_SIZE - SLOB_UNIT) {
		m = slob_alloc(size + SLOB_UNIT, gfp, 0);
		return m ? (void *)(m + 1) : 0;
	}

	bb = slob_alloc(sizeof(bigblock_t), gfp, 0);
	if (!bb)
		return 0;

	bb->order = find_order(size);
	bb->pages = (void *)__get_free_pages(gfp, bb->order);

	if (bb->pages) {
		spin_lock_irqsave(&block_lock, flags);
		bb->next = bigblocks;
		bigblocks = bb;
		spin_unlock_irqrestore(&block_lock, flags);
		return bb->pages;
	}

	slob_free(bb, sizeof(bigblock_t));
	return 0;
}
EXPORT_SYMBOL(__kmalloc);

void kfree(const void *block)
{
	bigblock_t *bb, **last = &bigblocks;
	unsigned long flags;

	if (!block)
		return;

	if (!((unsigned long)block & (PAGE_SIZE-1))) {
		/* might be on the big block list */
		spin_lock_irqsave(&block_lock, flags);
		for (bb = bigblocks; bb; last = &bb->next, bb = bb->next) {
			if (bb->pages == block) {
				*last = bb->next;
				spin_unlock_irqrestore(&block_lock, flags);
				free_pages((unsigned long)block, bb->order);
				slob_free(bb, sizeof(bigblock_t));
				return;
			}
		}
		spin_unlock_irqrestore(&block_lock, flags);
	}

	slob_free((slob_t *)block - 1, 0);
	return;
}

EXPORT_SYMBOL(kfree);

unsigned int ksize(const void *block)
{
	bigblock_t *bb;
	unsigned long flags;

	if (!block)
		return 0;

	if (!((unsigned long)block & (PAGE_SIZE-1))) {
		spin_lock_irqsave(&block_lock, flags);
		for (bb = bigblocks; bb; bb = bb->next)
			if (bb->pages == block) {
				spin_unlock_irqrestore(&slob_lock, flags);
				return PAGE_SIZE << bb->order;
			}
		spin_unlock_irqrestore(&block_lock, flags);
	}

	return ((slob_t *)block - 1)->units * SLOB_UNIT;
}

struct kmem_cache {
	unsigned int size, align;
	const char *name;
	void (*ctor)(void *, struct kmem_cache *, unsigned long);
	void (*dtor)(void *, struct kmem_cache *, unsigned long);
};

struct kmem_cache *kmem_cache_create(const char *name, size_t size,
	size_t align, unsigned long flags,
	void (*ctor)(void*, struct kmem_cache *, unsigned long),
	void (*dtor)(void*, struct kmem_cache *, unsigned long))
{
	struct kmem_cache *c;

	c = slob_alloc(sizeof(struct kmem_cache), flags, 0);

	if (c) {
		c->name = name;
		c->size = size;
		c->ctor = ctor;
		c->dtor = dtor;
		/* ignore alignment unless it's forced */
		c->align = (flags & SLAB_MUST_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
		if (c->align < align)
			c->align = align;
	}

	return c;
}
EXPORT_SYMBOL(kmem_cache_create);

void kmem_cache_destroy(struct kmem_cache *c)
{
	slob_free(c, sizeof(struct kmem_cache));
}
EXPORT_SYMBOL(kmem_cache_destroy);

void *kmem_cache_alloc(struct kmem_cache *c, gfp_t flags)
{
	void *b;

	if (c->size < PAGE_SIZE)
		b = slob_alloc(c->size, flags, c->align);
	else
		b = (void *)__get_free_pages(flags, find_order(c->size));

	if (c->ctor)
		c->ctor(b, c, SLAB_CTOR_CONSTRUCTOR);

	return b;
}
EXPORT_SYMBOL(kmem_cache_alloc);

void *kmem_cache_zalloc(struct kmem_cache *c, gfp_t flags)
{
	void *ret = kmem_cache_alloc(c, flags);
	if (ret)
		memset(ret, 0, c->size);

	return ret;
}
EXPORT_SYMBOL(kmem_cache_zalloc);

void kmem_cache_free(struct kmem_cache *c, void *b)
{
	if (c->dtor)
		c->dtor(b, c, 0);

	if (c->size < PAGE_SIZE)
		slob_free(b, c->size);
	else
		free_pages((unsigned long)b, find_order(c->size));
}
EXPORT_SYMBOL(kmem_cache_free);

unsigned int kmem_cache_size(struct kmem_cache *c)
{
	return c->size;
}
EXPORT_SYMBOL(kmem_cache_size);

const char *kmem_cache_name(struct kmem_cache *c)
{
	return c->name;
}
EXPORT_SYMBOL(kmem_cache_name);

static struct timer_list slob_timer = TIMER_INITIALIZER(
	(void (*)(unsigned long))slob_timer_cbk, 0, 0);

int kmem_cache_shrink(struct kmem_cache *d)
{
	return 0;
}
EXPORT_SYMBOL(kmem_cache_shrink);

int kmem_ptr_validate(struct kmem_cache *a, const void *b)
{
	return 0;
}

void __init kmem_cache_init(void)
{
	slob_timer_cbk();
}

static void slob_timer_cbk(void)
{
	void *p = slob_alloc(PAGE_SIZE, 0, PAGE_SIZE-1);

	if (p)
		free_page((unsigned long)p);

	mod_timer(&slob_timer, jiffies + HZ);
}
Commit	Line	Data
10cef602 MM	1	/*
	2	* SLOB Allocator: Simple List Of Blocks
	3	*
	4	* Matt Mackall <mpm@selenic.com> 12/30/03
	5	*
	6	* How SLOB works:
	7	*
	8	* The core of SLOB is a traditional K&R style heap allocator, with
	9	* support for returning aligned objects. The granularity of this
	10	* allocator is 8 bytes on x86, though it's perhaps possible to reduce
	11	* this to 4 if it's deemed worth the effort. The slob heap is a
	12	* singly-linked list of pages from __get_free_page, grown on demand
	13	* and allocation from the heap is currently first-fit.
	14	*
	15	* Above this is an implementation of kmalloc/kfree. Blocks returned
	16	* from kmalloc are 8-byte aligned and prepended with a 8-byte header.
	17	* If kmalloc is asked for objects of PAGE_SIZE or larger, it calls
	18	* __get_free_pages directly so that it can return page-aligned blocks
	19	* and keeps a linked list of such pages and their orders. These
	20	* objects are detected in kfree() by their page alignment.
	21	*
	22	* SLAB is emulated on top of SLOB by simply calling constructors and
	23	* destructors for every SLAB allocation. Objects are returned with
	24	* the 8-byte alignment unless the SLAB_MUST_HWCACHE_ALIGN flag is
	25	* set, in which case the low-level allocator will fragment blocks to
	26	* create the proper alignment. Again, objects of page-size or greater
	27	* are allocated by calling __get_free_pages. As SLAB objects know
	28	* their size, no separate size bookkeeping is necessary and there is
	29	* essentially no allocation space overhead.
	30	*/
	31
10cef602 MM	32	#include <linux/slab.h>
	33	#include <linux/mm.h>
	34	#include <linux/cache.h>
	35	#include <linux/init.h>
	36	#include <linux/module.h>
	37	#include <linux/timer.h>
	38
	39	struct slob_block {
	40	int units;
	41	struct slob_block *next;
	42	};
	43	typedef struct slob_block slob_t;
	44
	45	#define SLOB_UNIT sizeof(slob_t)
	46	#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
	47	#define SLOB_ALIGN L1_CACHE_BYTES
	48
	49	struct bigblock {
	50	int order;
	51	void *pages;
	52	struct bigblock *next;
	53	};
	54	typedef struct bigblock bigblock_t;
	55
	56	static slob_t arena = { .next = &arena, .units = 1 };
	57	static slob_t *slobfree = &arena;
	58	static bigblock_t *bigblocks;
	59	static DEFINE_SPINLOCK(slob_lock);
	60	static DEFINE_SPINLOCK(block_lock);
	61
	62	static void slob_free(void *b, int size);
bcb4ddb4 DG	63	static void slob_timer_cbk(void);
bcb4ddb4 DG	64
10cef602 MM	65
	66	static void *slob_alloc(size_t size, gfp_t gfp, int align)
	67	{
	68	slob_t prev, cur, *aligned = 0;
	69	int delta = 0, units = SLOB_UNITS(size);
	70	unsigned long flags;
	71
	72	spin_lock_irqsave(&slob_lock, flags);
	73	prev = slobfree;
	74	for (cur = prev->next; ; prev = cur, cur = cur->next) {
	75	if (align) {
	76	aligned = (slob_t *)ALIGN((unsigned long)cur, align);
	77	delta = aligned - cur;
	78	}
	79	if (cur->units >= units + delta) { /* room enough? */
	80	if (delta) { /* need to fragment head to align? */
	81	aligned->units = cur->units - delta;
	82	aligned->next = cur->next;
	83	cur->next = aligned;
	84	cur->units = delta;
	85	prev = cur;
	86	cur = aligned;
	87	}
	88
	89	if (cur->units == units) /* exact fit? */
	90	prev->next = cur->next; /* unlink */
	91	else { /* fragment */
	92	prev->next = cur + units;
	93	prev->next->units = cur->units - units;
	94	prev->next->next = cur->next;
	95	cur->units = units;
	96	}
	97
	98	slobfree = prev;
	99	spin_unlock_irqrestore(&slob_lock, flags);
	100	return cur;
	101	}
	102	if (cur == slobfree) {
	103	spin_unlock_irqrestore(&slob_lock, flags);
	104
	105	if (size == PAGE_SIZE) /* trying to shrink arena? */
	106	return 0;
	107
	108	cur = (slob_t *)__get_free_page(gfp);
	109	if (!cur)
	110	return 0;
	111
	112	slob_free(cur, PAGE_SIZE);
	113	spin_lock_irqsave(&slob_lock, flags);
	114	cur = slobfree;
	115	}
	116	}
	117	}
	118
	119	static void slob_free(void *block, int size)
	120	{
	121	slob_t cur, b = (slob_t *)block;
	122	unsigned long flags;
	123
	124	if (!block)
	125	return;
	126
	127	if (size)
	128	b->units = SLOB_UNITS(size);
129
130	/* Find reinsertion point */
131	spin_lock_irqsave(&slob_lock, flags);
132	for (cur = slobfree; !(b > cur && b < cur->next); cur = cur->next)
133	if (cur >= cur->next && (b > cur \|\| b < cur->next))
134	break;
135
136	if (b + b->units == cur->next) {
137	b->units += cur->next->units;
138	b->next = cur->next->next;
139	} else
140	b->next = cur->next;
141
142	if (cur + cur->units == b) {
143	cur->units += b->units;
144	cur->next = b->next;
145	} else
146	cur->next = b;
147
148	slobfree = cur;
149
150	spin_unlock_irqrestore(&slob_lock, flags);
151	}
152
153	static int FASTCALL(find_order(int size));
154	static int fastcall find_order(int size)
155	{
156	int order = 0;
157	for ( ; size > 4096 ; size >>=1)
158	order++;
159	return order;
160	}
161
2e892f43	162	void *__kmalloc(size_t size, gfp_t gfp)
10cef602 MM	163	{
	164	slob_t *m;
	165	bigblock_t *bb;
	166	unsigned long flags;
	167
	168	if (size < PAGE_SIZE - SLOB_UNIT) {
	169	m = slob_alloc(size + SLOB_UNIT, gfp, 0);
	170	return m ? (void *)(m + 1) : 0;
	171	}
	172
	173	bb = slob_alloc(sizeof(bigblock_t), gfp, 0);
	174	if (!bb)
	175	return 0;
	176
	177	bb->order = find_order(size);
	178	bb->pages = (void *)__get_free_pages(gfp, bb->order);
	179
	180	if (bb->pages) {
	181	spin_lock_irqsave(&block_lock, flags);
	182	bb->next = bigblocks;
	183	bigblocks = bb;
	184	spin_unlock_irqrestore(&block_lock, flags);
	185	return bb->pages;
	186	}
	187
	188	slob_free(bb, sizeof(bigblock_t));
	189	return 0;
	190	}
2e892f43	191	EXPORT_SYMBOL(__kmalloc);
10cef602 MM	192
	193	void kfree(const void *block)
	194	{
	195	bigblock_t bb, *last = &bigblocks;
	196	unsigned long flags;
	197
	198	if (!block)
	199	return;
	200
	201	if (!((unsigned long)block & (PAGE_SIZE-1))) {
	202	/* might be on the big block list */
	203	spin_lock_irqsave(&block_lock, flags);
	204	for (bb = bigblocks; bb; last = &bb->next, bb = bb->next) {
	205	if (bb->pages == block) {
	206	*last = bb->next;
	207	spin_unlock_irqrestore(&block_lock, flags);
	208	free_pages((unsigned long)block, bb->order);
	209	slob_free(bb, sizeof(bigblock_t));
	210	return;
	211	}
	212	}
	213	spin_unlock_irqrestore(&block_lock, flags);
	214	}
	215
	216	slob_free((slob_t *)block - 1, 0);
	217	return;
	218	}
	219
	220	EXPORT_SYMBOL(kfree);
	221
	222	unsigned int ksize(const void *block)
	223	{
	224	bigblock_t *bb;
	225	unsigned long flags;
	226
	227	if (!block)
	228	return 0;
	229
	230	if (!((unsigned long)block & (PAGE_SIZE-1))) {
	231	spin_lock_irqsave(&block_lock, flags);
	232	for (bb = bigblocks; bb; bb = bb->next)
	233	if (bb->pages == block) {
	234	spin_unlock_irqrestore(&slob_lock, flags);
	235	return PAGE_SIZE << bb->order;
	236	}
	237	spin_unlock_irqrestore(&block_lock, flags);
	238	}
	239
	240	return ((slob_t )block - 1)->units SLOB_UNIT;
	241	}
	242
	243	struct kmem_cache {
	244	unsigned int size, align;
	245	const char *name;
	246	void (ctor)(void , struct kmem_cache *, unsigned long);
	247	void (dtor)(void , struct kmem_cache *, unsigned long);
	248	};
	249
	250	struct kmem_cache kmem_cache_create(const char name, size_t size,
	251	size_t align, unsigned long flags,
	252	void (ctor)(void, struct kmem_cache *, unsigned long),
	253	void (dtor)(void, struct kmem_cache *, unsigned long))
	254	{
	255	struct kmem_cache *c;
256
257	c = slob_alloc(sizeof(struct kmem_cache), flags, 0);
258
259	if (c) {
260	c->name = name;
261	c->size = size;
262	c->ctor = ctor;
263	c->dtor = dtor;
264	/* ignore alignment unless it's forced */
265	c->align = (flags & SLAB_MUST_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
266	if (c->align < align)
267	c->align = align;
268	}
269
270	return c;
271	}
272	EXPORT_SYMBOL(kmem_cache_create);
273
133d205a	274	void kmem_cache_destroy(struct kmem_cache *c)
10cef602 MM	275	{
10cef602 MM	276	slob_free(c, sizeof(struct kmem_cache));
10cef602 MM	277	}
	278	EXPORT_SYMBOL(kmem_cache_destroy);
	279
	280	void kmem_cache_alloc(struct kmem_cache c, gfp_t flags)
	281	{
	282	void *b;
	283
	284	if (c->size < PAGE_SIZE)
	285	b = slob_alloc(c->size, flags, c->align);
	286	else
	287	b = (void *)__get_free_pages(flags, find_order(c->size));
	288
	289	if (c->ctor)
	290	c->ctor(b, c, SLAB_CTOR_CONSTRUCTOR);
	291
	292	return b;
	293	}
	294	EXPORT_SYMBOL(kmem_cache_alloc);
	295
a8c0f9a4 PE	296	void kmem_cache_zalloc(struct kmem_cache c, gfp_t flags)
	297	{
	298	void *ret = kmem_cache_alloc(c, flags);
	299	if (ret)
	300	memset(ret, 0, c->size);
	301
	302	return ret;
	303	}
	304	EXPORT_SYMBOL(kmem_cache_zalloc);
	305
10cef602 MM	306	void kmem_cache_free(struct kmem_cache c, void b)
	307	{
	308	if (c->dtor)
	309	c->dtor(b, c, 0);
	310
	311	if (c->size < PAGE_SIZE)
	312	slob_free(b, c->size);
	313	else
	314	free_pages((unsigned long)b, find_order(c->size));
	315	}
	316	EXPORT_SYMBOL(kmem_cache_free);
	317
	318	unsigned int kmem_cache_size(struct kmem_cache *c)
	319	{
	320	return c->size;
	321	}
	322	EXPORT_SYMBOL(kmem_cache_size);
	323
	324	const char kmem_cache_name(struct kmem_cache c)
	325	{
	326	return c->name;
	327	}
	328	EXPORT_SYMBOL(kmem_cache_name);
	329
	330	static struct timer_list slob_timer = TIMER_INITIALIZER(
bcb4ddb4	331	(void (*)(unsigned long))slob_timer_cbk, 0, 0);
10cef602	332
2e892f43 CL	333	int kmem_cache_shrink(struct kmem_cache *d)
	334	{
	335	return 0;
	336	}
	337	EXPORT_SYMBOL(kmem_cache_shrink);
	338
55935a34	339	int kmem_ptr_validate(struct kmem_cache a, const void b)
2e892f43 CL	340	{
	341	return 0;
	342	}
	343
bcb4ddb4 DG	344	void __init kmem_cache_init(void)
	345	{
	346	slob_timer_cbk();
	347	}
	348
	349	static void slob_timer_cbk(void)
10cef602 MM	350	{
	351	void *p = slob_alloc(PAGE_SIZE, 0, PAGE_SIZE-1);
	352
	353	if (p)
	354	free_page((unsigned long)p);
	355
	356	mod_timer(&slob_timer, jiffies + HZ);
	357	}