[linux-2.6-block.git] / mm / slab.h

#ifndef MM_SLAB_H
#define MM_SLAB_H
/*
 * Internal slab definitions
 */

/*
 * State of the slab allocator.
 *
 * This is used to describe the states of the allocator during bootup.
 * Allocators use this to gradually bootstrap themselves. Most allocators
 * have the problem that the structures used for managing slab caches are
 * allocated from slab caches themselves.
 */
enum slab_state {
	DOWN,			/* No slab functionality yet */
	PARTIAL,		/* SLUB: kmem_cache_node available */
	PARTIAL_ARRAYCACHE,	/* SLAB: kmalloc size for arraycache available */
	PARTIAL_L3,		/* SLAB: kmalloc size for l3 struct available */
	UP,			/* Slab caches usable but not all extras yet */
	FULL			/* Everything is working */
};

extern enum slab_state slab_state;

/* The slab cache mutex protects the management structures during changes */
extern struct mutex slab_mutex;

/* The list of all slab caches on the system */
extern struct list_head slab_caches;

/* The slab cache that manages slab cache information */
extern struct kmem_cache *kmem_cache;

unsigned long calculate_alignment(unsigned long flags,
		unsigned long align, unsigned long size);

#ifndef CONFIG_SLOB
/* Kmalloc array related functions */
void create_kmalloc_caches(unsigned long);
#endif


/* Functions provided by the slab allocators */
extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);

extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
			unsigned long flags);
extern void create_boot_cache(struct kmem_cache *, const char *name,
			size_t size, unsigned long flags);

struct mem_cgroup;
#ifdef CONFIG_SLUB
struct kmem_cache *
__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
		   size_t align, unsigned long flags, void (*ctor)(void *));
#else
static inline struct kmem_cache *
__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
		   size_t align, unsigned long flags, void (*ctor)(void *))
{ return NULL; }
#endif


/* Legal flag mask for kmem_cache_create(), for various configurations */
#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
			 SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )

#if defined(CONFIG_DEBUG_SLAB)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
#elif defined(CONFIG_SLUB_DEBUG)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
			  SLAB_TRACE | SLAB_DEBUG_FREE)
#else
#define SLAB_DEBUG_FLAGS (0)
#endif

#if defined(CONFIG_SLAB)
#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
			  SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK)
#elif defined(CONFIG_SLUB)
#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
			  SLAB_TEMPORARY | SLAB_NOTRACK)
#else
#define SLAB_CACHE_FLAGS (0)
#endif

#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)

int __kmem_cache_shutdown(struct kmem_cache *);

struct seq_file;
struct file;

struct slabinfo {
	unsigned long active_objs;
	unsigned long num_objs;
	unsigned long active_slabs;
	unsigned long num_slabs;
	unsigned long shared_avail;
	unsigned int limit;
	unsigned int batchcount;
	unsigned int shared;
	unsigned int objects_per_slab;
	unsigned int cache_order;
};

void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos);

#ifdef CONFIG_MEMCG_KMEM
static inline bool is_root_cache(struct kmem_cache *s)
{
	return !s->memcg_params || s->memcg_params->is_root_cache;
}

static inline bool cache_match_memcg(struct kmem_cache *cachep,
				     struct mem_cgroup *memcg)
{
	return (is_root_cache(cachep) && !memcg) ||
				(cachep->memcg_params->memcg == memcg);
}

static inline void memcg_bind_pages(struct kmem_cache *s, int order)
{
	if (!is_root_cache(s))
		atomic_add(1 << order, &s->memcg_params->nr_pages);
}

static inline void memcg_release_pages(struct kmem_cache *s, int order)
{
	if (is_root_cache(s))
		return;

	if (atomic_sub_and_test((1 << order), &s->memcg_params->nr_pages))
		mem_cgroup_destroy_cache(s);
}

static inline bool slab_equal_or_root(struct kmem_cache *s,
					struct kmem_cache *p)
{
	return (p == s) ||
		(s->memcg_params && (p == s->memcg_params->root_cache));
}

/*
 * We use suffixes to the name in memcg because we can't have caches
 * created in the system with the same name. But when we print them
 * locally, better refer to them with the base name
 */
static inline const char *cache_name(struct kmem_cache *s)
{
	if (!is_root_cache(s))
		return s->memcg_params->root_cache->name;
	return s->name;
}

static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx)
{
	return s->memcg_params->memcg_caches[idx];
}

static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
{
	if (is_root_cache(s))
		return s;
	return s->memcg_params->root_cache;
}
#else
static inline bool is_root_cache(struct kmem_cache *s)
{
	return true;
}

static inline bool cache_match_memcg(struct kmem_cache *cachep,
				     struct mem_cgroup *memcg)
{
	return true;
}

static inline void memcg_bind_pages(struct kmem_cache *s, int order)
{
}

static inline void memcg_release_pages(struct kmem_cache *s, int order)
{
}

static inline bool slab_equal_or_root(struct kmem_cache *s,
				      struct kmem_cache *p)
{
	return true;
}

static inline const char *cache_name(struct kmem_cache *s)
{
	return s->name;
}

static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx)
{
	return NULL;
}

static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
{
	return s;
}
#endif

static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
{
	struct kmem_cache *cachep;
	struct page *page;

	/*
	 * When kmemcg is not being used, both assignments should return the
	 * same value. but we don't want to pay the assignment price in that
	 * case. If it is not compiled in, the compiler should be smart enough
	 * to not do even the assignment. In that case, slab_equal_or_root
	 * will also be a constant.
	 */
	if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE))
		return s;

	page = virt_to_head_page(x);
	cachep = page->slab_cache;
	if (slab_equal_or_root(cachep, s))
		return cachep;

	pr_err("%s: Wrong slab cache. %s but object is from %s\n",
		__FUNCTION__, cachep->name, s->name);
	WARN_ON_ONCE(1);
	return s;
}
#endif
Commit	Line	Data
97d06609 CL	1	#ifndef MM_SLAB_H
	2	#define MM_SLAB_H
	3	/*
	4	* Internal slab definitions
	5	*/
	6
	7	/*
	8	* State of the slab allocator.
	9	*
	10	* This is used to describe the states of the allocator during bootup.
	11	* Allocators use this to gradually bootstrap themselves. Most allocators
	12	* have the problem that the structures used for managing slab caches are
	13	* allocated from slab caches themselves.
	14	*/
	15	enum slab_state {
	16	DOWN, /* No slab functionality yet */
	17	PARTIAL, /* SLUB: kmem_cache_node available */
	18	PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */
	19	PARTIAL_L3, /* SLAB: kmalloc size for l3 struct available */
	20	UP, /* Slab caches usable but not all extras yet */
	21	FULL /* Everything is working */
	22	};
	23
	24	extern enum slab_state slab_state;
	25
18004c5d CL	26	/* The slab cache mutex protects the management structures during changes */
18004c5d CL	27	extern struct mutex slab_mutex;
9b030cb8 CL	28
9b030cb8 CL	29	/* The list of all slab caches on the system */
18004c5d CL	30	extern struct list_head slab_caches;
18004c5d CL	31
9b030cb8 CL	32	/* The slab cache that manages slab cache information */
	33	extern struct kmem_cache *kmem_cache;
	34
45906855 CL	35	unsigned long calculate_alignment(unsigned long flags,
	36	unsigned long align, unsigned long size);
	37
f97d5f63 CL	38	#ifndef CONFIG_SLOB
	39	/* Kmalloc array related functions */
	40	void create_kmalloc_caches(unsigned long);
	41	#endif
	42
	43
9b030cb8	44	/* Functions provided by the slab allocators */
8a13a4cc	45	extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
97d06609	46
45530c44 CL	47	extern struct kmem_cache create_kmalloc_cache(const char name, size_t size,
	48	unsigned long flags);
	49	extern void create_boot_cache(struct kmem_cache , const char name,
	50	size_t size, unsigned long flags);
	51
2633d7a0	52	struct mem_cgroup;
cbb79694	53	#ifdef CONFIG_SLUB
2633d7a0 GC	54	struct kmem_cache *
	55	__kmem_cache_alias(struct mem_cgroup memcg, const char name, size_t size,
	56	size_t align, unsigned long flags, void (ctor)(void ));
cbb79694	57	#else
2633d7a0 GC	58	static inline struct kmem_cache *
	59	__kmem_cache_alias(struct mem_cgroup memcg, const char name, size_t size,
	60	size_t align, unsigned long flags, void (ctor)(void ))
cbb79694 CL	61	{ return NULL; }
	62	#endif
	63
	64
d8843922 GC	65	/* Legal flag mask for kmem_cache_create(), for various configurations */
	66	#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN \| SLAB_CACHE_DMA \| SLAB_PANIC \| \
	67	SLAB_DESTROY_BY_RCU \| SLAB_DEBUG_OBJECTS )
	68
	69	#if defined(CONFIG_DEBUG_SLAB)
	70	#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER)
	71	#elif defined(CONFIG_SLUB_DEBUG)
	72	#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER \| \
	73	SLAB_TRACE \| SLAB_DEBUG_FREE)
	74	#else
	75	#define SLAB_DEBUG_FLAGS (0)
	76	#endif
	77
	78	#if defined(CONFIG_SLAB)
	79	#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD \| SLAB_NOLEAKTRACE \| \
	80	SLAB_RECLAIM_ACCOUNT \| SLAB_TEMPORARY \| SLAB_NOTRACK)
	81	#elif defined(CONFIG_SLUB)
	82	#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE \| SLAB_RECLAIM_ACCOUNT \| \
	83	SLAB_TEMPORARY \| SLAB_NOTRACK)
	84	#else
	85	#define SLAB_CACHE_FLAGS (0)
	86	#endif
	87
	88	#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS \| SLAB_DEBUG_FLAGS \| SLAB_CACHE_FLAGS)
	89
945cf2b6	90	int __kmem_cache_shutdown(struct kmem_cache *);
945cf2b6	91
b7454ad3 GC	92	struct seq_file;
b7454ad3 GC	93	struct file;
b7454ad3	94
0d7561c6 GC	95	struct slabinfo {
	96	unsigned long active_objs;
	97	unsigned long num_objs;
	98	unsigned long active_slabs;
	99	unsigned long num_slabs;
	100	unsigned long shared_avail;
	101	unsigned int limit;
	102	unsigned int batchcount;
	103	unsigned int shared;
	104	unsigned int objects_per_slab;
	105	unsigned int cache_order;
	106	};
	107
	108	void get_slabinfo(struct kmem_cache s, struct slabinfo sinfo);
	109	void slabinfo_show_stats(struct seq_file m, struct kmem_cache s);
b7454ad3 GC	110	ssize_t slabinfo_write(struct file file, const char __user buffer,
b7454ad3 GC	111	size_t count, loff_t *ppos);
ba6c496e GC	112
	113	#ifdef CONFIG_MEMCG_KMEM
	114	static inline bool is_root_cache(struct kmem_cache *s)
	115	{
	116	return !s->memcg_params \|\| s->memcg_params->is_root_cache;
	117	}
2633d7a0 GC	118
	119	static inline bool cache_match_memcg(struct kmem_cache *cachep,
	120	struct mem_cgroup *memcg)
	121	{
	122	return (is_root_cache(cachep) && !memcg) \|\|
	123	(cachep->memcg_params->memcg == memcg);
	124	}
b9ce5ef4	125
1f458cbf GC	126	static inline void memcg_bind_pages(struct kmem_cache *s, int order)
	127	{
	128	if (!is_root_cache(s))
	129	atomic_add(1 << order, &s->memcg_params->nr_pages);
	130	}
	131
	132	static inline void memcg_release_pages(struct kmem_cache *s, int order)
	133	{
	134	if (is_root_cache(s))
	135	return;
	136
	137	if (atomic_sub_and_test((1 << order), &s->memcg_params->nr_pages))
	138	mem_cgroup_destroy_cache(s);
	139	}
	140
b9ce5ef4 GC	141	static inline bool slab_equal_or_root(struct kmem_cache *s,
	142	struct kmem_cache *p)
	143	{
	144	return (p == s) \|\|
	145	(s->memcg_params && (p == s->memcg_params->root_cache));
	146	}
749c5415 GC	147
	148	/*
	149	* We use suffixes to the name in memcg because we can't have caches
	150	* created in the system with the same name. But when we print them
	151	* locally, better refer to them with the base name
	152	*/
	153	static inline const char cache_name(struct kmem_cache s)
	154	{
	155	if (!is_root_cache(s))
	156	return s->memcg_params->root_cache->name;
	157	return s->name;
	158	}
	159
	160	static inline struct kmem_cache cache_from_memcg(struct kmem_cache s, int idx)
	161	{
	162	return s->memcg_params->memcg_caches[idx];
	163	}
943a451a GC	164
	165	static inline struct kmem_cache memcg_root_cache(struct kmem_cache s)
	166	{
	167	if (is_root_cache(s))
	168	return s;
	169	return s->memcg_params->root_cache;
	170	}
ba6c496e GC	171	#else
	172	static inline bool is_root_cache(struct kmem_cache *s)
	173	{
	174	return true;
	175	}
	176
2633d7a0 GC	177	static inline bool cache_match_memcg(struct kmem_cache *cachep,
	178	struct mem_cgroup *memcg)
	179	{
	180	return true;
	181	}
b9ce5ef4	182
1f458cbf GC	183	static inline void memcg_bind_pages(struct kmem_cache *s, int order)
	184	{
	185	}
	186
	187	static inline void memcg_release_pages(struct kmem_cache *s, int order)
	188	{
	189	}
	190
b9ce5ef4 GC	191	static inline bool slab_equal_or_root(struct kmem_cache *s,
	192	struct kmem_cache *p)
	193	{
	194	return true;
	195	}
749c5415 GC	196
	197	static inline const char cache_name(struct kmem_cache s)
	198	{
	199	return s->name;
	200	}
	201
	202	static inline struct kmem_cache cache_from_memcg(struct kmem_cache s, int idx)
	203	{
	204	return NULL;
	205	}
943a451a GC	206
	207	static inline struct kmem_cache memcg_root_cache(struct kmem_cache s)
	208	{
	209	return s;
	210	}
ba6c496e	211	#endif
b9ce5ef4 GC	212
	213	static inline struct kmem_cache cache_from_obj(struct kmem_cache s, void *x)
	214	{
	215	struct kmem_cache *cachep;
	216	struct page *page;
	217
	218	/*
	219	* When kmemcg is not being used, both assignments should return the
	220	* same value. but we don't want to pay the assignment price in that
	221	* case. If it is not compiled in, the compiler should be smart enough
	222	* to not do even the assignment. In that case, slab_equal_or_root
	223	* will also be a constant.
	224	*/
	225	if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE))
	226	return s;
	227
	228	page = virt_to_head_page(x);
	229	cachep = page->slab_cache;
	230	if (slab_equal_or_root(cachep, s))
	231	return cachep;
	232
	233	pr_err("%s: Wrong slab cache. %s but object is from %s\n",
	234	__FUNCTION__, cachep->name, s->name);
	235	WARN_ON_ONCE(1);
	236	return s;
	237	}
97d06609	238	#endif