mm/slab: activate debug_pagealloc in SLAB when it is actually enabled
[linux-2.6-block.git] / mm / slab.c
CommitLineData
1da177e4
LT
1/*
2 * linux/mm/slab.c
3 * Written by Mark Hemment, 1996/97.
4 * (markhe@nextd.demon.co.uk)
5 *
6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7 *
8 * Major cleanup, different bufctl logic, per-cpu arrays
9 * (c) 2000 Manfred Spraul
10 *
11 * Cleanup, make the head arrays unconditional, preparation for NUMA
12 * (c) 2002 Manfred Spraul
13 *
14 * An implementation of the Slab Allocator as described in outline in;
15 * UNIX Internals: The New Frontiers by Uresh Vahalia
16 * Pub: Prentice Hall ISBN 0-13-101908-2
17 * or with a little more detail in;
18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator
19 * Jeff Bonwick (Sun Microsystems).
20 * Presented at: USENIX Summer 1994 Technical Conference
21 *
22 * The memory is organized in caches, one cache for each object type.
23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24 * Each cache consists out of many slabs (they are small (usually one
25 * page long) and always contiguous), and each slab contains multiple
26 * initialized objects.
27 *
28 * This means, that your constructor is used only for newly allocated
183ff22b 29 * slabs and you must pass objects with the same initializations to
1da177e4
LT
30 * kmem_cache_free.
31 *
32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33 * normal). If you need a special memory type, then must create a new
34 * cache for that memory type.
35 *
36 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
37 * full slabs with 0 free objects
38 * partial slabs
39 * empty slabs with no allocated objects
40 *
41 * If partial slabs exist, then new allocations come from these slabs,
42 * otherwise from empty slabs or new slabs are allocated.
43 *
44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46 *
47 * Each cache has a short per-cpu head array, most allocs
48 * and frees go into that array, and if that array overflows, then 1/2
49 * of the entries in the array are given back into the global cache.
50 * The head array is strictly LIFO and should improve the cache hit rates.
51 * On SMP, it additionally reduces the spinlock operations.
52 *
a737b3e2 53 * The c_cpuarray may not be read with enabled local interrupts -
1da177e4
LT
54 * it's changed with a smp_call_function().
55 *
56 * SMP synchronization:
57 * constructors and destructors are called without any locking.
343e0d7a 58 * Several members in struct kmem_cache and struct slab never change, they
1da177e4
LT
59 * are accessed without any locking.
60 * The per-cpu arrays are never accessed from the wrong cpu, no locking,
61 * and local interrupts are disabled so slab code is preempt-safe.
62 * The non-constant members are protected with a per-cache irq spinlock.
63 *
64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65 * in 2000 - many ideas in the current implementation are derived from
66 * his patch.
67 *
68 * Further notes from the original documentation:
69 *
70 * 11 April '97. Started multi-threading - markhe
18004c5d 71 * The global cache-chain is protected by the mutex 'slab_mutex'.
1da177e4
LT
72 * The sem is only needed when accessing/extending the cache-chain, which
73 * can never happen inside an interrupt (kmem_cache_create(),
74 * kmem_cache_shrink() and kmem_cache_reap()).
75 *
76 * At present, each engine can be growing a cache. This should be blocked.
77 *
e498be7d
CL
78 * 15 March 2005. NUMA slab allocator.
79 * Shai Fultheim <shai@scalex86.org>.
80 * Shobhit Dayal <shobhit@calsoftinc.com>
81 * Alok N Kataria <alokk@calsoftinc.com>
82 * Christoph Lameter <christoph@lameter.com>
83 *
84 * Modified the slab allocator to be node aware on NUMA systems.
85 * Each node has its own list of partial, free and full slabs.
86 * All object allocations for a node occur from node specific slab lists.
1da177e4
LT
87 */
88
1da177e4
LT
89#include <linux/slab.h>
90#include <linux/mm.h>
c9cf5528 91#include <linux/poison.h>
1da177e4
LT
92#include <linux/swap.h>
93#include <linux/cache.h>
94#include <linux/interrupt.h>
95#include <linux/init.h>
96#include <linux/compiler.h>
101a5001 97#include <linux/cpuset.h>
a0ec95a8 98#include <linux/proc_fs.h>
1da177e4
LT
99#include <linux/seq_file.h>
100#include <linux/notifier.h>
101#include <linux/kallsyms.h>
102#include <linux/cpu.h>
103#include <linux/sysctl.h>
104#include <linux/module.h>
105#include <linux/rcupdate.h>
543537bd 106#include <linux/string.h>
138ae663 107#include <linux/uaccess.h>
e498be7d 108#include <linux/nodemask.h>
d5cff635 109#include <linux/kmemleak.h>
dc85da15 110#include <linux/mempolicy.h>
fc0abb14 111#include <linux/mutex.h>
8a8b6502 112#include <linux/fault-inject.h>
e7eebaf6 113#include <linux/rtmutex.h>
6a2d7a95 114#include <linux/reciprocal_div.h>
3ac7fe5a 115#include <linux/debugobjects.h>
c175eea4 116#include <linux/kmemcheck.h>
8f9f8d9e 117#include <linux/memory.h>
268bb0ce 118#include <linux/prefetch.h>
1da177e4 119
381760ea
MG
120#include <net/sock.h>
121
1da177e4
LT
122#include <asm/cacheflush.h>
123#include <asm/tlbflush.h>
124#include <asm/page.h>
125
4dee6b64
SR
126#include <trace/events/kmem.h>
127
072bb0aa
MG
128#include "internal.h"
129
b9ce5ef4
GC
130#include "slab.h"
131
1da177e4 132/*
50953fe9 133 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
1da177e4
LT
134 * 0 for faster, smaller code (especially in the critical paths).
135 *
136 * STATS - 1 to collect stats for /proc/slabinfo.
137 * 0 for faster, smaller code (especially in the critical paths).
138 *
139 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
140 */
141
142#ifdef CONFIG_DEBUG_SLAB
143#define DEBUG 1
144#define STATS 1
145#define FORCED_DEBUG 1
146#else
147#define DEBUG 0
148#define STATS 0
149#define FORCED_DEBUG 0
150#endif
151
1da177e4
LT
152/* Shouldn't this be in a header file somewhere? */
153#define BYTES_PER_WORD sizeof(void *)
87a927c7 154#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
1da177e4 155
1da177e4
LT
156#ifndef ARCH_KMALLOC_FLAGS
157#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
158#endif
159
f315e3fa
JK
160#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
161 <= SLAB_OBJ_MIN_SIZE) ? 1 : 0)
162
163#if FREELIST_BYTE_INDEX
164typedef unsigned char freelist_idx_t;
165#else
166typedef unsigned short freelist_idx_t;
167#endif
168
30321c7b 169#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
f315e3fa 170
072bb0aa
MG
171/*
172 * true if a page was allocated from pfmemalloc reserves for network-based
173 * swap
174 */
175static bool pfmemalloc_active __read_mostly;
176
1da177e4
LT
177/*
178 * struct array_cache
179 *
1da177e4
LT
180 * Purpose:
181 * - LIFO ordering, to hand out cache-warm objects from _alloc
182 * - reduce the number of linked list operations
183 * - reduce spinlock operations
184 *
185 * The limit is stored in the per-cpu structure to reduce the data cache
186 * footprint.
187 *
188 */
189struct array_cache {
190 unsigned int avail;
191 unsigned int limit;
192 unsigned int batchcount;
193 unsigned int touched;
bda5b655 194 void *entry[]; /*
a737b3e2
AM
195 * Must have this definition in here for the proper
196 * alignment of array_cache. Also simplifies accessing
197 * the entries.
072bb0aa
MG
198 *
199 * Entries should not be directly dereferenced as
200 * entries belonging to slabs marked pfmemalloc will
201 * have the lower bits set SLAB_OBJ_PFMEMALLOC
a737b3e2 202 */
1da177e4
LT
203};
204
c8522a3a
JK
205struct alien_cache {
206 spinlock_t lock;
207 struct array_cache ac;
208};
209
072bb0aa
MG
210#define SLAB_OBJ_PFMEMALLOC 1
211static inline bool is_obj_pfmemalloc(void *objp)
212{
213 return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
214}
215
216static inline void set_obj_pfmemalloc(void **objp)
217{
218 *objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC);
219 return;
220}
221
222static inline void clear_obj_pfmemalloc(void **objp)
223{
224 *objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
225}
226
e498be7d
CL
227/*
228 * Need this for bootstrapping a per node allocator.
229 */
bf0dea23 230#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
ce8eb6c4 231static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
e498be7d 232#define CACHE_CACHE 0
bf0dea23 233#define SIZE_NODE (MAX_NUMNODES)
e498be7d 234
ed11d9eb 235static int drain_freelist(struct kmem_cache *cache,
ce8eb6c4 236 struct kmem_cache_node *n, int tofree);
ed11d9eb 237static void free_block(struct kmem_cache *cachep, void **objpp, int len,
97654dfa
JK
238 int node, struct list_head *list);
239static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
83b519e8 240static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
65f27f38 241static void cache_reap(struct work_struct *unused);
ed11d9eb 242
e0a42726
IM
243static int slab_early_init = 1;
244
ce8eb6c4 245#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
1da177e4 246
ce8eb6c4 247static void kmem_cache_node_init(struct kmem_cache_node *parent)
e498be7d
CL
248{
249 INIT_LIST_HEAD(&parent->slabs_full);
250 INIT_LIST_HEAD(&parent->slabs_partial);
251 INIT_LIST_HEAD(&parent->slabs_free);
252 parent->shared = NULL;
253 parent->alien = NULL;
2e1217cf 254 parent->colour_next = 0;
e498be7d
CL
255 spin_lock_init(&parent->list_lock);
256 parent->free_objects = 0;
257 parent->free_touched = 0;
258}
259
a737b3e2
AM
260#define MAKE_LIST(cachep, listp, slab, nodeid) \
261 do { \
262 INIT_LIST_HEAD(listp); \
18bf8541 263 list_splice(&get_node(cachep, nodeid)->slab, listp); \
e498be7d
CL
264 } while (0)
265
a737b3e2
AM
266#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
267 do { \
e498be7d
CL
268 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
269 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
270 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
271 } while (0)
1da177e4 272
1da177e4
LT
273#define CFLGS_OFF_SLAB (0x80000000UL)
274#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
d4322d88 275#define OFF_SLAB_MIN_SIZE (max_t(size_t, PAGE_SIZE >> 5, KMALLOC_MIN_SIZE + 1))
1da177e4
LT
276
277#define BATCHREFILL_LIMIT 16
a737b3e2
AM
278/*
279 * Optimization question: fewer reaps means less probability for unnessary
280 * cpucache drain/refill cycles.
1da177e4 281 *
dc6f3f27 282 * OTOH the cpuarrays can contain lots of objects,
1da177e4
LT
283 * which could lock up otherwise freeable slabs.
284 */
5f0985bb
JZ
285#define REAPTIMEOUT_AC (2*HZ)
286#define REAPTIMEOUT_NODE (4*HZ)
1da177e4
LT
287
288#if STATS
289#define STATS_INC_ACTIVE(x) ((x)->num_active++)
290#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
291#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
292#define STATS_INC_GROWN(x) ((x)->grown++)
ed11d9eb 293#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
a737b3e2
AM
294#define STATS_SET_HIGH(x) \
295 do { \
296 if ((x)->num_active > (x)->high_mark) \
297 (x)->high_mark = (x)->num_active; \
298 } while (0)
1da177e4
LT
299#define STATS_INC_ERR(x) ((x)->errors++)
300#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
e498be7d 301#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
fb7faf33 302#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
a737b3e2
AM
303#define STATS_SET_FREEABLE(x, i) \
304 do { \
305 if ((x)->max_freeable < i) \
306 (x)->max_freeable = i; \
307 } while (0)
1da177e4
LT
308#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
309#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
310#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
311#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
312#else
313#define STATS_INC_ACTIVE(x) do { } while (0)
314#define STATS_DEC_ACTIVE(x) do { } while (0)
315#define STATS_INC_ALLOCED(x) do { } while (0)
316#define STATS_INC_GROWN(x) do { } while (0)
4e60c86b 317#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
1da177e4
LT
318#define STATS_SET_HIGH(x) do { } while (0)
319#define STATS_INC_ERR(x) do { } while (0)
320#define STATS_INC_NODEALLOCS(x) do { } while (0)
e498be7d 321#define STATS_INC_NODEFREES(x) do { } while (0)
fb7faf33 322#define STATS_INC_ACOVERFLOW(x) do { } while (0)
a737b3e2 323#define STATS_SET_FREEABLE(x, i) do { } while (0)
1da177e4
LT
324#define STATS_INC_ALLOCHIT(x) do { } while (0)
325#define STATS_INC_ALLOCMISS(x) do { } while (0)
326#define STATS_INC_FREEHIT(x) do { } while (0)
327#define STATS_INC_FREEMISS(x) do { } while (0)
328#endif
329
330#if DEBUG
1da177e4 331
a737b3e2
AM
332/*
333 * memory layout of objects:
1da177e4 334 * 0 : objp
3dafccf2 335 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
1da177e4
LT
336 * the end of an object is aligned with the end of the real
337 * allocation. Catches writes behind the end of the allocation.
3dafccf2 338 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
1da177e4 339 * redzone word.
3dafccf2 340 * cachep->obj_offset: The real object.
3b0efdfa
CL
341 * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
342 * cachep->size - 1* BYTES_PER_WORD: last caller address
a737b3e2 343 * [BYTES_PER_WORD long]
1da177e4 344 */
343e0d7a 345static int obj_offset(struct kmem_cache *cachep)
1da177e4 346{
3dafccf2 347 return cachep->obj_offset;
1da177e4
LT
348}
349
b46b8f19 350static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
1da177e4
LT
351{
352 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
b46b8f19
DW
353 return (unsigned long long*) (objp + obj_offset(cachep) -
354 sizeof(unsigned long long));
1da177e4
LT
355}
356
b46b8f19 357static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
1da177e4
LT
358{
359 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
360 if (cachep->flags & SLAB_STORE_USER)
3b0efdfa 361 return (unsigned long long *)(objp + cachep->size -
b46b8f19 362 sizeof(unsigned long long) -
87a927c7 363 REDZONE_ALIGN);
3b0efdfa 364 return (unsigned long long *) (objp + cachep->size -
b46b8f19 365 sizeof(unsigned long long));
1da177e4
LT
366}
367
343e0d7a 368static void **dbg_userword(struct kmem_cache *cachep, void *objp)
1da177e4
LT
369{
370 BUG_ON(!(cachep->flags & SLAB_STORE_USER));
3b0efdfa 371 return (void **)(objp + cachep->size - BYTES_PER_WORD);
1da177e4
LT
372}
373
374#else
375
3dafccf2 376#define obj_offset(x) 0
b46b8f19
DW
377#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
378#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
1da177e4
LT
379#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
380
381#endif
382
03787301
JK
383#define OBJECT_FREE (0)
384#define OBJECT_ACTIVE (1)
385
386#ifdef CONFIG_DEBUG_SLAB_LEAK
387
388static void set_obj_status(struct page *page, int idx, int val)
389{
390 int freelist_size;
391 char *status;
392 struct kmem_cache *cachep = page->slab_cache;
393
394 freelist_size = cachep->num * sizeof(freelist_idx_t);
395 status = (char *)page->freelist + freelist_size;
396 status[idx] = val;
397}
398
399static inline unsigned int get_obj_status(struct page *page, int idx)
400{
401 int freelist_size;
402 char *status;
403 struct kmem_cache *cachep = page->slab_cache;
404
405 freelist_size = cachep->num * sizeof(freelist_idx_t);
406 status = (char *)page->freelist + freelist_size;
407
408 return status[idx];
409}
410
411#else
412static inline void set_obj_status(struct page *page, int idx, int val) {}
413
414#endif
415
1da177e4 416/*
3df1cccd
DR
417 * Do not go above this order unless 0 objects fit into the slab or
418 * overridden on the command line.
1da177e4 419 */
543585cc
DR
420#define SLAB_MAX_ORDER_HI 1
421#define SLAB_MAX_ORDER_LO 0
422static int slab_max_order = SLAB_MAX_ORDER_LO;
3df1cccd 423static bool slab_max_order_set __initdata;
1da177e4 424
6ed5eb22
PE
425static inline struct kmem_cache *virt_to_cache(const void *obj)
426{
b49af68f 427 struct page *page = virt_to_head_page(obj);
35026088 428 return page->slab_cache;
6ed5eb22
PE
429}
430
8456a648 431static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
8fea4e96
PE
432 unsigned int idx)
433{
8456a648 434 return page->s_mem + cache->size * idx;
8fea4e96
PE
435}
436
6a2d7a95 437/*
3b0efdfa
CL
438 * We want to avoid an expensive divide : (offset / cache->size)
439 * Using the fact that size is a constant for a particular cache,
440 * we can replace (offset / cache->size) by
6a2d7a95
ED
441 * reciprocal_divide(offset, cache->reciprocal_buffer_size)
442 */
443static inline unsigned int obj_to_index(const struct kmem_cache *cache,
8456a648 444 const struct page *page, void *obj)
8fea4e96 445{
8456a648 446 u32 offset = (obj - page->s_mem);
6a2d7a95 447 return reciprocal_divide(offset, cache->reciprocal_buffer_size);
8fea4e96
PE
448}
449
6fb92430 450#define BOOT_CPUCACHE_ENTRIES 1
1da177e4 451/* internal cache of cache description objs */
9b030cb8 452static struct kmem_cache kmem_cache_boot = {
b28a02de
PE
453 .batchcount = 1,
454 .limit = BOOT_CPUCACHE_ENTRIES,
455 .shared = 1,
3b0efdfa 456 .size = sizeof(struct kmem_cache),
b28a02de 457 .name = "kmem_cache",
1da177e4
LT
458};
459
edcad250
JK
460#define BAD_ALIEN_MAGIC 0x01020304ul
461
1871e52c 462static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
1da177e4 463
343e0d7a 464static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
1da177e4 465{
bf0dea23 466 return this_cpu_ptr(cachep->cpu_cache);
1da177e4
LT
467}
468
03787301
JK
469static size_t calculate_freelist_size(int nr_objs, size_t align)
470{
471 size_t freelist_size;
472
473 freelist_size = nr_objs * sizeof(freelist_idx_t);
474 if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
475 freelist_size += nr_objs * sizeof(char);
476
477 if (align)
478 freelist_size = ALIGN(freelist_size, align);
479
480 return freelist_size;
481}
482
9cef2e2b
JK
483static int calculate_nr_objs(size_t slab_size, size_t buffer_size,
484 size_t idx_size, size_t align)
1da177e4 485{
9cef2e2b 486 int nr_objs;
03787301 487 size_t remained_size;
9cef2e2b 488 size_t freelist_size;
03787301 489 int extra_space = 0;
9cef2e2b 490
03787301
JK
491 if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
492 extra_space = sizeof(char);
9cef2e2b
JK
493 /*
494 * Ignore padding for the initial guess. The padding
495 * is at most @align-1 bytes, and @buffer_size is at
496 * least @align. In the worst case, this result will
497 * be one greater than the number of objects that fit
498 * into the memory allocation when taking the padding
499 * into account.
500 */
03787301 501 nr_objs = slab_size / (buffer_size + idx_size + extra_space);
9cef2e2b
JK
502
503 /*
504 * This calculated number will be either the right
505 * amount, or one greater than what we want.
506 */
03787301
JK
507 remained_size = slab_size - nr_objs * buffer_size;
508 freelist_size = calculate_freelist_size(nr_objs, align);
509 if (remained_size < freelist_size)
9cef2e2b
JK
510 nr_objs--;
511
512 return nr_objs;
fbaccacf 513}
1da177e4 514
a737b3e2
AM
515/*
516 * Calculate the number of objects and left-over bytes for a given buffer size.
517 */
fbaccacf
SR
518static void cache_estimate(unsigned long gfporder, size_t buffer_size,
519 size_t align, int flags, size_t *left_over,
520 unsigned int *num)
521{
522 int nr_objs;
523 size_t mgmt_size;
524 size_t slab_size = PAGE_SIZE << gfporder;
1da177e4 525
fbaccacf
SR
526 /*
527 * The slab management structure can be either off the slab or
528 * on it. For the latter case, the memory allocated for a
529 * slab is used for:
530 *
12c61fe9 531 * - One freelist_idx_t for each object
fbaccacf
SR
532 * - Padding to respect alignment of @align
533 * - @buffer_size bytes for each object
534 *
535 * If the slab management structure is off the slab, then the
536 * alignment will already be calculated into the size. Because
537 * the slabs are all pages aligned, the objects will be at the
538 * correct alignment when allocated.
539 */
540 if (flags & CFLGS_OFF_SLAB) {
541 mgmt_size = 0;
542 nr_objs = slab_size / buffer_size;
543
fbaccacf 544 } else {
9cef2e2b 545 nr_objs = calculate_nr_objs(slab_size, buffer_size,
a41adfaa 546 sizeof(freelist_idx_t), align);
03787301 547 mgmt_size = calculate_freelist_size(nr_objs, align);
fbaccacf
SR
548 }
549 *num = nr_objs;
550 *left_over = slab_size - nr_objs*buffer_size - mgmt_size;
1da177e4
LT
551}
552
f28510d3 553#if DEBUG
d40cee24 554#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
1da177e4 555
a737b3e2
AM
556static void __slab_error(const char *function, struct kmem_cache *cachep,
557 char *msg)
1da177e4
LT
558{
559 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
b28a02de 560 function, cachep->name, msg);
1da177e4 561 dump_stack();
373d4d09 562 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1da177e4 563}
f28510d3 564#endif
1da177e4 565
3395ee05
PM
566/*
567 * By default on NUMA we use alien caches to stage the freeing of
568 * objects allocated from other nodes. This causes massive memory
569 * inefficiencies when using fake NUMA setup to split memory into a
570 * large number of small nodes, so it can be disabled on the command
571 * line
572 */
573
574static int use_alien_caches __read_mostly = 1;
575static int __init noaliencache_setup(char *s)
576{
577 use_alien_caches = 0;
578 return 1;
579}
580__setup("noaliencache", noaliencache_setup);
581
3df1cccd
DR
582static int __init slab_max_order_setup(char *str)
583{
584 get_option(&str, &slab_max_order);
585 slab_max_order = slab_max_order < 0 ? 0 :
586 min(slab_max_order, MAX_ORDER - 1);
587 slab_max_order_set = true;
588
589 return 1;
590}
591__setup("slab_max_order=", slab_max_order_setup);
592
8fce4d8e
CL
593#ifdef CONFIG_NUMA
594/*
595 * Special reaping functions for NUMA systems called from cache_reap().
596 * These take care of doing round robin flushing of alien caches (containing
597 * objects freed on different nodes from which they were allocated) and the
598 * flushing of remote pcps by calling drain_node_pages.
599 */
1871e52c 600static DEFINE_PER_CPU(unsigned long, slab_reap_node);
8fce4d8e
CL
601
602static void init_reap_node(int cpu)
603{
604 int node;
605
7d6e6d09 606 node = next_node(cpu_to_mem(cpu), node_online_map);
8fce4d8e 607 if (node == MAX_NUMNODES)
442295c9 608 node = first_node(node_online_map);
8fce4d8e 609
1871e52c 610 per_cpu(slab_reap_node, cpu) = node;
8fce4d8e
CL
611}
612
613static void next_reap_node(void)
614{
909ea964 615 int node = __this_cpu_read(slab_reap_node);
8fce4d8e 616
8fce4d8e
CL
617 node = next_node(node, node_online_map);
618 if (unlikely(node >= MAX_NUMNODES))
619 node = first_node(node_online_map);
909ea964 620 __this_cpu_write(slab_reap_node, node);
8fce4d8e
CL
621}
622
623#else
624#define init_reap_node(cpu) do { } while (0)
625#define next_reap_node(void) do { } while (0)
626#endif
627
1da177e4
LT
628/*
629 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
630 * via the workqueue/eventd.
631 * Add the CPU number into the expiration time to minimize the possibility of
632 * the CPUs getting into lockstep and contending for the global cache chain
633 * lock.
634 */
0db0628d 635static void start_cpu_timer(int cpu)
1da177e4 636{
1871e52c 637 struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
1da177e4
LT
638
639 /*
640 * When this gets called from do_initcalls via cpucache_init(),
641 * init_workqueues() has already run, so keventd will be setup
642 * at that time.
643 */
52bad64d 644 if (keventd_up() && reap_work->work.func == NULL) {
8fce4d8e 645 init_reap_node(cpu);
203b42f7 646 INIT_DEFERRABLE_WORK(reap_work, cache_reap);
2b284214
AV
647 schedule_delayed_work_on(cpu, reap_work,
648 __round_jiffies_relative(HZ, cpu));
1da177e4
LT
649 }
650}
651
1fe00d50 652static void init_arraycache(struct array_cache *ac, int limit, int batch)
1da177e4 653{
d5cff635
CM
654 /*
655 * The array_cache structures contain pointers to free object.
25985edc 656 * However, when such objects are allocated or transferred to another
d5cff635
CM
657 * cache the pointers are not cleared and they could be counted as
658 * valid references during a kmemleak scan. Therefore, kmemleak must
659 * not scan such objects.
660 */
1fe00d50
JK
661 kmemleak_no_scan(ac);
662 if (ac) {
663 ac->avail = 0;
664 ac->limit = limit;
665 ac->batchcount = batch;
666 ac->touched = 0;
1da177e4 667 }
1fe00d50
JK
668}
669
670static struct array_cache *alloc_arraycache(int node, int entries,
671 int batchcount, gfp_t gfp)
672{
5e804789 673 size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
1fe00d50
JK
674 struct array_cache *ac = NULL;
675
676 ac = kmalloc_node(memsize, gfp, node);
677 init_arraycache(ac, entries, batchcount);
678 return ac;
1da177e4
LT
679}
680
8456a648 681static inline bool is_slab_pfmemalloc(struct page *page)
072bb0aa 682{
072bb0aa
MG
683 return PageSlabPfmemalloc(page);
684}
685
686/* Clears pfmemalloc_active if no slabs have pfmalloc set */
687static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
688 struct array_cache *ac)
689{
18bf8541 690 struct kmem_cache_node *n = get_node(cachep, numa_mem_id());
8456a648 691 struct page *page;
072bb0aa
MG
692 unsigned long flags;
693
694 if (!pfmemalloc_active)
695 return;
696
ce8eb6c4 697 spin_lock_irqsave(&n->list_lock, flags);
8456a648
JK
698 list_for_each_entry(page, &n->slabs_full, lru)
699 if (is_slab_pfmemalloc(page))
072bb0aa
MG
700 goto out;
701
8456a648
JK
702 list_for_each_entry(page, &n->slabs_partial, lru)
703 if (is_slab_pfmemalloc(page))
072bb0aa
MG
704 goto out;
705
8456a648
JK
706 list_for_each_entry(page, &n->slabs_free, lru)
707 if (is_slab_pfmemalloc(page))
072bb0aa
MG
708 goto out;
709
710 pfmemalloc_active = false;
711out:
ce8eb6c4 712 spin_unlock_irqrestore(&n->list_lock, flags);
072bb0aa
MG
713}
714
381760ea 715static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,
072bb0aa
MG
716 gfp_t flags, bool force_refill)
717{
718 int i;
719 void *objp = ac->entry[--ac->avail];
720
721 /* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
722 if (unlikely(is_obj_pfmemalloc(objp))) {
ce8eb6c4 723 struct kmem_cache_node *n;
072bb0aa
MG
724
725 if (gfp_pfmemalloc_allowed(flags)) {
726 clear_obj_pfmemalloc(&objp);
727 return objp;
728 }
729
730 /* The caller cannot use PFMEMALLOC objects, find another one */
d014dc2e 731 for (i = 0; i < ac->avail; i++) {
072bb0aa
MG
732 /* If a !PFMEMALLOC object is found, swap them */
733 if (!is_obj_pfmemalloc(ac->entry[i])) {
734 objp = ac->entry[i];
735 ac->entry[i] = ac->entry[ac->avail];
736 ac->entry[ac->avail] = objp;
737 return objp;
738 }
739 }
740
741 /*
742 * If there are empty slabs on the slabs_free list and we are
743 * being forced to refill the cache, mark this one !pfmemalloc.
744 */
18bf8541 745 n = get_node(cachep, numa_mem_id());
ce8eb6c4 746 if (!list_empty(&n->slabs_free) && force_refill) {
8456a648 747 struct page *page = virt_to_head_page(objp);
7ecccf9d 748 ClearPageSlabPfmemalloc(page);
072bb0aa
MG
749 clear_obj_pfmemalloc(&objp);
750 recheck_pfmemalloc_active(cachep, ac);
751 return objp;
752 }
753
754 /* No !PFMEMALLOC objects available */
755 ac->avail++;
756 objp = NULL;
757 }
758
759 return objp;
760}
761
381760ea
MG
762static inline void *ac_get_obj(struct kmem_cache *cachep,
763 struct array_cache *ac, gfp_t flags, bool force_refill)
764{
765 void *objp;
766
767 if (unlikely(sk_memalloc_socks()))
768 objp = __ac_get_obj(cachep, ac, flags, force_refill);
769 else
770 objp = ac->entry[--ac->avail];
771
772 return objp;
773}
774
d3aec344
JK
775static noinline void *__ac_put_obj(struct kmem_cache *cachep,
776 struct array_cache *ac, void *objp)
072bb0aa
MG
777{
778 if (unlikely(pfmemalloc_active)) {
779 /* Some pfmemalloc slabs exist, check if this is one */
30c29bea 780 struct page *page = virt_to_head_page(objp);
072bb0aa
MG
781 if (PageSlabPfmemalloc(page))
782 set_obj_pfmemalloc(&objp);
783 }
784
381760ea
MG
785 return objp;
786}
787
788static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
789 void *objp)
790{
791 if (unlikely(sk_memalloc_socks()))
792 objp = __ac_put_obj(cachep, ac, objp);
793
072bb0aa
MG
794 ac->entry[ac->avail++] = objp;
795}
796
3ded175a
CL
797/*
798 * Transfer objects in one arraycache to another.
799 * Locking must be handled by the caller.
800 *
801 * Return the number of entries transferred.
802 */
803static int transfer_objects(struct array_cache *to,
804 struct array_cache *from, unsigned int max)
805{
806 /* Figure out how many entries to transfer */
732eacc0 807 int nr = min3(from->avail, max, to->limit - to->avail);
3ded175a
CL
808
809 if (!nr)
810 return 0;
811
812 memcpy(to->entry + to->avail, from->entry + from->avail -nr,
813 sizeof(void *) *nr);
814
815 from->avail -= nr;
816 to->avail += nr;
3ded175a
CL
817 return nr;
818}
819
765c4507
CL
820#ifndef CONFIG_NUMA
821
822#define drain_alien_cache(cachep, alien) do { } while (0)
ce8eb6c4 823#define reap_alien(cachep, n) do { } while (0)
765c4507 824
c8522a3a
JK
825static inline struct alien_cache **alloc_alien_cache(int node,
826 int limit, gfp_t gfp)
765c4507 827{
edcad250 828 return (struct alien_cache **)BAD_ALIEN_MAGIC;
765c4507
CL
829}
830
c8522a3a 831static inline void free_alien_cache(struct alien_cache **ac_ptr)
765c4507
CL
832{
833}
834
835static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
836{
837 return 0;
838}
839
840static inline void *alternate_node_alloc(struct kmem_cache *cachep,
841 gfp_t flags)
842{
843 return NULL;
844}
845
8b98c169 846static inline void *____cache_alloc_node(struct kmem_cache *cachep,
765c4507
CL
847 gfp_t flags, int nodeid)
848{
849 return NULL;
850}
851
4167e9b2
DR
852static inline gfp_t gfp_exact_node(gfp_t flags)
853{
854 return flags;
855}
856
765c4507
CL
857#else /* CONFIG_NUMA */
858
8b98c169 859static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
c61afb18 860static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
dc85da15 861
c8522a3a
JK
862static struct alien_cache *__alloc_alien_cache(int node, int entries,
863 int batch, gfp_t gfp)
864{
5e804789 865 size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
c8522a3a
JK
866 struct alien_cache *alc = NULL;
867
868 alc = kmalloc_node(memsize, gfp, node);
869 init_arraycache(&alc->ac, entries, batch);
49dfc304 870 spin_lock_init(&alc->lock);
c8522a3a
JK
871 return alc;
872}
873
874static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
e498be7d 875{
c8522a3a 876 struct alien_cache **alc_ptr;
5e804789 877 size_t memsize = sizeof(void *) * nr_node_ids;
e498be7d
CL
878 int i;
879
880 if (limit > 1)
881 limit = 12;
c8522a3a
JK
882 alc_ptr = kzalloc_node(memsize, gfp, node);
883 if (!alc_ptr)
884 return NULL;
885
886 for_each_node(i) {
887 if (i == node || !node_online(i))
888 continue;
889 alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
890 if (!alc_ptr[i]) {
891 for (i--; i >= 0; i--)
892 kfree(alc_ptr[i]);
893 kfree(alc_ptr);
894 return NULL;
e498be7d
CL
895 }
896 }
c8522a3a 897 return alc_ptr;
e498be7d
CL
898}
899
c8522a3a 900static void free_alien_cache(struct alien_cache **alc_ptr)
e498be7d
CL
901{
902 int i;
903
c8522a3a 904 if (!alc_ptr)
e498be7d 905 return;
e498be7d 906 for_each_node(i)
c8522a3a
JK
907 kfree(alc_ptr[i]);
908 kfree(alc_ptr);
e498be7d
CL
909}
910
343e0d7a 911static void __drain_alien_cache(struct kmem_cache *cachep,
833b706c
JK
912 struct array_cache *ac, int node,
913 struct list_head *list)
e498be7d 914{
18bf8541 915 struct kmem_cache_node *n = get_node(cachep, node);
e498be7d
CL
916
917 if (ac->avail) {
ce8eb6c4 918 spin_lock(&n->list_lock);
e00946fe
CL
919 /*
920 * Stuff objects into the remote nodes shared array first.
921 * That way we could avoid the overhead of putting the objects
922 * into the free lists and getting them back later.
923 */
ce8eb6c4
CL
924 if (n->shared)
925 transfer_objects(n->shared, ac, ac->limit);
e00946fe 926
833b706c 927 free_block(cachep, ac->entry, ac->avail, node, list);
e498be7d 928 ac->avail = 0;
ce8eb6c4 929 spin_unlock(&n->list_lock);
e498be7d
CL
930 }
931}
932
8fce4d8e
CL
933/*
934 * Called from cache_reap() to regularly drain alien caches round robin.
935 */
ce8eb6c4 936static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
8fce4d8e 937{
909ea964 938 int node = __this_cpu_read(slab_reap_node);
8fce4d8e 939
ce8eb6c4 940 if (n->alien) {
c8522a3a
JK
941 struct alien_cache *alc = n->alien[node];
942 struct array_cache *ac;
943
944 if (alc) {
945 ac = &alc->ac;
49dfc304 946 if (ac->avail && spin_trylock_irq(&alc->lock)) {
833b706c
JK
947 LIST_HEAD(list);
948
949 __drain_alien_cache(cachep, ac, node, &list);
49dfc304 950 spin_unlock_irq(&alc->lock);
833b706c 951 slabs_destroy(cachep, &list);
c8522a3a 952 }
8fce4d8e
CL
953 }
954 }
955}
956
a737b3e2 957static void drain_alien_cache(struct kmem_cache *cachep,
c8522a3a 958 struct alien_cache **alien)
e498be7d 959{
b28a02de 960 int i = 0;
c8522a3a 961 struct alien_cache *alc;
e498be7d
CL
962 struct array_cache *ac;
963 unsigned long flags;
964
965 for_each_online_node(i) {
c8522a3a
JK
966 alc = alien[i];
967 if (alc) {
833b706c
JK
968 LIST_HEAD(list);
969
c8522a3a 970 ac = &alc->ac;
49dfc304 971 spin_lock_irqsave(&alc->lock, flags);
833b706c 972 __drain_alien_cache(cachep, ac, i, &list);
49dfc304 973 spin_unlock_irqrestore(&alc->lock, flags);
833b706c 974 slabs_destroy(cachep, &list);
e498be7d
CL
975 }
976 }
977}
729bd0b7 978
25c4f304
JK
979static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
980 int node, int page_node)
729bd0b7 981{
ce8eb6c4 982 struct kmem_cache_node *n;
c8522a3a
JK
983 struct alien_cache *alien = NULL;
984 struct array_cache *ac;
97654dfa 985 LIST_HEAD(list);
1ca4cb24 986
18bf8541 987 n = get_node(cachep, node);
729bd0b7 988 STATS_INC_NODEFREES(cachep);
25c4f304
JK
989 if (n->alien && n->alien[page_node]) {
990 alien = n->alien[page_node];
c8522a3a 991 ac = &alien->ac;
49dfc304 992 spin_lock(&alien->lock);
c8522a3a 993 if (unlikely(ac->avail == ac->limit)) {
729bd0b7 994 STATS_INC_ACOVERFLOW(cachep);
25c4f304 995 __drain_alien_cache(cachep, ac, page_node, &list);
729bd0b7 996 }
c8522a3a 997 ac_put_obj(cachep, ac, objp);
49dfc304 998 spin_unlock(&alien->lock);
833b706c 999 slabs_destroy(cachep, &list);
729bd0b7 1000 } else {
25c4f304 1001 n = get_node(cachep, page_node);
18bf8541 1002 spin_lock(&n->list_lock);
25c4f304 1003 free_block(cachep, &objp, 1, page_node, &list);
18bf8541 1004 spin_unlock(&n->list_lock);
97654dfa 1005 slabs_destroy(cachep, &list);
729bd0b7
PE
1006 }
1007 return 1;
1008}
25c4f304
JK
1009
1010static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
1011{
1012 int page_node = page_to_nid(virt_to_page(objp));
1013 int node = numa_mem_id();
1014 /*
1015 * Make sure we are not freeing a object from another node to the array
1016 * cache on this cpu.
1017 */
1018 if (likely(node == page_node))
1019 return 0;
1020
1021 return __cache_free_alien(cachep, objp, node, page_node);
1022}
4167e9b2
DR
1023
1024/*
d0164adc
MG
1025 * Construct gfp mask to allocate from a specific node but do not direct reclaim
1026 * or warn about failures. kswapd may still wake to reclaim in the background.
4167e9b2
DR
1027 */
1028static inline gfp_t gfp_exact_node(gfp_t flags)
1029{
d0164adc 1030 return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~__GFP_DIRECT_RECLAIM;
4167e9b2 1031}
e498be7d
CL
1032#endif
1033
8f9f8d9e 1034/*
6a67368c 1035 * Allocates and initializes node for a node on each slab cache, used for
ce8eb6c4 1036 * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
8f9f8d9e 1037 * will be allocated off-node since memory is not yet online for the new node.
6a67368c 1038 * When hotplugging memory or a cpu, existing node are not replaced if
8f9f8d9e
DR
1039 * already in use.
1040 *
18004c5d 1041 * Must hold slab_mutex.
8f9f8d9e 1042 */
6a67368c 1043static int init_cache_node_node(int node)
8f9f8d9e
DR
1044{
1045 struct kmem_cache *cachep;
ce8eb6c4 1046 struct kmem_cache_node *n;
5e804789 1047 const size_t memsize = sizeof(struct kmem_cache_node);
8f9f8d9e 1048
18004c5d 1049 list_for_each_entry(cachep, &slab_caches, list) {
8f9f8d9e 1050 /*
5f0985bb 1051 * Set up the kmem_cache_node for cpu before we can
8f9f8d9e
DR
1052 * begin anything. Make sure some other cpu on this
1053 * node has not already allocated this
1054 */
18bf8541
CL
1055 n = get_node(cachep, node);
1056 if (!n) {
ce8eb6c4
CL
1057 n = kmalloc_node(memsize, GFP_KERNEL, node);
1058 if (!n)
8f9f8d9e 1059 return -ENOMEM;
ce8eb6c4 1060 kmem_cache_node_init(n);
5f0985bb
JZ
1061 n->next_reap = jiffies + REAPTIMEOUT_NODE +
1062 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
8f9f8d9e
DR
1063
1064 /*
5f0985bb
JZ
1065 * The kmem_cache_nodes don't come and go as CPUs
1066 * come and go. slab_mutex is sufficient
8f9f8d9e
DR
1067 * protection here.
1068 */
ce8eb6c4 1069 cachep->node[node] = n;
8f9f8d9e
DR
1070 }
1071
18bf8541
CL
1072 spin_lock_irq(&n->list_lock);
1073 n->free_limit =
8f9f8d9e
DR
1074 (1 + nr_cpus_node(node)) *
1075 cachep->batchcount + cachep->num;
18bf8541 1076 spin_unlock_irq(&n->list_lock);
8f9f8d9e
DR
1077 }
1078 return 0;
1079}
1080
0fa8103b
WL
1081static inline int slabs_tofree(struct kmem_cache *cachep,
1082 struct kmem_cache_node *n)
1083{
1084 return (n->free_objects + cachep->num - 1) / cachep->num;
1085}
1086
0db0628d 1087static void cpuup_canceled(long cpu)
fbf1e473
AM
1088{
1089 struct kmem_cache *cachep;
ce8eb6c4 1090 struct kmem_cache_node *n = NULL;
7d6e6d09 1091 int node = cpu_to_mem(cpu);
a70f7302 1092 const struct cpumask *mask = cpumask_of_node(node);
fbf1e473 1093
18004c5d 1094 list_for_each_entry(cachep, &slab_caches, list) {
fbf1e473
AM
1095 struct array_cache *nc;
1096 struct array_cache *shared;
c8522a3a 1097 struct alien_cache **alien;
97654dfa 1098 LIST_HEAD(list);
fbf1e473 1099
18bf8541 1100 n = get_node(cachep, node);
ce8eb6c4 1101 if (!n)
bf0dea23 1102 continue;
fbf1e473 1103
ce8eb6c4 1104 spin_lock_irq(&n->list_lock);
fbf1e473 1105
ce8eb6c4
CL
1106 /* Free limit for this kmem_cache_node */
1107 n->free_limit -= cachep->batchcount;
bf0dea23
JK
1108
1109 /* cpu is dead; no one can alloc from it. */
1110 nc = per_cpu_ptr(cachep->cpu_cache, cpu);
1111 if (nc) {
97654dfa 1112 free_block(cachep, nc->entry, nc->avail, node, &list);
bf0dea23
JK
1113 nc->avail = 0;
1114 }
fbf1e473 1115
58463c1f 1116 if (!cpumask_empty(mask)) {
ce8eb6c4 1117 spin_unlock_irq(&n->list_lock);
bf0dea23 1118 goto free_slab;
fbf1e473
AM
1119 }
1120
ce8eb6c4 1121 shared = n->shared;
fbf1e473
AM
1122 if (shared) {
1123 free_block(cachep, shared->entry,
97654dfa 1124 shared->avail, node, &list);
ce8eb6c4 1125 n->shared = NULL;
fbf1e473
AM
1126 }
1127
ce8eb6c4
CL
1128 alien = n->alien;
1129 n->alien = NULL;
fbf1e473 1130
ce8eb6c4 1131 spin_unlock_irq(&n->list_lock);
fbf1e473
AM
1132
1133 kfree(shared);
1134 if (alien) {
1135 drain_alien_cache(cachep, alien);
1136 free_alien_cache(alien);
1137 }
bf0dea23
JK
1138
1139free_slab:
97654dfa 1140 slabs_destroy(cachep, &list);
fbf1e473
AM
1141 }
1142 /*
1143 * In the previous loop, all the objects were freed to
1144 * the respective cache's slabs, now we can go ahead and
1145 * shrink each nodelist to its limit.
1146 */
18004c5d 1147 list_for_each_entry(cachep, &slab_caches, list) {
18bf8541 1148 n = get_node(cachep, node);
ce8eb6c4 1149 if (!n)
fbf1e473 1150 continue;
0fa8103b 1151 drain_freelist(cachep, n, slabs_tofree(cachep, n));
fbf1e473
AM
1152 }
1153}
1154
0db0628d 1155static int cpuup_prepare(long cpu)
1da177e4 1156{
343e0d7a 1157 struct kmem_cache *cachep;
ce8eb6c4 1158 struct kmem_cache_node *n = NULL;
7d6e6d09 1159 int node = cpu_to_mem(cpu);
8f9f8d9e 1160 int err;
1da177e4 1161
fbf1e473
AM
1162 /*
1163 * We need to do this right in the beginning since
1164 * alloc_arraycache's are going to use this list.
1165 * kmalloc_node allows us to add the slab to the right
ce8eb6c4 1166 * kmem_cache_node and not this cpu's kmem_cache_node
fbf1e473 1167 */
6a67368c 1168 err = init_cache_node_node(node);
8f9f8d9e
DR
1169 if (err < 0)
1170 goto bad;
fbf1e473
AM
1171
1172 /*
1173 * Now we can go ahead with allocating the shared arrays and
1174 * array caches
1175 */
18004c5d 1176 list_for_each_entry(cachep, &slab_caches, list) {
fbf1e473 1177 struct array_cache *shared = NULL;
c8522a3a 1178 struct alien_cache **alien = NULL;
fbf1e473 1179
fbf1e473
AM
1180 if (cachep->shared) {
1181 shared = alloc_arraycache(node,
1182 cachep->shared * cachep->batchcount,
83b519e8 1183 0xbaadf00d, GFP_KERNEL);
bf0dea23 1184 if (!shared)
1da177e4 1185 goto bad;
fbf1e473
AM
1186 }
1187 if (use_alien_caches) {
83b519e8 1188 alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
12d00f6a
AM
1189 if (!alien) {
1190 kfree(shared);
fbf1e473 1191 goto bad;
12d00f6a 1192 }
fbf1e473 1193 }
18bf8541 1194 n = get_node(cachep, node);
ce8eb6c4 1195 BUG_ON(!n);
fbf1e473 1196
ce8eb6c4
CL
1197 spin_lock_irq(&n->list_lock);
1198 if (!n->shared) {
fbf1e473
AM
1199 /*
1200 * We are serialised from CPU_DEAD or
1201 * CPU_UP_CANCELLED by the cpucontrol lock
1202 */
ce8eb6c4 1203 n->shared = shared;
fbf1e473
AM
1204 shared = NULL;
1205 }
4484ebf1 1206#ifdef CONFIG_NUMA
ce8eb6c4
CL
1207 if (!n->alien) {
1208 n->alien = alien;
fbf1e473 1209 alien = NULL;
1da177e4 1210 }
fbf1e473 1211#endif
ce8eb6c4 1212 spin_unlock_irq(&n->list_lock);
fbf1e473
AM
1213 kfree(shared);
1214 free_alien_cache(alien);
1215 }
ce79ddc8 1216
fbf1e473
AM
1217 return 0;
1218bad:
12d00f6a 1219 cpuup_canceled(cpu);
fbf1e473
AM
1220 return -ENOMEM;
1221}
1222
0db0628d 1223static int cpuup_callback(struct notifier_block *nfb,
fbf1e473
AM
1224 unsigned long action, void *hcpu)
1225{
1226 long cpu = (long)hcpu;
1227 int err = 0;
1228
1229 switch (action) {
fbf1e473
AM
1230 case CPU_UP_PREPARE:
1231 case CPU_UP_PREPARE_FROZEN:
18004c5d 1232 mutex_lock(&slab_mutex);
fbf1e473 1233 err = cpuup_prepare(cpu);
18004c5d 1234 mutex_unlock(&slab_mutex);
1da177e4
LT
1235 break;
1236 case CPU_ONLINE:
8bb78442 1237 case CPU_ONLINE_FROZEN:
1da177e4
LT
1238 start_cpu_timer(cpu);
1239 break;
1240#ifdef CONFIG_HOTPLUG_CPU
5830c590 1241 case CPU_DOWN_PREPARE:
8bb78442 1242 case CPU_DOWN_PREPARE_FROZEN:
5830c590 1243 /*
18004c5d 1244 * Shutdown cache reaper. Note that the slab_mutex is
5830c590
CL
1245 * held so that if cache_reap() is invoked it cannot do
1246 * anything expensive but will only modify reap_work
1247 * and reschedule the timer.
1248 */
afe2c511 1249 cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
5830c590 1250 /* Now the cache_reaper is guaranteed to be not running. */
1871e52c 1251 per_cpu(slab_reap_work, cpu).work.func = NULL;
5830c590
CL
1252 break;
1253 case CPU_DOWN_FAILED:
8bb78442 1254 case CPU_DOWN_FAILED_FROZEN:
5830c590
CL
1255 start_cpu_timer(cpu);
1256 break;
1da177e4 1257 case CPU_DEAD:
8bb78442 1258 case CPU_DEAD_FROZEN:
4484ebf1
RT
1259 /*
1260 * Even if all the cpus of a node are down, we don't free the
ce8eb6c4 1261 * kmem_cache_node of any cache. This to avoid a race between
4484ebf1 1262 * cpu_down, and a kmalloc allocation from another cpu for
ce8eb6c4 1263 * memory from the node of the cpu going down. The node
4484ebf1
RT
1264 * structure is usually allocated from kmem_cache_create() and
1265 * gets destroyed at kmem_cache_destroy().
1266 */
183ff22b 1267 /* fall through */
8f5be20b 1268#endif
1da177e4 1269 case CPU_UP_CANCELED:
8bb78442 1270 case CPU_UP_CANCELED_FROZEN:
18004c5d 1271 mutex_lock(&slab_mutex);
fbf1e473 1272 cpuup_canceled(cpu);
18004c5d 1273 mutex_unlock(&slab_mutex);
1da177e4 1274 break;
1da177e4 1275 }
eac40680 1276 return notifier_from_errno(err);
1da177e4
LT
1277}
1278
0db0628d 1279static struct notifier_block cpucache_notifier = {
74b85f37
CS
1280 &cpuup_callback, NULL, 0
1281};
1da177e4 1282
8f9f8d9e
DR
1283#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1284/*
1285 * Drains freelist for a node on each slab cache, used for memory hot-remove.
1286 * Returns -EBUSY if all objects cannot be drained so that the node is not
1287 * removed.
1288 *
18004c5d 1289 * Must hold slab_mutex.
8f9f8d9e 1290 */
6a67368c 1291static int __meminit drain_cache_node_node(int node)
8f9f8d9e
DR
1292{
1293 struct kmem_cache *cachep;
1294 int ret = 0;
1295
18004c5d 1296 list_for_each_entry(cachep, &slab_caches, list) {
ce8eb6c4 1297 struct kmem_cache_node *n;
8f9f8d9e 1298
18bf8541 1299 n = get_node(cachep, node);
ce8eb6c4 1300 if (!n)
8f9f8d9e
DR
1301 continue;
1302
0fa8103b 1303 drain_freelist(cachep, n, slabs_tofree(cachep, n));
8f9f8d9e 1304
ce8eb6c4
CL
1305 if (!list_empty(&n->slabs_full) ||
1306 !list_empty(&n->slabs_partial)) {
8f9f8d9e
DR
1307 ret = -EBUSY;
1308 break;
1309 }
1310 }
1311 return ret;
1312}
1313
1314static int __meminit slab_memory_callback(struct notifier_block *self,
1315 unsigned long action, void *arg)
1316{
1317 struct memory_notify *mnb = arg;
1318 int ret = 0;
1319 int nid;
1320
1321 nid = mnb->status_change_nid;
1322 if (nid < 0)
1323 goto out;
1324
1325 switch (action) {
1326 case MEM_GOING_ONLINE:
18004c5d 1327 mutex_lock(&slab_mutex);
6a67368c 1328 ret = init_cache_node_node(nid);
18004c5d 1329 mutex_unlock(&slab_mutex);
8f9f8d9e
DR
1330 break;
1331 case MEM_GOING_OFFLINE:
18004c5d 1332 mutex_lock(&slab_mutex);
6a67368c 1333 ret = drain_cache_node_node(nid);
18004c5d 1334 mutex_unlock(&slab_mutex);
8f9f8d9e
DR
1335 break;
1336 case MEM_ONLINE:
1337 case MEM_OFFLINE:
1338 case MEM_CANCEL_ONLINE:
1339 case MEM_CANCEL_OFFLINE:
1340 break;
1341 }
1342out:
5fda1bd5 1343 return notifier_from_errno(ret);
8f9f8d9e
DR
1344}
1345#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1346
e498be7d 1347/*
ce8eb6c4 1348 * swap the static kmem_cache_node with kmalloced memory
e498be7d 1349 */
6744f087 1350static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
8f9f8d9e 1351 int nodeid)
e498be7d 1352{
6744f087 1353 struct kmem_cache_node *ptr;
e498be7d 1354
6744f087 1355 ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
e498be7d
CL
1356 BUG_ON(!ptr);
1357
6744f087 1358 memcpy(ptr, list, sizeof(struct kmem_cache_node));
2b2d5493
IM
1359 /*
1360 * Do not assume that spinlocks can be initialized via memcpy:
1361 */
1362 spin_lock_init(&ptr->list_lock);
1363
e498be7d 1364 MAKE_ALL_LISTS(cachep, ptr, nodeid);
6a67368c 1365 cachep->node[nodeid] = ptr;
e498be7d
CL
1366}
1367
556a169d 1368/*
ce8eb6c4
CL
1369 * For setting up all the kmem_cache_node for cache whose buffer_size is same as
1370 * size of kmem_cache_node.
556a169d 1371 */
ce8eb6c4 1372static void __init set_up_node(struct kmem_cache *cachep, int index)
556a169d
PE
1373{
1374 int node;
1375
1376 for_each_online_node(node) {
ce8eb6c4 1377 cachep->node[node] = &init_kmem_cache_node[index + node];
6a67368c 1378 cachep->node[node]->next_reap = jiffies +
5f0985bb
JZ
1379 REAPTIMEOUT_NODE +
1380 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
556a169d
PE
1381 }
1382}
1383
a737b3e2
AM
1384/*
1385 * Initialisation. Called after the page allocator have been initialised and
1386 * before smp_init().
1da177e4
LT
1387 */
1388void __init kmem_cache_init(void)
1389{
e498be7d
CL
1390 int i;
1391
68126702
JK
1392 BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
1393 sizeof(struct rcu_head));
9b030cb8
CL
1394 kmem_cache = &kmem_cache_boot;
1395
b6e68bc1 1396 if (num_possible_nodes() == 1)
62918a03
SS
1397 use_alien_caches = 0;
1398
3c583465 1399 for (i = 0; i < NUM_INIT_LISTS; i++)
ce8eb6c4 1400 kmem_cache_node_init(&init_kmem_cache_node[i]);
3c583465 1401
1da177e4
LT
1402 /*
1403 * Fragmentation resistance on low memory - only use bigger
3df1cccd
DR
1404 * page orders on machines with more than 32MB of memory if
1405 * not overridden on the command line.
1da177e4 1406 */
3df1cccd 1407 if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
543585cc 1408 slab_max_order = SLAB_MAX_ORDER_HI;
1da177e4 1409
1da177e4
LT
1410 /* Bootstrap is tricky, because several objects are allocated
1411 * from caches that do not exist yet:
9b030cb8
CL
1412 * 1) initialize the kmem_cache cache: it contains the struct
1413 * kmem_cache structures of all caches, except kmem_cache itself:
1414 * kmem_cache is statically allocated.
e498be7d 1415 * Initially an __init data area is used for the head array and the
ce8eb6c4 1416 * kmem_cache_node structures, it's replaced with a kmalloc allocated
e498be7d 1417 * array at the end of the bootstrap.
1da177e4 1418 * 2) Create the first kmalloc cache.
343e0d7a 1419 * The struct kmem_cache for the new cache is allocated normally.
e498be7d
CL
1420 * An __init data area is used for the head array.
1421 * 3) Create the remaining kmalloc caches, with minimally sized
1422 * head arrays.
9b030cb8 1423 * 4) Replace the __init data head arrays for kmem_cache and the first
1da177e4 1424 * kmalloc cache with kmalloc allocated arrays.
ce8eb6c4 1425 * 5) Replace the __init data for kmem_cache_node for kmem_cache and
e498be7d
CL
1426 * the other cache's with kmalloc allocated memory.
1427 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
1da177e4
LT
1428 */
1429
9b030cb8 1430 /* 1) create the kmem_cache */
1da177e4 1431
8da3430d 1432 /*
b56efcf0 1433 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
8da3430d 1434 */
2f9baa9f 1435 create_boot_cache(kmem_cache, "kmem_cache",
bf0dea23 1436 offsetof(struct kmem_cache, node) +
6744f087 1437 nr_node_ids * sizeof(struct kmem_cache_node *),
2f9baa9f
CL
1438 SLAB_HWCACHE_ALIGN);
1439 list_add(&kmem_cache->list, &slab_caches);
bf0dea23 1440 slab_state = PARTIAL;
1da177e4 1441
a737b3e2 1442 /*
bf0dea23
JK
1443 * Initialize the caches that provide memory for the kmem_cache_node
1444 * structures first. Without this, further allocations will bug.
e498be7d 1445 */
bf0dea23 1446 kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
ce8eb6c4 1447 kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
bf0dea23 1448 slab_state = PARTIAL_NODE;
34cc6990 1449 setup_kmalloc_cache_index_table();
e498be7d 1450
e0a42726
IM
1451 slab_early_init = 0;
1452
ce8eb6c4 1453 /* 5) Replace the bootstrap kmem_cache_node */
e498be7d 1454 {
1ca4cb24
PE
1455 int nid;
1456
9c09a95c 1457 for_each_online_node(nid) {
ce8eb6c4 1458 init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
556a169d 1459
bf0dea23 1460 init_list(kmalloc_caches[INDEX_NODE],
ce8eb6c4 1461 &init_kmem_cache_node[SIZE_NODE + nid], nid);
e498be7d
CL
1462 }
1463 }
1da177e4 1464
f97d5f63 1465 create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
8429db5c
PE
1466}
1467
1468void __init kmem_cache_init_late(void)
1469{
1470 struct kmem_cache *cachep;
1471
97d06609 1472 slab_state = UP;
52cef189 1473
8429db5c 1474 /* 6) resize the head arrays to their final sizes */
18004c5d
CL
1475 mutex_lock(&slab_mutex);
1476 list_for_each_entry(cachep, &slab_caches, list)
8429db5c
PE
1477 if (enable_cpucache(cachep, GFP_NOWAIT))
1478 BUG();
18004c5d 1479 mutex_unlock(&slab_mutex);
056c6241 1480
97d06609
CL
1481 /* Done! */
1482 slab_state = FULL;
1483
a737b3e2
AM
1484 /*
1485 * Register a cpu startup notifier callback that initializes
1486 * cpu_cache_get for all new cpus
1da177e4
LT
1487 */
1488 register_cpu_notifier(&cpucache_notifier);
1da177e4 1489
8f9f8d9e
DR
1490#ifdef CONFIG_NUMA
1491 /*
1492 * Register a memory hotplug callback that initializes and frees
6a67368c 1493 * node.
8f9f8d9e
DR
1494 */
1495 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1496#endif
1497
a737b3e2
AM
1498 /*
1499 * The reap timers are started later, with a module init call: That part
1500 * of the kernel is not yet operational.
1da177e4
LT
1501 */
1502}
1503
1504static int __init cpucache_init(void)
1505{
1506 int cpu;
1507
a737b3e2
AM
1508 /*
1509 * Register the timers that return unneeded pages to the page allocator
1da177e4 1510 */
e498be7d 1511 for_each_online_cpu(cpu)
a737b3e2 1512 start_cpu_timer(cpu);
a164f896
GC
1513
1514 /* Done! */
97d06609 1515 slab_state = FULL;
1da177e4
LT
1516 return 0;
1517}
1da177e4
LT
1518__initcall(cpucache_init);
1519
8bdec192
RA
1520static noinline void
1521slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
1522{
9a02d699 1523#if DEBUG
ce8eb6c4 1524 struct kmem_cache_node *n;
8456a648 1525 struct page *page;
8bdec192
RA
1526 unsigned long flags;
1527 int node;
9a02d699
DR
1528 static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
1529 DEFAULT_RATELIMIT_BURST);
1530
1531 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
1532 return;
8bdec192
RA
1533
1534 printk(KERN_WARNING
1535 "SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
1536 nodeid, gfpflags);
1537 printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n",
3b0efdfa 1538 cachep->name, cachep->size, cachep->gfporder);
8bdec192 1539
18bf8541 1540 for_each_kmem_cache_node(cachep, node, n) {
8bdec192
RA
1541 unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
1542 unsigned long active_slabs = 0, num_slabs = 0;
1543
ce8eb6c4 1544 spin_lock_irqsave(&n->list_lock, flags);
8456a648 1545 list_for_each_entry(page, &n->slabs_full, lru) {
8bdec192
RA
1546 active_objs += cachep->num;
1547 active_slabs++;
1548 }
8456a648
JK
1549 list_for_each_entry(page, &n->slabs_partial, lru) {
1550 active_objs += page->active;
8bdec192
RA
1551 active_slabs++;
1552 }
8456a648 1553 list_for_each_entry(page, &n->slabs_free, lru)
8bdec192
RA
1554 num_slabs++;
1555
ce8eb6c4
CL
1556 free_objects += n->free_objects;
1557 spin_unlock_irqrestore(&n->list_lock, flags);
8bdec192
RA
1558
1559 num_slabs += active_slabs;
1560 num_objs = num_slabs * cachep->num;
1561 printk(KERN_WARNING
1562 " node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
1563 node, active_slabs, num_slabs, active_objs, num_objs,
1564 free_objects);
1565 }
9a02d699 1566#endif
8bdec192
RA
1567}
1568
1da177e4 1569/*
8a7d9b43
WSH
1570 * Interface to system's page allocator. No need to hold the
1571 * kmem_cache_node ->list_lock.
1da177e4
LT
1572 *
1573 * If we requested dmaable memory, we will get it. Even if we
1574 * did not request dmaable memory, we might get it, but that
1575 * would be relatively rare and ignorable.
1576 */
0c3aa83e
JK
1577static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
1578 int nodeid)
1da177e4
LT
1579{
1580 struct page *page;
e1b6aa6f 1581 int nr_pages;
765c4507 1582
a618e89f 1583 flags |= cachep->allocflags;
e12ba74d
MG
1584 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1585 flags |= __GFP_RECLAIMABLE;
e1b6aa6f 1586
96db800f 1587 page = __alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
8bdec192 1588 if (!page) {
9a02d699 1589 slab_out_of_memory(cachep, flags, nodeid);
1da177e4 1590 return NULL;
8bdec192 1591 }
1da177e4 1592
f3ccb2c4
VD
1593 if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
1594 __free_pages(page, cachep->gfporder);
1595 return NULL;
1596 }
1597
b37f1dd0 1598 /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
2f064f34 1599 if (page_is_pfmemalloc(page))
072bb0aa
MG
1600 pfmemalloc_active = true;
1601
e1b6aa6f 1602 nr_pages = (1 << cachep->gfporder);
1da177e4 1603 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
972d1a7b
CL
1604 add_zone_page_state(page_zone(page),
1605 NR_SLAB_RECLAIMABLE, nr_pages);
1606 else
1607 add_zone_page_state(page_zone(page),
1608 NR_SLAB_UNRECLAIMABLE, nr_pages);
a57a4988 1609 __SetPageSlab(page);
2f064f34 1610 if (page_is_pfmemalloc(page))
a57a4988 1611 SetPageSlabPfmemalloc(page);
072bb0aa 1612
b1eeab67
VN
1613 if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
1614 kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
1615
1616 if (cachep->ctor)
1617 kmemcheck_mark_uninitialized_pages(page, nr_pages);
1618 else
1619 kmemcheck_mark_unallocated_pages(page, nr_pages);
1620 }
c175eea4 1621
0c3aa83e 1622 return page;
1da177e4
LT
1623}
1624
1625/*
1626 * Interface to system's page release.
1627 */
0c3aa83e 1628static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
1da177e4 1629{
a57a4988 1630 const unsigned long nr_freed = (1 << cachep->gfporder);
1da177e4 1631
b1eeab67 1632 kmemcheck_free_shadow(page, cachep->gfporder);
c175eea4 1633
972d1a7b
CL
1634 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1635 sub_zone_page_state(page_zone(page),
1636 NR_SLAB_RECLAIMABLE, nr_freed);
1637 else
1638 sub_zone_page_state(page_zone(page),
1639 NR_SLAB_UNRECLAIMABLE, nr_freed);
73293c2f 1640
a57a4988 1641 BUG_ON(!PageSlab(page));
73293c2f 1642 __ClearPageSlabPfmemalloc(page);
a57a4988 1643 __ClearPageSlab(page);
8456a648
JK
1644 page_mapcount_reset(page);
1645 page->mapping = NULL;
1f458cbf 1646
1da177e4
LT
1647 if (current->reclaim_state)
1648 current->reclaim_state->reclaimed_slab += nr_freed;
f3ccb2c4 1649 __free_kmem_pages(page, cachep->gfporder);
1da177e4
LT
1650}
1651
1652static void kmem_rcu_free(struct rcu_head *head)
1653{
68126702
JK
1654 struct kmem_cache *cachep;
1655 struct page *page;
1da177e4 1656
68126702
JK
1657 page = container_of(head, struct page, rcu_head);
1658 cachep = page->slab_cache;
1659
1660 kmem_freepages(cachep, page);
1da177e4
LT
1661}
1662
1663#if DEBUG
1664
1665#ifdef CONFIG_DEBUG_PAGEALLOC
343e0d7a 1666static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
b28a02de 1667 unsigned long caller)
1da177e4 1668{
8c138bc0 1669 int size = cachep->object_size;
1da177e4 1670
3dafccf2 1671 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
1da177e4 1672
b28a02de 1673 if (size < 5 * sizeof(unsigned long))
1da177e4
LT
1674 return;
1675
b28a02de
PE
1676 *addr++ = 0x12345678;
1677 *addr++ = caller;
1678 *addr++ = smp_processor_id();
1679 size -= 3 * sizeof(unsigned long);
1da177e4
LT
1680 {
1681 unsigned long *sptr = &caller;
1682 unsigned long svalue;
1683
1684 while (!kstack_end(sptr)) {
1685 svalue = *sptr++;
1686 if (kernel_text_address(svalue)) {
b28a02de 1687 *addr++ = svalue;
1da177e4
LT
1688 size -= sizeof(unsigned long);
1689 if (size <= sizeof(unsigned long))
1690 break;
1691 }
1692 }
1693
1694 }
b28a02de 1695 *addr++ = 0x87654321;
1da177e4
LT
1696}
1697#endif
1698
343e0d7a 1699static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
1da177e4 1700{
8c138bc0 1701 int size = cachep->object_size;
3dafccf2 1702 addr = &((char *)addr)[obj_offset(cachep)];
1da177e4
LT
1703
1704 memset(addr, val, size);
b28a02de 1705 *(unsigned char *)(addr + size - 1) = POISON_END;
1da177e4
LT
1706}
1707
1708static void dump_line(char *data, int offset, int limit)
1709{
1710 int i;
aa83aa40
DJ
1711 unsigned char error = 0;
1712 int bad_count = 0;
1713
fdde6abb 1714 printk(KERN_ERR "%03x: ", offset);
aa83aa40
DJ
1715 for (i = 0; i < limit; i++) {
1716 if (data[offset + i] != POISON_FREE) {
1717 error = data[offset + i];
1718 bad_count++;
1719 }
aa83aa40 1720 }
fdde6abb
SAS
1721 print_hex_dump(KERN_CONT, "", 0, 16, 1,
1722 &data[offset], limit, 1);
aa83aa40
DJ
1723
1724 if (bad_count == 1) {
1725 error ^= POISON_FREE;
1726 if (!(error & (error - 1))) {
1727 printk(KERN_ERR "Single bit error detected. Probably "
1728 "bad RAM.\n");
1729#ifdef CONFIG_X86
1730 printk(KERN_ERR "Run memtest86+ or a similar memory "
1731 "test tool.\n");
1732#else
1733 printk(KERN_ERR "Run a memory test tool.\n");
1734#endif
1735 }
1736 }
1da177e4
LT
1737}
1738#endif
1739
1740#if DEBUG
1741
343e0d7a 1742static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
1da177e4
LT
1743{
1744 int i, size;
1745 char *realobj;
1746
1747 if (cachep->flags & SLAB_RED_ZONE) {
b46b8f19 1748 printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
a737b3e2
AM
1749 *dbg_redzone1(cachep, objp),
1750 *dbg_redzone2(cachep, objp));
1da177e4
LT
1751 }
1752
1753 if (cachep->flags & SLAB_STORE_USER) {
071361d3
JP
1754 printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
1755 *dbg_userword(cachep, objp),
1756 *dbg_userword(cachep, objp));
1da177e4 1757 }
3dafccf2 1758 realobj = (char *)objp + obj_offset(cachep);
8c138bc0 1759 size = cachep->object_size;
b28a02de 1760 for (i = 0; i < size && lines; i += 16, lines--) {
1da177e4
LT
1761 int limit;
1762 limit = 16;
b28a02de
PE
1763 if (i + limit > size)
1764 limit = size - i;
1da177e4
LT
1765 dump_line(realobj, i, limit);
1766 }
1767}
1768
343e0d7a 1769static void check_poison_obj(struct kmem_cache *cachep, void *objp)
1da177e4
LT
1770{
1771 char *realobj;
1772 int size, i;
1773 int lines = 0;
1774
3dafccf2 1775 realobj = (char *)objp + obj_offset(cachep);
8c138bc0 1776 size = cachep->object_size;
1da177e4 1777
b28a02de 1778 for (i = 0; i < size; i++) {
1da177e4 1779 char exp = POISON_FREE;
b28a02de 1780 if (i == size - 1)
1da177e4
LT
1781 exp = POISON_END;
1782 if (realobj[i] != exp) {
1783 int limit;
1784 /* Mismatch ! */
1785 /* Print header */
1786 if (lines == 0) {
b28a02de 1787 printk(KERN_ERR
face37f5
DJ
1788 "Slab corruption (%s): %s start=%p, len=%d\n",
1789 print_tainted(), cachep->name, realobj, size);
1da177e4
LT
1790 print_objinfo(cachep, objp, 0);
1791 }
1792 /* Hexdump the affected line */
b28a02de 1793 i = (i / 16) * 16;
1da177e4 1794 limit = 16;
b28a02de
PE
1795 if (i + limit > size)
1796 limit = size - i;
1da177e4
LT
1797 dump_line(realobj, i, limit);
1798 i += 16;
1799 lines++;
1800 /* Limit to 5 lines */
1801 if (lines > 5)
1802 break;
1803 }
1804 }
1805 if (lines != 0) {
1806 /* Print some data about the neighboring objects, if they
1807 * exist:
1808 */
8456a648 1809 struct page *page = virt_to_head_page(objp);
8fea4e96 1810 unsigned int objnr;
1da177e4 1811
8456a648 1812 objnr = obj_to_index(cachep, page, objp);
1da177e4 1813 if (objnr) {
8456a648 1814 objp = index_to_obj(cachep, page, objnr - 1);
3dafccf2 1815 realobj = (char *)objp + obj_offset(cachep);
1da177e4 1816 printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
b28a02de 1817 realobj, size);
1da177e4
LT
1818 print_objinfo(cachep, objp, 2);
1819 }
b28a02de 1820 if (objnr + 1 < cachep->num) {
8456a648 1821 objp = index_to_obj(cachep, page, objnr + 1);
3dafccf2 1822 realobj = (char *)objp + obj_offset(cachep);
1da177e4 1823 printk(KERN_ERR "Next obj: start=%p, len=%d\n",
b28a02de 1824 realobj, size);
1da177e4
LT
1825 print_objinfo(cachep, objp, 2);
1826 }
1827 }
1828}
1829#endif
1830
12dd36fa 1831#if DEBUG
8456a648
JK
1832static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1833 struct page *page)
1da177e4 1834{
1da177e4
LT
1835 int i;
1836 for (i = 0; i < cachep->num; i++) {
8456a648 1837 void *objp = index_to_obj(cachep, page, i);
1da177e4
LT
1838
1839 if (cachep->flags & SLAB_POISON) {
1840#ifdef CONFIG_DEBUG_PAGEALLOC
a307ebd4
JK
1841 if (debug_pagealloc_enabled() &&
1842 cachep->size % PAGE_SIZE == 0 &&
a737b3e2 1843 OFF_SLAB(cachep))
b28a02de 1844 kernel_map_pages(virt_to_page(objp),
3b0efdfa 1845 cachep->size / PAGE_SIZE, 1);
1da177e4
LT
1846 else
1847 check_poison_obj(cachep, objp);
1848#else
1849 check_poison_obj(cachep, objp);
1850#endif
1851 }
1852 if (cachep->flags & SLAB_RED_ZONE) {
1853 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1854 slab_error(cachep, "start of a freed object "
b28a02de 1855 "was overwritten");
1da177e4
LT
1856 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1857 slab_error(cachep, "end of a freed object "
b28a02de 1858 "was overwritten");
1da177e4 1859 }
1da177e4 1860 }
12dd36fa 1861}
1da177e4 1862#else
8456a648
JK
1863static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1864 struct page *page)
12dd36fa 1865{
12dd36fa 1866}
1da177e4
LT
1867#endif
1868
911851e6
RD
1869/**
1870 * slab_destroy - destroy and release all objects in a slab
1871 * @cachep: cache pointer being destroyed
cb8ee1a3 1872 * @page: page pointer being destroyed
911851e6 1873 *
8a7d9b43
WSH
1874 * Destroy all the objs in a slab page, and release the mem back to the system.
1875 * Before calling the slab page must have been unlinked from the cache. The
1876 * kmem_cache_node ->list_lock is not held/needed.
12dd36fa 1877 */
8456a648 1878static void slab_destroy(struct kmem_cache *cachep, struct page *page)
12dd36fa 1879{
7e007355 1880 void *freelist;
12dd36fa 1881
8456a648
JK
1882 freelist = page->freelist;
1883 slab_destroy_debugcheck(cachep, page);
bc4f610d
KS
1884 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
1885 call_rcu(&page->rcu_head, kmem_rcu_free);
1886 else
0c3aa83e 1887 kmem_freepages(cachep, page);
68126702
JK
1888
1889 /*
8456a648 1890 * From now on, we don't use freelist
68126702
JK
1891 * although actual page can be freed in rcu context
1892 */
1893 if (OFF_SLAB(cachep))
8456a648 1894 kmem_cache_free(cachep->freelist_cache, freelist);
1da177e4
LT
1895}
1896
97654dfa
JK
1897static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
1898{
1899 struct page *page, *n;
1900
1901 list_for_each_entry_safe(page, n, list, lru) {
1902 list_del(&page->lru);
1903 slab_destroy(cachep, page);
1904 }
1905}
1906
4d268eba 1907/**
a70773dd
RD
1908 * calculate_slab_order - calculate size (page order) of slabs
1909 * @cachep: pointer to the cache that is being created
1910 * @size: size of objects to be created in this cache.
1911 * @align: required alignment for the objects.
1912 * @flags: slab allocation flags
1913 *
1914 * Also calculates the number of objects per slab.
4d268eba
PE
1915 *
1916 * This could be made much more intelligent. For now, try to avoid using
1917 * high order pages for slabs. When the gfp() functions are more friendly
1918 * towards high-order requests, this should be changed.
1919 */
a737b3e2 1920static size_t calculate_slab_order(struct kmem_cache *cachep,
ee13d785 1921 size_t size, size_t align, unsigned long flags)
4d268eba 1922{
b1ab41c4 1923 unsigned long offslab_limit;
4d268eba 1924 size_t left_over = 0;
9888e6fa 1925 int gfporder;
4d268eba 1926
0aa817f0 1927 for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
4d268eba
PE
1928 unsigned int num;
1929 size_t remainder;
1930
9888e6fa 1931 cache_estimate(gfporder, size, align, flags, &remainder, &num);
4d268eba
PE
1932 if (!num)
1933 continue;
9888e6fa 1934
f315e3fa
JK
1935 /* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
1936 if (num > SLAB_OBJ_MAX_NUM)
1937 break;
1938
b1ab41c4 1939 if (flags & CFLGS_OFF_SLAB) {
03787301 1940 size_t freelist_size_per_obj = sizeof(freelist_idx_t);
b1ab41c4
IM
1941 /*
1942 * Max number of objs-per-slab for caches which
1943 * use off-slab slabs. Needed to avoid a possible
1944 * looping condition in cache_grow().
1945 */
03787301
JK
1946 if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
1947 freelist_size_per_obj += sizeof(char);
8456a648 1948 offslab_limit = size;
03787301 1949 offslab_limit /= freelist_size_per_obj;
b1ab41c4
IM
1950
1951 if (num > offslab_limit)
1952 break;
1953 }
4d268eba 1954
9888e6fa 1955 /* Found something acceptable - save it away */
4d268eba 1956 cachep->num = num;
9888e6fa 1957 cachep->gfporder = gfporder;
4d268eba
PE
1958 left_over = remainder;
1959
f78bb8ad
LT
1960 /*
1961 * A VFS-reclaimable slab tends to have most allocations
1962 * as GFP_NOFS and we really don't want to have to be allocating
1963 * higher-order pages when we are unable to shrink dcache.
1964 */
1965 if (flags & SLAB_RECLAIM_ACCOUNT)
1966 break;
1967
4d268eba
PE
1968 /*
1969 * Large number of objects is good, but very large slabs are
1970 * currently bad for the gfp()s.
1971 */
543585cc 1972 if (gfporder >= slab_max_order)
4d268eba
PE
1973 break;
1974
9888e6fa
LT
1975 /*
1976 * Acceptable internal fragmentation?
1977 */
a737b3e2 1978 if (left_over * 8 <= (PAGE_SIZE << gfporder))
4d268eba
PE
1979 break;
1980 }
1981 return left_over;
1982}
1983
bf0dea23
JK
1984static struct array_cache __percpu *alloc_kmem_cache_cpus(
1985 struct kmem_cache *cachep, int entries, int batchcount)
1986{
1987 int cpu;
1988 size_t size;
1989 struct array_cache __percpu *cpu_cache;
1990
1991 size = sizeof(void *) * entries + sizeof(struct array_cache);
85c9f4b0 1992 cpu_cache = __alloc_percpu(size, sizeof(void *));
bf0dea23
JK
1993
1994 if (!cpu_cache)
1995 return NULL;
1996
1997 for_each_possible_cpu(cpu) {
1998 init_arraycache(per_cpu_ptr(cpu_cache, cpu),
1999 entries, batchcount);
2000 }
2001
2002 return cpu_cache;
2003}
2004
83b519e8 2005static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
f30cf7d1 2006{
97d06609 2007 if (slab_state >= FULL)
83b519e8 2008 return enable_cpucache(cachep, gfp);
2ed3a4ef 2009
bf0dea23
JK
2010 cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
2011 if (!cachep->cpu_cache)
2012 return 1;
2013
97d06609 2014 if (slab_state == DOWN) {
bf0dea23
JK
2015 /* Creation of first cache (kmem_cache). */
2016 set_up_node(kmem_cache, CACHE_CACHE);
2f9baa9f 2017 } else if (slab_state == PARTIAL) {
bf0dea23
JK
2018 /* For kmem_cache_node */
2019 set_up_node(cachep, SIZE_NODE);
f30cf7d1 2020 } else {
bf0dea23 2021 int node;
f30cf7d1 2022
bf0dea23
JK
2023 for_each_online_node(node) {
2024 cachep->node[node] = kmalloc_node(
2025 sizeof(struct kmem_cache_node), gfp, node);
2026 BUG_ON(!cachep->node[node]);
2027 kmem_cache_node_init(cachep->node[node]);
f30cf7d1
PE
2028 }
2029 }
bf0dea23 2030
6a67368c 2031 cachep->node[numa_mem_id()]->next_reap =
5f0985bb
JZ
2032 jiffies + REAPTIMEOUT_NODE +
2033 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
f30cf7d1
PE
2034
2035 cpu_cache_get(cachep)->avail = 0;
2036 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
2037 cpu_cache_get(cachep)->batchcount = 1;
2038 cpu_cache_get(cachep)->touched = 0;
2039 cachep->batchcount = 1;
2040 cachep->limit = BOOT_CPUCACHE_ENTRIES;
2ed3a4ef 2041 return 0;
f30cf7d1
PE
2042}
2043
12220dea
JK
2044unsigned long kmem_cache_flags(unsigned long object_size,
2045 unsigned long flags, const char *name,
2046 void (*ctor)(void *))
2047{
2048 return flags;
2049}
2050
2051struct kmem_cache *
2052__kmem_cache_alias(const char *name, size_t size, size_t align,
2053 unsigned long flags, void (*ctor)(void *))
2054{
2055 struct kmem_cache *cachep;
2056
2057 cachep = find_mergeable(size, align, flags, name, ctor);
2058 if (cachep) {
2059 cachep->refcount++;
2060
2061 /*
2062 * Adjust the object sizes so that we clear
2063 * the complete object on kzalloc.
2064 */
2065 cachep->object_size = max_t(int, cachep->object_size, size);
2066 }
2067 return cachep;
2068}
2069
1da177e4 2070/**
039363f3 2071 * __kmem_cache_create - Create a cache.
a755b76a 2072 * @cachep: cache management descriptor
1da177e4 2073 * @flags: SLAB flags
1da177e4
LT
2074 *
2075 * Returns a ptr to the cache on success, NULL on failure.
2076 * Cannot be called within a int, but can be interrupted.
20c2df83 2077 * The @ctor is run when new pages are allocated by the cache.
1da177e4 2078 *
1da177e4
LT
2079 * The flags are
2080 *
2081 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
2082 * to catch references to uninitialised memory.
2083 *
2084 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
2085 * for buffer overruns.
2086 *
1da177e4
LT
2087 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
2088 * cacheline. This can be beneficial if you're counting cycles as closely
2089 * as davem.
2090 */
278b1bb1 2091int
8a13a4cc 2092__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
1da177e4 2093{
d4a5fca5
DR
2094 size_t left_over, freelist_size;
2095 size_t ralign = BYTES_PER_WORD;
83b519e8 2096 gfp_t gfp;
278b1bb1 2097 int err;
8a13a4cc 2098 size_t size = cachep->size;
1da177e4 2099
1da177e4 2100#if DEBUG
1da177e4
LT
2101#if FORCED_DEBUG
2102 /*
2103 * Enable redzoning and last user accounting, except for caches with
2104 * large objects, if the increased size would increase the object size
2105 * above the next power of two: caches with object sizes just above a
2106 * power of two have a significant amount of internal fragmentation.
2107 */
87a927c7
DW
2108 if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
2109 2 * sizeof(unsigned long long)))
b28a02de 2110 flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
1da177e4
LT
2111 if (!(flags & SLAB_DESTROY_BY_RCU))
2112 flags |= SLAB_POISON;
2113#endif
1da177e4 2114#endif
1da177e4 2115
a737b3e2
AM
2116 /*
2117 * Check that size is in terms of words. This is needed to avoid
1da177e4
LT
2118 * unaligned accesses for some archs when redzoning is used, and makes
2119 * sure any on-slab bufctl's are also correctly aligned.
2120 */
b28a02de
PE
2121 if (size & (BYTES_PER_WORD - 1)) {
2122 size += (BYTES_PER_WORD - 1);
2123 size &= ~(BYTES_PER_WORD - 1);
1da177e4
LT
2124 }
2125
87a927c7
DW
2126 if (flags & SLAB_RED_ZONE) {
2127 ralign = REDZONE_ALIGN;
2128 /* If redzoning, ensure that the second redzone is suitably
2129 * aligned, by adjusting the object size accordingly. */
2130 size += REDZONE_ALIGN - 1;
2131 size &= ~(REDZONE_ALIGN - 1);
2132 }
ca5f9703 2133
a44b56d3 2134 /* 3) caller mandated alignment */
8a13a4cc
CL
2135 if (ralign < cachep->align) {
2136 ralign = cachep->align;
1da177e4 2137 }
3ff84a7f
PE
2138 /* disable debug if necessary */
2139 if (ralign > __alignof__(unsigned long long))
a44b56d3 2140 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
a737b3e2 2141 /*
ca5f9703 2142 * 4) Store it.
1da177e4 2143 */
8a13a4cc 2144 cachep->align = ralign;
1da177e4 2145
83b519e8
PE
2146 if (slab_is_available())
2147 gfp = GFP_KERNEL;
2148 else
2149 gfp = GFP_NOWAIT;
2150
1da177e4 2151#if DEBUG
1da177e4 2152
ca5f9703
PE
2153 /*
2154 * Both debugging options require word-alignment which is calculated
2155 * into align above.
2156 */
1da177e4 2157 if (flags & SLAB_RED_ZONE) {
1da177e4 2158 /* add space for red zone words */
3ff84a7f
PE
2159 cachep->obj_offset += sizeof(unsigned long long);
2160 size += 2 * sizeof(unsigned long long);
1da177e4
LT
2161 }
2162 if (flags & SLAB_STORE_USER) {
ca5f9703 2163 /* user store requires one word storage behind the end of
87a927c7
DW
2164 * the real object. But if the second red zone needs to be
2165 * aligned to 64 bits, we must allow that much space.
1da177e4 2166 */
87a927c7
DW
2167 if (flags & SLAB_RED_ZONE)
2168 size += REDZONE_ALIGN;
2169 else
2170 size += BYTES_PER_WORD;
1da177e4
LT
2171 }
2172#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
03a2d2a3
JK
2173 /*
2174 * To activate debug pagealloc, off-slab management is necessary
2175 * requirement. In early phase of initialization, small sized slab
2176 * doesn't get initialized so it would not be possible. So, we need
2177 * to check size >= 256. It guarantees that all necessary small
2178 * sized slab is initialized in current slab initialization sequence.
2179 */
a307ebd4
JK
2180 if (debug_pagealloc_enabled() &&
2181 !slab_early_init && size >= kmalloc_size(INDEX_NODE) &&
03a2d2a3
JK
2182 size >= 256 && cachep->object_size > cache_line_size() &&
2183 ALIGN(size, cachep->align) < PAGE_SIZE) {
608da7e3 2184 cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align);
1da177e4
LT
2185 size = PAGE_SIZE;
2186 }
2187#endif
2188#endif
2189
e0a42726
IM
2190 /*
2191 * Determine if the slab management is 'on' or 'off' slab.
2192 * (bootstrapping cannot cope with offslab caches so don't do
e7cb55b9
CM
2193 * it too early on. Always use on-slab management when
2194 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
e0a42726 2195 */
d4322d88 2196 if (size >= OFF_SLAB_MIN_SIZE && !slab_early_init &&
e7cb55b9 2197 !(flags & SLAB_NOLEAKTRACE))
1da177e4
LT
2198 /*
2199 * Size is large, assume best to place the slab management obj
2200 * off-slab (should allow better packing of objs).
2201 */
2202 flags |= CFLGS_OFF_SLAB;
2203
8a13a4cc 2204 size = ALIGN(size, cachep->align);
f315e3fa
JK
2205 /*
2206 * We should restrict the number of objects in a slab to implement
2207 * byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
2208 */
2209 if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
2210 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
1da177e4 2211
8a13a4cc 2212 left_over = calculate_slab_order(cachep, size, cachep->align, flags);
1da177e4 2213
8a13a4cc 2214 if (!cachep->num)
278b1bb1 2215 return -E2BIG;
1da177e4 2216
03787301 2217 freelist_size = calculate_freelist_size(cachep->num, cachep->align);
1da177e4
LT
2218
2219 /*
2220 * If the slab has been placed off-slab, and we have enough space then
2221 * move it on-slab. This is at the expense of any extra colouring.
2222 */
8456a648 2223 if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {
1da177e4 2224 flags &= ~CFLGS_OFF_SLAB;
8456a648 2225 left_over -= freelist_size;
1da177e4
LT
2226 }
2227
2228 if (flags & CFLGS_OFF_SLAB) {
2229 /* really off slab. No need for manual alignment */
03787301 2230 freelist_size = calculate_freelist_size(cachep->num, 0);
67461365
RL
2231
2232#ifdef CONFIG_PAGE_POISONING
2233 /* If we're going to use the generic kernel_map_pages()
2234 * poisoning, then it's going to smash the contents of
2235 * the redzone and userword anyhow, so switch them off.
2236 */
a307ebd4
JK
2237 if (debug_pagealloc_enabled() &&
2238 size % PAGE_SIZE == 0 && flags & SLAB_POISON)
67461365
RL
2239 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2240#endif
1da177e4
LT
2241 }
2242
2243 cachep->colour_off = cache_line_size();
2244 /* Offset must be a multiple of the alignment. */
8a13a4cc
CL
2245 if (cachep->colour_off < cachep->align)
2246 cachep->colour_off = cachep->align;
b28a02de 2247 cachep->colour = left_over / cachep->colour_off;
8456a648 2248 cachep->freelist_size = freelist_size;
1da177e4 2249 cachep->flags = flags;
a57a4988 2250 cachep->allocflags = __GFP_COMP;
4b51d669 2251 if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
a618e89f 2252 cachep->allocflags |= GFP_DMA;
3b0efdfa 2253 cachep->size = size;
6a2d7a95 2254 cachep->reciprocal_buffer_size = reciprocal_value(size);
1da177e4 2255
e5ac9c5a 2256 if (flags & CFLGS_OFF_SLAB) {
8456a648 2257 cachep->freelist_cache = kmalloc_slab(freelist_size, 0u);
e5ac9c5a 2258 /*
5f0985bb 2259 * This is a possibility for one of the kmalloc_{dma,}_caches.
e5ac9c5a 2260 * But since we go off slab only for object size greater than
d4322d88 2261 * OFF_SLAB_MIN_SIZE, and kmalloc_{dma,}_caches get created
5f0985bb 2262 * in ascending order,this should not happen at all.
e5ac9c5a
RT
2263 * But leave a BUG_ON for some lucky dude.
2264 */
8456a648 2265 BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));
e5ac9c5a 2266 }
1da177e4 2267
278b1bb1
CL
2268 err = setup_cpu_cache(cachep, gfp);
2269 if (err) {
52b4b950 2270 __kmem_cache_release(cachep);
278b1bb1 2271 return err;
2ed3a4ef 2272 }
1da177e4 2273
278b1bb1 2274 return 0;
1da177e4 2275}
1da177e4
LT
2276
2277#if DEBUG
2278static void check_irq_off(void)
2279{
2280 BUG_ON(!irqs_disabled());
2281}
2282
2283static void check_irq_on(void)
2284{
2285 BUG_ON(irqs_disabled());
2286}
2287
343e0d7a 2288static void check_spinlock_acquired(struct kmem_cache *cachep)
1da177e4
LT
2289{
2290#ifdef CONFIG_SMP
2291 check_irq_off();
18bf8541 2292 assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
1da177e4
LT
2293#endif
2294}
e498be7d 2295
343e0d7a 2296static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
e498be7d
CL
2297{
2298#ifdef CONFIG_SMP
2299 check_irq_off();
18bf8541 2300 assert_spin_locked(&get_node(cachep, node)->list_lock);
e498be7d
CL
2301#endif
2302}
2303
1da177e4
LT
2304#else
2305#define check_irq_off() do { } while(0)
2306#define check_irq_on() do { } while(0)
2307#define check_spinlock_acquired(x) do { } while(0)
e498be7d 2308#define check_spinlock_acquired_node(x, y) do { } while(0)
1da177e4
LT
2309#endif
2310
ce8eb6c4 2311static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
aab2207c
CL
2312 struct array_cache *ac,
2313 int force, int node);
2314
1da177e4
LT
2315static void do_drain(void *arg)
2316{
a737b3e2 2317 struct kmem_cache *cachep = arg;
1da177e4 2318 struct array_cache *ac;
7d6e6d09 2319 int node = numa_mem_id();
18bf8541 2320 struct kmem_cache_node *n;
97654dfa 2321 LIST_HEAD(list);
1da177e4
LT
2322
2323 check_irq_off();
9a2dba4b 2324 ac = cpu_cache_get(cachep);
18bf8541
CL
2325 n = get_node(cachep, node);
2326 spin_lock(&n->list_lock);
97654dfa 2327 free_block(cachep, ac->entry, ac->avail, node, &list);
18bf8541 2328 spin_unlock(&n->list_lock);
97654dfa 2329 slabs_destroy(cachep, &list);
1da177e4
LT
2330 ac->avail = 0;
2331}
2332
343e0d7a 2333static void drain_cpu_caches(struct kmem_cache *cachep)
1da177e4 2334{
ce8eb6c4 2335 struct kmem_cache_node *n;
e498be7d
CL
2336 int node;
2337
15c8b6c1 2338 on_each_cpu(do_drain, cachep, 1);
1da177e4 2339 check_irq_on();
18bf8541
CL
2340 for_each_kmem_cache_node(cachep, node, n)
2341 if (n->alien)
ce8eb6c4 2342 drain_alien_cache(cachep, n->alien);
a4523a8b 2343
18bf8541
CL
2344 for_each_kmem_cache_node(cachep, node, n)
2345 drain_array(cachep, n, n->shared, 1, node);
1da177e4
LT
2346}
2347
ed11d9eb
CL
2348/*
2349 * Remove slabs from the list of free slabs.
2350 * Specify the number of slabs to drain in tofree.
2351 *
2352 * Returns the actual number of slabs released.
2353 */
2354static int drain_freelist(struct kmem_cache *cache,
ce8eb6c4 2355 struct kmem_cache_node *n, int tofree)
1da177e4 2356{
ed11d9eb
CL
2357 struct list_head *p;
2358 int nr_freed;
8456a648 2359 struct page *page;
1da177e4 2360
ed11d9eb 2361 nr_freed = 0;
ce8eb6c4 2362 while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
1da177e4 2363
ce8eb6c4
CL
2364 spin_lock_irq(&n->list_lock);
2365 p = n->slabs_free.prev;
2366 if (p == &n->slabs_free) {
2367 spin_unlock_irq(&n->list_lock);
ed11d9eb
CL
2368 goto out;
2369 }
1da177e4 2370
8456a648 2371 page = list_entry(p, struct page, lru);
8456a648 2372 list_del(&page->lru);
ed11d9eb
CL
2373 /*
2374 * Safe to drop the lock. The slab is no longer linked
2375 * to the cache.
2376 */
ce8eb6c4
CL
2377 n->free_objects -= cache->num;
2378 spin_unlock_irq(&n->list_lock);
8456a648 2379 slab_destroy(cache, page);
ed11d9eb 2380 nr_freed++;
1da177e4 2381 }
ed11d9eb
CL
2382out:
2383 return nr_freed;
1da177e4
LT
2384}
2385
d6e0b7fa 2386int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
e498be7d 2387{
18bf8541
CL
2388 int ret = 0;
2389 int node;
ce8eb6c4 2390 struct kmem_cache_node *n;
e498be7d
CL
2391
2392 drain_cpu_caches(cachep);
2393
2394 check_irq_on();
18bf8541 2395 for_each_kmem_cache_node(cachep, node, n) {
0fa8103b 2396 drain_freelist(cachep, n, slabs_tofree(cachep, n));
ed11d9eb 2397
ce8eb6c4
CL
2398 ret += !list_empty(&n->slabs_full) ||
2399 !list_empty(&n->slabs_partial);
e498be7d
CL
2400 }
2401 return (ret ? 1 : 0);
2402}
2403
945cf2b6 2404int __kmem_cache_shutdown(struct kmem_cache *cachep)
52b4b950
DS
2405{
2406 return __kmem_cache_shrink(cachep, false);
2407}
2408
2409void __kmem_cache_release(struct kmem_cache *cachep)
1da177e4 2410{
12c3667f 2411 int i;
ce8eb6c4 2412 struct kmem_cache_node *n;
1da177e4 2413
bf0dea23 2414 free_percpu(cachep->cpu_cache);
1da177e4 2415
ce8eb6c4 2416 /* NUMA: free the node structures */
18bf8541
CL
2417 for_each_kmem_cache_node(cachep, i, n) {
2418 kfree(n->shared);
2419 free_alien_cache(n->alien);
2420 kfree(n);
2421 cachep->node[i] = NULL;
12c3667f 2422 }
1da177e4 2423}
1da177e4 2424
e5ac9c5a
RT
2425/*
2426 * Get the memory for a slab management obj.
5f0985bb
JZ
2427 *
2428 * For a slab cache when the slab descriptor is off-slab, the
2429 * slab descriptor can't come from the same cache which is being created,
2430 * Because if it is the case, that means we defer the creation of
2431 * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
2432 * And we eventually call down to __kmem_cache_create(), which
2433 * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
2434 * This is a "chicken-and-egg" problem.
2435 *
2436 * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
2437 * which are all initialized during kmem_cache_init().
e5ac9c5a 2438 */
7e007355 2439static void *alloc_slabmgmt(struct kmem_cache *cachep,
0c3aa83e
JK
2440 struct page *page, int colour_off,
2441 gfp_t local_flags, int nodeid)
1da177e4 2442{
7e007355 2443 void *freelist;
0c3aa83e 2444 void *addr = page_address(page);
b28a02de 2445
1da177e4
LT
2446 if (OFF_SLAB(cachep)) {
2447 /* Slab management obj is off-slab. */
8456a648 2448 freelist = kmem_cache_alloc_node(cachep->freelist_cache,
8759ec50 2449 local_flags, nodeid);
8456a648 2450 if (!freelist)
1da177e4
LT
2451 return NULL;
2452 } else {
8456a648
JK
2453 freelist = addr + colour_off;
2454 colour_off += cachep->freelist_size;
1da177e4 2455 }
8456a648
JK
2456 page->active = 0;
2457 page->s_mem = addr + colour_off;
2458 return freelist;
1da177e4
LT
2459}
2460
7cc68973 2461static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
1da177e4 2462{
a41adfaa 2463 return ((freelist_idx_t *)page->freelist)[idx];
e5c58dfd
JK
2464}
2465
2466static inline void set_free_obj(struct page *page,
7cc68973 2467 unsigned int idx, freelist_idx_t val)
e5c58dfd 2468{
a41adfaa 2469 ((freelist_idx_t *)(page->freelist))[idx] = val;
1da177e4
LT
2470}
2471
343e0d7a 2472static void cache_init_objs(struct kmem_cache *cachep,
8456a648 2473 struct page *page)
1da177e4
LT
2474{
2475 int i;
2476
2477 for (i = 0; i < cachep->num; i++) {
8456a648 2478 void *objp = index_to_obj(cachep, page, i);
1da177e4
LT
2479#if DEBUG
2480 /* need to poison the objs? */
2481 if (cachep->flags & SLAB_POISON)
2482 poison_obj(cachep, objp, POISON_FREE);
2483 if (cachep->flags & SLAB_STORE_USER)
2484 *dbg_userword(cachep, objp) = NULL;
2485
2486 if (cachep->flags & SLAB_RED_ZONE) {
2487 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2488 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2489 }
2490 /*
a737b3e2
AM
2491 * Constructors are not allowed to allocate memory from the same
2492 * cache which they are a constructor for. Otherwise, deadlock.
2493 * They must also be threaded.
1da177e4
LT
2494 */
2495 if (cachep->ctor && !(cachep->flags & SLAB_POISON))
51cc5068 2496 cachep->ctor(objp + obj_offset(cachep));
1da177e4
LT
2497
2498 if (cachep->flags & SLAB_RED_ZONE) {
2499 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2500 slab_error(cachep, "constructor overwrote the"
b28a02de 2501 " end of an object");
1da177e4
LT
2502 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2503 slab_error(cachep, "constructor overwrote the"
b28a02de 2504 " start of an object");
1da177e4 2505 }
3b0efdfa 2506 if ((cachep->size % PAGE_SIZE) == 0 &&
a737b3e2 2507 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
b28a02de 2508 kernel_map_pages(virt_to_page(objp),
3b0efdfa 2509 cachep->size / PAGE_SIZE, 0);
1da177e4
LT
2510#else
2511 if (cachep->ctor)
51cc5068 2512 cachep->ctor(objp);
1da177e4 2513#endif
03787301 2514 set_obj_status(page, i, OBJECT_FREE);
e5c58dfd 2515 set_free_obj(page, i, i);
1da177e4 2516 }
1da177e4
LT
2517}
2518
343e0d7a 2519static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
1da177e4 2520{
4b51d669
CL
2521 if (CONFIG_ZONE_DMA_FLAG) {
2522 if (flags & GFP_DMA)
a618e89f 2523 BUG_ON(!(cachep->allocflags & GFP_DMA));
4b51d669 2524 else
a618e89f 2525 BUG_ON(cachep->allocflags & GFP_DMA);
4b51d669 2526 }
1da177e4
LT
2527}
2528
260b61dd 2529static void *slab_get_obj(struct kmem_cache *cachep, struct page *page)
78d382d7 2530{
b1cb0982 2531 void *objp;
78d382d7 2532
e5c58dfd 2533 objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
8456a648 2534 page->active++;
78d382d7
MD
2535
2536 return objp;
2537}
2538
260b61dd
JK
2539static void slab_put_obj(struct kmem_cache *cachep,
2540 struct page *page, void *objp)
78d382d7 2541{
8456a648 2542 unsigned int objnr = obj_to_index(cachep, page, objp);
78d382d7 2543#if DEBUG
16025177 2544 unsigned int i;
b1cb0982 2545
b1cb0982 2546 /* Verify double free bug */
8456a648 2547 for (i = page->active; i < cachep->num; i++) {
e5c58dfd 2548 if (get_free_obj(page, i) == objnr) {
b1cb0982
JK
2549 printk(KERN_ERR "slab: double free detected in cache "
2550 "'%s', objp %p\n", cachep->name, objp);
2551 BUG();
2552 }
78d382d7
MD
2553 }
2554#endif
8456a648 2555 page->active--;
e5c58dfd 2556 set_free_obj(page, page->active, objnr);
78d382d7
MD
2557}
2558
4776874f
PE
2559/*
2560 * Map pages beginning at addr to the given cache and slab. This is required
2561 * for the slab allocator to be able to lookup the cache and slab of a
ccd35fb9 2562 * virtual address for kfree, ksize, and slab debugging.
4776874f 2563 */
8456a648 2564static void slab_map_pages(struct kmem_cache *cache, struct page *page,
7e007355 2565 void *freelist)
1da177e4 2566{
a57a4988 2567 page->slab_cache = cache;
8456a648 2568 page->freelist = freelist;
1da177e4
LT
2569}
2570
2571/*
2572 * Grow (by 1) the number of slabs within a cache. This is called by
2573 * kmem_cache_alloc() when there are no active objs left in a cache.
2574 */
3c517a61 2575static int cache_grow(struct kmem_cache *cachep,
0c3aa83e 2576 gfp_t flags, int nodeid, struct page *page)
1da177e4 2577{
7e007355 2578 void *freelist;
b28a02de
PE
2579 size_t offset;
2580 gfp_t local_flags;
ce8eb6c4 2581 struct kmem_cache_node *n;
1da177e4 2582
a737b3e2
AM
2583 /*
2584 * Be lazy and only check for valid flags here, keeping it out of the
2585 * critical path in kmem_cache_alloc().
1da177e4 2586 */
c871ac4e
AM
2587 if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
2588 pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
2589 BUG();
2590 }
6cb06229 2591 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
1da177e4 2592
ce8eb6c4 2593 /* Take the node list lock to change the colour_next on this node */
1da177e4 2594 check_irq_off();
18bf8541 2595 n = get_node(cachep, nodeid);
ce8eb6c4 2596 spin_lock(&n->list_lock);
1da177e4
LT
2597
2598 /* Get colour for the slab, and cal the next value. */
ce8eb6c4
CL
2599 offset = n->colour_next;
2600 n->colour_next++;
2601 if (n->colour_next >= cachep->colour)
2602 n->colour_next = 0;
2603 spin_unlock(&n->list_lock);
1da177e4 2604
2e1217cf 2605 offset *= cachep->colour_off;
1da177e4 2606
d0164adc 2607 if (gfpflags_allow_blocking(local_flags))
1da177e4
LT
2608 local_irq_enable();
2609
2610 /*
2611 * The test for missing atomic flag is performed here, rather than
2612 * the more obvious place, simply to reduce the critical path length
2613 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
2614 * will eventually be caught here (where it matters).
2615 */
2616 kmem_flagcheck(cachep, flags);
2617
a737b3e2
AM
2618 /*
2619 * Get mem for the objs. Attempt to allocate a physical page from
2620 * 'nodeid'.
e498be7d 2621 */
0c3aa83e
JK
2622 if (!page)
2623 page = kmem_getpages(cachep, local_flags, nodeid);
2624 if (!page)
1da177e4
LT
2625 goto failed;
2626
2627 /* Get slab management. */
8456a648 2628 freelist = alloc_slabmgmt(cachep, page, offset,
6cb06229 2629 local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
8456a648 2630 if (!freelist)
1da177e4
LT
2631 goto opps1;
2632
8456a648 2633 slab_map_pages(cachep, page, freelist);
1da177e4 2634
8456a648 2635 cache_init_objs(cachep, page);
1da177e4 2636
d0164adc 2637 if (gfpflags_allow_blocking(local_flags))
1da177e4
LT
2638 local_irq_disable();
2639 check_irq_off();
ce8eb6c4 2640 spin_lock(&n->list_lock);
1da177e4
LT
2641
2642 /* Make slab active. */
8456a648 2643 list_add_tail(&page->lru, &(n->slabs_free));
1da177e4 2644 STATS_INC_GROWN(cachep);
ce8eb6c4
CL
2645 n->free_objects += cachep->num;
2646 spin_unlock(&n->list_lock);
1da177e4 2647 return 1;
a737b3e2 2648opps1:
0c3aa83e 2649 kmem_freepages(cachep, page);
a737b3e2 2650failed:
d0164adc 2651 if (gfpflags_allow_blocking(local_flags))
1da177e4
LT
2652 local_irq_disable();
2653 return 0;
2654}
2655
2656#if DEBUG
2657
2658/*
2659 * Perform extra freeing checks:
2660 * - detect bad pointers.
2661 * - POISON/RED_ZONE checking
1da177e4
LT
2662 */
2663static void kfree_debugcheck(const void *objp)
2664{
1da177e4
LT
2665 if (!virt_addr_valid(objp)) {
2666 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
b28a02de
PE
2667 (unsigned long)objp);
2668 BUG();
1da177e4 2669 }
1da177e4
LT
2670}
2671
58ce1fd5
PE
2672static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
2673{
b46b8f19 2674 unsigned long long redzone1, redzone2;
58ce1fd5
PE
2675
2676 redzone1 = *dbg_redzone1(cache, obj);
2677 redzone2 = *dbg_redzone2(cache, obj);
2678
2679 /*
2680 * Redzone is ok.
2681 */
2682 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2683 return;
2684
2685 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2686 slab_error(cache, "double free detected");
2687 else
2688 slab_error(cache, "memory outside object was overwritten");
2689
b46b8f19 2690 printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
58ce1fd5
PE
2691 obj, redzone1, redzone2);
2692}
2693
343e0d7a 2694static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
7c0cb9c6 2695 unsigned long caller)
1da177e4 2696{
1da177e4 2697 unsigned int objnr;
8456a648 2698 struct page *page;
1da177e4 2699
80cbd911
MW
2700 BUG_ON(virt_to_cache(objp) != cachep);
2701
3dafccf2 2702 objp -= obj_offset(cachep);
1da177e4 2703 kfree_debugcheck(objp);
b49af68f 2704 page = virt_to_head_page(objp);
1da177e4 2705
1da177e4 2706 if (cachep->flags & SLAB_RED_ZONE) {
58ce1fd5 2707 verify_redzone_free(cachep, objp);
1da177e4
LT
2708 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2709 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2710 }
2711 if (cachep->flags & SLAB_STORE_USER)
7c0cb9c6 2712 *dbg_userword(cachep, objp) = (void *)caller;
1da177e4 2713
8456a648 2714 objnr = obj_to_index(cachep, page, objp);
1da177e4
LT
2715
2716 BUG_ON(objnr >= cachep->num);
8456a648 2717 BUG_ON(objp != index_to_obj(cachep, page, objnr));
1da177e4 2718
03787301 2719 set_obj_status(page, objnr, OBJECT_FREE);
1da177e4
LT
2720 if (cachep->flags & SLAB_POISON) {
2721#ifdef CONFIG_DEBUG_PAGEALLOC
a307ebd4
JK
2722 if (debug_pagealloc_enabled() &&
2723 (cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) {
7c0cb9c6 2724 store_stackinfo(cachep, objp, caller);
b28a02de 2725 kernel_map_pages(virt_to_page(objp),
3b0efdfa 2726 cachep->size / PAGE_SIZE, 0);
1da177e4
LT
2727 } else {
2728 poison_obj(cachep, objp, POISON_FREE);
2729 }
2730#else
2731 poison_obj(cachep, objp, POISON_FREE);
2732#endif
2733 }
2734 return objp;
2735}
2736
1da177e4
LT
2737#else
2738#define kfree_debugcheck(x) do { } while(0)
2739#define cache_free_debugcheck(x,objp,z) (objp)
1da177e4
LT
2740#endif
2741
7aa0d227
GT
2742static struct page *get_first_slab(struct kmem_cache_node *n)
2743{
2744 struct page *page;
2745
2746 page = list_first_entry_or_null(&n->slabs_partial,
2747 struct page, lru);
2748 if (!page) {
2749 n->free_touched = 1;
2750 page = list_first_entry_or_null(&n->slabs_free,
2751 struct page, lru);
2752 }
2753
2754 return page;
2755}
2756
072bb0aa
MG
2757static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags,
2758 bool force_refill)
1da177e4
LT
2759{
2760 int batchcount;
ce8eb6c4 2761 struct kmem_cache_node *n;
1da177e4 2762 struct array_cache *ac;
1ca4cb24
PE
2763 int node;
2764
1da177e4 2765 check_irq_off();
7d6e6d09 2766 node = numa_mem_id();
072bb0aa
MG
2767 if (unlikely(force_refill))
2768 goto force_grow;
2769retry:
9a2dba4b 2770 ac = cpu_cache_get(cachep);
1da177e4
LT
2771 batchcount = ac->batchcount;
2772 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
a737b3e2
AM
2773 /*
2774 * If there was little recent activity on this cache, then
2775 * perform only a partial refill. Otherwise we could generate
2776 * refill bouncing.
1da177e4
LT
2777 */
2778 batchcount = BATCHREFILL_LIMIT;
2779 }
18bf8541 2780 n = get_node(cachep, node);
e498be7d 2781
ce8eb6c4
CL
2782 BUG_ON(ac->avail > 0 || !n);
2783 spin_lock(&n->list_lock);
1da177e4 2784
3ded175a 2785 /* See if we can refill from the shared array */
ce8eb6c4
CL
2786 if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
2787 n->shared->touched = 1;
3ded175a 2788 goto alloc_done;
44b57f1c 2789 }
3ded175a 2790
1da177e4 2791 while (batchcount > 0) {
8456a648 2792 struct page *page;
1da177e4 2793 /* Get slab alloc is to come from. */
7aa0d227
GT
2794 page = get_first_slab(n);
2795 if (!page)
2796 goto must_grow;
1da177e4 2797
1da177e4 2798 check_spinlock_acquired(cachep);
714b8171
PE
2799
2800 /*
2801 * The slab was either on partial or free list so
2802 * there must be at least one object available for
2803 * allocation.
2804 */
8456a648 2805 BUG_ON(page->active >= cachep->num);
714b8171 2806
8456a648 2807 while (page->active < cachep->num && batchcount--) {
1da177e4
LT
2808 STATS_INC_ALLOCED(cachep);
2809 STATS_INC_ACTIVE(cachep);
2810 STATS_SET_HIGH(cachep);
2811
260b61dd 2812 ac_put_obj(cachep, ac, slab_get_obj(cachep, page));
1da177e4 2813 }
1da177e4
LT
2814
2815 /* move slabp to correct slabp list: */
8456a648
JK
2816 list_del(&page->lru);
2817 if (page->active == cachep->num)
34bf6ef9 2818 list_add(&page->lru, &n->slabs_full);
1da177e4 2819 else
34bf6ef9 2820 list_add(&page->lru, &n->slabs_partial);
1da177e4
LT
2821 }
2822
a737b3e2 2823must_grow:
ce8eb6c4 2824 n->free_objects -= ac->avail;
a737b3e2 2825alloc_done:
ce8eb6c4 2826 spin_unlock(&n->list_lock);
1da177e4
LT
2827
2828 if (unlikely(!ac->avail)) {
2829 int x;
072bb0aa 2830force_grow:
4167e9b2 2831 x = cache_grow(cachep, gfp_exact_node(flags), node, NULL);
e498be7d 2832
a737b3e2 2833 /* cache_grow can reenable interrupts, then ac could change. */
9a2dba4b 2834 ac = cpu_cache_get(cachep);
51cd8e6f 2835 node = numa_mem_id();
072bb0aa
MG
2836
2837 /* no objects in sight? abort */
2838 if (!x && (ac->avail == 0 || force_refill))
1da177e4
LT
2839 return NULL;
2840
a737b3e2 2841 if (!ac->avail) /* objects refilled by interrupt? */
1da177e4
LT
2842 goto retry;
2843 }
2844 ac->touched = 1;
072bb0aa
MG
2845
2846 return ac_get_obj(cachep, ac, flags, force_refill);
1da177e4
LT
2847}
2848
a737b3e2
AM
2849static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
2850 gfp_t flags)
1da177e4 2851{
d0164adc 2852 might_sleep_if(gfpflags_allow_blocking(flags));
1da177e4
LT
2853#if DEBUG
2854 kmem_flagcheck(cachep, flags);
2855#endif
2856}
2857
2858#if DEBUG
a737b3e2 2859static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
7c0cb9c6 2860 gfp_t flags, void *objp, unsigned long caller)
1da177e4 2861{
03787301
JK
2862 struct page *page;
2863
b28a02de 2864 if (!objp)
1da177e4 2865 return objp;
b28a02de 2866 if (cachep->flags & SLAB_POISON) {
1da177e4 2867#ifdef CONFIG_DEBUG_PAGEALLOC
a307ebd4
JK
2868 if (debug_pagealloc_enabled() &&
2869 (cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
b28a02de 2870 kernel_map_pages(virt_to_page(objp),
3b0efdfa 2871 cachep->size / PAGE_SIZE, 1);
1da177e4
LT
2872 else
2873 check_poison_obj(cachep, objp);
2874#else
2875 check_poison_obj(cachep, objp);
2876#endif
2877 poison_obj(cachep, objp, POISON_INUSE);
2878 }
2879 if (cachep->flags & SLAB_STORE_USER)
7c0cb9c6 2880 *dbg_userword(cachep, objp) = (void *)caller;
1da177e4
LT
2881
2882 if (cachep->flags & SLAB_RED_ZONE) {
a737b3e2
AM
2883 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
2884 *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
2885 slab_error(cachep, "double free, or memory outside"
2886 " object was overwritten");
b28a02de 2887 printk(KERN_ERR
b46b8f19 2888 "%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
a737b3e2
AM
2889 objp, *dbg_redzone1(cachep, objp),
2890 *dbg_redzone2(cachep, objp));
1da177e4
LT
2891 }
2892 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
2893 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
2894 }
03787301
JK
2895
2896 page = virt_to_head_page(objp);
2897 set_obj_status(page, obj_to_index(cachep, page, objp), OBJECT_ACTIVE);
3dafccf2 2898 objp += obj_offset(cachep);
4f104934 2899 if (cachep->ctor && cachep->flags & SLAB_POISON)
51cc5068 2900 cachep->ctor(objp);
7ea466f2
TH
2901 if (ARCH_SLAB_MINALIGN &&
2902 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
a44b56d3 2903 printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
c225150b 2904 objp, (int)ARCH_SLAB_MINALIGN);
a44b56d3 2905 }
1da177e4
LT
2906 return objp;
2907}
2908#else
2909#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
2910#endif
2911
343e0d7a 2912static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
1da177e4 2913{
b28a02de 2914 void *objp;
1da177e4 2915 struct array_cache *ac;
072bb0aa 2916 bool force_refill = false;
1da177e4 2917
5c382300 2918 check_irq_off();
8a8b6502 2919
9a2dba4b 2920 ac = cpu_cache_get(cachep);
1da177e4 2921 if (likely(ac->avail)) {
1da177e4 2922 ac->touched = 1;
072bb0aa
MG
2923 objp = ac_get_obj(cachep, ac, flags, false);
2924
ddbf2e83 2925 /*
072bb0aa
MG
2926 * Allow for the possibility all avail objects are not allowed
2927 * by the current flags
ddbf2e83 2928 */
072bb0aa
MG
2929 if (objp) {
2930 STATS_INC_ALLOCHIT(cachep);
2931 goto out;
2932 }
2933 force_refill = true;
1da177e4 2934 }
072bb0aa
MG
2935
2936 STATS_INC_ALLOCMISS(cachep);
2937 objp = cache_alloc_refill(cachep, flags, force_refill);
2938 /*
2939 * the 'ac' may be updated by cache_alloc_refill(),
2940 * and kmemleak_erase() requires its correct value.
2941 */
2942 ac = cpu_cache_get(cachep);
2943
2944out:
d5cff635
CM
2945 /*
2946 * To avoid a false negative, if an object that is in one of the
2947 * per-CPU caches is leaked, we need to make sure kmemleak doesn't
2948 * treat the array pointers as a reference to the object.
2949 */
f3d8b53a
O
2950 if (objp)
2951 kmemleak_erase(&ac->entry[ac->avail]);
5c382300
AK
2952 return objp;
2953}
2954
e498be7d 2955#ifdef CONFIG_NUMA
c61afb18 2956/*
2ad654bc 2957 * Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
c61afb18
PJ
2958 *
2959 * If we are in_interrupt, then process context, including cpusets and
2960 * mempolicy, may not apply and should not be used for allocation policy.
2961 */
2962static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
2963{
2964 int nid_alloc, nid_here;
2965
765c4507 2966 if (in_interrupt() || (flags & __GFP_THISNODE))
c61afb18 2967 return NULL;
7d6e6d09 2968 nid_alloc = nid_here = numa_mem_id();
c61afb18 2969 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
6adef3eb 2970 nid_alloc = cpuset_slab_spread_node();
c61afb18 2971 else if (current->mempolicy)
2a389610 2972 nid_alloc = mempolicy_slab_node();
c61afb18 2973 if (nid_alloc != nid_here)
8b98c169 2974 return ____cache_alloc_node(cachep, flags, nid_alloc);
c61afb18
PJ
2975 return NULL;
2976}
2977
765c4507
CL
2978/*
2979 * Fallback function if there was no memory available and no objects on a
3c517a61 2980 * certain node and fall back is permitted. First we scan all the
6a67368c 2981 * available node for available objects. If that fails then we
3c517a61
CL
2982 * perform an allocation without specifying a node. This allows the page
2983 * allocator to do its reclaim / fallback magic. We then insert the
2984 * slab into the proper nodelist and then allocate from it.
765c4507 2985 */
8c8cc2c1 2986static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
765c4507 2987{
8c8cc2c1
PE
2988 struct zonelist *zonelist;
2989 gfp_t local_flags;
dd1a239f 2990 struct zoneref *z;
54a6eb5c
MG
2991 struct zone *zone;
2992 enum zone_type high_zoneidx = gfp_zone(flags);
765c4507 2993 void *obj = NULL;
3c517a61 2994 int nid;
cc9a6c87 2995 unsigned int cpuset_mems_cookie;
8c8cc2c1
PE
2996
2997 if (flags & __GFP_THISNODE)
2998 return NULL;
2999
6cb06229 3000 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
765c4507 3001
cc9a6c87 3002retry_cpuset:
d26914d1 3003 cpuset_mems_cookie = read_mems_allowed_begin();
2a389610 3004 zonelist = node_zonelist(mempolicy_slab_node(), flags);
cc9a6c87 3005
3c517a61
CL
3006retry:
3007 /*
3008 * Look through allowed nodes for objects available
3009 * from existing per node queues.
3010 */
54a6eb5c
MG
3011 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3012 nid = zone_to_nid(zone);
aedb0eb1 3013
061d7074 3014 if (cpuset_zone_allowed(zone, flags) &&
18bf8541
CL
3015 get_node(cache, nid) &&
3016 get_node(cache, nid)->free_objects) {
3c517a61 3017 obj = ____cache_alloc_node(cache,
4167e9b2 3018 gfp_exact_node(flags), nid);
481c5346
CL
3019 if (obj)
3020 break;
3021 }
3c517a61
CL
3022 }
3023
cfce6604 3024 if (!obj) {
3c517a61
CL
3025 /*
3026 * This allocation will be performed within the constraints
3027 * of the current cpuset / memory policy requirements.
3028 * We may trigger various forms of reclaim on the allowed
3029 * set and go into memory reserves if necessary.
3030 */
0c3aa83e
JK
3031 struct page *page;
3032
d0164adc 3033 if (gfpflags_allow_blocking(local_flags))
dd47ea75
CL
3034 local_irq_enable();
3035 kmem_flagcheck(cache, flags);
0c3aa83e 3036 page = kmem_getpages(cache, local_flags, numa_mem_id());
d0164adc 3037 if (gfpflags_allow_blocking(local_flags))
dd47ea75 3038 local_irq_disable();
0c3aa83e 3039 if (page) {
3c517a61
CL
3040 /*
3041 * Insert into the appropriate per node queues
3042 */
0c3aa83e
JK
3043 nid = page_to_nid(page);
3044 if (cache_grow(cache, flags, nid, page)) {
3c517a61 3045 obj = ____cache_alloc_node(cache,
4167e9b2 3046 gfp_exact_node(flags), nid);
3c517a61
CL
3047 if (!obj)
3048 /*
3049 * Another processor may allocate the
3050 * objects in the slab since we are
3051 * not holding any locks.
3052 */
3053 goto retry;
3054 } else {
b6a60451 3055 /* cache_grow already freed obj */
3c517a61
CL
3056 obj = NULL;
3057 }
3058 }
aedb0eb1 3059 }
cc9a6c87 3060
d26914d1 3061 if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
cc9a6c87 3062 goto retry_cpuset;
765c4507
CL
3063 return obj;
3064}
3065
e498be7d
CL
3066/*
3067 * A interface to enable slab creation on nodeid
1da177e4 3068 */
8b98c169 3069static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
a737b3e2 3070 int nodeid)
e498be7d 3071{
8456a648 3072 struct page *page;
ce8eb6c4 3073 struct kmem_cache_node *n;
b28a02de 3074 void *obj;
b28a02de
PE
3075 int x;
3076
7c3fbbdd 3077 VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
18bf8541 3078 n = get_node(cachep, nodeid);
ce8eb6c4 3079 BUG_ON(!n);
b28a02de 3080
a737b3e2 3081retry:
ca3b9b91 3082 check_irq_off();
ce8eb6c4 3083 spin_lock(&n->list_lock);
7aa0d227
GT
3084 page = get_first_slab(n);
3085 if (!page)
3086 goto must_grow;
b28a02de 3087
b28a02de 3088 check_spinlock_acquired_node(cachep, nodeid);
b28a02de
PE
3089
3090 STATS_INC_NODEALLOCS(cachep);
3091 STATS_INC_ACTIVE(cachep);
3092 STATS_SET_HIGH(cachep);
3093
8456a648 3094 BUG_ON(page->active == cachep->num);
b28a02de 3095
260b61dd 3096 obj = slab_get_obj(cachep, page);
ce8eb6c4 3097 n->free_objects--;
b28a02de 3098 /* move slabp to correct slabp list: */
8456a648 3099 list_del(&page->lru);
b28a02de 3100
8456a648
JK
3101 if (page->active == cachep->num)
3102 list_add(&page->lru, &n->slabs_full);
a737b3e2 3103 else
8456a648 3104 list_add(&page->lru, &n->slabs_partial);
e498be7d 3105
ce8eb6c4 3106 spin_unlock(&n->list_lock);
b28a02de 3107 goto done;
e498be7d 3108
a737b3e2 3109must_grow:
ce8eb6c4 3110 spin_unlock(&n->list_lock);
4167e9b2 3111 x = cache_grow(cachep, gfp_exact_node(flags), nodeid, NULL);
765c4507
CL
3112 if (x)
3113 goto retry;
1da177e4 3114
8c8cc2c1 3115 return fallback_alloc(cachep, flags);
e498be7d 3116
a737b3e2 3117done:
b28a02de 3118 return obj;
e498be7d 3119}
8c8cc2c1 3120
8c8cc2c1 3121static __always_inline void *
48356303 3122slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
7c0cb9c6 3123 unsigned long caller)
8c8cc2c1
PE
3124{
3125 unsigned long save_flags;
3126 void *ptr;
7d6e6d09 3127 int slab_node = numa_mem_id();
8c8cc2c1 3128
dcce284a 3129 flags &= gfp_allowed_mask;
011eceaf
JDB
3130 cachep = slab_pre_alloc_hook(cachep, flags);
3131 if (unlikely(!cachep))
824ebef1
AM
3132 return NULL;
3133
8c8cc2c1
PE
3134 cache_alloc_debugcheck_before(cachep, flags);
3135 local_irq_save(save_flags);
3136
eacbbae3 3137 if (nodeid == NUMA_NO_NODE)
7d6e6d09 3138 nodeid = slab_node;
8c8cc2c1 3139
18bf8541 3140 if (unlikely(!get_node(cachep, nodeid))) {
8c8cc2c1
PE
3141 /* Node not bootstrapped yet */
3142 ptr = fallback_alloc(cachep, flags);
3143 goto out;
3144 }
3145
7d6e6d09 3146 if (nodeid == slab_node) {
8c8cc2c1
PE
3147 /*
3148 * Use the locally cached objects if possible.
3149 * However ____cache_alloc does not allow fallback
3150 * to other nodes. It may fail while we still have
3151 * objects on other nodes available.
3152 */
3153 ptr = ____cache_alloc(cachep, flags);
3154 if (ptr)
3155 goto out;
3156 }
3157 /* ___cache_alloc_node can fall back to other nodes */
3158 ptr = ____cache_alloc_node(cachep, flags, nodeid);
3159 out:
3160 local_irq_restore(save_flags);
3161 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
3162
d5e3ed66
JDB
3163 if (unlikely(flags & __GFP_ZERO) && ptr)
3164 memset(ptr, 0, cachep->object_size);
d07dbea4 3165
d5e3ed66 3166 slab_post_alloc_hook(cachep, flags, 1, &ptr);
8c8cc2c1
PE
3167 return ptr;
3168}
3169
3170static __always_inline void *
3171__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3172{
3173 void *objp;
3174
2ad654bc 3175 if (current->mempolicy || cpuset_do_slab_mem_spread()) {
8c8cc2c1
PE
3176 objp = alternate_node_alloc(cache, flags);
3177 if (objp)
3178 goto out;
3179 }
3180 objp = ____cache_alloc(cache, flags);
3181
3182 /*
3183 * We may just have run out of memory on the local node.
3184 * ____cache_alloc_node() knows how to locate memory on other nodes
3185 */
7d6e6d09
LS
3186 if (!objp)
3187 objp = ____cache_alloc_node(cache, flags, numa_mem_id());
8c8cc2c1
PE
3188
3189 out:
3190 return objp;
3191}
3192#else
3193
3194static __always_inline void *
3195__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3196{
3197 return ____cache_alloc(cachep, flags);
3198}
3199
3200#endif /* CONFIG_NUMA */
3201
3202static __always_inline void *
48356303 3203slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
8c8cc2c1
PE
3204{
3205 unsigned long save_flags;
3206 void *objp;
3207
dcce284a 3208 flags &= gfp_allowed_mask;
011eceaf
JDB
3209 cachep = slab_pre_alloc_hook(cachep, flags);
3210 if (unlikely(!cachep))
824ebef1
AM
3211 return NULL;
3212
8c8cc2c1
PE
3213 cache_alloc_debugcheck_before(cachep, flags);
3214 local_irq_save(save_flags);
3215 objp = __do_cache_alloc(cachep, flags);
3216 local_irq_restore(save_flags);
3217 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3218 prefetchw(objp);
3219
d5e3ed66
JDB
3220 if (unlikely(flags & __GFP_ZERO) && objp)
3221 memset(objp, 0, cachep->object_size);
d07dbea4 3222
d5e3ed66 3223 slab_post_alloc_hook(cachep, flags, 1, &objp);
8c8cc2c1
PE
3224 return objp;
3225}
e498be7d
CL
3226
3227/*
5f0985bb 3228 * Caller needs to acquire correct kmem_cache_node's list_lock
97654dfa 3229 * @list: List of detached free slabs should be freed by caller
e498be7d 3230 */
97654dfa
JK
3231static void free_block(struct kmem_cache *cachep, void **objpp,
3232 int nr_objects, int node, struct list_head *list)
1da177e4
LT
3233{
3234 int i;
25c063fb 3235 struct kmem_cache_node *n = get_node(cachep, node);
1da177e4
LT
3236
3237 for (i = 0; i < nr_objects; i++) {
072bb0aa 3238 void *objp;
8456a648 3239 struct page *page;
1da177e4 3240
072bb0aa
MG
3241 clear_obj_pfmemalloc(&objpp[i]);
3242 objp = objpp[i];
3243
8456a648 3244 page = virt_to_head_page(objp);
8456a648 3245 list_del(&page->lru);
ff69416e 3246 check_spinlock_acquired_node(cachep, node);
260b61dd 3247 slab_put_obj(cachep, page, objp);
1da177e4 3248 STATS_DEC_ACTIVE(cachep);
ce8eb6c4 3249 n->free_objects++;
1da177e4
LT
3250
3251 /* fixup slab chains */
8456a648 3252 if (page->active == 0) {
ce8eb6c4
CL
3253 if (n->free_objects > n->free_limit) {
3254 n->free_objects -= cachep->num;
97654dfa 3255 list_add_tail(&page->lru, list);
1da177e4 3256 } else {
8456a648 3257 list_add(&page->lru, &n->slabs_free);
1da177e4
LT
3258 }
3259 } else {
3260 /* Unconditionally move a slab to the end of the
3261 * partial list on free - maximum time for the
3262 * other objects to be freed, too.
3263 */
8456a648 3264 list_add_tail(&page->lru, &n->slabs_partial);
1da177e4
LT
3265 }
3266 }
3267}
3268
343e0d7a 3269static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
1da177e4
LT
3270{
3271 int batchcount;
ce8eb6c4 3272 struct kmem_cache_node *n;
7d6e6d09 3273 int node = numa_mem_id();
97654dfa 3274 LIST_HEAD(list);
1da177e4
LT
3275
3276 batchcount = ac->batchcount;
260b61dd 3277
1da177e4 3278 check_irq_off();
18bf8541 3279 n = get_node(cachep, node);
ce8eb6c4
CL
3280 spin_lock(&n->list_lock);
3281 if (n->shared) {
3282 struct array_cache *shared_array = n->shared;
b28a02de 3283 int max = shared_array->limit - shared_array->avail;
1da177e4
LT
3284 if (max) {
3285 if (batchcount > max)
3286 batchcount = max;
e498be7d 3287 memcpy(&(shared_array->entry[shared_array->avail]),
b28a02de 3288 ac->entry, sizeof(void *) * batchcount);
1da177e4
LT
3289 shared_array->avail += batchcount;
3290 goto free_done;
3291 }
3292 }
3293
97654dfa 3294 free_block(cachep, ac->entry, batchcount, node, &list);
a737b3e2 3295free_done:
1da177e4
LT
3296#if STATS
3297 {
3298 int i = 0;
73c0219d 3299 struct page *page;
1da177e4 3300
73c0219d 3301 list_for_each_entry(page, &n->slabs_free, lru) {
8456a648 3302 BUG_ON(page->active);
1da177e4
LT
3303
3304 i++;
1da177e4
LT
3305 }
3306 STATS_SET_FREEABLE(cachep, i);
3307 }
3308#endif
ce8eb6c4 3309 spin_unlock(&n->list_lock);
97654dfa 3310 slabs_destroy(cachep, &list);
1da177e4 3311 ac->avail -= batchcount;
a737b3e2 3312 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
1da177e4
LT
3313}
3314
3315/*
a737b3e2
AM
3316 * Release an obj back to its cache. If the obj has a constructed state, it must
3317 * be in this state _before_ it is released. Called with disabled ints.
1da177e4 3318 */
a947eb95 3319static inline void __cache_free(struct kmem_cache *cachep, void *objp,
7c0cb9c6 3320 unsigned long caller)
1da177e4 3321{
9a2dba4b 3322 struct array_cache *ac = cpu_cache_get(cachep);
1da177e4
LT
3323
3324 check_irq_off();
d5cff635 3325 kmemleak_free_recursive(objp, cachep->flags);
a947eb95 3326 objp = cache_free_debugcheck(cachep, objp, caller);
1da177e4 3327
8c138bc0 3328 kmemcheck_slab_free(cachep, objp, cachep->object_size);
c175eea4 3329
1807a1aa
SS
3330 /*
3331 * Skip calling cache_free_alien() when the platform is not numa.
3332 * This will avoid cache misses that happen while accessing slabp (which
3333 * is per page memory reference) to get nodeid. Instead use a global
3334 * variable to skip the call, which is mostly likely to be present in
3335 * the cache.
3336 */
b6e68bc1 3337 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
729bd0b7
PE
3338 return;
3339
3d880194 3340 if (ac->avail < ac->limit) {
1da177e4 3341 STATS_INC_FREEHIT(cachep);
1da177e4
LT
3342 } else {
3343 STATS_INC_FREEMISS(cachep);
3344 cache_flusharray(cachep, ac);
1da177e4 3345 }
42c8c99c 3346
072bb0aa 3347 ac_put_obj(cachep, ac, objp);
1da177e4
LT
3348}
3349
3350/**
3351 * kmem_cache_alloc - Allocate an object
3352 * @cachep: The cache to allocate from.
3353 * @flags: See kmalloc().
3354 *
3355 * Allocate an object from this cache. The flags are only relevant
3356 * if the cache has no available objects.
3357 */
343e0d7a 3358void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
1da177e4 3359{
48356303 3360 void *ret = slab_alloc(cachep, flags, _RET_IP_);
36555751 3361
ca2b84cb 3362 trace_kmem_cache_alloc(_RET_IP_, ret,
8c138bc0 3363 cachep->object_size, cachep->size, flags);
36555751
EGM
3364
3365 return ret;
1da177e4
LT
3366}
3367EXPORT_SYMBOL(kmem_cache_alloc);
3368
7b0501dd
JDB
3369static __always_inline void
3370cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
3371 size_t size, void **p, unsigned long caller)
3372{
3373 size_t i;
3374
3375 for (i = 0; i < size; i++)
3376 p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
3377}
3378
865762a8 3379int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
2a777eac 3380 void **p)
484748f0 3381{
2a777eac
JDB
3382 size_t i;
3383
3384 s = slab_pre_alloc_hook(s, flags);
3385 if (!s)
3386 return 0;
3387
3388 cache_alloc_debugcheck_before(s, flags);
3389
3390 local_irq_disable();
3391 for (i = 0; i < size; i++) {
3392 void *objp = __do_cache_alloc(s, flags);
3393
2a777eac
JDB
3394 if (unlikely(!objp))
3395 goto error;
3396 p[i] = objp;
3397 }
3398 local_irq_enable();
3399
7b0501dd
JDB
3400 cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
3401
2a777eac
JDB
3402 /* Clear memory outside IRQ disabled section */
3403 if (unlikely(flags & __GFP_ZERO))
3404 for (i = 0; i < size; i++)
3405 memset(p[i], 0, s->object_size);
3406
3407 slab_post_alloc_hook(s, flags, size, p);
3408 /* FIXME: Trace call missing. Christoph would like a bulk variant */
3409 return size;
3410error:
3411 local_irq_enable();
7b0501dd 3412 cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
2a777eac
JDB
3413 slab_post_alloc_hook(s, flags, i, p);
3414 __kmem_cache_free_bulk(s, i, p);
3415 return 0;
484748f0
CL
3416}
3417EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3418
0f24f128 3419#ifdef CONFIG_TRACING
85beb586 3420void *
4052147c 3421kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
36555751 3422{
85beb586
SR
3423 void *ret;
3424
48356303 3425 ret = slab_alloc(cachep, flags, _RET_IP_);
85beb586
SR
3426
3427 trace_kmalloc(_RET_IP_, ret,
ff4fcd01 3428 size, cachep->size, flags);
85beb586 3429 return ret;
36555751 3430}
85beb586 3431EXPORT_SYMBOL(kmem_cache_alloc_trace);
36555751
EGM
3432#endif
3433
1da177e4 3434#ifdef CONFIG_NUMA
d0d04b78
ZL
3435/**
3436 * kmem_cache_alloc_node - Allocate an object on the specified node
3437 * @cachep: The cache to allocate from.
3438 * @flags: See kmalloc().
3439 * @nodeid: node number of the target node.
3440 *
3441 * Identical to kmem_cache_alloc but it will allocate memory on the given
3442 * node, which can improve the performance for cpu bound structures.
3443 *
3444 * Fallback to other node is possible if __GFP_THISNODE is not set.
3445 */
8b98c169
CH
3446void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
3447{
48356303 3448 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
36555751 3449
ca2b84cb 3450 trace_kmem_cache_alloc_node(_RET_IP_, ret,
8c138bc0 3451 cachep->object_size, cachep->size,
ca2b84cb 3452 flags, nodeid);
36555751
EGM
3453
3454 return ret;
8b98c169 3455}
1da177e4
LT
3456EXPORT_SYMBOL(kmem_cache_alloc_node);
3457
0f24f128 3458#ifdef CONFIG_TRACING
4052147c 3459void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
85beb586 3460 gfp_t flags,
4052147c
EG
3461 int nodeid,
3462 size_t size)
36555751 3463{
85beb586
SR
3464 void *ret;
3465
592f4145 3466 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
7c0cb9c6 3467
85beb586 3468 trace_kmalloc_node(_RET_IP_, ret,
ff4fcd01 3469 size, cachep->size,
85beb586
SR
3470 flags, nodeid);
3471 return ret;
36555751 3472}
85beb586 3473EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
36555751
EGM
3474#endif
3475
8b98c169 3476static __always_inline void *
7c0cb9c6 3477__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
97e2bde4 3478{
343e0d7a 3479 struct kmem_cache *cachep;
97e2bde4 3480
2c59dd65 3481 cachep = kmalloc_slab(size, flags);
6cb8f913
CL
3482 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3483 return cachep;
4052147c 3484 return kmem_cache_alloc_node_trace(cachep, flags, node, size);
97e2bde4 3485}
8b98c169 3486
8b98c169
CH
3487void *__kmalloc_node(size_t size, gfp_t flags, int node)
3488{
7c0cb9c6 3489 return __do_kmalloc_node(size, flags, node, _RET_IP_);
8b98c169 3490}
dbe5e69d 3491EXPORT_SYMBOL(__kmalloc_node);
8b98c169
CH
3492
3493void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
ce71e27c 3494 int node, unsigned long caller)
8b98c169 3495{
7c0cb9c6 3496 return __do_kmalloc_node(size, flags, node, caller);
8b98c169
CH
3497}
3498EXPORT_SYMBOL(__kmalloc_node_track_caller);
8b98c169 3499#endif /* CONFIG_NUMA */
1da177e4
LT
3500
3501/**
800590f5 3502 * __do_kmalloc - allocate memory
1da177e4 3503 * @size: how many bytes of memory are required.
800590f5 3504 * @flags: the type of memory to allocate (see kmalloc).
911851e6 3505 * @caller: function caller for debug tracking of the caller
1da177e4 3506 */
7fd6b141 3507static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
7c0cb9c6 3508 unsigned long caller)
1da177e4 3509{
343e0d7a 3510 struct kmem_cache *cachep;
36555751 3511 void *ret;
1da177e4 3512
2c59dd65 3513 cachep = kmalloc_slab(size, flags);
a5c96d8a
LT
3514 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3515 return cachep;
48356303 3516 ret = slab_alloc(cachep, flags, caller);
36555751 3517
7c0cb9c6 3518 trace_kmalloc(caller, ret,
3b0efdfa 3519 size, cachep->size, flags);
36555751
EGM
3520
3521 return ret;
7fd6b141
PE
3522}
3523
7fd6b141
PE
3524void *__kmalloc(size_t size, gfp_t flags)
3525{
7c0cb9c6 3526 return __do_kmalloc(size, flags, _RET_IP_);
1da177e4
LT
3527}
3528EXPORT_SYMBOL(__kmalloc);
3529
ce71e27c 3530void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
7fd6b141 3531{
7c0cb9c6 3532 return __do_kmalloc(size, flags, caller);
7fd6b141
PE
3533}
3534EXPORT_SYMBOL(__kmalloc_track_caller);
1d2c8eea 3535
1da177e4
LT
3536/**
3537 * kmem_cache_free - Deallocate an object
3538 * @cachep: The cache the allocation was from.
3539 * @objp: The previously allocated object.
3540 *
3541 * Free an object which was previously allocated from this
3542 * cache.
3543 */
343e0d7a 3544void kmem_cache_free(struct kmem_cache *cachep, void *objp)
1da177e4
LT
3545{
3546 unsigned long flags;
b9ce5ef4
GC
3547 cachep = cache_from_obj(cachep, objp);
3548 if (!cachep)
3549 return;
1da177e4
LT
3550
3551 local_irq_save(flags);
d97d476b 3552 debug_check_no_locks_freed(objp, cachep->object_size);
3ac7fe5a 3553 if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
8c138bc0 3554 debug_check_no_obj_freed(objp, cachep->object_size);
7c0cb9c6 3555 __cache_free(cachep, objp, _RET_IP_);
1da177e4 3556 local_irq_restore(flags);
36555751 3557
ca2b84cb 3558 trace_kmem_cache_free(_RET_IP_, objp);
1da177e4
LT
3559}
3560EXPORT_SYMBOL(kmem_cache_free);
3561
e6cdb58d
JDB
3562void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
3563{
3564 struct kmem_cache *s;
3565 size_t i;
3566
3567 local_irq_disable();
3568 for (i = 0; i < size; i++) {
3569 void *objp = p[i];
3570
ca257195
JDB
3571 if (!orig_s) /* called via kfree_bulk */
3572 s = virt_to_cache(objp);
3573 else
3574 s = cache_from_obj(orig_s, objp);
e6cdb58d
JDB
3575
3576 debug_check_no_locks_freed(objp, s->object_size);
3577 if (!(s->flags & SLAB_DEBUG_OBJECTS))
3578 debug_check_no_obj_freed(objp, s->object_size);
3579
3580 __cache_free(s, objp, _RET_IP_);
3581 }
3582 local_irq_enable();
3583
3584 /* FIXME: add tracing */
3585}
3586EXPORT_SYMBOL(kmem_cache_free_bulk);
3587
1da177e4
LT
3588/**
3589 * kfree - free previously allocated memory
3590 * @objp: pointer returned by kmalloc.
3591 *
80e93eff
PE
3592 * If @objp is NULL, no operation is performed.
3593 *
1da177e4
LT
3594 * Don't free memory not originally allocated by kmalloc()
3595 * or you will run into trouble.
3596 */
3597void kfree(const void *objp)
3598{
343e0d7a 3599 struct kmem_cache *c;
1da177e4
LT
3600 unsigned long flags;
3601
2121db74
PE
3602 trace_kfree(_RET_IP_, objp);
3603
6cb8f913 3604 if (unlikely(ZERO_OR_NULL_PTR(objp)))
1da177e4
LT
3605 return;
3606 local_irq_save(flags);
3607 kfree_debugcheck(objp);
6ed5eb22 3608 c = virt_to_cache(objp);
8c138bc0
CL
3609 debug_check_no_locks_freed(objp, c->object_size);
3610
3611 debug_check_no_obj_freed(objp, c->object_size);
7c0cb9c6 3612 __cache_free(c, (void *)objp, _RET_IP_);
1da177e4
LT
3613 local_irq_restore(flags);
3614}
3615EXPORT_SYMBOL(kfree);
3616
e498be7d 3617/*
ce8eb6c4 3618 * This initializes kmem_cache_node or resizes various caches for all nodes.
e498be7d 3619 */
5f0985bb 3620static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp)
e498be7d
CL
3621{
3622 int node;
ce8eb6c4 3623 struct kmem_cache_node *n;
cafeb02e 3624 struct array_cache *new_shared;
c8522a3a 3625 struct alien_cache **new_alien = NULL;
e498be7d 3626
9c09a95c 3627 for_each_online_node(node) {
cafeb02e 3628
b455def2
L
3629 if (use_alien_caches) {
3630 new_alien = alloc_alien_cache(node, cachep->limit, gfp);
3631 if (!new_alien)
3632 goto fail;
3633 }
cafeb02e 3634
63109846
ED
3635 new_shared = NULL;
3636 if (cachep->shared) {
3637 new_shared = alloc_arraycache(node,
0718dc2a 3638 cachep->shared*cachep->batchcount,
83b519e8 3639 0xbaadf00d, gfp);
63109846
ED
3640 if (!new_shared) {
3641 free_alien_cache(new_alien);
3642 goto fail;
3643 }
0718dc2a 3644 }
cafeb02e 3645
18bf8541 3646 n = get_node(cachep, node);
ce8eb6c4
CL
3647 if (n) {
3648 struct array_cache *shared = n->shared;
97654dfa 3649 LIST_HEAD(list);
cafeb02e 3650
ce8eb6c4 3651 spin_lock_irq(&n->list_lock);
e498be7d 3652
cafeb02e 3653 if (shared)
0718dc2a 3654 free_block(cachep, shared->entry,
97654dfa 3655 shared->avail, node, &list);
e498be7d 3656
ce8eb6c4
CL
3657 n->shared = new_shared;
3658 if (!n->alien) {
3659 n->alien = new_alien;
e498be7d
CL
3660 new_alien = NULL;
3661 }
ce8eb6c4 3662 n->free_limit = (1 + nr_cpus_node(node)) *
a737b3e2 3663 cachep->batchcount + cachep->num;
ce8eb6c4 3664 spin_unlock_irq(&n->list_lock);
97654dfa 3665 slabs_destroy(cachep, &list);
cafeb02e 3666 kfree(shared);
e498be7d
CL
3667 free_alien_cache(new_alien);
3668 continue;
3669 }
ce8eb6c4
CL
3670 n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
3671 if (!n) {
0718dc2a
CL
3672 free_alien_cache(new_alien);
3673 kfree(new_shared);
e498be7d 3674 goto fail;
0718dc2a 3675 }
e498be7d 3676
ce8eb6c4 3677 kmem_cache_node_init(n);
5f0985bb
JZ
3678 n->next_reap = jiffies + REAPTIMEOUT_NODE +
3679 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
ce8eb6c4
CL
3680 n->shared = new_shared;
3681 n->alien = new_alien;
3682 n->free_limit = (1 + nr_cpus_node(node)) *
a737b3e2 3683 cachep->batchcount + cachep->num;
ce8eb6c4 3684 cachep->node[node] = n;
e498be7d 3685 }
cafeb02e 3686 return 0;
0718dc2a 3687
a737b3e2 3688fail:
3b0efdfa 3689 if (!cachep->list.next) {
0718dc2a
CL
3690 /* Cache is not active yet. Roll back what we did */
3691 node--;
3692 while (node >= 0) {
18bf8541
CL
3693 n = get_node(cachep, node);
3694 if (n) {
ce8eb6c4
CL
3695 kfree(n->shared);
3696 free_alien_cache(n->alien);
3697 kfree(n);
6a67368c 3698 cachep->node[node] = NULL;
0718dc2a
CL
3699 }
3700 node--;
3701 }
3702 }
cafeb02e 3703 return -ENOMEM;
e498be7d
CL
3704}
3705
18004c5d 3706/* Always called with the slab_mutex held */
943a451a 3707static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
83b519e8 3708 int batchcount, int shared, gfp_t gfp)
1da177e4 3709{
bf0dea23
JK
3710 struct array_cache __percpu *cpu_cache, *prev;
3711 int cpu;
1da177e4 3712
bf0dea23
JK
3713 cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
3714 if (!cpu_cache)
d2e7b7d0
SS
3715 return -ENOMEM;
3716
bf0dea23
JK
3717 prev = cachep->cpu_cache;
3718 cachep->cpu_cache = cpu_cache;
3719 kick_all_cpus_sync();
e498be7d 3720
1da177e4 3721 check_irq_on();
1da177e4
LT
3722 cachep->batchcount = batchcount;
3723 cachep->limit = limit;
e498be7d 3724 cachep->shared = shared;
1da177e4 3725
bf0dea23
JK
3726 if (!prev)
3727 goto alloc_node;
3728
3729 for_each_online_cpu(cpu) {
97654dfa 3730 LIST_HEAD(list);
18bf8541
CL
3731 int node;
3732 struct kmem_cache_node *n;
bf0dea23 3733 struct array_cache *ac = per_cpu_ptr(prev, cpu);
18bf8541 3734
bf0dea23 3735 node = cpu_to_mem(cpu);
18bf8541
CL
3736 n = get_node(cachep, node);
3737 spin_lock_irq(&n->list_lock);
bf0dea23 3738 free_block(cachep, ac->entry, ac->avail, node, &list);
18bf8541 3739 spin_unlock_irq(&n->list_lock);
97654dfa 3740 slabs_destroy(cachep, &list);
1da177e4 3741 }
bf0dea23
JK
3742 free_percpu(prev);
3743
3744alloc_node:
5f0985bb 3745 return alloc_kmem_cache_node(cachep, gfp);
1da177e4
LT
3746}
3747
943a451a
GC
3748static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3749 int batchcount, int shared, gfp_t gfp)
3750{
3751 int ret;
426589f5 3752 struct kmem_cache *c;
943a451a
GC
3753
3754 ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3755
3756 if (slab_state < FULL)
3757 return ret;
3758
3759 if ((ret < 0) || !is_root_cache(cachep))
3760 return ret;
3761
426589f5
VD
3762 lockdep_assert_held(&slab_mutex);
3763 for_each_memcg_cache(c, cachep) {
3764 /* return value determined by the root cache only */
3765 __do_tune_cpucache(c, limit, batchcount, shared, gfp);
943a451a
GC
3766 }
3767
3768 return ret;
3769}
3770
18004c5d 3771/* Called with slab_mutex held always */
83b519e8 3772static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
1da177e4
LT
3773{
3774 int err;
943a451a
GC
3775 int limit = 0;
3776 int shared = 0;
3777 int batchcount = 0;
3778
3779 if (!is_root_cache(cachep)) {
3780 struct kmem_cache *root = memcg_root_cache(cachep);
3781 limit = root->limit;
3782 shared = root->shared;
3783 batchcount = root->batchcount;
3784 }
1da177e4 3785
943a451a
GC
3786 if (limit && shared && batchcount)
3787 goto skip_setup;
a737b3e2
AM
3788 /*
3789 * The head array serves three purposes:
1da177e4
LT
3790 * - create a LIFO ordering, i.e. return objects that are cache-warm
3791 * - reduce the number of spinlock operations.
a737b3e2 3792 * - reduce the number of linked list operations on the slab and
1da177e4
LT
3793 * bufctl chains: array operations are cheaper.
3794 * The numbers are guessed, we should auto-tune as described by
3795 * Bonwick.
3796 */
3b0efdfa 3797 if (cachep->size > 131072)
1da177e4 3798 limit = 1;
3b0efdfa 3799 else if (cachep->size > PAGE_SIZE)
1da177e4 3800 limit = 8;
3b0efdfa 3801 else if (cachep->size > 1024)
1da177e4 3802 limit = 24;
3b0efdfa 3803 else if (cachep->size > 256)
1da177e4
LT
3804 limit = 54;
3805 else
3806 limit = 120;
3807
a737b3e2
AM
3808 /*
3809 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
1da177e4
LT
3810 * allocation behaviour: Most allocs on one cpu, most free operations
3811 * on another cpu. For these cases, an efficient object passing between
3812 * cpus is necessary. This is provided by a shared array. The array
3813 * replaces Bonwick's magazine layer.
3814 * On uniprocessor, it's functionally equivalent (but less efficient)
3815 * to a larger limit. Thus disabled by default.
3816 */
3817 shared = 0;
3b0efdfa 3818 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
1da177e4 3819 shared = 8;
1da177e4
LT
3820
3821#if DEBUG
a737b3e2
AM
3822 /*
3823 * With debugging enabled, large batchcount lead to excessively long
3824 * periods with disabled local interrupts. Limit the batchcount
1da177e4
LT
3825 */
3826 if (limit > 32)
3827 limit = 32;
3828#endif
943a451a
GC
3829 batchcount = (limit + 1) / 2;
3830skip_setup:
3831 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
1da177e4
LT
3832 if (err)
3833 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
b28a02de 3834 cachep->name, -err);
2ed3a4ef 3835 return err;
1da177e4
LT
3836}
3837
1b55253a 3838/*
ce8eb6c4
CL
3839 * Drain an array if it contains any elements taking the node lock only if
3840 * necessary. Note that the node listlock also protects the array_cache
b18e7e65 3841 * if drain_array() is used on the shared array.
1b55253a 3842 */
ce8eb6c4 3843static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
1b55253a 3844 struct array_cache *ac, int force, int node)
1da177e4 3845{
97654dfa 3846 LIST_HEAD(list);
1da177e4
LT
3847 int tofree;
3848
1b55253a
CL
3849 if (!ac || !ac->avail)
3850 return;
1da177e4
LT
3851 if (ac->touched && !force) {
3852 ac->touched = 0;
b18e7e65 3853 } else {
ce8eb6c4 3854 spin_lock_irq(&n->list_lock);
b18e7e65
CL
3855 if (ac->avail) {
3856 tofree = force ? ac->avail : (ac->limit + 4) / 5;
3857 if (tofree > ac->avail)
3858 tofree = (ac->avail + 1) / 2;
97654dfa 3859 free_block(cachep, ac->entry, tofree, node, &list);
b18e7e65
CL
3860 ac->avail -= tofree;
3861 memmove(ac->entry, &(ac->entry[tofree]),
3862 sizeof(void *) * ac->avail);
3863 }
ce8eb6c4 3864 spin_unlock_irq(&n->list_lock);
97654dfa 3865 slabs_destroy(cachep, &list);
1da177e4
LT
3866 }
3867}
3868
3869/**
3870 * cache_reap - Reclaim memory from caches.
05fb6bf0 3871 * @w: work descriptor
1da177e4
LT
3872 *
3873 * Called from workqueue/eventd every few seconds.
3874 * Purpose:
3875 * - clear the per-cpu caches for this CPU.
3876 * - return freeable pages to the main free memory pool.
3877 *
a737b3e2
AM
3878 * If we cannot acquire the cache chain mutex then just give up - we'll try
3879 * again on the next iteration.
1da177e4 3880 */
7c5cae36 3881static void cache_reap(struct work_struct *w)
1da177e4 3882{
7a7c381d 3883 struct kmem_cache *searchp;
ce8eb6c4 3884 struct kmem_cache_node *n;
7d6e6d09 3885 int node = numa_mem_id();
bf6aede7 3886 struct delayed_work *work = to_delayed_work(w);
1da177e4 3887
18004c5d 3888 if (!mutex_trylock(&slab_mutex))
1da177e4 3889 /* Give up. Setup the next iteration. */
7c5cae36 3890 goto out;
1da177e4 3891
18004c5d 3892 list_for_each_entry(searchp, &slab_caches, list) {
1da177e4
LT
3893 check_irq_on();
3894
35386e3b 3895 /*
ce8eb6c4 3896 * We only take the node lock if absolutely necessary and we
35386e3b
CL
3897 * have established with reasonable certainty that
3898 * we can do some work if the lock was obtained.
3899 */
18bf8541 3900 n = get_node(searchp, node);
35386e3b 3901
ce8eb6c4 3902 reap_alien(searchp, n);
1da177e4 3903
ce8eb6c4 3904 drain_array(searchp, n, cpu_cache_get(searchp), 0, node);
1da177e4 3905
35386e3b
CL
3906 /*
3907 * These are racy checks but it does not matter
3908 * if we skip one check or scan twice.
3909 */
ce8eb6c4 3910 if (time_after(n->next_reap, jiffies))
35386e3b 3911 goto next;
1da177e4 3912
5f0985bb 3913 n->next_reap = jiffies + REAPTIMEOUT_NODE;
1da177e4 3914
ce8eb6c4 3915 drain_array(searchp, n, n->shared, 0, node);
1da177e4 3916
ce8eb6c4
CL
3917 if (n->free_touched)
3918 n->free_touched = 0;
ed11d9eb
CL
3919 else {
3920 int freed;
1da177e4 3921
ce8eb6c4 3922 freed = drain_freelist(searchp, n, (n->free_limit +
ed11d9eb
CL
3923 5 * searchp->num - 1) / (5 * searchp->num));
3924 STATS_ADD_REAPED(searchp, freed);
3925 }
35386e3b 3926next:
1da177e4
LT
3927 cond_resched();
3928 }
3929 check_irq_on();
18004c5d 3930 mutex_unlock(&slab_mutex);
8fce4d8e 3931 next_reap_node();
7c5cae36 3932out:
a737b3e2 3933 /* Set up the next iteration */
5f0985bb 3934 schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC));
1da177e4
LT
3935}
3936
158a9624 3937#ifdef CONFIG_SLABINFO
0d7561c6 3938void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
1da177e4 3939{
8456a648 3940 struct page *page;
b28a02de
PE
3941 unsigned long active_objs;
3942 unsigned long num_objs;
3943 unsigned long active_slabs = 0;
3944 unsigned long num_slabs, free_objects = 0, shared_avail = 0;
e498be7d 3945 const char *name;
1da177e4 3946 char *error = NULL;
e498be7d 3947 int node;
ce8eb6c4 3948 struct kmem_cache_node *n;
1da177e4 3949
1da177e4
LT
3950 active_objs = 0;
3951 num_slabs = 0;
18bf8541 3952 for_each_kmem_cache_node(cachep, node, n) {
e498be7d 3953
ca3b9b91 3954 check_irq_on();
ce8eb6c4 3955 spin_lock_irq(&n->list_lock);
e498be7d 3956
8456a648
JK
3957 list_for_each_entry(page, &n->slabs_full, lru) {
3958 if (page->active != cachep->num && !error)
e498be7d
CL
3959 error = "slabs_full accounting error";
3960 active_objs += cachep->num;
3961 active_slabs++;
3962 }
8456a648
JK
3963 list_for_each_entry(page, &n->slabs_partial, lru) {
3964 if (page->active == cachep->num && !error)
106a74e1 3965 error = "slabs_partial accounting error";
8456a648 3966 if (!page->active && !error)
106a74e1 3967 error = "slabs_partial accounting error";
8456a648 3968 active_objs += page->active;
e498be7d
CL
3969 active_slabs++;
3970 }
8456a648
JK
3971 list_for_each_entry(page, &n->slabs_free, lru) {
3972 if (page->active && !error)
106a74e1 3973 error = "slabs_free accounting error";
e498be7d
CL
3974 num_slabs++;
3975 }
ce8eb6c4
CL
3976 free_objects += n->free_objects;
3977 if (n->shared)
3978 shared_avail += n->shared->avail;
e498be7d 3979
ce8eb6c4 3980 spin_unlock_irq(&n->list_lock);
1da177e4 3981 }
b28a02de
PE
3982 num_slabs += active_slabs;
3983 num_objs = num_slabs * cachep->num;
e498be7d 3984 if (num_objs - active_objs != free_objects && !error)
1da177e4
LT
3985 error = "free_objects accounting error";
3986
b28a02de 3987 name = cachep->name;
1da177e4
LT
3988 if (error)
3989 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
3990
0d7561c6
GC
3991 sinfo->active_objs = active_objs;
3992 sinfo->num_objs = num_objs;
3993 sinfo->active_slabs = active_slabs;
3994 sinfo->num_slabs = num_slabs;
3995 sinfo->shared_avail = shared_avail;
3996 sinfo->limit = cachep->limit;
3997 sinfo->batchcount = cachep->batchcount;
3998 sinfo->shared = cachep->shared;
3999 sinfo->objects_per_slab = cachep->num;
4000 sinfo->cache_order = cachep->gfporder;
4001}
4002
4003void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
4004{
1da177e4 4005#if STATS
ce8eb6c4 4006 { /* node stats */
1da177e4
LT
4007 unsigned long high = cachep->high_mark;
4008 unsigned long allocs = cachep->num_allocations;
4009 unsigned long grown = cachep->grown;
4010 unsigned long reaped = cachep->reaped;
4011 unsigned long errors = cachep->errors;
4012 unsigned long max_freeable = cachep->max_freeable;
1da177e4 4013 unsigned long node_allocs = cachep->node_allocs;
e498be7d 4014 unsigned long node_frees = cachep->node_frees;
fb7faf33 4015 unsigned long overflows = cachep->node_overflow;
1da177e4 4016
e92dd4fd
JP
4017 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4018 "%4lu %4lu %4lu %4lu %4lu",
4019 allocs, high, grown,
4020 reaped, errors, max_freeable, node_allocs,
4021 node_frees, overflows);
1da177e4
LT
4022 }
4023 /* cpu stats */
4024 {
4025 unsigned long allochit = atomic_read(&cachep->allochit);
4026 unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4027 unsigned long freehit = atomic_read(&cachep->freehit);
4028 unsigned long freemiss = atomic_read(&cachep->freemiss);
4029
4030 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
b28a02de 4031 allochit, allocmiss, freehit, freemiss);
1da177e4
LT
4032 }
4033#endif
1da177e4
LT
4034}
4035
1da177e4
LT
4036#define MAX_SLABINFO_WRITE 128
4037/**
4038 * slabinfo_write - Tuning for the slab allocator
4039 * @file: unused
4040 * @buffer: user buffer
4041 * @count: data length
4042 * @ppos: unused
4043 */
b7454ad3 4044ssize_t slabinfo_write(struct file *file, const char __user *buffer,
b28a02de 4045 size_t count, loff_t *ppos)
1da177e4 4046{
b28a02de 4047 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
1da177e4 4048 int limit, batchcount, shared, res;
7a7c381d 4049 struct kmem_cache *cachep;
b28a02de 4050
1da177e4
LT
4051 if (count > MAX_SLABINFO_WRITE)
4052 return -EINVAL;
4053 if (copy_from_user(&kbuf, buffer, count))
4054 return -EFAULT;
b28a02de 4055 kbuf[MAX_SLABINFO_WRITE] = '\0';
1da177e4
LT
4056
4057 tmp = strchr(kbuf, ' ');
4058 if (!tmp)
4059 return -EINVAL;
4060 *tmp = '\0';
4061 tmp++;
4062 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4063 return -EINVAL;
4064
4065 /* Find the cache in the chain of caches. */
18004c5d 4066 mutex_lock(&slab_mutex);
1da177e4 4067 res = -EINVAL;
18004c5d 4068 list_for_each_entry(cachep, &slab_caches, list) {
1da177e4 4069 if (!strcmp(cachep->name, kbuf)) {
a737b3e2
AM
4070 if (limit < 1 || batchcount < 1 ||
4071 batchcount > limit || shared < 0) {
e498be7d 4072 res = 0;
1da177e4 4073 } else {
e498be7d 4074 res = do_tune_cpucache(cachep, limit,
83b519e8
PE
4075 batchcount, shared,
4076 GFP_KERNEL);
1da177e4
LT
4077 }
4078 break;
4079 }
4080 }
18004c5d 4081 mutex_unlock(&slab_mutex);
1da177e4
LT
4082 if (res >= 0)
4083 res = count;
4084 return res;
4085}
871751e2
AV
4086
4087#ifdef CONFIG_DEBUG_SLAB_LEAK
4088
871751e2
AV
4089static inline int add_caller(unsigned long *n, unsigned long v)
4090{
4091 unsigned long *p;
4092 int l;
4093 if (!v)
4094 return 1;
4095 l = n[1];
4096 p = n + 2;
4097 while (l) {
4098 int i = l/2;
4099 unsigned long *q = p + 2 * i;
4100 if (*q == v) {
4101 q[1]++;
4102 return 1;
4103 }
4104 if (*q > v) {
4105 l = i;
4106 } else {
4107 p = q + 2;
4108 l -= i + 1;
4109 }
4110 }
4111 if (++n[1] == n[0])
4112 return 0;
4113 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4114 p[0] = v;
4115 p[1] = 1;
4116 return 1;
4117}
4118
8456a648
JK
4119static void handle_slab(unsigned long *n, struct kmem_cache *c,
4120 struct page *page)
871751e2
AV
4121{
4122 void *p;
03787301 4123 int i;
b1cb0982 4124
871751e2
AV
4125 if (n[0] == n[1])
4126 return;
8456a648 4127 for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
03787301 4128 if (get_obj_status(page, i) != OBJECT_ACTIVE)
871751e2 4129 continue;
b1cb0982 4130
871751e2
AV
4131 if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4132 return;
4133 }
4134}
4135
4136static void show_symbol(struct seq_file *m, unsigned long address)
4137{
4138#ifdef CONFIG_KALLSYMS
871751e2 4139 unsigned long offset, size;
9281acea 4140 char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
871751e2 4141
a5c43dae 4142 if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
871751e2 4143 seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
a5c43dae 4144 if (modname[0])
871751e2
AV
4145 seq_printf(m, " [%s]", modname);
4146 return;
4147 }
4148#endif
4149 seq_printf(m, "%p", (void *)address);
4150}
4151
4152static int leaks_show(struct seq_file *m, void *p)
4153{
0672aa7c 4154 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
8456a648 4155 struct page *page;
ce8eb6c4 4156 struct kmem_cache_node *n;
871751e2 4157 const char *name;
db845067 4158 unsigned long *x = m->private;
871751e2
AV
4159 int node;
4160 int i;
4161
4162 if (!(cachep->flags & SLAB_STORE_USER))
4163 return 0;
4164 if (!(cachep->flags & SLAB_RED_ZONE))
4165 return 0;
4166
4167 /* OK, we can do it */
4168
db845067 4169 x[1] = 0;
871751e2 4170
18bf8541 4171 for_each_kmem_cache_node(cachep, node, n) {
871751e2
AV
4172
4173 check_irq_on();
ce8eb6c4 4174 spin_lock_irq(&n->list_lock);
871751e2 4175
8456a648
JK
4176 list_for_each_entry(page, &n->slabs_full, lru)
4177 handle_slab(x, cachep, page);
4178 list_for_each_entry(page, &n->slabs_partial, lru)
4179 handle_slab(x, cachep, page);
ce8eb6c4 4180 spin_unlock_irq(&n->list_lock);
871751e2
AV
4181 }
4182 name = cachep->name;
db845067 4183 if (x[0] == x[1]) {
871751e2 4184 /* Increase the buffer size */
18004c5d 4185 mutex_unlock(&slab_mutex);
db845067 4186 m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
871751e2
AV
4187 if (!m->private) {
4188 /* Too bad, we are really out */
db845067 4189 m->private = x;
18004c5d 4190 mutex_lock(&slab_mutex);
871751e2
AV
4191 return -ENOMEM;
4192 }
db845067
CL
4193 *(unsigned long *)m->private = x[0] * 2;
4194 kfree(x);
18004c5d 4195 mutex_lock(&slab_mutex);
871751e2
AV
4196 /* Now make sure this entry will be retried */
4197 m->count = m->size;
4198 return 0;
4199 }
db845067
CL
4200 for (i = 0; i < x[1]; i++) {
4201 seq_printf(m, "%s: %lu ", name, x[2*i+3]);
4202 show_symbol(m, x[2*i+2]);
871751e2
AV
4203 seq_putc(m, '\n');
4204 }
d2e7b7d0 4205
871751e2
AV
4206 return 0;
4207}
4208
a0ec95a8 4209static const struct seq_operations slabstats_op = {
1df3b26f 4210 .start = slab_start,
276a2439
WL
4211 .next = slab_next,
4212 .stop = slab_stop,
871751e2
AV
4213 .show = leaks_show,
4214};
a0ec95a8
AD
4215
4216static int slabstats_open(struct inode *inode, struct file *file)
4217{
b208ce32
RJ
4218 unsigned long *n;
4219
4220 n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
4221 if (!n)
4222 return -ENOMEM;
4223
4224 *n = PAGE_SIZE / (2 * sizeof(unsigned long));
4225
4226 return 0;
a0ec95a8
AD
4227}
4228
4229static const struct file_operations proc_slabstats_operations = {
4230 .open = slabstats_open,
4231 .read = seq_read,
4232 .llseek = seq_lseek,
4233 .release = seq_release_private,
4234};
4235#endif
4236
4237static int __init slab_proc_init(void)
4238{
4239#ifdef CONFIG_DEBUG_SLAB_LEAK
4240 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
871751e2 4241#endif
a0ec95a8
AD
4242 return 0;
4243}
4244module_init(slab_proc_init);
1da177e4
LT
4245#endif
4246
00e145b6
MS
4247/**
4248 * ksize - get the actual amount of memory allocated for a given object
4249 * @objp: Pointer to the object
4250 *
4251 * kmalloc may internally round up allocations and return more memory
4252 * than requested. ksize() can be used to determine the actual amount of
4253 * memory allocated. The caller may use this additional memory, even though
4254 * a smaller amount of memory was initially specified with the kmalloc call.
4255 * The caller must guarantee that objp points to a valid object previously
4256 * allocated with either kmalloc() or kmem_cache_alloc(). The object
4257 * must not be freed during the duration of the call.
4258 */
fd76bab2 4259size_t ksize(const void *objp)
1da177e4 4260{
ef8b4520
CL
4261 BUG_ON(!objp);
4262 if (unlikely(objp == ZERO_SIZE_PTR))
00e145b6 4263 return 0;
1da177e4 4264
8c138bc0 4265 return virt_to_cache(objp)->object_size;
1da177e4 4266}
b1aabecd 4267EXPORT_SYMBOL(ksize);