Merge tag 'driver-core-4.9-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-block.git] / mm / slub.c
CommitLineData
81819f0f
CL
1/*
2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists.
4 *
881db7fb
CL
5 * The allocator synchronizes using per slab locks or atomic operatios
6 * and only uses a centralized lock to manage a pool of partial slabs.
81819f0f 7 *
cde53535 8 * (C) 2007 SGI, Christoph Lameter
881db7fb 9 * (C) 2011 Linux Foundation, Christoph Lameter
81819f0f
CL
10 */
11
12#include <linux/mm.h>
1eb5ac64 13#include <linux/swap.h> /* struct reclaim_state */
81819f0f
CL
14#include <linux/module.h>
15#include <linux/bit_spinlock.h>
16#include <linux/interrupt.h>
17#include <linux/bitops.h>
18#include <linux/slab.h>
97d06609 19#include "slab.h"
7b3c3a50 20#include <linux/proc_fs.h>
3ac38faa 21#include <linux/notifier.h>
81819f0f 22#include <linux/seq_file.h>
a79316c6 23#include <linux/kasan.h>
5a896d9e 24#include <linux/kmemcheck.h>
81819f0f
CL
25#include <linux/cpu.h>
26#include <linux/cpuset.h>
27#include <linux/mempolicy.h>
28#include <linux/ctype.h>
3ac7fe5a 29#include <linux/debugobjects.h>
81819f0f 30#include <linux/kallsyms.h>
b9049e23 31#include <linux/memory.h>
f8bd2258 32#include <linux/math64.h>
773ff60e 33#include <linux/fault-inject.h>
bfa71457 34#include <linux/stacktrace.h>
4de900b4 35#include <linux/prefetch.h>
2633d7a0 36#include <linux/memcontrol.h>
81819f0f 37
4a92379b
RK
38#include <trace/events/kmem.h>
39
072bb0aa
MG
40#include "internal.h"
41
81819f0f
CL
42/*
43 * Lock order:
18004c5d 44 * 1. slab_mutex (Global Mutex)
881db7fb
CL
45 * 2. node->list_lock
46 * 3. slab_lock(page) (Only on some arches and for debugging)
81819f0f 47 *
18004c5d 48 * slab_mutex
881db7fb 49 *
18004c5d 50 * The role of the slab_mutex is to protect the list of all the slabs
881db7fb
CL
51 * and to synchronize major metadata changes to slab cache structures.
52 *
53 * The slab_lock is only used for debugging and on arches that do not
54 * have the ability to do a cmpxchg_double. It only protects the second
55 * double word in the page struct. Meaning
56 * A. page->freelist -> List of object free in a page
57 * B. page->counters -> Counters of objects
58 * C. page->frozen -> frozen state
59 *
60 * If a slab is frozen then it is exempt from list management. It is not
61 * on any list. The processor that froze the slab is the one who can
62 * perform list operations on the page. Other processors may put objects
63 * onto the freelist but the processor that froze the slab is the only
64 * one that can retrieve the objects from the page's freelist.
81819f0f
CL
65 *
66 * The list_lock protects the partial and full list on each node and
67 * the partial slab counter. If taken then no new slabs may be added or
68 * removed from the lists nor make the number of partial slabs be modified.
69 * (Note that the total number of slabs is an atomic value that may be
70 * modified without taking the list lock).
71 *
72 * The list_lock is a centralized lock and thus we avoid taking it as
73 * much as possible. As long as SLUB does not have to handle partial
74 * slabs, operations can continue without any centralized lock. F.e.
75 * allocating a long series of objects that fill up slabs does not require
76 * the list lock.
81819f0f
CL
77 * Interrupts are disabled during allocation and deallocation in order to
78 * make the slab allocator safe to use in the context of an irq. In addition
79 * interrupts are disabled to ensure that the processor does not change
80 * while handling per_cpu slabs, due to kernel preemption.
81 *
82 * SLUB assigns one slab for allocation to each processor.
83 * Allocations only occur from these slabs called cpu slabs.
84 *
672bba3a
CL
85 * Slabs with free elements are kept on a partial list and during regular
86 * operations no list for full slabs is used. If an object in a full slab is
81819f0f 87 * freed then the slab will show up again on the partial lists.
672bba3a
CL
88 * We track full slabs for debugging purposes though because otherwise we
89 * cannot scan all objects.
81819f0f
CL
90 *
91 * Slabs are freed when they become empty. Teardown and setup is
92 * minimal so we rely on the page allocators per cpu caches for
93 * fast frees and allocs.
94 *
95 * Overloading of page flags that are otherwise used for LRU management.
96 *
4b6f0750
CL
97 * PageActive The slab is frozen and exempt from list processing.
98 * This means that the slab is dedicated to a purpose
99 * such as satisfying allocations for a specific
100 * processor. Objects may be freed in the slab while
101 * it is frozen but slab_free will then skip the usual
102 * list operations. It is up to the processor holding
103 * the slab to integrate the slab into the slab lists
104 * when the slab is no longer needed.
105 *
106 * One use of this flag is to mark slabs that are
107 * used for allocations. Then such a slab becomes a cpu
108 * slab. The cpu slab may be equipped with an additional
dfb4f096 109 * freelist that allows lockless access to
894b8788
CL
110 * free objects in addition to the regular freelist
111 * that requires the slab lock.
81819f0f
CL
112 *
113 * PageError Slab requires special handling due to debug
114 * options set. This moves slab handling out of
894b8788 115 * the fast path and disables lockless freelists.
81819f0f
CL
116 */
117
af537b0a
CL
118static inline int kmem_cache_debug(struct kmem_cache *s)
119{
5577bd8a 120#ifdef CONFIG_SLUB_DEBUG
af537b0a 121 return unlikely(s->flags & SLAB_DEBUG_FLAGS);
5577bd8a 122#else
af537b0a 123 return 0;
5577bd8a 124#endif
af537b0a 125}
5577bd8a 126
117d54df 127void *fixup_red_left(struct kmem_cache *s, void *p)
d86bd1be
JK
128{
129 if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE)
130 p += s->red_left_pad;
131
132 return p;
133}
134
345c905d
JK
135static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
136{
137#ifdef CONFIG_SLUB_CPU_PARTIAL
138 return !kmem_cache_debug(s);
139#else
140 return false;
141#endif
142}
143
81819f0f
CL
144/*
145 * Issues still to be resolved:
146 *
81819f0f
CL
147 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
148 *
81819f0f
CL
149 * - Variable sizing of the per node arrays
150 */
151
152/* Enable to test recovery from slab corruption on boot */
153#undef SLUB_RESILIENCY_TEST
154
b789ef51
CL
155/* Enable to log cmpxchg failures */
156#undef SLUB_DEBUG_CMPXCHG
157
2086d26a
CL
158/*
159 * Mininum number of partial slabs. These will be left on the partial
160 * lists even if they are empty. kmem_cache_shrink may reclaim them.
161 */
76be8950 162#define MIN_PARTIAL 5
e95eed57 163
2086d26a
CL
164/*
165 * Maximum number of desirable partial slabs.
166 * The existence of more partial slabs makes kmem_cache_shrink
721ae22a 167 * sort the partial list by the number of objects in use.
2086d26a
CL
168 */
169#define MAX_PARTIAL 10
170
becfda68 171#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
81819f0f 172 SLAB_POISON | SLAB_STORE_USER)
672bba3a 173
149daaf3
LA
174/*
175 * These debug flags cannot use CMPXCHG because there might be consistency
176 * issues when checking or reading debug information
177 */
178#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
179 SLAB_TRACE)
180
181
fa5ec8a1 182/*
3de47213
DR
183 * Debugging flags that require metadata to be stored in the slab. These get
184 * disabled when slub_debug=O is used and a cache's min order increases with
185 * metadata.
fa5ec8a1 186 */
3de47213 187#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
fa5ec8a1 188
210b5c06
CG
189#define OO_SHIFT 16
190#define OO_MASK ((1 << OO_SHIFT) - 1)
50d5c41c 191#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
210b5c06 192
81819f0f 193/* Internal SLUB flags */
f90ec390 194#define __OBJECT_POISON 0x80000000UL /* Poison object */
b789ef51 195#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
81819f0f 196
02cbc874
CL
197/*
198 * Tracking user of a slab.
199 */
d6543e39 200#define TRACK_ADDRS_COUNT 16
02cbc874 201struct track {
ce71e27c 202 unsigned long addr; /* Called from address */
d6543e39
BG
203#ifdef CONFIG_STACKTRACE
204 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
205#endif
02cbc874
CL
206 int cpu; /* Was running on cpu */
207 int pid; /* Pid context */
208 unsigned long when; /* When did the operation occur */
209};
210
211enum track_item { TRACK_ALLOC, TRACK_FREE };
212
ab4d5ed5 213#ifdef CONFIG_SYSFS
81819f0f
CL
214static int sysfs_slab_add(struct kmem_cache *);
215static int sysfs_slab_alias(struct kmem_cache *, const char *);
107dab5c 216static void memcg_propagate_slab_attrs(struct kmem_cache *s);
81819f0f 217#else
0c710013
CL
218static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
219static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
220 { return 0; }
107dab5c 221static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
81819f0f
CL
222#endif
223
4fdccdfb 224static inline void stat(const struct kmem_cache *s, enum stat_item si)
8ff12cfc
CL
225{
226#ifdef CONFIG_SLUB_STATS
88da03a6
CL
227 /*
228 * The rmw is racy on a preemptible kernel but this is acceptable, so
229 * avoid this_cpu_add()'s irq-disable overhead.
230 */
231 raw_cpu_inc(s->cpu_slab->stat[si]);
8ff12cfc
CL
232#endif
233}
234
81819f0f
CL
235/********************************************************************
236 * Core slab cache functions
237 *******************************************************************/
238
7656c72b
CL
239static inline void *get_freepointer(struct kmem_cache *s, void *object)
240{
241 return *(void **)(object + s->offset);
242}
243
0ad9500e
ED
244static void prefetch_freepointer(const struct kmem_cache *s, void *object)
245{
246 prefetch(object + s->offset);
247}
248
1393d9a1
CL
249static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
250{
251 void *p;
252
922d566c
JK
253 if (!debug_pagealloc_enabled())
254 return get_freepointer(s, object);
255
1393d9a1 256 probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
1393d9a1
CL
257 return p;
258}
259
7656c72b
CL
260static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
261{
262 *(void **)(object + s->offset) = fp;
263}
264
265/* Loop over all objects in a slab */
224a88be 266#define for_each_object(__p, __s, __addr, __objects) \
d86bd1be
JK
267 for (__p = fixup_red_left(__s, __addr); \
268 __p < (__addr) + (__objects) * (__s)->size; \
269 __p += (__s)->size)
7656c72b 270
54266640 271#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
d86bd1be
JK
272 for (__p = fixup_red_left(__s, __addr), __idx = 1; \
273 __idx <= __objects; \
274 __p += (__s)->size, __idx++)
54266640 275
7656c72b
CL
276/* Determine object index from a given position */
277static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
278{
279 return (p - addr) / s->size;
280}
281
ab9a0f19
LJ
282static inline int order_objects(int order, unsigned long size, int reserved)
283{
284 return ((PAGE_SIZE << order) - reserved) / size;
285}
286
834f3d11 287static inline struct kmem_cache_order_objects oo_make(int order,
ab9a0f19 288 unsigned long size, int reserved)
834f3d11
CL
289{
290 struct kmem_cache_order_objects x = {
ab9a0f19 291 (order << OO_SHIFT) + order_objects(order, size, reserved)
834f3d11
CL
292 };
293
294 return x;
295}
296
297static inline int oo_order(struct kmem_cache_order_objects x)
298{
210b5c06 299 return x.x >> OO_SHIFT;
834f3d11
CL
300}
301
302static inline int oo_objects(struct kmem_cache_order_objects x)
303{
210b5c06 304 return x.x & OO_MASK;
834f3d11
CL
305}
306
881db7fb
CL
307/*
308 * Per slab locking using the pagelock
309 */
310static __always_inline void slab_lock(struct page *page)
311{
48c935ad 312 VM_BUG_ON_PAGE(PageTail(page), page);
881db7fb
CL
313 bit_spin_lock(PG_locked, &page->flags);
314}
315
316static __always_inline void slab_unlock(struct page *page)
317{
48c935ad 318 VM_BUG_ON_PAGE(PageTail(page), page);
881db7fb
CL
319 __bit_spin_unlock(PG_locked, &page->flags);
320}
321
a0320865
DH
322static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
323{
324 struct page tmp;
325 tmp.counters = counters_new;
326 /*
327 * page->counters can cover frozen/inuse/objects as well
0139aa7b
JK
328 * as page->_refcount. If we assign to ->counters directly
329 * we run the risk of losing updates to page->_refcount, so
a0320865
DH
330 * be careful and only assign to the fields we need.
331 */
332 page->frozen = tmp.frozen;
333 page->inuse = tmp.inuse;
334 page->objects = tmp.objects;
335}
336
1d07171c
CL
337/* Interrupts must be disabled (for the fallback code to work right) */
338static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
339 void *freelist_old, unsigned long counters_old,
340 void *freelist_new, unsigned long counters_new,
341 const char *n)
342{
343 VM_BUG_ON(!irqs_disabled());
2565409f
HC
344#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
345 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
1d07171c 346 if (s->flags & __CMPXCHG_DOUBLE) {
cdcd6298 347 if (cmpxchg_double(&page->freelist, &page->counters,
0aa9a13d
DC
348 freelist_old, counters_old,
349 freelist_new, counters_new))
6f6528a1 350 return true;
1d07171c
CL
351 } else
352#endif
353 {
354 slab_lock(page);
d0e0ac97
CG
355 if (page->freelist == freelist_old &&
356 page->counters == counters_old) {
1d07171c 357 page->freelist = freelist_new;
a0320865 358 set_page_slub_counters(page, counters_new);
1d07171c 359 slab_unlock(page);
6f6528a1 360 return true;
1d07171c
CL
361 }
362 slab_unlock(page);
363 }
364
365 cpu_relax();
366 stat(s, CMPXCHG_DOUBLE_FAIL);
367
368#ifdef SLUB_DEBUG_CMPXCHG
f9f58285 369 pr_info("%s %s: cmpxchg double redo ", n, s->name);
1d07171c
CL
370#endif
371
6f6528a1 372 return false;
1d07171c
CL
373}
374
b789ef51
CL
375static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
376 void *freelist_old, unsigned long counters_old,
377 void *freelist_new, unsigned long counters_new,
378 const char *n)
379{
2565409f
HC
380#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
381 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
b789ef51 382 if (s->flags & __CMPXCHG_DOUBLE) {
cdcd6298 383 if (cmpxchg_double(&page->freelist, &page->counters,
0aa9a13d
DC
384 freelist_old, counters_old,
385 freelist_new, counters_new))
6f6528a1 386 return true;
b789ef51
CL
387 } else
388#endif
389 {
1d07171c
CL
390 unsigned long flags;
391
392 local_irq_save(flags);
881db7fb 393 slab_lock(page);
d0e0ac97
CG
394 if (page->freelist == freelist_old &&
395 page->counters == counters_old) {
b789ef51 396 page->freelist = freelist_new;
a0320865 397 set_page_slub_counters(page, counters_new);
881db7fb 398 slab_unlock(page);
1d07171c 399 local_irq_restore(flags);
6f6528a1 400 return true;
b789ef51 401 }
881db7fb 402 slab_unlock(page);
1d07171c 403 local_irq_restore(flags);
b789ef51
CL
404 }
405
406 cpu_relax();
407 stat(s, CMPXCHG_DOUBLE_FAIL);
408
409#ifdef SLUB_DEBUG_CMPXCHG
f9f58285 410 pr_info("%s %s: cmpxchg double redo ", n, s->name);
b789ef51
CL
411#endif
412
6f6528a1 413 return false;
b789ef51
CL
414}
415
41ecc55b 416#ifdef CONFIG_SLUB_DEBUG
5f80b13a
CL
417/*
418 * Determine a map of object in use on a page.
419 *
881db7fb 420 * Node listlock must be held to guarantee that the page does
5f80b13a
CL
421 * not vanish from under us.
422 */
423static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
424{
425 void *p;
426 void *addr = page_address(page);
427
428 for (p = page->freelist; p; p = get_freepointer(s, p))
429 set_bit(slab_index(p, s, addr), map);
430}
431
d86bd1be
JK
432static inline int size_from_object(struct kmem_cache *s)
433{
434 if (s->flags & SLAB_RED_ZONE)
435 return s->size - s->red_left_pad;
436
437 return s->size;
438}
439
440static inline void *restore_red_left(struct kmem_cache *s, void *p)
441{
442 if (s->flags & SLAB_RED_ZONE)
443 p -= s->red_left_pad;
444
445 return p;
446}
447
41ecc55b
CL
448/*
449 * Debug settings:
450 */
89d3c87e 451#if defined(CONFIG_SLUB_DEBUG_ON)
f0630fff
CL
452static int slub_debug = DEBUG_DEFAULT_FLAGS;
453#else
41ecc55b 454static int slub_debug;
f0630fff 455#endif
41ecc55b
CL
456
457static char *slub_debug_slabs;
fa5ec8a1 458static int disable_higher_order_debug;
41ecc55b 459
a79316c6
AR
460/*
461 * slub is about to manipulate internal object metadata. This memory lies
462 * outside the range of the allocated object, so accessing it would normally
463 * be reported by kasan as a bounds error. metadata_access_enable() is used
464 * to tell kasan that these accesses are OK.
465 */
466static inline void metadata_access_enable(void)
467{
468 kasan_disable_current();
469}
470
471static inline void metadata_access_disable(void)
472{
473 kasan_enable_current();
474}
475
81819f0f
CL
476/*
477 * Object debugging
478 */
d86bd1be
JK
479
480/* Verify that a pointer has an address that is valid within a slab page */
481static inline int check_valid_pointer(struct kmem_cache *s,
482 struct page *page, void *object)
483{
484 void *base;
485
486 if (!object)
487 return 1;
488
489 base = page_address(page);
490 object = restore_red_left(s, object);
491 if (object < base || object >= base + page->objects * s->size ||
492 (object - base) % s->size) {
493 return 0;
494 }
495
496 return 1;
497}
498
81819f0f
CL
499static void print_section(char *text, u8 *addr, unsigned int length)
500{
a79316c6 501 metadata_access_enable();
ffc79d28
SAS
502 print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
503 length, 1);
a79316c6 504 metadata_access_disable();
81819f0f
CL
505}
506
81819f0f
CL
507static struct track *get_track(struct kmem_cache *s, void *object,
508 enum track_item alloc)
509{
510 struct track *p;
511
512 if (s->offset)
513 p = object + s->offset + sizeof(void *);
514 else
515 p = object + s->inuse;
516
517 return p + alloc;
518}
519
520static void set_track(struct kmem_cache *s, void *object,
ce71e27c 521 enum track_item alloc, unsigned long addr)
81819f0f 522{
1a00df4a 523 struct track *p = get_track(s, object, alloc);
81819f0f 524
81819f0f 525 if (addr) {
d6543e39
BG
526#ifdef CONFIG_STACKTRACE
527 struct stack_trace trace;
528 int i;
529
530 trace.nr_entries = 0;
531 trace.max_entries = TRACK_ADDRS_COUNT;
532 trace.entries = p->addrs;
533 trace.skip = 3;
a79316c6 534 metadata_access_enable();
d6543e39 535 save_stack_trace(&trace);
a79316c6 536 metadata_access_disable();
d6543e39
BG
537
538 /* See rant in lockdep.c */
539 if (trace.nr_entries != 0 &&
540 trace.entries[trace.nr_entries - 1] == ULONG_MAX)
541 trace.nr_entries--;
542
543 for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
544 p->addrs[i] = 0;
545#endif
81819f0f
CL
546 p->addr = addr;
547 p->cpu = smp_processor_id();
88e4ccf2 548 p->pid = current->pid;
81819f0f
CL
549 p->when = jiffies;
550 } else
551 memset(p, 0, sizeof(struct track));
552}
553
81819f0f
CL
554static void init_tracking(struct kmem_cache *s, void *object)
555{
24922684
CL
556 if (!(s->flags & SLAB_STORE_USER))
557 return;
558
ce71e27c
EGM
559 set_track(s, object, TRACK_FREE, 0UL);
560 set_track(s, object, TRACK_ALLOC, 0UL);
81819f0f
CL
561}
562
563static void print_track(const char *s, struct track *t)
564{
565 if (!t->addr)
566 return;
567
f9f58285
FF
568 pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
569 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
d6543e39
BG
570#ifdef CONFIG_STACKTRACE
571 {
572 int i;
573 for (i = 0; i < TRACK_ADDRS_COUNT; i++)
574 if (t->addrs[i])
f9f58285 575 pr_err("\t%pS\n", (void *)t->addrs[i]);
d6543e39
BG
576 else
577 break;
578 }
579#endif
24922684
CL
580}
581
582static void print_tracking(struct kmem_cache *s, void *object)
583{
584 if (!(s->flags & SLAB_STORE_USER))
585 return;
586
587 print_track("Allocated", get_track(s, object, TRACK_ALLOC));
588 print_track("Freed", get_track(s, object, TRACK_FREE));
589}
590
591static void print_page_info(struct page *page)
592{
f9f58285 593 pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
d0e0ac97 594 page, page->objects, page->inuse, page->freelist, page->flags);
24922684
CL
595
596}
597
598static void slab_bug(struct kmem_cache *s, char *fmt, ...)
599{
ecc42fbe 600 struct va_format vaf;
24922684 601 va_list args;
24922684
CL
602
603 va_start(args, fmt);
ecc42fbe
FF
604 vaf.fmt = fmt;
605 vaf.va = &args;
f9f58285 606 pr_err("=============================================================================\n");
ecc42fbe 607 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
f9f58285 608 pr_err("-----------------------------------------------------------------------------\n\n");
645df230 609
373d4d09 610 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
ecc42fbe 611 va_end(args);
81819f0f
CL
612}
613
24922684
CL
614static void slab_fix(struct kmem_cache *s, char *fmt, ...)
615{
ecc42fbe 616 struct va_format vaf;
24922684 617 va_list args;
24922684
CL
618
619 va_start(args, fmt);
ecc42fbe
FF
620 vaf.fmt = fmt;
621 vaf.va = &args;
622 pr_err("FIX %s: %pV\n", s->name, &vaf);
24922684 623 va_end(args);
24922684
CL
624}
625
626static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
81819f0f
CL
627{
628 unsigned int off; /* Offset of last byte */
a973e9dd 629 u8 *addr = page_address(page);
24922684
CL
630
631 print_tracking(s, p);
632
633 print_page_info(page);
634
f9f58285
FF
635 pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
636 p, p - addr, get_freepointer(s, p));
24922684 637
d86bd1be
JK
638 if (s->flags & SLAB_RED_ZONE)
639 print_section("Redzone ", p - s->red_left_pad, s->red_left_pad);
640 else if (p > addr + 16)
ffc79d28 641 print_section("Bytes b4 ", p - 16, 16);
81819f0f 642
3b0efdfa 643 print_section("Object ", p, min_t(unsigned long, s->object_size,
ffc79d28 644 PAGE_SIZE));
81819f0f 645 if (s->flags & SLAB_RED_ZONE)
3b0efdfa
CL
646 print_section("Redzone ", p + s->object_size,
647 s->inuse - s->object_size);
81819f0f 648
81819f0f
CL
649 if (s->offset)
650 off = s->offset + sizeof(void *);
651 else
652 off = s->inuse;
653
24922684 654 if (s->flags & SLAB_STORE_USER)
81819f0f 655 off += 2 * sizeof(struct track);
81819f0f 656
80a9201a
AP
657 off += kasan_metadata_size(s);
658
d86bd1be 659 if (off != size_from_object(s))
81819f0f 660 /* Beginning of the filler is the free pointer */
d86bd1be 661 print_section("Padding ", p + off, size_from_object(s) - off);
24922684
CL
662
663 dump_stack();
81819f0f
CL
664}
665
75c66def 666void object_err(struct kmem_cache *s, struct page *page,
81819f0f
CL
667 u8 *object, char *reason)
668{
3dc50637 669 slab_bug(s, "%s", reason);
24922684 670 print_trailer(s, page, object);
81819f0f
CL
671}
672
d0e0ac97
CG
673static void slab_err(struct kmem_cache *s, struct page *page,
674 const char *fmt, ...)
81819f0f
CL
675{
676 va_list args;
677 char buf[100];
678
24922684
CL
679 va_start(args, fmt);
680 vsnprintf(buf, sizeof(buf), fmt, args);
81819f0f 681 va_end(args);
3dc50637 682 slab_bug(s, "%s", buf);
24922684 683 print_page_info(page);
81819f0f
CL
684 dump_stack();
685}
686
f7cb1933 687static void init_object(struct kmem_cache *s, void *object, u8 val)
81819f0f
CL
688{
689 u8 *p = object;
690
d86bd1be
JK
691 if (s->flags & SLAB_RED_ZONE)
692 memset(p - s->red_left_pad, val, s->red_left_pad);
693
81819f0f 694 if (s->flags & __OBJECT_POISON) {
3b0efdfa
CL
695 memset(p, POISON_FREE, s->object_size - 1);
696 p[s->object_size - 1] = POISON_END;
81819f0f
CL
697 }
698
699 if (s->flags & SLAB_RED_ZONE)
3b0efdfa 700 memset(p + s->object_size, val, s->inuse - s->object_size);
81819f0f
CL
701}
702
24922684
CL
703static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
704 void *from, void *to)
705{
706 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
707 memset(from, data, to - from);
708}
709
710static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
711 u8 *object, char *what,
06428780 712 u8 *start, unsigned int value, unsigned int bytes)
24922684
CL
713{
714 u8 *fault;
715 u8 *end;
716
a79316c6 717 metadata_access_enable();
79824820 718 fault = memchr_inv(start, value, bytes);
a79316c6 719 metadata_access_disable();
24922684
CL
720 if (!fault)
721 return 1;
722
723 end = start + bytes;
724 while (end > fault && end[-1] == value)
725 end--;
726
727 slab_bug(s, "%s overwritten", what);
f9f58285 728 pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
24922684
CL
729 fault, end - 1, fault[0], value);
730 print_trailer(s, page, object);
731
732 restore_bytes(s, what, value, fault, end);
733 return 0;
81819f0f
CL
734}
735
81819f0f
CL
736/*
737 * Object layout:
738 *
739 * object address
740 * Bytes of the object to be managed.
741 * If the freepointer may overlay the object then the free
742 * pointer is the first word of the object.
672bba3a 743 *
81819f0f
CL
744 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
745 * 0xa5 (POISON_END)
746 *
3b0efdfa 747 * object + s->object_size
81819f0f 748 * Padding to reach word boundary. This is also used for Redzoning.
672bba3a 749 * Padding is extended by another word if Redzoning is enabled and
3b0efdfa 750 * object_size == inuse.
672bba3a 751 *
81819f0f
CL
752 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
753 * 0xcc (RED_ACTIVE) for objects in use.
754 *
755 * object + s->inuse
672bba3a
CL
756 * Meta data starts here.
757 *
81819f0f
CL
758 * A. Free pointer (if we cannot overwrite object on free)
759 * B. Tracking data for SLAB_STORE_USER
672bba3a 760 * C. Padding to reach required alignment boundary or at mininum
6446faa2 761 * one word if debugging is on to be able to detect writes
672bba3a
CL
762 * before the word boundary.
763 *
764 * Padding is done using 0x5a (POISON_INUSE)
81819f0f
CL
765 *
766 * object + s->size
672bba3a 767 * Nothing is used beyond s->size.
81819f0f 768 *
3b0efdfa 769 * If slabcaches are merged then the object_size and inuse boundaries are mostly
672bba3a 770 * ignored. And therefore no slab options that rely on these boundaries
81819f0f
CL
771 * may be used with merged slabcaches.
772 */
773
81819f0f
CL
774static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
775{
776 unsigned long off = s->inuse; /* The end of info */
777
778 if (s->offset)
779 /* Freepointer is placed after the object. */
780 off += sizeof(void *);
781
782 if (s->flags & SLAB_STORE_USER)
783 /* We also have user information there */
784 off += 2 * sizeof(struct track);
785
80a9201a
AP
786 off += kasan_metadata_size(s);
787
d86bd1be 788 if (size_from_object(s) == off)
81819f0f
CL
789 return 1;
790
24922684 791 return check_bytes_and_report(s, page, p, "Object padding",
d86bd1be 792 p + off, POISON_INUSE, size_from_object(s) - off);
81819f0f
CL
793}
794
39b26464 795/* Check the pad bytes at the end of a slab page */
81819f0f
CL
796static int slab_pad_check(struct kmem_cache *s, struct page *page)
797{
24922684
CL
798 u8 *start;
799 u8 *fault;
800 u8 *end;
801 int length;
802 int remainder;
81819f0f
CL
803
804 if (!(s->flags & SLAB_POISON))
805 return 1;
806
a973e9dd 807 start = page_address(page);
ab9a0f19 808 length = (PAGE_SIZE << compound_order(page)) - s->reserved;
39b26464
CL
809 end = start + length;
810 remainder = length % s->size;
81819f0f
CL
811 if (!remainder)
812 return 1;
813
a79316c6 814 metadata_access_enable();
79824820 815 fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
a79316c6 816 metadata_access_disable();
24922684
CL
817 if (!fault)
818 return 1;
819 while (end > fault && end[-1] == POISON_INUSE)
820 end--;
821
822 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
ffc79d28 823 print_section("Padding ", end - remainder, remainder);
24922684 824
8a3d271d 825 restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
24922684 826 return 0;
81819f0f
CL
827}
828
829static int check_object(struct kmem_cache *s, struct page *page,
f7cb1933 830 void *object, u8 val)
81819f0f
CL
831{
832 u8 *p = object;
3b0efdfa 833 u8 *endobject = object + s->object_size;
81819f0f
CL
834
835 if (s->flags & SLAB_RED_ZONE) {
d86bd1be
JK
836 if (!check_bytes_and_report(s, page, object, "Redzone",
837 object - s->red_left_pad, val, s->red_left_pad))
838 return 0;
839
24922684 840 if (!check_bytes_and_report(s, page, object, "Redzone",
3b0efdfa 841 endobject, val, s->inuse - s->object_size))
81819f0f 842 return 0;
81819f0f 843 } else {
3b0efdfa 844 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
3adbefee 845 check_bytes_and_report(s, page, p, "Alignment padding",
d0e0ac97
CG
846 endobject, POISON_INUSE,
847 s->inuse - s->object_size);
3adbefee 848 }
81819f0f
CL
849 }
850
851 if (s->flags & SLAB_POISON) {
f7cb1933 852 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
24922684 853 (!check_bytes_and_report(s, page, p, "Poison", p,
3b0efdfa 854 POISON_FREE, s->object_size - 1) ||
24922684 855 !check_bytes_and_report(s, page, p, "Poison",
3b0efdfa 856 p + s->object_size - 1, POISON_END, 1)))
81819f0f 857 return 0;
81819f0f
CL
858 /*
859 * check_pad_bytes cleans up on its own.
860 */
861 check_pad_bytes(s, page, p);
862 }
863
f7cb1933 864 if (!s->offset && val == SLUB_RED_ACTIVE)
81819f0f
CL
865 /*
866 * Object and freepointer overlap. Cannot check
867 * freepointer while object is allocated.
868 */
869 return 1;
870
871 /* Check free pointer validity */
872 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
873 object_err(s, page, p, "Freepointer corrupt");
874 /*
9f6c708e 875 * No choice but to zap it and thus lose the remainder
81819f0f 876 * of the free objects in this slab. May cause
672bba3a 877 * another error because the object count is now wrong.
81819f0f 878 */
a973e9dd 879 set_freepointer(s, p, NULL);
81819f0f
CL
880 return 0;
881 }
882 return 1;
883}
884
885static int check_slab(struct kmem_cache *s, struct page *page)
886{
39b26464
CL
887 int maxobj;
888
81819f0f
CL
889 VM_BUG_ON(!irqs_disabled());
890
891 if (!PageSlab(page)) {
24922684 892 slab_err(s, page, "Not a valid slab page");
81819f0f
CL
893 return 0;
894 }
39b26464 895
ab9a0f19 896 maxobj = order_objects(compound_order(page), s->size, s->reserved);
39b26464
CL
897 if (page->objects > maxobj) {
898 slab_err(s, page, "objects %u > max %u",
f6edde9c 899 page->objects, maxobj);
39b26464
CL
900 return 0;
901 }
902 if (page->inuse > page->objects) {
24922684 903 slab_err(s, page, "inuse %u > max %u",
f6edde9c 904 page->inuse, page->objects);
81819f0f
CL
905 return 0;
906 }
907 /* Slab_pad_check fixes things up after itself */
908 slab_pad_check(s, page);
909 return 1;
910}
911
912/*
672bba3a
CL
913 * Determine if a certain object on a page is on the freelist. Must hold the
914 * slab lock to guarantee that the chains are in a consistent state.
81819f0f
CL
915 */
916static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
917{
918 int nr = 0;
881db7fb 919 void *fp;
81819f0f 920 void *object = NULL;
f6edde9c 921 int max_objects;
81819f0f 922
881db7fb 923 fp = page->freelist;
39b26464 924 while (fp && nr <= page->objects) {
81819f0f
CL
925 if (fp == search)
926 return 1;
927 if (!check_valid_pointer(s, page, fp)) {
928 if (object) {
929 object_err(s, page, object,
930 "Freechain corrupt");
a973e9dd 931 set_freepointer(s, object, NULL);
81819f0f 932 } else {
24922684 933 slab_err(s, page, "Freepointer corrupt");
a973e9dd 934 page->freelist = NULL;
39b26464 935 page->inuse = page->objects;
24922684 936 slab_fix(s, "Freelist cleared");
81819f0f
CL
937 return 0;
938 }
939 break;
940 }
941 object = fp;
942 fp = get_freepointer(s, object);
943 nr++;
944 }
945
ab9a0f19 946 max_objects = order_objects(compound_order(page), s->size, s->reserved);
210b5c06
CG
947 if (max_objects > MAX_OBJS_PER_PAGE)
948 max_objects = MAX_OBJS_PER_PAGE;
224a88be
CL
949
950 if (page->objects != max_objects) {
756a025f
JP
951 slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
952 page->objects, max_objects);
224a88be
CL
953 page->objects = max_objects;
954 slab_fix(s, "Number of objects adjusted.");
955 }
39b26464 956 if (page->inuse != page->objects - nr) {
756a025f
JP
957 slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
958 page->inuse, page->objects - nr);
39b26464 959 page->inuse = page->objects - nr;
24922684 960 slab_fix(s, "Object count adjusted.");
81819f0f
CL
961 }
962 return search == NULL;
963}
964
0121c619
CL
965static void trace(struct kmem_cache *s, struct page *page, void *object,
966 int alloc)
3ec09742
CL
967{
968 if (s->flags & SLAB_TRACE) {
f9f58285 969 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
3ec09742
CL
970 s->name,
971 alloc ? "alloc" : "free",
972 object, page->inuse,
973 page->freelist);
974
975 if (!alloc)
d0e0ac97
CG
976 print_section("Object ", (void *)object,
977 s->object_size);
3ec09742
CL
978
979 dump_stack();
980 }
981}
982
643b1138 983/*
672bba3a 984 * Tracking of fully allocated slabs for debugging purposes.
643b1138 985 */
5cc6eee8
CL
986static void add_full(struct kmem_cache *s,
987 struct kmem_cache_node *n, struct page *page)
643b1138 988{
5cc6eee8
CL
989 if (!(s->flags & SLAB_STORE_USER))
990 return;
991
255d0884 992 lockdep_assert_held(&n->list_lock);
643b1138 993 list_add(&page->lru, &n->full);
643b1138
CL
994}
995
c65c1877 996static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
643b1138 997{
643b1138
CL
998 if (!(s->flags & SLAB_STORE_USER))
999 return;
1000
255d0884 1001 lockdep_assert_held(&n->list_lock);
643b1138 1002 list_del(&page->lru);
643b1138
CL
1003}
1004
0f389ec6
CL
1005/* Tracking of the number of slabs for debugging purposes */
1006static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1007{
1008 struct kmem_cache_node *n = get_node(s, node);
1009
1010 return atomic_long_read(&n->nr_slabs);
1011}
1012
26c02cf0
AB
1013static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1014{
1015 return atomic_long_read(&n->nr_slabs);
1016}
1017
205ab99d 1018static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
0f389ec6
CL
1019{
1020 struct kmem_cache_node *n = get_node(s, node);
1021
1022 /*
1023 * May be called early in order to allocate a slab for the
1024 * kmem_cache_node structure. Solve the chicken-egg
1025 * dilemma by deferring the increment of the count during
1026 * bootstrap (see early_kmem_cache_node_alloc).
1027 */
338b2642 1028 if (likely(n)) {
0f389ec6 1029 atomic_long_inc(&n->nr_slabs);
205ab99d
CL
1030 atomic_long_add(objects, &n->total_objects);
1031 }
0f389ec6 1032}
205ab99d 1033static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
0f389ec6
CL
1034{
1035 struct kmem_cache_node *n = get_node(s, node);
1036
1037 atomic_long_dec(&n->nr_slabs);
205ab99d 1038 atomic_long_sub(objects, &n->total_objects);
0f389ec6
CL
1039}
1040
1041/* Object debug checks for alloc/free paths */
3ec09742
CL
1042static void setup_object_debug(struct kmem_cache *s, struct page *page,
1043 void *object)
1044{
1045 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
1046 return;
1047
f7cb1933 1048 init_object(s, object, SLUB_RED_INACTIVE);
3ec09742
CL
1049 init_tracking(s, object);
1050}
1051
becfda68 1052static inline int alloc_consistency_checks(struct kmem_cache *s,
d0e0ac97 1053 struct page *page,
ce71e27c 1054 void *object, unsigned long addr)
81819f0f
CL
1055{
1056 if (!check_slab(s, page))
becfda68 1057 return 0;
81819f0f 1058
81819f0f
CL
1059 if (!check_valid_pointer(s, page, object)) {
1060 object_err(s, page, object, "Freelist Pointer check fails");
becfda68 1061 return 0;
81819f0f
CL
1062 }
1063
f7cb1933 1064 if (!check_object(s, page, object, SLUB_RED_INACTIVE))
becfda68
LA
1065 return 0;
1066
1067 return 1;
1068}
1069
1070static noinline int alloc_debug_processing(struct kmem_cache *s,
1071 struct page *page,
1072 void *object, unsigned long addr)
1073{
1074 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1075 if (!alloc_consistency_checks(s, page, object, addr))
1076 goto bad;
1077 }
81819f0f 1078
3ec09742
CL
1079 /* Success perform special debug activities for allocs */
1080 if (s->flags & SLAB_STORE_USER)
1081 set_track(s, object, TRACK_ALLOC, addr);
1082 trace(s, page, object, 1);
f7cb1933 1083 init_object(s, object, SLUB_RED_ACTIVE);
81819f0f 1084 return 1;
3ec09742 1085
81819f0f
CL
1086bad:
1087 if (PageSlab(page)) {
1088 /*
1089 * If this is a slab page then lets do the best we can
1090 * to avoid issues in the future. Marking all objects
672bba3a 1091 * as used avoids touching the remaining objects.
81819f0f 1092 */
24922684 1093 slab_fix(s, "Marking all objects used");
39b26464 1094 page->inuse = page->objects;
a973e9dd 1095 page->freelist = NULL;
81819f0f
CL
1096 }
1097 return 0;
1098}
1099
becfda68
LA
1100static inline int free_consistency_checks(struct kmem_cache *s,
1101 struct page *page, void *object, unsigned long addr)
81819f0f 1102{
81819f0f 1103 if (!check_valid_pointer(s, page, object)) {
70d71228 1104 slab_err(s, page, "Invalid object pointer 0x%p", object);
becfda68 1105 return 0;
81819f0f
CL
1106 }
1107
1108 if (on_freelist(s, page, object)) {
24922684 1109 object_err(s, page, object, "Object already free");
becfda68 1110 return 0;
81819f0f
CL
1111 }
1112
f7cb1933 1113 if (!check_object(s, page, object, SLUB_RED_ACTIVE))
becfda68 1114 return 0;
81819f0f 1115
1b4f59e3 1116 if (unlikely(s != page->slab_cache)) {
3adbefee 1117 if (!PageSlab(page)) {
756a025f
JP
1118 slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
1119 object);
1b4f59e3 1120 } else if (!page->slab_cache) {
f9f58285
FF
1121 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1122 object);
70d71228 1123 dump_stack();
06428780 1124 } else
24922684
CL
1125 object_err(s, page, object,
1126 "page slab pointer corrupt.");
becfda68
LA
1127 return 0;
1128 }
1129 return 1;
1130}
1131
1132/* Supports checking bulk free of a constructed freelist */
1133static noinline int free_debug_processing(
1134 struct kmem_cache *s, struct page *page,
1135 void *head, void *tail, int bulk_cnt,
1136 unsigned long addr)
1137{
1138 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1139 void *object = head;
1140 int cnt = 0;
1141 unsigned long uninitialized_var(flags);
1142 int ret = 0;
1143
1144 spin_lock_irqsave(&n->list_lock, flags);
1145 slab_lock(page);
1146
1147 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1148 if (!check_slab(s, page))
1149 goto out;
1150 }
1151
1152next_object:
1153 cnt++;
1154
1155 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1156 if (!free_consistency_checks(s, page, object, addr))
1157 goto out;
81819f0f 1158 }
3ec09742 1159
3ec09742
CL
1160 if (s->flags & SLAB_STORE_USER)
1161 set_track(s, object, TRACK_FREE, addr);
1162 trace(s, page, object, 0);
81084651 1163 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
f7cb1933 1164 init_object(s, object, SLUB_RED_INACTIVE);
81084651
JDB
1165
1166 /* Reached end of constructed freelist yet? */
1167 if (object != tail) {
1168 object = get_freepointer(s, object);
1169 goto next_object;
1170 }
804aa132
LA
1171 ret = 1;
1172
5c2e4bbb 1173out:
81084651
JDB
1174 if (cnt != bulk_cnt)
1175 slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
1176 bulk_cnt, cnt);
1177
881db7fb 1178 slab_unlock(page);
282acb43 1179 spin_unlock_irqrestore(&n->list_lock, flags);
804aa132
LA
1180 if (!ret)
1181 slab_fix(s, "Object at 0x%p not freed", object);
1182 return ret;
81819f0f
CL
1183}
1184
41ecc55b
CL
1185static int __init setup_slub_debug(char *str)
1186{
f0630fff
CL
1187 slub_debug = DEBUG_DEFAULT_FLAGS;
1188 if (*str++ != '=' || !*str)
1189 /*
1190 * No options specified. Switch on full debugging.
1191 */
1192 goto out;
1193
1194 if (*str == ',')
1195 /*
1196 * No options but restriction on slabs. This means full
1197 * debugging for slabs matching a pattern.
1198 */
1199 goto check_slabs;
1200
1201 slub_debug = 0;
1202 if (*str == '-')
1203 /*
1204 * Switch off all debugging measures.
1205 */
1206 goto out;
1207
1208 /*
1209 * Determine which debug features should be switched on
1210 */
06428780 1211 for (; *str && *str != ','; str++) {
f0630fff
CL
1212 switch (tolower(*str)) {
1213 case 'f':
becfda68 1214 slub_debug |= SLAB_CONSISTENCY_CHECKS;
f0630fff
CL
1215 break;
1216 case 'z':
1217 slub_debug |= SLAB_RED_ZONE;
1218 break;
1219 case 'p':
1220 slub_debug |= SLAB_POISON;
1221 break;
1222 case 'u':
1223 slub_debug |= SLAB_STORE_USER;
1224 break;
1225 case 't':
1226 slub_debug |= SLAB_TRACE;
1227 break;
4c13dd3b
DM
1228 case 'a':
1229 slub_debug |= SLAB_FAILSLAB;
1230 break;
08303a73
CA
1231 case 'o':
1232 /*
1233 * Avoid enabling debugging on caches if its minimum
1234 * order would increase as a result.
1235 */
1236 disable_higher_order_debug = 1;
1237 break;
f0630fff 1238 default:
f9f58285
FF
1239 pr_err("slub_debug option '%c' unknown. skipped\n",
1240 *str);
f0630fff 1241 }
41ecc55b
CL
1242 }
1243
f0630fff 1244check_slabs:
41ecc55b
CL
1245 if (*str == ',')
1246 slub_debug_slabs = str + 1;
f0630fff 1247out:
41ecc55b
CL
1248 return 1;
1249}
1250
1251__setup("slub_debug", setup_slub_debug);
1252
423c929c 1253unsigned long kmem_cache_flags(unsigned long object_size,
ba0268a8 1254 unsigned long flags, const char *name,
51cc5068 1255 void (*ctor)(void *))
41ecc55b
CL
1256{
1257 /*
e153362a 1258 * Enable debugging if selected on the kernel commandline.
41ecc55b 1259 */
c6f58d9b
CL
1260 if (slub_debug && (!slub_debug_slabs || (name &&
1261 !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
3de47213 1262 flags |= slub_debug;
ba0268a8
CL
1263
1264 return flags;
41ecc55b 1265}
b4a64718 1266#else /* !CONFIG_SLUB_DEBUG */
3ec09742
CL
1267static inline void setup_object_debug(struct kmem_cache *s,
1268 struct page *page, void *object) {}
41ecc55b 1269
3ec09742 1270static inline int alloc_debug_processing(struct kmem_cache *s,
ce71e27c 1271 struct page *page, void *object, unsigned long addr) { return 0; }
41ecc55b 1272
282acb43 1273static inline int free_debug_processing(
81084651
JDB
1274 struct kmem_cache *s, struct page *page,
1275 void *head, void *tail, int bulk_cnt,
282acb43 1276 unsigned long addr) { return 0; }
41ecc55b 1277
41ecc55b
CL
1278static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1279 { return 1; }
1280static inline int check_object(struct kmem_cache *s, struct page *page,
f7cb1933 1281 void *object, u8 val) { return 1; }
5cc6eee8
CL
1282static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1283 struct page *page) {}
c65c1877
PZ
1284static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1285 struct page *page) {}
423c929c 1286unsigned long kmem_cache_flags(unsigned long object_size,
ba0268a8 1287 unsigned long flags, const char *name,
51cc5068 1288 void (*ctor)(void *))
ba0268a8
CL
1289{
1290 return flags;
1291}
41ecc55b 1292#define slub_debug 0
0f389ec6 1293
fdaa45e9
IM
1294#define disable_higher_order_debug 0
1295
0f389ec6
CL
1296static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1297 { return 0; }
26c02cf0
AB
1298static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1299 { return 0; }
205ab99d
CL
1300static inline void inc_slabs_node(struct kmem_cache *s, int node,
1301 int objects) {}
1302static inline void dec_slabs_node(struct kmem_cache *s, int node,
1303 int objects) {}
7d550c56 1304
02e72cc6
AR
1305#endif /* CONFIG_SLUB_DEBUG */
1306
1307/*
1308 * Hooks for other subsystems that check memory allocations. In a typical
1309 * production configuration these hooks all should produce no code at all.
1310 */
d56791b3
RB
1311static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
1312{
1313 kmemleak_alloc(ptr, size, 1, flags);
505f5dcb 1314 kasan_kmalloc_large(ptr, size, flags);
d56791b3
RB
1315}
1316
1317static inline void kfree_hook(const void *x)
1318{
1319 kmemleak_free(x);
0316bec2 1320 kasan_kfree_large(x);
d56791b3
RB
1321}
1322
80a9201a 1323static inline void *slab_free_hook(struct kmem_cache *s, void *x)
d56791b3 1324{
80a9201a
AP
1325 void *freeptr;
1326
d56791b3 1327 kmemleak_free_recursive(x, s->flags);
7d550c56 1328
02e72cc6
AR
1329 /*
1330 * Trouble is that we may no longer disable interrupts in the fast path
1331 * So in order to make the debug calls that expect irqs to be
1332 * disabled we need to disable interrupts temporarily.
1333 */
1334#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
1335 {
1336 unsigned long flags;
1337
1338 local_irq_save(flags);
1339 kmemcheck_slab_free(s, x, s->object_size);
1340 debug_check_no_locks_freed(x, s->object_size);
1341 local_irq_restore(flags);
1342 }
1343#endif
1344 if (!(s->flags & SLAB_DEBUG_OBJECTS))
1345 debug_check_no_obj_freed(x, s->object_size);
0316bec2 1346
80a9201a
AP
1347 freeptr = get_freepointer(s, x);
1348 /*
1349 * kasan_slab_free() may put x into memory quarantine, delaying its
1350 * reuse. In this case the object's freelist pointer is changed.
1351 */
0316bec2 1352 kasan_slab_free(s, x);
80a9201a 1353 return freeptr;
02e72cc6 1354}
205ab99d 1355
81084651
JDB
1356static inline void slab_free_freelist_hook(struct kmem_cache *s,
1357 void *head, void *tail)
1358{
1359/*
1360 * Compiler cannot detect this function can be removed if slab_free_hook()
1361 * evaluates to nothing. Thus, catch all relevant config debug options here.
1362 */
1363#if defined(CONFIG_KMEMCHECK) || \
1364 defined(CONFIG_LOCKDEP) || \
1365 defined(CONFIG_DEBUG_KMEMLEAK) || \
1366 defined(CONFIG_DEBUG_OBJECTS_FREE) || \
1367 defined(CONFIG_KASAN)
1368
1369 void *object = head;
1370 void *tail_obj = tail ? : head;
80a9201a 1371 void *freeptr;
81084651
JDB
1372
1373 do {
80a9201a
AP
1374 freeptr = slab_free_hook(s, object);
1375 } while ((object != tail_obj) && (object = freeptr));
81084651
JDB
1376#endif
1377}
1378
588f8ba9
TG
1379static void setup_object(struct kmem_cache *s, struct page *page,
1380 void *object)
1381{
1382 setup_object_debug(s, page, object);
b3cbd9bf 1383 kasan_init_slab_obj(s, object);
588f8ba9
TG
1384 if (unlikely(s->ctor)) {
1385 kasan_unpoison_object_data(s, object);
1386 s->ctor(object);
1387 kasan_poison_object_data(s, object);
1388 }
1389}
1390
81819f0f
CL
1391/*
1392 * Slab allocation and freeing
1393 */
5dfb4175
VD
1394static inline struct page *alloc_slab_page(struct kmem_cache *s,
1395 gfp_t flags, int node, struct kmem_cache_order_objects oo)
65c3376a 1396{
5dfb4175 1397 struct page *page;
65c3376a
CL
1398 int order = oo_order(oo);
1399
b1eeab67
VN
1400 flags |= __GFP_NOTRACK;
1401
2154a336 1402 if (node == NUMA_NO_NODE)
5dfb4175 1403 page = alloc_pages(flags, order);
65c3376a 1404 else
96db800f 1405 page = __alloc_pages_node(node, flags, order);
5dfb4175 1406
f3ccb2c4
VD
1407 if (page && memcg_charge_slab(page, flags, order, s)) {
1408 __free_pages(page, order);
1409 page = NULL;
1410 }
5dfb4175
VD
1411
1412 return page;
65c3376a
CL
1413}
1414
210e7a43
TG
1415#ifdef CONFIG_SLAB_FREELIST_RANDOM
1416/* Pre-initialize the random sequence cache */
1417static int init_cache_random_seq(struct kmem_cache *s)
1418{
1419 int err;
1420 unsigned long i, count = oo_objects(s->oo);
1421
1422 err = cache_random_seq_create(s, count, GFP_KERNEL);
1423 if (err) {
1424 pr_err("SLUB: Unable to initialize free list for %s\n",
1425 s->name);
1426 return err;
1427 }
1428
1429 /* Transform to an offset on the set of pages */
1430 if (s->random_seq) {
1431 for (i = 0; i < count; i++)
1432 s->random_seq[i] *= s->size;
1433 }
1434 return 0;
1435}
1436
1437/* Initialize each random sequence freelist per cache */
1438static void __init init_freelist_randomization(void)
1439{
1440 struct kmem_cache *s;
1441
1442 mutex_lock(&slab_mutex);
1443
1444 list_for_each_entry(s, &slab_caches, list)
1445 init_cache_random_seq(s);
1446
1447 mutex_unlock(&slab_mutex);
1448}
1449
1450/* Get the next entry on the pre-computed freelist randomized */
1451static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
1452 unsigned long *pos, void *start,
1453 unsigned long page_limit,
1454 unsigned long freelist_count)
1455{
1456 unsigned int idx;
1457
1458 /*
1459 * If the target page allocation failed, the number of objects on the
1460 * page might be smaller than the usual size defined by the cache.
1461 */
1462 do {
1463 idx = s->random_seq[*pos];
1464 *pos += 1;
1465 if (*pos >= freelist_count)
1466 *pos = 0;
1467 } while (unlikely(idx >= page_limit));
1468
1469 return (char *)start + idx;
1470}
1471
1472/* Shuffle the single linked freelist based on a random pre-computed sequence */
1473static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
1474{
1475 void *start;
1476 void *cur;
1477 void *next;
1478 unsigned long idx, pos, page_limit, freelist_count;
1479
1480 if (page->objects < 2 || !s->random_seq)
1481 return false;
1482
1483 freelist_count = oo_objects(s->oo);
1484 pos = get_random_int() % freelist_count;
1485
1486 page_limit = page->objects * s->size;
1487 start = fixup_red_left(s, page_address(page));
1488
1489 /* First entry is used as the base of the freelist */
1490 cur = next_freelist_entry(s, page, &pos, start, page_limit,
1491 freelist_count);
1492 page->freelist = cur;
1493
1494 for (idx = 1; idx < page->objects; idx++) {
1495 setup_object(s, page, cur);
1496 next = next_freelist_entry(s, page, &pos, start, page_limit,
1497 freelist_count);
1498 set_freepointer(s, cur, next);
1499 cur = next;
1500 }
1501 setup_object(s, page, cur);
1502 set_freepointer(s, cur, NULL);
1503
1504 return true;
1505}
1506#else
1507static inline int init_cache_random_seq(struct kmem_cache *s)
1508{
1509 return 0;
1510}
1511static inline void init_freelist_randomization(void) { }
1512static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)
1513{
1514 return false;
1515}
1516#endif /* CONFIG_SLAB_FREELIST_RANDOM */
1517
81819f0f
CL
1518static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1519{
06428780 1520 struct page *page;
834f3d11 1521 struct kmem_cache_order_objects oo = s->oo;
ba52270d 1522 gfp_t alloc_gfp;
588f8ba9
TG
1523 void *start, *p;
1524 int idx, order;
210e7a43 1525 bool shuffle;
81819f0f 1526
7e0528da
CL
1527 flags &= gfp_allowed_mask;
1528
d0164adc 1529 if (gfpflags_allow_blocking(flags))
7e0528da
CL
1530 local_irq_enable();
1531
b7a49f0d 1532 flags |= s->allocflags;
e12ba74d 1533
ba52270d
PE
1534 /*
1535 * Let the initial higher-order allocation fail under memory pressure
1536 * so we fall-back to the minimum order allocation.
1537 */
1538 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
d0164adc 1539 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
444eb2a4 1540 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
ba52270d 1541
5dfb4175 1542 page = alloc_slab_page(s, alloc_gfp, node, oo);
65c3376a
CL
1543 if (unlikely(!page)) {
1544 oo = s->min;
80c3a998 1545 alloc_gfp = flags;
65c3376a
CL
1546 /*
1547 * Allocation may have failed due to fragmentation.
1548 * Try a lower order alloc if possible
1549 */
5dfb4175 1550 page = alloc_slab_page(s, alloc_gfp, node, oo);
588f8ba9
TG
1551 if (unlikely(!page))
1552 goto out;
1553 stat(s, ORDER_FALLBACK);
65c3376a 1554 }
5a896d9e 1555
588f8ba9
TG
1556 if (kmemcheck_enabled &&
1557 !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
b1eeab67
VN
1558 int pages = 1 << oo_order(oo);
1559
80c3a998 1560 kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
b1eeab67
VN
1561
1562 /*
1563 * Objects from caches that have a constructor don't get
1564 * cleared when they're allocated, so we need to do it here.
1565 */
1566 if (s->ctor)
1567 kmemcheck_mark_uninitialized_pages(page, pages);
1568 else
1569 kmemcheck_mark_unallocated_pages(page, pages);
5a896d9e
VN
1570 }
1571
834f3d11 1572 page->objects = oo_objects(oo);
81819f0f 1573
1f458cbf 1574 order = compound_order(page);
1b4f59e3 1575 page->slab_cache = s;
c03f94cc 1576 __SetPageSlab(page);
2f064f34 1577 if (page_is_pfmemalloc(page))
072bb0aa 1578 SetPageSlabPfmemalloc(page);
81819f0f
CL
1579
1580 start = page_address(page);
81819f0f
CL
1581
1582 if (unlikely(s->flags & SLAB_POISON))
1f458cbf 1583 memset(start, POISON_INUSE, PAGE_SIZE << order);
81819f0f 1584
0316bec2
AR
1585 kasan_poison_slab(page);
1586
210e7a43
TG
1587 shuffle = shuffle_freelist(s, page);
1588
1589 if (!shuffle) {
1590 for_each_object_idx(p, idx, s, start, page->objects) {
1591 setup_object(s, page, p);
1592 if (likely(idx < page->objects))
1593 set_freepointer(s, p, p + s->size);
1594 else
1595 set_freepointer(s, p, NULL);
1596 }
1597 page->freelist = fixup_red_left(s, start);
81819f0f 1598 }
81819f0f 1599
e6e82ea1 1600 page->inuse = page->objects;
8cb0a506 1601 page->frozen = 1;
588f8ba9 1602
81819f0f 1603out:
d0164adc 1604 if (gfpflags_allow_blocking(flags))
588f8ba9
TG
1605 local_irq_disable();
1606 if (!page)
1607 return NULL;
1608
1609 mod_zone_page_state(page_zone(page),
1610 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1611 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1612 1 << oo_order(oo));
1613
1614 inc_slabs_node(s, page_to_nid(page), page->objects);
1615
81819f0f
CL
1616 return page;
1617}
1618
588f8ba9
TG
1619static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1620{
1621 if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
bacdcb34 1622 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
72baeef0
MH
1623 flags &= ~GFP_SLAB_BUG_MASK;
1624 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
1625 invalid_mask, &invalid_mask, flags, &flags);
588f8ba9
TG
1626 }
1627
1628 return allocate_slab(s,
1629 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
1630}
1631
81819f0f
CL
1632static void __free_slab(struct kmem_cache *s, struct page *page)
1633{
834f3d11
CL
1634 int order = compound_order(page);
1635 int pages = 1 << order;
81819f0f 1636
becfda68 1637 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
81819f0f
CL
1638 void *p;
1639
1640 slab_pad_check(s, page);
224a88be
CL
1641 for_each_object(p, s, page_address(page),
1642 page->objects)
f7cb1933 1643 check_object(s, page, p, SLUB_RED_INACTIVE);
81819f0f
CL
1644 }
1645
b1eeab67 1646 kmemcheck_free_shadow(page, compound_order(page));
5a896d9e 1647
81819f0f
CL
1648 mod_zone_page_state(page_zone(page),
1649 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1650 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
06428780 1651 -pages);
81819f0f 1652
072bb0aa 1653 __ClearPageSlabPfmemalloc(page);
49bd5221 1654 __ClearPageSlab(page);
1f458cbf 1655
22b751c3 1656 page_mapcount_reset(page);
1eb5ac64
NP
1657 if (current->reclaim_state)
1658 current->reclaim_state->reclaimed_slab += pages;
27ee57c9
VD
1659 memcg_uncharge_slab(page, order, s);
1660 __free_pages(page, order);
81819f0f
CL
1661}
1662
da9a638c
LJ
1663#define need_reserve_slab_rcu \
1664 (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
1665
81819f0f
CL
1666static void rcu_free_slab(struct rcu_head *h)
1667{
1668 struct page *page;
1669
da9a638c
LJ
1670 if (need_reserve_slab_rcu)
1671 page = virt_to_head_page(h);
1672 else
1673 page = container_of((struct list_head *)h, struct page, lru);
1674
1b4f59e3 1675 __free_slab(page->slab_cache, page);
81819f0f
CL
1676}
1677
1678static void free_slab(struct kmem_cache *s, struct page *page)
1679{
1680 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
da9a638c
LJ
1681 struct rcu_head *head;
1682
1683 if (need_reserve_slab_rcu) {
1684 int order = compound_order(page);
1685 int offset = (PAGE_SIZE << order) - s->reserved;
1686
1687 VM_BUG_ON(s->reserved != sizeof(*head));
1688 head = page_address(page) + offset;
1689 } else {
bc4f610d 1690 head = &page->rcu_head;
da9a638c 1691 }
81819f0f
CL
1692
1693 call_rcu(head, rcu_free_slab);
1694 } else
1695 __free_slab(s, page);
1696}
1697
1698static void discard_slab(struct kmem_cache *s, struct page *page)
1699{
205ab99d 1700 dec_slabs_node(s, page_to_nid(page), page->objects);
81819f0f
CL
1701 free_slab(s, page);
1702}
1703
1704/*
5cc6eee8 1705 * Management of partially allocated slabs.
81819f0f 1706 */
1e4dd946
SR
1707static inline void
1708__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
81819f0f 1709{
e95eed57 1710 n->nr_partial++;
136333d1 1711 if (tail == DEACTIVATE_TO_TAIL)
7c2e132c
CL
1712 list_add_tail(&page->lru, &n->partial);
1713 else
1714 list_add(&page->lru, &n->partial);
81819f0f
CL
1715}
1716
1e4dd946
SR
1717static inline void add_partial(struct kmem_cache_node *n,
1718 struct page *page, int tail)
62e346a8 1719{
c65c1877 1720 lockdep_assert_held(&n->list_lock);
1e4dd946
SR
1721 __add_partial(n, page, tail);
1722}
c65c1877 1723
1e4dd946
SR
1724static inline void remove_partial(struct kmem_cache_node *n,
1725 struct page *page)
1726{
1727 lockdep_assert_held(&n->list_lock);
52b4b950
DS
1728 list_del(&page->lru);
1729 n->nr_partial--;
1e4dd946
SR
1730}
1731
81819f0f 1732/*
7ced3719
CL
1733 * Remove slab from the partial list, freeze it and
1734 * return the pointer to the freelist.
81819f0f 1735 *
497b66f2 1736 * Returns a list of objects or NULL if it fails.
81819f0f 1737 */
497b66f2 1738static inline void *acquire_slab(struct kmem_cache *s,
acd19fd1 1739 struct kmem_cache_node *n, struct page *page,
633b0764 1740 int mode, int *objects)
81819f0f 1741{
2cfb7455
CL
1742 void *freelist;
1743 unsigned long counters;
1744 struct page new;
1745
c65c1877
PZ
1746 lockdep_assert_held(&n->list_lock);
1747
2cfb7455
CL
1748 /*
1749 * Zap the freelist and set the frozen bit.
1750 * The old freelist is the list of objects for the
1751 * per cpu allocation list.
1752 */
7ced3719
CL
1753 freelist = page->freelist;
1754 counters = page->counters;
1755 new.counters = counters;
633b0764 1756 *objects = new.objects - new.inuse;
23910c50 1757 if (mode) {
7ced3719 1758 new.inuse = page->objects;
23910c50
PE
1759 new.freelist = NULL;
1760 } else {
1761 new.freelist = freelist;
1762 }
2cfb7455 1763
a0132ac0 1764 VM_BUG_ON(new.frozen);
7ced3719 1765 new.frozen = 1;
2cfb7455 1766
7ced3719 1767 if (!__cmpxchg_double_slab(s, page,
2cfb7455 1768 freelist, counters,
02d7633f 1769 new.freelist, new.counters,
7ced3719 1770 "acquire_slab"))
7ced3719 1771 return NULL;
2cfb7455
CL
1772
1773 remove_partial(n, page);
7ced3719 1774 WARN_ON(!freelist);
49e22585 1775 return freelist;
81819f0f
CL
1776}
1777
633b0764 1778static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
8ba00bb6 1779static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
49e22585 1780
81819f0f 1781/*
672bba3a 1782 * Try to allocate a partial slab from a specific node.
81819f0f 1783 */
8ba00bb6
JK
1784static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
1785 struct kmem_cache_cpu *c, gfp_t flags)
81819f0f 1786{
49e22585
CL
1787 struct page *page, *page2;
1788 void *object = NULL;
633b0764
JK
1789 int available = 0;
1790 int objects;
81819f0f
CL
1791
1792 /*
1793 * Racy check. If we mistakenly see no partial slabs then we
1794 * just allocate an empty slab. If we mistakenly try to get a
672bba3a
CL
1795 * partial slab and there is none available then get_partials()
1796 * will return NULL.
81819f0f
CL
1797 */
1798 if (!n || !n->nr_partial)
1799 return NULL;
1800
1801 spin_lock(&n->list_lock);
49e22585 1802 list_for_each_entry_safe(page, page2, &n->partial, lru) {
8ba00bb6 1803 void *t;
49e22585 1804
8ba00bb6
JK
1805 if (!pfmemalloc_match(page, flags))
1806 continue;
1807
633b0764 1808 t = acquire_slab(s, n, page, object == NULL, &objects);
49e22585
CL
1809 if (!t)
1810 break;
1811
633b0764 1812 available += objects;
12d79634 1813 if (!object) {
49e22585 1814 c->page = page;
49e22585 1815 stat(s, ALLOC_FROM_PARTIAL);
49e22585 1816 object = t;
49e22585 1817 } else {
633b0764 1818 put_cpu_partial(s, page, 0);
8028dcea 1819 stat(s, CPU_PARTIAL_NODE);
49e22585 1820 }
345c905d
JK
1821 if (!kmem_cache_has_cpu_partial(s)
1822 || available > s->cpu_partial / 2)
49e22585
CL
1823 break;
1824
497b66f2 1825 }
81819f0f 1826 spin_unlock(&n->list_lock);
497b66f2 1827 return object;
81819f0f
CL
1828}
1829
1830/*
672bba3a 1831 * Get a page from somewhere. Search in increasing NUMA distances.
81819f0f 1832 */
de3ec035 1833static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
acd19fd1 1834 struct kmem_cache_cpu *c)
81819f0f
CL
1835{
1836#ifdef CONFIG_NUMA
1837 struct zonelist *zonelist;
dd1a239f 1838 struct zoneref *z;
54a6eb5c
MG
1839 struct zone *zone;
1840 enum zone_type high_zoneidx = gfp_zone(flags);
497b66f2 1841 void *object;
cc9a6c87 1842 unsigned int cpuset_mems_cookie;
81819f0f
CL
1843
1844 /*
672bba3a
CL
1845 * The defrag ratio allows a configuration of the tradeoffs between
1846 * inter node defragmentation and node local allocations. A lower
1847 * defrag_ratio increases the tendency to do local allocations
1848 * instead of attempting to obtain partial slabs from other nodes.
81819f0f 1849 *
672bba3a
CL
1850 * If the defrag_ratio is set to 0 then kmalloc() always
1851 * returns node local objects. If the ratio is higher then kmalloc()
1852 * may return off node objects because partial slabs are obtained
1853 * from other nodes and filled up.
81819f0f 1854 *
43efd3ea
LP
1855 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
1856 * (which makes defrag_ratio = 1000) then every (well almost)
1857 * allocation will first attempt to defrag slab caches on other nodes.
1858 * This means scanning over all nodes to look for partial slabs which
1859 * may be expensive if we do it every time we are trying to find a slab
672bba3a 1860 * with available objects.
81819f0f 1861 */
9824601e
CL
1862 if (!s->remote_node_defrag_ratio ||
1863 get_cycles() % 1024 > s->remote_node_defrag_ratio)
81819f0f
CL
1864 return NULL;
1865
cc9a6c87 1866 do {
d26914d1 1867 cpuset_mems_cookie = read_mems_allowed_begin();
2a389610 1868 zonelist = node_zonelist(mempolicy_slab_node(), flags);
cc9a6c87
MG
1869 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1870 struct kmem_cache_node *n;
1871
1872 n = get_node(s, zone_to_nid(zone));
1873
dee2f8aa 1874 if (n && cpuset_zone_allowed(zone, flags) &&
cc9a6c87 1875 n->nr_partial > s->min_partial) {
8ba00bb6 1876 object = get_partial_node(s, n, c, flags);
cc9a6c87
MG
1877 if (object) {
1878 /*
d26914d1
MG
1879 * Don't check read_mems_allowed_retry()
1880 * here - if mems_allowed was updated in
1881 * parallel, that was a harmless race
1882 * between allocation and the cpuset
1883 * update
cc9a6c87 1884 */
cc9a6c87
MG
1885 return object;
1886 }
c0ff7453 1887 }
81819f0f 1888 }
d26914d1 1889 } while (read_mems_allowed_retry(cpuset_mems_cookie));
81819f0f
CL
1890#endif
1891 return NULL;
1892}
1893
1894/*
1895 * Get a partial page, lock it and return it.
1896 */
497b66f2 1897static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
acd19fd1 1898 struct kmem_cache_cpu *c)
81819f0f 1899{
497b66f2 1900 void *object;
a561ce00
JK
1901 int searchnode = node;
1902
1903 if (node == NUMA_NO_NODE)
1904 searchnode = numa_mem_id();
1905 else if (!node_present_pages(node))
1906 searchnode = node_to_mem_node(node);
81819f0f 1907
8ba00bb6 1908 object = get_partial_node(s, get_node(s, searchnode), c, flags);
497b66f2
CL
1909 if (object || node != NUMA_NO_NODE)
1910 return object;
81819f0f 1911
acd19fd1 1912 return get_any_partial(s, flags, c);
81819f0f
CL
1913}
1914
8a5ec0ba
CL
1915#ifdef CONFIG_PREEMPT
1916/*
1917 * Calculate the next globally unique transaction for disambiguiation
1918 * during cmpxchg. The transactions start with the cpu number and are then
1919 * incremented by CONFIG_NR_CPUS.
1920 */
1921#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
1922#else
1923/*
1924 * No preemption supported therefore also no need to check for
1925 * different cpus.
1926 */
1927#define TID_STEP 1
1928#endif
1929
1930static inline unsigned long next_tid(unsigned long tid)
1931{
1932 return tid + TID_STEP;
1933}
1934
1935static inline unsigned int tid_to_cpu(unsigned long tid)
1936{
1937 return tid % TID_STEP;
1938}
1939
1940static inline unsigned long tid_to_event(unsigned long tid)
1941{
1942 return tid / TID_STEP;
1943}
1944
1945static inline unsigned int init_tid(int cpu)
1946{
1947 return cpu;
1948}
1949
1950static inline void note_cmpxchg_failure(const char *n,
1951 const struct kmem_cache *s, unsigned long tid)
1952{
1953#ifdef SLUB_DEBUG_CMPXCHG
1954 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
1955
f9f58285 1956 pr_info("%s %s: cmpxchg redo ", n, s->name);
8a5ec0ba
CL
1957
1958#ifdef CONFIG_PREEMPT
1959 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
f9f58285 1960 pr_warn("due to cpu change %d -> %d\n",
8a5ec0ba
CL
1961 tid_to_cpu(tid), tid_to_cpu(actual_tid));
1962 else
1963#endif
1964 if (tid_to_event(tid) != tid_to_event(actual_tid))
f9f58285 1965 pr_warn("due to cpu running other code. Event %ld->%ld\n",
8a5ec0ba
CL
1966 tid_to_event(tid), tid_to_event(actual_tid));
1967 else
f9f58285 1968 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
8a5ec0ba
CL
1969 actual_tid, tid, next_tid(tid));
1970#endif
4fdccdfb 1971 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
8a5ec0ba
CL
1972}
1973
788e1aad 1974static void init_kmem_cache_cpus(struct kmem_cache *s)
8a5ec0ba 1975{
8a5ec0ba
CL
1976 int cpu;
1977
1978 for_each_possible_cpu(cpu)
1979 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
8a5ec0ba 1980}
2cfb7455 1981
81819f0f
CL
1982/*
1983 * Remove the cpu slab
1984 */
d0e0ac97
CG
1985static void deactivate_slab(struct kmem_cache *s, struct page *page,
1986 void *freelist)
81819f0f 1987{
2cfb7455 1988 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
2cfb7455
CL
1989 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1990 int lock = 0;
1991 enum slab_modes l = M_NONE, m = M_NONE;
2cfb7455 1992 void *nextfree;
136333d1 1993 int tail = DEACTIVATE_TO_HEAD;
2cfb7455
CL
1994 struct page new;
1995 struct page old;
1996
1997 if (page->freelist) {
84e554e6 1998 stat(s, DEACTIVATE_REMOTE_FREES);
136333d1 1999 tail = DEACTIVATE_TO_TAIL;
2cfb7455
CL
2000 }
2001
894b8788 2002 /*
2cfb7455
CL
2003 * Stage one: Free all available per cpu objects back
2004 * to the page freelist while it is still frozen. Leave the
2005 * last one.
2006 *
2007 * There is no need to take the list->lock because the page
2008 * is still frozen.
2009 */
2010 while (freelist && (nextfree = get_freepointer(s, freelist))) {
2011 void *prior;
2012 unsigned long counters;
2013
2014 do {
2015 prior = page->freelist;
2016 counters = page->counters;
2017 set_freepointer(s, freelist, prior);
2018 new.counters = counters;
2019 new.inuse--;
a0132ac0 2020 VM_BUG_ON(!new.frozen);
2cfb7455 2021
1d07171c 2022 } while (!__cmpxchg_double_slab(s, page,
2cfb7455
CL
2023 prior, counters,
2024 freelist, new.counters,
2025 "drain percpu freelist"));
2026
2027 freelist = nextfree;
2028 }
2029
894b8788 2030 /*
2cfb7455
CL
2031 * Stage two: Ensure that the page is unfrozen while the
2032 * list presence reflects the actual number of objects
2033 * during unfreeze.
2034 *
2035 * We setup the list membership and then perform a cmpxchg
2036 * with the count. If there is a mismatch then the page
2037 * is not unfrozen but the page is on the wrong list.
2038 *
2039 * Then we restart the process which may have to remove
2040 * the page from the list that we just put it on again
2041 * because the number of objects in the slab may have
2042 * changed.
894b8788 2043 */
2cfb7455 2044redo:
894b8788 2045
2cfb7455
CL
2046 old.freelist = page->freelist;
2047 old.counters = page->counters;
a0132ac0 2048 VM_BUG_ON(!old.frozen);
7c2e132c 2049
2cfb7455
CL
2050 /* Determine target state of the slab */
2051 new.counters = old.counters;
2052 if (freelist) {
2053 new.inuse--;
2054 set_freepointer(s, freelist, old.freelist);
2055 new.freelist = freelist;
2056 } else
2057 new.freelist = old.freelist;
2058
2059 new.frozen = 0;
2060
8a5b20ae 2061 if (!new.inuse && n->nr_partial >= s->min_partial)
2cfb7455
CL
2062 m = M_FREE;
2063 else if (new.freelist) {
2064 m = M_PARTIAL;
2065 if (!lock) {
2066 lock = 1;
2067 /*
2068 * Taking the spinlock removes the possiblity
2069 * that acquire_slab() will see a slab page that
2070 * is frozen
2071 */
2072 spin_lock(&n->list_lock);
2073 }
2074 } else {
2075 m = M_FULL;
2076 if (kmem_cache_debug(s) && !lock) {
2077 lock = 1;
2078 /*
2079 * This also ensures that the scanning of full
2080 * slabs from diagnostic functions will not see
2081 * any frozen slabs.
2082 */
2083 spin_lock(&n->list_lock);
2084 }
2085 }
2086
2087 if (l != m) {
2088
2089 if (l == M_PARTIAL)
2090
2091 remove_partial(n, page);
2092
2093 else if (l == M_FULL)
894b8788 2094
c65c1877 2095 remove_full(s, n, page);
2cfb7455
CL
2096
2097 if (m == M_PARTIAL) {
2098
2099 add_partial(n, page, tail);
136333d1 2100 stat(s, tail);
2cfb7455
CL
2101
2102 } else if (m == M_FULL) {
894b8788 2103
2cfb7455
CL
2104 stat(s, DEACTIVATE_FULL);
2105 add_full(s, n, page);
2106
2107 }
2108 }
2109
2110 l = m;
1d07171c 2111 if (!__cmpxchg_double_slab(s, page,
2cfb7455
CL
2112 old.freelist, old.counters,
2113 new.freelist, new.counters,
2114 "unfreezing slab"))
2115 goto redo;
2116
2cfb7455
CL
2117 if (lock)
2118 spin_unlock(&n->list_lock);
2119
2120 if (m == M_FREE) {
2121 stat(s, DEACTIVATE_EMPTY);
2122 discard_slab(s, page);
2123 stat(s, FREE_SLAB);
894b8788 2124 }
81819f0f
CL
2125}
2126
d24ac77f
JK
2127/*
2128 * Unfreeze all the cpu partial slabs.
2129 *
59a09917
CL
2130 * This function must be called with interrupts disabled
2131 * for the cpu using c (or some other guarantee must be there
2132 * to guarantee no concurrent accesses).
d24ac77f 2133 */
59a09917
CL
2134static void unfreeze_partials(struct kmem_cache *s,
2135 struct kmem_cache_cpu *c)
49e22585 2136{
345c905d 2137#ifdef CONFIG_SLUB_CPU_PARTIAL
43d77867 2138 struct kmem_cache_node *n = NULL, *n2 = NULL;
9ada1934 2139 struct page *page, *discard_page = NULL;
49e22585
CL
2140
2141 while ((page = c->partial)) {
49e22585
CL
2142 struct page new;
2143 struct page old;
2144
2145 c->partial = page->next;
43d77867
JK
2146
2147 n2 = get_node(s, page_to_nid(page));
2148 if (n != n2) {
2149 if (n)
2150 spin_unlock(&n->list_lock);
2151
2152 n = n2;
2153 spin_lock(&n->list_lock);
2154 }
49e22585
CL
2155
2156 do {
2157
2158 old.freelist = page->freelist;
2159 old.counters = page->counters;
a0132ac0 2160 VM_BUG_ON(!old.frozen);
49e22585
CL
2161
2162 new.counters = old.counters;
2163 new.freelist = old.freelist;
2164
2165 new.frozen = 0;
2166
d24ac77f 2167 } while (!__cmpxchg_double_slab(s, page,
49e22585
CL
2168 old.freelist, old.counters,
2169 new.freelist, new.counters,
2170 "unfreezing slab"));
2171
8a5b20ae 2172 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
9ada1934
SL
2173 page->next = discard_page;
2174 discard_page = page;
43d77867
JK
2175 } else {
2176 add_partial(n, page, DEACTIVATE_TO_TAIL);
2177 stat(s, FREE_ADD_PARTIAL);
49e22585
CL
2178 }
2179 }
2180
2181 if (n)
2182 spin_unlock(&n->list_lock);
9ada1934
SL
2183
2184 while (discard_page) {
2185 page = discard_page;
2186 discard_page = discard_page->next;
2187
2188 stat(s, DEACTIVATE_EMPTY);
2189 discard_slab(s, page);
2190 stat(s, FREE_SLAB);
2191 }
345c905d 2192#endif
49e22585
CL
2193}
2194
2195/*
2196 * Put a page that was just frozen (in __slab_free) into a partial page
2197 * slot if available. This is done without interrupts disabled and without
2198 * preemption disabled. The cmpxchg is racy and may put the partial page
2199 * onto a random cpus partial slot.
2200 *
2201 * If we did not find a slot then simply move all the partials to the
2202 * per node partial list.
2203 */
633b0764 2204static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
49e22585 2205{
345c905d 2206#ifdef CONFIG_SLUB_CPU_PARTIAL
49e22585
CL
2207 struct page *oldpage;
2208 int pages;
2209 int pobjects;
2210
d6e0b7fa 2211 preempt_disable();
49e22585
CL
2212 do {
2213 pages = 0;
2214 pobjects = 0;
2215 oldpage = this_cpu_read(s->cpu_slab->partial);
2216
2217 if (oldpage) {
2218 pobjects = oldpage->pobjects;
2219 pages = oldpage->pages;
2220 if (drain && pobjects > s->cpu_partial) {
2221 unsigned long flags;
2222 /*
2223 * partial array is full. Move the existing
2224 * set to the per node partial list.
2225 */
2226 local_irq_save(flags);
59a09917 2227 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
49e22585 2228 local_irq_restore(flags);
e24fc410 2229 oldpage = NULL;
49e22585
CL
2230 pobjects = 0;
2231 pages = 0;
8028dcea 2232 stat(s, CPU_PARTIAL_DRAIN);
49e22585
CL
2233 }
2234 }
2235
2236 pages++;
2237 pobjects += page->objects - page->inuse;
2238
2239 page->pages = pages;
2240 page->pobjects = pobjects;
2241 page->next = oldpage;
2242
d0e0ac97
CG
2243 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
2244 != oldpage);
d6e0b7fa
VD
2245 if (unlikely(!s->cpu_partial)) {
2246 unsigned long flags;
2247
2248 local_irq_save(flags);
2249 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2250 local_irq_restore(flags);
2251 }
2252 preempt_enable();
345c905d 2253#endif
49e22585
CL
2254}
2255
dfb4f096 2256static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
81819f0f 2257{
84e554e6 2258 stat(s, CPUSLAB_FLUSH);
c17dda40
CL
2259 deactivate_slab(s, c->page, c->freelist);
2260
2261 c->tid = next_tid(c->tid);
2262 c->page = NULL;
2263 c->freelist = NULL;
81819f0f
CL
2264}
2265
2266/*
2267 * Flush cpu slab.
6446faa2 2268 *
81819f0f
CL
2269 * Called from IPI handler with interrupts disabled.
2270 */
0c710013 2271static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
81819f0f 2272{
9dfc6e68 2273 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
81819f0f 2274
49e22585
CL
2275 if (likely(c)) {
2276 if (c->page)
2277 flush_slab(s, c);
2278
59a09917 2279 unfreeze_partials(s, c);
49e22585 2280 }
81819f0f
CL
2281}
2282
2283static void flush_cpu_slab(void *d)
2284{
2285 struct kmem_cache *s = d;
81819f0f 2286
dfb4f096 2287 __flush_cpu_slab(s, smp_processor_id());
81819f0f
CL
2288}
2289
a8364d55
GBY
2290static bool has_cpu_slab(int cpu, void *info)
2291{
2292 struct kmem_cache *s = info;
2293 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2294
02e1a9cd 2295 return c->page || c->partial;
a8364d55
GBY
2296}
2297
81819f0f
CL
2298static void flush_all(struct kmem_cache *s)
2299{
a8364d55 2300 on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
81819f0f
CL
2301}
2302
a96a87bf
SAS
2303/*
2304 * Use the cpu notifier to insure that the cpu slabs are flushed when
2305 * necessary.
2306 */
2307static int slub_cpu_dead(unsigned int cpu)
2308{
2309 struct kmem_cache *s;
2310 unsigned long flags;
2311
2312 mutex_lock(&slab_mutex);
2313 list_for_each_entry(s, &slab_caches, list) {
2314 local_irq_save(flags);
2315 __flush_cpu_slab(s, cpu);
2316 local_irq_restore(flags);
2317 }
2318 mutex_unlock(&slab_mutex);
2319 return 0;
2320}
2321
dfb4f096
CL
2322/*
2323 * Check if the objects in a per cpu structure fit numa
2324 * locality expectations.
2325 */
57d437d2 2326static inline int node_match(struct page *page, int node)
dfb4f096
CL
2327{
2328#ifdef CONFIG_NUMA
4d7868e6 2329 if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node))
dfb4f096
CL
2330 return 0;
2331#endif
2332 return 1;
2333}
2334
9a02d699 2335#ifdef CONFIG_SLUB_DEBUG
781b2ba6
PE
2336static int count_free(struct page *page)
2337{
2338 return page->objects - page->inuse;
2339}
2340
9a02d699
DR
2341static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2342{
2343 return atomic_long_read(&n->total_objects);
2344}
2345#endif /* CONFIG_SLUB_DEBUG */
2346
2347#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
781b2ba6
PE
2348static unsigned long count_partial(struct kmem_cache_node *n,
2349 int (*get_count)(struct page *))
2350{
2351 unsigned long flags;
2352 unsigned long x = 0;
2353 struct page *page;
2354
2355 spin_lock_irqsave(&n->list_lock, flags);
2356 list_for_each_entry(page, &n->partial, lru)
2357 x += get_count(page);
2358 spin_unlock_irqrestore(&n->list_lock, flags);
2359 return x;
2360}
9a02d699 2361#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
26c02cf0 2362
781b2ba6
PE
2363static noinline void
2364slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2365{
9a02d699
DR
2366#ifdef CONFIG_SLUB_DEBUG
2367 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
2368 DEFAULT_RATELIMIT_BURST);
781b2ba6 2369 int node;
fa45dc25 2370 struct kmem_cache_node *n;
781b2ba6 2371
9a02d699
DR
2372 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
2373 return;
2374
5b3810e5
VB
2375 pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
2376 nid, gfpflags, &gfpflags);
f9f58285
FF
2377 pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n",
2378 s->name, s->object_size, s->size, oo_order(s->oo),
2379 oo_order(s->min));
781b2ba6 2380
3b0efdfa 2381 if (oo_order(s->min) > get_order(s->object_size))
f9f58285
FF
2382 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
2383 s->name);
fa5ec8a1 2384
fa45dc25 2385 for_each_kmem_cache_node(s, node, n) {
781b2ba6
PE
2386 unsigned long nr_slabs;
2387 unsigned long nr_objs;
2388 unsigned long nr_free;
2389
26c02cf0
AB
2390 nr_free = count_partial(n, count_free);
2391 nr_slabs = node_nr_slabs(n);
2392 nr_objs = node_nr_objs(n);
781b2ba6 2393
f9f58285 2394 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
781b2ba6
PE
2395 node, nr_slabs, nr_objs, nr_free);
2396 }
9a02d699 2397#endif
781b2ba6
PE
2398}
2399
497b66f2
CL
2400static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2401 int node, struct kmem_cache_cpu **pc)
2402{
6faa6833 2403 void *freelist;
188fd063
CL
2404 struct kmem_cache_cpu *c = *pc;
2405 struct page *page;
497b66f2 2406
188fd063 2407 freelist = get_partial(s, flags, node, c);
497b66f2 2408
188fd063
CL
2409 if (freelist)
2410 return freelist;
2411
2412 page = new_slab(s, flags, node);
497b66f2 2413 if (page) {
7c8e0181 2414 c = raw_cpu_ptr(s->cpu_slab);
497b66f2
CL
2415 if (c->page)
2416 flush_slab(s, c);
2417
2418 /*
2419 * No other reference to the page yet so we can
2420 * muck around with it freely without cmpxchg
2421 */
6faa6833 2422 freelist = page->freelist;
497b66f2
CL
2423 page->freelist = NULL;
2424
2425 stat(s, ALLOC_SLAB);
497b66f2
CL
2426 c->page = page;
2427 *pc = c;
2428 } else
6faa6833 2429 freelist = NULL;
497b66f2 2430
6faa6833 2431 return freelist;
497b66f2
CL
2432}
2433
072bb0aa
MG
2434static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
2435{
2436 if (unlikely(PageSlabPfmemalloc(page)))
2437 return gfp_pfmemalloc_allowed(gfpflags);
2438
2439 return true;
2440}
2441
213eeb9f 2442/*
d0e0ac97
CG
2443 * Check the page->freelist of a page and either transfer the freelist to the
2444 * per cpu freelist or deactivate the page.
213eeb9f
CL
2445 *
2446 * The page is still frozen if the return value is not NULL.
2447 *
2448 * If this function returns NULL then the page has been unfrozen.
d24ac77f
JK
2449 *
2450 * This function must be called with interrupt disabled.
213eeb9f
CL
2451 */
2452static inline void *get_freelist(struct kmem_cache *s, struct page *page)
2453{
2454 struct page new;
2455 unsigned long counters;
2456 void *freelist;
2457
2458 do {
2459 freelist = page->freelist;
2460 counters = page->counters;
6faa6833 2461
213eeb9f 2462 new.counters = counters;
a0132ac0 2463 VM_BUG_ON(!new.frozen);
213eeb9f
CL
2464
2465 new.inuse = page->objects;
2466 new.frozen = freelist != NULL;
2467
d24ac77f 2468 } while (!__cmpxchg_double_slab(s, page,
213eeb9f
CL
2469 freelist, counters,
2470 NULL, new.counters,
2471 "get_freelist"));
2472
2473 return freelist;
2474}
2475
81819f0f 2476/*
894b8788
CL
2477 * Slow path. The lockless freelist is empty or we need to perform
2478 * debugging duties.
2479 *
894b8788
CL
2480 * Processing is still very fast if new objects have been freed to the
2481 * regular freelist. In that case we simply take over the regular freelist
2482 * as the lockless freelist and zap the regular freelist.
81819f0f 2483 *
894b8788
CL
2484 * If that is not working then we fall back to the partial lists. We take the
2485 * first element of the freelist as the object to allocate now and move the
2486 * rest of the freelist to the lockless freelist.
81819f0f 2487 *
894b8788 2488 * And if we were unable to get a new slab from the partial slab lists then
6446faa2
CL
2489 * we need to allocate a new slab. This is the slowest path since it involves
2490 * a call to the page allocator and the setup of a new slab.
a380a3c7
CL
2491 *
2492 * Version of __slab_alloc to use when we know that interrupts are
2493 * already disabled (which is the case for bulk allocation).
81819f0f 2494 */
a380a3c7 2495static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
ce71e27c 2496 unsigned long addr, struct kmem_cache_cpu *c)
81819f0f 2497{
6faa6833 2498 void *freelist;
f6e7def7 2499 struct page *page;
81819f0f 2500
f6e7def7
CL
2501 page = c->page;
2502 if (!page)
81819f0f 2503 goto new_slab;
49e22585 2504redo:
6faa6833 2505
57d437d2 2506 if (unlikely(!node_match(page, node))) {
a561ce00
JK
2507 int searchnode = node;
2508
2509 if (node != NUMA_NO_NODE && !node_present_pages(node))
2510 searchnode = node_to_mem_node(node);
2511
2512 if (unlikely(!node_match(page, searchnode))) {
2513 stat(s, ALLOC_NODE_MISMATCH);
2514 deactivate_slab(s, page, c->freelist);
2515 c->page = NULL;
2516 c->freelist = NULL;
2517 goto new_slab;
2518 }
fc59c053 2519 }
6446faa2 2520
072bb0aa
MG
2521 /*
2522 * By rights, we should be searching for a slab page that was
2523 * PFMEMALLOC but right now, we are losing the pfmemalloc
2524 * information when the page leaves the per-cpu allocator
2525 */
2526 if (unlikely(!pfmemalloc_match(page, gfpflags))) {
2527 deactivate_slab(s, page, c->freelist);
2528 c->page = NULL;
2529 c->freelist = NULL;
2530 goto new_slab;
2531 }
2532
73736e03 2533 /* must check again c->freelist in case of cpu migration or IRQ */
6faa6833
CL
2534 freelist = c->freelist;
2535 if (freelist)
73736e03 2536 goto load_freelist;
03e404af 2537
f6e7def7 2538 freelist = get_freelist(s, page);
6446faa2 2539
6faa6833 2540 if (!freelist) {
03e404af
CL
2541 c->page = NULL;
2542 stat(s, DEACTIVATE_BYPASS);
fc59c053 2543 goto new_slab;
03e404af 2544 }
6446faa2 2545
84e554e6 2546 stat(s, ALLOC_REFILL);
6446faa2 2547
894b8788 2548load_freelist:
507effea
CL
2549 /*
2550 * freelist is pointing to the list of objects to be used.
2551 * page is pointing to the page from which the objects are obtained.
2552 * That page must be frozen for per cpu allocations to work.
2553 */
a0132ac0 2554 VM_BUG_ON(!c->page->frozen);
6faa6833 2555 c->freelist = get_freepointer(s, freelist);
8a5ec0ba 2556 c->tid = next_tid(c->tid);
6faa6833 2557 return freelist;
81819f0f 2558
81819f0f 2559new_slab:
2cfb7455 2560
49e22585 2561 if (c->partial) {
f6e7def7
CL
2562 page = c->page = c->partial;
2563 c->partial = page->next;
49e22585
CL
2564 stat(s, CPU_PARTIAL_ALLOC);
2565 c->freelist = NULL;
2566 goto redo;
81819f0f
CL
2567 }
2568
188fd063 2569 freelist = new_slab_objects(s, gfpflags, node, &c);
01ad8a7b 2570
f4697436 2571 if (unlikely(!freelist)) {
9a02d699 2572 slab_out_of_memory(s, gfpflags, node);
f4697436 2573 return NULL;
81819f0f 2574 }
2cfb7455 2575
f6e7def7 2576 page = c->page;
5091b74a 2577 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
4b6f0750 2578 goto load_freelist;
2cfb7455 2579
497b66f2 2580 /* Only entered in the debug case */
d0e0ac97
CG
2581 if (kmem_cache_debug(s) &&
2582 !alloc_debug_processing(s, page, freelist, addr))
497b66f2 2583 goto new_slab; /* Slab failed checks. Next slab needed */
894b8788 2584
f6e7def7 2585 deactivate_slab(s, page, get_freepointer(s, freelist));
c17dda40
CL
2586 c->page = NULL;
2587 c->freelist = NULL;
6faa6833 2588 return freelist;
894b8788
CL
2589}
2590
a380a3c7
CL
2591/*
2592 * Another one that disabled interrupt and compensates for possible
2593 * cpu changes by refetching the per cpu area pointer.
2594 */
2595static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2596 unsigned long addr, struct kmem_cache_cpu *c)
2597{
2598 void *p;
2599 unsigned long flags;
2600
2601 local_irq_save(flags);
2602#ifdef CONFIG_PREEMPT
2603 /*
2604 * We may have been preempted and rescheduled on a different
2605 * cpu before disabling interrupts. Need to reload cpu area
2606 * pointer.
2607 */
2608 c = this_cpu_ptr(s->cpu_slab);
2609#endif
2610
2611 p = ___slab_alloc(s, gfpflags, node, addr, c);
2612 local_irq_restore(flags);
2613 return p;
2614}
2615
894b8788
CL
2616/*
2617 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2618 * have the fastpath folded into their functions. So no function call
2619 * overhead for requests that can be satisfied on the fastpath.
2620 *
2621 * The fastpath works by first checking if the lockless freelist can be used.
2622 * If not then __slab_alloc is called for slow processing.
2623 *
2624 * Otherwise we can simply pick the next object from the lockless free list.
2625 */
2b847c3c 2626static __always_inline void *slab_alloc_node(struct kmem_cache *s,
ce71e27c 2627 gfp_t gfpflags, int node, unsigned long addr)
894b8788 2628{
03ec0ed5 2629 void *object;
dfb4f096 2630 struct kmem_cache_cpu *c;
57d437d2 2631 struct page *page;
8a5ec0ba 2632 unsigned long tid;
1f84260c 2633
8135be5a
VD
2634 s = slab_pre_alloc_hook(s, gfpflags);
2635 if (!s)
773ff60e 2636 return NULL;
8a5ec0ba 2637redo:
8a5ec0ba
CL
2638 /*
2639 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
2640 * enabled. We may switch back and forth between cpus while
2641 * reading from one cpu area. That does not matter as long
2642 * as we end up on the original cpu again when doing the cmpxchg.
7cccd80b 2643 *
9aabf810
JK
2644 * We should guarantee that tid and kmem_cache are retrieved on
2645 * the same cpu. It could be different if CONFIG_PREEMPT so we need
2646 * to check if it is matched or not.
8a5ec0ba 2647 */
9aabf810
JK
2648 do {
2649 tid = this_cpu_read(s->cpu_slab->tid);
2650 c = raw_cpu_ptr(s->cpu_slab);
859b7a0e
MR
2651 } while (IS_ENABLED(CONFIG_PREEMPT) &&
2652 unlikely(tid != READ_ONCE(c->tid)));
9aabf810
JK
2653
2654 /*
2655 * Irqless object alloc/free algorithm used here depends on sequence
2656 * of fetching cpu_slab's data. tid should be fetched before anything
2657 * on c to guarantee that object and page associated with previous tid
2658 * won't be used with current tid. If we fetch tid first, object and
2659 * page could be one associated with next tid and our alloc/free
2660 * request will be failed. In this case, we will retry. So, no problem.
2661 */
2662 barrier();
8a5ec0ba 2663
8a5ec0ba
CL
2664 /*
2665 * The transaction ids are globally unique per cpu and per operation on
2666 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2667 * occurs on the right processor and that there was no operation on the
2668 * linked list in between.
2669 */
8a5ec0ba 2670
9dfc6e68 2671 object = c->freelist;
57d437d2 2672 page = c->page;
8eae1492 2673 if (unlikely(!object || !node_match(page, node))) {
dfb4f096 2674 object = __slab_alloc(s, gfpflags, node, addr, c);
8eae1492
DH
2675 stat(s, ALLOC_SLOWPATH);
2676 } else {
0ad9500e
ED
2677 void *next_object = get_freepointer_safe(s, object);
2678
8a5ec0ba 2679 /*
25985edc 2680 * The cmpxchg will only match if there was no additional
8a5ec0ba
CL
2681 * operation and if we are on the right processor.
2682 *
d0e0ac97
CG
2683 * The cmpxchg does the following atomically (without lock
2684 * semantics!)
8a5ec0ba
CL
2685 * 1. Relocate first pointer to the current per cpu area.
2686 * 2. Verify that tid and freelist have not been changed
2687 * 3. If they were not changed replace tid and freelist
2688 *
d0e0ac97
CG
2689 * Since this is without lock semantics the protection is only
2690 * against code executing on this cpu *not* from access by
2691 * other cpus.
8a5ec0ba 2692 */
933393f5 2693 if (unlikely(!this_cpu_cmpxchg_double(
8a5ec0ba
CL
2694 s->cpu_slab->freelist, s->cpu_slab->tid,
2695 object, tid,
0ad9500e 2696 next_object, next_tid(tid)))) {
8a5ec0ba
CL
2697
2698 note_cmpxchg_failure("slab_alloc", s, tid);
2699 goto redo;
2700 }
0ad9500e 2701 prefetch_freepointer(s, next_object);
84e554e6 2702 stat(s, ALLOC_FASTPATH);
894b8788 2703 }
8a5ec0ba 2704
74e2134f 2705 if (unlikely(gfpflags & __GFP_ZERO) && object)
3b0efdfa 2706 memset(object, 0, s->object_size);
d07dbea4 2707
03ec0ed5 2708 slab_post_alloc_hook(s, gfpflags, 1, &object);
5a896d9e 2709
894b8788 2710 return object;
81819f0f
CL
2711}
2712
2b847c3c
EG
2713static __always_inline void *slab_alloc(struct kmem_cache *s,
2714 gfp_t gfpflags, unsigned long addr)
2715{
2716 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
2717}
2718
81819f0f
CL
2719void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
2720{
2b847c3c 2721 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
5b882be4 2722
d0e0ac97
CG
2723 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
2724 s->size, gfpflags);
5b882be4
EGM
2725
2726 return ret;
81819f0f
CL
2727}
2728EXPORT_SYMBOL(kmem_cache_alloc);
2729
0f24f128 2730#ifdef CONFIG_TRACING
4a92379b
RK
2731void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
2732{
2b847c3c 2733 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
4a92379b 2734 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
505f5dcb 2735 kasan_kmalloc(s, ret, size, gfpflags);
4a92379b
RK
2736 return ret;
2737}
2738EXPORT_SYMBOL(kmem_cache_alloc_trace);
5b882be4
EGM
2739#endif
2740
81819f0f
CL
2741#ifdef CONFIG_NUMA
2742void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
2743{
2b847c3c 2744 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
5b882be4 2745
ca2b84cb 2746 trace_kmem_cache_alloc_node(_RET_IP_, ret,
3b0efdfa 2747 s->object_size, s->size, gfpflags, node);
5b882be4
EGM
2748
2749 return ret;
81819f0f
CL
2750}
2751EXPORT_SYMBOL(kmem_cache_alloc_node);
81819f0f 2752
0f24f128 2753#ifdef CONFIG_TRACING
4a92379b 2754void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
5b882be4 2755 gfp_t gfpflags,
4a92379b 2756 int node, size_t size)
5b882be4 2757{
2b847c3c 2758 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
4a92379b
RK
2759
2760 trace_kmalloc_node(_RET_IP_, ret,
2761 size, s->size, gfpflags, node);
0316bec2 2762
505f5dcb 2763 kasan_kmalloc(s, ret, size, gfpflags);
4a92379b 2764 return ret;
5b882be4 2765}
4a92379b 2766EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
5b882be4 2767#endif
5d1f57e4 2768#endif
5b882be4 2769
81819f0f 2770/*
94e4d712 2771 * Slow path handling. This may still be called frequently since objects
894b8788 2772 * have a longer lifetime than the cpu slabs in most processing loads.
81819f0f 2773 *
894b8788
CL
2774 * So we still attempt to reduce cache line usage. Just take the slab
2775 * lock and free the item. If there is no additional partial page
2776 * handling required then we can return immediately.
81819f0f 2777 */
894b8788 2778static void __slab_free(struct kmem_cache *s, struct page *page,
81084651
JDB
2779 void *head, void *tail, int cnt,
2780 unsigned long addr)
2781
81819f0f
CL
2782{
2783 void *prior;
2cfb7455 2784 int was_frozen;
2cfb7455
CL
2785 struct page new;
2786 unsigned long counters;
2787 struct kmem_cache_node *n = NULL;
61728d1e 2788 unsigned long uninitialized_var(flags);
81819f0f 2789
8a5ec0ba 2790 stat(s, FREE_SLOWPATH);
81819f0f 2791
19c7ff9e 2792 if (kmem_cache_debug(s) &&
282acb43 2793 !free_debug_processing(s, page, head, tail, cnt, addr))
80f08c19 2794 return;
6446faa2 2795
2cfb7455 2796 do {
837d678d
JK
2797 if (unlikely(n)) {
2798 spin_unlock_irqrestore(&n->list_lock, flags);
2799 n = NULL;
2800 }
2cfb7455
CL
2801 prior = page->freelist;
2802 counters = page->counters;
81084651 2803 set_freepointer(s, tail, prior);
2cfb7455
CL
2804 new.counters = counters;
2805 was_frozen = new.frozen;
81084651 2806 new.inuse -= cnt;
837d678d 2807 if ((!new.inuse || !prior) && !was_frozen) {
49e22585 2808
c65c1877 2809 if (kmem_cache_has_cpu_partial(s) && !prior) {
49e22585
CL
2810
2811 /*
d0e0ac97
CG
2812 * Slab was on no list before and will be
2813 * partially empty
2814 * We can defer the list move and instead
2815 * freeze it.
49e22585
CL
2816 */
2817 new.frozen = 1;
2818
c65c1877 2819 } else { /* Needs to be taken off a list */
49e22585 2820
b455def2 2821 n = get_node(s, page_to_nid(page));
49e22585
CL
2822 /*
2823 * Speculatively acquire the list_lock.
2824 * If the cmpxchg does not succeed then we may
2825 * drop the list_lock without any processing.
2826 *
2827 * Otherwise the list_lock will synchronize with
2828 * other processors updating the list of slabs.
2829 */
2830 spin_lock_irqsave(&n->list_lock, flags);
2831
2832 }
2cfb7455 2833 }
81819f0f 2834
2cfb7455
CL
2835 } while (!cmpxchg_double_slab(s, page,
2836 prior, counters,
81084651 2837 head, new.counters,
2cfb7455 2838 "__slab_free"));
81819f0f 2839
2cfb7455 2840 if (likely(!n)) {
49e22585
CL
2841
2842 /*
2843 * If we just froze the page then put it onto the
2844 * per cpu partial list.
2845 */
8028dcea 2846 if (new.frozen && !was_frozen) {
49e22585 2847 put_cpu_partial(s, page, 1);
8028dcea
AS
2848 stat(s, CPU_PARTIAL_FREE);
2849 }
49e22585 2850 /*
2cfb7455
CL
2851 * The list lock was not taken therefore no list
2852 * activity can be necessary.
2853 */
b455def2
L
2854 if (was_frozen)
2855 stat(s, FREE_FROZEN);
2856 return;
2857 }
81819f0f 2858
8a5b20ae 2859 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
837d678d
JK
2860 goto slab_empty;
2861
81819f0f 2862 /*
837d678d
JK
2863 * Objects left in the slab. If it was not on the partial list before
2864 * then add it.
81819f0f 2865 */
345c905d
JK
2866 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
2867 if (kmem_cache_debug(s))
c65c1877 2868 remove_full(s, n, page);
837d678d
JK
2869 add_partial(n, page, DEACTIVATE_TO_TAIL);
2870 stat(s, FREE_ADD_PARTIAL);
8ff12cfc 2871 }
80f08c19 2872 spin_unlock_irqrestore(&n->list_lock, flags);
81819f0f
CL
2873 return;
2874
2875slab_empty:
a973e9dd 2876 if (prior) {
81819f0f 2877 /*
6fbabb20 2878 * Slab on the partial list.
81819f0f 2879 */
5cc6eee8 2880 remove_partial(n, page);
84e554e6 2881 stat(s, FREE_REMOVE_PARTIAL);
c65c1877 2882 } else {
6fbabb20 2883 /* Slab must be on the full list */
c65c1877
PZ
2884 remove_full(s, n, page);
2885 }
2cfb7455 2886
80f08c19 2887 spin_unlock_irqrestore(&n->list_lock, flags);
84e554e6 2888 stat(s, FREE_SLAB);
81819f0f 2889 discard_slab(s, page);
81819f0f
CL
2890}
2891
894b8788
CL
2892/*
2893 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
2894 * can perform fastpath freeing without additional function calls.
2895 *
2896 * The fastpath is only possible if we are freeing to the current cpu slab
2897 * of this processor. This typically the case if we have just allocated
2898 * the item before.
2899 *
2900 * If fastpath is not possible then fall back to __slab_free where we deal
2901 * with all sorts of special processing.
81084651
JDB
2902 *
2903 * Bulk free of a freelist with several objects (all pointing to the
2904 * same page) possible by specifying head and tail ptr, plus objects
2905 * count (cnt). Bulk free indicated by tail pointer being set.
894b8788 2906 */
80a9201a
AP
2907static __always_inline void do_slab_free(struct kmem_cache *s,
2908 struct page *page, void *head, void *tail,
2909 int cnt, unsigned long addr)
894b8788 2910{
81084651 2911 void *tail_obj = tail ? : head;
dfb4f096 2912 struct kmem_cache_cpu *c;
8a5ec0ba 2913 unsigned long tid;
8a5ec0ba
CL
2914redo:
2915 /*
2916 * Determine the currently cpus per cpu slab.
2917 * The cpu may change afterward. However that does not matter since
2918 * data is retrieved via this pointer. If we are on the same cpu
2ae44005 2919 * during the cmpxchg then the free will succeed.
8a5ec0ba 2920 */
9aabf810
JK
2921 do {
2922 tid = this_cpu_read(s->cpu_slab->tid);
2923 c = raw_cpu_ptr(s->cpu_slab);
859b7a0e
MR
2924 } while (IS_ENABLED(CONFIG_PREEMPT) &&
2925 unlikely(tid != READ_ONCE(c->tid)));
c016b0bd 2926
9aabf810
JK
2927 /* Same with comment on barrier() in slab_alloc_node() */
2928 barrier();
c016b0bd 2929
442b06bc 2930 if (likely(page == c->page)) {
81084651 2931 set_freepointer(s, tail_obj, c->freelist);
8a5ec0ba 2932
933393f5 2933 if (unlikely(!this_cpu_cmpxchg_double(
8a5ec0ba
CL
2934 s->cpu_slab->freelist, s->cpu_slab->tid,
2935 c->freelist, tid,
81084651 2936 head, next_tid(tid)))) {
8a5ec0ba
CL
2937
2938 note_cmpxchg_failure("slab_free", s, tid);
2939 goto redo;
2940 }
84e554e6 2941 stat(s, FREE_FASTPATH);
894b8788 2942 } else
81084651 2943 __slab_free(s, page, head, tail_obj, cnt, addr);
894b8788 2944
894b8788
CL
2945}
2946
80a9201a
AP
2947static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
2948 void *head, void *tail, int cnt,
2949 unsigned long addr)
2950{
2951 slab_free_freelist_hook(s, head, tail);
2952 /*
2953 * slab_free_freelist_hook() could have put the items into quarantine.
2954 * If so, no need to free them.
2955 */
2956 if (s->flags & SLAB_KASAN && !(s->flags & SLAB_DESTROY_BY_RCU))
2957 return;
2958 do_slab_free(s, page, head, tail, cnt, addr);
2959}
2960
2961#ifdef CONFIG_KASAN
2962void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
2963{
2964 do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
2965}
2966#endif
2967
81819f0f
CL
2968void kmem_cache_free(struct kmem_cache *s, void *x)
2969{
b9ce5ef4
GC
2970 s = cache_from_obj(s, x);
2971 if (!s)
79576102 2972 return;
81084651 2973 slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
ca2b84cb 2974 trace_kmem_cache_free(_RET_IP_, x);
81819f0f
CL
2975}
2976EXPORT_SYMBOL(kmem_cache_free);
2977
d0ecd894 2978struct detached_freelist {
fbd02630 2979 struct page *page;
d0ecd894
JDB
2980 void *tail;
2981 void *freelist;
2982 int cnt;
376bf125 2983 struct kmem_cache *s;
d0ecd894 2984};
fbd02630 2985
d0ecd894
JDB
2986/*
2987 * This function progressively scans the array with free objects (with
2988 * a limited look ahead) and extract objects belonging to the same
2989 * page. It builds a detached freelist directly within the given
2990 * page/objects. This can happen without any need for
2991 * synchronization, because the objects are owned by running process.
2992 * The freelist is build up as a single linked list in the objects.
2993 * The idea is, that this detached freelist can then be bulk
2994 * transferred to the real freelist(s), but only requiring a single
2995 * synchronization primitive. Look ahead in the array is limited due
2996 * to performance reasons.
2997 */
376bf125
JDB
2998static inline
2999int build_detached_freelist(struct kmem_cache *s, size_t size,
3000 void **p, struct detached_freelist *df)
d0ecd894
JDB
3001{
3002 size_t first_skipped_index = 0;
3003 int lookahead = 3;
3004 void *object;
ca257195 3005 struct page *page;
fbd02630 3006
d0ecd894
JDB
3007 /* Always re-init detached_freelist */
3008 df->page = NULL;
fbd02630 3009
d0ecd894
JDB
3010 do {
3011 object = p[--size];
ca257195 3012 /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
d0ecd894 3013 } while (!object && size);
3eed034d 3014
d0ecd894
JDB
3015 if (!object)
3016 return 0;
fbd02630 3017
ca257195
JDB
3018 page = virt_to_head_page(object);
3019 if (!s) {
3020 /* Handle kalloc'ed objects */
3021 if (unlikely(!PageSlab(page))) {
3022 BUG_ON(!PageCompound(page));
3023 kfree_hook(object);
4949148a 3024 __free_pages(page, compound_order(page));
ca257195
JDB
3025 p[size] = NULL; /* mark object processed */
3026 return size;
3027 }
3028 /* Derive kmem_cache from object */
3029 df->s = page->slab_cache;
3030 } else {
3031 df->s = cache_from_obj(s, object); /* Support for memcg */
3032 }
376bf125 3033
d0ecd894 3034 /* Start new detached freelist */
ca257195 3035 df->page = page;
376bf125 3036 set_freepointer(df->s, object, NULL);
d0ecd894
JDB
3037 df->tail = object;
3038 df->freelist = object;
3039 p[size] = NULL; /* mark object processed */
3040 df->cnt = 1;
3041
3042 while (size) {
3043 object = p[--size];
3044 if (!object)
3045 continue; /* Skip processed objects */
3046
3047 /* df->page is always set at this point */
3048 if (df->page == virt_to_head_page(object)) {
3049 /* Opportunity build freelist */
376bf125 3050 set_freepointer(df->s, object, df->freelist);
d0ecd894
JDB
3051 df->freelist = object;
3052 df->cnt++;
3053 p[size] = NULL; /* mark object processed */
3054
3055 continue;
fbd02630 3056 }
d0ecd894
JDB
3057
3058 /* Limit look ahead search */
3059 if (!--lookahead)
3060 break;
3061
3062 if (!first_skipped_index)
3063 first_skipped_index = size + 1;
fbd02630 3064 }
d0ecd894
JDB
3065
3066 return first_skipped_index;
3067}
3068
d0ecd894 3069/* Note that interrupts must be enabled when calling this function. */
376bf125 3070void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
d0ecd894
JDB
3071{
3072 if (WARN_ON(!size))
3073 return;
3074
3075 do {
3076 struct detached_freelist df;
3077
3078 size = build_detached_freelist(s, size, p, &df);
3079 if (unlikely(!df.page))
3080 continue;
3081
376bf125 3082 slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
d0ecd894 3083 } while (likely(size));
484748f0
CL
3084}
3085EXPORT_SYMBOL(kmem_cache_free_bulk);
3086
994eb764 3087/* Note that interrupts must be enabled when calling this function. */
865762a8
JDB
3088int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3089 void **p)
484748f0 3090{
994eb764
JDB
3091 struct kmem_cache_cpu *c;
3092 int i;
3093
03ec0ed5
JDB
3094 /* memcg and kmem_cache debug support */
3095 s = slab_pre_alloc_hook(s, flags);
3096 if (unlikely(!s))
3097 return false;
994eb764
JDB
3098 /*
3099 * Drain objects in the per cpu slab, while disabling local
3100 * IRQs, which protects against PREEMPT and interrupts
3101 * handlers invoking normal fastpath.
3102 */
3103 local_irq_disable();
3104 c = this_cpu_ptr(s->cpu_slab);
3105
3106 for (i = 0; i < size; i++) {
3107 void *object = c->freelist;
3108
ebe909e0 3109 if (unlikely(!object)) {
ebe909e0
JDB
3110 /*
3111 * Invoking slow path likely have side-effect
3112 * of re-populating per CPU c->freelist
3113 */
87098373 3114 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
ebe909e0 3115 _RET_IP_, c);
87098373
CL
3116 if (unlikely(!p[i]))
3117 goto error;
3118
ebe909e0
JDB
3119 c = this_cpu_ptr(s->cpu_slab);
3120 continue; /* goto for-loop */
3121 }
994eb764
JDB
3122 c->freelist = get_freepointer(s, object);
3123 p[i] = object;
3124 }
3125 c->tid = next_tid(c->tid);
3126 local_irq_enable();
3127
3128 /* Clear memory outside IRQ disabled fastpath loop */
3129 if (unlikely(flags & __GFP_ZERO)) {
3130 int j;
3131
3132 for (j = 0; j < i; j++)
3133 memset(p[j], 0, s->object_size);
3134 }
3135
03ec0ed5
JDB
3136 /* memcg and kmem_cache debug support */
3137 slab_post_alloc_hook(s, flags, size, p);
865762a8 3138 return i;
87098373 3139error:
87098373 3140 local_irq_enable();
03ec0ed5
JDB
3141 slab_post_alloc_hook(s, flags, i, p);
3142 __kmem_cache_free_bulk(s, i, p);
865762a8 3143 return 0;
484748f0
CL
3144}
3145EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3146
3147
81819f0f 3148/*
672bba3a
CL
3149 * Object placement in a slab is made very easy because we always start at
3150 * offset 0. If we tune the size of the object to the alignment then we can
3151 * get the required alignment by putting one properly sized object after
3152 * another.
81819f0f
CL
3153 *
3154 * Notice that the allocation order determines the sizes of the per cpu
3155 * caches. Each processor has always one slab available for allocations.
3156 * Increasing the allocation order reduces the number of times that slabs
672bba3a 3157 * must be moved on and off the partial lists and is therefore a factor in
81819f0f 3158 * locking overhead.
81819f0f
CL
3159 */
3160
3161/*
3162 * Mininum / Maximum order of slab pages. This influences locking overhead
3163 * and slab fragmentation. A higher order reduces the number of partial slabs
3164 * and increases the number of allocations possible without having to
3165 * take the list_lock.
3166 */
3167static int slub_min_order;
114e9e89 3168static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
9b2cd506 3169static int slub_min_objects;
81819f0f 3170
81819f0f
CL
3171/*
3172 * Calculate the order of allocation given an slab object size.
3173 *
672bba3a
CL
3174 * The order of allocation has significant impact on performance and other
3175 * system components. Generally order 0 allocations should be preferred since
3176 * order 0 does not cause fragmentation in the page allocator. Larger objects
3177 * be problematic to put into order 0 slabs because there may be too much
c124f5b5 3178 * unused space left. We go to a higher order if more than 1/16th of the slab
672bba3a
CL
3179 * would be wasted.
3180 *
3181 * In order to reach satisfactory performance we must ensure that a minimum
3182 * number of objects is in one slab. Otherwise we may generate too much
3183 * activity on the partial lists which requires taking the list_lock. This is
3184 * less a concern for large slabs though which are rarely used.
81819f0f 3185 *
672bba3a
CL
3186 * slub_max_order specifies the order where we begin to stop considering the
3187 * number of objects in a slab as critical. If we reach slub_max_order then
3188 * we try to keep the page order as low as possible. So we accept more waste
3189 * of space in favor of a small page order.
81819f0f 3190 *
672bba3a
CL
3191 * Higher order allocations also allow the placement of more objects in a
3192 * slab and thereby reduce object handling overhead. If the user has
3193 * requested a higher mininum order then we start with that one instead of
3194 * the smallest order which will fit the object.
81819f0f 3195 */
5e6d444e 3196static inline int slab_order(int size, int min_objects,
ab9a0f19 3197 int max_order, int fract_leftover, int reserved)
81819f0f
CL
3198{
3199 int order;
3200 int rem;
6300ea75 3201 int min_order = slub_min_order;
81819f0f 3202
ab9a0f19 3203 if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
210b5c06 3204 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
39b26464 3205
9f835703 3206 for (order = max(min_order, get_order(min_objects * size + reserved));
5e6d444e 3207 order <= max_order; order++) {
81819f0f 3208
5e6d444e 3209 unsigned long slab_size = PAGE_SIZE << order;
81819f0f 3210
ab9a0f19 3211 rem = (slab_size - reserved) % size;
81819f0f 3212
5e6d444e 3213 if (rem <= slab_size / fract_leftover)
81819f0f 3214 break;
81819f0f 3215 }
672bba3a 3216
81819f0f
CL
3217 return order;
3218}
3219
ab9a0f19 3220static inline int calculate_order(int size, int reserved)
5e6d444e
CL
3221{
3222 int order;
3223 int min_objects;
3224 int fraction;
e8120ff1 3225 int max_objects;
5e6d444e
CL
3226
3227 /*
3228 * Attempt to find best configuration for a slab. This
3229 * works by first attempting to generate a layout with
3230 * the best configuration and backing off gradually.
3231 *
422ff4d7 3232 * First we increase the acceptable waste in a slab. Then
5e6d444e
CL
3233 * we reduce the minimum objects required in a slab.
3234 */
3235 min_objects = slub_min_objects;
9b2cd506
CL
3236 if (!min_objects)
3237 min_objects = 4 * (fls(nr_cpu_ids) + 1);
ab9a0f19 3238 max_objects = order_objects(slub_max_order, size, reserved);
e8120ff1
ZY
3239 min_objects = min(min_objects, max_objects);
3240
5e6d444e 3241 while (min_objects > 1) {
c124f5b5 3242 fraction = 16;
5e6d444e
CL
3243 while (fraction >= 4) {
3244 order = slab_order(size, min_objects,
ab9a0f19 3245 slub_max_order, fraction, reserved);
5e6d444e
CL
3246 if (order <= slub_max_order)
3247 return order;
3248 fraction /= 2;
3249 }
5086c389 3250 min_objects--;
5e6d444e
CL
3251 }
3252
3253 /*
3254 * We were unable to place multiple objects in a slab. Now
3255 * lets see if we can place a single object there.
3256 */
ab9a0f19 3257 order = slab_order(size, 1, slub_max_order, 1, reserved);
5e6d444e
CL
3258 if (order <= slub_max_order)
3259 return order;
3260
3261 /*
3262 * Doh this slab cannot be placed using slub_max_order.
3263 */
ab9a0f19 3264 order = slab_order(size, 1, MAX_ORDER, 1, reserved);
818cf590 3265 if (order < MAX_ORDER)
5e6d444e
CL
3266 return order;
3267 return -ENOSYS;
3268}
3269
5595cffc 3270static void
4053497d 3271init_kmem_cache_node(struct kmem_cache_node *n)
81819f0f
CL
3272{
3273 n->nr_partial = 0;
81819f0f
CL
3274 spin_lock_init(&n->list_lock);
3275 INIT_LIST_HEAD(&n->partial);
8ab1372f 3276#ifdef CONFIG_SLUB_DEBUG
0f389ec6 3277 atomic_long_set(&n->nr_slabs, 0);
02b71b70 3278 atomic_long_set(&n->total_objects, 0);
643b1138 3279 INIT_LIST_HEAD(&n->full);
8ab1372f 3280#endif
81819f0f
CL
3281}
3282
55136592 3283static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4c93c355 3284{
6c182dc0 3285 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
95a05b42 3286 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
4c93c355 3287
8a5ec0ba 3288 /*
d4d84fef
CM
3289 * Must align to double word boundary for the double cmpxchg
3290 * instructions to work; see __pcpu_double_call_return_bool().
8a5ec0ba 3291 */
d4d84fef
CM
3292 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
3293 2 * sizeof(void *));
8a5ec0ba
CL
3294
3295 if (!s->cpu_slab)
3296 return 0;
3297
3298 init_kmem_cache_cpus(s);
4c93c355 3299
8a5ec0ba 3300 return 1;
4c93c355 3301}
4c93c355 3302
51df1142
CL
3303static struct kmem_cache *kmem_cache_node;
3304
81819f0f
CL
3305/*
3306 * No kmalloc_node yet so do it by hand. We know that this is the first
3307 * slab on the node for this slabcache. There are no concurrent accesses
3308 * possible.
3309 *
721ae22a
ZYW
3310 * Note that this function only works on the kmem_cache_node
3311 * when allocating for the kmem_cache_node. This is used for bootstrapping
4c93c355 3312 * memory on a fresh node that has no slab structures yet.
81819f0f 3313 */
55136592 3314static void early_kmem_cache_node_alloc(int node)
81819f0f
CL
3315{
3316 struct page *page;
3317 struct kmem_cache_node *n;
3318
51df1142 3319 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
81819f0f 3320
51df1142 3321 page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
81819f0f
CL
3322
3323 BUG_ON(!page);
a2f92ee7 3324 if (page_to_nid(page) != node) {
f9f58285
FF
3325 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
3326 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
a2f92ee7
CL
3327 }
3328
81819f0f
CL
3329 n = page->freelist;
3330 BUG_ON(!n);
51df1142 3331 page->freelist = get_freepointer(kmem_cache_node, n);
e6e82ea1 3332 page->inuse = 1;
8cb0a506 3333 page->frozen = 0;
51df1142 3334 kmem_cache_node->node[node] = n;
8ab1372f 3335#ifdef CONFIG_SLUB_DEBUG
f7cb1933 3336 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
51df1142 3337 init_tracking(kmem_cache_node, n);
8ab1372f 3338#endif
505f5dcb
AP
3339 kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
3340 GFP_KERNEL);
4053497d 3341 init_kmem_cache_node(n);
51df1142 3342 inc_slabs_node(kmem_cache_node, node, page->objects);
6446faa2 3343
67b6c900 3344 /*
1e4dd946
SR
3345 * No locks need to be taken here as it has just been
3346 * initialized and there is no concurrent access.
67b6c900 3347 */
1e4dd946 3348 __add_partial(n, page, DEACTIVATE_TO_HEAD);
81819f0f
CL
3349}
3350
3351static void free_kmem_cache_nodes(struct kmem_cache *s)
3352{
3353 int node;
fa45dc25 3354 struct kmem_cache_node *n;
81819f0f 3355
fa45dc25
CL
3356 for_each_kmem_cache_node(s, node, n) {
3357 kmem_cache_free(kmem_cache_node, n);
81819f0f
CL
3358 s->node[node] = NULL;
3359 }
3360}
3361
52b4b950
DS
3362void __kmem_cache_release(struct kmem_cache *s)
3363{
210e7a43 3364 cache_random_seq_destroy(s);
52b4b950
DS
3365 free_percpu(s->cpu_slab);
3366 free_kmem_cache_nodes(s);
3367}
3368
55136592 3369static int init_kmem_cache_nodes(struct kmem_cache *s)
81819f0f
CL
3370{
3371 int node;
81819f0f 3372
f64dc58c 3373 for_each_node_state(node, N_NORMAL_MEMORY) {
81819f0f
CL
3374 struct kmem_cache_node *n;
3375
73367bd8 3376 if (slab_state == DOWN) {
55136592 3377 early_kmem_cache_node_alloc(node);
73367bd8
AD
3378 continue;
3379 }
51df1142 3380 n = kmem_cache_alloc_node(kmem_cache_node,
55136592 3381 GFP_KERNEL, node);
81819f0f 3382
73367bd8
AD
3383 if (!n) {
3384 free_kmem_cache_nodes(s);
3385 return 0;
81819f0f 3386 }
73367bd8 3387
81819f0f 3388 s->node[node] = n;
4053497d 3389 init_kmem_cache_node(n);
81819f0f
CL
3390 }
3391 return 1;
3392}
81819f0f 3393
c0bdb232 3394static void set_min_partial(struct kmem_cache *s, unsigned long min)
3b89d7d8
DR
3395{
3396 if (min < MIN_PARTIAL)
3397 min = MIN_PARTIAL;
3398 else if (min > MAX_PARTIAL)
3399 min = MAX_PARTIAL;
3400 s->min_partial = min;
3401}
3402
81819f0f
CL
3403/*
3404 * calculate_sizes() determines the order and the distribution of data within
3405 * a slab object.
3406 */
06b285dc 3407static int calculate_sizes(struct kmem_cache *s, int forced_order)
81819f0f
CL
3408{
3409 unsigned long flags = s->flags;
80a9201a 3410 size_t size = s->object_size;
834f3d11 3411 int order;
81819f0f 3412
d8b42bf5
CL
3413 /*
3414 * Round up object size to the next word boundary. We can only
3415 * place the free pointer at word boundaries and this determines
3416 * the possible location of the free pointer.
3417 */
3418 size = ALIGN(size, sizeof(void *));
3419
3420#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
3421 /*
3422 * Determine if we can poison the object itself. If the user of
3423 * the slab may touch the object after free or before allocation
3424 * then we should never poison the object itself.
3425 */
3426 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
c59def9f 3427 !s->ctor)
81819f0f
CL
3428 s->flags |= __OBJECT_POISON;
3429 else
3430 s->flags &= ~__OBJECT_POISON;
3431
81819f0f
CL
3432
3433 /*
672bba3a 3434 * If we are Redzoning then check if there is some space between the
81819f0f 3435 * end of the object and the free pointer. If not then add an
672bba3a 3436 * additional word to have some bytes to store Redzone information.
81819f0f 3437 */
3b0efdfa 3438 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
81819f0f 3439 size += sizeof(void *);
41ecc55b 3440#endif
81819f0f
CL
3441
3442 /*
672bba3a
CL
3443 * With that we have determined the number of bytes in actual use
3444 * by the object. This is the potential offset to the free pointer.
81819f0f
CL
3445 */
3446 s->inuse = size;
3447
3448 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
c59def9f 3449 s->ctor)) {
81819f0f
CL
3450 /*
3451 * Relocate free pointer after the object if it is not
3452 * permitted to overwrite the first word of the object on
3453 * kmem_cache_free.
3454 *
3455 * This is the case if we do RCU, have a constructor or
3456 * destructor or are poisoning the objects.
3457 */
3458 s->offset = size;
3459 size += sizeof(void *);
3460 }
3461
c12b3c62 3462#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
3463 if (flags & SLAB_STORE_USER)
3464 /*
3465 * Need to store information about allocs and frees after
3466 * the object.
3467 */
3468 size += 2 * sizeof(struct track);
80a9201a 3469#endif
81819f0f 3470
80a9201a
AP
3471 kasan_cache_create(s, &size, &s->flags);
3472#ifdef CONFIG_SLUB_DEBUG
d86bd1be 3473 if (flags & SLAB_RED_ZONE) {
81819f0f
CL
3474 /*
3475 * Add some empty padding so that we can catch
3476 * overwrites from earlier objects rather than let
3477 * tracking information or the free pointer be
0211a9c8 3478 * corrupted if a user writes before the start
81819f0f
CL
3479 * of the object.
3480 */
3481 size += sizeof(void *);
d86bd1be
JK
3482
3483 s->red_left_pad = sizeof(void *);
3484 s->red_left_pad = ALIGN(s->red_left_pad, s->align);
3485 size += s->red_left_pad;
3486 }
41ecc55b 3487#endif
672bba3a 3488
81819f0f
CL
3489 /*
3490 * SLUB stores one object immediately after another beginning from
3491 * offset 0. In order to align the objects we have to simply size
3492 * each object to conform to the alignment.
3493 */
45906855 3494 size = ALIGN(size, s->align);
81819f0f 3495 s->size = size;
06b285dc
CL
3496 if (forced_order >= 0)
3497 order = forced_order;
3498 else
ab9a0f19 3499 order = calculate_order(size, s->reserved);
81819f0f 3500
834f3d11 3501 if (order < 0)
81819f0f
CL
3502 return 0;
3503
b7a49f0d 3504 s->allocflags = 0;
834f3d11 3505 if (order)
b7a49f0d
CL
3506 s->allocflags |= __GFP_COMP;
3507
3508 if (s->flags & SLAB_CACHE_DMA)
2c59dd65 3509 s->allocflags |= GFP_DMA;
b7a49f0d
CL
3510
3511 if (s->flags & SLAB_RECLAIM_ACCOUNT)
3512 s->allocflags |= __GFP_RECLAIMABLE;
3513
81819f0f
CL
3514 /*
3515 * Determine the number of objects per slab
3516 */
ab9a0f19
LJ
3517 s->oo = oo_make(order, size, s->reserved);
3518 s->min = oo_make(get_order(size), size, s->reserved);
205ab99d
CL
3519 if (oo_objects(s->oo) > oo_objects(s->max))
3520 s->max = s->oo;
81819f0f 3521
834f3d11 3522 return !!oo_objects(s->oo);
81819f0f
CL
3523}
3524
8a13a4cc 3525static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
81819f0f 3526{
8a13a4cc 3527 s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
ab9a0f19 3528 s->reserved = 0;
81819f0f 3529
da9a638c
LJ
3530 if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
3531 s->reserved = sizeof(struct rcu_head);
81819f0f 3532
06b285dc 3533 if (!calculate_sizes(s, -1))
81819f0f 3534 goto error;
3de47213
DR
3535 if (disable_higher_order_debug) {
3536 /*
3537 * Disable debugging flags that store metadata if the min slab
3538 * order increased.
3539 */
3b0efdfa 3540 if (get_order(s->size) > get_order(s->object_size)) {
3de47213
DR
3541 s->flags &= ~DEBUG_METADATA_FLAGS;
3542 s->offset = 0;
3543 if (!calculate_sizes(s, -1))
3544 goto error;
3545 }
3546 }
81819f0f 3547
2565409f
HC
3548#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3549 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
149daaf3 3550 if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
b789ef51
CL
3551 /* Enable fast mode */
3552 s->flags |= __CMPXCHG_DOUBLE;
3553#endif
3554
3b89d7d8
DR
3555 /*
3556 * The larger the object size is, the more pages we want on the partial
3557 * list to avoid pounding the page allocator excessively.
3558 */
49e22585
CL
3559 set_min_partial(s, ilog2(s->size) / 2);
3560
3561 /*
3562 * cpu_partial determined the maximum number of objects kept in the
3563 * per cpu partial lists of a processor.
3564 *
3565 * Per cpu partial lists mainly contain slabs that just have one
3566 * object freed. If they are used for allocation then they can be
3567 * filled up again with minimal effort. The slab will never hit the
3568 * per node partial lists and therefore no locking will be required.
3569 *
3570 * This setting also determines
3571 *
3572 * A) The number of objects from per cpu partial slabs dumped to the
3573 * per node list when we reach the limit.
9f264904 3574 * B) The number of objects in cpu partial slabs to extract from the
d0e0ac97
CG
3575 * per node list when we run out of per cpu objects. We only fetch
3576 * 50% to keep some capacity around for frees.
49e22585 3577 */
345c905d 3578 if (!kmem_cache_has_cpu_partial(s))
8f1e33da
CL
3579 s->cpu_partial = 0;
3580 else if (s->size >= PAGE_SIZE)
49e22585
CL
3581 s->cpu_partial = 2;
3582 else if (s->size >= 1024)
3583 s->cpu_partial = 6;
3584 else if (s->size >= 256)
3585 s->cpu_partial = 13;
3586 else
3587 s->cpu_partial = 30;
3588
81819f0f 3589#ifdef CONFIG_NUMA
e2cb96b7 3590 s->remote_node_defrag_ratio = 1000;
81819f0f 3591#endif
210e7a43
TG
3592
3593 /* Initialize the pre-computed randomized freelist if slab is up */
3594 if (slab_state >= UP) {
3595 if (init_cache_random_seq(s))
3596 goto error;
3597 }
3598
55136592 3599 if (!init_kmem_cache_nodes(s))
dfb4f096 3600 goto error;
81819f0f 3601
55136592 3602 if (alloc_kmem_cache_cpus(s))
278b1bb1 3603 return 0;
ff12059e 3604
4c93c355 3605 free_kmem_cache_nodes(s);
81819f0f
CL
3606error:
3607 if (flags & SLAB_PANIC)
756a025f
JP
3608 panic("Cannot create slab %s size=%lu realsize=%u order=%u offset=%u flags=%lx\n",
3609 s->name, (unsigned long)s->size, s->size,
3610 oo_order(s->oo), s->offset, flags);
278b1bb1 3611 return -EINVAL;
81819f0f 3612}
81819f0f 3613
33b12c38
CL
3614static void list_slab_objects(struct kmem_cache *s, struct page *page,
3615 const char *text)
3616{
3617#ifdef CONFIG_SLUB_DEBUG
3618 void *addr = page_address(page);
3619 void *p;
a5dd5c11
NK
3620 unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
3621 sizeof(long), GFP_ATOMIC);
bbd7d57b
ED
3622 if (!map)
3623 return;
945cf2b6 3624 slab_err(s, page, text, s->name);
33b12c38 3625 slab_lock(page);
33b12c38 3626
5f80b13a 3627 get_map(s, page, map);
33b12c38
CL
3628 for_each_object(p, s, addr, page->objects) {
3629
3630 if (!test_bit(slab_index(p, s, addr), map)) {
f9f58285 3631 pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
33b12c38
CL
3632 print_tracking(s, p);
3633 }
3634 }
3635 slab_unlock(page);
bbd7d57b 3636 kfree(map);
33b12c38
CL
3637#endif
3638}
3639
81819f0f 3640/*
599870b1 3641 * Attempt to free all partial slabs on a node.
52b4b950
DS
3642 * This is called from __kmem_cache_shutdown(). We must take list_lock
3643 * because sysfs file might still access partial list after the shutdowning.
81819f0f 3644 */
599870b1 3645static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
81819f0f 3646{
60398923 3647 LIST_HEAD(discard);
81819f0f
CL
3648 struct page *page, *h;
3649
52b4b950
DS
3650 BUG_ON(irqs_disabled());
3651 spin_lock_irq(&n->list_lock);
33b12c38 3652 list_for_each_entry_safe(page, h, &n->partial, lru) {
81819f0f 3653 if (!page->inuse) {
52b4b950 3654 remove_partial(n, page);
60398923 3655 list_add(&page->lru, &discard);
33b12c38
CL
3656 } else {
3657 list_slab_objects(s, page,
52b4b950 3658 "Objects remaining in %s on __kmem_cache_shutdown()");
599870b1 3659 }
33b12c38 3660 }
52b4b950 3661 spin_unlock_irq(&n->list_lock);
60398923
CW
3662
3663 list_for_each_entry_safe(page, h, &discard, lru)
3664 discard_slab(s, page);
81819f0f
CL
3665}
3666
3667/*
672bba3a 3668 * Release all resources used by a slab cache.
81819f0f 3669 */
52b4b950 3670int __kmem_cache_shutdown(struct kmem_cache *s)
81819f0f
CL
3671{
3672 int node;
fa45dc25 3673 struct kmem_cache_node *n;
81819f0f
CL
3674
3675 flush_all(s);
81819f0f 3676 /* Attempt to free all objects */
fa45dc25 3677 for_each_kmem_cache_node(s, node, n) {
599870b1
CL
3678 free_partial(s, n);
3679 if (n->nr_partial || slabs_node(s, node))
81819f0f
CL
3680 return 1;
3681 }
81819f0f
CL
3682 return 0;
3683}
3684
81819f0f
CL
3685/********************************************************************
3686 * Kmalloc subsystem
3687 *******************************************************************/
3688
81819f0f
CL
3689static int __init setup_slub_min_order(char *str)
3690{
06428780 3691 get_option(&str, &slub_min_order);
81819f0f
CL
3692
3693 return 1;
3694}
3695
3696__setup("slub_min_order=", setup_slub_min_order);
3697
3698static int __init setup_slub_max_order(char *str)
3699{
06428780 3700 get_option(&str, &slub_max_order);
818cf590 3701 slub_max_order = min(slub_max_order, MAX_ORDER - 1);
81819f0f
CL
3702
3703 return 1;
3704}
3705
3706__setup("slub_max_order=", setup_slub_max_order);
3707
3708static int __init setup_slub_min_objects(char *str)
3709{
06428780 3710 get_option(&str, &slub_min_objects);
81819f0f
CL
3711
3712 return 1;
3713}
3714
3715__setup("slub_min_objects=", setup_slub_min_objects);
3716
81819f0f
CL
3717void *__kmalloc(size_t size, gfp_t flags)
3718{
aadb4bc4 3719 struct kmem_cache *s;
5b882be4 3720 void *ret;
81819f0f 3721
95a05b42 3722 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
eada35ef 3723 return kmalloc_large(size, flags);
aadb4bc4 3724
2c59dd65 3725 s = kmalloc_slab(size, flags);
aadb4bc4
CL
3726
3727 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913
CL
3728 return s;
3729
2b847c3c 3730 ret = slab_alloc(s, flags, _RET_IP_);
5b882be4 3731
ca2b84cb 3732 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
5b882be4 3733
505f5dcb 3734 kasan_kmalloc(s, ret, size, flags);
0316bec2 3735
5b882be4 3736 return ret;
81819f0f
CL
3737}
3738EXPORT_SYMBOL(__kmalloc);
3739
5d1f57e4 3740#ifdef CONFIG_NUMA
f619cfe1
CL
3741static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3742{
b1eeab67 3743 struct page *page;
e4f7c0b4 3744 void *ptr = NULL;
f619cfe1 3745
52383431 3746 flags |= __GFP_COMP | __GFP_NOTRACK;
4949148a 3747 page = alloc_pages_node(node, flags, get_order(size));
f619cfe1 3748 if (page)
e4f7c0b4
CM
3749 ptr = page_address(page);
3750
d56791b3 3751 kmalloc_large_node_hook(ptr, size, flags);
e4f7c0b4 3752 return ptr;
f619cfe1
CL
3753}
3754
81819f0f
CL
3755void *__kmalloc_node(size_t size, gfp_t flags, int node)
3756{
aadb4bc4 3757 struct kmem_cache *s;
5b882be4 3758 void *ret;
81819f0f 3759
95a05b42 3760 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
5b882be4
EGM
3761 ret = kmalloc_large_node(size, flags, node);
3762
ca2b84cb
EGM
3763 trace_kmalloc_node(_RET_IP_, ret,
3764 size, PAGE_SIZE << get_order(size),
3765 flags, node);
5b882be4
EGM
3766
3767 return ret;
3768 }
aadb4bc4 3769
2c59dd65 3770 s = kmalloc_slab(size, flags);
aadb4bc4
CL
3771
3772 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913
CL
3773 return s;
3774
2b847c3c 3775 ret = slab_alloc_node(s, flags, node, _RET_IP_);
5b882be4 3776
ca2b84cb 3777 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
5b882be4 3778
505f5dcb 3779 kasan_kmalloc(s, ret, size, flags);
0316bec2 3780
5b882be4 3781 return ret;
81819f0f
CL
3782}
3783EXPORT_SYMBOL(__kmalloc_node);
3784#endif
3785
ed18adc1
KC
3786#ifdef CONFIG_HARDENED_USERCOPY
3787/*
3788 * Rejects objects that are incorrectly sized.
3789 *
3790 * Returns NULL if check passes, otherwise const char * to name of cache
3791 * to indicate an error.
3792 */
3793const char *__check_heap_object(const void *ptr, unsigned long n,
3794 struct page *page)
3795{
3796 struct kmem_cache *s;
3797 unsigned long offset;
3798 size_t object_size;
3799
3800 /* Find object and usable object size. */
3801 s = page->slab_cache;
3802 object_size = slab_ksize(s);
3803
3804 /* Reject impossible pointers. */
3805 if (ptr < page_address(page))
3806 return s->name;
3807
3808 /* Find offset within object. */
3809 offset = (ptr - page_address(page)) % s->size;
3810
3811 /* Adjust for redzone and reject if within the redzone. */
3812 if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE) {
3813 if (offset < s->red_left_pad)
3814 return s->name;
3815 offset -= s->red_left_pad;
3816 }
3817
3818 /* Allow address range falling entirely within object size. */
3819 if (offset <= object_size && n <= object_size - offset)
3820 return NULL;
3821
3822 return s->name;
3823}
3824#endif /* CONFIG_HARDENED_USERCOPY */
3825
0316bec2 3826static size_t __ksize(const void *object)
81819f0f 3827{
272c1d21 3828 struct page *page;
81819f0f 3829
ef8b4520 3830 if (unlikely(object == ZERO_SIZE_PTR))
272c1d21
CL
3831 return 0;
3832
294a80a8 3833 page = virt_to_head_page(object);
294a80a8 3834
76994412
PE
3835 if (unlikely(!PageSlab(page))) {
3836 WARN_ON(!PageCompound(page));
294a80a8 3837 return PAGE_SIZE << compound_order(page);
76994412 3838 }
81819f0f 3839
1b4f59e3 3840 return slab_ksize(page->slab_cache);
81819f0f 3841}
0316bec2
AR
3842
3843size_t ksize(const void *object)
3844{
3845 size_t size = __ksize(object);
3846 /* We assume that ksize callers could use whole allocated area,
4ebb31a4
AP
3847 * so we need to unpoison this area.
3848 */
3849 kasan_unpoison_shadow(object, size);
0316bec2
AR
3850 return size;
3851}
b1aabecd 3852EXPORT_SYMBOL(ksize);
81819f0f
CL
3853
3854void kfree(const void *x)
3855{
81819f0f 3856 struct page *page;
5bb983b0 3857 void *object = (void *)x;
81819f0f 3858
2121db74
PE
3859 trace_kfree(_RET_IP_, x);
3860
2408c550 3861 if (unlikely(ZERO_OR_NULL_PTR(x)))
81819f0f
CL
3862 return;
3863
b49af68f 3864 page = virt_to_head_page(x);
aadb4bc4 3865 if (unlikely(!PageSlab(page))) {
0937502a 3866 BUG_ON(!PageCompound(page));
d56791b3 3867 kfree_hook(x);
4949148a 3868 __free_pages(page, compound_order(page));
aadb4bc4
CL
3869 return;
3870 }
81084651 3871 slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
81819f0f
CL
3872}
3873EXPORT_SYMBOL(kfree);
3874
832f37f5
VD
3875#define SHRINK_PROMOTE_MAX 32
3876
2086d26a 3877/*
832f37f5
VD
3878 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
3879 * up most to the head of the partial lists. New allocations will then
3880 * fill those up and thus they can be removed from the partial lists.
672bba3a
CL
3881 *
3882 * The slabs with the least items are placed last. This results in them
3883 * being allocated from last increasing the chance that the last objects
3884 * are freed in them.
2086d26a 3885 */
d6e0b7fa 3886int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
2086d26a
CL
3887{
3888 int node;
3889 int i;
3890 struct kmem_cache_node *n;
3891 struct page *page;
3892 struct page *t;
832f37f5
VD
3893 struct list_head discard;
3894 struct list_head promote[SHRINK_PROMOTE_MAX];
2086d26a 3895 unsigned long flags;
ce3712d7 3896 int ret = 0;
2086d26a 3897
d6e0b7fa
VD
3898 if (deactivate) {
3899 /*
3900 * Disable empty slabs caching. Used to avoid pinning offline
3901 * memory cgroups by kmem pages that can be freed.
3902 */
3903 s->cpu_partial = 0;
3904 s->min_partial = 0;
3905
3906 /*
3907 * s->cpu_partial is checked locklessly (see put_cpu_partial),
3908 * so we have to make sure the change is visible.
3909 */
81ae6d03 3910 synchronize_sched();
d6e0b7fa
VD
3911 }
3912
2086d26a 3913 flush_all(s);
fa45dc25 3914 for_each_kmem_cache_node(s, node, n) {
832f37f5
VD
3915 INIT_LIST_HEAD(&discard);
3916 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
3917 INIT_LIST_HEAD(promote + i);
2086d26a
CL
3918
3919 spin_lock_irqsave(&n->list_lock, flags);
3920
3921 /*
832f37f5 3922 * Build lists of slabs to discard or promote.
2086d26a 3923 *
672bba3a
CL
3924 * Note that concurrent frees may occur while we hold the
3925 * list_lock. page->inuse here is the upper limit.
2086d26a
CL
3926 */
3927 list_for_each_entry_safe(page, t, &n->partial, lru) {
832f37f5
VD
3928 int free = page->objects - page->inuse;
3929
3930 /* Do not reread page->inuse */
3931 barrier();
3932
3933 /* We do not keep full slabs on the list */
3934 BUG_ON(free <= 0);
3935
3936 if (free == page->objects) {
3937 list_move(&page->lru, &discard);
69cb8e6b 3938 n->nr_partial--;
832f37f5
VD
3939 } else if (free <= SHRINK_PROMOTE_MAX)
3940 list_move(&page->lru, promote + free - 1);
2086d26a
CL
3941 }
3942
2086d26a 3943 /*
832f37f5
VD
3944 * Promote the slabs filled up most to the head of the
3945 * partial list.
2086d26a 3946 */
832f37f5
VD
3947 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
3948 list_splice(promote + i, &n->partial);
2086d26a 3949
2086d26a 3950 spin_unlock_irqrestore(&n->list_lock, flags);
69cb8e6b
CL
3951
3952 /* Release empty slabs */
832f37f5 3953 list_for_each_entry_safe(page, t, &discard, lru)
69cb8e6b 3954 discard_slab(s, page);
ce3712d7
VD
3955
3956 if (slabs_node(s, node))
3957 ret = 1;
2086d26a
CL
3958 }
3959
ce3712d7 3960 return ret;
2086d26a 3961}
2086d26a 3962
b9049e23
YG
3963static int slab_mem_going_offline_callback(void *arg)
3964{
3965 struct kmem_cache *s;
3966
18004c5d 3967 mutex_lock(&slab_mutex);
b9049e23 3968 list_for_each_entry(s, &slab_caches, list)
d6e0b7fa 3969 __kmem_cache_shrink(s, false);
18004c5d 3970 mutex_unlock(&slab_mutex);
b9049e23
YG
3971
3972 return 0;
3973}
3974
3975static void slab_mem_offline_callback(void *arg)
3976{
3977 struct kmem_cache_node *n;
3978 struct kmem_cache *s;
3979 struct memory_notify *marg = arg;
3980 int offline_node;
3981
b9d5ab25 3982 offline_node = marg->status_change_nid_normal;
b9049e23
YG
3983
3984 /*
3985 * If the node still has available memory. we need kmem_cache_node
3986 * for it yet.
3987 */
3988 if (offline_node < 0)
3989 return;
3990
18004c5d 3991 mutex_lock(&slab_mutex);
b9049e23
YG
3992 list_for_each_entry(s, &slab_caches, list) {
3993 n = get_node(s, offline_node);
3994 if (n) {
3995 /*
3996 * if n->nr_slabs > 0, slabs still exist on the node
3997 * that is going down. We were unable to free them,
c9404c9c 3998 * and offline_pages() function shouldn't call this
b9049e23
YG
3999 * callback. So, we must fail.
4000 */
0f389ec6 4001 BUG_ON(slabs_node(s, offline_node));
b9049e23
YG
4002
4003 s->node[offline_node] = NULL;
8de66a0c 4004 kmem_cache_free(kmem_cache_node, n);
b9049e23
YG
4005 }
4006 }
18004c5d 4007 mutex_unlock(&slab_mutex);
b9049e23
YG
4008}
4009
4010static int slab_mem_going_online_callback(void *arg)
4011{
4012 struct kmem_cache_node *n;
4013 struct kmem_cache *s;
4014 struct memory_notify *marg = arg;
b9d5ab25 4015 int nid = marg->status_change_nid_normal;
b9049e23
YG
4016 int ret = 0;
4017
4018 /*
4019 * If the node's memory is already available, then kmem_cache_node is
4020 * already created. Nothing to do.
4021 */
4022 if (nid < 0)
4023 return 0;
4024
4025 /*
0121c619 4026 * We are bringing a node online. No memory is available yet. We must
b9049e23
YG
4027 * allocate a kmem_cache_node structure in order to bring the node
4028 * online.
4029 */
18004c5d 4030 mutex_lock(&slab_mutex);
b9049e23
YG
4031 list_for_each_entry(s, &slab_caches, list) {
4032 /*
4033 * XXX: kmem_cache_alloc_node will fallback to other nodes
4034 * since memory is not yet available from the node that
4035 * is brought up.
4036 */
8de66a0c 4037 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
b9049e23
YG
4038 if (!n) {
4039 ret = -ENOMEM;
4040 goto out;
4041 }
4053497d 4042 init_kmem_cache_node(n);
b9049e23
YG
4043 s->node[nid] = n;
4044 }
4045out:
18004c5d 4046 mutex_unlock(&slab_mutex);
b9049e23
YG
4047 return ret;
4048}
4049
4050static int slab_memory_callback(struct notifier_block *self,
4051 unsigned long action, void *arg)
4052{
4053 int ret = 0;
4054
4055 switch (action) {
4056 case MEM_GOING_ONLINE:
4057 ret = slab_mem_going_online_callback(arg);
4058 break;
4059 case MEM_GOING_OFFLINE:
4060 ret = slab_mem_going_offline_callback(arg);
4061 break;
4062 case MEM_OFFLINE:
4063 case MEM_CANCEL_ONLINE:
4064 slab_mem_offline_callback(arg);
4065 break;
4066 case MEM_ONLINE:
4067 case MEM_CANCEL_OFFLINE:
4068 break;
4069 }
dc19f9db
KH
4070 if (ret)
4071 ret = notifier_from_errno(ret);
4072 else
4073 ret = NOTIFY_OK;
b9049e23
YG
4074 return ret;
4075}
4076
3ac38faa
AM
4077static struct notifier_block slab_memory_callback_nb = {
4078 .notifier_call = slab_memory_callback,
4079 .priority = SLAB_CALLBACK_PRI,
4080};
b9049e23 4081
81819f0f
CL
4082/********************************************************************
4083 * Basic setup of slabs
4084 *******************************************************************/
4085
51df1142
CL
4086/*
4087 * Used for early kmem_cache structures that were allocated using
dffb4d60
CL
4088 * the page allocator. Allocate them properly then fix up the pointers
4089 * that may be pointing to the wrong kmem_cache structure.
51df1142
CL
4090 */
4091
dffb4d60 4092static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
51df1142
CL
4093{
4094 int node;
dffb4d60 4095 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
fa45dc25 4096 struct kmem_cache_node *n;
51df1142 4097
dffb4d60 4098 memcpy(s, static_cache, kmem_cache->object_size);
51df1142 4099
7d557b3c
GC
4100 /*
4101 * This runs very early, and only the boot processor is supposed to be
4102 * up. Even if it weren't true, IRQs are not up so we couldn't fire
4103 * IPIs around.
4104 */
4105 __flush_cpu_slab(s, smp_processor_id());
fa45dc25 4106 for_each_kmem_cache_node(s, node, n) {
51df1142
CL
4107 struct page *p;
4108
fa45dc25
CL
4109 list_for_each_entry(p, &n->partial, lru)
4110 p->slab_cache = s;
51df1142 4111
607bf324 4112#ifdef CONFIG_SLUB_DEBUG
fa45dc25
CL
4113 list_for_each_entry(p, &n->full, lru)
4114 p->slab_cache = s;
51df1142 4115#endif
51df1142 4116 }
f7ce3190 4117 slab_init_memcg_params(s);
dffb4d60
CL
4118 list_add(&s->list, &slab_caches);
4119 return s;
51df1142
CL
4120}
4121
81819f0f
CL
4122void __init kmem_cache_init(void)
4123{
dffb4d60
CL
4124 static __initdata struct kmem_cache boot_kmem_cache,
4125 boot_kmem_cache_node;
51df1142 4126
fc8d8620
SG
4127 if (debug_guardpage_minorder())
4128 slub_max_order = 0;
4129
dffb4d60
CL
4130 kmem_cache_node = &boot_kmem_cache_node;
4131 kmem_cache = &boot_kmem_cache;
51df1142 4132
dffb4d60
CL
4133 create_boot_cache(kmem_cache_node, "kmem_cache_node",
4134 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
b9049e23 4135
3ac38faa 4136 register_hotmemory_notifier(&slab_memory_callback_nb);
81819f0f
CL
4137
4138 /* Able to allocate the per node structures */
4139 slab_state = PARTIAL;
4140
dffb4d60
CL
4141 create_boot_cache(kmem_cache, "kmem_cache",
4142 offsetof(struct kmem_cache, node) +
4143 nr_node_ids * sizeof(struct kmem_cache_node *),
4144 SLAB_HWCACHE_ALIGN);
8a13a4cc 4145
dffb4d60 4146 kmem_cache = bootstrap(&boot_kmem_cache);
81819f0f 4147
51df1142
CL
4148 /*
4149 * Allocate kmem_cache_node properly from the kmem_cache slab.
4150 * kmem_cache_node is separately allocated so no need to
4151 * update any list pointers.
4152 */
dffb4d60 4153 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
51df1142
CL
4154
4155 /* Now we can use the kmem_cache to allocate kmalloc slabs */
34cc6990 4156 setup_kmalloc_cache_index_table();
f97d5f63 4157 create_kmalloc_caches(0);
81819f0f 4158
210e7a43
TG
4159 /* Setup random freelists for each cache */
4160 init_freelist_randomization();
4161
a96a87bf
SAS
4162 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
4163 slub_cpu_dead);
81819f0f 4164
f9f58285 4165 pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n",
f97d5f63 4166 cache_line_size(),
81819f0f
CL
4167 slub_min_order, slub_max_order, slub_min_objects,
4168 nr_cpu_ids, nr_node_ids);
4169}
4170
7e85ee0c
PE
4171void __init kmem_cache_init_late(void)
4172{
7e85ee0c
PE
4173}
4174
2633d7a0 4175struct kmem_cache *
a44cb944
VD
4176__kmem_cache_alias(const char *name, size_t size, size_t align,
4177 unsigned long flags, void (*ctor)(void *))
81819f0f 4178{
426589f5 4179 struct kmem_cache *s, *c;
81819f0f 4180
a44cb944 4181 s = find_mergeable(size, align, flags, name, ctor);
81819f0f
CL
4182 if (s) {
4183 s->refcount++;
84d0ddd6 4184
81819f0f
CL
4185 /*
4186 * Adjust the object sizes so that we clear
4187 * the complete object on kzalloc.
4188 */
3b0efdfa 4189 s->object_size = max(s->object_size, (int)size);
81819f0f 4190 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
6446faa2 4191
426589f5 4192 for_each_memcg_cache(c, s) {
84d0ddd6
VD
4193 c->object_size = s->object_size;
4194 c->inuse = max_t(int, c->inuse,
4195 ALIGN(size, sizeof(void *)));
4196 }
4197
7b8f3b66 4198 if (sysfs_slab_alias(s, name)) {
7b8f3b66 4199 s->refcount--;
cbb79694 4200 s = NULL;
7b8f3b66 4201 }
a0e1d1be 4202 }
6446faa2 4203
cbb79694
CL
4204 return s;
4205}
84c1cf62 4206
8a13a4cc 4207int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
cbb79694 4208{
aac3a166
PE
4209 int err;
4210
4211 err = kmem_cache_open(s, flags);
4212 if (err)
4213 return err;
20cea968 4214
45530c44
CL
4215 /* Mutex is not taken during early boot */
4216 if (slab_state <= UP)
4217 return 0;
4218
107dab5c 4219 memcg_propagate_slab_attrs(s);
aac3a166 4220 err = sysfs_slab_add(s);
aac3a166 4221 if (err)
52b4b950 4222 __kmem_cache_release(s);
20cea968 4223
aac3a166 4224 return err;
81819f0f 4225}
81819f0f 4226
ce71e27c 4227void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
81819f0f 4228{
aadb4bc4 4229 struct kmem_cache *s;
94b528d0 4230 void *ret;
aadb4bc4 4231
95a05b42 4232 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
eada35ef
PE
4233 return kmalloc_large(size, gfpflags);
4234
2c59dd65 4235 s = kmalloc_slab(size, gfpflags);
81819f0f 4236
2408c550 4237 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913 4238 return s;
81819f0f 4239
2b847c3c 4240 ret = slab_alloc(s, gfpflags, caller);
94b528d0 4241
25985edc 4242 /* Honor the call site pointer we received. */
ca2b84cb 4243 trace_kmalloc(caller, ret, size, s->size, gfpflags);
94b528d0
EGM
4244
4245 return ret;
81819f0f
CL
4246}
4247
5d1f57e4 4248#ifdef CONFIG_NUMA
81819f0f 4249void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
ce71e27c 4250 int node, unsigned long caller)
81819f0f 4251{
aadb4bc4 4252 struct kmem_cache *s;
94b528d0 4253 void *ret;
aadb4bc4 4254
95a05b42 4255 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
d3e14aa3
XF
4256 ret = kmalloc_large_node(size, gfpflags, node);
4257
4258 trace_kmalloc_node(caller, ret,
4259 size, PAGE_SIZE << get_order(size),
4260 gfpflags, node);
4261
4262 return ret;
4263 }
eada35ef 4264
2c59dd65 4265 s = kmalloc_slab(size, gfpflags);
81819f0f 4266
2408c550 4267 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913 4268 return s;
81819f0f 4269
2b847c3c 4270 ret = slab_alloc_node(s, gfpflags, node, caller);
94b528d0 4271
25985edc 4272 /* Honor the call site pointer we received. */
ca2b84cb 4273 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
94b528d0
EGM
4274
4275 return ret;
81819f0f 4276}
5d1f57e4 4277#endif
81819f0f 4278
ab4d5ed5 4279#ifdef CONFIG_SYSFS
205ab99d
CL
4280static int count_inuse(struct page *page)
4281{
4282 return page->inuse;
4283}
4284
4285static int count_total(struct page *page)
4286{
4287 return page->objects;
4288}
ab4d5ed5 4289#endif
205ab99d 4290
ab4d5ed5 4291#ifdef CONFIG_SLUB_DEBUG
434e245d
CL
4292static int validate_slab(struct kmem_cache *s, struct page *page,
4293 unsigned long *map)
53e15af0
CL
4294{
4295 void *p;
a973e9dd 4296 void *addr = page_address(page);
53e15af0
CL
4297
4298 if (!check_slab(s, page) ||
4299 !on_freelist(s, page, NULL))
4300 return 0;
4301
4302 /* Now we know that a valid freelist exists */
39b26464 4303 bitmap_zero(map, page->objects);
53e15af0 4304
5f80b13a
CL
4305 get_map(s, page, map);
4306 for_each_object(p, s, addr, page->objects) {
4307 if (test_bit(slab_index(p, s, addr), map))
4308 if (!check_object(s, page, p, SLUB_RED_INACTIVE))
4309 return 0;
53e15af0
CL
4310 }
4311
224a88be 4312 for_each_object(p, s, addr, page->objects)
7656c72b 4313 if (!test_bit(slab_index(p, s, addr), map))
37d57443 4314 if (!check_object(s, page, p, SLUB_RED_ACTIVE))
53e15af0
CL
4315 return 0;
4316 return 1;
4317}
4318
434e245d
CL
4319static void validate_slab_slab(struct kmem_cache *s, struct page *page,
4320 unsigned long *map)
53e15af0 4321{
881db7fb
CL
4322 slab_lock(page);
4323 validate_slab(s, page, map);
4324 slab_unlock(page);
53e15af0
CL
4325}
4326
434e245d
CL
4327static int validate_slab_node(struct kmem_cache *s,
4328 struct kmem_cache_node *n, unsigned long *map)
53e15af0
CL
4329{
4330 unsigned long count = 0;
4331 struct page *page;
4332 unsigned long flags;
4333
4334 spin_lock_irqsave(&n->list_lock, flags);
4335
4336 list_for_each_entry(page, &n->partial, lru) {
434e245d 4337 validate_slab_slab(s, page, map);
53e15af0
CL
4338 count++;
4339 }
4340 if (count != n->nr_partial)
f9f58285
FF
4341 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
4342 s->name, count, n->nr_partial);
53e15af0
CL
4343
4344 if (!(s->flags & SLAB_STORE_USER))
4345 goto out;
4346
4347 list_for_each_entry(page, &n->full, lru) {
434e245d 4348 validate_slab_slab(s, page, map);
53e15af0
CL
4349 count++;
4350 }
4351 if (count != atomic_long_read(&n->nr_slabs))
f9f58285
FF
4352 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
4353 s->name, count, atomic_long_read(&n->nr_slabs));
53e15af0
CL
4354
4355out:
4356 spin_unlock_irqrestore(&n->list_lock, flags);
4357 return count;
4358}
4359
434e245d 4360static long validate_slab_cache(struct kmem_cache *s)
53e15af0
CL
4361{
4362 int node;
4363 unsigned long count = 0;
205ab99d 4364 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
434e245d 4365 sizeof(unsigned long), GFP_KERNEL);
fa45dc25 4366 struct kmem_cache_node *n;
434e245d
CL
4367
4368 if (!map)
4369 return -ENOMEM;
53e15af0
CL
4370
4371 flush_all(s);
fa45dc25 4372 for_each_kmem_cache_node(s, node, n)
434e245d 4373 count += validate_slab_node(s, n, map);
434e245d 4374 kfree(map);
53e15af0
CL
4375 return count;
4376}
88a420e4 4377/*
672bba3a 4378 * Generate lists of code addresses where slabcache objects are allocated
88a420e4
CL
4379 * and freed.
4380 */
4381
4382struct location {
4383 unsigned long count;
ce71e27c 4384 unsigned long addr;
45edfa58
CL
4385 long long sum_time;
4386 long min_time;
4387 long max_time;
4388 long min_pid;
4389 long max_pid;
174596a0 4390 DECLARE_BITMAP(cpus, NR_CPUS);
45edfa58 4391 nodemask_t nodes;
88a420e4
CL
4392};
4393
4394struct loc_track {
4395 unsigned long max;
4396 unsigned long count;
4397 struct location *loc;
4398};
4399
4400static void free_loc_track(struct loc_track *t)
4401{
4402 if (t->max)
4403 free_pages((unsigned long)t->loc,
4404 get_order(sizeof(struct location) * t->max));
4405}
4406
68dff6a9 4407static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
88a420e4
CL
4408{
4409 struct location *l;
4410 int order;
4411
88a420e4
CL
4412 order = get_order(sizeof(struct location) * max);
4413
68dff6a9 4414 l = (void *)__get_free_pages(flags, order);
88a420e4
CL
4415 if (!l)
4416 return 0;
4417
4418 if (t->count) {
4419 memcpy(l, t->loc, sizeof(struct location) * t->count);
4420 free_loc_track(t);
4421 }
4422 t->max = max;
4423 t->loc = l;
4424 return 1;
4425}
4426
4427static int add_location(struct loc_track *t, struct kmem_cache *s,
45edfa58 4428 const struct track *track)
88a420e4
CL
4429{
4430 long start, end, pos;
4431 struct location *l;
ce71e27c 4432 unsigned long caddr;
45edfa58 4433 unsigned long age = jiffies - track->when;
88a420e4
CL
4434
4435 start = -1;
4436 end = t->count;
4437
4438 for ( ; ; ) {
4439 pos = start + (end - start + 1) / 2;
4440
4441 /*
4442 * There is nothing at "end". If we end up there
4443 * we need to add something to before end.
4444 */
4445 if (pos == end)
4446 break;
4447
4448 caddr = t->loc[pos].addr;
45edfa58
CL
4449 if (track->addr == caddr) {
4450
4451 l = &t->loc[pos];
4452 l->count++;
4453 if (track->when) {
4454 l->sum_time += age;
4455 if (age < l->min_time)
4456 l->min_time = age;
4457 if (age > l->max_time)
4458 l->max_time = age;
4459
4460 if (track->pid < l->min_pid)
4461 l->min_pid = track->pid;
4462 if (track->pid > l->max_pid)
4463 l->max_pid = track->pid;
4464
174596a0
RR
4465 cpumask_set_cpu(track->cpu,
4466 to_cpumask(l->cpus));
45edfa58
CL
4467 }
4468 node_set(page_to_nid(virt_to_page(track)), l->nodes);
88a420e4
CL
4469 return 1;
4470 }
4471
45edfa58 4472 if (track->addr < caddr)
88a420e4
CL
4473 end = pos;
4474 else
4475 start = pos;
4476 }
4477
4478 /*
672bba3a 4479 * Not found. Insert new tracking element.
88a420e4 4480 */
68dff6a9 4481 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
88a420e4
CL
4482 return 0;
4483
4484 l = t->loc + pos;
4485 if (pos < t->count)
4486 memmove(l + 1, l,
4487 (t->count - pos) * sizeof(struct location));
4488 t->count++;
4489 l->count = 1;
45edfa58
CL
4490 l->addr = track->addr;
4491 l->sum_time = age;
4492 l->min_time = age;
4493 l->max_time = age;
4494 l->min_pid = track->pid;
4495 l->max_pid = track->pid;
174596a0
RR
4496 cpumask_clear(to_cpumask(l->cpus));
4497 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
45edfa58
CL
4498 nodes_clear(l->nodes);
4499 node_set(page_to_nid(virt_to_page(track)), l->nodes);
88a420e4
CL
4500 return 1;
4501}
4502
4503static void process_slab(struct loc_track *t, struct kmem_cache *s,
bbd7d57b 4504 struct page *page, enum track_item alloc,
a5dd5c11 4505 unsigned long *map)
88a420e4 4506{
a973e9dd 4507 void *addr = page_address(page);
88a420e4
CL
4508 void *p;
4509
39b26464 4510 bitmap_zero(map, page->objects);
5f80b13a 4511 get_map(s, page, map);
88a420e4 4512
224a88be 4513 for_each_object(p, s, addr, page->objects)
45edfa58
CL
4514 if (!test_bit(slab_index(p, s, addr), map))
4515 add_location(t, s, get_track(s, p, alloc));
88a420e4
CL
4516}
4517
4518static int list_locations(struct kmem_cache *s, char *buf,
4519 enum track_item alloc)
4520{
e374d483 4521 int len = 0;
88a420e4 4522 unsigned long i;
68dff6a9 4523 struct loc_track t = { 0, 0, NULL };
88a420e4 4524 int node;
bbd7d57b
ED
4525 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4526 sizeof(unsigned long), GFP_KERNEL);
fa45dc25 4527 struct kmem_cache_node *n;
88a420e4 4528
bbd7d57b
ED
4529 if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4530 GFP_TEMPORARY)) {
4531 kfree(map);
68dff6a9 4532 return sprintf(buf, "Out of memory\n");
bbd7d57b 4533 }
88a420e4
CL
4534 /* Push back cpu slabs */
4535 flush_all(s);
4536
fa45dc25 4537 for_each_kmem_cache_node(s, node, n) {
88a420e4
CL
4538 unsigned long flags;
4539 struct page *page;
4540
9e86943b 4541 if (!atomic_long_read(&n->nr_slabs))
88a420e4
CL
4542 continue;
4543
4544 spin_lock_irqsave(&n->list_lock, flags);
4545 list_for_each_entry(page, &n->partial, lru)
bbd7d57b 4546 process_slab(&t, s, page, alloc, map);
88a420e4 4547 list_for_each_entry(page, &n->full, lru)
bbd7d57b 4548 process_slab(&t, s, page, alloc, map);
88a420e4
CL
4549 spin_unlock_irqrestore(&n->list_lock, flags);
4550 }
4551
4552 for (i = 0; i < t.count; i++) {
45edfa58 4553 struct location *l = &t.loc[i];
88a420e4 4554
9c246247 4555 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
88a420e4 4556 break;
e374d483 4557 len += sprintf(buf + len, "%7ld ", l->count);
45edfa58
CL
4558
4559 if (l->addr)
62c70bce 4560 len += sprintf(buf + len, "%pS", (void *)l->addr);
88a420e4 4561 else
e374d483 4562 len += sprintf(buf + len, "<not-available>");
45edfa58
CL
4563
4564 if (l->sum_time != l->min_time) {
e374d483 4565 len += sprintf(buf + len, " age=%ld/%ld/%ld",
f8bd2258
RZ
4566 l->min_time,
4567 (long)div_u64(l->sum_time, l->count),
4568 l->max_time);
45edfa58 4569 } else
e374d483 4570 len += sprintf(buf + len, " age=%ld",
45edfa58
CL
4571 l->min_time);
4572
4573 if (l->min_pid != l->max_pid)
e374d483 4574 len += sprintf(buf + len, " pid=%ld-%ld",
45edfa58
CL
4575 l->min_pid, l->max_pid);
4576 else
e374d483 4577 len += sprintf(buf + len, " pid=%ld",
45edfa58
CL
4578 l->min_pid);
4579
174596a0
RR
4580 if (num_online_cpus() > 1 &&
4581 !cpumask_empty(to_cpumask(l->cpus)) &&
5024c1d7
TH
4582 len < PAGE_SIZE - 60)
4583 len += scnprintf(buf + len, PAGE_SIZE - len - 50,
4584 " cpus=%*pbl",
4585 cpumask_pr_args(to_cpumask(l->cpus)));
45edfa58 4586
62bc62a8 4587 if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
5024c1d7
TH
4588 len < PAGE_SIZE - 60)
4589 len += scnprintf(buf + len, PAGE_SIZE - len - 50,
4590 " nodes=%*pbl",
4591 nodemask_pr_args(&l->nodes));
45edfa58 4592
e374d483 4593 len += sprintf(buf + len, "\n");
88a420e4
CL
4594 }
4595
4596 free_loc_track(&t);
bbd7d57b 4597 kfree(map);
88a420e4 4598 if (!t.count)
e374d483
HH
4599 len += sprintf(buf, "No data\n");
4600 return len;
88a420e4 4601}
ab4d5ed5 4602#endif
88a420e4 4603
a5a84755 4604#ifdef SLUB_RESILIENCY_TEST
c07b8183 4605static void __init resiliency_test(void)
a5a84755
CL
4606{
4607 u8 *p;
4608
95a05b42 4609 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
a5a84755 4610
f9f58285
FF
4611 pr_err("SLUB resiliency testing\n");
4612 pr_err("-----------------------\n");
4613 pr_err("A. Corruption after allocation\n");
a5a84755
CL
4614
4615 p = kzalloc(16, GFP_KERNEL);
4616 p[16] = 0x12;
f9f58285
FF
4617 pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
4618 p + 16);
a5a84755
CL
4619
4620 validate_slab_cache(kmalloc_caches[4]);
4621
4622 /* Hmmm... The next two are dangerous */
4623 p = kzalloc(32, GFP_KERNEL);
4624 p[32 + sizeof(void *)] = 0x34;
f9f58285
FF
4625 pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
4626 p);
4627 pr_err("If allocated object is overwritten then not detectable\n\n");
a5a84755
CL
4628
4629 validate_slab_cache(kmalloc_caches[5]);
4630 p = kzalloc(64, GFP_KERNEL);
4631 p += 64 + (get_cycles() & 0xff) * sizeof(void *);
4632 *p = 0x56;
f9f58285
FF
4633 pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
4634 p);
4635 pr_err("If allocated object is overwritten then not detectable\n\n");
a5a84755
CL
4636 validate_slab_cache(kmalloc_caches[6]);
4637
f9f58285 4638 pr_err("\nB. Corruption after free\n");
a5a84755
CL
4639 p = kzalloc(128, GFP_KERNEL);
4640 kfree(p);
4641 *p = 0x78;
f9f58285 4642 pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
a5a84755
CL
4643 validate_slab_cache(kmalloc_caches[7]);
4644
4645 p = kzalloc(256, GFP_KERNEL);
4646 kfree(p);
4647 p[50] = 0x9a;
f9f58285 4648 pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
a5a84755
CL
4649 validate_slab_cache(kmalloc_caches[8]);
4650
4651 p = kzalloc(512, GFP_KERNEL);
4652 kfree(p);
4653 p[512] = 0xab;
f9f58285 4654 pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
a5a84755
CL
4655 validate_slab_cache(kmalloc_caches[9]);
4656}
4657#else
4658#ifdef CONFIG_SYSFS
4659static void resiliency_test(void) {};
4660#endif
4661#endif
4662
ab4d5ed5 4663#ifdef CONFIG_SYSFS
81819f0f 4664enum slab_stat_type {
205ab99d
CL
4665 SL_ALL, /* All slabs */
4666 SL_PARTIAL, /* Only partially allocated slabs */
4667 SL_CPU, /* Only slabs used for cpu caches */
4668 SL_OBJECTS, /* Determine allocated objects not slabs */
4669 SL_TOTAL /* Determine object capacity not slabs */
81819f0f
CL
4670};
4671
205ab99d 4672#define SO_ALL (1 << SL_ALL)
81819f0f
CL
4673#define SO_PARTIAL (1 << SL_PARTIAL)
4674#define SO_CPU (1 << SL_CPU)
4675#define SO_OBJECTS (1 << SL_OBJECTS)
205ab99d 4676#define SO_TOTAL (1 << SL_TOTAL)
81819f0f 4677
62e5c4b4
CG
4678static ssize_t show_slab_objects(struct kmem_cache *s,
4679 char *buf, unsigned long flags)
81819f0f
CL
4680{
4681 unsigned long total = 0;
81819f0f
CL
4682 int node;
4683 int x;
4684 unsigned long *nodes;
81819f0f 4685
e35e1a97 4686 nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
62e5c4b4
CG
4687 if (!nodes)
4688 return -ENOMEM;
81819f0f 4689
205ab99d
CL
4690 if (flags & SO_CPU) {
4691 int cpu;
81819f0f 4692
205ab99d 4693 for_each_possible_cpu(cpu) {
d0e0ac97
CG
4694 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
4695 cpu);
ec3ab083 4696 int node;
49e22585 4697 struct page *page;
dfb4f096 4698
4db0c3c2 4699 page = READ_ONCE(c->page);
ec3ab083
CL
4700 if (!page)
4701 continue;
205ab99d 4702
ec3ab083
CL
4703 node = page_to_nid(page);
4704 if (flags & SO_TOTAL)
4705 x = page->objects;
4706 else if (flags & SO_OBJECTS)
4707 x = page->inuse;
4708 else
4709 x = 1;
49e22585 4710
ec3ab083
CL
4711 total += x;
4712 nodes[node] += x;
4713
4db0c3c2 4714 page = READ_ONCE(c->partial);
49e22585 4715 if (page) {
8afb1474
LZ
4716 node = page_to_nid(page);
4717 if (flags & SO_TOTAL)
4718 WARN_ON_ONCE(1);
4719 else if (flags & SO_OBJECTS)
4720 WARN_ON_ONCE(1);
4721 else
4722 x = page->pages;
bc6697d8
ED
4723 total += x;
4724 nodes[node] += x;
49e22585 4725 }
81819f0f
CL
4726 }
4727 }
4728
bfc8c901 4729 get_online_mems();
ab4d5ed5 4730#ifdef CONFIG_SLUB_DEBUG
205ab99d 4731 if (flags & SO_ALL) {
fa45dc25
CL
4732 struct kmem_cache_node *n;
4733
4734 for_each_kmem_cache_node(s, node, n) {
205ab99d 4735
d0e0ac97
CG
4736 if (flags & SO_TOTAL)
4737 x = atomic_long_read(&n->total_objects);
4738 else if (flags & SO_OBJECTS)
4739 x = atomic_long_read(&n->total_objects) -
4740 count_partial(n, count_free);
81819f0f 4741 else
205ab99d 4742 x = atomic_long_read(&n->nr_slabs);
81819f0f
CL
4743 total += x;
4744 nodes[node] += x;
4745 }
4746
ab4d5ed5
CL
4747 } else
4748#endif
4749 if (flags & SO_PARTIAL) {
fa45dc25 4750 struct kmem_cache_node *n;
81819f0f 4751
fa45dc25 4752 for_each_kmem_cache_node(s, node, n) {
205ab99d
CL
4753 if (flags & SO_TOTAL)
4754 x = count_partial(n, count_total);
4755 else if (flags & SO_OBJECTS)
4756 x = count_partial(n, count_inuse);
81819f0f 4757 else
205ab99d 4758 x = n->nr_partial;
81819f0f
CL
4759 total += x;
4760 nodes[node] += x;
4761 }
4762 }
81819f0f
CL
4763 x = sprintf(buf, "%lu", total);
4764#ifdef CONFIG_NUMA
fa45dc25 4765 for (node = 0; node < nr_node_ids; node++)
81819f0f
CL
4766 if (nodes[node])
4767 x += sprintf(buf + x, " N%d=%lu",
4768 node, nodes[node]);
4769#endif
bfc8c901 4770 put_online_mems();
81819f0f
CL
4771 kfree(nodes);
4772 return x + sprintf(buf + x, "\n");
4773}
4774
ab4d5ed5 4775#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
4776static int any_slab_objects(struct kmem_cache *s)
4777{
4778 int node;
fa45dc25 4779 struct kmem_cache_node *n;
81819f0f 4780
fa45dc25 4781 for_each_kmem_cache_node(s, node, n)
4ea33e2d 4782 if (atomic_long_read(&n->total_objects))
81819f0f 4783 return 1;
fa45dc25 4784
81819f0f
CL
4785 return 0;
4786}
ab4d5ed5 4787#endif
81819f0f
CL
4788
4789#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
497888cf 4790#define to_slab(n) container_of(n, struct kmem_cache, kobj)
81819f0f
CL
4791
4792struct slab_attribute {
4793 struct attribute attr;
4794 ssize_t (*show)(struct kmem_cache *s, char *buf);
4795 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
4796};
4797
4798#define SLAB_ATTR_RO(_name) \
ab067e99
VK
4799 static struct slab_attribute _name##_attr = \
4800 __ATTR(_name, 0400, _name##_show, NULL)
81819f0f
CL
4801
4802#define SLAB_ATTR(_name) \
4803 static struct slab_attribute _name##_attr = \
ab067e99 4804 __ATTR(_name, 0600, _name##_show, _name##_store)
81819f0f 4805
81819f0f
CL
4806static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
4807{
4808 return sprintf(buf, "%d\n", s->size);
4809}
4810SLAB_ATTR_RO(slab_size);
4811
4812static ssize_t align_show(struct kmem_cache *s, char *buf)
4813{
4814 return sprintf(buf, "%d\n", s->align);
4815}
4816SLAB_ATTR_RO(align);
4817
4818static ssize_t object_size_show(struct kmem_cache *s, char *buf)
4819{
3b0efdfa 4820 return sprintf(buf, "%d\n", s->object_size);
81819f0f
CL
4821}
4822SLAB_ATTR_RO(object_size);
4823
4824static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
4825{
834f3d11 4826 return sprintf(buf, "%d\n", oo_objects(s->oo));
81819f0f
CL
4827}
4828SLAB_ATTR_RO(objs_per_slab);
4829
06b285dc
CL
4830static ssize_t order_store(struct kmem_cache *s,
4831 const char *buf, size_t length)
4832{
0121c619
CL
4833 unsigned long order;
4834 int err;
4835
3dbb95f7 4836 err = kstrtoul(buf, 10, &order);
0121c619
CL
4837 if (err)
4838 return err;
06b285dc
CL
4839
4840 if (order > slub_max_order || order < slub_min_order)
4841 return -EINVAL;
4842
4843 calculate_sizes(s, order);
4844 return length;
4845}
4846
81819f0f
CL
4847static ssize_t order_show(struct kmem_cache *s, char *buf)
4848{
834f3d11 4849 return sprintf(buf, "%d\n", oo_order(s->oo));
81819f0f 4850}
06b285dc 4851SLAB_ATTR(order);
81819f0f 4852
73d342b1
DR
4853static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
4854{
4855 return sprintf(buf, "%lu\n", s->min_partial);
4856}
4857
4858static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
4859 size_t length)
4860{
4861 unsigned long min;
4862 int err;
4863
3dbb95f7 4864 err = kstrtoul(buf, 10, &min);
73d342b1
DR
4865 if (err)
4866 return err;
4867
c0bdb232 4868 set_min_partial(s, min);
73d342b1
DR
4869 return length;
4870}
4871SLAB_ATTR(min_partial);
4872
49e22585
CL
4873static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
4874{
4875 return sprintf(buf, "%u\n", s->cpu_partial);
4876}
4877
4878static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
4879 size_t length)
4880{
4881 unsigned long objects;
4882 int err;
4883
3dbb95f7 4884 err = kstrtoul(buf, 10, &objects);
49e22585
CL
4885 if (err)
4886 return err;
345c905d 4887 if (objects && !kmem_cache_has_cpu_partial(s))
74ee4ef1 4888 return -EINVAL;
49e22585
CL
4889
4890 s->cpu_partial = objects;
4891 flush_all(s);
4892 return length;
4893}
4894SLAB_ATTR(cpu_partial);
4895
81819f0f
CL
4896static ssize_t ctor_show(struct kmem_cache *s, char *buf)
4897{
62c70bce
JP
4898 if (!s->ctor)
4899 return 0;
4900 return sprintf(buf, "%pS\n", s->ctor);
81819f0f
CL
4901}
4902SLAB_ATTR_RO(ctor);
4903
81819f0f
CL
4904static ssize_t aliases_show(struct kmem_cache *s, char *buf)
4905{
4307c14f 4906 return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
81819f0f
CL
4907}
4908SLAB_ATTR_RO(aliases);
4909
81819f0f
CL
4910static ssize_t partial_show(struct kmem_cache *s, char *buf)
4911{
d9acf4b7 4912 return show_slab_objects(s, buf, SO_PARTIAL);
81819f0f
CL
4913}
4914SLAB_ATTR_RO(partial);
4915
4916static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
4917{
d9acf4b7 4918 return show_slab_objects(s, buf, SO_CPU);
81819f0f
CL
4919}
4920SLAB_ATTR_RO(cpu_slabs);
4921
4922static ssize_t objects_show(struct kmem_cache *s, char *buf)
4923{
205ab99d 4924 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
81819f0f
CL
4925}
4926SLAB_ATTR_RO(objects);
4927
205ab99d
CL
4928static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
4929{
4930 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
4931}
4932SLAB_ATTR_RO(objects_partial);
4933
49e22585
CL
4934static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
4935{
4936 int objects = 0;
4937 int pages = 0;
4938 int cpu;
4939 int len;
4940
4941 for_each_online_cpu(cpu) {
4942 struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
4943
4944 if (page) {
4945 pages += page->pages;
4946 objects += page->pobjects;
4947 }
4948 }
4949
4950 len = sprintf(buf, "%d(%d)", objects, pages);
4951
4952#ifdef CONFIG_SMP
4953 for_each_online_cpu(cpu) {
4954 struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
4955
4956 if (page && len < PAGE_SIZE - 20)
4957 len += sprintf(buf + len, " C%d=%d(%d)", cpu,
4958 page->pobjects, page->pages);
4959 }
4960#endif
4961 return len + sprintf(buf + len, "\n");
4962}
4963SLAB_ATTR_RO(slabs_cpu_partial);
4964
a5a84755
CL
4965static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4966{
4967 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4968}
4969
4970static ssize_t reclaim_account_store(struct kmem_cache *s,
4971 const char *buf, size_t length)
4972{
4973 s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4974 if (buf[0] == '1')
4975 s->flags |= SLAB_RECLAIM_ACCOUNT;
4976 return length;
4977}
4978SLAB_ATTR(reclaim_account);
4979
4980static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4981{
4982 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
4983}
4984SLAB_ATTR_RO(hwcache_align);
4985
4986#ifdef CONFIG_ZONE_DMA
4987static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4988{
4989 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4990}
4991SLAB_ATTR_RO(cache_dma);
4992#endif
4993
4994static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4995{
4996 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4997}
4998SLAB_ATTR_RO(destroy_by_rcu);
4999
ab9a0f19
LJ
5000static ssize_t reserved_show(struct kmem_cache *s, char *buf)
5001{
5002 return sprintf(buf, "%d\n", s->reserved);
5003}
5004SLAB_ATTR_RO(reserved);
5005
ab4d5ed5 5006#ifdef CONFIG_SLUB_DEBUG
a5a84755
CL
5007static ssize_t slabs_show(struct kmem_cache *s, char *buf)
5008{
5009 return show_slab_objects(s, buf, SO_ALL);
5010}
5011SLAB_ATTR_RO(slabs);
5012
205ab99d
CL
5013static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
5014{
5015 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
5016}
5017SLAB_ATTR_RO(total_objects);
5018
81819f0f
CL
5019static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
5020{
becfda68 5021 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
81819f0f
CL
5022}
5023
5024static ssize_t sanity_checks_store(struct kmem_cache *s,
5025 const char *buf, size_t length)
5026{
becfda68 5027 s->flags &= ~SLAB_CONSISTENCY_CHECKS;
b789ef51
CL
5028 if (buf[0] == '1') {
5029 s->flags &= ~__CMPXCHG_DOUBLE;
becfda68 5030 s->flags |= SLAB_CONSISTENCY_CHECKS;
b789ef51 5031 }
81819f0f
CL
5032 return length;
5033}
5034SLAB_ATTR(sanity_checks);
5035
5036static ssize_t trace_show(struct kmem_cache *s, char *buf)
5037{
5038 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
5039}
5040
5041static ssize_t trace_store(struct kmem_cache *s, const char *buf,
5042 size_t length)
5043{
c9e16131
CL
5044 /*
5045 * Tracing a merged cache is going to give confusing results
5046 * as well as cause other issues like converting a mergeable
5047 * cache into an umergeable one.
5048 */
5049 if (s->refcount > 1)
5050 return -EINVAL;
5051
81819f0f 5052 s->flags &= ~SLAB_TRACE;
b789ef51
CL
5053 if (buf[0] == '1') {
5054 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 5055 s->flags |= SLAB_TRACE;
b789ef51 5056 }
81819f0f
CL
5057 return length;
5058}
5059SLAB_ATTR(trace);
5060
81819f0f
CL
5061static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
5062{
5063 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
5064}
5065
5066static ssize_t red_zone_store(struct kmem_cache *s,
5067 const char *buf, size_t length)
5068{
5069 if (any_slab_objects(s))
5070 return -EBUSY;
5071
5072 s->flags &= ~SLAB_RED_ZONE;
b789ef51 5073 if (buf[0] == '1') {
81819f0f 5074 s->flags |= SLAB_RED_ZONE;
b789ef51 5075 }
06b285dc 5076 calculate_sizes(s, -1);
81819f0f
CL
5077 return length;
5078}
5079SLAB_ATTR(red_zone);
5080
5081static ssize_t poison_show(struct kmem_cache *s, char *buf)
5082{
5083 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
5084}
5085
5086static ssize_t poison_store(struct kmem_cache *s,
5087 const char *buf, size_t length)
5088{
5089 if (any_slab_objects(s))
5090 return -EBUSY;
5091
5092 s->flags &= ~SLAB_POISON;
b789ef51 5093 if (buf[0] == '1') {
81819f0f 5094 s->flags |= SLAB_POISON;
b789ef51 5095 }
06b285dc 5096 calculate_sizes(s, -1);
81819f0f
CL
5097 return length;
5098}
5099SLAB_ATTR(poison);
5100
5101static ssize_t store_user_show(struct kmem_cache *s, char *buf)
5102{
5103 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
5104}
5105
5106static ssize_t store_user_store(struct kmem_cache *s,
5107 const char *buf, size_t length)
5108{
5109 if (any_slab_objects(s))
5110 return -EBUSY;
5111
5112 s->flags &= ~SLAB_STORE_USER;
b789ef51
CL
5113 if (buf[0] == '1') {
5114 s->flags &= ~__CMPXCHG_DOUBLE;
81819f0f 5115 s->flags |= SLAB_STORE_USER;
b789ef51 5116 }
06b285dc 5117 calculate_sizes(s, -1);
81819f0f
CL
5118 return length;
5119}
5120SLAB_ATTR(store_user);
5121
53e15af0
CL
5122static ssize_t validate_show(struct kmem_cache *s, char *buf)
5123{
5124 return 0;
5125}
5126
5127static ssize_t validate_store(struct kmem_cache *s,
5128 const char *buf, size_t length)
5129{
434e245d
CL
5130 int ret = -EINVAL;
5131
5132 if (buf[0] == '1') {
5133 ret = validate_slab_cache(s);
5134 if (ret >= 0)
5135 ret = length;
5136 }
5137 return ret;
53e15af0
CL
5138}
5139SLAB_ATTR(validate);
a5a84755
CL
5140
5141static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
5142{
5143 if (!(s->flags & SLAB_STORE_USER))
5144 return -ENOSYS;
5145 return list_locations(s, buf, TRACK_ALLOC);
5146}
5147SLAB_ATTR_RO(alloc_calls);
5148
5149static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
5150{
5151 if (!(s->flags & SLAB_STORE_USER))
5152 return -ENOSYS;
5153 return list_locations(s, buf, TRACK_FREE);
5154}
5155SLAB_ATTR_RO(free_calls);
5156#endif /* CONFIG_SLUB_DEBUG */
5157
5158#ifdef CONFIG_FAILSLAB
5159static ssize_t failslab_show(struct kmem_cache *s, char *buf)
5160{
5161 return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
5162}
5163
5164static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
5165 size_t length)
5166{
c9e16131
CL
5167 if (s->refcount > 1)
5168 return -EINVAL;
5169
a5a84755
CL
5170 s->flags &= ~SLAB_FAILSLAB;
5171 if (buf[0] == '1')
5172 s->flags |= SLAB_FAILSLAB;
5173 return length;
5174}
5175SLAB_ATTR(failslab);
ab4d5ed5 5176#endif
53e15af0 5177
2086d26a
CL
5178static ssize_t shrink_show(struct kmem_cache *s, char *buf)
5179{
5180 return 0;
5181}
5182
5183static ssize_t shrink_store(struct kmem_cache *s,
5184 const char *buf, size_t length)
5185{
832f37f5
VD
5186 if (buf[0] == '1')
5187 kmem_cache_shrink(s);
5188 else
2086d26a
CL
5189 return -EINVAL;
5190 return length;
5191}
5192SLAB_ATTR(shrink);
5193
81819f0f 5194#ifdef CONFIG_NUMA
9824601e 5195static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
81819f0f 5196{
9824601e 5197 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
81819f0f
CL
5198}
5199
9824601e 5200static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
81819f0f
CL
5201 const char *buf, size_t length)
5202{
0121c619
CL
5203 unsigned long ratio;
5204 int err;
5205
3dbb95f7 5206 err = kstrtoul(buf, 10, &ratio);
0121c619
CL
5207 if (err)
5208 return err;
5209
e2cb96b7 5210 if (ratio <= 100)
0121c619 5211 s->remote_node_defrag_ratio = ratio * 10;
81819f0f 5212
81819f0f
CL
5213 return length;
5214}
9824601e 5215SLAB_ATTR(remote_node_defrag_ratio);
81819f0f
CL
5216#endif
5217
8ff12cfc 5218#ifdef CONFIG_SLUB_STATS
8ff12cfc
CL
5219static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5220{
5221 unsigned long sum = 0;
5222 int cpu;
5223 int len;
5224 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
5225
5226 if (!data)
5227 return -ENOMEM;
5228
5229 for_each_online_cpu(cpu) {
9dfc6e68 5230 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
8ff12cfc
CL
5231
5232 data[cpu] = x;
5233 sum += x;
5234 }
5235
5236 len = sprintf(buf, "%lu", sum);
5237
50ef37b9 5238#ifdef CONFIG_SMP
8ff12cfc
CL
5239 for_each_online_cpu(cpu) {
5240 if (data[cpu] && len < PAGE_SIZE - 20)
50ef37b9 5241 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
8ff12cfc 5242 }
50ef37b9 5243#endif
8ff12cfc
CL
5244 kfree(data);
5245 return len + sprintf(buf + len, "\n");
5246}
5247
78eb00cc
DR
5248static void clear_stat(struct kmem_cache *s, enum stat_item si)
5249{
5250 int cpu;
5251
5252 for_each_online_cpu(cpu)
9dfc6e68 5253 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
78eb00cc
DR
5254}
5255
8ff12cfc
CL
5256#define STAT_ATTR(si, text) \
5257static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5258{ \
5259 return show_stat(s, buf, si); \
5260} \
78eb00cc
DR
5261static ssize_t text##_store(struct kmem_cache *s, \
5262 const char *buf, size_t length) \
5263{ \
5264 if (buf[0] != '0') \
5265 return -EINVAL; \
5266 clear_stat(s, si); \
5267 return length; \
5268} \
5269SLAB_ATTR(text); \
8ff12cfc
CL
5270
5271STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5272STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5273STAT_ATTR(FREE_FASTPATH, free_fastpath);
5274STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5275STAT_ATTR(FREE_FROZEN, free_frozen);
5276STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5277STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5278STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5279STAT_ATTR(ALLOC_SLAB, alloc_slab);
5280STAT_ATTR(ALLOC_REFILL, alloc_refill);
e36a2652 5281STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
8ff12cfc
CL
5282STAT_ATTR(FREE_SLAB, free_slab);
5283STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5284STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5285STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5286STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5287STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5288STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
03e404af 5289STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
65c3376a 5290STAT_ATTR(ORDER_FALLBACK, order_fallback);
b789ef51
CL
5291STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5292STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
49e22585
CL
5293STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5294STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
8028dcea
AS
5295STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5296STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
8ff12cfc
CL
5297#endif
5298
06428780 5299static struct attribute *slab_attrs[] = {
81819f0f
CL
5300 &slab_size_attr.attr,
5301 &object_size_attr.attr,
5302 &objs_per_slab_attr.attr,
5303 &order_attr.attr,
73d342b1 5304 &min_partial_attr.attr,
49e22585 5305 &cpu_partial_attr.attr,
81819f0f 5306 &objects_attr.attr,
205ab99d 5307 &objects_partial_attr.attr,
81819f0f
CL
5308 &partial_attr.attr,
5309 &cpu_slabs_attr.attr,
5310 &ctor_attr.attr,
81819f0f
CL
5311 &aliases_attr.attr,
5312 &align_attr.attr,
81819f0f
CL
5313 &hwcache_align_attr.attr,
5314 &reclaim_account_attr.attr,
5315 &destroy_by_rcu_attr.attr,
a5a84755 5316 &shrink_attr.attr,
ab9a0f19 5317 &reserved_attr.attr,
49e22585 5318 &slabs_cpu_partial_attr.attr,
ab4d5ed5 5319#ifdef CONFIG_SLUB_DEBUG
a5a84755
CL
5320 &total_objects_attr.attr,
5321 &slabs_attr.attr,
5322 &sanity_checks_attr.attr,
5323 &trace_attr.attr,
81819f0f
CL
5324 &red_zone_attr.attr,
5325 &poison_attr.attr,
5326 &store_user_attr.attr,
53e15af0 5327 &validate_attr.attr,
88a420e4
CL
5328 &alloc_calls_attr.attr,
5329 &free_calls_attr.attr,
ab4d5ed5 5330#endif
81819f0f
CL
5331#ifdef CONFIG_ZONE_DMA
5332 &cache_dma_attr.attr,
5333#endif
5334#ifdef CONFIG_NUMA
9824601e 5335 &remote_node_defrag_ratio_attr.attr,
8ff12cfc
CL
5336#endif
5337#ifdef CONFIG_SLUB_STATS
5338 &alloc_fastpath_attr.attr,
5339 &alloc_slowpath_attr.attr,
5340 &free_fastpath_attr.attr,
5341 &free_slowpath_attr.attr,
5342 &free_frozen_attr.attr,
5343 &free_add_partial_attr.attr,
5344 &free_remove_partial_attr.attr,
5345 &alloc_from_partial_attr.attr,
5346 &alloc_slab_attr.attr,
5347 &alloc_refill_attr.attr,
e36a2652 5348 &alloc_node_mismatch_attr.attr,
8ff12cfc
CL
5349 &free_slab_attr.attr,
5350 &cpuslab_flush_attr.attr,
5351 &deactivate_full_attr.attr,
5352 &deactivate_empty_attr.attr,
5353 &deactivate_to_head_attr.attr,
5354 &deactivate_to_tail_attr.attr,
5355 &deactivate_remote_frees_attr.attr,
03e404af 5356 &deactivate_bypass_attr.attr,
65c3376a 5357 &order_fallback_attr.attr,
b789ef51
CL
5358 &cmpxchg_double_fail_attr.attr,
5359 &cmpxchg_double_cpu_fail_attr.attr,
49e22585
CL
5360 &cpu_partial_alloc_attr.attr,
5361 &cpu_partial_free_attr.attr,
8028dcea
AS
5362 &cpu_partial_node_attr.attr,
5363 &cpu_partial_drain_attr.attr,
81819f0f 5364#endif
4c13dd3b
DM
5365#ifdef CONFIG_FAILSLAB
5366 &failslab_attr.attr,
5367#endif
5368
81819f0f
CL
5369 NULL
5370};
5371
5372static struct attribute_group slab_attr_group = {
5373 .attrs = slab_attrs,
5374};
5375
5376static ssize_t slab_attr_show(struct kobject *kobj,
5377 struct attribute *attr,
5378 char *buf)
5379{
5380 struct slab_attribute *attribute;
5381 struct kmem_cache *s;
5382 int err;
5383
5384 attribute = to_slab_attr(attr);
5385 s = to_slab(kobj);
5386
5387 if (!attribute->show)
5388 return -EIO;
5389
5390 err = attribute->show(s, buf);
5391
5392 return err;
5393}
5394
5395static ssize_t slab_attr_store(struct kobject *kobj,
5396 struct attribute *attr,
5397 const char *buf, size_t len)
5398{
5399 struct slab_attribute *attribute;
5400 struct kmem_cache *s;
5401 int err;
5402
5403 attribute = to_slab_attr(attr);
5404 s = to_slab(kobj);
5405
5406 if (!attribute->store)
5407 return -EIO;
5408
5409 err = attribute->store(s, buf, len);
127424c8 5410#ifdef CONFIG_MEMCG
107dab5c 5411 if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
426589f5 5412 struct kmem_cache *c;
81819f0f 5413
107dab5c
GC
5414 mutex_lock(&slab_mutex);
5415 if (s->max_attr_size < len)
5416 s->max_attr_size = len;
5417
ebe945c2
GC
5418 /*
5419 * This is a best effort propagation, so this function's return
5420 * value will be determined by the parent cache only. This is
5421 * basically because not all attributes will have a well
5422 * defined semantics for rollbacks - most of the actions will
5423 * have permanent effects.
5424 *
5425 * Returning the error value of any of the children that fail
5426 * is not 100 % defined, in the sense that users seeing the
5427 * error code won't be able to know anything about the state of
5428 * the cache.
5429 *
5430 * Only returning the error code for the parent cache at least
5431 * has well defined semantics. The cache being written to
5432 * directly either failed or succeeded, in which case we loop
5433 * through the descendants with best-effort propagation.
5434 */
426589f5
VD
5435 for_each_memcg_cache(c, s)
5436 attribute->store(c, buf, len);
107dab5c
GC
5437 mutex_unlock(&slab_mutex);
5438 }
5439#endif
81819f0f
CL
5440 return err;
5441}
5442
107dab5c
GC
5443static void memcg_propagate_slab_attrs(struct kmem_cache *s)
5444{
127424c8 5445#ifdef CONFIG_MEMCG
107dab5c
GC
5446 int i;
5447 char *buffer = NULL;
93030d83 5448 struct kmem_cache *root_cache;
107dab5c 5449
93030d83 5450 if (is_root_cache(s))
107dab5c
GC
5451 return;
5452
f7ce3190 5453 root_cache = s->memcg_params.root_cache;
93030d83 5454
107dab5c
GC
5455 /*
5456 * This mean this cache had no attribute written. Therefore, no point
5457 * in copying default values around
5458 */
93030d83 5459 if (!root_cache->max_attr_size)
107dab5c
GC
5460 return;
5461
5462 for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
5463 char mbuf[64];
5464 char *buf;
5465 struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
5466
5467 if (!attr || !attr->store || !attr->show)
5468 continue;
5469
5470 /*
5471 * It is really bad that we have to allocate here, so we will
5472 * do it only as a fallback. If we actually allocate, though,
5473 * we can just use the allocated buffer until the end.
5474 *
5475 * Most of the slub attributes will tend to be very small in
5476 * size, but sysfs allows buffers up to a page, so they can
5477 * theoretically happen.
5478 */
5479 if (buffer)
5480 buf = buffer;
93030d83 5481 else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
107dab5c
GC
5482 buf = mbuf;
5483 else {
5484 buffer = (char *) get_zeroed_page(GFP_KERNEL);
5485 if (WARN_ON(!buffer))
5486 continue;
5487 buf = buffer;
5488 }
5489
93030d83 5490 attr->show(root_cache, buf);
107dab5c
GC
5491 attr->store(s, buf, strlen(buf));
5492 }
5493
5494 if (buffer)
5495 free_page((unsigned long)buffer);
5496#endif
5497}
5498
41a21285
CL
5499static void kmem_cache_release(struct kobject *k)
5500{
5501 slab_kmem_cache_release(to_slab(k));
5502}
5503
52cf25d0 5504static const struct sysfs_ops slab_sysfs_ops = {
81819f0f
CL
5505 .show = slab_attr_show,
5506 .store = slab_attr_store,
5507};
5508
5509static struct kobj_type slab_ktype = {
5510 .sysfs_ops = &slab_sysfs_ops,
41a21285 5511 .release = kmem_cache_release,
81819f0f
CL
5512};
5513
5514static int uevent_filter(struct kset *kset, struct kobject *kobj)
5515{
5516 struct kobj_type *ktype = get_ktype(kobj);
5517
5518 if (ktype == &slab_ktype)
5519 return 1;
5520 return 0;
5521}
5522
9cd43611 5523static const struct kset_uevent_ops slab_uevent_ops = {
81819f0f
CL
5524 .filter = uevent_filter,
5525};
5526
27c3a314 5527static struct kset *slab_kset;
81819f0f 5528
9a41707b
VD
5529static inline struct kset *cache_kset(struct kmem_cache *s)
5530{
127424c8 5531#ifdef CONFIG_MEMCG
9a41707b 5532 if (!is_root_cache(s))
f7ce3190 5533 return s->memcg_params.root_cache->memcg_kset;
9a41707b
VD
5534#endif
5535 return slab_kset;
5536}
5537
81819f0f
CL
5538#define ID_STR_LENGTH 64
5539
5540/* Create a unique string id for a slab cache:
6446faa2
CL
5541 *
5542 * Format :[flags-]size
81819f0f
CL
5543 */
5544static char *create_unique_id(struct kmem_cache *s)
5545{
5546 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5547 char *p = name;
5548
5549 BUG_ON(!name);
5550
5551 *p++ = ':';
5552 /*
5553 * First flags affecting slabcache operations. We will only
5554 * get here for aliasable slabs so we do not need to support
5555 * too many flags. The flags here must cover all flags that
5556 * are matched during merging to guarantee that the id is
5557 * unique.
5558 */
5559 if (s->flags & SLAB_CACHE_DMA)
5560 *p++ = 'd';
5561 if (s->flags & SLAB_RECLAIM_ACCOUNT)
5562 *p++ = 'a';
becfda68 5563 if (s->flags & SLAB_CONSISTENCY_CHECKS)
81819f0f 5564 *p++ = 'F';
5a896d9e
VN
5565 if (!(s->flags & SLAB_NOTRACK))
5566 *p++ = 't';
230e9fc2
VD
5567 if (s->flags & SLAB_ACCOUNT)
5568 *p++ = 'A';
81819f0f
CL
5569 if (p != name + 1)
5570 *p++ = '-';
5571 p += sprintf(p, "%07d", s->size);
2633d7a0 5572
81819f0f
CL
5573 BUG_ON(p > name + ID_STR_LENGTH - 1);
5574 return name;
5575}
5576
5577static int sysfs_slab_add(struct kmem_cache *s)
5578{
5579 int err;
5580 const char *name;
45530c44 5581 int unmergeable = slab_unmergeable(s);
81819f0f 5582
81819f0f
CL
5583 if (unmergeable) {
5584 /*
5585 * Slabcache can never be merged so we can use the name proper.
5586 * This is typically the case for debug situations. In that
5587 * case we can catch duplicate names easily.
5588 */
27c3a314 5589 sysfs_remove_link(&slab_kset->kobj, s->name);
81819f0f
CL
5590 name = s->name;
5591 } else {
5592 /*
5593 * Create a unique name for the slab as a target
5594 * for the symlinks.
5595 */
5596 name = create_unique_id(s);
5597 }
5598
9a41707b 5599 s->kobj.kset = cache_kset(s);
26e4f205 5600 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
54b6a731 5601 if (err)
80da026a 5602 goto out;
81819f0f
CL
5603
5604 err = sysfs_create_group(&s->kobj, &slab_attr_group);
54b6a731
DJ
5605 if (err)
5606 goto out_del_kobj;
9a41707b 5607
127424c8 5608#ifdef CONFIG_MEMCG
9a41707b
VD
5609 if (is_root_cache(s)) {
5610 s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
5611 if (!s->memcg_kset) {
54b6a731
DJ
5612 err = -ENOMEM;
5613 goto out_del_kobj;
9a41707b
VD
5614 }
5615 }
5616#endif
5617
81819f0f
CL
5618 kobject_uevent(&s->kobj, KOBJ_ADD);
5619 if (!unmergeable) {
5620 /* Setup first alias */
5621 sysfs_slab_alias(s, s->name);
81819f0f 5622 }
54b6a731
DJ
5623out:
5624 if (!unmergeable)
5625 kfree(name);
5626 return err;
5627out_del_kobj:
5628 kobject_del(&s->kobj);
54b6a731 5629 goto out;
81819f0f
CL
5630}
5631
41a21285 5632void sysfs_slab_remove(struct kmem_cache *s)
81819f0f 5633{
97d06609 5634 if (slab_state < FULL)
2bce6485
CL
5635 /*
5636 * Sysfs has not been setup yet so no need to remove the
5637 * cache from sysfs.
5638 */
5639 return;
5640
127424c8 5641#ifdef CONFIG_MEMCG
9a41707b
VD
5642 kset_unregister(s->memcg_kset);
5643#endif
81819f0f
CL
5644 kobject_uevent(&s->kobj, KOBJ_REMOVE);
5645 kobject_del(&s->kobj);
151c602f 5646 kobject_put(&s->kobj);
81819f0f
CL
5647}
5648
5649/*
5650 * Need to buffer aliases during bootup until sysfs becomes
9f6c708e 5651 * available lest we lose that information.
81819f0f
CL
5652 */
5653struct saved_alias {
5654 struct kmem_cache *s;
5655 const char *name;
5656 struct saved_alias *next;
5657};
5658
5af328a5 5659static struct saved_alias *alias_list;
81819f0f
CL
5660
5661static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5662{
5663 struct saved_alias *al;
5664
97d06609 5665 if (slab_state == FULL) {
81819f0f
CL
5666 /*
5667 * If we have a leftover link then remove it.
5668 */
27c3a314
GKH
5669 sysfs_remove_link(&slab_kset->kobj, name);
5670 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
81819f0f
CL
5671 }
5672
5673 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5674 if (!al)
5675 return -ENOMEM;
5676
5677 al->s = s;
5678 al->name = name;
5679 al->next = alias_list;
5680 alias_list = al;
5681 return 0;
5682}
5683
5684static int __init slab_sysfs_init(void)
5685{
5b95a4ac 5686 struct kmem_cache *s;
81819f0f
CL
5687 int err;
5688
18004c5d 5689 mutex_lock(&slab_mutex);
2bce6485 5690
0ff21e46 5691 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
27c3a314 5692 if (!slab_kset) {
18004c5d 5693 mutex_unlock(&slab_mutex);
f9f58285 5694 pr_err("Cannot register slab subsystem.\n");
81819f0f
CL
5695 return -ENOSYS;
5696 }
5697
97d06609 5698 slab_state = FULL;
26a7bd03 5699
5b95a4ac 5700 list_for_each_entry(s, &slab_caches, list) {
26a7bd03 5701 err = sysfs_slab_add(s);
5d540fb7 5702 if (err)
f9f58285
FF
5703 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
5704 s->name);
26a7bd03 5705 }
81819f0f
CL
5706
5707 while (alias_list) {
5708 struct saved_alias *al = alias_list;
5709
5710 alias_list = alias_list->next;
5711 err = sysfs_slab_alias(al->s, al->name);
5d540fb7 5712 if (err)
f9f58285
FF
5713 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
5714 al->name);
81819f0f
CL
5715 kfree(al);
5716 }
5717
18004c5d 5718 mutex_unlock(&slab_mutex);
81819f0f
CL
5719 resiliency_test();
5720 return 0;
5721}
5722
5723__initcall(slab_sysfs_init);
ab4d5ed5 5724#endif /* CONFIG_SYSFS */
57ed3eda
PE
5725
5726/*
5727 * The /proc/slabinfo ABI
5728 */
158a9624 5729#ifdef CONFIG_SLABINFO
0d7561c6 5730void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
57ed3eda 5731{
57ed3eda 5732 unsigned long nr_slabs = 0;
205ab99d
CL
5733 unsigned long nr_objs = 0;
5734 unsigned long nr_free = 0;
57ed3eda 5735 int node;
fa45dc25 5736 struct kmem_cache_node *n;
57ed3eda 5737
fa45dc25 5738 for_each_kmem_cache_node(s, node, n) {
c17fd13e
WL
5739 nr_slabs += node_nr_slabs(n);
5740 nr_objs += node_nr_objs(n);
205ab99d 5741 nr_free += count_partial(n, count_free);
57ed3eda
PE
5742 }
5743
0d7561c6
GC
5744 sinfo->active_objs = nr_objs - nr_free;
5745 sinfo->num_objs = nr_objs;
5746 sinfo->active_slabs = nr_slabs;
5747 sinfo->num_slabs = nr_slabs;
5748 sinfo->objects_per_slab = oo_objects(s->oo);
5749 sinfo->cache_order = oo_order(s->oo);
57ed3eda
PE
5750}
5751
0d7561c6 5752void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
7b3c3a50 5753{
7b3c3a50
AD
5754}
5755
b7454ad3
GC
5756ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5757 size_t count, loff_t *ppos)
7b3c3a50 5758{
b7454ad3 5759 return -EIO;
7b3c3a50 5760}
158a9624 5761#endif /* CONFIG_SLABINFO */