HID: i2c-hid: use "struct i2c_hid" as argument in most calls
[linux-2.6-block.git] / mm / slub.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
81819f0f
CL
2/*
3 * SLUB: A slab allocator that limits cache line use instead of queuing
4 * objects in per cpu and per node lists.
5 *
dc84207d 6 * The allocator synchronizes using per slab locks or atomic operations
881db7fb 7 * and only uses a centralized lock to manage a pool of partial slabs.
81819f0f 8 *
cde53535 9 * (C) 2007 SGI, Christoph Lameter
881db7fb 10 * (C) 2011 Linux Foundation, Christoph Lameter
81819f0f
CL
11 */
12
13#include <linux/mm.h>
1eb5ac64 14#include <linux/swap.h> /* struct reclaim_state */
81819f0f
CL
15#include <linux/module.h>
16#include <linux/bit_spinlock.h>
17#include <linux/interrupt.h>
1b3865d0 18#include <linux/swab.h>
81819f0f
CL
19#include <linux/bitops.h>
20#include <linux/slab.h>
97d06609 21#include "slab.h"
7b3c3a50 22#include <linux/proc_fs.h>
81819f0f 23#include <linux/seq_file.h>
a79316c6 24#include <linux/kasan.h>
81819f0f
CL
25#include <linux/cpu.h>
26#include <linux/cpuset.h>
27#include <linux/mempolicy.h>
28#include <linux/ctype.h>
3ac7fe5a 29#include <linux/debugobjects.h>
81819f0f 30#include <linux/kallsyms.h>
b89fb5ef 31#include <linux/kfence.h>
b9049e23 32#include <linux/memory.h>
f8bd2258 33#include <linux/math64.h>
773ff60e 34#include <linux/fault-inject.h>
bfa71457 35#include <linux/stacktrace.h>
4de900b4 36#include <linux/prefetch.h>
2633d7a0 37#include <linux/memcontrol.h>
2482ddec 38#include <linux/random.h>
1f9f78b1 39#include <kunit/test.h>
81819f0f 40
64dd6849 41#include <linux/debugfs.h>
4a92379b
RK
42#include <trace/events/kmem.h>
43
072bb0aa
MG
44#include "internal.h"
45
81819f0f
CL
46/*
47 * Lock order:
18004c5d 48 * 1. slab_mutex (Global Mutex)
bd0e7491
VB
49 * 2. node->list_lock (Spinlock)
50 * 3. kmem_cache->cpu_slab->lock (Local lock)
c2092c12 51 * 4. slab_lock(slab) (Only on some arches or for debugging)
bd0e7491 52 * 5. object_map_lock (Only for debugging)
81819f0f 53 *
18004c5d 54 * slab_mutex
881db7fb 55 *
18004c5d 56 * The role of the slab_mutex is to protect the list of all the slabs
881db7fb 57 * and to synchronize major metadata changes to slab cache structures.
bd0e7491
VB
58 * Also synchronizes memory hotplug callbacks.
59 *
60 * slab_lock
61 *
62 * The slab_lock is a wrapper around the page lock, thus it is a bit
63 * spinlock.
881db7fb
CL
64 *
65 * The slab_lock is only used for debugging and on arches that do not
b7ccc7f8 66 * have the ability to do a cmpxchg_double. It only protects:
c2092c12
VB
67 * A. slab->freelist -> List of free objects in a slab
68 * B. slab->inuse -> Number of objects in use
69 * C. slab->objects -> Number of objects in slab
70 * D. slab->frozen -> frozen state
881db7fb 71 *
bd0e7491
VB
72 * Frozen slabs
73 *
881db7fb 74 * If a slab is frozen then it is exempt from list management. It is not
632b2ef0 75 * on any list except per cpu partial list. The processor that froze the
c2092c12 76 * slab is the one who can perform list operations on the slab. Other
632b2ef0
LX
77 * processors may put objects onto the freelist but the processor that
78 * froze the slab is the only one that can retrieve the objects from the
c2092c12 79 * slab's freelist.
81819f0f 80 *
bd0e7491
VB
81 * list_lock
82 *
81819f0f
CL
83 * The list_lock protects the partial and full list on each node and
84 * the partial slab counter. If taken then no new slabs may be added or
85 * removed from the lists nor make the number of partial slabs be modified.
86 * (Note that the total number of slabs is an atomic value that may be
87 * modified without taking the list lock).
88 *
89 * The list_lock is a centralized lock and thus we avoid taking it as
90 * much as possible. As long as SLUB does not have to handle partial
91 * slabs, operations can continue without any centralized lock. F.e.
92 * allocating a long series of objects that fill up slabs does not require
93 * the list lock.
bd0e7491
VB
94 *
95 * cpu_slab->lock local lock
96 *
97 * This locks protect slowpath manipulation of all kmem_cache_cpu fields
98 * except the stat counters. This is a percpu structure manipulated only by
99 * the local cpu, so the lock protects against being preempted or interrupted
100 * by an irq. Fast path operations rely on lockless operations instead.
101 * On PREEMPT_RT, the local lock does not actually disable irqs (and thus
102 * prevent the lockless operations), so fastpath operations also need to take
103 * the lock and are no longer lockless.
104 *
105 * lockless fastpaths
106 *
107 * The fast path allocation (slab_alloc_node()) and freeing (do_slab_free())
108 * are fully lockless when satisfied from the percpu slab (and when
109 * cmpxchg_double is possible to use, otherwise slab_lock is taken).
110 * They also don't disable preemption or migration or irqs. They rely on
111 * the transaction id (tid) field to detect being preempted or moved to
112 * another cpu.
113 *
114 * irq, preemption, migration considerations
115 *
116 * Interrupts are disabled as part of list_lock or local_lock operations, or
117 * around the slab_lock operation, in order to make the slab allocator safe
118 * to use in the context of an irq.
119 *
120 * In addition, preemption (or migration on PREEMPT_RT) is disabled in the
121 * allocation slowpath, bulk allocation, and put_cpu_partial(), so that the
122 * local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer
123 * doesn't have to be revalidated in each section protected by the local lock.
81819f0f
CL
124 *
125 * SLUB assigns one slab for allocation to each processor.
126 * Allocations only occur from these slabs called cpu slabs.
127 *
672bba3a
CL
128 * Slabs with free elements are kept on a partial list and during regular
129 * operations no list for full slabs is used. If an object in a full slab is
81819f0f 130 * freed then the slab will show up again on the partial lists.
672bba3a
CL
131 * We track full slabs for debugging purposes though because otherwise we
132 * cannot scan all objects.
81819f0f
CL
133 *
134 * Slabs are freed when they become empty. Teardown and setup is
135 * minimal so we rely on the page allocators per cpu caches for
136 * fast frees and allocs.
137 *
c2092c12 138 * slab->frozen The slab is frozen and exempt from list processing.
4b6f0750
CL
139 * This means that the slab is dedicated to a purpose
140 * such as satisfying allocations for a specific
141 * processor. Objects may be freed in the slab while
142 * it is frozen but slab_free will then skip the usual
143 * list operations. It is up to the processor holding
144 * the slab to integrate the slab into the slab lists
145 * when the slab is no longer needed.
146 *
147 * One use of this flag is to mark slabs that are
148 * used for allocations. Then such a slab becomes a cpu
149 * slab. The cpu slab may be equipped with an additional
dfb4f096 150 * freelist that allows lockless access to
894b8788
CL
151 * free objects in addition to the regular freelist
152 * that requires the slab lock.
81819f0f 153 *
aed68148 154 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
81819f0f 155 * options set. This moves slab handling out of
894b8788 156 * the fast path and disables lockless freelists.
81819f0f
CL
157 */
158
25c00c50
VB
159/*
160 * We could simply use migrate_disable()/enable() but as long as it's a
161 * function call even on !PREEMPT_RT, use inline preempt_disable() there.
162 */
163#ifndef CONFIG_PREEMPT_RT
164#define slub_get_cpu_ptr(var) get_cpu_ptr(var)
165#define slub_put_cpu_ptr(var) put_cpu_ptr(var)
166#else
167#define slub_get_cpu_ptr(var) \
168({ \
169 migrate_disable(); \
170 this_cpu_ptr(var); \
171})
172#define slub_put_cpu_ptr(var) \
173do { \
174 (void)(var); \
175 migrate_enable(); \
176} while (0)
177#endif
178
ca0cab65
VB
179#ifdef CONFIG_SLUB_DEBUG
180#ifdef CONFIG_SLUB_DEBUG_ON
181DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
182#else
183DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
184#endif
79270291 185#endif /* CONFIG_SLUB_DEBUG */
ca0cab65 186
59052e89
VB
187static inline bool kmem_cache_debug(struct kmem_cache *s)
188{
189 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
af537b0a 190}
5577bd8a 191
117d54df 192void *fixup_red_left(struct kmem_cache *s, void *p)
d86bd1be 193{
59052e89 194 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
d86bd1be
JK
195 p += s->red_left_pad;
196
197 return p;
198}
199
345c905d
JK
200static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
201{
202#ifdef CONFIG_SLUB_CPU_PARTIAL
203 return !kmem_cache_debug(s);
204#else
205 return false;
206#endif
207}
208
81819f0f
CL
209/*
210 * Issues still to be resolved:
211 *
81819f0f
CL
212 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
213 *
81819f0f
CL
214 * - Variable sizing of the per node arrays
215 */
216
b789ef51
CL
217/* Enable to log cmpxchg failures */
218#undef SLUB_DEBUG_CMPXCHG
219
2086d26a 220/*
dc84207d 221 * Minimum number of partial slabs. These will be left on the partial
2086d26a
CL
222 * lists even if they are empty. kmem_cache_shrink may reclaim them.
223 */
76be8950 224#define MIN_PARTIAL 5
e95eed57 225
2086d26a
CL
226/*
227 * Maximum number of desirable partial slabs.
228 * The existence of more partial slabs makes kmem_cache_shrink
721ae22a 229 * sort the partial list by the number of objects in use.
2086d26a
CL
230 */
231#define MAX_PARTIAL 10
232
becfda68 233#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
81819f0f 234 SLAB_POISON | SLAB_STORE_USER)
672bba3a 235
149daaf3
LA
236/*
237 * These debug flags cannot use CMPXCHG because there might be consistency
238 * issues when checking or reading debug information
239 */
240#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
241 SLAB_TRACE)
242
243
fa5ec8a1 244/*
3de47213
DR
245 * Debugging flags that require metadata to be stored in the slab. These get
246 * disabled when slub_debug=O is used and a cache's min order increases with
247 * metadata.
fa5ec8a1 248 */
3de47213 249#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
fa5ec8a1 250
210b5c06
CG
251#define OO_SHIFT 16
252#define OO_MASK ((1 << OO_SHIFT) - 1)
c2092c12 253#define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */
210b5c06 254
81819f0f 255/* Internal SLUB flags */
d50112ed 256/* Poison object */
4fd0b46e 257#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
d50112ed 258/* Use cmpxchg_double */
4fd0b46e 259#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
81819f0f 260
02cbc874
CL
261/*
262 * Tracking user of a slab.
263 */
d6543e39 264#define TRACK_ADDRS_COUNT 16
02cbc874 265struct track {
ce71e27c 266 unsigned long addr; /* Called from address */
ae14c63a
LT
267#ifdef CONFIG_STACKTRACE
268 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
d6543e39 269#endif
02cbc874
CL
270 int cpu; /* Was running on cpu */
271 int pid; /* Pid context */
272 unsigned long when; /* When did the operation occur */
273};
274
275enum track_item { TRACK_ALLOC, TRACK_FREE };
276
ab4d5ed5 277#ifdef CONFIG_SYSFS
81819f0f
CL
278static int sysfs_slab_add(struct kmem_cache *);
279static int sysfs_slab_alias(struct kmem_cache *, const char *);
81819f0f 280#else
0c710013
CL
281static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
282static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
283 { return 0; }
81819f0f
CL
284#endif
285
64dd6849
FM
286#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
287static void debugfs_slab_add(struct kmem_cache *);
288#else
289static inline void debugfs_slab_add(struct kmem_cache *s) { }
290#endif
291
4fdccdfb 292static inline void stat(const struct kmem_cache *s, enum stat_item si)
8ff12cfc
CL
293{
294#ifdef CONFIG_SLUB_STATS
88da03a6
CL
295 /*
296 * The rmw is racy on a preemptible kernel but this is acceptable, so
297 * avoid this_cpu_add()'s irq-disable overhead.
298 */
299 raw_cpu_inc(s->cpu_slab->stat[si]);
8ff12cfc
CL
300#endif
301}
302
7e1fa93d
VB
303/*
304 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
305 * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
306 * differ during memory hotplug/hotremove operations.
307 * Protected by slab_mutex.
308 */
309static nodemask_t slab_nodes;
310
81819f0f
CL
311/********************************************************************
312 * Core slab cache functions
313 *******************************************************************/
314
2482ddec
KC
315/*
316 * Returns freelist pointer (ptr). With hardening, this is obfuscated
317 * with an XOR of the address where the pointer is held and a per-cache
318 * random number.
319 */
320static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
321 unsigned long ptr_addr)
322{
323#ifdef CONFIG_SLAB_FREELIST_HARDENED
d36a63a9 324 /*
aa1ef4d7 325 * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
d36a63a9
AK
326 * Normally, this doesn't cause any issues, as both set_freepointer()
327 * and get_freepointer() are called with a pointer with the same tag.
328 * However, there are some issues with CONFIG_SLUB_DEBUG code. For
329 * example, when __free_slub() iterates over objects in a cache, it
330 * passes untagged pointers to check_object(). check_object() in turns
331 * calls get_freepointer() with an untagged pointer, which causes the
332 * freepointer to be restored incorrectly.
333 */
334 return (void *)((unsigned long)ptr ^ s->random ^
1ad53d9f 335 swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));
2482ddec
KC
336#else
337 return ptr;
338#endif
339}
340
341/* Returns the freelist pointer recorded at location ptr_addr. */
342static inline void *freelist_dereference(const struct kmem_cache *s,
343 void *ptr_addr)
344{
345 return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
346 (unsigned long)ptr_addr);
347}
348
7656c72b
CL
349static inline void *get_freepointer(struct kmem_cache *s, void *object)
350{
aa1ef4d7 351 object = kasan_reset_tag(object);
2482ddec 352 return freelist_dereference(s, object + s->offset);
7656c72b
CL
353}
354
0ad9500e
ED
355static void prefetch_freepointer(const struct kmem_cache *s, void *object)
356{
04b4b006 357 prefetchw(object + s->offset);
0ad9500e
ED
358}
359
1393d9a1
CL
360static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
361{
2482ddec 362 unsigned long freepointer_addr;
1393d9a1
CL
363 void *p;
364
8e57f8ac 365 if (!debug_pagealloc_enabled_static())
922d566c
JK
366 return get_freepointer(s, object);
367
f70b0049 368 object = kasan_reset_tag(object);
2482ddec 369 freepointer_addr = (unsigned long)object + s->offset;
fe557319 370 copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
2482ddec 371 return freelist_ptr(s, p, freepointer_addr);
1393d9a1
CL
372}
373
7656c72b
CL
374static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
375{
2482ddec
KC
376 unsigned long freeptr_addr = (unsigned long)object + s->offset;
377
ce6fa91b
AP
378#ifdef CONFIG_SLAB_FREELIST_HARDENED
379 BUG_ON(object == fp); /* naive detection of double free or corruption */
380#endif
381
aa1ef4d7 382 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
2482ddec 383 *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
7656c72b
CL
384}
385
386/* Loop over all objects in a slab */
224a88be 387#define for_each_object(__p, __s, __addr, __objects) \
d86bd1be
JK
388 for (__p = fixup_red_left(__s, __addr); \
389 __p < (__addr) + (__objects) * (__s)->size; \
390 __p += (__s)->size)
7656c72b 391
9736d2a9 392static inline unsigned int order_objects(unsigned int order, unsigned int size)
ab9a0f19 393{
9736d2a9 394 return ((unsigned int)PAGE_SIZE << order) / size;
ab9a0f19
LJ
395}
396
19af27af 397static inline struct kmem_cache_order_objects oo_make(unsigned int order,
9736d2a9 398 unsigned int size)
834f3d11
CL
399{
400 struct kmem_cache_order_objects x = {
9736d2a9 401 (order << OO_SHIFT) + order_objects(order, size)
834f3d11
CL
402 };
403
404 return x;
405}
406
19af27af 407static inline unsigned int oo_order(struct kmem_cache_order_objects x)
834f3d11 408{
210b5c06 409 return x.x >> OO_SHIFT;
834f3d11
CL
410}
411
19af27af 412static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
834f3d11 413{
210b5c06 414 return x.x & OO_MASK;
834f3d11
CL
415}
416
b47291ef
VB
417#ifdef CONFIG_SLUB_CPU_PARTIAL
418static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
419{
bb192ed9 420 unsigned int nr_slabs;
b47291ef
VB
421
422 s->cpu_partial = nr_objects;
423
424 /*
425 * We take the number of objects but actually limit the number of
c2092c12
VB
426 * slabs on the per cpu partial list, in order to limit excessive
427 * growth of the list. For simplicity we assume that the slabs will
b47291ef
VB
428 * be half-full.
429 */
bb192ed9
VB
430 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
431 s->cpu_partial_slabs = nr_slabs;
b47291ef
VB
432}
433#else
434static inline void
435slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
436{
437}
438#endif /* CONFIG_SLUB_CPU_PARTIAL */
439
881db7fb
CL
440/*
441 * Per slab locking using the pagelock
442 */
0393895b 443static __always_inline void __slab_lock(struct slab *slab)
881db7fb 444{
0393895b
VB
445 struct page *page = slab_page(slab);
446
48c935ad 447 VM_BUG_ON_PAGE(PageTail(page), page);
881db7fb
CL
448 bit_spin_lock(PG_locked, &page->flags);
449}
450
0393895b 451static __always_inline void __slab_unlock(struct slab *slab)
881db7fb 452{
0393895b
VB
453 struct page *page = slab_page(slab);
454
48c935ad 455 VM_BUG_ON_PAGE(PageTail(page), page);
881db7fb
CL
456 __bit_spin_unlock(PG_locked, &page->flags);
457}
458
bb192ed9 459static __always_inline void slab_lock(struct slab *slab, unsigned long *flags)
a2b4ae8b
VB
460{
461 if (IS_ENABLED(CONFIG_PREEMPT_RT))
462 local_irq_save(*flags);
bb192ed9 463 __slab_lock(slab);
a2b4ae8b
VB
464}
465
bb192ed9 466static __always_inline void slab_unlock(struct slab *slab, unsigned long *flags)
a2b4ae8b 467{
bb192ed9 468 __slab_unlock(slab);
a2b4ae8b
VB
469 if (IS_ENABLED(CONFIG_PREEMPT_RT))
470 local_irq_restore(*flags);
471}
472
473/*
474 * Interrupts must be disabled (for the fallback code to work right), typically
475 * by an _irqsave() lock variant. Except on PREEMPT_RT where locks are different
476 * so we disable interrupts as part of slab_[un]lock().
477 */
bb192ed9 478static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,
1d07171c
CL
479 void *freelist_old, unsigned long counters_old,
480 void *freelist_new, unsigned long counters_new,
481 const char *n)
482{
a2b4ae8b
VB
483 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
484 lockdep_assert_irqs_disabled();
2565409f
HC
485#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
486 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
1d07171c 487 if (s->flags & __CMPXCHG_DOUBLE) {
bb192ed9 488 if (cmpxchg_double(&slab->freelist, &slab->counters,
0aa9a13d
DC
489 freelist_old, counters_old,
490 freelist_new, counters_new))
6f6528a1 491 return true;
1d07171c
CL
492 } else
493#endif
494 {
a2b4ae8b
VB
495 /* init to 0 to prevent spurious warnings */
496 unsigned long flags = 0;
497
bb192ed9
VB
498 slab_lock(slab, &flags);
499 if (slab->freelist == freelist_old &&
500 slab->counters == counters_old) {
501 slab->freelist = freelist_new;
502 slab->counters = counters_new;
503 slab_unlock(slab, &flags);
6f6528a1 504 return true;
1d07171c 505 }
bb192ed9 506 slab_unlock(slab, &flags);
1d07171c
CL
507 }
508
509 cpu_relax();
510 stat(s, CMPXCHG_DOUBLE_FAIL);
511
512#ifdef SLUB_DEBUG_CMPXCHG
f9f58285 513 pr_info("%s %s: cmpxchg double redo ", n, s->name);
1d07171c
CL
514#endif
515
6f6528a1 516 return false;
1d07171c
CL
517}
518
bb192ed9 519static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,
b789ef51
CL
520 void *freelist_old, unsigned long counters_old,
521 void *freelist_new, unsigned long counters_new,
522 const char *n)
523{
2565409f
HC
524#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
525 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
b789ef51 526 if (s->flags & __CMPXCHG_DOUBLE) {
bb192ed9 527 if (cmpxchg_double(&slab->freelist, &slab->counters,
0aa9a13d
DC
528 freelist_old, counters_old,
529 freelist_new, counters_new))
6f6528a1 530 return true;
b789ef51
CL
531 } else
532#endif
533 {
1d07171c
CL
534 unsigned long flags;
535
536 local_irq_save(flags);
bb192ed9
VB
537 __slab_lock(slab);
538 if (slab->freelist == freelist_old &&
539 slab->counters == counters_old) {
540 slab->freelist = freelist_new;
541 slab->counters = counters_new;
542 __slab_unlock(slab);
1d07171c 543 local_irq_restore(flags);
6f6528a1 544 return true;
b789ef51 545 }
bb192ed9 546 __slab_unlock(slab);
1d07171c 547 local_irq_restore(flags);
b789ef51
CL
548 }
549
550 cpu_relax();
551 stat(s, CMPXCHG_DOUBLE_FAIL);
552
553#ifdef SLUB_DEBUG_CMPXCHG
f9f58285 554 pr_info("%s %s: cmpxchg double redo ", n, s->name);
b789ef51
CL
555#endif
556
6f6528a1 557 return false;
b789ef51
CL
558}
559
41ecc55b 560#ifdef CONFIG_SLUB_DEBUG
90e9f6a6 561static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
94ef0304 562static DEFINE_RAW_SPINLOCK(object_map_lock);
90e9f6a6 563
b3fd64e1 564static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
bb192ed9 565 struct slab *slab)
b3fd64e1 566{
bb192ed9 567 void *addr = slab_address(slab);
b3fd64e1
VB
568 void *p;
569
bb192ed9 570 bitmap_zero(obj_map, slab->objects);
b3fd64e1 571
bb192ed9 572 for (p = slab->freelist; p; p = get_freepointer(s, p))
b3fd64e1
VB
573 set_bit(__obj_to_index(s, addr, p), obj_map);
574}
575
1f9f78b1
OG
576#if IS_ENABLED(CONFIG_KUNIT)
577static bool slab_add_kunit_errors(void)
578{
579 struct kunit_resource *resource;
580
581 if (likely(!current->kunit_test))
582 return false;
583
584 resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
585 if (!resource)
586 return false;
587
588 (*(int *)resource->data)++;
589 kunit_put_resource(resource);
590 return true;
591}
592#else
593static inline bool slab_add_kunit_errors(void) { return false; }
594#endif
595
5f80b13a 596/*
c2092c12 597 * Determine a map of objects in use in a slab.
5f80b13a 598 *
c2092c12 599 * Node listlock must be held to guarantee that the slab does
5f80b13a
CL
600 * not vanish from under us.
601 */
bb192ed9 602static unsigned long *get_map(struct kmem_cache *s, struct slab *slab)
31364c2e 603 __acquires(&object_map_lock)
5f80b13a 604{
90e9f6a6
YZ
605 VM_BUG_ON(!irqs_disabled());
606
94ef0304 607 raw_spin_lock(&object_map_lock);
90e9f6a6 608
bb192ed9 609 __fill_map(object_map, s, slab);
90e9f6a6
YZ
610
611 return object_map;
612}
613
81aba9e0 614static void put_map(unsigned long *map) __releases(&object_map_lock)
90e9f6a6
YZ
615{
616 VM_BUG_ON(map != object_map);
94ef0304 617 raw_spin_unlock(&object_map_lock);
5f80b13a
CL
618}
619
870b1fbb 620static inline unsigned int size_from_object(struct kmem_cache *s)
d86bd1be
JK
621{
622 if (s->flags & SLAB_RED_ZONE)
623 return s->size - s->red_left_pad;
624
625 return s->size;
626}
627
628static inline void *restore_red_left(struct kmem_cache *s, void *p)
629{
630 if (s->flags & SLAB_RED_ZONE)
631 p -= s->red_left_pad;
632
633 return p;
634}
635
41ecc55b
CL
636/*
637 * Debug settings:
638 */
89d3c87e 639#if defined(CONFIG_SLUB_DEBUG_ON)
d50112ed 640static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
f0630fff 641#else
d50112ed 642static slab_flags_t slub_debug;
f0630fff 643#endif
41ecc55b 644
e17f1dfb 645static char *slub_debug_string;
fa5ec8a1 646static int disable_higher_order_debug;
41ecc55b 647
a79316c6
AR
648/*
649 * slub is about to manipulate internal object metadata. This memory lies
650 * outside the range of the allocated object, so accessing it would normally
651 * be reported by kasan as a bounds error. metadata_access_enable() is used
652 * to tell kasan that these accesses are OK.
653 */
654static inline void metadata_access_enable(void)
655{
656 kasan_disable_current();
657}
658
659static inline void metadata_access_disable(void)
660{
661 kasan_enable_current();
662}
663
81819f0f
CL
664/*
665 * Object debugging
666 */
d86bd1be
JK
667
668/* Verify that a pointer has an address that is valid within a slab page */
669static inline int check_valid_pointer(struct kmem_cache *s,
bb192ed9 670 struct slab *slab, void *object)
d86bd1be
JK
671{
672 void *base;
673
674 if (!object)
675 return 1;
676
bb192ed9 677 base = slab_address(slab);
338cfaad 678 object = kasan_reset_tag(object);
d86bd1be 679 object = restore_red_left(s, object);
bb192ed9 680 if (object < base || object >= base + slab->objects * s->size ||
d86bd1be
JK
681 (object - base) % s->size) {
682 return 0;
683 }
684
685 return 1;
686}
687
aa2efd5e
DT
688static void print_section(char *level, char *text, u8 *addr,
689 unsigned int length)
81819f0f 690{
a79316c6 691 metadata_access_enable();
340caf17
KYL
692 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
693 16, 1, kasan_reset_tag((void *)addr), length, 1);
a79316c6 694 metadata_access_disable();
81819f0f
CL
695}
696
cbfc35a4
WL
697/*
698 * See comment in calculate_sizes().
699 */
700static inline bool freeptr_outside_object(struct kmem_cache *s)
701{
702 return s->offset >= s->inuse;
703}
704
705/*
706 * Return offset of the end of info block which is inuse + free pointer if
707 * not overlapping with object.
708 */
709static inline unsigned int get_info_end(struct kmem_cache *s)
710{
711 if (freeptr_outside_object(s))
712 return s->inuse + sizeof(void *);
713 else
714 return s->inuse;
715}
716
81819f0f
CL
717static struct track *get_track(struct kmem_cache *s, void *object,
718 enum track_item alloc)
719{
720 struct track *p;
721
cbfc35a4 722 p = object + get_info_end(s);
81819f0f 723
aa1ef4d7 724 return kasan_reset_tag(p + alloc);
81819f0f
CL
725}
726
727static void set_track(struct kmem_cache *s, void *object,
ce71e27c 728 enum track_item alloc, unsigned long addr)
81819f0f 729{
1a00df4a 730 struct track *p = get_track(s, object, alloc);
81819f0f 731
81819f0f 732 if (addr) {
ae14c63a
LT
733#ifdef CONFIG_STACKTRACE
734 unsigned int nr_entries;
735
736 metadata_access_enable();
737 nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
738 TRACK_ADDRS_COUNT, 3);
739 metadata_access_disable();
740
741 if (nr_entries < TRACK_ADDRS_COUNT)
742 p->addrs[nr_entries] = 0;
d6543e39 743#endif
81819f0f
CL
744 p->addr = addr;
745 p->cpu = smp_processor_id();
88e4ccf2 746 p->pid = current->pid;
81819f0f 747 p->when = jiffies;
b8ca7ff7 748 } else {
81819f0f 749 memset(p, 0, sizeof(struct track));
b8ca7ff7 750 }
81819f0f
CL
751}
752
81819f0f
CL
753static void init_tracking(struct kmem_cache *s, void *object)
754{
24922684
CL
755 if (!(s->flags & SLAB_STORE_USER))
756 return;
757
ce71e27c
EGM
758 set_track(s, object, TRACK_FREE, 0UL);
759 set_track(s, object, TRACK_ALLOC, 0UL);
81819f0f
CL
760}
761
86609d33 762static void print_track(const char *s, struct track *t, unsigned long pr_time)
81819f0f
CL
763{
764 if (!t->addr)
765 return;
766
96b94abc 767 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
86609d33 768 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
ae14c63a 769#ifdef CONFIG_STACKTRACE
d6543e39 770 {
ae14c63a
LT
771 int i;
772 for (i = 0; i < TRACK_ADDRS_COUNT; i++)
773 if (t->addrs[i])
774 pr_err("\t%pS\n", (void *)t->addrs[i]);
775 else
776 break;
d6543e39
BG
777 }
778#endif
24922684
CL
779}
780
e42f174e 781void print_tracking(struct kmem_cache *s, void *object)
24922684 782{
86609d33 783 unsigned long pr_time = jiffies;
24922684
CL
784 if (!(s->flags & SLAB_STORE_USER))
785 return;
786
86609d33
CP
787 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
788 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
24922684
CL
789}
790
fb012e27 791static void print_slab_info(const struct slab *slab)
24922684 792{
fb012e27 793 struct folio *folio = (struct folio *)slab_folio(slab);
24922684 794
fb012e27
MWO
795 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
796 slab, slab->objects, slab->inuse, slab->freelist,
797 folio_flags(folio, 0));
24922684
CL
798}
799
800static void slab_bug(struct kmem_cache *s, char *fmt, ...)
801{
ecc42fbe 802 struct va_format vaf;
24922684 803 va_list args;
24922684
CL
804
805 va_start(args, fmt);
ecc42fbe
FF
806 vaf.fmt = fmt;
807 vaf.va = &args;
f9f58285 808 pr_err("=============================================================================\n");
ecc42fbe 809 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
f9f58285 810 pr_err("-----------------------------------------------------------------------------\n\n");
ecc42fbe 811 va_end(args);
81819f0f
CL
812}
813
582d1212 814__printf(2, 3)
24922684
CL
815static void slab_fix(struct kmem_cache *s, char *fmt, ...)
816{
ecc42fbe 817 struct va_format vaf;
24922684 818 va_list args;
24922684 819
1f9f78b1
OG
820 if (slab_add_kunit_errors())
821 return;
822
24922684 823 va_start(args, fmt);
ecc42fbe
FF
824 vaf.fmt = fmt;
825 vaf.va = &args;
826 pr_err("FIX %s: %pV\n", s->name, &vaf);
24922684 827 va_end(args);
24922684
CL
828}
829
bb192ed9 830static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
81819f0f
CL
831{
832 unsigned int off; /* Offset of last byte */
bb192ed9 833 u8 *addr = slab_address(slab);
24922684
CL
834
835 print_tracking(s, p);
836
bb192ed9 837 print_slab_info(slab);
24922684 838
96b94abc 839 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
f9f58285 840 p, p - addr, get_freepointer(s, p));
24922684 841
d86bd1be 842 if (s->flags & SLAB_RED_ZONE)
8669dbab 843 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
aa2efd5e 844 s->red_left_pad);
d86bd1be 845 else if (p > addr + 16)
aa2efd5e 846 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
81819f0f 847
8669dbab 848 print_section(KERN_ERR, "Object ", p,
1b473f29 849 min_t(unsigned int, s->object_size, PAGE_SIZE));
81819f0f 850 if (s->flags & SLAB_RED_ZONE)
8669dbab 851 print_section(KERN_ERR, "Redzone ", p + s->object_size,
3b0efdfa 852 s->inuse - s->object_size);
81819f0f 853
cbfc35a4 854 off = get_info_end(s);
81819f0f 855
24922684 856 if (s->flags & SLAB_STORE_USER)
81819f0f 857 off += 2 * sizeof(struct track);
81819f0f 858
80a9201a
AP
859 off += kasan_metadata_size(s);
860
d86bd1be 861 if (off != size_from_object(s))
81819f0f 862 /* Beginning of the filler is the free pointer */
8669dbab 863 print_section(KERN_ERR, "Padding ", p + off,
aa2efd5e 864 size_from_object(s) - off);
24922684
CL
865
866 dump_stack();
81819f0f
CL
867}
868
bb192ed9 869static void object_err(struct kmem_cache *s, struct slab *slab,
81819f0f
CL
870 u8 *object, char *reason)
871{
1f9f78b1
OG
872 if (slab_add_kunit_errors())
873 return;
874
3dc50637 875 slab_bug(s, "%s", reason);
bb192ed9 876 print_trailer(s, slab, object);
65ebdeef 877 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
81819f0f
CL
878}
879
bb192ed9 880static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
ae16d059
VB
881 void **freelist, void *nextfree)
882{
883 if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
bb192ed9
VB
884 !check_valid_pointer(s, slab, nextfree) && freelist) {
885 object_err(s, slab, *freelist, "Freechain corrupt");
ae16d059
VB
886 *freelist = NULL;
887 slab_fix(s, "Isolate corrupted freechain");
888 return true;
889 }
890
891 return false;
892}
893
bb192ed9 894static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
d0e0ac97 895 const char *fmt, ...)
81819f0f
CL
896{
897 va_list args;
898 char buf[100];
899
1f9f78b1
OG
900 if (slab_add_kunit_errors())
901 return;
902
24922684
CL
903 va_start(args, fmt);
904 vsnprintf(buf, sizeof(buf), fmt, args);
81819f0f 905 va_end(args);
3dc50637 906 slab_bug(s, "%s", buf);
bb192ed9 907 print_slab_info(slab);
81819f0f 908 dump_stack();
65ebdeef 909 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
81819f0f
CL
910}
911
f7cb1933 912static void init_object(struct kmem_cache *s, void *object, u8 val)
81819f0f 913{
aa1ef4d7 914 u8 *p = kasan_reset_tag(object);
81819f0f 915
d86bd1be
JK
916 if (s->flags & SLAB_RED_ZONE)
917 memset(p - s->red_left_pad, val, s->red_left_pad);
918
81819f0f 919 if (s->flags & __OBJECT_POISON) {
3b0efdfa
CL
920 memset(p, POISON_FREE, s->object_size - 1);
921 p[s->object_size - 1] = POISON_END;
81819f0f
CL
922 }
923
924 if (s->flags & SLAB_RED_ZONE)
3b0efdfa 925 memset(p + s->object_size, val, s->inuse - s->object_size);
81819f0f
CL
926}
927
24922684
CL
928static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
929 void *from, void *to)
930{
582d1212 931 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
24922684
CL
932 memset(from, data, to - from);
933}
934
bb192ed9 935static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
24922684 936 u8 *object, char *what,
06428780 937 u8 *start, unsigned int value, unsigned int bytes)
24922684
CL
938{
939 u8 *fault;
940 u8 *end;
bb192ed9 941 u8 *addr = slab_address(slab);
24922684 942
a79316c6 943 metadata_access_enable();
aa1ef4d7 944 fault = memchr_inv(kasan_reset_tag(start), value, bytes);
a79316c6 945 metadata_access_disable();
24922684
CL
946 if (!fault)
947 return 1;
948
949 end = start + bytes;
950 while (end > fault && end[-1] == value)
951 end--;
952
1f9f78b1
OG
953 if (slab_add_kunit_errors())
954 goto skip_bug_print;
955
24922684 956 slab_bug(s, "%s overwritten", what);
96b94abc 957 pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
e1b70dd1
MC
958 fault, end - 1, fault - addr,
959 fault[0], value);
bb192ed9 960 print_trailer(s, slab, object);
65ebdeef 961 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
24922684 962
1f9f78b1 963skip_bug_print:
24922684
CL
964 restore_bytes(s, what, value, fault, end);
965 return 0;
81819f0f
CL
966}
967
81819f0f
CL
968/*
969 * Object layout:
970 *
971 * object address
972 * Bytes of the object to be managed.
973 * If the freepointer may overlay the object then the free
cbfc35a4 974 * pointer is at the middle of the object.
672bba3a 975 *
81819f0f
CL
976 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
977 * 0xa5 (POISON_END)
978 *
3b0efdfa 979 * object + s->object_size
81819f0f 980 * Padding to reach word boundary. This is also used for Redzoning.
672bba3a 981 * Padding is extended by another word if Redzoning is enabled and
3b0efdfa 982 * object_size == inuse.
672bba3a 983 *
81819f0f
CL
984 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
985 * 0xcc (RED_ACTIVE) for objects in use.
986 *
987 * object + s->inuse
672bba3a
CL
988 * Meta data starts here.
989 *
81819f0f
CL
990 * A. Free pointer (if we cannot overwrite object on free)
991 * B. Tracking data for SLAB_STORE_USER
dc84207d 992 * C. Padding to reach required alignment boundary or at minimum
6446faa2 993 * one word if debugging is on to be able to detect writes
672bba3a
CL
994 * before the word boundary.
995 *
996 * Padding is done using 0x5a (POISON_INUSE)
81819f0f
CL
997 *
998 * object + s->size
672bba3a 999 * Nothing is used beyond s->size.
81819f0f 1000 *
3b0efdfa 1001 * If slabcaches are merged then the object_size and inuse boundaries are mostly
672bba3a 1002 * ignored. And therefore no slab options that rely on these boundaries
81819f0f
CL
1003 * may be used with merged slabcaches.
1004 */
1005
bb192ed9 1006static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
81819f0f 1007{
cbfc35a4 1008 unsigned long off = get_info_end(s); /* The end of info */
81819f0f
CL
1009
1010 if (s->flags & SLAB_STORE_USER)
1011 /* We also have user information there */
1012 off += 2 * sizeof(struct track);
1013
80a9201a
AP
1014 off += kasan_metadata_size(s);
1015
d86bd1be 1016 if (size_from_object(s) == off)
81819f0f
CL
1017 return 1;
1018
bb192ed9 1019 return check_bytes_and_report(s, slab, p, "Object padding",
d86bd1be 1020 p + off, POISON_INUSE, size_from_object(s) - off);
81819f0f
CL
1021}
1022
39b26464 1023/* Check the pad bytes at the end of a slab page */
bb192ed9 1024static int slab_pad_check(struct kmem_cache *s, struct slab *slab)
81819f0f 1025{
24922684
CL
1026 u8 *start;
1027 u8 *fault;
1028 u8 *end;
5d682681 1029 u8 *pad;
24922684
CL
1030 int length;
1031 int remainder;
81819f0f
CL
1032
1033 if (!(s->flags & SLAB_POISON))
1034 return 1;
1035
bb192ed9
VB
1036 start = slab_address(slab);
1037 length = slab_size(slab);
39b26464
CL
1038 end = start + length;
1039 remainder = length % s->size;
81819f0f
CL
1040 if (!remainder)
1041 return 1;
1042
5d682681 1043 pad = end - remainder;
a79316c6 1044 metadata_access_enable();
aa1ef4d7 1045 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
a79316c6 1046 metadata_access_disable();
24922684
CL
1047 if (!fault)
1048 return 1;
1049 while (end > fault && end[-1] == POISON_INUSE)
1050 end--;
1051
bb192ed9 1052 slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
e1b70dd1 1053 fault, end - 1, fault - start);
5d682681 1054 print_section(KERN_ERR, "Padding ", pad, remainder);
24922684 1055
5d682681 1056 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
24922684 1057 return 0;
81819f0f
CL
1058}
1059
bb192ed9 1060static int check_object(struct kmem_cache *s, struct slab *slab,
f7cb1933 1061 void *object, u8 val)
81819f0f
CL
1062{
1063 u8 *p = object;
3b0efdfa 1064 u8 *endobject = object + s->object_size;
81819f0f
CL
1065
1066 if (s->flags & SLAB_RED_ZONE) {
bb192ed9 1067 if (!check_bytes_and_report(s, slab, object, "Left Redzone",
d86bd1be
JK
1068 object - s->red_left_pad, val, s->red_left_pad))
1069 return 0;
1070
bb192ed9 1071 if (!check_bytes_and_report(s, slab, object, "Right Redzone",
3b0efdfa 1072 endobject, val, s->inuse - s->object_size))
81819f0f 1073 return 0;
81819f0f 1074 } else {
3b0efdfa 1075 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
bb192ed9 1076 check_bytes_and_report(s, slab, p, "Alignment padding",
d0e0ac97
CG
1077 endobject, POISON_INUSE,
1078 s->inuse - s->object_size);
3adbefee 1079 }
81819f0f
CL
1080 }
1081
1082 if (s->flags & SLAB_POISON) {
f7cb1933 1083 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
bb192ed9 1084 (!check_bytes_and_report(s, slab, p, "Poison", p,
3b0efdfa 1085 POISON_FREE, s->object_size - 1) ||
bb192ed9 1086 !check_bytes_and_report(s, slab, p, "End Poison",
3b0efdfa 1087 p + s->object_size - 1, POISON_END, 1)))
81819f0f 1088 return 0;
81819f0f
CL
1089 /*
1090 * check_pad_bytes cleans up on its own.
1091 */
bb192ed9 1092 check_pad_bytes(s, slab, p);
81819f0f
CL
1093 }
1094
cbfc35a4 1095 if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
81819f0f
CL
1096 /*
1097 * Object and freepointer overlap. Cannot check
1098 * freepointer while object is allocated.
1099 */
1100 return 1;
1101
1102 /* Check free pointer validity */
bb192ed9
VB
1103 if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
1104 object_err(s, slab, p, "Freepointer corrupt");
81819f0f 1105 /*
9f6c708e 1106 * No choice but to zap it and thus lose the remainder
81819f0f 1107 * of the free objects in this slab. May cause
672bba3a 1108 * another error because the object count is now wrong.
81819f0f 1109 */
a973e9dd 1110 set_freepointer(s, p, NULL);
81819f0f
CL
1111 return 0;
1112 }
1113 return 1;
1114}
1115
bb192ed9 1116static int check_slab(struct kmem_cache *s, struct slab *slab)
81819f0f 1117{
39b26464
CL
1118 int maxobj;
1119
bb192ed9
VB
1120 if (!folio_test_slab(slab_folio(slab))) {
1121 slab_err(s, slab, "Not a valid slab page");
81819f0f
CL
1122 return 0;
1123 }
39b26464 1124
bb192ed9
VB
1125 maxobj = order_objects(slab_order(slab), s->size);
1126 if (slab->objects > maxobj) {
1127 slab_err(s, slab, "objects %u > max %u",
1128 slab->objects, maxobj);
39b26464
CL
1129 return 0;
1130 }
bb192ed9
VB
1131 if (slab->inuse > slab->objects) {
1132 slab_err(s, slab, "inuse %u > max %u",
1133 slab->inuse, slab->objects);
81819f0f
CL
1134 return 0;
1135 }
1136 /* Slab_pad_check fixes things up after itself */
bb192ed9 1137 slab_pad_check(s, slab);
81819f0f
CL
1138 return 1;
1139}
1140
1141/*
c2092c12 1142 * Determine if a certain object in a slab is on the freelist. Must hold the
672bba3a 1143 * slab lock to guarantee that the chains are in a consistent state.
81819f0f 1144 */
bb192ed9 1145static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
81819f0f
CL
1146{
1147 int nr = 0;
881db7fb 1148 void *fp;
81819f0f 1149 void *object = NULL;
f6edde9c 1150 int max_objects;
81819f0f 1151
bb192ed9
VB
1152 fp = slab->freelist;
1153 while (fp && nr <= slab->objects) {
81819f0f
CL
1154 if (fp == search)
1155 return 1;
bb192ed9 1156 if (!check_valid_pointer(s, slab, fp)) {
81819f0f 1157 if (object) {
bb192ed9 1158 object_err(s, slab, object,
81819f0f 1159 "Freechain corrupt");
a973e9dd 1160 set_freepointer(s, object, NULL);
81819f0f 1161 } else {
bb192ed9
VB
1162 slab_err(s, slab, "Freepointer corrupt");
1163 slab->freelist = NULL;
1164 slab->inuse = slab->objects;
24922684 1165 slab_fix(s, "Freelist cleared");
81819f0f
CL
1166 return 0;
1167 }
1168 break;
1169 }
1170 object = fp;
1171 fp = get_freepointer(s, object);
1172 nr++;
1173 }
1174
bb192ed9 1175 max_objects = order_objects(slab_order(slab), s->size);
210b5c06
CG
1176 if (max_objects > MAX_OBJS_PER_PAGE)
1177 max_objects = MAX_OBJS_PER_PAGE;
224a88be 1178
bb192ed9
VB
1179 if (slab->objects != max_objects) {
1180 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1181 slab->objects, max_objects);
1182 slab->objects = max_objects;
582d1212 1183 slab_fix(s, "Number of objects adjusted");
224a88be 1184 }
bb192ed9
VB
1185 if (slab->inuse != slab->objects - nr) {
1186 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1187 slab->inuse, slab->objects - nr);
1188 slab->inuse = slab->objects - nr;
582d1212 1189 slab_fix(s, "Object count adjusted");
81819f0f
CL
1190 }
1191 return search == NULL;
1192}
1193
bb192ed9 1194static void trace(struct kmem_cache *s, struct slab *slab, void *object,
0121c619 1195 int alloc)
3ec09742
CL
1196{
1197 if (s->flags & SLAB_TRACE) {
f9f58285 1198 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
3ec09742
CL
1199 s->name,
1200 alloc ? "alloc" : "free",
bb192ed9
VB
1201 object, slab->inuse,
1202 slab->freelist);
3ec09742
CL
1203
1204 if (!alloc)
aa2efd5e 1205 print_section(KERN_INFO, "Object ", (void *)object,
d0e0ac97 1206 s->object_size);
3ec09742
CL
1207
1208 dump_stack();
1209 }
1210}
1211
643b1138 1212/*
672bba3a 1213 * Tracking of fully allocated slabs for debugging purposes.
643b1138 1214 */
5cc6eee8 1215static void add_full(struct kmem_cache *s,
bb192ed9 1216 struct kmem_cache_node *n, struct slab *slab)
643b1138 1217{
5cc6eee8
CL
1218 if (!(s->flags & SLAB_STORE_USER))
1219 return;
1220
255d0884 1221 lockdep_assert_held(&n->list_lock);
bb192ed9 1222 list_add(&slab->slab_list, &n->full);
643b1138
CL
1223}
1224
bb192ed9 1225static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
643b1138 1226{
643b1138
CL
1227 if (!(s->flags & SLAB_STORE_USER))
1228 return;
1229
255d0884 1230 lockdep_assert_held(&n->list_lock);
bb192ed9 1231 list_del(&slab->slab_list);
643b1138
CL
1232}
1233
0f389ec6
CL
1234/* Tracking of the number of slabs for debugging purposes */
1235static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1236{
1237 struct kmem_cache_node *n = get_node(s, node);
1238
1239 return atomic_long_read(&n->nr_slabs);
1240}
1241
26c02cf0
AB
1242static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1243{
1244 return atomic_long_read(&n->nr_slabs);
1245}
1246
205ab99d 1247static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
0f389ec6
CL
1248{
1249 struct kmem_cache_node *n = get_node(s, node);
1250
1251 /*
1252 * May be called early in order to allocate a slab for the
1253 * kmem_cache_node structure. Solve the chicken-egg
1254 * dilemma by deferring the increment of the count during
1255 * bootstrap (see early_kmem_cache_node_alloc).
1256 */
338b2642 1257 if (likely(n)) {
0f389ec6 1258 atomic_long_inc(&n->nr_slabs);
205ab99d
CL
1259 atomic_long_add(objects, &n->total_objects);
1260 }
0f389ec6 1261}
205ab99d 1262static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
0f389ec6
CL
1263{
1264 struct kmem_cache_node *n = get_node(s, node);
1265
1266 atomic_long_dec(&n->nr_slabs);
205ab99d 1267 atomic_long_sub(objects, &n->total_objects);
0f389ec6
CL
1268}
1269
1270/* Object debug checks for alloc/free paths */
bb192ed9 1271static void setup_object_debug(struct kmem_cache *s, struct slab *slab,
3ec09742
CL
1272 void *object)
1273{
8fc8d666 1274 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
3ec09742
CL
1275 return;
1276
f7cb1933 1277 init_object(s, object, SLUB_RED_INACTIVE);
3ec09742
CL
1278 init_tracking(s, object);
1279}
1280
a50b854e 1281static
bb192ed9 1282void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
a7101224 1283{
8fc8d666 1284 if (!kmem_cache_debug_flags(s, SLAB_POISON))
a7101224
AK
1285 return;
1286
1287 metadata_access_enable();
bb192ed9 1288 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
a7101224
AK
1289 metadata_access_disable();
1290}
1291
becfda68 1292static inline int alloc_consistency_checks(struct kmem_cache *s,
bb192ed9 1293 struct slab *slab, void *object)
81819f0f 1294{
bb192ed9 1295 if (!check_slab(s, slab))
becfda68 1296 return 0;
81819f0f 1297
bb192ed9
VB
1298 if (!check_valid_pointer(s, slab, object)) {
1299 object_err(s, slab, object, "Freelist Pointer check fails");
becfda68 1300 return 0;
81819f0f
CL
1301 }
1302
bb192ed9 1303 if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
becfda68
LA
1304 return 0;
1305
1306 return 1;
1307}
1308
1309static noinline int alloc_debug_processing(struct kmem_cache *s,
bb192ed9 1310 struct slab *slab,
becfda68
LA
1311 void *object, unsigned long addr)
1312{
1313 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
bb192ed9 1314 if (!alloc_consistency_checks(s, slab, object))
becfda68
LA
1315 goto bad;
1316 }
81819f0f 1317
3ec09742
CL
1318 /* Success perform special debug activities for allocs */
1319 if (s->flags & SLAB_STORE_USER)
1320 set_track(s, object, TRACK_ALLOC, addr);
bb192ed9 1321 trace(s, slab, object, 1);
f7cb1933 1322 init_object(s, object, SLUB_RED_ACTIVE);
81819f0f 1323 return 1;
3ec09742 1324
81819f0f 1325bad:
bb192ed9 1326 if (folio_test_slab(slab_folio(slab))) {
81819f0f
CL
1327 /*
1328 * If this is a slab page then lets do the best we can
1329 * to avoid issues in the future. Marking all objects
672bba3a 1330 * as used avoids touching the remaining objects.
81819f0f 1331 */
24922684 1332 slab_fix(s, "Marking all objects used");
bb192ed9
VB
1333 slab->inuse = slab->objects;
1334 slab->freelist = NULL;
81819f0f
CL
1335 }
1336 return 0;
1337}
1338
becfda68 1339static inline int free_consistency_checks(struct kmem_cache *s,
bb192ed9 1340 struct slab *slab, void *object, unsigned long addr)
81819f0f 1341{
bb192ed9
VB
1342 if (!check_valid_pointer(s, slab, object)) {
1343 slab_err(s, slab, "Invalid object pointer 0x%p", object);
becfda68 1344 return 0;
81819f0f
CL
1345 }
1346
bb192ed9
VB
1347 if (on_freelist(s, slab, object)) {
1348 object_err(s, slab, object, "Object already free");
becfda68 1349 return 0;
81819f0f
CL
1350 }
1351
bb192ed9 1352 if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
becfda68 1353 return 0;
81819f0f 1354
bb192ed9
VB
1355 if (unlikely(s != slab->slab_cache)) {
1356 if (!folio_test_slab(slab_folio(slab))) {
1357 slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
756a025f 1358 object);
bb192ed9 1359 } else if (!slab->slab_cache) {
f9f58285
FF
1360 pr_err("SLUB <none>: no slab for object 0x%p.\n",
1361 object);
70d71228 1362 dump_stack();
06428780 1363 } else
bb192ed9 1364 object_err(s, slab, object,
24922684 1365 "page slab pointer corrupt.");
becfda68
LA
1366 return 0;
1367 }
1368 return 1;
1369}
1370
1371/* Supports checking bulk free of a constructed freelist */
1372static noinline int free_debug_processing(
bb192ed9 1373 struct kmem_cache *s, struct slab *slab,
becfda68
LA
1374 void *head, void *tail, int bulk_cnt,
1375 unsigned long addr)
1376{
bb192ed9 1377 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
becfda68
LA
1378 void *object = head;
1379 int cnt = 0;
a2b4ae8b 1380 unsigned long flags, flags2;
becfda68
LA
1381 int ret = 0;
1382
1383 spin_lock_irqsave(&n->list_lock, flags);
bb192ed9 1384 slab_lock(slab, &flags2);
becfda68
LA
1385
1386 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
bb192ed9 1387 if (!check_slab(s, slab))
becfda68
LA
1388 goto out;
1389 }
1390
1391next_object:
1392 cnt++;
1393
1394 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
bb192ed9 1395 if (!free_consistency_checks(s, slab, object, addr))
becfda68 1396 goto out;
81819f0f 1397 }
3ec09742 1398
3ec09742
CL
1399 if (s->flags & SLAB_STORE_USER)
1400 set_track(s, object, TRACK_FREE, addr);
bb192ed9 1401 trace(s, slab, object, 0);
81084651 1402 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
f7cb1933 1403 init_object(s, object, SLUB_RED_INACTIVE);
81084651
JDB
1404
1405 /* Reached end of constructed freelist yet? */
1406 if (object != tail) {
1407 object = get_freepointer(s, object);
1408 goto next_object;
1409 }
804aa132
LA
1410 ret = 1;
1411
5c2e4bbb 1412out:
81084651 1413 if (cnt != bulk_cnt)
bb192ed9 1414 slab_err(s, slab, "Bulk freelist count(%d) invalid(%d)\n",
81084651
JDB
1415 bulk_cnt, cnt);
1416
bb192ed9 1417 slab_unlock(slab, &flags2);
282acb43 1418 spin_unlock_irqrestore(&n->list_lock, flags);
804aa132
LA
1419 if (!ret)
1420 slab_fix(s, "Object at 0x%p not freed", object);
1421 return ret;
81819f0f
CL
1422}
1423
e17f1dfb
VB
1424/*
1425 * Parse a block of slub_debug options. Blocks are delimited by ';'
1426 *
1427 * @str: start of block
1428 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
1429 * @slabs: return start of list of slabs, or NULL when there's no list
1430 * @init: assume this is initial parsing and not per-kmem-create parsing
1431 *
1432 * returns the start of next block if there's any, or NULL
1433 */
1434static char *
1435parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
41ecc55b 1436{
e17f1dfb 1437 bool higher_order_disable = false;
f0630fff 1438
e17f1dfb
VB
1439 /* Skip any completely empty blocks */
1440 while (*str && *str == ';')
1441 str++;
1442
1443 if (*str == ',') {
f0630fff
CL
1444 /*
1445 * No options but restriction on slabs. This means full
1446 * debugging for slabs matching a pattern.
1447 */
e17f1dfb 1448 *flags = DEBUG_DEFAULT_FLAGS;
f0630fff 1449 goto check_slabs;
e17f1dfb
VB
1450 }
1451 *flags = 0;
f0630fff 1452
e17f1dfb
VB
1453 /* Determine which debug features should be switched on */
1454 for (; *str && *str != ',' && *str != ';'; str++) {
f0630fff 1455 switch (tolower(*str)) {
e17f1dfb
VB
1456 case '-':
1457 *flags = 0;
1458 break;
f0630fff 1459 case 'f':
e17f1dfb 1460 *flags |= SLAB_CONSISTENCY_CHECKS;
f0630fff
CL
1461 break;
1462 case 'z':
e17f1dfb 1463 *flags |= SLAB_RED_ZONE;
f0630fff
CL
1464 break;
1465 case 'p':
e17f1dfb 1466 *flags |= SLAB_POISON;
f0630fff
CL
1467 break;
1468 case 'u':
e17f1dfb 1469 *flags |= SLAB_STORE_USER;
f0630fff
CL
1470 break;
1471 case 't':
e17f1dfb 1472 *flags |= SLAB_TRACE;
f0630fff 1473 break;
4c13dd3b 1474 case 'a':
e17f1dfb 1475 *flags |= SLAB_FAILSLAB;
4c13dd3b 1476 break;
08303a73
CA
1477 case 'o':
1478 /*
1479 * Avoid enabling debugging on caches if its minimum
1480 * order would increase as a result.
1481 */
e17f1dfb 1482 higher_order_disable = true;
08303a73 1483 break;
f0630fff 1484 default:
e17f1dfb
VB
1485 if (init)
1486 pr_err("slub_debug option '%c' unknown. skipped\n", *str);
f0630fff 1487 }
41ecc55b 1488 }
f0630fff 1489check_slabs:
41ecc55b 1490 if (*str == ',')
e17f1dfb
VB
1491 *slabs = ++str;
1492 else
1493 *slabs = NULL;
1494
1495 /* Skip over the slab list */
1496 while (*str && *str != ';')
1497 str++;
1498
1499 /* Skip any completely empty blocks */
1500 while (*str && *str == ';')
1501 str++;
1502
1503 if (init && higher_order_disable)
1504 disable_higher_order_debug = 1;
1505
1506 if (*str)
1507 return str;
1508 else
1509 return NULL;
1510}
1511
1512static int __init setup_slub_debug(char *str)
1513{
1514 slab_flags_t flags;
a7f1d485 1515 slab_flags_t global_flags;
e17f1dfb
VB
1516 char *saved_str;
1517 char *slab_list;
1518 bool global_slub_debug_changed = false;
1519 bool slab_list_specified = false;
1520
a7f1d485 1521 global_flags = DEBUG_DEFAULT_FLAGS;
e17f1dfb
VB
1522 if (*str++ != '=' || !*str)
1523 /*
1524 * No options specified. Switch on full debugging.
1525 */
1526 goto out;
1527
1528 saved_str = str;
1529 while (str) {
1530 str = parse_slub_debug_flags(str, &flags, &slab_list, true);
1531
1532 if (!slab_list) {
a7f1d485 1533 global_flags = flags;
e17f1dfb
VB
1534 global_slub_debug_changed = true;
1535 } else {
1536 slab_list_specified = true;
1537 }
1538 }
1539
1540 /*
1541 * For backwards compatibility, a single list of flags with list of
a7f1d485
VB
1542 * slabs means debugging is only changed for those slabs, so the global
1543 * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1544 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
e17f1dfb
VB
1545 * long as there is no option specifying flags without a slab list.
1546 */
1547 if (slab_list_specified) {
1548 if (!global_slub_debug_changed)
a7f1d485 1549 global_flags = slub_debug;
e17f1dfb
VB
1550 slub_debug_string = saved_str;
1551 }
f0630fff 1552out:
a7f1d485 1553 slub_debug = global_flags;
ca0cab65
VB
1554 if (slub_debug != 0 || slub_debug_string)
1555 static_branch_enable(&slub_debug_enabled);
02ac47d0
SB
1556 else
1557 static_branch_disable(&slub_debug_enabled);
6471384a
AP
1558 if ((static_branch_unlikely(&init_on_alloc) ||
1559 static_branch_unlikely(&init_on_free)) &&
1560 (slub_debug & SLAB_POISON))
1561 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
41ecc55b
CL
1562 return 1;
1563}
1564
1565__setup("slub_debug", setup_slub_debug);
1566
c5fd3ca0
AT
1567/*
1568 * kmem_cache_flags - apply debugging options to the cache
1569 * @object_size: the size of an object without meta data
1570 * @flags: flags to set
1571 * @name: name of the cache
c5fd3ca0
AT
1572 *
1573 * Debug option(s) are applied to @flags. In addition to the debug
1574 * option(s), if a slab name (or multiple) is specified i.e.
1575 * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1576 * then only the select slabs will receive the debug option(s).
1577 */
0293d1fd 1578slab_flags_t kmem_cache_flags(unsigned int object_size,
37540008 1579 slab_flags_t flags, const char *name)
41ecc55b 1580{
c5fd3ca0
AT
1581 char *iter;
1582 size_t len;
e17f1dfb
VB
1583 char *next_block;
1584 slab_flags_t block_flags;
ca220593
JB
1585 slab_flags_t slub_debug_local = slub_debug;
1586
1587 /*
1588 * If the slab cache is for debugging (e.g. kmemleak) then
1589 * don't store user (stack trace) information by default,
1590 * but let the user enable it via the command line below.
1591 */
1592 if (flags & SLAB_NOLEAKTRACE)
1593 slub_debug_local &= ~SLAB_STORE_USER;
c5fd3ca0 1594
c5fd3ca0 1595 len = strlen(name);
e17f1dfb
VB
1596 next_block = slub_debug_string;
1597 /* Go through all blocks of debug options, see if any matches our slab's name */
1598 while (next_block) {
1599 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
1600 if (!iter)
1601 continue;
1602 /* Found a block that has a slab list, search it */
1603 while (*iter) {
1604 char *end, *glob;
1605 size_t cmplen;
1606
1607 end = strchrnul(iter, ',');
1608 if (next_block && next_block < end)
1609 end = next_block - 1;
1610
1611 glob = strnchr(iter, end - iter, '*');
1612 if (glob)
1613 cmplen = glob - iter;
1614 else
1615 cmplen = max_t(size_t, len, (end - iter));
c5fd3ca0 1616
e17f1dfb
VB
1617 if (!strncmp(name, iter, cmplen)) {
1618 flags |= block_flags;
1619 return flags;
1620 }
c5fd3ca0 1621
e17f1dfb
VB
1622 if (!*end || *end == ';')
1623 break;
1624 iter = end + 1;
c5fd3ca0 1625 }
c5fd3ca0 1626 }
ba0268a8 1627
ca220593 1628 return flags | slub_debug_local;
41ecc55b 1629}
b4a64718 1630#else /* !CONFIG_SLUB_DEBUG */
3ec09742 1631static inline void setup_object_debug(struct kmem_cache *s,
bb192ed9 1632 struct slab *slab, void *object) {}
a50b854e 1633static inline
bb192ed9 1634void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
41ecc55b 1635
3ec09742 1636static inline int alloc_debug_processing(struct kmem_cache *s,
bb192ed9 1637 struct slab *slab, void *object, unsigned long addr) { return 0; }
41ecc55b 1638
282acb43 1639static inline int free_debug_processing(
bb192ed9 1640 struct kmem_cache *s, struct slab *slab,
81084651 1641 void *head, void *tail, int bulk_cnt,
282acb43 1642 unsigned long addr) { return 0; }
41ecc55b 1643
bb192ed9 1644static inline int slab_pad_check(struct kmem_cache *s, struct slab *slab)
41ecc55b 1645 { return 1; }
bb192ed9 1646static inline int check_object(struct kmem_cache *s, struct slab *slab,
f7cb1933 1647 void *object, u8 val) { return 1; }
5cc6eee8 1648static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
bb192ed9 1649 struct slab *slab) {}
c65c1877 1650static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
bb192ed9 1651 struct slab *slab) {}
0293d1fd 1652slab_flags_t kmem_cache_flags(unsigned int object_size,
37540008 1653 slab_flags_t flags, const char *name)
ba0268a8
CL
1654{
1655 return flags;
1656}
41ecc55b 1657#define slub_debug 0
0f389ec6 1658
fdaa45e9
IM
1659#define disable_higher_order_debug 0
1660
0f389ec6
CL
1661static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1662 { return 0; }
26c02cf0
AB
1663static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1664 { return 0; }
205ab99d
CL
1665static inline void inc_slabs_node(struct kmem_cache *s, int node,
1666 int objects) {}
1667static inline void dec_slabs_node(struct kmem_cache *s, int node,
1668 int objects) {}
7d550c56 1669
bb192ed9 1670static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
dc07a728 1671 void **freelist, void *nextfree)
52f23478
DZ
1672{
1673 return false;
1674}
02e72cc6
AR
1675#endif /* CONFIG_SLUB_DEBUG */
1676
1677/*
1678 * Hooks for other subsystems that check memory allocations. In a typical
1679 * production configuration these hooks all should produce no code at all.
1680 */
0116523c 1681static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
d56791b3 1682{
53128245 1683 ptr = kasan_kmalloc_large(ptr, size, flags);
a2f77575 1684 /* As ptr might get tagged, call kmemleak hook after KASAN. */
d56791b3 1685 kmemleak_alloc(ptr, size, 1, flags);
53128245 1686 return ptr;
d56791b3
RB
1687}
1688
ee3ce779 1689static __always_inline void kfree_hook(void *x)
d56791b3
RB
1690{
1691 kmemleak_free(x);
027b37b5 1692 kasan_kfree_large(x);
d56791b3
RB
1693}
1694
d57a964e
AK
1695static __always_inline bool slab_free_hook(struct kmem_cache *s,
1696 void *x, bool init)
d56791b3
RB
1697{
1698 kmemleak_free_recursive(x, s->flags);
7d550c56 1699
84048039 1700 debug_check_no_locks_freed(x, s->object_size);
02e72cc6 1701
02e72cc6
AR
1702 if (!(s->flags & SLAB_DEBUG_OBJECTS))
1703 debug_check_no_obj_freed(x, s->object_size);
0316bec2 1704
cfbe1636
ME
1705 /* Use KCSAN to help debug racy use-after-free. */
1706 if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
1707 __kcsan_check_access(x, s->object_size,
1708 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
1709
d57a964e
AK
1710 /*
1711 * As memory initialization might be integrated into KASAN,
1712 * kasan_slab_free and initialization memset's must be
1713 * kept together to avoid discrepancies in behavior.
1714 *
1715 * The initialization memset's clear the object and the metadata,
1716 * but don't touch the SLAB redzone.
1717 */
1718 if (init) {
1719 int rsize;
1720
1721 if (!kasan_has_integrated_init())
1722 memset(kasan_reset_tag(x), 0, s->object_size);
1723 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
1724 memset((char *)kasan_reset_tag(x) + s->inuse, 0,
1725 s->size - s->inuse - rsize);
1726 }
1727 /* KASAN might put x into memory quarantine, delaying its reuse. */
1728 return kasan_slab_free(s, x, init);
02e72cc6 1729}
205ab99d 1730
c3895391 1731static inline bool slab_free_freelist_hook(struct kmem_cache *s,
899447f6
ML
1732 void **head, void **tail,
1733 int *cnt)
81084651 1734{
6471384a
AP
1735
1736 void *object;
1737 void *next = *head;
1738 void *old_tail = *tail ? *tail : *head;
6471384a 1739
b89fb5ef 1740 if (is_kfence_address(next)) {
d57a964e 1741 slab_free_hook(s, next, false);
b89fb5ef
AP
1742 return true;
1743 }
1744
aea4df4c
LA
1745 /* Head and tail of the reconstructed freelist */
1746 *head = NULL;
1747 *tail = NULL;
1b7e816f 1748
aea4df4c
LA
1749 do {
1750 object = next;
1751 next = get_freepointer(s, object);
1752
c3895391 1753 /* If object's reuse doesn't have to be delayed */
d57a964e 1754 if (!slab_free_hook(s, object, slab_want_init_on_free(s))) {
c3895391
AK
1755 /* Move object to the new freelist */
1756 set_freepointer(s, object, *head);
1757 *head = object;
1758 if (!*tail)
1759 *tail = object;
899447f6
ML
1760 } else {
1761 /*
1762 * Adjust the reconstructed freelist depth
1763 * accordingly if object's reuse is delayed.
1764 */
1765 --(*cnt);
c3895391
AK
1766 }
1767 } while (object != old_tail);
1768
1769 if (*head == *tail)
1770 *tail = NULL;
1771
1772 return *head != NULL;
81084651
JDB
1773}
1774
bb192ed9 1775static void *setup_object(struct kmem_cache *s, struct slab *slab,
588f8ba9
TG
1776 void *object)
1777{
bb192ed9 1778 setup_object_debug(s, slab, object);
4d176711 1779 object = kasan_init_slab_obj(s, object);
588f8ba9
TG
1780 if (unlikely(s->ctor)) {
1781 kasan_unpoison_object_data(s, object);
1782 s->ctor(object);
1783 kasan_poison_object_data(s, object);
1784 }
4d176711 1785 return object;
588f8ba9
TG
1786}
1787
81819f0f
CL
1788/*
1789 * Slab allocation and freeing
1790 */
45387b8c 1791static inline struct slab *alloc_slab_page(struct kmem_cache *s,
5dfb4175 1792 gfp_t flags, int node, struct kmem_cache_order_objects oo)
65c3376a 1793{
45387b8c
VB
1794 struct folio *folio;
1795 struct slab *slab;
19af27af 1796 unsigned int order = oo_order(oo);
65c3376a 1797
2154a336 1798 if (node == NUMA_NO_NODE)
45387b8c 1799 folio = (struct folio *)alloc_pages(flags, order);
65c3376a 1800 else
45387b8c 1801 folio = (struct folio *)__alloc_pages_node(node, flags, order);
5dfb4175 1802
45387b8c
VB
1803 if (!folio)
1804 return NULL;
1805
1806 slab = folio_slab(folio);
1807 __folio_set_slab(folio);
1808 if (page_is_pfmemalloc(folio_page(folio, 0)))
1809 slab_set_pfmemalloc(slab);
1810
1811 return slab;
65c3376a
CL
1812}
1813
210e7a43
TG
1814#ifdef CONFIG_SLAB_FREELIST_RANDOM
1815/* Pre-initialize the random sequence cache */
1816static int init_cache_random_seq(struct kmem_cache *s)
1817{
19af27af 1818 unsigned int count = oo_objects(s->oo);
210e7a43 1819 int err;
210e7a43 1820
a810007a
SR
1821 /* Bailout if already initialised */
1822 if (s->random_seq)
1823 return 0;
1824
210e7a43
TG
1825 err = cache_random_seq_create(s, count, GFP_KERNEL);
1826 if (err) {
1827 pr_err("SLUB: Unable to initialize free list for %s\n",
1828 s->name);
1829 return err;
1830 }
1831
1832 /* Transform to an offset on the set of pages */
1833 if (s->random_seq) {
19af27af
AD
1834 unsigned int i;
1835
210e7a43
TG
1836 for (i = 0; i < count; i++)
1837 s->random_seq[i] *= s->size;
1838 }
1839 return 0;
1840}
1841
1842/* Initialize each random sequence freelist per cache */
1843static void __init init_freelist_randomization(void)
1844{
1845 struct kmem_cache *s;
1846
1847 mutex_lock(&slab_mutex);
1848
1849 list_for_each_entry(s, &slab_caches, list)
1850 init_cache_random_seq(s);
1851
1852 mutex_unlock(&slab_mutex);
1853}
1854
1855/* Get the next entry on the pre-computed freelist randomized */
bb192ed9 1856static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab,
210e7a43
TG
1857 unsigned long *pos, void *start,
1858 unsigned long page_limit,
1859 unsigned long freelist_count)
1860{
1861 unsigned int idx;
1862
1863 /*
1864 * If the target page allocation failed, the number of objects on the
1865 * page might be smaller than the usual size defined by the cache.
1866 */
1867 do {
1868 idx = s->random_seq[*pos];
1869 *pos += 1;
1870 if (*pos >= freelist_count)
1871 *pos = 0;
1872 } while (unlikely(idx >= page_limit));
1873
1874 return (char *)start + idx;
1875}
1876
1877/* Shuffle the single linked freelist based on a random pre-computed sequence */
bb192ed9 1878static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
210e7a43
TG
1879{
1880 void *start;
1881 void *cur;
1882 void *next;
1883 unsigned long idx, pos, page_limit, freelist_count;
1884
bb192ed9 1885 if (slab->objects < 2 || !s->random_seq)
210e7a43
TG
1886 return false;
1887
1888 freelist_count = oo_objects(s->oo);
1889 pos = get_random_int() % freelist_count;
1890
bb192ed9
VB
1891 page_limit = slab->objects * s->size;
1892 start = fixup_red_left(s, slab_address(slab));
210e7a43
TG
1893
1894 /* First entry is used as the base of the freelist */
bb192ed9 1895 cur = next_freelist_entry(s, slab, &pos, start, page_limit,
210e7a43 1896 freelist_count);
bb192ed9
VB
1897 cur = setup_object(s, slab, cur);
1898 slab->freelist = cur;
210e7a43 1899
bb192ed9
VB
1900 for (idx = 1; idx < slab->objects; idx++) {
1901 next = next_freelist_entry(s, slab, &pos, start, page_limit,
210e7a43 1902 freelist_count);
bb192ed9 1903 next = setup_object(s, slab, next);
210e7a43
TG
1904 set_freepointer(s, cur, next);
1905 cur = next;
1906 }
210e7a43
TG
1907 set_freepointer(s, cur, NULL);
1908
1909 return true;
1910}
1911#else
1912static inline int init_cache_random_seq(struct kmem_cache *s)
1913{
1914 return 0;
1915}
1916static inline void init_freelist_randomization(void) { }
bb192ed9 1917static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
210e7a43
TG
1918{
1919 return false;
1920}
1921#endif /* CONFIG_SLAB_FREELIST_RANDOM */
1922
bb192ed9 1923static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
81819f0f 1924{
bb192ed9 1925 struct slab *slab;
834f3d11 1926 struct kmem_cache_order_objects oo = s->oo;
ba52270d 1927 gfp_t alloc_gfp;
4d176711 1928 void *start, *p, *next;
a50b854e 1929 int idx;
210e7a43 1930 bool shuffle;
81819f0f 1931
7e0528da
CL
1932 flags &= gfp_allowed_mask;
1933
b7a49f0d 1934 flags |= s->allocflags;
e12ba74d 1935
ba52270d
PE
1936 /*
1937 * Let the initial higher-order allocation fail under memory pressure
1938 * so we fall-back to the minimum order allocation.
1939 */
1940 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
d0164adc 1941 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
444eb2a4 1942 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
ba52270d 1943
bb192ed9
VB
1944 slab = alloc_slab_page(s, alloc_gfp, node, oo);
1945 if (unlikely(!slab)) {
65c3376a 1946 oo = s->min;
80c3a998 1947 alloc_gfp = flags;
65c3376a
CL
1948 /*
1949 * Allocation may have failed due to fragmentation.
1950 * Try a lower order alloc if possible
1951 */
bb192ed9
VB
1952 slab = alloc_slab_page(s, alloc_gfp, node, oo);
1953 if (unlikely(!slab))
588f8ba9
TG
1954 goto out;
1955 stat(s, ORDER_FALLBACK);
65c3376a 1956 }
5a896d9e 1957
bb192ed9 1958 slab->objects = oo_objects(oo);
81819f0f 1959
bb192ed9 1960 account_slab(slab, oo_order(oo), s, flags);
1f3147b4 1961
bb192ed9 1962 slab->slab_cache = s;
81819f0f 1963
6e48a966 1964 kasan_poison_slab(slab);
81819f0f 1965
bb192ed9 1966 start = slab_address(slab);
81819f0f 1967
bb192ed9 1968 setup_slab_debug(s, slab, start);
0316bec2 1969
bb192ed9 1970 shuffle = shuffle_freelist(s, slab);
210e7a43
TG
1971
1972 if (!shuffle) {
4d176711 1973 start = fixup_red_left(s, start);
bb192ed9
VB
1974 start = setup_object(s, slab, start);
1975 slab->freelist = start;
1976 for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
18e50661 1977 next = p + s->size;
bb192ed9 1978 next = setup_object(s, slab, next);
18e50661
AK
1979 set_freepointer(s, p, next);
1980 p = next;
1981 }
1982 set_freepointer(s, p, NULL);
81819f0f 1983 }
81819f0f 1984
bb192ed9
VB
1985 slab->inuse = slab->objects;
1986 slab->frozen = 1;
588f8ba9 1987
81819f0f 1988out:
bb192ed9 1989 if (!slab)
588f8ba9
TG
1990 return NULL;
1991
bb192ed9 1992 inc_slabs_node(s, slab_nid(slab), slab->objects);
588f8ba9 1993
bb192ed9 1994 return slab;
81819f0f
CL
1995}
1996
bb192ed9 1997static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
588f8ba9 1998{
44405099
LL
1999 if (unlikely(flags & GFP_SLAB_BUG_MASK))
2000 flags = kmalloc_fix_flags(flags);
588f8ba9 2001
53a0de06
VB
2002 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
2003
588f8ba9
TG
2004 return allocate_slab(s,
2005 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
2006}
2007
4020b4a2 2008static void __free_slab(struct kmem_cache *s, struct slab *slab)
81819f0f 2009{
4020b4a2
VB
2010 struct folio *folio = slab_folio(slab);
2011 int order = folio_order(folio);
834f3d11 2012 int pages = 1 << order;
81819f0f 2013
8fc8d666 2014 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
81819f0f
CL
2015 void *p;
2016
bb192ed9 2017 slab_pad_check(s, slab);
4020b4a2 2018 for_each_object(p, s, slab_address(slab), slab->objects)
bb192ed9 2019 check_object(s, slab, p, SLUB_RED_INACTIVE);
81819f0f
CL
2020 }
2021
4020b4a2
VB
2022 __slab_clear_pfmemalloc(slab);
2023 __folio_clear_slab(folio);
2024 folio->mapping = NULL;
1eb5ac64
NP
2025 if (current->reclaim_state)
2026 current->reclaim_state->reclaimed_slab += pages;
4020b4a2
VB
2027 unaccount_slab(slab, order, s);
2028 __free_pages(folio_page(folio, 0), order);
81819f0f
CL
2029}
2030
2031static void rcu_free_slab(struct rcu_head *h)
2032{
bb192ed9 2033 struct slab *slab = container_of(h, struct slab, rcu_head);
da9a638c 2034
bb192ed9 2035 __free_slab(slab->slab_cache, slab);
81819f0f
CL
2036}
2037
bb192ed9 2038static void free_slab(struct kmem_cache *s, struct slab *slab)
81819f0f 2039{
5f0d5a3a 2040 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
bb192ed9 2041 call_rcu(&slab->rcu_head, rcu_free_slab);
81819f0f 2042 } else
bb192ed9 2043 __free_slab(s, slab);
81819f0f
CL
2044}
2045
bb192ed9 2046static void discard_slab(struct kmem_cache *s, struct slab *slab)
81819f0f 2047{
bb192ed9
VB
2048 dec_slabs_node(s, slab_nid(slab), slab->objects);
2049 free_slab(s, slab);
81819f0f
CL
2050}
2051
2052/*
5cc6eee8 2053 * Management of partially allocated slabs.
81819f0f 2054 */
1e4dd946 2055static inline void
bb192ed9 2056__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
81819f0f 2057{
e95eed57 2058 n->nr_partial++;
136333d1 2059 if (tail == DEACTIVATE_TO_TAIL)
bb192ed9 2060 list_add_tail(&slab->slab_list, &n->partial);
7c2e132c 2061 else
bb192ed9 2062 list_add(&slab->slab_list, &n->partial);
81819f0f
CL
2063}
2064
1e4dd946 2065static inline void add_partial(struct kmem_cache_node *n,
bb192ed9 2066 struct slab *slab, int tail)
62e346a8 2067{
c65c1877 2068 lockdep_assert_held(&n->list_lock);
bb192ed9 2069 __add_partial(n, slab, tail);
1e4dd946 2070}
c65c1877 2071
1e4dd946 2072static inline void remove_partial(struct kmem_cache_node *n,
bb192ed9 2073 struct slab *slab)
1e4dd946
SR
2074{
2075 lockdep_assert_held(&n->list_lock);
bb192ed9 2076 list_del(&slab->slab_list);
52b4b950 2077 n->nr_partial--;
1e4dd946
SR
2078}
2079
81819f0f 2080/*
7ced3719
CL
2081 * Remove slab from the partial list, freeze it and
2082 * return the pointer to the freelist.
81819f0f 2083 *
497b66f2 2084 * Returns a list of objects or NULL if it fails.
81819f0f 2085 */
497b66f2 2086static inline void *acquire_slab(struct kmem_cache *s,
bb192ed9 2087 struct kmem_cache_node *n, struct slab *slab,
b47291ef 2088 int mode)
81819f0f 2089{
2cfb7455
CL
2090 void *freelist;
2091 unsigned long counters;
bb192ed9 2092 struct slab new;
2cfb7455 2093
c65c1877
PZ
2094 lockdep_assert_held(&n->list_lock);
2095
2cfb7455
CL
2096 /*
2097 * Zap the freelist and set the frozen bit.
2098 * The old freelist is the list of objects for the
2099 * per cpu allocation list.
2100 */
bb192ed9
VB
2101 freelist = slab->freelist;
2102 counters = slab->counters;
7ced3719 2103 new.counters = counters;
23910c50 2104 if (mode) {
bb192ed9 2105 new.inuse = slab->objects;
23910c50
PE
2106 new.freelist = NULL;
2107 } else {
2108 new.freelist = freelist;
2109 }
2cfb7455 2110
a0132ac0 2111 VM_BUG_ON(new.frozen);
7ced3719 2112 new.frozen = 1;
2cfb7455 2113
bb192ed9 2114 if (!__cmpxchg_double_slab(s, slab,
2cfb7455 2115 freelist, counters,
02d7633f 2116 new.freelist, new.counters,
7ced3719 2117 "acquire_slab"))
7ced3719 2118 return NULL;
2cfb7455 2119
bb192ed9 2120 remove_partial(n, slab);
7ced3719 2121 WARN_ON(!freelist);
49e22585 2122 return freelist;
81819f0f
CL
2123}
2124
e0a043aa 2125#ifdef CONFIG_SLUB_CPU_PARTIAL
bb192ed9 2126static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
e0a043aa 2127#else
bb192ed9 2128static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
e0a043aa
VB
2129 int drain) { }
2130#endif
01b34d16 2131static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
49e22585 2132
81819f0f 2133/*
672bba3a 2134 * Try to allocate a partial slab from a specific node.
81819f0f 2135 */
8ba00bb6 2136static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
bb192ed9 2137 struct slab **ret_slab, gfp_t gfpflags)
81819f0f 2138{
bb192ed9 2139 struct slab *slab, *slab2;
49e22585 2140 void *object = NULL;
4b1f449d 2141 unsigned long flags;
bb192ed9 2142 unsigned int partial_slabs = 0;
81819f0f
CL
2143
2144 /*
2145 * Racy check. If we mistakenly see no partial slabs then we
2146 * just allocate an empty slab. If we mistakenly try to get a
70b6d25e 2147 * partial slab and there is none available then get_partial()
672bba3a 2148 * will return NULL.
81819f0f
CL
2149 */
2150 if (!n || !n->nr_partial)
2151 return NULL;
2152
4b1f449d 2153 spin_lock_irqsave(&n->list_lock, flags);
bb192ed9 2154 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
8ba00bb6 2155 void *t;
49e22585 2156
bb192ed9 2157 if (!pfmemalloc_match(slab, gfpflags))
8ba00bb6
JK
2158 continue;
2159
bb192ed9 2160 t = acquire_slab(s, n, slab, object == NULL);
49e22585 2161 if (!t)
9b1ea29b 2162 break;
49e22585 2163
12d79634 2164 if (!object) {
bb192ed9 2165 *ret_slab = slab;
49e22585 2166 stat(s, ALLOC_FROM_PARTIAL);
49e22585 2167 object = t;
49e22585 2168 } else {
bb192ed9 2169 put_cpu_partial(s, slab, 0);
8028dcea 2170 stat(s, CPU_PARTIAL_NODE);
bb192ed9 2171 partial_slabs++;
49e22585 2172 }
b47291ef 2173#ifdef CONFIG_SLUB_CPU_PARTIAL
345c905d 2174 if (!kmem_cache_has_cpu_partial(s)
bb192ed9 2175 || partial_slabs > s->cpu_partial_slabs / 2)
49e22585 2176 break;
b47291ef
VB
2177#else
2178 break;
2179#endif
49e22585 2180
497b66f2 2181 }
4b1f449d 2182 spin_unlock_irqrestore(&n->list_lock, flags);
497b66f2 2183 return object;
81819f0f
CL
2184}
2185
2186/*
c2092c12 2187 * Get a slab from somewhere. Search in increasing NUMA distances.
81819f0f 2188 */
de3ec035 2189static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
bb192ed9 2190 struct slab **ret_slab)
81819f0f
CL
2191{
2192#ifdef CONFIG_NUMA
2193 struct zonelist *zonelist;
dd1a239f 2194 struct zoneref *z;
54a6eb5c 2195 struct zone *zone;
97a225e6 2196 enum zone_type highest_zoneidx = gfp_zone(flags);
497b66f2 2197 void *object;
cc9a6c87 2198 unsigned int cpuset_mems_cookie;
81819f0f
CL
2199
2200 /*
672bba3a
CL
2201 * The defrag ratio allows a configuration of the tradeoffs between
2202 * inter node defragmentation and node local allocations. A lower
2203 * defrag_ratio increases the tendency to do local allocations
2204 * instead of attempting to obtain partial slabs from other nodes.
81819f0f 2205 *
672bba3a
CL
2206 * If the defrag_ratio is set to 0 then kmalloc() always
2207 * returns node local objects. If the ratio is higher then kmalloc()
2208 * may return off node objects because partial slabs are obtained
2209 * from other nodes and filled up.
81819f0f 2210 *
43efd3ea
LP
2211 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
2212 * (which makes defrag_ratio = 1000) then every (well almost)
2213 * allocation will first attempt to defrag slab caches on other nodes.
2214 * This means scanning over all nodes to look for partial slabs which
2215 * may be expensive if we do it every time we are trying to find a slab
672bba3a 2216 * with available objects.
81819f0f 2217 */
9824601e
CL
2218 if (!s->remote_node_defrag_ratio ||
2219 get_cycles() % 1024 > s->remote_node_defrag_ratio)
81819f0f
CL
2220 return NULL;
2221
cc9a6c87 2222 do {
d26914d1 2223 cpuset_mems_cookie = read_mems_allowed_begin();
2a389610 2224 zonelist = node_zonelist(mempolicy_slab_node(), flags);
97a225e6 2225 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
cc9a6c87
MG
2226 struct kmem_cache_node *n;
2227
2228 n = get_node(s, zone_to_nid(zone));
2229
dee2f8aa 2230 if (n && cpuset_zone_allowed(zone, flags) &&
cc9a6c87 2231 n->nr_partial > s->min_partial) {
bb192ed9 2232 object = get_partial_node(s, n, ret_slab, flags);
cc9a6c87
MG
2233 if (object) {
2234 /*
d26914d1
MG
2235 * Don't check read_mems_allowed_retry()
2236 * here - if mems_allowed was updated in
2237 * parallel, that was a harmless race
2238 * between allocation and the cpuset
2239 * update
cc9a6c87 2240 */
cc9a6c87
MG
2241 return object;
2242 }
c0ff7453 2243 }
81819f0f 2244 }
d26914d1 2245 } while (read_mems_allowed_retry(cpuset_mems_cookie));
6dfd1b65 2246#endif /* CONFIG_NUMA */
81819f0f
CL
2247 return NULL;
2248}
2249
2250/*
c2092c12 2251 * Get a partial slab, lock it and return it.
81819f0f 2252 */
497b66f2 2253static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
bb192ed9 2254 struct slab **ret_slab)
81819f0f 2255{
497b66f2 2256 void *object;
a561ce00
JK
2257 int searchnode = node;
2258
2259 if (node == NUMA_NO_NODE)
2260 searchnode = numa_mem_id();
81819f0f 2261
bb192ed9 2262 object = get_partial_node(s, get_node(s, searchnode), ret_slab, flags);
497b66f2
CL
2263 if (object || node != NUMA_NO_NODE)
2264 return object;
81819f0f 2265
bb192ed9 2266 return get_any_partial(s, flags, ret_slab);
81819f0f
CL
2267}
2268
923717cb 2269#ifdef CONFIG_PREEMPTION
8a5ec0ba 2270/*
0d645ed1 2271 * Calculate the next globally unique transaction for disambiguation
8a5ec0ba
CL
2272 * during cmpxchg. The transactions start with the cpu number and are then
2273 * incremented by CONFIG_NR_CPUS.
2274 */
2275#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
2276#else
2277/*
2278 * No preemption supported therefore also no need to check for
2279 * different cpus.
2280 */
2281#define TID_STEP 1
2282#endif
2283
2284static inline unsigned long next_tid(unsigned long tid)
2285{
2286 return tid + TID_STEP;
2287}
2288
9d5f0be0 2289#ifdef SLUB_DEBUG_CMPXCHG
8a5ec0ba
CL
2290static inline unsigned int tid_to_cpu(unsigned long tid)
2291{
2292 return tid % TID_STEP;
2293}
2294
2295static inline unsigned long tid_to_event(unsigned long tid)
2296{
2297 return tid / TID_STEP;
2298}
9d5f0be0 2299#endif
8a5ec0ba
CL
2300
2301static inline unsigned int init_tid(int cpu)
2302{
2303 return cpu;
2304}
2305
2306static inline void note_cmpxchg_failure(const char *n,
2307 const struct kmem_cache *s, unsigned long tid)
2308{
2309#ifdef SLUB_DEBUG_CMPXCHG
2310 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
2311
f9f58285 2312 pr_info("%s %s: cmpxchg redo ", n, s->name);
8a5ec0ba 2313
923717cb 2314#ifdef CONFIG_PREEMPTION
8a5ec0ba 2315 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
f9f58285 2316 pr_warn("due to cpu change %d -> %d\n",
8a5ec0ba
CL
2317 tid_to_cpu(tid), tid_to_cpu(actual_tid));
2318 else
2319#endif
2320 if (tid_to_event(tid) != tid_to_event(actual_tid))
f9f58285 2321 pr_warn("due to cpu running other code. Event %ld->%ld\n",
8a5ec0ba
CL
2322 tid_to_event(tid), tid_to_event(actual_tid));
2323 else
f9f58285 2324 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
8a5ec0ba
CL
2325 actual_tid, tid, next_tid(tid));
2326#endif
4fdccdfb 2327 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
8a5ec0ba
CL
2328}
2329
788e1aad 2330static void init_kmem_cache_cpus(struct kmem_cache *s)
8a5ec0ba 2331{
8a5ec0ba 2332 int cpu;
bd0e7491 2333 struct kmem_cache_cpu *c;
8a5ec0ba 2334
bd0e7491
VB
2335 for_each_possible_cpu(cpu) {
2336 c = per_cpu_ptr(s->cpu_slab, cpu);
2337 local_lock_init(&c->lock);
2338 c->tid = init_tid(cpu);
2339 }
8a5ec0ba 2340}
2cfb7455 2341
81819f0f 2342/*
c2092c12 2343 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
a019d201
VB
2344 * unfreezes the slabs and puts it on the proper list.
2345 * Assumes the slab has been already safely taken away from kmem_cache_cpu
2346 * by the caller.
81819f0f 2347 */
bb192ed9 2348static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
a019d201 2349 void *freelist)
81819f0f 2350{
2cfb7455 2351 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
bb192ed9 2352 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
d930ff03 2353 int lock = 0, free_delta = 0;
2cfb7455 2354 enum slab_modes l = M_NONE, m = M_NONE;
d930ff03 2355 void *nextfree, *freelist_iter, *freelist_tail;
136333d1 2356 int tail = DEACTIVATE_TO_HEAD;
3406e91b 2357 unsigned long flags = 0;
bb192ed9
VB
2358 struct slab new;
2359 struct slab old;
2cfb7455 2360
bb192ed9 2361 if (slab->freelist) {
84e554e6 2362 stat(s, DEACTIVATE_REMOTE_FREES);
136333d1 2363 tail = DEACTIVATE_TO_TAIL;
2cfb7455
CL
2364 }
2365
894b8788 2366 /*
d930ff03
VB
2367 * Stage one: Count the objects on cpu's freelist as free_delta and
2368 * remember the last object in freelist_tail for later splicing.
2cfb7455 2369 */
d930ff03
VB
2370 freelist_tail = NULL;
2371 freelist_iter = freelist;
2372 while (freelist_iter) {
2373 nextfree = get_freepointer(s, freelist_iter);
2cfb7455 2374
52f23478
DZ
2375 /*
2376 * If 'nextfree' is invalid, it is possible that the object at
d930ff03
VB
2377 * 'freelist_iter' is already corrupted. So isolate all objects
2378 * starting at 'freelist_iter' by skipping them.
52f23478 2379 */
bb192ed9 2380 if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
52f23478
DZ
2381 break;
2382
d930ff03
VB
2383 freelist_tail = freelist_iter;
2384 free_delta++;
2cfb7455 2385
d930ff03 2386 freelist_iter = nextfree;
2cfb7455
CL
2387 }
2388
894b8788 2389 /*
c2092c12
VB
2390 * Stage two: Unfreeze the slab while splicing the per-cpu
2391 * freelist to the head of slab's freelist.
d930ff03 2392 *
c2092c12 2393 * Ensure that the slab is unfrozen while the list presence
d930ff03 2394 * reflects the actual number of objects during unfreeze.
2cfb7455
CL
2395 *
2396 * We setup the list membership and then perform a cmpxchg
c2092c12
VB
2397 * with the count. If there is a mismatch then the slab
2398 * is not unfrozen but the slab is on the wrong list.
2cfb7455
CL
2399 *
2400 * Then we restart the process which may have to remove
c2092c12 2401 * the slab from the list that we just put it on again
2cfb7455
CL
2402 * because the number of objects in the slab may have
2403 * changed.
894b8788 2404 */
2cfb7455 2405redo:
894b8788 2406
bb192ed9
VB
2407 old.freelist = READ_ONCE(slab->freelist);
2408 old.counters = READ_ONCE(slab->counters);
a0132ac0 2409 VM_BUG_ON(!old.frozen);
7c2e132c 2410
2cfb7455
CL
2411 /* Determine target state of the slab */
2412 new.counters = old.counters;
d930ff03
VB
2413 if (freelist_tail) {
2414 new.inuse -= free_delta;
2415 set_freepointer(s, freelist_tail, old.freelist);
2cfb7455
CL
2416 new.freelist = freelist;
2417 } else
2418 new.freelist = old.freelist;
2419
2420 new.frozen = 0;
2421
8a5b20ae 2422 if (!new.inuse && n->nr_partial >= s->min_partial)
2cfb7455
CL
2423 m = M_FREE;
2424 else if (new.freelist) {
2425 m = M_PARTIAL;
2426 if (!lock) {
2427 lock = 1;
2428 /*
c2092c12
VB
2429 * Taking the spinlock removes the possibility that
2430 * acquire_slab() will see a slab that is frozen
2cfb7455 2431 */
3406e91b 2432 spin_lock_irqsave(&n->list_lock, flags);
2cfb7455
CL
2433 }
2434 } else {
2435 m = M_FULL;
965c4848 2436 if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {
2cfb7455
CL
2437 lock = 1;
2438 /*
2439 * This also ensures that the scanning of full
2440 * slabs from diagnostic functions will not see
2441 * any frozen slabs.
2442 */
3406e91b 2443 spin_lock_irqsave(&n->list_lock, flags);
2cfb7455
CL
2444 }
2445 }
2446
2447 if (l != m) {
2cfb7455 2448 if (l == M_PARTIAL)
bb192ed9 2449 remove_partial(n, slab);
2cfb7455 2450 else if (l == M_FULL)
bb192ed9 2451 remove_full(s, n, slab);
2cfb7455 2452
88349a28 2453 if (m == M_PARTIAL)
bb192ed9 2454 add_partial(n, slab, tail);
88349a28 2455 else if (m == M_FULL)
bb192ed9 2456 add_full(s, n, slab);
2cfb7455
CL
2457 }
2458
2459 l = m;
bb192ed9 2460 if (!cmpxchg_double_slab(s, slab,
2cfb7455
CL
2461 old.freelist, old.counters,
2462 new.freelist, new.counters,
2463 "unfreezing slab"))
2464 goto redo;
2465
2cfb7455 2466 if (lock)
3406e91b 2467 spin_unlock_irqrestore(&n->list_lock, flags);
2cfb7455 2468
88349a28
WY
2469 if (m == M_PARTIAL)
2470 stat(s, tail);
2471 else if (m == M_FULL)
2472 stat(s, DEACTIVATE_FULL);
2473 else if (m == M_FREE) {
2cfb7455 2474 stat(s, DEACTIVATE_EMPTY);
bb192ed9 2475 discard_slab(s, slab);
2cfb7455 2476 stat(s, FREE_SLAB);
894b8788 2477 }
81819f0f
CL
2478}
2479
345c905d 2480#ifdef CONFIG_SLUB_CPU_PARTIAL
bb192ed9 2481static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)
fc1455f4 2482{
43d77867 2483 struct kmem_cache_node *n = NULL, *n2 = NULL;
bb192ed9 2484 struct slab *slab, *slab_to_discard = NULL;
7cf9f3ba 2485 unsigned long flags = 0;
49e22585 2486
bb192ed9
VB
2487 while (partial_slab) {
2488 struct slab new;
2489 struct slab old;
49e22585 2490
bb192ed9
VB
2491 slab = partial_slab;
2492 partial_slab = slab->next;
43d77867 2493
bb192ed9 2494 n2 = get_node(s, slab_nid(slab));
43d77867
JK
2495 if (n != n2) {
2496 if (n)
7cf9f3ba 2497 spin_unlock_irqrestore(&n->list_lock, flags);
43d77867
JK
2498
2499 n = n2;
7cf9f3ba 2500 spin_lock_irqsave(&n->list_lock, flags);
43d77867 2501 }
49e22585
CL
2502
2503 do {
2504
bb192ed9
VB
2505 old.freelist = slab->freelist;
2506 old.counters = slab->counters;
a0132ac0 2507 VM_BUG_ON(!old.frozen);
49e22585
CL
2508
2509 new.counters = old.counters;
2510 new.freelist = old.freelist;
2511
2512 new.frozen = 0;
2513
bb192ed9 2514 } while (!__cmpxchg_double_slab(s, slab,
49e22585
CL
2515 old.freelist, old.counters,
2516 new.freelist, new.counters,
2517 "unfreezing slab"));
2518
8a5b20ae 2519 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
bb192ed9
VB
2520 slab->next = slab_to_discard;
2521 slab_to_discard = slab;
43d77867 2522 } else {
bb192ed9 2523 add_partial(n, slab, DEACTIVATE_TO_TAIL);
43d77867 2524 stat(s, FREE_ADD_PARTIAL);
49e22585
CL
2525 }
2526 }
2527
2528 if (n)
7cf9f3ba 2529 spin_unlock_irqrestore(&n->list_lock, flags);
8de06a6f 2530
bb192ed9
VB
2531 while (slab_to_discard) {
2532 slab = slab_to_discard;
2533 slab_to_discard = slab_to_discard->next;
9ada1934
SL
2534
2535 stat(s, DEACTIVATE_EMPTY);
bb192ed9 2536 discard_slab(s, slab);
9ada1934
SL
2537 stat(s, FREE_SLAB);
2538 }
fc1455f4 2539}
f3ab8b6b 2540
fc1455f4
VB
2541/*
2542 * Unfreeze all the cpu partial slabs.
2543 */
2544static void unfreeze_partials(struct kmem_cache *s)
2545{
bb192ed9 2546 struct slab *partial_slab;
fc1455f4
VB
2547 unsigned long flags;
2548
bd0e7491 2549 local_lock_irqsave(&s->cpu_slab->lock, flags);
bb192ed9 2550 partial_slab = this_cpu_read(s->cpu_slab->partial);
fc1455f4 2551 this_cpu_write(s->cpu_slab->partial, NULL);
bd0e7491 2552 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
fc1455f4 2553
bb192ed9
VB
2554 if (partial_slab)
2555 __unfreeze_partials(s, partial_slab);
fc1455f4
VB
2556}
2557
2558static void unfreeze_partials_cpu(struct kmem_cache *s,
2559 struct kmem_cache_cpu *c)
2560{
bb192ed9 2561 struct slab *partial_slab;
fc1455f4 2562
bb192ed9 2563 partial_slab = slub_percpu_partial(c);
fc1455f4
VB
2564 c->partial = NULL;
2565
bb192ed9
VB
2566 if (partial_slab)
2567 __unfreeze_partials(s, partial_slab);
49e22585
CL
2568}
2569
2570/*
c2092c12
VB
2571 * Put a slab that was just frozen (in __slab_free|get_partial_node) into a
2572 * partial slab slot if available.
49e22585
CL
2573 *
2574 * If we did not find a slot then simply move all the partials to the
2575 * per node partial list.
2576 */
bb192ed9 2577static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
49e22585 2578{
bb192ed9
VB
2579 struct slab *oldslab;
2580 struct slab *slab_to_unfreeze = NULL;
e0a043aa 2581 unsigned long flags;
bb192ed9 2582 int slabs = 0;
49e22585 2583
bd0e7491 2584 local_lock_irqsave(&s->cpu_slab->lock, flags);
49e22585 2585
bb192ed9 2586 oldslab = this_cpu_read(s->cpu_slab->partial);
e0a043aa 2587
bb192ed9
VB
2588 if (oldslab) {
2589 if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
e0a043aa
VB
2590 /*
2591 * Partial array is full. Move the existing set to the
2592 * per node partial list. Postpone the actual unfreezing
2593 * outside of the critical section.
2594 */
bb192ed9
VB
2595 slab_to_unfreeze = oldslab;
2596 oldslab = NULL;
e0a043aa 2597 } else {
bb192ed9 2598 slabs = oldslab->slabs;
49e22585 2599 }
e0a043aa 2600 }
49e22585 2601
bb192ed9 2602 slabs++;
49e22585 2603
bb192ed9
VB
2604 slab->slabs = slabs;
2605 slab->next = oldslab;
49e22585 2606
bb192ed9 2607 this_cpu_write(s->cpu_slab->partial, slab);
e0a043aa 2608
bd0e7491 2609 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
e0a043aa 2610
bb192ed9
VB
2611 if (slab_to_unfreeze) {
2612 __unfreeze_partials(s, slab_to_unfreeze);
e0a043aa
VB
2613 stat(s, CPU_PARTIAL_DRAIN);
2614 }
49e22585
CL
2615}
2616
e0a043aa
VB
2617#else /* CONFIG_SLUB_CPU_PARTIAL */
2618
2619static inline void unfreeze_partials(struct kmem_cache *s) { }
2620static inline void unfreeze_partials_cpu(struct kmem_cache *s,
2621 struct kmem_cache_cpu *c) { }
2622
2623#endif /* CONFIG_SLUB_CPU_PARTIAL */
2624
dfb4f096 2625static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
81819f0f 2626{
5a836bf6 2627 unsigned long flags;
bb192ed9 2628 struct slab *slab;
5a836bf6
SAS
2629 void *freelist;
2630
bd0e7491 2631 local_lock_irqsave(&s->cpu_slab->lock, flags);
5a836bf6 2632
bb192ed9 2633 slab = c->slab;
5a836bf6 2634 freelist = c->freelist;
c17dda40 2635
bb192ed9 2636 c->slab = NULL;
a019d201 2637 c->freelist = NULL;
c17dda40 2638 c->tid = next_tid(c->tid);
a019d201 2639
bd0e7491 2640 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
a019d201 2641
bb192ed9
VB
2642 if (slab) {
2643 deactivate_slab(s, slab, freelist);
5a836bf6
SAS
2644 stat(s, CPUSLAB_FLUSH);
2645 }
81819f0f
CL
2646}
2647
0c710013 2648static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
81819f0f 2649{
9dfc6e68 2650 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
08beb547 2651 void *freelist = c->freelist;
bb192ed9 2652 struct slab *slab = c->slab;
81819f0f 2653
bb192ed9 2654 c->slab = NULL;
08beb547
VB
2655 c->freelist = NULL;
2656 c->tid = next_tid(c->tid);
2657
bb192ed9
VB
2658 if (slab) {
2659 deactivate_slab(s, slab, freelist);
08beb547
VB
2660 stat(s, CPUSLAB_FLUSH);
2661 }
49e22585 2662
fc1455f4 2663 unfreeze_partials_cpu(s, c);
81819f0f
CL
2664}
2665
5a836bf6
SAS
2666struct slub_flush_work {
2667 struct work_struct work;
2668 struct kmem_cache *s;
2669 bool skip;
2670};
2671
fc1455f4
VB
2672/*
2673 * Flush cpu slab.
2674 *
5a836bf6 2675 * Called from CPU work handler with migration disabled.
fc1455f4 2676 */
5a836bf6 2677static void flush_cpu_slab(struct work_struct *w)
81819f0f 2678{
5a836bf6
SAS
2679 struct kmem_cache *s;
2680 struct kmem_cache_cpu *c;
2681 struct slub_flush_work *sfw;
2682
2683 sfw = container_of(w, struct slub_flush_work, work);
2684
2685 s = sfw->s;
2686 c = this_cpu_ptr(s->cpu_slab);
fc1455f4 2687
bb192ed9 2688 if (c->slab)
fc1455f4 2689 flush_slab(s, c);
81819f0f 2690
fc1455f4 2691 unfreeze_partials(s);
81819f0f
CL
2692}
2693
5a836bf6 2694static bool has_cpu_slab(int cpu, struct kmem_cache *s)
a8364d55 2695{
a8364d55
GBY
2696 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2697
bb192ed9 2698 return c->slab || slub_percpu_partial(c);
a8364d55
GBY
2699}
2700
5a836bf6
SAS
2701static DEFINE_MUTEX(flush_lock);
2702static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);
2703
2704static void flush_all_cpus_locked(struct kmem_cache *s)
2705{
2706 struct slub_flush_work *sfw;
2707 unsigned int cpu;
2708
2709 lockdep_assert_cpus_held();
2710 mutex_lock(&flush_lock);
2711
2712 for_each_online_cpu(cpu) {
2713 sfw = &per_cpu(slub_flush, cpu);
2714 if (!has_cpu_slab(cpu, s)) {
2715 sfw->skip = true;
2716 continue;
2717 }
2718 INIT_WORK(&sfw->work, flush_cpu_slab);
2719 sfw->skip = false;
2720 sfw->s = s;
2721 schedule_work_on(cpu, &sfw->work);
2722 }
2723
2724 for_each_online_cpu(cpu) {
2725 sfw = &per_cpu(slub_flush, cpu);
2726 if (sfw->skip)
2727 continue;
2728 flush_work(&sfw->work);
2729 }
2730
2731 mutex_unlock(&flush_lock);
2732}
2733
81819f0f
CL
2734static void flush_all(struct kmem_cache *s)
2735{
5a836bf6
SAS
2736 cpus_read_lock();
2737 flush_all_cpus_locked(s);
2738 cpus_read_unlock();
81819f0f
CL
2739}
2740
a96a87bf
SAS
2741/*
2742 * Use the cpu notifier to insure that the cpu slabs are flushed when
2743 * necessary.
2744 */
2745static int slub_cpu_dead(unsigned int cpu)
2746{
2747 struct kmem_cache *s;
a96a87bf
SAS
2748
2749 mutex_lock(&slab_mutex);
0e7ac738 2750 list_for_each_entry(s, &slab_caches, list)
a96a87bf 2751 __flush_cpu_slab(s, cpu);
a96a87bf
SAS
2752 mutex_unlock(&slab_mutex);
2753 return 0;
2754}
2755
dfb4f096
CL
2756/*
2757 * Check if the objects in a per cpu structure fit numa
2758 * locality expectations.
2759 */
bb192ed9 2760static inline int node_match(struct slab *slab, int node)
dfb4f096
CL
2761{
2762#ifdef CONFIG_NUMA
bb192ed9 2763 if (node != NUMA_NO_NODE && slab_nid(slab) != node)
dfb4f096
CL
2764 return 0;
2765#endif
2766 return 1;
2767}
2768
9a02d699 2769#ifdef CONFIG_SLUB_DEBUG
bb192ed9 2770static int count_free(struct slab *slab)
781b2ba6 2771{
bb192ed9 2772 return slab->objects - slab->inuse;
781b2ba6
PE
2773}
2774
9a02d699
DR
2775static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2776{
2777 return atomic_long_read(&n->total_objects);
2778}
2779#endif /* CONFIG_SLUB_DEBUG */
2780
2781#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
781b2ba6 2782static unsigned long count_partial(struct kmem_cache_node *n,
bb192ed9 2783 int (*get_count)(struct slab *))
781b2ba6
PE
2784{
2785 unsigned long flags;
2786 unsigned long x = 0;
bb192ed9 2787 struct slab *slab;
781b2ba6
PE
2788
2789 spin_lock_irqsave(&n->list_lock, flags);
bb192ed9
VB
2790 list_for_each_entry(slab, &n->partial, slab_list)
2791 x += get_count(slab);
781b2ba6
PE
2792 spin_unlock_irqrestore(&n->list_lock, flags);
2793 return x;
2794}
9a02d699 2795#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
26c02cf0 2796
781b2ba6
PE
2797static noinline void
2798slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2799{
9a02d699
DR
2800#ifdef CONFIG_SLUB_DEBUG
2801 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
2802 DEFAULT_RATELIMIT_BURST);
781b2ba6 2803 int node;
fa45dc25 2804 struct kmem_cache_node *n;
781b2ba6 2805
9a02d699
DR
2806 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
2807 return;
2808
5b3810e5
VB
2809 pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
2810 nid, gfpflags, &gfpflags);
19af27af 2811 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
f9f58285
FF
2812 s->name, s->object_size, s->size, oo_order(s->oo),
2813 oo_order(s->min));
781b2ba6 2814
3b0efdfa 2815 if (oo_order(s->min) > get_order(s->object_size))
f9f58285
FF
2816 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
2817 s->name);
fa5ec8a1 2818
fa45dc25 2819 for_each_kmem_cache_node(s, node, n) {
781b2ba6
PE
2820 unsigned long nr_slabs;
2821 unsigned long nr_objs;
2822 unsigned long nr_free;
2823
26c02cf0
AB
2824 nr_free = count_partial(n, count_free);
2825 nr_slabs = node_nr_slabs(n);
2826 nr_objs = node_nr_objs(n);
781b2ba6 2827
f9f58285 2828 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
781b2ba6
PE
2829 node, nr_slabs, nr_objs, nr_free);
2830 }
9a02d699 2831#endif
781b2ba6
PE
2832}
2833
01b34d16 2834static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
072bb0aa 2835{
01b34d16 2836 if (unlikely(slab_test_pfmemalloc(slab)))
0b303fb4
VB
2837 return gfp_pfmemalloc_allowed(gfpflags);
2838
2839 return true;
2840}
2841
213eeb9f 2842/*
c2092c12
VB
2843 * Check the slab->freelist and either transfer the freelist to the
2844 * per cpu freelist or deactivate the slab.
213eeb9f 2845 *
c2092c12 2846 * The slab is still frozen if the return value is not NULL.
213eeb9f 2847 *
c2092c12 2848 * If this function returns NULL then the slab has been unfrozen.
213eeb9f 2849 */
bb192ed9 2850static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
213eeb9f 2851{
bb192ed9 2852 struct slab new;
213eeb9f
CL
2853 unsigned long counters;
2854 void *freelist;
2855
bd0e7491
VB
2856 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
2857
213eeb9f 2858 do {
bb192ed9
VB
2859 freelist = slab->freelist;
2860 counters = slab->counters;
6faa6833 2861
213eeb9f 2862 new.counters = counters;
a0132ac0 2863 VM_BUG_ON(!new.frozen);
213eeb9f 2864
bb192ed9 2865 new.inuse = slab->objects;
213eeb9f
CL
2866 new.frozen = freelist != NULL;
2867
bb192ed9 2868 } while (!__cmpxchg_double_slab(s, slab,
213eeb9f
CL
2869 freelist, counters,
2870 NULL, new.counters,
2871 "get_freelist"));
2872
2873 return freelist;
2874}
2875
81819f0f 2876/*
894b8788
CL
2877 * Slow path. The lockless freelist is empty or we need to perform
2878 * debugging duties.
2879 *
894b8788
CL
2880 * Processing is still very fast if new objects have been freed to the
2881 * regular freelist. In that case we simply take over the regular freelist
2882 * as the lockless freelist and zap the regular freelist.
81819f0f 2883 *
894b8788
CL
2884 * If that is not working then we fall back to the partial lists. We take the
2885 * first element of the freelist as the object to allocate now and move the
2886 * rest of the freelist to the lockless freelist.
81819f0f 2887 *
894b8788 2888 * And if we were unable to get a new slab from the partial slab lists then
6446faa2
CL
2889 * we need to allocate a new slab. This is the slowest path since it involves
2890 * a call to the page allocator and the setup of a new slab.
a380a3c7 2891 *
e500059b 2892 * Version of __slab_alloc to use when we know that preemption is
a380a3c7 2893 * already disabled (which is the case for bulk allocation).
81819f0f 2894 */
a380a3c7 2895static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
ce71e27c 2896 unsigned long addr, struct kmem_cache_cpu *c)
81819f0f 2897{
6faa6833 2898 void *freelist;
bb192ed9 2899 struct slab *slab;
e500059b 2900 unsigned long flags;
81819f0f 2901
9f986d99
AW
2902 stat(s, ALLOC_SLOWPATH);
2903
c2092c12 2904reread_slab:
0b303fb4 2905
bb192ed9
VB
2906 slab = READ_ONCE(c->slab);
2907 if (!slab) {
0715e6c5
VB
2908 /*
2909 * if the node is not online or has no normal memory, just
2910 * ignore the node constraint
2911 */
2912 if (unlikely(node != NUMA_NO_NODE &&
7e1fa93d 2913 !node_isset(node, slab_nodes)))
0715e6c5 2914 node = NUMA_NO_NODE;
81819f0f 2915 goto new_slab;
0715e6c5 2916 }
49e22585 2917redo:
6faa6833 2918
bb192ed9 2919 if (unlikely(!node_match(slab, node))) {
0715e6c5
VB
2920 /*
2921 * same as above but node_match() being false already
2922 * implies node != NUMA_NO_NODE
2923 */
7e1fa93d 2924 if (!node_isset(node, slab_nodes)) {
0715e6c5
VB
2925 node = NUMA_NO_NODE;
2926 goto redo;
2927 } else {
a561ce00 2928 stat(s, ALLOC_NODE_MISMATCH);
0b303fb4 2929 goto deactivate_slab;
a561ce00 2930 }
fc59c053 2931 }
6446faa2 2932
072bb0aa
MG
2933 /*
2934 * By rights, we should be searching for a slab page that was
2935 * PFMEMALLOC but right now, we are losing the pfmemalloc
2936 * information when the page leaves the per-cpu allocator
2937 */
bb192ed9 2938 if (unlikely(!pfmemalloc_match(slab, gfpflags)))
0b303fb4 2939 goto deactivate_slab;
072bb0aa 2940
c2092c12 2941 /* must check again c->slab in case we got preempted and it changed */
bd0e7491 2942 local_lock_irqsave(&s->cpu_slab->lock, flags);
bb192ed9 2943 if (unlikely(slab != c->slab)) {
bd0e7491 2944 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
c2092c12 2945 goto reread_slab;
0b303fb4 2946 }
6faa6833
CL
2947 freelist = c->freelist;
2948 if (freelist)
73736e03 2949 goto load_freelist;
03e404af 2950
bb192ed9 2951 freelist = get_freelist(s, slab);
6446faa2 2952
6faa6833 2953 if (!freelist) {
bb192ed9 2954 c->slab = NULL;
bd0e7491 2955 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
03e404af 2956 stat(s, DEACTIVATE_BYPASS);
fc59c053 2957 goto new_slab;
03e404af 2958 }
6446faa2 2959
84e554e6 2960 stat(s, ALLOC_REFILL);
6446faa2 2961
894b8788 2962load_freelist:
0b303fb4 2963
bd0e7491 2964 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
0b303fb4 2965
507effea
CL
2966 /*
2967 * freelist is pointing to the list of objects to be used.
c2092c12
VB
2968 * slab is pointing to the slab from which the objects are obtained.
2969 * That slab must be frozen for per cpu allocations to work.
507effea 2970 */
bb192ed9 2971 VM_BUG_ON(!c->slab->frozen);
6faa6833 2972 c->freelist = get_freepointer(s, freelist);
8a5ec0ba 2973 c->tid = next_tid(c->tid);
bd0e7491 2974 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
6faa6833 2975 return freelist;
81819f0f 2976
0b303fb4
VB
2977deactivate_slab:
2978
bd0e7491 2979 local_lock_irqsave(&s->cpu_slab->lock, flags);
bb192ed9 2980 if (slab != c->slab) {
bd0e7491 2981 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
c2092c12 2982 goto reread_slab;
0b303fb4 2983 }
a019d201 2984 freelist = c->freelist;
bb192ed9 2985 c->slab = NULL;
a019d201 2986 c->freelist = NULL;
bd0e7491 2987 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
bb192ed9 2988 deactivate_slab(s, slab, freelist);
0b303fb4 2989
81819f0f 2990new_slab:
2cfb7455 2991
a93cf07b 2992 if (slub_percpu_partial(c)) {
bd0e7491 2993 local_lock_irqsave(&s->cpu_slab->lock, flags);
bb192ed9 2994 if (unlikely(c->slab)) {
bd0e7491 2995 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
c2092c12 2996 goto reread_slab;
fa417ab7 2997 }
4b1f449d 2998 if (unlikely(!slub_percpu_partial(c))) {
bd0e7491 2999 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
25c00c50
VB
3000 /* we were preempted and partial list got empty */
3001 goto new_objects;
4b1f449d 3002 }
fa417ab7 3003
bb192ed9
VB
3004 slab = c->slab = slub_percpu_partial(c);
3005 slub_set_percpu_partial(c, slab);
bd0e7491 3006 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
49e22585 3007 stat(s, CPU_PARTIAL_ALLOC);
49e22585 3008 goto redo;
81819f0f
CL
3009 }
3010
fa417ab7
VB
3011new_objects:
3012
bb192ed9 3013 freelist = get_partial(s, gfpflags, node, &slab);
3f2b77e3 3014 if (freelist)
c2092c12 3015 goto check_new_slab;
2a904905 3016
25c00c50 3017 slub_put_cpu_ptr(s->cpu_slab);
bb192ed9 3018 slab = new_slab(s, gfpflags, node);
25c00c50 3019 c = slub_get_cpu_ptr(s->cpu_slab);
01ad8a7b 3020
bb192ed9 3021 if (unlikely(!slab)) {
9a02d699 3022 slab_out_of_memory(s, gfpflags, node);
f4697436 3023 return NULL;
81819f0f 3024 }
2cfb7455 3025
53a0de06 3026 /*
c2092c12 3027 * No other reference to the slab yet so we can
53a0de06
VB
3028 * muck around with it freely without cmpxchg
3029 */
bb192ed9
VB
3030 freelist = slab->freelist;
3031 slab->freelist = NULL;
53a0de06
VB
3032
3033 stat(s, ALLOC_SLAB);
53a0de06 3034
c2092c12 3035check_new_slab:
2cfb7455 3036
1572df7c 3037 if (kmem_cache_debug(s)) {
bb192ed9 3038 if (!alloc_debug_processing(s, slab, freelist, addr)) {
1572df7c
VB
3039 /* Slab failed checks. Next slab needed */
3040 goto new_slab;
fa417ab7 3041 } else {
1572df7c
VB
3042 /*
3043 * For debug case, we don't load freelist so that all
3044 * allocations go through alloc_debug_processing()
3045 */
3046 goto return_single;
fa417ab7 3047 }
1572df7c
VB
3048 }
3049
bb192ed9 3050 if (unlikely(!pfmemalloc_match(slab, gfpflags)))
1572df7c
VB
3051 /*
3052 * For !pfmemalloc_match() case we don't load freelist so that
3053 * we don't make further mismatched allocations easier.
3054 */
3055 goto return_single;
3056
c2092c12 3057retry_load_slab:
cfdf836e 3058
bd0e7491 3059 local_lock_irqsave(&s->cpu_slab->lock, flags);
bb192ed9 3060 if (unlikely(c->slab)) {
cfdf836e 3061 void *flush_freelist = c->freelist;
bb192ed9 3062 struct slab *flush_slab = c->slab;
cfdf836e 3063
bb192ed9 3064 c->slab = NULL;
cfdf836e
VB
3065 c->freelist = NULL;
3066 c->tid = next_tid(c->tid);
3067
bd0e7491 3068 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
cfdf836e 3069
bb192ed9 3070 deactivate_slab(s, flush_slab, flush_freelist);
cfdf836e
VB
3071
3072 stat(s, CPUSLAB_FLUSH);
3073
c2092c12 3074 goto retry_load_slab;
cfdf836e 3075 }
bb192ed9 3076 c->slab = slab;
3f2b77e3 3077
1572df7c
VB
3078 goto load_freelist;
3079
3080return_single:
894b8788 3081
bb192ed9 3082 deactivate_slab(s, slab, get_freepointer(s, freelist));
6faa6833 3083 return freelist;
894b8788
CL
3084}
3085
a380a3c7 3086/*
e500059b
VB
3087 * A wrapper for ___slab_alloc() for contexts where preemption is not yet
3088 * disabled. Compensates for possible cpu changes by refetching the per cpu area
3089 * pointer.
a380a3c7
CL
3090 */
3091static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3092 unsigned long addr, struct kmem_cache_cpu *c)
3093{
3094 void *p;
a380a3c7 3095
e500059b 3096#ifdef CONFIG_PREEMPT_COUNT
a380a3c7
CL
3097 /*
3098 * We may have been preempted and rescheduled on a different
e500059b 3099 * cpu before disabling preemption. Need to reload cpu area
a380a3c7
CL
3100 * pointer.
3101 */
25c00c50 3102 c = slub_get_cpu_ptr(s->cpu_slab);
a380a3c7
CL
3103#endif
3104
3105 p = ___slab_alloc(s, gfpflags, node, addr, c);
e500059b 3106#ifdef CONFIG_PREEMPT_COUNT
25c00c50 3107 slub_put_cpu_ptr(s->cpu_slab);
e500059b 3108#endif
a380a3c7
CL
3109 return p;
3110}
3111
0f181f9f
AP
3112/*
3113 * If the object has been wiped upon free, make sure it's fully initialized by
3114 * zeroing out freelist pointer.
3115 */
3116static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
3117 void *obj)
3118{
3119 if (unlikely(slab_want_init_on_free(s)) && obj)
ce5716c6
AK
3120 memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
3121 0, sizeof(void *));
0f181f9f
AP
3122}
3123
894b8788
CL
3124/*
3125 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
3126 * have the fastpath folded into their functions. So no function call
3127 * overhead for requests that can be satisfied on the fastpath.
3128 *
3129 * The fastpath works by first checking if the lockless freelist can be used.
3130 * If not then __slab_alloc is called for slow processing.
3131 *
3132 * Otherwise we can simply pick the next object from the lockless free list.
3133 */
2b847c3c 3134static __always_inline void *slab_alloc_node(struct kmem_cache *s,
b89fb5ef 3135 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
894b8788 3136{
03ec0ed5 3137 void *object;
dfb4f096 3138 struct kmem_cache_cpu *c;
bb192ed9 3139 struct slab *slab;
8a5ec0ba 3140 unsigned long tid;
964d4bd3 3141 struct obj_cgroup *objcg = NULL;
da844b78 3142 bool init = false;
1f84260c 3143
964d4bd3 3144 s = slab_pre_alloc_hook(s, &objcg, 1, gfpflags);
8135be5a 3145 if (!s)
773ff60e 3146 return NULL;
b89fb5ef
AP
3147
3148 object = kfence_alloc(s, orig_size, gfpflags);
3149 if (unlikely(object))
3150 goto out;
3151
8a5ec0ba 3152redo:
8a5ec0ba
CL
3153 /*
3154 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
3155 * enabled. We may switch back and forth between cpus while
3156 * reading from one cpu area. That does not matter as long
3157 * as we end up on the original cpu again when doing the cmpxchg.
7cccd80b 3158 *
9b4bc85a
VB
3159 * We must guarantee that tid and kmem_cache_cpu are retrieved on the
3160 * same cpu. We read first the kmem_cache_cpu pointer and use it to read
3161 * the tid. If we are preempted and switched to another cpu between the
3162 * two reads, it's OK as the two are still associated with the same cpu
3163 * and cmpxchg later will validate the cpu.
8a5ec0ba 3164 */
9b4bc85a
VB
3165 c = raw_cpu_ptr(s->cpu_slab);
3166 tid = READ_ONCE(c->tid);
9aabf810
JK
3167
3168 /*
3169 * Irqless object alloc/free algorithm used here depends on sequence
3170 * of fetching cpu_slab's data. tid should be fetched before anything
c2092c12 3171 * on c to guarantee that object and slab associated with previous tid
9aabf810 3172 * won't be used with current tid. If we fetch tid first, object and
c2092c12 3173 * slab could be one associated with next tid and our alloc/free
9aabf810
JK
3174 * request will be failed. In this case, we will retry. So, no problem.
3175 */
3176 barrier();
8a5ec0ba 3177
8a5ec0ba
CL
3178 /*
3179 * The transaction ids are globally unique per cpu and per operation on
3180 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
3181 * occurs on the right processor and that there was no operation on the
3182 * linked list in between.
3183 */
8a5ec0ba 3184
9dfc6e68 3185 object = c->freelist;
bb192ed9 3186 slab = c->slab;
bd0e7491
VB
3187 /*
3188 * We cannot use the lockless fastpath on PREEMPT_RT because if a
3189 * slowpath has taken the local_lock_irqsave(), it is not protected
3190 * against a fast path operation in an irq handler. So we need to take
3191 * the slow path which uses local_lock. It is still relatively fast if
3192 * there is a suitable cpu freelist.
3193 */
3194 if (IS_ENABLED(CONFIG_PREEMPT_RT) ||
bb192ed9 3195 unlikely(!object || !slab || !node_match(slab, node))) {
dfb4f096 3196 object = __slab_alloc(s, gfpflags, node, addr, c);
8eae1492 3197 } else {
0ad9500e
ED
3198 void *next_object = get_freepointer_safe(s, object);
3199
8a5ec0ba 3200 /*
25985edc 3201 * The cmpxchg will only match if there was no additional
8a5ec0ba
CL
3202 * operation and if we are on the right processor.
3203 *
d0e0ac97
CG
3204 * The cmpxchg does the following atomically (without lock
3205 * semantics!)
8a5ec0ba
CL
3206 * 1. Relocate first pointer to the current per cpu area.
3207 * 2. Verify that tid and freelist have not been changed
3208 * 3. If they were not changed replace tid and freelist
3209 *
d0e0ac97
CG
3210 * Since this is without lock semantics the protection is only
3211 * against code executing on this cpu *not* from access by
3212 * other cpus.
8a5ec0ba 3213 */
933393f5 3214 if (unlikely(!this_cpu_cmpxchg_double(
8a5ec0ba
CL
3215 s->cpu_slab->freelist, s->cpu_slab->tid,
3216 object, tid,
0ad9500e 3217 next_object, next_tid(tid)))) {
8a5ec0ba
CL
3218
3219 note_cmpxchg_failure("slab_alloc", s, tid);
3220 goto redo;
3221 }
0ad9500e 3222 prefetch_freepointer(s, next_object);
84e554e6 3223 stat(s, ALLOC_FASTPATH);
894b8788 3224 }
0f181f9f 3225
ce5716c6 3226 maybe_wipe_obj_freeptr(s, object);
da844b78 3227 init = slab_want_init_on_alloc(gfpflags, s);
d07dbea4 3228
b89fb5ef 3229out:
da844b78 3230 slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init);
5a896d9e 3231
894b8788 3232 return object;
81819f0f
CL
3233}
3234
2b847c3c 3235static __always_inline void *slab_alloc(struct kmem_cache *s,
b89fb5ef 3236 gfp_t gfpflags, unsigned long addr, size_t orig_size)
2b847c3c 3237{
b89fb5ef 3238 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr, orig_size);
2b847c3c
EG
3239}
3240
81819f0f
CL
3241void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
3242{
b89fb5ef 3243 void *ret = slab_alloc(s, gfpflags, _RET_IP_, s->object_size);
5b882be4 3244
d0e0ac97
CG
3245 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
3246 s->size, gfpflags);
5b882be4
EGM
3247
3248 return ret;
81819f0f
CL
3249}
3250EXPORT_SYMBOL(kmem_cache_alloc);
3251
0f24f128 3252#ifdef CONFIG_TRACING
4a92379b
RK
3253void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
3254{
b89fb5ef 3255 void *ret = slab_alloc(s, gfpflags, _RET_IP_, size);
4a92379b 3256 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
0116523c 3257 ret = kasan_kmalloc(s, ret, size, gfpflags);
4a92379b
RK
3258 return ret;
3259}
3260EXPORT_SYMBOL(kmem_cache_alloc_trace);
5b882be4
EGM
3261#endif
3262
81819f0f
CL
3263#ifdef CONFIG_NUMA
3264void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
3265{
b89fb5ef 3266 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, s->object_size);
5b882be4 3267
ca2b84cb 3268 trace_kmem_cache_alloc_node(_RET_IP_, ret,
3b0efdfa 3269 s->object_size, s->size, gfpflags, node);
5b882be4
EGM
3270
3271 return ret;
81819f0f
CL
3272}
3273EXPORT_SYMBOL(kmem_cache_alloc_node);
81819f0f 3274
0f24f128 3275#ifdef CONFIG_TRACING
4a92379b 3276void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
5b882be4 3277 gfp_t gfpflags,
4a92379b 3278 int node, size_t size)
5b882be4 3279{
b89fb5ef 3280 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, size);
4a92379b
RK
3281
3282 trace_kmalloc_node(_RET_IP_, ret,
3283 size, s->size, gfpflags, node);
0316bec2 3284
0116523c 3285 ret = kasan_kmalloc(s, ret, size, gfpflags);
4a92379b 3286 return ret;
5b882be4 3287}
4a92379b 3288EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
5b882be4 3289#endif
6dfd1b65 3290#endif /* CONFIG_NUMA */
5b882be4 3291
81819f0f 3292/*
94e4d712 3293 * Slow path handling. This may still be called frequently since objects
894b8788 3294 * have a longer lifetime than the cpu slabs in most processing loads.
81819f0f 3295 *
894b8788 3296 * So we still attempt to reduce cache line usage. Just take the slab
c2092c12 3297 * lock and free the item. If there is no additional partial slab
894b8788 3298 * handling required then we can return immediately.
81819f0f 3299 */
bb192ed9 3300static void __slab_free(struct kmem_cache *s, struct slab *slab,
81084651
JDB
3301 void *head, void *tail, int cnt,
3302 unsigned long addr)
3303
81819f0f
CL
3304{
3305 void *prior;
2cfb7455 3306 int was_frozen;
bb192ed9 3307 struct slab new;
2cfb7455
CL
3308 unsigned long counters;
3309 struct kmem_cache_node *n = NULL;
3f649ab7 3310 unsigned long flags;
81819f0f 3311
8a5ec0ba 3312 stat(s, FREE_SLOWPATH);
81819f0f 3313
b89fb5ef
AP
3314 if (kfence_free(head))
3315 return;
3316
19c7ff9e 3317 if (kmem_cache_debug(s) &&
bb192ed9 3318 !free_debug_processing(s, slab, head, tail, cnt, addr))
80f08c19 3319 return;
6446faa2 3320
2cfb7455 3321 do {
837d678d
JK
3322 if (unlikely(n)) {
3323 spin_unlock_irqrestore(&n->list_lock, flags);
3324 n = NULL;
3325 }
bb192ed9
VB
3326 prior = slab->freelist;
3327 counters = slab->counters;
81084651 3328 set_freepointer(s, tail, prior);
2cfb7455
CL
3329 new.counters = counters;
3330 was_frozen = new.frozen;
81084651 3331 new.inuse -= cnt;
837d678d 3332 if ((!new.inuse || !prior) && !was_frozen) {
49e22585 3333
c65c1877 3334 if (kmem_cache_has_cpu_partial(s) && !prior) {
49e22585
CL
3335
3336 /*
d0e0ac97
CG
3337 * Slab was on no list before and will be
3338 * partially empty
3339 * We can defer the list move and instead
3340 * freeze it.
49e22585
CL
3341 */
3342 new.frozen = 1;
3343
c65c1877 3344 } else { /* Needs to be taken off a list */
49e22585 3345
bb192ed9 3346 n = get_node(s, slab_nid(slab));
49e22585
CL
3347 /*
3348 * Speculatively acquire the list_lock.
3349 * If the cmpxchg does not succeed then we may
3350 * drop the list_lock without any processing.
3351 *
3352 * Otherwise the list_lock will synchronize with
3353 * other processors updating the list of slabs.
3354 */
3355 spin_lock_irqsave(&n->list_lock, flags);
3356
3357 }
2cfb7455 3358 }
81819f0f 3359
bb192ed9 3360 } while (!cmpxchg_double_slab(s, slab,
2cfb7455 3361 prior, counters,
81084651 3362 head, new.counters,
2cfb7455 3363 "__slab_free"));
81819f0f 3364
2cfb7455 3365 if (likely(!n)) {
49e22585 3366
c270cf30
AW
3367 if (likely(was_frozen)) {
3368 /*
3369 * The list lock was not taken therefore no list
3370 * activity can be necessary.
3371 */
3372 stat(s, FREE_FROZEN);
3373 } else if (new.frozen) {
3374 /*
c2092c12 3375 * If we just froze the slab then put it onto the
c270cf30
AW
3376 * per cpu partial list.
3377 */
bb192ed9 3378 put_cpu_partial(s, slab, 1);
8028dcea
AS
3379 stat(s, CPU_PARTIAL_FREE);
3380 }
c270cf30 3381
b455def2
L
3382 return;
3383 }
81819f0f 3384
8a5b20ae 3385 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
837d678d
JK
3386 goto slab_empty;
3387
81819f0f 3388 /*
837d678d
JK
3389 * Objects left in the slab. If it was not on the partial list before
3390 * then add it.
81819f0f 3391 */
345c905d 3392 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
bb192ed9
VB
3393 remove_full(s, n, slab);
3394 add_partial(n, slab, DEACTIVATE_TO_TAIL);
837d678d 3395 stat(s, FREE_ADD_PARTIAL);
8ff12cfc 3396 }
80f08c19 3397 spin_unlock_irqrestore(&n->list_lock, flags);
81819f0f
CL
3398 return;
3399
3400slab_empty:
a973e9dd 3401 if (prior) {
81819f0f 3402 /*
6fbabb20 3403 * Slab on the partial list.
81819f0f 3404 */
bb192ed9 3405 remove_partial(n, slab);
84e554e6 3406 stat(s, FREE_REMOVE_PARTIAL);
c65c1877 3407 } else {
6fbabb20 3408 /* Slab must be on the full list */
bb192ed9 3409 remove_full(s, n, slab);
c65c1877 3410 }
2cfb7455 3411
80f08c19 3412 spin_unlock_irqrestore(&n->list_lock, flags);
84e554e6 3413 stat(s, FREE_SLAB);
bb192ed9 3414 discard_slab(s, slab);
81819f0f
CL
3415}
3416
894b8788
CL
3417/*
3418 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
3419 * can perform fastpath freeing without additional function calls.
3420 *
3421 * The fastpath is only possible if we are freeing to the current cpu slab
3422 * of this processor. This typically the case if we have just allocated
3423 * the item before.
3424 *
3425 * If fastpath is not possible then fall back to __slab_free where we deal
3426 * with all sorts of special processing.
81084651
JDB
3427 *
3428 * Bulk free of a freelist with several objects (all pointing to the
c2092c12 3429 * same slab) possible by specifying head and tail ptr, plus objects
81084651 3430 * count (cnt). Bulk free indicated by tail pointer being set.
894b8788 3431 */
80a9201a 3432static __always_inline void do_slab_free(struct kmem_cache *s,
bb192ed9 3433 struct slab *slab, void *head, void *tail,
80a9201a 3434 int cnt, unsigned long addr)
894b8788 3435{
81084651 3436 void *tail_obj = tail ? : head;
dfb4f096 3437 struct kmem_cache_cpu *c;
8a5ec0ba 3438 unsigned long tid;
964d4bd3 3439
3ddd6026
ML
3440 /* memcg_slab_free_hook() is already called for bulk free. */
3441 if (!tail)
3442 memcg_slab_free_hook(s, &head, 1);
8a5ec0ba
CL
3443redo:
3444 /*
3445 * Determine the currently cpus per cpu slab.
3446 * The cpu may change afterward. However that does not matter since
3447 * data is retrieved via this pointer. If we are on the same cpu
2ae44005 3448 * during the cmpxchg then the free will succeed.
8a5ec0ba 3449 */
9b4bc85a
VB
3450 c = raw_cpu_ptr(s->cpu_slab);
3451 tid = READ_ONCE(c->tid);
c016b0bd 3452
9aabf810
JK
3453 /* Same with comment on barrier() in slab_alloc_node() */
3454 barrier();
c016b0bd 3455
bb192ed9 3456 if (likely(slab == c->slab)) {
bd0e7491 3457#ifndef CONFIG_PREEMPT_RT
5076190d
LT
3458 void **freelist = READ_ONCE(c->freelist);
3459
3460 set_freepointer(s, tail_obj, freelist);
8a5ec0ba 3461
933393f5 3462 if (unlikely(!this_cpu_cmpxchg_double(
8a5ec0ba 3463 s->cpu_slab->freelist, s->cpu_slab->tid,
5076190d 3464 freelist, tid,
81084651 3465 head, next_tid(tid)))) {
8a5ec0ba
CL
3466
3467 note_cmpxchg_failure("slab_free", s, tid);
3468 goto redo;
3469 }
bd0e7491
VB
3470#else /* CONFIG_PREEMPT_RT */
3471 /*
3472 * We cannot use the lockless fastpath on PREEMPT_RT because if
3473 * a slowpath has taken the local_lock_irqsave(), it is not
3474 * protected against a fast path operation in an irq handler. So
3475 * we need to take the local_lock. We shouldn't simply defer to
3476 * __slab_free() as that wouldn't use the cpu freelist at all.
3477 */
3478 void **freelist;
3479
3480 local_lock(&s->cpu_slab->lock);
3481 c = this_cpu_ptr(s->cpu_slab);
bb192ed9 3482 if (unlikely(slab != c->slab)) {
bd0e7491
VB
3483 local_unlock(&s->cpu_slab->lock);
3484 goto redo;
3485 }
3486 tid = c->tid;
3487 freelist = c->freelist;
3488
3489 set_freepointer(s, tail_obj, freelist);
3490 c->freelist = head;
3491 c->tid = next_tid(tid);
3492
3493 local_unlock(&s->cpu_slab->lock);
3494#endif
84e554e6 3495 stat(s, FREE_FASTPATH);
894b8788 3496 } else
bb192ed9 3497 __slab_free(s, slab, head, tail_obj, cnt, addr);
894b8788 3498
894b8788
CL
3499}
3500
bb192ed9 3501static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
80a9201a
AP
3502 void *head, void *tail, int cnt,
3503 unsigned long addr)
3504{
80a9201a 3505 /*
c3895391
AK
3506 * With KASAN enabled slab_free_freelist_hook modifies the freelist
3507 * to remove objects, whose reuse must be delayed.
80a9201a 3508 */
899447f6 3509 if (slab_free_freelist_hook(s, &head, &tail, &cnt))
bb192ed9 3510 do_slab_free(s, slab, head, tail, cnt, addr);
80a9201a
AP
3511}
3512
2bd926b4 3513#ifdef CONFIG_KASAN_GENERIC
80a9201a
AP
3514void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
3515{
bb192ed9 3516 do_slab_free(cache, virt_to_slab(x), x, NULL, 1, addr);
80a9201a
AP
3517}
3518#endif
3519
81819f0f
CL
3520void kmem_cache_free(struct kmem_cache *s, void *x)
3521{
b9ce5ef4
GC
3522 s = cache_from_obj(s, x);
3523 if (!s)
79576102 3524 return;
3544de8e 3525 trace_kmem_cache_free(_RET_IP_, x, s->name);
bb192ed9 3526 slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_);
81819f0f
CL
3527}
3528EXPORT_SYMBOL(kmem_cache_free);
3529
d0ecd894 3530struct detached_freelist {
cc465c3b 3531 struct slab *slab;
d0ecd894
JDB
3532 void *tail;
3533 void *freelist;
3534 int cnt;
376bf125 3535 struct kmem_cache *s;
d0ecd894 3536};
fbd02630 3537
d835eef4 3538static inline void free_large_kmalloc(struct folio *folio, void *object)
f227f0fa 3539{
d835eef4 3540 unsigned int order = folio_order(folio);
f227f0fa 3541
d835eef4 3542 if (WARN_ON_ONCE(order == 0))
d0fe47c6
KW
3543 pr_warn_once("object pointer: 0x%p\n", object);
3544
1ed7ce57 3545 kfree_hook(object);
d835eef4
MWO
3546 mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
3547 -(PAGE_SIZE << order));
3548 __free_pages(folio_page(folio, 0), order);
f227f0fa
SB
3549}
3550
d0ecd894
JDB
3551/*
3552 * This function progressively scans the array with free objects (with
3553 * a limited look ahead) and extract objects belonging to the same
cc465c3b
MWO
3554 * slab. It builds a detached freelist directly within the given
3555 * slab/objects. This can happen without any need for
d0ecd894
JDB
3556 * synchronization, because the objects are owned by running process.
3557 * The freelist is build up as a single linked list in the objects.
3558 * The idea is, that this detached freelist can then be bulk
3559 * transferred to the real freelist(s), but only requiring a single
3560 * synchronization primitive. Look ahead in the array is limited due
3561 * to performance reasons.
3562 */
376bf125
JDB
3563static inline
3564int build_detached_freelist(struct kmem_cache *s, size_t size,
3565 void **p, struct detached_freelist *df)
d0ecd894
JDB
3566{
3567 size_t first_skipped_index = 0;
3568 int lookahead = 3;
3569 void *object;
cc465c3b
MWO
3570 struct folio *folio;
3571 struct slab *slab;
fbd02630 3572
d0ecd894 3573 /* Always re-init detached_freelist */
cc465c3b 3574 df->slab = NULL;
fbd02630 3575
d0ecd894
JDB
3576 do {
3577 object = p[--size];
ca257195 3578 /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
d0ecd894 3579 } while (!object && size);
3eed034d 3580
d0ecd894
JDB
3581 if (!object)
3582 return 0;
fbd02630 3583
cc465c3b 3584 folio = virt_to_folio(object);
ca257195
JDB
3585 if (!s) {
3586 /* Handle kalloc'ed objects */
cc465c3b 3587 if (unlikely(!folio_test_slab(folio))) {
d835eef4 3588 free_large_kmalloc(folio, object);
ca257195
JDB
3589 p[size] = NULL; /* mark object processed */
3590 return size;
3591 }
3592 /* Derive kmem_cache from object */
cc465c3b
MWO
3593 slab = folio_slab(folio);
3594 df->s = slab->slab_cache;
ca257195 3595 } else {
cc465c3b 3596 slab = folio_slab(folio);
ca257195
JDB
3597 df->s = cache_from_obj(s, object); /* Support for memcg */
3598 }
376bf125 3599
b89fb5ef 3600 if (is_kfence_address(object)) {
d57a964e 3601 slab_free_hook(df->s, object, false);
b89fb5ef
AP
3602 __kfence_free(object);
3603 p[size] = NULL; /* mark object processed */
3604 return size;
3605 }
3606
d0ecd894 3607 /* Start new detached freelist */
cc465c3b 3608 df->slab = slab;
376bf125 3609 set_freepointer(df->s, object, NULL);
d0ecd894
JDB
3610 df->tail = object;
3611 df->freelist = object;
3612 p[size] = NULL; /* mark object processed */
3613 df->cnt = 1;
3614
3615 while (size) {
3616 object = p[--size];
3617 if (!object)
3618 continue; /* Skip processed objects */
3619
cc465c3b
MWO
3620 /* df->slab is always set at this point */
3621 if (df->slab == virt_to_slab(object)) {
d0ecd894 3622 /* Opportunity build freelist */
376bf125 3623 set_freepointer(df->s, object, df->freelist);
d0ecd894
JDB
3624 df->freelist = object;
3625 df->cnt++;
3626 p[size] = NULL; /* mark object processed */
3627
3628 continue;
fbd02630 3629 }
d0ecd894
JDB
3630
3631 /* Limit look ahead search */
3632 if (!--lookahead)
3633 break;
3634
3635 if (!first_skipped_index)
3636 first_skipped_index = size + 1;
fbd02630 3637 }
d0ecd894
JDB
3638
3639 return first_skipped_index;
3640}
3641
d0ecd894 3642/* Note that interrupts must be enabled when calling this function. */
376bf125 3643void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
d0ecd894
JDB
3644{
3645 if (WARN_ON(!size))
3646 return;
3647
d1b2cf6c 3648 memcg_slab_free_hook(s, p, size);
d0ecd894
JDB
3649 do {
3650 struct detached_freelist df;
3651
3652 size = build_detached_freelist(s, size, p, &df);
cc465c3b 3653 if (!df.slab)
d0ecd894
JDB
3654 continue;
3655
bb192ed9 3656 slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, _RET_IP_);
d0ecd894 3657 } while (likely(size));
484748f0
CL
3658}
3659EXPORT_SYMBOL(kmem_cache_free_bulk);
3660
994eb764 3661/* Note that interrupts must be enabled when calling this function. */
865762a8
JDB
3662int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3663 void **p)
484748f0 3664{
994eb764
JDB
3665 struct kmem_cache_cpu *c;
3666 int i;
964d4bd3 3667 struct obj_cgroup *objcg = NULL;
994eb764 3668
03ec0ed5 3669 /* memcg and kmem_cache debug support */
964d4bd3 3670 s = slab_pre_alloc_hook(s, &objcg, size, flags);
03ec0ed5
JDB
3671 if (unlikely(!s))
3672 return false;
994eb764
JDB
3673 /*
3674 * Drain objects in the per cpu slab, while disabling local
3675 * IRQs, which protects against PREEMPT and interrupts
3676 * handlers invoking normal fastpath.
3677 */
25c00c50 3678 c = slub_get_cpu_ptr(s->cpu_slab);
bd0e7491 3679 local_lock_irq(&s->cpu_slab->lock);
994eb764
JDB
3680
3681 for (i = 0; i < size; i++) {
b89fb5ef 3682 void *object = kfence_alloc(s, s->object_size, flags);
994eb764 3683
b89fb5ef
AP
3684 if (unlikely(object)) {
3685 p[i] = object;
3686 continue;
3687 }
3688
3689 object = c->freelist;
ebe909e0 3690 if (unlikely(!object)) {
fd4d9c7d
JH
3691 /*
3692 * We may have removed an object from c->freelist using
3693 * the fastpath in the previous iteration; in that case,
3694 * c->tid has not been bumped yet.
3695 * Since ___slab_alloc() may reenable interrupts while
3696 * allocating memory, we should bump c->tid now.
3697 */
3698 c->tid = next_tid(c->tid);
3699
bd0e7491 3700 local_unlock_irq(&s->cpu_slab->lock);
e500059b 3701
ebe909e0
JDB
3702 /*
3703 * Invoking slow path likely have side-effect
3704 * of re-populating per CPU c->freelist
3705 */
87098373 3706 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
ebe909e0 3707 _RET_IP_, c);
87098373
CL
3708 if (unlikely(!p[i]))
3709 goto error;
3710
ebe909e0 3711 c = this_cpu_ptr(s->cpu_slab);
0f181f9f
AP
3712 maybe_wipe_obj_freeptr(s, p[i]);
3713
bd0e7491 3714 local_lock_irq(&s->cpu_slab->lock);
e500059b 3715
ebe909e0
JDB
3716 continue; /* goto for-loop */
3717 }
994eb764
JDB
3718 c->freelist = get_freepointer(s, object);
3719 p[i] = object;
0f181f9f 3720 maybe_wipe_obj_freeptr(s, p[i]);
994eb764
JDB
3721 }
3722 c->tid = next_tid(c->tid);
bd0e7491 3723 local_unlock_irq(&s->cpu_slab->lock);
25c00c50 3724 slub_put_cpu_ptr(s->cpu_slab);
994eb764 3725
da844b78
AK
3726 /*
3727 * memcg and kmem_cache debug support and memory initialization.
3728 * Done outside of the IRQ disabled fastpath loop.
3729 */
3730 slab_post_alloc_hook(s, objcg, flags, size, p,
3731 slab_want_init_on_alloc(flags, s));
865762a8 3732 return i;
87098373 3733error:
25c00c50 3734 slub_put_cpu_ptr(s->cpu_slab);
da844b78 3735 slab_post_alloc_hook(s, objcg, flags, i, p, false);
03ec0ed5 3736 __kmem_cache_free_bulk(s, i, p);
865762a8 3737 return 0;
484748f0
CL
3738}
3739EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3740
3741
81819f0f 3742/*
672bba3a
CL
3743 * Object placement in a slab is made very easy because we always start at
3744 * offset 0. If we tune the size of the object to the alignment then we can
3745 * get the required alignment by putting one properly sized object after
3746 * another.
81819f0f
CL
3747 *
3748 * Notice that the allocation order determines the sizes of the per cpu
3749 * caches. Each processor has always one slab available for allocations.
3750 * Increasing the allocation order reduces the number of times that slabs
672bba3a 3751 * must be moved on and off the partial lists and is therefore a factor in
81819f0f 3752 * locking overhead.
81819f0f
CL
3753 */
3754
3755/*
f0953a1b 3756 * Minimum / Maximum order of slab pages. This influences locking overhead
81819f0f
CL
3757 * and slab fragmentation. A higher order reduces the number of partial slabs
3758 * and increases the number of allocations possible without having to
3759 * take the list_lock.
3760 */
19af27af
AD
3761static unsigned int slub_min_order;
3762static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
3763static unsigned int slub_min_objects;
81819f0f 3764
81819f0f
CL
3765/*
3766 * Calculate the order of allocation given an slab object size.
3767 *
672bba3a
CL
3768 * The order of allocation has significant impact on performance and other
3769 * system components. Generally order 0 allocations should be preferred since
3770 * order 0 does not cause fragmentation in the page allocator. Larger objects
3771 * be problematic to put into order 0 slabs because there may be too much
c124f5b5 3772 * unused space left. We go to a higher order if more than 1/16th of the slab
672bba3a
CL
3773 * would be wasted.
3774 *
3775 * In order to reach satisfactory performance we must ensure that a minimum
3776 * number of objects is in one slab. Otherwise we may generate too much
3777 * activity on the partial lists which requires taking the list_lock. This is
3778 * less a concern for large slabs though which are rarely used.
81819f0f 3779 *
672bba3a
CL
3780 * slub_max_order specifies the order where we begin to stop considering the
3781 * number of objects in a slab as critical. If we reach slub_max_order then
3782 * we try to keep the page order as low as possible. So we accept more waste
3783 * of space in favor of a small page order.
81819f0f 3784 *
672bba3a
CL
3785 * Higher order allocations also allow the placement of more objects in a
3786 * slab and thereby reduce object handling overhead. If the user has
dc84207d 3787 * requested a higher minimum order then we start with that one instead of
672bba3a 3788 * the smallest order which will fit the object.
81819f0f 3789 */
d122019b 3790static inline unsigned int calc_slab_order(unsigned int size,
19af27af 3791 unsigned int min_objects, unsigned int max_order,
9736d2a9 3792 unsigned int fract_leftover)
81819f0f 3793{
19af27af
AD
3794 unsigned int min_order = slub_min_order;
3795 unsigned int order;
81819f0f 3796
9736d2a9 3797 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
210b5c06 3798 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
39b26464 3799
9736d2a9 3800 for (order = max(min_order, (unsigned int)get_order(min_objects * size));
5e6d444e 3801 order <= max_order; order++) {
81819f0f 3802
19af27af
AD
3803 unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
3804 unsigned int rem;
81819f0f 3805
9736d2a9 3806 rem = slab_size % size;
81819f0f 3807
5e6d444e 3808 if (rem <= slab_size / fract_leftover)
81819f0f 3809 break;
81819f0f 3810 }
672bba3a 3811
81819f0f
CL
3812 return order;
3813}
3814
9736d2a9 3815static inline int calculate_order(unsigned int size)
5e6d444e 3816{
19af27af
AD
3817 unsigned int order;
3818 unsigned int min_objects;
3819 unsigned int max_objects;
3286222f 3820 unsigned int nr_cpus;
5e6d444e
CL
3821
3822 /*
3823 * Attempt to find best configuration for a slab. This
3824 * works by first attempting to generate a layout with
3825 * the best configuration and backing off gradually.
3826 *
422ff4d7 3827 * First we increase the acceptable waste in a slab. Then
5e6d444e
CL
3828 * we reduce the minimum objects required in a slab.
3829 */
3830 min_objects = slub_min_objects;
3286222f
VB
3831 if (!min_objects) {
3832 /*
3833 * Some architectures will only update present cpus when
3834 * onlining them, so don't trust the number if it's just 1. But
3835 * we also don't want to use nr_cpu_ids always, as on some other
3836 * architectures, there can be many possible cpus, but never
3837 * onlined. Here we compromise between trying to avoid too high
3838 * order on systems that appear larger than they are, and too
3839 * low order on systems that appear smaller than they are.
3840 */
3841 nr_cpus = num_present_cpus();
3842 if (nr_cpus <= 1)
3843 nr_cpus = nr_cpu_ids;
3844 min_objects = 4 * (fls(nr_cpus) + 1);
3845 }
9736d2a9 3846 max_objects = order_objects(slub_max_order, size);
e8120ff1
ZY
3847 min_objects = min(min_objects, max_objects);
3848
5e6d444e 3849 while (min_objects > 1) {
19af27af
AD
3850 unsigned int fraction;
3851
c124f5b5 3852 fraction = 16;
5e6d444e 3853 while (fraction >= 4) {
d122019b 3854 order = calc_slab_order(size, min_objects,
9736d2a9 3855 slub_max_order, fraction);
5e6d444e
CL
3856 if (order <= slub_max_order)
3857 return order;
3858 fraction /= 2;
3859 }
5086c389 3860 min_objects--;
5e6d444e
CL
3861 }
3862
3863 /*
3864 * We were unable to place multiple objects in a slab. Now
3865 * lets see if we can place a single object there.
3866 */
d122019b 3867 order = calc_slab_order(size, 1, slub_max_order, 1);
5e6d444e
CL
3868 if (order <= slub_max_order)
3869 return order;
3870
3871 /*
3872 * Doh this slab cannot be placed using slub_max_order.
3873 */
d122019b 3874 order = calc_slab_order(size, 1, MAX_ORDER, 1);
818cf590 3875 if (order < MAX_ORDER)
5e6d444e
CL
3876 return order;
3877 return -ENOSYS;
3878}
3879
5595cffc 3880static void
4053497d 3881init_kmem_cache_node(struct kmem_cache_node *n)
81819f0f
CL
3882{
3883 n->nr_partial = 0;
81819f0f
CL
3884 spin_lock_init(&n->list_lock);
3885 INIT_LIST_HEAD(&n->partial);
8ab1372f 3886#ifdef CONFIG_SLUB_DEBUG
0f389ec6 3887 atomic_long_set(&n->nr_slabs, 0);
02b71b70 3888 atomic_long_set(&n->total_objects, 0);
643b1138 3889 INIT_LIST_HEAD(&n->full);
8ab1372f 3890#endif
81819f0f
CL
3891}
3892
55136592 3893static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
4c93c355 3894{
6c182dc0 3895 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
95a05b42 3896 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
4c93c355 3897
8a5ec0ba 3898 /*
d4d84fef
CM
3899 * Must align to double word boundary for the double cmpxchg
3900 * instructions to work; see __pcpu_double_call_return_bool().
8a5ec0ba 3901 */
d4d84fef
CM
3902 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
3903 2 * sizeof(void *));
8a5ec0ba
CL
3904
3905 if (!s->cpu_slab)
3906 return 0;
3907
3908 init_kmem_cache_cpus(s);
4c93c355 3909
8a5ec0ba 3910 return 1;
4c93c355 3911}
4c93c355 3912
51df1142
CL
3913static struct kmem_cache *kmem_cache_node;
3914
81819f0f
CL
3915/*
3916 * No kmalloc_node yet so do it by hand. We know that this is the first
3917 * slab on the node for this slabcache. There are no concurrent accesses
3918 * possible.
3919 *
721ae22a
ZYW
3920 * Note that this function only works on the kmem_cache_node
3921 * when allocating for the kmem_cache_node. This is used for bootstrapping
4c93c355 3922 * memory on a fresh node that has no slab structures yet.
81819f0f 3923 */
55136592 3924static void early_kmem_cache_node_alloc(int node)
81819f0f 3925{
bb192ed9 3926 struct slab *slab;
81819f0f
CL
3927 struct kmem_cache_node *n;
3928
51df1142 3929 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
81819f0f 3930
bb192ed9 3931 slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
81819f0f 3932
bb192ed9
VB
3933 BUG_ON(!slab);
3934 if (slab_nid(slab) != node) {
f9f58285
FF
3935 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
3936 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
a2f92ee7
CL
3937 }
3938
bb192ed9 3939 n = slab->freelist;
81819f0f 3940 BUG_ON(!n);
8ab1372f 3941#ifdef CONFIG_SLUB_DEBUG
f7cb1933 3942 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
51df1142 3943 init_tracking(kmem_cache_node, n);
8ab1372f 3944#endif
da844b78 3945 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
bb192ed9
VB
3946 slab->freelist = get_freepointer(kmem_cache_node, n);
3947 slab->inuse = 1;
3948 slab->frozen = 0;
12b22386 3949 kmem_cache_node->node[node] = n;
4053497d 3950 init_kmem_cache_node(n);
bb192ed9 3951 inc_slabs_node(kmem_cache_node, node, slab->objects);
6446faa2 3952
67b6c900 3953 /*
1e4dd946
SR
3954 * No locks need to be taken here as it has just been
3955 * initialized and there is no concurrent access.
67b6c900 3956 */
bb192ed9 3957 __add_partial(n, slab, DEACTIVATE_TO_HEAD);
81819f0f
CL
3958}
3959
3960static void free_kmem_cache_nodes(struct kmem_cache *s)
3961{
3962 int node;
fa45dc25 3963 struct kmem_cache_node *n;
81819f0f 3964
fa45dc25 3965 for_each_kmem_cache_node(s, node, n) {
81819f0f 3966 s->node[node] = NULL;
ea37df54 3967 kmem_cache_free(kmem_cache_node, n);
81819f0f
CL
3968 }
3969}
3970
52b4b950
DS
3971void __kmem_cache_release(struct kmem_cache *s)
3972{
210e7a43 3973 cache_random_seq_destroy(s);
52b4b950
DS
3974 free_percpu(s->cpu_slab);
3975 free_kmem_cache_nodes(s);
3976}
3977
55136592 3978static int init_kmem_cache_nodes(struct kmem_cache *s)
81819f0f
CL
3979{
3980 int node;
81819f0f 3981
7e1fa93d 3982 for_each_node_mask(node, slab_nodes) {
81819f0f
CL
3983 struct kmem_cache_node *n;
3984
73367bd8 3985 if (slab_state == DOWN) {
55136592 3986 early_kmem_cache_node_alloc(node);
73367bd8
AD
3987 continue;
3988 }
51df1142 3989 n = kmem_cache_alloc_node(kmem_cache_node,
55136592 3990 GFP_KERNEL, node);
81819f0f 3991
73367bd8
AD
3992 if (!n) {
3993 free_kmem_cache_nodes(s);
3994 return 0;
81819f0f 3995 }
73367bd8 3996
4053497d 3997 init_kmem_cache_node(n);
ea37df54 3998 s->node[node] = n;
81819f0f
CL
3999 }
4000 return 1;
4001}
81819f0f 4002
c0bdb232 4003static void set_min_partial(struct kmem_cache *s, unsigned long min)
3b89d7d8
DR
4004{
4005 if (min < MIN_PARTIAL)
4006 min = MIN_PARTIAL;
4007 else if (min > MAX_PARTIAL)
4008 min = MAX_PARTIAL;
4009 s->min_partial = min;
4010}
4011
e6d0e1dc
WY
4012static void set_cpu_partial(struct kmem_cache *s)
4013{
4014#ifdef CONFIG_SLUB_CPU_PARTIAL
b47291ef
VB
4015 unsigned int nr_objects;
4016
e6d0e1dc
WY
4017 /*
4018 * cpu_partial determined the maximum number of objects kept in the
4019 * per cpu partial lists of a processor.
4020 *
4021 * Per cpu partial lists mainly contain slabs that just have one
4022 * object freed. If they are used for allocation then they can be
4023 * filled up again with minimal effort. The slab will never hit the
4024 * per node partial lists and therefore no locking will be required.
4025 *
b47291ef
VB
4026 * For backwards compatibility reasons, this is determined as number
4027 * of objects, even though we now limit maximum number of pages, see
4028 * slub_set_cpu_partial()
e6d0e1dc
WY
4029 */
4030 if (!kmem_cache_has_cpu_partial(s))
b47291ef 4031 nr_objects = 0;
e6d0e1dc 4032 else if (s->size >= PAGE_SIZE)
b47291ef 4033 nr_objects = 6;
e6d0e1dc 4034 else if (s->size >= 1024)
23e98ad1 4035 nr_objects = 24;
e6d0e1dc 4036 else if (s->size >= 256)
23e98ad1 4037 nr_objects = 52;
e6d0e1dc 4038 else
23e98ad1 4039 nr_objects = 120;
b47291ef
VB
4040
4041 slub_set_cpu_partial(s, nr_objects);
e6d0e1dc
WY
4042#endif
4043}
4044
81819f0f
CL
4045/*
4046 * calculate_sizes() determines the order and the distribution of data within
4047 * a slab object.
4048 */
06b285dc 4049static int calculate_sizes(struct kmem_cache *s, int forced_order)
81819f0f 4050{
d50112ed 4051 slab_flags_t flags = s->flags;
be4a7988 4052 unsigned int size = s->object_size;
19af27af 4053 unsigned int order;
81819f0f 4054
d8b42bf5
CL
4055 /*
4056 * Round up object size to the next word boundary. We can only
4057 * place the free pointer at word boundaries and this determines
4058 * the possible location of the free pointer.
4059 */
4060 size = ALIGN(size, sizeof(void *));
4061
4062#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
4063 /*
4064 * Determine if we can poison the object itself. If the user of
4065 * the slab may touch the object after free or before allocation
4066 * then we should never poison the object itself.
4067 */
5f0d5a3a 4068 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
c59def9f 4069 !s->ctor)
81819f0f
CL
4070 s->flags |= __OBJECT_POISON;
4071 else
4072 s->flags &= ~__OBJECT_POISON;
4073
81819f0f
CL
4074
4075 /*
672bba3a 4076 * If we are Redzoning then check if there is some space between the
81819f0f 4077 * end of the object and the free pointer. If not then add an
672bba3a 4078 * additional word to have some bytes to store Redzone information.
81819f0f 4079 */
3b0efdfa 4080 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
81819f0f 4081 size += sizeof(void *);
41ecc55b 4082#endif
81819f0f
CL
4083
4084 /*
672bba3a 4085 * With that we have determined the number of bytes in actual use
e41a49fa 4086 * by the object and redzoning.
81819f0f
CL
4087 */
4088 s->inuse = size;
4089
74c1d3e0
KC
4090 if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
4091 ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
4092 s->ctor) {
81819f0f
CL
4093 /*
4094 * Relocate free pointer after the object if it is not
4095 * permitted to overwrite the first word of the object on
4096 * kmem_cache_free.
4097 *
4098 * This is the case if we do RCU, have a constructor or
74c1d3e0
KC
4099 * destructor, are poisoning the objects, or are
4100 * redzoning an object smaller than sizeof(void *).
cbfc35a4
WL
4101 *
4102 * The assumption that s->offset >= s->inuse means free
4103 * pointer is outside of the object is used in the
4104 * freeptr_outside_object() function. If that is no
4105 * longer true, the function needs to be modified.
81819f0f
CL
4106 */
4107 s->offset = size;
4108 size += sizeof(void *);
e41a49fa 4109 } else {
3202fa62
KC
4110 /*
4111 * Store freelist pointer near middle of object to keep
4112 * it away from the edges of the object to avoid small
4113 * sized over/underflows from neighboring allocations.
4114 */
e41a49fa 4115 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
81819f0f
CL
4116 }
4117
c12b3c62 4118#ifdef CONFIG_SLUB_DEBUG
81819f0f
CL
4119 if (flags & SLAB_STORE_USER)
4120 /*
4121 * Need to store information about allocs and frees after
4122 * the object.
4123 */
4124 size += 2 * sizeof(struct track);
80a9201a 4125#endif
81819f0f 4126
80a9201a
AP
4127 kasan_cache_create(s, &size, &s->flags);
4128#ifdef CONFIG_SLUB_DEBUG
d86bd1be 4129 if (flags & SLAB_RED_ZONE) {
81819f0f
CL
4130 /*
4131 * Add some empty padding so that we can catch
4132 * overwrites from earlier objects rather than let
4133 * tracking information or the free pointer be
0211a9c8 4134 * corrupted if a user writes before the start
81819f0f
CL
4135 * of the object.
4136 */
4137 size += sizeof(void *);
d86bd1be
JK
4138
4139 s->red_left_pad = sizeof(void *);
4140 s->red_left_pad = ALIGN(s->red_left_pad, s->align);
4141 size += s->red_left_pad;
4142 }
41ecc55b 4143#endif
672bba3a 4144
81819f0f
CL
4145 /*
4146 * SLUB stores one object immediately after another beginning from
4147 * offset 0. In order to align the objects we have to simply size
4148 * each object to conform to the alignment.
4149 */
45906855 4150 size = ALIGN(size, s->align);
81819f0f 4151 s->size = size;
4138fdfc 4152 s->reciprocal_size = reciprocal_value(size);
06b285dc
CL
4153 if (forced_order >= 0)
4154 order = forced_order;
4155 else
9736d2a9 4156 order = calculate_order(size);
81819f0f 4157
19af27af 4158 if ((int)order < 0)
81819f0f
CL
4159 return 0;
4160
b7a49f0d 4161 s->allocflags = 0;
834f3d11 4162 if (order)
b7a49f0d
CL
4163 s->allocflags |= __GFP_COMP;
4164
4165 if (s->flags & SLAB_CACHE_DMA)
2c59dd65 4166 s->allocflags |= GFP_DMA;
b7a49f0d 4167
6d6ea1e9
NB
4168 if (s->flags & SLAB_CACHE_DMA32)
4169 s->allocflags |= GFP_DMA32;
4170
b7a49f0d
CL
4171 if (s->flags & SLAB_RECLAIM_ACCOUNT)
4172 s->allocflags |= __GFP_RECLAIMABLE;
4173
81819f0f
CL
4174 /*
4175 * Determine the number of objects per slab
4176 */
9736d2a9
MW
4177 s->oo = oo_make(order, size);
4178 s->min = oo_make(get_order(size), size);
205ab99d
CL
4179 if (oo_objects(s->oo) > oo_objects(s->max))
4180 s->max = s->oo;
81819f0f 4181
834f3d11 4182 return !!oo_objects(s->oo);
81819f0f
CL
4183}
4184
d50112ed 4185static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
81819f0f 4186{
37540008 4187 s->flags = kmem_cache_flags(s->size, flags, s->name);
2482ddec
KC
4188#ifdef CONFIG_SLAB_FREELIST_HARDENED
4189 s->random = get_random_long();
4190#endif
81819f0f 4191
06b285dc 4192 if (!calculate_sizes(s, -1))
81819f0f 4193 goto error;
3de47213
DR
4194 if (disable_higher_order_debug) {
4195 /*
4196 * Disable debugging flags that store metadata if the min slab
4197 * order increased.
4198 */
3b0efdfa 4199 if (get_order(s->size) > get_order(s->object_size)) {
3de47213
DR
4200 s->flags &= ~DEBUG_METADATA_FLAGS;
4201 s->offset = 0;
4202 if (!calculate_sizes(s, -1))
4203 goto error;
4204 }
4205 }
81819f0f 4206
2565409f
HC
4207#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
4208 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
149daaf3 4209 if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
b789ef51
CL
4210 /* Enable fast mode */
4211 s->flags |= __CMPXCHG_DOUBLE;
4212#endif
4213
3b89d7d8 4214 /*
c2092c12 4215 * The larger the object size is, the more slabs we want on the partial
3b89d7d8
DR
4216 * list to avoid pounding the page allocator excessively.
4217 */
49e22585
CL
4218 set_min_partial(s, ilog2(s->size) / 2);
4219
e6d0e1dc 4220 set_cpu_partial(s);
49e22585 4221
81819f0f 4222#ifdef CONFIG_NUMA
e2cb96b7 4223 s->remote_node_defrag_ratio = 1000;
81819f0f 4224#endif
210e7a43
TG
4225
4226 /* Initialize the pre-computed randomized freelist if slab is up */
4227 if (slab_state >= UP) {
4228 if (init_cache_random_seq(s))
4229 goto error;
4230 }
4231
55136592 4232 if (!init_kmem_cache_nodes(s))
dfb4f096 4233 goto error;
81819f0f 4234
55136592 4235 if (alloc_kmem_cache_cpus(s))
278b1bb1 4236 return 0;
ff12059e 4237
81819f0f 4238error:
9037c576 4239 __kmem_cache_release(s);
278b1bb1 4240 return -EINVAL;
81819f0f 4241}
81819f0f 4242
bb192ed9 4243static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
55860d96 4244 const char *text)
33b12c38
CL
4245{
4246#ifdef CONFIG_SLUB_DEBUG
bb192ed9 4247 void *addr = slab_address(slab);
a2b4ae8b 4248 unsigned long flags;
55860d96 4249 unsigned long *map;
33b12c38 4250 void *p;
aa456c7a 4251
bb192ed9
VB
4252 slab_err(s, slab, text, s->name);
4253 slab_lock(slab, &flags);
33b12c38 4254
bb192ed9
VB
4255 map = get_map(s, slab);
4256 for_each_object(p, s, addr, slab->objects) {
33b12c38 4257
4138fdfc 4258 if (!test_bit(__obj_to_index(s, addr, p), map)) {
96b94abc 4259 pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
33b12c38
CL
4260 print_tracking(s, p);
4261 }
4262 }
55860d96 4263 put_map(map);
bb192ed9 4264 slab_unlock(slab, &flags);
33b12c38
CL
4265#endif
4266}
4267
81819f0f 4268/*
599870b1 4269 * Attempt to free all partial slabs on a node.
52b4b950
DS
4270 * This is called from __kmem_cache_shutdown(). We must take list_lock
4271 * because sysfs file might still access partial list after the shutdowning.
81819f0f 4272 */
599870b1 4273static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
81819f0f 4274{
60398923 4275 LIST_HEAD(discard);
bb192ed9 4276 struct slab *slab, *h;
81819f0f 4277
52b4b950
DS
4278 BUG_ON(irqs_disabled());
4279 spin_lock_irq(&n->list_lock);
bb192ed9
VB
4280 list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
4281 if (!slab->inuse) {
4282 remove_partial(n, slab);
4283 list_add(&slab->slab_list, &discard);
33b12c38 4284 } else {
bb192ed9 4285 list_slab_objects(s, slab,
55860d96 4286 "Objects remaining in %s on __kmem_cache_shutdown()");
599870b1 4287 }
33b12c38 4288 }
52b4b950 4289 spin_unlock_irq(&n->list_lock);
60398923 4290
bb192ed9
VB
4291 list_for_each_entry_safe(slab, h, &discard, slab_list)
4292 discard_slab(s, slab);
81819f0f
CL
4293}
4294
f9e13c0a
SB
4295bool __kmem_cache_empty(struct kmem_cache *s)
4296{
4297 int node;
4298 struct kmem_cache_node *n;
4299
4300 for_each_kmem_cache_node(s, node, n)
4301 if (n->nr_partial || slabs_node(s, node))
4302 return false;
4303 return true;
4304}
4305
81819f0f 4306/*
672bba3a 4307 * Release all resources used by a slab cache.
81819f0f 4308 */
52b4b950 4309int __kmem_cache_shutdown(struct kmem_cache *s)
81819f0f
CL
4310{
4311 int node;
fa45dc25 4312 struct kmem_cache_node *n;
81819f0f 4313
5a836bf6 4314 flush_all_cpus_locked(s);
81819f0f 4315 /* Attempt to free all objects */
fa45dc25 4316 for_each_kmem_cache_node(s, node, n) {
599870b1
CL
4317 free_partial(s, n);
4318 if (n->nr_partial || slabs_node(s, node))
81819f0f
CL
4319 return 1;
4320 }
81819f0f
CL
4321 return 0;
4322}
4323
5bb1bb35 4324#ifdef CONFIG_PRINTK
7213230a 4325void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
8e7f37f2
PM
4326{
4327 void *base;
4328 int __maybe_unused i;
4329 unsigned int objnr;
4330 void *objp;
4331 void *objp0;
7213230a 4332 struct kmem_cache *s = slab->slab_cache;
8e7f37f2
PM
4333 struct track __maybe_unused *trackp;
4334
4335 kpp->kp_ptr = object;
7213230a 4336 kpp->kp_slab = slab;
8e7f37f2 4337 kpp->kp_slab_cache = s;
7213230a 4338 base = slab_address(slab);
8e7f37f2
PM
4339 objp0 = kasan_reset_tag(object);
4340#ifdef CONFIG_SLUB_DEBUG
4341 objp = restore_red_left(s, objp0);
4342#else
4343 objp = objp0;
4344#endif
40f3bf0c 4345 objnr = obj_to_index(s, slab, objp);
8e7f37f2
PM
4346 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
4347 objp = base + s->size * objnr;
4348 kpp->kp_objp = objp;
7213230a
MWO
4349 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
4350 || (objp - base) % s->size) ||
8e7f37f2
PM
4351 !(s->flags & SLAB_STORE_USER))
4352 return;
4353#ifdef CONFIG_SLUB_DEBUG
0cbc124b 4354 objp = fixup_red_left(s, objp);
8e7f37f2
PM
4355 trackp = get_track(s, objp, TRACK_ALLOC);
4356 kpp->kp_ret = (void *)trackp->addr;
ae14c63a
LT
4357#ifdef CONFIG_STACKTRACE
4358 for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
4359 kpp->kp_stack[i] = (void *)trackp->addrs[i];
4360 if (!kpp->kp_stack[i])
4361 break;
4362 }
78869146 4363
ae14c63a
LT
4364 trackp = get_track(s, objp, TRACK_FREE);
4365 for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) {
4366 kpp->kp_free_stack[i] = (void *)trackp->addrs[i];
4367 if (!kpp->kp_free_stack[i])
4368 break;
e548eaa1 4369 }
8e7f37f2
PM
4370#endif
4371#endif
4372}
5bb1bb35 4373#endif
8e7f37f2 4374
81819f0f
CL
4375/********************************************************************
4376 * Kmalloc subsystem
4377 *******************************************************************/
4378
81819f0f
CL
4379static int __init setup_slub_min_order(char *str)
4380{
19af27af 4381 get_option(&str, (int *)&slub_min_order);
81819f0f
CL
4382
4383 return 1;
4384}
4385
4386__setup("slub_min_order=", setup_slub_min_order);
4387
4388static int __init setup_slub_max_order(char *str)
4389{
19af27af
AD
4390 get_option(&str, (int *)&slub_max_order);
4391 slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
81819f0f
CL
4392
4393 return 1;
4394}
4395
4396__setup("slub_max_order=", setup_slub_max_order);
4397
4398static int __init setup_slub_min_objects(char *str)
4399{
19af27af 4400 get_option(&str, (int *)&slub_min_objects);
81819f0f
CL
4401
4402 return 1;
4403}
4404
4405__setup("slub_min_objects=", setup_slub_min_objects);
4406
81819f0f
CL
4407void *__kmalloc(size_t size, gfp_t flags)
4408{
aadb4bc4 4409 struct kmem_cache *s;
5b882be4 4410 void *ret;
81819f0f 4411
95a05b42 4412 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
eada35ef 4413 return kmalloc_large(size, flags);
aadb4bc4 4414
2c59dd65 4415 s = kmalloc_slab(size, flags);
aadb4bc4
CL
4416
4417 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913
CL
4418 return s;
4419
b89fb5ef 4420 ret = slab_alloc(s, flags, _RET_IP_, size);
5b882be4 4421
ca2b84cb 4422 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
5b882be4 4423
0116523c 4424 ret = kasan_kmalloc(s, ret, size, flags);
0316bec2 4425
5b882be4 4426 return ret;
81819f0f
CL
4427}
4428EXPORT_SYMBOL(__kmalloc);
4429
5d1f57e4 4430#ifdef CONFIG_NUMA
f619cfe1
CL
4431static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
4432{
b1eeab67 4433 struct page *page;
e4f7c0b4 4434 void *ptr = NULL;
6a486c0a 4435 unsigned int order = get_order(size);
f619cfe1 4436
75f296d9 4437 flags |= __GFP_COMP;
6a486c0a
VB
4438 page = alloc_pages_node(node, flags, order);
4439 if (page) {
e4f7c0b4 4440 ptr = page_address(page);
96403bfe
MS
4441 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
4442 PAGE_SIZE << order);
6a486c0a 4443 }
e4f7c0b4 4444
0116523c 4445 return kmalloc_large_node_hook(ptr, size, flags);
f619cfe1
CL
4446}
4447
81819f0f
CL
4448void *__kmalloc_node(size_t size, gfp_t flags, int node)
4449{
aadb4bc4 4450 struct kmem_cache *s;
5b882be4 4451 void *ret;
81819f0f 4452
95a05b42 4453 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
5b882be4
EGM
4454 ret = kmalloc_large_node(size, flags, node);
4455
ca2b84cb
EGM
4456 trace_kmalloc_node(_RET_IP_, ret,
4457 size, PAGE_SIZE << get_order(size),
4458 flags, node);
5b882be4
EGM
4459
4460 return ret;
4461 }
aadb4bc4 4462
2c59dd65 4463 s = kmalloc_slab(size, flags);
aadb4bc4
CL
4464
4465 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913
CL
4466 return s;
4467
b89fb5ef 4468 ret = slab_alloc_node(s, flags, node, _RET_IP_, size);
5b882be4 4469
ca2b84cb 4470 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
5b882be4 4471
0116523c 4472 ret = kasan_kmalloc(s, ret, size, flags);
0316bec2 4473
5b882be4 4474 return ret;
81819f0f
CL
4475}
4476EXPORT_SYMBOL(__kmalloc_node);
6dfd1b65 4477#endif /* CONFIG_NUMA */
81819f0f 4478
ed18adc1
KC
4479#ifdef CONFIG_HARDENED_USERCOPY
4480/*
afcc90f8
KC
4481 * Rejects incorrectly sized objects and objects that are to be copied
4482 * to/from userspace but do not fall entirely within the containing slab
4483 * cache's usercopy region.
ed18adc1
KC
4484 *
4485 * Returns NULL if check passes, otherwise const char * to name of cache
4486 * to indicate an error.
4487 */
0b3eb091
MWO
4488void __check_heap_object(const void *ptr, unsigned long n,
4489 const struct slab *slab, bool to_user)
ed18adc1
KC
4490{
4491 struct kmem_cache *s;
44065b2e 4492 unsigned int offset;
b89fb5ef 4493 bool is_kfence = is_kfence_address(ptr);
ed18adc1 4494
96fedce2
AK
4495 ptr = kasan_reset_tag(ptr);
4496
ed18adc1 4497 /* Find object and usable object size. */
0b3eb091 4498 s = slab->slab_cache;
ed18adc1
KC
4499
4500 /* Reject impossible pointers. */
0b3eb091 4501 if (ptr < slab_address(slab))
f4e6e289
KC
4502 usercopy_abort("SLUB object not in SLUB page?!", NULL,
4503 to_user, 0, n);
ed18adc1
KC
4504
4505 /* Find offset within object. */
b89fb5ef
AP
4506 if (is_kfence)
4507 offset = ptr - kfence_object_start(ptr);
4508 else
0b3eb091 4509 offset = (ptr - slab_address(slab)) % s->size;
ed18adc1
KC
4510
4511 /* Adjust for redzone and reject if within the redzone. */
b89fb5ef 4512 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
ed18adc1 4513 if (offset < s->red_left_pad)
f4e6e289
KC
4514 usercopy_abort("SLUB object in left red zone",
4515 s->name, to_user, offset, n);
ed18adc1
KC
4516 offset -= s->red_left_pad;
4517 }
4518
afcc90f8
KC
4519 /* Allow address range falling entirely within usercopy region. */
4520 if (offset >= s->useroffset &&
4521 offset - s->useroffset <= s->usersize &&
4522 n <= s->useroffset - offset + s->usersize)
f4e6e289 4523 return;
ed18adc1 4524
f4e6e289 4525 usercopy_abort("SLUB object", s->name, to_user, offset, n);
ed18adc1
KC
4526}
4527#endif /* CONFIG_HARDENED_USERCOPY */
4528
10d1f8cb 4529size_t __ksize(const void *object)
81819f0f 4530{
0c24811b 4531 struct folio *folio;
81819f0f 4532
ef8b4520 4533 if (unlikely(object == ZERO_SIZE_PTR))
272c1d21
CL
4534 return 0;
4535
0c24811b 4536 folio = virt_to_folio(object);
294a80a8 4537
0c24811b
MWO
4538 if (unlikely(!folio_test_slab(folio)))
4539 return folio_size(folio);
81819f0f 4540
0c24811b 4541 return slab_ksize(folio_slab(folio)->slab_cache);
81819f0f 4542}
10d1f8cb 4543EXPORT_SYMBOL(__ksize);
81819f0f
CL
4544
4545void kfree(const void *x)
4546{
d835eef4
MWO
4547 struct folio *folio;
4548 struct slab *slab;
5bb983b0 4549 void *object = (void *)x;
81819f0f 4550
2121db74
PE
4551 trace_kfree(_RET_IP_, x);
4552
2408c550 4553 if (unlikely(ZERO_OR_NULL_PTR(x)))
81819f0f
CL
4554 return;
4555
d835eef4
MWO
4556 folio = virt_to_folio(x);
4557 if (unlikely(!folio_test_slab(folio))) {
4558 free_large_kmalloc(folio, object);
aadb4bc4
CL
4559 return;
4560 }
d835eef4 4561 slab = folio_slab(folio);
bb192ed9 4562 slab_free(slab->slab_cache, slab, object, NULL, 1, _RET_IP_);
81819f0f
CL
4563}
4564EXPORT_SYMBOL(kfree);
4565
832f37f5
VD
4566#define SHRINK_PROMOTE_MAX 32
4567
2086d26a 4568/*
832f37f5
VD
4569 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
4570 * up most to the head of the partial lists. New allocations will then
4571 * fill those up and thus they can be removed from the partial lists.
672bba3a
CL
4572 *
4573 * The slabs with the least items are placed last. This results in them
4574 * being allocated from last increasing the chance that the last objects
4575 * are freed in them.
2086d26a 4576 */
5a836bf6 4577static int __kmem_cache_do_shrink(struct kmem_cache *s)
2086d26a
CL
4578{
4579 int node;
4580 int i;
4581 struct kmem_cache_node *n;
bb192ed9
VB
4582 struct slab *slab;
4583 struct slab *t;
832f37f5
VD
4584 struct list_head discard;
4585 struct list_head promote[SHRINK_PROMOTE_MAX];
2086d26a 4586 unsigned long flags;
ce3712d7 4587 int ret = 0;
2086d26a 4588
fa45dc25 4589 for_each_kmem_cache_node(s, node, n) {
832f37f5
VD
4590 INIT_LIST_HEAD(&discard);
4591 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
4592 INIT_LIST_HEAD(promote + i);
2086d26a
CL
4593
4594 spin_lock_irqsave(&n->list_lock, flags);
4595
4596 /*
832f37f5 4597 * Build lists of slabs to discard or promote.
2086d26a 4598 *
672bba3a 4599 * Note that concurrent frees may occur while we hold the
c2092c12 4600 * list_lock. slab->inuse here is the upper limit.
2086d26a 4601 */
bb192ed9
VB
4602 list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
4603 int free = slab->objects - slab->inuse;
832f37f5 4604
c2092c12 4605 /* Do not reread slab->inuse */
832f37f5
VD
4606 barrier();
4607
4608 /* We do not keep full slabs on the list */
4609 BUG_ON(free <= 0);
4610
bb192ed9
VB
4611 if (free == slab->objects) {
4612 list_move(&slab->slab_list, &discard);
69cb8e6b 4613 n->nr_partial--;
832f37f5 4614 } else if (free <= SHRINK_PROMOTE_MAX)
bb192ed9 4615 list_move(&slab->slab_list, promote + free - 1);
2086d26a
CL
4616 }
4617
2086d26a 4618 /*
832f37f5
VD
4619 * Promote the slabs filled up most to the head of the
4620 * partial list.
2086d26a 4621 */
832f37f5
VD
4622 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
4623 list_splice(promote + i, &n->partial);
2086d26a 4624
2086d26a 4625 spin_unlock_irqrestore(&n->list_lock, flags);
69cb8e6b
CL
4626
4627 /* Release empty slabs */
bb192ed9
VB
4628 list_for_each_entry_safe(slab, t, &discard, slab_list)
4629 discard_slab(s, slab);
ce3712d7
VD
4630
4631 if (slabs_node(s, node))
4632 ret = 1;
2086d26a
CL
4633 }
4634
ce3712d7 4635 return ret;
2086d26a 4636}
2086d26a 4637
5a836bf6
SAS
4638int __kmem_cache_shrink(struct kmem_cache *s)
4639{
4640 flush_all(s);
4641 return __kmem_cache_do_shrink(s);
4642}
4643
b9049e23
YG
4644static int slab_mem_going_offline_callback(void *arg)
4645{
4646 struct kmem_cache *s;
4647
18004c5d 4648 mutex_lock(&slab_mutex);
5a836bf6
SAS
4649 list_for_each_entry(s, &slab_caches, list) {
4650 flush_all_cpus_locked(s);
4651 __kmem_cache_do_shrink(s);
4652 }
18004c5d 4653 mutex_unlock(&slab_mutex);
b9049e23
YG
4654
4655 return 0;
4656}
4657
4658static void slab_mem_offline_callback(void *arg)
4659{
b9049e23
YG
4660 struct memory_notify *marg = arg;
4661 int offline_node;
4662
b9d5ab25 4663 offline_node = marg->status_change_nid_normal;
b9049e23
YG
4664
4665 /*
4666 * If the node still has available memory. we need kmem_cache_node
4667 * for it yet.
4668 */
4669 if (offline_node < 0)
4670 return;
4671
18004c5d 4672 mutex_lock(&slab_mutex);
7e1fa93d 4673 node_clear(offline_node, slab_nodes);
666716fd
VB
4674 /*
4675 * We no longer free kmem_cache_node structures here, as it would be
4676 * racy with all get_node() users, and infeasible to protect them with
4677 * slab_mutex.
4678 */
18004c5d 4679 mutex_unlock(&slab_mutex);
b9049e23
YG
4680}
4681
4682static int slab_mem_going_online_callback(void *arg)
4683{
4684 struct kmem_cache_node *n;
4685 struct kmem_cache *s;
4686 struct memory_notify *marg = arg;
b9d5ab25 4687 int nid = marg->status_change_nid_normal;
b9049e23
YG
4688 int ret = 0;
4689
4690 /*
4691 * If the node's memory is already available, then kmem_cache_node is
4692 * already created. Nothing to do.
4693 */
4694 if (nid < 0)
4695 return 0;
4696
4697 /*
0121c619 4698 * We are bringing a node online. No memory is available yet. We must
b9049e23
YG
4699 * allocate a kmem_cache_node structure in order to bring the node
4700 * online.
4701 */
18004c5d 4702 mutex_lock(&slab_mutex);
b9049e23 4703 list_for_each_entry(s, &slab_caches, list) {
666716fd
VB
4704 /*
4705 * The structure may already exist if the node was previously
4706 * onlined and offlined.
4707 */
4708 if (get_node(s, nid))
4709 continue;
b9049e23
YG
4710 /*
4711 * XXX: kmem_cache_alloc_node will fallback to other nodes
4712 * since memory is not yet available from the node that
4713 * is brought up.
4714 */
8de66a0c 4715 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
b9049e23
YG
4716 if (!n) {
4717 ret = -ENOMEM;
4718 goto out;
4719 }
4053497d 4720 init_kmem_cache_node(n);
b9049e23
YG
4721 s->node[nid] = n;
4722 }
7e1fa93d
VB
4723 /*
4724 * Any cache created after this point will also have kmem_cache_node
4725 * initialized for the new node.
4726 */
4727 node_set(nid, slab_nodes);
b9049e23 4728out:
18004c5d 4729 mutex_unlock(&slab_mutex);
b9049e23
YG
4730 return ret;
4731}
4732
4733static int slab_memory_callback(struct notifier_block *self,
4734 unsigned long action, void *arg)
4735{
4736 int ret = 0;
4737
4738 switch (action) {
4739 case MEM_GOING_ONLINE:
4740 ret = slab_mem_going_online_callback(arg);
4741 break;
4742 case MEM_GOING_OFFLINE:
4743 ret = slab_mem_going_offline_callback(arg);
4744 break;
4745 case MEM_OFFLINE:
4746 case MEM_CANCEL_ONLINE:
4747 slab_mem_offline_callback(arg);
4748 break;
4749 case MEM_ONLINE:
4750 case MEM_CANCEL_OFFLINE:
4751 break;
4752 }
dc19f9db
KH
4753 if (ret)
4754 ret = notifier_from_errno(ret);
4755 else
4756 ret = NOTIFY_OK;
b9049e23
YG
4757 return ret;
4758}
4759
3ac38faa
AM
4760static struct notifier_block slab_memory_callback_nb = {
4761 .notifier_call = slab_memory_callback,
4762 .priority = SLAB_CALLBACK_PRI,
4763};
b9049e23 4764
81819f0f
CL
4765/********************************************************************
4766 * Basic setup of slabs
4767 *******************************************************************/
4768
51df1142
CL
4769/*
4770 * Used for early kmem_cache structures that were allocated using
dffb4d60
CL
4771 * the page allocator. Allocate them properly then fix up the pointers
4772 * that may be pointing to the wrong kmem_cache structure.
51df1142
CL
4773 */
4774
dffb4d60 4775static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
51df1142
CL
4776{
4777 int node;
dffb4d60 4778 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
fa45dc25 4779 struct kmem_cache_node *n;
51df1142 4780
dffb4d60 4781 memcpy(s, static_cache, kmem_cache->object_size);
51df1142 4782
7d557b3c
GC
4783 /*
4784 * This runs very early, and only the boot processor is supposed to be
4785 * up. Even if it weren't true, IRQs are not up so we couldn't fire
4786 * IPIs around.
4787 */
4788 __flush_cpu_slab(s, smp_processor_id());
fa45dc25 4789 for_each_kmem_cache_node(s, node, n) {
bb192ed9 4790 struct slab *p;
51df1142 4791
916ac052 4792 list_for_each_entry(p, &n->partial, slab_list)
fa45dc25 4793 p->slab_cache = s;
51df1142 4794
607bf324 4795#ifdef CONFIG_SLUB_DEBUG
916ac052 4796 list_for_each_entry(p, &n->full, slab_list)
fa45dc25 4797 p->slab_cache = s;
51df1142 4798#endif
51df1142 4799 }
dffb4d60
CL
4800 list_add(&s->list, &slab_caches);
4801 return s;
51df1142
CL
4802}
4803
81819f0f
CL
4804void __init kmem_cache_init(void)
4805{
dffb4d60
CL
4806 static __initdata struct kmem_cache boot_kmem_cache,
4807 boot_kmem_cache_node;
7e1fa93d 4808 int node;
51df1142 4809
fc8d8620
SG
4810 if (debug_guardpage_minorder())
4811 slub_max_order = 0;
4812
79270291
SB
4813 /* Print slub debugging pointers without hashing */
4814 if (__slub_debug_enabled())
4815 no_hash_pointers_enable(NULL);
4816
dffb4d60
CL
4817 kmem_cache_node = &boot_kmem_cache_node;
4818 kmem_cache = &boot_kmem_cache;
51df1142 4819
7e1fa93d
VB
4820 /*
4821 * Initialize the nodemask for which we will allocate per node
4822 * structures. Here we don't need taking slab_mutex yet.
4823 */
4824 for_each_node_state(node, N_NORMAL_MEMORY)
4825 node_set(node, slab_nodes);
4826
dffb4d60 4827 create_boot_cache(kmem_cache_node, "kmem_cache_node",
8eb8284b 4828 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
b9049e23 4829
3ac38faa 4830 register_hotmemory_notifier(&slab_memory_callback_nb);
81819f0f
CL
4831
4832 /* Able to allocate the per node structures */
4833 slab_state = PARTIAL;
4834
dffb4d60
CL
4835 create_boot_cache(kmem_cache, "kmem_cache",
4836 offsetof(struct kmem_cache, node) +
4837 nr_node_ids * sizeof(struct kmem_cache_node *),
8eb8284b 4838 SLAB_HWCACHE_ALIGN, 0, 0);
8a13a4cc 4839
dffb4d60 4840 kmem_cache = bootstrap(&boot_kmem_cache);
dffb4d60 4841 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
51df1142
CL
4842
4843 /* Now we can use the kmem_cache to allocate kmalloc slabs */
34cc6990 4844 setup_kmalloc_cache_index_table();
f97d5f63 4845 create_kmalloc_caches(0);
81819f0f 4846
210e7a43
TG
4847 /* Setup random freelists for each cache */
4848 init_freelist_randomization();
4849
a96a87bf
SAS
4850 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
4851 slub_cpu_dead);
81819f0f 4852
b9726c26 4853 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
f97d5f63 4854 cache_line_size(),
81819f0f
CL
4855 slub_min_order, slub_max_order, slub_min_objects,
4856 nr_cpu_ids, nr_node_ids);
4857}
4858
7e85ee0c
PE
4859void __init kmem_cache_init_late(void)
4860{
7e85ee0c
PE
4861}
4862
2633d7a0 4863struct kmem_cache *
f4957d5b 4864__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
d50112ed 4865 slab_flags_t flags, void (*ctor)(void *))
81819f0f 4866{
10befea9 4867 struct kmem_cache *s;
81819f0f 4868
a44cb944 4869 s = find_mergeable(size, align, flags, name, ctor);
81819f0f
CL
4870 if (s) {
4871 s->refcount++;
84d0ddd6 4872
81819f0f
CL
4873 /*
4874 * Adjust the object sizes so that we clear
4875 * the complete object on kzalloc.
4876 */
1b473f29 4877 s->object_size = max(s->object_size, size);
52ee6d74 4878 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
6446faa2 4879
7b8f3b66 4880 if (sysfs_slab_alias(s, name)) {
7b8f3b66 4881 s->refcount--;
cbb79694 4882 s = NULL;
7b8f3b66 4883 }
a0e1d1be 4884 }
6446faa2 4885
cbb79694
CL
4886 return s;
4887}
84c1cf62 4888
d50112ed 4889int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
cbb79694 4890{
aac3a166
PE
4891 int err;
4892
4893 err = kmem_cache_open(s, flags);
4894 if (err)
4895 return err;
20cea968 4896
45530c44
CL
4897 /* Mutex is not taken during early boot */
4898 if (slab_state <= UP)
4899 return 0;
4900
aac3a166 4901 err = sysfs_slab_add(s);
67823a54 4902 if (err) {
52b4b950 4903 __kmem_cache_release(s);
67823a54
ML
4904 return err;
4905 }
20cea968 4906
64dd6849
FM
4907 if (s->flags & SLAB_STORE_USER)
4908 debugfs_slab_add(s);
4909
67823a54 4910 return 0;
81819f0f 4911}
81819f0f 4912
ce71e27c 4913void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
81819f0f 4914{
aadb4bc4 4915 struct kmem_cache *s;
94b528d0 4916 void *ret;
aadb4bc4 4917
95a05b42 4918 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
eada35ef
PE
4919 return kmalloc_large(size, gfpflags);
4920
2c59dd65 4921 s = kmalloc_slab(size, gfpflags);
81819f0f 4922
2408c550 4923 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913 4924 return s;
81819f0f 4925
b89fb5ef 4926 ret = slab_alloc(s, gfpflags, caller, size);
94b528d0 4927
25985edc 4928 /* Honor the call site pointer we received. */
ca2b84cb 4929 trace_kmalloc(caller, ret, size, s->size, gfpflags);
94b528d0
EGM
4930
4931 return ret;
81819f0f 4932}
fd7cb575 4933EXPORT_SYMBOL(__kmalloc_track_caller);
81819f0f 4934
5d1f57e4 4935#ifdef CONFIG_NUMA
81819f0f 4936void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
ce71e27c 4937 int node, unsigned long caller)
81819f0f 4938{
aadb4bc4 4939 struct kmem_cache *s;
94b528d0 4940 void *ret;
aadb4bc4 4941
95a05b42 4942 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
d3e14aa3
XF
4943 ret = kmalloc_large_node(size, gfpflags, node);
4944
4945 trace_kmalloc_node(caller, ret,
4946 size, PAGE_SIZE << get_order(size),
4947 gfpflags, node);
4948
4949 return ret;
4950 }
eada35ef 4951
2c59dd65 4952 s = kmalloc_slab(size, gfpflags);
81819f0f 4953
2408c550 4954 if (unlikely(ZERO_OR_NULL_PTR(s)))
6cb8f913 4955 return s;
81819f0f 4956
b89fb5ef 4957 ret = slab_alloc_node(s, gfpflags, node, caller, size);
94b528d0 4958
25985edc 4959 /* Honor the call site pointer we received. */
ca2b84cb 4960 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
94b528d0
EGM
4961
4962 return ret;
81819f0f 4963}
fd7cb575 4964EXPORT_SYMBOL(__kmalloc_node_track_caller);
5d1f57e4 4965#endif
81819f0f 4966
ab4d5ed5 4967#ifdef CONFIG_SYSFS
bb192ed9 4968static int count_inuse(struct slab *slab)
205ab99d 4969{
bb192ed9 4970 return slab->inuse;
205ab99d
CL
4971}
4972
bb192ed9 4973static int count_total(struct slab *slab)
205ab99d 4974{
bb192ed9 4975 return slab->objects;
205ab99d 4976}
ab4d5ed5 4977#endif
205ab99d 4978
ab4d5ed5 4979#ifdef CONFIG_SLUB_DEBUG
bb192ed9 4980static void validate_slab(struct kmem_cache *s, struct slab *slab,
0a19e7dd 4981 unsigned long *obj_map)
53e15af0
CL
4982{
4983 void *p;
bb192ed9 4984 void *addr = slab_address(slab);
a2b4ae8b 4985 unsigned long flags;
90e9f6a6 4986
bb192ed9 4987 slab_lock(slab, &flags);
53e15af0 4988
bb192ed9 4989 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
90e9f6a6 4990 goto unlock;
53e15af0
CL
4991
4992 /* Now we know that a valid freelist exists */
bb192ed9
VB
4993 __fill_map(obj_map, s, slab);
4994 for_each_object(p, s, addr, slab->objects) {
0a19e7dd 4995 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
dd98afd4 4996 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
53e15af0 4997
bb192ed9 4998 if (!check_object(s, slab, p, val))
dd98afd4
YZ
4999 break;
5000 }
90e9f6a6 5001unlock:
bb192ed9 5002 slab_unlock(slab, &flags);
53e15af0
CL
5003}
5004
434e245d 5005static int validate_slab_node(struct kmem_cache *s,
0a19e7dd 5006 struct kmem_cache_node *n, unsigned long *obj_map)
53e15af0
CL
5007{
5008 unsigned long count = 0;
bb192ed9 5009 struct slab *slab;
53e15af0
CL
5010 unsigned long flags;
5011
5012 spin_lock_irqsave(&n->list_lock, flags);
5013
bb192ed9
VB
5014 list_for_each_entry(slab, &n->partial, slab_list) {
5015 validate_slab(s, slab, obj_map);
53e15af0
CL
5016 count++;
5017 }
1f9f78b1 5018 if (count != n->nr_partial) {
f9f58285
FF
5019 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
5020 s->name, count, n->nr_partial);
1f9f78b1
OG
5021 slab_add_kunit_errors();
5022 }
53e15af0
CL
5023
5024 if (!(s->flags & SLAB_STORE_USER))
5025 goto out;
5026
bb192ed9
VB
5027 list_for_each_entry(slab, &n->full, slab_list) {
5028 validate_slab(s, slab, obj_map);
53e15af0
CL
5029 count++;
5030 }
1f9f78b1 5031 if (count != atomic_long_read(&n->nr_slabs)) {
f9f58285
FF
5032 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
5033 s->name, count, atomic_long_read(&n->nr_slabs));
1f9f78b1
OG
5034 slab_add_kunit_errors();
5035 }
53e15af0
CL
5036
5037out:
5038 spin_unlock_irqrestore(&n->list_lock, flags);
5039 return count;
5040}
5041
1f9f78b1 5042long validate_slab_cache(struct kmem_cache *s)
53e15af0
CL
5043{
5044 int node;
5045 unsigned long count = 0;
fa45dc25 5046 struct kmem_cache_node *n;
0a19e7dd
VB
5047 unsigned long *obj_map;
5048
5049 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
5050 if (!obj_map)
5051 return -ENOMEM;
53e15af0
CL
5052
5053 flush_all(s);
fa45dc25 5054 for_each_kmem_cache_node(s, node, n)
0a19e7dd
VB
5055 count += validate_slab_node(s, n, obj_map);
5056
5057 bitmap_free(obj_map);
90e9f6a6 5058
53e15af0
CL
5059 return count;
5060}
1f9f78b1
OG
5061EXPORT_SYMBOL(validate_slab_cache);
5062
64dd6849 5063#ifdef CONFIG_DEBUG_FS
88a420e4 5064/*
672bba3a 5065 * Generate lists of code addresses where slabcache objects are allocated
88a420e4
CL
5066 * and freed.
5067 */
5068
5069struct location {
5070 unsigned long count;
ce71e27c 5071 unsigned long addr;
45edfa58
CL
5072 long long sum_time;
5073 long min_time;
5074 long max_time;
5075 long min_pid;
5076 long max_pid;
174596a0 5077 DECLARE_BITMAP(cpus, NR_CPUS);
45edfa58 5078 nodemask_t nodes;
88a420e4
CL
5079};
5080
5081struct loc_track {
5082 unsigned long max;
5083 unsigned long count;
5084 struct location *loc;
005a79e5 5085 loff_t idx;
88a420e4
CL
5086};
5087
64dd6849
FM
5088static struct dentry *slab_debugfs_root;
5089
88a420e4
CL
5090static void free_loc_track(struct loc_track *t)
5091{
5092 if (t->max)
5093 free_pages((unsigned long)t->loc,
5094 get_order(sizeof(struct location) * t->max));
5095}
5096
68dff6a9 5097static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
88a420e4
CL
5098{
5099 struct location *l;
5100 int order;
5101
88a420e4
CL
5102 order = get_order(sizeof(struct location) * max);
5103
68dff6a9 5104 l = (void *)__get_free_pages(flags, order);
88a420e4
CL
5105 if (!l)
5106 return 0;
5107
5108 if (t->count) {
5109 memcpy(l, t->loc, sizeof(struct location) * t->count);
5110 free_loc_track(t);
5111 }
5112 t->max = max;
5113 t->loc = l;
5114 return 1;
5115}
5116
5117static int add_location(struct loc_track *t, struct kmem_cache *s,
45edfa58 5118 const struct track *track)
88a420e4
CL
5119{
5120 long start, end, pos;
5121 struct location *l;
ce71e27c 5122 unsigned long caddr;
45edfa58 5123 unsigned long age = jiffies - track->when;
88a420e4
CL
5124
5125 start = -1;
5126 end = t->count;
5127
5128 for ( ; ; ) {
5129 pos = start + (end - start + 1) / 2;
5130
5131 /*
5132 * There is nothing at "end". If we end up there
5133 * we need to add something to before end.
5134 */
5135 if (pos == end)
5136 break;
5137
5138 caddr = t->loc[pos].addr;
45edfa58
CL
5139 if (track->addr == caddr) {
5140
5141 l = &t->loc[pos];
5142 l->count++;
5143 if (track->when) {
5144 l->sum_time += age;
5145 if (age < l->min_time)
5146 l->min_time = age;
5147 if (age > l->max_time)
5148 l->max_time = age;
5149
5150 if (track->pid < l->min_pid)
5151 l->min_pid = track->pid;
5152 if (track->pid > l->max_pid)
5153 l->max_pid = track->pid;
5154
174596a0
RR
5155 cpumask_set_cpu(track->cpu,
5156 to_cpumask(l->cpus));
45edfa58
CL
5157 }
5158 node_set(page_to_nid(virt_to_page(track)), l->nodes);
88a420e4
CL
5159 return 1;
5160 }
5161
45edfa58 5162 if (track->addr < caddr)
88a420e4
CL
5163 end = pos;
5164 else
5165 start = pos;
5166 }
5167
5168 /*
672bba3a 5169 * Not found. Insert new tracking element.
88a420e4 5170 */
68dff6a9 5171 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
88a420e4
CL
5172 return 0;
5173
5174 l = t->loc + pos;
5175 if (pos < t->count)
5176 memmove(l + 1, l,
5177 (t->count - pos) * sizeof(struct location));
5178 t->count++;
5179 l->count = 1;
45edfa58
CL
5180 l->addr = track->addr;
5181 l->sum_time = age;
5182 l->min_time = age;
5183 l->max_time = age;
5184 l->min_pid = track->pid;
5185 l->max_pid = track->pid;
174596a0
RR
5186 cpumask_clear(to_cpumask(l->cpus));
5187 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
45edfa58
CL
5188 nodes_clear(l->nodes);
5189 node_set(page_to_nid(virt_to_page(track)), l->nodes);
88a420e4
CL
5190 return 1;
5191}
5192
5193static void process_slab(struct loc_track *t, struct kmem_cache *s,
bb192ed9 5194 struct slab *slab, enum track_item alloc,
b3fd64e1 5195 unsigned long *obj_map)
88a420e4 5196{
bb192ed9 5197 void *addr = slab_address(slab);
88a420e4
CL
5198 void *p;
5199
bb192ed9 5200 __fill_map(obj_map, s, slab);
b3fd64e1 5201
bb192ed9 5202 for_each_object(p, s, addr, slab->objects)
b3fd64e1 5203 if (!test_bit(__obj_to_index(s, addr, p), obj_map))
45edfa58 5204 add_location(t, s, get_track(s, p, alloc));
88a420e4 5205}
64dd6849 5206#endif /* CONFIG_DEBUG_FS */
6dfd1b65 5207#endif /* CONFIG_SLUB_DEBUG */
88a420e4 5208
ab4d5ed5 5209#ifdef CONFIG_SYSFS
81819f0f 5210enum slab_stat_type {
205ab99d
CL
5211 SL_ALL, /* All slabs */
5212 SL_PARTIAL, /* Only partially allocated slabs */
5213 SL_CPU, /* Only slabs used for cpu caches */
5214 SL_OBJECTS, /* Determine allocated objects not slabs */
5215 SL_TOTAL /* Determine object capacity not slabs */
81819f0f
CL
5216};
5217
205ab99d 5218#define SO_ALL (1 << SL_ALL)
81819f0f
CL
5219#define SO_PARTIAL (1 << SL_PARTIAL)
5220#define SO_CPU (1 << SL_CPU)
5221#define SO_OBJECTS (1 << SL_OBJECTS)
205ab99d 5222#define SO_TOTAL (1 << SL_TOTAL)
81819f0f 5223
62e5c4b4 5224static ssize_t show_slab_objects(struct kmem_cache *s,
bf16d19a 5225 char *buf, unsigned long flags)
81819f0f
CL
5226{
5227 unsigned long total = 0;
81819f0f
CL
5228 int node;
5229 int x;
5230 unsigned long *nodes;
bf16d19a 5231 int len = 0;
81819f0f 5232
6396bb22 5233 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
62e5c4b4
CG
5234 if (!nodes)
5235 return -ENOMEM;
81819f0f 5236
205ab99d
CL
5237 if (flags & SO_CPU) {
5238 int cpu;
81819f0f 5239
205ab99d 5240 for_each_possible_cpu(cpu) {
d0e0ac97
CG
5241 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
5242 cpu);
ec3ab083 5243 int node;
bb192ed9 5244 struct slab *slab;
dfb4f096 5245
bb192ed9
VB
5246 slab = READ_ONCE(c->slab);
5247 if (!slab)
ec3ab083 5248 continue;
205ab99d 5249
bb192ed9 5250 node = slab_nid(slab);
ec3ab083 5251 if (flags & SO_TOTAL)
bb192ed9 5252 x = slab->objects;
ec3ab083 5253 else if (flags & SO_OBJECTS)
bb192ed9 5254 x = slab->inuse;
ec3ab083
CL
5255 else
5256 x = 1;
49e22585 5257
ec3ab083
CL
5258 total += x;
5259 nodes[node] += x;
5260
9c01e9af 5261#ifdef CONFIG_SLUB_CPU_PARTIAL
bb192ed9
VB
5262 slab = slub_percpu_partial_read_once(c);
5263 if (slab) {
5264 node = slab_nid(slab);
8afb1474
LZ
5265 if (flags & SO_TOTAL)
5266 WARN_ON_ONCE(1);
5267 else if (flags & SO_OBJECTS)
5268 WARN_ON_ONCE(1);
5269 else
bb192ed9 5270 x = slab->slabs;
bc6697d8
ED
5271 total += x;
5272 nodes[node] += x;
49e22585 5273 }
9c01e9af 5274#endif
81819f0f
CL
5275 }
5276 }
5277
e4f8e513
QC
5278 /*
5279 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
5280 * already held which will conflict with an existing lock order:
5281 *
5282 * mem_hotplug_lock->slab_mutex->kernfs_mutex
5283 *
5284 * We don't really need mem_hotplug_lock (to hold off
5285 * slab_mem_going_offline_callback) here because slab's memory hot
5286 * unplug code doesn't destroy the kmem_cache->node[] data.
5287 */
5288
ab4d5ed5 5289#ifdef CONFIG_SLUB_DEBUG
205ab99d 5290 if (flags & SO_ALL) {
fa45dc25
CL
5291 struct kmem_cache_node *n;
5292
5293 for_each_kmem_cache_node(s, node, n) {
205ab99d 5294
d0e0ac97
CG
5295 if (flags & SO_TOTAL)
5296 x = atomic_long_read(&n->total_objects);
5297 else if (flags & SO_OBJECTS)
5298 x = atomic_long_read(&n->total_objects) -
5299 count_partial(n, count_free);
81819f0f 5300 else
205ab99d 5301 x = atomic_long_read(&n->nr_slabs);
81819f0f
CL
5302 total += x;
5303 nodes[node] += x;
5304 }
5305
ab4d5ed5
CL
5306 } else
5307#endif
5308 if (flags & SO_PARTIAL) {
fa45dc25 5309 struct kmem_cache_node *n;
81819f0f 5310
fa45dc25 5311 for_each_kmem_cache_node(s, node, n) {
205ab99d
CL
5312 if (flags & SO_TOTAL)
5313 x = count_partial(n, count_total);
5314 else if (flags & SO_OBJECTS)
5315 x = count_partial(n, count_inuse);
81819f0f 5316 else
205ab99d 5317 x = n->nr_partial;
81819f0f
CL
5318 total += x;
5319 nodes[node] += x;
5320 }
5321 }
bf16d19a
JP
5322
5323 len += sysfs_emit_at(buf, len, "%lu", total);
81819f0f 5324#ifdef CONFIG_NUMA
bf16d19a 5325 for (node = 0; node < nr_node_ids; node++) {
81819f0f 5326 if (nodes[node])
bf16d19a
JP
5327 len += sysfs_emit_at(buf, len, " N%d=%lu",
5328 node, nodes[node]);
5329 }
81819f0f 5330#endif
bf16d19a 5331 len += sysfs_emit_at(buf, len, "\n");
81819f0f 5332 kfree(nodes);
bf16d19a
JP
5333
5334 return len;
81819f0f
CL
5335}
5336
81819f0f 5337#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
497888cf 5338#define to_slab(n) container_of(n, struct kmem_cache, kobj)
81819f0f
CL
5339
5340struct slab_attribute {
5341 struct attribute attr;
5342 ssize_t (*show)(struct kmem_cache *s, char *buf);
5343 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
5344};
5345
5346#define SLAB_ATTR_RO(_name) \
ab067e99
VK
5347 static struct slab_attribute _name##_attr = \
5348 __ATTR(_name, 0400, _name##_show, NULL)
81819f0f
CL
5349
5350#define SLAB_ATTR(_name) \
5351 static struct slab_attribute _name##_attr = \
ab067e99 5352 __ATTR(_name, 0600, _name##_show, _name##_store)
81819f0f 5353
81819f0f
CL
5354static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
5355{
bf16d19a 5356 return sysfs_emit(buf, "%u\n", s->size);
81819f0f
CL
5357}
5358SLAB_ATTR_RO(slab_size);
5359
5360static ssize_t align_show(struct kmem_cache *s, char *buf)
5361{
bf16d19a 5362 return sysfs_emit(buf, "%u\n", s->align);
81819f0f
CL
5363}
5364SLAB_ATTR_RO(align);
5365
5366static ssize_t object_size_show(struct kmem_cache *s, char *buf)
5367{
bf16d19a 5368 return sysfs_emit(buf, "%u\n", s->object_size);
81819f0f
CL
5369}
5370SLAB_ATTR_RO(object_size);
5371
5372static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
5373{
bf16d19a 5374 return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
81819f0f
CL
5375}
5376SLAB_ATTR_RO(objs_per_slab);
5377
5378static ssize_t order_show(struct kmem_cache *s, char *buf)
5379{
bf16d19a 5380 return sysfs_emit(buf, "%u\n", oo_order(s->oo));
81819f0f 5381}
32a6f409 5382SLAB_ATTR_RO(order);
81819f0f 5383
73d342b1
DR
5384static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
5385{
bf16d19a 5386 return sysfs_emit(buf, "%lu\n", s->min_partial);
73d342b1
DR
5387}
5388
5389static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
5390 size_t length)
5391{
5392 unsigned long min;
5393 int err;
5394
3dbb95f7 5395 err = kstrtoul(buf, 10, &min);
73d342b1
DR
5396 if (err)
5397 return err;
5398
c0bdb232 5399 set_min_partial(s, min);
73d342b1
DR
5400 return length;
5401}
5402SLAB_ATTR(min_partial);
5403
49e22585
CL
5404static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
5405{
b47291ef
VB
5406 unsigned int nr_partial = 0;
5407#ifdef CONFIG_SLUB_CPU_PARTIAL
5408 nr_partial = s->cpu_partial;
5409#endif
5410
5411 return sysfs_emit(buf, "%u\n", nr_partial);
49e22585
CL
5412}
5413
5414static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
5415 size_t length)
5416{
e5d9998f 5417 unsigned int objects;
49e22585
CL
5418 int err;
5419
e5d9998f 5420 err = kstrtouint(buf, 10, &objects);
49e22585
CL
5421 if (err)
5422 return err;
345c905d 5423 if (objects && !kmem_cache_has_cpu_partial(s))
74ee4ef1 5424 return -EINVAL;
49e22585 5425
e6d0e1dc 5426 slub_set_cpu_partial(s, objects);
49e22585
CL
5427 flush_all(s);
5428 return length;
5429}
5430SLAB_ATTR(cpu_partial);
5431
81819f0f
CL
5432static ssize_t ctor_show(struct kmem_cache *s, char *buf)
5433{
62c70bce
JP
5434 if (!s->ctor)
5435 return 0;
bf16d19a 5436 return sysfs_emit(buf, "%pS\n", s->ctor);
81819f0f
CL
5437}
5438SLAB_ATTR_RO(ctor);
5439
81819f0f
CL
5440static ssize_t aliases_show(struct kmem_cache *s, char *buf)
5441{
bf16d19a 5442 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
81819f0f
CL
5443}
5444SLAB_ATTR_RO(aliases);
5445
81819f0f
CL
5446static ssize_t partial_show(struct kmem_cache *s, char *buf)
5447{
d9acf4b7 5448 return show_slab_objects(s, buf, SO_PARTIAL);
81819f0f
CL
5449}
5450SLAB_ATTR_RO(partial);
5451
5452static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
5453{
d9acf4b7 5454 return show_slab_objects(s, buf, SO_CPU);
81819f0f
CL
5455}
5456SLAB_ATTR_RO(cpu_slabs);
5457
5458static ssize_t objects_show(struct kmem_cache *s, char *buf)
5459{
205ab99d 5460 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
81819f0f
CL
5461}
5462SLAB_ATTR_RO(objects);
5463
205ab99d
CL
5464static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
5465{
5466 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
5467}
5468SLAB_ATTR_RO(objects_partial);
5469
49e22585
CL
5470static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
5471{
5472 int objects = 0;
bb192ed9 5473 int slabs = 0;
9c01e9af 5474 int cpu __maybe_unused;
bf16d19a 5475 int len = 0;
49e22585 5476
9c01e9af 5477#ifdef CONFIG_SLUB_CPU_PARTIAL
49e22585 5478 for_each_online_cpu(cpu) {
bb192ed9 5479 struct slab *slab;
a93cf07b 5480
bb192ed9 5481 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
49e22585 5482
bb192ed9
VB
5483 if (slab)
5484 slabs += slab->slabs;
49e22585 5485 }
9c01e9af 5486#endif
49e22585 5487
c2092c12 5488 /* Approximate half-full slabs, see slub_set_cpu_partial() */
bb192ed9
VB
5489 objects = (slabs * oo_objects(s->oo)) / 2;
5490 len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
49e22585 5491
9c01e9af 5492#if defined(CONFIG_SLUB_CPU_PARTIAL) && defined(CONFIG_SMP)
49e22585 5493 for_each_online_cpu(cpu) {
bb192ed9 5494 struct slab *slab;
a93cf07b 5495
bb192ed9
VB
5496 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
5497 if (slab) {
5498 slabs = READ_ONCE(slab->slabs);
5499 objects = (slabs * oo_objects(s->oo)) / 2;
bf16d19a 5500 len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
bb192ed9 5501 cpu, objects, slabs);
b47291ef 5502 }
49e22585
CL
5503 }
5504#endif
bf16d19a
JP
5505 len += sysfs_emit_at(buf, len, "\n");
5506
5507 return len;
49e22585
CL
5508}
5509SLAB_ATTR_RO(slabs_cpu_partial);
5510
a5a84755
CL
5511static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
5512{
bf16d19a 5513 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
a5a84755 5514}
8f58119a 5515SLAB_ATTR_RO(reclaim_account);
a5a84755
CL
5516
5517static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
5518{
bf16d19a 5519 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
a5a84755
CL
5520}
5521SLAB_ATTR_RO(hwcache_align);
5522
5523#ifdef CONFIG_ZONE_DMA
5524static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
5525{
bf16d19a 5526 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
a5a84755
CL
5527}
5528SLAB_ATTR_RO(cache_dma);
5529#endif
5530
8eb8284b
DW
5531static ssize_t usersize_show(struct kmem_cache *s, char *buf)
5532{
bf16d19a 5533 return sysfs_emit(buf, "%u\n", s->usersize);
8eb8284b
DW
5534}
5535SLAB_ATTR_RO(usersize);
5536
a5a84755
CL
5537static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
5538{
bf16d19a 5539 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
a5a84755
CL
5540}
5541SLAB_ATTR_RO(destroy_by_rcu);
5542
ab4d5ed5 5543#ifdef CONFIG_SLUB_DEBUG
a5a84755
CL
5544static ssize_t slabs_show(struct kmem_cache *s, char *buf)
5545{
5546 return show_slab_objects(s, buf, SO_ALL);
5547}
5548SLAB_ATTR_RO(slabs);
5549
205ab99d
CL
5550static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
5551{
5552 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
5553}
5554SLAB_ATTR_RO(total_objects);
5555
81819f0f
CL
5556static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
5557{
bf16d19a 5558 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
81819f0f 5559}
060807f8 5560SLAB_ATTR_RO(sanity_checks);
81819f0f
CL
5561
5562static ssize_t trace_show(struct kmem_cache *s, char *buf)
5563{
bf16d19a 5564 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
81819f0f 5565}
060807f8 5566SLAB_ATTR_RO(trace);
81819f0f 5567
81819f0f
CL
5568static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
5569{
bf16d19a 5570 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
81819f0f
CL
5571}
5572
ad38b5b1 5573SLAB_ATTR_RO(red_zone);
81819f0f
CL
5574
5575static ssize_t poison_show(struct kmem_cache *s, char *buf)
5576{
bf16d19a 5577 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
81819f0f
CL
5578}
5579
ad38b5b1 5580SLAB_ATTR_RO(poison);
81819f0f
CL
5581
5582static ssize_t store_user_show(struct kmem_cache *s, char *buf)
5583{
bf16d19a 5584 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
81819f0f
CL
5585}
5586
ad38b5b1 5587SLAB_ATTR_RO(store_user);
81819f0f 5588
53e15af0
CL
5589static ssize_t validate_show(struct kmem_cache *s, char *buf)
5590{
5591 return 0;
5592}
5593
5594static ssize_t validate_store(struct kmem_cache *s,
5595 const char *buf, size_t length)
5596{
434e245d
CL
5597 int ret = -EINVAL;
5598
5599 if (buf[0] == '1') {
5600 ret = validate_slab_cache(s);
5601 if (ret >= 0)
5602 ret = length;
5603 }
5604 return ret;
53e15af0
CL
5605}
5606SLAB_ATTR(validate);
a5a84755 5607
a5a84755
CL
5608#endif /* CONFIG_SLUB_DEBUG */
5609
5610#ifdef CONFIG_FAILSLAB
5611static ssize_t failslab_show(struct kmem_cache *s, char *buf)
5612{
bf16d19a 5613 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
a5a84755 5614}
060807f8 5615SLAB_ATTR_RO(failslab);
ab4d5ed5 5616#endif
53e15af0 5617
2086d26a
CL
5618static ssize_t shrink_show(struct kmem_cache *s, char *buf)
5619{
5620 return 0;
5621}
5622
5623static ssize_t shrink_store(struct kmem_cache *s,
5624 const char *buf, size_t length)
5625{
832f37f5 5626 if (buf[0] == '1')
10befea9 5627 kmem_cache_shrink(s);
832f37f5 5628 else
2086d26a
CL
5629 return -EINVAL;
5630 return length;
5631}
5632SLAB_ATTR(shrink);
5633
81819f0f 5634#ifdef CONFIG_NUMA
9824601e 5635static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
81819f0f 5636{
bf16d19a 5637 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
81819f0f
CL
5638}
5639
9824601e 5640static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
81819f0f
CL
5641 const char *buf, size_t length)
5642{
eb7235eb 5643 unsigned int ratio;
0121c619
CL
5644 int err;
5645
eb7235eb 5646 err = kstrtouint(buf, 10, &ratio);
0121c619
CL
5647 if (err)
5648 return err;
eb7235eb
AD
5649 if (ratio > 100)
5650 return -ERANGE;
0121c619 5651
eb7235eb 5652 s->remote_node_defrag_ratio = ratio * 10;
81819f0f 5653
81819f0f
CL
5654 return length;
5655}
9824601e 5656SLAB_ATTR(remote_node_defrag_ratio);
81819f0f
CL
5657#endif
5658
8ff12cfc 5659#ifdef CONFIG_SLUB_STATS
8ff12cfc
CL
5660static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5661{
5662 unsigned long sum = 0;
5663 int cpu;
bf16d19a 5664 int len = 0;
6da2ec56 5665 int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
8ff12cfc
CL
5666
5667 if (!data)
5668 return -ENOMEM;
5669
5670 for_each_online_cpu(cpu) {
9dfc6e68 5671 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
8ff12cfc
CL
5672
5673 data[cpu] = x;
5674 sum += x;
5675 }
5676
bf16d19a 5677 len += sysfs_emit_at(buf, len, "%lu", sum);
8ff12cfc 5678
50ef37b9 5679#ifdef CONFIG_SMP
8ff12cfc 5680 for_each_online_cpu(cpu) {
bf16d19a
JP
5681 if (data[cpu])
5682 len += sysfs_emit_at(buf, len, " C%d=%u",
5683 cpu, data[cpu]);
8ff12cfc 5684 }
50ef37b9 5685#endif
8ff12cfc 5686 kfree(data);
bf16d19a
JP
5687 len += sysfs_emit_at(buf, len, "\n");
5688
5689 return len;
8ff12cfc
CL
5690}
5691
78eb00cc
DR
5692static void clear_stat(struct kmem_cache *s, enum stat_item si)
5693{
5694 int cpu;
5695
5696 for_each_online_cpu(cpu)
9dfc6e68 5697 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
78eb00cc
DR
5698}
5699
8ff12cfc
CL
5700#define STAT_ATTR(si, text) \
5701static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5702{ \
5703 return show_stat(s, buf, si); \
5704} \
78eb00cc
DR
5705static ssize_t text##_store(struct kmem_cache *s, \
5706 const char *buf, size_t length) \
5707{ \
5708 if (buf[0] != '0') \
5709 return -EINVAL; \
5710 clear_stat(s, si); \
5711 return length; \
5712} \
5713SLAB_ATTR(text); \
8ff12cfc
CL
5714
5715STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5716STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5717STAT_ATTR(FREE_FASTPATH, free_fastpath);
5718STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5719STAT_ATTR(FREE_FROZEN, free_frozen);
5720STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5721STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5722STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5723STAT_ATTR(ALLOC_SLAB, alloc_slab);
5724STAT_ATTR(ALLOC_REFILL, alloc_refill);
e36a2652 5725STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
8ff12cfc
CL
5726STAT_ATTR(FREE_SLAB, free_slab);
5727STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5728STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5729STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5730STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5731STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5732STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
03e404af 5733STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
65c3376a 5734STAT_ATTR(ORDER_FALLBACK, order_fallback);
b789ef51
CL
5735STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5736STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
49e22585
CL
5737STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5738STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
8028dcea
AS
5739STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5740STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
6dfd1b65 5741#endif /* CONFIG_SLUB_STATS */
8ff12cfc 5742
06428780 5743static struct attribute *slab_attrs[] = {
81819f0f
CL
5744 &slab_size_attr.attr,
5745 &object_size_attr.attr,
5746 &objs_per_slab_attr.attr,
5747 &order_attr.attr,
73d342b1 5748 &min_partial_attr.attr,
49e22585 5749 &cpu_partial_attr.attr,
81819f0f 5750 &objects_attr.attr,
205ab99d 5751 &objects_partial_attr.attr,
81819f0f
CL
5752 &partial_attr.attr,
5753 &cpu_slabs_attr.attr,
5754 &ctor_attr.attr,
81819f0f
CL
5755 &aliases_attr.attr,
5756 &align_attr.attr,
81819f0f
CL
5757 &hwcache_align_attr.attr,
5758 &reclaim_account_attr.attr,
5759 &destroy_by_rcu_attr.attr,
a5a84755 5760 &shrink_attr.attr,
49e22585 5761 &slabs_cpu_partial_attr.attr,
ab4d5ed5 5762#ifdef CONFIG_SLUB_DEBUG
a5a84755
CL
5763 &total_objects_attr.attr,
5764 &slabs_attr.attr,
5765 &sanity_checks_attr.attr,
5766 &trace_attr.attr,
81819f0f
CL
5767 &red_zone_attr.attr,
5768 &poison_attr.attr,
5769 &store_user_attr.attr,
53e15af0 5770 &validate_attr.attr,
ab4d5ed5 5771#endif
81819f0f
CL
5772#ifdef CONFIG_ZONE_DMA
5773 &cache_dma_attr.attr,
5774#endif
5775#ifdef CONFIG_NUMA
9824601e 5776 &remote_node_defrag_ratio_attr.attr,
8ff12cfc
CL
5777#endif
5778#ifdef CONFIG_SLUB_STATS
5779 &alloc_fastpath_attr.attr,
5780 &alloc_slowpath_attr.attr,
5781 &free_fastpath_attr.attr,
5782 &free_slowpath_attr.attr,
5783 &free_frozen_attr.attr,
5784 &free_add_partial_attr.attr,
5785 &free_remove_partial_attr.attr,
5786 &alloc_from_partial_attr.attr,
5787 &alloc_slab_attr.attr,
5788 &alloc_refill_attr.attr,
e36a2652 5789 &alloc_node_mismatch_attr.attr,
8ff12cfc
CL
5790 &free_slab_attr.attr,
5791 &cpuslab_flush_attr.attr,
5792 &deactivate_full_attr.attr,
5793 &deactivate_empty_attr.attr,
5794 &deactivate_to_head_attr.attr,
5795 &deactivate_to_tail_attr.attr,
5796 &deactivate_remote_frees_attr.attr,
03e404af 5797 &deactivate_bypass_attr.attr,
65c3376a 5798 &order_fallback_attr.attr,
b789ef51
CL
5799 &cmpxchg_double_fail_attr.attr,
5800 &cmpxchg_double_cpu_fail_attr.attr,
49e22585
CL
5801 &cpu_partial_alloc_attr.attr,
5802 &cpu_partial_free_attr.attr,
8028dcea
AS
5803 &cpu_partial_node_attr.attr,
5804 &cpu_partial_drain_attr.attr,
81819f0f 5805#endif
4c13dd3b
DM
5806#ifdef CONFIG_FAILSLAB
5807 &failslab_attr.attr,
5808#endif
8eb8284b 5809 &usersize_attr.attr,
4c13dd3b 5810
81819f0f
CL
5811 NULL
5812};
5813
1fdaaa23 5814static const struct attribute_group slab_attr_group = {
81819f0f
CL
5815 .attrs = slab_attrs,
5816};
5817
5818static ssize_t slab_attr_show(struct kobject *kobj,
5819 struct attribute *attr,
5820 char *buf)
5821{
5822 struct slab_attribute *attribute;
5823 struct kmem_cache *s;
5824 int err;
5825
5826 attribute = to_slab_attr(attr);
5827 s = to_slab(kobj);
5828
5829 if (!attribute->show)
5830 return -EIO;
5831
5832 err = attribute->show(s, buf);
5833
5834 return err;
5835}
5836
5837static ssize_t slab_attr_store(struct kobject *kobj,
5838 struct attribute *attr,
5839 const char *buf, size_t len)
5840{
5841 struct slab_attribute *attribute;
5842 struct kmem_cache *s;
5843 int err;
5844
5845 attribute = to_slab_attr(attr);
5846 s = to_slab(kobj);
5847
5848 if (!attribute->store)
5849 return -EIO;
5850
5851 err = attribute->store(s, buf, len);
81819f0f
CL
5852 return err;
5853}
5854
41a21285
CL
5855static void kmem_cache_release(struct kobject *k)
5856{
5857 slab_kmem_cache_release(to_slab(k));
5858}
5859
52cf25d0 5860static const struct sysfs_ops slab_sysfs_ops = {
81819f0f
CL
5861 .show = slab_attr_show,
5862 .store = slab_attr_store,
5863};
5864
5865static struct kobj_type slab_ktype = {
5866 .sysfs_ops = &slab_sysfs_ops,
41a21285 5867 .release = kmem_cache_release,
81819f0f
CL
5868};
5869
27c3a314 5870static struct kset *slab_kset;
81819f0f 5871
9a41707b
VD
5872static inline struct kset *cache_kset(struct kmem_cache *s)
5873{
9a41707b
VD
5874 return slab_kset;
5875}
5876
81819f0f
CL
5877#define ID_STR_LENGTH 64
5878
5879/* Create a unique string id for a slab cache:
6446faa2
CL
5880 *
5881 * Format :[flags-]size
81819f0f
CL
5882 */
5883static char *create_unique_id(struct kmem_cache *s)
5884{
5885 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5886 char *p = name;
5887
5888 BUG_ON(!name);
5889
5890 *p++ = ':';
5891 /*
5892 * First flags affecting slabcache operations. We will only
5893 * get here for aliasable slabs so we do not need to support
5894 * too many flags. The flags here must cover all flags that
5895 * are matched during merging to guarantee that the id is
5896 * unique.
5897 */
5898 if (s->flags & SLAB_CACHE_DMA)
5899 *p++ = 'd';
6d6ea1e9
NB
5900 if (s->flags & SLAB_CACHE_DMA32)
5901 *p++ = 'D';
81819f0f
CL
5902 if (s->flags & SLAB_RECLAIM_ACCOUNT)
5903 *p++ = 'a';
becfda68 5904 if (s->flags & SLAB_CONSISTENCY_CHECKS)
81819f0f 5905 *p++ = 'F';
230e9fc2
VD
5906 if (s->flags & SLAB_ACCOUNT)
5907 *p++ = 'A';
81819f0f
CL
5908 if (p != name + 1)
5909 *p++ = '-';
44065b2e 5910 p += sprintf(p, "%07u", s->size);
2633d7a0 5911
81819f0f
CL
5912 BUG_ON(p > name + ID_STR_LENGTH - 1);
5913 return name;
5914}
5915
5916static int sysfs_slab_add(struct kmem_cache *s)
5917{
5918 int err;
5919 const char *name;
1663f26d 5920 struct kset *kset = cache_kset(s);
45530c44 5921 int unmergeable = slab_unmergeable(s);
81819f0f 5922
1663f26d
TH
5923 if (!kset) {
5924 kobject_init(&s->kobj, &slab_ktype);
5925 return 0;
5926 }
5927
11066386
MC
5928 if (!unmergeable && disable_higher_order_debug &&
5929 (slub_debug & DEBUG_METADATA_FLAGS))
5930 unmergeable = 1;
5931
81819f0f
CL
5932 if (unmergeable) {
5933 /*
5934 * Slabcache can never be merged so we can use the name proper.
5935 * This is typically the case for debug situations. In that
5936 * case we can catch duplicate names easily.
5937 */
27c3a314 5938 sysfs_remove_link(&slab_kset->kobj, s->name);
81819f0f
CL
5939 name = s->name;
5940 } else {
5941 /*
5942 * Create a unique name for the slab as a target
5943 * for the symlinks.
5944 */
5945 name = create_unique_id(s);
5946 }
5947
1663f26d 5948 s->kobj.kset = kset;
26e4f205 5949 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
757fed1d 5950 if (err)
80da026a 5951 goto out;
81819f0f
CL
5952
5953 err = sysfs_create_group(&s->kobj, &slab_attr_group);
54b6a731
DJ
5954 if (err)
5955 goto out_del_kobj;
9a41707b 5956
81819f0f
CL
5957 if (!unmergeable) {
5958 /* Setup first alias */
5959 sysfs_slab_alias(s, s->name);
81819f0f 5960 }
54b6a731
DJ
5961out:
5962 if (!unmergeable)
5963 kfree(name);
5964 return err;
5965out_del_kobj:
5966 kobject_del(&s->kobj);
54b6a731 5967 goto out;
81819f0f
CL
5968}
5969
d50d82fa
MP
5970void sysfs_slab_unlink(struct kmem_cache *s)
5971{
5972 if (slab_state >= FULL)
5973 kobject_del(&s->kobj);
5974}
5975
bf5eb3de
TH
5976void sysfs_slab_release(struct kmem_cache *s)
5977{
5978 if (slab_state >= FULL)
5979 kobject_put(&s->kobj);
81819f0f
CL
5980}
5981
5982/*
5983 * Need to buffer aliases during bootup until sysfs becomes
9f6c708e 5984 * available lest we lose that information.
81819f0f
CL
5985 */
5986struct saved_alias {
5987 struct kmem_cache *s;
5988 const char *name;
5989 struct saved_alias *next;
5990};
5991
5af328a5 5992static struct saved_alias *alias_list;
81819f0f
CL
5993
5994static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5995{
5996 struct saved_alias *al;
5997
97d06609 5998 if (slab_state == FULL) {
81819f0f
CL
5999 /*
6000 * If we have a leftover link then remove it.
6001 */
27c3a314
GKH
6002 sysfs_remove_link(&slab_kset->kobj, name);
6003 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
81819f0f
CL
6004 }
6005
6006 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
6007 if (!al)
6008 return -ENOMEM;
6009
6010 al->s = s;
6011 al->name = name;
6012 al->next = alias_list;
6013 alias_list = al;
6014 return 0;
6015}
6016
6017static int __init slab_sysfs_init(void)
6018{
5b95a4ac 6019 struct kmem_cache *s;
81819f0f
CL
6020 int err;
6021
18004c5d 6022 mutex_lock(&slab_mutex);
2bce6485 6023
d7660ce5 6024 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
27c3a314 6025 if (!slab_kset) {
18004c5d 6026 mutex_unlock(&slab_mutex);
f9f58285 6027 pr_err("Cannot register slab subsystem.\n");
81819f0f
CL
6028 return -ENOSYS;
6029 }
6030
97d06609 6031 slab_state = FULL;
26a7bd03 6032
5b95a4ac 6033 list_for_each_entry(s, &slab_caches, list) {
26a7bd03 6034 err = sysfs_slab_add(s);
5d540fb7 6035 if (err)
f9f58285
FF
6036 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
6037 s->name);
26a7bd03 6038 }
81819f0f
CL
6039
6040 while (alias_list) {
6041 struct saved_alias *al = alias_list;
6042
6043 alias_list = alias_list->next;
6044 err = sysfs_slab_alias(al->s, al->name);
5d540fb7 6045 if (err)
f9f58285
FF
6046 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
6047 al->name);
81819f0f
CL
6048 kfree(al);
6049 }
6050
18004c5d 6051 mutex_unlock(&slab_mutex);
81819f0f
CL
6052 return 0;
6053}
6054
6055__initcall(slab_sysfs_init);
ab4d5ed5 6056#endif /* CONFIG_SYSFS */
57ed3eda 6057
64dd6849
FM
6058#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
6059static int slab_debugfs_show(struct seq_file *seq, void *v)
6060{
64dd6849 6061 struct loc_track *t = seq->private;
005a79e5
GS
6062 struct location *l;
6063 unsigned long idx;
64dd6849 6064
005a79e5 6065 idx = (unsigned long) t->idx;
64dd6849
FM
6066 if (idx < t->count) {
6067 l = &t->loc[idx];
6068
6069 seq_printf(seq, "%7ld ", l->count);
6070
6071 if (l->addr)
6072 seq_printf(seq, "%pS", (void *)l->addr);
6073 else
6074 seq_puts(seq, "<not-available>");
6075
6076 if (l->sum_time != l->min_time) {
6077 seq_printf(seq, " age=%ld/%llu/%ld",
6078 l->min_time, div_u64(l->sum_time, l->count),
6079 l->max_time);
6080 } else
6081 seq_printf(seq, " age=%ld", l->min_time);
6082
6083 if (l->min_pid != l->max_pid)
6084 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
6085 else
6086 seq_printf(seq, " pid=%ld",
6087 l->min_pid);
6088
6089 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
6090 seq_printf(seq, " cpus=%*pbl",
6091 cpumask_pr_args(to_cpumask(l->cpus)));
6092
6093 if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
6094 seq_printf(seq, " nodes=%*pbl",
6095 nodemask_pr_args(&l->nodes));
6096
6097 seq_puts(seq, "\n");
6098 }
6099
6100 if (!idx && !t->count)
6101 seq_puts(seq, "No data\n");
6102
6103 return 0;
6104}
6105
6106static void slab_debugfs_stop(struct seq_file *seq, void *v)
6107{
6108}
6109
6110static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
6111{
6112 struct loc_track *t = seq->private;
6113
005a79e5 6114 t->idx = ++(*ppos);
64dd6849 6115 if (*ppos <= t->count)
005a79e5 6116 return ppos;
64dd6849
FM
6117
6118 return NULL;
6119}
6120
6121static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
6122{
005a79e5
GS
6123 struct loc_track *t = seq->private;
6124
6125 t->idx = *ppos;
64dd6849
FM
6126 return ppos;
6127}
6128
6129static const struct seq_operations slab_debugfs_sops = {
6130 .start = slab_debugfs_start,
6131 .next = slab_debugfs_next,
6132 .stop = slab_debugfs_stop,
6133 .show = slab_debugfs_show,
6134};
6135
6136static int slab_debug_trace_open(struct inode *inode, struct file *filep)
6137{
6138
6139 struct kmem_cache_node *n;
6140 enum track_item alloc;
6141 int node;
6142 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
6143 sizeof(struct loc_track));
6144 struct kmem_cache *s = file_inode(filep)->i_private;
b3fd64e1
VB
6145 unsigned long *obj_map;
6146
2127d225
ML
6147 if (!t)
6148 return -ENOMEM;
6149
b3fd64e1 6150 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
2127d225
ML
6151 if (!obj_map) {
6152 seq_release_private(inode, filep);
b3fd64e1 6153 return -ENOMEM;
2127d225 6154 }
64dd6849
FM
6155
6156 if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
6157 alloc = TRACK_ALLOC;
6158 else
6159 alloc = TRACK_FREE;
6160
b3fd64e1
VB
6161 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
6162 bitmap_free(obj_map);
2127d225 6163 seq_release_private(inode, filep);
64dd6849 6164 return -ENOMEM;
b3fd64e1 6165 }
64dd6849 6166
64dd6849
FM
6167 for_each_kmem_cache_node(s, node, n) {
6168 unsigned long flags;
bb192ed9 6169 struct slab *slab;
64dd6849
FM
6170
6171 if (!atomic_long_read(&n->nr_slabs))
6172 continue;
6173
6174 spin_lock_irqsave(&n->list_lock, flags);
bb192ed9
VB
6175 list_for_each_entry(slab, &n->partial, slab_list)
6176 process_slab(t, s, slab, alloc, obj_map);
6177 list_for_each_entry(slab, &n->full, slab_list)
6178 process_slab(t, s, slab, alloc, obj_map);
64dd6849
FM
6179 spin_unlock_irqrestore(&n->list_lock, flags);
6180 }
6181
b3fd64e1 6182 bitmap_free(obj_map);
64dd6849
FM
6183 return 0;
6184}
6185
6186static int slab_debug_trace_release(struct inode *inode, struct file *file)
6187{
6188 struct seq_file *seq = file->private_data;
6189 struct loc_track *t = seq->private;
6190
6191 free_loc_track(t);
6192 return seq_release_private(inode, file);
6193}
6194
6195static const struct file_operations slab_debugfs_fops = {
6196 .open = slab_debug_trace_open,
6197 .read = seq_read,
6198 .llseek = seq_lseek,
6199 .release = slab_debug_trace_release,
6200};
6201
6202static void debugfs_slab_add(struct kmem_cache *s)
6203{
6204 struct dentry *slab_cache_dir;
6205
6206 if (unlikely(!slab_debugfs_root))
6207 return;
6208
6209 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
6210
6211 debugfs_create_file("alloc_traces", 0400,
6212 slab_cache_dir, s, &slab_debugfs_fops);
6213
6214 debugfs_create_file("free_traces", 0400,
6215 slab_cache_dir, s, &slab_debugfs_fops);
6216}
6217
6218void debugfs_slab_release(struct kmem_cache *s)
6219{
6220 debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root));
6221}
6222
6223static int __init slab_debugfs_init(void)
6224{
6225 struct kmem_cache *s;
6226
6227 slab_debugfs_root = debugfs_create_dir("slab", NULL);
6228
6229 list_for_each_entry(s, &slab_caches, list)
6230 if (s->flags & SLAB_STORE_USER)
6231 debugfs_slab_add(s);
6232
6233 return 0;
6234
6235}
6236__initcall(slab_debugfs_init);
6237#endif
57ed3eda
PE
6238/*
6239 * The /proc/slabinfo ABI
6240 */
5b365771 6241#ifdef CONFIG_SLUB_DEBUG
0d7561c6 6242void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
57ed3eda 6243{
57ed3eda 6244 unsigned long nr_slabs = 0;
205ab99d
CL
6245 unsigned long nr_objs = 0;
6246 unsigned long nr_free = 0;
57ed3eda 6247 int node;
fa45dc25 6248 struct kmem_cache_node *n;
57ed3eda 6249
fa45dc25 6250 for_each_kmem_cache_node(s, node, n) {
c17fd13e
WL
6251 nr_slabs += node_nr_slabs(n);
6252 nr_objs += node_nr_objs(n);
205ab99d 6253 nr_free += count_partial(n, count_free);
57ed3eda
PE
6254 }
6255
0d7561c6
GC
6256 sinfo->active_objs = nr_objs - nr_free;
6257 sinfo->num_objs = nr_objs;
6258 sinfo->active_slabs = nr_slabs;
6259 sinfo->num_slabs = nr_slabs;
6260 sinfo->objects_per_slab = oo_objects(s->oo);
6261 sinfo->cache_order = oo_order(s->oo);
57ed3eda
PE
6262}
6263
0d7561c6 6264void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
7b3c3a50 6265{
7b3c3a50
AD
6266}
6267
b7454ad3
GC
6268ssize_t slabinfo_write(struct file *file, const char __user *buffer,
6269 size_t count, loff_t *ppos)
7b3c3a50 6270{
b7454ad3 6271 return -EIO;
7b3c3a50 6272}
5b365771 6273#endif /* CONFIG_SLUB_DEBUG */