Commit | Line | Data |
---|---|---|
1802d0be | 1 | // SPDX-License-Identifier: GPL-2.0-only |
cd11016e AP |
2 | /* |
3 | * Generic stack depot for storing stack traces. | |
4 | * | |
5 | * Some debugging tools need to save stack traces of certain events which can | |
6 | * be later presented to the user. For example, KASAN needs to safe alloc and | |
7 | * free stacks for each object, but storing two stack traces per object | |
8 | * requires too much memory (e.g. SLUB_DEBUG needs 256 bytes per object for | |
9 | * that). | |
10 | * | |
11 | * Instead, stack depot maintains a hashtable of unique stacktraces. Since alloc | |
12 | * and free stacks repeat a lot, we save about 100x space. | |
13 | * Stacks are never removed from depot, so we store them contiguously one after | |
9dbbc3b9 | 14 | * another in a contiguous memory allocation. |
cd11016e AP |
15 | * |
16 | * Author: Alexander Potapenko <glider@google.com> | |
17 | * Copyright (C) 2016 Google, Inc. | |
18 | * | |
19 | * Based on code by Dmitry Chernenkov. | |
cd11016e AP |
20 | */ |
21 | ||
4a6b5314 AK |
22 | #define pr_fmt(fmt) "stackdepot: " fmt |
23 | ||
cd11016e AP |
24 | #include <linux/gfp.h> |
25 | #include <linux/jhash.h> | |
26 | #include <linux/kernel.h> | |
27 | #include <linux/mm.h> | |
2dba5eb1 | 28 | #include <linux/mutex.h> |
cd11016e AP |
29 | #include <linux/percpu.h> |
30 | #include <linux/printk.h> | |
31 | #include <linux/slab.h> | |
32 | #include <linux/stacktrace.h> | |
33 | #include <linux/stackdepot.h> | |
34 | #include <linux/string.h> | |
35 | #include <linux/types.h> | |
e1fdc403 | 36 | #include <linux/memblock.h> |
f9987921 | 37 | #include <linux/kasan-enabled.h> |
cd11016e | 38 | |
424cafee AK |
39 | #define DEPOT_HANDLE_BITS (sizeof(depot_stack_handle_t) * 8) |
40 | ||
41 | #define DEPOT_VALID_BITS 1 | |
42 | #define DEPOT_POOL_ORDER 2 /* Pool size order, 4 pages */ | |
43 | #define DEPOT_POOL_SIZE (1LL << (PAGE_SHIFT + DEPOT_POOL_ORDER)) | |
44 | #define DEPOT_STACK_ALIGN 4 | |
45 | #define DEPOT_OFFSET_BITS (DEPOT_POOL_ORDER + PAGE_SHIFT - DEPOT_STACK_ALIGN) | |
46 | #define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_VALID_BITS - \ | |
47 | DEPOT_OFFSET_BITS - STACK_DEPOT_EXTRA_BITS) | |
48 | #define DEPOT_POOLS_CAP 8192 | |
49 | #define DEPOT_MAX_POOLS \ | |
50 | (((1LL << (DEPOT_POOL_INDEX_BITS)) < DEPOT_POOLS_CAP) ? \ | |
51 | (1LL << (DEPOT_POOL_INDEX_BITS)) : DEPOT_POOLS_CAP) | |
cd11016e AP |
52 | |
53 | /* The compact structure to store the reference to stacks. */ | |
54 | union handle_parts { | |
55 | depot_stack_handle_t handle; | |
56 | struct { | |
424cafee AK |
57 | u32 pool_index : DEPOT_POOL_INDEX_BITS; |
58 | u32 offset : DEPOT_OFFSET_BITS; | |
59 | u32 valid : DEPOT_VALID_BITS; | |
60 | u32 extra : STACK_DEPOT_EXTRA_BITS; | |
cd11016e AP |
61 | }; |
62 | }; | |
63 | ||
64 | struct stack_record { | |
65 | struct stack_record *next; /* Link in the hashtable */ | |
66 | u32 hash; /* Hash in the hastable */ | |
67 | u32 size; /* Number of frames in the stack */ | |
68 | union handle_parts handle; | |
3a2b67e6 | 69 | unsigned long entries[]; /* Variable-sized array of entries. */ |
cd11016e AP |
70 | }; |
71 | ||
735df3c3 | 72 | static bool stack_depot_disabled; |
1c0310ad | 73 | static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); |
a5f1783b VB |
74 | static bool __stack_depot_early_init_passed __initdata; |
75 | ||
0d249ac0 | 76 | /* Use one hash table bucket per 16 KB of memory. */ |
4c2e9a67 | 77 | #define STACK_HASH_TABLE_SCALE 14 |
0d249ac0 | 78 | /* Limit the number of buckets between 4K and 1M. */ |
4c2e9a67 AK |
79 | #define STACK_BUCKET_NUMBER_ORDER_MIN 12 |
80 | #define STACK_BUCKET_NUMBER_ORDER_MAX 20 | |
0d249ac0 | 81 | /* Initial seed for jhash2. */ |
cd11016e AP |
82 | #define STACK_HASH_SEED 0x9747b28c |
83 | ||
0d249ac0 AK |
84 | /* Hash table of pointers to stored stack traces. */ |
85 | static struct stack_record **stack_table; | |
86 | /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ | |
4c2e9a67 | 87 | static unsigned int stack_bucket_number_order; |
0d249ac0 | 88 | /* Hash mask for indexing the table. */ |
f9987921 VB |
89 | static unsigned int stack_hash_mask; |
90 | ||
0d249ac0 | 91 | /* Array of memory regions that store stack traces. */ |
424cafee | 92 | static void *stack_pools[DEPOT_MAX_POOLS]; |
961c949b AK |
93 | /* Currently used pool in stack_pools. */ |
94 | static int pool_index; | |
95 | /* Offset to the unused space in the currently used pool. */ | |
96 | static size_t pool_offset; | |
0d249ac0 | 97 | /* Lock that protects the variables above. */ |
961c949b AK |
98 | static DEFINE_RAW_SPINLOCK(pool_lock); |
99 | /* Whether the next pool is initialized. */ | |
100 | static int next_pool_inited; | |
e1fdc403 | 101 | |
735df3c3 | 102 | static int __init disable_stack_depot(char *str) |
e1fdc403 | 103 | { |
64427985 VJ |
104 | int ret; |
105 | ||
735df3c3 AK |
106 | ret = kstrtobool(str, &stack_depot_disabled); |
107 | if (!ret && stack_depot_disabled) { | |
4a6b5314 | 108 | pr_info("disabled\n"); |
e1fdc403 VJ |
109 | stack_table = NULL; |
110 | } | |
111 | return 0; | |
112 | } | |
735df3c3 | 113 | early_param("stack_depot_disable", disable_stack_depot); |
e1fdc403 | 114 | |
1c0310ad | 115 | void __init stack_depot_request_early_init(void) |
a5f1783b | 116 | { |
1c0310ad | 117 | /* Too late to request early init now. */ |
a5f1783b VB |
118 | WARN_ON(__stack_depot_early_init_passed); |
119 | ||
1c0310ad | 120 | __stack_depot_early_init_requested = true; |
a5f1783b VB |
121 | } |
122 | ||
df225c87 | 123 | /* Allocates a hash table via memblock. Can only be used during early boot. */ |
a5f1783b VB |
124 | int __init stack_depot_early_init(void) |
125 | { | |
f9987921 | 126 | unsigned long entries = 0; |
a5f1783b | 127 | |
df225c87 | 128 | /* This function must be called only once, from mm_init(). */ |
a5f1783b VB |
129 | if (WARN_ON(__stack_depot_early_init_passed)) |
130 | return 0; | |
a5f1783b VB |
131 | __stack_depot_early_init_passed = true; |
132 | ||
df225c87 AK |
133 | /* |
134 | * If KASAN is enabled, use the maximum order: KASAN is frequently used | |
135 | * in fuzzing scenarios, which leads to a large number of different | |
136 | * stack traces being stored in stack depot. | |
137 | */ | |
4c2e9a67 AK |
138 | if (kasan_enabled() && !stack_bucket_number_order) |
139 | stack_bucket_number_order = STACK_BUCKET_NUMBER_ORDER_MAX; | |
f9987921 | 140 | |
735df3c3 | 141 | if (!__stack_depot_early_init_requested || stack_depot_disabled) |
a5f1783b VB |
142 | return 0; |
143 | ||
df225c87 | 144 | /* |
4c2e9a67 AK |
145 | * If stack_bucket_number_order is not set, leave entries as 0 to rely |
146 | * on the automatic calculations performed by alloc_large_system_hash. | |
df225c87 | 147 | */ |
4c2e9a67 AK |
148 | if (stack_bucket_number_order) |
149 | entries = 1UL << stack_bucket_number_order; | |
df225c87 | 150 | pr_info("allocating hash table via alloc_large_system_hash\n"); |
f9987921 VB |
151 | stack_table = alloc_large_system_hash("stackdepot", |
152 | sizeof(struct stack_record *), | |
153 | entries, | |
4c2e9a67 | 154 | STACK_HASH_TABLE_SCALE, |
f9987921 VB |
155 | HASH_EARLY | HASH_ZERO, |
156 | NULL, | |
157 | &stack_hash_mask, | |
4c2e9a67 AK |
158 | 1UL << STACK_BUCKET_NUMBER_ORDER_MIN, |
159 | 1UL << STACK_BUCKET_NUMBER_ORDER_MAX); | |
a5f1783b | 160 | if (!stack_table) { |
4a6b5314 | 161 | pr_err("hash table allocation failed, disabling\n"); |
735df3c3 | 162 | stack_depot_disabled = true; |
a5f1783b VB |
163 | return -ENOMEM; |
164 | } | |
165 | ||
166 | return 0; | |
167 | } | |
168 | ||
df225c87 | 169 | /* Allocates a hash table via kvcalloc. Can be used after boot. */ |
a5f1783b | 170 | int stack_depot_init(void) |
e1fdc403 | 171 | { |
2dba5eb1 | 172 | static DEFINE_MUTEX(stack_depot_init_mutex); |
c60324fb | 173 | unsigned long entries; |
a5f1783b | 174 | int ret = 0; |
2dba5eb1 VB |
175 | |
176 | mutex_lock(&stack_depot_init_mutex); | |
f9987921 | 177 | |
c60324fb AK |
178 | if (stack_depot_disabled || stack_table) |
179 | goto out_unlock; | |
f9987921 | 180 | |
c60324fb | 181 | /* |
4c2e9a67 | 182 | * Similarly to stack_depot_early_init, use stack_bucket_number_order |
c60324fb AK |
183 | * if assigned, and rely on automatic scaling otherwise. |
184 | */ | |
4c2e9a67 AK |
185 | if (stack_bucket_number_order) { |
186 | entries = 1UL << stack_bucket_number_order; | |
c60324fb | 187 | } else { |
4c2e9a67 | 188 | int scale = STACK_HASH_TABLE_SCALE; |
c60324fb AK |
189 | |
190 | entries = nr_free_buffer_pages(); | |
191 | entries = roundup_pow_of_two(entries); | |
192 | ||
193 | if (scale > PAGE_SHIFT) | |
194 | entries >>= (scale - PAGE_SHIFT); | |
195 | else | |
196 | entries <<= (PAGE_SHIFT - scale); | |
e1fdc403 | 197 | } |
c60324fb | 198 | |
4c2e9a67 AK |
199 | if (entries < 1UL << STACK_BUCKET_NUMBER_ORDER_MIN) |
200 | entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MIN; | |
201 | if (entries > 1UL << STACK_BUCKET_NUMBER_ORDER_MAX) | |
202 | entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MAX; | |
c60324fb AK |
203 | |
204 | pr_info("allocating hash table of %lu entries via kvcalloc\n", entries); | |
205 | stack_table = kvcalloc(entries, sizeof(struct stack_record *), GFP_KERNEL); | |
206 | if (!stack_table) { | |
207 | pr_err("hash table allocation failed, disabling\n"); | |
208 | stack_depot_disabled = true; | |
209 | ret = -ENOMEM; | |
210 | goto out_unlock; | |
211 | } | |
212 | stack_hash_mask = entries - 1; | |
213 | ||
214 | out_unlock: | |
2dba5eb1 | 215 | mutex_unlock(&stack_depot_init_mutex); |
c60324fb | 216 | |
a5f1783b | 217 | return ret; |
e1fdc403 | 218 | } |
2dba5eb1 | 219 | EXPORT_SYMBOL_GPL(stack_depot_init); |
cd11016e | 220 | |
cd0fc64e | 221 | /* Uses preallocated memory to initialize a new stack depot pool. */ |
514d5c55 | 222 | static void depot_init_pool(void **prealloc) |
15ef6a98 | 223 | { |
15ef6a98 | 224 | /* |
cd0fc64e AK |
225 | * smp_load_acquire() here pairs with smp_store_release() below and |
226 | * in depot_alloc_stack(). | |
15ef6a98 | 227 | */ |
961c949b | 228 | if (smp_load_acquire(&next_pool_inited)) |
514d5c55 | 229 | return; |
cd0fc64e AK |
230 | |
231 | /* Check if the current pool is not yet allocated. */ | |
961c949b | 232 | if (stack_pools[pool_index] == NULL) { |
cd0fc64e | 233 | /* Use the preallocated memory for the current pool. */ |
961c949b | 234 | stack_pools[pool_index] = *prealloc; |
15ef6a98 AK |
235 | *prealloc = NULL; |
236 | } else { | |
cd0fc64e AK |
237 | /* |
238 | * Otherwise, use the preallocated memory for the next pool | |
239 | * as long as we do not exceed the maximum number of pools. | |
240 | */ | |
424cafee | 241 | if (pool_index + 1 < DEPOT_MAX_POOLS) { |
961c949b | 242 | stack_pools[pool_index + 1] = *prealloc; |
15ef6a98 AK |
243 | *prealloc = NULL; |
244 | } | |
245 | /* | |
cd0fc64e AK |
246 | * This smp_store_release pairs with smp_load_acquire() above |
247 | * and in stack_depot_save(). | |
15ef6a98 | 248 | */ |
961c949b | 249 | smp_store_release(&next_pool_inited, 1); |
15ef6a98 | 250 | } |
15ef6a98 AK |
251 | } |
252 | ||
cd0fc64e | 253 | /* Allocates a new stack in a stack depot pool. */ |
15ef6a98 AK |
254 | static struct stack_record * |
255 | depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **prealloc) | |
256 | { | |
257 | struct stack_record *stack; | |
258 | size_t required_size = struct_size(stack, entries, size); | |
259 | ||
424cafee | 260 | required_size = ALIGN(required_size, 1 << DEPOT_STACK_ALIGN); |
15ef6a98 | 261 | |
cd0fc64e | 262 | /* Check if there is not enough space in the current pool. */ |
424cafee | 263 | if (unlikely(pool_offset + required_size > DEPOT_POOL_SIZE)) { |
cd0fc64e | 264 | /* Bail out if we reached the pool limit. */ |
424cafee | 265 | if (unlikely(pool_index + 1 >= DEPOT_MAX_POOLS)) { |
15ef6a98 AK |
266 | WARN_ONCE(1, "Stack depot reached limit capacity"); |
267 | return NULL; | |
268 | } | |
cd0fc64e AK |
269 | |
270 | /* Move on to the next pool. */ | |
961c949b AK |
271 | pool_index++; |
272 | pool_offset = 0; | |
15ef6a98 | 273 | /* |
cd0fc64e AK |
274 | * smp_store_release() here pairs with smp_load_acquire() in |
275 | * stack_depot_save() and depot_init_pool(). | |
15ef6a98 | 276 | */ |
424cafee | 277 | if (pool_index + 1 < DEPOT_MAX_POOLS) |
961c949b | 278 | smp_store_release(&next_pool_inited, 0); |
15ef6a98 | 279 | } |
cd0fc64e AK |
280 | |
281 | /* Assign the preallocated memory to a pool if required. */ | |
514d5c55 AK |
282 | if (*prealloc) |
283 | depot_init_pool(prealloc); | |
cd0fc64e AK |
284 | |
285 | /* Check if we have a pool to save the stack trace. */ | |
961c949b | 286 | if (stack_pools[pool_index] == NULL) |
15ef6a98 AK |
287 | return NULL; |
288 | ||
cd0fc64e | 289 | /* Save the stack trace. */ |
961c949b | 290 | stack = stack_pools[pool_index] + pool_offset; |
15ef6a98 AK |
291 | stack->hash = hash; |
292 | stack->size = size; | |
961c949b | 293 | stack->handle.pool_index = pool_index; |
424cafee | 294 | stack->handle.offset = pool_offset >> DEPOT_STACK_ALIGN; |
15ef6a98 AK |
295 | stack->handle.valid = 1; |
296 | stack->handle.extra = 0; | |
297 | memcpy(stack->entries, entries, flex_array_size(stack, entries, size)); | |
961c949b | 298 | pool_offset += required_size; |
15ef6a98 AK |
299 | |
300 | return stack; | |
301 | } | |
302 | ||
cd11016e AP |
303 | /* Calculate hash for a stack */ |
304 | static inline u32 hash_stack(unsigned long *entries, unsigned int size) | |
305 | { | |
306 | return jhash2((u32 *)entries, | |
180644f8 GS |
307 | array_size(size, sizeof(*entries)) / sizeof(u32), |
308 | STACK_HASH_SEED); | |
cd11016e AP |
309 | } |
310 | ||
a571b272 AP |
311 | /* Use our own, non-instrumented version of memcmp(). |
312 | * | |
313 | * We actually don't care about the order, just the equality. | |
314 | */ | |
315 | static inline | |
316 | int stackdepot_memcmp(const unsigned long *u1, const unsigned long *u2, | |
317 | unsigned int n) | |
318 | { | |
319 | for ( ; n-- ; u1++, u2++) { | |
320 | if (*u1 != *u2) | |
321 | return 1; | |
322 | } | |
323 | return 0; | |
324 | } | |
325 | ||
cd11016e AP |
326 | /* Find a stack that is equal to the one stored in entries in the hash */ |
327 | static inline struct stack_record *find_stack(struct stack_record *bucket, | |
328 | unsigned long *entries, int size, | |
329 | u32 hash) | |
330 | { | |
331 | struct stack_record *found; | |
332 | ||
333 | for (found = bucket; found; found = found->next) { | |
334 | if (found->hash == hash && | |
335 | found->size == size && | |
a571b272 | 336 | !stackdepot_memcmp(entries, found->entries, size)) |
cd11016e | 337 | return found; |
cd11016e AP |
338 | } |
339 | return NULL; | |
340 | } | |
341 | ||
cd11016e | 342 | /** |
11ac25c6 | 343 | * __stack_depot_save - Save a stack trace from an array |
c0cfc337 TG |
344 | * |
345 | * @entries: Pointer to storage array | |
346 | * @nr_entries: Size of the storage array | |
83a4f1ef | 347 | * @extra_bits: Flags to store in unused bits of depot_stack_handle_t |
c0cfc337 | 348 | * @alloc_flags: Allocation gfp flags |
961c949b | 349 | * @can_alloc: Allocate stack pools (increased chance of failure if false) |
11ac25c6 ME |
350 | * |
351 | * Saves a stack trace from @entries array of size @nr_entries. If @can_alloc is | |
961c949b | 352 | * %true, is allowed to replenish the stack pool in case no space is left |
11ac25c6 ME |
353 | * (allocates using GFP flags of @alloc_flags). If @can_alloc is %false, avoids |
354 | * any allocations and will fail if no space is left to store the stack trace. | |
355 | * | |
e9400660 ME |
356 | * If the stack trace in @entries is from an interrupt, only the portion up to |
357 | * interrupt entry is saved. | |
358 | * | |
83a4f1ef AP |
359 | * Additional opaque flags can be passed in @extra_bits, stored in the unused |
360 | * bits of the stack handle, and retrieved using stack_depot_get_extra_bits() | |
361 | * without calling stack_depot_fetch(). | |
362 | * | |
11ac25c6 ME |
363 | * Context: Any context, but setting @can_alloc to %false is required if |
364 | * alloc_pages() cannot be used from the current context. Currently | |
365 | * this is the case from contexts where neither %GFP_ATOMIC nor | |
366 | * %GFP_NOWAIT can be used (NMI, raw_spin_lock). | |
cd11016e | 367 | * |
11ac25c6 | 368 | * Return: The handle of the stack struct stored in depot, 0 on failure. |
cd11016e | 369 | */ |
11ac25c6 ME |
370 | depot_stack_handle_t __stack_depot_save(unsigned long *entries, |
371 | unsigned int nr_entries, | |
83a4f1ef | 372 | unsigned int extra_bits, |
11ac25c6 | 373 | gfp_t alloc_flags, bool can_alloc) |
cd11016e | 374 | { |
cd11016e | 375 | struct stack_record *found = NULL, **bucket; |
83a4f1ef | 376 | union handle_parts retval = { .handle = 0 }; |
cd11016e AP |
377 | struct page *page = NULL; |
378 | void *prealloc = NULL; | |
c0cfc337 TG |
379 | unsigned long flags; |
380 | u32 hash; | |
cd11016e | 381 | |
e9400660 ME |
382 | /* |
383 | * If this stack trace is from an interrupt, including anything before | |
384 | * interrupt entry usually leads to unbounded stackdepot growth. | |
385 | * | |
386 | * Because use of filter_irq_stacks() is a requirement to ensure | |
387 | * stackdepot can efficiently deduplicate interrupt stacks, always | |
388 | * filter_irq_stacks() to simplify all callers' use of stackdepot. | |
389 | */ | |
390 | nr_entries = filter_irq_stacks(entries, nr_entries); | |
391 | ||
735df3c3 | 392 | if (unlikely(nr_entries == 0) || stack_depot_disabled) |
cd11016e AP |
393 | goto fast_exit; |
394 | ||
c0cfc337 | 395 | hash = hash_stack(entries, nr_entries); |
f9987921 | 396 | bucket = &stack_table[hash & stack_hash_mask]; |
cd11016e AP |
397 | |
398 | /* | |
399 | * Fast path: look the stack trace up without locking. | |
400 | * The smp_load_acquire() here pairs with smp_store_release() to | |
401 | * |bucket| below. | |
402 | */ | |
c0cfc337 TG |
403 | found = find_stack(smp_load_acquire(bucket), entries, |
404 | nr_entries, hash); | |
cd11016e AP |
405 | if (found) |
406 | goto exit; | |
407 | ||
408 | /* | |
961c949b | 409 | * Check if the current or the next stack pool need to be initialized. |
cd11016e AP |
410 | * If so, allocate the memory - we won't be able to do that under the |
411 | * lock. | |
412 | * | |
413 | * The smp_load_acquire() here pairs with smp_store_release() to | |
cb788e84 | 414 | * |next_pool_inited| in depot_alloc_stack() and depot_init_pool(). |
cd11016e | 415 | */ |
961c949b | 416 | if (unlikely(can_alloc && !smp_load_acquire(&next_pool_inited))) { |
cd11016e AP |
417 | /* |
418 | * Zero out zone modifiers, as we don't have specific zone | |
419 | * requirements. Keep the flags related to allocation in atomic | |
420 | * contexts and I/O. | |
421 | */ | |
422 | alloc_flags &= ~GFP_ZONEMASK; | |
423 | alloc_flags &= (GFP_ATOMIC | GFP_KERNEL); | |
87cc271d | 424 | alloc_flags |= __GFP_NOWARN; |
424cafee | 425 | page = alloc_pages(alloc_flags, DEPOT_POOL_ORDER); |
cd11016e AP |
426 | if (page) |
427 | prealloc = page_address(page); | |
428 | } | |
429 | ||
961c949b | 430 | raw_spin_lock_irqsave(&pool_lock, flags); |
cd11016e | 431 | |
c0cfc337 | 432 | found = find_stack(*bucket, entries, nr_entries, hash); |
cd11016e | 433 | if (!found) { |
7f2b8818 ME |
434 | struct stack_record *new = depot_alloc_stack(entries, nr_entries, hash, &prealloc); |
435 | ||
cd11016e AP |
436 | if (new) { |
437 | new->next = *bucket; | |
438 | /* | |
439 | * This smp_store_release() pairs with | |
440 | * smp_load_acquire() from |bucket| above. | |
441 | */ | |
442 | smp_store_release(bucket, new); | |
443 | found = new; | |
444 | } | |
445 | } else if (prealloc) { | |
446 | /* | |
447 | * We didn't need to store this stack trace, but let's keep | |
448 | * the preallocated memory for the future. | |
449 | */ | |
514d5c55 | 450 | depot_init_pool(&prealloc); |
cd11016e AP |
451 | } |
452 | ||
961c949b | 453 | raw_spin_unlock_irqrestore(&pool_lock, flags); |
cd11016e AP |
454 | exit: |
455 | if (prealloc) { | |
456 | /* Nobody used this memory, ok to free it. */ | |
424cafee | 457 | free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); |
cd11016e AP |
458 | } |
459 | if (found) | |
83a4f1ef | 460 | retval.handle = found->handle.handle; |
cd11016e | 461 | fast_exit: |
83a4f1ef AP |
462 | retval.extra = extra_bits; |
463 | ||
464 | return retval.handle; | |
cd11016e | 465 | } |
11ac25c6 ME |
466 | EXPORT_SYMBOL_GPL(__stack_depot_save); |
467 | ||
468 | /** | |
469 | * stack_depot_save - Save a stack trace from an array | |
470 | * | |
471 | * @entries: Pointer to storage array | |
472 | * @nr_entries: Size of the storage array | |
473 | * @alloc_flags: Allocation gfp flags | |
474 | * | |
475 | * Context: Contexts where allocations via alloc_pages() are allowed. | |
476 | * See __stack_depot_save() for more details. | |
477 | * | |
478 | * Return: The handle of the stack struct stored in depot, 0 on failure. | |
479 | */ | |
480 | depot_stack_handle_t stack_depot_save(unsigned long *entries, | |
481 | unsigned int nr_entries, | |
482 | gfp_t alloc_flags) | |
483 | { | |
83a4f1ef | 484 | return __stack_depot_save(entries, nr_entries, 0, alloc_flags, true); |
11ac25c6 | 485 | } |
c0cfc337 | 486 | EXPORT_SYMBOL_GPL(stack_depot_save); |
15ef6a98 AK |
487 | |
488 | /** | |
489 | * stack_depot_fetch - Fetch stack entries from a depot | |
490 | * | |
491 | * @handle: Stack depot handle which was returned from | |
492 | * stack_depot_save(). | |
493 | * @entries: Pointer to store the entries address | |
494 | * | |
495 | * Return: The number of trace entries for this depot. | |
496 | */ | |
497 | unsigned int stack_depot_fetch(depot_stack_handle_t handle, | |
498 | unsigned long **entries) | |
499 | { | |
500 | union handle_parts parts = { .handle = handle }; | |
961c949b | 501 | void *pool; |
424cafee | 502 | size_t offset = parts.offset << DEPOT_STACK_ALIGN; |
15ef6a98 AK |
503 | struct stack_record *stack; |
504 | ||
505 | *entries = NULL; | |
506 | if (!handle) | |
507 | return 0; | |
508 | ||
961c949b AK |
509 | if (parts.pool_index > pool_index) { |
510 | WARN(1, "pool index %d out of bounds (%d) for stack id %08x\n", | |
511 | parts.pool_index, pool_index, handle); | |
15ef6a98 AK |
512 | return 0; |
513 | } | |
961c949b AK |
514 | pool = stack_pools[parts.pool_index]; |
515 | if (!pool) | |
15ef6a98 | 516 | return 0; |
961c949b | 517 | stack = pool + offset; |
15ef6a98 AK |
518 | |
519 | *entries = stack->entries; | |
520 | return stack->size; | |
521 | } | |
522 | EXPORT_SYMBOL_GPL(stack_depot_fetch); | |
523 | ||
524 | /** | |
525 | * stack_depot_print - print stack entries from a depot | |
526 | * | |
527 | * @stack: Stack depot handle which was returned from | |
528 | * stack_depot_save(). | |
529 | * | |
530 | */ | |
531 | void stack_depot_print(depot_stack_handle_t stack) | |
532 | { | |
533 | unsigned long *entries; | |
534 | unsigned int nr_entries; | |
535 | ||
536 | nr_entries = stack_depot_fetch(stack, &entries); | |
537 | if (nr_entries > 0) | |
538 | stack_trace_print(entries, nr_entries, 0); | |
539 | } | |
540 | EXPORT_SYMBOL_GPL(stack_depot_print); | |
541 | ||
542 | /** | |
543 | * stack_depot_snprint - print stack entries from a depot into a buffer | |
544 | * | |
545 | * @handle: Stack depot handle which was returned from | |
546 | * stack_depot_save(). | |
547 | * @buf: Pointer to the print buffer | |
548 | * | |
549 | * @size: Size of the print buffer | |
550 | * | |
551 | * @spaces: Number of leading spaces to print | |
552 | * | |
553 | * Return: Number of bytes printed. | |
554 | */ | |
555 | int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, | |
556 | int spaces) | |
557 | { | |
558 | unsigned long *entries; | |
559 | unsigned int nr_entries; | |
560 | ||
561 | nr_entries = stack_depot_fetch(handle, &entries); | |
562 | return nr_entries ? stack_trace_snprint(buf, size, entries, nr_entries, | |
563 | spaces) : 0; | |
564 | } | |
565 | EXPORT_SYMBOL_GPL(stack_depot_snprint); | |
566 | ||
567 | unsigned int stack_depot_get_extra_bits(depot_stack_handle_t handle) | |
568 | { | |
569 | union handle_parts parts = { .handle = handle }; | |
570 | ||
571 | return parts.extra; | |
572 | } | |
573 | EXPORT_SYMBOL(stack_depot_get_extra_bits); |