Commit | Line | Data |
---|---|---|
a0503b8a KYL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * This file contains common tag-based KASAN code. | |
4 | * | |
5 | * Copyright (c) 2018 Google, Inc. | |
6 | * Copyright (c) 2020 Google, Inc. | |
7 | */ | |
8 | ||
7bc0584e | 9 | #include <linux/atomic.h> |
a0503b8a KYL |
10 | #include <linux/init.h> |
11 | #include <linux/kasan.h> | |
12 | #include <linux/kernel.h> | |
80b92bfe | 13 | #include <linux/memblock.h> |
a0503b8a KYL |
14 | #include <linux/memory.h> |
15 | #include <linux/mm.h> | |
16 | #include <linux/static_key.h> | |
17 | #include <linux/string.h> | |
18 | #include <linux/types.h> | |
19 | ||
20 | #include "kasan.h" | |
7bc0584e AK |
21 | #include "../slab.h" |
22 | ||
80b92bfe AK |
23 | #define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10) |
24 | ||
7ebfce33 AK |
25 | enum kasan_arg_stacktrace { |
26 | KASAN_ARG_STACKTRACE_DEFAULT, | |
27 | KASAN_ARG_STACKTRACE_OFF, | |
28 | KASAN_ARG_STACKTRACE_ON, | |
29 | }; | |
30 | ||
31 | static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; | |
32 | ||
33 | /* Whether to collect alloc/free stack traces. */ | |
34 | DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); | |
35 | ||
7bc0584e AK |
36 | /* Non-zero, as initial pointer values are 0. */ |
37 | #define STACK_RING_BUSY_PTR ((void *)1) | |
38 | ||
39 | struct kasan_stack_ring stack_ring = { | |
40 | .lock = __RW_LOCK_UNLOCKED(stack_ring.lock) | |
41 | }; | |
42 | ||
7ebfce33 AK |
43 | /* kasan.stacktrace=off/on */ |
44 | static int __init early_kasan_flag_stacktrace(char *arg) | |
45 | { | |
46 | if (!arg) | |
47 | return -EINVAL; | |
48 | ||
49 | if (!strcmp(arg, "off")) | |
50 | kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; | |
51 | else if (!strcmp(arg, "on")) | |
52 | kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; | |
53 | else | |
54 | return -EINVAL; | |
55 | ||
56 | return 0; | |
57 | } | |
58 | early_param("kasan.stacktrace", early_kasan_flag_stacktrace); | |
59 | ||
80b92bfe AK |
60 | /* kasan.stack_ring_size=<number of entries> */ |
61 | static int __init early_kasan_flag_stack_ring_size(char *arg) | |
62 | { | |
63 | if (!arg) | |
64 | return -EINVAL; | |
65 | ||
66 | return kstrtoul(arg, 0, &stack_ring.size); | |
67 | } | |
68 | early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size); | |
69 | ||
7ebfce33 AK |
70 | void __init kasan_init_tags(void) |
71 | { | |
72 | switch (kasan_arg_stacktrace) { | |
73 | case KASAN_ARG_STACKTRACE_DEFAULT: | |
74 | /* Default is specified by kasan_flag_stacktrace definition. */ | |
75 | break; | |
76 | case KASAN_ARG_STACKTRACE_OFF: | |
77 | static_branch_disable(&kasan_flag_stacktrace); | |
78 | break; | |
79 | case KASAN_ARG_STACKTRACE_ON: | |
80 | static_branch_enable(&kasan_flag_stacktrace); | |
81 | break; | |
82 | } | |
80b92bfe AK |
83 | |
84 | if (kasan_stack_collection_enabled()) { | |
85 | if (!stack_ring.size) | |
86 | stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT; | |
87 | stack_ring.entries = memblock_alloc( | |
88 | sizeof(stack_ring.entries[0]) * stack_ring.size, | |
89 | SMP_CACHE_BYTES); | |
90 | if (WARN_ON(!stack_ring.entries)) | |
91 | static_branch_disable(&kasan_flag_stacktrace); | |
92 | } | |
7ebfce33 AK |
93 | } |
94 | ||
7bc0584e AK |
95 | static void save_stack_info(struct kmem_cache *cache, void *object, |
96 | gfp_t gfp_flags, bool is_free) | |
97 | { | |
98 | unsigned long flags; | |
99 | depot_stack_handle_t stack; | |
100 | u64 pos; | |
101 | struct kasan_stack_ring_entry *entry; | |
102 | void *old_ptr; | |
103 | ||
104 | stack = kasan_save_stack(gfp_flags, true); | |
105 | ||
106 | /* | |
107 | * Prevent save_stack_info() from modifying stack ring | |
108 | * when kasan_complete_mode_report_info() is walking it. | |
109 | */ | |
110 | read_lock_irqsave(&stack_ring.lock, flags); | |
111 | ||
112 | next: | |
113 | pos = atomic64_fetch_add(1, &stack_ring.pos); | |
80b92bfe | 114 | entry = &stack_ring.entries[pos % stack_ring.size]; |
7bc0584e AK |
115 | |
116 | /* Detect stack ring entry slots that are being written to. */ | |
117 | old_ptr = READ_ONCE(entry->ptr); | |
118 | if (old_ptr == STACK_RING_BUSY_PTR) | |
119 | goto next; /* Busy slot. */ | |
120 | if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR)) | |
121 | goto next; /* Busy slot. */ | |
122 | ||
123 | WRITE_ONCE(entry->size, cache->object_size); | |
124 | WRITE_ONCE(entry->pid, current->pid); | |
125 | WRITE_ONCE(entry->stack, stack); | |
126 | WRITE_ONCE(entry->is_free, is_free); | |
127 | ||
128 | /* | |
129 | * Paired with smp_load_acquire() in kasan_complete_mode_report_info(). | |
130 | */ | |
131 | smp_store_release(&entry->ptr, (s64)object); | |
132 | ||
133 | read_unlock_irqrestore(&stack_ring.lock, flags); | |
134 | } | |
a0503b8a | 135 | |
ccf643e6 AK |
136 | void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags) |
137 | { | |
7bc0584e | 138 | save_stack_info(cache, object, flags, false); |
ccf643e6 AK |
139 | } |
140 | ||
6b074349 | 141 | void kasan_save_free_info(struct kmem_cache *cache, void *object) |
a0503b8a | 142 | { |
7bc0584e | 143 | save_stack_info(cache, object, GFP_NOWAIT, true); |
a0503b8a | 144 | } |