Commit | Line | Data |
---|---|---|
0ce20dd8 AP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * KFENCE reporting. | |
4 | * | |
5 | * Copyright (C) 2020, Google LLC. | |
6 | */ | |
7 | ||
8 | #include <stdarg.h> | |
9 | ||
10 | #include <linux/kernel.h> | |
11 | #include <linux/lockdep.h> | |
12 | #include <linux/printk.h> | |
d438fabc | 13 | #include <linux/sched/debug.h> |
0ce20dd8 AP |
14 | #include <linux/seq_file.h> |
15 | #include <linux/stacktrace.h> | |
16 | #include <linux/string.h> | |
f2b84d2e | 17 | #include <trace/events/error_report.h> |
0ce20dd8 AP |
18 | |
19 | #include <asm/kfence.h> | |
20 | ||
21 | #include "kfence.h" | |
22 | ||
0aa41cae ME |
23 | /* May be overridden by <asm/kfence.h>. */ |
24 | #ifndef ARCH_FUNC_PREFIX | |
25 | #define ARCH_FUNC_PREFIX "" | |
26 | #endif | |
27 | ||
35beccf0 ME |
28 | extern bool no_hash_pointers; |
29 | ||
0ce20dd8 AP |
30 | /* Helper function to either print to a seq_file or to console. */ |
31 | __printf(2, 3) | |
32 | static void seq_con_printf(struct seq_file *seq, const char *fmt, ...) | |
33 | { | |
34 | va_list args; | |
35 | ||
36 | va_start(args, fmt); | |
37 | if (seq) | |
38 | seq_vprintf(seq, fmt, args); | |
39 | else | |
40 | vprintk(fmt, args); | |
41 | va_end(args); | |
42 | } | |
43 | ||
44 | /* | |
45 | * Get the number of stack entries to skip to get out of MM internals. @type is | |
46 | * optional, and if set to NULL, assumes an allocation or free stack. | |
47 | */ | |
48 | static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries, | |
49 | const enum kfence_error_type *type) | |
50 | { | |
51 | char buf[64]; | |
52 | int skipnr, fallback = 0; | |
0ce20dd8 AP |
53 | |
54 | if (type) { | |
55 | /* Depending on error type, find different stack entries. */ | |
56 | switch (*type) { | |
57 | case KFENCE_ERROR_UAF: | |
58 | case KFENCE_ERROR_OOB: | |
59 | case KFENCE_ERROR_INVALID: | |
d438fabc ME |
60 | /* |
61 | * kfence_handle_page_fault() may be called with pt_regs | |
62 | * set to NULL; in that case we'll simply show the full | |
63 | * stack trace. | |
64 | */ | |
65 | return 0; | |
0ce20dd8 AP |
66 | case KFENCE_ERROR_CORRUPTION: |
67 | case KFENCE_ERROR_INVALID_FREE: | |
68 | break; | |
69 | } | |
70 | } | |
71 | ||
72 | for (skipnr = 0; skipnr < num_entries; skipnr++) { | |
73 | int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]); | |
74 | ||
0aa41cae ME |
75 | if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") || |
76 | str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") || | |
77 | !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) { | |
d438fabc ME |
78 | /* |
79 | * In case of tail calls from any of the below | |
80 | * to any of the above. | |
81 | */ | |
82 | fallback = skipnr + 1; | |
0ce20dd8 | 83 | } |
d438fabc ME |
84 | |
85 | /* Also the *_bulk() variants by only checking prefixes. */ | |
0aa41cae ME |
86 | if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") || |
87 | str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") || | |
88 | str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") || | |
89 | str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc")) | |
d438fabc | 90 | goto found; |
0ce20dd8 AP |
91 | } |
92 | if (fallback < num_entries) | |
93 | return fallback; | |
94 | found: | |
95 | skipnr++; | |
96 | return skipnr < num_entries ? skipnr : 0; | |
97 | } | |
98 | ||
99 | static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta, | |
100 | bool show_alloc) | |
101 | { | |
102 | const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; | |
103 | ||
104 | if (track->num_stack_entries) { | |
105 | /* Skip allocation/free internals stack. */ | |
106 | int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); | |
107 | ||
108 | /* stack_trace_seq_print() does not exist; open code our own. */ | |
109 | for (; i < track->num_stack_entries; i++) | |
110 | seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]); | |
111 | } else { | |
112 | seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation"); | |
113 | } | |
114 | } | |
115 | ||
116 | void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) | |
117 | { | |
118 | const int size = abs(meta->size); | |
119 | const unsigned long start = meta->addr; | |
120 | const struct kmem_cache *const cache = meta->cache; | |
121 | ||
122 | lockdep_assert_held(&meta->lock); | |
123 | ||
124 | if (meta->state == KFENCE_OBJECT_UNUSED) { | |
702b16d7 | 125 | seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata); |
0ce20dd8 AP |
126 | return; |
127 | } | |
128 | ||
129 | seq_con_printf(seq, | |
702b16d7 | 130 | "kfence-#%td [0x%p-0x%p" |
0ce20dd8 AP |
131 | ", size=%d, cache=%s] allocated by task %d:\n", |
132 | meta - kfence_metadata, (void *)start, (void *)(start + size - 1), size, | |
133 | (cache && cache->name) ? cache->name : "<destroyed>", meta->alloc_track.pid); | |
134 | kfence_print_stack(seq, meta, true); | |
135 | ||
136 | if (meta->state == KFENCE_OBJECT_FREED) { | |
137 | seq_con_printf(seq, "\nfreed by task %d:\n", meta->free_track.pid); | |
138 | kfence_print_stack(seq, meta, false); | |
139 | } | |
140 | } | |
141 | ||
142 | /* | |
143 | * Show bytes at @addr that are different from the expected canary values, up to | |
144 | * @max_bytes. | |
145 | */ | |
146 | static void print_diff_canary(unsigned long address, size_t bytes_to_show, | |
147 | const struct kfence_metadata *meta) | |
148 | { | |
149 | const unsigned long show_until_addr = address + bytes_to_show; | |
150 | const u8 *cur, *end; | |
151 | ||
152 | /* Do not show contents of object nor read into following guard page. */ | |
153 | end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr) | |
154 | : min(show_until_addr, PAGE_ALIGN(address))); | |
155 | ||
156 | pr_cont("["); | |
157 | for (cur = (const u8 *)address; cur < end; cur++) { | |
158 | if (*cur == KFENCE_CANARY_PATTERN(cur)) | |
159 | pr_cont(" ."); | |
35beccf0 | 160 | else if (no_hash_pointers) |
0ce20dd8 AP |
161 | pr_cont(" 0x%02x", *cur); |
162 | else /* Do not leak kernel memory in non-debug builds. */ | |
163 | pr_cont(" !"); | |
164 | } | |
165 | pr_cont(" ]"); | |
166 | } | |
167 | ||
bc8fbc5f ME |
168 | static const char *get_access_type(bool is_write) |
169 | { | |
170 | return is_write ? "write" : "read"; | |
171 | } | |
172 | ||
173 | void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, | |
d438fabc | 174 | const struct kfence_metadata *meta, enum kfence_error_type type) |
0ce20dd8 AP |
175 | { |
176 | unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 }; | |
0ce20dd8 | 177 | const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1; |
d438fabc ME |
178 | int num_stack_entries; |
179 | int skipnr = 0; | |
180 | ||
181 | if (regs) { | |
182 | num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0); | |
183 | } else { | |
184 | num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1); | |
185 | skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type); | |
186 | } | |
0ce20dd8 AP |
187 | |
188 | /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */ | |
189 | if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta)) | |
190 | return; | |
191 | ||
192 | if (meta) | |
193 | lockdep_assert_held(&meta->lock); | |
194 | /* | |
195 | * Because we may generate reports in printk-unfriendly parts of the | |
196 | * kernel, such as scheduler code, the use of printk() could deadlock. | |
197 | * Until such time that all printing code here is safe in all parts of | |
198 | * the kernel, accept the risk, and just get our message out (given the | |
199 | * system might already behave unpredictably due to the memory error). | |
200 | * As such, also disable lockdep to hide warnings, and avoid disabling | |
201 | * lockdep for the rest of the kernel. | |
202 | */ | |
203 | lockdep_off(); | |
204 | ||
205 | pr_err("==================================================================\n"); | |
206 | /* Print report header. */ | |
207 | switch (type) { | |
208 | case KFENCE_ERROR_OOB: { | |
209 | const bool left_of_object = address < meta->addr; | |
210 | ||
bc8fbc5f ME |
211 | pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write), |
212 | (void *)stack_entries[skipnr]); | |
702b16d7 | 213 | pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n", |
bc8fbc5f | 214 | get_access_type(is_write), (void *)address, |
0ce20dd8 AP |
215 | left_of_object ? meta->addr - address : address - meta->addr, |
216 | left_of_object ? "left" : "right", object_index); | |
217 | break; | |
218 | } | |
219 | case KFENCE_ERROR_UAF: | |
bc8fbc5f ME |
220 | pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write), |
221 | (void *)stack_entries[skipnr]); | |
702b16d7 | 222 | pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n", |
bc8fbc5f | 223 | get_access_type(is_write), (void *)address, object_index); |
0ce20dd8 AP |
224 | break; |
225 | case KFENCE_ERROR_CORRUPTION: | |
226 | pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]); | |
35beccf0 | 227 | pr_err("Corrupted memory at 0x%p ", (void *)address); |
0ce20dd8 | 228 | print_diff_canary(address, 16, meta); |
702b16d7 | 229 | pr_cont(" (in kfence-#%td):\n", object_index); |
0ce20dd8 AP |
230 | break; |
231 | case KFENCE_ERROR_INVALID: | |
bc8fbc5f ME |
232 | pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write), |
233 | (void *)stack_entries[skipnr]); | |
35beccf0 | 234 | pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write), |
bc8fbc5f | 235 | (void *)address); |
0ce20dd8 AP |
236 | break; |
237 | case KFENCE_ERROR_INVALID_FREE: | |
238 | pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]); | |
702b16d7 | 239 | pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address, |
0ce20dd8 AP |
240 | object_index); |
241 | break; | |
242 | } | |
243 | ||
244 | /* Print stack trace and object info. */ | |
245 | stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0); | |
246 | ||
247 | if (meta) { | |
248 | pr_err("\n"); | |
249 | kfence_print_object(NULL, meta); | |
250 | } | |
251 | ||
252 | /* Print report footer. */ | |
253 | pr_err("\n"); | |
35beccf0 | 254 | if (no_hash_pointers && regs) |
d438fabc ME |
255 | show_regs(regs); |
256 | else | |
257 | dump_stack_print_info(KERN_ERR); | |
f2b84d2e | 258 | trace_error_report_end(ERROR_DETECTOR_KFENCE, address); |
0ce20dd8 AP |
259 | pr_err("==================================================================\n"); |
260 | ||
261 | lockdep_on(); | |
262 | ||
263 | if (panic_on_warn) | |
264 | panic("panic_on_warn set ...\n"); | |
265 | ||
266 | /* We encountered a memory unsafety error, taint the kernel! */ | |
267 | add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK); | |
268 | } |