1 // SPDX-License-Identifier: GPL-2.0
3 * This file contains core hardware tag-based KASAN code.
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
9 #define pr_fmt(fmt) "kasan: " fmt
11 #include <linux/init.h>
12 #include <linux/kasan.h>
13 #include <linux/kernel.h>
14 #include <linux/memory.h>
16 #include <linux/static_key.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
29 KASAN_ARG_MODE_DEFAULT,
35 enum kasan_arg_vmalloc {
36 KASAN_ARG_VMALLOC_DEFAULT,
37 KASAN_ARG_VMALLOC_OFF,
41 static enum kasan_arg kasan_arg __ro_after_init;
42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
43 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
46 * Whether KASAN is enabled at all.
47 * The value remains false until KASAN is initialized by kasan_init_hw_tags().
49 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
50 EXPORT_SYMBOL(kasan_flag_enabled);
53 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
54 * Defaults to KASAN_MODE_SYNC.
56 enum kasan_mode kasan_mode __ro_after_init;
57 EXPORT_SYMBOL_GPL(kasan_mode);
59 /* Whether to enable vmalloc tagging. */
60 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
62 #define PAGE_ALLOC_SAMPLE_DEFAULT 1
63 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3
66 * Sampling interval of page_alloc allocation (un)poisoning.
67 * Defaults to no sampling.
69 unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
72 * Minimum order of page_alloc allocations to be affected by sampling.
73 * The default value is chosen to match both
74 * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
76 unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
78 DEFINE_PER_CPU(long, kasan_page_alloc_skip);
81 static int __init early_kasan_flag(char *arg)
86 if (!strcmp(arg, "off"))
87 kasan_arg = KASAN_ARG_OFF;
88 else if (!strcmp(arg, "on"))
89 kasan_arg = KASAN_ARG_ON;
95 early_param("kasan", early_kasan_flag);
97 /* kasan.mode=sync/async/asymm */
98 static int __init early_kasan_mode(char *arg)
103 if (!strcmp(arg, "sync"))
104 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
105 else if (!strcmp(arg, "async"))
106 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
107 else if (!strcmp(arg, "asymm"))
108 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
114 early_param("kasan.mode", early_kasan_mode);
116 /* kasan.vmalloc=off/on */
117 static int __init early_kasan_flag_vmalloc(char *arg)
122 if (!strcmp(arg, "off"))
123 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
124 else if (!strcmp(arg, "on"))
125 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
131 early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
133 static inline const char *kasan_mode_info(void)
135 if (kasan_mode == KASAN_MODE_ASYNC)
137 else if (kasan_mode == KASAN_MODE_ASYMM)
143 /* kasan.page_alloc.sample=<sampling interval> */
144 static int __init early_kasan_flag_page_alloc_sample(char *arg)
151 rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
155 if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
156 kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
162 early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
164 /* kasan.page_alloc.sample.order=<minimum page order> */
165 static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
172 rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
176 if (kasan_page_alloc_sample_order > INT_MAX) {
177 kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
183 early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
186 * kasan_init_hw_tags_cpu() is called for each CPU.
187 * Not marked as __init as a CPU can be hot-plugged after boot.
189 void kasan_init_hw_tags_cpu(void)
192 * There's no need to check that the hardware is MTE-capable here,
193 * as this function is only called for MTE-capable hardware.
197 * If KASAN is disabled via command line, don't initialize it.
198 * When this function is called, kasan_flag_enabled is not yet
199 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
201 if (kasan_arg == KASAN_ARG_OFF)
205 * Enable async or asymm modes only when explicitly requested
206 * through the command line.
208 kasan_enable_hw_tags();
211 /* kasan_init_hw_tags() is called once on boot CPU. */
212 void __init kasan_init_hw_tags(void)
214 /* If hardware doesn't support MTE, don't initialize KASAN. */
215 if (!system_supports_mte())
218 /* If KASAN is disabled via command line, don't initialize it. */
219 if (kasan_arg == KASAN_ARG_OFF)
222 switch (kasan_arg_mode) {
223 case KASAN_ARG_MODE_DEFAULT:
224 /* Default is specified by kasan_mode definition. */
226 case KASAN_ARG_MODE_SYNC:
227 kasan_mode = KASAN_MODE_SYNC;
229 case KASAN_ARG_MODE_ASYNC:
230 kasan_mode = KASAN_MODE_ASYNC;
232 case KASAN_ARG_MODE_ASYMM:
233 kasan_mode = KASAN_MODE_ASYMM;
237 switch (kasan_arg_vmalloc) {
238 case KASAN_ARG_VMALLOC_DEFAULT:
239 /* Default is specified by kasan_flag_vmalloc definition. */
241 case KASAN_ARG_VMALLOC_OFF:
242 static_branch_disable(&kasan_flag_vmalloc);
244 case KASAN_ARG_VMALLOC_ON:
245 static_branch_enable(&kasan_flag_vmalloc);
251 /* KASAN is now initialized, enable it. */
252 static_branch_enable(&kasan_flag_enabled);
254 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
256 kasan_vmalloc_enabled() ? "on" : "off",
257 kasan_stack_collection_enabled() ? "on" : "off");
260 #ifdef CONFIG_KASAN_VMALLOC
262 static void unpoison_vmalloc_pages(const void *addr, u8 tag)
264 struct vm_struct *area;
268 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
269 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
270 * should belong to a single area.
272 area = find_vm_area((void *)addr);
276 for (i = 0; i < area->nr_pages; i++) {
277 struct page *page = area->pages[i];
279 page_kasan_tag_set(page, tag);
283 static void init_vmalloc_pages(const void *start, unsigned long size)
287 for (addr = start; addr < start + size; addr += PAGE_SIZE) {
288 struct page *page = virt_to_page(addr);
290 clear_highpage_kasan_tagged(page);
294 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
295 kasan_vmalloc_flags_t flags)
298 unsigned long redzone_start, redzone_size;
300 if (!kasan_vmalloc_enabled() || !is_vmalloc_or_module_addr(start)) {
301 if (flags & KASAN_VMALLOC_INIT)
302 init_vmalloc_pages(start, size);
303 return (void *)start;
307 * Don't tag non-VM_ALLOC mappings, as:
309 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
310 * supports tagging physical memory. Therefore, it can only tag a
311 * single mapping of normal physical pages.
312 * 2. Hardware tag-based KASAN can only tag memory mapped with special
313 * mapping protection bits, see arch_vmap_pgprot_tagged().
314 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
315 * providing these bits would require tracking all non-VM_ALLOC
318 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
319 * the first virtual mapping, which is created by vmalloc().
320 * Tagging the page_alloc memory backing that vmalloc() allocation is
321 * skipped, see ___GFP_SKIP_KASAN.
323 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
325 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
326 WARN_ON(flags & KASAN_VMALLOC_INIT);
327 return (void *)start;
331 * Don't tag executable memory.
332 * The kernel doesn't tolerate having the PC register tagged.
334 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
335 WARN_ON(flags & KASAN_VMALLOC_INIT);
336 return (void *)start;
339 tag = kasan_random_tag();
340 start = set_tag(start, tag);
342 /* Unpoison and initialize memory up to size. */
343 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
346 * Explicitly poison and initialize the in-page vmalloc() redzone.
347 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
348 * unpoison memory when populating shadow for vmalloc() space.
350 redzone_start = round_up((unsigned long)start + size,
352 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
353 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
354 flags & KASAN_VMALLOC_INIT);
357 * Set per-page tag flags to allow accessing physical memory for the
358 * vmalloc() mapping through page_address(vmalloc_to_page()).
360 unpoison_vmalloc_pages(start, tag);
362 return (void *)start;
365 void __kasan_poison_vmalloc(const void *start, unsigned long size)
369 * The physical pages backing the vmalloc() allocation are poisoned
370 * through the usual page_alloc paths.
376 void kasan_enable_hw_tags(void)
378 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
379 hw_enable_tag_checks_async();
380 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
381 hw_enable_tag_checks_asymm();
383 hw_enable_tag_checks_sync();
386 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
388 EXPORT_SYMBOL_GPL(kasan_enable_hw_tags);
390 void kasan_force_async_fault(void)
392 hw_force_async_tag_fault();
394 EXPORT_SYMBOL_GPL(kasan_force_async_fault);