Commit | Line | Data |
---|---|---|
d08b9f0c ST |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Shadow Call Stack support. | |
4 | * | |
5 | * Copyright (C) 2019 Google LLC | |
6 | */ | |
7 | ||
8 | #include <linux/kasan.h> | |
628d06a4 | 9 | #include <linux/mm.h> |
d08b9f0c ST |
10 | #include <linux/scs.h> |
11 | #include <linux/slab.h> | |
628d06a4 | 12 | #include <linux/vmstat.h> |
d08b9f0c ST |
13 | |
14 | static struct kmem_cache *scs_cache; | |
15 | ||
bee348fa WD |
16 | static void __scs_account(void *s, int account) |
17 | { | |
18 | struct page *scs_page = virt_to_page(s); | |
19 | ||
20 | mod_zone_page_state(page_zone(scs_page), NR_KERNEL_SCS_KB, | |
21 | account * (SCS_SIZE / SZ_1K)); | |
22 | } | |
23 | ||
d08b9f0c ST |
24 | static void *scs_alloc(int node) |
25 | { | |
bee348fa WD |
26 | void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node); |
27 | ||
28 | if (!s) | |
29 | return NULL; | |
d08b9f0c | 30 | |
bee348fa WD |
31 | *__scs_magic(s) = SCS_END_MAGIC; |
32 | ||
33 | /* | |
34 | * Poison the allocation to catch unintentional accesses to | |
35 | * the shadow stack when KASAN is enabled. | |
36 | */ | |
37 | kasan_poison_object_data(scs_cache, s); | |
38 | __scs_account(s, 1); | |
d08b9f0c ST |
39 | return s; |
40 | } | |
41 | ||
42 | static void scs_free(void *s) | |
43 | { | |
bee348fa | 44 | __scs_account(s, -1); |
d08b9f0c ST |
45 | kasan_unpoison_object_data(scs_cache, s); |
46 | kmem_cache_free(scs_cache, s); | |
47 | } | |
48 | ||
49 | void __init scs_init(void) | |
50 | { | |
51 | scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL); | |
52 | } | |
53 | ||
54 | int scs_prepare(struct task_struct *tsk, int node) | |
55 | { | |
56 | void *s = scs_alloc(node); | |
57 | ||
58 | if (!s) | |
59 | return -ENOMEM; | |
60 | ||
51189c7a | 61 | task_scs(tsk) = task_scs_sp(tsk) = s; |
d08b9f0c ST |
62 | return 0; |
63 | } | |
64 | ||
5bbaf9d1 ST |
65 | static void scs_check_usage(struct task_struct *tsk) |
66 | { | |
67 | static unsigned long highest; | |
68 | ||
69 | unsigned long *p, prev, curr = highest, used = 0; | |
70 | ||
71 | if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) | |
72 | return; | |
73 | ||
74 | for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { | |
75 | if (!READ_ONCE_NOCHECK(*p)) | |
76 | break; | |
333ed746 | 77 | used += sizeof(*p); |
5bbaf9d1 ST |
78 | } |
79 | ||
80 | while (used > curr) { | |
81 | prev = cmpxchg_relaxed(&highest, curr, used); | |
82 | ||
83 | if (prev == curr) { | |
84 | pr_info("%s (%d): highest shadow stack usage: %lu bytes\n", | |
85 | tsk->comm, task_pid_nr(tsk), used); | |
86 | break; | |
87 | } | |
88 | ||
89 | curr = prev; | |
90 | } | |
91 | } | |
92 | ||
d08b9f0c ST |
93 | void scs_release(struct task_struct *tsk) |
94 | { | |
95 | void *s = task_scs(tsk); | |
96 | ||
97 | if (!s) | |
98 | return; | |
99 | ||
88485be5 WD |
100 | WARN(task_scs_end_corrupted(tsk), |
101 | "corrupted shadow stack detected when freeing task\n"); | |
5bbaf9d1 | 102 | scs_check_usage(tsk); |
d08b9f0c ST |
103 | scs_free(s); |
104 | } |