Commit | Line | Data |
---|---|---|
d08b9f0c ST |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Shadow Call Stack support. | |
4 | * | |
5 | * Copyright (C) 2019 Google LLC | |
6 | */ | |
7 | ||
8 | #include <linux/kasan.h> | |
628d06a4 | 9 | #include <linux/mm.h> |
d08b9f0c ST |
10 | #include <linux/scs.h> |
11 | #include <linux/slab.h> | |
628d06a4 | 12 | #include <linux/vmstat.h> |
d08b9f0c ST |
13 | #include <asm/scs.h> |
14 | ||
15 | static struct kmem_cache *scs_cache; | |
16 | ||
17 | static void *scs_alloc(int node) | |
18 | { | |
19 | void *s; | |
20 | ||
21 | s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node); | |
22 | if (s) { | |
23 | *__scs_magic(s) = SCS_END_MAGIC; | |
24 | /* | |
25 | * Poison the allocation to catch unintentional accesses to | |
26 | * the shadow stack when KASAN is enabled. | |
27 | */ | |
28 | kasan_poison_object_data(scs_cache, s); | |
29 | } | |
30 | ||
31 | return s; | |
32 | } | |
33 | ||
34 | static void scs_free(void *s) | |
35 | { | |
36 | kasan_unpoison_object_data(scs_cache, s); | |
37 | kmem_cache_free(scs_cache, s); | |
38 | } | |
39 | ||
40 | void __init scs_init(void) | |
41 | { | |
42 | scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL); | |
43 | } | |
44 | ||
628d06a4 ST |
45 | static struct page *__scs_page(struct task_struct *tsk) |
46 | { | |
47 | return virt_to_page(task_scs(tsk)); | |
48 | } | |
49 | ||
50 | static void scs_account(struct task_struct *tsk, int account) | |
51 | { | |
52 | mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_KB, | |
53 | account * (SCS_SIZE / 1024)); | |
54 | } | |
55 | ||
d08b9f0c ST |
56 | int scs_prepare(struct task_struct *tsk, int node) |
57 | { | |
58 | void *s = scs_alloc(node); | |
59 | ||
60 | if (!s) | |
61 | return -ENOMEM; | |
62 | ||
63 | task_scs(tsk) = s; | |
64 | task_scs_offset(tsk) = 0; | |
628d06a4 | 65 | scs_account(tsk, 1); |
d08b9f0c ST |
66 | return 0; |
67 | } | |
68 | ||
5bbaf9d1 ST |
69 | static void scs_check_usage(struct task_struct *tsk) |
70 | { | |
71 | static unsigned long highest; | |
72 | ||
73 | unsigned long *p, prev, curr = highest, used = 0; | |
74 | ||
75 | if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) | |
76 | return; | |
77 | ||
78 | for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { | |
79 | if (!READ_ONCE_NOCHECK(*p)) | |
80 | break; | |
81 | used++; | |
82 | } | |
83 | ||
84 | while (used > curr) { | |
85 | prev = cmpxchg_relaxed(&highest, curr, used); | |
86 | ||
87 | if (prev == curr) { | |
88 | pr_info("%s (%d): highest shadow stack usage: %lu bytes\n", | |
89 | tsk->comm, task_pid_nr(tsk), used); | |
90 | break; | |
91 | } | |
92 | ||
93 | curr = prev; | |
94 | } | |
95 | } | |
96 | ||
d08b9f0c ST |
97 | void scs_release(struct task_struct *tsk) |
98 | { | |
99 | void *s = task_scs(tsk); | |
100 | ||
101 | if (!s) | |
102 | return; | |
103 | ||
104 | WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n"); | |
5bbaf9d1 | 105 | scs_check_usage(tsk); |
628d06a4 | 106 | scs_account(tsk, -1); |
d08b9f0c ST |
107 | scs_free(s); |
108 | } |