Commit | Line | Data |
---|---|---|
dfd402a4 ME |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
178a1877 ME |
3 | #define pr_fmt(fmt) "kcsan: " fmt |
4 | ||
dfd402a4 ME |
5 | #include <linux/atomic.h> |
6 | #include <linux/bsearch.h> | |
7 | #include <linux/bug.h> | |
8 | #include <linux/debugfs.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/kallsyms.h> | |
a3120135 | 11 | #include <linux/sched.h> |
dfd402a4 ME |
12 | #include <linux/seq_file.h> |
13 | #include <linux/slab.h> | |
14 | #include <linux/sort.h> | |
15 | #include <linux/string.h> | |
16 | #include <linux/uaccess.h> | |
17 | ||
18 | #include "kcsan.h" | |
19 | ||
2e986b81 | 20 | atomic_long_t kcsan_counters[KCSAN_COUNTER_COUNT]; |
69b2c81b ME |
21 | static const char *const counter_names[] = { |
22 | [KCSAN_COUNTER_USED_WATCHPOINTS] = "used_watchpoints", | |
23 | [KCSAN_COUNTER_SETUP_WATCHPOINTS] = "setup_watchpoints", | |
24 | [KCSAN_COUNTER_DATA_RACES] = "data_races", | |
25 | [KCSAN_COUNTER_ASSERT_FAILURES] = "assert_failures", | |
26 | [KCSAN_COUNTER_NO_CAPACITY] = "no_capacity", | |
27 | [KCSAN_COUNTER_REPORT_RACES] = "report_races", | |
28 | [KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN] = "races_unknown_origin", | |
29 | [KCSAN_COUNTER_UNENCODABLE_ACCESSES] = "unencodable_accesses", | |
30 | [KCSAN_COUNTER_ENCODING_FALSE_POSITIVES] = "encoding_false_positives", | |
31 | }; | |
32 | static_assert(ARRAY_SIZE(counter_names) == KCSAN_COUNTER_COUNT); | |
dfd402a4 ME |
33 | |
34 | /* | |
35 | * Addresses for filtering functions from reporting. This list can be used as a | |
36 | * whitelist or blacklist. | |
37 | */ | |
38 | static struct { | |
5cbaefe9 IM |
39 | unsigned long *addrs; /* array of addresses */ |
40 | size_t size; /* current size */ | |
41 | int used; /* number of elements used */ | |
42 | bool sorted; /* if elements are sorted */ | |
43 | bool whitelist; /* if list is a blacklist or whitelist */ | |
dfd402a4 | 44 | } report_filterlist = { |
5cbaefe9 IM |
45 | .addrs = NULL, |
46 | .size = 8, /* small initial size */ | |
47 | .used = 0, | |
48 | .sorted = false, | |
49 | .whitelist = false, /* default is blacklist */ | |
dfd402a4 ME |
50 | }; |
51 | static DEFINE_SPINLOCK(report_filterlist_lock); | |
52 | ||
dfd402a4 ME |
53 | /* |
54 | * The microbenchmark allows benchmarking KCSAN core runtime only. To run | |
55 | * multiple threads, pipe 'microbench=<iters>' from multiple tasks into the | |
a3120135 | 56 | * debugfs file. This will not generate any conflicts, and tests fast-path only. |
dfd402a4 | 57 | */ |
a3120135 | 58 | static noinline void microbenchmark(unsigned long iters) |
dfd402a4 | 59 | { |
44656d3d ME |
60 | const struct kcsan_ctx ctx_save = current->kcsan_ctx; |
61 | const bool was_enabled = READ_ONCE(kcsan_enabled); | |
dfd402a4 ME |
62 | cycles_t cycles; |
63 | ||
44656d3d ME |
64 | /* We may have been called from an atomic region; reset context. */ |
65 | memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx)); | |
66 | /* | |
67 | * Disable to benchmark fast-path for all accesses, and (expected | |
68 | * negligible) call into slow-path, but never set up watchpoints. | |
69 | */ | |
70 | WRITE_ONCE(kcsan_enabled, false); | |
71 | ||
178a1877 | 72 | pr_info("%s begin | iters: %lu\n", __func__, iters); |
dfd402a4 ME |
73 | |
74 | cycles = get_cycles(); | |
75 | while (iters--) { | |
44656d3d ME |
76 | unsigned long addr = iters & ((PAGE_SIZE << 8) - 1); |
77 | int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC : | |
78 | (!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0); | |
79 | __kcsan_check_access((void *)addr, sizeof(long), type); | |
dfd402a4 ME |
80 | } |
81 | cycles = get_cycles() - cycles; | |
82 | ||
178a1877 | 83 | pr_info("%s end | cycles: %llu\n", __func__, cycles); |
44656d3d ME |
84 | |
85 | WRITE_ONCE(kcsan_enabled, was_enabled); | |
86 | /* restore context */ | |
87 | current->kcsan_ctx = ctx_save; | |
dfd402a4 ME |
88 | } |
89 | ||
90 | static int cmp_filterlist_addrs(const void *rhs, const void *lhs) | |
91 | { | |
92 | const unsigned long a = *(const unsigned long *)rhs; | |
93 | const unsigned long b = *(const unsigned long *)lhs; | |
94 | ||
95 | return a < b ? -1 : a == b ? 0 : 1; | |
96 | } | |
97 | ||
98 | bool kcsan_skip_report_debugfs(unsigned long func_addr) | |
99 | { | |
100 | unsigned long symbolsize, offset; | |
101 | unsigned long flags; | |
102 | bool ret = false; | |
103 | ||
104 | if (!kallsyms_lookup_size_offset(func_addr, &symbolsize, &offset)) | |
105 | return false; | |
5cbaefe9 | 106 | func_addr -= offset; /* Get function start */ |
dfd402a4 ME |
107 | |
108 | spin_lock_irqsave(&report_filterlist_lock, flags); | |
109 | if (report_filterlist.used == 0) | |
110 | goto out; | |
111 | ||
112 | /* Sort array if it is unsorted, and then do a binary search. */ | |
113 | if (!report_filterlist.sorted) { | |
114 | sort(report_filterlist.addrs, report_filterlist.used, | |
115 | sizeof(unsigned long), cmp_filterlist_addrs, NULL); | |
116 | report_filterlist.sorted = true; | |
117 | } | |
118 | ret = !!bsearch(&func_addr, report_filterlist.addrs, | |
119 | report_filterlist.used, sizeof(unsigned long), | |
120 | cmp_filterlist_addrs); | |
121 | if (report_filterlist.whitelist) | |
122 | ret = !ret; | |
123 | ||
124 | out: | |
125 | spin_unlock_irqrestore(&report_filterlist_lock, flags); | |
126 | return ret; | |
127 | } | |
128 | ||
129 | static void set_report_filterlist_whitelist(bool whitelist) | |
130 | { | |
131 | unsigned long flags; | |
132 | ||
133 | spin_lock_irqsave(&report_filterlist_lock, flags); | |
134 | report_filterlist.whitelist = whitelist; | |
135 | spin_unlock_irqrestore(&report_filterlist_lock, flags); | |
136 | } | |
137 | ||
138 | /* Returns 0 on success, error-code otherwise. */ | |
139 | static ssize_t insert_report_filterlist(const char *func) | |
140 | { | |
141 | unsigned long flags; | |
142 | unsigned long addr = kallsyms_lookup_name(func); | |
143 | ssize_t ret = 0; | |
144 | ||
145 | if (!addr) { | |
178a1877 | 146 | pr_err("could not find function: '%s'\n", func); |
dfd402a4 ME |
147 | return -ENOENT; |
148 | } | |
149 | ||
150 | spin_lock_irqsave(&report_filterlist_lock, flags); | |
151 | ||
152 | if (report_filterlist.addrs == NULL) { | |
153 | /* initial allocation */ | |
154 | report_filterlist.addrs = | |
155 | kmalloc_array(report_filterlist.size, | |
52785b6a | 156 | sizeof(unsigned long), GFP_ATOMIC); |
dfd402a4 ME |
157 | if (report_filterlist.addrs == NULL) { |
158 | ret = -ENOMEM; | |
159 | goto out; | |
160 | } | |
161 | } else if (report_filterlist.used == report_filterlist.size) { | |
162 | /* resize filterlist */ | |
163 | size_t new_size = report_filterlist.size * 2; | |
164 | unsigned long *new_addrs = | |
165 | krealloc(report_filterlist.addrs, | |
52785b6a | 166 | new_size * sizeof(unsigned long), GFP_ATOMIC); |
dfd402a4 ME |
167 | |
168 | if (new_addrs == NULL) { | |
169 | /* leave filterlist itself untouched */ | |
170 | ret = -ENOMEM; | |
171 | goto out; | |
172 | } | |
173 | ||
174 | report_filterlist.size = new_size; | |
175 | report_filterlist.addrs = new_addrs; | |
176 | } | |
177 | ||
178 | /* Note: deduplicating should be done in userspace. */ | |
179 | report_filterlist.addrs[report_filterlist.used++] = | |
180 | kallsyms_lookup_name(func); | |
181 | report_filterlist.sorted = false; | |
182 | ||
183 | out: | |
184 | spin_unlock_irqrestore(&report_filterlist_lock, flags); | |
5cbaefe9 | 185 | |
dfd402a4 ME |
186 | return ret; |
187 | } | |
188 | ||
189 | static int show_info(struct seq_file *file, void *v) | |
190 | { | |
191 | int i; | |
192 | unsigned long flags; | |
193 | ||
194 | /* show stats */ | |
195 | seq_printf(file, "enabled: %i\n", READ_ONCE(kcsan_enabled)); | |
2e986b81 ME |
196 | for (i = 0; i < KCSAN_COUNTER_COUNT; ++i) { |
197 | seq_printf(file, "%s: %ld\n", counter_names[i], | |
198 | atomic_long_read(&kcsan_counters[i])); | |
199 | } | |
dfd402a4 ME |
200 | |
201 | /* show filter functions, and filter type */ | |
202 | spin_lock_irqsave(&report_filterlist_lock, flags); | |
203 | seq_printf(file, "\n%s functions: %s\n", | |
204 | report_filterlist.whitelist ? "whitelisted" : "blacklisted", | |
205 | report_filterlist.used == 0 ? "none" : ""); | |
206 | for (i = 0; i < report_filterlist.used; ++i) | |
207 | seq_printf(file, " %ps\n", (void *)report_filterlist.addrs[i]); | |
208 | spin_unlock_irqrestore(&report_filterlist_lock, flags); | |
209 | ||
210 | return 0; | |
211 | } | |
212 | ||
213 | static int debugfs_open(struct inode *inode, struct file *file) | |
214 | { | |
215 | return single_open(file, show_info, NULL); | |
216 | } | |
217 | ||
5cbaefe9 IM |
218 | static ssize_t |
219 | debugfs_write(struct file *file, const char __user *buf, size_t count, loff_t *off) | |
dfd402a4 ME |
220 | { |
221 | char kbuf[KSYM_NAME_LEN]; | |
222 | char *arg; | |
223 | int read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1); | |
224 | ||
225 | if (copy_from_user(kbuf, buf, read_len)) | |
226 | return -EFAULT; | |
227 | kbuf[read_len] = '\0'; | |
228 | arg = strstrip(kbuf); | |
229 | ||
230 | if (!strcmp(arg, "on")) { | |
231 | WRITE_ONCE(kcsan_enabled, true); | |
232 | } else if (!strcmp(arg, "off")) { | |
233 | WRITE_ONCE(kcsan_enabled, false); | |
a4e74fa5 | 234 | } else if (str_has_prefix(arg, "microbench=")) { |
dfd402a4 ME |
235 | unsigned long iters; |
236 | ||
a4e74fa5 | 237 | if (kstrtoul(&arg[strlen("microbench=")], 0, &iters)) |
dfd402a4 ME |
238 | return -EINVAL; |
239 | microbenchmark(iters); | |
240 | } else if (!strcmp(arg, "whitelist")) { | |
241 | set_report_filterlist_whitelist(true); | |
242 | } else if (!strcmp(arg, "blacklist")) { | |
243 | set_report_filterlist_whitelist(false); | |
244 | } else if (arg[0] == '!') { | |
245 | ssize_t ret = insert_report_filterlist(&arg[1]); | |
246 | ||
247 | if (ret < 0) | |
248 | return ret; | |
249 | } else { | |
250 | return -EINVAL; | |
251 | } | |
252 | ||
253 | return count; | |
254 | } | |
255 | ||
5cbaefe9 IM |
256 | static const struct file_operations debugfs_ops = |
257 | { | |
258 | .read = seq_read, | |
259 | .open = debugfs_open, | |
260 | .write = debugfs_write, | |
261 | .release = single_release | |
262 | }; | |
dfd402a4 ME |
263 | |
264 | void __init kcsan_debugfs_init(void) | |
265 | { | |
266 | debugfs_create_file("kcsan", 0644, NULL, NULL, &debugfs_ops); | |
267 | } |