Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5c9a8750 DV |
2 | #define pr_fmt(fmt) "kcov: " fmt |
3 | ||
36f05ae8 | 4 | #define DISABLE_BRANCH_PROFILING |
db862358 | 5 | #include <linux/atomic.h> |
5c9a8750 | 6 | #include <linux/compiler.h> |
db862358 KW |
7 | #include <linux/errno.h> |
8 | #include <linux/export.h> | |
5c9a8750 DV |
9 | #include <linux/types.h> |
10 | #include <linux/file.h> | |
11 | #include <linux/fs.h> | |
eec028c9 | 12 | #include <linux/hashtable.h> |
db862358 | 13 | #include <linux/init.h> |
6cd0dd93 | 14 | #include <linux/jiffies.h> |
74d89909 | 15 | #include <linux/kmsan-checks.h> |
5c9a8750 | 16 | #include <linux/mm.h> |
db862358 | 17 | #include <linux/preempt.h> |
5c9a8750 | 18 | #include <linux/printk.h> |
166ad0e1 | 19 | #include <linux/sched.h> |
5c9a8750 DV |
20 | #include <linux/slab.h> |
21 | #include <linux/spinlock.h> | |
22 | #include <linux/vmalloc.h> | |
23 | #include <linux/debugfs.h> | |
24 | #include <linux/uaccess.h> | |
25 | #include <linux/kcov.h> | |
39e07cb6 | 26 | #include <linux/refcount.h> |
eec028c9 | 27 | #include <linux/log2.h> |
4983f0ab | 28 | #include <asm/setup.h> |
5c9a8750 | 29 | |
eec028c9 AK |
30 | #define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__) |
31 | ||
ded97d2c VC |
32 | /* Number of 64-bit words written per one comparison: */ |
33 | #define KCOV_WORDS_PER_CMP 4 | |
34 | ||
5c9a8750 DV |
35 | /* |
36 | * kcov descriptor (one per opened debugfs file). | |
37 | * State transitions of the descriptor: | |
38 | * - initial state after open() | |
39 | * - then there must be a single ioctl(KCOV_INIT_TRACE) call | |
40 | * - then, mmap() call (several calls are allowed but not useful) | |
ded97d2c VC |
41 | * - then, ioctl(KCOV_ENABLE, arg), where arg is |
42 | * KCOV_TRACE_PC - to trace only the PCs | |
43 | * or | |
44 | * KCOV_TRACE_CMP - to trace only the comparison operands | |
45 | * - then, ioctl(KCOV_DISABLE) to disable the task. | |
46 | * Enabling/disabling ioctls can be repeated (only one task a time allowed). | |
5c9a8750 DV |
47 | */ |
48 | struct kcov { | |
49 | /* | |
50 | * Reference counter. We keep one for: | |
51 | * - opened file descriptor | |
52 | * - task with enabled coverage (we can't unwire it from another task) | |
eec028c9 | 53 | * - each code section for remote coverage collection |
5c9a8750 | 54 | */ |
39e07cb6 | 55 | refcount_t refcount; |
5c9a8750 DV |
56 | /* The lock protects mode, size, area and t. */ |
57 | spinlock_t lock; | |
58 | enum kcov_mode mode; | |
eec028c9 AK |
59 | /* Size of arena (in long's). */ |
60 | unsigned int size; | |
5c9a8750 DV |
61 | /* Coverage buffer shared with user space. */ |
62 | void *area; | |
63 | /* Task for which we collect coverage, or NULL. */ | |
64 | struct task_struct *t; | |
eec028c9 AK |
65 | /* Collecting coverage from remote (background) threads. */ |
66 | bool remote; | |
67 | /* Size of remote area (in long's). */ | |
68 | unsigned int remote_size; | |
69 | /* | |
70 | * Sequence is incremented each time kcov is reenabled, used by | |
71 | * kcov_remote_stop(), see the comment there. | |
72 | */ | |
73 | int sequence; | |
5c9a8750 DV |
74 | }; |
75 | ||
eec028c9 AK |
76 | struct kcov_remote_area { |
77 | struct list_head list; | |
78 | unsigned int size; | |
79 | }; | |
80 | ||
81 | struct kcov_remote { | |
82 | u64 handle; | |
83 | struct kcov *kcov; | |
84 | struct hlist_node hnode; | |
85 | }; | |
86 | ||
87 | static DEFINE_SPINLOCK(kcov_remote_lock); | |
88 | static DEFINE_HASHTABLE(kcov_remote_map, 4); | |
89 | static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas); | |
90 | ||
5ff3b30a AK |
91 | struct kcov_percpu_data { |
92 | void *irq_area; | |
d5d2c51f | 93 | local_lock_t lock; |
5ff3b30a AK |
94 | |
95 | unsigned int saved_mode; | |
96 | unsigned int saved_size; | |
97 | void *saved_area; | |
98 | struct kcov *saved_kcov; | |
99 | int saved_sequence; | |
100 | }; | |
101 | ||
d5d2c51f SAS |
102 | static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data) = { |
103 | .lock = INIT_LOCAL_LOCK(lock), | |
104 | }; | |
5ff3b30a | 105 | |
eec028c9 AK |
106 | /* Must be called with kcov_remote_lock locked. */ |
107 | static struct kcov_remote *kcov_remote_find(u64 handle) | |
108 | { | |
109 | struct kcov_remote *remote; | |
110 | ||
111 | hash_for_each_possible(kcov_remote_map, remote, hnode, handle) { | |
112 | if (remote->handle == handle) | |
113 | return remote; | |
114 | } | |
115 | return NULL; | |
116 | } | |
117 | ||
3c61df38 | 118 | /* Must be called with kcov_remote_lock locked. */ |
eec028c9 AK |
119 | static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle) |
120 | { | |
121 | struct kcov_remote *remote; | |
122 | ||
123 | if (kcov_remote_find(handle)) | |
124 | return ERR_PTR(-EEXIST); | |
125 | remote = kmalloc(sizeof(*remote), GFP_ATOMIC); | |
126 | if (!remote) | |
127 | return ERR_PTR(-ENOMEM); | |
128 | remote->handle = handle; | |
129 | remote->kcov = kcov; | |
130 | hash_add(kcov_remote_map, &remote->hnode, handle); | |
131 | return remote; | |
132 | } | |
133 | ||
134 | /* Must be called with kcov_remote_lock locked. */ | |
135 | static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) | |
136 | { | |
137 | struct kcov_remote_area *area; | |
138 | struct list_head *pos; | |
139 | ||
eec028c9 AK |
140 | list_for_each(pos, &kcov_remote_areas) { |
141 | area = list_entry(pos, struct kcov_remote_area, list); | |
142 | if (area->size == size) { | |
143 | list_del(&area->list); | |
eec028c9 AK |
144 | return area; |
145 | } | |
146 | } | |
eec028c9 AK |
147 | return NULL; |
148 | } | |
149 | ||
150 | /* Must be called with kcov_remote_lock locked. */ | |
151 | static void kcov_remote_area_put(struct kcov_remote_area *area, | |
152 | unsigned int size) | |
153 | { | |
eec028c9 AK |
154 | INIT_LIST_HEAD(&area->list); |
155 | area->size = size; | |
156 | list_add(&area->list, &kcov_remote_areas); | |
74d89909 AP |
157 | /* |
158 | * KMSAN doesn't instrument this file, so it may not know area->list | |
159 | * is initialized. Unpoison it explicitly to avoid reports in | |
160 | * kcov_remote_area_get(). | |
161 | */ | |
162 | kmsan_unpoison_memory(&area->list, sizeof(area->list)); | |
eec028c9 AK |
163 | } |
164 | ||
7d4df2da AK |
165 | /* |
166 | * Unlike in_serving_softirq(), this function returns false when called during | |
167 | * a hardirq or an NMI that happened in the softirq context. | |
168 | */ | |
169 | static inline bool in_softirq_really(void) | |
170 | { | |
171 | return in_serving_softirq() && !in_hardirq() && !in_nmi(); | |
172 | } | |
173 | ||
903e8ff8 | 174 | static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) |
5c9a8750 | 175 | { |
0ed557aa | 176 | unsigned int mode; |
5c9a8750 | 177 | |
5c9a8750 DV |
178 | /* |
179 | * We are interested in code coverage as a function of a syscall inputs, | |
5ff3b30a AK |
180 | * so we ignore code executed in interrupts, unless we are in a remote |
181 | * coverage collection section in a softirq. | |
5c9a8750 | 182 | */ |
7d4df2da | 183 | if (!in_task() && !(in_softirq_really() && t->kcov_softirq)) |
ded97d2c | 184 | return false; |
5c9a8750 | 185 | mode = READ_ONCE(t->kcov_mode); |
ded97d2c VC |
186 | /* |
187 | * There is some code that runs in interrupts but for which | |
188 | * in_interrupt() returns false (e.g. preempt_schedule_irq()). | |
189 | * READ_ONCE()/barrier() effectively provides load-acquire wrt | |
190 | * interrupts, there are paired barrier()/WRITE_ONCE() in | |
eec028c9 | 191 | * kcov_start(). |
ded97d2c VC |
192 | */ |
193 | barrier(); | |
194 | return mode == needed_mode; | |
195 | } | |
4983f0ab | 196 | |
903e8ff8 | 197 | static notrace unsigned long canonicalize_ip(unsigned long ip) |
ded97d2c | 198 | { |
4983f0ab | 199 | #ifdef CONFIG_RANDOMIZE_BASE |
ded97d2c | 200 | ip -= kaslr_offset(); |
4983f0ab | 201 | #endif |
ded97d2c VC |
202 | return ip; |
203 | } | |
5c9a8750 | 204 | |
ded97d2c VC |
205 | /* |
206 | * Entry point from instrumented code. | |
207 | * This is called once per basic-block/edge. | |
208 | */ | |
209 | void notrace __sanitizer_cov_trace_pc(void) | |
210 | { | |
211 | struct task_struct *t; | |
212 | unsigned long *area; | |
213 | unsigned long ip = canonicalize_ip(_RET_IP_); | |
214 | unsigned long pos; | |
215 | ||
216 | t = current; | |
217 | if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t)) | |
218 | return; | |
219 | ||
220 | area = t->kcov_area; | |
221 | /* The first 64-bit word is the number of subsequent PCs. */ | |
222 | pos = READ_ONCE(area[0]) + 1; | |
223 | if (likely(pos < t->kcov_size)) { | |
3159d79b CL |
224 | /* Previously we write pc before updating pos. However, some |
225 | * early interrupt code could bypass check_kcov_mode() check | |
226 | * and invoke __sanitizer_cov_trace_pc(). If such interrupt is | |
227 | * raised between writing pc and updating pos, the pc could be | |
228 | * overitten by the recursive __sanitizer_cov_trace_pc(). | |
229 | * Update pos before writing pc to avoid such interleaving. | |
230 | */ | |
ded97d2c | 231 | WRITE_ONCE(area[0], pos); |
3159d79b CL |
232 | barrier(); |
233 | area[pos] = ip; | |
5c9a8750 DV |
234 | } |
235 | } | |
236 | EXPORT_SYMBOL(__sanitizer_cov_trace_pc); | |
237 | ||
ded97d2c | 238 | #ifdef CONFIG_KCOV_ENABLE_COMPARISONS |
63472443 | 239 | static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip) |
ded97d2c VC |
240 | { |
241 | struct task_struct *t; | |
242 | u64 *area; | |
243 | u64 count, start_index, end_pos, max_pos; | |
244 | ||
245 | t = current; | |
246 | if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t)) | |
247 | return; | |
248 | ||
249 | ip = canonicalize_ip(ip); | |
250 | ||
251 | /* | |
252 | * We write all comparison arguments and types as u64. | |
253 | * The buffer was allocated for t->kcov_size unsigned longs. | |
254 | */ | |
255 | area = (u64 *)t->kcov_area; | |
256 | max_pos = t->kcov_size * sizeof(unsigned long); | |
257 | ||
258 | count = READ_ONCE(area[0]); | |
259 | ||
260 | /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */ | |
261 | start_index = 1 + count * KCOV_WORDS_PER_CMP; | |
262 | end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64); | |
263 | if (likely(end_pos <= max_pos)) { | |
3159d79b CL |
264 | /* See comment in __sanitizer_cov_trace_pc(). */ |
265 | WRITE_ONCE(area[0], count + 1); | |
266 | barrier(); | |
ded97d2c VC |
267 | area[start_index] = type; |
268 | area[start_index + 1] = arg1; | |
269 | area[start_index + 2] = arg2; | |
270 | area[start_index + 3] = ip; | |
ded97d2c VC |
271 | } |
272 | } | |
273 | ||
274 | void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2) | |
275 | { | |
276 | write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_); | |
277 | } | |
278 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1); | |
279 | ||
280 | void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2) | |
281 | { | |
282 | write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_); | |
283 | } | |
284 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2); | |
285 | ||
689d77f0 | 286 | void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2) |
ded97d2c VC |
287 | { |
288 | write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_); | |
289 | } | |
290 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4); | |
291 | ||
e0ddec73 | 292 | void notrace __sanitizer_cov_trace_cmp8(kcov_u64 arg1, kcov_u64 arg2) |
ded97d2c VC |
293 | { |
294 | write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_); | |
295 | } | |
296 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8); | |
297 | ||
298 | void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2) | |
299 | { | |
300 | write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, | |
301 | _RET_IP_); | |
302 | } | |
303 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1); | |
304 | ||
305 | void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2) | |
306 | { | |
307 | write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, | |
308 | _RET_IP_); | |
309 | } | |
310 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2); | |
311 | ||
689d77f0 | 312 | void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2) |
ded97d2c VC |
313 | { |
314 | write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, | |
315 | _RET_IP_); | |
316 | } | |
317 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4); | |
318 | ||
e0ddec73 | 319 | void notrace __sanitizer_cov_trace_const_cmp8(kcov_u64 arg1, kcov_u64 arg2) |
ded97d2c VC |
320 | { |
321 | write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, | |
322 | _RET_IP_); | |
323 | } | |
324 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8); | |
325 | ||
e0ddec73 | 326 | void notrace __sanitizer_cov_trace_switch(kcov_u64 val, void *arg) |
ded97d2c VC |
327 | { |
328 | u64 i; | |
e0ddec73 | 329 | u64 *cases = arg; |
ded97d2c VC |
330 | u64 count = cases[0]; |
331 | u64 size = cases[1]; | |
332 | u64 type = KCOV_CMP_CONST; | |
333 | ||
334 | switch (size) { | |
335 | case 8: | |
336 | type |= KCOV_CMP_SIZE(0); | |
337 | break; | |
338 | case 16: | |
339 | type |= KCOV_CMP_SIZE(1); | |
340 | break; | |
341 | case 32: | |
342 | type |= KCOV_CMP_SIZE(2); | |
343 | break; | |
344 | case 64: | |
345 | type |= KCOV_CMP_SIZE(3); | |
346 | break; | |
347 | default: | |
348 | return; | |
349 | } | |
350 | for (i = 0; i < count; i++) | |
351 | write_comp_data(type, cases[i + 2], val, _RET_IP_); | |
352 | } | |
353 | EXPORT_SYMBOL(__sanitizer_cov_trace_switch); | |
354 | #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */ | |
355 | ||
76484b1c AK |
356 | static void kcov_start(struct task_struct *t, struct kcov *kcov, |
357 | unsigned int size, void *area, enum kcov_mode mode, | |
358 | int sequence) | |
eec028c9 AK |
359 | { |
360 | kcov_debug("t = %px, size = %u, area = %px\n", t, size, area); | |
76484b1c | 361 | t->kcov = kcov; |
eec028c9 AK |
362 | /* Cache in task struct for performance. */ |
363 | t->kcov_size = size; | |
364 | t->kcov_area = area; | |
eeb91f9a | 365 | t->kcov_sequence = sequence; |
eec028c9 AK |
366 | /* See comment in check_kcov_mode(). */ |
367 | barrier(); | |
368 | WRITE_ONCE(t->kcov_mode, mode); | |
eec028c9 AK |
369 | } |
370 | ||
371 | static void kcov_stop(struct task_struct *t) | |
372 | { | |
373 | WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); | |
374 | barrier(); | |
76484b1c | 375 | t->kcov = NULL; |
eec028c9 AK |
376 | t->kcov_size = 0; |
377 | t->kcov_area = NULL; | |
378 | } | |
379 | ||
380 | static void kcov_task_reset(struct task_struct *t) | |
381 | { | |
382 | kcov_stop(t); | |
eec028c9 AK |
383 | t->kcov_sequence = 0; |
384 | t->kcov_handle = 0; | |
385 | } | |
386 | ||
387 | void kcov_task_init(struct task_struct *t) | |
388 | { | |
389 | kcov_task_reset(t); | |
390 | t->kcov_handle = current->kcov_handle; | |
391 | } | |
392 | ||
393 | static void kcov_reset(struct kcov *kcov) | |
394 | { | |
395 | kcov->t = NULL; | |
396 | kcov->mode = KCOV_MODE_INIT; | |
397 | kcov->remote = false; | |
398 | kcov->remote_size = 0; | |
399 | kcov->sequence++; | |
400 | } | |
401 | ||
402 | static void kcov_remote_reset(struct kcov *kcov) | |
403 | { | |
404 | int bkt; | |
405 | struct kcov_remote *remote; | |
406 | struct hlist_node *tmp; | |
5ff3b30a | 407 | unsigned long flags; |
eec028c9 | 408 | |
5ff3b30a | 409 | spin_lock_irqsave(&kcov_remote_lock, flags); |
eec028c9 AK |
410 | hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) { |
411 | if (remote->kcov != kcov) | |
412 | continue; | |
eec028c9 AK |
413 | hash_del(&remote->hnode); |
414 | kfree(remote); | |
415 | } | |
416 | /* Do reset before unlock to prevent races with kcov_remote_start(). */ | |
417 | kcov_reset(kcov); | |
5ff3b30a | 418 | spin_unlock_irqrestore(&kcov_remote_lock, flags); |
eec028c9 AK |
419 | } |
420 | ||
421 | static void kcov_disable(struct task_struct *t, struct kcov *kcov) | |
422 | { | |
423 | kcov_task_reset(t); | |
424 | if (kcov->remote) | |
425 | kcov_remote_reset(kcov); | |
426 | else | |
427 | kcov_reset(kcov); | |
428 | } | |
429 | ||
5c9a8750 DV |
430 | static void kcov_get(struct kcov *kcov) |
431 | { | |
39e07cb6 | 432 | refcount_inc(&kcov->refcount); |
5c9a8750 DV |
433 | } |
434 | ||
435 | static void kcov_put(struct kcov *kcov) | |
436 | { | |
39e07cb6 | 437 | if (refcount_dec_and_test(&kcov->refcount)) { |
eec028c9 | 438 | kcov_remote_reset(kcov); |
5c9a8750 DV |
439 | vfree(kcov->area); |
440 | kfree(kcov); | |
441 | } | |
442 | } | |
443 | ||
5c9a8750 DV |
444 | void kcov_task_exit(struct task_struct *t) |
445 | { | |
446 | struct kcov *kcov; | |
5ff3b30a | 447 | unsigned long flags; |
5c9a8750 DV |
448 | |
449 | kcov = t->kcov; | |
450 | if (kcov == NULL) | |
451 | return; | |
eec028c9 | 452 | |
5ff3b30a | 453 | spin_lock_irqsave(&kcov->lock, flags); |
eec028c9 AK |
454 | kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); |
455 | /* | |
456 | * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t, | |
457 | * which comes down to: | |
458 | * WARN_ON(!kcov->remote && kcov->t != t); | |
459 | * | |
460 | * For KCOV_REMOTE_ENABLE devices, the exiting task is either: | |
3021e692 AK |
461 | * |
462 | * 1. A remote task between kcov_remote_start() and kcov_remote_stop(). | |
eec028c9 AK |
463 | * In this case we should print a warning right away, since a task |
464 | * shouldn't be exiting when it's in a kcov coverage collection | |
465 | * section. Here t points to the task that is collecting remote | |
466 | * coverage, and t->kcov->t points to the thread that created the | |
467 | * kcov device. Which means that to detect this case we need to | |
468 | * check that t != t->kcov->t, and this gives us the following: | |
469 | * WARN_ON(kcov->remote && kcov->t != t); | |
470 | * | |
471 | * 2. The task that created kcov exiting without calling KCOV_DISABLE, | |
3021e692 | 472 | * and then again we make sure that t->kcov->t == t: |
eec028c9 AK |
473 | * WARN_ON(kcov->remote && kcov->t != t); |
474 | * | |
475 | * By combining all three checks into one we get: | |
476 | */ | |
5c9a8750 | 477 | if (WARN_ON(kcov->t != t)) { |
5ff3b30a | 478 | spin_unlock_irqrestore(&kcov->lock, flags); |
5c9a8750 DV |
479 | return; |
480 | } | |
481 | /* Just to not leave dangling references behind. */ | |
eec028c9 | 482 | kcov_disable(t, kcov); |
5ff3b30a | 483 | spin_unlock_irqrestore(&kcov->lock, flags); |
5c9a8750 DV |
484 | kcov_put(kcov); |
485 | } | |
486 | ||
487 | static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) | |
488 | { | |
489 | int res = 0; | |
5c9a8750 DV |
490 | struct kcov *kcov = vma->vm_file->private_data; |
491 | unsigned long size, off; | |
492 | struct page *page; | |
5ff3b30a | 493 | unsigned long flags; |
5c9a8750 | 494 | |
5ff3b30a | 495 | spin_lock_irqsave(&kcov->lock, flags); |
5c9a8750 | 496 | size = kcov->size * sizeof(unsigned long); |
b3d7fe86 | 497 | if (kcov->area == NULL || vma->vm_pgoff != 0 || |
5c9a8750 DV |
498 | vma->vm_end - vma->vm_start != size) { |
499 | res = -EINVAL; | |
500 | goto exit; | |
501 | } | |
b3d7fe86 | 502 | spin_unlock_irqrestore(&kcov->lock, flags); |
1c71222e | 503 | vm_flags_set(vma, VM_DONTEXPAND); |
b3d7fe86 AN |
504 | for (off = 0; off < size; off += PAGE_SIZE) { |
505 | page = vmalloc_to_page(kcov->area + off); | |
ecc04463 AN |
506 | res = vm_insert_page(vma, vma->vm_start + off, page); |
507 | if (res) { | |
508 | pr_warn_once("kcov: vm_insert_page() failed\n"); | |
509 | return res; | |
510 | } | |
5c9a8750 | 511 | } |
b3d7fe86 | 512 | return 0; |
5c9a8750 | 513 | exit: |
5ff3b30a | 514 | spin_unlock_irqrestore(&kcov->lock, flags); |
5c9a8750 DV |
515 | return res; |
516 | } | |
517 | ||
518 | static int kcov_open(struct inode *inode, struct file *filep) | |
519 | { | |
520 | struct kcov *kcov; | |
521 | ||
522 | kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); | |
523 | if (!kcov) | |
524 | return -ENOMEM; | |
ded97d2c | 525 | kcov->mode = KCOV_MODE_DISABLED; |
eec028c9 | 526 | kcov->sequence = 1; |
39e07cb6 | 527 | refcount_set(&kcov->refcount, 1); |
5c9a8750 DV |
528 | spin_lock_init(&kcov->lock); |
529 | filep->private_data = kcov; | |
530 | return nonseekable_open(inode, filep); | |
531 | } | |
532 | ||
533 | static int kcov_close(struct inode *inode, struct file *filep) | |
534 | { | |
535 | kcov_put(filep->private_data); | |
536 | return 0; | |
537 | } | |
538 | ||
eec028c9 AK |
539 | static int kcov_get_mode(unsigned long arg) |
540 | { | |
541 | if (arg == KCOV_TRACE_PC) | |
542 | return KCOV_MODE_TRACE_PC; | |
543 | else if (arg == KCOV_TRACE_CMP) | |
544 | #ifdef CONFIG_KCOV_ENABLE_COMPARISONS | |
545 | return KCOV_MODE_TRACE_CMP; | |
546 | #else | |
547 | return -ENOTSUPP; | |
548 | #endif | |
549 | else | |
550 | return -EINVAL; | |
551 | } | |
552 | ||
dc55daff MR |
553 | /* |
554 | * Fault in a lazily-faulted vmalloc area before it can be used by | |
555 | * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the | |
556 | * vmalloc fault handling path is instrumented. | |
557 | */ | |
558 | static void kcov_fault_in_area(struct kcov *kcov) | |
559 | { | |
560 | unsigned long stride = PAGE_SIZE / sizeof(unsigned long); | |
561 | unsigned long *area = kcov->area; | |
562 | unsigned long offset; | |
563 | ||
564 | for (offset = 0; offset < kcov->size; offset += stride) | |
565 | READ_ONCE(area[offset]); | |
566 | } | |
567 | ||
eec028c9 AK |
568 | static inline bool kcov_check_handle(u64 handle, bool common_valid, |
569 | bool uncommon_valid, bool zero_valid) | |
570 | { | |
571 | if (handle & ~(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK)) | |
572 | return false; | |
573 | switch (handle & KCOV_SUBSYSTEM_MASK) { | |
574 | case KCOV_SUBSYSTEM_COMMON: | |
575 | return (handle & KCOV_INSTANCE_MASK) ? | |
576 | common_valid : zero_valid; | |
577 | case KCOV_SUBSYSTEM_USB: | |
578 | return uncommon_valid; | |
579 | default: | |
580 | return false; | |
581 | } | |
582 | return false; | |
583 | } | |
584 | ||
5c9a8750 DV |
585 | static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, |
586 | unsigned long arg) | |
587 | { | |
588 | struct task_struct *t; | |
17581aa1 | 589 | unsigned long flags, unused; |
eec028c9 AK |
590 | int mode, i; |
591 | struct kcov_remote_arg *remote_arg; | |
592 | struct kcov_remote *remote; | |
5c9a8750 DV |
593 | |
594 | switch (cmd) { | |
5c9a8750 DV |
595 | case KCOV_ENABLE: |
596 | /* | |
597 | * Enable coverage for the current task. | |
598 | * At this point user must have been enabled trace mode, | |
599 | * and mmapped the file. Coverage collection is disabled only | |
600 | * at task exit or voluntary by KCOV_DISABLE. After that it can | |
601 | * be enabled for another task. | |
602 | */ | |
ded97d2c | 603 | if (kcov->mode != KCOV_MODE_INIT || !kcov->area) |
5c9a8750 | 604 | return -EINVAL; |
a77660d2 DV |
605 | t = current; |
606 | if (kcov->t != NULL || t->kcov != NULL) | |
5c9a8750 | 607 | return -EBUSY; |
eec028c9 AK |
608 | mode = kcov_get_mode(arg); |
609 | if (mode < 0) | |
610 | return mode; | |
dc55daff | 611 | kcov_fault_in_area(kcov); |
eec028c9 | 612 | kcov->mode = mode; |
76484b1c | 613 | kcov_start(t, kcov, kcov->size, kcov->area, kcov->mode, |
eec028c9 | 614 | kcov->sequence); |
5c9a8750 | 615 | kcov->t = t; |
eec028c9 | 616 | /* Put either in kcov_task_exit() or in KCOV_DISABLE. */ |
5c9a8750 DV |
617 | kcov_get(kcov); |
618 | return 0; | |
619 | case KCOV_DISABLE: | |
620 | /* Disable coverage for the current task. */ | |
621 | unused = arg; | |
622 | if (unused != 0 || current->kcov != kcov) | |
623 | return -EINVAL; | |
624 | t = current; | |
625 | if (WARN_ON(kcov->t != t)) | |
626 | return -EINVAL; | |
eec028c9 | 627 | kcov_disable(t, kcov); |
5c9a8750 DV |
628 | kcov_put(kcov); |
629 | return 0; | |
eec028c9 | 630 | case KCOV_REMOTE_ENABLE: |
eec028c9 AK |
631 | if (kcov->mode != KCOV_MODE_INIT || !kcov->area) |
632 | return -EINVAL; | |
633 | t = current; | |
634 | if (kcov->t != NULL || t->kcov != NULL) | |
635 | return -EBUSY; | |
636 | remote_arg = (struct kcov_remote_arg *)arg; | |
637 | mode = kcov_get_mode(remote_arg->trace_mode); | |
638 | if (mode < 0) | |
639 | return mode; | |
56fd6162 AB |
640 | if ((unsigned long)remote_arg->area_size > |
641 | LONG_MAX / sizeof(unsigned long)) | |
eec028c9 AK |
642 | return -EINVAL; |
643 | kcov->mode = mode; | |
644 | t->kcov = kcov; | |
01c8f980 | 645 | t->kcov_mode = KCOV_MODE_REMOTE; |
eec028c9 AK |
646 | kcov->t = t; |
647 | kcov->remote = true; | |
648 | kcov->remote_size = remote_arg->area_size; | |
5ff3b30a | 649 | spin_lock_irqsave(&kcov_remote_lock, flags); |
eec028c9 | 650 | for (i = 0; i < remote_arg->num_handles; i++) { |
eec028c9 AK |
651 | if (!kcov_check_handle(remote_arg->handles[i], |
652 | false, true, false)) { | |
5ff3b30a AK |
653 | spin_unlock_irqrestore(&kcov_remote_lock, |
654 | flags); | |
eec028c9 AK |
655 | kcov_disable(t, kcov); |
656 | return -EINVAL; | |
657 | } | |
658 | remote = kcov_remote_add(kcov, remote_arg->handles[i]); | |
659 | if (IS_ERR(remote)) { | |
5ff3b30a AK |
660 | spin_unlock_irqrestore(&kcov_remote_lock, |
661 | flags); | |
eec028c9 AK |
662 | kcov_disable(t, kcov); |
663 | return PTR_ERR(remote); | |
664 | } | |
665 | } | |
666 | if (remote_arg->common_handle) { | |
eec028c9 AK |
667 | if (!kcov_check_handle(remote_arg->common_handle, |
668 | true, false, false)) { | |
5ff3b30a AK |
669 | spin_unlock_irqrestore(&kcov_remote_lock, |
670 | flags); | |
eec028c9 AK |
671 | kcov_disable(t, kcov); |
672 | return -EINVAL; | |
673 | } | |
674 | remote = kcov_remote_add(kcov, | |
675 | remote_arg->common_handle); | |
676 | if (IS_ERR(remote)) { | |
5ff3b30a AK |
677 | spin_unlock_irqrestore(&kcov_remote_lock, |
678 | flags); | |
eec028c9 AK |
679 | kcov_disable(t, kcov); |
680 | return PTR_ERR(remote); | |
681 | } | |
682 | t->kcov_handle = remote_arg->common_handle; | |
683 | } | |
5ff3b30a | 684 | spin_unlock_irqrestore(&kcov_remote_lock, flags); |
eec028c9 AK |
685 | /* Put either in kcov_task_exit() or in KCOV_DISABLE. */ |
686 | kcov_get(kcov); | |
687 | return 0; | |
5c9a8750 DV |
688 | default: |
689 | return -ENOTTY; | |
690 | } | |
691 | } | |
692 | ||
693 | static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | |
694 | { | |
695 | struct kcov *kcov; | |
696 | int res; | |
eec028c9 AK |
697 | struct kcov_remote_arg *remote_arg = NULL; |
698 | unsigned int remote_num_handles; | |
699 | unsigned long remote_arg_size; | |
17581aa1 | 700 | unsigned long size, flags; |
b3d7fe86 | 701 | void *area; |
eec028c9 | 702 | |
17581aa1 AN |
703 | kcov = filep->private_data; |
704 | switch (cmd) { | |
705 | case KCOV_INIT_TRACE: | |
706 | /* | |
707 | * Enable kcov in trace mode and setup buffer size. | |
708 | * Must happen before anything else. | |
709 | * | |
710 | * First check the size argument - it must be at least 2 | |
b3d7fe86 | 711 | * to hold the current position and one PC. |
17581aa1 AN |
712 | */ |
713 | size = arg; | |
714 | if (size < 2 || size > INT_MAX / sizeof(unsigned long)) | |
715 | return -EINVAL; | |
b3d7fe86 AN |
716 | area = vmalloc_user(size * sizeof(unsigned long)); |
717 | if (area == NULL) | |
718 | return -ENOMEM; | |
17581aa1 AN |
719 | spin_lock_irqsave(&kcov->lock, flags); |
720 | if (kcov->mode != KCOV_MODE_DISABLED) { | |
721 | spin_unlock_irqrestore(&kcov->lock, flags); | |
b3d7fe86 | 722 | vfree(area); |
17581aa1 AN |
723 | return -EBUSY; |
724 | } | |
b3d7fe86 | 725 | kcov->area = area; |
17581aa1 AN |
726 | kcov->size = size; |
727 | kcov->mode = KCOV_MODE_INIT; | |
728 | spin_unlock_irqrestore(&kcov->lock, flags); | |
729 | return 0; | |
730 | case KCOV_REMOTE_ENABLE: | |
eec028c9 AK |
731 | if (get_user(remote_num_handles, (unsigned __user *)(arg + |
732 | offsetof(struct kcov_remote_arg, num_handles)))) | |
733 | return -EFAULT; | |
734 | if (remote_num_handles > KCOV_REMOTE_MAX_HANDLES) | |
735 | return -EINVAL; | |
736 | remote_arg_size = struct_size(remote_arg, handles, | |
737 | remote_num_handles); | |
738 | remote_arg = memdup_user((void __user *)arg, remote_arg_size); | |
739 | if (IS_ERR(remote_arg)) | |
740 | return PTR_ERR(remote_arg); | |
741 | if (remote_arg->num_handles != remote_num_handles) { | |
742 | kfree(remote_arg); | |
743 | return -EINVAL; | |
744 | } | |
745 | arg = (unsigned long)remote_arg; | |
17581aa1 AN |
746 | fallthrough; |
747 | default: | |
748 | /* | |
749 | * All other commands can be normally executed under a spin lock, so we | |
750 | * obtain and release it here in order to simplify kcov_ioctl_locked(). | |
751 | */ | |
752 | spin_lock_irqsave(&kcov->lock, flags); | |
753 | res = kcov_ioctl_locked(kcov, cmd, arg); | |
754 | spin_unlock_irqrestore(&kcov->lock, flags); | |
755 | kfree(remote_arg); | |
756 | return res; | |
eec028c9 | 757 | } |
5c9a8750 DV |
758 | } |
759 | ||
760 | static const struct file_operations kcov_fops = { | |
761 | .open = kcov_open, | |
762 | .unlocked_ioctl = kcov_ioctl, | |
7483e5d4 | 763 | .compat_ioctl = kcov_ioctl, |
5c9a8750 DV |
764 | .mmap = kcov_mmap, |
765 | .release = kcov_close, | |
766 | }; | |
767 | ||
eec028c9 AK |
768 | /* |
769 | * kcov_remote_start() and kcov_remote_stop() can be used to annotate a section | |
5ff3b30a AK |
770 | * of code in a kernel background thread or in a softirq to allow kcov to be |
771 | * used to collect coverage from that part of code. | |
eec028c9 AK |
772 | * |
773 | * The handle argument of kcov_remote_start() identifies a code section that is | |
774 | * used for coverage collection. A userspace process passes this handle to | |
775 | * KCOV_REMOTE_ENABLE ioctl to make the used kcov device start collecting | |
776 | * coverage for the code section identified by this handle. | |
777 | * | |
778 | * The usage of these annotations in the kernel code is different depending on | |
779 | * the type of the kernel thread whose code is being annotated. | |
780 | * | |
781 | * For global kernel threads that are spawned in a limited number of instances | |
5ff3b30a AK |
782 | * (e.g. one USB hub_event() worker thread is spawned per USB HCD) and for |
783 | * softirqs, each instance must be assigned a unique 4-byte instance id. The | |
784 | * instance id is then combined with a 1-byte subsystem id to get a handle via | |
eec028c9 AK |
785 | * kcov_remote_handle(subsystem_id, instance_id). |
786 | * | |
787 | * For local kernel threads that are spawned from system calls handler when a | |
788 | * user interacts with some kernel interface (e.g. vhost workers), a handle is | |
789 | * passed from a userspace process as the common_handle field of the | |
790 | * kcov_remote_arg struct (note, that the user must generate a handle by using | |
791 | * kcov_remote_handle() with KCOV_SUBSYSTEM_COMMON as the subsystem id and an | |
792 | * arbitrary 4-byte non-zero number as the instance id). This common handle | |
793 | * then gets saved into the task_struct of the process that issued the | |
324cfb19 MG |
794 | * KCOV_REMOTE_ENABLE ioctl. When this process issues system calls that spawn |
795 | * kernel threads, the common handle must be retrieved via kcov_common_handle() | |
eec028c9 AK |
796 | * and passed to the spawned threads via custom annotations. Those kernel |
797 | * threads must in turn be annotated with kcov_remote_start(common_handle) and | |
798 | * kcov_remote_stop(). All of the threads that are spawned by the same process | |
799 | * obtain the same handle, hence the name "common". | |
800 | * | |
801 | * See Documentation/dev-tools/kcov.rst for more details. | |
802 | * | |
5ff3b30a | 803 | * Internally, kcov_remote_start() looks up the kcov device associated with the |
eec028c9 AK |
804 | * provided handle, allocates an area for coverage collection, and saves the |
805 | * pointers to kcov and area into the current task_struct to allow coverage to | |
3021e692 | 806 | * be collected via __sanitizer_cov_trace_pc(). |
eec028c9 AK |
807 | * In turns kcov_remote_stop() clears those pointers from task_struct to stop |
808 | * collecting coverage and copies all collected coverage into the kcov area. | |
809 | */ | |
5fe7042d AK |
810 | |
811 | static inline bool kcov_mode_enabled(unsigned int mode) | |
812 | { | |
813 | return (mode & ~KCOV_IN_CTXSW) != KCOV_MODE_DISABLED; | |
814 | } | |
815 | ||
fed79d05 | 816 | static void kcov_remote_softirq_start(struct task_struct *t) |
5ff3b30a AK |
817 | { |
818 | struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); | |
819 | unsigned int mode; | |
820 | ||
821 | mode = READ_ONCE(t->kcov_mode); | |
822 | barrier(); | |
823 | if (kcov_mode_enabled(mode)) { | |
824 | data->saved_mode = mode; | |
825 | data->saved_size = t->kcov_size; | |
826 | data->saved_area = t->kcov_area; | |
827 | data->saved_sequence = t->kcov_sequence; | |
828 | data->saved_kcov = t->kcov; | |
829 | kcov_stop(t); | |
830 | } | |
831 | } | |
832 | ||
fed79d05 | 833 | static void kcov_remote_softirq_stop(struct task_struct *t) |
5ff3b30a AK |
834 | { |
835 | struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); | |
836 | ||
837 | if (data->saved_kcov) { | |
838 | kcov_start(t, data->saved_kcov, data->saved_size, | |
839 | data->saved_area, data->saved_mode, | |
840 | data->saved_sequence); | |
841 | data->saved_mode = 0; | |
842 | data->saved_size = 0; | |
843 | data->saved_area = NULL; | |
844 | data->saved_sequence = 0; | |
845 | data->saved_kcov = NULL; | |
846 | } | |
847 | } | |
848 | ||
eec028c9 AK |
849 | void kcov_remote_start(u64 handle) |
850 | { | |
5fe7042d | 851 | struct task_struct *t = current; |
eec028c9 | 852 | struct kcov_remote *remote; |
67b3d3cc | 853 | struct kcov *kcov; |
5fe7042d | 854 | unsigned int mode; |
eec028c9 | 855 | void *area; |
eec028c9 | 856 | unsigned int size; |
eec028c9 | 857 | int sequence; |
5ff3b30a | 858 | unsigned long flags; |
eec028c9 AK |
859 | |
860 | if (WARN_ON(!kcov_check_handle(handle, true, true, true))) | |
861 | return; | |
7d4df2da | 862 | if (!in_task() && !in_softirq_really()) |
eec028c9 | 863 | return; |
5fe7042d | 864 | |
d5d2c51f | 865 | local_lock_irqsave(&kcov_percpu_data.lock, flags); |
5ff3b30a | 866 | |
eec028c9 | 867 | /* |
5ff3b30a AK |
868 | * Check that kcov_remote_start() is not called twice in background |
869 | * threads nor called by user tasks (with enabled kcov). | |
eec028c9 | 870 | */ |
5fe7042d | 871 | mode = READ_ONCE(t->kcov_mode); |
5ff3b30a | 872 | if (WARN_ON(in_task() && kcov_mode_enabled(mode))) { |
d5d2c51f | 873 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
eec028c9 | 874 | return; |
5ff3b30a AK |
875 | } |
876 | /* | |
877 | * Check that kcov_remote_start() is not called twice in softirqs. | |
878 | * Note, that kcov_remote_start() can be called from a softirq that | |
879 | * happened while collecting coverage from a background thread. | |
880 | */ | |
881 | if (WARN_ON(in_serving_softirq() && t->kcov_softirq)) { | |
d5d2c51f | 882 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
5ff3b30a AK |
883 | return; |
884 | } | |
eec028c9 AK |
885 | |
886 | spin_lock(&kcov_remote_lock); | |
887 | remote = kcov_remote_find(handle); | |
888 | if (!remote) { | |
d5d2c51f SAS |
889 | spin_unlock(&kcov_remote_lock); |
890 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); | |
eec028c9 AK |
891 | return; |
892 | } | |
5ff3b30a AK |
893 | kcov_debug("handle = %llx, context: %s\n", handle, |
894 | in_task() ? "task" : "softirq"); | |
67b3d3cc | 895 | kcov = remote->kcov; |
eec028c9 | 896 | /* Put in kcov_remote_stop(). */ |
67b3d3cc | 897 | kcov_get(kcov); |
eec028c9 AK |
898 | /* |
899 | * Read kcov fields before unlock to prevent races with | |
900 | * KCOV_DISABLE / kcov_remote_reset(). | |
901 | */ | |
67b3d3cc AK |
902 | mode = kcov->mode; |
903 | sequence = kcov->sequence; | |
5ff3b30a AK |
904 | if (in_task()) { |
905 | size = kcov->remote_size; | |
906 | area = kcov_remote_area_get(size); | |
907 | } else { | |
908 | size = CONFIG_KCOV_IRQ_AREA_SIZE; | |
909 | area = this_cpu_ptr(&kcov_percpu_data)->irq_area; | |
910 | } | |
22036abe | 911 | spin_unlock(&kcov_remote_lock); |
eec028c9 | 912 | |
5ff3b30a | 913 | /* Can only happen when in_task(). */ |
eec028c9 | 914 | if (!area) { |
d5d2c51f | 915 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
eec028c9 AK |
916 | area = vmalloc(size * sizeof(unsigned long)); |
917 | if (!area) { | |
67b3d3cc | 918 | kcov_put(kcov); |
eec028c9 AK |
919 | return; |
920 | } | |
d5d2c51f | 921 | local_lock_irqsave(&kcov_percpu_data.lock, flags); |
eec028c9 | 922 | } |
5ff3b30a | 923 | |
eec028c9 AK |
924 | /* Reset coverage size. */ |
925 | *(u64 *)area = 0; | |
926 | ||
5ff3b30a AK |
927 | if (in_serving_softirq()) { |
928 | kcov_remote_softirq_start(t); | |
929 | t->kcov_softirq = 1; | |
930 | } | |
76484b1c | 931 | kcov_start(t, kcov, size, area, mode, sequence); |
eec028c9 | 932 | |
d5d2c51f | 933 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
5ff3b30a | 934 | |
eec028c9 AK |
935 | } |
936 | EXPORT_SYMBOL(kcov_remote_start); | |
937 | ||
938 | static void kcov_move_area(enum kcov_mode mode, void *dst_area, | |
939 | unsigned int dst_area_size, void *src_area) | |
940 | { | |
941 | u64 word_size = sizeof(unsigned long); | |
942 | u64 count_size, entry_size_log; | |
943 | u64 dst_len, src_len; | |
944 | void *dst_entries, *src_entries; | |
945 | u64 dst_occupied, dst_free, bytes_to_move, entries_moved; | |
946 | ||
947 | kcov_debug("%px %u <= %px %lu\n", | |
948 | dst_area, dst_area_size, src_area, *(unsigned long *)src_area); | |
949 | ||
950 | switch (mode) { | |
951 | case KCOV_MODE_TRACE_PC: | |
952 | dst_len = READ_ONCE(*(unsigned long *)dst_area); | |
953 | src_len = *(unsigned long *)src_area; | |
954 | count_size = sizeof(unsigned long); | |
955 | entry_size_log = __ilog2_u64(sizeof(unsigned long)); | |
956 | break; | |
957 | case KCOV_MODE_TRACE_CMP: | |
958 | dst_len = READ_ONCE(*(u64 *)dst_area); | |
959 | src_len = *(u64 *)src_area; | |
960 | count_size = sizeof(u64); | |
961 | BUILD_BUG_ON(!is_power_of_2(KCOV_WORDS_PER_CMP)); | |
962 | entry_size_log = __ilog2_u64(sizeof(u64) * KCOV_WORDS_PER_CMP); | |
963 | break; | |
964 | default: | |
965 | WARN_ON(1); | |
966 | return; | |
967 | } | |
968 | ||
969 | /* As arm can't divide u64 integers use log of entry size. */ | |
970 | if (dst_len > ((dst_area_size * word_size - count_size) >> | |
971 | entry_size_log)) | |
972 | return; | |
973 | dst_occupied = count_size + (dst_len << entry_size_log); | |
974 | dst_free = dst_area_size * word_size - dst_occupied; | |
975 | bytes_to_move = min(dst_free, src_len << entry_size_log); | |
976 | dst_entries = dst_area + dst_occupied; | |
977 | src_entries = src_area + count_size; | |
978 | memcpy(dst_entries, src_entries, bytes_to_move); | |
979 | entries_moved = bytes_to_move >> entry_size_log; | |
980 | ||
981 | switch (mode) { | |
982 | case KCOV_MODE_TRACE_PC: | |
983 | WRITE_ONCE(*(unsigned long *)dst_area, dst_len + entries_moved); | |
984 | break; | |
985 | case KCOV_MODE_TRACE_CMP: | |
986 | WRITE_ONCE(*(u64 *)dst_area, dst_len + entries_moved); | |
987 | break; | |
988 | default: | |
989 | break; | |
990 | } | |
991 | } | |
992 | ||
993 | /* See the comment before kcov_remote_start() for usage details. */ | |
994 | void kcov_remote_stop(void) | |
995 | { | |
996 | struct task_struct *t = current; | |
5fe7042d AK |
997 | struct kcov *kcov; |
998 | unsigned int mode; | |
999 | void *area; | |
1000 | unsigned int size; | |
1001 | int sequence; | |
5ff3b30a AK |
1002 | unsigned long flags; |
1003 | ||
7d4df2da | 1004 | if (!in_task() && !in_softirq_really()) |
5ff3b30a AK |
1005 | return; |
1006 | ||
d5d2c51f | 1007 | local_lock_irqsave(&kcov_percpu_data.lock, flags); |
eec028c9 | 1008 | |
5fe7042d AK |
1009 | mode = READ_ONCE(t->kcov_mode); |
1010 | barrier(); | |
5ff3b30a | 1011 | if (!kcov_mode_enabled(mode)) { |
d5d2c51f | 1012 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
eec028c9 | 1013 | return; |
5ff3b30a | 1014 | } |
3021e692 AK |
1015 | /* |
1016 | * When in softirq, check if the corresponding kcov_remote_start() | |
1017 | * actually found the remote handle and started collecting coverage. | |
1018 | */ | |
1019 | if (in_serving_softirq() && !t->kcov_softirq) { | |
d5d2c51f | 1020 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
3021e692 AK |
1021 | return; |
1022 | } | |
1023 | /* Make sure that kcov_softirq is only set when in softirq. */ | |
5ff3b30a | 1024 | if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) { |
d5d2c51f | 1025 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
5ff3b30a AK |
1026 | return; |
1027 | } | |
1028 | ||
3021e692 AK |
1029 | kcov = t->kcov; |
1030 | area = t->kcov_area; | |
1031 | size = t->kcov_size; | |
1032 | sequence = t->kcov_sequence; | |
1033 | ||
eec028c9 | 1034 | kcov_stop(t); |
5ff3b30a AK |
1035 | if (in_serving_softirq()) { |
1036 | t->kcov_softirq = 0; | |
1037 | kcov_remote_softirq_stop(t); | |
1038 | } | |
eec028c9 AK |
1039 | |
1040 | spin_lock(&kcov->lock); | |
1041 | /* | |
1042 | * KCOV_DISABLE could have been called between kcov_remote_start() | |
5ff3b30a | 1043 | * and kcov_remote_stop(), hence the sequence check. |
eec028c9 | 1044 | */ |
eec028c9 AK |
1045 | if (sequence == kcov->sequence && kcov->remote) |
1046 | kcov_move_area(kcov->mode, kcov->area, kcov->size, area); | |
1047 | spin_unlock(&kcov->lock); | |
1048 | ||
5ff3b30a AK |
1049 | if (in_task()) { |
1050 | spin_lock(&kcov_remote_lock); | |
1051 | kcov_remote_area_put(area, size); | |
1052 | spin_unlock(&kcov_remote_lock); | |
1053 | } | |
eec028c9 | 1054 | |
d5d2c51f | 1055 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
5ff3b30a AK |
1056 | |
1057 | /* Get in kcov_remote_start(). */ | |
eec028c9 AK |
1058 | kcov_put(kcov); |
1059 | } | |
1060 | EXPORT_SYMBOL(kcov_remote_stop); | |
1061 | ||
1062 | /* See the comment before kcov_remote_start() for usage details. */ | |
1063 | u64 kcov_common_handle(void) | |
1064 | { | |
b08e84da AN |
1065 | if (!in_task()) |
1066 | return 0; | |
eec028c9 AK |
1067 | return current->kcov_handle; |
1068 | } | |
1069 | EXPORT_SYMBOL(kcov_common_handle); | |
1070 | ||
6cd0dd93 DV |
1071 | #ifdef CONFIG_KCOV_SELFTEST |
1072 | static void __init selftest(void) | |
1073 | { | |
1074 | unsigned long start; | |
1075 | ||
1076 | pr_err("running self test\n"); | |
1077 | /* | |
1078 | * Test that interrupts don't produce spurious coverage. | |
1079 | * The coverage callback filters out interrupt code, but only | |
1080 | * after the handler updates preempt count. Some code periodically | |
1081 | * leaks out of that section and leads to spurious coverage. | |
1082 | * It's hard to call the actual interrupt handler directly, | |
1083 | * so we just loop here for a bit waiting for a timer interrupt. | |
1084 | * We set kcov_mode to enable tracing, but don't setup the area, | |
1085 | * so any attempt to trace will crash. Note: we must not call any | |
1086 | * potentially traced functions in this region. | |
1087 | */ | |
1088 | start = jiffies; | |
1089 | current->kcov_mode = KCOV_MODE_TRACE_PC; | |
1090 | while ((jiffies - start) * MSEC_PER_SEC / HZ < 300) | |
1091 | ; | |
1092 | current->kcov_mode = 0; | |
1093 | pr_err("done running self test\n"); | |
1094 | } | |
1095 | #endif | |
1096 | ||
5c9a8750 DV |
1097 | static int __init kcov_init(void) |
1098 | { | |
5ff3b30a AK |
1099 | int cpu; |
1100 | ||
1101 | for_each_possible_cpu(cpu) { | |
741ddd45 SAS |
1102 | void *area = vmalloc_node(CONFIG_KCOV_IRQ_AREA_SIZE * |
1103 | sizeof(unsigned long), cpu_to_node(cpu)); | |
5ff3b30a AK |
1104 | if (!area) |
1105 | return -ENOMEM; | |
1106 | per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area; | |
1107 | } | |
1108 | ||
df4565f9 NS |
1109 | /* |
1110 | * The kcov debugfs file won't ever get removed and thus, | |
1111 | * there is no need to protect it against removal races. The | |
1112 | * use of debugfs_create_file_unsafe() is actually safe here. | |
1113 | */ | |
ec9672d5 GKH |
1114 | debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops); |
1115 | ||
6cd0dd93 DV |
1116 | #ifdef CONFIG_KCOV_SELFTEST |
1117 | selftest(); | |
1118 | #endif | |
1119 | ||
5c9a8750 DV |
1120 | return 0; |
1121 | } | |
1122 | ||
1123 | device_initcall(kcov_init); |