| 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | #define pr_fmt(fmt) "kcov: " fmt |
| 3 | |
| 4 | #define DISABLE_BRANCH_PROFILING |
| 5 | #include <linux/atomic.h> |
| 6 | #include <linux/compiler.h> |
| 7 | #include <linux/errno.h> |
| 8 | #include <linux/export.h> |
| 9 | #include <linux/types.h> |
| 10 | #include <linux/file.h> |
| 11 | #include <linux/fs.h> |
| 12 | #include <linux/hashtable.h> |
| 13 | #include <linux/init.h> |
| 14 | #include <linux/jiffies.h> |
| 15 | #include <linux/kmsan-checks.h> |
| 16 | #include <linux/mm.h> |
| 17 | #include <linux/preempt.h> |
| 18 | #include <linux/printk.h> |
| 19 | #include <linux/sched.h> |
| 20 | #include <linux/slab.h> |
| 21 | #include <linux/spinlock.h> |
| 22 | #include <linux/vmalloc.h> |
| 23 | #include <linux/debugfs.h> |
| 24 | #include <linux/uaccess.h> |
| 25 | #include <linux/kcov.h> |
| 26 | #include <linux/refcount.h> |
| 27 | #include <linux/log2.h> |
| 28 | #include <asm/setup.h> |
| 29 | |
| 30 | #define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__) |
| 31 | |
| 32 | /* Number of 64-bit words written per one comparison: */ |
| 33 | #define KCOV_WORDS_PER_CMP 4 |
| 34 | |
| 35 | /* |
| 36 | * kcov descriptor (one per opened debugfs file). |
| 37 | * State transitions of the descriptor: |
| 38 | * - initial state after open() |
| 39 | * - then there must be a single ioctl(KCOV_INIT_TRACE) call |
| 40 | * - then, mmap() call (several calls are allowed but not useful) |
| 41 | * - then, ioctl(KCOV_ENABLE, arg), where arg is |
| 42 | * KCOV_TRACE_PC - to trace only the PCs |
| 43 | * or |
| 44 | * KCOV_TRACE_CMP - to trace only the comparison operands |
| 45 | * - then, ioctl(KCOV_DISABLE) to disable the task. |
| 46 | * Enabling/disabling ioctls can be repeated (only one task a time allowed). |
| 47 | */ |
| 48 | struct kcov { |
| 49 | /* |
| 50 | * Reference counter. We keep one for: |
| 51 | * - opened file descriptor |
| 52 | * - task with enabled coverage (we can't unwire it from another task) |
| 53 | * - each code section for remote coverage collection |
| 54 | */ |
| 55 | refcount_t refcount; |
| 56 | /* The lock protects mode, size, area and t. */ |
| 57 | spinlock_t lock; |
| 58 | enum kcov_mode mode; |
| 59 | /* Size of arena (in long's). */ |
| 60 | unsigned int size; |
| 61 | /* Coverage buffer shared with user space. */ |
| 62 | void *area; |
| 63 | /* Task for which we collect coverage, or NULL. */ |
| 64 | struct task_struct *t; |
| 65 | /* Collecting coverage from remote (background) threads. */ |
| 66 | bool remote; |
| 67 | /* Size of remote area (in long's). */ |
| 68 | unsigned int remote_size; |
| 69 | /* |
| 70 | * Sequence is incremented each time kcov is reenabled, used by |
| 71 | * kcov_remote_stop(), see the comment there. |
| 72 | */ |
| 73 | int sequence; |
| 74 | }; |
| 75 | |
| 76 | struct kcov_remote_area { |
| 77 | struct list_head list; |
| 78 | unsigned int size; |
| 79 | }; |
| 80 | |
| 81 | struct kcov_remote { |
| 82 | u64 handle; |
| 83 | struct kcov *kcov; |
| 84 | struct hlist_node hnode; |
| 85 | }; |
| 86 | |
| 87 | static DEFINE_SPINLOCK(kcov_remote_lock); |
| 88 | static DEFINE_HASHTABLE(kcov_remote_map, 4); |
| 89 | static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas); |
| 90 | |
| 91 | struct kcov_percpu_data { |
| 92 | void *irq_area; |
| 93 | local_lock_t lock; |
| 94 | |
| 95 | unsigned int saved_mode; |
| 96 | unsigned int saved_size; |
| 97 | void *saved_area; |
| 98 | struct kcov *saved_kcov; |
| 99 | int saved_sequence; |
| 100 | }; |
| 101 | |
| 102 | static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data) = { |
| 103 | .lock = INIT_LOCAL_LOCK(lock), |
| 104 | }; |
| 105 | |
| 106 | /* Must be called with kcov_remote_lock locked. */ |
| 107 | static struct kcov_remote *kcov_remote_find(u64 handle) |
| 108 | { |
| 109 | struct kcov_remote *remote; |
| 110 | |
| 111 | hash_for_each_possible(kcov_remote_map, remote, hnode, handle) { |
| 112 | if (remote->handle == handle) |
| 113 | return remote; |
| 114 | } |
| 115 | return NULL; |
| 116 | } |
| 117 | |
| 118 | /* Must be called with kcov_remote_lock locked. */ |
| 119 | static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle) |
| 120 | { |
| 121 | struct kcov_remote *remote; |
| 122 | |
| 123 | if (kcov_remote_find(handle)) |
| 124 | return ERR_PTR(-EEXIST); |
| 125 | remote = kmalloc(sizeof(*remote), GFP_ATOMIC); |
| 126 | if (!remote) |
| 127 | return ERR_PTR(-ENOMEM); |
| 128 | remote->handle = handle; |
| 129 | remote->kcov = kcov; |
| 130 | hash_add(kcov_remote_map, &remote->hnode, handle); |
| 131 | return remote; |
| 132 | } |
| 133 | |
| 134 | /* Must be called with kcov_remote_lock locked. */ |
| 135 | static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) |
| 136 | { |
| 137 | struct kcov_remote_area *area; |
| 138 | struct list_head *pos; |
| 139 | |
| 140 | list_for_each(pos, &kcov_remote_areas) { |
| 141 | area = list_entry(pos, struct kcov_remote_area, list); |
| 142 | if (area->size == size) { |
| 143 | list_del(&area->list); |
| 144 | return area; |
| 145 | } |
| 146 | } |
| 147 | return NULL; |
| 148 | } |
| 149 | |
| 150 | /* Must be called with kcov_remote_lock locked. */ |
| 151 | static void kcov_remote_area_put(struct kcov_remote_area *area, |
| 152 | unsigned int size) |
| 153 | { |
| 154 | INIT_LIST_HEAD(&area->list); |
| 155 | area->size = size; |
| 156 | list_add(&area->list, &kcov_remote_areas); |
| 157 | /* |
| 158 | * KMSAN doesn't instrument this file, so it may not know area->list |
| 159 | * is initialized. Unpoison it explicitly to avoid reports in |
| 160 | * kcov_remote_area_get(). |
| 161 | */ |
| 162 | kmsan_unpoison_memory(&area->list, sizeof(area->list)); |
| 163 | } |
| 164 | |
| 165 | /* |
| 166 | * Unlike in_serving_softirq(), this function returns false when called during |
| 167 | * a hardirq or an NMI that happened in the softirq context. |
| 168 | */ |
| 169 | static inline bool in_softirq_really(void) |
| 170 | { |
| 171 | return in_serving_softirq() && !in_hardirq() && !in_nmi(); |
| 172 | } |
| 173 | |
| 174 | static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) |
| 175 | { |
| 176 | unsigned int mode; |
| 177 | |
| 178 | /* |
| 179 | * We are interested in code coverage as a function of a syscall inputs, |
| 180 | * so we ignore code executed in interrupts, unless we are in a remote |
| 181 | * coverage collection section in a softirq. |
| 182 | */ |
| 183 | if (!in_task() && !(in_softirq_really() && t->kcov_softirq)) |
| 184 | return false; |
| 185 | mode = READ_ONCE(t->kcov_mode); |
| 186 | /* |
| 187 | * There is some code that runs in interrupts but for which |
| 188 | * in_interrupt() returns false (e.g. preempt_schedule_irq()). |
| 189 | * READ_ONCE()/barrier() effectively provides load-acquire wrt |
| 190 | * interrupts, there are paired barrier()/WRITE_ONCE() in |
| 191 | * kcov_start(). |
| 192 | */ |
| 193 | barrier(); |
| 194 | return mode == needed_mode; |
| 195 | } |
| 196 | |
| 197 | static notrace unsigned long canonicalize_ip(unsigned long ip) |
| 198 | { |
| 199 | #ifdef CONFIG_RANDOMIZE_BASE |
| 200 | ip -= kaslr_offset(); |
| 201 | #endif |
| 202 | return ip; |
| 203 | } |
| 204 | |
| 205 | /* |
| 206 | * Entry point from instrumented code. |
| 207 | * This is called once per basic-block/edge. |
| 208 | */ |
| 209 | void notrace __sanitizer_cov_trace_pc(void) |
| 210 | { |
| 211 | struct task_struct *t; |
| 212 | unsigned long *area; |
| 213 | unsigned long ip = canonicalize_ip(_RET_IP_); |
| 214 | unsigned long pos; |
| 215 | |
| 216 | t = current; |
| 217 | if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t)) |
| 218 | return; |
| 219 | |
| 220 | area = t->kcov_area; |
| 221 | /* The first 64-bit word is the number of subsequent PCs. */ |
| 222 | pos = READ_ONCE(area[0]) + 1; |
| 223 | if (likely(pos < t->kcov_size)) { |
| 224 | /* Previously we write pc before updating pos. However, some |
| 225 | * early interrupt code could bypass check_kcov_mode() check |
| 226 | * and invoke __sanitizer_cov_trace_pc(). If such interrupt is |
| 227 | * raised between writing pc and updating pos, the pc could be |
| 228 | * overitten by the recursive __sanitizer_cov_trace_pc(). |
| 229 | * Update pos before writing pc to avoid such interleaving. |
| 230 | */ |
| 231 | WRITE_ONCE(area[0], pos); |
| 232 | barrier(); |
| 233 | area[pos] = ip; |
| 234 | } |
| 235 | } |
| 236 | EXPORT_SYMBOL(__sanitizer_cov_trace_pc); |
| 237 | |
| 238 | #ifdef CONFIG_KCOV_ENABLE_COMPARISONS |
| 239 | static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip) |
| 240 | { |
| 241 | struct task_struct *t; |
| 242 | u64 *area; |
| 243 | u64 count, start_index, end_pos, max_pos; |
| 244 | |
| 245 | t = current; |
| 246 | if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t)) |
| 247 | return; |
| 248 | |
| 249 | ip = canonicalize_ip(ip); |
| 250 | |
| 251 | /* |
| 252 | * We write all comparison arguments and types as u64. |
| 253 | * The buffer was allocated for t->kcov_size unsigned longs. |
| 254 | */ |
| 255 | area = (u64 *)t->kcov_area; |
| 256 | max_pos = t->kcov_size * sizeof(unsigned long); |
| 257 | |
| 258 | count = READ_ONCE(area[0]); |
| 259 | |
| 260 | /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */ |
| 261 | start_index = 1 + count * KCOV_WORDS_PER_CMP; |
| 262 | end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64); |
| 263 | if (likely(end_pos <= max_pos)) { |
| 264 | /* See comment in __sanitizer_cov_trace_pc(). */ |
| 265 | WRITE_ONCE(area[0], count + 1); |
| 266 | barrier(); |
| 267 | area[start_index] = type; |
| 268 | area[start_index + 1] = arg1; |
| 269 | area[start_index + 2] = arg2; |
| 270 | area[start_index + 3] = ip; |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2) |
| 275 | { |
| 276 | write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_); |
| 277 | } |
| 278 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1); |
| 279 | |
| 280 | void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2) |
| 281 | { |
| 282 | write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_); |
| 283 | } |
| 284 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2); |
| 285 | |
| 286 | void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2) |
| 287 | { |
| 288 | write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_); |
| 289 | } |
| 290 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4); |
| 291 | |
| 292 | void notrace __sanitizer_cov_trace_cmp8(kcov_u64 arg1, kcov_u64 arg2) |
| 293 | { |
| 294 | write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_); |
| 295 | } |
| 296 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8); |
| 297 | |
| 298 | void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2) |
| 299 | { |
| 300 | write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, |
| 301 | _RET_IP_); |
| 302 | } |
| 303 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1); |
| 304 | |
| 305 | void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2) |
| 306 | { |
| 307 | write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, |
| 308 | _RET_IP_); |
| 309 | } |
| 310 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2); |
| 311 | |
| 312 | void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2) |
| 313 | { |
| 314 | write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, |
| 315 | _RET_IP_); |
| 316 | } |
| 317 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4); |
| 318 | |
| 319 | void notrace __sanitizer_cov_trace_const_cmp8(kcov_u64 arg1, kcov_u64 arg2) |
| 320 | { |
| 321 | write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, |
| 322 | _RET_IP_); |
| 323 | } |
| 324 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8); |
| 325 | |
| 326 | void notrace __sanitizer_cov_trace_switch(kcov_u64 val, void *arg) |
| 327 | { |
| 328 | u64 i; |
| 329 | u64 *cases = arg; |
| 330 | u64 count = cases[0]; |
| 331 | u64 size = cases[1]; |
| 332 | u64 type = KCOV_CMP_CONST; |
| 333 | |
| 334 | switch (size) { |
| 335 | case 8: |
| 336 | type |= KCOV_CMP_SIZE(0); |
| 337 | break; |
| 338 | case 16: |
| 339 | type |= KCOV_CMP_SIZE(1); |
| 340 | break; |
| 341 | case 32: |
| 342 | type |= KCOV_CMP_SIZE(2); |
| 343 | break; |
| 344 | case 64: |
| 345 | type |= KCOV_CMP_SIZE(3); |
| 346 | break; |
| 347 | default: |
| 348 | return; |
| 349 | } |
| 350 | for (i = 0; i < count; i++) |
| 351 | write_comp_data(type, cases[i + 2], val, _RET_IP_); |
| 352 | } |
| 353 | EXPORT_SYMBOL(__sanitizer_cov_trace_switch); |
| 354 | #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */ |
| 355 | |
| 356 | static void kcov_start(struct task_struct *t, struct kcov *kcov, |
| 357 | unsigned int size, void *area, enum kcov_mode mode, |
| 358 | int sequence) |
| 359 | { |
| 360 | kcov_debug("t = %px, size = %u, area = %px\n", t, size, area); |
| 361 | t->kcov = kcov; |
| 362 | /* Cache in task struct for performance. */ |
| 363 | t->kcov_size = size; |
| 364 | t->kcov_area = area; |
| 365 | t->kcov_sequence = sequence; |
| 366 | /* See comment in check_kcov_mode(). */ |
| 367 | barrier(); |
| 368 | WRITE_ONCE(t->kcov_mode, mode); |
| 369 | } |
| 370 | |
| 371 | static void kcov_stop(struct task_struct *t) |
| 372 | { |
| 373 | WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); |
| 374 | barrier(); |
| 375 | t->kcov = NULL; |
| 376 | t->kcov_size = 0; |
| 377 | t->kcov_area = NULL; |
| 378 | } |
| 379 | |
| 380 | static void kcov_task_reset(struct task_struct *t) |
| 381 | { |
| 382 | kcov_stop(t); |
| 383 | t->kcov_sequence = 0; |
| 384 | t->kcov_handle = 0; |
| 385 | } |
| 386 | |
| 387 | void kcov_task_init(struct task_struct *t) |
| 388 | { |
| 389 | kcov_task_reset(t); |
| 390 | t->kcov_handle = current->kcov_handle; |
| 391 | } |
| 392 | |
| 393 | static void kcov_reset(struct kcov *kcov) |
| 394 | { |
| 395 | kcov->t = NULL; |
| 396 | kcov->mode = KCOV_MODE_INIT; |
| 397 | kcov->remote = false; |
| 398 | kcov->remote_size = 0; |
| 399 | kcov->sequence++; |
| 400 | } |
| 401 | |
| 402 | static void kcov_remote_reset(struct kcov *kcov) |
| 403 | { |
| 404 | int bkt; |
| 405 | struct kcov_remote *remote; |
| 406 | struct hlist_node *tmp; |
| 407 | unsigned long flags; |
| 408 | |
| 409 | spin_lock_irqsave(&kcov_remote_lock, flags); |
| 410 | hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) { |
| 411 | if (remote->kcov != kcov) |
| 412 | continue; |
| 413 | hash_del(&remote->hnode); |
| 414 | kfree(remote); |
| 415 | } |
| 416 | /* Do reset before unlock to prevent races with kcov_remote_start(). */ |
| 417 | kcov_reset(kcov); |
| 418 | spin_unlock_irqrestore(&kcov_remote_lock, flags); |
| 419 | } |
| 420 | |
| 421 | static void kcov_disable(struct task_struct *t, struct kcov *kcov) |
| 422 | { |
| 423 | kcov_task_reset(t); |
| 424 | if (kcov->remote) |
| 425 | kcov_remote_reset(kcov); |
| 426 | else |
| 427 | kcov_reset(kcov); |
| 428 | } |
| 429 | |
| 430 | static void kcov_get(struct kcov *kcov) |
| 431 | { |
| 432 | refcount_inc(&kcov->refcount); |
| 433 | } |
| 434 | |
| 435 | static void kcov_put(struct kcov *kcov) |
| 436 | { |
| 437 | if (refcount_dec_and_test(&kcov->refcount)) { |
| 438 | kcov_remote_reset(kcov); |
| 439 | vfree(kcov->area); |
| 440 | kfree(kcov); |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | void kcov_task_exit(struct task_struct *t) |
| 445 | { |
| 446 | struct kcov *kcov; |
| 447 | unsigned long flags; |
| 448 | |
| 449 | kcov = t->kcov; |
| 450 | if (kcov == NULL) |
| 451 | return; |
| 452 | |
| 453 | spin_lock_irqsave(&kcov->lock, flags); |
| 454 | kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); |
| 455 | /* |
| 456 | * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t, |
| 457 | * which comes down to: |
| 458 | * WARN_ON(!kcov->remote && kcov->t != t); |
| 459 | * |
| 460 | * For KCOV_REMOTE_ENABLE devices, the exiting task is either: |
| 461 | * |
| 462 | * 1. A remote task between kcov_remote_start() and kcov_remote_stop(). |
| 463 | * In this case we should print a warning right away, since a task |
| 464 | * shouldn't be exiting when it's in a kcov coverage collection |
| 465 | * section. Here t points to the task that is collecting remote |
| 466 | * coverage, and t->kcov->t points to the thread that created the |
| 467 | * kcov device. Which means that to detect this case we need to |
| 468 | * check that t != t->kcov->t, and this gives us the following: |
| 469 | * WARN_ON(kcov->remote && kcov->t != t); |
| 470 | * |
| 471 | * 2. The task that created kcov exiting without calling KCOV_DISABLE, |
| 472 | * and then again we make sure that t->kcov->t == t: |
| 473 | * WARN_ON(kcov->remote && kcov->t != t); |
| 474 | * |
| 475 | * By combining all three checks into one we get: |
| 476 | */ |
| 477 | if (WARN_ON(kcov->t != t)) { |
| 478 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 479 | return; |
| 480 | } |
| 481 | /* Just to not leave dangling references behind. */ |
| 482 | kcov_disable(t, kcov); |
| 483 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 484 | kcov_put(kcov); |
| 485 | } |
| 486 | |
| 487 | static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) |
| 488 | { |
| 489 | int res = 0; |
| 490 | struct kcov *kcov = vma->vm_file->private_data; |
| 491 | unsigned long size, off; |
| 492 | struct page *page; |
| 493 | unsigned long flags; |
| 494 | |
| 495 | spin_lock_irqsave(&kcov->lock, flags); |
| 496 | size = kcov->size * sizeof(unsigned long); |
| 497 | if (kcov->area == NULL || vma->vm_pgoff != 0 || |
| 498 | vma->vm_end - vma->vm_start != size) { |
| 499 | res = -EINVAL; |
| 500 | goto exit; |
| 501 | } |
| 502 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 503 | vm_flags_set(vma, VM_DONTEXPAND); |
| 504 | for (off = 0; off < size; off += PAGE_SIZE) { |
| 505 | page = vmalloc_to_page(kcov->area + off); |
| 506 | res = vm_insert_page(vma, vma->vm_start + off, page); |
| 507 | if (res) { |
| 508 | pr_warn_once("kcov: vm_insert_page() failed\n"); |
| 509 | return res; |
| 510 | } |
| 511 | } |
| 512 | return 0; |
| 513 | exit: |
| 514 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 515 | return res; |
| 516 | } |
| 517 | |
| 518 | static int kcov_open(struct inode *inode, struct file *filep) |
| 519 | { |
| 520 | struct kcov *kcov; |
| 521 | |
| 522 | kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); |
| 523 | if (!kcov) |
| 524 | return -ENOMEM; |
| 525 | kcov->mode = KCOV_MODE_DISABLED; |
| 526 | kcov->sequence = 1; |
| 527 | refcount_set(&kcov->refcount, 1); |
| 528 | spin_lock_init(&kcov->lock); |
| 529 | filep->private_data = kcov; |
| 530 | return nonseekable_open(inode, filep); |
| 531 | } |
| 532 | |
| 533 | static int kcov_close(struct inode *inode, struct file *filep) |
| 534 | { |
| 535 | kcov_put(filep->private_data); |
| 536 | return 0; |
| 537 | } |
| 538 | |
| 539 | static int kcov_get_mode(unsigned long arg) |
| 540 | { |
| 541 | if (arg == KCOV_TRACE_PC) |
| 542 | return KCOV_MODE_TRACE_PC; |
| 543 | else if (arg == KCOV_TRACE_CMP) |
| 544 | #ifdef CONFIG_KCOV_ENABLE_COMPARISONS |
| 545 | return KCOV_MODE_TRACE_CMP; |
| 546 | #else |
| 547 | return -ENOTSUPP; |
| 548 | #endif |
| 549 | else |
| 550 | return -EINVAL; |
| 551 | } |
| 552 | |
| 553 | /* |
| 554 | * Fault in a lazily-faulted vmalloc area before it can be used by |
| 555 | * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the |
| 556 | * vmalloc fault handling path is instrumented. |
| 557 | */ |
| 558 | static void kcov_fault_in_area(struct kcov *kcov) |
| 559 | { |
| 560 | unsigned long stride = PAGE_SIZE / sizeof(unsigned long); |
| 561 | unsigned long *area = kcov->area; |
| 562 | unsigned long offset; |
| 563 | |
| 564 | for (offset = 0; offset < kcov->size; offset += stride) |
| 565 | READ_ONCE(area[offset]); |
| 566 | } |
| 567 | |
| 568 | static inline bool kcov_check_handle(u64 handle, bool common_valid, |
| 569 | bool uncommon_valid, bool zero_valid) |
| 570 | { |
| 571 | if (handle & ~(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK)) |
| 572 | return false; |
| 573 | switch (handle & KCOV_SUBSYSTEM_MASK) { |
| 574 | case KCOV_SUBSYSTEM_COMMON: |
| 575 | return (handle & KCOV_INSTANCE_MASK) ? |
| 576 | common_valid : zero_valid; |
| 577 | case KCOV_SUBSYSTEM_USB: |
| 578 | return uncommon_valid; |
| 579 | default: |
| 580 | return false; |
| 581 | } |
| 582 | return false; |
| 583 | } |
| 584 | |
| 585 | static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, |
| 586 | unsigned long arg) |
| 587 | { |
| 588 | struct task_struct *t; |
| 589 | unsigned long flags, unused; |
| 590 | int mode, i; |
| 591 | struct kcov_remote_arg *remote_arg; |
| 592 | struct kcov_remote *remote; |
| 593 | |
| 594 | switch (cmd) { |
| 595 | case KCOV_ENABLE: |
| 596 | /* |
| 597 | * Enable coverage for the current task. |
| 598 | * At this point user must have been enabled trace mode, |
| 599 | * and mmapped the file. Coverage collection is disabled only |
| 600 | * at task exit or voluntary by KCOV_DISABLE. After that it can |
| 601 | * be enabled for another task. |
| 602 | */ |
| 603 | if (kcov->mode != KCOV_MODE_INIT || !kcov->area) |
| 604 | return -EINVAL; |
| 605 | t = current; |
| 606 | if (kcov->t != NULL || t->kcov != NULL) |
| 607 | return -EBUSY; |
| 608 | mode = kcov_get_mode(arg); |
| 609 | if (mode < 0) |
| 610 | return mode; |
| 611 | kcov_fault_in_area(kcov); |
| 612 | kcov->mode = mode; |
| 613 | kcov_start(t, kcov, kcov->size, kcov->area, kcov->mode, |
| 614 | kcov->sequence); |
| 615 | kcov->t = t; |
| 616 | /* Put either in kcov_task_exit() or in KCOV_DISABLE. */ |
| 617 | kcov_get(kcov); |
| 618 | return 0; |
| 619 | case KCOV_DISABLE: |
| 620 | /* Disable coverage for the current task. */ |
| 621 | unused = arg; |
| 622 | if (unused != 0 || current->kcov != kcov) |
| 623 | return -EINVAL; |
| 624 | t = current; |
| 625 | if (WARN_ON(kcov->t != t)) |
| 626 | return -EINVAL; |
| 627 | kcov_disable(t, kcov); |
| 628 | kcov_put(kcov); |
| 629 | return 0; |
| 630 | case KCOV_REMOTE_ENABLE: |
| 631 | if (kcov->mode != KCOV_MODE_INIT || !kcov->area) |
| 632 | return -EINVAL; |
| 633 | t = current; |
| 634 | if (kcov->t != NULL || t->kcov != NULL) |
| 635 | return -EBUSY; |
| 636 | remote_arg = (struct kcov_remote_arg *)arg; |
| 637 | mode = kcov_get_mode(remote_arg->trace_mode); |
| 638 | if (mode < 0) |
| 639 | return mode; |
| 640 | if ((unsigned long)remote_arg->area_size > |
| 641 | LONG_MAX / sizeof(unsigned long)) |
| 642 | return -EINVAL; |
| 643 | kcov->mode = mode; |
| 644 | t->kcov = kcov; |
| 645 | t->kcov_mode = KCOV_MODE_REMOTE; |
| 646 | kcov->t = t; |
| 647 | kcov->remote = true; |
| 648 | kcov->remote_size = remote_arg->area_size; |
| 649 | spin_lock_irqsave(&kcov_remote_lock, flags); |
| 650 | for (i = 0; i < remote_arg->num_handles; i++) { |
| 651 | if (!kcov_check_handle(remote_arg->handles[i], |
| 652 | false, true, false)) { |
| 653 | spin_unlock_irqrestore(&kcov_remote_lock, |
| 654 | flags); |
| 655 | kcov_disable(t, kcov); |
| 656 | return -EINVAL; |
| 657 | } |
| 658 | remote = kcov_remote_add(kcov, remote_arg->handles[i]); |
| 659 | if (IS_ERR(remote)) { |
| 660 | spin_unlock_irqrestore(&kcov_remote_lock, |
| 661 | flags); |
| 662 | kcov_disable(t, kcov); |
| 663 | return PTR_ERR(remote); |
| 664 | } |
| 665 | } |
| 666 | if (remote_arg->common_handle) { |
| 667 | if (!kcov_check_handle(remote_arg->common_handle, |
| 668 | true, false, false)) { |
| 669 | spin_unlock_irqrestore(&kcov_remote_lock, |
| 670 | flags); |
| 671 | kcov_disable(t, kcov); |
| 672 | return -EINVAL; |
| 673 | } |
| 674 | remote = kcov_remote_add(kcov, |
| 675 | remote_arg->common_handle); |
| 676 | if (IS_ERR(remote)) { |
| 677 | spin_unlock_irqrestore(&kcov_remote_lock, |
| 678 | flags); |
| 679 | kcov_disable(t, kcov); |
| 680 | return PTR_ERR(remote); |
| 681 | } |
| 682 | t->kcov_handle = remote_arg->common_handle; |
| 683 | } |
| 684 | spin_unlock_irqrestore(&kcov_remote_lock, flags); |
| 685 | /* Put either in kcov_task_exit() or in KCOV_DISABLE. */ |
| 686 | kcov_get(kcov); |
| 687 | return 0; |
| 688 | default: |
| 689 | return -ENOTTY; |
| 690 | } |
| 691 | } |
| 692 | |
| 693 | static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) |
| 694 | { |
| 695 | struct kcov *kcov; |
| 696 | int res; |
| 697 | struct kcov_remote_arg *remote_arg = NULL; |
| 698 | unsigned int remote_num_handles; |
| 699 | unsigned long remote_arg_size; |
| 700 | unsigned long size, flags; |
| 701 | void *area; |
| 702 | |
| 703 | kcov = filep->private_data; |
| 704 | switch (cmd) { |
| 705 | case KCOV_INIT_TRACE: |
| 706 | /* |
| 707 | * Enable kcov in trace mode and setup buffer size. |
| 708 | * Must happen before anything else. |
| 709 | * |
| 710 | * First check the size argument - it must be at least 2 |
| 711 | * to hold the current position and one PC. |
| 712 | */ |
| 713 | size = arg; |
| 714 | if (size < 2 || size > INT_MAX / sizeof(unsigned long)) |
| 715 | return -EINVAL; |
| 716 | area = vmalloc_user(size * sizeof(unsigned long)); |
| 717 | if (area == NULL) |
| 718 | return -ENOMEM; |
| 719 | spin_lock_irqsave(&kcov->lock, flags); |
| 720 | if (kcov->mode != KCOV_MODE_DISABLED) { |
| 721 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 722 | vfree(area); |
| 723 | return -EBUSY; |
| 724 | } |
| 725 | kcov->area = area; |
| 726 | kcov->size = size; |
| 727 | kcov->mode = KCOV_MODE_INIT; |
| 728 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 729 | return 0; |
| 730 | case KCOV_REMOTE_ENABLE: |
| 731 | if (get_user(remote_num_handles, (unsigned __user *)(arg + |
| 732 | offsetof(struct kcov_remote_arg, num_handles)))) |
| 733 | return -EFAULT; |
| 734 | if (remote_num_handles > KCOV_REMOTE_MAX_HANDLES) |
| 735 | return -EINVAL; |
| 736 | remote_arg_size = struct_size(remote_arg, handles, |
| 737 | remote_num_handles); |
| 738 | remote_arg = memdup_user((void __user *)arg, remote_arg_size); |
| 739 | if (IS_ERR(remote_arg)) |
| 740 | return PTR_ERR(remote_arg); |
| 741 | if (remote_arg->num_handles != remote_num_handles) { |
| 742 | kfree(remote_arg); |
| 743 | return -EINVAL; |
| 744 | } |
| 745 | arg = (unsigned long)remote_arg; |
| 746 | fallthrough; |
| 747 | default: |
| 748 | /* |
| 749 | * All other commands can be normally executed under a spin lock, so we |
| 750 | * obtain and release it here in order to simplify kcov_ioctl_locked(). |
| 751 | */ |
| 752 | spin_lock_irqsave(&kcov->lock, flags); |
| 753 | res = kcov_ioctl_locked(kcov, cmd, arg); |
| 754 | spin_unlock_irqrestore(&kcov->lock, flags); |
| 755 | kfree(remote_arg); |
| 756 | return res; |
| 757 | } |
| 758 | } |
| 759 | |
| 760 | static const struct file_operations kcov_fops = { |
| 761 | .open = kcov_open, |
| 762 | .unlocked_ioctl = kcov_ioctl, |
| 763 | .compat_ioctl = kcov_ioctl, |
| 764 | .mmap = kcov_mmap, |
| 765 | .release = kcov_close, |
| 766 | }; |
| 767 | |
| 768 | /* |
| 769 | * kcov_remote_start() and kcov_remote_stop() can be used to annotate a section |
| 770 | * of code in a kernel background thread or in a softirq to allow kcov to be |
| 771 | * used to collect coverage from that part of code. |
| 772 | * |
| 773 | * The handle argument of kcov_remote_start() identifies a code section that is |
| 774 | * used for coverage collection. A userspace process passes this handle to |
| 775 | * KCOV_REMOTE_ENABLE ioctl to make the used kcov device start collecting |
| 776 | * coverage for the code section identified by this handle. |
| 777 | * |
| 778 | * The usage of these annotations in the kernel code is different depending on |
| 779 | * the type of the kernel thread whose code is being annotated. |
| 780 | * |
| 781 | * For global kernel threads that are spawned in a limited number of instances |
| 782 | * (e.g. one USB hub_event() worker thread is spawned per USB HCD) and for |
| 783 | * softirqs, each instance must be assigned a unique 4-byte instance id. The |
| 784 | * instance id is then combined with a 1-byte subsystem id to get a handle via |
| 785 | * kcov_remote_handle(subsystem_id, instance_id). |
| 786 | * |
| 787 | * For local kernel threads that are spawned from system calls handler when a |
| 788 | * user interacts with some kernel interface (e.g. vhost workers), a handle is |
| 789 | * passed from a userspace process as the common_handle field of the |
| 790 | * kcov_remote_arg struct (note, that the user must generate a handle by using |
| 791 | * kcov_remote_handle() with KCOV_SUBSYSTEM_COMMON as the subsystem id and an |
| 792 | * arbitrary 4-byte non-zero number as the instance id). This common handle |
| 793 | * then gets saved into the task_struct of the process that issued the |
| 794 | * KCOV_REMOTE_ENABLE ioctl. When this process issues system calls that spawn |
| 795 | * kernel threads, the common handle must be retrieved via kcov_common_handle() |
| 796 | * and passed to the spawned threads via custom annotations. Those kernel |
| 797 | * threads must in turn be annotated with kcov_remote_start(common_handle) and |
| 798 | * kcov_remote_stop(). All of the threads that are spawned by the same process |
| 799 | * obtain the same handle, hence the name "common". |
| 800 | * |
| 801 | * See Documentation/dev-tools/kcov.rst for more details. |
| 802 | * |
| 803 | * Internally, kcov_remote_start() looks up the kcov device associated with the |
| 804 | * provided handle, allocates an area for coverage collection, and saves the |
| 805 | * pointers to kcov and area into the current task_struct to allow coverage to |
| 806 | * be collected via __sanitizer_cov_trace_pc(). |
| 807 | * In turns kcov_remote_stop() clears those pointers from task_struct to stop |
| 808 | * collecting coverage and copies all collected coverage into the kcov area. |
| 809 | */ |
| 810 | |
| 811 | static inline bool kcov_mode_enabled(unsigned int mode) |
| 812 | { |
| 813 | return (mode & ~KCOV_IN_CTXSW) != KCOV_MODE_DISABLED; |
| 814 | } |
| 815 | |
| 816 | static void kcov_remote_softirq_start(struct task_struct *t) |
| 817 | { |
| 818 | struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); |
| 819 | unsigned int mode; |
| 820 | |
| 821 | mode = READ_ONCE(t->kcov_mode); |
| 822 | barrier(); |
| 823 | if (kcov_mode_enabled(mode)) { |
| 824 | data->saved_mode = mode; |
| 825 | data->saved_size = t->kcov_size; |
| 826 | data->saved_area = t->kcov_area; |
| 827 | data->saved_sequence = t->kcov_sequence; |
| 828 | data->saved_kcov = t->kcov; |
| 829 | kcov_stop(t); |
| 830 | } |
| 831 | } |
| 832 | |
| 833 | static void kcov_remote_softirq_stop(struct task_struct *t) |
| 834 | { |
| 835 | struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); |
| 836 | |
| 837 | if (data->saved_kcov) { |
| 838 | kcov_start(t, data->saved_kcov, data->saved_size, |
| 839 | data->saved_area, data->saved_mode, |
| 840 | data->saved_sequence); |
| 841 | data->saved_mode = 0; |
| 842 | data->saved_size = 0; |
| 843 | data->saved_area = NULL; |
| 844 | data->saved_sequence = 0; |
| 845 | data->saved_kcov = NULL; |
| 846 | } |
| 847 | } |
| 848 | |
| 849 | void kcov_remote_start(u64 handle) |
| 850 | { |
| 851 | struct task_struct *t = current; |
| 852 | struct kcov_remote *remote; |
| 853 | struct kcov *kcov; |
| 854 | unsigned int mode; |
| 855 | void *area; |
| 856 | unsigned int size; |
| 857 | int sequence; |
| 858 | unsigned long flags; |
| 859 | |
| 860 | if (WARN_ON(!kcov_check_handle(handle, true, true, true))) |
| 861 | return; |
| 862 | if (!in_task() && !in_softirq_really()) |
| 863 | return; |
| 864 | |
| 865 | local_lock_irqsave(&kcov_percpu_data.lock, flags); |
| 866 | |
| 867 | /* |
| 868 | * Check that kcov_remote_start() is not called twice in background |
| 869 | * threads nor called by user tasks (with enabled kcov). |
| 870 | */ |
| 871 | mode = READ_ONCE(t->kcov_mode); |
| 872 | if (WARN_ON(in_task() && kcov_mode_enabled(mode))) { |
| 873 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 874 | return; |
| 875 | } |
| 876 | /* |
| 877 | * Check that kcov_remote_start() is not called twice in softirqs. |
| 878 | * Note, that kcov_remote_start() can be called from a softirq that |
| 879 | * happened while collecting coverage from a background thread. |
| 880 | */ |
| 881 | if (WARN_ON(in_serving_softirq() && t->kcov_softirq)) { |
| 882 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 883 | return; |
| 884 | } |
| 885 | |
| 886 | spin_lock(&kcov_remote_lock); |
| 887 | remote = kcov_remote_find(handle); |
| 888 | if (!remote) { |
| 889 | spin_unlock(&kcov_remote_lock); |
| 890 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 891 | return; |
| 892 | } |
| 893 | kcov_debug("handle = %llx, context: %s\n", handle, |
| 894 | in_task() ? "task" : "softirq"); |
| 895 | kcov = remote->kcov; |
| 896 | /* Put in kcov_remote_stop(). */ |
| 897 | kcov_get(kcov); |
| 898 | /* |
| 899 | * Read kcov fields before unlock to prevent races with |
| 900 | * KCOV_DISABLE / kcov_remote_reset(). |
| 901 | */ |
| 902 | mode = kcov->mode; |
| 903 | sequence = kcov->sequence; |
| 904 | if (in_task()) { |
| 905 | size = kcov->remote_size; |
| 906 | area = kcov_remote_area_get(size); |
| 907 | } else { |
| 908 | size = CONFIG_KCOV_IRQ_AREA_SIZE; |
| 909 | area = this_cpu_ptr(&kcov_percpu_data)->irq_area; |
| 910 | } |
| 911 | spin_unlock(&kcov_remote_lock); |
| 912 | |
| 913 | /* Can only happen when in_task(). */ |
| 914 | if (!area) { |
| 915 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 916 | area = vmalloc(size * sizeof(unsigned long)); |
| 917 | if (!area) { |
| 918 | kcov_put(kcov); |
| 919 | return; |
| 920 | } |
| 921 | local_lock_irqsave(&kcov_percpu_data.lock, flags); |
| 922 | } |
| 923 | |
| 924 | /* Reset coverage size. */ |
| 925 | *(u64 *)area = 0; |
| 926 | |
| 927 | if (in_serving_softirq()) { |
| 928 | kcov_remote_softirq_start(t); |
| 929 | t->kcov_softirq = 1; |
| 930 | } |
| 931 | kcov_start(t, kcov, size, area, mode, sequence); |
| 932 | |
| 933 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 934 | |
| 935 | } |
| 936 | EXPORT_SYMBOL(kcov_remote_start); |
| 937 | |
| 938 | static void kcov_move_area(enum kcov_mode mode, void *dst_area, |
| 939 | unsigned int dst_area_size, void *src_area) |
| 940 | { |
| 941 | u64 word_size = sizeof(unsigned long); |
| 942 | u64 count_size, entry_size_log; |
| 943 | u64 dst_len, src_len; |
| 944 | void *dst_entries, *src_entries; |
| 945 | u64 dst_occupied, dst_free, bytes_to_move, entries_moved; |
| 946 | |
| 947 | kcov_debug("%px %u <= %px %lu\n", |
| 948 | dst_area, dst_area_size, src_area, *(unsigned long *)src_area); |
| 949 | |
| 950 | switch (mode) { |
| 951 | case KCOV_MODE_TRACE_PC: |
| 952 | dst_len = READ_ONCE(*(unsigned long *)dst_area); |
| 953 | src_len = *(unsigned long *)src_area; |
| 954 | count_size = sizeof(unsigned long); |
| 955 | entry_size_log = __ilog2_u64(sizeof(unsigned long)); |
| 956 | break; |
| 957 | case KCOV_MODE_TRACE_CMP: |
| 958 | dst_len = READ_ONCE(*(u64 *)dst_area); |
| 959 | src_len = *(u64 *)src_area; |
| 960 | count_size = sizeof(u64); |
| 961 | BUILD_BUG_ON(!is_power_of_2(KCOV_WORDS_PER_CMP)); |
| 962 | entry_size_log = __ilog2_u64(sizeof(u64) * KCOV_WORDS_PER_CMP); |
| 963 | break; |
| 964 | default: |
| 965 | WARN_ON(1); |
| 966 | return; |
| 967 | } |
| 968 | |
| 969 | /* As arm can't divide u64 integers use log of entry size. */ |
| 970 | if (dst_len > ((dst_area_size * word_size - count_size) >> |
| 971 | entry_size_log)) |
| 972 | return; |
| 973 | dst_occupied = count_size + (dst_len << entry_size_log); |
| 974 | dst_free = dst_area_size * word_size - dst_occupied; |
| 975 | bytes_to_move = min(dst_free, src_len << entry_size_log); |
| 976 | dst_entries = dst_area + dst_occupied; |
| 977 | src_entries = src_area + count_size; |
| 978 | memcpy(dst_entries, src_entries, bytes_to_move); |
| 979 | entries_moved = bytes_to_move >> entry_size_log; |
| 980 | |
| 981 | switch (mode) { |
| 982 | case KCOV_MODE_TRACE_PC: |
| 983 | WRITE_ONCE(*(unsigned long *)dst_area, dst_len + entries_moved); |
| 984 | break; |
| 985 | case KCOV_MODE_TRACE_CMP: |
| 986 | WRITE_ONCE(*(u64 *)dst_area, dst_len + entries_moved); |
| 987 | break; |
| 988 | default: |
| 989 | break; |
| 990 | } |
| 991 | } |
| 992 | |
| 993 | /* See the comment before kcov_remote_start() for usage details. */ |
| 994 | void kcov_remote_stop(void) |
| 995 | { |
| 996 | struct task_struct *t = current; |
| 997 | struct kcov *kcov; |
| 998 | unsigned int mode; |
| 999 | void *area; |
| 1000 | unsigned int size; |
| 1001 | int sequence; |
| 1002 | unsigned long flags; |
| 1003 | |
| 1004 | if (!in_task() && !in_softirq_really()) |
| 1005 | return; |
| 1006 | |
| 1007 | local_lock_irqsave(&kcov_percpu_data.lock, flags); |
| 1008 | |
| 1009 | mode = READ_ONCE(t->kcov_mode); |
| 1010 | barrier(); |
| 1011 | if (!kcov_mode_enabled(mode)) { |
| 1012 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 1013 | return; |
| 1014 | } |
| 1015 | /* |
| 1016 | * When in softirq, check if the corresponding kcov_remote_start() |
| 1017 | * actually found the remote handle and started collecting coverage. |
| 1018 | */ |
| 1019 | if (in_serving_softirq() && !t->kcov_softirq) { |
| 1020 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 1021 | return; |
| 1022 | } |
| 1023 | /* Make sure that kcov_softirq is only set when in softirq. */ |
| 1024 | if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) { |
| 1025 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 1026 | return; |
| 1027 | } |
| 1028 | |
| 1029 | kcov = t->kcov; |
| 1030 | area = t->kcov_area; |
| 1031 | size = t->kcov_size; |
| 1032 | sequence = t->kcov_sequence; |
| 1033 | |
| 1034 | kcov_stop(t); |
| 1035 | if (in_serving_softirq()) { |
| 1036 | t->kcov_softirq = 0; |
| 1037 | kcov_remote_softirq_stop(t); |
| 1038 | } |
| 1039 | |
| 1040 | spin_lock(&kcov->lock); |
| 1041 | /* |
| 1042 | * KCOV_DISABLE could have been called between kcov_remote_start() |
| 1043 | * and kcov_remote_stop(), hence the sequence check. |
| 1044 | */ |
| 1045 | if (sequence == kcov->sequence && kcov->remote) |
| 1046 | kcov_move_area(kcov->mode, kcov->area, kcov->size, area); |
| 1047 | spin_unlock(&kcov->lock); |
| 1048 | |
| 1049 | if (in_task()) { |
| 1050 | spin_lock(&kcov_remote_lock); |
| 1051 | kcov_remote_area_put(area, size); |
| 1052 | spin_unlock(&kcov_remote_lock); |
| 1053 | } |
| 1054 | |
| 1055 | local_unlock_irqrestore(&kcov_percpu_data.lock, flags); |
| 1056 | |
| 1057 | /* Get in kcov_remote_start(). */ |
| 1058 | kcov_put(kcov); |
| 1059 | } |
| 1060 | EXPORT_SYMBOL(kcov_remote_stop); |
| 1061 | |
| 1062 | /* See the comment before kcov_remote_start() for usage details. */ |
| 1063 | u64 kcov_common_handle(void) |
| 1064 | { |
| 1065 | if (!in_task()) |
| 1066 | return 0; |
| 1067 | return current->kcov_handle; |
| 1068 | } |
| 1069 | EXPORT_SYMBOL(kcov_common_handle); |
| 1070 | |
| 1071 | #ifdef CONFIG_KCOV_SELFTEST |
| 1072 | static void __init selftest(void) |
| 1073 | { |
| 1074 | unsigned long start; |
| 1075 | |
| 1076 | pr_err("running self test\n"); |
| 1077 | /* |
| 1078 | * Test that interrupts don't produce spurious coverage. |
| 1079 | * The coverage callback filters out interrupt code, but only |
| 1080 | * after the handler updates preempt count. Some code periodically |
| 1081 | * leaks out of that section and leads to spurious coverage. |
| 1082 | * It's hard to call the actual interrupt handler directly, |
| 1083 | * so we just loop here for a bit waiting for a timer interrupt. |
| 1084 | * We set kcov_mode to enable tracing, but don't setup the area, |
| 1085 | * so any attempt to trace will crash. Note: we must not call any |
| 1086 | * potentially traced functions in this region. |
| 1087 | */ |
| 1088 | start = jiffies; |
| 1089 | current->kcov_mode = KCOV_MODE_TRACE_PC; |
| 1090 | while ((jiffies - start) * MSEC_PER_SEC / HZ < 300) |
| 1091 | ; |
| 1092 | current->kcov_mode = 0; |
| 1093 | pr_err("done running self test\n"); |
| 1094 | } |
| 1095 | #endif |
| 1096 | |
| 1097 | static int __init kcov_init(void) |
| 1098 | { |
| 1099 | int cpu; |
| 1100 | |
| 1101 | for_each_possible_cpu(cpu) { |
| 1102 | void *area = vmalloc_node(CONFIG_KCOV_IRQ_AREA_SIZE * |
| 1103 | sizeof(unsigned long), cpu_to_node(cpu)); |
| 1104 | if (!area) |
| 1105 | return -ENOMEM; |
| 1106 | per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area; |
| 1107 | } |
| 1108 | |
| 1109 | /* |
| 1110 | * The kcov debugfs file won't ever get removed and thus, |
| 1111 | * there is no need to protect it against removal races. The |
| 1112 | * use of debugfs_create_file_unsafe() is actually safe here. |
| 1113 | */ |
| 1114 | debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops); |
| 1115 | |
| 1116 | #ifdef CONFIG_KCOV_SELFTEST |
| 1117 | selftest(); |
| 1118 | #endif |
| 1119 | |
| 1120 | return 0; |
| 1121 | } |
| 1122 | |
| 1123 | device_initcall(kcov_init); |