mm/kasan/tags.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains common tag-based KASAN code.
   4  *
   5  * Copyright (c) 2018 Google, Inc.
   6  * Copyright (c) 2020 Google, Inc.
   7  */
   8
   9 #include <linux/atomic.h>
  10 #include <linux/init.h>
  11 #include <linux/kasan.h>
  12 #include <linux/kernel.h>
  13 #include <linux/memblock.h>
  14 #include <linux/memory.h>
  15 #include <linux/mm.h>
  16 #include <linux/static_key.h>
  17 #include <linux/string.h>
  18 #include <linux/types.h>
  19
  20 #include "kasan.h"
  21 #include "../slab.h"
  22
  23 #define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10)
  24
  25 enum kasan_arg_stacktrace {
  26         KASAN_ARG_STACKTRACE_DEFAULT,
  27         KASAN_ARG_STACKTRACE_OFF,
  28         KASAN_ARG_STACKTRACE_ON,
  29 };
  30
  31 static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
  32
  33 /* Whether to collect alloc/free stack traces. */
  34 DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
  35
  36 /* Non-zero, as initial pointer values are 0. */
  37 #define STACK_RING_BUSY_PTR ((void *)1)
  38
  39 struct kasan_stack_ring stack_ring = {
  40         .lock = __RW_LOCK_UNLOCKED(stack_ring.lock)
  41 };
  42
  43 /* kasan.stacktrace=off/on */
  44 static int __init early_kasan_flag_stacktrace(char *arg)
  45 {
  46         if (!arg)
  47                 return -EINVAL;
  48
  49         if (!strcmp(arg, "off"))
  50                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
  51         else if (!strcmp(arg, "on"))
  52                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
  53         else
  54                 return -EINVAL;
  55
  56         return 0;
  57 }
  58 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
  59
  60 /* kasan.stack_ring_size=<number of entries> */
  61 static int __init early_kasan_flag_stack_ring_size(char *arg)
  62 {
  63         if (!arg)
  64                 return -EINVAL;
  65
  66         return kstrtoul(arg, 0, &stack_ring.size);
  67 }
  68 early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size);
  69
  70 void __init kasan_init_tags(void)
  71 {
  72         switch (kasan_arg_stacktrace) {
  73         case KASAN_ARG_STACKTRACE_DEFAULT:
  74                 /* Default is specified by kasan_flag_stacktrace definition. */
  75                 break;
  76         case KASAN_ARG_STACKTRACE_OFF:
  77                 static_branch_disable(&kasan_flag_stacktrace);
  78                 break;
  79         case KASAN_ARG_STACKTRACE_ON:
  80                 static_branch_enable(&kasan_flag_stacktrace);
  81                 break;
  82         }
  83
  84         if (kasan_stack_collection_enabled()) {
  85                 if (!stack_ring.size)
  86                         stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT;
  87                 stack_ring.entries = memblock_alloc(
  88                         sizeof(stack_ring.entries[0]) * stack_ring.size,
  89                         SMP_CACHE_BYTES);
  90                 if (WARN_ON(!stack_ring.entries))
  91                         static_branch_disable(&kasan_flag_stacktrace);
  92         }
  93 }
  94
  95 static void save_stack_info(struct kmem_cache *cache, void *object,
  96                         gfp_t gfp_flags, bool is_free)
  97 {
  98         unsigned long flags;
  99         depot_stack_handle_t stack;
 100         u64 pos;
 101         struct kasan_stack_ring_entry *entry;
 102         void *old_ptr;
 103
 104         stack = kasan_save_stack(gfp_flags, true);
 105
 106         /*
 107          * Prevent save_stack_info() from modifying stack ring
 108          * when kasan_complete_mode_report_info() is walking it.
 109          */
 110         read_lock_irqsave(&stack_ring.lock, flags);
 111
 112 next:
 113         pos = atomic64_fetch_add(1, &stack_ring.pos);
 114         entry = &stack_ring.entries[pos % stack_ring.size];
 115
 116         /* Detect stack ring entry slots that are being written to. */
 117         old_ptr = READ_ONCE(entry->ptr);
 118         if (old_ptr == STACK_RING_BUSY_PTR)
 119                 goto next; /* Busy slot. */
 120         if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
 121                 goto next; /* Busy slot. */
 122
 123         WRITE_ONCE(entry->size, cache->object_size);
 124         WRITE_ONCE(entry->pid, current->pid);
 125         WRITE_ONCE(entry->stack, stack);
 126         WRITE_ONCE(entry->is_free, is_free);
 127
 128         /*
 129          * Paired with smp_load_acquire() in kasan_complete_mode_report_info().
 130          */
 131         smp_store_release(&entry->ptr, (s64)object);
 132
 133         read_unlock_irqrestore(&stack_ring.lock, flags);
 134 }
 135
 136 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
 137 {
 138         save_stack_info(cache, object, flags, false);
 139 }
 140
 141 void kasan_save_free_info(struct kmem_cache *cache, void *object)
 142 {
 143         save_stack_info(cache, object, GFP_NOWAIT, true);
 144 }