mm/kmsan/instrumentation.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KMSAN compiler API.
   4  *
   5  * This file implements __msan_XXX hooks that Clang inserts into the code
   6  * compiled with -fsanitize=kernel-memory.
   7  * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN
   8  * instrumentation works.
   9  *
  10  * Copyright (C) 2017-2022 Google LLC
  11  * Author: Alexander Potapenko <glider@google.com>
  12  *
  13  */
  14
  15 #include "kmsan.h"
  16 #include <linux/gfp.h>
  17 #include <linux/kmsan_string.h>
  18 #include <linux/mm.h>
  19 #include <linux/uaccess.h>
  20
  21 static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store)
  22 {
  23         if ((u64)addr < TASK_SIZE)
  24                 return true;
  25         if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW))
  26                 return true;
  27         return false;
  28 }
  29
  30 static inline struct shadow_origin_ptr
  31 get_shadow_origin_ptr(void *addr, u64 size, bool store)
  32 {
  33         unsigned long ua_flags = user_access_save();
  34         struct shadow_origin_ptr ret;
  35
  36         ret = kmsan_get_shadow_origin_ptr(addr, size, store);
  37         user_access_restore(ua_flags);
  38         return ret;
  39 }
  40
  41 /*
  42  * KMSAN instrumentation functions follow. They are not declared elsewhere in
  43  * the kernel code, so they are preceded by prototypes, to silence
  44  * -Wmissing-prototypes warnings.
  45  */
  46
  47 /* Get shadow and origin pointers for a memory load with non-standard size. */
  48 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
  49                                                         uintptr_t size);
  50 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
  51                                                         uintptr_t size)
  52 {
  53         return get_shadow_origin_ptr(addr, size, /*store*/ false);
  54 }
  55 EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);
  56
  57 /* Get shadow and origin pointers for a memory store with non-standard size. */
  58 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
  59                                                          uintptr_t size);
  60 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
  61                                                          uintptr_t size)
  62 {
  63         return get_shadow_origin_ptr(addr, size, /*store*/ true);
  64 }
  65 EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);
  66
  67 /*
  68  * Declare functions that obtain shadow/origin pointers for loads and stores
  69  * with fixed size.
  70  */
  71 #define DECLARE_METADATA_PTR_GETTER(size)                                  \
  72         struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(      \
  73                 void *addr);                                               \
  74         struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(      \
  75                 void *addr)                                                \
  76         {                                                                  \
  77                 return get_shadow_origin_ptr(addr, size, /*store*/ false); \
  78         }                                                                  \
  79         EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size);                \
  80         struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(     \
  81                 void *addr);                                               \
  82         struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(     \
  83                 void *addr)                                                \
  84         {                                                                  \
  85                 return get_shadow_origin_ptr(addr, size, /*store*/ true);  \
  86         }                                                                  \
  87         EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)
  88
  89 DECLARE_METADATA_PTR_GETTER(1);
  90 DECLARE_METADATA_PTR_GETTER(2);
  91 DECLARE_METADATA_PTR_GETTER(4);
  92 DECLARE_METADATA_PTR_GETTER(8);
  93
  94 /*
  95  * Handle a memory store performed by inline assembly. KMSAN conservatively
  96  * attempts to unpoison the outputs of asm() directives to prevent false
  97  * positives caused by missed stores.
  98  *
  99  * __msan_instrument_asm_store() may be called for inline assembly code when
 100  * entering or leaving IRQ. We omit the check for kmsan_in_runtime() to ensure
 101  * the memory written to in these cases is also marked as initialized.
 102  */
 103 void __msan_instrument_asm_store(void *addr, uintptr_t size);
 104 void __msan_instrument_asm_store(void *addr, uintptr_t size)
 105 {
 106         unsigned long ua_flags;
 107
 108         if (!kmsan_enabled)
 109                 return;
 110
 111         ua_flags = user_access_save();
 112         /*
 113          * Most of the accesses are below 32 bytes. The two exceptions so far
 114          * are clwb() (64 bytes) and FPU state (512 bytes).
 115          * It's unlikely that the assembly will touch more than 512 bytes.
 116          */
 117         if (size > 512) {
 118                 WARN_ONCE(1, "assembly store size too big: %ld\n", size);
 119                 size = 8;
 120         }
 121         if (is_bad_asm_addr(addr, size, /*is_store*/ true)) {
 122                 user_access_restore(ua_flags);
 123                 return;
 124         }
 125         /* Unpoisoning the memory on best effort. */
 126         kmsan_internal_unpoison_memory(addr, size, /*checked*/ false);
 127         user_access_restore(ua_flags);
 128 }
 129 EXPORT_SYMBOL(__msan_instrument_asm_store);
 130
 131 /*
 132  * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset
 133  * intrinsics with calls to respective __msan_ functions. We use
 134  * get_param0_metadata() and set_retval_metadata() to store the shadow/origin
 135  * values for the destination argument of these functions and use them for the
 136  * functions' return values.
 137  */
 138 static inline void get_param0_metadata(u64 *shadow,
 139                                        depot_stack_handle_t *origin)
 140 {
 141         struct kmsan_ctx *ctx = kmsan_get_context();
 142
 143         *shadow = *(u64 *)(ctx->cstate.param_tls);
 144         *origin = ctx->cstate.param_origin_tls[0];
 145 }
 146
 147 static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin)
 148 {
 149         struct kmsan_ctx *ctx = kmsan_get_context();
 150
 151         *(u64 *)(ctx->cstate.retval_tls) = shadow;
 152         ctx->cstate.retval_origin_tls = origin;
 153 }
 154
 155 /* Handle llvm.memmove intrinsic. */
 156 void *__msan_memmove(void *dst, const void *src, uintptr_t n);
 157 void *__msan_memmove(void *dst, const void *src, uintptr_t n)
 158 {
 159         depot_stack_handle_t origin;
 160         void *result;
 161         u64 shadow;
 162
 163         get_param0_metadata(&shadow, &origin);
 164         result = __memmove(dst, src, n);
 165         if (!n)
 166                 /* Some people call memmove() with zero length. */
 167                 return result;
 168         if (!kmsan_enabled || kmsan_in_runtime())
 169                 return result;
 170
 171         kmsan_enter_runtime();
 172         kmsan_internal_memmove_metadata(dst, (void *)src, n);
 173         kmsan_leave_runtime();
 174
 175         set_retval_metadata(shadow, origin);
 176         return result;
 177 }
 178 EXPORT_SYMBOL(__msan_memmove);
 179
 180 /* Handle llvm.memcpy intrinsic. */
 181 void *__msan_memcpy(void *dst, const void *src, uintptr_t n);
 182 void *__msan_memcpy(void *dst, const void *src, uintptr_t n)
 183 {
 184         depot_stack_handle_t origin;
 185         void *result;
 186         u64 shadow;
 187
 188         get_param0_metadata(&shadow, &origin);
 189         result = __memcpy(dst, src, n);
 190         if (!n)
 191                 /* Some people call memcpy() with zero length. */
 192                 return result;
 193
 194         if (!kmsan_enabled || kmsan_in_runtime())
 195                 return result;
 196
 197         kmsan_enter_runtime();
 198         /* Using memmove instead of memcpy doesn't affect correctness. */
 199         kmsan_internal_memmove_metadata(dst, (void *)src, n);
 200         kmsan_leave_runtime();
 201
 202         set_retval_metadata(shadow, origin);
 203         return result;
 204 }
 205 EXPORT_SYMBOL(__msan_memcpy);
 206
 207 /* Handle llvm.memset intrinsic. */
 208 void *__msan_memset(void *dst, int c, uintptr_t n);
 209 void *__msan_memset(void *dst, int c, uintptr_t n)
 210 {
 211         depot_stack_handle_t origin;
 212         void *result;
 213         u64 shadow;
 214
 215         get_param0_metadata(&shadow, &origin);
 216         result = __memset(dst, c, n);
 217         if (!kmsan_enabled || kmsan_in_runtime())
 218                 return result;
 219
 220         kmsan_enter_runtime();
 221         /*
 222          * Clang doesn't pass parameter metadata here, so it is impossible to
 223          * use shadow of @c to set up the shadow for @dst.
 224          */
 225         kmsan_internal_unpoison_memory(dst, n, /*checked*/ false);
 226         kmsan_leave_runtime();
 227
 228         set_retval_metadata(shadow, origin);
 229         return result;
 230 }
 231 EXPORT_SYMBOL(__msan_memset);
 232
 233 /*
 234  * Create a new origin from an old one. This is done when storing an
 235  * uninitialized value to memory. When reporting an error, KMSAN unrolls and
 236  * prints the whole chain of stores that preceded the use of this value.
 237  */
 238 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin);
 239 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)
 240 {
 241         depot_stack_handle_t ret = 0;
 242         unsigned long ua_flags;
 243
 244         if (!kmsan_enabled || kmsan_in_runtime())
 245                 return ret;
 246
 247         ua_flags = user_access_save();
 248
 249         /* Creating new origins may allocate memory. */
 250         kmsan_enter_runtime();
 251         ret = kmsan_internal_chain_origin(origin);
 252         kmsan_leave_runtime();
 253         user_access_restore(ua_flags);
 254         return ret;
 255 }
 256 EXPORT_SYMBOL(__msan_chain_origin);
 257
 258 /* Poison a local variable when entering a function. */
 259 void __msan_poison_alloca(void *address, uintptr_t size, char *descr);
 260 void __msan_poison_alloca(void *address, uintptr_t size, char *descr)
 261 {
 262         depot_stack_handle_t handle;
 263         unsigned long entries[4];
 264         unsigned long ua_flags;
 265
 266         if (!kmsan_enabled || kmsan_in_runtime())
 267                 return;
 268
 269         ua_flags = user_access_save();
 270         entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;
 271         entries[1] = (u64)descr;
 272         entries[2] = (u64)__builtin_return_address(0);
 273         /*
 274          * With frame pointers enabled, it is possible to quickly fetch the
 275          * second frame of the caller stack without calling the unwinder.
 276          * Without them, simply do not bother.
 277          */
 278         if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER))
 279                 entries[3] = (u64)__builtin_return_address(1);
 280         else
 281                 entries[3] = 0;
 282
 283         /* stack_depot_save() may allocate memory. */
 284         kmsan_enter_runtime();
 285         handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_ATOMIC);
 286         kmsan_leave_runtime();
 287
 288         kmsan_internal_set_shadow_origin(address, size, -1, handle,
 289                                          /*checked*/ true);
 290         user_access_restore(ua_flags);
 291 }
 292 EXPORT_SYMBOL(__msan_poison_alloca);
 293
 294 /* Unpoison a local variable. */
 295 void __msan_unpoison_alloca(void *address, uintptr_t size);
 296 void __msan_unpoison_alloca(void *address, uintptr_t size)
 297 {
 298         if (!kmsan_enabled || kmsan_in_runtime())
 299                 return;
 300
 301         kmsan_enter_runtime();
 302         kmsan_internal_unpoison_memory(address, size, /*checked*/ true);
 303         kmsan_leave_runtime();
 304 }
 305 EXPORT_SYMBOL(__msan_unpoison_alloca);
 306
 307 /*
 308  * Report that an uninitialized value with the given origin was used in a way
 309  * that constituted undefined behavior.
 310  */
 311 void __msan_warning(u32 origin);
 312 void __msan_warning(u32 origin)
 313 {
 314         if (!kmsan_enabled || kmsan_in_runtime())
 315                 return;
 316         kmsan_enter_runtime();
 317         kmsan_report(origin, /*address*/ 0, /*size*/ 0,
 318                      /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ 0,
 319                      REASON_ANY);
 320         kmsan_leave_runtime();
 321 }
 322 EXPORT_SYMBOL(__msan_warning);
 323
 324 /*
 325  * At the beginning of an instrumented function, obtain the pointer to
 326  * `struct kmsan_context_state` holding the metadata for function parameters.
 327  */
 328 struct kmsan_context_state *__msan_get_context_state(void);
 329 struct kmsan_context_state *__msan_get_context_state(void)
 330 {
 331         return &kmsan_get_context()->cstate;
 332 }
 333 EXPORT_SYMBOL(__msan_get_context_state);