Commit | Line | Data |
---|---|---|
e886bf9d | 1 | // SPDX-License-Identifier: GPL-2.0 |
0b24becc | 2 | /* |
2bd926b4 | 3 | * This file contains core generic KASAN code. |
0b24becc AR |
4 | * |
5 | * Copyright (c) 2014 Samsung Electronics Co., Ltd. | |
2baf9e89 | 6 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> |
0b24becc | 7 | * |
5d0926ef | 8 | * Some code borrowed from https://github.com/xairy/kasan-prototype by |
5f21f3a8 | 9 | * Andrey Konovalov <andreyknvl@gmail.com> |
0b24becc AR |
10 | */ |
11 | ||
0b24becc | 12 | #include <linux/export.h> |
cd11016e | 13 | #include <linux/interrupt.h> |
0b24becc | 14 | #include <linux/init.h> |
cd11016e | 15 | #include <linux/kasan.h> |
0b24becc | 16 | #include <linux/kernel.h> |
2b830526 | 17 | #include <linux/kfence.h> |
45937254 | 18 | #include <linux/kmemleak.h> |
e3ae1163 | 19 | #include <linux/linkage.h> |
0b24becc | 20 | #include <linux/memblock.h> |
786a8959 | 21 | #include <linux/memory.h> |
0b24becc | 22 | #include <linux/mm.h> |
bebf56a1 | 23 | #include <linux/module.h> |
0b24becc AR |
24 | #include <linux/printk.h> |
25 | #include <linux/sched.h> | |
68db0cf1 | 26 | #include <linux/sched/task_stack.h> |
0b24becc AR |
27 | #include <linux/slab.h> |
28 | #include <linux/stacktrace.h> | |
29 | #include <linux/string.h> | |
30 | #include <linux/types.h> | |
a5af5aa8 | 31 | #include <linux/vmalloc.h> |
9f7d416c | 32 | #include <linux/bug.h> |
0b24becc AR |
33 | |
34 | #include "kasan.h" | |
0316bec2 | 35 | #include "../slab.h" |
0b24becc | 36 | |
0b24becc AR |
37 | /* |
38 | * All functions below always inlined so compiler could | |
39 | * perform better optimizations in each of __asan_loadX/__assn_storeX | |
40 | * depending on memory access size X. | |
41 | */ | |
42 | ||
43 | static __always_inline bool memory_is_poisoned_1(unsigned long addr) | |
44 | { | |
45 | s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr); | |
46 | ||
47 | if (unlikely(shadow_value)) { | |
1f600626 | 48 | s8 last_accessible_byte = addr & KASAN_GRANULE_MASK; |
0b24becc AR |
49 | return unlikely(last_accessible_byte >= shadow_value); |
50 | } | |
51 | ||
52 | return false; | |
53 | } | |
54 | ||
c634d807 AR |
55 | static __always_inline bool memory_is_poisoned_2_4_8(unsigned long addr, |
56 | unsigned long size) | |
0b24becc | 57 | { |
c634d807 | 58 | u8 *shadow_addr = (u8 *)kasan_mem_to_shadow((void *)addr); |
0b24becc | 59 | |
c634d807 AR |
60 | /* |
61 | * Access crosses 8(shadow size)-byte boundary. Such access maps | |
62 | * into 2 shadow bytes, so we need to check them both. | |
63 | */ | |
1f600626 | 64 | if (unlikely(((addr + size - 1) & KASAN_GRANULE_MASK) < size - 1)) |
c634d807 | 65 | return *shadow_addr || memory_is_poisoned_1(addr + size - 1); |
0b24becc | 66 | |
c634d807 | 67 | return memory_is_poisoned_1(addr + size - 1); |
0b24becc AR |
68 | } |
69 | ||
70 | static __always_inline bool memory_is_poisoned_16(unsigned long addr) | |
71 | { | |
c634d807 | 72 | u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); |
0b24becc | 73 | |
c634d807 | 74 | /* Unaligned 16-bytes access maps into 3 shadow bytes. */ |
1f600626 | 75 | if (unlikely(!IS_ALIGNED(addr, KASAN_GRANULE_SIZE))) |
c634d807 | 76 | return *shadow_addr || memory_is_poisoned_1(addr + 15); |
0b24becc | 77 | |
c634d807 | 78 | return *shadow_addr; |
0b24becc AR |
79 | } |
80 | ||
f5bd62cd | 81 | static __always_inline unsigned long bytes_is_nonzero(const u8 *start, |
0b24becc AR |
82 | size_t size) |
83 | { | |
84 | while (size) { | |
85 | if (unlikely(*start)) | |
86 | return (unsigned long)start; | |
87 | start++; | |
88 | size--; | |
89 | } | |
90 | ||
91 | return 0; | |
92 | } | |
93 | ||
f5bd62cd | 94 | static __always_inline unsigned long memory_is_nonzero(const void *start, |
0b24becc AR |
95 | const void *end) |
96 | { | |
97 | unsigned int words; | |
98 | unsigned long ret; | |
99 | unsigned int prefix = (unsigned long)start % 8; | |
100 | ||
101 | if (end - start <= 16) | |
f5bd62cd | 102 | return bytes_is_nonzero(start, end - start); |
0b24becc AR |
103 | |
104 | if (prefix) { | |
105 | prefix = 8 - prefix; | |
f5bd62cd | 106 | ret = bytes_is_nonzero(start, prefix); |
0b24becc AR |
107 | if (unlikely(ret)) |
108 | return ret; | |
109 | start += prefix; | |
110 | } | |
111 | ||
112 | words = (end - start) / 8; | |
113 | while (words) { | |
114 | if (unlikely(*(u64 *)start)) | |
f5bd62cd | 115 | return bytes_is_nonzero(start, 8); |
0b24becc AR |
116 | start += 8; |
117 | words--; | |
118 | } | |
119 | ||
f5bd62cd | 120 | return bytes_is_nonzero(start, (end - start) % 8); |
0b24becc AR |
121 | } |
122 | ||
123 | static __always_inline bool memory_is_poisoned_n(unsigned long addr, | |
124 | size_t size) | |
125 | { | |
126 | unsigned long ret; | |
127 | ||
f5bd62cd | 128 | ret = memory_is_nonzero(kasan_mem_to_shadow((void *)addr), |
0b24becc AR |
129 | kasan_mem_to_shadow((void *)addr + size - 1) + 1); |
130 | ||
131 | if (unlikely(ret)) { | |
132 | unsigned long last_byte = addr + size - 1; | |
133 | s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte); | |
134 | ||
135 | if (unlikely(ret != (unsigned long)last_shadow || | |
1f600626 | 136 | ((long)(last_byte & KASAN_GRANULE_MASK) >= *last_shadow))) |
0b24becc AR |
137 | return true; |
138 | } | |
139 | return false; | |
140 | } | |
141 | ||
142 | static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size) | |
143 | { | |
144 | if (__builtin_constant_p(size)) { | |
145 | switch (size) { | |
146 | case 1: | |
147 | return memory_is_poisoned_1(addr); | |
148 | case 2: | |
0b24becc | 149 | case 4: |
0b24becc | 150 | case 8: |
c634d807 | 151 | return memory_is_poisoned_2_4_8(addr, size); |
0b24becc AR |
152 | case 16: |
153 | return memory_is_poisoned_16(addr); | |
154 | default: | |
155 | BUILD_BUG(); | |
156 | } | |
157 | } | |
158 | ||
159 | return memory_is_poisoned_n(addr, size); | |
160 | } | |
161 | ||
f00748bf | 162 | static __always_inline bool check_region_inline(unsigned long addr, |
936bb4bb AR |
163 | size_t size, bool write, |
164 | unsigned long ret_ip) | |
0b24becc | 165 | { |
af3751f3 DA |
166 | if (!kasan_arch_is_ready()) |
167 | return true; | |
168 | ||
0b24becc | 169 | if (unlikely(size == 0)) |
b5f6e0fc | 170 | return true; |
0b24becc | 171 | |
8cceeff4 WW |
172 | if (unlikely(addr + size < addr)) |
173 | return !kasan_report(addr, size, write, ret_ip); | |
174 | ||
0b24becc AR |
175 | if (unlikely((void *)addr < |
176 | kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) { | |
8cceeff4 | 177 | return !kasan_report(addr, size, write, ret_ip); |
0b24becc AR |
178 | } |
179 | ||
180 | if (likely(!memory_is_poisoned(addr, size))) | |
b5f6e0fc | 181 | return true; |
0b24becc | 182 | |
8cceeff4 | 183 | return !kasan_report(addr, size, write, ret_ip); |
0b24becc AR |
184 | } |
185 | ||
f00748bf AK |
186 | bool kasan_check_range(unsigned long addr, size_t size, bool write, |
187 | unsigned long ret_ip) | |
936bb4bb | 188 | { |
f00748bf | 189 | return check_region_inline(addr, size, write, ret_ip); |
936bb4bb | 190 | } |
393f203f | 191 | |
611806b4 | 192 | bool kasan_byte_accessible(const void *addr) |
2cdbed63 AK |
193 | { |
194 | s8 shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(addr)); | |
195 | ||
611806b4 | 196 | return shadow_byte >= 0 && shadow_byte < KASAN_GRANULE_SIZE; |
2cdbed63 AK |
197 | } |
198 | ||
55834c59 AP |
199 | void kasan_cache_shrink(struct kmem_cache *cache) |
200 | { | |
f00748bf | 201 | kasan_quarantine_remove_cache(cache); |
55834c59 AP |
202 | } |
203 | ||
f9fa1d91 | 204 | void kasan_cache_shutdown(struct kmem_cache *cache) |
55834c59 | 205 | { |
f9e13c0a | 206 | if (!__kmem_cache_empty(cache)) |
f00748bf | 207 | kasan_quarantine_remove_cache(cache); |
55834c59 AP |
208 | } |
209 | ||
bebf56a1 AR |
210 | static void register_global(struct kasan_global *global) |
211 | { | |
1f600626 | 212 | size_t aligned_size = round_up(global->size, KASAN_GRANULE_SIZE); |
bebf56a1 | 213 | |
aa5c219c | 214 | kasan_unpoison(global->beg, global->size, false); |
bebf56a1 | 215 | |
f00748bf | 216 | kasan_poison(global->beg + aligned_size, |
cebd0eb2 | 217 | global->size_with_redzone - aligned_size, |
aa5c219c | 218 | KASAN_GLOBAL_REDZONE, false); |
bebf56a1 AR |
219 | } |
220 | ||
221 | void __asan_register_globals(struct kasan_global *globals, size_t size) | |
222 | { | |
223 | int i; | |
224 | ||
225 | for (i = 0; i < size; i++) | |
226 | register_global(&globals[i]); | |
227 | } | |
228 | EXPORT_SYMBOL(__asan_register_globals); | |
229 | ||
230 | void __asan_unregister_globals(struct kasan_global *globals, size_t size) | |
231 | { | |
232 | } | |
233 | EXPORT_SYMBOL(__asan_unregister_globals); | |
234 | ||
936bb4bb AR |
235 | #define DEFINE_ASAN_LOAD_STORE(size) \ |
236 | void __asan_load##size(unsigned long addr) \ | |
237 | { \ | |
f00748bf | 238 | check_region_inline(addr, size, false, _RET_IP_); \ |
936bb4bb AR |
239 | } \ |
240 | EXPORT_SYMBOL(__asan_load##size); \ | |
241 | __alias(__asan_load##size) \ | |
242 | void __asan_load##size##_noabort(unsigned long); \ | |
243 | EXPORT_SYMBOL(__asan_load##size##_noabort); \ | |
244 | void __asan_store##size(unsigned long addr) \ | |
245 | { \ | |
f00748bf | 246 | check_region_inline(addr, size, true, _RET_IP_); \ |
936bb4bb AR |
247 | } \ |
248 | EXPORT_SYMBOL(__asan_store##size); \ | |
249 | __alias(__asan_store##size) \ | |
250 | void __asan_store##size##_noabort(unsigned long); \ | |
0b24becc AR |
251 | EXPORT_SYMBOL(__asan_store##size##_noabort) |
252 | ||
253 | DEFINE_ASAN_LOAD_STORE(1); | |
254 | DEFINE_ASAN_LOAD_STORE(2); | |
255 | DEFINE_ASAN_LOAD_STORE(4); | |
256 | DEFINE_ASAN_LOAD_STORE(8); | |
257 | DEFINE_ASAN_LOAD_STORE(16); | |
258 | ||
259 | void __asan_loadN(unsigned long addr, size_t size) | |
260 | { | |
f00748bf | 261 | kasan_check_range(addr, size, false, _RET_IP_); |
0b24becc AR |
262 | } |
263 | EXPORT_SYMBOL(__asan_loadN); | |
264 | ||
265 | __alias(__asan_loadN) | |
266 | void __asan_loadN_noabort(unsigned long, size_t); | |
267 | EXPORT_SYMBOL(__asan_loadN_noabort); | |
268 | ||
269 | void __asan_storeN(unsigned long addr, size_t size) | |
270 | { | |
f00748bf | 271 | kasan_check_range(addr, size, true, _RET_IP_); |
0b24becc AR |
272 | } |
273 | EXPORT_SYMBOL(__asan_storeN); | |
274 | ||
275 | __alias(__asan_storeN) | |
276 | void __asan_storeN_noabort(unsigned long, size_t); | |
277 | EXPORT_SYMBOL(__asan_storeN_noabort); | |
278 | ||
279 | /* to shut up compiler complaints */ | |
280 | void __asan_handle_no_return(void) {} | |
281 | EXPORT_SYMBOL(__asan_handle_no_return); | |
786a8959 | 282 | |
342061ee PL |
283 | /* Emitted by compiler to poison alloca()ed objects. */ |
284 | void __asan_alloca_poison(unsigned long addr, size_t size) | |
285 | { | |
1f600626 | 286 | size_t rounded_up_size = round_up(size, KASAN_GRANULE_SIZE); |
342061ee PL |
287 | size_t padding_size = round_up(size, KASAN_ALLOCA_REDZONE_SIZE) - |
288 | rounded_up_size; | |
1f600626 | 289 | size_t rounded_down_size = round_down(size, KASAN_GRANULE_SIZE); |
342061ee PL |
290 | |
291 | const void *left_redzone = (const void *)(addr - | |
292 | KASAN_ALLOCA_REDZONE_SIZE); | |
293 | const void *right_redzone = (const void *)(addr + rounded_up_size); | |
294 | ||
295 | WARN_ON(!IS_ALIGNED(addr, KASAN_ALLOCA_REDZONE_SIZE)); | |
296 | ||
f00748bf | 297 | kasan_unpoison((const void *)(addr + rounded_down_size), |
aa5c219c | 298 | size - rounded_down_size, false); |
f00748bf | 299 | kasan_poison(left_redzone, KASAN_ALLOCA_REDZONE_SIZE, |
aa5c219c | 300 | KASAN_ALLOCA_LEFT, false); |
f00748bf | 301 | kasan_poison(right_redzone, padding_size + KASAN_ALLOCA_REDZONE_SIZE, |
aa5c219c | 302 | KASAN_ALLOCA_RIGHT, false); |
342061ee PL |
303 | } |
304 | EXPORT_SYMBOL(__asan_alloca_poison); | |
305 | ||
306 | /* Emitted by compiler to unpoison alloca()ed areas when the stack unwinds. */ | |
307 | void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom) | |
308 | { | |
309 | if (unlikely(!stack_top || stack_top > stack_bottom)) | |
310 | return; | |
311 | ||
aa5c219c | 312 | kasan_unpoison(stack_top, stack_bottom - stack_top, false); |
342061ee PL |
313 | } |
314 | EXPORT_SYMBOL(__asan_allocas_unpoison); | |
315 | ||
d321599c AP |
316 | /* Emitted by the compiler to [un]poison local variables. */ |
317 | #define DEFINE_ASAN_SET_SHADOW(byte) \ | |
318 | void __asan_set_shadow_##byte(const void *addr, size_t size) \ | |
319 | { \ | |
320 | __memset((void *)addr, 0x##byte, size); \ | |
321 | } \ | |
322 | EXPORT_SYMBOL(__asan_set_shadow_##byte) | |
323 | ||
324 | DEFINE_ASAN_SET_SHADOW(00); | |
325 | DEFINE_ASAN_SET_SHADOW(f1); | |
326 | DEFINE_ASAN_SET_SHADOW(f2); | |
327 | DEFINE_ASAN_SET_SHADOW(f3); | |
328 | DEFINE_ASAN_SET_SHADOW(f5); | |
329 | DEFINE_ASAN_SET_SHADOW(f8); | |
26e760c9 | 330 | |
3b7f8813 AK |
331 | /* Only allow cache merging when no per-object metadata is present. */ |
332 | slab_flags_t kasan_never_merge(void) | |
333 | { | |
334 | if (!kasan_requires_meta()) | |
335 | return 0; | |
336 | return SLAB_KASAN; | |
337 | } | |
338 | ||
5935143d AK |
339 | /* |
340 | * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. | |
341 | * For larger allocations larger redzones are used. | |
342 | */ | |
343 | static inline unsigned int optimal_redzone(unsigned int object_size) | |
344 | { | |
345 | return | |
346 | object_size <= 64 - 16 ? 16 : | |
347 | object_size <= 128 - 32 ? 32 : | |
348 | object_size <= 512 - 64 ? 64 : | |
349 | object_size <= 4096 - 128 ? 128 : | |
350 | object_size <= (1 << 14) - 256 ? 256 : | |
351 | object_size <= (1 << 15) - 512 ? 512 : | |
352 | object_size <= (1 << 16) - 1024 ? 1024 : 2048; | |
353 | } | |
354 | ||
682ed089 AK |
355 | void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, |
356 | slab_flags_t *flags) | |
5935143d AK |
357 | { |
358 | unsigned int ok_size; | |
359 | unsigned int optimal_size; | |
360 | ||
682ed089 AK |
361 | if (!kasan_requires_meta()) |
362 | return; | |
363 | ||
364 | /* | |
365 | * SLAB_KASAN is used to mark caches that are sanitized by KASAN | |
366 | * and that thus have per-object metadata. | |
367 | * Currently this flag is used in two places: | |
368 | * 1. In slab_ksize() to account for per-object metadata when | |
369 | * calculating the size of the accessible memory within the object. | |
370 | * 2. In slab_common.c via kasan_never_merge() to prevent merging of | |
371 | * caches with per-object metadata. | |
372 | */ | |
373 | *flags |= SLAB_KASAN; | |
374 | ||
5935143d AK |
375 | ok_size = *size; |
376 | ||
377 | /* Add alloc meta into redzone. */ | |
378 | cache->kasan_info.alloc_meta_offset = *size; | |
379 | *size += sizeof(struct kasan_alloc_meta); | |
380 | ||
381 | /* | |
382 | * If alloc meta doesn't fit, don't add it. | |
383 | * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal | |
384 | * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for | |
385 | * larger sizes. | |
386 | */ | |
387 | if (*size > KMALLOC_MAX_SIZE) { | |
388 | cache->kasan_info.alloc_meta_offset = 0; | |
389 | *size = ok_size; | |
390 | /* Continue, since free meta might still fit. */ | |
391 | } | |
392 | ||
5935143d AK |
393 | /* |
394 | * Add free meta into redzone when it's not possible to store | |
395 | * it in the object. This is the case when: | |
396 | * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can | |
397 | * be touched after it was freed, or | |
398 | * 2. Object has a constructor, which means it's expected to | |
399 | * retain its content until the next allocation, or | |
400 | * 3. Object is too small. | |
401 | * Otherwise cache->kasan_info.free_meta_offset = 0 is implied. | |
402 | */ | |
403 | if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor || | |
404 | cache->object_size < sizeof(struct kasan_free_meta)) { | |
405 | ok_size = *size; | |
406 | ||
407 | cache->kasan_info.free_meta_offset = *size; | |
408 | *size += sizeof(struct kasan_free_meta); | |
409 | ||
410 | /* If free meta doesn't fit, don't add it. */ | |
411 | if (*size > KMALLOC_MAX_SIZE) { | |
412 | cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META; | |
413 | *size = ok_size; | |
414 | } | |
415 | } | |
416 | ||
417 | /* Calculate size with optimal redzone. */ | |
418 | optimal_size = cache->object_size + optimal_redzone(cache->object_size); | |
419 | /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */ | |
420 | if (optimal_size > KMALLOC_MAX_SIZE) | |
421 | optimal_size = KMALLOC_MAX_SIZE; | |
422 | /* Use optimal size if the size with added metas is not large enough. */ | |
423 | if (*size < optimal_size) | |
424 | *size = optimal_size; | |
425 | } | |
426 | ||
2f356801 AK |
427 | struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, |
428 | const void *object) | |
429 | { | |
430 | if (!cache->kasan_info.alloc_meta_offset) | |
431 | return NULL; | |
432 | return (void *)object + cache->kasan_info.alloc_meta_offset; | |
433 | } | |
434 | ||
435 | struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, | |
436 | const void *object) | |
437 | { | |
438 | BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); | |
439 | if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META) | |
440 | return NULL; | |
441 | return (void *)object + cache->kasan_info.free_meta_offset; | |
442 | } | |
443 | ||
836daba0 AK |
444 | void kasan_init_object_meta(struct kmem_cache *cache, const void *object) |
445 | { | |
446 | struct kasan_alloc_meta *alloc_meta; | |
447 | ||
448 | alloc_meta = kasan_get_alloc_meta(cache, object); | |
449 | if (alloc_meta) | |
450 | __memset(alloc_meta, 0, sizeof(*alloc_meta)); | |
451 | } | |
452 | ||
5d1ba310 | 453 | size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object) |
f372bde9 | 454 | { |
5d1ba310 FT |
455 | struct kasan_cache *info = &cache->kasan_info; |
456 | ||
f372bde9 AK |
457 | if (!kasan_requires_meta()) |
458 | return 0; | |
5d1ba310 FT |
459 | |
460 | if (in_object) | |
461 | return (info->free_meta_offset ? | |
462 | 0 : sizeof(struct kasan_free_meta)); | |
463 | else | |
464 | return (info->alloc_meta_offset ? | |
465 | sizeof(struct kasan_alloc_meta) : 0) + | |
466 | ((info->free_meta_offset && | |
467 | info->free_meta_offset != KASAN_NO_FREE_META) ? | |
468 | sizeof(struct kasan_free_meta) : 0); | |
f372bde9 AK |
469 | } |
470 | ||
7cb3007c | 471 | static void __kasan_record_aux_stack(void *addr, bool can_alloc) |
26e760c9 | 472 | { |
6e48a966 | 473 | struct slab *slab = kasan_addr_to_slab(addr); |
26e760c9 | 474 | struct kmem_cache *cache; |
6476792f | 475 | struct kasan_alloc_meta *alloc_meta; |
26e760c9 WW |
476 | void *object; |
477 | ||
6e48a966 | 478 | if (is_kfence_address(addr) || !slab) |
26e760c9 WW |
479 | return; |
480 | ||
6e48a966 MWO |
481 | cache = slab->slab_cache; |
482 | object = nearest_obj(cache, slab, addr); | |
6476792f | 483 | alloc_meta = kasan_get_alloc_meta(cache, object); |
13384f61 WW |
484 | if (!alloc_meta) |
485 | return; | |
26e760c9 | 486 | |
6476792f | 487 | alloc_meta->aux_stack[1] = alloc_meta->aux_stack[0]; |
7cb3007c ME |
488 | alloc_meta->aux_stack[0] = kasan_save_stack(GFP_NOWAIT, can_alloc); |
489 | } | |
490 | ||
491 | void kasan_record_aux_stack(void *addr) | |
492 | { | |
493 | return __kasan_record_aux_stack(addr, true); | |
494 | } | |
495 | ||
496 | void kasan_record_aux_stack_noalloc(void *addr) | |
497 | { | |
498 | return __kasan_record_aux_stack(addr, false); | |
26e760c9 | 499 | } |
e4b7818b | 500 | |
ccf643e6 AK |
501 | void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags) |
502 | { | |
503 | struct kasan_alloc_meta *alloc_meta; | |
504 | ||
505 | alloc_meta = kasan_get_alloc_meta(cache, object); | |
506 | if (alloc_meta) | |
507 | kasan_set_track(&alloc_meta->alloc_track, flags); | |
508 | } | |
509 | ||
6b074349 | 510 | void kasan_save_free_info(struct kmem_cache *cache, void *object) |
e4b7818b WW |
511 | { |
512 | struct kasan_free_meta *free_meta; | |
513 | ||
6476792f | 514 | free_meta = kasan_get_free_meta(cache, object); |
97593cad AK |
515 | if (!free_meta) |
516 | return; | |
e4b7818b | 517 | |
97593cad AK |
518 | kasan_set_track(&free_meta->free_track, GFP_NOWAIT); |
519 | /* The object was freed and has free track set. */ | |
06bc4cf6 | 520 | *(u8 *)kasan_mem_to_shadow(object) = KASAN_SLAB_FREETRACK; |
e4b7818b | 521 | } |