Commit | Line | Data |
---|---|---|
e886bf9d | 1 | // SPDX-License-Identifier: GPL-2.0 |
bffa986c AK |
2 | /* |
3 | * This file contains common generic and tag-based KASAN code. | |
4 | * | |
5 | * Copyright (c) 2014 Samsung Electronics Co., Ltd. | |
6 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> | |
7 | * | |
8 | * Some code borrowed from https://github.com/xairy/kasan-prototype by | |
9 | * Andrey Konovalov <andreyknvl@gmail.com> | |
bffa986c AK |
10 | */ |
11 | ||
12 | #include <linux/export.h> | |
bffa986c AK |
13 | #include <linux/init.h> |
14 | #include <linux/kasan.h> | |
15 | #include <linux/kernel.h> | |
16 | #include <linux/kmemleak.h> | |
17 | #include <linux/linkage.h> | |
18 | #include <linux/memblock.h> | |
19 | #include <linux/memory.h> | |
20 | #include <linux/mm.h> | |
21 | #include <linux/module.h> | |
22 | #include <linux/printk.h> | |
23 | #include <linux/sched.h> | |
24 | #include <linux/sched/task_stack.h> | |
25 | #include <linux/slab.h> | |
26 | #include <linux/stacktrace.h> | |
27 | #include <linux/string.h> | |
28 | #include <linux/types.h> | |
29 | #include <linux/vmalloc.h> | |
30 | #include <linux/bug.h> | |
31 | ||
2e7d3170 | 32 | #include <asm/cacheflush.h> |
3c5c3cfb DA |
33 | #include <asm/tlbflush.h> |
34 | ||
bffa986c AK |
35 | #include "kasan.h" |
36 | #include "../slab.h" | |
37 | ||
26e760c9 | 38 | depot_stack_handle_t kasan_save_stack(gfp_t flags) |
bffa986c AK |
39 | { |
40 | unsigned long entries[KASAN_STACK_DEPTH]; | |
880e049c | 41 | unsigned int nr_entries; |
bffa986c | 42 | |
880e049c TG |
43 | nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); |
44 | nr_entries = filter_irq_stacks(entries, nr_entries); | |
45 | return stack_depot_save(entries, nr_entries, flags); | |
bffa986c AK |
46 | } |
47 | ||
e4b7818b | 48 | void kasan_set_track(struct kasan_track *track, gfp_t flags) |
bffa986c AK |
49 | { |
50 | track->pid = current->pid; | |
26e760c9 | 51 | track->stack = kasan_save_stack(flags); |
bffa986c AK |
52 | } |
53 | ||
54 | void kasan_enable_current(void) | |
55 | { | |
56 | current->kasan_depth++; | |
57 | } | |
58 | ||
59 | void kasan_disable_current(void) | |
60 | { | |
61 | current->kasan_depth--; | |
62 | } | |
63 | ||
b5f6e0fc | 64 | bool __kasan_check_read(const volatile void *p, unsigned int size) |
bffa986c | 65 | { |
b5f6e0fc | 66 | return check_memory_region((unsigned long)p, size, false, _RET_IP_); |
bffa986c | 67 | } |
7d8ad890 | 68 | EXPORT_SYMBOL(__kasan_check_read); |
bffa986c | 69 | |
b5f6e0fc | 70 | bool __kasan_check_write(const volatile void *p, unsigned int size) |
bffa986c | 71 | { |
b5f6e0fc | 72 | return check_memory_region((unsigned long)p, size, true, _RET_IP_); |
bffa986c | 73 | } |
7d8ad890 | 74 | EXPORT_SYMBOL(__kasan_check_write); |
bffa986c AK |
75 | |
76 | #undef memset | |
77 | void *memset(void *addr, int c, size_t len) | |
78 | { | |
8cceeff4 WW |
79 | if (!check_memory_region((unsigned long)addr, len, true, _RET_IP_)) |
80 | return NULL; | |
bffa986c AK |
81 | |
82 | return __memset(addr, c, len); | |
83 | } | |
84 | ||
57ee58e3 | 85 | #ifdef __HAVE_ARCH_MEMMOVE |
bffa986c AK |
86 | #undef memmove |
87 | void *memmove(void *dest, const void *src, size_t len) | |
88 | { | |
8cceeff4 WW |
89 | if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || |
90 | !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) | |
91 | return NULL; | |
bffa986c AK |
92 | |
93 | return __memmove(dest, src, len); | |
94 | } | |
57ee58e3 | 95 | #endif |
bffa986c AK |
96 | |
97 | #undef memcpy | |
98 | void *memcpy(void *dest, const void *src, size_t len) | |
99 | { | |
8cceeff4 WW |
100 | if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || |
101 | !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) | |
102 | return NULL; | |
bffa986c AK |
103 | |
104 | return __memcpy(dest, src, len); | |
105 | } | |
106 | ||
107 | /* | |
108 | * Poisons the shadow memory for 'size' bytes starting from 'addr'. | |
1f600626 | 109 | * Memory addresses should be aligned to KASAN_GRANULE_SIZE. |
bffa986c | 110 | */ |
cebd0eb2 | 111 | void poison_range(const void *address, size_t size, u8 value) |
bffa986c AK |
112 | { |
113 | void *shadow_start, *shadow_end; | |
114 | ||
7f94ffbc AK |
115 | /* |
116 | * Perform shadow offset calculation based on untagged address, as | |
117 | * some of the callers (e.g. kasan_poison_object_data) pass tagged | |
118 | * addresses to this function. | |
119 | */ | |
120 | address = reset_tag(address); | |
121 | ||
bffa986c AK |
122 | shadow_start = kasan_mem_to_shadow(address); |
123 | shadow_end = kasan_mem_to_shadow(address + size); | |
124 | ||
125 | __memset(shadow_start, value, shadow_end - shadow_start); | |
126 | } | |
127 | ||
cebd0eb2 | 128 | void unpoison_range(const void *address, size_t size) |
bffa986c | 129 | { |
7f94ffbc AK |
130 | u8 tag = get_tag(address); |
131 | ||
132 | /* | |
133 | * Perform shadow offset calculation based on untagged address, as | |
134 | * some of the callers (e.g. kasan_unpoison_object_data) pass tagged | |
135 | * addresses to this function. | |
136 | */ | |
137 | address = reset_tag(address); | |
138 | ||
cebd0eb2 | 139 | poison_range(address, size, tag); |
bffa986c | 140 | |
1f600626 | 141 | if (size & KASAN_GRANULE_MASK) { |
bffa986c | 142 | u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size); |
7f94ffbc AK |
143 | |
144 | if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) | |
145 | *shadow = tag; | |
146 | else | |
1f600626 | 147 | *shadow = size & KASAN_GRANULE_MASK; |
bffa986c AK |
148 | } |
149 | } | |
150 | ||
cebd0eb2 AK |
151 | void kasan_unpoison_range(const void *address, size_t size) |
152 | { | |
153 | unpoison_range(address, size); | |
154 | } | |
155 | ||
bffa986c AK |
156 | static void __kasan_unpoison_stack(struct task_struct *task, const void *sp) |
157 | { | |
158 | void *base = task_stack_page(task); | |
159 | size_t size = sp - base; | |
160 | ||
cebd0eb2 | 161 | unpoison_range(base, size); |
bffa986c AK |
162 | } |
163 | ||
164 | /* Unpoison the entire stack for a task. */ | |
165 | void kasan_unpoison_task_stack(struct task_struct *task) | |
166 | { | |
167 | __kasan_unpoison_stack(task, task_stack_page(task) + THREAD_SIZE); | |
168 | } | |
169 | ||
170 | /* Unpoison the stack for the current task beyond a watermark sp value. */ | |
171 | asmlinkage void kasan_unpoison_task_stack_below(const void *watermark) | |
172 | { | |
173 | /* | |
174 | * Calculate the task stack base address. Avoid using 'current' | |
175 | * because this function is called by early resume code which hasn't | |
176 | * yet set up the percpu register (%gs). | |
177 | */ | |
178 | void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1)); | |
179 | ||
cebd0eb2 | 180 | unpoison_range(base, watermark - base); |
bffa986c AK |
181 | } |
182 | ||
bffa986c AK |
183 | void kasan_alloc_pages(struct page *page, unsigned int order) |
184 | { | |
2813b9c0 AK |
185 | u8 tag; |
186 | unsigned long i; | |
187 | ||
7f94ffbc AK |
188 | if (unlikely(PageHighMem(page))) |
189 | return; | |
2813b9c0 AK |
190 | |
191 | tag = random_tag(); | |
192 | for (i = 0; i < (1 << order); i++) | |
193 | page_kasan_tag_set(page + i, tag); | |
cebd0eb2 | 194 | unpoison_range(page_address(page), PAGE_SIZE << order); |
bffa986c AK |
195 | } |
196 | ||
197 | void kasan_free_pages(struct page *page, unsigned int order) | |
198 | { | |
199 | if (likely(!PageHighMem(page))) | |
cebd0eb2 | 200 | poison_range(page_address(page), |
bffa986c AK |
201 | PAGE_SIZE << order, |
202 | KASAN_FREE_PAGE); | |
203 | } | |
204 | ||
205 | /* | |
206 | * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. | |
207 | * For larger allocations larger redzones are used. | |
208 | */ | |
209 | static inline unsigned int optimal_redzone(unsigned int object_size) | |
210 | { | |
7f94ffbc AK |
211 | if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) |
212 | return 0; | |
213 | ||
bffa986c AK |
214 | return |
215 | object_size <= 64 - 16 ? 16 : | |
216 | object_size <= 128 - 32 ? 32 : | |
217 | object_size <= 512 - 64 ? 64 : | |
218 | object_size <= 4096 - 128 ? 128 : | |
219 | object_size <= (1 << 14) - 256 ? 256 : | |
220 | object_size <= (1 << 15) - 512 ? 512 : | |
221 | object_size <= (1 << 16) - 1024 ? 1024 : 2048; | |
222 | } | |
223 | ||
224 | void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, | |
225 | slab_flags_t *flags) | |
226 | { | |
227 | unsigned int orig_size = *size; | |
7f94ffbc | 228 | unsigned int redzone_size; |
bffa986c AK |
229 | int redzone_adjust; |
230 | ||
231 | /* Add alloc meta. */ | |
232 | cache->kasan_info.alloc_meta_offset = *size; | |
233 | *size += sizeof(struct kasan_alloc_meta); | |
234 | ||
235 | /* Add free meta. */ | |
7f94ffbc AK |
236 | if (IS_ENABLED(CONFIG_KASAN_GENERIC) && |
237 | (cache->flags & SLAB_TYPESAFE_BY_RCU || cache->ctor || | |
238 | cache->object_size < sizeof(struct kasan_free_meta))) { | |
bffa986c AK |
239 | cache->kasan_info.free_meta_offset = *size; |
240 | *size += sizeof(struct kasan_free_meta); | |
241 | } | |
bffa986c | 242 | |
7f94ffbc AK |
243 | redzone_size = optimal_redzone(cache->object_size); |
244 | redzone_adjust = redzone_size - (*size - cache->object_size); | |
bffa986c AK |
245 | if (redzone_adjust > 0) |
246 | *size += redzone_adjust; | |
247 | ||
248 | *size = min_t(unsigned int, KMALLOC_MAX_SIZE, | |
7f94ffbc | 249 | max(*size, cache->object_size + redzone_size)); |
bffa986c AK |
250 | |
251 | /* | |
252 | * If the metadata doesn't fit, don't enable KASAN at all. | |
253 | */ | |
254 | if (*size <= cache->kasan_info.alloc_meta_offset || | |
255 | *size <= cache->kasan_info.free_meta_offset) { | |
256 | cache->kasan_info.alloc_meta_offset = 0; | |
257 | cache->kasan_info.free_meta_offset = 0; | |
258 | *size = orig_size; | |
259 | return; | |
260 | } | |
261 | ||
262 | *flags |= SLAB_KASAN; | |
263 | } | |
264 | ||
265 | size_t kasan_metadata_size(struct kmem_cache *cache) | |
266 | { | |
267 | return (cache->kasan_info.alloc_meta_offset ? | |
268 | sizeof(struct kasan_alloc_meta) : 0) + | |
269 | (cache->kasan_info.free_meta_offset ? | |
270 | sizeof(struct kasan_free_meta) : 0); | |
271 | } | |
272 | ||
273 | struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache, | |
274 | const void *object) | |
275 | { | |
bffa986c AK |
276 | return (void *)object + cache->kasan_info.alloc_meta_offset; |
277 | } | |
278 | ||
279 | struct kasan_free_meta *get_free_info(struct kmem_cache *cache, | |
280 | const void *object) | |
281 | { | |
282 | BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); | |
283 | return (void *)object + cache->kasan_info.free_meta_offset; | |
284 | } | |
285 | ||
286 | void kasan_poison_slab(struct page *page) | |
287 | { | |
2813b9c0 AK |
288 | unsigned long i; |
289 | ||
d8c6546b | 290 | for (i = 0; i < compound_nr(page); i++) |
2813b9c0 | 291 | page_kasan_tag_reset(page + i); |
cebd0eb2 AK |
292 | poison_range(page_address(page), page_size(page), |
293 | KASAN_KMALLOC_REDZONE); | |
bffa986c AK |
294 | } |
295 | ||
296 | void kasan_unpoison_object_data(struct kmem_cache *cache, void *object) | |
297 | { | |
cebd0eb2 | 298 | unpoison_range(object, cache->object_size); |
bffa986c AK |
299 | } |
300 | ||
301 | void kasan_poison_object_data(struct kmem_cache *cache, void *object) | |
302 | { | |
cebd0eb2 | 303 | poison_range(object, |
1f600626 | 304 | round_up(cache->object_size, KASAN_GRANULE_SIZE), |
bffa986c AK |
305 | KASAN_KMALLOC_REDZONE); |
306 | } | |
307 | ||
7f94ffbc | 308 | /* |
a3fe7cdf AK |
309 | * This function assigns a tag to an object considering the following: |
310 | * 1. A cache might have a constructor, which might save a pointer to a slab | |
311 | * object somewhere (e.g. in the object itself). We preassign a tag for | |
312 | * each object in caches with constructors during slab creation and reuse | |
313 | * the same tag each time a particular object is allocated. | |
314 | * 2. A cache might be SLAB_TYPESAFE_BY_RCU, which means objects can be | |
315 | * accessed after being freed. We preassign tags for objects in these | |
316 | * caches as well. | |
317 | * 3. For SLAB allocator we can't preassign tags randomly since the freelist | |
318 | * is stored as an array of indexes instead of a linked list. Assign tags | |
319 | * based on objects indexes, so that objects that are next to each other | |
320 | * get different tags. | |
7f94ffbc | 321 | */ |
a3fe7cdf | 322 | static u8 assign_tag(struct kmem_cache *cache, const void *object, |
e1db95be | 323 | bool init, bool keep_tag) |
7f94ffbc | 324 | { |
e1db95be AK |
325 | /* |
326 | * 1. When an object is kmalloc()'ed, two hooks are called: | |
327 | * kasan_slab_alloc() and kasan_kmalloc(). We assign the | |
328 | * tag only in the first one. | |
329 | * 2. We reuse the same tag for krealloc'ed objects. | |
330 | */ | |
331 | if (keep_tag) | |
a3fe7cdf AK |
332 | return get_tag(object); |
333 | ||
334 | /* | |
335 | * If the cache neither has a constructor nor has SLAB_TYPESAFE_BY_RCU | |
336 | * set, assign a tag when the object is being allocated (init == false). | |
337 | */ | |
7f94ffbc | 338 | if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU)) |
a3fe7cdf | 339 | return init ? KASAN_TAG_KERNEL : random_tag(); |
7f94ffbc | 340 | |
a3fe7cdf | 341 | /* For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU: */ |
7f94ffbc | 342 | #ifdef CONFIG_SLAB |
a3fe7cdf | 343 | /* For SLAB assign tags based on the object index in the freelist. */ |
7f94ffbc AK |
344 | return (u8)obj_to_index(cache, virt_to_page(object), (void *)object); |
345 | #else | |
a3fe7cdf AK |
346 | /* |
347 | * For SLUB assign a random tag during slab creation, otherwise reuse | |
348 | * the already assigned tag. | |
349 | */ | |
350 | return init ? random_tag() : get_tag(object); | |
7f94ffbc AK |
351 | #endif |
352 | } | |
353 | ||
66afc7f1 AK |
354 | void * __must_check kasan_init_slab_obj(struct kmem_cache *cache, |
355 | const void *object) | |
bffa986c AK |
356 | { |
357 | struct kasan_alloc_meta *alloc_info; | |
358 | ||
359 | if (!(cache->flags & SLAB_KASAN)) | |
360 | return (void *)object; | |
361 | ||
362 | alloc_info = get_alloc_info(cache, object); | |
363 | __memset(alloc_info, 0, sizeof(*alloc_info)); | |
364 | ||
7f94ffbc | 365 | if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) |
a3fe7cdf AK |
366 | object = set_tag(object, |
367 | assign_tag(cache, object, true, false)); | |
7f94ffbc | 368 | |
bffa986c AK |
369 | return (void *)object; |
370 | } | |
371 | ||
7f94ffbc AK |
372 | static inline bool shadow_invalid(u8 tag, s8 shadow_byte) |
373 | { | |
374 | if (IS_ENABLED(CONFIG_KASAN_GENERIC)) | |
375 | return shadow_byte < 0 || | |
1f600626 | 376 | shadow_byte >= KASAN_GRANULE_SIZE; |
00fb24a4 AR |
377 | |
378 | /* else CONFIG_KASAN_SW_TAGS: */ | |
379 | if ((u8)shadow_byte == KASAN_TAG_INVALID) | |
380 | return true; | |
381 | if ((tag != KASAN_TAG_KERNEL) && (tag != (u8)shadow_byte)) | |
382 | return true; | |
383 | ||
384 | return false; | |
7f94ffbc AK |
385 | } |
386 | ||
bffa986c AK |
387 | static bool __kasan_slab_free(struct kmem_cache *cache, void *object, |
388 | unsigned long ip, bool quarantine) | |
389 | { | |
390 | s8 shadow_byte; | |
7f94ffbc AK |
391 | u8 tag; |
392 | void *tagged_object; | |
bffa986c AK |
393 | unsigned long rounded_up_size; |
394 | ||
7f94ffbc AK |
395 | tag = get_tag(object); |
396 | tagged_object = object; | |
397 | object = reset_tag(object); | |
398 | ||
bffa986c AK |
399 | if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) != |
400 | object)) { | |
7f94ffbc | 401 | kasan_report_invalid_free(tagged_object, ip); |
bffa986c AK |
402 | return true; |
403 | } | |
404 | ||
405 | /* RCU slabs could be legally used after free within the RCU period */ | |
406 | if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU)) | |
407 | return false; | |
408 | ||
409 | shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(object)); | |
7f94ffbc AK |
410 | if (shadow_invalid(tag, shadow_byte)) { |
411 | kasan_report_invalid_free(tagged_object, ip); | |
bffa986c AK |
412 | return true; |
413 | } | |
414 | ||
1f600626 | 415 | rounded_up_size = round_up(cache->object_size, KASAN_GRANULE_SIZE); |
cebd0eb2 | 416 | poison_range(object, rounded_up_size, KASAN_KMALLOC_FREE); |
bffa986c | 417 | |
7f94ffbc AK |
418 | if ((IS_ENABLED(CONFIG_KASAN_GENERIC) && !quarantine) || |
419 | unlikely(!(cache->flags & SLAB_KASAN))) | |
bffa986c AK |
420 | return false; |
421 | ||
ae8f06b3 WW |
422 | kasan_set_free_info(cache, object, tag); |
423 | ||
bffa986c | 424 | quarantine_put(get_free_info(cache, object), cache); |
7f94ffbc AK |
425 | |
426 | return IS_ENABLED(CONFIG_KASAN_GENERIC); | |
bffa986c AK |
427 | } |
428 | ||
429 | bool kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip) | |
430 | { | |
431 | return __kasan_slab_free(cache, object, ip, true); | |
432 | } | |
433 | ||
a3fe7cdf | 434 | static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object, |
e1db95be | 435 | size_t size, gfp_t flags, bool keep_tag) |
bffa986c AK |
436 | { |
437 | unsigned long redzone_start; | |
438 | unsigned long redzone_end; | |
0600597c | 439 | u8 tag = 0xff; |
bffa986c AK |
440 | |
441 | if (gfpflags_allow_blocking(flags)) | |
442 | quarantine_reduce(); | |
443 | ||
444 | if (unlikely(object == NULL)) | |
445 | return NULL; | |
446 | ||
447 | redzone_start = round_up((unsigned long)(object + size), | |
1f600626 | 448 | KASAN_GRANULE_SIZE); |
bffa986c | 449 | redzone_end = round_up((unsigned long)object + cache->object_size, |
1f600626 | 450 | KASAN_GRANULE_SIZE); |
bffa986c | 451 | |
7f94ffbc | 452 | if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) |
e1db95be | 453 | tag = assign_tag(cache, object, false, keep_tag); |
7f94ffbc AK |
454 | |
455 | /* Tag is ignored in set_tag without CONFIG_KASAN_SW_TAGS */ | |
cebd0eb2 AK |
456 | unpoison_range(set_tag(object, tag), size); |
457 | poison_range((void *)redzone_start, redzone_end - redzone_start, | |
458 | KASAN_KMALLOC_REDZONE); | |
bffa986c AK |
459 | |
460 | if (cache->flags & SLAB_KASAN) | |
e4b7818b | 461 | kasan_set_track(&get_alloc_info(cache, object)->alloc_track, flags); |
bffa986c | 462 | |
7f94ffbc | 463 | return set_tag(object, tag); |
bffa986c | 464 | } |
a3fe7cdf | 465 | |
e1db95be AK |
466 | void * __must_check kasan_slab_alloc(struct kmem_cache *cache, void *object, |
467 | gfp_t flags) | |
468 | { | |
469 | return __kasan_kmalloc(cache, object, cache->object_size, flags, false); | |
470 | } | |
471 | ||
a3fe7cdf AK |
472 | void * __must_check kasan_kmalloc(struct kmem_cache *cache, const void *object, |
473 | size_t size, gfp_t flags) | |
474 | { | |
e1db95be | 475 | return __kasan_kmalloc(cache, object, size, flags, true); |
a3fe7cdf | 476 | } |
bffa986c AK |
477 | EXPORT_SYMBOL(kasan_kmalloc); |
478 | ||
66afc7f1 AK |
479 | void * __must_check kasan_kmalloc_large(const void *ptr, size_t size, |
480 | gfp_t flags) | |
bffa986c AK |
481 | { |
482 | struct page *page; | |
483 | unsigned long redzone_start; | |
484 | unsigned long redzone_end; | |
485 | ||
486 | if (gfpflags_allow_blocking(flags)) | |
487 | quarantine_reduce(); | |
488 | ||
489 | if (unlikely(ptr == NULL)) | |
490 | return NULL; | |
491 | ||
492 | page = virt_to_page(ptr); | |
493 | redzone_start = round_up((unsigned long)(ptr + size), | |
1f600626 | 494 | KASAN_GRANULE_SIZE); |
a50b854e | 495 | redzone_end = (unsigned long)ptr + page_size(page); |
bffa986c | 496 | |
cebd0eb2 AK |
497 | unpoison_range(ptr, size); |
498 | poison_range((void *)redzone_start, redzone_end - redzone_start, | |
499 | KASAN_PAGE_REDZONE); | |
bffa986c AK |
500 | |
501 | return (void *)ptr; | |
502 | } | |
503 | ||
66afc7f1 | 504 | void * __must_check kasan_krealloc(const void *object, size_t size, gfp_t flags) |
bffa986c AK |
505 | { |
506 | struct page *page; | |
507 | ||
508 | if (unlikely(object == ZERO_SIZE_PTR)) | |
509 | return (void *)object; | |
510 | ||
511 | page = virt_to_head_page(object); | |
512 | ||
513 | if (unlikely(!PageSlab(page))) | |
514 | return kasan_kmalloc_large(object, size, flags); | |
515 | else | |
a3fe7cdf AK |
516 | return __kasan_kmalloc(page->slab_cache, object, size, |
517 | flags, true); | |
bffa986c AK |
518 | } |
519 | ||
520 | void kasan_poison_kfree(void *ptr, unsigned long ip) | |
521 | { | |
522 | struct page *page; | |
523 | ||
524 | page = virt_to_head_page(ptr); | |
525 | ||
526 | if (unlikely(!PageSlab(page))) { | |
2813b9c0 | 527 | if (ptr != page_address(page)) { |
bffa986c AK |
528 | kasan_report_invalid_free(ptr, ip); |
529 | return; | |
530 | } | |
cebd0eb2 | 531 | poison_range(ptr, page_size(page), KASAN_FREE_PAGE); |
bffa986c AK |
532 | } else { |
533 | __kasan_slab_free(page->slab_cache, ptr, ip, false); | |
534 | } | |
535 | } | |
536 | ||
537 | void kasan_kfree_large(void *ptr, unsigned long ip) | |
538 | { | |
2813b9c0 | 539 | if (ptr != page_address(virt_to_head_page(ptr))) |
bffa986c AK |
540 | kasan_report_invalid_free(ptr, ip); |
541 | /* The object will be poisoned by page_alloc. */ | |
542 | } | |
543 | ||
bffa986c AK |
544 | #ifdef CONFIG_MEMORY_HOTPLUG |
545 | static bool shadow_mapped(unsigned long addr) | |
546 | { | |
547 | pgd_t *pgd = pgd_offset_k(addr); | |
548 | p4d_t *p4d; | |
549 | pud_t *pud; | |
550 | pmd_t *pmd; | |
551 | pte_t *pte; | |
552 | ||
553 | if (pgd_none(*pgd)) | |
554 | return false; | |
555 | p4d = p4d_offset(pgd, addr); | |
556 | if (p4d_none(*p4d)) | |
557 | return false; | |
558 | pud = pud_offset(p4d, addr); | |
559 | if (pud_none(*pud)) | |
560 | return false; | |
561 | ||
562 | /* | |
563 | * We can't use pud_large() or pud_huge(), the first one is | |
564 | * arch-specific, the last one depends on HUGETLB_PAGE. So let's abuse | |
565 | * pud_bad(), if pud is bad then it's bad because it's huge. | |
566 | */ | |
567 | if (pud_bad(*pud)) | |
568 | return true; | |
569 | pmd = pmd_offset(pud, addr); | |
570 | if (pmd_none(*pmd)) | |
571 | return false; | |
572 | ||
573 | if (pmd_bad(*pmd)) | |
574 | return true; | |
575 | pte = pte_offset_kernel(pmd, addr); | |
576 | return !pte_none(*pte); | |
577 | } | |
578 | ||
579 | static int __meminit kasan_mem_notifier(struct notifier_block *nb, | |
580 | unsigned long action, void *data) | |
581 | { | |
582 | struct memory_notify *mem_data = data; | |
583 | unsigned long nr_shadow_pages, start_kaddr, shadow_start; | |
584 | unsigned long shadow_end, shadow_size; | |
585 | ||
586 | nr_shadow_pages = mem_data->nr_pages >> KASAN_SHADOW_SCALE_SHIFT; | |
587 | start_kaddr = (unsigned long)pfn_to_kaddr(mem_data->start_pfn); | |
588 | shadow_start = (unsigned long)kasan_mem_to_shadow((void *)start_kaddr); | |
589 | shadow_size = nr_shadow_pages << PAGE_SHIFT; | |
590 | shadow_end = shadow_start + shadow_size; | |
591 | ||
1f600626 AK |
592 | if (WARN_ON(mem_data->nr_pages % KASAN_GRANULE_SIZE) || |
593 | WARN_ON(start_kaddr % (KASAN_GRANULE_SIZE << PAGE_SHIFT))) | |
bffa986c AK |
594 | return NOTIFY_BAD; |
595 | ||
596 | switch (action) { | |
597 | case MEM_GOING_ONLINE: { | |
598 | void *ret; | |
599 | ||
600 | /* | |
601 | * If shadow is mapped already than it must have been mapped | |
602 | * during the boot. This could happen if we onlining previously | |
603 | * offlined memory. | |
604 | */ | |
605 | if (shadow_mapped(shadow_start)) | |
606 | return NOTIFY_OK; | |
607 | ||
608 | ret = __vmalloc_node_range(shadow_size, PAGE_SIZE, shadow_start, | |
609 | shadow_end, GFP_KERNEL, | |
610 | PAGE_KERNEL, VM_NO_GUARD, | |
611 | pfn_to_nid(mem_data->start_pfn), | |
612 | __builtin_return_address(0)); | |
613 | if (!ret) | |
614 | return NOTIFY_BAD; | |
615 | ||
616 | kmemleak_ignore(ret); | |
617 | return NOTIFY_OK; | |
618 | } | |
619 | case MEM_CANCEL_ONLINE: | |
620 | case MEM_OFFLINE: { | |
621 | struct vm_struct *vm; | |
622 | ||
623 | /* | |
624 | * shadow_start was either mapped during boot by kasan_init() | |
625 | * or during memory online by __vmalloc_node_range(). | |
626 | * In the latter case we can use vfree() to free shadow. | |
627 | * Non-NULL result of the find_vm_area() will tell us if | |
628 | * that was the second case. | |
629 | * | |
630 | * Currently it's not possible to free shadow mapped | |
631 | * during boot by kasan_init(). It's because the code | |
632 | * to do that hasn't been written yet. So we'll just | |
633 | * leak the memory. | |
634 | */ | |
635 | vm = find_vm_area((void *)shadow_start); | |
636 | if (vm) | |
637 | vfree((void *)shadow_start); | |
638 | } | |
639 | } | |
640 | ||
641 | return NOTIFY_OK; | |
642 | } | |
643 | ||
644 | static int __init kasan_memhotplug_init(void) | |
645 | { | |
646 | hotplug_memory_notifier(kasan_mem_notifier, 0); | |
647 | ||
648 | return 0; | |
649 | } | |
650 | ||
651 | core_initcall(kasan_memhotplug_init); | |
652 | #endif | |
3c5c3cfb DA |
653 | |
654 | #ifdef CONFIG_KASAN_VMALLOC | |
3b1a4a86 | 655 | |
3c5c3cfb DA |
656 | static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr, |
657 | void *unused) | |
658 | { | |
659 | unsigned long page; | |
660 | pte_t pte; | |
661 | ||
662 | if (likely(!pte_none(*ptep))) | |
663 | return 0; | |
664 | ||
665 | page = __get_free_page(GFP_KERNEL); | |
666 | if (!page) | |
667 | return -ENOMEM; | |
668 | ||
669 | memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE); | |
670 | pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL); | |
671 | ||
672 | spin_lock(&init_mm.page_table_lock); | |
673 | if (likely(pte_none(*ptep))) { | |
674 | set_pte_at(&init_mm, addr, ptep, pte); | |
675 | page = 0; | |
676 | } | |
677 | spin_unlock(&init_mm.page_table_lock); | |
678 | if (page) | |
679 | free_page(page); | |
680 | return 0; | |
681 | } | |
682 | ||
d98c9e83 | 683 | int kasan_populate_vmalloc(unsigned long addr, unsigned long size) |
3c5c3cfb DA |
684 | { |
685 | unsigned long shadow_start, shadow_end; | |
686 | int ret; | |
687 | ||
d98c9e83 AR |
688 | if (!is_vmalloc_or_module_addr((void *)addr)) |
689 | return 0; | |
690 | ||
691 | shadow_start = (unsigned long)kasan_mem_to_shadow((void *)addr); | |
3c5c3cfb | 692 | shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE); |
d98c9e83 | 693 | shadow_end = (unsigned long)kasan_mem_to_shadow((void *)addr + size); |
3c5c3cfb DA |
694 | shadow_end = ALIGN(shadow_end, PAGE_SIZE); |
695 | ||
696 | ret = apply_to_page_range(&init_mm, shadow_start, | |
697 | shadow_end - shadow_start, | |
698 | kasan_populate_vmalloc_pte, NULL); | |
699 | if (ret) | |
700 | return ret; | |
701 | ||
702 | flush_cache_vmap(shadow_start, shadow_end); | |
703 | ||
3c5c3cfb DA |
704 | /* |
705 | * We need to be careful about inter-cpu effects here. Consider: | |
706 | * | |
707 | * CPU#0 CPU#1 | |
708 | * WRITE_ONCE(p, vmalloc(100)); while (x = READ_ONCE(p)) ; | |
709 | * p[99] = 1; | |
710 | * | |
711 | * With compiler instrumentation, that ends up looking like this: | |
712 | * | |
713 | * CPU#0 CPU#1 | |
714 | * // vmalloc() allocates memory | |
715 | * // let a = area->addr | |
716 | * // we reach kasan_populate_vmalloc | |
cebd0eb2 | 717 | * // and call unpoison_range: |
3c5c3cfb DA |
718 | * STORE shadow(a), unpoison_val |
719 | * ... | |
720 | * STORE shadow(a+99), unpoison_val x = LOAD p | |
721 | * // rest of vmalloc process <data dependency> | |
722 | * STORE p, a LOAD shadow(x+99) | |
723 | * | |
724 | * If there is no barrier between the end of unpoisioning the shadow | |
725 | * and the store of the result to p, the stores could be committed | |
726 | * in a different order by CPU#0, and CPU#1 could erroneously observe | |
727 | * poison in the shadow. | |
728 | * | |
729 | * We need some sort of barrier between the stores. | |
730 | * | |
731 | * In the vmalloc() case, this is provided by a smp_wmb() in | |
732 | * clear_vm_uninitialized_flag(). In the per-cpu allocator and in | |
733 | * get_vm_area() and friends, the caller gets shadow allocated but | |
734 | * doesn't have any pages mapped into the virtual address space that | |
735 | * has been reserved. Mapping those pages in will involve taking and | |
736 | * releasing a page-table lock, which will provide the barrier. | |
737 | */ | |
738 | ||
739 | return 0; | |
740 | } | |
741 | ||
742 | /* | |
743 | * Poison the shadow for a vmalloc region. Called as part of the | |
744 | * freeing process at the time the region is freed. | |
745 | */ | |
d98c9e83 | 746 | void kasan_poison_vmalloc(const void *start, unsigned long size) |
3c5c3cfb | 747 | { |
d98c9e83 AR |
748 | if (!is_vmalloc_or_module_addr(start)) |
749 | return; | |
750 | ||
1f600626 | 751 | size = round_up(size, KASAN_GRANULE_SIZE); |
cebd0eb2 | 752 | poison_range(start, size, KASAN_VMALLOC_INVALID); |
3c5c3cfb DA |
753 | } |
754 | ||
d98c9e83 AR |
755 | void kasan_unpoison_vmalloc(const void *start, unsigned long size) |
756 | { | |
757 | if (!is_vmalloc_or_module_addr(start)) | |
758 | return; | |
759 | ||
cebd0eb2 | 760 | unpoison_range(start, size); |
d98c9e83 AR |
761 | } |
762 | ||
3c5c3cfb DA |
763 | static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr, |
764 | void *unused) | |
765 | { | |
766 | unsigned long page; | |
767 | ||
768 | page = (unsigned long)__va(pte_pfn(*ptep) << PAGE_SHIFT); | |
769 | ||
770 | spin_lock(&init_mm.page_table_lock); | |
771 | ||
772 | if (likely(!pte_none(*ptep))) { | |
773 | pte_clear(&init_mm, addr, ptep); | |
774 | free_page(page); | |
775 | } | |
776 | spin_unlock(&init_mm.page_table_lock); | |
777 | ||
778 | return 0; | |
779 | } | |
780 | ||
781 | /* | |
782 | * Release the backing for the vmalloc region [start, end), which | |
783 | * lies within the free region [free_region_start, free_region_end). | |
784 | * | |
785 | * This can be run lazily, long after the region was freed. It runs | |
786 | * under vmap_area_lock, so it's not safe to interact with the vmalloc/vmap | |
787 | * infrastructure. | |
788 | * | |
789 | * How does this work? | |
790 | * ------------------- | |
791 | * | |
792 | * We have a region that is page aligned, labelled as A. | |
793 | * That might not map onto the shadow in a way that is page-aligned: | |
794 | * | |
795 | * start end | |
796 | * v v | |
797 | * |????????|????????|AAAAAAAA|AA....AA|AAAAAAAA|????????| < vmalloc | |
798 | * -------- -------- -------- -------- -------- | |
799 | * | | | | | | |
800 | * | | | /-------/ | | |
801 | * \-------\|/------/ |/---------------/ | |
802 | * ||| || | |
803 | * |??AAAAAA|AAAAAAAA|AA??????| < shadow | |
804 | * (1) (2) (3) | |
805 | * | |
806 | * First we align the start upwards and the end downwards, so that the | |
807 | * shadow of the region aligns with shadow page boundaries. In the | |
808 | * example, this gives us the shadow page (2). This is the shadow entirely | |
809 | * covered by this allocation. | |
810 | * | |
811 | * Then we have the tricky bits. We want to know if we can free the | |
812 | * partially covered shadow pages - (1) and (3) in the example. For this, | |
813 | * we are given the start and end of the free region that contains this | |
814 | * allocation. Extending our previous example, we could have: | |
815 | * | |
816 | * free_region_start free_region_end | |
817 | * | start end | | |
818 | * v v v v | |
819 | * |FFFFFFFF|FFFFFFFF|AAAAAAAA|AA....AA|AAAAAAAA|FFFFFFFF| < vmalloc | |
820 | * -------- -------- -------- -------- -------- | |
821 | * | | | | | | |
822 | * | | | /-------/ | | |
823 | * \-------\|/------/ |/---------------/ | |
824 | * ||| || | |
825 | * |FFAAAAAA|AAAAAAAA|AAF?????| < shadow | |
826 | * (1) (2) (3) | |
827 | * | |
828 | * Once again, we align the start of the free region up, and the end of | |
829 | * the free region down so that the shadow is page aligned. So we can free | |
830 | * page (1) - we know no allocation currently uses anything in that page, | |
831 | * because all of it is in the vmalloc free region. But we cannot free | |
832 | * page (3), because we can't be sure that the rest of it is unused. | |
833 | * | |
834 | * We only consider pages that contain part of the original region for | |
835 | * freeing: we don't try to free other pages from the free region or we'd | |
836 | * end up trying to free huge chunks of virtual address space. | |
837 | * | |
838 | * Concurrency | |
839 | * ----------- | |
840 | * | |
841 | * How do we know that we're not freeing a page that is simultaneously | |
842 | * being used for a fresh allocation in kasan_populate_vmalloc(_pte)? | |
843 | * | |
844 | * We _can_ have kasan_release_vmalloc and kasan_populate_vmalloc running | |
845 | * at the same time. While we run under free_vmap_area_lock, the population | |
846 | * code does not. | |
847 | * | |
848 | * free_vmap_area_lock instead operates to ensure that the larger range | |
849 | * [free_region_start, free_region_end) is safe: because __alloc_vmap_area and | |
850 | * the per-cpu region-finding algorithm both run under free_vmap_area_lock, | |
851 | * no space identified as free will become used while we are running. This | |
852 | * means that so long as we are careful with alignment and only free shadow | |
853 | * pages entirely covered by the free region, we will not run in to any | |
854 | * trouble - any simultaneous allocations will be for disjoint regions. | |
855 | */ | |
856 | void kasan_release_vmalloc(unsigned long start, unsigned long end, | |
857 | unsigned long free_region_start, | |
858 | unsigned long free_region_end) | |
859 | { | |
860 | void *shadow_start, *shadow_end; | |
861 | unsigned long region_start, region_end; | |
e218f1ca | 862 | unsigned long size; |
3c5c3cfb | 863 | |
1f600626 AK |
864 | region_start = ALIGN(start, PAGE_SIZE * KASAN_GRANULE_SIZE); |
865 | region_end = ALIGN_DOWN(end, PAGE_SIZE * KASAN_GRANULE_SIZE); | |
3c5c3cfb DA |
866 | |
867 | free_region_start = ALIGN(free_region_start, | |
1f600626 | 868 | PAGE_SIZE * KASAN_GRANULE_SIZE); |
3c5c3cfb DA |
869 | |
870 | if (start != region_start && | |
871 | free_region_start < region_start) | |
1f600626 | 872 | region_start -= PAGE_SIZE * KASAN_GRANULE_SIZE; |
3c5c3cfb DA |
873 | |
874 | free_region_end = ALIGN_DOWN(free_region_end, | |
1f600626 | 875 | PAGE_SIZE * KASAN_GRANULE_SIZE); |
3c5c3cfb DA |
876 | |
877 | if (end != region_end && | |
878 | free_region_end > region_end) | |
1f600626 | 879 | region_end += PAGE_SIZE * KASAN_GRANULE_SIZE; |
3c5c3cfb DA |
880 | |
881 | shadow_start = kasan_mem_to_shadow((void *)region_start); | |
882 | shadow_end = kasan_mem_to_shadow((void *)region_end); | |
883 | ||
884 | if (shadow_end > shadow_start) { | |
e218f1ca DA |
885 | size = shadow_end - shadow_start; |
886 | apply_to_existing_page_range(&init_mm, | |
887 | (unsigned long)shadow_start, | |
888 | size, kasan_depopulate_vmalloc_pte, | |
889 | NULL); | |
3c5c3cfb DA |
890 | flush_tlb_kernel_range((unsigned long)shadow_start, |
891 | (unsigned long)shadow_end); | |
892 | } | |
893 | } | |
3b1a4a86 AK |
894 | |
895 | #else /* CONFIG_KASAN_VMALLOC */ | |
896 | ||
897 | int kasan_module_alloc(void *addr, size_t size) | |
898 | { | |
899 | void *ret; | |
900 | size_t scaled_size; | |
901 | size_t shadow_size; | |
902 | unsigned long shadow_start; | |
903 | ||
904 | shadow_start = (unsigned long)kasan_mem_to_shadow(addr); | |
1f600626 AK |
905 | scaled_size = (size + KASAN_GRANULE_SIZE - 1) >> |
906 | KASAN_SHADOW_SCALE_SHIFT; | |
3b1a4a86 AK |
907 | shadow_size = round_up(scaled_size, PAGE_SIZE); |
908 | ||
909 | if (WARN_ON(!PAGE_ALIGNED(shadow_start))) | |
910 | return -EINVAL; | |
911 | ||
912 | ret = __vmalloc_node_range(shadow_size, 1, shadow_start, | |
913 | shadow_start + shadow_size, | |
914 | GFP_KERNEL, | |
915 | PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE, | |
916 | __builtin_return_address(0)); | |
917 | ||
918 | if (ret) { | |
919 | __memset(ret, KASAN_SHADOW_INIT, shadow_size); | |
920 | find_vm_area(addr)->flags |= VM_KASAN; | |
921 | kmemleak_ignore(ret); | |
922 | return 0; | |
923 | } | |
924 | ||
925 | return -ENOMEM; | |
926 | } | |
927 | ||
928 | void kasan_free_shadow(const struct vm_struct *vm) | |
929 | { | |
930 | if (vm->flags & VM_KASAN) | |
931 | vfree(kasan_mem_to_shadow(vm->addr)); | |
932 | } | |
933 | ||
3c5c3cfb | 934 | #endif |