mm/kasan: add API to check memory regions

[linux-2.6-block.git] / mm / kasan / kasan.c

diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c
index 38f1dd79acdbc4eab8f6226a968317f764814f5b..18b6a2b8d183550daa6c3e64ae7ba50610ead79c 100644 (file)
--- a/mm/kasan/kasan.c
+++ b/mm/kasan/kasan.c
@@ -273,32 +273,48 @@ static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
        return memory_is_poisoned_n(addr, size);
 }
 
-
-static __always_inline void check_memory_region(unsigned long addr,
-                                               size_t size, bool write)
+static __always_inline void check_memory_region_inline(unsigned long addr,
+                                               size_t size, bool write,
+                                               unsigned long ret_ip)
 {
        if (unlikely(size == 0))
                return;
 
        if (unlikely((void *)addr <
                kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
-               kasan_report(addr, size, write, _RET_IP_);
+               kasan_report(addr, size, write, ret_ip);
                return;
        }
 
        if (likely(!memory_is_poisoned(addr, size)))
                return;
 
-       kasan_report(addr, size, write, _RET_IP_);
+       kasan_report(addr, size, write, ret_ip);
+}
+
+static void check_memory_region(unsigned long addr,
+                               size_t size, bool write,
+                               unsigned long ret_ip)
+{
+       check_memory_region_inline(addr, size, write, ret_ip);
+}
+
+void kasan_check_read(const void *p, unsigned int size)
+{
+       check_memory_region((unsigned long)p, size, false, _RET_IP_);
 }
+EXPORT_SYMBOL(kasan_check_read);
 
-void __asan_loadN(unsigned long addr, size_t size);
-void __asan_storeN(unsigned long addr, size_t size);
+void kasan_check_write(const void *p, unsigned int size)
+{
+       check_memory_region((unsigned long)p, size, true, _RET_IP_);
+}
+EXPORT_SYMBOL(kasan_check_write);
 
 #undef memset
 void *memset(void *addr, int c, size_t len)
 {
-       __asan_storeN((unsigned long)addr, len);
+       check_memory_region((unsigned long)addr, len, true, _RET_IP_);
 
        return __memset(addr, c, len);
 }
@@ -306,8 +322,8 @@ void *memset(void *addr, int c, size_t len)
 #undef memmove
 void *memmove(void *dest, const void *src, size_t len)
 {
-       __asan_loadN((unsigned long)src, len);
-       __asan_storeN((unsigned long)dest, len);
+       check_memory_region((unsigned long)src, len, false, _RET_IP_);
+       check_memory_region((unsigned long)dest, len, true, _RET_IP_);
 
        return __memmove(dest, src, len);
 }
@@ -315,8 +331,8 @@ void *memmove(void *dest, const void *src, size_t len)
 #undef memcpy
 void *memcpy(void *dest, const void *src, size_t len)
 {
-       __asan_loadN((unsigned long)src, len);
-       __asan_storeN((unsigned long)dest, len);
+       check_memory_region((unsigned long)src, len, false, _RET_IP_);
+       check_memory_region((unsigned long)dest, len, true, _RET_IP_);
 
        return __memcpy(dest, src, len);
 }
@@ -388,6 +404,16 @@ void kasan_cache_create(struct kmem_cache *cache, size_t *size,
 }
 #endif
 
+void kasan_cache_shrink(struct kmem_cache *cache)
+{
+       quarantine_remove_cache(cache);
+}
+
+void kasan_cache_destroy(struct kmem_cache *cache)
+{
+       quarantine_remove_cache(cache);
+}
+
 void kasan_poison_slab(struct page *page)
 {
        kasan_poison_shadow(page_address(page),
@@ -482,7 +508,7 @@ void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags)
        kasan_kmalloc(cache, object, cache->object_size, flags);
 }
 
-void kasan_slab_free(struct kmem_cache *cache, void *object)
+void kasan_poison_slab_free(struct kmem_cache *cache, void *object)
 {
        unsigned long size = cache->object_size;
        unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE);
@@ -491,18 +517,43 @@ void kasan_slab_free(struct kmem_cache *cache, void *object)
        if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
                return;
 
+       kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
+}
+
+bool kasan_slab_free(struct kmem_cache *cache, void *object)
+{
 #ifdef CONFIG_SLAB
-       if (cache->flags & SLAB_KASAN) {
-               struct kasan_free_meta *free_info =
-                       get_free_info(cache, object);
+       /* RCU slabs could be legally used after free within the RCU period */
+       if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
+               return false;
+
+       if (likely(cache->flags & SLAB_KASAN)) {
                struct kasan_alloc_meta *alloc_info =
                        get_alloc_info(cache, object);
-               alloc_info->state = KASAN_STATE_FREE;
-               set_track(&free_info->track, GFP_NOWAIT);
+               struct kasan_free_meta *free_info =
+                       get_free_info(cache, object);
+
+               switch (alloc_info->state) {
+               case KASAN_STATE_ALLOC:
+                       alloc_info->state = KASAN_STATE_QUARANTINE;
+                       quarantine_put(free_info, cache);
+                       set_track(&free_info->track, GFP_NOWAIT);
+                       kasan_poison_slab_free(cache, object);
+                       return true;
+               case KASAN_STATE_QUARANTINE:
+               case KASAN_STATE_FREE:
+                       pr_err("Double free");
+                       dump_stack();
+                       break;
+               default:
+                       break;
+               }
        }
+       return false;
+#else
+       kasan_poison_slab_free(cache, object);
+       return false;
 #endif
-
-       kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
 }
 
 void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size,
@@ -511,6 +562,9 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size,
        unsigned long redzone_start;
        unsigned long redzone_end;
 
+       if (flags & __GFP_RECLAIM)
+               quarantine_reduce();
+
        if (unlikely(object == NULL))
                return;
 
@@ -541,6 +595,9 @@ void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
        unsigned long redzone_start;
        unsigned long redzone_end;
 
+       if (flags & __GFP_RECLAIM)
+               quarantine_reduce();
+
        if (unlikely(ptr == NULL))
                return;
 
@@ -649,22 +706,22 @@ void __asan_unregister_globals(struct kasan_global *globals, size_t size)
 }
 EXPORT_SYMBOL(__asan_unregister_globals);
 
-#define DEFINE_ASAN_LOAD_STORE(size)                           \
-       void __asan_load##size(unsigned long addr)              \
-       {                                                       \
-               check_memory_region(addr, size, false);         \
-       }                                                       \
-       EXPORT_SYMBOL(__asan_load##size);                       \
-       __alias(__asan_load##size)                              \
-       void __asan_load##size##_noabort(unsigned long);        \
-       EXPORT_SYMBOL(__asan_load##size##_noabort);             \
-       void __asan_store##size(unsigned long addr)             \
-       {                                                       \
-               check_memory_region(addr, size, true);          \
-       }                                                       \
-       EXPORT_SYMBOL(__asan_store##size);                      \
-       __alias(__asan_store##size)                             \
-       void __asan_store##size##_noabort(unsigned long);       \
+#define DEFINE_ASAN_LOAD_STORE(size)                                   \
+       void __asan_load##size(unsigned long addr)                      \
+       {                                                               \
+               check_memory_region_inline(addr, size, false, _RET_IP_);\
+       }                                                               \
+       EXPORT_SYMBOL(__asan_load##size);                               \
+       __alias(__asan_load##size)                                      \
+       void __asan_load##size##_noabort(unsigned long);                \
+       EXPORT_SYMBOL(__asan_load##size##_noabort);                     \
+       void __asan_store##size(unsigned long addr)                     \
+       {                                                               \
+               check_memory_region_inline(addr, size, true, _RET_IP_); \
+       }                                                               \
+       EXPORT_SYMBOL(__asan_store##size);                              \
+       __alias(__asan_store##size)                                     \
+       void __asan_store##size##_noabort(unsigned long);               \
        EXPORT_SYMBOL(__asan_store##size##_noabort)
 
 DEFINE_ASAN_LOAD_STORE(1);
@@ -675,7 +732,7 @@ DEFINE_ASAN_LOAD_STORE(16);
 
 void __asan_loadN(unsigned long addr, size_t size)
 {
-       check_memory_region(addr, size, false);
+       check_memory_region(addr, size, false, _RET_IP_);
 }
 EXPORT_SYMBOL(__asan_loadN);
 
@@ -685,7 +742,7 @@ EXPORT_SYMBOL(__asan_loadN_noabort);
 
 void __asan_storeN(unsigned long addr, size_t size)
 {
-       check_memory_region(addr, size, true);
+       check_memory_region(addr, size, true, _RET_IP_);
 }
 EXPORT_SYMBOL(__asan_storeN);