[linux-2.6-block.git] / include / linux / kasan.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_KASAN_H
#define _LINUX_KASAN_H

#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/static_key.h>
#include <linux/types.h>

struct kmem_cache;
struct page;
struct vm_struct;
struct task_struct;

#ifdef CONFIG_KASAN

#include <linux/linkage.h>
#include <asm/kasan.h>

/* kasan_data struct is used in KUnit tests for KASAN expected failures */
struct kunit_kasan_expectation {
	bool report_found;
};

#endif

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)

#include <linux/pgtable.h>

/* Software KASAN implementations use shadow memory. */

#ifdef CONFIG_KASAN_SW_TAGS
/* This matches KASAN_TAG_INVALID. */
#define KASAN_SHADOW_INIT 0xFE
#else
#define KASAN_SHADOW_INIT 0
#endif

#ifndef PTE_HWTABLE_PTRS
#define PTE_HWTABLE_PTRS 0
#endif

extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];

int kasan_populate_early_shadow(const void *shadow_start,
				const void *shadow_end);

static inline void *kasan_mem_to_shadow(const void *addr)
{
	return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
		+ KASAN_SHADOW_OFFSET;
}

int kasan_add_zero_shadow(void *start, unsigned long size);
void kasan_remove_zero_shadow(void *start, unsigned long size);

/* Enable reporting bugs after kasan_disable_current() */
extern void kasan_enable_current(void);

/* Disable reporting bugs for current task */
extern void kasan_disable_current(void);

#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

static inline int kasan_add_zero_shadow(void *start, unsigned long size)
{
	return 0;
}
static inline void kasan_remove_zero_shadow(void *start,
					unsigned long size)
{}

static inline void kasan_enable_current(void) {}
static inline void kasan_disable_current(void) {}

#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

#ifdef CONFIG_KASAN_HW_TAGS

DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);

static __always_inline bool kasan_enabled(void)
{
	return static_branch_likely(&kasan_flag_enabled);
}

static inline bool kasan_hw_tags_enabled(void)
{
	return kasan_enabled();
}

void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags);
void kasan_free_pages(struct page *page, unsigned int order);

#else /* CONFIG_KASAN_HW_TAGS */

static inline bool kasan_enabled(void)
{
	return IS_ENABLED(CONFIG_KASAN);
}

static inline bool kasan_hw_tags_enabled(void)
{
	return false;
}

static __always_inline void kasan_alloc_pages(struct page *page,
					      unsigned int order, gfp_t flags)
{
	/* Only available for integrated init. */
	BUILD_BUG();
}

static __always_inline void kasan_free_pages(struct page *page,
					     unsigned int order)
{
	/* Only available for integrated init. */
	BUILD_BUG();
}

#endif /* CONFIG_KASAN_HW_TAGS */

static inline bool kasan_has_integrated_init(void)
{
	return kasan_hw_tags_enabled();
}

#ifdef CONFIG_KASAN

struct kasan_cache {
	int alloc_meta_offset;
	int free_meta_offset;
	bool is_kmalloc;
};

slab_flags_t __kasan_never_merge(void);
static __always_inline slab_flags_t kasan_never_merge(void)
{
	if (kasan_enabled())
		return __kasan_never_merge();
	return 0;
}

void __kasan_unpoison_range(const void *addr, size_t size);
static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
{
	if (kasan_enabled())
		__kasan_unpoison_range(addr, size);
}

void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
static __always_inline void kasan_poison_pages(struct page *page,
						unsigned int order, bool init)
{
	if (kasan_enabled())
		__kasan_poison_pages(page, order, init);
}

void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
static __always_inline void kasan_unpoison_pages(struct page *page,
						 unsigned int order, bool init)
{
	if (kasan_enabled())
		__kasan_unpoison_pages(page, order, init);
}

void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
				slab_flags_t *flags);
static __always_inline void kasan_cache_create(struct kmem_cache *cache,
				unsigned int *size, slab_flags_t *flags)
{
	if (kasan_enabled())
		__kasan_cache_create(cache, size, flags);
}

void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
{
	if (kasan_enabled())
		__kasan_cache_create_kmalloc(cache);
}

size_t __kasan_metadata_size(struct kmem_cache *cache);
static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
{
	if (kasan_enabled())
		return __kasan_metadata_size(cache);
	return 0;
}

void __kasan_poison_slab(struct page *page);
static __always_inline void kasan_poison_slab(struct page *page)
{
	if (kasan_enabled())
		__kasan_poison_slab(page);
}

void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
							void *object)
{
	if (kasan_enabled())
		__kasan_unpoison_object_data(cache, object);
}

void __kasan_poison_object_data(struct kmem_cache *cache, void *object);
static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
							void *object)
{
	if (kasan_enabled())
		__kasan_poison_object_data(cache, object);
}

void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
					  const void *object);
static __always_inline void * __must_check kasan_init_slab_obj(
				struct kmem_cache *cache, const void *object)
{
	if (kasan_enabled())
		return __kasan_init_slab_obj(cache, object);
	return (void *)object;
}

bool __kasan_slab_free(struct kmem_cache *s, void *object,
			unsigned long ip, bool init);
static __always_inline bool kasan_slab_free(struct kmem_cache *s,
						void *object, bool init)
{
	if (kasan_enabled())
		return __kasan_slab_free(s, object, _RET_IP_, init);
	return false;
}

void __kasan_kfree_large(void *ptr, unsigned long ip);
static __always_inline void kasan_kfree_large(void *ptr)
{
	if (kasan_enabled())
		__kasan_kfree_large(ptr, _RET_IP_);
}

void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
static __always_inline void kasan_slab_free_mempool(void *ptr)
{
	if (kasan_enabled())
		__kasan_slab_free_mempool(ptr, _RET_IP_);
}

void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
				       void *object, gfp_t flags, bool init);
static __always_inline void * __must_check kasan_slab_alloc(
		struct kmem_cache *s, void *object, gfp_t flags, bool init)
{
	if (kasan_enabled())
		return __kasan_slab_alloc(s, object, flags, init);
	return object;
}

void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
				    size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
				const void *object, size_t size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_kmalloc(s, object, size, flags);
	return (void *)object;
}

void * __must_check __kasan_kmalloc_large(const void *ptr,
					  size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
						      size_t size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_kmalloc_large(ptr, size, flags);
	return (void *)ptr;
}

void * __must_check __kasan_krealloc(const void *object,
				     size_t new_size, gfp_t flags);
static __always_inline void * __must_check kasan_krealloc(const void *object,
						 size_t new_size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_krealloc(object, new_size, flags);
	return (void *)object;
}

/*
 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
 */
bool __kasan_check_byte(const void *addr, unsigned long ip);
static __always_inline bool kasan_check_byte(const void *addr)
{
	if (kasan_enabled())
		return __kasan_check_byte(addr, _RET_IP_);
	return true;
}


bool kasan_save_enable_multi_shot(void);
void kasan_restore_multi_shot(bool enabled);

#else /* CONFIG_KASAN */

static inline slab_flags_t kasan_never_merge(void)
{
	return 0;
}
static inline void kasan_unpoison_range(const void *address, size_t size) {}
static inline void kasan_poison_pages(struct page *page, unsigned int order,
				      bool init) {}
static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
					bool init) {}
static inline void kasan_cache_create(struct kmem_cache *cache,
				      unsigned int *size,
				      slab_flags_t *flags) {}
static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
static inline void kasan_poison_slab(struct page *page) {}
static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
					void *object) {}
static inline void kasan_poison_object_data(struct kmem_cache *cache,
					void *object) {}
static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
				const void *object)
{
	return (void *)object;
}
static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init)
{
	return false;
}
static inline void kasan_kfree_large(void *ptr) {}
static inline void kasan_slab_free_mempool(void *ptr) {}
static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
				   gfp_t flags, bool init)
{
	return object;
}
static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
				size_t size, gfp_t flags)
{
	return (void *)object;
}
static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
	return (void *)ptr;
}
static inline void *kasan_krealloc(const void *object, size_t new_size,
				 gfp_t flags)
{
	return (void *)object;
}
static inline bool kasan_check_byte(const void *address)
{
	return true;
}

#endif /* CONFIG_KASAN */

#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
void kasan_unpoison_task_stack(struct task_struct *task);
#else
static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
#endif

#ifdef CONFIG_KASAN_GENERIC

void kasan_cache_shrink(struct kmem_cache *cache);
void kasan_cache_shutdown(struct kmem_cache *cache);
void kasan_record_aux_stack(void *ptr);
void kasan_record_aux_stack_noalloc(void *ptr);

#else /* CONFIG_KASAN_GENERIC */

static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
static inline void kasan_record_aux_stack(void *ptr) {}
static inline void kasan_record_aux_stack_noalloc(void *ptr) {}

#endif /* CONFIG_KASAN_GENERIC */

#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)

static inline void *kasan_reset_tag(const void *addr)
{
	return (void *)arch_kasan_reset_tag(addr);
}

/**
 * kasan_report - print a report about a bad memory access detected by KASAN
 * @addr: address of the bad access
 * @size: size of the bad access
 * @is_write: whether the bad access is a write or a read
 * @ip: instruction pointer for the accessibility check or the bad access itself
 */
bool kasan_report(unsigned long addr, size_t size,
		bool is_write, unsigned long ip);

#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */

static inline void *kasan_reset_tag(const void *addr)
{
	return (void *)addr;
}

#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/

#ifdef CONFIG_KASAN_HW_TAGS

void kasan_report_async(void);

#endif /* CONFIG_KASAN_HW_TAGS */

#ifdef CONFIG_KASAN_SW_TAGS
void __init kasan_init_sw_tags(void);
#else
static inline void kasan_init_sw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_HW_TAGS
void kasan_init_hw_tags_cpu(void);
void __init kasan_init_hw_tags(void);
#else
static inline void kasan_init_hw_tags_cpu(void) { }
static inline void kasan_init_hw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_VMALLOC

int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
void kasan_poison_vmalloc(const void *start, unsigned long size);
void kasan_unpoison_vmalloc(const void *start, unsigned long size);
void kasan_release_vmalloc(unsigned long start, unsigned long end,
			   unsigned long free_region_start,
			   unsigned long free_region_end);

void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);

#else /* CONFIG_KASAN_VMALLOC */

static inline int kasan_populate_vmalloc(unsigned long start,
					unsigned long size)
{
	return 0;
}

static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
{ }
static inline void kasan_unpoison_vmalloc(const void *start, unsigned long size)
{ }
static inline void kasan_release_vmalloc(unsigned long start,
					 unsigned long end,
					 unsigned long free_region_start,
					 unsigned long free_region_end) {}

static inline void kasan_populate_early_vm_area_shadow(void *start,
						       unsigned long size)
{ }

#endif /* CONFIG_KASAN_VMALLOC */

#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
		!defined(CONFIG_KASAN_VMALLOC)

/*
 * These functions provide a special case to support backing module
 * allocations with real shadow memory. With KASAN vmalloc, the special
 * case is unnecessary, as the work is handled in the generic case.
 */
int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask);
void kasan_free_shadow(const struct vm_struct *vm);

#else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

static inline int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
static inline void kasan_free_shadow(const struct vm_struct *vm) {}

#endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

#ifdef CONFIG_KASAN_INLINE
void kasan_non_canonical_hook(unsigned long addr);
#else /* CONFIG_KASAN_INLINE */
static inline void kasan_non_canonical_hook(unsigned long addr) { }
#endif /* CONFIG_KASAN_INLINE */

#endif /* LINUX_KASAN_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
0b24becc AR	2	#ifndef _LINUX_KASAN_H
	3	#define _LINUX_KASAN_H
	4
7a3b8353	5	#include <linux/bug.h>
2db710cc	6	#include <linux/kernel.h>
34303244	7	#include <linux/static_key.h>
0b24becc AR	8	#include <linux/types.h>
	9
	10	struct kmem_cache;
	11	struct page;
a5af5aa8	12	struct vm_struct;
5be9b730	13	struct task_struct;
0b24becc AR	14
	15	#ifdef CONFIG_KASAN
	16
d5750edf	17	#include <linux/linkage.h>
65fddcfc	18	#include <asm/kasan.h>
0b24becc	19
83c4e7a0 PA	20	/* kasan_data struct is used in KUnit tests for KASAN expected failures */
83c4e7a0 PA	21	struct kunit_kasan_expectation {
83c4e7a0 PA	22	bool report_found;
	23	};
	24
d5750edf AK	25	#endif
	26
	27	#if defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)
	28
	29	#include <linux/pgtable.h>
	30
	31	/* Software KASAN implementations use shadow memory. */
	32
	33	#ifdef CONFIG_KASAN_SW_TAGS
a064cb00 AK	34	/* This matches KASAN_TAG_INVALID. */
a064cb00 AK	35	#define KASAN_SHADOW_INIT 0xFE
d5750edf AK	36	#else
	37	#define KASAN_SHADOW_INIT 0
	38	#endif
	39
29970dc2 HL	40	#ifndef PTE_HWTABLE_PTRS
	41	#define PTE_HWTABLE_PTRS 0
	42	#endif
	43
9577dd74	44	extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
cb32c9c5 DA	45	extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
	46	extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
	47	extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
9577dd74	48	extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
69786cdb	49
9577dd74	50	int kasan_populate_early_shadow(const void *shadow_start,
69786cdb AR	51	const void *shadow_end);
69786cdb AR	52
0b24becc AR	53	static inline void kasan_mem_to_shadow(const void addr)
	54	{
	55	return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
	56	+ KASAN_SHADOW_OFFSET;
	57	}
	58
d5750edf AK	59	int kasan_add_zero_shadow(void *start, unsigned long size);
	60	void kasan_remove_zero_shadow(void *start, unsigned long size);
	61
d73b4936 AK	62	/* Enable reporting bugs after kasan_disable_current() */
	63	extern void kasan_enable_current(void);
	64
	65	/* Disable reporting bugs for current task */
	66	extern void kasan_disable_current(void);
	67
d5750edf AK	68	#else /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
	69
	70	static inline int kasan_add_zero_shadow(void *start, unsigned long size)
	71	{
	72	return 0;
	73	}
	74	static inline void kasan_remove_zero_shadow(void *start,
	75	unsigned long size)
	76	{}
	77
d73b4936 AK	78	static inline void kasan_enable_current(void) {}
	79	static inline void kasan_disable_current(void) {}
	80
d5750edf AK	81	#endif /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
d5750edf AK	82
34303244	83	#ifdef CONFIG_KASAN_HW_TAGS
e86f8b09	84
34303244	85	DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);
e86f8b09	86
34303244 AK	87	static __always_inline bool kasan_enabled(void)
	88	{
	89	return static_branch_likely(&kasan_flag_enabled);
	90	}
e86f8b09	91
e5af50a5	92	static inline bool kasan_hw_tags_enabled(void)
1bb5eab3 AK	93	{
	94	return kasan_enabled();
	95	}
	96
7a3b8353 PC	97	void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags);
	98	void kasan_free_pages(struct page *page, unsigned int order);
	99
e86f8b09 AK	100	#else /* CONFIG_KASAN_HW_TAGS */
e86f8b09 AK	101
34303244 AK	102	static inline bool kasan_enabled(void)
34303244 AK	103	{
7a3b8353	104	return IS_ENABLED(CONFIG_KASAN);
34303244	105	}
e86f8b09	106
e5af50a5	107	static inline bool kasan_hw_tags_enabled(void)
1bb5eab3 AK	108	{
	109	return false;
	110	}
	111
7a3b8353 PC	112	static __always_inline void kasan_alloc_pages(struct page *page,
	113	unsigned int order, gfp_t flags)
	114	{
	115	/* Only available for integrated init. */
	116	BUILD_BUG();
	117	}
	118
	119	static __always_inline void kasan_free_pages(struct page *page,
	120	unsigned int order)
	121	{
	122	/* Only available for integrated init. */
	123	BUILD_BUG();
	124	}
	125
e86f8b09 AK	126	#endif /* CONFIG_KASAN_HW_TAGS */
e86f8b09 AK	127
e5af50a5 PC	128	static inline bool kasan_has_integrated_init(void)
	129	{
	130	return kasan_hw_tags_enabled();
	131	}
	132
7a3b8353 PC	133	#ifdef CONFIG_KASAN
	134
	135	struct kasan_cache {
	136	int alloc_meta_offset;
	137	int free_meta_offset;
	138	bool is_kmalloc;
	139	};
	140
e86f8b09 AK	141	slab_flags_t __kasan_never_merge(void);
	142	static __always_inline slab_flags_t kasan_never_merge(void)
	143	{
	144	if (kasan_enabled())
	145	return __kasan_never_merge();
	146	return 0;
	147	}
b8c73fc2	148
34303244 AK	149	void __kasan_unpoison_range(const void *addr, size_t size);
	150	static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
	151	{
	152	if (kasan_enabled())
	153	__kasan_unpoison_range(addr, size);
	154	}
7ed2f9e6	155
7a3b8353 PC	156	void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
7a3b8353 PC	157	static __always_inline void kasan_poison_pages(struct page *page,
1bb5eab3	158	unsigned int order, bool init)
34303244 AK	159	{
34303244 AK	160	if (kasan_enabled())
7a3b8353	161	__kasan_poison_pages(page, order, init);
34303244	162	}
0316bec2	163
7a3b8353 PC	164	void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
	165	static __always_inline void kasan_unpoison_pages(struct page *page,
	166	unsigned int order, bool init)
34303244 AK	167	{
34303244 AK	168	if (kasan_enabled())
7a3b8353	169	__kasan_unpoison_pages(page, order, init);
34303244	170	}
0316bec2	171
34303244 AK	172	void __kasan_cache_create(struct kmem_cache cache, unsigned int size,
	173	slab_flags_t *flags);
	174	static __always_inline void kasan_cache_create(struct kmem_cache *cache,
	175	unsigned int size, slab_flags_t flags)
	176	{
	177	if (kasan_enabled())
	178	__kasan_cache_create(cache, size, flags);
	179	}
0316bec2	180
92850134 AK	181	void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
	182	static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
	183	{
	184	if (kasan_enabled())
	185	__kasan_cache_create_kmalloc(cache);
	186	}
	187
34303244 AK	188	size_t __kasan_metadata_size(struct kmem_cache *cache);
	189	static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
	190	{
	191	if (kasan_enabled())
	192	return __kasan_metadata_size(cache);
	193	return 0;
	194	}
7ed2f9e6	195
34303244 AK	196	void __kasan_poison_slab(struct page *page);
	197	static __always_inline void kasan_poison_slab(struct page *page)
	198	{
	199	if (kasan_enabled())
	200	__kasan_poison_slab(page);
	201	}
	202
	203	void __kasan_unpoison_object_data(struct kmem_cache cache, void object);
	204	static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
	205	void *object)
	206	{
	207	if (kasan_enabled())
	208	__kasan_unpoison_object_data(cache, object);
	209	}
	210
	211	void __kasan_poison_object_data(struct kmem_cache cache, void object);
	212	static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
	213	void *object)
	214	{
	215	if (kasan_enabled())
	216	__kasan_poison_object_data(cache, object);
	217	}
	218
	219	void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
	220	const void *object);
	221	static __always_inline void * __must_check kasan_init_slab_obj(
	222	struct kmem_cache cache, const void object)
	223	{
	224	if (kasan_enabled())
	225	return __kasan_init_slab_obj(cache, object);
	226	return (void *)object;
	227	}
	228
d57a964e AK	229	bool __kasan_slab_free(struct kmem_cache s, void object,
	230	unsigned long ip, bool init);
	231	static __always_inline bool kasan_slab_free(struct kmem_cache *s,
	232	void *object, bool init)
34303244 AK	233	{
34303244 AK	234	if (kasan_enabled())
d57a964e	235	return __kasan_slab_free(s, object, _RET_IP_, init);
34303244 AK	236	return false;
	237	}
	238
200072ce AK	239	void __kasan_kfree_large(void *ptr, unsigned long ip);
	240	static __always_inline void kasan_kfree_large(void *ptr)
	241	{
	242	if (kasan_enabled())
	243	__kasan_kfree_large(ptr, _RET_IP_);
	244	}
	245
eeb3160c	246	void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
027b37b5	247	static __always_inline void kasan_slab_free_mempool(void *ptr)
eeb3160c AK	248	{
eeb3160c AK	249	if (kasan_enabled())
027b37b5	250	__kasan_slab_free_mempool(ptr, _RET_IP_);
eeb3160c AK	251	}
eeb3160c AK	252
34303244	253	void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
da844b78	254	void *object, gfp_t flags, bool init);
34303244	255	static __always_inline void * __must_check kasan_slab_alloc(
da844b78	256	struct kmem_cache s, void object, gfp_t flags, bool init)
34303244 AK	257	{
34303244 AK	258	if (kasan_enabled())
da844b78	259	return __kasan_slab_alloc(s, object, flags, init);
34303244 AK	260	return object;
	261	}
	262
	263	void * __must_check __kasan_kmalloc(struct kmem_cache s, const void object,
	264	size_t size, gfp_t flags);
	265	static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
	266	const void *object, size_t size, gfp_t flags)
	267	{
	268	if (kasan_enabled())
	269	return __kasan_kmalloc(s, object, size, flags);
	270	return (void *)object;
	271	}
	272
	273	void * __must_check __kasan_kmalloc_large(const void *ptr,
	274	size_t size, gfp_t flags);
	275	static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
	276	size_t size, gfp_t flags)
	277	{
	278	if (kasan_enabled())
	279	return __kasan_kmalloc_large(ptr, size, flags);
	280	return (void *)ptr;
	281	}
	282
	283	void * __must_check __kasan_krealloc(const void *object,
	284	size_t new_size, gfp_t flags);
	285	static __always_inline void * __must_check kasan_krealloc(const void *object,
	286	size_t new_size, gfp_t flags)
	287	{
	288	if (kasan_enabled())
	289	return __kasan_krealloc(object, new_size, flags);
	290	return (void *)object;
	291	}
	292
611806b4 AK	293	/*
	294	* Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
	295	* the hardware tag-based mode that doesn't rely on compiler instrumentation.
	296	*/
	297	bool __kasan_check_byte(const void *addr, unsigned long ip);
	298	static __always_inline bool kasan_check_byte(const void *addr)
	299	{
	300	if (kasan_enabled())
	301	return __kasan_check_byte(addr, _RET_IP_);
	302	return true;
	303	}
	304
	305
b0845ce5 MR	306	bool kasan_save_enable_multi_shot(void);
	307	void kasan_restore_multi_shot(bool enabled);
	308
0b24becc AR	309	#else /* CONFIG_KASAN */
0b24becc AR	310
e86f8b09 AK	311	static inline slab_flags_t kasan_never_merge(void)
	312	{
	313	return 0;
	314	}
cebd0eb2	315	static inline void kasan_unpoison_range(const void *address, size_t size) {}
7a3b8353 PC	316	static inline void kasan_poison_pages(struct page *page, unsigned int order,
	317	bool init) {}
	318	static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
	319	bool init) {}
7ed2f9e6	320	static inline void kasan_cache_create(struct kmem_cache *cache,
be4a7988	321	unsigned int *size,
d50112ed	322	slab_flags_t *flags) {}
92850134	323	static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
34303244	324	static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
0316bec2 AR	325	static inline void kasan_poison_slab(struct page *page) {}
	326	static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
	327	void *object) {}
	328	static inline void kasan_poison_object_data(struct kmem_cache *cache,
	329	void *object) {}
0116523c AK	330	static inline void kasan_init_slab_obj(struct kmem_cache cache,
	331	const void *object)
	332	{
	333	return (void *)object;
	334	}
d57a964e	335	static inline bool kasan_slab_free(struct kmem_cache s, void object, bool init)
34303244 AK	336	{
	337	return false;
	338	}
200072ce	339	static inline void kasan_kfree_large(void *ptr) {}
027b37b5	340	static inline void kasan_slab_free_mempool(void *ptr) {}
34303244	341	static inline void kasan_slab_alloc(struct kmem_cache s, void *object,
da844b78	342	gfp_t flags, bool init)
0116523c	343	{
34303244	344	return object;
0116523c	345	}
0116523c AK	346	static inline void kasan_kmalloc(struct kmem_cache s, const void *object,
	347	size_t size, gfp_t flags)
	348	{
	349	return (void *)object;
	350	}
34303244 AK	351	static inline void kasan_kmalloc_large(const void ptr, size_t size, gfp_t flags)
	352	{
	353	return (void *)ptr;
	354	}
0116523c AK	355	static inline void kasan_krealloc(const void object, size_t new_size,
	356	gfp_t flags)
	357	{
	358	return (void *)object;
	359	}
611806b4 AK	360	static inline bool kasan_check_byte(const void *address)
	361	{
	362	return true;
	363	}
9b75a867	364
0b24becc AR	365	#endif /* CONFIG_KASAN */
0b24becc AR	366
02c58773	367	#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
d56a9ef8 AK	368	void kasan_unpoison_task_stack(struct task_struct *task);
	369	#else
	370	static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
	371	#endif
	372
2bd926b4 AK	373	#ifdef CONFIG_KASAN_GENERIC
	374
	375	void kasan_cache_shrink(struct kmem_cache *cache);
	376	void kasan_cache_shutdown(struct kmem_cache *cache);
26e760c9	377	void kasan_record_aux_stack(void *ptr);
7cb3007c	378	void kasan_record_aux_stack_noalloc(void *ptr);
2bd926b4 AK	379
	380	#else /* CONFIG_KASAN_GENERIC */
	381
	382	static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
	383	static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
26e760c9	384	static inline void kasan_record_aux_stack(void *ptr) {}
7cb3007c	385	static inline void kasan_record_aux_stack_noalloc(void *ptr) {}
2bd926b4 AK	386
	387	#endif /* CONFIG_KASAN_GENERIC */
	388
2e903b91	389	#if defined(CONFIG_KASAN_SW_TAGS) \|\| defined(CONFIG_KASAN_HW_TAGS)
3c9e3aa1	390
c0054c56 AK	391	static inline void kasan_reset_tag(const void addr)
	392	{
	393	return (void *)arch_kasan_reset_tag(addr);
	394	}
3c9e3aa1	395
49c6631d VF	396	/**
	397	* kasan_report - print a report about a bad memory access detected by KASAN
	398	* @addr: address of the bad access
	399	* @size: size of the bad access
	400	* @is_write: whether the bad access is a write or a read
	401	* @ip: instruction pointer for the accessibility check or the bad access itself
	402	*/
8cceeff4	403	bool kasan_report(unsigned long addr, size_t size,
41eea9cd AK	404	bool is_write, unsigned long ip);
41eea9cd AK	405
2e903b91	406	#else /* CONFIG_KASAN_SW_TAGS \|\| CONFIG_KASAN_HW_TAGS */
3c9e3aa1 AK	407
	408	static inline void kasan_reset_tag(const void addr)
	409	{
	410	return (void *)addr;
	411	}
	412
2e903b91 AK	413	#endif /* CONFIG_KASAN_SW_TAGS \|\| CONFIG_KASAN_HW_TAGS*/
2e903b91 AK	414
8f7b5054 VF	415	#ifdef CONFIG_KASAN_HW_TAGS
	416
	417	void kasan_report_async(void);
	418
	419	#endif /* CONFIG_KASAN_HW_TAGS */
	420
2e903b91 AK	421	#ifdef CONFIG_KASAN_SW_TAGS
	422	void __init kasan_init_sw_tags(void);
	423	#else
	424	static inline void kasan_init_sw_tags(void) { }
	425	#endif
	426
	427	#ifdef CONFIG_KASAN_HW_TAGS
	428	void kasan_init_hw_tags_cpu(void);
	429	void __init kasan_init_hw_tags(void);
	430	#else
	431	static inline void kasan_init_hw_tags_cpu(void) { }
	432	static inline void kasan_init_hw_tags(void) { }
	433	#endif
080eb83f	434
3c5c3cfb	435	#ifdef CONFIG_KASAN_VMALLOC
3b1a4a86	436
d98c9e83 AR	437	int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
	438	void kasan_poison_vmalloc(const void *start, unsigned long size);
	439	void kasan_unpoison_vmalloc(const void *start, unsigned long size);
3c5c3cfb DA	440	void kasan_release_vmalloc(unsigned long start, unsigned long end,
	441	unsigned long free_region_start,
	442	unsigned long free_region_end);
3b1a4a86	443
3252b1d8 KW	444	void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
3252b1d8 KW	445
3b1a4a86 AK	446	#else /* CONFIG_KASAN_VMALLOC */
3b1a4a86 AK	447
d98c9e83 AR	448	static inline int kasan_populate_vmalloc(unsigned long start,
d98c9e83 AR	449	unsigned long size)
3c5c3cfb DA	450	{
	451	return 0;
	452	}
	453
d98c9e83 AR	454	static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
	455	{ }
	456	static inline void kasan_unpoison_vmalloc(const void *start, unsigned long size)
	457	{ }
3c5c3cfb DA	458	static inline void kasan_release_vmalloc(unsigned long start,
	459	unsigned long end,
	460	unsigned long free_region_start,
	461	unsigned long free_region_end) {}
3b1a4a86	462
3252b1d8 KW	463	static inline void kasan_populate_early_vm_area_shadow(void *start,
	464	unsigned long size)
	465	{ }
	466
3b1a4a86 AK	467	#endif /* CONFIG_KASAN_VMALLOC */
3b1a4a86 AK	468
0fea6e9a AK	469	#if (defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)) && \
0fea6e9a AK	470	!defined(CONFIG_KASAN_VMALLOC)
3b1a4a86 AK	471
	472	/*
	473	* These functions provide a special case to support backing module
	474	* allocations with real shadow memory. With KASAN vmalloc, the special
	475	* case is unnecessary, as the work is handled in the generic case.
	476	*/
60115fa5	477	int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask);
3b1a4a86 AK	478	void kasan_free_shadow(const struct vm_struct *vm);
3b1a4a86 AK	479
0fea6e9a	480	#else /* (CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
3b1a4a86	481
60115fa5	482	static inline int kasan_module_alloc(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
3b1a4a86 AK	483	static inline void kasan_free_shadow(const struct vm_struct *vm) {}
3b1a4a86 AK	484
0fea6e9a	485	#endif /* (CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
3c5c3cfb	486
2f004eea JH	487	#ifdef CONFIG_KASAN_INLINE
	488	void kasan_non_canonical_hook(unsigned long addr);
	489	#else /* CONFIG_KASAN_INLINE */
	490	static inline void kasan_non_canonical_hook(unsigned long addr) { }
	491	#endif /* CONFIG_KASAN_INLINE */
	492
0b24becc	493	#endif /* LINUX_KASAN_H */