[linux-block.git] / include / linux / kasan.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_KASAN_H
#define _LINUX_KASAN_H

#include <linux/bug.h>
#include <linux/kasan-enabled.h>
#include <linux/kernel.h>
#include <linux/static_key.h>
#include <linux/types.h>

struct kmem_cache;
struct page;
struct slab;
struct vm_struct;
struct task_struct;

#ifdef CONFIG_KASAN

#include <linux/linkage.h>
#include <asm/kasan.h>

#endif

typedef unsigned int __bitwise kasan_vmalloc_flags_t;

#define KASAN_VMALLOC_NONE		((__force kasan_vmalloc_flags_t)0x00u)
#define KASAN_VMALLOC_INIT		((__force kasan_vmalloc_flags_t)0x01u)
#define KASAN_VMALLOC_VM_ALLOC		((__force kasan_vmalloc_flags_t)0x02u)
#define KASAN_VMALLOC_PROT_NORMAL	((__force kasan_vmalloc_flags_t)0x04u)

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)

#include <linux/pgtable.h>

/* Software KASAN implementations use shadow memory. */

#ifdef CONFIG_KASAN_SW_TAGS
/* This matches KASAN_TAG_INVALID. */
#define KASAN_SHADOW_INIT 0xFE
#else
#define KASAN_SHADOW_INIT 0
#endif

#ifndef PTE_HWTABLE_PTRS
#define PTE_HWTABLE_PTRS 0
#endif

extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];

int kasan_populate_early_shadow(const void *shadow_start,
				const void *shadow_end);

static inline void *kasan_mem_to_shadow(const void *addr)
{
	return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
		+ KASAN_SHADOW_OFFSET;
}

int kasan_add_zero_shadow(void *start, unsigned long size);
void kasan_remove_zero_shadow(void *start, unsigned long size);

/* Enable reporting bugs after kasan_disable_current() */
extern void kasan_enable_current(void);

/* Disable reporting bugs for current task */
extern void kasan_disable_current(void);

#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

static inline int kasan_add_zero_shadow(void *start, unsigned long size)
{
	return 0;
}
static inline void kasan_remove_zero_shadow(void *start,
					unsigned long size)
{}

static inline void kasan_enable_current(void) {}
static inline void kasan_disable_current(void) {}

#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

#ifdef CONFIG_KASAN_HW_TAGS

#else /* CONFIG_KASAN_HW_TAGS */

#endif /* CONFIG_KASAN_HW_TAGS */

static inline bool kasan_has_integrated_init(void)
{
	return kasan_hw_tags_enabled();
}

#ifdef CONFIG_KASAN
void __kasan_unpoison_range(const void *addr, size_t size);
static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
{
	if (kasan_enabled())
		__kasan_unpoison_range(addr, size);
}

void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
static __always_inline void kasan_poison_pages(struct page *page,
						unsigned int order, bool init)
{
	if (kasan_enabled())
		__kasan_poison_pages(page, order, init);
}

bool __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
static __always_inline bool kasan_unpoison_pages(struct page *page,
						 unsigned int order, bool init)
{
	if (kasan_enabled())
		return __kasan_unpoison_pages(page, order, init);
	return false;
}

void __kasan_poison_slab(struct slab *slab);
static __always_inline void kasan_poison_slab(struct slab *slab)
{
	if (kasan_enabled())
		__kasan_poison_slab(slab);
}

void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
							void *object)
{
	if (kasan_enabled())
		__kasan_unpoison_object_data(cache, object);
}

void __kasan_poison_object_data(struct kmem_cache *cache, void *object);
static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
							void *object)
{
	if (kasan_enabled())
		__kasan_poison_object_data(cache, object);
}

void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
					  const void *object);
static __always_inline void * __must_check kasan_init_slab_obj(
				struct kmem_cache *cache, const void *object)
{
	if (kasan_enabled())
		return __kasan_init_slab_obj(cache, object);
	return (void *)object;
}

bool __kasan_slab_free(struct kmem_cache *s, void *object,
			unsigned long ip, bool init);
static __always_inline bool kasan_slab_free(struct kmem_cache *s,
						void *object, bool init)
{
	if (kasan_enabled())
		return __kasan_slab_free(s, object, _RET_IP_, init);
	return false;
}

void __kasan_kfree_large(void *ptr, unsigned long ip);
static __always_inline void kasan_kfree_large(void *ptr)
{
	if (kasan_enabled())
		__kasan_kfree_large(ptr, _RET_IP_);
}

void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
static __always_inline void kasan_slab_free_mempool(void *ptr)
{
	if (kasan_enabled())
		__kasan_slab_free_mempool(ptr, _RET_IP_);
}

void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
				       void *object, gfp_t flags, bool init);
static __always_inline void * __must_check kasan_slab_alloc(
		struct kmem_cache *s, void *object, gfp_t flags, bool init)
{
	if (kasan_enabled())
		return __kasan_slab_alloc(s, object, flags, init);
	return object;
}

void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
				    size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
				const void *object, size_t size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_kmalloc(s, object, size, flags);
	return (void *)object;
}

void * __must_check __kasan_kmalloc_large(const void *ptr,
					  size_t size, gfp_t flags);
static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
						      size_t size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_kmalloc_large(ptr, size, flags);
	return (void *)ptr;
}

void * __must_check __kasan_krealloc(const void *object,
				     size_t new_size, gfp_t flags);
static __always_inline void * __must_check kasan_krealloc(const void *object,
						 size_t new_size, gfp_t flags)
{
	if (kasan_enabled())
		return __kasan_krealloc(object, new_size, flags);
	return (void *)object;
}

/*
 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
 */
bool __kasan_check_byte(const void *addr, unsigned long ip);
static __always_inline bool kasan_check_byte(const void *addr)
{
	if (kasan_enabled())
		return __kasan_check_byte(addr, _RET_IP_);
	return true;
}

#else /* CONFIG_KASAN */

static inline void kasan_unpoison_range(const void *address, size_t size) {}
static inline void kasan_poison_pages(struct page *page, unsigned int order,
				      bool init) {}
static inline bool kasan_unpoison_pages(struct page *page, unsigned int order,
					bool init)
{
	return false;
}
static inline void kasan_poison_slab(struct slab *slab) {}
static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
					void *object) {}
static inline void kasan_poison_object_data(struct kmem_cache *cache,
					void *object) {}
static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
				const void *object)
{
	return (void *)object;
}
static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init)
{
	return false;
}
static inline void kasan_kfree_large(void *ptr) {}
static inline void kasan_slab_free_mempool(void *ptr) {}
static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
				   gfp_t flags, bool init)
{
	return object;
}
static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
				size_t size, gfp_t flags)
{
	return (void *)object;
}
static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
	return (void *)ptr;
}
static inline void *kasan_krealloc(const void *object, size_t new_size,
				 gfp_t flags)
{
	return (void *)object;
}
static inline bool kasan_check_byte(const void *address)
{
	return true;
}

#endif /* CONFIG_KASAN */

#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
void kasan_unpoison_task_stack(struct task_struct *task);
#else
static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
#endif

#ifdef CONFIG_KASAN_GENERIC

struct kasan_cache {
	int alloc_meta_offset;
	int free_meta_offset;
};

size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object);
slab_flags_t kasan_never_merge(void);
void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
			slab_flags_t *flags);

void kasan_cache_shrink(struct kmem_cache *cache);
void kasan_cache_shutdown(struct kmem_cache *cache);
void kasan_record_aux_stack(void *ptr);
void kasan_record_aux_stack_noalloc(void *ptr);

#else /* CONFIG_KASAN_GENERIC */

/* Tag-based KASAN modes do not use per-object metadata. */
static inline size_t kasan_metadata_size(struct kmem_cache *cache,
						bool in_object)
{
	return 0;
}
/* And thus nothing prevents cache merging. */
static inline slab_flags_t kasan_never_merge(void)
{
	return 0;
}
/* And no cache-related metadata initialization is required. */
static inline void kasan_cache_create(struct kmem_cache *cache,
				      unsigned int *size,
				      slab_flags_t *flags) {}

static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
static inline void kasan_record_aux_stack(void *ptr) {}
static inline void kasan_record_aux_stack_noalloc(void *ptr) {}

#endif /* CONFIG_KASAN_GENERIC */

#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)

static inline void *kasan_reset_tag(const void *addr)
{
	return (void *)arch_kasan_reset_tag(addr);
}

/**
 * kasan_report - print a report about a bad memory access detected by KASAN
 * @addr: address of the bad access
 * @size: size of the bad access
 * @is_write: whether the bad access is a write or a read
 * @ip: instruction pointer for the accessibility check or the bad access itself
 */
bool kasan_report(unsigned long addr, size_t size,
		bool is_write, unsigned long ip);

#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */

static inline void *kasan_reset_tag(const void *addr)
{
	return (void *)addr;
}

#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/

#ifdef CONFIG_KASAN_HW_TAGS

void kasan_report_async(void);

#endif /* CONFIG_KASAN_HW_TAGS */

#ifdef CONFIG_KASAN_SW_TAGS
void __init kasan_init_sw_tags(void);
#else
static inline void kasan_init_sw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_HW_TAGS
void kasan_init_hw_tags_cpu(void);
void __init kasan_init_hw_tags(void);
#else
static inline void kasan_init_hw_tags_cpu(void) { }
static inline void kasan_init_hw_tags(void) { }
#endif

#ifdef CONFIG_KASAN_VMALLOC

#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)

void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
void kasan_release_vmalloc(unsigned long start, unsigned long end,
			   unsigned long free_region_start,
			   unsigned long free_region_end);

#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

static inline void kasan_populate_early_vm_area_shadow(void *start,
						       unsigned long size)
{ }
static inline int kasan_populate_vmalloc(unsigned long start,
					unsigned long size)
{
	return 0;
}
static inline void kasan_release_vmalloc(unsigned long start,
					 unsigned long end,
					 unsigned long free_region_start,
					 unsigned long free_region_end) { }

#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */

void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
			       kasan_vmalloc_flags_t flags);
static __always_inline void *kasan_unpoison_vmalloc(const void *start,
						unsigned long size,
						kasan_vmalloc_flags_t flags)
{
	if (kasan_enabled())
		return __kasan_unpoison_vmalloc(start, size, flags);
	return (void *)start;
}

void __kasan_poison_vmalloc(const void *start, unsigned long size);
static __always_inline void kasan_poison_vmalloc(const void *start,
						 unsigned long size)
{
	if (kasan_enabled())
		__kasan_poison_vmalloc(start, size);
}

#else /* CONFIG_KASAN_VMALLOC */

static inline void kasan_populate_early_vm_area_shadow(void *start,
						       unsigned long size) { }
static inline int kasan_populate_vmalloc(unsigned long start,
					unsigned long size)
{
	return 0;
}
static inline void kasan_release_vmalloc(unsigned long start,
					 unsigned long end,
					 unsigned long free_region_start,
					 unsigned long free_region_end) { }

static inline void *kasan_unpoison_vmalloc(const void *start,
					   unsigned long size,
					   kasan_vmalloc_flags_t flags)
{
	return (void *)start;
}
static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
{ }

#endif /* CONFIG_KASAN_VMALLOC */

#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
		!defined(CONFIG_KASAN_VMALLOC)

/*
 * These functions allocate and free shadow memory for kernel modules.
 * They are only required when KASAN_VMALLOC is not supported, as otherwise
 * shadow memory is allocated by the generic vmalloc handlers.
 */
int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask);
void kasan_free_module_shadow(const struct vm_struct *vm);

#else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

static inline int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
static inline void kasan_free_module_shadow(const struct vm_struct *vm) {}

#endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */

#ifdef CONFIG_KASAN_INLINE
void kasan_non_canonical_hook(unsigned long addr);
#else /* CONFIG_KASAN_INLINE */
static inline void kasan_non_canonical_hook(unsigned long addr) { }
#endif /* CONFIG_KASAN_INLINE */

#endif /* LINUX_KASAN_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
0b24becc AR	2	#ifndef _LINUX_KASAN_H
	3	#define _LINUX_KASAN_H
	4
7a3b8353	5	#include <linux/bug.h>
f9b5e46f	6	#include <linux/kasan-enabled.h>
2db710cc	7	#include <linux/kernel.h>
34303244	8	#include <linux/static_key.h>
0b24becc AR	9	#include <linux/types.h>
	10
	11	struct kmem_cache;
	12	struct page;
6e48a966	13	struct slab;
a5af5aa8	14	struct vm_struct;
5be9b730	15	struct task_struct;
0b24becc AR	16
	17	#ifdef CONFIG_KASAN
	18
d5750edf	19	#include <linux/linkage.h>
65fddcfc	20	#include <asm/kasan.h>
0b24becc	21
d5750edf AK	22	#endif
d5750edf AK	23
23689e91 AK	24	typedef unsigned int __bitwise kasan_vmalloc_flags_t;
23689e91 AK	25
ec2a0f9c AK	26	#define KASAN_VMALLOC_NONE ((__force kasan_vmalloc_flags_t)0x00u)
	27	#define KASAN_VMALLOC_INIT ((__force kasan_vmalloc_flags_t)0x01u)
	28	#define KASAN_VMALLOC_VM_ALLOC ((__force kasan_vmalloc_flags_t)0x02u)
	29	#define KASAN_VMALLOC_PROT_NORMAL ((__force kasan_vmalloc_flags_t)0x04u)
23689e91	30
d5750edf AK	31	#if defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)
	32
	33	#include <linux/pgtable.h>
	34
	35	/* Software KASAN implementations use shadow memory. */
	36
	37	#ifdef CONFIG_KASAN_SW_TAGS
a064cb00 AK	38	/* This matches KASAN_TAG_INVALID. */
a064cb00 AK	39	#define KASAN_SHADOW_INIT 0xFE
d5750edf AK	40	#else
	41	#define KASAN_SHADOW_INIT 0
	42	#endif
	43
29970dc2 HL	44	#ifndef PTE_HWTABLE_PTRS
	45	#define PTE_HWTABLE_PTRS 0
	46	#endif
	47
9577dd74	48	extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
cb32c9c5 DA	49	extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];
	50	extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];
	51	extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];
9577dd74	52	extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
69786cdb	53
9577dd74	54	int kasan_populate_early_shadow(const void *shadow_start,
69786cdb AR	55	const void *shadow_end);
69786cdb AR	56
0b24becc AR	57	static inline void kasan_mem_to_shadow(const void addr)
	58	{
	59	return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
	60	+ KASAN_SHADOW_OFFSET;
	61	}
	62
d5750edf AK	63	int kasan_add_zero_shadow(void *start, unsigned long size);
	64	void kasan_remove_zero_shadow(void *start, unsigned long size);
	65
d73b4936 AK	66	/* Enable reporting bugs after kasan_disable_current() */
	67	extern void kasan_enable_current(void);
	68
	69	/* Disable reporting bugs for current task */
	70	extern void kasan_disable_current(void);
	71
d5750edf AK	72	#else /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
	73
	74	static inline int kasan_add_zero_shadow(void *start, unsigned long size)
	75	{
	76	return 0;
	77	}
	78	static inline void kasan_remove_zero_shadow(void *start,
	79	unsigned long size)
	80	{}
	81
d73b4936 AK	82	static inline void kasan_enable_current(void) {}
	83	static inline void kasan_disable_current(void) {}
	84
d5750edf AK	85	#endif /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
d5750edf AK	86
34303244	87	#ifdef CONFIG_KASAN_HW_TAGS
e86f8b09	88
e86f8b09 AK	89	#else /* CONFIG_KASAN_HW_TAGS */
e86f8b09 AK	90
e86f8b09 AK	91	#endif /* CONFIG_KASAN_HW_TAGS */
e86f8b09 AK	92
e5af50a5 PC	93	static inline bool kasan_has_integrated_init(void)
	94	{
	95	return kasan_hw_tags_enabled();
	96	}
	97
7a3b8353	98	#ifdef CONFIG_KASAN
34303244 AK	99	void __kasan_unpoison_range(const void *addr, size_t size);
	100	static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
	101	{
	102	if (kasan_enabled())
	103	__kasan_unpoison_range(addr, size);
	104	}
7ed2f9e6	105
7a3b8353 PC	106	void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
7a3b8353 PC	107	static __always_inline void kasan_poison_pages(struct page *page,
1bb5eab3	108	unsigned int order, bool init)
34303244 AK	109	{
34303244 AK	110	if (kasan_enabled())
7a3b8353	111	__kasan_poison_pages(page, order, init);
34303244	112	}
0316bec2	113
44383cef AK	114	bool __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
44383cef AK	115	static __always_inline bool kasan_unpoison_pages(struct page *page,
7a3b8353	116	unsigned int order, bool init)
34303244 AK	117	{
34303244 AK	118	if (kasan_enabled())
44383cef AK	119	return __kasan_unpoison_pages(page, order, init);
44383cef AK	120	return false;
34303244	121	}
0316bec2	122
6e48a966 MWO	123	void __kasan_poison_slab(struct slab *slab);
6e48a966 MWO	124	static __always_inline void kasan_poison_slab(struct slab *slab)
34303244 AK	125	{
34303244 AK	126	if (kasan_enabled())
6e48a966	127	__kasan_poison_slab(slab);
34303244 AK	128	}
	129
	130	void __kasan_unpoison_object_data(struct kmem_cache cache, void object);
	131	static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
	132	void *object)
	133	{
	134	if (kasan_enabled())
	135	__kasan_unpoison_object_data(cache, object);
	136	}
	137
	138	void __kasan_poison_object_data(struct kmem_cache cache, void object);
	139	static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
	140	void *object)
	141	{
	142	if (kasan_enabled())
	143	__kasan_poison_object_data(cache, object);
	144	}
	145
	146	void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
	147	const void *object);
	148	static __always_inline void * __must_check kasan_init_slab_obj(
	149	struct kmem_cache cache, const void object)
	150	{
	151	if (kasan_enabled())
	152	return __kasan_init_slab_obj(cache, object);
	153	return (void *)object;
	154	}
	155
d57a964e AK	156	bool __kasan_slab_free(struct kmem_cache s, void object,
	157	unsigned long ip, bool init);
	158	static __always_inline bool kasan_slab_free(struct kmem_cache *s,
	159	void *object, bool init)
34303244 AK	160	{
34303244 AK	161	if (kasan_enabled())
d57a964e	162	return __kasan_slab_free(s, object, _RET_IP_, init);
34303244 AK	163	return false;
	164	}
	165
200072ce AK	166	void __kasan_kfree_large(void *ptr, unsigned long ip);
	167	static __always_inline void kasan_kfree_large(void *ptr)
	168	{
	169	if (kasan_enabled())
	170	__kasan_kfree_large(ptr, _RET_IP_);
	171	}
	172
eeb3160c	173	void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
027b37b5	174	static __always_inline void kasan_slab_free_mempool(void *ptr)
eeb3160c AK	175	{
eeb3160c AK	176	if (kasan_enabled())
027b37b5	177	__kasan_slab_free_mempool(ptr, _RET_IP_);
eeb3160c AK	178	}
eeb3160c AK	179
34303244	180	void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
da844b78	181	void *object, gfp_t flags, bool init);
34303244	182	static __always_inline void * __must_check kasan_slab_alloc(
da844b78	183	struct kmem_cache s, void object, gfp_t flags, bool init)
34303244 AK	184	{
34303244 AK	185	if (kasan_enabled())
da844b78	186	return __kasan_slab_alloc(s, object, flags, init);
34303244 AK	187	return object;
	188	}
	189
	190	void * __must_check __kasan_kmalloc(struct kmem_cache s, const void object,
	191	size_t size, gfp_t flags);
	192	static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
	193	const void *object, size_t size, gfp_t flags)
	194	{
	195	if (kasan_enabled())
	196	return __kasan_kmalloc(s, object, size, flags);
	197	return (void *)object;
	198	}
	199
	200	void * __must_check __kasan_kmalloc_large(const void *ptr,
	201	size_t size, gfp_t flags);
	202	static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
	203	size_t size, gfp_t flags)
	204	{
	205	if (kasan_enabled())
	206	return __kasan_kmalloc_large(ptr, size, flags);
	207	return (void *)ptr;
	208	}
	209
	210	void * __must_check __kasan_krealloc(const void *object,
	211	size_t new_size, gfp_t flags);
	212	static __always_inline void * __must_check kasan_krealloc(const void *object,
	213	size_t new_size, gfp_t flags)
	214	{
	215	if (kasan_enabled())
	216	return __kasan_krealloc(object, new_size, flags);
	217	return (void *)object;
	218	}
	219
611806b4 AK	220	/*
	221	* Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
	222	* the hardware tag-based mode that doesn't rely on compiler instrumentation.
	223	*/
	224	bool __kasan_check_byte(const void *addr, unsigned long ip);
	225	static __always_inline bool kasan_check_byte(const void *addr)
	226	{
	227	if (kasan_enabled())
	228	return __kasan_check_byte(addr, _RET_IP_);
	229	return true;
	230	}
	231
0b24becc AR	232	#else /* CONFIG_KASAN */
0b24becc AR	233
cebd0eb2	234	static inline void kasan_unpoison_range(const void *address, size_t size) {}
7a3b8353 PC	235	static inline void kasan_poison_pages(struct page *page, unsigned int order,
7a3b8353 PC	236	bool init) {}
44383cef AK	237	static inline bool kasan_unpoison_pages(struct page *page, unsigned int order,
	238	bool init)
	239	{
	240	return false;
	241	}
6e48a966	242	static inline void kasan_poison_slab(struct slab *slab) {}
0316bec2 AR	243	static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
	244	void *object) {}
	245	static inline void kasan_poison_object_data(struct kmem_cache *cache,
	246	void *object) {}
0116523c AK	247	static inline void kasan_init_slab_obj(struct kmem_cache cache,
	248	const void *object)
	249	{
	250	return (void *)object;
	251	}
d57a964e	252	static inline bool kasan_slab_free(struct kmem_cache s, void object, bool init)
34303244 AK	253	{
	254	return false;
	255	}
200072ce	256	static inline void kasan_kfree_large(void *ptr) {}
027b37b5	257	static inline void kasan_slab_free_mempool(void *ptr) {}
34303244	258	static inline void kasan_slab_alloc(struct kmem_cache s, void *object,
da844b78	259	gfp_t flags, bool init)
0116523c	260	{
34303244	261	return object;
0116523c	262	}
0116523c AK	263	static inline void kasan_kmalloc(struct kmem_cache s, const void *object,
	264	size_t size, gfp_t flags)
	265	{
	266	return (void *)object;
	267	}
34303244 AK	268	static inline void kasan_kmalloc_large(const void ptr, size_t size, gfp_t flags)
	269	{
	270	return (void *)ptr;
	271	}
0116523c AK	272	static inline void kasan_krealloc(const void object, size_t new_size,
	273	gfp_t flags)
	274	{
	275	return (void *)object;
	276	}
611806b4 AK	277	static inline bool kasan_check_byte(const void *address)
	278	{
	279	return true;
	280	}
9b75a867	281
0b24becc AR	282	#endif /* CONFIG_KASAN */
0b24becc AR	283
02c58773	284	#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
d56a9ef8 AK	285	void kasan_unpoison_task_stack(struct task_struct *task);
	286	#else
	287	static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
	288	#endif
	289
2bd926b4 AK	290	#ifdef CONFIG_KASAN_GENERIC
2bd926b4 AK	291
bbc61844 FT	292	struct kasan_cache {
	293	int alloc_meta_offset;
	294	int free_meta_offset;
	295	};
	296
5d1ba310	297	size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object);
3b7f8813	298	slab_flags_t kasan_never_merge(void);
682ed089 AK	299	void kasan_cache_create(struct kmem_cache cache, unsigned int size,
682ed089 AK	300	slab_flags_t *flags);
f372bde9	301
2bd926b4 AK	302	void kasan_cache_shrink(struct kmem_cache *cache);
2bd926b4 AK	303	void kasan_cache_shutdown(struct kmem_cache *cache);
26e760c9	304	void kasan_record_aux_stack(void *ptr);
7cb3007c	305	void kasan_record_aux_stack_noalloc(void *ptr);
2bd926b4 AK	306
	307	#else /* CONFIG_KASAN_GENERIC */
	308
f372bde9	309	/* Tag-based KASAN modes do not use per-object metadata. */
5d1ba310 FT	310	static inline size_t kasan_metadata_size(struct kmem_cache *cache,
5d1ba310 FT	311	bool in_object)
f372bde9 AK	312	{
	313	return 0;
	314	}
3b7f8813 AK	315	/* And thus nothing prevents cache merging. */
	316	static inline slab_flags_t kasan_never_merge(void)
	317	{
	318	return 0;
	319	}
682ed089 AK	320	/* And no cache-related metadata initialization is required. */
	321	static inline void kasan_cache_create(struct kmem_cache *cache,
	322	unsigned int *size,
	323	slab_flags_t *flags) {}
f372bde9	324
2bd926b4 AK	325	static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
2bd926b4 AK	326	static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
26e760c9	327	static inline void kasan_record_aux_stack(void *ptr) {}
7cb3007c	328	static inline void kasan_record_aux_stack_noalloc(void *ptr) {}
2bd926b4 AK	329
	330	#endif /* CONFIG_KASAN_GENERIC */
	331
2e903b91	332	#if defined(CONFIG_KASAN_SW_TAGS) \|\| defined(CONFIG_KASAN_HW_TAGS)
3c9e3aa1	333
c0054c56 AK	334	static inline void kasan_reset_tag(const void addr)
	335	{
	336	return (void *)arch_kasan_reset_tag(addr);
	337	}
3c9e3aa1	338
49c6631d VF	339	/**
	340	* kasan_report - print a report about a bad memory access detected by KASAN
	341	* @addr: address of the bad access
	342	* @size: size of the bad access
	343	* @is_write: whether the bad access is a write or a read
	344	* @ip: instruction pointer for the accessibility check or the bad access itself
	345	*/
8cceeff4	346	bool kasan_report(unsigned long addr, size_t size,
41eea9cd AK	347	bool is_write, unsigned long ip);
41eea9cd AK	348
2e903b91	349	#else /* CONFIG_KASAN_SW_TAGS \|\| CONFIG_KASAN_HW_TAGS */
3c9e3aa1 AK	350
	351	static inline void kasan_reset_tag(const void addr)
	352	{
	353	return (void *)addr;
	354	}
	355
2e903b91 AK	356	#endif /* CONFIG_KASAN_SW_TAGS \|\| CONFIG_KASAN_HW_TAGS*/
2e903b91 AK	357
8f7b5054 VF	358	#ifdef CONFIG_KASAN_HW_TAGS
	359
	360	void kasan_report_async(void);
	361
	362	#endif /* CONFIG_KASAN_HW_TAGS */
	363
2e903b91 AK	364	#ifdef CONFIG_KASAN_SW_TAGS
	365	void __init kasan_init_sw_tags(void);
	366	#else
	367	static inline void kasan_init_sw_tags(void) { }
	368	#endif
	369
	370	#ifdef CONFIG_KASAN_HW_TAGS
	371	void kasan_init_hw_tags_cpu(void);
	372	void __init kasan_init_hw_tags(void);
	373	#else
	374	static inline void kasan_init_hw_tags_cpu(void) { }
	375	static inline void kasan_init_hw_tags(void) { }
	376	#endif
080eb83f	377
3c5c3cfb	378	#ifdef CONFIG_KASAN_VMALLOC
3b1a4a86	379
23689e91 AK	380	#if defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)
23689e91 AK	381
5bd9bae2	382	void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
d98c9e83	383	int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
3c5c3cfb DA	384	void kasan_release_vmalloc(unsigned long start, unsigned long end,
	385	unsigned long free_region_start,
	386	unsigned long free_region_end);
3b1a4a86	387
23689e91 AK	388	#else /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
	389
	390	static inline void kasan_populate_early_vm_area_shadow(void *start,
	391	unsigned long size)
	392	{ }
	393	static inline int kasan_populate_vmalloc(unsigned long start,
	394	unsigned long size)
	395	{
	396	return 0;
	397	}
	398	static inline void kasan_release_vmalloc(unsigned long start,
	399	unsigned long end,
	400	unsigned long free_region_start,
	401	unsigned long free_region_end) { }
	402
	403	#endif /* CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS */
	404
	405	void __kasan_unpoison_vmalloc(const void start, unsigned long size,
	406	kasan_vmalloc_flags_t flags);
1d96320f	407	static __always_inline void kasan_unpoison_vmalloc(const void start,
23689e91 AK	408	unsigned long size,
23689e91 AK	409	kasan_vmalloc_flags_t flags)
579fb0ac AK	410	{
579fb0ac AK	411	if (kasan_enabled())
23689e91	412	return __kasan_unpoison_vmalloc(start, size, flags);
1d96320f	413	return (void *)start;
579fb0ac AK	414	}
	415
	416	void __kasan_poison_vmalloc(const void *start, unsigned long size);
	417	static __always_inline void kasan_poison_vmalloc(const void *start,
	418	unsigned long size)
	419	{
	420	if (kasan_enabled())
	421	__kasan_poison_vmalloc(start, size);
	422	}
3252b1d8	423
3b1a4a86 AK	424	#else /* CONFIG_KASAN_VMALLOC */
3b1a4a86 AK	425
5bd9bae2 AK	426	static inline void kasan_populate_early_vm_area_shadow(void *start,
5bd9bae2 AK	427	unsigned long size) { }
d98c9e83 AR	428	static inline int kasan_populate_vmalloc(unsigned long start,
d98c9e83 AR	429	unsigned long size)
3c5c3cfb DA	430	{
	431	return 0;
	432	}
3c5c3cfb DA	433	static inline void kasan_release_vmalloc(unsigned long start,
	434	unsigned long end,
	435	unsigned long free_region_start,
5bd9bae2	436	unsigned long free_region_end) { }
3b1a4a86	437
1d96320f	438	static inline void kasan_unpoison_vmalloc(const void start,
23689e91 AK	439	unsigned long size,
23689e91 AK	440	kasan_vmalloc_flags_t flags)
1d96320f AK	441	{
	442	return (void *)start;
	443	}
5bd9bae2	444	static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
3252b1d8 KW	445	{ }
3252b1d8 KW	446
3b1a4a86 AK	447	#endif /* CONFIG_KASAN_VMALLOC */
3b1a4a86 AK	448
0fea6e9a AK	449	#if (defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KASAN_SW_TAGS)) && \
0fea6e9a AK	450	!defined(CONFIG_KASAN_VMALLOC)
3b1a4a86 AK	451
3b1a4a86 AK	452	/*
63840de2 AK	453	* These functions allocate and free shadow memory for kernel modules.
	454	* They are only required when KASAN_VMALLOC is not supported, as otherwise
	455	* shadow memory is allocated by the generic vmalloc handlers.
3b1a4a86	456	*/
63840de2 AK	457	int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask);
63840de2 AK	458	void kasan_free_module_shadow(const struct vm_struct *vm);
3b1a4a86	459
0fea6e9a	460	#else /* (CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
3b1a4a86	461
63840de2 AK	462	static inline int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { return 0; }
63840de2 AK	463	static inline void kasan_free_module_shadow(const struct vm_struct *vm) {}
3b1a4a86	464
0fea6e9a	465	#endif /* (CONFIG_KASAN_GENERIC \|\| CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
3c5c3cfb	466
2f004eea JH	467	#ifdef CONFIG_KASAN_INLINE
	468	void kasan_non_canonical_hook(unsigned long addr);
	469	#else /* CONFIG_KASAN_INLINE */
	470	static inline void kasan_non_canonical_hook(unsigned long addr) { }
	471	#endif /* CONFIG_KASAN_INLINE */
	472
0b24becc	473	#endif /* LINUX_KASAN_H */