[linux-2.6-block.git] / arch / x86 / mm / kasan_init_64.c

// SPDX-License-Identifier: GPL-2.0
#define DISABLE_BRANCH_PROFILING
#define pr_fmt(fmt) "kasan: " fmt

/* cpu_feature_enabled() cannot be used this early */
#define USE_EARLY_PGTABLE_L5

#include <linux/bootmem.h>
#include <linux/kasan.h>
#include <linux/kdebug.h>
#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/vmalloc.h>

#include <asm/e820/types.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/cpu_entry_area.h>

extern struct range pfn_mapped[E820_MAX_ENTRIES];

static p4d_t tmp_p4d_table[MAX_PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);

static __init void *early_alloc(size_t size, int nid, bool panic)
{
	if (panic)
		return memblock_alloc_try_nid(size, size,
			__pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
	else
		return memblock_alloc_try_nid_nopanic(size, size,
			__pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
}

static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr,
				      unsigned long end, int nid)
{
	pte_t *pte;

	if (pmd_none(*pmd)) {
		void *p;

		if (boot_cpu_has(X86_FEATURE_PSE) &&
		    ((end - addr) == PMD_SIZE) &&
		    IS_ALIGNED(addr, PMD_SIZE)) {
			p = early_alloc(PMD_SIZE, nid, false);
			if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL))
				return;
			else if (p)
				memblock_free(__pa(p), PMD_SIZE);
		}

		p = early_alloc(PAGE_SIZE, nid, true);
		pmd_populate_kernel(&init_mm, pmd, p);
	}

	pte = pte_offset_kernel(pmd, addr);
	do {
		pte_t entry;
		void *p;

		if (!pte_none(*pte))
			continue;

		p = early_alloc(PAGE_SIZE, nid, true);
		entry = pfn_pte(PFN_DOWN(__pa(p)), PAGE_KERNEL);
		set_pte_at(&init_mm, addr, pte, entry);
	} while (pte++, addr += PAGE_SIZE, addr != end);
}

static void __init kasan_populate_pud(pud_t *pud, unsigned long addr,
				      unsigned long end, int nid)
{
	pmd_t *pmd;
	unsigned long next;

	if (pud_none(*pud)) {
		void *p;

		if (boot_cpu_has(X86_FEATURE_GBPAGES) &&
		    ((end - addr) == PUD_SIZE) &&
		    IS_ALIGNED(addr, PUD_SIZE)) {
			p = early_alloc(PUD_SIZE, nid, false);
			if (p && pud_set_huge(pud, __pa(p), PAGE_KERNEL))
				return;
			else if (p)
				memblock_free(__pa(p), PUD_SIZE);
		}

		p = early_alloc(PAGE_SIZE, nid, true);
		pud_populate(&init_mm, pud, p);
	}

	pmd = pmd_offset(pud, addr);
	do {
		next = pmd_addr_end(addr, end);
		if (!pmd_large(*pmd))
			kasan_populate_pmd(pmd, addr, next, nid);
	} while (pmd++, addr = next, addr != end);
}

static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr,
				      unsigned long end, int nid)
{
	pud_t *pud;
	unsigned long next;

	if (p4d_none(*p4d)) {
		void *p = early_alloc(PAGE_SIZE, nid, true);

		p4d_populate(&init_mm, p4d, p);
	}

	pud = pud_offset(p4d, addr);
	do {
		next = pud_addr_end(addr, end);
		if (!pud_large(*pud))
			kasan_populate_pud(pud, addr, next, nid);
	} while (pud++, addr = next, addr != end);
}

static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr,
				      unsigned long end, int nid)
{
	void *p;
	p4d_t *p4d;
	unsigned long next;

	if (pgd_none(*pgd)) {
		p = early_alloc(PAGE_SIZE, nid, true);
		pgd_populate(&init_mm, pgd, p);
	}

	p4d = p4d_offset(pgd, addr);
	do {
		next = p4d_addr_end(addr, end);
		kasan_populate_p4d(p4d, addr, next, nid);
	} while (p4d++, addr = next, addr != end);
}

static void __init kasan_populate_shadow(unsigned long addr, unsigned long end,
					 int nid)
{
	pgd_t *pgd;
	unsigned long next;

	addr = addr & PAGE_MASK;
	end = round_up(end, PAGE_SIZE);
	pgd = pgd_offset_k(addr);
	do {
		next = pgd_addr_end(addr, end);
		kasan_populate_pgd(pgd, addr, next, nid);
	} while (pgd++, addr = next, addr != end);
}

static void __init map_range(struct range *range)
{
	unsigned long start;
	unsigned long end;

	start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start));
	end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end));

	kasan_populate_shadow(start, end, early_pfn_to_nid(range->start));
}

static void __init clear_pgds(unsigned long start,
			unsigned long end)
{
	pgd_t *pgd;
	/* See comment in kasan_init() */
	unsigned long pgd_end = end & PGDIR_MASK;

	for (; start < pgd_end; start += PGDIR_SIZE) {
		pgd = pgd_offset_k(start);
		/*
		 * With folded p4d, pgd_clear() is nop, use p4d_clear()
		 * instead.
		 */
		if (pgtable_l5_enabled())
			pgd_clear(pgd);
		else
			p4d_clear(p4d_offset(pgd, start));
	}

	pgd = pgd_offset_k(start);
	for (; start < end; start += P4D_SIZE)
		p4d_clear(p4d_offset(pgd, start));
}

static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr)
{
	unsigned long p4d;

	if (!pgtable_l5_enabled())
		return (p4d_t *)pgd;

	p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK;
	p4d += __START_KERNEL_map - phys_base;
	return (p4d_t *)p4d + p4d_index(addr);
}

static void __init kasan_early_p4d_populate(pgd_t *pgd,
		unsigned long addr,
		unsigned long end)
{
	pgd_t pgd_entry;
	p4d_t *p4d, p4d_entry;
	unsigned long next;

	if (pgd_none(*pgd)) {
		pgd_entry = __pgd(_KERNPG_TABLE | __pa_nodebug(kasan_zero_p4d));
		set_pgd(pgd, pgd_entry);
	}

	p4d = early_p4d_offset(pgd, addr);
	do {
		next = p4d_addr_end(addr, end);

		if (!p4d_none(*p4d))
			continue;

		p4d_entry = __p4d(_KERNPG_TABLE | __pa_nodebug(kasan_zero_pud));
		set_p4d(p4d, p4d_entry);
	} while (p4d++, addr = next, addr != end && p4d_none(*p4d));
}

static void __init kasan_map_early_shadow(pgd_t *pgd)
{
	/* See comment in kasan_init() */
	unsigned long addr = KASAN_SHADOW_START & PGDIR_MASK;
	unsigned long end = KASAN_SHADOW_END;
	unsigned long next;

	pgd += pgd_index(addr);
	do {
		next = pgd_addr_end(addr, end);
		kasan_early_p4d_populate(pgd, addr, next);
	} while (pgd++, addr = next, addr != end);
}

#ifdef CONFIG_KASAN_INLINE
static int kasan_die_handler(struct notifier_block *self,
			     unsigned long val,
			     void *data)
{
	if (val == DIE_GPF) {
		pr_emerg("CONFIG_KASAN_INLINE enabled\n");
		pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n");
	}
	return NOTIFY_OK;
}

static struct notifier_block kasan_die_notifier = {
	.notifier_call = kasan_die_handler,
};
#endif

void __init kasan_early_init(void)
{
	int i;
	pteval_t pte_val = __pa_nodebug(kasan_zero_page) | __PAGE_KERNEL | _PAGE_ENC;
	pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) | _KERNPG_TABLE;
	pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) | _KERNPG_TABLE;
	p4dval_t p4d_val = __pa_nodebug(kasan_zero_pud) | _KERNPG_TABLE;

	/* Mask out unsupported __PAGE_KERNEL bits: */
	pte_val &= __default_kernel_pte_mask;
	pmd_val &= __default_kernel_pte_mask;
	pud_val &= __default_kernel_pte_mask;
	p4d_val &= __default_kernel_pte_mask;

	for (i = 0; i < PTRS_PER_PTE; i++)
		kasan_zero_pte[i] = __pte(pte_val);

	for (i = 0; i < PTRS_PER_PMD; i++)
		kasan_zero_pmd[i] = __pmd(pmd_val);

	for (i = 0; i < PTRS_PER_PUD; i++)
		kasan_zero_pud[i] = __pud(pud_val);

	for (i = 0; pgtable_l5_enabled() && i < PTRS_PER_P4D; i++)
		kasan_zero_p4d[i] = __p4d(p4d_val);

	kasan_map_early_shadow(early_top_pgt);
	kasan_map_early_shadow(init_top_pgt);
}

void __init kasan_init(void)
{
	int i;
	void *shadow_cpu_entry_begin, *shadow_cpu_entry_end;

#ifdef CONFIG_KASAN_INLINE
	register_die_notifier(&kasan_die_notifier);
#endif

	memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt));

	/*
	 * We use the same shadow offset for 4- and 5-level paging to
	 * facilitate boot-time switching between paging modes.
	 * As result in 5-level paging mode KASAN_SHADOW_START and
	 * KASAN_SHADOW_END are not aligned to PGD boundary.
	 *
	 * KASAN_SHADOW_START doesn't share PGD with anything else.
	 * We claim whole PGD entry to make things easier.
	 *
	 * KASAN_SHADOW_END lands in the last PGD entry and it collides with
	 * bunch of things like kernel code, modules, EFI mapping, etc.
	 * We need to take extra steps to not overwrite them.
	 */
	if (pgtable_l5_enabled()) {
		void *ptr;

		ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END));
		memcpy(tmp_p4d_table, (void *)ptr, sizeof(tmp_p4d_table));
		set_pgd(&early_top_pgt[pgd_index(KASAN_SHADOW_END)],
				__pgd(__pa(tmp_p4d_table) | _KERNPG_TABLE));
	}

	load_cr3(early_top_pgt);
	__flush_tlb_all();

	clear_pgds(KASAN_SHADOW_START & PGDIR_MASK, KASAN_SHADOW_END);

	kasan_populate_zero_shadow((void *)(KASAN_SHADOW_START & PGDIR_MASK),
			kasan_mem_to_shadow((void *)PAGE_OFFSET));

	for (i = 0; i < E820_MAX_ENTRIES; i++) {
		if (pfn_mapped[i].end == 0)
			break;

		map_range(&pfn_mapped[i]);
	}

	shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE;
	shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin);
	shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin,
						PAGE_SIZE);

	shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE +
					CPU_ENTRY_AREA_MAP_SIZE);
	shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end);
	shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end,
					PAGE_SIZE);

	kasan_populate_zero_shadow(
		kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
		shadow_cpu_entry_begin);

	kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,
			      (unsigned long)shadow_cpu_entry_end, 0);

	kasan_populate_zero_shadow(shadow_cpu_entry_end,
				kasan_mem_to_shadow((void *)__START_KERNEL_map));

	kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
			      (unsigned long)kasan_mem_to_shadow(_end),
			      early_pfn_to_nid(__pa(_stext)));

	kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END),
				(void *)KASAN_SHADOW_END);

	load_cr3(init_top_pgt);
	__flush_tlb_all();

	/*
	 * kasan_zero_page has been used as early shadow memory, thus it may
	 * contain some garbage. Now we can clear and write protect it, since
	 * after the TLB flush no one should write to it.
	 */
	memset(kasan_zero_page, 0, PAGE_SIZE);
	for (i = 0; i < PTRS_PER_PTE; i++) {
		pte_t pte;
		pgprot_t prot;

		prot = __pgprot(__PAGE_KERNEL_RO | _PAGE_ENC);
		pgprot_val(prot) &= __default_kernel_pte_mask;

		pte = __pte(__pa(kasan_zero_page) | pgprot_val(prot));
		set_pte(&kasan_zero_pte[i], pte);
	}
	/* Flush TLBs again to be sure that write protection applied. */
	__flush_tlb_all();

	init_task.kasan_depth = 0;
	pr_info("KernelAddressSanitizer initialized\n");
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
be3606ff	2	#define DISABLE_BRANCH_PROFILING
85155229	3	#define pr_fmt(fmt) "kasan: " fmt
39b95522	4
ad3fe525 KS	5	/* cpu_feature_enabled() cannot be used this early */
ad3fe525 KS	6	#define USE_EARLY_PGTABLE_L5
39b95522	7
ef7f0d6a AR	8	#include <linux/bootmem.h>
	9	#include <linux/kasan.h>
	10	#include <linux/kdebug.h>
2aeb0736	11	#include <linux/memblock.h>
ef7f0d6a AR	12	#include <linux/mm.h>
ef7f0d6a AR	13	#include <linux/sched.h>
9164bb4a	14	#include <linux/sched/task.h>
ef7f0d6a AR	15	#include <linux/vmalloc.h>
ef7f0d6a AR	16
5520b7e7	17	#include <asm/e820/types.h>
2aeb0736	18	#include <asm/pgalloc.h>
ef7f0d6a AR	19	#include <asm/tlbflush.h>
ef7f0d6a AR	20	#include <asm/sections.h>
b9d05200	21	#include <asm/pgtable.h>
92a0f81d	22	#include <asm/cpu_entry_area.h>
ef7f0d6a	23
08b46d5d	24	extern struct range pfn_mapped[E820_MAX_ENTRIES];
ef7f0d6a	25
c65e774f	26	static p4d_t tmp_p4d_table[MAX_PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
12a8cc7f	27
0d39e266	28	static __init void *early_alloc(size_t size, int nid, bool panic)
2aeb0736	29	{
0d39e266	30	if (panic)
eb31d559	31	return memblock_alloc_try_nid(size, size,
97ad1087	32	__pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
0d39e266	33	else
eb31d559	34	return memblock_alloc_try_nid_nopanic(size, size,
97ad1087	35	__pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
2aeb0736 AR	36	}
	37
	38	static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr,
	39	unsigned long end, int nid)
	40	{
	41	pte_t *pte;
	42
	43	if (pmd_none(*pmd)) {
	44	void *p;
	45
	46	if (boot_cpu_has(X86_FEATURE_PSE) &&
	47	((end - addr) == PMD_SIZE) &&
	48	IS_ALIGNED(addr, PMD_SIZE)) {
0d39e266	49	p = early_alloc(PMD_SIZE, nid, false);
2aeb0736 AR	50	if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL))
	51	return;
	52	else if (p)
	53	memblock_free(__pa(p), PMD_SIZE);
	54	}
	55
0d39e266	56	p = early_alloc(PAGE_SIZE, nid, true);
2aeb0736 AR	57	pmd_populate_kernel(&init_mm, pmd, p);
	58	}
	59
	60	pte = pte_offset_kernel(pmd, addr);
	61	do {
	62	pte_t entry;
	63	void *p;
	64
	65	if (!pte_none(*pte))
	66	continue;
	67
0d39e266	68	p = early_alloc(PAGE_SIZE, nid, true);
2aeb0736 AR	69	entry = pfn_pte(PFN_DOWN(__pa(p)), PAGE_KERNEL);
	70	set_pte_at(&init_mm, addr, pte, entry);
	71	} while (pte++, addr += PAGE_SIZE, addr != end);
	72	}
	73
	74	static void __init kasan_populate_pud(pud_t *pud, unsigned long addr,
	75	unsigned long end, int nid)
	76	{
	77	pmd_t *pmd;
	78	unsigned long next;
	79
	80	if (pud_none(*pud)) {
	81	void *p;
	82
	83	if (boot_cpu_has(X86_FEATURE_GBPAGES) &&
	84	((end - addr) == PUD_SIZE) &&
	85	IS_ALIGNED(addr, PUD_SIZE)) {
0d39e266	86	p = early_alloc(PUD_SIZE, nid, false);
2aeb0736 AR	87	if (p && pud_set_huge(pud, __pa(p), PAGE_KERNEL))
	88	return;
	89	else if (p)
	90	memblock_free(__pa(p), PUD_SIZE);
	91	}
	92
0d39e266	93	p = early_alloc(PAGE_SIZE, nid, true);
2aeb0736 AR	94	pud_populate(&init_mm, pud, p);
	95	}
	96
	97	pmd = pmd_offset(pud, addr);
	98	do {
	99	next = pmd_addr_end(addr, end);
	100	if (!pmd_large(*pmd))
	101	kasan_populate_pmd(pmd, addr, next, nid);
	102	} while (pmd++, addr = next, addr != end);
	103	}
	104
	105	static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr,
	106	unsigned long end, int nid)
	107	{
	108	pud_t *pud;
	109	unsigned long next;
	110
	111	if (p4d_none(*p4d)) {
0d39e266	112	void *p = early_alloc(PAGE_SIZE, nid, true);
2aeb0736 AR	113
	114	p4d_populate(&init_mm, p4d, p);
	115	}
	116
	117	pud = pud_offset(p4d, addr);
	118	do {
	119	next = pud_addr_end(addr, end);
	120	if (!pud_large(*pud))
	121	kasan_populate_pud(pud, addr, next, nid);
	122	} while (pud++, addr = next, addr != end);
	123	}
	124
	125	static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr,
	126	unsigned long end, int nid)
	127	{
	128	void *p;
	129	p4d_t *p4d;
	130	unsigned long next;
	131
	132	if (pgd_none(*pgd)) {
0d39e266	133	p = early_alloc(PAGE_SIZE, nid, true);
2aeb0736 AR	134	pgd_populate(&init_mm, pgd, p);
	135	}
	136
	137	p4d = p4d_offset(pgd, addr);
	138	do {
	139	next = p4d_addr_end(addr, end);
	140	kasan_populate_p4d(p4d, addr, next, nid);
	141	} while (p4d++, addr = next, addr != end);
	142	}
	143
	144	static void __init kasan_populate_shadow(unsigned long addr, unsigned long end,
	145	int nid)
	146	{
	147	pgd_t *pgd;
	148	unsigned long next;
	149
	150	addr = addr & PAGE_MASK;
	151	end = round_up(end, PAGE_SIZE);
	152	pgd = pgd_offset_k(addr);
	153	do {
	154	next = pgd_addr_end(addr, end);
	155	kasan_populate_pgd(pgd, addr, next, nid);
	156	} while (pgd++, addr = next, addr != end);
	157	}
	158
	159	static void __init map_range(struct range *range)
ef7f0d6a AR	160	{
	161	unsigned long start;
	162	unsigned long end;
	163
	164	start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start));
	165	end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end));
	166
2aeb0736	167	kasan_populate_shadow(start, end, early_pfn_to_nid(range->start));
ef7f0d6a AR	168	}
	169
	170	static void __init clear_pgds(unsigned long start,
	171	unsigned long end)
	172	{
d691a3cf	173	pgd_t *pgd;
12a8cc7f AR	174	/* See comment in kasan_init() */
12a8cc7f AR	175	unsigned long pgd_end = end & PGDIR_MASK;
d691a3cf	176
12a8cc7f	177	for (; start < pgd_end; start += PGDIR_SIZE) {
d691a3cf KS	178	pgd = pgd_offset_k(start);
	179	/*
	180	* With folded p4d, pgd_clear() is nop, use p4d_clear()
	181	* instead.
	182	*/
ed7588d5	183	if (pgtable_l5_enabled())
d691a3cf	184	pgd_clear(pgd);
91f606a8 KS	185	else
91f606a8 KS	186	p4d_clear(p4d_offset(pgd, start));
d691a3cf	187	}
12a8cc7f AR	188
	189	pgd = pgd_offset_k(start);
	190	for (; start < end; start += P4D_SIZE)
	191	p4d_clear(p4d_offset(pgd, start));
	192	}
	193
	194	static inline p4d_t early_p4d_offset(pgd_t pgd, unsigned long addr)
	195	{
	196	unsigned long p4d;
	197
ed7588d5	198	if (!pgtable_l5_enabled())
12a8cc7f AR	199	return (p4d_t *)pgd;
	200
	201	p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK;
	202	p4d += __START_KERNEL_map - phys_base;
	203	return (p4d_t *)p4d + p4d_index(addr);
	204	}
	205
	206	static void __init kasan_early_p4d_populate(pgd_t *pgd,
	207	unsigned long addr,
	208	unsigned long end)
	209	{
	210	pgd_t pgd_entry;
	211	p4d_t *p4d, p4d_entry;
	212	unsigned long next;
	213
	214	if (pgd_none(*pgd)) {
	215	pgd_entry = __pgd(_KERNPG_TABLE \| __pa_nodebug(kasan_zero_p4d));
	216	set_pgd(pgd, pgd_entry);
	217	}
	218
	219	p4d = early_p4d_offset(pgd, addr);
	220	do {
	221	next = p4d_addr_end(addr, end);
	222
	223	if (!p4d_none(*p4d))
	224	continue;
	225
	226	p4d_entry = __p4d(_KERNPG_TABLE \| __pa_nodebug(kasan_zero_pud));
	227	set_p4d(p4d, p4d_entry);
	228	} while (p4d++, addr = next, addr != end && p4d_none(*p4d));
ef7f0d6a AR	229	}
ef7f0d6a AR	230
5d5aa3cf	231	static void __init kasan_map_early_shadow(pgd_t *pgd)
ef7f0d6a	232	{
12a8cc7f AR	233	/* See comment in kasan_init() */
12a8cc7f AR	234	unsigned long addr = KASAN_SHADOW_START & PGDIR_MASK;
ef7f0d6a	235	unsigned long end = KASAN_SHADOW_END;
12a8cc7f	236	unsigned long next;
ef7f0d6a	237
12a8cc7f AR	238	pgd += pgd_index(addr);
	239	do {
	240	next = pgd_addr_end(addr, end);
	241	kasan_early_p4d_populate(pgd, addr, next);
	242	} while (pgd++, addr = next, addr != end);
ef7f0d6a AR	243	}
ef7f0d6a AR	244
ef7f0d6a AR	245	#ifdef CONFIG_KASAN_INLINE
	246	static int kasan_die_handler(struct notifier_block *self,
	247	unsigned long val,
	248	void *data)
	249	{
	250	if (val == DIE_GPF) {
2ba78056 DV	251	pr_emerg("CONFIG_KASAN_INLINE enabled\n");
2ba78056 DV	252	pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n");
ef7f0d6a AR	253	}
	254	return NOTIFY_OK;
	255	}
	256
	257	static struct notifier_block kasan_die_notifier = {
	258	.notifier_call = kasan_die_handler,
	259	};
	260	#endif
	261
5d5aa3cf AP	262	void __init kasan_early_init(void)
	263	{
	264	int i;
21729f81	265	pteval_t pte_val = __pa_nodebug(kasan_zero_page) \| __PAGE_KERNEL \| _PAGE_ENC;
5d5aa3cf AP	266	pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) \| _KERNPG_TABLE;
5d5aa3cf AP	267	pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) \| _KERNPG_TABLE;
5480bb61	268	p4dval_t p4d_val = __pa_nodebug(kasan_zero_pud) \| _KERNPG_TABLE;
5d5aa3cf	269
fb43d6cb DH	270	/* Mask out unsupported __PAGE_KERNEL bits: */
	271	pte_val &= __default_kernel_pte_mask;
	272	pmd_val &= __default_kernel_pte_mask;
	273	pud_val &= __default_kernel_pte_mask;
	274	p4d_val &= __default_kernel_pte_mask;
	275
5d5aa3cf AP	276	for (i = 0; i < PTRS_PER_PTE; i++)
	277	kasan_zero_pte[i] = __pte(pte_val);
	278
	279	for (i = 0; i < PTRS_PER_PMD; i++)
	280	kasan_zero_pmd[i] = __pmd(pmd_val);
	281
	282	for (i = 0; i < PTRS_PER_PUD; i++)
	283	kasan_zero_pud[i] = __pud(pud_val);
	284
ed7588d5	285	for (i = 0; pgtable_l5_enabled() && i < PTRS_PER_P4D; i++)
5480bb61 KS	286	kasan_zero_p4d[i] = __p4d(p4d_val);
5480bb61 KS	287
65ade2f8 KS	288	kasan_map_early_shadow(early_top_pgt);
65ade2f8 KS	289	kasan_map_early_shadow(init_top_pgt);
5d5aa3cf AP	290	}
5d5aa3cf AP	291
ef7f0d6a AR	292	void __init kasan_init(void)
	293	{
	294	int i;
21506525	295	void shadow_cpu_entry_begin, shadow_cpu_entry_end;
ef7f0d6a AR	296
	297	#ifdef CONFIG_KASAN_INLINE
	298	register_die_notifier(&kasan_die_notifier);
	299	#endif
	300
65ade2f8	301	memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt));
12a8cc7f AR	302
	303	/*
	304	* We use the same shadow offset for 4- and 5-level paging to
	305	* facilitate boot-time switching between paging modes.
	306	* As result in 5-level paging mode KASAN_SHADOW_START and
	307	* KASAN_SHADOW_END are not aligned to PGD boundary.
	308	*
	309	* KASAN_SHADOW_START doesn't share PGD with anything else.
	310	* We claim whole PGD entry to make things easier.
	311	*
	312	* KASAN_SHADOW_END lands in the last PGD entry and it collides with
	313	* bunch of things like kernel code, modules, EFI mapping, etc.
	314	* We need to take extra steps to not overwrite them.
	315	*/
ed7588d5	316	if (pgtable_l5_enabled()) {
12a8cc7f AR	317	void *ptr;
	318
	319	ptr = (void )pgd_page_vaddr(pgd_offset_k(KASAN_SHADOW_END));
	320	memcpy(tmp_p4d_table, (void *)ptr, sizeof(tmp_p4d_table));
	321	set_pgd(&early_top_pgt[pgd_index(KASAN_SHADOW_END)],
	322	__pgd(__pa(tmp_p4d_table) \| _KERNPG_TABLE));
	323	}
	324
65ade2f8	325	load_cr3(early_top_pgt);
241d2c54	326	__flush_tlb_all();
ef7f0d6a	327
12a8cc7f	328	clear_pgds(KASAN_SHADOW_START & PGDIR_MASK, KASAN_SHADOW_END);
ef7f0d6a	329
12a8cc7f	330	kasan_populate_zero_shadow((void *)(KASAN_SHADOW_START & PGDIR_MASK),
ef7f0d6a AR	331	kasan_mem_to_shadow((void *)PAGE_OFFSET));
ef7f0d6a AR	332
08b46d5d	333	for (i = 0; i < E820_MAX_ENTRIES; i++) {
ef7f0d6a AR	334	if (pfn_mapped[i].end == 0)
	335	break;
	336
2aeb0736	337	map_range(&pfn_mapped[i]);
ef7f0d6a	338	}
2aeb0736	339
92a0f81d	340	shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE;
21506525 AL	341	shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin);
	342	shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin,
	343	PAGE_SIZE);
	344
92a0f81d TG	345	shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE +
92a0f81d TG	346	CPU_ENTRY_AREA_MAP_SIZE);
21506525 AL	347	shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end);
	348	shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end,
	349	PAGE_SIZE);
	350
92a0f81d TG	351	kasan_populate_zero_shadow(
	352	kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
	353	shadow_cpu_entry_begin);
21506525 AL	354
	355	kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,
	356	(unsigned long)shadow_cpu_entry_end, 0);
	357
92a0f81d TG	358	kasan_populate_zero_shadow(shadow_cpu_entry_end,
	359	kasan_mem_to_shadow((void *)__START_KERNEL_map));
	360
	361	kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
	362	(unsigned long)kasan_mem_to_shadow(_end),
	363	early_pfn_to_nid(__pa(_stext)));
	364
	365	kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END),
	366	(void *)KASAN_SHADOW_END);
ef7f0d6a	367
65ade2f8	368	load_cr3(init_top_pgt);
241d2c54	369	__flush_tlb_all();
85155229	370
69e0210f AR	371	/*
69e0210f AR	372	* kasan_zero_page has been used as early shadow memory, thus it may
063fb3e5 AR	373	* contain some garbage. Now we can clear and write protect it, since
063fb3e5 AR	374	* after the TLB flush no one should write to it.
69e0210f AR	375	*/
69e0210f AR	376	memset(kasan_zero_page, 0, PAGE_SIZE);
063fb3e5	377	for (i = 0; i < PTRS_PER_PTE; i++) {
fb43d6cb DH	378	pte_t pte;
	379	pgprot_t prot;
	380
	381	prot = __pgprot(__PAGE_KERNEL_RO \| _PAGE_ENC);
	382	pgprot_val(prot) &= __default_kernel_pte_mask;
	383
	384	pte = __pte(__pa(kasan_zero_page) \| pgprot_val(prot));
063fb3e5 AR	385	set_pte(&kasan_zero_pte[i], pte);
	386	}
	387	/* Flush TLBs again to be sure that write protection applied. */
	388	__flush_tlb_all();
69e0210f AR	389
69e0210f AR	390	init_task.kasan_depth = 0;
25add7ec	391	pr_info("KernelAddressSanitizer initialized\n");
ef7f0d6a	392	}