1 // SPDX-License-Identifier: GPL-2.0
3 #define DISABLE_BRANCH_PROFILING
5 #include <linux/kasan.h>
6 #include <linux/printk.h>
7 #include <linux/memblock.h>
8 #include <linux/sched/task.h>
9 #include <linux/vmalloc.h>
10 #include <asm/pgalloc.h>
11 #include <asm/code-patching.h>
12 #include <mm/mmu_decl.h>
14 static void kasan_populate_pte(pte_t *ptep, pgprot_t prot)
16 unsigned long va = (unsigned long)kasan_early_shadow_page;
17 phys_addr_t pa = __pa(kasan_early_shadow_page);
20 for (i = 0; i < PTRS_PER_PTE; i++, ptep++)
21 __set_pte_at(&init_mm, va, ptep, pfn_pte(PHYS_PFN(pa), prot), 0);
24 static int kasan_init_shadow_page_tables(unsigned long k_start, unsigned long k_end)
27 unsigned long k_cur, k_next;
29 pmd = pmd_offset(pud_offset(pgd_offset_k(k_start), k_start), k_start);
31 for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {
34 k_next = pgd_addr_end(k_cur, k_end);
35 if ((void *)pmd_page_vaddr(*pmd) != kasan_early_shadow_pte)
38 new = pte_alloc_one_kernel(&init_mm);
42 if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE))
43 kasan_populate_pte(new, PAGE_READONLY);
45 kasan_populate_pte(new, PAGE_KERNEL_RO);
46 pmd_populate_kernel(&init_mm, pmd, new);
51 static void __ref *kasan_get_one_page(void)
53 if (slab_is_available())
54 return (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
56 return memblock_alloc(PAGE_SIZE, PAGE_SIZE);
59 static int __ref kasan_init_region(void *start, size_t size)
61 unsigned long k_start = (unsigned long)kasan_mem_to_shadow(start);
62 unsigned long k_end = (unsigned long)kasan_mem_to_shadow(start + size);
67 ret = kasan_init_shadow_page_tables(k_start, k_end);
71 if (!slab_is_available())
72 block = memblock_alloc(k_end - k_start, PAGE_SIZE);
74 for (k_cur = k_start; k_cur < k_end; k_cur += PAGE_SIZE) {
75 pmd_t *pmd = pmd_offset(pud_offset(pgd_offset_k(k_cur), k_cur), k_cur);
76 void *va = block ? block + k_cur - k_start : kasan_get_one_page();
77 pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL);
82 __set_pte_at(&init_mm, k_cur, pte_offset_kernel(pmd, k_cur), pte, 0);
84 flush_tlb_kernel_range(k_start, k_end);
88 static void __init kasan_remap_early_shadow_ro(void)
90 if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE))
91 kasan_populate_pte(kasan_early_shadow_pte, PAGE_READONLY);
93 kasan_populate_pte(kasan_early_shadow_pte, PAGE_KERNEL_RO);
95 flush_tlb_kernel_range(KASAN_SHADOW_START, KASAN_SHADOW_END);
98 void __init kasan_mmu_init(void)
101 struct memblock_region *reg;
103 if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) {
104 ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END);
107 panic("kasan: kasan_init_shadow_page_tables() failed");
110 for_each_memblock(memory, reg) {
111 phys_addr_t base = reg->base;
112 phys_addr_t top = min(base + reg->size, total_lowmem);
117 ret = kasan_init_region(__va(base), top - base);
119 panic("kasan: kasan_init_region() failed");
123 void __init kasan_init(void)
125 kasan_remap_early_shadow_ro();
127 clear_page(kasan_early_shadow_page);
129 /* At this point kasan is fully initialized. Enable error messages */
130 init_task.kasan_depth = 0;
131 pr_info("KASAN init done\n");
134 #ifdef CONFIG_MODULES
135 void *module_alloc(unsigned long size)
137 void *base = vmalloc_exec(size);
142 if (!kasan_init_region(base, size))
151 #ifdef CONFIG_PPC_BOOK3S_32
152 u8 __initdata early_hash[256 << 10] __aligned(256 << 10) = {0};
154 static void __init kasan_early_hash_table(void)
156 modify_instruction_site(&patch__hash_page_A0, 0xffff, __pa(early_hash) >> 16);
157 modify_instruction_site(&patch__flush_hash_A0, 0xffff, __pa(early_hash) >> 16);
159 Hash = (struct hash_pte *)early_hash;
162 static void __init kasan_early_hash_table(void) {}
165 void __init kasan_early_init(void)
167 unsigned long addr = KASAN_SHADOW_START;
168 unsigned long end = KASAN_SHADOW_END;
170 pmd_t *pmd = pmd_offset(pud_offset(pgd_offset_k(addr), addr), addr);
172 BUILD_BUG_ON(KASAN_SHADOW_START & ~PGDIR_MASK);
174 kasan_populate_pte(kasan_early_shadow_pte, PAGE_KERNEL);
177 next = pgd_addr_end(addr, end);
178 pmd_populate_kernel(&init_mm, pmd, kasan_early_shadow_pte);
179 } while (pmd++, addr = next, addr != end);
181 if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE))
182 kasan_early_hash_table();