Commit | Line | Data |
---|---|---|
2edb16ef CL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #define DISABLE_BRANCH_PROFILING | |
4 | ||
5 | #include <linux/kasan.h> | |
6 | #include <linux/printk.h> | |
7 | #include <linux/memblock.h> | |
8 | #include <linux/sched/task.h> | |
2edb16ef CL |
9 | #include <asm/pgalloc.h> |
10 | #include <asm/code-patching.h> | |
11 | #include <mm/mmu_decl.h> | |
12 | ||
47febbee | 13 | static pgprot_t __init kasan_prot_ro(void) |
4c0f5d1e CL |
14 | { |
15 | if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) | |
16 | return PAGE_READONLY; | |
17 | ||
18 | return PAGE_KERNEL_RO; | |
19 | } | |
20 | ||
47febbee | 21 | static void __init kasan_populate_pte(pte_t *ptep, pgprot_t prot) |
2edb16ef CL |
22 | { |
23 | unsigned long va = (unsigned long)kasan_early_shadow_page; | |
24 | phys_addr_t pa = __pa(kasan_early_shadow_page); | |
25 | int i; | |
26 | ||
27 | for (i = 0; i < PTRS_PER_PTE; i++, ptep++) | |
6042a165 | 28 | __set_pte_at(&init_mm, va, ptep, pfn_pte(PHYS_PFN(pa), prot), 1); |
2edb16ef CL |
29 | } |
30 | ||
ec97d022 | 31 | int __init kasan_init_shadow_page_tables(unsigned long k_start, unsigned long k_end) |
2edb16ef CL |
32 | { |
33 | pmd_t *pmd; | |
34 | unsigned long k_cur, k_next; | |
35 | ||
e05c7b1f | 36 | pmd = pmd_off_k(k_start); |
2edb16ef CL |
37 | |
38 | for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) { | |
7c31c05e CL |
39 | pte_t *new; |
40 | ||
2edb16ef CL |
41 | k_next = pgd_addr_end(k_cur, k_end); |
42 | if ((void *)pmd_page_vaddr(*pmd) != kasan_early_shadow_pte) | |
43 | continue; | |
44 | ||
7c31c05e | 45 | new = memblock_alloc(PTE_FRAG_SIZE, PTE_FRAG_SIZE); |
2edb16ef CL |
46 | |
47 | if (!new) | |
48 | return -ENOMEM; | |
509cd3f2 | 49 | kasan_populate_pte(new, PAGE_KERNEL); |
7c31c05e | 50 | pmd_populate_kernel(&init_mm, pmd, new); |
2edb16ef CL |
51 | } |
52 | return 0; | |
53 | } | |
54 | ||
ec97d022 | 55 | int __init __weak kasan_init_region(void *start, size_t size) |
2edb16ef CL |
56 | { |
57 | unsigned long k_start = (unsigned long)kasan_mem_to_shadow(start); | |
58 | unsigned long k_end = (unsigned long)kasan_mem_to_shadow(start + size); | |
59 | unsigned long k_cur; | |
60 | int ret; | |
509cd3f2 | 61 | void *block; |
2edb16ef CL |
62 | |
63 | ret = kasan_init_shadow_page_tables(k_start, k_end); | |
64 | if (ret) | |
65 | return ret; | |
66 | ||
4a7aee96 | 67 | k_start = k_start & PAGE_MASK; |
509cd3f2 | 68 | block = memblock_alloc(k_end - k_start, PAGE_SIZE); |
d132443a CL |
69 | if (!block) |
70 | return -ENOMEM; | |
2edb16ef | 71 | |
663c0c94 | 72 | for (k_cur = k_start & PAGE_MASK; k_cur < k_end; k_cur += PAGE_SIZE) { |
e05c7b1f | 73 | pmd_t *pmd = pmd_off_k(k_cur); |
509cd3f2 | 74 | void *va = block + k_cur - k_start; |
2edb16ef CL |
75 | pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL); |
76 | ||
2edb16ef CL |
77 | __set_pte_at(&init_mm, k_cur, pte_offset_kernel(pmd, k_cur), pte, 0); |
78 | } | |
79 | flush_tlb_kernel_range(k_start, k_end); | |
80 | return 0; | |
81 | } | |
82 | ||
7974c473 | 83 | void __init |
7dec42ab | 84 | kasan_update_early_region(unsigned long k_start, unsigned long k_end, pte_t pte) |
2edb16ef | 85 | { |
cbd18991 | 86 | unsigned long k_cur; |
4c0f5d1e | 87 | |
7dec42ab | 88 | for (k_cur = k_start; k_cur != k_end; k_cur += PAGE_SIZE) { |
e05c7b1f | 89 | pmd_t *pmd = pmd_off_k(k_cur); |
cbd18991 CL |
90 | pte_t *ptep = pte_offset_kernel(pmd, k_cur); |
91 | ||
dd75080a | 92 | if (pte_page(*ptep) != virt_to_page(lm_alias(kasan_early_shadow_page))) |
cbd18991 CL |
93 | continue; |
94 | ||
7dec42ab | 95 | __set_pte_at(&init_mm, k_cur, ptep, pte, 0); |
cbd18991 | 96 | } |
7dec42ab CL |
97 | |
98 | flush_tlb_kernel_range(k_start, k_end); | |
2edb16ef CL |
99 | } |
100 | ||
7dec42ab | 101 | static void __init kasan_remap_early_shadow_ro(void) |
3d4247fc | 102 | { |
7dec42ab | 103 | pgprot_t prot = kasan_prot_ro(); |
3d4247fc CL |
104 | phys_addr_t pa = __pa(kasan_early_shadow_page); |
105 | ||
7dec42ab | 106 | kasan_populate_pte(kasan_early_shadow_pte, prot); |
3d4247fc | 107 | |
7dec42ab CL |
108 | kasan_update_early_region(KASAN_SHADOW_START, KASAN_SHADOW_END, |
109 | pfn_pte(PHYS_PFN(pa), prot)); | |
110 | } | |
3d4247fc | 111 | |
7dec42ab CL |
112 | static void __init kasan_unmap_early_shadow_vmalloc(void) |
113 | { | |
114 | unsigned long k_start = (unsigned long)kasan_mem_to_shadow((void *)VMALLOC_START); | |
115 | unsigned long k_end = (unsigned long)kasan_mem_to_shadow((void *)VMALLOC_END); | |
116 | ||
117 | kasan_update_early_region(k_start, k_end, __pte(0)); | |
7fbc22ce CL |
118 | |
119 | #ifdef MODULES_VADDR | |
120 | k_start = (unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR); | |
121 | k_end = (unsigned long)kasan_mem_to_shadow((void *)MODULES_END); | |
122 | kasan_update_early_region(k_start, k_end, __pte(0)); | |
123 | #endif | |
3d4247fc CL |
124 | } |
125 | ||
b506923e | 126 | void __init kasan_mmu_init(void) |
2edb16ef CL |
127 | { |
128 | int ret; | |
41ea93cf | 129 | |
4c42dc5c | 130 | if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE)) { |
41ea93cf CL |
131 | ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END); |
132 | ||
133 | if (ret) | |
134 | panic("kasan: kasan_init_shadow_page_tables() failed"); | |
135 | } | |
136 | } | |
137 | ||
138 | void __init kasan_init(void) | |
139 | { | |
b10d6bca MR |
140 | phys_addr_t base, end; |
141 | u64 i; | |
4c42dc5c | 142 | int ret; |
2edb16ef | 143 | |
b10d6bca MR |
144 | for_each_mem_range(i, &base, &end) { |
145 | phys_addr_t top = min(end, total_lowmem); | |
2edb16ef CL |
146 | |
147 | if (base >= top) | |
148 | continue; | |
149 | ||
150 | ret = kasan_init_region(__va(base), top - base); | |
151 | if (ret) | |
152 | panic("kasan: kasan_init_region() failed"); | |
153 | } | |
ec97d022 | 154 | |
4c42dc5c CL |
155 | if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { |
156 | ret = kasan_init_shadow_page_tables(KASAN_SHADOW_START, KASAN_SHADOW_END); | |
157 | ||
158 | if (ret) | |
159 | panic("kasan: kasan_init_shadow_page_tables() failed"); | |
160 | } | |
161 | ||
2edb16ef CL |
162 | kasan_remap_early_shadow_ro(); |
163 | ||
164 | clear_page(kasan_early_shadow_page); | |
165 | ||
166 | /* At this point kasan is fully initialized. Enable error messages */ | |
167 | init_task.kasan_depth = 0; | |
168 | pr_info("KASAN init done\n"); | |
169 | } | |
170 | ||
3d4247fc CL |
171 | void __init kasan_late_init(void) |
172 | { | |
173 | if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) | |
174 | kasan_unmap_early_shadow_vmalloc(); | |
175 | } | |
176 | ||
2edb16ef CL |
177 | void __init kasan_early_init(void) |
178 | { | |
179 | unsigned long addr = KASAN_SHADOW_START; | |
180 | unsigned long end = KASAN_SHADOW_END; | |
181 | unsigned long next; | |
e05c7b1f | 182 | pmd_t *pmd = pmd_off_k(addr); |
2edb16ef CL |
183 | |
184 | BUILD_BUG_ON(KASAN_SHADOW_START & ~PGDIR_MASK); | |
185 | ||
186 | kasan_populate_pte(kasan_early_shadow_pte, PAGE_KERNEL); | |
187 | ||
188 | do { | |
189 | next = pgd_addr_end(addr, end); | |
190 | pmd_populate_kernel(&init_mm, pmd, kasan_early_shadow_pte); | |
191 | } while (pmd++, addr = next, addr != end); | |
2edb16ef | 192 | } |