Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
39d114dd AR |
2 | /* |
3 | * This file contains kasan initialization code for ARM64. | |
4 | * | |
5 | * Copyright (c) 2015 Samsung Electronics Co., Ltd. | |
6 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> | |
39d114dd AR |
7 | */ |
8 | ||
9 | #define pr_fmt(fmt) "kasan: " fmt | |
10 | #include <linux/kasan.h> | |
11 | #include <linux/kernel.h> | |
9164bb4a | 12 | #include <linux/sched/task.h> |
39d114dd AR |
13 | #include <linux/memblock.h> |
14 | #include <linux/start_kernel.h> | |
2077be67 | 15 | #include <linux/mm.h> |
39d114dd | 16 | |
c1a88e91 | 17 | #include <asm/mmu_context.h> |
f9040773 | 18 | #include <asm/kernel-pgtable.h> |
39d114dd AR |
19 | #include <asm/page.h> |
20 | #include <asm/pgalloc.h> | |
f9040773 | 21 | #include <asm/sections.h> |
39d114dd AR |
22 | #include <asm/tlbflush.h> |
23 | ||
24 | static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE); | |
25 | ||
2077be67 LA |
26 | /* |
27 | * The p*d_populate functions call virt_to_phys implicitly so they can't be used | |
28 | * directly on kernel symbols (bm_p*d). All the early functions are called too | |
29 | * early to use lm_alias so __p*d_populate functions must be used to populate | |
30 | * with the physical address from __pa_symbol. | |
31 | */ | |
32 | ||
e17d8025 | 33 | static phys_addr_t __init kasan_alloc_zeroed_page(int node) |
39d114dd | 34 | { |
eb31d559 | 35 | void *p = memblock_alloc_try_nid(PAGE_SIZE, PAGE_SIZE, |
e17d8025 | 36 | __pa(MAX_DMA_ADDRESS), |
fed84c78 | 37 | MEMBLOCK_ALLOC_KASAN, node); |
8a7f97b9 MR |
38 | if (!p) |
39 | panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n", | |
40 | __func__, PAGE_SIZE, PAGE_SIZE, node, | |
41 | __pa(MAX_DMA_ADDRESS)); | |
42 | ||
e17d8025 WD |
43 | return __pa(p); |
44 | } | |
45 | ||
080eb83f AK |
46 | static phys_addr_t __init kasan_alloc_raw_page(int node) |
47 | { | |
48 | void *p = memblock_alloc_try_nid_raw(PAGE_SIZE, PAGE_SIZE, | |
49 | __pa(MAX_DMA_ADDRESS), | |
50 | MEMBLOCK_ALLOC_KASAN, node); | |
8a7f97b9 MR |
51 | if (!p) |
52 | panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n", | |
53 | __func__, PAGE_SIZE, PAGE_SIZE, node, | |
54 | __pa(MAX_DMA_ADDRESS)); | |
55 | ||
080eb83f AK |
56 | return __pa(p); |
57 | } | |
58 | ||
20a004e7 | 59 | static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, |
e17d8025 WD |
60 | bool early) |
61 | { | |
20a004e7 | 62 | if (pmd_none(READ_ONCE(*pmdp))) { |
9577dd74 AK |
63 | phys_addr_t pte_phys = early ? |
64 | __pa_symbol(kasan_early_shadow_pte) | |
65 | : kasan_alloc_zeroed_page(node); | |
20a004e7 | 66 | __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); |
e17d8025 WD |
67 | } |
68 | ||
20a004e7 WD |
69 | return early ? pte_offset_kimg(pmdp, addr) |
70 | : pte_offset_kernel(pmdp, addr); | |
e17d8025 | 71 | } |
39d114dd | 72 | |
20a004e7 | 73 | static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, |
e17d8025 WD |
74 | bool early) |
75 | { | |
20a004e7 | 76 | if (pud_none(READ_ONCE(*pudp))) { |
9577dd74 AK |
77 | phys_addr_t pmd_phys = early ? |
78 | __pa_symbol(kasan_early_shadow_pmd) | |
79 | : kasan_alloc_zeroed_page(node); | |
20a004e7 | 80 | __pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE); |
e17d8025 WD |
81 | } |
82 | ||
20a004e7 | 83 | return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); |
e17d8025 WD |
84 | } |
85 | ||
e9f63768 | 86 | static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, |
e17d8025 WD |
87 | bool early) |
88 | { | |
e9f63768 | 89 | if (p4d_none(READ_ONCE(*p4dp))) { |
9577dd74 AK |
90 | phys_addr_t pud_phys = early ? |
91 | __pa_symbol(kasan_early_shadow_pud) | |
92 | : kasan_alloc_zeroed_page(node); | |
e9f63768 | 93 | __p4d_populate(p4dp, pud_phys, PMD_TYPE_TABLE); |
e17d8025 WD |
94 | } |
95 | ||
e9f63768 | 96 | return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr); |
e17d8025 WD |
97 | } |
98 | ||
20a004e7 | 99 | static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, |
e17d8025 WD |
100 | unsigned long end, int node, bool early) |
101 | { | |
102 | unsigned long next; | |
20a004e7 | 103 | pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); |
39d114dd | 104 | |
39d114dd | 105 | do { |
9577dd74 AK |
106 | phys_addr_t page_phys = early ? |
107 | __pa_symbol(kasan_early_shadow_page) | |
080eb83f AK |
108 | : kasan_alloc_raw_page(node); |
109 | if (!early) | |
110 | memset(__va(page_phys), KASAN_SHADOW_INIT, PAGE_SIZE); | |
39d114dd | 111 | next = addr + PAGE_SIZE; |
20a004e7 WD |
112 | set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL)); |
113 | } while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep))); | |
39d114dd AR |
114 | } |
115 | ||
20a004e7 | 116 | static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, |
e17d8025 | 117 | unsigned long end, int node, bool early) |
39d114dd | 118 | { |
39d114dd | 119 | unsigned long next; |
20a004e7 | 120 | pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); |
39d114dd | 121 | |
39d114dd AR |
122 | do { |
123 | next = pmd_addr_end(addr, end); | |
20a004e7 WD |
124 | kasan_pte_populate(pmdp, addr, next, node, early); |
125 | } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp))); | |
39d114dd AR |
126 | } |
127 | ||
e9f63768 | 128 | static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, |
e17d8025 | 129 | unsigned long end, int node, bool early) |
39d114dd | 130 | { |
39d114dd | 131 | unsigned long next; |
e9f63768 | 132 | pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); |
39d114dd | 133 | |
39d114dd AR |
134 | do { |
135 | next = pud_addr_end(addr, end); | |
20a004e7 WD |
136 | kasan_pmd_populate(pudp, addr, next, node, early); |
137 | } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); | |
39d114dd AR |
138 | } |
139 | ||
e9f63768 MR |
140 | static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr, |
141 | unsigned long end, int node, bool early) | |
142 | { | |
143 | unsigned long next; | |
144 | p4d_t *p4dp = p4d_offset(pgdp, addr); | |
145 | ||
146 | do { | |
147 | next = p4d_addr_end(addr, end); | |
148 | kasan_pud_populate(p4dp, addr, next, node, early); | |
149 | } while (p4dp++, addr = next, addr != end); | |
150 | } | |
151 | ||
e17d8025 WD |
152 | static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, |
153 | int node, bool early) | |
39d114dd | 154 | { |
39d114dd | 155 | unsigned long next; |
20a004e7 | 156 | pgd_t *pgdp; |
39d114dd | 157 | |
20a004e7 | 158 | pgdp = pgd_offset_k(addr); |
39d114dd AR |
159 | do { |
160 | next = pgd_addr_end(addr, end); | |
e9f63768 | 161 | kasan_p4d_populate(pgdp, addr, next, node, early); |
20a004e7 | 162 | } while (pgdp++, addr = next, addr != end); |
39d114dd AR |
163 | } |
164 | ||
e17d8025 | 165 | /* The early shadow maps everything to a single page of zeroes */ |
83040123 | 166 | asmlinkage void __init kasan_early_init(void) |
39d114dd | 167 | { |
917538e2 AK |
168 | BUILD_BUG_ON(KASAN_SHADOW_OFFSET != |
169 | KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT))); | |
90ec95cd SC |
170 | BUILD_BUG_ON(!IS_ALIGNED(_KASAN_SHADOW_START(VA_BITS), PGDIR_SIZE)); |
171 | BUILD_BUG_ON(!IS_ALIGNED(_KASAN_SHADOW_START(VA_BITS_MIN), PGDIR_SIZE)); | |
39d114dd | 172 | BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE)); |
e17d8025 WD |
173 | kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, |
174 | true); | |
175 | } | |
176 | ||
177 | /* Set up full kasan mappings, ensuring that the mapped pages are zeroed */ | |
178 | static void __init kasan_map_populate(unsigned long start, unsigned long end, | |
179 | int node) | |
180 | { | |
181 | kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false); | |
39d114dd AR |
182 | } |
183 | ||
068a17a5 MR |
184 | /* |
185 | * Copy the current shadow region into a new pgdir. | |
186 | */ | |
187 | void __init kasan_copy_shadow(pgd_t *pgdir) | |
188 | { | |
20a004e7 | 189 | pgd_t *pgdp, *pgdp_new, *pgdp_end; |
068a17a5 | 190 | |
20a004e7 WD |
191 | pgdp = pgd_offset_k(KASAN_SHADOW_START); |
192 | pgdp_end = pgd_offset_k(KASAN_SHADOW_END); | |
974b9b2c | 193 | pgdp_new = pgd_offset_pgd(pgdir, KASAN_SHADOW_START); |
068a17a5 | 194 | do { |
20a004e7 WD |
195 | set_pgd(pgdp_new, READ_ONCE(*pgdp)); |
196 | } while (pgdp++, pgdp_new++, pgdp != pgdp_end); | |
068a17a5 MR |
197 | } |
198 | ||
39d114dd AR |
199 | static void __init clear_pgds(unsigned long start, |
200 | unsigned long end) | |
201 | { | |
202 | /* | |
203 | * Remove references to kasan page tables from | |
204 | * swapper_pg_dir. pgd_clear() can't be used | |
205 | * here because it's nop on 2,3-level pagetable setups | |
206 | */ | |
207 | for (; start < end; start += PGDIR_SIZE) | |
208 | set_pgd(pgd_offset_k(start), __pgd(0)); | |
209 | } | |
210 | ||
39d114dd AR |
211 | void __init kasan_init(void) |
212 | { | |
f9040773 | 213 | u64 kimg_shadow_start, kimg_shadow_end; |
f80fb3a3 | 214 | u64 mod_shadow_start, mod_shadow_end; |
b10d6bca MR |
215 | phys_addr_t pa_start, pa_end; |
216 | u64 i; | |
39d114dd | 217 | |
e17d8025 WD |
218 | kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK; |
219 | kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end)); | |
f9040773 | 220 | |
f80fb3a3 AB |
221 | mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR); |
222 | mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END); | |
223 | ||
39d114dd AR |
224 | /* |
225 | * We are going to perform proper setup of shadow memory. | |
0293c8ba | 226 | * At first we should unmap early shadow (clear_pgds() call below). |
39d114dd AR |
227 | * However, instrumented code couldn't execute without shadow memory. |
228 | * tmp_pg_dir used to keep early shadow mapped until full shadow | |
229 | * setup will be finished. | |
230 | */ | |
231 | memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir)); | |
c1a88e91 | 232 | dsb(ishst); |
2077be67 | 233 | cpu_replace_ttbr1(lm_alias(tmp_pg_dir)); |
39d114dd AR |
234 | |
235 | clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); | |
236 | ||
e17d8025 | 237 | kasan_map_populate(kimg_shadow_start, kimg_shadow_end, |
800cb2e5 | 238 | early_pfn_to_nid(virt_to_pfn(lm_alias(_text)))); |
f9040773 | 239 | |
77ad4ce6 | 240 | kasan_populate_early_shadow(kasan_mem_to_shadow((void *)PAGE_END), |
14c127c9 | 241 | (void *)mod_shadow_start); |
9577dd74 | 242 | kasan_populate_early_shadow((void *)kimg_shadow_end, |
14c127c9 | 243 | (void *)KASAN_SHADOW_END); |
f80fb3a3 AB |
244 | |
245 | if (kimg_shadow_start > mod_shadow_end) | |
9577dd74 AK |
246 | kasan_populate_early_shadow((void *)mod_shadow_end, |
247 | (void *)kimg_shadow_start); | |
39d114dd | 248 | |
b10d6bca MR |
249 | for_each_mem_range(i, &pa_start, &pa_end) { |
250 | void *start = (void *)__phys_to_virt(pa_start); | |
251 | void *end = (void *)__phys_to_virt(pa_end); | |
39d114dd AR |
252 | |
253 | if (start >= end) | |
254 | break; | |
255 | ||
e17d8025 WD |
256 | kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), |
257 | (unsigned long)kasan_mem_to_shadow(end), | |
800cb2e5 | 258 | early_pfn_to_nid(virt_to_pfn(start))); |
39d114dd AR |
259 | } |
260 | ||
7b1af979 | 261 | /* |
9577dd74 AK |
262 | * KAsan may reuse the contents of kasan_early_shadow_pte directly, |
263 | * so we should make sure that it maps the zero page read-only. | |
7b1af979 AB |
264 | */ |
265 | for (i = 0; i < PTRS_PER_PTE; i++) | |
9577dd74 AK |
266 | set_pte(&kasan_early_shadow_pte[i], |
267 | pfn_pte(sym_to_pfn(kasan_early_shadow_page), | |
268 | PAGE_KERNEL_RO)); | |
7b1af979 | 269 | |
080eb83f | 270 | memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE); |
2077be67 | 271 | cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); |
39d114dd AR |
272 | |
273 | /* At this point kasan is fully initialized. Enable error messages */ | |
274 | init_task.kasan_depth = 0; | |
275 | pr_info("KernelAddressSanitizer initialized\n"); | |
276 | } |