Commit | Line | Data |
---|---|---|
39d114dd AR |
1 | /* |
2 | * This file contains kasan initialization code for ARM64. | |
3 | * | |
4 | * Copyright (c) 2015 Samsung Electronics Co., Ltd. | |
5 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | */ | |
12 | ||
13 | #define pr_fmt(fmt) "kasan: " fmt | |
e17d8025 | 14 | #include <linux/bootmem.h> |
39d114dd AR |
15 | #include <linux/kasan.h> |
16 | #include <linux/kernel.h> | |
9164bb4a | 17 | #include <linux/sched/task.h> |
39d114dd AR |
18 | #include <linux/memblock.h> |
19 | #include <linux/start_kernel.h> | |
2077be67 | 20 | #include <linux/mm.h> |
39d114dd | 21 | |
c1a88e91 | 22 | #include <asm/mmu_context.h> |
f9040773 | 23 | #include <asm/kernel-pgtable.h> |
39d114dd AR |
24 | #include <asm/page.h> |
25 | #include <asm/pgalloc.h> | |
26 | #include <asm/pgtable.h> | |
f9040773 | 27 | #include <asm/sections.h> |
39d114dd AR |
28 | #include <asm/tlbflush.h> |
29 | ||
30 | static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE); | |
31 | ||
2077be67 LA |
32 | /* |
33 | * The p*d_populate functions call virt_to_phys implicitly so they can't be used | |
34 | * directly on kernel symbols (bm_p*d). All the early functions are called too | |
35 | * early to use lm_alias so __p*d_populate functions must be used to populate | |
36 | * with the physical address from __pa_symbol. | |
37 | */ | |
38 | ||
e17d8025 | 39 | static phys_addr_t __init kasan_alloc_zeroed_page(int node) |
39d114dd | 40 | { |
e17d8025 WD |
41 | void *p = memblock_virt_alloc_try_nid(PAGE_SIZE, PAGE_SIZE, |
42 | __pa(MAX_DMA_ADDRESS), | |
43 | MEMBLOCK_ALLOC_ACCESSIBLE, node); | |
44 | return __pa(p); | |
45 | } | |
46 | ||
20a004e7 | 47 | static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, |
e17d8025 WD |
48 | bool early) |
49 | { | |
20a004e7 | 50 | if (pmd_none(READ_ONCE(*pmdp))) { |
e17d8025 WD |
51 | phys_addr_t pte_phys = early ? __pa_symbol(kasan_zero_pte) |
52 | : kasan_alloc_zeroed_page(node); | |
20a004e7 | 53 | __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); |
e17d8025 WD |
54 | } |
55 | ||
20a004e7 WD |
56 | return early ? pte_offset_kimg(pmdp, addr) |
57 | : pte_offset_kernel(pmdp, addr); | |
e17d8025 | 58 | } |
39d114dd | 59 | |
20a004e7 | 60 | static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, |
e17d8025 WD |
61 | bool early) |
62 | { | |
20a004e7 | 63 | if (pud_none(READ_ONCE(*pudp))) { |
e17d8025 WD |
64 | phys_addr_t pmd_phys = early ? __pa_symbol(kasan_zero_pmd) |
65 | : kasan_alloc_zeroed_page(node); | |
20a004e7 | 66 | __pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE); |
e17d8025 WD |
67 | } |
68 | ||
20a004e7 | 69 | return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); |
e17d8025 WD |
70 | } |
71 | ||
20a004e7 | 72 | static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node, |
e17d8025 WD |
73 | bool early) |
74 | { | |
20a004e7 | 75 | if (pgd_none(READ_ONCE(*pgdp))) { |
e17d8025 WD |
76 | phys_addr_t pud_phys = early ? __pa_symbol(kasan_zero_pud) |
77 | : kasan_alloc_zeroed_page(node); | |
20a004e7 | 78 | __pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE); |
e17d8025 WD |
79 | } |
80 | ||
20a004e7 | 81 | return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr); |
e17d8025 WD |
82 | } |
83 | ||
20a004e7 | 84 | static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, |
e17d8025 WD |
85 | unsigned long end, int node, bool early) |
86 | { | |
87 | unsigned long next; | |
20a004e7 | 88 | pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); |
39d114dd | 89 | |
39d114dd | 90 | do { |
e17d8025 WD |
91 | phys_addr_t page_phys = early ? __pa_symbol(kasan_zero_page) |
92 | : kasan_alloc_zeroed_page(node); | |
39d114dd | 93 | next = addr + PAGE_SIZE; |
20a004e7 WD |
94 | set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL)); |
95 | } while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep))); | |
39d114dd AR |
96 | } |
97 | ||
20a004e7 | 98 | static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, |
e17d8025 | 99 | unsigned long end, int node, bool early) |
39d114dd | 100 | { |
39d114dd | 101 | unsigned long next; |
20a004e7 | 102 | pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); |
39d114dd | 103 | |
39d114dd AR |
104 | do { |
105 | next = pmd_addr_end(addr, end); | |
20a004e7 WD |
106 | kasan_pte_populate(pmdp, addr, next, node, early); |
107 | } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp))); | |
39d114dd AR |
108 | } |
109 | ||
20a004e7 | 110 | static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr, |
e17d8025 | 111 | unsigned long end, int node, bool early) |
39d114dd | 112 | { |
39d114dd | 113 | unsigned long next; |
20a004e7 | 114 | pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early); |
39d114dd | 115 | |
39d114dd AR |
116 | do { |
117 | next = pud_addr_end(addr, end); | |
20a004e7 WD |
118 | kasan_pmd_populate(pudp, addr, next, node, early); |
119 | } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); | |
39d114dd AR |
120 | } |
121 | ||
e17d8025 WD |
122 | static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, |
123 | int node, bool early) | |
39d114dd | 124 | { |
39d114dd | 125 | unsigned long next; |
20a004e7 | 126 | pgd_t *pgdp; |
39d114dd | 127 | |
20a004e7 | 128 | pgdp = pgd_offset_k(addr); |
39d114dd AR |
129 | do { |
130 | next = pgd_addr_end(addr, end); | |
20a004e7 WD |
131 | kasan_pud_populate(pgdp, addr, next, node, early); |
132 | } while (pgdp++, addr = next, addr != end); | |
39d114dd AR |
133 | } |
134 | ||
e17d8025 | 135 | /* The early shadow maps everything to a single page of zeroes */ |
83040123 | 136 | asmlinkage void __init kasan_early_init(void) |
39d114dd | 137 | { |
917538e2 AK |
138 | BUILD_BUG_ON(KASAN_SHADOW_OFFSET != |
139 | KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT))); | |
39d114dd AR |
140 | BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE)); |
141 | BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE)); | |
e17d8025 WD |
142 | kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, |
143 | true); | |
144 | } | |
145 | ||
146 | /* Set up full kasan mappings, ensuring that the mapped pages are zeroed */ | |
147 | static void __init kasan_map_populate(unsigned long start, unsigned long end, | |
148 | int node) | |
149 | { | |
150 | kasan_pgd_populate(start & PAGE_MASK, PAGE_ALIGN(end), node, false); | |
39d114dd AR |
151 | } |
152 | ||
068a17a5 MR |
153 | /* |
154 | * Copy the current shadow region into a new pgdir. | |
155 | */ | |
156 | void __init kasan_copy_shadow(pgd_t *pgdir) | |
157 | { | |
20a004e7 | 158 | pgd_t *pgdp, *pgdp_new, *pgdp_end; |
068a17a5 | 159 | |
20a004e7 WD |
160 | pgdp = pgd_offset_k(KASAN_SHADOW_START); |
161 | pgdp_end = pgd_offset_k(KASAN_SHADOW_END); | |
162 | pgdp_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START); | |
068a17a5 | 163 | do { |
20a004e7 WD |
164 | set_pgd(pgdp_new, READ_ONCE(*pgdp)); |
165 | } while (pgdp++, pgdp_new++, pgdp != pgdp_end); | |
068a17a5 MR |
166 | } |
167 | ||
39d114dd AR |
168 | static void __init clear_pgds(unsigned long start, |
169 | unsigned long end) | |
170 | { | |
171 | /* | |
172 | * Remove references to kasan page tables from | |
173 | * swapper_pg_dir. pgd_clear() can't be used | |
174 | * here because it's nop on 2,3-level pagetable setups | |
175 | */ | |
176 | for (; start < end; start += PGDIR_SIZE) | |
177 | set_pgd(pgd_offset_k(start), __pgd(0)); | |
178 | } | |
179 | ||
39d114dd AR |
180 | void __init kasan_init(void) |
181 | { | |
f9040773 | 182 | u64 kimg_shadow_start, kimg_shadow_end; |
f80fb3a3 | 183 | u64 mod_shadow_start, mod_shadow_end; |
39d114dd | 184 | struct memblock_region *reg; |
7b1af979 | 185 | int i; |
39d114dd | 186 | |
e17d8025 WD |
187 | kimg_shadow_start = (u64)kasan_mem_to_shadow(_text) & PAGE_MASK; |
188 | kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(_end)); | |
f9040773 | 189 | |
f80fb3a3 AB |
190 | mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR); |
191 | mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END); | |
192 | ||
39d114dd AR |
193 | /* |
194 | * We are going to perform proper setup of shadow memory. | |
195 | * At first we should unmap early shadow (clear_pgds() call bellow). | |
196 | * However, instrumented code couldn't execute without shadow memory. | |
197 | * tmp_pg_dir used to keep early shadow mapped until full shadow | |
198 | * setup will be finished. | |
199 | */ | |
200 | memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir)); | |
c1a88e91 | 201 | dsb(ishst); |
2077be67 | 202 | cpu_replace_ttbr1(lm_alias(tmp_pg_dir)); |
39d114dd AR |
203 | |
204 | clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); | |
205 | ||
e17d8025 | 206 | kasan_map_populate(kimg_shadow_start, kimg_shadow_end, |
800cb2e5 | 207 | early_pfn_to_nid(virt_to_pfn(lm_alias(_text)))); |
f9040773 | 208 | |
39d114dd | 209 | kasan_populate_zero_shadow((void *)KASAN_SHADOW_START, |
f80fb3a3 | 210 | (void *)mod_shadow_start); |
f9040773 | 211 | kasan_populate_zero_shadow((void *)kimg_shadow_end, |
f80fb3a3 AB |
212 | kasan_mem_to_shadow((void *)PAGE_OFFSET)); |
213 | ||
214 | if (kimg_shadow_start > mod_shadow_end) | |
215 | kasan_populate_zero_shadow((void *)mod_shadow_end, | |
216 | (void *)kimg_shadow_start); | |
39d114dd AR |
217 | |
218 | for_each_memblock(memory, reg) { | |
219 | void *start = (void *)__phys_to_virt(reg->base); | |
220 | void *end = (void *)__phys_to_virt(reg->base + reg->size); | |
221 | ||
222 | if (start >= end) | |
223 | break; | |
224 | ||
e17d8025 WD |
225 | kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), |
226 | (unsigned long)kasan_mem_to_shadow(end), | |
800cb2e5 | 227 | early_pfn_to_nid(virt_to_pfn(start))); |
39d114dd AR |
228 | } |
229 | ||
7b1af979 AB |
230 | /* |
231 | * KAsan may reuse the contents of kasan_zero_pte directly, so we | |
232 | * should make sure that it maps the zero page read-only. | |
233 | */ | |
234 | for (i = 0; i < PTRS_PER_PTE; i++) | |
235 | set_pte(&kasan_zero_pte[i], | |
2077be67 | 236 | pfn_pte(sym_to_pfn(kasan_zero_page), PAGE_KERNEL_RO)); |
7b1af979 | 237 | |
39d114dd | 238 | memset(kasan_zero_page, 0, PAGE_SIZE); |
2077be67 | 239 | cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); |
39d114dd AR |
240 | |
241 | /* At this point kasan is fully initialized. Enable error messages */ | |
242 | init_task.kasan_depth = 0; | |
243 | pr_info("KernelAddressSanitizer initialized\n"); | |
244 | } |