Commit | Line | Data |
---|---|---|
c7b9ed7c CL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * KASAN for 64-bit Book3e powerpc | |
4 | * | |
5 | * Copyright 2022, Christophe Leroy, CS GROUP France | |
6 | */ | |
7 | ||
8 | #define DISABLE_BRANCH_PROFILING | |
9 | ||
10 | #include <linux/kasan.h> | |
11 | #include <linux/printk.h> | |
12 | #include <linux/memblock.h> | |
13 | #include <linux/set_memory.h> | |
14 | ||
15 | #include <asm/pgalloc.h> | |
16 | ||
17 | static inline bool kasan_pud_table(p4d_t p4d) | |
18 | { | |
19 | return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); | |
20 | } | |
21 | ||
22 | static inline bool kasan_pmd_table(pud_t pud) | |
23 | { | |
24 | return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd)); | |
25 | } | |
26 | ||
27 | static inline bool kasan_pte_table(pmd_t pmd) | |
28 | { | |
29 | return pmd_page(pmd) == virt_to_page(lm_alias(kasan_early_shadow_pte)); | |
30 | } | |
31 | ||
32 | static int __init kasan_map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) | |
33 | { | |
34 | pgd_t *pgdp; | |
35 | p4d_t *p4dp; | |
36 | pud_t *pudp; | |
37 | pmd_t *pmdp; | |
38 | pte_t *ptep; | |
39 | ||
40 | pgdp = pgd_offset_k(ea); | |
41 | p4dp = p4d_offset(pgdp, ea); | |
42 | if (kasan_pud_table(*p4dp)) { | |
43 | pudp = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); | |
44 | memcpy(pudp, kasan_early_shadow_pud, PUD_TABLE_SIZE); | |
45 | p4d_populate(&init_mm, p4dp, pudp); | |
46 | } | |
47 | pudp = pud_offset(p4dp, ea); | |
48 | if (kasan_pmd_table(*pudp)) { | |
49 | pmdp = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); | |
50 | memcpy(pmdp, kasan_early_shadow_pmd, PMD_TABLE_SIZE); | |
51 | pud_populate(&init_mm, pudp, pmdp); | |
52 | } | |
53 | pmdp = pmd_offset(pudp, ea); | |
54 | if (kasan_pte_table(*pmdp)) { | |
55 | ptep = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); | |
56 | memcpy(ptep, kasan_early_shadow_pte, PTE_TABLE_SIZE); | |
57 | pmd_populate_kernel(&init_mm, pmdp, ptep); | |
58 | } | |
59 | ptep = pte_offset_kernel(pmdp, ea); | |
60 | ||
61 | __set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot), 0); | |
62 | ||
63 | return 0; | |
64 | } | |
65 | ||
66 | static void __init kasan_init_phys_region(void *start, void *end) | |
67 | { | |
68 | unsigned long k_start, k_end, k_cur; | |
69 | void *va; | |
70 | ||
71 | if (start >= end) | |
72 | return; | |
73 | ||
74 | k_start = ALIGN_DOWN((unsigned long)kasan_mem_to_shadow(start), PAGE_SIZE); | |
75 | k_end = ALIGN((unsigned long)kasan_mem_to_shadow(end), PAGE_SIZE); | |
76 | ||
77 | va = memblock_alloc(k_end - k_start, PAGE_SIZE); | |
78 | for (k_cur = k_start; k_cur < k_end; k_cur += PAGE_SIZE, va += PAGE_SIZE) | |
79 | kasan_map_kernel_page(k_cur, __pa(va), PAGE_KERNEL); | |
80 | } | |
81 | ||
82 | void __init kasan_early_init(void) | |
83 | { | |
84 | int i; | |
85 | unsigned long addr; | |
86 | pgd_t *pgd = pgd_offset_k(KASAN_SHADOW_START); | |
87 | pte_t zero_pte = pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL); | |
88 | ||
89 | BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE)); | |
90 | BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE)); | |
91 | ||
92 | for (i = 0; i < PTRS_PER_PTE; i++) | |
93 | __set_pte_at(&init_mm, (unsigned long)kasan_early_shadow_page, | |
94 | &kasan_early_shadow_pte[i], zero_pte, 0); | |
95 | ||
96 | for (i = 0; i < PTRS_PER_PMD; i++) | |
97 | pmd_populate_kernel(&init_mm, &kasan_early_shadow_pmd[i], | |
98 | kasan_early_shadow_pte); | |
99 | ||
100 | for (i = 0; i < PTRS_PER_PUD; i++) | |
101 | pud_populate(&init_mm, &kasan_early_shadow_pud[i], | |
102 | kasan_early_shadow_pmd); | |
103 | ||
104 | for (addr = KASAN_SHADOW_START; addr != KASAN_SHADOW_END; addr += PGDIR_SIZE) | |
105 | p4d_populate(&init_mm, p4d_offset(pgd++, addr), kasan_early_shadow_pud); | |
106 | } | |
107 | ||
108 | void __init kasan_init(void) | |
109 | { | |
110 | phys_addr_t start, end; | |
111 | u64 i; | |
112 | pte_t zero_pte = pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL_RO); | |
113 | ||
114 | for_each_mem_range(i, &start, &end) | |
295454ed | 115 | kasan_init_phys_region(phys_to_virt(start), phys_to_virt(end)); |
c7b9ed7c CL |
116 | |
117 | if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) | |
118 | kasan_remove_zero_shadow((void *)VMALLOC_START, VMALLOC_SIZE); | |
119 | ||
120 | for (i = 0; i < PTRS_PER_PTE; i++) | |
121 | __set_pte_at(&init_mm, (unsigned long)kasan_early_shadow_page, | |
122 | &kasan_early_shadow_pte[i], zero_pte, 0); | |
123 | ||
124 | flush_tlb_kernel_range(KASAN_SHADOW_START, KASAN_SHADOW_END); | |
125 | ||
126 | memset(kasan_early_shadow_page, 0, PAGE_SIZE); | |
127 | ||
128 | /* Enable error messages */ | |
129 | init_task.kasan_depth = 0; | |
130 | pr_info("KASAN init done\n"); | |
131 | } | |
132 | ||
133 | void __init kasan_late_init(void) { } |