Commit | Line | Data |
---|---|---|
df4e817b PT |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | /* | |
4 | * Copyright (c) 2021, Google LLC. | |
5 | * Pasha Tatashin <pasha.tatashin@soleen.com> | |
6 | */ | |
7 | #include <linux/mm.h> | |
8 | #include <linux/page_table_check.h> | |
9 | ||
10 | #undef pr_fmt | |
11 | #define pr_fmt(fmt) "page_table_check: " fmt | |
12 | ||
13 | struct page_table_check { | |
14 | atomic_t anon_map_count; | |
15 | atomic_t file_map_count; | |
16 | }; | |
17 | ||
18 | static bool __page_table_check_enabled __initdata = | |
19 | IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); | |
20 | ||
21 | DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); | |
22 | EXPORT_SYMBOL(page_table_check_disabled); | |
23 | ||
24 | static int __init early_page_table_check_param(char *buf) | |
25 | { | |
26 | if (!buf) | |
27 | return -EINVAL; | |
28 | ||
29 | if (strcmp(buf, "on") == 0) | |
30 | __page_table_check_enabled = true; | |
31 | else if (strcmp(buf, "off") == 0) | |
32 | __page_table_check_enabled = false; | |
33 | ||
34 | return 0; | |
35 | } | |
36 | ||
37 | early_param("page_table_check", early_page_table_check_param); | |
38 | ||
39 | static bool __init need_page_table_check(void) | |
40 | { | |
41 | return __page_table_check_enabled; | |
42 | } | |
43 | ||
44 | static void __init init_page_table_check(void) | |
45 | { | |
46 | if (!__page_table_check_enabled) | |
47 | return; | |
48 | static_branch_disable(&page_table_check_disabled); | |
49 | } | |
50 | ||
51 | struct page_ext_operations page_table_check_ops = { | |
52 | .size = sizeof(struct page_table_check), | |
53 | .need = need_page_table_check, | |
54 | .init = init_page_table_check, | |
55 | }; | |
56 | ||
57 | static struct page_table_check *get_page_table_check(struct page_ext *page_ext) | |
58 | { | |
59 | BUG_ON(!page_ext); | |
60 | return (void *)(page_ext) + page_table_check_ops.offset; | |
61 | } | |
62 | ||
63 | static inline bool pte_user_accessible_page(pte_t pte) | |
64 | { | |
65 | return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER); | |
66 | } | |
67 | ||
68 | static inline bool pmd_user_accessible_page(pmd_t pmd) | |
69 | { | |
70 | return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && | |
71 | (pmd_val(pmd) & _PAGE_USER); | |
72 | } | |
73 | ||
74 | static inline bool pud_user_accessible_page(pud_t pud) | |
75 | { | |
76 | return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && | |
77 | (pud_val(pud) & _PAGE_USER); | |
78 | } | |
79 | ||
80 | /* | |
81 | * An enty is removed from the page table, decrement the counters for that page | |
82 | * verify that it is of correct type and counters do not become negative. | |
83 | */ | |
84 | static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, | |
85 | unsigned long pfn, unsigned long pgcnt) | |
86 | { | |
87 | struct page_ext *page_ext; | |
88 | struct page *page; | |
64d8b9e1 | 89 | unsigned long i; |
df4e817b | 90 | bool anon; |
df4e817b PT |
91 | |
92 | if (!pfn_valid(pfn)) | |
93 | return; | |
94 | ||
95 | page = pfn_to_page(pfn); | |
96 | page_ext = lookup_page_ext(page); | |
97 | anon = PageAnon(page); | |
98 | ||
99 | for (i = 0; i < pgcnt; i++) { | |
100 | struct page_table_check *ptc = get_page_table_check(page_ext); | |
101 | ||
102 | if (anon) { | |
103 | BUG_ON(atomic_read(&ptc->file_map_count)); | |
104 | BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); | |
105 | } else { | |
106 | BUG_ON(atomic_read(&ptc->anon_map_count)); | |
107 | BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); | |
108 | } | |
109 | page_ext = page_ext_next(page_ext); | |
110 | } | |
111 | } | |
112 | ||
113 | /* | |
114 | * A new enty is added to the page table, increment the counters for that page | |
115 | * verify that it is of correct type and is not being mapped with a different | |
116 | * type to a different process. | |
117 | */ | |
118 | static void page_table_check_set(struct mm_struct *mm, unsigned long addr, | |
119 | unsigned long pfn, unsigned long pgcnt, | |
120 | bool rw) | |
121 | { | |
122 | struct page_ext *page_ext; | |
123 | struct page *page; | |
64d8b9e1 | 124 | unsigned long i; |
df4e817b | 125 | bool anon; |
df4e817b PT |
126 | |
127 | if (!pfn_valid(pfn)) | |
128 | return; | |
129 | ||
130 | page = pfn_to_page(pfn); | |
131 | page_ext = lookup_page_ext(page); | |
132 | anon = PageAnon(page); | |
133 | ||
134 | for (i = 0; i < pgcnt; i++) { | |
135 | struct page_table_check *ptc = get_page_table_check(page_ext); | |
136 | ||
137 | if (anon) { | |
138 | BUG_ON(atomic_read(&ptc->file_map_count)); | |
139 | BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); | |
140 | } else { | |
141 | BUG_ON(atomic_read(&ptc->anon_map_count)); | |
142 | BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); | |
143 | } | |
144 | page_ext = page_ext_next(page_ext); | |
145 | } | |
146 | } | |
147 | ||
148 | /* | |
149 | * page is on free list, or is being allocated, verify that counters are zeroes | |
150 | * crash if they are not. | |
151 | */ | |
152 | void __page_table_check_zero(struct page *page, unsigned int order) | |
153 | { | |
154 | struct page_ext *page_ext = lookup_page_ext(page); | |
64d8b9e1 | 155 | unsigned long i; |
df4e817b PT |
156 | |
157 | BUG_ON(!page_ext); | |
64d8b9e1 | 158 | for (i = 0; i < (1ul << order); i++) { |
df4e817b PT |
159 | struct page_table_check *ptc = get_page_table_check(page_ext); |
160 | ||
161 | BUG_ON(atomic_read(&ptc->anon_map_count)); | |
162 | BUG_ON(atomic_read(&ptc->file_map_count)); | |
163 | page_ext = page_ext_next(page_ext); | |
164 | } | |
165 | } | |
166 | ||
167 | void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, | |
168 | pte_t pte) | |
169 | { | |
170 | if (&init_mm == mm) | |
171 | return; | |
172 | ||
173 | if (pte_user_accessible_page(pte)) { | |
174 | page_table_check_clear(mm, addr, pte_pfn(pte), | |
175 | PAGE_SIZE >> PAGE_SHIFT); | |
176 | } | |
177 | } | |
178 | EXPORT_SYMBOL(__page_table_check_pte_clear); | |
179 | ||
180 | void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, | |
181 | pmd_t pmd) | |
182 | { | |
183 | if (&init_mm == mm) | |
184 | return; | |
185 | ||
186 | if (pmd_user_accessible_page(pmd)) { | |
187 | page_table_check_clear(mm, addr, pmd_pfn(pmd), | |
188 | PMD_PAGE_SIZE >> PAGE_SHIFT); | |
189 | } | |
190 | } | |
191 | EXPORT_SYMBOL(__page_table_check_pmd_clear); | |
192 | ||
193 | void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, | |
194 | pud_t pud) | |
195 | { | |
196 | if (&init_mm == mm) | |
197 | return; | |
198 | ||
199 | if (pud_user_accessible_page(pud)) { | |
200 | page_table_check_clear(mm, addr, pud_pfn(pud), | |
201 | PUD_PAGE_SIZE >> PAGE_SHIFT); | |
202 | } | |
203 | } | |
204 | EXPORT_SYMBOL(__page_table_check_pud_clear); | |
205 | ||
206 | void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, | |
207 | pte_t *ptep, pte_t pte) | |
208 | { | |
df4e817b PT |
209 | if (&init_mm == mm) |
210 | return; | |
211 | ||
64d8b9e1 | 212 | __page_table_check_pte_clear(mm, addr, *ptep); |
df4e817b PT |
213 | if (pte_user_accessible_page(pte)) { |
214 | page_table_check_set(mm, addr, pte_pfn(pte), | |
215 | PAGE_SIZE >> PAGE_SHIFT, | |
216 | pte_write(pte)); | |
217 | } | |
218 | } | |
219 | EXPORT_SYMBOL(__page_table_check_pte_set); | |
220 | ||
221 | void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, | |
222 | pmd_t *pmdp, pmd_t pmd) | |
223 | { | |
df4e817b PT |
224 | if (&init_mm == mm) |
225 | return; | |
226 | ||
64d8b9e1 | 227 | __page_table_check_pmd_clear(mm, addr, *pmdp); |
df4e817b PT |
228 | if (pmd_user_accessible_page(pmd)) { |
229 | page_table_check_set(mm, addr, pmd_pfn(pmd), | |
230 | PMD_PAGE_SIZE >> PAGE_SHIFT, | |
231 | pmd_write(pmd)); | |
232 | } | |
233 | } | |
234 | EXPORT_SYMBOL(__page_table_check_pmd_set); | |
235 | ||
236 | void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, | |
237 | pud_t *pudp, pud_t pud) | |
238 | { | |
df4e817b PT |
239 | if (&init_mm == mm) |
240 | return; | |
241 | ||
64d8b9e1 | 242 | __page_table_check_pud_clear(mm, addr, *pudp); |
df4e817b PT |
243 | if (pud_user_accessible_page(pud)) { |
244 | page_table_check_set(mm, addr, pud_pfn(pud), | |
245 | PUD_PAGE_SIZE >> PAGE_SHIFT, | |
246 | pud_write(pud)); | |
247 | } | |
248 | } | |
249 | EXPORT_SYMBOL(__page_table_check_pud_set); | |
80110bbf PT |
250 | |
251 | void __page_table_check_pte_clear_range(struct mm_struct *mm, | |
252 | unsigned long addr, | |
253 | pmd_t pmd) | |
254 | { | |
255 | if (&init_mm == mm) | |
256 | return; | |
257 | ||
258 | if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { | |
259 | pte_t *ptep = pte_offset_map(&pmd, addr); | |
260 | unsigned long i; | |
261 | ||
262 | pte_unmap(ptep); | |
263 | for (i = 0; i < PTRS_PER_PTE; i++) { | |
264 | __page_table_check_pte_clear(mm, addr, *ptep); | |
265 | addr += PAGE_SIZE; | |
266 | ptep++; | |
267 | } | |
268 | } | |
269 | } |