Commit | Line | Data |
---|---|---|
926e5392 AV |
1 | /* |
2 | * Debug helper to dump the current kernel pagetables of the system | |
3 | * so that we can see what the various memory ranges are set to. | |
4 | * | |
5 | * (C) Copyright 2008 Intel Corporation | |
6 | * | |
7 | * Author: Arjan van de Ven <arjan@linux.intel.com> | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; version 2 | |
12 | * of the License. | |
13 | */ | |
14 | ||
fe770bf0 | 15 | #include <linux/debugfs.h> |
04b67022 | 16 | #include <linux/kasan.h> |
fe770bf0 | 17 | #include <linux/mm.h> |
84e629b6 | 18 | #include <linux/init.h> |
146fbb76 | 19 | #include <linux/sched.h> |
926e5392 | 20 | #include <linux/seq_file.h> |
926e5392 AV |
21 | |
22 | #include <asm/pgtable.h> | |
23 | ||
24 | /* | |
25 | * The dumper groups pagetable entries of the same type into one, and for | |
26 | * that it needs to keep some state when walking, and flush this state | |
27 | * when a "break" in the continuity is found. | |
28 | */ | |
29 | struct pg_state { | |
30 | int level; | |
31 | pgprot_t current_prot; | |
32 | unsigned long start_address; | |
33 | unsigned long current_address; | |
fe770bf0 | 34 | const struct addr_marker *marker; |
3891a04a | 35 | unsigned long lines; |
ef6bea6d | 36 | bool to_dmesg; |
e1a58320 SS |
37 | bool check_wx; |
38 | unsigned long wx_pages; | |
926e5392 AV |
39 | }; |
40 | ||
fe770bf0 PA |
41 | struct addr_marker { |
42 | unsigned long start_address; | |
43 | const char *name; | |
3891a04a | 44 | unsigned long max_lines; |
fe770bf0 PA |
45 | }; |
46 | ||
146122e2 TG |
47 | /* Address space markers hints */ |
48 | ||
49 | #ifdef CONFIG_X86_64 | |
50 | ||
92851e2f AS |
51 | enum address_markers_idx { |
52 | USER_SPACE_NR = 0, | |
92851e2f AS |
53 | KERNEL_SPACE_NR, |
54 | LOW_KERNEL_NR, | |
f55f0501 AL |
55 | #if defined(CONFIG_MODIFY_LDT_SYSCALL) && defined(CONFIG_X86_5LEVEL) |
56 | LDT_NR, | |
57 | #endif | |
92851e2f AS |
58 | VMALLOC_START_NR, |
59 | VMEMMAP_START_NR, | |
025205f8 AR |
60 | #ifdef CONFIG_KASAN |
61 | KASAN_SHADOW_START_NR, | |
62 | KASAN_SHADOW_END_NR, | |
f55f0501 AL |
63 | #endif |
64 | #if defined(CONFIG_MODIFY_LDT_SYSCALL) && !defined(CONFIG_X86_5LEVEL) | |
65 | LDT_NR, | |
025205f8 | 66 | #endif |
92a0f81d | 67 | CPU_ENTRY_AREA_NR, |
146122e2 | 68 | #ifdef CONFIG_X86_ESPFIX64 |
3891a04a | 69 | ESPFIX_START_NR, |
146122e2 TG |
70 | #endif |
71 | #ifdef CONFIG_EFI | |
72 | EFI_END_NR, | |
73 | #endif | |
92851e2f AS |
74 | HIGH_KERNEL_NR, |
75 | MODULES_VADDR_NR, | |
76 | MODULES_END_NR, | |
146122e2 TG |
77 | FIXADDR_START_NR, |
78 | END_OF_SPACE_NR, | |
79 | }; | |
80 | ||
81 | static struct addr_marker address_markers[] = { | |
82 | [USER_SPACE_NR] = { 0, "User Space" }, | |
83 | [KERNEL_SPACE_NR] = { (1UL << 63), "Kernel Space" }, | |
84 | [LOW_KERNEL_NR] = { 0UL, "Low Kernel Mapping" }, | |
85 | [VMALLOC_START_NR] = { 0UL, "vmalloc() Area" }, | |
86 | [VMEMMAP_START_NR] = { 0UL, "Vmemmap" }, | |
87 | #ifdef CONFIG_KASAN | |
88 | [KASAN_SHADOW_START_NR] = { KASAN_SHADOW_START, "KASAN shadow" }, | |
89 | [KASAN_SHADOW_END_NR] = { KASAN_SHADOW_END, "KASAN shadow end" }, | |
f55f0501 AL |
90 | #endif |
91 | #ifdef CONFIG_MODIFY_LDT_SYSCALL | |
92 | [LDT_NR] = { LDT_BASE_ADDR, "LDT remap" }, | |
146122e2 | 93 | #endif |
92a0f81d | 94 | [CPU_ENTRY_AREA_NR] = { CPU_ENTRY_AREA_BASE,"CPU entry Area" }, |
146122e2 TG |
95 | #ifdef CONFIG_X86_ESPFIX64 |
96 | [ESPFIX_START_NR] = { ESPFIX_BASE_ADDR, "ESPfix Area", 16 }, | |
97 | #endif | |
98 | #ifdef CONFIG_EFI | |
99 | [EFI_END_NR] = { EFI_VA_END, "EFI Runtime Services" }, | |
100 | #endif | |
101 | [HIGH_KERNEL_NR] = { __START_KERNEL_map, "High Kernel Mapping" }, | |
102 | [MODULES_VADDR_NR] = { MODULES_VADDR, "Modules" }, | |
103 | [MODULES_END_NR] = { MODULES_END, "End Modules" }, | |
104 | [FIXADDR_START_NR] = { FIXADDR_START, "Fixmap Area" }, | |
105 | [END_OF_SPACE_NR] = { -1, NULL } | |
106 | }; | |
107 | ||
108 | #else /* CONFIG_X86_64 */ | |
109 | ||
110 | enum address_markers_idx { | |
111 | USER_SPACE_NR = 0, | |
92851e2f AS |
112 | KERNEL_SPACE_NR, |
113 | VMALLOC_START_NR, | |
114 | VMALLOC_END_NR, | |
146122e2 | 115 | #ifdef CONFIG_HIGHMEM |
92851e2f | 116 | PKMAP_BASE_NR, |
92851e2f | 117 | #endif |
92a0f81d | 118 | CPU_ENTRY_AREA_NR, |
146122e2 TG |
119 | FIXADDR_START_NR, |
120 | END_OF_SPACE_NR, | |
92851e2f AS |
121 | }; |
122 | ||
fe770bf0 | 123 | static struct addr_marker address_markers[] = { |
146122e2 TG |
124 | [USER_SPACE_NR] = { 0, "User Space" }, |
125 | [KERNEL_SPACE_NR] = { PAGE_OFFSET, "Kernel Mapping" }, | |
126 | [VMALLOC_START_NR] = { 0UL, "vmalloc() Area" }, | |
127 | [VMALLOC_END_NR] = { 0UL, "vmalloc() End" }, | |
128 | #ifdef CONFIG_HIGHMEM | |
129 | [PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" }, | |
fe770bf0 | 130 | #endif |
92a0f81d | 131 | [CPU_ENTRY_AREA_NR] = { 0UL, "CPU entry area" }, |
146122e2 TG |
132 | [FIXADDR_START_NR] = { 0UL, "Fixmap area" }, |
133 | [END_OF_SPACE_NR] = { -1, NULL } | |
fe770bf0 | 134 | }; |
926e5392 | 135 | |
146122e2 TG |
136 | #endif /* !CONFIG_X86_64 */ |
137 | ||
fe770bf0 PA |
138 | /* Multipliers for offsets within the PTEs */ |
139 | #define PTE_LEVEL_MULT (PAGE_SIZE) | |
140 | #define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT) | |
141 | #define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT) | |
fdd3d8ce | 142 | #define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT) |
84bbabc3 | 143 | #define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT) |
926e5392 | 144 | |
ef6bea6d BP |
145 | #define pt_dump_seq_printf(m, to_dmesg, fmt, args...) \ |
146 | ({ \ | |
147 | if (to_dmesg) \ | |
148 | printk(KERN_INFO fmt, ##args); \ | |
149 | else \ | |
150 | if (m) \ | |
151 | seq_printf(m, fmt, ##args); \ | |
152 | }) | |
153 | ||
154 | #define pt_dump_cont_printf(m, to_dmesg, fmt, args...) \ | |
155 | ({ \ | |
156 | if (to_dmesg) \ | |
157 | printk(KERN_CONT fmt, ##args); \ | |
158 | else \ | |
159 | if (m) \ | |
160 | seq_printf(m, fmt, ##args); \ | |
161 | }) | |
162 | ||
926e5392 AV |
163 | /* |
164 | * Print a readable form of a pgprot_t to the seq_file | |
165 | */ | |
ef6bea6d | 166 | static void printk_prot(struct seq_file *m, pgprot_t prot, int level, bool dmsg) |
926e5392 | 167 | { |
fe770bf0 PA |
168 | pgprotval_t pr = pgprot_val(prot); |
169 | static const char * const level_name[] = | |
45dcd209 | 170 | { "cr3", "pgd", "p4d", "pud", "pmd", "pte" }; |
fe770bf0 | 171 | |
c0534494 | 172 | if (!(pr & _PAGE_PRESENT)) { |
fe770bf0 | 173 | /* Not present */ |
f439c429 | 174 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 PA |
175 | } else { |
176 | if (pr & _PAGE_USER) | |
ef6bea6d | 177 | pt_dump_cont_printf(m, dmsg, "USR "); |
926e5392 | 178 | else |
ef6bea6d | 179 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 180 | if (pr & _PAGE_RW) |
ef6bea6d | 181 | pt_dump_cont_printf(m, dmsg, "RW "); |
fe770bf0 | 182 | else |
ef6bea6d | 183 | pt_dump_cont_printf(m, dmsg, "ro "); |
fe770bf0 | 184 | if (pr & _PAGE_PWT) |
ef6bea6d | 185 | pt_dump_cont_printf(m, dmsg, "PWT "); |
fe770bf0 | 186 | else |
ef6bea6d | 187 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 188 | if (pr & _PAGE_PCD) |
ef6bea6d | 189 | pt_dump_cont_printf(m, dmsg, "PCD "); |
926e5392 | 190 | else |
ef6bea6d | 191 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 192 | |
f439c429 | 193 | /* Bit 7 has a different meaning on level 3 vs 4 */ |
45dcd209 | 194 | if (level <= 4 && pr & _PAGE_PSE) |
f439c429 JG |
195 | pt_dump_cont_printf(m, dmsg, "PSE "); |
196 | else | |
197 | pt_dump_cont_printf(m, dmsg, " "); | |
45dcd209 KS |
198 | if ((level == 5 && pr & _PAGE_PAT) || |
199 | ((level == 4 || level == 3) && pr & _PAGE_PAT_LARGE)) | |
da25e628 | 200 | pt_dump_cont_printf(m, dmsg, "PAT "); |
f439c429 JG |
201 | else |
202 | pt_dump_cont_printf(m, dmsg, " "); | |
fe770bf0 | 203 | if (pr & _PAGE_GLOBAL) |
ef6bea6d | 204 | pt_dump_cont_printf(m, dmsg, "GLB "); |
fe770bf0 | 205 | else |
ef6bea6d | 206 | pt_dump_cont_printf(m, dmsg, " "); |
fe770bf0 | 207 | if (pr & _PAGE_NX) |
ef6bea6d | 208 | pt_dump_cont_printf(m, dmsg, "NX "); |
fe770bf0 | 209 | else |
ef6bea6d | 210 | pt_dump_cont_printf(m, dmsg, "x "); |
926e5392 | 211 | } |
ef6bea6d | 212 | pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]); |
926e5392 AV |
213 | } |
214 | ||
215 | /* | |
fe770bf0 | 216 | * On 64 bits, sign-extend the 48 bit address to 64 bit |
926e5392 | 217 | */ |
fe770bf0 | 218 | static unsigned long normalize_addr(unsigned long u) |
926e5392 | 219 | { |
3a366f79 KS |
220 | int shift; |
221 | if (!IS_ENABLED(CONFIG_X86_64)) | |
222 | return u; | |
223 | ||
224 | shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); | |
225 | return (signed long)(u << shift) >> shift; | |
926e5392 AV |
226 | } |
227 | ||
228 | /* | |
229 | * This function gets called on a break in a continuous series | |
230 | * of PTE entries; the next one is different so we need to | |
231 | * print what we collected so far. | |
232 | */ | |
233 | static void note_page(struct seq_file *m, struct pg_state *st, | |
fe770bf0 | 234 | pgprot_t new_prot, int level) |
926e5392 | 235 | { |
fe770bf0 | 236 | pgprotval_t prot, cur; |
3891a04a | 237 | static const char units[] = "BKMGTPE"; |
926e5392 AV |
238 | |
239 | /* | |
240 | * If we have a "break" in the series, we need to flush the state that | |
fe770bf0 PA |
241 | * we have now. "break" is either changing perms, levels or |
242 | * address space marker. | |
926e5392 | 243 | */ |
da25e628 TK |
244 | prot = pgprot_val(new_prot); |
245 | cur = pgprot_val(st->current_prot); | |
926e5392 | 246 | |
fe770bf0 PA |
247 | if (!st->level) { |
248 | /* First entry */ | |
249 | st->current_prot = new_prot; | |
250 | st->level = level; | |
251 | st->marker = address_markers; | |
3891a04a | 252 | st->lines = 0; |
ef6bea6d BP |
253 | pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n", |
254 | st->marker->name); | |
fe770bf0 PA |
255 | } else if (prot != cur || level != st->level || |
256 | st->current_address >= st->marker[1].start_address) { | |
257 | const char *unit = units; | |
926e5392 | 258 | unsigned long delta; |
6424fb38 | 259 | int width = sizeof(unsigned long) * 2; |
e1a58320 SS |
260 | pgprotval_t pr = pgprot_val(st->current_prot); |
261 | ||
262 | if (st->check_wx && (pr & _PAGE_RW) && !(pr & _PAGE_NX)) { | |
263 | WARN_ONCE(1, | |
264 | "x86/mm: Found insecure W+X mapping at address %p/%pS\n", | |
265 | (void *)st->start_address, | |
266 | (void *)st->start_address); | |
267 | st->wx_pages += (st->current_address - | |
268 | st->start_address) / PAGE_SIZE; | |
269 | } | |
926e5392 | 270 | |
926e5392 AV |
271 | /* |
272 | * Now print the actual finished series | |
273 | */ | |
3891a04a PA |
274 | if (!st->marker->max_lines || |
275 | st->lines < st->marker->max_lines) { | |
276 | pt_dump_seq_printf(m, st->to_dmesg, | |
277 | "0x%0*lx-0x%0*lx ", | |
278 | width, st->start_address, | |
279 | width, st->current_address); | |
926e5392 | 280 | |
3891a04a PA |
281 | delta = st->current_address - st->start_address; |
282 | while (!(delta & 1023) && unit[1]) { | |
283 | delta >>= 10; | |
284 | unit++; | |
285 | } | |
286 | pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ", | |
287 | delta, *unit); | |
288 | printk_prot(m, st->current_prot, st->level, | |
289 | st->to_dmesg); | |
926e5392 | 290 | } |
3891a04a | 291 | st->lines++; |
fe770bf0 PA |
292 | |
293 | /* | |
294 | * We print markers for special areas of address space, | |
295 | * such as the start of vmalloc space etc. | |
296 | * This helps in the interpretation. | |
297 | */ | |
298 | if (st->current_address >= st->marker[1].start_address) { | |
3891a04a PA |
299 | if (st->marker->max_lines && |
300 | st->lines > st->marker->max_lines) { | |
301 | unsigned long nskip = | |
302 | st->lines - st->marker->max_lines; | |
303 | pt_dump_seq_printf(m, st->to_dmesg, | |
304 | "... %lu entr%s skipped ... \n", | |
305 | nskip, | |
306 | nskip == 1 ? "y" : "ies"); | |
307 | } | |
fe770bf0 | 308 | st->marker++; |
3891a04a | 309 | st->lines = 0; |
ef6bea6d BP |
310 | pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n", |
311 | st->marker->name); | |
926e5392 | 312 | } |
fe770bf0 | 313 | |
926e5392 AV |
314 | st->start_address = st->current_address; |
315 | st->current_prot = new_prot; | |
316 | st->level = level; | |
fe770bf0 | 317 | } |
926e5392 AV |
318 | } |
319 | ||
fdd3d8ce | 320 | static void walk_pte_level(struct seq_file *m, struct pg_state *st, pmd_t addr, unsigned long P) |
926e5392 AV |
321 | { |
322 | int i; | |
323 | pte_t *start; | |
da25e628 | 324 | pgprotval_t prot; |
926e5392 | 325 | |
fdd3d8ce | 326 | start = (pte_t *)pmd_page_vaddr(addr); |
926e5392 | 327 | for (i = 0; i < PTRS_PER_PTE; i++) { |
da25e628 | 328 | prot = pte_flags(*start); |
fe770bf0 | 329 | st->current_address = normalize_addr(P + i * PTE_LEVEL_MULT); |
45dcd209 | 330 | note_page(m, st, __pgprot(prot), 5); |
926e5392 AV |
331 | start++; |
332 | } | |
333 | } | |
04b67022 AR |
334 | #ifdef CONFIG_KASAN |
335 | ||
336 | /* | |
337 | * This is an optimization for KASAN=y case. Since all kasan page tables | |
338 | * eventually point to the kasan_zero_page we could call note_page() | |
339 | * right away without walking through lower level page tables. This saves | |
340 | * us dozens of seconds (minutes for 5-level config) while checking for | |
341 | * W+X mapping or reading kernel_page_tables debugfs file. | |
342 | */ | |
343 | static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st, | |
344 | void *pt) | |
345 | { | |
346 | if (__pa(pt) == __pa(kasan_zero_pmd) || | |
347 | #ifdef CONFIG_X86_5LEVEL | |
348 | __pa(pt) == __pa(kasan_zero_p4d) || | |
349 | #endif | |
350 | __pa(pt) == __pa(kasan_zero_pud)) { | |
351 | pgprotval_t prot = pte_flags(kasan_zero_pte[0]); | |
352 | note_page(m, st, __pgprot(prot), 5); | |
353 | return true; | |
354 | } | |
355 | return false; | |
356 | } | |
357 | #else | |
358 | static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st, | |
359 | void *pt) | |
360 | { | |
361 | return false; | |
362 | } | |
363 | #endif | |
926e5392 | 364 | |
fe770bf0 | 365 | #if PTRS_PER_PMD > 1 |
926e5392 | 366 | |
fdd3d8ce | 367 | static void walk_pmd_level(struct seq_file *m, struct pg_state *st, pud_t addr, unsigned long P) |
926e5392 AV |
368 | { |
369 | int i; | |
04b67022 | 370 | pmd_t *start, *pmd_start; |
da25e628 | 371 | pgprotval_t prot; |
926e5392 | 372 | |
04b67022 | 373 | pmd_start = start = (pmd_t *)pud_page_vaddr(addr); |
926e5392 | 374 | for (i = 0; i < PTRS_PER_PMD; i++) { |
fe770bf0 | 375 | st->current_address = normalize_addr(P + i * PMD_LEVEL_MULT); |
926e5392 | 376 | if (!pmd_none(*start)) { |
da25e628 TK |
377 | if (pmd_large(*start) || !pmd_present(*start)) { |
378 | prot = pmd_flags(*start); | |
45dcd209 | 379 | note_page(m, st, __pgprot(prot), 4); |
04b67022 | 380 | } else if (!kasan_page_table(m, st, pmd_start)) { |
fe770bf0 PA |
381 | walk_pte_level(m, st, *start, |
382 | P + i * PMD_LEVEL_MULT); | |
da25e628 | 383 | } |
926e5392 | 384 | } else |
45dcd209 | 385 | note_page(m, st, __pgprot(0), 4); |
926e5392 AV |
386 | start++; |
387 | } | |
388 | } | |
389 | ||
fe770bf0 PA |
390 | #else |
391 | #define walk_pmd_level(m,s,a,p) walk_pte_level(m,s,__pmd(pud_val(a)),p) | |
392 | #define pud_large(a) pmd_large(__pmd(pud_val(a))) | |
393 | #define pud_none(a) pmd_none(__pmd(pud_val(a))) | |
394 | #endif | |
926e5392 | 395 | |
fe770bf0 PA |
396 | #if PTRS_PER_PUD > 1 |
397 | ||
fdd3d8ce | 398 | static void walk_pud_level(struct seq_file *m, struct pg_state *st, p4d_t addr, unsigned long P) |
926e5392 AV |
399 | { |
400 | int i; | |
04b67022 | 401 | pud_t *start, *pud_start; |
da25e628 | 402 | pgprotval_t prot; |
243b72aa | 403 | pud_t *prev_pud = NULL; |
926e5392 | 404 | |
04b67022 | 405 | pud_start = start = (pud_t *)p4d_page_vaddr(addr); |
926e5392 AV |
406 | |
407 | for (i = 0; i < PTRS_PER_PUD; i++) { | |
fe770bf0 | 408 | st->current_address = normalize_addr(P + i * PUD_LEVEL_MULT); |
04b67022 | 409 | if (!pud_none(*start)) { |
da25e628 TK |
410 | if (pud_large(*start) || !pud_present(*start)) { |
411 | prot = pud_flags(*start); | |
45dcd209 | 412 | note_page(m, st, __pgprot(prot), 3); |
04b67022 | 413 | } else if (!kasan_page_table(m, st, pud_start)) { |
fe770bf0 PA |
414 | walk_pmd_level(m, st, *start, |
415 | P + i * PUD_LEVEL_MULT); | |
da25e628 | 416 | } |
926e5392 | 417 | } else |
45dcd209 | 418 | note_page(m, st, __pgprot(0), 3); |
926e5392 | 419 | |
243b72aa | 420 | prev_pud = start; |
926e5392 AV |
421 | start++; |
422 | } | |
423 | } | |
424 | ||
fe770bf0 | 425 | #else |
fdd3d8ce KS |
426 | #define walk_pud_level(m,s,a,p) walk_pmd_level(m,s,__pud(p4d_val(a)),p) |
427 | #define p4d_large(a) pud_large(__pud(p4d_val(a))) | |
428 | #define p4d_none(a) pud_none(__pud(p4d_val(a))) | |
429 | #endif | |
430 | ||
431 | #if PTRS_PER_P4D > 1 | |
432 | ||
433 | static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr, unsigned long P) | |
434 | { | |
435 | int i; | |
04b67022 | 436 | p4d_t *start, *p4d_start; |
fdd3d8ce KS |
437 | pgprotval_t prot; |
438 | ||
04b67022 | 439 | p4d_start = start = (p4d_t *)pgd_page_vaddr(addr); |
fdd3d8ce KS |
440 | |
441 | for (i = 0; i < PTRS_PER_P4D; i++) { | |
442 | st->current_address = normalize_addr(P + i * P4D_LEVEL_MULT); | |
443 | if (!p4d_none(*start)) { | |
444 | if (p4d_large(*start) || !p4d_present(*start)) { | |
445 | prot = p4d_flags(*start); | |
446 | note_page(m, st, __pgprot(prot), 2); | |
04b67022 | 447 | } else if (!kasan_page_table(m, st, p4d_start)) { |
fdd3d8ce KS |
448 | walk_pud_level(m, st, *start, |
449 | P + i * P4D_LEVEL_MULT); | |
450 | } | |
451 | } else | |
452 | note_page(m, st, __pgprot(0), 2); | |
453 | ||
454 | start++; | |
455 | } | |
456 | } | |
457 | ||
458 | #else | |
459 | #define walk_p4d_level(m,s,a,p) walk_pud_level(m,s,__p4d(pgd_val(a)),p) | |
460 | #define pgd_large(a) p4d_large(__p4d(pgd_val(a))) | |
461 | #define pgd_none(a) p4d_none(__p4d(pgd_val(a))) | |
fe770bf0 PA |
462 | #endif |
463 | ||
f4e342c8 BO |
464 | static inline bool is_hypervisor_range(int idx) |
465 | { | |
b176862f | 466 | #ifdef CONFIG_X86_64 |
f4e342c8 BO |
467 | /* |
468 | * ffff800000000000 - ffff87ffffffffff is reserved for | |
469 | * the hypervisor. | |
470 | */ | |
b176862f BP |
471 | return (idx >= pgd_index(__PAGE_OFFSET) - 16) && |
472 | (idx < pgd_index(__PAGE_OFFSET)); | |
f4e342c8 | 473 | #else |
b176862f | 474 | return false; |
f4e342c8 | 475 | #endif |
b176862f | 476 | } |
f4e342c8 | 477 | |
e1a58320 | 478 | static void ptdump_walk_pgd_level_core(struct seq_file *m, pgd_t *pgd, |
b4bf4f92 | 479 | bool checkwx, bool dmesg) |
926e5392 | 480 | { |
fe770bf0 | 481 | #ifdef CONFIG_X86_64 |
65ade2f8 | 482 | pgd_t *start = (pgd_t *) &init_top_pgt; |
fe770bf0 PA |
483 | #else |
484 | pgd_t *start = swapper_pg_dir; | |
485 | #endif | |
da25e628 | 486 | pgprotval_t prot; |
926e5392 | 487 | int i; |
ef6bea6d | 488 | struct pg_state st = {}; |
926e5392 | 489 | |
ef6bea6d BP |
490 | if (pgd) { |
491 | start = pgd; | |
b4bf4f92 | 492 | st.to_dmesg = dmesg; |
ef6bea6d | 493 | } |
926e5392 | 494 | |
e1a58320 SS |
495 | st.check_wx = checkwx; |
496 | if (checkwx) | |
497 | st.wx_pages = 0; | |
498 | ||
926e5392 | 499 | for (i = 0; i < PTRS_PER_PGD; i++) { |
fe770bf0 | 500 | st.current_address = normalize_addr(i * PGD_LEVEL_MULT); |
f4e342c8 | 501 | if (!pgd_none(*start) && !is_hypervisor_range(i)) { |
da25e628 TK |
502 | if (pgd_large(*start) || !pgd_present(*start)) { |
503 | prot = pgd_flags(*start); | |
fe770bf0 | 504 | note_page(m, &st, __pgprot(prot), 1); |
da25e628 | 505 | } else { |
fdd3d8ce | 506 | walk_p4d_level(m, &st, *start, |
fe770bf0 | 507 | i * PGD_LEVEL_MULT); |
da25e628 | 508 | } |
fe770bf0 | 509 | } else |
926e5392 | 510 | note_page(m, &st, __pgprot(0), 1); |
fe770bf0 | 511 | |
146fbb76 | 512 | cond_resched(); |
926e5392 AV |
513 | start++; |
514 | } | |
fe770bf0 PA |
515 | |
516 | /* Flush out the last page */ | |
517 | st.current_address = normalize_addr(PTRS_PER_PGD*PGD_LEVEL_MULT); | |
518 | note_page(m, &st, __pgprot(0), 0); | |
e1a58320 SS |
519 | if (!checkwx) |
520 | return; | |
521 | if (st.wx_pages) | |
522 | pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n", | |
523 | st.wx_pages); | |
524 | else | |
525 | pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n"); | |
526 | } | |
527 | ||
528 | void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd) | |
529 | { | |
b4bf4f92 TG |
530 | ptdump_walk_pgd_level_core(m, pgd, false, true); |
531 | } | |
532 | ||
a4b51ef6 | 533 | void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user) |
b4bf4f92 | 534 | { |
a4b51ef6 TG |
535 | #ifdef CONFIG_PAGE_TABLE_ISOLATION |
536 | if (user && static_cpu_has(X86_FEATURE_PTI)) | |
537 | pgd = kernel_to_user_pgdp(pgd); | |
538 | #endif | |
b4bf4f92 TG |
539 | ptdump_walk_pgd_level_core(m, pgd, false, false); |
540 | } | |
541 | EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs); | |
542 | ||
543 | static void ptdump_walk_user_pgd_level_checkwx(void) | |
544 | { | |
545 | #ifdef CONFIG_PAGE_TABLE_ISOLATION | |
546 | pgd_t *pgd = (pgd_t *) &init_top_pgt; | |
547 | ||
548 | if (!static_cpu_has(X86_FEATURE_PTI)) | |
549 | return; | |
550 | ||
551 | pr_info("x86/mm: Checking user space page tables\n"); | |
552 | pgd = kernel_to_user_pgdp(pgd); | |
553 | ptdump_walk_pgd_level_core(NULL, pgd, true, false); | |
554 | #endif | |
926e5392 AV |
555 | } |
556 | ||
e1a58320 SS |
557 | void ptdump_walk_pgd_level_checkwx(void) |
558 | { | |
b4bf4f92 TG |
559 | ptdump_walk_pgd_level_core(NULL, NULL, true, false); |
560 | ptdump_walk_user_pgd_level_checkwx(); | |
e1a58320 SS |
561 | } |
562 | ||
8609d1b5 | 563 | static int __init pt_dump_init(void) |
926e5392 | 564 | { |
0483e1fa TG |
565 | /* |
566 | * Various markers are not compile-time constants, so assign them | |
567 | * here. | |
568 | */ | |
569 | #ifdef CONFIG_X86_64 | |
570 | address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET; | |
571 | address_markers[VMALLOC_START_NR].start_address = VMALLOC_START; | |
572 | address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START; | |
573 | #endif | |
fe770bf0 | 574 | #ifdef CONFIG_X86_32 |
92851e2f AS |
575 | address_markers[VMALLOC_START_NR].start_address = VMALLOC_START; |
576 | address_markers[VMALLOC_END_NR].start_address = VMALLOC_END; | |
fe770bf0 | 577 | # ifdef CONFIG_HIGHMEM |
92851e2f | 578 | address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE; |
fe770bf0 | 579 | # endif |
92851e2f | 580 | address_markers[FIXADDR_START_NR].start_address = FIXADDR_START; |
92a0f81d | 581 | address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE; |
fe770bf0 | 582 | #endif |
926e5392 AV |
583 | return 0; |
584 | } | |
926e5392 | 585 | __initcall(pt_dump_init); |