Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cb18ef0d KC |
2 | /* |
3 | * Helper routines for building identity mapping page tables. This is | |
4 | * included by both the compressed kernel and the regular kernel. | |
5 | */ | |
cf4fb15b | 6 | |
e4630fdd | 7 | static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, |
cf4fb15b YL |
8 | unsigned long addr, unsigned long end) |
9 | { | |
10 | addr &= PMD_MASK; | |
11 | for (; addr < end; addr += PMD_SIZE) { | |
12 | pmd_t *pmd = pmd_page + pmd_index(addr); | |
13 | ||
e4630fdd RW |
14 | if (pmd_present(*pmd)) |
15 | continue; | |
16 | ||
66aad4fd | 17 | set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag)); |
cf4fb15b YL |
18 | } |
19 | } | |
cb18ef0d | 20 | |
cf4fb15b YL |
21 | static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, |
22 | unsigned long addr, unsigned long end) | |
23 | { | |
24 | unsigned long next; | |
25 | ||
26 | for (; addr < end; addr = next) { | |
27 | pud_t *pud = pud_page + pud_index(addr); | |
28 | pmd_t *pmd; | |
29 | ||
30 | next = (addr & PUD_MASK) + PUD_SIZE; | |
31 | if (next > end) | |
32 | next = end; | |
33 | ||
66aad4fd XP |
34 | if (info->direct_gbpages) { |
35 | pud_t pudval; | |
36 | ||
37 | if (pud_present(*pud)) | |
38 | continue; | |
39 | ||
40 | addr &= PUD_MASK; | |
41 | pudval = __pud((addr - info->offset) | info->page_flag); | |
42 | set_pud(pud, pudval); | |
43 | continue; | |
44 | } | |
45 | ||
cf4fb15b YL |
46 | if (pud_present(*pud)) { |
47 | pmd = pmd_offset(pud, 0); | |
e4630fdd | 48 | ident_pmd_init(info, pmd, addr, next); |
cf4fb15b YL |
49 | continue; |
50 | } | |
51 | pmd = (pmd_t *)info->alloc_pgt_page(info->context); | |
52 | if (!pmd) | |
53 | return -ENOMEM; | |
e4630fdd | 54 | ident_pmd_init(info, pmd, addr, next); |
bba4ed01 | 55 | set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag)); |
cf4fb15b YL |
56 | } |
57 | ||
58 | return 0; | |
59 | } | |
60 | ||
ea3b5e60 KS |
61 | static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, |
62 | unsigned long addr, unsigned long end) | |
63 | { | |
64 | unsigned long next; | |
65 | ||
66 | for (; addr < end; addr = next) { | |
67 | p4d_t *p4d = p4d_page + p4d_index(addr); | |
68 | pud_t *pud; | |
69 | ||
70 | next = (addr & P4D_MASK) + P4D_SIZE; | |
71 | if (next > end) | |
72 | next = end; | |
73 | ||
74 | if (p4d_present(*p4d)) { | |
75 | pud = pud_offset(p4d, 0); | |
76 | ident_pud_init(info, pud, addr, next); | |
77 | continue; | |
78 | } | |
79 | pud = (pud_t *)info->alloc_pgt_page(info->context); | |
80 | if (!pud) | |
81 | return -ENOMEM; | |
82 | ident_pud_init(info, pud, addr, next); | |
bba4ed01 | 83 | set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); |
ea3b5e60 KS |
84 | } |
85 | ||
86 | return 0; | |
87 | } | |
88 | ||
cf4fb15b | 89 | int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, |
e4630fdd | 90 | unsigned long pstart, unsigned long pend) |
cf4fb15b | 91 | { |
e4630fdd RW |
92 | unsigned long addr = pstart + info->offset; |
93 | unsigned long end = pend + info->offset; | |
cf4fb15b YL |
94 | unsigned long next; |
95 | int result; | |
cf4fb15b | 96 | |
bba4ed01 TL |
97 | /* Set the default pagetable flags if not supplied */ |
98 | if (!info->kernpg_flag) | |
99 | info->kernpg_flag = _KERNPG_TABLE; | |
100 | ||
fb43d6cb DH |
101 | /* Filter out unsupported __PAGE_KERNEL_* bits: */ |
102 | info->kernpg_flag &= __default_kernel_pte_mask; | |
103 | ||
cf4fb15b | 104 | for (; addr < end; addr = next) { |
e4630fdd | 105 | pgd_t *pgd = pgd_page + pgd_index(addr); |
ea3b5e60 | 106 | p4d_t *p4d; |
cf4fb15b YL |
107 | |
108 | next = (addr & PGDIR_MASK) + PGDIR_SIZE; | |
109 | if (next > end) | |
110 | next = end; | |
111 | ||
112 | if (pgd_present(*pgd)) { | |
ea3b5e60 KS |
113 | p4d = p4d_offset(pgd, 0); |
114 | result = ident_p4d_init(info, p4d, addr, next); | |
cf4fb15b YL |
115 | if (result) |
116 | return result; | |
117 | continue; | |
118 | } | |
119 | ||
ea3b5e60 KS |
120 | p4d = (p4d_t *)info->alloc_pgt_page(info->context); |
121 | if (!p4d) | |
cf4fb15b | 122 | return -ENOMEM; |
ea3b5e60 | 123 | result = ident_p4d_init(info, p4d, addr, next); |
cf4fb15b YL |
124 | if (result) |
125 | return result; | |
ed7588d5 | 126 | if (pgtable_l5_enabled()) { |
bba4ed01 | 127 | set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag)); |
ea3b5e60 KS |
128 | } else { |
129 | /* | |
130 | * With p4d folded, pgd is equal to p4d. | |
131 | * The pgd entry has to point to the pud page table in this case. | |
132 | */ | |
133 | pud_t *pud = pud_offset(p4d, 0); | |
bba4ed01 | 134 | set_pgd(pgd, __pgd(__pa(pud) | info->kernpg_flag)); |
ea3b5e60 | 135 | } |
cf4fb15b YL |
136 | } |
137 | ||
138 | return 0; | |
139 | } |