Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
8d19c99f JF |
2 | #ifndef _ASM_X86_PGTABLE_DEFS_H |
3 | #define _ASM_X86_PGTABLE_DEFS_H | |
4 | ||
5 | #include <linux/const.h> | |
21729f81 TL |
6 | #include <linux/mem_encrypt.h> |
7 | ||
e43623b4 | 8 | #include <asm/page_types.h> |
8d19c99f | 9 | |
8d19c99f JF |
10 | #define _PAGE_BIT_PRESENT 0 /* is present */ |
11 | #define _PAGE_BIT_RW 1 /* writeable */ | |
12 | #define _PAGE_BIT_USER 2 /* userspace addressable */ | |
13 | #define _PAGE_BIT_PWT 3 /* page write through */ | |
14 | #define _PAGE_BIT_PCD 4 /* page cache disabled */ | |
15 | #define _PAGE_BIT_ACCESSED 5 /* was accessed (raised by CPU) */ | |
16 | #define _PAGE_BIT_DIRTY 6 /* was written to (raised by CPU) */ | |
17 | #define _PAGE_BIT_PSE 7 /* 4 MB (or 2MB) page */ | |
18 | #define _PAGE_BIT_PAT 7 /* on 4KB pages */ | |
19 | #define _PAGE_BIT_GLOBAL 8 /* Global TLB entry PPro+ */ | |
c46a7c81 MG |
20 | #define _PAGE_BIT_SOFTW1 9 /* available for programmer */ |
21 | #define _PAGE_BIT_SOFTW2 10 /* " */ | |
22 | #define _PAGE_BIT_SOFTW3 11 /* " */ | |
8d19c99f | 23 | #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ |
fca4d413 RE |
24 | #define _PAGE_BIT_SOFTW4 57 /* available for programmer */ |
25 | #define _PAGE_BIT_SOFTW5 58 /* available for programmer */ | |
5c1d90f5 DH |
26 | #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ |
27 | #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ | |
28 | #define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */ | |
29 | #define _PAGE_BIT_PKEY_BIT3 62 /* Protection Keys, bit 4/4 */ | |
30 | #define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ | |
31 | ||
c46a7c81 MG |
32 | #define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 |
33 | #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 | |
5a281062 | 34 | #define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */ |
c46a7c81 | 35 | #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ |
5c1d90f5 | 36 | #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 |
8d19c99f | 37 | |
fca4d413 RE |
38 | #ifdef CONFIG_X86_64 |
39 | #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit */ | |
40 | #else | |
41 | /* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */ | |
42 | #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit */ | |
43 | #endif | |
44 | ||
8d19c99f JF |
45 | /* If _PAGE_BIT_PRESENT is clear, we use these: */ |
46 | /* - if the user mapped it with PROT_NONE; pte_present gives true */ | |
47 | #define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL | |
8d19c99f JF |
48 | |
49 | #define _PAGE_PRESENT (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT) | |
50 | #define _PAGE_RW (_AT(pteval_t, 1) << _PAGE_BIT_RW) | |
51 | #define _PAGE_USER (_AT(pteval_t, 1) << _PAGE_BIT_USER) | |
52 | #define _PAGE_PWT (_AT(pteval_t, 1) << _PAGE_BIT_PWT) | |
53 | #define _PAGE_PCD (_AT(pteval_t, 1) << _PAGE_BIT_PCD) | |
54 | #define _PAGE_ACCESSED (_AT(pteval_t, 1) << _PAGE_BIT_ACCESSED) | |
55 | #define _PAGE_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY) | |
56 | #define _PAGE_PSE (_AT(pteval_t, 1) << _PAGE_BIT_PSE) | |
57 | #define _PAGE_GLOBAL (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL) | |
c46a7c81 | 58 | #define _PAGE_SOFTW1 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1) |
f955371c | 59 | #define _PAGE_SOFTW2 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW2) |
6c0df868 | 60 | #define _PAGE_SOFTW3 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW3) |
8d19c99f JF |
61 | #define _PAGE_PAT (_AT(pteval_t, 1) << _PAGE_BIT_PAT) |
62 | #define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE) | |
63 | #define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL) | |
64 | #define _PAGE_CPA_TEST (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST) | |
5c1d90f5 DH |
65 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
66 | #define _PAGE_PKEY_BIT0 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT0) | |
67 | #define _PAGE_PKEY_BIT1 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT1) | |
68 | #define _PAGE_PKEY_BIT2 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT2) | |
69 | #define _PAGE_PKEY_BIT3 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT3) | |
70 | #else | |
71 | #define _PAGE_PKEY_BIT0 (_AT(pteval_t, 0)) | |
72 | #define _PAGE_PKEY_BIT1 (_AT(pteval_t, 0)) | |
73 | #define _PAGE_PKEY_BIT2 (_AT(pteval_t, 0)) | |
74 | #define _PAGE_PKEY_BIT3 (_AT(pteval_t, 0)) | |
75 | #endif | |
8d19c99f | 76 | |
019132ff DH |
77 | #define _PAGE_PKEY_MASK (_PAGE_PKEY_BIT0 | \ |
78 | _PAGE_PKEY_BIT1 | \ | |
79 | _PAGE_PKEY_BIT2 | \ | |
80 | _PAGE_PKEY_BIT3) | |
81 | ||
97e3c602 DH |
82 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
83 | #define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) | |
84 | #else | |
85 | #define _PAGE_KNL_ERRATUM_MASK 0 | |
86 | #endif | |
87 | ||
0f8975ec | 88 | #ifdef CONFIG_MEM_SOFT_DIRTY |
41bb3476 | 89 | #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SOFT_DIRTY) |
0f8975ec PE |
90 | #else |
91 | #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0)) | |
92 | #endif | |
93 | ||
179ef71c CG |
94 | /* |
95 | * Tracking soft dirty bit when a page goes to a swap is tricky. | |
96 | * We need a bit which can be stored in pte _and_ not conflict | |
eee4818b NH |
97 | * with swap entry format. On x86 bits 1-4 are *not* involved |
98 | * into swap entry computation, but bit 7 is used for thp migration, | |
99 | * so we borrow bit 1 for soft dirty tracking. | |
fa0f281c CG |
100 | * |
101 | * Please note that this bit must be treated as swap dirty page | |
eee4818b | 102 | * mark if and only if the PTE/PMD has present bit clear! |
179ef71c CG |
103 | */ |
104 | #ifdef CONFIG_MEM_SOFT_DIRTY | |
eee4818b | 105 | #define _PAGE_SWP_SOFT_DIRTY _PAGE_RW |
179ef71c CG |
106 | #else |
107 | #define _PAGE_SWP_SOFT_DIRTY (_AT(pteval_t, 0)) | |
108 | #endif | |
109 | ||
5a281062 AA |
110 | #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP |
111 | #define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP) | |
112 | #define _PAGE_SWP_UFFD_WP _PAGE_USER | |
113 | #else | |
114 | #define _PAGE_UFFD_WP (_AT(pteval_t, 0)) | |
115 | #define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0)) | |
116 | #endif | |
117 | ||
8d19c99f JF |
118 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
119 | #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) | |
69660fd7 | 120 | #define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) |
c9fe6656 | 121 | #define _PAGE_SOFTW4 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW4) |
8d19c99f JF |
122 | #else |
123 | #define _PAGE_NX (_AT(pteval_t, 0)) | |
69660fd7 | 124 | #define _PAGE_DEVMAP (_AT(pteval_t, 0)) |
c9fe6656 | 125 | #define _PAGE_SOFTW4 (_AT(pteval_t, 0)) |
8d19c99f JF |
126 | #endif |
127 | ||
fca4d413 RE |
128 | /* |
129 | * The hardware requires shadow stack to be Write=0,Dirty=1. However, | |
130 | * there are valid cases where the kernel might create read-only PTEs that | |
131 | * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In | |
132 | * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit, | |
133 | * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have | |
134 | * (Write=0,SavedDirty=1,Dirty=0) set. | |
135 | */ | |
136 | #define _PAGE_SAVED_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SAVED_DIRTY) | |
137 | ||
138 | #define _PAGE_DIRTY_BITS (_PAGE_DIRTY | _PAGE_SAVED_DIRTY) | |
139 | ||
8d19c99f JF |
140 | #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) |
141 | ||
8f62c883 DH |
142 | /* |
143 | * Set of bits not changed in pte_modify. The pte's | |
144 | * protection key is treated like _PAGE_RW, for | |
145 | * instance, and is *not* included in this mask since | |
146 | * pte_modify() does modify it. | |
147 | */ | |
df57721f LT |
148 | #define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ |
149 | _PAGE_SPECIAL | _PAGE_ACCESSED | \ | |
150 | _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \ | |
a0a8d15a | 151 | _PAGE_DEVMAP | _PAGE_CC | _PAGE_UFFD_WP) |
548cb932 JK |
152 | #define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT) |
153 | #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE) | |
8d19c99f | 154 | |
281d4078 JG |
155 | /* |
156 | * The cache modes defined here are used to translate between pure SW usage | |
157 | * and the HW defined cache mode bits and/or PAT entries. | |
158 | * | |
159 | * The resulting bits for PWT, PCD and PAT should be chosen in a way | |
160 | * to have the WB mode at index 0 (all bits clear). This is the default | |
161 | * right now and likely would break too much if changed. | |
162 | */ | |
163 | #ifndef __ASSEMBLY__ | |
164 | enum page_cache_mode { | |
4efb5664 IM |
165 | _PAGE_CACHE_MODE_WB = 0, |
166 | _PAGE_CACHE_MODE_WC = 1, | |
281d4078 | 167 | _PAGE_CACHE_MODE_UC_MINUS = 2, |
4efb5664 IM |
168 | _PAGE_CACHE_MODE_UC = 3, |
169 | _PAGE_CACHE_MODE_WT = 4, | |
170 | _PAGE_CACHE_MODE_WP = 5, | |
171 | ||
172 | _PAGE_CACHE_MODE_NUM = 8 | |
281d4078 JG |
173 | }; |
174 | #endif | |
175 | ||
a0a8d15a | 176 | #define _PAGE_CC (_AT(pteval_t, cc_mask)) |
4efb5664 | 177 | #define _PAGE_ENC (_AT(pteval_t, sme_me_mask)) |
281d4078 | 178 | |
4efb5664 | 179 | #define _PAGE_CACHE_MASK (_PAGE_PWT | _PAGE_PCD | _PAGE_PAT) |
29ac40cb | 180 | #define _PAGE_LARGE_CACHE_MASK (_PAGE_PWT | _PAGE_PCD | _PAGE_PAT_LARGE) |
8d19c99f | 181 | |
4efb5664 IM |
182 | #define _PAGE_NOCACHE (cachemode2protval(_PAGE_CACHE_MODE_UC)) |
183 | #define _PAGE_CACHE_WP (cachemode2protval(_PAGE_CACHE_MODE_WP)) | |
21729f81 | 184 | |
4efb5664 IM |
185 | #define __PP _PAGE_PRESENT |
186 | #define __RW _PAGE_RW | |
187 | #define _USR _PAGE_USER | |
188 | #define ___A _PAGE_ACCESSED | |
189 | #define ___D _PAGE_DIRTY | |
190 | #define ___G _PAGE_GLOBAL | |
191 | #define __NX _PAGE_NX | |
192 | ||
193 | #define _ENC _PAGE_ENC | |
194 | #define __WP _PAGE_CACHE_WP | |
195 | #define __NC _PAGE_NOCACHE | |
196 | #define _PSE _PAGE_PSE | |
197 | ||
198 | #define pgprot_val(x) ((x).pgprot) | |
199 | #define __pgprot(x) ((pgprot_t) { (x) } ) | |
200 | #define __pg(x) __pgprot(x) | |
201 | ||
4efb5664 IM |
202 | #define PAGE_NONE __pg( 0| 0| 0|___A| 0| 0| 0|___G) |
203 | #define PAGE_SHARED __pg(__PP|__RW|_USR|___A|__NX| 0| 0| 0) | |
204 | #define PAGE_SHARED_EXEC __pg(__PP|__RW|_USR|___A| 0| 0| 0| 0) | |
205 | #define PAGE_COPY_NOEXEC __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0) | |
206 | #define PAGE_COPY_EXEC __pg(__PP| 0|_USR|___A| 0| 0| 0| 0) | |
207 | #define PAGE_COPY __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0) | |
208 | #define PAGE_READONLY __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0) | |
209 | #define PAGE_READONLY_EXEC __pg(__PP| 0|_USR|___A| 0| 0| 0| 0) | |
210 | ||
211 | #define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G) | |
212 | #define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G) | |
fca4d413 RE |
213 | |
214 | /* | |
215 | * Page tables needs to have Write=1 in order for any lower PTEs to be | |
216 | * writable. This includes shadow stack memory (Write=0, Dirty=1) | |
217 | */ | |
4efb5664 IM |
218 | #define _KERNPG_TABLE_NOENC (__PP|__RW| 0|___A| 0|___D| 0| 0) |
219 | #define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC) | |
220 | #define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0) | |
221 | #define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC) | |
fca4d413 | 222 | |
f788b717 RE |
223 | #define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX| 0| 0|___G) |
224 | #define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0| 0| 0|___G) | |
225 | #define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G) | |
226 | #define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G) | |
4efb5664 | 227 | #define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC) |
f788b717 | 228 | #define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX| 0| 0|___G) |
4efb5664 IM |
229 | #define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G) |
230 | #define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G) | |
231 | #define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP) | |
232 | ||
233 | ||
234 | #define __PAGE_KERNEL_IO __PAGE_KERNEL | |
235 | #define __PAGE_KERNEL_IO_NOCACHE __PAGE_KERNEL_NOCACHE | |
21729f81 | 236 | |
21729f81 | 237 | |
4efb5664 | 238 | #ifndef __ASSEMBLY__ |
f88a68fa | 239 | |
4efb5664 IM |
240 | #define __PAGE_KERNEL_ENC (__PAGE_KERNEL | _ENC) |
241 | #define __PAGE_KERNEL_ENC_WP (__PAGE_KERNEL_WP | _ENC) | |
242 | #define __PAGE_KERNEL_NOENC (__PAGE_KERNEL | 0) | |
243 | #define __PAGE_KERNEL_NOENC_WP (__PAGE_KERNEL_WP | 0) | |
f88a68fa | 244 | |
4efb5664 | 245 | #define __pgprot_mask(x) __pgprot((x) & __default_kernel_pte_mask) |
8a57f484 | 246 | |
4efb5664 IM |
247 | #define PAGE_KERNEL __pgprot_mask(__PAGE_KERNEL | _ENC) |
248 | #define PAGE_KERNEL_NOENC __pgprot_mask(__PAGE_KERNEL | 0) | |
249 | #define PAGE_KERNEL_RO __pgprot_mask(__PAGE_KERNEL_RO | _ENC) | |
250 | #define PAGE_KERNEL_EXEC __pgprot_mask(__PAGE_KERNEL_EXEC | _ENC) | |
251 | #define PAGE_KERNEL_EXEC_NOENC __pgprot_mask(__PAGE_KERNEL_EXEC | 0) | |
800e26b8 | 252 | #define PAGE_KERNEL_ROX __pgprot_mask(__PAGE_KERNEL_ROX | _ENC) |
4efb5664 IM |
253 | #define PAGE_KERNEL_NOCACHE __pgprot_mask(__PAGE_KERNEL_NOCACHE | _ENC) |
254 | #define PAGE_KERNEL_LARGE __pgprot_mask(__PAGE_KERNEL_LARGE | _ENC) | |
255 | #define PAGE_KERNEL_LARGE_EXEC __pgprot_mask(__PAGE_KERNEL_LARGE_EXEC | _ENC) | |
256 | #define PAGE_KERNEL_VVAR __pgprot_mask(__PAGE_KERNEL_VVAR | _ENC) | |
8a57f484 | 257 | |
4efb5664 IM |
258 | #define PAGE_KERNEL_IO __pgprot_mask(__PAGE_KERNEL_IO) |
259 | #define PAGE_KERNEL_IO_NOCACHE __pgprot_mask(__PAGE_KERNEL_IO_NOCACHE) | |
21729f81 TL |
260 | |
261 | #endif /* __ASSEMBLY__ */ | |
8d19c99f | 262 | |
8d19c99f JF |
263 | /* |
264 | * early identity mapping pte attrib macros. | |
265 | */ | |
266 | #ifdef CONFIG_X86_64 | |
267 | #define __PAGE_KERNEL_IDENT_LARGE_EXEC __PAGE_KERNEL_LARGE_EXEC | |
268 | #else | |
8d19c99f | 269 | #define PTE_IDENT_ATTR 0x003 /* PRESENT+RW */ |
7dda0387 | 270 | #define PDE_IDENT_ATTR 0x063 /* PRESENT+RW+DIRTY+ACCESSED */ |
8d19c99f JF |
271 | #define PGD_IDENT_ATTR 0x001 /* PRESENT (no other attributes) */ |
272 | #endif | |
273 | ||
54321d94 | 274 | #ifdef CONFIG_X86_32 |
a1ce3928 | 275 | # include <asm/pgtable_32_types.h> |
54321d94 | 276 | #else |
a1ce3928 | 277 | # include <asm/pgtable_64_types.h> |
54321d94 JF |
278 | #endif |
279 | ||
8d19c99f JF |
280 | #ifndef __ASSEMBLY__ |
281 | ||
54321d94 JF |
282 | #include <linux/types.h> |
283 | ||
4be4c1fb | 284 | /* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */ |
9b3651cb JF |
285 | #define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK) |
286 | ||
8f62c883 DH |
287 | /* |
288 | * Extracts the flags from a (pte|pmd|pud|pgd)val_t | |
289 | * This includes the protection key value. | |
290 | */ | |
9b3651cb JF |
291 | #define PTE_FLAGS_MASK (~PTE_PFN_MASK) |
292 | ||
54321d94 JF |
293 | typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; |
294 | ||
295 | typedef struct { pgdval_t pgd; } pgd_t; | |
296 | ||
cca98e9f CH |
297 | static inline pgprot_t pgprot_nx(pgprot_t prot) |
298 | { | |
299 | return __pgprot(pgprot_val(prot) | _PAGE_NX); | |
300 | } | |
301 | #define pgprot_nx pgprot_nx | |
302 | ||
6c0df868 JR |
303 | #ifdef CONFIG_X86_PAE |
304 | ||
305 | /* | |
306 | * PHYSICAL_PAGE_MASK might be non-constant when SME is compiled in, so we can't | |
307 | * use it here. | |
308 | */ | |
309 | ||
310 | #define PGD_PAE_PAGE_MASK ((signed long)PAGE_MASK) | |
311 | #define PGD_PAE_PHYS_MASK (((1ULL << __PHYSICAL_MASK_SHIFT)-1) & PGD_PAE_PAGE_MASK) | |
312 | ||
313 | /* | |
314 | * PAE allows Base Address, P, PWT, PCD and AVL bits to be set in PGD entries. | |
315 | * All other bits are Reserved MBZ | |
316 | */ | |
317 | #define PGD_ALLOWED_BITS (PGD_PAE_PHYS_MASK | _PAGE_PRESENT | \ | |
318 | _PAGE_PWT | _PAGE_PCD | \ | |
319 | _PAGE_SOFTW1 | _PAGE_SOFTW2 | _PAGE_SOFTW3) | |
320 | ||
321 | #else | |
322 | /* No need to mask any bits for !PAE */ | |
323 | #define PGD_ALLOWED_BITS (~0ULL) | |
324 | #endif | |
325 | ||
54321d94 JF |
326 | static inline pgd_t native_make_pgd(pgdval_t val) |
327 | { | |
6c0df868 | 328 | return (pgd_t) { val & PGD_ALLOWED_BITS }; |
54321d94 JF |
329 | } |
330 | ||
331 | static inline pgdval_t native_pgd_val(pgd_t pgd) | |
332 | { | |
6c0df868 | 333 | return pgd.pgd & PGD_ALLOWED_BITS; |
54321d94 JF |
334 | } |
335 | ||
336 | static inline pgdval_t pgd_flags(pgd_t pgd) | |
337 | { | |
338 | return native_pgd_val(pgd) & PTE_FLAGS_MASK; | |
339 | } | |
340 | ||
fe1e8c3e | 341 | #if CONFIG_PGTABLE_LEVELS > 4 |
b8504058 | 342 | typedef struct { p4dval_t p4d; } p4d_t; |
fe1e8c3e | 343 | |
b8504058 KS |
344 | static inline p4d_t native_make_p4d(pudval_t val) |
345 | { | |
346 | return (p4d_t) { val }; | |
347 | } | |
fe1e8c3e | 348 | |
b8504058 KS |
349 | static inline p4dval_t native_p4d_val(p4d_t p4d) |
350 | { | |
351 | return p4d.p4d; | |
352 | } | |
fe1e8c3e | 353 | #else |
f2a6a705 | 354 | #include <asm-generic/pgtable-nop4d.h> |
9849a569 | 355 | |
db516997 TL |
356 | static inline p4d_t native_make_p4d(pudval_t val) |
357 | { | |
358 | return (p4d_t) { .pgd = native_make_pgd((pgdval_t)val) }; | |
359 | } | |
360 | ||
fe1e8c3e KS |
361 | static inline p4dval_t native_p4d_val(p4d_t p4d) |
362 | { | |
f2a6a705 | 363 | return native_pgd_val(p4d.pgd); |
fe1e8c3e KS |
364 | } |
365 | #endif | |
366 | ||
367 | #if CONFIG_PGTABLE_LEVELS > 3 | |
54321d94 JF |
368 | typedef struct { pudval_t pud; } pud_t; |
369 | ||
370 | static inline pud_t native_make_pud(pmdval_t val) | |
371 | { | |
372 | return (pud_t) { val }; | |
373 | } | |
374 | ||
375 | static inline pudval_t native_pud_val(pud_t pud) | |
376 | { | |
377 | return pud.pud; | |
378 | } | |
379 | #else | |
380 | #include <asm-generic/pgtable-nopud.h> | |
381 | ||
842cef91 JB |
382 | static inline pud_t native_make_pud(pudval_t val) |
383 | { | |
384 | return (pud_t) { .p4d.pgd = native_make_pgd(val) }; | |
385 | } | |
386 | ||
54321d94 JF |
387 | static inline pudval_t native_pud_val(pud_t pud) |
388 | { | |
f2a6a705 | 389 | return native_pgd_val(pud.p4d.pgd); |
54321d94 JF |
390 | } |
391 | #endif | |
392 | ||
98233368 | 393 | #if CONFIG_PGTABLE_LEVELS > 2 |
54321d94 JF |
394 | static inline pmd_t native_make_pmd(pmdval_t val) |
395 | { | |
fbfdec99 | 396 | return (pmd_t) { .pmd = val }; |
54321d94 JF |
397 | } |
398 | ||
399 | static inline pmdval_t native_pmd_val(pmd_t pmd) | |
400 | { | |
401 | return pmd.pmd; | |
402 | } | |
403 | #else | |
404 | #include <asm-generic/pgtable-nopmd.h> | |
405 | ||
842cef91 JB |
406 | static inline pmd_t native_make_pmd(pmdval_t val) |
407 | { | |
408 | return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; | |
409 | } | |
410 | ||
54321d94 JF |
411 | static inline pmdval_t native_pmd_val(pmd_t pmd) |
412 | { | |
f2a6a705 | 413 | return native_pgd_val(pmd.pud.p4d.pgd); |
54321d94 JF |
414 | } |
415 | #endif | |
416 | ||
fe1e8c3e KS |
417 | static inline p4dval_t p4d_pfn_mask(p4d_t p4d) |
418 | { | |
419 | /* No 512 GiB huge pages yet */ | |
420 | return PTE_PFN_MASK; | |
421 | } | |
422 | ||
423 | static inline p4dval_t p4d_flags_mask(p4d_t p4d) | |
424 | { | |
425 | return ~p4d_pfn_mask(p4d); | |
426 | } | |
427 | ||
428 | static inline p4dval_t p4d_flags(p4d_t p4d) | |
429 | { | |
430 | return native_p4d_val(p4d) & p4d_flags_mask(p4d); | |
431 | } | |
432 | ||
4be4c1fb TK |
433 | static inline pudval_t pud_pfn_mask(pud_t pud) |
434 | { | |
435 | if (native_pud_val(pud) & _PAGE_PSE) | |
70f15287 | 436 | return PHYSICAL_PUD_PAGE_MASK; |
4be4c1fb TK |
437 | else |
438 | return PTE_PFN_MASK; | |
439 | } | |
440 | ||
441 | static inline pudval_t pud_flags_mask(pud_t pud) | |
442 | { | |
70f15287 | 443 | return ~pud_pfn_mask(pud); |
4be4c1fb TK |
444 | } |
445 | ||
54321d94 JF |
446 | static inline pudval_t pud_flags(pud_t pud) |
447 | { | |
f70abb0f | 448 | return native_pud_val(pud) & pud_flags_mask(pud); |
54321d94 JF |
449 | } |
450 | ||
4be4c1fb TK |
451 | static inline pmdval_t pmd_pfn_mask(pmd_t pmd) |
452 | { | |
453 | if (native_pmd_val(pmd) & _PAGE_PSE) | |
70f15287 | 454 | return PHYSICAL_PMD_PAGE_MASK; |
4be4c1fb TK |
455 | else |
456 | return PTE_PFN_MASK; | |
457 | } | |
458 | ||
459 | static inline pmdval_t pmd_flags_mask(pmd_t pmd) | |
460 | { | |
70f15287 | 461 | return ~pmd_pfn_mask(pmd); |
4be4c1fb TK |
462 | } |
463 | ||
54321d94 JF |
464 | static inline pmdval_t pmd_flags(pmd_t pmd) |
465 | { | |
f70abb0f | 466 | return native_pmd_val(pmd) & pmd_flags_mask(pmd); |
54321d94 JF |
467 | } |
468 | ||
469 | static inline pte_t native_make_pte(pteval_t val) | |
470 | { | |
471 | return (pte_t) { .pte = val }; | |
472 | } | |
473 | ||
474 | static inline pteval_t native_pte_val(pte_t pte) | |
475 | { | |
476 | return pte.pte; | |
477 | } | |
478 | ||
479 | static inline pteval_t pte_flags(pte_t pte) | |
480 | { | |
481 | return native_pte_val(pte) & PTE_FLAGS_MASK; | |
482 | } | |
483 | ||
281d4078 JG |
484 | #define __pte2cm_idx(cb) \ |
485 | ((((cb) >> (_PAGE_BIT_PAT - 2)) & 4) | \ | |
486 | (((cb) >> (_PAGE_BIT_PCD - 1)) & 2) | \ | |
487 | (((cb) >> _PAGE_BIT_PWT) & 1)) | |
bd809af1 JG |
488 | #define __cm_idx2pte(i) \ |
489 | ((((i) & 4) << (_PAGE_BIT_PAT - 2)) | \ | |
490 | (((i) & 2) << (_PAGE_BIT_PCD - 1)) | \ | |
491 | (((i) & 1) << _PAGE_BIT_PWT)) | |
281d4078 | 492 | |
de17a378 CH |
493 | unsigned long cachemode2protval(enum page_cache_mode pcm); |
494 | ||
325518e9 | 495 | static inline pgprotval_t protval_4k_2_large(pgprotval_t val) |
281d4078 | 496 | { |
d0735693 | 497 | return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | |
281d4078 | 498 | ((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); |
281d4078 | 499 | } |
d0735693 | 500 | static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot) |
281d4078 | 501 | { |
d0735693 | 502 | return __pgprot(protval_4k_2_large(pgprot_val(pgprot))); |
281d4078 | 503 | } |
325518e9 | 504 | static inline pgprotval_t protval_large_2_4k(pgprotval_t val) |
281d4078 | 505 | { |
d0735693 CH |
506 | return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | |
507 | ((val & _PAGE_PAT_LARGE) >> | |
508 | (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); | |
281d4078 JG |
509 | } |
510 | static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot) | |
511 | { | |
d0735693 | 512 | return __pgprot(protval_large_2_4k(pgprot_val(pgprot))); |
281d4078 JG |
513 | } |
514 | ||
54321d94 JF |
515 | |
516 | typedef struct page *pgtable_t; | |
517 | ||
8d19c99f | 518 | extern pteval_t __supported_pte_mask; |
8a57f484 | 519 | extern pteval_t __default_kernel_pte_mask; |
c44c9ec0 | 520 | extern void set_nx(void); |
54321d94 | 521 | extern int nx_enabled; |
8d19c99f JF |
522 | |
523 | #define pgprot_writecombine pgprot_writecombine | |
524 | extern pgprot_t pgprot_writecombine(pgprot_t prot); | |
525 | ||
d1b4bfbf TK |
526 | #define pgprot_writethrough pgprot_writethrough |
527 | extern pgprot_t pgprot_writethrough(pgprot_t prot); | |
528 | ||
8d19c99f JF |
529 | /* Indicate that x86 has its own track and untrack pfn vma functions */ |
530 | #define __HAVE_PFNMAP_TRACKING | |
531 | ||
532 | #define __HAVE_PHYS_MEM_ACCESS_PROT | |
533 | struct file; | |
534 | pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, | |
535 | unsigned long size, pgprot_t vma_prot); | |
8d19c99f JF |
536 | |
537 | /* Install a pte for a particular vaddr in kernel space. */ | |
538 | void set_pte_vaddr(unsigned long vaddr, pte_t pte); | |
539 | ||
540 | #ifdef CONFIG_X86_32 | |
7737b215 | 541 | extern void native_pagetable_init(void); |
8d19c99f | 542 | #else |
843b8ed2 | 543 | #define native_pagetable_init paging_init |
8d19c99f JF |
544 | #endif |
545 | ||
4cbeb51b | 546 | enum pg_level { |
8d19c99f JF |
547 | PG_LEVEL_NONE, |
548 | PG_LEVEL_4K, | |
549 | PG_LEVEL_2M, | |
550 | PG_LEVEL_1G, | |
fe1e8c3e | 551 | PG_LEVEL_512G, |
8d19c99f JF |
552 | PG_LEVEL_NUM |
553 | }; | |
554 | ||
555 | #ifdef CONFIG_PROC_FS | |
556 | extern void update_page_count(int level, unsigned long pages); | |
557 | #else | |
558 | static inline void update_page_count(int level, unsigned long pages) { } | |
559 | #endif | |
560 | ||
561 | /* | |
562 | * Helper function that returns the kernel pagetable entry controlling | |
563 | * the virtual address 'address'. NULL means no pagetable entry present. | |
564 | * NOTE: the return type is pte_t but if the pmd is PSE then we return it | |
565 | * as a pte too. | |
566 | */ | |
567 | extern pte_t *lookup_address(unsigned long address, unsigned int *level); | |
426e34cc MF |
568 | extern pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address, |
569 | unsigned int *level); | |
792230c3 | 570 | extern pmd_t *lookup_pmd_address(unsigned long address); |
d7656534 | 571 | extern phys_addr_t slow_virt_to_phys(void *__address); |
7e0dabd3 SPP |
572 | extern int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, |
573 | unsigned long address, | |
574 | unsigned numpages, | |
575 | unsigned long page_flags); | |
576 | extern int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address, | |
577 | unsigned long numpages); | |
8d19c99f JF |
578 | #endif /* !__ASSEMBLY__ */ |
579 | ||
580 | #endif /* _ASM_X86_PGTABLE_DEFS_H */ |