Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_MMU_CONTEXT_H |
2 | #define _ASM_X86_MMU_CONTEXT_H | |
c3c2fee3 JF |
3 | |
4 | #include <asm/desc.h> | |
60063497 | 5 | #include <linux/atomic.h> |
d17d8f9d | 6 | #include <linux/mm_types.h> |
7d06d9c9 | 7 | #include <linux/pkeys.h> |
d17d8f9d DH |
8 | |
9 | #include <trace/events/tlb.h> | |
10 | ||
c3c2fee3 JF |
11 | #include <asm/pgalloc.h> |
12 | #include <asm/tlbflush.h> | |
13 | #include <asm/paravirt.h> | |
fe3d197f | 14 | #include <asm/mpx.h> |
c3c2fee3 | 15 | #ifndef CONFIG_PARAVIRT |
c3c2fee3 JF |
16 | static inline void paravirt_activate_mm(struct mm_struct *prev, |
17 | struct mm_struct *next) | |
18 | { | |
19 | } | |
20 | #endif /* !CONFIG_PARAVIRT */ | |
21 | ||
7911d3f7 | 22 | #ifdef CONFIG_PERF_EVENTS |
a6673429 AL |
23 | extern struct static_key rdpmc_always_available; |
24 | ||
7911d3f7 AL |
25 | static inline void load_mm_cr4(struct mm_struct *mm) |
26 | { | |
a833581e | 27 | if (static_key_false(&rdpmc_always_available) || |
a6673429 | 28 | atomic_read(&mm->context.perf_rdpmc_allowed)) |
7911d3f7 AL |
29 | cr4_set_bits(X86_CR4_PCE); |
30 | else | |
31 | cr4_clear_bits(X86_CR4_PCE); | |
32 | } | |
33 | #else | |
34 | static inline void load_mm_cr4(struct mm_struct *mm) {} | |
35 | #endif | |
36 | ||
a5b9e5a2 | 37 | #ifdef CONFIG_MODIFY_LDT_SYSCALL |
37868fe1 AL |
38 | /* |
39 | * ldt_structs can be allocated, used, and freed, but they are never | |
40 | * modified while live. | |
41 | */ | |
42 | struct ldt_struct { | |
43 | /* | |
44 | * Xen requires page-aligned LDTs with special permissions. This is | |
45 | * needed to prevent us from installing evil descriptors such as | |
46 | * call gates. On native, we could merge the ldt_struct and LDT | |
47 | * allocations, but it's not worth trying to optimize. | |
48 | */ | |
49 | struct desc_struct *entries; | |
bbf79d21 | 50 | unsigned int nr_entries; |
37868fe1 AL |
51 | }; |
52 | ||
a5b9e5a2 AL |
53 | /* |
54 | * Used for LDT copy/destruction. | |
55 | */ | |
39a0526f DH |
56 | int init_new_context_ldt(struct task_struct *tsk, struct mm_struct *mm); |
57 | void destroy_context_ldt(struct mm_struct *mm); | |
a5b9e5a2 | 58 | #else /* CONFIG_MODIFY_LDT_SYSCALL */ |
39a0526f DH |
59 | static inline int init_new_context_ldt(struct task_struct *tsk, |
60 | struct mm_struct *mm) | |
a5b9e5a2 AL |
61 | { |
62 | return 0; | |
63 | } | |
39a0526f | 64 | static inline void destroy_context_ldt(struct mm_struct *mm) {} |
a5b9e5a2 AL |
65 | #endif |
66 | ||
37868fe1 AL |
67 | static inline void load_mm_ldt(struct mm_struct *mm) |
68 | { | |
a5b9e5a2 | 69 | #ifdef CONFIG_MODIFY_LDT_SYSCALL |
37868fe1 AL |
70 | struct ldt_struct *ldt; |
71 | ||
72 | /* lockless_dereference synchronizes with smp_store_release */ | |
73 | ldt = lockless_dereference(mm->context.ldt); | |
74 | ||
75 | /* | |
76 | * Any change to mm->context.ldt is followed by an IPI to all | |
77 | * CPUs with the mm active. The LDT will not be freed until | |
78 | * after the IPI is handled by all such CPUs. This means that, | |
79 | * if the ldt_struct changes before we return, the values we see | |
80 | * will be safe, and the new values will be loaded before we run | |
81 | * any user code. | |
82 | * | |
83 | * NB: don't try to convert this to use RCU without extreme care. | |
84 | * We would still need IRQs off, because we don't want to change | |
85 | * the local LDT after an IPI loaded a newer value than the one | |
86 | * that we can see. | |
87 | */ | |
88 | ||
89 | if (unlikely(ldt)) | |
bbf79d21 | 90 | set_ldt(ldt->entries, ldt->nr_entries); |
37868fe1 AL |
91 | else |
92 | clear_LDT(); | |
a5b9e5a2 AL |
93 | #else |
94 | clear_LDT(); | |
95 | #endif | |
37868fe1 AL |
96 | |
97 | DEBUG_LOCKS_WARN_ON(preemptible()); | |
98 | } | |
99 | ||
6826c8ff BG |
100 | static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) |
101 | { | |
c6ae41e7 AS |
102 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) |
103 | this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY); | |
6826c8ff BG |
104 | } |
105 | ||
39a0526f DH |
106 | static inline int init_new_context(struct task_struct *tsk, |
107 | struct mm_struct *mm) | |
108 | { | |
e8c24d3a DH |
109 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
110 | if (cpu_feature_enabled(X86_FEATURE_OSPKE)) { | |
111 | /* pkey 0 is the default and always allocated */ | |
112 | mm->context.pkey_allocation_map = 0x1; | |
113 | /* -1 means unallocated or invalid */ | |
114 | mm->context.execute_only_pkey = -1; | |
115 | } | |
116 | #endif | |
39a0526f | 117 | init_new_context_ldt(tsk, mm); |
e8c24d3a | 118 | |
39a0526f DH |
119 | return 0; |
120 | } | |
121 | static inline void destroy_context(struct mm_struct *mm) | |
122 | { | |
123 | destroy_context_ldt(mm); | |
124 | } | |
125 | ||
69c0319a AL |
126 | extern void switch_mm(struct mm_struct *prev, struct mm_struct *next, |
127 | struct task_struct *tsk); | |
6826c8ff | 128 | |
078194f8 AL |
129 | extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, |
130 | struct task_struct *tsk); | |
131 | #define switch_mm_irqs_off switch_mm_irqs_off | |
c3c2fee3 JF |
132 | |
133 | #define activate_mm(prev, next) \ | |
134 | do { \ | |
135 | paravirt_activate_mm((prev), (next)); \ | |
136 | switch_mm((prev), (next), NULL); \ | |
137 | } while (0); | |
138 | ||
6826c8ff BG |
139 | #ifdef CONFIG_X86_32 |
140 | #define deactivate_mm(tsk, mm) \ | |
141 | do { \ | |
ccbeed3a | 142 | lazy_load_gs(0); \ |
6826c8ff BG |
143 | } while (0) |
144 | #else | |
145 | #define deactivate_mm(tsk, mm) \ | |
146 | do { \ | |
147 | load_gs_index(0); \ | |
148 | loadsegment(fs, 0); \ | |
149 | } while (0) | |
150 | #endif | |
c3c2fee3 | 151 | |
a1ea1c03 DH |
152 | static inline void arch_dup_mmap(struct mm_struct *oldmm, |
153 | struct mm_struct *mm) | |
154 | { | |
155 | paravirt_arch_dup_mmap(oldmm, mm); | |
156 | } | |
157 | ||
158 | static inline void arch_exit_mmap(struct mm_struct *mm) | |
159 | { | |
160 | paravirt_arch_exit_mmap(mm); | |
161 | } | |
162 | ||
b0e9b09b DH |
163 | #ifdef CONFIG_X86_64 |
164 | static inline bool is_64bit_mm(struct mm_struct *mm) | |
165 | { | |
97f2645f | 166 | return !IS_ENABLED(CONFIG_IA32_EMULATION) || |
b0e9b09b DH |
167 | !(mm->context.ia32_compat == TIF_IA32); |
168 | } | |
169 | #else | |
170 | static inline bool is_64bit_mm(struct mm_struct *mm) | |
171 | { | |
172 | return false; | |
173 | } | |
174 | #endif | |
175 | ||
fe3d197f DH |
176 | static inline void arch_bprm_mm_init(struct mm_struct *mm, |
177 | struct vm_area_struct *vma) | |
178 | { | |
179 | mpx_mm_init(mm); | |
180 | } | |
181 | ||
1de4fa14 DH |
182 | static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma, |
183 | unsigned long start, unsigned long end) | |
184 | { | |
c922228e DH |
185 | /* |
186 | * mpx_notify_unmap() goes and reads a rarely-hot | |
187 | * cacheline in the mm_struct. That can be expensive | |
188 | * enough to be seen in profiles. | |
189 | * | |
190 | * The mpx_notify_unmap() call and its contents have been | |
191 | * observed to affect munmap() performance on hardware | |
192 | * where MPX is not present. | |
193 | * | |
194 | * The unlikely() optimizes for the fast case: no MPX | |
195 | * in the CPU, or no MPX use in the process. Even if | |
196 | * we get this wrong (in the unlikely event that MPX | |
197 | * is widely enabled on some system) the overhead of | |
198 | * MPX itself (reading bounds tables) is expected to | |
199 | * overwhelm the overhead of getting this unlikely() | |
200 | * consistently wrong. | |
201 | */ | |
202 | if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX))) | |
203 | mpx_notify_unmap(mm, vma, start, end); | |
1de4fa14 DH |
204 | } |
205 | ||
7d06d9c9 | 206 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
8f62c883 DH |
207 | static inline int vma_pkey(struct vm_area_struct *vma) |
208 | { | |
8f62c883 DH |
209 | unsigned long vma_pkey_mask = VM_PKEY_BIT0 | VM_PKEY_BIT1 | |
210 | VM_PKEY_BIT2 | VM_PKEY_BIT3; | |
7d06d9c9 DH |
211 | |
212 | return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT; | |
213 | } | |
214 | #else | |
215 | static inline int vma_pkey(struct vm_area_struct *vma) | |
216 | { | |
217 | return 0; | |
8f62c883 | 218 | } |
7d06d9c9 | 219 | #endif |
8f62c883 | 220 | |
33a709b2 DH |
221 | /* |
222 | * We only want to enforce protection keys on the current process | |
223 | * because we effectively have no access to PKRU for other | |
224 | * processes or any way to tell *which * PKRU in a threaded | |
225 | * process we could use. | |
226 | * | |
227 | * So do not enforce things if the VMA is not from the current | |
228 | * mm, or if we are in a kernel thread. | |
229 | */ | |
230 | static inline bool vma_is_foreign(struct vm_area_struct *vma) | |
231 | { | |
232 | if (!current->mm) | |
233 | return true; | |
234 | /* | |
235 | * Should PKRU be enforced on the access to this VMA? If | |
236 | * the VMA is from another process, then PKRU has no | |
237 | * relevance and should not be enforced. | |
238 | */ | |
239 | if (current->mm != vma->vm_mm) | |
240 | return true; | |
241 | ||
242 | return false; | |
243 | } | |
244 | ||
1b2ee126 | 245 | static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, |
d61172b4 | 246 | bool write, bool execute, bool foreign) |
33a709b2 | 247 | { |
d61172b4 DH |
248 | /* pkeys never affect instruction fetches */ |
249 | if (execute) | |
250 | return true; | |
33a709b2 | 251 | /* allow access if the VMA is not one from this process */ |
1b2ee126 | 252 | if (foreign || vma_is_foreign(vma)) |
33a709b2 DH |
253 | return true; |
254 | return __pkru_allows_pkey(vma_pkey(vma), write); | |
255 | } | |
256 | ||
d6e41f11 AL |
257 | |
258 | /* | |
259 | * This can be used from process context to figure out what the value of | |
6c690ee1 | 260 | * CR3 is without needing to do a (slow) __read_cr3(). |
d6e41f11 AL |
261 | * |
262 | * It's intended to be used for code like KVM that sneakily changes CR3 | |
263 | * and needs to restore it. It needs to be used very carefully. | |
264 | */ | |
265 | static inline unsigned long __get_current_cr3_fast(void) | |
266 | { | |
267 | unsigned long cr3 = __pa(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd); | |
268 | ||
269 | /* For now, be very restrictive about when this can be called. */ | |
270 | VM_WARN_ON(in_nmi() || !in_atomic()); | |
271 | ||
6c690ee1 | 272 | VM_BUG_ON(cr3 != __read_cr3()); |
d6e41f11 AL |
273 | return cr3; |
274 | } | |
275 | ||
1965aae3 | 276 | #endif /* _ASM_X86_MMU_CONTEXT_H */ |