Commit | Line | Data |
---|---|---|
fd534e9b | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
d3561b7f RR |
2 | /* Paravirtualization interfaces |
3 | Copyright (C) 2006 Rusty Russell IBM Corporation | |
4 | ||
b1df07bd GOC |
5 | |
6 | 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc | |
d3561b7f | 7 | */ |
b1df07bd | 8 | |
d3561b7f | 9 | #include <linux/errno.h> |
186f4360 PG |
10 | #include <linux/init.h> |
11 | #include <linux/export.h> | |
d3561b7f RR |
12 | #include <linux/efi.h> |
13 | #include <linux/bcd.h> | |
ce6234b5 | 14 | #include <linux/highmem.h> |
376e2424 | 15 | #include <linux/kprobes.h> |
65fddcfc | 16 | #include <linux/pgtable.h> |
a0e2bf7c | 17 | #include <linux/static_call.h> |
d3561b7f RR |
18 | |
19 | #include <asm/bug.h> | |
20 | #include <asm/paravirt.h> | |
50af5ead | 21 | #include <asm/debugreg.h> |
d3561b7f RR |
22 | #include <asm/desc.h> |
23 | #include <asm/setup.h> | |
d3561b7f | 24 | #include <asm/time.h> |
eba0045f | 25 | #include <asm/pgalloc.h> |
d3561b7f RR |
26 | #include <asm/irq.h> |
27 | #include <asm/delay.h> | |
13623d79 RR |
28 | #include <asm/fixmap.h> |
29 | #include <asm/apic.h> | |
da181a8b | 30 | #include <asm/tlbflush.h> |
6cb9a835 | 31 | #include <asm/timer.h> |
f05e798a | 32 | #include <asm/special_insns.h> |
48a8b97c | 33 | #include <asm/tlb.h> |
99bcd4a6 | 34 | #include <asm/io_bitmap.h> |
ae53fa18 | 35 | #include <asm/gsseg.h> |
d3561b7f | 36 | |
fc57a7c6 AL |
37 | /* |
38 | * nop stub, which must not clobber anything *including the stack* to | |
39 | * avoid confusing the entry prologues. | |
40 | */ | |
f1a033cc | 41 | DEFINE_PARAVIRT_ASM(_paravirt_nop, "", .entry.text); |
d3561b7f | 42 | |
e453f872 | 43 | /* stub always returning 0. */ |
f1a033cc | 44 | DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax", .entry.text); |
e453f872 | 45 | |
6f30c1ac | 46 | void __init default_banner(void) |
d3561b7f RR |
47 | { |
48 | printk(KERN_INFO "Booting paravirtualized kernel on %s\n", | |
93b1eab3 | 49 | pv_info.name); |
d3561b7f RR |
50 | } |
51 | ||
93b1eab3 | 52 | /* Undefined instruction for dealing with missing ops pointers. */ |
e453f872 | 53 | noinstr void paravirt_BUG(void) |
fafe5e74 JG |
54 | { |
55 | BUG(); | |
56 | } | |
139ec7c4 | 57 | |
1fc654cf | 58 | static unsigned paravirt_patch_call(void *insn_buff, const void *target, |
abc745f8 | 59 | unsigned long addr, unsigned len) |
63f70270 | 60 | { |
ba27d1a8 PZ |
61 | __text_gen_insn(insn_buff, CALL_INSN_OPCODE, |
62 | (void *)addr, target, CALL_INSN_SIZE); | |
63 | return CALL_INSN_SIZE; | |
63f70270 JF |
64 | } |
65 | ||
9bad5658 | 66 | #ifdef CONFIG_PARAVIRT_XXL |
11af36cb JG |
67 | DEFINE_PARAVIRT_ASM(_paravirt_ident_64, "mov %rdi, %rax", .text); |
68 | DEFINE_PARAVIRT_ASM(pv_native_save_fl, "pushf; pop %rax", .noinstr.text); | |
69 | DEFINE_PARAVIRT_ASM(pv_native_irq_disable, "cli", .noinstr.text); | |
70 | DEFINE_PARAVIRT_ASM(pv_native_irq_enable, "sti", .noinstr.text); | |
71 | DEFINE_PARAVIRT_ASM(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text); | |
9bad5658 | 72 | #endif |
63f70270 | 73 | |
9043442b JG |
74 | DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key); |
75 | ||
76 | void __init native_pv_lock_init(void) | |
77 | { | |
67e87d43 | 78 | if (!boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
9043442b JG |
79 | static_branch_disable(&virt_spin_lock_key); |
80 | } | |
81 | ||
054ac8ad JG |
82 | unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, |
83 | unsigned int len) | |
63f70270 | 84 | { |
5c83511b JG |
85 | /* |
86 | * Neat trick to map patch type back to the call within the | |
87 | * corresponding structure. | |
88 | */ | |
89 | void *opfunc = *((void **)&pv_ops + type); | |
63f70270 JF |
90 | unsigned ret; |
91 | ||
92 | if (opfunc == NULL) | |
fafe5e74 JG |
93 | /* If there's no function, patch it with paravirt_BUG() */ |
94 | ret = paravirt_patch_call(insn_buff, paravirt_BUG, addr, len); | |
41edafdb | 95 | else if (opfunc == _paravirt_nop) |
79f1d836 | 96 | ret = 0; |
63f70270 | 97 | else |
abc745f8 | 98 | /* Otherwise call the function. */ |
1fc654cf | 99 | ret = paravirt_patch_call(insn_buff, opfunc, addr, len); |
63f70270 JF |
100 | |
101 | return ret; | |
102 | } | |
103 | ||
c5905afb IM |
104 | struct static_key paravirt_steal_enabled; |
105 | struct static_key paravirt_steal_rq_enabled; | |
3c404b57 GC |
106 | |
107 | static u64 native_steal_clock(int cpu) | |
108 | { | |
109 | return 0; | |
110 | } | |
111 | ||
a0e2bf7c JG |
112 | DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); |
113 | DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); | |
114 | ||
115 | void paravirt_set_sched_clock(u64 (*func)(void)) | |
116 | { | |
117 | static_call_update(pv_sched_clock, func); | |
118 | } | |
119 | ||
d3561b7f | 120 | /* These are in entry.S */ |
d572929c JF |
121 | static struct resource reserve_ioports = { |
122 | .start = 0, | |
123 | .end = IO_SPACE_LIMIT, | |
124 | .name = "paravirt-ioport", | |
125 | .flags = IORESOURCE_IO | IORESOURCE_BUSY, | |
126 | }; | |
127 | ||
d572929c JF |
128 | /* |
129 | * Reserve the whole legacy IO space to prevent any legacy drivers | |
130 | * from wasting time probing for their hardware. This is a fairly | |
131 | * brute-force approach to disabling all non-virtual drivers. | |
132 | * | |
133 | * Note that this must be called very early to have any effect. | |
134 | */ | |
135 | int paravirt_disable_iospace(void) | |
136 | { | |
f7743fe6 | 137 | return request_resource(&ioport_resource, &reserve_ioports); |
d572929c JF |
138 | } |
139 | ||
8965c1c0 JF |
140 | static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; |
141 | ||
142 | static inline void enter_lazy(enum paravirt_lazy_mode mode) | |
143 | { | |
c6ae41e7 | 144 | BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE); |
8965c1c0 | 145 | |
c6ae41e7 | 146 | this_cpu_write(paravirt_lazy_mode, mode); |
8965c1c0 JF |
147 | } |
148 | ||
b407fc57 | 149 | static void leave_lazy(enum paravirt_lazy_mode mode) |
8965c1c0 | 150 | { |
c6ae41e7 | 151 | BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode); |
8965c1c0 | 152 | |
c6ae41e7 | 153 | this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE); |
8965c1c0 JF |
154 | } |
155 | ||
156 | void paravirt_enter_lazy_mmu(void) | |
157 | { | |
158 | enter_lazy(PARAVIRT_LAZY_MMU); | |
159 | } | |
160 | ||
161 | void paravirt_leave_lazy_mmu(void) | |
162 | { | |
b407fc57 | 163 | leave_lazy(PARAVIRT_LAZY_MMU); |
8965c1c0 JF |
164 | } |
165 | ||
511ba86e BO |
166 | void paravirt_flush_lazy_mmu(void) |
167 | { | |
168 | preempt_disable(); | |
169 | ||
170 | if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) { | |
171 | arch_leave_lazy_mmu_mode(); | |
172 | arch_enter_lazy_mmu_mode(); | |
173 | } | |
174 | ||
175 | preempt_enable(); | |
176 | } | |
177 | ||
9bad5658 | 178 | #ifdef CONFIG_PARAVIRT_XXL |
224101ed | 179 | void paravirt_start_context_switch(struct task_struct *prev) |
8965c1c0 | 180 | { |
2829b449 JF |
181 | BUG_ON(preemptible()); |
182 | ||
c6ae41e7 | 183 | if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) { |
b407fc57 | 184 | arch_leave_lazy_mmu_mode(); |
224101ed | 185 | set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); |
b407fc57 | 186 | } |
8965c1c0 JF |
187 | enter_lazy(PARAVIRT_LAZY_CPU); |
188 | } | |
189 | ||
224101ed | 190 | void paravirt_end_context_switch(struct task_struct *next) |
8965c1c0 | 191 | { |
2829b449 JF |
192 | BUG_ON(preemptible()); |
193 | ||
b407fc57 JF |
194 | leave_lazy(PARAVIRT_LAZY_CPU); |
195 | ||
224101ed | 196 | if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) |
b407fc57 | 197 | arch_enter_lazy_mmu_mode(); |
8965c1c0 | 198 | } |
0a53c9ac | 199 | |
209cfd0c PZ |
200 | static noinstr void pv_native_write_cr2(unsigned long val) |
201 | { | |
202 | native_write_cr2(val); | |
203 | } | |
f4afb713 PZ |
204 | |
205 | static noinstr unsigned long pv_native_get_debugreg(int regno) | |
206 | { | |
207 | return native_get_debugreg(regno); | |
208 | } | |
7361fac0 PZ |
209 | |
210 | static noinstr void pv_native_set_debugreg(int regno, unsigned long val) | |
211 | { | |
212 | native_set_debugreg(regno, val); | |
213 | } | |
d7bfc7d5 | 214 | |
10a09940 PZ |
215 | noinstr void pv_native_wbinvd(void) |
216 | { | |
217 | native_wbinvd(); | |
218 | } | |
219 | ||
10a09940 PZ |
220 | static noinstr void pv_native_safe_halt(void) |
221 | { | |
222 | native_safe_halt(); | |
223 | } | |
9bad5658 | 224 | #endif |
8965c1c0 JF |
225 | |
226 | enum paravirt_lazy_mode paravirt_get_lazy_mode(void) | |
227 | { | |
b8bcfe99 JF |
228 | if (in_interrupt()) |
229 | return PARAVIRT_LAZY_NONE; | |
230 | ||
c6ae41e7 | 231 | return this_cpu_read(paravirt_lazy_mode); |
8965c1c0 JF |
232 | } |
233 | ||
93b1eab3 | 234 | struct pv_info pv_info = { |
d3561b7f | 235 | .name = "bare hardware", |
40181646 | 236 | #ifdef CONFIG_PARAVIRT_XXL |
318f5a2a AL |
237 | .extra_user_64bit_cs = __USER_CS, |
238 | #endif | |
93b1eab3 | 239 | }; |
d3561b7f | 240 | |
41edafdb | 241 | /* 64-bit pagetable entries */ |
da5de7c2 | 242 | #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) |
41edafdb | 243 | |
5c83511b | 244 | struct paravirt_patch_template pv_ops = { |
5c83511b | 245 | /* Cpu ops. */ |
9bad5658 | 246 | .cpu.io_delay = native_io_delay, |
da181a8b | 247 | |
9bad5658 | 248 | #ifdef CONFIG_PARAVIRT_XXL |
5c83511b | 249 | .cpu.cpuid = native_cpuid, |
f4afb713 | 250 | .cpu.get_debugreg = pv_native_get_debugreg, |
7361fac0 | 251 | .cpu.set_debugreg = pv_native_set_debugreg, |
5c83511b JG |
252 | .cpu.read_cr0 = native_read_cr0, |
253 | .cpu.write_cr0 = native_write_cr0, | |
254 | .cpu.write_cr4 = native_write_cr4, | |
10a09940 | 255 | .cpu.wbinvd = pv_native_wbinvd, |
5c83511b JG |
256 | .cpu.read_msr = native_read_msr, |
257 | .cpu.write_msr = native_write_msr, | |
258 | .cpu.read_msr_safe = native_read_msr_safe, | |
259 | .cpu.write_msr_safe = native_write_msr_safe, | |
260 | .cpu.read_pmc = native_read_pmc, | |
261 | .cpu.load_tr_desc = native_load_tr_desc, | |
262 | .cpu.set_ldt = native_set_ldt, | |
263 | .cpu.load_gdt = native_load_gdt, | |
264 | .cpu.load_idt = native_load_idt, | |
265 | .cpu.store_tr = native_store_tr, | |
266 | .cpu.load_tls = native_load_tls, | |
5c83511b | 267 | .cpu.load_gs_index = native_load_gs_index, |
5c83511b JG |
268 | .cpu.write_ldt_entry = native_write_ldt_entry, |
269 | .cpu.write_gdt_entry = native_write_gdt_entry, | |
270 | .cpu.write_idt_entry = native_write_idt_entry, | |
eba0045f | 271 | |
5c83511b JG |
272 | .cpu.alloc_ldt = paravirt_nop, |
273 | .cpu.free_ldt = paravirt_nop, | |
c119ecce | 274 | |
5c83511b | 275 | .cpu.load_sp0 = native_load_sp0, |
3dc494e8 | 276 | |
99bcd4a6 | 277 | #ifdef CONFIG_X86_IOPL_IOPERM |
cadfad87 AL |
278 | .cpu.invalidate_io_bitmap = native_tss_invalidate_io_bitmap, |
279 | .cpu.update_io_bitmap = native_tss_update_io_bitmap, | |
99bcd4a6 JG |
280 | #endif |
281 | ||
5c83511b JG |
282 | .cpu.start_context_switch = paravirt_nop, |
283 | .cpu.end_context_switch = paravirt_nop, | |
284 | ||
285 | /* Irq ops. */ | |
11af36cb | 286 | .irq.save_fl = __PV_IS_CALLEE_SAVE(pv_native_save_fl), |
09c41307 | 287 | .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable), |
d7bfc7d5 | 288 | .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable), |
10a09940 | 289 | .irq.safe_halt = pv_native_safe_halt, |
5c83511b | 290 | .irq.halt = native_halt, |
6da63eb2 | 291 | #endif /* CONFIG_PARAVIRT_XXL */ |
5c83511b JG |
292 | |
293 | /* Mmu ops. */ | |
2faf153b | 294 | .mmu.flush_tlb_user = native_flush_tlb_local, |
5c83511b JG |
295 | .mmu.flush_tlb_kernel = native_flush_tlb_global, |
296 | .mmu.flush_tlb_one_user = native_flush_tlb_one_user, | |
4ce94eab | 297 | .mmu.flush_tlb_multi = native_flush_tlb_multi, |
5c83511b JG |
298 | .mmu.tlb_remove_table = |
299 | (void (*)(struct mmu_gather *, void *))tlb_remove_page, | |
300 | ||
fdc0269e | 301 | .mmu.exit_mmap = paravirt_nop, |
064ce6c5 | 302 | .mmu.notify_page_enc_status_changed = paravirt_nop, |
fdc0269e JG |
303 | |
304 | #ifdef CONFIG_PARAVIRT_XXL | |
0a53c9ac | 305 | .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), |
209cfd0c | 306 | .mmu.write_cr2 = pv_native_write_cr2, |
fdc0269e JG |
307 | .mmu.read_cr3 = __native_read_cr3, |
308 | .mmu.write_cr3 = native_write_cr3, | |
309 | ||
5c83511b JG |
310 | .mmu.pgd_alloc = __paravirt_pgd_alloc, |
311 | .mmu.pgd_free = paravirt_nop, | |
312 | ||
313 | .mmu.alloc_pte = paravirt_nop, | |
314 | .mmu.alloc_pmd = paravirt_nop, | |
315 | .mmu.alloc_pud = paravirt_nop, | |
316 | .mmu.alloc_p4d = paravirt_nop, | |
317 | .mmu.release_pte = paravirt_nop, | |
318 | .mmu.release_pmd = paravirt_nop, | |
319 | .mmu.release_pud = paravirt_nop, | |
320 | .mmu.release_p4d = paravirt_nop, | |
321 | ||
322 | .mmu.set_pte = native_set_pte, | |
5c83511b JG |
323 | .mmu.set_pmd = native_set_pmd, |
324 | ||
325 | .mmu.ptep_modify_prot_start = __ptep_modify_prot_start, | |
326 | .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit, | |
08b882c6 | 327 | |
5c83511b | 328 | .mmu.set_pud = native_set_pud, |
da5de7c2 | 329 | |
5c83511b JG |
330 | .mmu.pmd_val = PTE_IDENT, |
331 | .mmu.make_pmd = PTE_IDENT, | |
f95f2f7b | 332 | |
5c83511b JG |
333 | .mmu.pud_val = PTE_IDENT, |
334 | .mmu.make_pud = PTE_IDENT, | |
da5de7c2 | 335 | |
5c83511b | 336 | .mmu.set_p4d = native_set_p4d, |
f2a6a705 KS |
337 | |
338 | #if CONFIG_PGTABLE_LEVELS >= 5 | |
5c83511b JG |
339 | .mmu.p4d_val = PTE_IDENT, |
340 | .mmu.make_p4d = PTE_IDENT, | |
335437fb | 341 | |
5c83511b | 342 | .mmu.set_pgd = native_set_pgd, |
335437fb | 343 | #endif /* CONFIG_PGTABLE_LEVELS >= 5 */ |
da181a8b | 344 | |
5c83511b JG |
345 | .mmu.pte_val = PTE_IDENT, |
346 | .mmu.pgd_val = PTE_IDENT, | |
3dc494e8 | 347 | |
5c83511b JG |
348 | .mmu.make_pte = PTE_IDENT, |
349 | .mmu.make_pgd = PTE_IDENT, | |
3dc494e8 | 350 | |
c9ae1b10 | 351 | .mmu.enter_mmap = paravirt_nop, |
8965c1c0 | 352 | |
5c83511b JG |
353 | .mmu.lazy_mode = { |
354 | .enter = paravirt_nop, | |
355 | .leave = paravirt_nop, | |
356 | .flush = paravirt_nop, | |
8965c1c0 | 357 | }, |
aeaaa59c | 358 | |
5c83511b | 359 | .mmu.set_fixmap = native_set_fixmap, |
fdc0269e | 360 | #endif /* CONFIG_PARAVIRT_XXL */ |
5c83511b JG |
361 | |
362 | #if defined(CONFIG_PARAVIRT_SPINLOCKS) | |
363 | /* Lock ops. */ | |
364 | #ifdef CONFIG_SMP | |
365 | .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, | |
366 | .lock.queued_spin_unlock = | |
367 | PV_CALLEE_SAVE(__native_queued_spin_unlock), | |
368 | .lock.wait = paravirt_nop, | |
369 | .lock.kick = paravirt_nop, | |
370 | .lock.vcpu_is_preempted = | |
371 | PV_CALLEE_SAVE(__native_vcpu_is_preempted), | |
372 | #endif /* SMP */ | |
373 | #endif | |
d3561b7f | 374 | }; |
0dbe5a11 | 375 | |
9bad5658 | 376 | #ifdef CONFIG_PARAVIRT_XXL |
5c83511b | 377 | NOKPROBE_SYMBOL(native_load_idt); |
9bad5658 | 378 | #endif |
5c83511b | 379 | |
8af19095 | 380 | EXPORT_SYMBOL(pv_ops); |
93b1eab3 | 381 | EXPORT_SYMBOL_GPL(pv_info); |