Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
cdacc127 | 2 | /* |
edcb5cf8 | 3 | * Asm versions of Xen pv-ops, suitable for direct use. |
130ace11 TH |
4 | * |
5 | * We only bother with direct forms (ie, vcpu in pda) of the | |
edcb5cf8 | 6 | * operations here; the indirect forms are better handled in C. |
cdacc127 JF |
7 | */ |
8 | ||
6fcac6d3 | 9 | #include <asm/errno.h> |
9af45651 | 10 | #include <asm/percpu.h> |
5393744b JF |
11 | #include <asm/processor-flags.h> |
12 | #include <asm/segment.h> | |
63332a84 | 13 | #include <asm/asm-offsets.h> |
3a23208e | 14 | #include <asm/thread_info.h> |
cdacc127 JF |
15 | |
16 | #include <xen/interface/xen.h> | |
17 | ||
42b3a4cb | 18 | #include <linux/init.h> |
edcb5cf8 | 19 | #include <linux/linkage.h> |
cdacc127 | 20 | |
5878d5d6 JG |
21 | .macro xen_pv_trap name |
22 | ENTRY(xen_\name) | |
23 | pop %rcx | |
24 | pop %r11 | |
25 | jmp \name | |
26 | END(xen_\name) | |
27 | .endm | |
28 | ||
29 | xen_pv_trap divide_error | |
30 | xen_pv_trap debug | |
31 | xen_pv_trap xendebug | |
32 | xen_pv_trap int3 | |
33 | xen_pv_trap xenint3 | |
43e41110 | 34 | xen_pv_trap xennmi |
5878d5d6 JG |
35 | xen_pv_trap overflow |
36 | xen_pv_trap bounds | |
37 | xen_pv_trap invalid_op | |
38 | xen_pv_trap device_not_available | |
39 | xen_pv_trap double_fault | |
40 | xen_pv_trap coprocessor_segment_overrun | |
41 | xen_pv_trap invalid_TSS | |
42 | xen_pv_trap segment_not_present | |
43 | xen_pv_trap stack_segment | |
44 | xen_pv_trap general_protection | |
45 | xen_pv_trap page_fault | |
46 | xen_pv_trap spurious_interrupt_bug | |
47 | xen_pv_trap coprocessor_error | |
48 | xen_pv_trap alignment_check | |
49 | #ifdef CONFIG_X86_MCE | |
50 | xen_pv_trap machine_check | |
51 | #endif /* CONFIG_X86_MCE */ | |
52 | xen_pv_trap simd_coprocessor_error | |
53 | #ifdef CONFIG_IA32_EMULATION | |
54 | xen_pv_trap entry_INT80_compat | |
55 | #endif | |
56 | xen_pv_trap hypervisor_callback | |
997409d3 | 57 | |
42b3a4cb JG |
58 | __INIT |
59 | ENTRY(xen_early_idt_handler_array) | |
60 | i = 0 | |
61 | .rept NUM_EXCEPTION_VECTORS | |
62 | pop %rcx | |
63 | pop %r11 | |
64 | jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE | |
65 | i = i + 1 | |
66 | .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc | |
67 | .endr | |
68 | END(xen_early_idt_handler_array) | |
69 | __FINIT | |
70 | ||
6fcac6d3 JF |
71 | hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 |
72 | /* | |
130ace11 TH |
73 | * Xen64 iret frame: |
74 | * | |
75 | * ss | |
76 | * rsp | |
77 | * rflags | |
78 | * cs | |
79 | * rip <-- standard iret frame | |
80 | * | |
81 | * flags | |
82 | * | |
83 | * rcx } | |
84 | * r11 }<-- pushed by hypercall page | |
85 | * rsp->rax } | |
6fcac6d3 | 86 | */ |
cdacc127 JF |
87 | ENTRY(xen_iret) |
88 | pushq $0 | |
edcb5cf8 | 89 | jmp hypercall_iret |
cdacc127 | 90 | |
6fcac6d3 | 91 | ENTRY(xen_sysret64) |
130ace11 TH |
92 | /* |
93 | * We're already on the usermode stack at this point, but | |
94 | * still with the kernel gs, so we can easily switch back | |
95 | */ | |
c38e5038 | 96 | movq %rsp, PER_CPU_VAR(rsp_scratch) |
3a23208e | 97 | movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp |
6fcac6d3 JF |
98 | |
99 | pushq $__USER_DS | |
c38e5038 | 100 | pushq PER_CPU_VAR(rsp_scratch) |
6fcac6d3 JF |
101 | pushq %r11 |
102 | pushq $__USER_CS | |
103 | pushq %rcx | |
104 | ||
105 | pushq $VGCF_in_syscall | |
edcb5cf8 | 106 | jmp hypercall_iret |
6fcac6d3 | 107 | |
6fcac6d3 | 108 | /* |
130ace11 TH |
109 | * Xen handles syscall callbacks much like ordinary exceptions, which |
110 | * means we have: | |
111 | * - kernel gs | |
112 | * - kernel rsp | |
113 | * - an iret-like stack frame on the stack (including rcx and r11): | |
114 | * ss | |
115 | * rsp | |
116 | * rflags | |
117 | * cs | |
118 | * rip | |
119 | * r11 | |
120 | * rsp->rcx | |
6fcac6d3 JF |
121 | */ |
122 | ||
6fcac6d3 JF |
123 | /* Normal 64-bit system call target */ |
124 | ENTRY(xen_syscall_target) | |
8a9949bc AL |
125 | popq %rcx |
126 | popq %r11 | |
fa2016a8 AL |
127 | |
128 | /* | |
129 | * Neither Xen nor the kernel really knows what the old SS and | |
130 | * CS were. The kernel expects __USER_DS and __USER_CS, so | |
131 | * report those values even though Xen will guess its own values. | |
132 | */ | |
133 | movq $__USER_DS, 4*8(%rsp) | |
134 | movq $__USER_CS, 1*8(%rsp) | |
135 | ||
8a9949bc | 136 | jmp entry_SYSCALL_64_after_hwframe |
6fcac6d3 JF |
137 | ENDPROC(xen_syscall_target) |
138 | ||
139 | #ifdef CONFIG_IA32_EMULATION | |
140 | ||
141 | /* 32-bit compat syscall target */ | |
142 | ENTRY(xen_syscall32_target) | |
8a9949bc AL |
143 | popq %rcx |
144 | popq %r11 | |
fa2016a8 AL |
145 | |
146 | /* | |
147 | * Neither Xen nor the kernel really knows what the old SS and | |
148 | * CS were. The kernel expects __USER32_DS and __USER32_CS, so | |
149 | * report those values even though Xen will guess its own values. | |
150 | */ | |
151 | movq $__USER32_DS, 4*8(%rsp) | |
152 | movq $__USER32_CS, 1*8(%rsp) | |
153 | ||
8a9949bc | 154 | jmp entry_SYSCALL_compat_after_hwframe |
6fcac6d3 JF |
155 | ENDPROC(xen_syscall32_target) |
156 | ||
157 | /* 32-bit compat sysenter target */ | |
158 | ENTRY(xen_sysenter_target) | |
8a9949bc AL |
159 | mov 0*8(%rsp), %rcx |
160 | mov 1*8(%rsp), %r11 | |
161 | mov 5*8(%rsp), %rsp | |
4c8cd0c5 | 162 | jmp entry_SYSENTER_compat |
6fcac6d3 JF |
163 | ENDPROC(xen_sysenter_target) |
164 | ||
165 | #else /* !CONFIG_IA32_EMULATION */ | |
166 | ||
167 | ENTRY(xen_syscall32_target) | |
168 | ENTRY(xen_sysenter_target) | |
130ace11 | 169 | lea 16(%rsp), %rsp /* strip %rcx, %r11 */ |
6fcac6d3 | 170 | mov $-ENOSYS, %rax |
6aaf5d63 | 171 | pushq $0 |
6fcac6d3 JF |
172 | jmp hypercall_iret |
173 | ENDPROC(xen_syscall32_target) | |
174 | ENDPROC(xen_sysenter_target) | |
175 | ||
176 | #endif /* CONFIG_IA32_EMULATION */ |