Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 | 2 | /* |
1da177e4 LT |
3 | * S390 low-level entry points. |
4 | * | |
a53c8fab | 5 | * Copyright IBM Corp. 1999, 2012 |
1da177e4 | 6 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), |
25d83cbf HC |
7 | * Hartmut Penner (hp@de.ibm.com), |
8 | * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), | |
1da177e4 LT |
9 | */ |
10 | ||
b8c723f1 | 11 | #include <linux/export.h> |
2bc89b5e | 12 | #include <linux/init.h> |
144d634a | 13 | #include <linux/linkage.h> |
d09a307f | 14 | #include <asm/asm-extable.h> |
b058661a | 15 | #include <asm/alternative-asm.h> |
eb608fb3 | 16 | #include <asm/processor.h> |
1da177e4 | 17 | #include <asm/cache.h> |
dc24b7b4 | 18 | #include <asm/dwarf.h> |
1da177e4 LT |
19 | #include <asm/errno.h> |
20 | #include <asm/ptrace.h> | |
21 | #include <asm/thread_info.h> | |
0013a854 | 22 | #include <asm/asm-offsets.h> |
1da177e4 LT |
23 | #include <asm/unistd.h> |
24 | #include <asm/page.h> | |
eb546195 | 25 | #include <asm/sigp.h> |
1f44a225 | 26 | #include <asm/irq.h> |
fd2527f2 | 27 | #include <asm/fpu-insn.h> |
83abeffb HB |
28 | #include <asm/setup.h> |
29 | #include <asm/nmi.h> | |
6dd85fbb | 30 | #include <asm/nospec-insn.h> |
1da177e4 | 31 | |
e5b98199 MS |
32 | _LPP_OFFSET = __LC_LPP |
33 | ||
3b051e89 | 34 | .macro STBEAR address |
fad442d3 | 35 | ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193 |
3b051e89 SS |
36 | .endm |
37 | ||
38 | .macro LBEAR address | |
fad442d3 | 39 | ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193 |
3b051e89 SS |
40 | .endm |
41 | ||
42 | .macro LPSWEY address,lpswe | |
fad442d3 | 43 | ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193 |
3b051e89 SS |
44 | .endm |
45 | ||
46 | .macro MBEAR reg | |
fad442d3 | 47 | ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193 |
3b051e89 SS |
48 | .endm |
49 | ||
ce3dc447 | 50 | .macro CHECK_STACK savearea |
63b12246 | 51 | #ifdef CONFIG_CHECK_STACK |
c2c3258f | 52 | tml %r15,THREAD_SIZE - CONFIG_STACK_GUARD |
c5328901 MS |
53 | lghi %r14,\savearea |
54 | jz stack_overflow | |
63b12246 | 55 | #endif |
63b12246 MS |
56 | .endm |
57 | ||
ce3dc447 MS |
58 | .macro CHECK_VMAP_STACK savearea,oklabel |
59 | #ifdef CONFIG_VMAP_STACK | |
60 | lgr %r14,%r15 | |
c2c3258f HC |
61 | nill %r14,0x10000 - THREAD_SIZE |
62 | oill %r14,STACK_INIT_OFFSET | |
ce3dc447 MS |
63 | clg %r14,__LC_KERNEL_STACK |
64 | je \oklabel | |
65 | clg %r14,__LC_ASYNC_STACK | |
66 | je \oklabel | |
b61b1595 SS |
67 | clg %r14,__LC_MCCK_STACK |
68 | je \oklabel | |
ce3dc447 MS |
69 | clg %r14,__LC_NODAT_STACK |
70 | je \oklabel | |
71 | clg %r14,__LC_RESTART_STACK | |
72 | je \oklabel | |
73 | lghi %r14,\savearea | |
74 | j stack_overflow | |
75 | #else | |
76 | j \oklabel | |
77 | #endif | |
78 | .endm | |
79 | ||
83abeffb HB |
80 | /* |
81 | * The TSTMSK macro generates a test-under-mask instruction by | |
82 | * calculating the memory offset for the specified mask value. | |
83 | * Mask value can be any constant. The macro shifts the mask | |
84 | * value to calculate the memory offset for the test-under-mask | |
85 | * instruction. | |
86 | */ | |
87 | .macro TSTMSK addr, mask, size=8, bytepos=0 | |
88 | .if (\bytepos < \size) && (\mask >> 8) | |
89 | .if (\mask & 0xff) | |
90 | .error "Mask exceeds byte boundary" | |
91 | .endif | |
92 | TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)" | |
93 | .exitm | |
94 | .endif | |
95 | .ifeq \mask | |
96 | .error "Mask must not be zero" | |
97 | .endif | |
98 | off = \size - \bytepos - 1 | |
99 | tm off+\addr, \mask | |
100 | .endm | |
101 | ||
d768bd89 | 102 | .macro BPOFF |
fad442d3 | 103 | ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82 |
d768bd89 MS |
104 | .endm |
105 | ||
106 | .macro BPON | |
fad442d3 | 107 | ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82 |
d768bd89 MS |
108 | .endm |
109 | ||
6b73044b | 110 | .macro BPENTER tif_ptr,tif_mask |
6982dba1 | 111 | ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \ |
fad442d3 | 112 | "j .+12; nop; nop", 82 |
6b73044b MS |
113 | .endm |
114 | ||
115 | .macro BPEXIT tif_ptr,tif_mask | |
116 | TSTMSK \tif_ptr,\tif_mask | |
6982dba1 HC |
117 | ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \ |
118 | "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82 | |
6b73044b MS |
119 | .endm |
120 | ||
b5415c8f AG |
121 | #if IS_ENABLED(CONFIG_KVM) |
122 | /* | |
123 | * The OUTSIDE macro jumps to the provided label in case the value | |
124 | * in the provided register is outside of the provided range. The | |
125 | * macro is useful for checking whether a PSW stored in a register | |
126 | * pair points inside or outside of a block of instructions. | |
127 | * @reg: register to check | |
128 | * @start: start of the range | |
129 | * @end: end of the range | |
130 | * @outside_label: jump here if @reg is outside of [@start..@end) | |
131 | */ | |
132 | .macro OUTSIDE reg,start,end,outside_label | |
133 | lgr %r14,\reg | |
134 | larl %r13,\start | |
135 | slgr %r14,%r13 | |
4c25f0ff | 136 | clgfrl %r14,.Lrange_size\@ |
b5415c8f | 137 | jhe \outside_label |
4c25f0ff | 138 | .section .rodata, "a" |
27d45655 | 139 | .balign 4 |
4c25f0ff HC |
140 | .Lrange_size\@: |
141 | .long \end - \start | |
142 | .previous | |
b5415c8f | 143 | .endm |
fbbdfca5 AG |
144 | |
145 | .macro SIEEXIT | |
146 | lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer | |
147 | ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE | |
148 | lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce | |
c239c83e | 149 | ni __LC_CPU_FLAGS+7,255-_CIF_SIE |
fbbdfca5 AG |
150 | larl %r9,sie_exit # skip forward to sie_exit |
151 | .endm | |
b5415c8f AG |
152 | #endif |
153 | ||
b94c0ebb HC |
154 | .macro STACKLEAK_ERASE |
155 | #ifdef CONFIG_GCC_PLUGIN_STACKLEAK | |
156 | brasl %r14,stackleak_erase_on_task_stack | |
157 | #endif | |
158 | .endm | |
159 | ||
6dd85fbb | 160 | GEN_BR_THUNK %r14 |
f19fbd5e | 161 | |
860dba45 | 162 | .section .kprobes.text, "ax" |
46210c44 HC |
163 | .Ldummy: |
164 | /* | |
69a407bf HC |
165 | * The following nop exists only in order to avoid that the next |
166 | * symbol starts at the beginning of the kprobes text section. | |
167 | * In that case there would be several symbols at the same address. | |
168 | * E.g. objdump would take an arbitrary symbol when disassembling | |
169 | * the code. | |
170 | * With the added nop in between this cannot happen. | |
46210c44 HC |
171 | */ |
172 | nop 0 | |
860dba45 | 173 | |
1da177e4 | 174 | /* |
340750c1 HC |
175 | * Scheduler resume function, called by __switch_to |
176 | * gpr2 = (task_struct *)prev | |
177 | * gpr3 = (task_struct *)next | |
1da177e4 LT |
178 | * Returns: |
179 | * gpr2 = prev | |
180 | */ | |
340750c1 | 181 | SYM_FUNC_START(__switch_to_asm) |
eda0c6d6 | 182 | stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task |
3241d3eb HC |
183 | lghi %r4,__TASK_stack |
184 | lghi %r1,__TASK_thread | |
c2c3258f | 185 | llill %r5,STACK_INIT_OFFSET |
3241d3eb | 186 | stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev |
9fed920e VG |
187 | lg %r15,0(%r4,%r3) # start of kernel stack of next |
188 | agr %r15,%r5 # end of kernel stack of next | |
eda0c6d6 | 189 | stg %r3,__LC_CURRENT # store task struct of next |
eda0c6d6 | 190 | stg %r15,__LC_KERNEL_STACK # store end of kernel stack |
3241d3eb HC |
191 | lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next |
192 | aghi %r3,__TASK_pid | |
193 | mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next | |
d3a73acb | 194 | lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task |
fad442d3 | 195 | ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40 |
6dd85fbb | 196 | BR_EX %r14 |
340750c1 | 197 | SYM_FUNC_END(__switch_to_asm) |
1da177e4 | 198 | |
d0fc4107 MS |
199 | #if IS_ENABLED(CONFIG_KVM) |
200 | /* | |
6b33e68a NB |
201 | * __sie64a calling convention: |
202 | * %r2 pointer to sie control block phys | |
203 | * %r3 pointer to sie control block virt | |
204 | * %r4 guest register save area | |
d0fc4107 | 205 | */ |
fda1dffa | 206 | SYM_FUNC_START(__sie64a) |
d0fc4107 | 207 | stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers |
6b73044b | 208 | lg %r12,__LC_CURRENT |
6b33e68a NB |
209 | stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical.. |
210 | stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses | |
211 | stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area | |
92fa7a13 MS |
212 | xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 |
213 | mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags | |
6b33e68a | 214 | lmg %r0,%r13,0(%r4) # load guest gprs 0-13 |
d0fc4107 MS |
215 | lg %r14,__LC_GMAP # get gmap pointer |
216 | ltgr %r14,%r14 | |
217 | jz .Lsie_gmap | |
c239c83e | 218 | oi __LC_CPU_FLAGS+7,_CIF_SIE |
d0fc4107 MS |
219 | lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce |
220 | .Lsie_gmap: | |
92fa7a13 | 221 | lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer |
d0fc4107 MS |
222 | oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now |
223 | tm __SIE_PROG20+3(%r14),3 # last exit... | |
224 | jnz .Lsie_skip | |
6b33e68a | 225 | lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr |
f33f2d4c | 226 | BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST |
c929500d | 227 | .Lsie_entry: |
d0fc4107 | 228 | sie 0(%r14) |
29ccaa4b AG |
229 | # Let the next instruction be NOP to avoid triggering a machine check |
230 | # and handling it in a guest as result of the instruction execution. | |
231 | nopr 7 | |
232 | .Lsie_leave: | |
d768bd89 | 233 | BPOFF |
f33f2d4c | 234 | BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST |
d0fc4107 | 235 | .Lsie_skip: |
6b33e68a | 236 | lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer |
d0fc4107 | 237 | ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE |
87d59863 | 238 | lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce |
d0fc4107 | 239 | .Lsie_done: |
c239c83e | 240 | ni __LC_CPU_FLAGS+7,255-_CIF_SIE |
d0fc4107 | 241 | # some program checks are suppressing. C code (e.g. do_protection_exception) |
c0e7bb38 CB |
242 | # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There |
243 | # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable. | |
6b33e68a | 244 | # Other instructions between __sie64a and .Lsie_done should not cause program |
c0e7bb38 | 245 | # interrupts. So lets use 3 nops as a landing pad for all possible rewinds. |
c0e7bb38 CB |
246 | .Lrewind_pad6: |
247 | nopr 7 | |
248 | .Lrewind_pad4: | |
249 | nopr 7 | |
250 | .Lrewind_pad2: | |
251 | nopr 7 | |
fda1dffa | 252 | SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL) |
92fa7a13 | 253 | lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area |
d0fc4107 | 254 | stmg %r0,%r13,0(%r14) # save guest gprs 0-13 |
7041d281 MS |
255 | xgr %r0,%r0 # clear guest registers to |
256 | xgr %r1,%r1 # prevent speculative use | |
7041d281 MS |
257 | xgr %r3,%r3 |
258 | xgr %r4,%r4 | |
259 | xgr %r5,%r5 | |
d0fc4107 | 260 | lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers |
92fa7a13 | 261 | lg %r2,__SF_SIE_REASON(%r15) # return exit reason code |
6dd85fbb | 262 | BR_EX %r14 |
d0fc4107 MS |
263 | .Lsie_fault: |
264 | lghi %r14,-EFAULT | |
92fa7a13 | 265 | stg %r14,__SF_SIE_REASON(%r15) # set exit reason code |
d0fc4107 MS |
266 | j sie_exit |
267 | ||
c0e7bb38 CB |
268 | EX_TABLE(.Lrewind_pad6,.Lsie_fault) |
269 | EX_TABLE(.Lrewind_pad4,.Lsie_fault) | |
270 | EX_TABLE(.Lrewind_pad2,.Lsie_fault) | |
d0fc4107 | 271 | EX_TABLE(sie_exit,.Lsie_fault) |
fda1dffa | 272 | SYM_FUNC_END(__sie64a) |
6b33e68a | 273 | EXPORT_SYMBOL(__sie64a) |
711f5df7 | 274 | EXPORT_SYMBOL(sie_exit) |
d0fc4107 MS |
275 | #endif |
276 | ||
1da177e4 LT |
277 | /* |
278 | * SVC interrupt handler routine. System calls are synchronous events and | |
7b7735c5 | 279 | * are entered with interrupts disabled. |
1da177e4 LT |
280 | */ |
281 | ||
fda1dffa | 282 | SYM_CODE_START(system_call) |
56e62a73 | 283 | stpt __LC_SYS_ENTER_TIMER |
c5328901 | 284 | stmg %r8,%r15,__LC_SAVE_AREA_SYNC |
d768bd89 | 285 | BPOFF |
56e62a73 | 286 | lghi %r14,0 |
86ed42f4 | 287 | .Lsysc_per: |
3b051e89 | 288 | STBEAR __LC_LAST_BREAK |
87d59863 | 289 | lctlg %c1,%c1,__LC_KERNEL_ASCE |
c5328901 | 290 | lg %r15,__LC_KERNEL_STACK |
9365965d | 291 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
56e62a73 | 292 | stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15) |
d3f46896 CB |
293 | # clear user controlled register to prevent speculative use |
294 | xgr %r0,%r0 | |
56e62a73 SS |
295 | xgr %r1,%r1 |
296 | xgr %r4,%r4 | |
297 | xgr %r5,%r5 | |
298 | xgr %r6,%r6 | |
299 | xgr %r7,%r7 | |
300 | xgr %r8,%r8 | |
301 | xgr %r9,%r9 | |
302 | xgr %r10,%r10 | |
303 | xgr %r11,%r11 | |
304 | la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs | |
af9ad822 | 305 | mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC |
3b051e89 | 306 | MBEAR %r2 |
56e62a73 SS |
307 | lgr %r3,%r14 |
308 | brasl %r14,__do_syscall | |
b94c0ebb | 309 | STACKLEAK_ERASE |
87d59863 | 310 | lctlg %c1,%c1,__LC_USER_ASCE |
56e62a73 | 311 | mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) |
f33f2d4c | 312 | BPON |
3b051e89 | 313 | LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15) |
56e62a73 | 314 | lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) |
c5328901 | 315 | stpt __LC_EXIT_TIMER |
3b051e89 | 316 | LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE |
fda1dffa | 317 | SYM_CODE_END(system_call) |
1da177e4 LT |
318 | |
319 | # | |
320 | # a new process exits the kernel with ret_from_fork | |
321 | # | |
fda1dffa | 322 | SYM_CODE_START(ret_from_fork) |
56e62a73 SS |
323 | lgr %r3,%r11 |
324 | brasl %r14,__ret_from_fork | |
b94c0ebb | 325 | STACKLEAK_ERASE |
56e62a73 SS |
326 | lctlg %c1,%c1,__LC_USER_ASCE |
327 | mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) | |
f33f2d4c | 328 | BPON |
3b051e89 | 329 | LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15) |
56e62a73 SS |
330 | lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) |
331 | stpt __LC_EXIT_TIMER | |
3b051e89 | 332 | LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE |
fda1dffa | 333 | SYM_CODE_END(ret_from_fork) |
26a374ae | 334 | |
1da177e4 LT |
335 | /* |
336 | * Program check handler routine | |
337 | */ | |
338 | ||
fda1dffa | 339 | SYM_CODE_START(pgm_check_handler) |
56e62a73 | 340 | stpt __LC_SYS_ENTER_TIMER |
d768bd89 | 341 | BPOFF |
c5328901 | 342 | stmg %r8,%r15,__LC_SAVE_AREA_SYNC |
56e62a73 | 343 | lghi %r10,0 |
c5328901 | 344 | lmg %r8,%r9,__LC_PGM_OLD_PSW |
87d59863 HC |
345 | tmhh %r8,0x0001 # coming from user space? |
346 | jno .Lpgm_skip_asce | |
347 | lctlg %c1,%c1,__LC_KERNEL_ASCE | |
56e62a73 | 348 | j 3f # -> fault in user space |
87d59863 | 349 | .Lpgm_skip_asce: |
d0fc4107 | 350 | #if IS_ENABLED(CONFIG_KVM) |
6b33e68a | 351 | # cleanup critical section for program checks in __sie64a |
c239c83e SS |
352 | TSTMSK __LC_CPU_FLAGS,_CIF_SIE |
353 | jz 1f | |
f33f2d4c | 354 | BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST |
fbbdfca5 | 355 | SIEEXIT |
56e62a73 | 356 | lghi %r10,_PIF_GUEST_FAULT |
d0fc4107 | 357 | #endif |
0b38b5e1 SS |
358 | 1: tmhh %r8,0x4000 # PER bit set in old PSW ? |
359 | jnz 2f # -> enabled, can't be a double fault | |
c5328901 | 360 | tm __LC_PGM_ILC+3,0x80 # check for per exception |
86ed42f4 | 361 | jnz .Lpgm_svcper # -> single stepped svc |
0b38b5e1 | 362 | 2: CHECK_STACK __LC_SAVE_AREA_SYNC |
dc7ee00d | 363 | aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) |
56e62a73 SS |
364 | # CHECK_VMAP_STACK branches to stack_overflow or 4f |
365 | CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f | |
f33f2d4c | 366 | 3: lg %r15,__LC_KERNEL_STACK |
56e62a73 SS |
367 | 4: la %r11,STACK_FRAME_OVERHEAD(%r15) |
368 | stg %r10,__PT_FLAGS(%r11) | |
369 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) | |
c5328901 | 370 | stmg %r0,%r7,__PT_R0(%r11) |
56e62a73 | 371 | mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC |
3b051e89 | 372 | mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK |
56e62a73 SS |
373 | stmg %r8,%r9,__PT_PSW(%r11) |
374 | ||
7041d281 MS |
375 | # clear user controlled registers to prevent speculative use |
376 | xgr %r0,%r0 | |
377 | xgr %r1,%r1 | |
7041d281 MS |
378 | xgr %r3,%r3 |
379 | xgr %r4,%r4 | |
380 | xgr %r5,%r5 | |
381 | xgr %r6,%r6 | |
382 | xgr %r7,%r7 | |
56e62a73 SS |
383 | lgr %r2,%r11 |
384 | brasl %r14,__do_pgm_check | |
385 | tmhh %r8,0x0001 # returning to user space? | |
386 | jno .Lpgm_exit_kernel | |
b94c0ebb | 387 | STACKLEAK_ERASE |
56e62a73 | 388 | lctlg %c1,%c1,__LC_USER_ASCE |
f33f2d4c | 389 | BPON |
0cd9b723 | 390 | stpt __LC_EXIT_TIMER |
56e62a73 SS |
391 | .Lpgm_exit_kernel: |
392 | mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) | |
3b051e89 | 393 | LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15) |
56e62a73 | 394 | lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) |
3b051e89 | 395 | LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE |
1da177e4 | 396 | |
4ba069b8 | 397 | # |
c5328901 | 398 | # single stepped system call |
4ba069b8 | 399 | # |
86ed42f4 | 400 | .Lpgm_svcper: |
c5328901 | 401 | mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW |
86ed42f4 | 402 | larl %r14,.Lsysc_per |
c5328901 | 403 | stg %r14,__LC_RETURN_PSW+8 |
56e62a73 | 404 | lghi %r14,1 |
3b051e89 SS |
405 | LBEAR __LC_PGM_LAST_BREAK |
406 | LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per | |
fda1dffa | 407 | SYM_CODE_END(pgm_check_handler) |
4ba069b8 | 408 | |
1da177e4 | 409 | /* |
56e62a73 | 410 | * Interrupt handler macro used for external and IO interrupts. |
1da177e4 | 411 | */ |
56e62a73 | 412 | .macro INT_HANDLER name,lc_old_psw,handler |
fda1dffa | 413 | SYM_CODE_START(\name) |
10bc15ba | 414 | stckf __LC_INT_CLOCK |
56e62a73 | 415 | stpt __LC_SYS_ENTER_TIMER |
3b051e89 | 416 | STBEAR __LC_LAST_BREAK |
d768bd89 | 417 | BPOFF |
c5328901 | 418 | stmg %r8,%r15,__LC_SAVE_AREA_ASYNC |
56e62a73 | 419 | lmg %r8,%r9,\lc_old_psw |
b0d31159 SS |
420 | tmhh %r8,0x0001 # interrupting from user ? |
421 | jnz 1f | |
422 | #if IS_ENABLED(CONFIG_KVM) | |
c239c83e SS |
423 | TSTMSK __LC_CPU_FLAGS,_CIF_SIE |
424 | jz 0f | |
f33f2d4c | 425 | BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST |
fbbdfca5 | 426 | SIEEXIT |
b0d31159 SS |
427 | #endif |
428 | 0: CHECK_STACK __LC_SAVE_AREA_ASYNC | |
b0d31159 | 429 | aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) |
b0d31159 | 430 | j 2f |
f33f2d4c | 431 | 1: lctlg %c1,%c1,__LC_KERNEL_ASCE |
b0d31159 | 432 | lg %r15,__LC_KERNEL_STACK |
b74e409e VG |
433 | 2: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
434 | la %r11,STACK_FRAME_OVERHEAD(%r15) | |
c5328901 | 435 | stmg %r0,%r7,__PT_R0(%r11) |
7041d281 MS |
436 | # clear user controlled registers to prevent speculative use |
437 | xgr %r0,%r0 | |
438 | xgr %r1,%r1 | |
7041d281 MS |
439 | xgr %r3,%r3 |
440 | xgr %r4,%r4 | |
441 | xgr %r5,%r5 | |
442 | xgr %r6,%r6 | |
443 | xgr %r7,%r7 | |
444 | xgr %r10,%r10 | |
ca1f4d70 | 445 | xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) |
c5328901 | 446 | mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC |
3b051e89 | 447 | MBEAR %r11 |
c5328901 | 448 | stmg %r8,%r9,__PT_PSW(%r11) |
29b06ad7 | 449 | lgr %r2,%r11 # pass pointer to pt_regs |
56e62a73 | 450 | brasl %r14,\handler |
c5328901 | 451 | mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) |
56e62a73 SS |
452 | tmhh %r8,0x0001 # returning to user ? |
453 | jno 2f | |
b94c0ebb | 454 | STACKLEAK_ERASE |
87d59863 | 455 | lctlg %c1,%c1,__LC_USER_ASCE |
f33f2d4c | 456 | BPON |
c5328901 | 457 | stpt __LC_EXIT_TIMER |
3b051e89 SS |
458 | 2: LBEAR __PT_LAST_BREAK(%r11) |
459 | lmg %r0,%r15,__PT_R0(%r11) | |
460 | LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE | |
fda1dffa | 461 | SYM_CODE_END(\name) |
56e62a73 | 462 | .endm |
916cda1a | 463 | |
56e62a73 SS |
464 | INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq |
465 | INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq | |
1da177e4 | 466 | |
4c1051e3 | 467 | /* |
0b0ed657 | 468 | * Load idle PSW. |
4c1051e3 | 469 | */ |
fda1dffa | 470 | SYM_FUNC_START(psw_idle) |
a994eddb | 471 | stg %r14,(__SF_GPRS+8*8)(%r15) |
27f6b416 | 472 | stg %r3,__SF_EMPTY(%r15) |
56e62a73 | 473 | larl %r1,psw_idle_exit |
4c1051e3 | 474 | stg %r1,__SF_EMPTY+8(%r15) |
72d38b19 MS |
475 | larl %r1,smp_cpu_mtid |
476 | llgf %r1,0(%r1) | |
477 | ltgr %r1,%r1 | |
478 | jz .Lpsw_idle_stcctm | |
56e62a73 | 479 | .insn rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2) |
72d38b19 | 480 | .Lpsw_idle_stcctm: |
419123f9 | 481 | oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT |
d768bd89 | 482 | BPON |
10bc15ba | 483 | stckf __CLOCK_IDLE_ENTER(%r2) |
27f6b416 | 484 | stpt __TIMER_IDLE_ENTER(%r2) |
4c1051e3 | 485 | lpswe __SF_EMPTY(%r15) |
fda1dffa | 486 | SYM_INNER_LABEL(psw_idle_exit, SYM_L_GLOBAL) |
6dd85fbb | 487 | BR_EX %r14 |
fda1dffa | 488 | SYM_FUNC_END(psw_idle) |
4c1051e3 | 489 | |
1da177e4 LT |
490 | /* |
491 | * Machine check handler routines | |
492 | */ | |
fda1dffa | 493 | SYM_CODE_START(mcck_int_handler) |
d768bd89 | 494 | BPOFF |
c5328901 | 495 | lmg %r8,%r9,__LC_MCK_OLD_PSW |
83abeffb | 496 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE |
86ed42f4 | 497 | jo .Lmcck_panic # yes -> rest of mcck code invalid |
3037a52f MS |
498 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID |
499 | jno .Lmcck_panic # control registers invalid -> panic | |
3037a52f | 500 | ptlb |
5fa2ea07 | 501 | lghi %r14,__LC_CPU_TIMER_SAVE_AREA |
c5328901 | 502 | mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) |
83abeffb | 503 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID |
c5328901 | 504 | jo 3f |
56e62a73 SS |
505 | la %r14,__LC_SYS_ENTER_TIMER |
506 | clc 0(8,%r14),__LC_EXIT_TIMER | |
c5328901 | 507 | jl 1f |
63b12246 | 508 | la %r14,__LC_EXIT_TIMER |
c5328901 MS |
509 | 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER |
510 | jl 2f | |
63b12246 | 511 | la %r14,__LC_LAST_UPDATE_TIMER |
c5328901 | 512 | 2: spt 0(%r14) |
6377981f | 513 | mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) |
3037a52f MS |
514 | 3: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID |
515 | jno .Lmcck_panic | |
516 | tmhh %r8,0x0001 # interrupting from user ? | |
742aed05 | 517 | jnz .Lmcck_user |
3037a52f MS |
518 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID |
519 | jno .Lmcck_panic | |
b0d31159 | 520 | #if IS_ENABLED(CONFIG_KVM) |
c239c83e SS |
521 | TSTMSK __LC_CPU_FLAGS,_CIF_SIE |
522 | jz .Lmcck_user | |
523 | # Need to compare the address instead of a CIF_SIE* flag. | |
524 | # Otherwise there would be a race between setting the flag | |
525 | # and entering SIE (or leaving and clearing the flag). This | |
526 | # would cause machine checks targeted at the guest to be | |
527 | # handled by the host. | |
29ccaa4b | 528 | OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f |
20232b18 | 529 | oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST |
f33f2d4c | 530 | 4: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST |
fbbdfca5 | 531 | SIEEXIT |
e2c13d64 | 532 | #endif |
742aed05 | 533 | .Lmcck_user: |
b61b1595 | 534 | lg %r15,__LC_MCCK_STACK |
b61b1595 | 535 | la %r11,STACK_FRAME_OVERHEAD(%r15) |
26521412 | 536 | stctg %c1,%c1,__PT_CR1(%r11) |
b0d31159 | 537 | lctlg %c1,%c1,__LC_KERNEL_ASCE |
b0d31159 | 538 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
6551fbdf MS |
539 | lghi %r14,__LC_GPREGS_SAVE_AREA+64 |
540 | stmg %r0,%r7,__PT_R0(%r11) | |
7041d281 MS |
541 | # clear user controlled registers to prevent speculative use |
542 | xgr %r0,%r0 | |
543 | xgr %r1,%r1 | |
7041d281 MS |
544 | xgr %r3,%r3 |
545 | xgr %r4,%r4 | |
546 | xgr %r5,%r5 | |
547 | xgr %r6,%r6 | |
548 | xgr %r7,%r7 | |
549 | xgr %r10,%r10 | |
6551fbdf | 550 | mvc __PT_R8(64,%r11),0(%r14) |
c5328901 | 551 | stmg %r8,%r9,__PT_PSW(%r11) |
d3a73acb | 552 | xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) |
c5328901 MS |
553 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
554 | lgr %r2,%r11 # pass pointer to pt_regs | |
77fa2245 | 555 | brasl %r14,s390_do_machine_check |
87d59863 | 556 | lctlg %c1,%c1,__PT_CR1(%r11) |
c5328901 MS |
557 | lmg %r0,%r10,__PT_R0(%r11) |
558 | mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW | |
63b12246 MS |
559 | tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? |
560 | jno 0f | |
f33f2d4c | 561 | BPON |
63b12246 | 562 | stpt __LC_EXIT_TIMER |
fad442d3 | 563 | 0: ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193 |
3b051e89 SS |
564 | LBEAR 0(%r12) |
565 | lmg %r11,%r15,__PT_R11(%r11) | |
566 | LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE | |
c5328901 | 567 | |
86ed42f4 | 568 | .Lmcck_panic: |
7f6dc8d4 AG |
569 | /* |
570 | * Iterate over all possible CPU addresses in the range 0..0xffff | |
571 | * and stop each CPU using signal processor. Use compare and swap | |
572 | * to allow just one CPU-stopper and prevent concurrent CPUs from | |
573 | * stopping each other while leaving the others running. | |
574 | */ | |
575 | lhi %r5,0 | |
576 | lhi %r6,1 | |
fda1dffa | 577 | larl %r7,stop_lock |
7f6dc8d4 AG |
578 | cs %r5,%r6,0(%r7) # single CPU-stopper only |
579 | jnz 4f | |
fda1dffa | 580 | larl %r7,this_cpu |
7f6dc8d4 AG |
581 | stap 0(%r7) # this CPU address |
582 | lh %r4,0(%r7) | |
583 | nilh %r4,0 | |
584 | lhi %r0,1 | |
585 | sll %r0,16 # CPU counter | |
586 | lhi %r3,0 # next CPU address | |
587 | 0: cr %r3,%r4 | |
588 | je 2f | |
589 | 1: sigp %r1,%r3,SIGP_STOP # stop next CPU | |
590 | brc SIGP_CC_BUSY,1b | |
591 | 2: ahi %r3,1 | |
592 | brct %r0,0b | |
593 | 3: sigp %r1,%r4,SIGP_STOP # stop this CPU | |
594 | brc SIGP_CC_BUSY,3b | |
595 | 4: j 4b | |
fda1dffa | 596 | SYM_CODE_END(mcck_int_handler) |
1da177e4 | 597 | |
fda1dffa | 598 | SYM_CODE_START(restart_int_handler) |
fad442d3 | 599 | ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40 |
e5b98199 | 600 | stg %r15,__LC_SAVE_AREA_RESTART |
915fea04 AG |
601 | TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4 |
602 | jz 0f | |
385bf43c | 603 | lctlg %c0,%c15,__LC_CREGS_SAVE_AREA |
edbe2898 AG |
604 | 0: larl %r15,daton_psw |
605 | lpswe 0(%r15) # turn dat on, keep irqs off | |
606 | .Ldaton: | |
8b646bd7 | 607 | lg %r15,__LC_RESTART_STACK |
ce3dc447 MS |
608 | xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15) |
609 | stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15) | |
610 | mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART | |
611 | mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW | |
8b646bd7 | 612 | xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) |
fbe76568 HC |
613 | lg %r1,__LC_RESTART_FN # load fn, parm & source cpu |
614 | lg %r2,__LC_RESTART_DATA | |
915fea04 | 615 | lgf %r3,__LC_RESTART_SOURCE |
8b646bd7 MS |
616 | ltgr %r3,%r3 # test source cpu address |
617 | jm 1f # negative -> skip source stop | |
eb546195 | 618 | 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu |
8b646bd7 MS |
619 | brc 10,0b # wait for status stored |
620 | 1: basr %r14,%r1 # call function | |
621 | stap __SF_EMPTY(%r15) # store cpu address | |
622 | llgh %r3,__SF_EMPTY(%r15) | |
eb546195 | 623 | 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu |
8b646bd7 MS |
624 | brc 2,2b |
625 | 3: j 3b | |
fda1dffa | 626 | SYM_CODE_END(restart_int_handler) |
7dd6b334 | 627 | |
860dba45 MS |
628 | .section .kprobes.text, "ax" |
629 | ||
ce3dc447 | 630 | #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK) |
1da177e4 LT |
631 | /* |
632 | * The synchronous or the asynchronous stack overflowed. We are dead. | |
633 | * No need to properly save the registers, we are going to panic anyway. | |
634 | * Setup a pt_regs so that show_trace can provide a good call trace. | |
635 | */ | |
fda1dffa | 636 | SYM_CODE_START(stack_overflow) |
ce3dc447 | 637 | lg %r15,__LC_NODAT_STACK # change to panic stack |
dc7ee00d | 638 | la %r11,STACK_FRAME_OVERHEAD(%r15) |
c5328901 MS |
639 | stmg %r0,%r7,__PT_R0(%r11) |
640 | stmg %r8,%r9,__PT_PSW(%r11) | |
641 | mvc __PT_R8(64,%r11),0(%r14) | |
642 | stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 | |
c5328901 MS |
643 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
644 | lgr %r2,%r11 # pass pointer to pt_regs | |
1da177e4 | 645 | jg kernel_stack_overflow |
fda1dffa | 646 | SYM_CODE_END(stack_overflow) |
1da177e4 LT |
647 | #endif |
648 | ||
7f6dc8d4 | 649 | .section .data, "aw" |
fda1dffa HC |
650 | .balign 4 |
651 | SYM_DATA_LOCAL(stop_lock, .long 0) | |
652 | SYM_DATA_LOCAL(this_cpu, .short 0) | |
edbe2898 AG |
653 | .balign 8 |
654 | SYM_DATA_START_LOCAL(daton_psw) | |
655 | .quad PSW_KERNEL_BITS | |
656 | .quad .Ldaton | |
657 | SYM_DATA_END(daton_psw) | |
fda1dffa | 658 | |
a876cb3f | 659 | .section .rodata, "a" |
ff4a742d | 660 | #define SYSCALL(esame,emu) .quad __s390x_ ## esame |
fda1dffa | 661 | SYM_DATA_START(sys_call_table) |
4381f9f1 | 662 | #include "asm/syscall_table.h" |
fda1dffa | 663 | SYM_DATA_END(sys_call_table) |
1da177e4 LT |
664 | #undef SYSCALL |
665 | ||
347a8dc3 | 666 | #ifdef CONFIG_COMPAT |
1da177e4 | 667 | |
ff4a742d | 668 | #define SYSCALL(esame,emu) .quad __s390_ ## emu |
fda1dffa | 669 | SYM_DATA_START(sys_call_table_emu) |
4381f9f1 | 670 | #include "asm/syscall_table.h" |
fda1dffa | 671 | SYM_DATA_END(sys_call_table_emu) |
1da177e4 LT |
672 | #undef SYSCALL |
673 | #endif |