Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 | 2 | /* |
1da177e4 LT |
3 | * S390 low-level entry points. |
4 | * | |
a53c8fab | 5 | * Copyright IBM Corp. 1999, 2012 |
1da177e4 | 6 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), |
25d83cbf HC |
7 | * Hartmut Penner (hp@de.ibm.com), |
8 | * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), | |
77fa2245 | 9 | * Heiko Carstens <heiko.carstens@de.ibm.com> |
1da177e4 LT |
10 | */ |
11 | ||
2bc89b5e | 12 | #include <linux/init.h> |
144d634a | 13 | #include <linux/linkage.h> |
b058661a | 14 | #include <asm/alternative-asm.h> |
eb608fb3 | 15 | #include <asm/processor.h> |
1da177e4 | 16 | #include <asm/cache.h> |
3037a52f | 17 | #include <asm/ctl_reg.h> |
dc24b7b4 | 18 | #include <asm/dwarf.h> |
1da177e4 LT |
19 | #include <asm/errno.h> |
20 | #include <asm/ptrace.h> | |
21 | #include <asm/thread_info.h> | |
0013a854 | 22 | #include <asm/asm-offsets.h> |
1da177e4 LT |
23 | #include <asm/unistd.h> |
24 | #include <asm/page.h> | |
eb546195 | 25 | #include <asm/sigp.h> |
1f44a225 | 26 | #include <asm/irq.h> |
9977e886 | 27 | #include <asm/vx-insn.h> |
83abeffb HB |
28 | #include <asm/setup.h> |
29 | #include <asm/nmi.h> | |
711f5df7 | 30 | #include <asm/export.h> |
6dd85fbb | 31 | #include <asm/nospec-insn.h> |
1da177e4 | 32 | |
c5328901 MS |
33 | __PT_R0 = __PT_GPRS |
34 | __PT_R1 = __PT_GPRS + 8 | |
35 | __PT_R2 = __PT_GPRS + 16 | |
36 | __PT_R3 = __PT_GPRS + 24 | |
37 | __PT_R4 = __PT_GPRS + 32 | |
38 | __PT_R5 = __PT_GPRS + 40 | |
39 | __PT_R6 = __PT_GPRS + 48 | |
40 | __PT_R7 = __PT_GPRS + 56 | |
41 | __PT_R8 = __PT_GPRS + 64 | |
42 | __PT_R9 = __PT_GPRS + 72 | |
43 | __PT_R10 = __PT_GPRS + 80 | |
44 | __PT_R11 = __PT_GPRS + 88 | |
45 | __PT_R12 = __PT_GPRS + 96 | |
46 | __PT_R13 = __PT_GPRS + 104 | |
47 | __PT_R14 = __PT_GPRS + 112 | |
48 | __PT_R15 = __PT_GPRS + 120 | |
1da177e4 | 49 | |
3a890380 | 50 | STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER |
1da177e4 | 51 | STACK_SIZE = 1 << STACK_SHIFT |
dc7ee00d | 52 | STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE |
1da177e4 | 53 | |
e5b98199 MS |
54 | _LPP_OFFSET = __LC_LPP |
55 | ||
ce3dc447 | 56 | .macro CHECK_STACK savearea |
63b12246 | 57 | #ifdef CONFIG_CHECK_STACK |
ce3dc447 | 58 | tml %r15,STACK_SIZE - CONFIG_STACK_GUARD |
c5328901 MS |
59 | lghi %r14,\savearea |
60 | jz stack_overflow | |
63b12246 | 61 | #endif |
63b12246 MS |
62 | .endm |
63 | ||
ce3dc447 MS |
64 | .macro CHECK_VMAP_STACK savearea,oklabel |
65 | #ifdef CONFIG_VMAP_STACK | |
66 | lgr %r14,%r15 | |
67 | nill %r14,0x10000 - STACK_SIZE | |
68 | oill %r14,STACK_INIT | |
69 | clg %r14,__LC_KERNEL_STACK | |
70 | je \oklabel | |
71 | clg %r14,__LC_ASYNC_STACK | |
72 | je \oklabel | |
73 | clg %r14,__LC_NODAT_STACK | |
74 | je \oklabel | |
75 | clg %r14,__LC_RESTART_STACK | |
76 | je \oklabel | |
77 | lghi %r14,\savearea | |
78 | j stack_overflow | |
79 | #else | |
80 | j \oklabel | |
81 | #endif | |
82 | .endm | |
83 | ||
473e66ba | 84 | .macro STCK savearea |
78f65709 HC |
85 | ALTERNATIVE ".insn s,0xb2050000,\savearea", \ |
86 | ".insn s,0xb27c0000,\savearea", 25 | |
473e66ba HC |
87 | .endm |
88 | ||
83abeffb HB |
89 | /* |
90 | * The TSTMSK macro generates a test-under-mask instruction by | |
91 | * calculating the memory offset for the specified mask value. | |
92 | * Mask value can be any constant. The macro shifts the mask | |
93 | * value to calculate the memory offset for the test-under-mask | |
94 | * instruction. | |
95 | */ | |
96 | .macro TSTMSK addr, mask, size=8, bytepos=0 | |
97 | .if (\bytepos < \size) && (\mask >> 8) | |
98 | .if (\mask & 0xff) | |
99 | .error "Mask exceeds byte boundary" | |
100 | .endif | |
101 | TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)" | |
102 | .exitm | |
103 | .endif | |
104 | .ifeq \mask | |
105 | .error "Mask must not be zero" | |
106 | .endif | |
107 | off = \size - \bytepos - 1 | |
108 | tm off+\addr, \mask | |
109 | .endm | |
110 | ||
d768bd89 | 111 | .macro BPOFF |
b058661a | 112 | ALTERNATIVE "", ".long 0xb2e8c000", 82 |
d768bd89 MS |
113 | .endm |
114 | ||
115 | .macro BPON | |
b058661a | 116 | ALTERNATIVE "", ".long 0xb2e8d000", 82 |
d768bd89 MS |
117 | .endm |
118 | ||
6b73044b | 119 | .macro BPENTER tif_ptr,tif_mask |
b058661a MS |
120 | ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \ |
121 | "", 82 | |
6b73044b MS |
122 | .endm |
123 | ||
124 | .macro BPEXIT tif_ptr,tif_mask | |
125 | TSTMSK \tif_ptr,\tif_mask | |
b058661a MS |
126 | ALTERNATIVE "jz .+8; .long 0xb2e8c000", \ |
127 | "jnz .+8; .long 0xb2e8d000", 82 | |
6b73044b MS |
128 | .endm |
129 | ||
6dd85fbb MS |
130 | GEN_BR_THUNK %r14 |
131 | GEN_BR_THUNK %r14,%r11 | |
f19fbd5e | 132 | |
860dba45 | 133 | .section .kprobes.text, "ax" |
46210c44 HC |
134 | .Ldummy: |
135 | /* | |
56e62a73 | 136 | * This nop exists only in order to avoid that __bpon starts at |
46210c44 HC |
137 | * the beginning of the kprobes text section. In that case we would |
138 | * have several symbols at the same address. E.g. objdump would take | |
139 | * an arbitrary symbol name when disassembling this code. | |
56e62a73 | 140 | * With the added nop in between the __bpon symbol is unique |
46210c44 HC |
141 | * again. |
142 | */ | |
143 | nop 0 | |
860dba45 | 144 | |
d768bd89 MS |
145 | ENTRY(__bpon) |
146 | .globl __bpon | |
147 | BPON | |
6dd85fbb | 148 | BR_EX %r14 |
26a374ae | 149 | ENDPROC(__bpon) |
d768bd89 | 150 | |
1da177e4 LT |
151 | /* |
152 | * Scheduler resume function, called by switch_to | |
153 | * gpr2 = (task_struct *) prev | |
154 | * gpr3 = (task_struct *) next | |
155 | * Returns: | |
156 | * gpr2 = prev | |
157 | */ | |
144d634a | 158 | ENTRY(__switch_to) |
eda0c6d6 | 159 | stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task |
3241d3eb HC |
160 | lghi %r4,__TASK_stack |
161 | lghi %r1,__TASK_thread | |
9fed920e | 162 | llill %r5,STACK_INIT |
3241d3eb | 163 | stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev |
9fed920e VG |
164 | lg %r15,0(%r4,%r3) # start of kernel stack of next |
165 | agr %r15,%r5 # end of kernel stack of next | |
eda0c6d6 | 166 | stg %r3,__LC_CURRENT # store task struct of next |
eda0c6d6 | 167 | stg %r15,__LC_KERNEL_STACK # store end of kernel stack |
3241d3eb HC |
168 | lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next |
169 | aghi %r3,__TASK_pid | |
170 | mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next | |
d3a73acb | 171 | lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task |
e5b98199 | 172 | ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 |
6dd85fbb | 173 | BR_EX %r14 |
26a374ae | 174 | ENDPROC(__switch_to) |
1da177e4 | 175 | |
d0fc4107 MS |
176 | #if IS_ENABLED(CONFIG_KVM) |
177 | /* | |
178 | * sie64a calling convention: | |
179 | * %r2 pointer to sie control block | |
180 | * %r3 guest register save area | |
181 | */ | |
182 | ENTRY(sie64a) | |
183 | stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers | |
6b73044b | 184 | lg %r12,__LC_CURRENT |
92fa7a13 MS |
185 | stg %r2,__SF_SIE_CONTROL(%r15) # save control block pointer |
186 | stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area | |
187 | xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 | |
188 | mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags | |
d0fc4107 MS |
189 | lmg %r0,%r13,0(%r3) # load guest gprs 0-13 |
190 | lg %r14,__LC_GMAP # get gmap pointer | |
191 | ltgr %r14,%r14 | |
192 | jz .Lsie_gmap | |
193 | lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce | |
194 | .Lsie_gmap: | |
92fa7a13 | 195 | lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer |
d0fc4107 MS |
196 | oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now |
197 | tm __SIE_PROG20+3(%r14),3 # last exit... | |
198 | jnz .Lsie_skip | |
83abeffb | 199 | TSTMSK __LC_CPU_FLAGS,_CIF_FPU |
d0fc4107 | 200 | jo .Lsie_skip # exit if fp/vx regs changed |
92fa7a13 | 201 | BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) |
c929500d | 202 | .Lsie_entry: |
d0fc4107 | 203 | sie 0(%r14) |
d768bd89 | 204 | BPOFF |
92fa7a13 | 205 | BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) |
d0fc4107 MS |
206 | .Lsie_skip: |
207 | ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE | |
87d59863 | 208 | lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce |
d0fc4107 MS |
209 | .Lsie_done: |
210 | # some program checks are suppressing. C code (e.g. do_protection_exception) | |
c0e7bb38 CB |
211 | # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There |
212 | # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable. | |
213 | # Other instructions between sie64a and .Lsie_done should not cause program | |
214 | # interrupts. So lets use 3 nops as a landing pad for all possible rewinds. | |
d0fc4107 | 215 | # See also .Lcleanup_sie |
c0e7bb38 CB |
216 | .Lrewind_pad6: |
217 | nopr 7 | |
218 | .Lrewind_pad4: | |
219 | nopr 7 | |
220 | .Lrewind_pad2: | |
221 | nopr 7 | |
d0fc4107 MS |
222 | .globl sie_exit |
223 | sie_exit: | |
92fa7a13 | 224 | lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area |
d0fc4107 | 225 | stmg %r0,%r13,0(%r14) # save guest gprs 0-13 |
7041d281 MS |
226 | xgr %r0,%r0 # clear guest registers to |
227 | xgr %r1,%r1 # prevent speculative use | |
7041d281 MS |
228 | xgr %r3,%r3 |
229 | xgr %r4,%r4 | |
230 | xgr %r5,%r5 | |
d0fc4107 | 231 | lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers |
92fa7a13 | 232 | lg %r2,__SF_SIE_REASON(%r15) # return exit reason code |
6dd85fbb | 233 | BR_EX %r14 |
d0fc4107 MS |
234 | .Lsie_fault: |
235 | lghi %r14,-EFAULT | |
92fa7a13 | 236 | stg %r14,__SF_SIE_REASON(%r15) # set exit reason code |
d0fc4107 MS |
237 | j sie_exit |
238 | ||
c0e7bb38 CB |
239 | EX_TABLE(.Lrewind_pad6,.Lsie_fault) |
240 | EX_TABLE(.Lrewind_pad4,.Lsie_fault) | |
241 | EX_TABLE(.Lrewind_pad2,.Lsie_fault) | |
d0fc4107 | 242 | EX_TABLE(sie_exit,.Lsie_fault) |
26a374ae | 243 | ENDPROC(sie64a) |
711f5df7 AV |
244 | EXPORT_SYMBOL(sie64a) |
245 | EXPORT_SYMBOL(sie_exit) | |
d0fc4107 MS |
246 | #endif |
247 | ||
1da177e4 LT |
248 | /* |
249 | * SVC interrupt handler routine. System calls are synchronous events and | |
7b7735c5 | 250 | * are entered with interrupts disabled. |
1da177e4 LT |
251 | */ |
252 | ||
144d634a | 253 | ENTRY(system_call) |
56e62a73 | 254 | stpt __LC_SYS_ENTER_TIMER |
c5328901 | 255 | stmg %r8,%r15,__LC_SAVE_AREA_SYNC |
d768bd89 | 256 | BPOFF |
56e62a73 | 257 | lghi %r14,0 |
86ed42f4 | 258 | .Lsysc_per: |
87d59863 | 259 | lctlg %c1,%c1,__LC_KERNEL_ASCE |
56e62a73 | 260 | lg %r12,__LC_CURRENT |
c5328901 | 261 | lg %r15,__LC_KERNEL_STACK |
9365965d | 262 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
56e62a73 SS |
263 | stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15) |
264 | BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP | |
d3f46896 CB |
265 | # clear user controlled register to prevent speculative use |
266 | xgr %r0,%r0 | |
56e62a73 SS |
267 | xgr %r1,%r1 |
268 | xgr %r4,%r4 | |
269 | xgr %r5,%r5 | |
270 | xgr %r6,%r6 | |
271 | xgr %r7,%r7 | |
272 | xgr %r8,%r8 | |
273 | xgr %r9,%r9 | |
274 | xgr %r10,%r10 | |
275 | xgr %r11,%r11 | |
276 | la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs | |
277 | lgr %r3,%r14 | |
278 | brasl %r14,__do_syscall | |
87d59863 | 279 | lctlg %c1,%c1,__LC_USER_ASCE |
56e62a73 SS |
280 | mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) |
281 | BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP | |
282 | lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) | |
c5328901 | 283 | stpt __LC_EXIT_TIMER |
0b0ed657 | 284 | b __LC_RETURN_LPSWE |
26a374ae | 285 | ENDPROC(system_call) |
1da177e4 LT |
286 | |
287 | # | |
288 | # a new process exits the kernel with ret_from_fork | |
289 | # | |
144d634a | 290 | ENTRY(ret_from_fork) |
56e62a73 SS |
291 | lgr %r3,%r11 |
292 | brasl %r14,__ret_from_fork | |
293 | lctlg %c1,%c1,__LC_USER_ASCE | |
294 | mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) | |
295 | BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP | |
296 | lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) | |
297 | stpt __LC_EXIT_TIMER | |
298 | b __LC_RETURN_LPSWE | |
26a374ae MS |
299 | ENDPROC(ret_from_fork) |
300 | ||
1da177e4 LT |
301 | /* |
302 | * Program check handler routine | |
303 | */ | |
304 | ||
144d634a | 305 | ENTRY(pgm_check_handler) |
56e62a73 | 306 | stpt __LC_SYS_ENTER_TIMER |
d768bd89 | 307 | BPOFF |
c5328901 | 308 | stmg %r8,%r15,__LC_SAVE_AREA_SYNC |
56e62a73 SS |
309 | lg %r12,__LC_CURRENT |
310 | lghi %r10,0 | |
c5328901 | 311 | lmg %r8,%r9,__LC_PGM_OLD_PSW |
87d59863 HC |
312 | tmhh %r8,0x0001 # coming from user space? |
313 | jno .Lpgm_skip_asce | |
314 | lctlg %c1,%c1,__LC_KERNEL_ASCE | |
56e62a73 | 315 | j 3f # -> fault in user space |
87d59863 | 316 | .Lpgm_skip_asce: |
d0fc4107 | 317 | #if IS_ENABLED(CONFIG_KVM) |
0a5e2ec2 | 318 | # cleanup critical section for program checks in sie64a |
d0fc4107 | 319 | lgr %r14,%r9 |
0b0ed657 SS |
320 | larl %r13,.Lsie_gmap |
321 | slgr %r14,%r13 | |
322 | lghi %r13,.Lsie_done - .Lsie_gmap | |
323 | clgr %r14,%r13 | |
0b38b5e1 | 324 | jhe 1f |
92fa7a13 | 325 | lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer |
0a5e2ec2 | 326 | ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE |
87d59863 | 327 | lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce |
0a5e2ec2 | 328 | larl %r9,sie_exit # skip forward to sie_exit |
56e62a73 | 329 | lghi %r10,_PIF_GUEST_FAULT |
d0fc4107 | 330 | #endif |
0b38b5e1 SS |
331 | 1: tmhh %r8,0x4000 # PER bit set in old PSW ? |
332 | jnz 2f # -> enabled, can't be a double fault | |
c5328901 | 333 | tm __LC_PGM_ILC+3,0x80 # check for per exception |
86ed42f4 | 334 | jnz .Lpgm_svcper # -> single stepped svc |
0b38b5e1 | 335 | 2: CHECK_STACK __LC_SAVE_AREA_SYNC |
dc7ee00d | 336 | aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) |
56e62a73 SS |
337 | # CHECK_VMAP_STACK branches to stack_overflow or 4f |
338 | CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f | |
339 | 3: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP | |
c5328901 | 340 | lg %r15,__LC_KERNEL_STACK |
56e62a73 SS |
341 | 4: la %r11,STACK_FRAME_OVERHEAD(%r15) |
342 | stg %r10,__PT_FLAGS(%r11) | |
343 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) | |
c5328901 | 344 | stmg %r0,%r7,__PT_R0(%r11) |
56e62a73 SS |
345 | mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC |
346 | stmg %r8,%r9,__PT_PSW(%r11) | |
347 | ||
7041d281 MS |
348 | # clear user controlled registers to prevent speculative use |
349 | xgr %r0,%r0 | |
350 | xgr %r1,%r1 | |
7041d281 MS |
351 | xgr %r3,%r3 |
352 | xgr %r4,%r4 | |
353 | xgr %r5,%r5 | |
354 | xgr %r6,%r6 | |
355 | xgr %r7,%r7 | |
56e62a73 SS |
356 | lgr %r2,%r11 |
357 | brasl %r14,__do_pgm_check | |
358 | tmhh %r8,0x0001 # returning to user space? | |
359 | jno .Lpgm_exit_kernel | |
360 | lctlg %c1,%c1,__LC_USER_ASCE | |
361 | BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP | |
0cd9b723 | 362 | stpt __LC_EXIT_TIMER |
56e62a73 SS |
363 | .Lpgm_exit_kernel: |
364 | mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15) | |
365 | lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15) | |
0cd9b723 | 366 | b __LC_RETURN_LPSWE |
1da177e4 | 367 | |
4ba069b8 | 368 | # |
c5328901 | 369 | # single stepped system call |
4ba069b8 | 370 | # |
86ed42f4 | 371 | .Lpgm_svcper: |
c5328901 | 372 | mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW |
86ed42f4 | 373 | larl %r14,.Lsysc_per |
c5328901 | 374 | stg %r14,__LC_RETURN_PSW+8 |
56e62a73 | 375 | lghi %r14,1 |
0b0ed657 | 376 | lpswe __LC_RETURN_PSW # branch to .Lsysc_per |
26a374ae | 377 | ENDPROC(pgm_check_handler) |
4ba069b8 | 378 | |
1da177e4 | 379 | /* |
56e62a73 | 380 | * Interrupt handler macro used for external and IO interrupts. |
1da177e4 | 381 | */ |
56e62a73 SS |
382 | .macro INT_HANDLER name,lc_old_psw,handler |
383 | ENTRY(\name) | |
473e66ba | 384 | STCK __LC_INT_CLOCK |
56e62a73 | 385 | stpt __LC_SYS_ENTER_TIMER |
d768bd89 | 386 | BPOFF |
c5328901 | 387 | stmg %r8,%r15,__LC_SAVE_AREA_ASYNC |
d5c352cd | 388 | lg %r12,__LC_CURRENT |
56e62a73 | 389 | lmg %r8,%r9,\lc_old_psw |
b0d31159 SS |
390 | tmhh %r8,0x0001 # interrupting from user ? |
391 | jnz 1f | |
392 | #if IS_ENABLED(CONFIG_KVM) | |
393 | lgr %r14,%r9 | |
394 | larl %r13,.Lsie_gmap | |
395 | slgr %r14,%r13 | |
396 | lghi %r13,.Lsie_done - .Lsie_gmap | |
397 | clgr %r14,%r13 | |
398 | jhe 0f | |
399 | lghi %r11,__LC_SAVE_AREA_ASYNC # inside critical section, do cleanup | |
400 | brasl %r14,.Lcleanup_sie | |
401 | #endif | |
402 | 0: CHECK_STACK __LC_SAVE_AREA_ASYNC | |
403 | lgr %r11,%r15 | |
404 | aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) | |
405 | stg %r11,__SF_BACKCHAIN(%r15) | |
406 | j 2f | |
407 | 1: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP | |
408 | lctlg %c1,%c1,__LC_KERNEL_ASCE | |
409 | lg %r15,__LC_KERNEL_STACK | |
410 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) | |
411 | 2: la %r11,STACK_FRAME_OVERHEAD(%r15) | |
c5328901 | 412 | stmg %r0,%r7,__PT_R0(%r11) |
7041d281 MS |
413 | # clear user controlled registers to prevent speculative use |
414 | xgr %r0,%r0 | |
415 | xgr %r1,%r1 | |
7041d281 MS |
416 | xgr %r3,%r3 |
417 | xgr %r4,%r4 | |
418 | xgr %r5,%r5 | |
419 | xgr %r6,%r6 | |
420 | xgr %r7,%r7 | |
421 | xgr %r10,%r10 | |
c5328901 MS |
422 | mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC |
423 | stmg %r8,%r9,__PT_PSW(%r11) | |
56e62a73 SS |
424 | tm %r8,0x0001 # coming from user space? |
425 | jno 1f | |
87d59863 | 426 | lctlg %c1,%c1,__LC_KERNEL_ASCE |
56e62a73 SS |
427 | 1: lgr %r2,%r11 # pass pointer to pt_regs |
428 | brasl %r14,\handler | |
c5328901 | 429 | mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) |
56e62a73 SS |
430 | tmhh %r8,0x0001 # returning to user ? |
431 | jno 2f | |
87d59863 | 432 | lctlg %c1,%c1,__LC_USER_ASCE |
6b73044b | 433 | BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP |
c5328901 | 434 | stpt __LC_EXIT_TIMER |
56e62a73 | 435 | 2: lmg %r0,%r15,__PT_R0(%r11) |
0b0ed657 | 436 | b __LC_RETURN_LPSWE |
56e62a73 SS |
437 | ENDPROC(\name) |
438 | .endm | |
916cda1a | 439 | |
56e62a73 SS |
440 | INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq |
441 | INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq | |
1da177e4 | 442 | |
4c1051e3 | 443 | /* |
0b0ed657 | 444 | * Load idle PSW. |
4c1051e3 MS |
445 | */ |
446 | ENTRY(psw_idle) | |
27f6b416 | 447 | stg %r3,__SF_EMPTY(%r15) |
56e62a73 | 448 | larl %r1,psw_idle_exit |
4c1051e3 | 449 | stg %r1,__SF_EMPTY+8(%r15) |
72d38b19 MS |
450 | larl %r1,smp_cpu_mtid |
451 | llgf %r1,0(%r1) | |
452 | ltgr %r1,%r1 | |
453 | jz .Lpsw_idle_stcctm | |
56e62a73 | 454 | .insn rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2) |
72d38b19 | 455 | .Lpsw_idle_stcctm: |
419123f9 | 456 | oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT |
d768bd89 | 457 | BPON |
27f6b416 MS |
458 | STCK __CLOCK_IDLE_ENTER(%r2) |
459 | stpt __TIMER_IDLE_ENTER(%r2) | |
4c1051e3 | 460 | lpswe __SF_EMPTY(%r15) |
56e62a73 SS |
461 | .globl psw_idle_exit |
462 | psw_idle_exit: | |
6dd85fbb | 463 | BR_EX %r14 |
26a374ae | 464 | ENDPROC(psw_idle) |
4c1051e3 | 465 | |
1da177e4 LT |
466 | /* |
467 | * Machine check handler routines | |
468 | */ | |
144d634a | 469 | ENTRY(mcck_int_handler) |
473e66ba | 470 | STCK __LC_MCCK_CLOCK |
d768bd89 | 471 | BPOFF |
3037a52f MS |
472 | la %r1,4095 # validate r1 |
473 | spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer | |
474 | sckc __LC_CLOCK_COMPARATOR # validate comparator | |
475 | lam %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs | |
476 | lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs | |
d5c352cd | 477 | lg %r12,__LC_CURRENT |
c5328901 | 478 | lmg %r8,%r9,__LC_MCK_OLD_PSW |
83abeffb | 479 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE |
86ed42f4 | 480 | jo .Lmcck_panic # yes -> rest of mcck code invalid |
3037a52f MS |
481 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID |
482 | jno .Lmcck_panic # control registers invalid -> panic | |
483 | la %r14,4095 | |
484 | lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs | |
485 | ptlb | |
2a2d7bef | 486 | lg %r11,__LC_MCESAD-4095(%r14) # extended machine check save area |
3037a52f MS |
487 | nill %r11,0xfc00 # MCESA_ORIGIN_MASK |
488 | TSTMSK __LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE | |
489 | jno 0f | |
490 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_GS_VALID | |
491 | jno 0f | |
492 | .insn rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC | |
493 | 0: l %r14,__LC_FP_CREG_SAVE_AREA-4095(%r14) | |
494 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_FC_VALID | |
495 | jo 0f | |
496 | sr %r14,%r14 | |
497 | 0: sfpc %r14 | |
498 | TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX | |
499 | jo 0f | |
500 | lghi %r14,__LC_FPREGS_SAVE_AREA | |
501 | ld %f0,0(%r14) | |
502 | ld %f1,8(%r14) | |
503 | ld %f2,16(%r14) | |
504 | ld %f3,24(%r14) | |
505 | ld %f4,32(%r14) | |
506 | ld %f5,40(%r14) | |
507 | ld %f6,48(%r14) | |
508 | ld %f7,56(%r14) | |
509 | ld %f8,64(%r14) | |
510 | ld %f9,72(%r14) | |
511 | ld %f10,80(%r14) | |
512 | ld %f11,88(%r14) | |
513 | ld %f12,96(%r14) | |
514 | ld %f13,104(%r14) | |
515 | ld %f14,112(%r14) | |
516 | ld %f15,120(%r14) | |
517 | j 1f | |
518 | 0: VLM %v0,%v15,0,%r11 | |
519 | VLM %v16,%v31,256,%r11 | |
520 | 1: lghi %r14,__LC_CPU_TIMER_SAVE_AREA | |
c5328901 | 521 | mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) |
83abeffb | 522 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID |
c5328901 | 523 | jo 3f |
56e62a73 SS |
524 | la %r14,__LC_SYS_ENTER_TIMER |
525 | clc 0(8,%r14),__LC_EXIT_TIMER | |
c5328901 | 526 | jl 1f |
63b12246 | 527 | la %r14,__LC_EXIT_TIMER |
c5328901 MS |
528 | 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER |
529 | jl 2f | |
63b12246 | 530 | la %r14,__LC_LAST_UPDATE_TIMER |
c5328901 | 531 | 2: spt 0(%r14) |
6377981f | 532 | mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) |
3037a52f MS |
533 | 3: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID |
534 | jno .Lmcck_panic | |
535 | tmhh %r8,0x0001 # interrupting from user ? | |
536 | jnz 4f | |
537 | TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID | |
538 | jno .Lmcck_panic | |
ce3dc447 | 539 | 4: ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off |
b0d31159 SS |
540 | tmhh %r8,0x0001 # interrupting from user ? |
541 | jnz .Lmcck_user | |
542 | #if IS_ENABLED(CONFIG_KVM) | |
543 | lgr %r14,%r9 | |
544 | larl %r13,.Lsie_gmap | |
545 | slgr %r14,%r13 | |
546 | lghi %r13,.Lsie_done - .Lsie_gmap | |
547 | clgr %r14,%r13 | |
548 | jhe .Lmcck_stack | |
549 | lghi %r11,__LC_GPREGS_SAVE_AREA+64 # inside critical section, do cleanup | |
550 | brasl %r14,.Lcleanup_sie | |
551 | .Lmcck_stack: | |
552 | #endif | |
553 | CHECK_STACK __LC_GPREGS_SAVE_AREA+64 | |
554 | lgr %r11,%r15 | |
555 | aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) | |
556 | stg %r11,__SF_BACKCHAIN(%r15) | |
557 | j 5f | |
558 | .Lmcck_user: | |
559 | BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP | |
560 | lctlg %c1,%c1,__LC_KERNEL_ASCE | |
561 | lg %r15,__LC_KERNEL_STACK | |
562 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) | |
563 | 5: la %r11,STACK_FRAME_OVERHEAD(%r15) | |
86ed42f4 | 564 | .Lmcck_skip: |
6551fbdf MS |
565 | lghi %r14,__LC_GPREGS_SAVE_AREA+64 |
566 | stmg %r0,%r7,__PT_R0(%r11) | |
7041d281 MS |
567 | # clear user controlled registers to prevent speculative use |
568 | xgr %r0,%r0 | |
569 | xgr %r1,%r1 | |
7041d281 MS |
570 | xgr %r3,%r3 |
571 | xgr %r4,%r4 | |
572 | xgr %r5,%r5 | |
573 | xgr %r6,%r6 | |
574 | xgr %r7,%r7 | |
575 | xgr %r10,%r10 | |
6551fbdf | 576 | mvc __PT_R8(64,%r11),0(%r14) |
c5328901 | 577 | stmg %r8,%r9,__PT_PSW(%r11) |
87d59863 HC |
578 | la %r14,4095 |
579 | mvc __PT_CR1(8,%r11),__LC_CREGS_SAVE_AREA-4095+8(%r14) | |
d3a73acb | 580 | xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) |
c5328901 MS |
581 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
582 | lgr %r2,%r11 # pass pointer to pt_regs | |
77fa2245 | 583 | brasl %r14,s390_do_machine_check |
0b0ed657 SS |
584 | cghi %r2,0 |
585 | je .Lmcck_return | |
77fa2245 | 586 | lg %r1,__LC_KERNEL_STACK # switch to kernel stack |
c5328901 MS |
587 | mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) |
588 | xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) | |
589 | la %r11,STACK_FRAME_OVERHEAD(%r1) | |
77fa2245 | 590 | lgr %r15,%r1 |
77fa2245 | 591 | brasl %r14,s390_handle_mcck |
86ed42f4 | 592 | .Lmcck_return: |
87d59863 | 593 | lctlg %c1,%c1,__PT_CR1(%r11) |
c5328901 MS |
594 | lmg %r0,%r10,__PT_R0(%r11) |
595 | mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW | |
63b12246 MS |
596 | tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? |
597 | jno 0f | |
6b73044b | 598 | BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP |
63b12246 | 599 | stpt __LC_EXIT_TIMER |
c5328901 | 600 | 0: lmg %r11,%r15,__PT_R11(%r11) |
0b38b5e1 | 601 | b __LC_RETURN_MCCK_LPSWE |
c5328901 | 602 | |
86ed42f4 | 603 | .Lmcck_panic: |
ce3dc447 | 604 | lg %r15,__LC_NODAT_STACK |
ce4dda3f | 605 | la %r11,STACK_FRAME_OVERHEAD(%r15) |
86ed42f4 | 606 | j .Lmcck_skip |
26a374ae | 607 | ENDPROC(mcck_int_handler) |
1da177e4 | 608 | |
7dd6b334 MH |
609 | # |
610 | # PSW restart interrupt handler | |
611 | # | |
8b646bd7 | 612 | ENTRY(restart_int_handler) |
e5b98199 MS |
613 | ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 |
614 | stg %r15,__LC_SAVE_AREA_RESTART | |
8b646bd7 | 615 | lg %r15,__LC_RESTART_STACK |
ce3dc447 MS |
616 | xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15) |
617 | stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15) | |
618 | mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART | |
619 | mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW | |
8b646bd7 | 620 | xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) |
fbe76568 HC |
621 | lg %r1,__LC_RESTART_FN # load fn, parm & source cpu |
622 | lg %r2,__LC_RESTART_DATA | |
623 | lg %r3,__LC_RESTART_SOURCE | |
8b646bd7 MS |
624 | ltgr %r3,%r3 # test source cpu address |
625 | jm 1f # negative -> skip source stop | |
eb546195 | 626 | 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu |
8b646bd7 MS |
627 | brc 10,0b # wait for status stored |
628 | 1: basr %r14,%r1 # call function | |
629 | stap __SF_EMPTY(%r15) # store cpu address | |
630 | llgh %r3,__SF_EMPTY(%r15) | |
eb546195 | 631 | 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu |
8b646bd7 MS |
632 | brc 2,2b |
633 | 3: j 3b | |
26a374ae | 634 | ENDPROC(restart_int_handler) |
7dd6b334 | 635 | |
860dba45 MS |
636 | .section .kprobes.text, "ax" |
637 | ||
ce3dc447 | 638 | #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK) |
1da177e4 LT |
639 | /* |
640 | * The synchronous or the asynchronous stack overflowed. We are dead. | |
641 | * No need to properly save the registers, we are going to panic anyway. | |
642 | * Setup a pt_regs so that show_trace can provide a good call trace. | |
643 | */ | |
26a374ae | 644 | ENTRY(stack_overflow) |
ce3dc447 | 645 | lg %r15,__LC_NODAT_STACK # change to panic stack |
dc7ee00d | 646 | la %r11,STACK_FRAME_OVERHEAD(%r15) |
c5328901 MS |
647 | stmg %r0,%r7,__PT_R0(%r11) |
648 | stmg %r8,%r9,__PT_PSW(%r11) | |
649 | mvc __PT_R8(64,%r11),0(%r14) | |
650 | stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 | |
c5328901 MS |
651 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
652 | lgr %r2,%r11 # pass pointer to pt_regs | |
1da177e4 | 653 | jg kernel_stack_overflow |
26a374ae | 654 | ENDPROC(stack_overflow) |
1da177e4 LT |
655 | #endif |
656 | ||
d0fc4107 | 657 | #if IS_ENABLED(CONFIG_KVM) |
d0fc4107 | 658 | .Lcleanup_sie: |
0b0ed657 SS |
659 | cghi %r11,__LC_SAVE_AREA_ASYNC #Is this in normal interrupt? |
660 | je 1f | |
661 | larl %r13,.Lsie_entry | |
662 | slgr %r9,%r13 | |
663 | larl %r13,.Lsie_skip | |
664 | clgr %r9,%r13 | |
665 | jh 1f | |
666 | oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST | |
667 | 1: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) | |
92fa7a13 | 668 | lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer |
e22cf8ca | 669 | ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE |
87d59863 | 670 | lctlg %c1,%c1,__LC_KERNEL_ASCE |
d0fc4107 | 671 | larl %r9,sie_exit # skip forward to sie_exit |
891f6a72 | 672 | BR_EX %r14,%r11 |
1da177e4 | 673 | |
603d1a50 | 674 | #endif |
a876cb3f | 675 | .section .rodata, "a" |
ff4a742d | 676 | #define SYSCALL(esame,emu) .quad __s390x_ ## esame |
9bf1226b | 677 | .globl sys_call_table |
1da177e4 | 678 | sys_call_table: |
4381f9f1 | 679 | #include "asm/syscall_table.h" |
1da177e4 LT |
680 | #undef SYSCALL |
681 | ||
347a8dc3 | 682 | #ifdef CONFIG_COMPAT |
1da177e4 | 683 | |
ff4a742d | 684 | #define SYSCALL(esame,emu) .quad __s390_ ## emu |
61649881 | 685 | .globl sys_call_table_emu |
1da177e4 | 686 | sys_call_table_emu: |
4381f9f1 | 687 | #include "asm/syscall_table.h" |
1da177e4 LT |
688 | #undef SYSCALL |
689 | #endif |