Merge tag 'pci-v6.16-fixes-3' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci
[linux-block.git] / arch / s390 / kernel / kprobes.c
CommitLineData
a17ae4c3 1// SPDX-License-Identifier: GPL-2.0+
4ba069b8
MG
2/*
3 * Kernel Probes (KProbes)
4 *
a53c8fab 5 * Copyright IBM Corp. 2002, 2006
4ba069b8
MG
6 *
7 * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
8 */
9
9c89bb8e
MH
10#define pr_fmt(fmt) "kprobes: " fmt
11
4ba069b8
MG
12#include <linux/kprobes.h>
13#include <linux/ptrace.h>
14#include <linux/preempt.h>
15#include <linux/stop_machine.h>
679b110b 16#include <linux/cpufeature.h>
1eeb66a1 17#include <linux/kdebug.h>
a2b53673 18#include <linux/uaccess.h>
dcc096c5 19#include <linux/extable.h>
4ba069b8 20#include <linux/module.h>
5a0e3ad6 21#include <linux/slab.h>
adb45839 22#include <linux/hardirq.h>
c933146a 23#include <linux/ftrace.h>
12af2b83 24#include <linux/execmem.h>
30799152 25#include <asm/text-patching.h>
e6c7c630 26#include <asm/set_memory.h>
a882b3b0
HC
27#include <asm/sections.h>
28#include <asm/dis.h>
b61e1f32 29#include "entry.h"
4ba069b8 30
4a188635 31DEFINE_PER_CPU(struct kprobe *, current_kprobe);
4ba069b8
MG
32DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
33
4a188635 34struct kretprobe_blackpoint kretprobe_blacklist[] = { };
f438d914 35
6c6687a4
HC
36void *alloc_insn_page(void)
37{
38 void *page;
39
12af2b83 40 page = execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE);
6c6687a4
HC
41 if (!page)
42 return NULL;
f9b2d96c 43 set_memory_rox((unsigned long)page, 1);
6c6687a4
HC
44 return page;
45}
46
7a5388de 47static void copy_instruction(struct kprobe *p)
63c40436 48{
6c6687a4 49 kprobe_opcode_t insn[MAX_INSN_SIZE];
63c40436
HC
50 s64 disp, new_disp;
51 u64 addr, new_addr;
6c6687a4 52 unsigned int len;
63c40436 53
6c6687a4
HC
54 len = insn_length(*p->addr >> 8);
55 memcpy(&insn, p->addr, len);
56 p->opcode = insn[0];
57 if (probe_is_insn_relative_long(&insn[0])) {
58 /*
59 * For pc-relative instructions in RIL-b or RIL-c format patch
d890e6af
HC
60 * the RI2 displacement field. The insn slot for the to be
61 * patched instruction is within the same 4GB area like the
62 * original instruction. Therefore the new displacement will
63 * always fit.
6c6687a4
HC
64 */
65 disp = *(s32 *)&insn[1];
66 addr = (u64)(unsigned long)p->addr;
67 new_addr = (u64)(unsigned long)p->ainsn.insn;
68 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
69 *(s32 *)&insn[1] = new_disp;
70 }
71 s390_kernel_write(p->ainsn.insn, &insn, len);
63c40436 72}
7a5388de 73NOKPROBE_SYMBOL(copy_instruction);
63c40436 74
4df898dc
SS
75/* Check if paddr is at an instruction boundary */
76static bool can_probe(unsigned long paddr)
77{
78 unsigned long addr, offset = 0;
79 kprobe_opcode_t insn;
80 struct kprobe *kp;
81
82 if (paddr & 0x01)
83 return false;
84
85 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
86 return false;
87
88 /* Decode instructions */
89 addr = paddr - offset;
90 while (addr < paddr) {
91 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn)))
92 return false;
93
94 if (insn >> 8 == 0) {
95 if (insn != BREAKPOINT_INSTRUCTION) {
96 /*
97 * Note that QEMU inserts opcode 0x0000 to implement
98 * software breakpoints for guests. Since the size of
99 * the original instruction is unknown, stop following
100 * instructions and prevent setting a kprobe.
101 */
102 return false;
103 }
104 /*
105 * Check if the instruction has been modified by another
106 * kprobe, in which case the original instruction is
107 * decoded.
108 */
109 kp = get_kprobe((void *)addr);
110 if (!kp) {
111 /* not a kprobe */
112 return false;
113 }
114 insn = kp->opcode;
115 }
116 addr += insn_length(insn >> 8);
117 }
118 return addr == paddr;
119}
120
7a5388de 121int arch_prepare_kprobe(struct kprobe *p)
ba640a59 122{
4df898dc 123 if (!can_probe((unsigned long)p->addr))
ba640a59 124 return -EINVAL;
ba640a59 125 /* Make sure the probe isn't going on a difficult instruction */
975fab17 126 if (probe_is_prohibited_opcode(p->addr))
ba640a59 127 return -EINVAL;
d890e6af
HC
128 p->ainsn.insn = get_insn_slot();
129 if (!p->ainsn.insn)
63c40436 130 return -ENOMEM;
63c40436 131 copy_instruction(p);
ba640a59 132 return 0;
4ba069b8 133}
7a5388de 134NOKPROBE_SYMBOL(arch_prepare_kprobe);
4ba069b8 135
c933146a
HC
136struct swap_insn_args {
137 struct kprobe *p;
138 unsigned int arm_kprobe : 1;
5a8b589f
MS
139};
140
7a5388de 141static int swap_instruction(void *data)
4ba069b8 142{
c933146a 143 struct swap_insn_args *args = data;
c933146a 144 struct kprobe *p = args->p;
657480d9
SS
145 u16 opc;
146
147 opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
148 s390_kernel_write(p->addr, &opc, sizeof(opc));
5a8b589f 149 return 0;
4ba069b8 150}
7a5388de 151NOKPROBE_SYMBOL(swap_instruction);
4ba069b8 152
7a5388de 153void arch_arm_kprobe(struct kprobe *p)
4ba069b8 154{
c933146a 155 struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
4ba069b8 156
679b110b 157 if (cpu_has_seq_insn()) {
30799152
VG
158 swap_instruction(&args);
159 text_poke_sync();
160 } else {
161 stop_machine_cpuslocked(swap_instruction, &args, NULL);
162 }
4ba069b8 163}
7a5388de 164NOKPROBE_SYMBOL(arch_arm_kprobe);
4ba069b8 165
7a5388de 166void arch_disarm_kprobe(struct kprobe *p)
4ba069b8 167{
c933146a 168 struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
4ba069b8 169
679b110b 170 if (cpu_has_seq_insn()) {
30799152
VG
171 swap_instruction(&args);
172 text_poke_sync();
173 } else {
174 stop_machine_cpuslocked(swap_instruction, &args, NULL);
175 }
4ba069b8 176}
7a5388de 177NOKPROBE_SYMBOL(arch_disarm_kprobe);
4ba069b8 178
7a5388de 179void arch_remove_kprobe(struct kprobe *p)
4ba069b8 180{
d890e6af
HC
181 if (!p->ainsn.insn)
182 return;
183 free_insn_slot(p->ainsn.insn, 0);
184 p->ainsn.insn = NULL;
4ba069b8 185}
7a5388de 186NOKPROBE_SYMBOL(arch_remove_kprobe);
4ba069b8 187
7a5388de
HC
188static void enable_singlestep(struct kprobe_ctlblk *kcb,
189 struct pt_regs *regs,
190 unsigned long ip)
4ba069b8 191{
4b440e01 192 union {
527618ab 193 struct ctlreg regs[3];
4b440e01 194 struct {
527618ab
HC
195 struct ctlreg control;
196 struct ctlreg start;
197 struct ctlreg end;
4b440e01
HC
198 };
199 } per_kprobe;
4ba069b8 200
5e9a2692 201 /* Set up the PER control registers %cr9-%cr11 */
527618ab
HC
202 per_kprobe.control.val = PER_EVENT_IFETCH;
203 per_kprobe.start.val = ip;
204 per_kprobe.end.val = ip;
4ba069b8 205
fc0a1fea 206 /* Save control regs and psw mask */
80725978 207 __local_ctl_store(9, 11, kcb->kprobe_saved_ctl);
fc0a1fea
MS
208 kcb->kprobe_saved_imask = regs->psw.mask &
209 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
210
211 /* Set PER control regs, turns on single step for the given address */
4b440e01 212 __local_ctl_load(9, 11, per_kprobe.regs);
4ba069b8 213 regs->psw.mask |= PSW_MASK_PER;
adb45839 214 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
fecc868a 215 regs->psw.addr = ip;
4ba069b8 216}
7a5388de 217NOKPROBE_SYMBOL(enable_singlestep);
4ba069b8 218
7a5388de
HC
219static void disable_singlestep(struct kprobe_ctlblk *kcb,
220 struct pt_regs *regs,
221 unsigned long ip)
fc0a1fea
MS
222{
223 /* Restore control regs and psw mask, set new psw address */
80725978 224 __local_ctl_load(9, 11, kcb->kprobe_saved_ctl);
fc0a1fea
MS
225 regs->psw.mask &= ~PSW_MASK_PER;
226 regs->psw.mask |= kcb->kprobe_saved_imask;
fecc868a 227 regs->psw.addr = ip;
fc0a1fea 228}
7a5388de 229NOKPROBE_SYMBOL(disable_singlestep);
fc0a1fea 230
b9599798
MS
231/*
232 * Activate a kprobe by storing its pointer to current_kprobe. The
233 * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
234 * two kprobes can be active, see KPROBE_REENTER.
235 */
7a5388de 236static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
4ba069b8 237{
eb7e7d76 238 kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
4ba069b8 239 kcb->prev_kprobe.status = kcb->kprobe_status;
eb7e7d76 240 __this_cpu_write(current_kprobe, p);
4ba069b8 241}
7a5388de 242NOKPROBE_SYMBOL(push_kprobe);
4ba069b8 243
b9599798
MS
244/*
245 * Deactivate a kprobe by backing up to the previous state. If the
246 * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
247 * for any other state prev_kprobe.kp will be NULL.
248 */
7a5388de 249static void pop_kprobe(struct kprobe_ctlblk *kcb)
4ba069b8 250{
eb7e7d76 251 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
4ba069b8 252 kcb->kprobe_status = kcb->prev_kprobe.status;
cd579539 253 kcb->prev_kprobe.kp = NULL;
4ba069b8 254}
7a5388de 255NOKPROBE_SYMBOL(pop_kprobe);
4ba069b8 256
7a5388de 257static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
0e917cc3
MS
258{
259 switch (kcb->kprobe_status) {
260 case KPROBE_HIT_SSDONE:
261 case KPROBE_HIT_ACTIVE:
262 kprobes_inc_nmissed_count(p);
263 break;
264 case KPROBE_HIT_SS:
265 case KPROBE_REENTER:
266 default:
267 /*
268 * A kprobe on the code path to single step an instruction
269 * is a BUG. The code path resides in the .kprobes.text
270 * section and is executed with interrupts disabled.
271 */
9c89bb8e 272 pr_err("Failed to recover from reentered kprobes.\n");
0e917cc3
MS
273 dump_kprobe(p);
274 BUG();
275 }
276}
7a5388de 277NOKPROBE_SYMBOL(kprobe_reenter_check);
0e917cc3 278
7a5388de 279static int kprobe_handler(struct pt_regs *regs)
4ba069b8 280{
4ba069b8 281 struct kprobe_ctlblk *kcb;
0e917cc3 282 struct kprobe *p;
4ba069b8
MG
283
284 /*
0e917cc3
MS
285 * We want to disable preemption for the entire duration of kprobe
286 * processing. That includes the calls to the pre/post handlers
287 * and single stepping the kprobe instruction.
4ba069b8
MG
288 */
289 preempt_disable();
290 kcb = get_kprobe_ctlblk();
9cb1ccec 291 p = get_kprobe((void *)(regs->psw.addr - 2));
4ba069b8 292
0e917cc3
MS
293 if (p) {
294 if (kprobe_running()) {
b9599798
MS
295 /*
296 * We have hit a kprobe while another is still
297 * active. This can happen in the pre and post
298 * handler. Single step the instruction of the
299 * new probe but do not call any handler function
300 * of this secondary kprobe.
301 * push_kprobe and pop_kprobe saves and restores
302 * the currently active kprobe.
4ba069b8 303 */
0e917cc3 304 kprobe_reenter_check(kcb, p);
b9599798 305 push_kprobe(kcb, p);
4ba069b8 306 kcb->kprobe_status = KPROBE_REENTER;
4ba069b8 307 } else {
0e917cc3
MS
308 /*
309 * If we have no pre-handler or it returned 0, we
310 * continue with single stepping. If we have a
311 * pre-handler and it returned non-zero, it prepped
fc682f7b
MH
312 * for changing execution path, so get out doing
313 * nothing more here.
0e917cc3
MS
314 */
315 push_kprobe(kcb, p);
316 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
cce188bd
MH
317 if (p->pre_handler && p->pre_handler(p, regs)) {
318 pop_kprobe(kcb);
319 preempt_enable_no_resched();
0e917cc3 320 return 1;
cce188bd 321 }
0e917cc3 322 kcb->kprobe_status = KPROBE_HIT_SS;
4ba069b8 323 }
0e917cc3 324 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
4ba069b8 325 return 1;
0e917cc3
MS
326 } /* else:
327 * No kprobe at this address and no active kprobe. The trap has
328 * not been caused by a kprobe breakpoint. The race of breakpoint
329 * vs. kprobe remove does not exist because on s390 as we use
330 * stop_machine to arm/disarm the breakpoints.
331 */
4ba069b8 332 preempt_enable_no_resched();
0e917cc3 333 return 0;
4ba069b8 334}
7a5388de 335NOKPROBE_SYMBOL(kprobe_handler);
4ba069b8 336
4ba069b8
MG
337/*
338 * Called after single-stepping. p->addr is the address of the
339 * instruction whose first byte has been replaced by the "breakpoint"
340 * instruction. To avoid the SMP problems that can occur when we
341 * temporarily put back the original opcode to single-step, we
342 * single-stepped a copy of the instruction. The address of this
343 * copy is p->ainsn.insn.
344 */
7a5388de 345static void resume_execution(struct kprobe *p, struct pt_regs *regs)
4ba069b8
MG
346{
347 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
9cb1ccec 348 unsigned long ip = regs->psw.addr;
975fab17 349 int fixup = probe_get_fixup_type(p->ainsn.insn);
4ba069b8 350
ba640a59 351 if (fixup & FIXUP_PSW_NORMAL)
fc0a1fea 352 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
4ba069b8 353
ba640a59 354 if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
a882b3b0 355 int ilen = insn_length(p->ainsn.insn[0] >> 8);
ba640a59
MS
356 if (ip - (unsigned long) p->ainsn.insn == ilen)
357 ip = (unsigned long) p->addr + ilen;
358 }
4ba069b8 359
ba640a59
MS
360 if (fixup & FIXUP_RETURN_REGISTER) {
361 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
362 regs->gprs[reg] += (unsigned long) p->addr -
363 (unsigned long) p->ainsn.insn;
364 }
4ba069b8 365
fc0a1fea 366 disable_singlestep(kcb, regs, ip);
4ba069b8 367}
7a5388de 368NOKPROBE_SYMBOL(resume_execution);
4ba069b8 369
7a5388de 370static int post_kprobe_handler(struct pt_regs *regs)
4ba069b8 371{
4ba069b8 372 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
4a188635 373 struct kprobe *p = kprobe_running();
4ba069b8 374
4a188635 375 if (!p)
4ba069b8
MG
376 return 0;
377
42e19e6f 378 resume_execution(p, regs);
4a188635 379 if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
4ba069b8 380 kcb->kprobe_status = KPROBE_HIT_SSDONE;
4a188635 381 p->post_handler(p, regs, 0);
4ba069b8 382 }
b9599798 383 pop_kprobe(kcb);
4ba069b8
MG
384 preempt_enable_no_resched();
385
386 /*
387 * if somebody else is singlestepping across a probe point, psw mask
388 * will have PER set, in which case, continue the remaining processing
389 * of do_single_step, as if this is not a probe hit.
390 */
4a188635 391 if (regs->psw.mask & PSW_MASK_PER)
4ba069b8 392 return 0;
4ba069b8
MG
393
394 return 1;
395}
7a5388de 396NOKPROBE_SYMBOL(post_kprobe_handler);
4ba069b8 397
7a5388de 398static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
4ba069b8 399{
4ba069b8 400 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
4a188635 401 struct kprobe *p = kprobe_running();
4ba069b8
MG
402
403 switch(kcb->kprobe_status) {
4ba069b8
MG
404 case KPROBE_HIT_SS:
405 case KPROBE_REENTER:
406 /*
407 * We are here because the instruction being single
408 * stepped caused a page fault. We reset the current
409 * kprobe and the nip points back to the probe address
410 * and allow the page fault handler to continue as a
411 * normal page fault.
412 */
4a188635 413 disable_singlestep(kcb, regs, (unsigned long) p->addr);
b9599798 414 pop_kprobe(kcb);
4ba069b8
MG
415 preempt_enable_no_resched();
416 break;
417 case KPROBE_HIT_ACTIVE:
418 case KPROBE_HIT_SSDONE:
4ba069b8
MG
419 /*
420 * In case the user-specified fault handler returned
421 * zero, try to fix up.
422 */
46fee16f 423 if (fixup_exception(regs))
4ba069b8 424 return 1;
4ba069b8
MG
425 /*
426 * fixup_exception() could not handle it,
427 * Let do_page_fault() fix it.
428 */
429 break;
430 default:
431 break;
432 }
433 return 0;
434}
7a5388de 435NOKPROBE_SYMBOL(kprobe_trap_handler);
4ba069b8 436
7a5388de 437int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
adb45839
MS
438{
439 int ret;
440
441 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
442 local_irq_disable();
443 ret = kprobe_trap_handler(regs, trapnr);
444 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
445 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
446 return ret;
447}
7a5388de 448NOKPROBE_SYMBOL(kprobe_fault_handler);
adb45839 449
4ba069b8
MG
450/*
451 * Wrapper routine to for handling exceptions.
452 */
7a5388de
HC
453int kprobe_exceptions_notify(struct notifier_block *self,
454 unsigned long val, void *data)
4ba069b8 455{
4a188635 456 struct die_args *args = (struct die_args *) data;
adb45839 457 struct pt_regs *regs = args->regs;
4ba069b8
MG
458 int ret = NOTIFY_DONE;
459
adb45839
MS
460 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
461 local_irq_disable();
462
4ba069b8
MG
463 switch (val) {
464 case DIE_BPT:
4a188635 465 if (kprobe_handler(regs))
4ba069b8
MG
466 ret = NOTIFY_STOP;
467 break;
468 case DIE_SSTEP:
4a188635 469 if (post_kprobe_handler(regs))
4ba069b8
MG
470 ret = NOTIFY_STOP;
471 break;
472 case DIE_TRAP:
adb45839 473 if (!preemptible() && kprobe_running() &&
4a188635 474 kprobe_trap_handler(regs, args->trapnr))
4ba069b8 475 ret = NOTIFY_STOP;
4ba069b8
MG
476 break;
477 default:
478 break;
479 }
adb45839
MS
480
481 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
482 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
483
4ba069b8
MG
484 return ret;
485}
7a5388de 486NOKPROBE_SYMBOL(kprobe_exceptions_notify);
4ba069b8 487
4ba069b8
MG
488int __init arch_init_kprobes(void)
489{
63bf38ff 490 return 0;
4ba069b8 491}
bf8f6e5b 492
45c9f2b8
VG
493int __init arch_populate_kprobe_blacklist(void)
494{
495 return kprobe_add_area_blacklist((unsigned long)__irqentry_text_start,
496 (unsigned long)__irqentry_text_end);
497}
498
7a5388de 499int arch_trampoline_kprobe(struct kprobe *p)
bf8f6e5b 500{
63bf38ff 501 return 0;
bf8f6e5b 502}
7a5388de 503NOKPROBE_SYMBOL(arch_trampoline_kprobe);