arch/s390/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright IBM Corp. 2002, 2006
   6  *
   7  * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
   8  */
   9
  10 #define pr_fmt(fmt) "kprobes: " fmt
  11
  12 #include <linux/kprobes.h>
  13 #include <linux/ptrace.h>
  14 #include <linux/preempt.h>
  15 #include <linux/stop_machine.h>
  16 #include <linux/cpufeature.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/extable.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/ftrace.h>
  24 #include <linux/execmem.h>
  25 #include <asm/text-patching.h>
  26 #include <asm/set_memory.h>
  27 #include <asm/sections.h>
  28 #include <asm/dis.h>
  29 #include "entry.h"
  30
  31 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  32 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  33
  34 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
  35
  36 void *alloc_insn_page(void)
  37 {
  38         void *page;
  39
  40         page = execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE);
  41         if (!page)
  42                 return NULL;
  43         set_memory_rox((unsigned long)page, 1);
  44         return page;
  45 }
  46
  47 static void copy_instruction(struct kprobe *p)
  48 {
  49         kprobe_opcode_t insn[MAX_INSN_SIZE];
  50         s64 disp, new_disp;
  51         u64 addr, new_addr;
  52         unsigned int len;
  53
  54         len = insn_length(*p->addr >> 8);
  55         memcpy(&insn, p->addr, len);
  56         p->opcode = insn[0];
  57         if (probe_is_insn_relative_long(&insn[0])) {
  58                 /*
  59                  * For pc-relative instructions in RIL-b or RIL-c format patch
  60                  * the RI2 displacement field. The insn slot for the to be
  61                  * patched instruction is within the same 4GB area like the
  62                  * original instruction. Therefore the new displacement will
  63                  * always fit.
  64                  */
  65                 disp = *(s32 *)&insn[1];
  66                 addr = (u64)(unsigned long)p->addr;
  67                 new_addr = (u64)(unsigned long)p->ainsn.insn;
  68                 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
  69                 *(s32 *)&insn[1] = new_disp;
  70         }
  71         s390_kernel_write(p->ainsn.insn, &insn, len);
  72 }
  73 NOKPROBE_SYMBOL(copy_instruction);
  74
  75 /* Check if paddr is at an instruction boundary */
  76 static bool can_probe(unsigned long paddr)
  77 {
  78         unsigned long addr, offset = 0;
  79         kprobe_opcode_t insn;
  80         struct kprobe *kp;
  81
  82         if (paddr & 0x01)
  83                 return false;
  84
  85         if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
  86                 return false;
  87
  88         /* Decode instructions */
  89         addr = paddr - offset;
  90         while (addr < paddr) {
  91                 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn)))
  92                         return false;
  93
  94                 if (insn >> 8 == 0) {
  95                         if (insn != BREAKPOINT_INSTRUCTION) {
  96                                 /*
  97                                  * Note that QEMU inserts opcode 0x0000 to implement
  98                                  * software breakpoints for guests. Since the size of
  99                                  * the original instruction is unknown, stop following
 100                                  * instructions and prevent setting a kprobe.
 101                                  */
 102                                 return false;
 103                         }
 104                         /*
 105                          * Check if the instruction has been modified by another
 106                          * kprobe, in which case the original instruction is
 107                          * decoded.
 108                          */
 109                         kp = get_kprobe((void *)addr);
 110                         if (!kp) {
 111                                 /* not a kprobe */
 112                                 return false;
 113                         }
 114                         insn = kp->opcode;
 115                 }
 116                 addr += insn_length(insn >> 8);
 117         }
 118         return addr == paddr;
 119 }
 120
 121 int arch_prepare_kprobe(struct kprobe *p)
 122 {
 123         if (!can_probe((unsigned long)p->addr))
 124                 return -EINVAL;
 125         /* Make sure the probe isn't going on a difficult instruction */
 126         if (probe_is_prohibited_opcode(p->addr))
 127                 return -EINVAL;
 128         p->ainsn.insn = get_insn_slot();
 129         if (!p->ainsn.insn)
 130                 return -ENOMEM;
 131         copy_instruction(p);
 132         return 0;
 133 }
 134 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 135
 136 struct swap_insn_args {
 137         struct kprobe *p;
 138         unsigned int arm_kprobe : 1;
 139 };
 140
 141 static int swap_instruction(void *data)
 142 {
 143         struct swap_insn_args *args = data;
 144         struct kprobe *p = args->p;
 145         u16 opc;
 146
 147         opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
 148         s390_kernel_write(p->addr, &opc, sizeof(opc));
 149         return 0;
 150 }
 151 NOKPROBE_SYMBOL(swap_instruction);
 152
 153 void arch_arm_kprobe(struct kprobe *p)
 154 {
 155         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 156
 157         if (cpu_has_seq_insn()) {
 158                 swap_instruction(&args);
 159                 text_poke_sync();
 160         } else {
 161                 stop_machine_cpuslocked(swap_instruction, &args, NULL);
 162         }
 163 }
 164 NOKPROBE_SYMBOL(arch_arm_kprobe);
 165
 166 void arch_disarm_kprobe(struct kprobe *p)
 167 {
 168         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 169
 170         if (cpu_has_seq_insn()) {
 171                 swap_instruction(&args);
 172                 text_poke_sync();
 173         } else {
 174                 stop_machine_cpuslocked(swap_instruction, &args, NULL);
 175         }
 176 }
 177 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 178
 179 void arch_remove_kprobe(struct kprobe *p)
 180 {
 181         if (!p->ainsn.insn)
 182                 return;
 183         free_insn_slot(p->ainsn.insn, 0);
 184         p->ainsn.insn = NULL;
 185 }
 186 NOKPROBE_SYMBOL(arch_remove_kprobe);
 187
 188 static void enable_singlestep(struct kprobe_ctlblk *kcb,
 189                               struct pt_regs *regs,
 190                               unsigned long ip)
 191 {
 192         union {
 193                 struct ctlreg regs[3];
 194                 struct {
 195                         struct ctlreg control;
 196                         struct ctlreg start;
 197                         struct ctlreg end;
 198                 };
 199         } per_kprobe;
 200
 201         /* Set up the PER control registers %cr9-%cr11 */
 202         per_kprobe.control.val = PER_EVENT_IFETCH;
 203         per_kprobe.start.val = ip;
 204         per_kprobe.end.val = ip;
 205
 206         /* Save control regs and psw mask */
 207         __local_ctl_store(9, 11, kcb->kprobe_saved_ctl);
 208         kcb->kprobe_saved_imask = regs->psw.mask &
 209                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
 210
 211         /* Set PER control regs, turns on single step for the given address */
 212         __local_ctl_load(9, 11, per_kprobe.regs);
 213         regs->psw.mask |= PSW_MASK_PER;
 214         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
 215         regs->psw.addr = ip;
 216 }
 217 NOKPROBE_SYMBOL(enable_singlestep);
 218
 219 static void disable_singlestep(struct kprobe_ctlblk *kcb,
 220                                struct pt_regs *regs,
 221                                unsigned long ip)
 222 {
 223         /* Restore control regs and psw mask, set new psw address */
 224         __local_ctl_load(9, 11, kcb->kprobe_saved_ctl);
 225         regs->psw.mask &= ~PSW_MASK_PER;
 226         regs->psw.mask |= kcb->kprobe_saved_imask;
 227         regs->psw.addr = ip;
 228 }
 229 NOKPROBE_SYMBOL(disable_singlestep);
 230
 231 /*
 232  * Activate a kprobe by storing its pointer to current_kprobe. The
 233  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
 234  * two kprobes can be active, see KPROBE_REENTER.
 235  */
 236 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 237 {
 238         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
 239         kcb->prev_kprobe.status = kcb->kprobe_status;
 240         __this_cpu_write(current_kprobe, p);
 241 }
 242 NOKPROBE_SYMBOL(push_kprobe);
 243
 244 /*
 245  * Deactivate a kprobe by backing up to the previous state. If the
 246  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
 247  * for any other state prev_kprobe.kp will be NULL.
 248  */
 249 static void pop_kprobe(struct kprobe_ctlblk *kcb)
 250 {
 251         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 252         kcb->kprobe_status = kcb->prev_kprobe.status;
 253         kcb->prev_kprobe.kp = NULL;
 254 }
 255 NOKPROBE_SYMBOL(pop_kprobe);
 256
 257 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 258 {
 259         switch (kcb->kprobe_status) {
 260         case KPROBE_HIT_SSDONE:
 261         case KPROBE_HIT_ACTIVE:
 262                 kprobes_inc_nmissed_count(p);
 263                 break;
 264         case KPROBE_HIT_SS:
 265         case KPROBE_REENTER:
 266         default:
 267                 /*
 268                  * A kprobe on the code path to single step an instruction
 269                  * is a BUG. The code path resides in the .kprobes.text
 270                  * section and is executed with interrupts disabled.
 271                  */
 272                 pr_err("Failed to recover from reentered kprobes.\n");
 273                 dump_kprobe(p);
 274                 BUG();
 275         }
 276 }
 277 NOKPROBE_SYMBOL(kprobe_reenter_check);
 278
 279 static int kprobe_handler(struct pt_regs *regs)
 280 {
 281         struct kprobe_ctlblk *kcb;
 282         struct kprobe *p;
 283
 284         /*
 285          * We want to disable preemption for the entire duration of kprobe
 286          * processing. That includes the calls to the pre/post handlers
 287          * and single stepping the kprobe instruction.
 288          */
 289         preempt_disable();
 290         kcb = get_kprobe_ctlblk();
 291         p = get_kprobe((void *)(regs->psw.addr - 2));
 292
 293         if (p) {
 294                 if (kprobe_running()) {
 295                         /*
 296                          * We have hit a kprobe while another is still
 297                          * active. This can happen in the pre and post
 298                          * handler. Single step the instruction of the
 299                          * new probe but do not call any handler function
 300                          * of this secondary kprobe.
 301                          * push_kprobe and pop_kprobe saves and restores
 302                          * the currently active kprobe.
 303                          */
 304                         kprobe_reenter_check(kcb, p);
 305                         push_kprobe(kcb, p);
 306                         kcb->kprobe_status = KPROBE_REENTER;
 307                 } else {
 308                         /*
 309                          * If we have no pre-handler or it returned 0, we
 310                          * continue with single stepping. If we have a
 311                          * pre-handler and it returned non-zero, it prepped
 312                          * for changing execution path, so get out doing
 313                          * nothing more here.
 314                          */
 315                         push_kprobe(kcb, p);
 316                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 317                         if (p->pre_handler && p->pre_handler(p, regs)) {
 318                                 pop_kprobe(kcb);
 319                                 preempt_enable_no_resched();
 320                                 return 1;
 321                         }
 322                         kcb->kprobe_status = KPROBE_HIT_SS;
 323                 }
 324                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
 325                 return 1;
 326         } /* else:
 327            * No kprobe at this address and no active kprobe. The trap has
 328            * not been caused by a kprobe breakpoint. The race of breakpoint
 329            * vs. kprobe remove does not exist because on s390 as we use
 330            * stop_machine to arm/disarm the breakpoints.
 331            */
 332         preempt_enable_no_resched();
 333         return 0;
 334 }
 335 NOKPROBE_SYMBOL(kprobe_handler);
 336
 337 /*
 338  * Called after single-stepping.  p->addr is the address of the
 339  * instruction whose first byte has been replaced by the "breakpoint"
 340  * instruction.  To avoid the SMP problems that can occur when we
 341  * temporarily put back the original opcode to single-step, we
 342  * single-stepped a copy of the instruction.  The address of this
 343  * copy is p->ainsn.insn.
 344  */
 345 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 346 {
 347         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 348         unsigned long ip = regs->psw.addr;
 349         int fixup = probe_get_fixup_type(p->ainsn.insn);
 350
 351         if (fixup & FIXUP_PSW_NORMAL)
 352                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 353
 354         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
 355                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
 356                 if (ip - (unsigned long) p->ainsn.insn == ilen)
 357                         ip = (unsigned long) p->addr + ilen;
 358         }
 359
 360         if (fixup & FIXUP_RETURN_REGISTER) {
 361                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
 362                 regs->gprs[reg] += (unsigned long) p->addr -
 363                                    (unsigned long) p->ainsn.insn;
 364         }
 365
 366         disable_singlestep(kcb, regs, ip);
 367 }
 368 NOKPROBE_SYMBOL(resume_execution);
 369
 370 static int post_kprobe_handler(struct pt_regs *regs)
 371 {
 372         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 373         struct kprobe *p = kprobe_running();
 374
 375         if (!p)
 376                 return 0;
 377
 378         resume_execution(p, regs);
 379         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
 380                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 381                 p->post_handler(p, regs, 0);
 382         }
 383         pop_kprobe(kcb);
 384         preempt_enable_no_resched();
 385
 386         /*
 387          * if somebody else is singlestepping across a probe point, psw mask
 388          * will have PER set, in which case, continue the remaining processing
 389          * of do_single_step, as if this is not a probe hit.
 390          */
 391         if (regs->psw.mask & PSW_MASK_PER)
 392                 return 0;
 393
 394         return 1;
 395 }
 396 NOKPROBE_SYMBOL(post_kprobe_handler);
 397
 398 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 399 {
 400         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 401         struct kprobe *p = kprobe_running();
 402
 403         switch(kcb->kprobe_status) {
 404         case KPROBE_HIT_SS:
 405         case KPROBE_REENTER:
 406                 /*
 407                  * We are here because the instruction being single
 408                  * stepped caused a page fault. We reset the current
 409                  * kprobe and the nip points back to the probe address
 410                  * and allow the page fault handler to continue as a
 411                  * normal page fault.
 412                  */
 413                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
 414                 pop_kprobe(kcb);
 415                 preempt_enable_no_resched();
 416                 break;
 417         case KPROBE_HIT_ACTIVE:
 418         case KPROBE_HIT_SSDONE:
 419                 /*
 420                  * In case the user-specified fault handler returned
 421                  * zero, try to fix up.
 422                  */
 423                 if (fixup_exception(regs))
 424                         return 1;
 425                 /*
 426                  * fixup_exception() could not handle it,
 427                  * Let do_page_fault() fix it.
 428                  */
 429                 break;
 430         default:
 431                 break;
 432         }
 433         return 0;
 434 }
 435 NOKPROBE_SYMBOL(kprobe_trap_handler);
 436
 437 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 438 {
 439         int ret;
 440
 441         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 442                 local_irq_disable();
 443         ret = kprobe_trap_handler(regs, trapnr);
 444         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 445                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 446         return ret;
 447 }
 448 NOKPROBE_SYMBOL(kprobe_fault_handler);
 449
 450 /*
 451  * Wrapper routine to for handling exceptions.
 452  */
 453 int kprobe_exceptions_notify(struct notifier_block *self,
 454                              unsigned long val, void *data)
 455 {
 456         struct die_args *args = (struct die_args *) data;
 457         struct pt_regs *regs = args->regs;
 458         int ret = NOTIFY_DONE;
 459
 460         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 461                 local_irq_disable();
 462
 463         switch (val) {
 464         case DIE_BPT:
 465                 if (kprobe_handler(regs))
 466                         ret = NOTIFY_STOP;
 467                 break;
 468         case DIE_SSTEP:
 469                 if (post_kprobe_handler(regs))
 470                         ret = NOTIFY_STOP;
 471                 break;
 472         case DIE_TRAP:
 473                 if (!preemptible() && kprobe_running() &&
 474                     kprobe_trap_handler(regs, args->trapnr))
 475                         ret = NOTIFY_STOP;
 476                 break;
 477         default:
 478                 break;
 479         }
 480
 481         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 482                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 483
 484         return ret;
 485 }
 486 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 487
 488 int __init arch_init_kprobes(void)
 489 {
 490         return 0;
 491 }
 492
 493 int __init arch_populate_kprobe_blacklist(void)
 494 {
 495         return kprobe_add_area_blacklist((unsigned long)__irqentry_text_start,
 496                                          (unsigned long)__irqentry_text_end);
 497 }
 498
 499 int arch_trampoline_kprobe(struct kprobe *p)
 500 {
 501         return 0;
 502 }
 503 NOKPROBE_SYMBOL(arch_trampoline_kprobe);