Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* |
3 | * Kernel Probes (KProbes) | |
1da177e4 | 4 | * |
1da177e4 LT |
5 | * Copyright (C) IBM Corporation, 2002, 2004 |
6 | * | |
7 | * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel | |
8 | * Probes initial implementation ( includes contributions from | |
9 | * Rusty Russell). | |
10 | * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes | |
11 | * interface to access function arguments. | |
12 | * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port | |
13 | * for PPC64 | |
14 | */ | |
15 | ||
1da177e4 LT |
16 | #include <linux/kprobes.h> |
17 | #include <linux/ptrace.h> | |
1da177e4 | 18 | #include <linux/preempt.h> |
8a39b05f | 19 | #include <linux/extable.h> |
1eeb66a1 | 20 | #include <linux/kdebug.h> |
5a0e3ad6 | 21 | #include <linux/slab.h> |
6a3a58e6 | 22 | #include <linux/moduleloader.h> |
2f0143c9 | 23 | #include <asm/code-patching.h> |
7e1048b1 | 24 | #include <asm/cacheflush.h> |
1da177e4 | 25 | #include <asm/sstep.h> |
7aa5b018 | 26 | #include <asm/sections.h> |
75346251 | 27 | #include <asm/inst.h> |
6a3a58e6 | 28 | #include <asm/set_memory.h> |
7c0f6ba6 | 29 | #include <linux/uaccess.h> |
f8279621 | 30 | |
0dc036c9 AM |
31 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
32 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); | |
1da177e4 | 33 | |
f438d914 MH |
34 | struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; |
35 | ||
7aa5b018 NR |
36 | bool arch_within_kprobe_blacklist(unsigned long addr) |
37 | { | |
38 | return (addr >= (unsigned long)__kprobes_text_start && | |
39 | addr < (unsigned long)__kprobes_text_end) || | |
40 | (addr >= (unsigned long)_stext && | |
41 | addr < (unsigned long)__head_end); | |
42 | } | |
43 | ||
290e3070 | 44 | kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset) |
49e0b465 | 45 | { |
acdfe931 | 46 | kprobe_opcode_t *addr = NULL; |
49e0b465 | 47 | |
7d40aff8 | 48 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
49e0b465 NR |
49 | /* PPC64 ABIv2 needs local entry point */ |
50 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); | |
24bd909e NR |
51 | if (addr && !offset) { |
52 | #ifdef CONFIG_KPROBES_ON_FTRACE | |
53 | unsigned long faddr; | |
54 | /* | |
55 | * Per livepatch.h, ftrace location is always within the first | |
56 | * 16 bytes of a function on powerpc with -mprofile-kernel. | |
57 | */ | |
58 | faddr = ftrace_location_range((unsigned long)addr, | |
59 | (unsigned long)addr + 16); | |
60 | if (faddr) | |
61 | addr = (kprobe_opcode_t *)faddr; | |
62 | else | |
63 | #endif | |
64 | addr = (kprobe_opcode_t *)ppc_function_entry(addr); | |
65 | } | |
7d40aff8 | 66 | #elif defined(CONFIG_PPC64_ELF_ABI_V1) |
49e0b465 NR |
67 | /* |
68 | * 64bit powerpc ABIv1 uses function descriptors: | |
69 | * - Check for the dot variant of the symbol first. | |
70 | * - If that fails, try looking up the symbol provided. | |
71 | * | |
72 | * This ensures we always get to the actual symbol and not | |
73 | * the descriptor. | |
74 | * | |
75 | * Also handle <module:symbol> format. | |
76 | */ | |
77 | char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN]; | |
49e0b465 | 78 | bool dot_appended = false; |
acdfe931 NR |
79 | const char *c; |
80 | ssize_t ret = 0; | |
81 | int len = 0; | |
82 | ||
83 | if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) { | |
84 | c++; | |
85 | len = c - name; | |
86 | memcpy(dot_name, name, len); | |
87 | } else | |
88 | c = name; | |
89 | ||
90 | if (*c != '\0' && *c != '.') { | |
91 | dot_name[len++] = '.'; | |
49e0b465 | 92 | dot_appended = true; |
49e0b465 | 93 | } |
acdfe931 NR |
94 | ret = strscpy(dot_name + len, c, KSYM_NAME_LEN); |
95 | if (ret > 0) | |
96 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name); | |
97 | ||
98 | /* Fallback to the original non-dot symbol lookup */ | |
99 | if (!addr && dot_appended) | |
49e0b465 | 100 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); |
49e0b465 NR |
101 | #else |
102 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); | |
103 | #endif | |
104 | ||
105 | return addr; | |
106 | } | |
107 | ||
cc66bb91 PZ |
108 | static bool arch_kprobe_on_func_entry(unsigned long offset) |
109 | { | |
7d40aff8 | 110 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
cc66bb91 PZ |
111 | #ifdef CONFIG_KPROBES_ON_FTRACE |
112 | return offset <= 16; | |
113 | #else | |
114 | return offset <= 8; | |
115 | #endif | |
116 | #else | |
117 | return !offset; | |
118 | #endif | |
119 | } | |
120 | ||
121 | /* XXX try and fold the magic of kprobe_lookup_name() in this */ | |
122 | kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, | |
123 | bool *on_func_entry) | |
124 | { | |
125 | *on_func_entry = arch_kprobe_on_func_entry(offset); | |
126 | return (kprobe_opcode_t *)(addr + offset); | |
127 | } | |
128 | ||
6a3a58e6 RC |
129 | void *alloc_insn_page(void) |
130 | { | |
131 | void *page; | |
132 | ||
133 | page = module_alloc(PAGE_SIZE); | |
134 | if (!page) | |
135 | return NULL; | |
136 | ||
137 | if (strict_module_rwx_enabled()) { | |
138 | set_memory_ro((unsigned long)page, 1); | |
139 | set_memory_x((unsigned long)page, 1); | |
140 | } | |
141 | return page; | |
142 | } | |
143 | ||
71f6e58e | 144 | int arch_prepare_kprobe(struct kprobe *p) |
1da177e4 | 145 | { |
63224d1e | 146 | int ret = 0; |
b4657f76 | 147 | struct kprobe *prev; |
c545b9f0 | 148 | ppc_inst_t insn = ppc_inst_read(p->addr); |
1da177e4 | 149 | |
63224d1e AM |
150 | if ((unsigned long)p->addr & 0x03) { |
151 | printk("Attempt to register kprobe at an unaligned address\n"); | |
152 | ret = -EINVAL; | |
54cdacd7 NR |
153 | } else if (!can_single_step(ppc_inst_val(insn))) { |
154 | printk("Cannot register a kprobe on instructions that can't be single stepped\n"); | |
63224d1e | 155 | ret = -EINVAL; |
82123a3d | 156 | } else if ((unsigned long)p->addr & ~PAGE_MASK && |
69d4d6e5 | 157 | ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) { |
b4657f76 JN |
158 | printk("Cannot register a kprobe on the second word of prefixed instruction\n"); |
159 | ret = -EINVAL; | |
160 | } | |
161 | preempt_disable(); | |
162 | prev = get_kprobe(p->addr - 1); | |
163 | preempt_enable_no_resched(); | |
97f88a3d LH |
164 | |
165 | /* | |
166 | * When prev is a ftrace-based kprobe, we don't have an insn, and it | |
167 | * doesn't probe for prefixed instruction. | |
168 | */ | |
169 | if (prev && !kprobe_ftrace(prev) && | |
170 | ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) { | |
b4657f76 JN |
171 | printk("Cannot register a kprobe on the second word of prefixed instruction\n"); |
172 | ret = -EINVAL; | |
63224d1e | 173 | } |
9ec4b1f3 | 174 | |
f8279621 KG |
175 | /* insn must be on a special executable page on ppc64. This is |
176 | * not explicitly required on ppc32 (right now), but it doesn't hurt */ | |
9ec4b1f3 | 177 | if (!ret) { |
2d8ab6ad | 178 | p->ainsn.insn = get_insn_slot(); |
9ec4b1f3 AM |
179 | if (!p->ainsn.insn) |
180 | ret = -ENOMEM; | |
181 | } | |
1da177e4 | 182 | |
49a2a1b8 | 183 | if (!ret) { |
69d4d6e5 | 184 | patch_instruction(p->ainsn.insn, insn); |
f8faaffa | 185 | p->opcode = ppc_inst_val(insn); |
49a2a1b8 AK |
186 | } |
187 | ||
e6349a95 | 188 | p->ainsn.boostable = 0; |
49a2a1b8 | 189 | return ret; |
1da177e4 | 190 | } |
71f6e58e | 191 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
1da177e4 | 192 | |
71f6e58e | 193 | void arch_arm_kprobe(struct kprobe *p) |
1da177e4 | 194 | { |
12b58492 | 195 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION))); |
1da177e4 | 196 | } |
71f6e58e | 197 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
1da177e4 | 198 | |
71f6e58e | 199 | void arch_disarm_kprobe(struct kprobe *p) |
1da177e4 | 200 | { |
12b58492 | 201 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode))); |
7e1048b1 | 202 | } |
71f6e58e | 203 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
7e1048b1 | 204 | |
71f6e58e | 205 | void arch_remove_kprobe(struct kprobe *p) |
7e1048b1 | 206 | { |
12941560 MH |
207 | if (p->ainsn.insn) { |
208 | free_insn_slot(p->ainsn.insn, 0); | |
209 | p->ainsn.insn = NULL; | |
210 | } | |
1da177e4 | 211 | } |
71f6e58e | 212 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
1da177e4 | 213 | |
71f6e58e | 214 | static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) |
1da177e4 | 215 | { |
35fd219a | 216 | enable_single_step(regs); |
9ec4b1f3 | 217 | |
0ccde0a2 AM |
218 | /* |
219 | * On powerpc we should single step on the original | |
220 | * instruction even if the probed insn is a trap | |
221 | * variant as values in regs could play a part in | |
222 | * if the trap is taken or not | |
223 | */ | |
59dc5bfc | 224 | regs_set_return_ip(regs, (unsigned long)p->ainsn.insn); |
1da177e4 LT |
225 | } |
226 | ||
71f6e58e | 227 | static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb) |
0dc036c9 AM |
228 | { |
229 | kcb->prev_kprobe.kp = kprobe_running(); | |
230 | kcb->prev_kprobe.status = kcb->kprobe_status; | |
231 | kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; | |
232 | } | |
233 | ||
71f6e58e | 234 | static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
42cc2060 | 235 | { |
69111bac | 236 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
0dc036c9 AM |
237 | kcb->kprobe_status = kcb->prev_kprobe.status; |
238 | kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; | |
42cc2060 PP |
239 | } |
240 | ||
71f6e58e | 241 | static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, |
0dc036c9 | 242 | struct kprobe_ctlblk *kcb) |
42cc2060 | 243 | { |
69111bac | 244 | __this_cpu_write(current_kprobe, p); |
0dc036c9 | 245 | kcb->kprobe_saved_msr = regs->msr; |
42cc2060 PP |
246 | } |
247 | ||
71f6e58e | 248 | void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
97f7943d | 249 | { |
4c4308cb | 250 | ri->ret_addr = (kprobe_opcode_t *)regs->link; |
b6c5a58d | 251 | ri->fp = NULL; |
4c4308cb CH |
252 | |
253 | /* Replace the return addr with trampoline addr */ | |
adf8a61a | 254 | regs->link = (unsigned long)__kretprobe_trampoline; |
97f7943d | 255 | } |
71f6e58e | 256 | NOKPROBE_SYMBOL(arch_prepare_kretprobe); |
97f7943d | 257 | |
22085337 | 258 | static int try_to_emulate(struct kprobe *p, struct pt_regs *regs) |
1cabd2f8 NR |
259 | { |
260 | int ret; | |
c545b9f0 | 261 | ppc_inst_t insn = ppc_inst_read(p->ainsn.insn); |
1cabd2f8 NR |
262 | |
263 | /* regs->nip is also adjusted if emulate_step returns 1 */ | |
264 | ret = emulate_step(regs, insn); | |
265 | if (ret > 0) { | |
266 | /* | |
267 | * Once this instruction has been boosted | |
268 | * successfully, set the boostable flag | |
269 | */ | |
270 | if (unlikely(p->ainsn.boostable == 0)) | |
271 | p->ainsn.boostable = 1; | |
272 | } else if (ret < 0) { | |
273 | /* | |
274 | * We don't allow kprobes on mtmsr(d)/rfi(d), etc. | |
275 | * So, we should never get here... but, its still | |
276 | * good to catch them, just in case... | |
277 | */ | |
2a83afe7 | 278 | printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn)); |
1cabd2f8 | 279 | BUG(); |
a7b44038 NR |
280 | } else { |
281 | /* | |
282 | * If we haven't previously emulated this instruction, then it | |
283 | * can't be boosted. Note it down so we don't try to do so again. | |
284 | * | |
285 | * If, however, we had emulated this instruction in the past, | |
286 | * then this is just an error with the current run (for | |
287 | * instance, exceptions due to a load/store). We return 0 so | |
288 | * that this is now single-stepped, but continue to try | |
289 | * emulating it in subsequent probe hits. | |
290 | */ | |
291 | if (unlikely(p->ainsn.boostable != 1)) | |
292 | p->ainsn.boostable = -1; | |
293 | } | |
1cabd2f8 NR |
294 | |
295 | return ret; | |
296 | } | |
9fc84914 | 297 | NOKPROBE_SYMBOL(try_to_emulate); |
1cabd2f8 | 298 | |
71f6e58e | 299 | int kprobe_handler(struct pt_regs *regs) |
1da177e4 LT |
300 | { |
301 | struct kprobe *p; | |
302 | int ret = 0; | |
303 | unsigned int *addr = (unsigned int *)regs->nip; | |
d217d545 AM |
304 | struct kprobe_ctlblk *kcb; |
305 | ||
6cc89bad NR |
306 | if (user_mode(regs)) |
307 | return 0; | |
308 | ||
43e8f760 PL |
309 | if (!IS_ENABLED(CONFIG_BOOKE) && |
310 | (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR))) | |
21f8b2fa CL |
311 | return 0; |
312 | ||
d217d545 AM |
313 | /* |
314 | * We don't want to be preempted for the entire | |
315 | * duration of kprobe processing | |
316 | */ | |
317 | preempt_disable(); | |
318 | kcb = get_kprobe_ctlblk(); | |
1da177e4 | 319 | |
1da177e4 LT |
320 | p = get_kprobe(addr); |
321 | if (!p) { | |
9ed5df69 CL |
322 | unsigned int instr; |
323 | ||
25f12ae4 | 324 | if (get_kernel_nofault(instr, addr)) |
9ed5df69 CL |
325 | goto no_kprobe; |
326 | ||
327 | if (instr != BREAKPOINT_INSTRUCTION) { | |
1da177e4 LT |
328 | /* |
329 | * PowerPC has multiple variants of the "trap" | |
330 | * instruction. If the current instruction is a | |
331 | * trap variant, it could belong to someone else | |
332 | */ | |
9ed5df69 | 333 | if (is_trap(instr)) |
6e5fd3a2 | 334 | goto no_kprobe; |
1da177e4 LT |
335 | /* |
336 | * The breakpoint instruction was removed right | |
337 | * after we hit it. Another cpu has removed | |
338 | * either a probepoint or a debugger breakpoint | |
339 | * at this address. In either case, no further | |
340 | * handling of this interrupt is appropriate. | |
341 | */ | |
342 | ret = 1; | |
343 | } | |
344 | /* Not one of ours: let kernel handle it */ | |
345 | goto no_kprobe; | |
346 | } | |
347 | ||
eb4f8e25 CL |
348 | /* Check we're not actually recursing */ |
349 | if (kprobe_running()) { | |
350 | kprobe_opcode_t insn = *p->ainsn.insn; | |
351 | if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) { | |
352 | /* Turn off 'trace' bits */ | |
59dc5bfc NP |
353 | regs_set_return_msr(regs, |
354 | (regs->msr & ~MSR_SINGLESTEP) | | |
355 | kcb->kprobe_saved_msr); | |
eb4f8e25 CL |
356 | goto no_kprobe; |
357 | } | |
358 | ||
359 | /* | |
360 | * We have reentered the kprobe_handler(), since another probe | |
361 | * was hit while within the handler. We here save the original | |
362 | * kprobes variables and just single step on the instruction of | |
363 | * the new probe without calling any user handlers. | |
364 | */ | |
365 | save_previous_kprobe(kcb); | |
366 | set_current_kprobe(p, regs, kcb); | |
367 | kprobes_inc_nmissed_count(p); | |
368 | kcb->kprobe_status = KPROBE_REENTER; | |
369 | if (p->ainsn.boostable >= 0) { | |
370 | ret = try_to_emulate(p, regs); | |
371 | ||
372 | if (ret > 0) { | |
373 | restore_previous_kprobe(kcb); | |
374 | preempt_enable_no_resched(); | |
375 | return 1; | |
376 | } | |
377 | } | |
378 | prepare_singlestep(p, regs); | |
379 | return 1; | |
380 | } | |
381 | ||
0dc036c9 AM |
382 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
383 | set_current_kprobe(p, regs, kcb); | |
cce188bd MH |
384 | if (p->pre_handler && p->pre_handler(p, regs)) { |
385 | /* handler changed execution path, so skip ss setup */ | |
386 | reset_current_kprobe(); | |
387 | preempt_enable_no_resched(); | |
1da177e4 | 388 | return 1; |
cce188bd | 389 | } |
1da177e4 | 390 | |
e6349a95 | 391 | if (p->ainsn.boostable >= 0) { |
1cabd2f8 | 392 | ret = try_to_emulate(p, regs); |
e6349a95 | 393 | |
e6349a95 | 394 | if (ret > 0) { |
e6349a95 AM |
395 | if (p->post_handler) |
396 | p->post_handler(p, regs, 0); | |
397 | ||
398 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
399 | reset_current_kprobe(); | |
400 | preempt_enable_no_resched(); | |
401 | return 1; | |
1cabd2f8 | 402 | } |
e6349a95 | 403 | } |
1da177e4 | 404 | prepare_singlestep(p, regs); |
0dc036c9 | 405 | kcb->kprobe_status = KPROBE_HIT_SS; |
1da177e4 LT |
406 | return 1; |
407 | ||
408 | no_kprobe: | |
d217d545 | 409 | preempt_enable_no_resched(); |
1da177e4 LT |
410 | return ret; |
411 | } | |
71f6e58e | 412 | NOKPROBE_SYMBOL(kprobe_handler); |
1da177e4 | 413 | |
97f7943d RL |
414 | /* |
415 | * Function return probe trampoline: | |
416 | * - init_kprobes() establishes a probepoint here | |
417 | * - When the probed function returns, this probe | |
418 | * causes the handlers to fire | |
419 | */ | |
adf8a61a MH |
420 | asm(".global __kretprobe_trampoline\n" |
421 | ".type __kretprobe_trampoline, @function\n" | |
422 | "__kretprobe_trampoline:\n" | |
61ed9cfb | 423 | "nop\n" |
762df10b | 424 | "blr\n" |
adf8a61a | 425 | ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"); |
97f7943d RL |
426 | |
427 | /* | |
428 | * Called when the probe at kretprobe trampoline is hit | |
429 | */ | |
71f6e58e | 430 | static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
97f7943d | 431 | { |
b6c5a58d | 432 | unsigned long orig_ret_address; |
e6e133c4 | 433 | |
96fed8ac | 434 | orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); |
762df10b | 435 | /* |
e6e133c4 NR |
436 | * We get here through one of two paths: |
437 | * 1. by taking a trap -> kprobe_handler() -> here | |
438 | * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here | |
439 | * | |
440 | * When going back through (1), we need regs->nip to be setup properly | |
441 | * as it is used to determine the return address from the trap. | |
442 | * For (2), since nip is not honoured with optprobes, we instead setup | |
443 | * the link register properly so that the subsequent 'blr' in | |
adf8a61a | 444 | * __kretprobe_trampoline jumps back to the right instruction. |
e6e133c4 NR |
445 | * |
446 | * For nip, we should set the address to the previous instruction since | |
447 | * we end up emulating it in kprobe_handler(), which increments the nip | |
448 | * again. | |
762df10b | 449 | */ |
59dc5bfc | 450 | regs_set_return_ip(regs, orig_ret_address - 4); |
762df10b | 451 | regs->link = orig_ret_address; |
97f7943d | 452 | |
e6e133c4 | 453 | return 0; |
97f7943d | 454 | } |
71f6e58e | 455 | NOKPROBE_SYMBOL(trampoline_probe_handler); |
97f7943d | 456 | |
1da177e4 LT |
457 | /* |
458 | * Called after single-stepping. p->addr is the address of the | |
459 | * instruction whose first byte has been replaced by the "breakpoint" | |
460 | * instruction. To avoid the SMP problems that can occur when we | |
461 | * temporarily put back the original opcode to single-step, we | |
462 | * single-stepped a copy of the instruction. The address of this | |
463 | * copy is p->ainsn.insn. | |
464 | */ | |
71f6e58e | 465 | int kprobe_post_handler(struct pt_regs *regs) |
1da177e4 | 466 | { |
622cf6f4 | 467 | int len; |
0dc036c9 AM |
468 | struct kprobe *cur = kprobe_running(); |
469 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
470 | ||
6cc89bad | 471 | if (!cur || user_mode(regs)) |
1da177e4 LT |
472 | return 0; |
473 | ||
69d4d6e5 | 474 | len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn)); |
b76e59d1 | 475 | /* make sure we got here for instruction we have a kprobe on */ |
622cf6f4 | 476 | if (((unsigned long)cur->ainsn.insn + len) != regs->nip) |
b76e59d1 KG |
477 | return 0; |
478 | ||
0dc036c9 AM |
479 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
480 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
481 | cur->post_handler(cur, regs, 0); | |
42cc2060 | 482 | } |
1da177e4 | 483 | |
db97bc7f | 484 | /* Adjust nip to after the single-stepped instruction */ |
59dc5bfc NP |
485 | regs_set_return_ip(regs, (unsigned long)cur->addr + len); |
486 | regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr); | |
1da177e4 | 487 | |
42cc2060 | 488 | /*Restore back the original saved kprobes variables and continue. */ |
0dc036c9 AM |
489 | if (kcb->kprobe_status == KPROBE_REENTER) { |
490 | restore_previous_kprobe(kcb); | |
42cc2060 PP |
491 | goto out; |
492 | } | |
0dc036c9 | 493 | reset_current_kprobe(); |
42cc2060 | 494 | out: |
1da177e4 LT |
495 | preempt_enable_no_resched(); |
496 | ||
497 | /* | |
498 | * if somebody else is singlestepping across a probe point, msr | |
f8279621 | 499 | * will have DE/SE set, in which case, continue the remaining processing |
1da177e4 LT |
500 | * of do_debug, as if this is not a probe hit. |
501 | */ | |
f8279621 | 502 | if (regs->msr & MSR_SINGLESTEP) |
1da177e4 LT |
503 | return 0; |
504 | ||
505 | return 1; | |
506 | } | |
71f6e58e | 507 | NOKPROBE_SYMBOL(kprobe_post_handler); |
1da177e4 | 508 | |
71f6e58e | 509 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
1da177e4 | 510 | { |
0dc036c9 AM |
511 | struct kprobe *cur = kprobe_running(); |
512 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
50e21f2b PP |
513 | const struct exception_table_entry *entry; |
514 | ||
515 | switch(kcb->kprobe_status) { | |
516 | case KPROBE_HIT_SS: | |
517 | case KPROBE_REENTER: | |
518 | /* | |
519 | * We are here because the instruction being single | |
520 | * stepped caused a page fault. We reset the current | |
521 | * kprobe and the nip points back to the probe address | |
522 | * and allow the page fault handler to continue as a | |
523 | * normal page fault. | |
524 | */ | |
59dc5bfc NP |
525 | regs_set_return_ip(regs, (unsigned long)cur->addr); |
526 | /* Turn off 'trace' bits */ | |
527 | regs_set_return_msr(regs, | |
528 | (regs->msr & ~MSR_SINGLESTEP) | | |
529 | kcb->kprobe_saved_msr); | |
50e21f2b PP |
530 | if (kcb->kprobe_status == KPROBE_REENTER) |
531 | restore_previous_kprobe(kcb); | |
532 | else | |
533 | reset_current_kprobe(); | |
1da177e4 | 534 | preempt_enable_no_resched(); |
50e21f2b PP |
535 | break; |
536 | case KPROBE_HIT_ACTIVE: | |
537 | case KPROBE_HIT_SSDONE: | |
538 | /* | |
50e21f2b PP |
539 | * In case the user-specified fault handler returned |
540 | * zero, try to fix up. | |
541 | */ | |
542 | if ((entry = search_exception_tables(regs->nip)) != NULL) { | |
59dc5bfc | 543 | regs_set_return_ip(regs, extable_fixup(entry)); |
50e21f2b PP |
544 | return 1; |
545 | } | |
546 | ||
547 | /* | |
548 | * fixup_exception() could not handle it, | |
549 | * Let do_page_fault() fix it. | |
550 | */ | |
551 | break; | |
552 | default: | |
553 | break; | |
1da177e4 LT |
554 | } |
555 | return 0; | |
556 | } | |
71f6e58e | 557 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
1da177e4 | 558 | |
97f7943d | 559 | static struct kprobe trampoline_p = { |
adf8a61a | 560 | .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, |
97f7943d RL |
561 | .pre_handler = trampoline_probe_handler |
562 | }; | |
563 | ||
6772926b | 564 | int __init arch_init_kprobes(void) |
97f7943d RL |
565 | { |
566 | return register_kprobe(&trampoline_p); | |
567 | } | |
bf8f6e5b | 568 | |
71f6e58e | 569 | int arch_trampoline_kprobe(struct kprobe *p) |
bf8f6e5b | 570 | { |
adf8a61a | 571 | if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) |
bf8f6e5b AM |
572 | return 1; |
573 | ||
574 | return 0; | |
575 | } | |
71f6e58e | 576 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |