Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* |
3 | * Kernel Probes (KProbes) | |
1da177e4 | 4 | * |
1da177e4 LT |
5 | * Copyright (C) IBM Corporation, 2002, 2004 |
6 | * | |
7 | * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel | |
8 | * Probes initial implementation ( includes contributions from | |
9 | * Rusty Russell). | |
10 | * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes | |
11 | * interface to access function arguments. | |
12 | * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port | |
13 | * for PPC64 | |
14 | */ | |
15 | ||
1da177e4 LT |
16 | #include <linux/kprobes.h> |
17 | #include <linux/ptrace.h> | |
1da177e4 | 18 | #include <linux/preempt.h> |
8a39b05f | 19 | #include <linux/extable.h> |
1eeb66a1 | 20 | #include <linux/kdebug.h> |
5a0e3ad6 | 21 | #include <linux/slab.h> |
d48567c9 | 22 | #include <linux/set_memory.h> |
12af2b83 | 23 | #include <linux/execmem.h> |
2f0143c9 | 24 | #include <asm/code-patching.h> |
7e1048b1 | 25 | #include <asm/cacheflush.h> |
1da177e4 | 26 | #include <asm/sstep.h> |
7aa5b018 | 27 | #include <asm/sections.h> |
75346251 | 28 | #include <asm/inst.h> |
7c0f6ba6 | 29 | #include <linux/uaccess.h> |
f8279621 | 30 | |
0dc036c9 AM |
31 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
32 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); | |
1da177e4 | 33 | |
f438d914 MH |
34 | struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; |
35 | ||
7aa5b018 NR |
36 | bool arch_within_kprobe_blacklist(unsigned long addr) |
37 | { | |
38 | return (addr >= (unsigned long)__kprobes_text_start && | |
39 | addr < (unsigned long)__kprobes_text_end) || | |
40 | (addr >= (unsigned long)_stext && | |
41 | addr < (unsigned long)__head_end); | |
42 | } | |
43 | ||
290e3070 | 44 | kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset) |
49e0b465 | 45 | { |
acdfe931 | 46 | kprobe_opcode_t *addr = NULL; |
49e0b465 | 47 | |
7d40aff8 | 48 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
49e0b465 NR |
49 | /* PPC64 ABIv2 needs local entry point */ |
50 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); | |
24bd909e NR |
51 | if (addr && !offset) { |
52 | #ifdef CONFIG_KPROBES_ON_FTRACE | |
53 | unsigned long faddr; | |
54 | /* | |
55 | * Per livepatch.h, ftrace location is always within the first | |
56 | * 16 bytes of a function on powerpc with -mprofile-kernel. | |
57 | */ | |
58 | faddr = ftrace_location_range((unsigned long)addr, | |
59 | (unsigned long)addr + 16); | |
60 | if (faddr) | |
61 | addr = (kprobe_opcode_t *)faddr; | |
62 | else | |
63 | #endif | |
64 | addr = (kprobe_opcode_t *)ppc_function_entry(addr); | |
65 | } | |
7d40aff8 | 66 | #elif defined(CONFIG_PPC64_ELF_ABI_V1) |
49e0b465 NR |
67 | /* |
68 | * 64bit powerpc ABIv1 uses function descriptors: | |
69 | * - Check for the dot variant of the symbol first. | |
70 | * - If that fails, try looking up the symbol provided. | |
71 | * | |
72 | * This ensures we always get to the actual symbol and not | |
73 | * the descriptor. | |
74 | * | |
75 | * Also handle <module:symbol> format. | |
76 | */ | |
77 | char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN]; | |
49e0b465 | 78 | bool dot_appended = false; |
acdfe931 NR |
79 | const char *c; |
80 | ssize_t ret = 0; | |
81 | int len = 0; | |
82 | ||
83 | if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) { | |
84 | c++; | |
85 | len = c - name; | |
86 | memcpy(dot_name, name, len); | |
87 | } else | |
88 | c = name; | |
89 | ||
90 | if (*c != '\0' && *c != '.') { | |
91 | dot_name[len++] = '.'; | |
49e0b465 | 92 | dot_appended = true; |
49e0b465 | 93 | } |
acdfe931 NR |
94 | ret = strscpy(dot_name + len, c, KSYM_NAME_LEN); |
95 | if (ret > 0) | |
96 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name); | |
97 | ||
98 | /* Fallback to the original non-dot symbol lookup */ | |
99 | if (!addr && dot_appended) | |
49e0b465 | 100 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); |
49e0b465 NR |
101 | #else |
102 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); | |
103 | #endif | |
104 | ||
105 | return addr; | |
106 | } | |
107 | ||
cc66bb91 PZ |
108 | static bool arch_kprobe_on_func_entry(unsigned long offset) |
109 | { | |
7d40aff8 | 110 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
cc66bb91 PZ |
111 | #ifdef CONFIG_KPROBES_ON_FTRACE |
112 | return offset <= 16; | |
113 | #else | |
114 | return offset <= 8; | |
115 | #endif | |
116 | #else | |
117 | return !offset; | |
118 | #endif | |
119 | } | |
120 | ||
121 | /* XXX try and fold the magic of kprobe_lookup_name() in this */ | |
122 | kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, | |
123 | bool *on_func_entry) | |
124 | { | |
125 | *on_func_entry = arch_kprobe_on_func_entry(offset); | |
126 | return (kprobe_opcode_t *)(addr + offset); | |
127 | } | |
128 | ||
6a3a58e6 RC |
129 | void *alloc_insn_page(void) |
130 | { | |
131 | void *page; | |
132 | ||
12af2b83 | 133 | page = execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE); |
6a3a58e6 RC |
134 | if (!page) |
135 | return NULL; | |
136 | ||
f7f18e30 CL |
137 | if (strict_module_rwx_enabled()) { |
138 | int err = set_memory_rox((unsigned long)page, 1); | |
d48567c9 | 139 | |
f7f18e30 CL |
140 | if (err) |
141 | goto error; | |
142 | } | |
6a3a58e6 | 143 | return page; |
f7f18e30 | 144 | error: |
12af2b83 | 145 | execmem_free(page); |
f7f18e30 | 146 | return NULL; |
6a3a58e6 RC |
147 | } |
148 | ||
71f6e58e | 149 | int arch_prepare_kprobe(struct kprobe *p) |
1da177e4 | 150 | { |
63224d1e | 151 | int ret = 0; |
b4657f76 | 152 | struct kprobe *prev; |
c545b9f0 | 153 | ppc_inst_t insn = ppc_inst_read(p->addr); |
1da177e4 | 154 | |
63224d1e AM |
155 | if ((unsigned long)p->addr & 0x03) { |
156 | printk("Attempt to register kprobe at an unaligned address\n"); | |
157 | ret = -EINVAL; | |
54cdacd7 NR |
158 | } else if (!can_single_step(ppc_inst_val(insn))) { |
159 | printk("Cannot register a kprobe on instructions that can't be single stepped\n"); | |
63224d1e | 160 | ret = -EINVAL; |
82123a3d | 161 | } else if ((unsigned long)p->addr & ~PAGE_MASK && |
69d4d6e5 | 162 | ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) { |
b4657f76 JN |
163 | printk("Cannot register a kprobe on the second word of prefixed instruction\n"); |
164 | ret = -EINVAL; | |
165 | } | |
b4657f76 | 166 | prev = get_kprobe(p->addr - 1); |
97f88a3d LH |
167 | |
168 | /* | |
169 | * When prev is a ftrace-based kprobe, we don't have an insn, and it | |
170 | * doesn't probe for prefixed instruction. | |
171 | */ | |
172 | if (prev && !kprobe_ftrace(prev) && | |
173 | ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) { | |
b4657f76 JN |
174 | printk("Cannot register a kprobe on the second word of prefixed instruction\n"); |
175 | ret = -EINVAL; | |
63224d1e | 176 | } |
9ec4b1f3 | 177 | |
f8279621 KG |
178 | /* insn must be on a special executable page on ppc64. This is |
179 | * not explicitly required on ppc32 (right now), but it doesn't hurt */ | |
9ec4b1f3 | 180 | if (!ret) { |
2d8ab6ad | 181 | p->ainsn.insn = get_insn_slot(); |
9ec4b1f3 AM |
182 | if (!p->ainsn.insn) |
183 | ret = -ENOMEM; | |
184 | } | |
1da177e4 | 185 | |
49a2a1b8 | 186 | if (!ret) { |
69d4d6e5 | 187 | patch_instruction(p->ainsn.insn, insn); |
f8faaffa | 188 | p->opcode = ppc_inst_val(insn); |
49a2a1b8 AK |
189 | } |
190 | ||
e6349a95 | 191 | p->ainsn.boostable = 0; |
49a2a1b8 | 192 | return ret; |
1da177e4 | 193 | } |
71f6e58e | 194 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
1da177e4 | 195 | |
71f6e58e | 196 | void arch_arm_kprobe(struct kprobe *p) |
1da177e4 | 197 | { |
12b58492 | 198 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION))); |
1da177e4 | 199 | } |
71f6e58e | 200 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
1da177e4 | 201 | |
71f6e58e | 202 | void arch_disarm_kprobe(struct kprobe *p) |
1da177e4 | 203 | { |
12b58492 | 204 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode))); |
7e1048b1 | 205 | } |
71f6e58e | 206 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
7e1048b1 | 207 | |
71f6e58e | 208 | void arch_remove_kprobe(struct kprobe *p) |
7e1048b1 | 209 | { |
12941560 MH |
210 | if (p->ainsn.insn) { |
211 | free_insn_slot(p->ainsn.insn, 0); | |
212 | p->ainsn.insn = NULL; | |
213 | } | |
1da177e4 | 214 | } |
71f6e58e | 215 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
1da177e4 | 216 | |
71f6e58e | 217 | static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) |
1da177e4 | 218 | { |
35fd219a | 219 | enable_single_step(regs); |
9ec4b1f3 | 220 | |
0ccde0a2 AM |
221 | /* |
222 | * On powerpc we should single step on the original | |
223 | * instruction even if the probed insn is a trap | |
224 | * variant as values in regs could play a part in | |
225 | * if the trap is taken or not | |
226 | */ | |
59dc5bfc | 227 | regs_set_return_ip(regs, (unsigned long)p->ainsn.insn); |
1da177e4 LT |
228 | } |
229 | ||
71f6e58e | 230 | static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb) |
0dc036c9 AM |
231 | { |
232 | kcb->prev_kprobe.kp = kprobe_running(); | |
233 | kcb->prev_kprobe.status = kcb->kprobe_status; | |
234 | kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; | |
235 | } | |
236 | ||
71f6e58e | 237 | static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
42cc2060 | 238 | { |
69111bac | 239 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
0dc036c9 AM |
240 | kcb->kprobe_status = kcb->prev_kprobe.status; |
241 | kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; | |
42cc2060 PP |
242 | } |
243 | ||
71f6e58e | 244 | static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, |
0dc036c9 | 245 | struct kprobe_ctlblk *kcb) |
42cc2060 | 246 | { |
69111bac | 247 | __this_cpu_write(current_kprobe, p); |
0dc036c9 | 248 | kcb->kprobe_saved_msr = regs->msr; |
42cc2060 PP |
249 | } |
250 | ||
71f6e58e | 251 | void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
97f7943d | 252 | { |
4c4308cb | 253 | ri->ret_addr = (kprobe_opcode_t *)regs->link; |
b6c5a58d | 254 | ri->fp = NULL; |
4c4308cb CH |
255 | |
256 | /* Replace the return addr with trampoline addr */ | |
adf8a61a | 257 | regs->link = (unsigned long)__kretprobe_trampoline; |
97f7943d | 258 | } |
71f6e58e | 259 | NOKPROBE_SYMBOL(arch_prepare_kretprobe); |
97f7943d | 260 | |
22085337 | 261 | static int try_to_emulate(struct kprobe *p, struct pt_regs *regs) |
1cabd2f8 NR |
262 | { |
263 | int ret; | |
c545b9f0 | 264 | ppc_inst_t insn = ppc_inst_read(p->ainsn.insn); |
1cabd2f8 NR |
265 | |
266 | /* regs->nip is also adjusted if emulate_step returns 1 */ | |
267 | ret = emulate_step(regs, insn); | |
268 | if (ret > 0) { | |
269 | /* | |
270 | * Once this instruction has been boosted | |
271 | * successfully, set the boostable flag | |
272 | */ | |
273 | if (unlikely(p->ainsn.boostable == 0)) | |
274 | p->ainsn.boostable = 1; | |
275 | } else if (ret < 0) { | |
276 | /* | |
277 | * We don't allow kprobes on mtmsr(d)/rfi(d), etc. | |
278 | * So, we should never get here... but, its still | |
279 | * good to catch them, just in case... | |
280 | */ | |
2a83afe7 | 281 | printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn)); |
1cabd2f8 | 282 | BUG(); |
a7b44038 NR |
283 | } else { |
284 | /* | |
285 | * If we haven't previously emulated this instruction, then it | |
286 | * can't be boosted. Note it down so we don't try to do so again. | |
287 | * | |
288 | * If, however, we had emulated this instruction in the past, | |
289 | * then this is just an error with the current run (for | |
290 | * instance, exceptions due to a load/store). We return 0 so | |
291 | * that this is now single-stepped, but continue to try | |
292 | * emulating it in subsequent probe hits. | |
293 | */ | |
294 | if (unlikely(p->ainsn.boostable != 1)) | |
295 | p->ainsn.boostable = -1; | |
296 | } | |
1cabd2f8 NR |
297 | |
298 | return ret; | |
299 | } | |
9fc84914 | 300 | NOKPROBE_SYMBOL(try_to_emulate); |
1cabd2f8 | 301 | |
71f6e58e | 302 | int kprobe_handler(struct pt_regs *regs) |
1da177e4 LT |
303 | { |
304 | struct kprobe *p; | |
305 | int ret = 0; | |
306 | unsigned int *addr = (unsigned int *)regs->nip; | |
d217d545 AM |
307 | struct kprobe_ctlblk *kcb; |
308 | ||
6cc89bad NR |
309 | if (user_mode(regs)) |
310 | return 0; | |
311 | ||
43e8f760 PL |
312 | if (!IS_ENABLED(CONFIG_BOOKE) && |
313 | (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR))) | |
21f8b2fa CL |
314 | return 0; |
315 | ||
d217d545 AM |
316 | /* |
317 | * We don't want to be preempted for the entire | |
318 | * duration of kprobe processing | |
319 | */ | |
320 | preempt_disable(); | |
321 | kcb = get_kprobe_ctlblk(); | |
1da177e4 | 322 | |
1da177e4 LT |
323 | p = get_kprobe(addr); |
324 | if (!p) { | |
9ed5df69 CL |
325 | unsigned int instr; |
326 | ||
25f12ae4 | 327 | if (get_kernel_nofault(instr, addr)) |
9ed5df69 CL |
328 | goto no_kprobe; |
329 | ||
330 | if (instr != BREAKPOINT_INSTRUCTION) { | |
1da177e4 LT |
331 | /* |
332 | * PowerPC has multiple variants of the "trap" | |
333 | * instruction. If the current instruction is a | |
334 | * trap variant, it could belong to someone else | |
335 | */ | |
9ed5df69 | 336 | if (is_trap(instr)) |
6e5fd3a2 | 337 | goto no_kprobe; |
1da177e4 LT |
338 | /* |
339 | * The breakpoint instruction was removed right | |
340 | * after we hit it. Another cpu has removed | |
341 | * either a probepoint or a debugger breakpoint | |
342 | * at this address. In either case, no further | |
343 | * handling of this interrupt is appropriate. | |
344 | */ | |
345 | ret = 1; | |
346 | } | |
347 | /* Not one of ours: let kernel handle it */ | |
348 | goto no_kprobe; | |
349 | } | |
350 | ||
eb4f8e25 CL |
351 | /* Check we're not actually recursing */ |
352 | if (kprobe_running()) { | |
353 | kprobe_opcode_t insn = *p->ainsn.insn; | |
354 | if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) { | |
355 | /* Turn off 'trace' bits */ | |
59dc5bfc NP |
356 | regs_set_return_msr(regs, |
357 | (regs->msr & ~MSR_SINGLESTEP) | | |
358 | kcb->kprobe_saved_msr); | |
eb4f8e25 CL |
359 | goto no_kprobe; |
360 | } | |
361 | ||
362 | /* | |
363 | * We have reentered the kprobe_handler(), since another probe | |
364 | * was hit while within the handler. We here save the original | |
365 | * kprobes variables and just single step on the instruction of | |
366 | * the new probe without calling any user handlers. | |
367 | */ | |
368 | save_previous_kprobe(kcb); | |
369 | set_current_kprobe(p, regs, kcb); | |
370 | kprobes_inc_nmissed_count(p); | |
371 | kcb->kprobe_status = KPROBE_REENTER; | |
372 | if (p->ainsn.boostable >= 0) { | |
373 | ret = try_to_emulate(p, regs); | |
374 | ||
375 | if (ret > 0) { | |
376 | restore_previous_kprobe(kcb); | |
266b1991 | 377 | preempt_enable(); |
eb4f8e25 CL |
378 | return 1; |
379 | } | |
380 | } | |
381 | prepare_singlestep(p, regs); | |
382 | return 1; | |
383 | } | |
384 | ||
0dc036c9 AM |
385 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
386 | set_current_kprobe(p, regs, kcb); | |
cce188bd MH |
387 | if (p->pre_handler && p->pre_handler(p, regs)) { |
388 | /* handler changed execution path, so skip ss setup */ | |
389 | reset_current_kprobe(); | |
266b1991 | 390 | preempt_enable(); |
1da177e4 | 391 | return 1; |
cce188bd | 392 | } |
1da177e4 | 393 | |
e6349a95 | 394 | if (p->ainsn.boostable >= 0) { |
1cabd2f8 | 395 | ret = try_to_emulate(p, regs); |
e6349a95 | 396 | |
e6349a95 | 397 | if (ret > 0) { |
e6349a95 AM |
398 | if (p->post_handler) |
399 | p->post_handler(p, regs, 0); | |
400 | ||
401 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
402 | reset_current_kprobe(); | |
266b1991 | 403 | preempt_enable(); |
e6349a95 | 404 | return 1; |
1cabd2f8 | 405 | } |
e6349a95 | 406 | } |
1da177e4 | 407 | prepare_singlestep(p, regs); |
0dc036c9 | 408 | kcb->kprobe_status = KPROBE_HIT_SS; |
1da177e4 LT |
409 | return 1; |
410 | ||
411 | no_kprobe: | |
266b1991 | 412 | preempt_enable(); |
1da177e4 LT |
413 | return ret; |
414 | } | |
71f6e58e | 415 | NOKPROBE_SYMBOL(kprobe_handler); |
1da177e4 | 416 | |
97f7943d RL |
417 | /* |
418 | * Function return probe trampoline: | |
419 | * - init_kprobes() establishes a probepoint here | |
420 | * - When the probed function returns, this probe | |
421 | * causes the handlers to fire | |
422 | */ | |
adf8a61a MH |
423 | asm(".global __kretprobe_trampoline\n" |
424 | ".type __kretprobe_trampoline, @function\n" | |
425 | "__kretprobe_trampoline:\n" | |
61ed9cfb | 426 | "nop\n" |
762df10b | 427 | "blr\n" |
adf8a61a | 428 | ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"); |
97f7943d RL |
429 | |
430 | /* | |
431 | * Called when the probe at kretprobe trampoline is hit | |
432 | */ | |
71f6e58e | 433 | static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
97f7943d | 434 | { |
b6c5a58d | 435 | unsigned long orig_ret_address; |
e6e133c4 | 436 | |
96fed8ac | 437 | orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); |
762df10b | 438 | /* |
e6e133c4 NR |
439 | * We get here through one of two paths: |
440 | * 1. by taking a trap -> kprobe_handler() -> here | |
441 | * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here | |
442 | * | |
443 | * When going back through (1), we need regs->nip to be setup properly | |
444 | * as it is used to determine the return address from the trap. | |
445 | * For (2), since nip is not honoured with optprobes, we instead setup | |
446 | * the link register properly so that the subsequent 'blr' in | |
adf8a61a | 447 | * __kretprobe_trampoline jumps back to the right instruction. |
e6e133c4 NR |
448 | * |
449 | * For nip, we should set the address to the previous instruction since | |
450 | * we end up emulating it in kprobe_handler(), which increments the nip | |
451 | * again. | |
762df10b | 452 | */ |
59dc5bfc | 453 | regs_set_return_ip(regs, orig_ret_address - 4); |
762df10b | 454 | regs->link = orig_ret_address; |
97f7943d | 455 | |
e6e133c4 | 456 | return 0; |
97f7943d | 457 | } |
71f6e58e | 458 | NOKPROBE_SYMBOL(trampoline_probe_handler); |
97f7943d | 459 | |
1da177e4 LT |
460 | /* |
461 | * Called after single-stepping. p->addr is the address of the | |
462 | * instruction whose first byte has been replaced by the "breakpoint" | |
463 | * instruction. To avoid the SMP problems that can occur when we | |
464 | * temporarily put back the original opcode to single-step, we | |
465 | * single-stepped a copy of the instruction. The address of this | |
466 | * copy is p->ainsn.insn. | |
467 | */ | |
71f6e58e | 468 | int kprobe_post_handler(struct pt_regs *regs) |
1da177e4 | 469 | { |
622cf6f4 | 470 | int len; |
0dc036c9 AM |
471 | struct kprobe *cur = kprobe_running(); |
472 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
473 | ||
6cc89bad | 474 | if (!cur || user_mode(regs)) |
1da177e4 LT |
475 | return 0; |
476 | ||
69d4d6e5 | 477 | len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn)); |
b76e59d1 | 478 | /* make sure we got here for instruction we have a kprobe on */ |
622cf6f4 | 479 | if (((unsigned long)cur->ainsn.insn + len) != regs->nip) |
b76e59d1 KG |
480 | return 0; |
481 | ||
0dc036c9 AM |
482 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
483 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
484 | cur->post_handler(cur, regs, 0); | |
42cc2060 | 485 | } |
1da177e4 | 486 | |
db97bc7f | 487 | /* Adjust nip to after the single-stepped instruction */ |
59dc5bfc NP |
488 | regs_set_return_ip(regs, (unsigned long)cur->addr + len); |
489 | regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr); | |
1da177e4 | 490 | |
42cc2060 | 491 | /*Restore back the original saved kprobes variables and continue. */ |
0dc036c9 AM |
492 | if (kcb->kprobe_status == KPROBE_REENTER) { |
493 | restore_previous_kprobe(kcb); | |
42cc2060 PP |
494 | goto out; |
495 | } | |
0dc036c9 | 496 | reset_current_kprobe(); |
42cc2060 | 497 | out: |
266b1991 | 498 | preempt_enable(); |
1da177e4 LT |
499 | |
500 | /* | |
501 | * if somebody else is singlestepping across a probe point, msr | |
f8279621 | 502 | * will have DE/SE set, in which case, continue the remaining processing |
1da177e4 LT |
503 | * of do_debug, as if this is not a probe hit. |
504 | */ | |
f8279621 | 505 | if (regs->msr & MSR_SINGLESTEP) |
1da177e4 LT |
506 | return 0; |
507 | ||
508 | return 1; | |
509 | } | |
71f6e58e | 510 | NOKPROBE_SYMBOL(kprobe_post_handler); |
1da177e4 | 511 | |
71f6e58e | 512 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
1da177e4 | 513 | { |
0dc036c9 AM |
514 | struct kprobe *cur = kprobe_running(); |
515 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
50e21f2b PP |
516 | const struct exception_table_entry *entry; |
517 | ||
518 | switch(kcb->kprobe_status) { | |
519 | case KPROBE_HIT_SS: | |
520 | case KPROBE_REENTER: | |
521 | /* | |
522 | * We are here because the instruction being single | |
523 | * stepped caused a page fault. We reset the current | |
524 | * kprobe and the nip points back to the probe address | |
525 | * and allow the page fault handler to continue as a | |
526 | * normal page fault. | |
527 | */ | |
59dc5bfc NP |
528 | regs_set_return_ip(regs, (unsigned long)cur->addr); |
529 | /* Turn off 'trace' bits */ | |
530 | regs_set_return_msr(regs, | |
531 | (regs->msr & ~MSR_SINGLESTEP) | | |
532 | kcb->kprobe_saved_msr); | |
50e21f2b PP |
533 | if (kcb->kprobe_status == KPROBE_REENTER) |
534 | restore_previous_kprobe(kcb); | |
535 | else | |
536 | reset_current_kprobe(); | |
266b1991 | 537 | preempt_enable(); |
50e21f2b PP |
538 | break; |
539 | case KPROBE_HIT_ACTIVE: | |
540 | case KPROBE_HIT_SSDONE: | |
541 | /* | |
50e21f2b PP |
542 | * In case the user-specified fault handler returned |
543 | * zero, try to fix up. | |
544 | */ | |
545 | if ((entry = search_exception_tables(regs->nip)) != NULL) { | |
59dc5bfc | 546 | regs_set_return_ip(regs, extable_fixup(entry)); |
50e21f2b PP |
547 | return 1; |
548 | } | |
549 | ||
550 | /* | |
551 | * fixup_exception() could not handle it, | |
552 | * Let do_page_fault() fix it. | |
553 | */ | |
554 | break; | |
555 | default: | |
556 | break; | |
1da177e4 LT |
557 | } |
558 | return 0; | |
559 | } | |
71f6e58e | 560 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
1da177e4 | 561 | |
97f7943d | 562 | static struct kprobe trampoline_p = { |
adf8a61a | 563 | .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, |
97f7943d RL |
564 | .pre_handler = trampoline_probe_handler |
565 | }; | |
566 | ||
6772926b | 567 | int __init arch_init_kprobes(void) |
97f7943d RL |
568 | { |
569 | return register_kprobe(&trampoline_p); | |
570 | } | |
bf8f6e5b | 571 | |
71f6e58e | 572 | int arch_trampoline_kprobe(struct kprobe *p) |
bf8f6e5b | 573 | { |
adf8a61a | 574 | if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) |
bf8f6e5b AM |
575 | return 1; |
576 | ||
577 | return 0; | |
578 | } | |
71f6e58e | 579 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |