Commit | Line | Data |
---|---|---|
24ba613c AS |
1 | /* |
2 | * arch/arm/kernel/kprobes.c | |
3 | * | |
4 | * Kprobes on ARM | |
5 | * | |
6 | * Abhishek Sagar <sagar.abhishek@gmail.com> | |
7 | * Copyright (C) 2006, 2007 Motorola Inc. | |
8 | * | |
9 | * Nicolas Pitre <nico@marvell.com> | |
10 | * Copyright (C) 2007 Marvell Ltd. | |
11 | * | |
12 | * This program is free software; you can redistribute it and/or modify | |
13 | * it under the terms of the GNU General Public License version 2 as | |
14 | * published by the Free Software Foundation. | |
15 | * | |
16 | * This program is distributed in the hope that it will be useful, | |
17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
19 | * General Public License for more details. | |
20 | */ | |
21 | ||
22 | #include <linux/kernel.h> | |
23 | #include <linux/kprobes.h> | |
24 | #include <linux/module.h> | |
25 | #include <linux/stringify.h> | |
26 | #include <asm/traps.h> | |
27 | #include <asm/cacheflush.h> | |
28 | ||
24ba613c AS |
29 | #define MIN_STACK_SIZE(addr) \ |
30 | min((unsigned long)MAX_STACK_SIZE, \ | |
31 | (unsigned long)current_thread_info() + THREAD_START_SP - (addr)) | |
32 | ||
33 | #define flush_insns(addr, cnt) \ | |
34 | flush_icache_range((unsigned long)(addr), \ | |
35 | (unsigned long)(addr) + \ | |
36 | sizeof(kprobe_opcode_t) * (cnt)) | |
37 | ||
38 | /* Used as a marker in ARM_pc to note when we're in a jprobe. */ | |
39 | #define JPROBE_MAGIC_ADDR 0xffffffff | |
40 | ||
41 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; | |
42 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); | |
43 | ||
44 | ||
45 | int __kprobes arch_prepare_kprobe(struct kprobe *p) | |
46 | { | |
47 | kprobe_opcode_t insn; | |
48 | kprobe_opcode_t tmp_insn[MAX_INSN_SIZE]; | |
49 | unsigned long addr = (unsigned long)p->addr; | |
50 | int is; | |
51 | ||
785d3cd2 | 52 | if (addr & 0x3 || in_exception_text(addr)) |
24ba613c AS |
53 | return -EINVAL; |
54 | ||
55 | insn = *p->addr; | |
56 | p->opcode = insn; | |
57 | p->ainsn.insn = tmp_insn; | |
58 | ||
59 | switch (arm_kprobe_decode_insn(insn, &p->ainsn)) { | |
60 | case INSN_REJECTED: /* not supported */ | |
61 | return -EINVAL; | |
62 | ||
63 | case INSN_GOOD: /* instruction uses slot */ | |
64 | p->ainsn.insn = get_insn_slot(); | |
65 | if (!p->ainsn.insn) | |
66 | return -ENOMEM; | |
67 | for (is = 0; is < MAX_INSN_SIZE; ++is) | |
68 | p->ainsn.insn[is] = tmp_insn[is]; | |
8f79ff0c | 69 | flush_insns(p->ainsn.insn, MAX_INSN_SIZE); |
24ba613c AS |
70 | break; |
71 | ||
72 | case INSN_GOOD_NO_SLOT: /* instruction doesn't need insn slot */ | |
73 | p->ainsn.insn = NULL; | |
74 | break; | |
75 | } | |
76 | ||
77 | return 0; | |
78 | } | |
79 | ||
80 | void __kprobes arch_arm_kprobe(struct kprobe *p) | |
81 | { | |
82 | *p->addr = KPROBE_BREAKPOINT_INSTRUCTION; | |
83 | flush_insns(p->addr, 1); | |
84 | } | |
85 | ||
86 | void __kprobes arch_disarm_kprobe(struct kprobe *p) | |
87 | { | |
88 | *p->addr = p->opcode; | |
89 | flush_insns(p->addr, 1); | |
90 | } | |
91 | ||
92 | void __kprobes arch_remove_kprobe(struct kprobe *p) | |
93 | { | |
94 | if (p->ainsn.insn) { | |
24ba613c | 95 | free_insn_slot(p->ainsn.insn, 0); |
24ba613c AS |
96 | p->ainsn.insn = NULL; |
97 | } | |
98 | } | |
99 | ||
100 | static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) | |
101 | { | |
102 | kcb->prev_kprobe.kp = kprobe_running(); | |
103 | kcb->prev_kprobe.status = kcb->kprobe_status; | |
104 | } | |
105 | ||
106 | static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) | |
107 | { | |
108 | __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; | |
109 | kcb->kprobe_status = kcb->prev_kprobe.status; | |
110 | } | |
111 | ||
112 | static void __kprobes set_current_kprobe(struct kprobe *p) | |
113 | { | |
114 | __get_cpu_var(current_kprobe) = p; | |
115 | } | |
116 | ||
117 | static void __kprobes singlestep(struct kprobe *p, struct pt_regs *regs, | |
118 | struct kprobe_ctlblk *kcb) | |
119 | { | |
120 | regs->ARM_pc += 4; | |
121 | p->ainsn.insn_handler(p, regs); | |
122 | } | |
123 | ||
124 | /* | |
125 | * Called with IRQs disabled. IRQs must remain disabled from that point | |
126 | * all the way until processing this kprobe is complete. The current | |
127 | * kprobes implementation cannot process more than one nested level of | |
128 | * kprobe, and that level is reserved for user kprobe handlers, so we can't | |
129 | * risk encountering a new kprobe in an interrupt handler. | |
130 | */ | |
131 | void __kprobes kprobe_handler(struct pt_regs *regs) | |
132 | { | |
133 | struct kprobe *p, *cur; | |
134 | struct kprobe_ctlblk *kcb; | |
135 | kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->ARM_pc; | |
136 | ||
137 | kcb = get_kprobe_ctlblk(); | |
138 | cur = kprobe_running(); | |
139 | p = get_kprobe(addr); | |
140 | ||
141 | if (p) { | |
142 | if (cur) { | |
143 | /* Kprobe is pending, so we're recursing. */ | |
144 | switch (kcb->kprobe_status) { | |
145 | case KPROBE_HIT_ACTIVE: | |
146 | case KPROBE_HIT_SSDONE: | |
147 | /* A pre- or post-handler probe got us here. */ | |
148 | kprobes_inc_nmissed_count(p); | |
149 | save_previous_kprobe(kcb); | |
150 | set_current_kprobe(p); | |
151 | kcb->kprobe_status = KPROBE_REENTER; | |
152 | singlestep(p, regs, kcb); | |
153 | restore_previous_kprobe(kcb); | |
154 | break; | |
155 | default: | |
156 | /* impossible cases */ | |
157 | BUG(); | |
158 | } | |
159 | } else { | |
160 | set_current_kprobe(p); | |
161 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; | |
162 | ||
163 | /* | |
164 | * If we have no pre-handler or it returned 0, we | |
165 | * continue with normal processing. If we have a | |
166 | * pre-handler and it returned non-zero, it prepped | |
167 | * for calling the break_handler below on re-entry, | |
168 | * so get out doing nothing more here. | |
169 | */ | |
170 | if (!p->pre_handler || !p->pre_handler(p, regs)) { | |
171 | kcb->kprobe_status = KPROBE_HIT_SS; | |
172 | singlestep(p, regs, kcb); | |
173 | if (p->post_handler) { | |
174 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
175 | p->post_handler(p, regs, 0); | |
176 | } | |
177 | reset_current_kprobe(); | |
178 | } | |
179 | } | |
180 | } else if (cur) { | |
181 | /* We probably hit a jprobe. Call its break handler. */ | |
182 | if (cur->break_handler && cur->break_handler(cur, regs)) { | |
183 | kcb->kprobe_status = KPROBE_HIT_SS; | |
184 | singlestep(cur, regs, kcb); | |
185 | if (cur->post_handler) { | |
186 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
187 | cur->post_handler(cur, regs, 0); | |
188 | } | |
189 | } | |
190 | reset_current_kprobe(); | |
191 | } else { | |
192 | /* | |
193 | * The probe was removed and a race is in progress. | |
194 | * There is nothing we can do about it. Let's restart | |
195 | * the instruction. By the time we can restart, the | |
196 | * real instruction will be there. | |
197 | */ | |
198 | } | |
199 | } | |
200 | ||
3305a607 | 201 | static int __kprobes kprobe_trap_handler(struct pt_regs *regs, unsigned int instr) |
24ba613c | 202 | { |
3305a607 NP |
203 | unsigned long flags; |
204 | local_irq_save(flags); | |
24ba613c | 205 | kprobe_handler(regs); |
3305a607 | 206 | local_irq_restore(flags); |
24ba613c AS |
207 | return 0; |
208 | } | |
209 | ||
210 | int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) | |
211 | { | |
212 | struct kprobe *cur = kprobe_running(); | |
213 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
214 | ||
215 | switch (kcb->kprobe_status) { | |
216 | case KPROBE_HIT_SS: | |
217 | case KPROBE_REENTER: | |
218 | /* | |
219 | * We are here because the instruction being single | |
220 | * stepped caused a page fault. We reset the current | |
221 | * kprobe and the PC to point back to the probe address | |
222 | * and allow the page fault handler to continue as a | |
223 | * normal page fault. | |
224 | */ | |
225 | regs->ARM_pc = (long)cur->addr; | |
226 | if (kcb->kprobe_status == KPROBE_REENTER) { | |
227 | restore_previous_kprobe(kcb); | |
228 | } else { | |
229 | reset_current_kprobe(); | |
230 | } | |
231 | break; | |
232 | ||
233 | case KPROBE_HIT_ACTIVE: | |
234 | case KPROBE_HIT_SSDONE: | |
235 | /* | |
236 | * We increment the nmissed count for accounting, | |
237 | * we can also use npre/npostfault count for accounting | |
238 | * these specific fault cases. | |
239 | */ | |
240 | kprobes_inc_nmissed_count(cur); | |
241 | ||
242 | /* | |
243 | * We come here because instructions in the pre/post | |
244 | * handler caused the page_fault, this could happen | |
245 | * if handler tries to access user space by | |
246 | * copy_from_user(), get_user() etc. Let the | |
247 | * user-specified handler try to fix it. | |
248 | */ | |
249 | if (cur->fault_handler && cur->fault_handler(cur, regs, fsr)) | |
250 | return 1; | |
251 | break; | |
252 | ||
253 | default: | |
254 | break; | |
255 | } | |
256 | ||
257 | return 0; | |
258 | } | |
259 | ||
260 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, | |
261 | unsigned long val, void *data) | |
262 | { | |
263 | /* | |
264 | * notify_die() is currently never called on ARM, | |
265 | * so this callback is currently empty. | |
266 | */ | |
267 | return NOTIFY_DONE; | |
268 | } | |
269 | ||
270 | /* | |
271 | * When a retprobed function returns, trampoline_handler() is called, | |
272 | * calling the kretprobe's handler. We construct a struct pt_regs to | |
273 | * give a view of registers r0-r11 to the user return-handler. This is | |
274 | * not a complete pt_regs structure, but that should be plenty sufficient | |
275 | * for kretprobe handlers which should normally be interested in r0 only | |
276 | * anyway. | |
277 | */ | |
e0773410 | 278 | void __naked __kprobes kretprobe_trampoline(void) |
24ba613c AS |
279 | { |
280 | __asm__ __volatile__ ( | |
281 | "stmdb sp!, {r0 - r11} \n\t" | |
282 | "mov r0, sp \n\t" | |
283 | "bl trampoline_handler \n\t" | |
284 | "mov lr, r0 \n\t" | |
285 | "ldmia sp!, {r0 - r11} \n\t" | |
286 | "mov pc, lr \n\t" | |
287 | : : : "memory"); | |
288 | } | |
289 | ||
290 | /* Called from kretprobe_trampoline */ | |
291 | static __used __kprobes void *trampoline_handler(struct pt_regs *regs) | |
292 | { | |
293 | struct kretprobe_instance *ri = NULL; | |
294 | struct hlist_head *head, empty_rp; | |
295 | struct hlist_node *node, *tmp; | |
296 | unsigned long flags, orig_ret_address = 0; | |
297 | unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline; | |
298 | ||
299 | INIT_HLIST_HEAD(&empty_rp); | |
ef53d9c5 | 300 | kretprobe_hash_lock(current, &head, &flags); |
24ba613c AS |
301 | |
302 | /* | |
303 | * It is possible to have multiple instances associated with a given | |
304 | * task either because multiple functions in the call path have | |
305 | * a return probe installed on them, and/or more than one return | |
306 | * probe was registered for a target function. | |
307 | * | |
308 | * We can handle this because: | |
309 | * - instances are always inserted at the head of the list | |
310 | * - when multiple return probes are registered for the same | |
311 | * function, the first instance's ret_addr will point to the | |
312 | * real return address, and all the rest will point to | |
313 | * kretprobe_trampoline | |
314 | */ | |
315 | hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { | |
316 | if (ri->task != current) | |
317 | /* another task is sharing our hash bucket */ | |
318 | continue; | |
319 | ||
320 | if (ri->rp && ri->rp->handler) { | |
321 | __get_cpu_var(current_kprobe) = &ri->rp->kp; | |
322 | get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; | |
323 | ri->rp->handler(ri, regs); | |
324 | __get_cpu_var(current_kprobe) = NULL; | |
325 | } | |
326 | ||
327 | orig_ret_address = (unsigned long)ri->ret_addr; | |
328 | recycle_rp_inst(ri, &empty_rp); | |
329 | ||
330 | if (orig_ret_address != trampoline_address) | |
331 | /* | |
332 | * This is the real return address. Any other | |
333 | * instances associated with this task are for | |
334 | * other calls deeper on the call stack | |
335 | */ | |
336 | break; | |
337 | } | |
338 | ||
339 | kretprobe_assert(ri, orig_ret_address, trampoline_address); | |
ef53d9c5 | 340 | kretprobe_hash_unlock(current, &flags); |
24ba613c AS |
341 | |
342 | hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { | |
343 | hlist_del(&ri->hlist); | |
344 | kfree(ri); | |
345 | } | |
346 | ||
347 | return (void *)orig_ret_address; | |
348 | } | |
349 | ||
24ba613c AS |
350 | void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, |
351 | struct pt_regs *regs) | |
352 | { | |
353 | ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr; | |
354 | ||
355 | /* Replace the return addr with trampoline addr. */ | |
356 | regs->ARM_lr = (unsigned long)&kretprobe_trampoline; | |
357 | } | |
358 | ||
359 | int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) | |
360 | { | |
361 | struct jprobe *jp = container_of(p, struct jprobe, kp); | |
362 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
363 | long sp_addr = regs->ARM_sp; | |
364 | ||
365 | kcb->jprobe_saved_regs = *regs; | |
366 | memcpy(kcb->jprobes_stack, (void *)sp_addr, MIN_STACK_SIZE(sp_addr)); | |
367 | regs->ARM_pc = (long)jp->entry; | |
368 | regs->ARM_cpsr |= PSR_I_BIT; | |
369 | preempt_disable(); | |
370 | return 1; | |
371 | } | |
372 | ||
373 | void __kprobes jprobe_return(void) | |
374 | { | |
375 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
376 | ||
377 | __asm__ __volatile__ ( | |
378 | /* | |
379 | * Setup an empty pt_regs. Fill SP and PC fields as | |
380 | * they're needed by longjmp_break_handler. | |
381 | */ | |
382 | "sub sp, %0, %1 \n\t" | |
383 | "ldr r0, ="__stringify(JPROBE_MAGIC_ADDR)"\n\t" | |
384 | "str %0, [sp, %2] \n\t" | |
385 | "str r0, [sp, %3] \n\t" | |
386 | "mov r0, sp \n\t" | |
387 | "bl kprobe_handler \n\t" | |
388 | ||
389 | /* | |
390 | * Return to the context saved by setjmp_pre_handler | |
391 | * and restored by longjmp_break_handler. | |
392 | */ | |
393 | "ldr r0, [sp, %4] \n\t" | |
394 | "msr cpsr_cxsf, r0 \n\t" | |
395 | "ldmia sp, {r0 - pc} \n\t" | |
396 | : | |
397 | : "r" (kcb->jprobe_saved_regs.ARM_sp), | |
398 | "I" (sizeof(struct pt_regs)), | |
399 | "J" (offsetof(struct pt_regs, ARM_sp)), | |
400 | "J" (offsetof(struct pt_regs, ARM_pc)), | |
401 | "J" (offsetof(struct pt_regs, ARM_cpsr)) | |
402 | : "memory", "cc"); | |
403 | } | |
404 | ||
405 | int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) | |
406 | { | |
407 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
408 | long stack_addr = kcb->jprobe_saved_regs.ARM_sp; | |
409 | long orig_sp = regs->ARM_sp; | |
410 | struct jprobe *jp = container_of(p, struct jprobe, kp); | |
411 | ||
412 | if (regs->ARM_pc == JPROBE_MAGIC_ADDR) { | |
413 | if (orig_sp != stack_addr) { | |
414 | struct pt_regs *saved_regs = | |
415 | (struct pt_regs *)kcb->jprobe_saved_regs.ARM_sp; | |
416 | printk("current sp %lx does not match saved sp %lx\n", | |
417 | orig_sp, stack_addr); | |
418 | printk("Saved registers for jprobe %p\n", jp); | |
419 | show_regs(saved_regs); | |
420 | printk("Current registers\n"); | |
421 | show_regs(regs); | |
422 | BUG(); | |
423 | } | |
424 | *regs = kcb->jprobe_saved_regs; | |
425 | memcpy((void *)stack_addr, kcb->jprobes_stack, | |
426 | MIN_STACK_SIZE(stack_addr)); | |
427 | preempt_enable_no_resched(); | |
428 | return 1; | |
429 | } | |
430 | return 0; | |
431 | } | |
432 | ||
b24061fa NP |
433 | int __kprobes arch_trampoline_kprobe(struct kprobe *p) |
434 | { | |
435 | return 0; | |
436 | } | |
437 | ||
24ba613c AS |
438 | static struct undef_hook kprobes_break_hook = { |
439 | .instr_mask = 0xffffffff, | |
440 | .instr_val = KPROBE_BREAKPOINT_INSTRUCTION, | |
441 | .cpsr_mask = MODE_MASK, | |
442 | .cpsr_val = SVC_MODE, | |
443 | .fn = kprobe_trap_handler, | |
444 | }; | |
445 | ||
446 | int __init arch_init_kprobes() | |
447 | { | |
448 | arm_kprobe_decode_init(); | |
449 | register_undef_hook(&kprobes_break_hook); | |
450 | return 0; | |
451 | } |