Commit | Line | Data |
---|---|---|
819e50e2 AT |
1 | /* |
2 | * arch/arm64/kernel/entry-ftrace.S | |
3 | * | |
4 | * Copyright (C) 2013 Linaro Limited | |
5 | * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | */ | |
11 | ||
12 | #include <linux/linkage.h> | |
f705d954 | 13 | #include <asm/assembler.h> |
819e50e2 AT |
14 | #include <asm/ftrace.h> |
15 | #include <asm/insn.h> | |
16 | ||
17 | /* | |
18 | * Gcc with -pg will put the following code in the beginning of each function: | |
19 | * mov x0, x30 | |
20 | * bl _mcount | |
21 | * [function's body ...] | |
22 | * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic | |
23 | * ftrace is enabled. | |
24 | * | |
25 | * Please note that x0 as an argument will not be used here because we can | |
26 | * get lr(x30) of instrumented function at any time by winding up call stack | |
27 | * as long as the kernel is compiled without -fomit-frame-pointer. | |
28 | * (or CONFIG_FRAME_POINTER, this is forced on arm64) | |
29 | * | |
30 | * stack layout after mcount_enter in _mcount(): | |
31 | * | |
32 | * current sp/fp => 0:+-----+ | |
33 | * in _mcount() | x29 | -> instrumented function's fp | |
34 | * +-----+ | |
35 | * | x30 | -> _mcount()'s lr (= instrumented function's pc) | |
36 | * old sp => +16:+-----+ | |
37 | * when instrumented | | | |
38 | * function calls | ... | | |
39 | * _mcount() | | | |
40 | * | | | |
41 | * instrumented => +xx:+-----+ | |
42 | * function's fp | x29 | -> parent's fp | |
43 | * +-----+ | |
44 | * | x30 | -> instrumented function's lr (= parent's pc) | |
45 | * +-----+ | |
46 | * | ... | | |
47 | */ | |
48 | ||
49 | .macro mcount_enter | |
50 | stp x29, x30, [sp, #-16]! | |
51 | mov x29, sp | |
52 | .endm | |
53 | ||
54 | .macro mcount_exit | |
55 | ldp x29, x30, [sp], #16 | |
56 | ret | |
57 | .endm | |
58 | ||
59 | .macro mcount_adjust_addr rd, rn | |
60 | sub \rd, \rn, #AARCH64_INSN_SIZE | |
61 | .endm | |
62 | ||
63 | /* for instrumented function's parent */ | |
64 | .macro mcount_get_parent_fp reg | |
65 | ldr \reg, [x29] | |
66 | ldr \reg, [\reg] | |
67 | .endm | |
68 | ||
69 | /* for instrumented function */ | |
70 | .macro mcount_get_pc0 reg | |
71 | mcount_adjust_addr \reg, x30 | |
72 | .endm | |
73 | ||
74 | .macro mcount_get_pc reg | |
75 | ldr \reg, [x29, #8] | |
76 | mcount_adjust_addr \reg, \reg | |
77 | .endm | |
78 | ||
79 | .macro mcount_get_lr reg | |
80 | ldr \reg, [x29] | |
81 | ldr \reg, [\reg, #8] | |
82 | mcount_adjust_addr \reg, \reg | |
83 | .endm | |
84 | ||
85 | .macro mcount_get_lr_addr reg | |
86 | ldr \reg, [x29] | |
87 | add \reg, \reg, #8 | |
88 | .endm | |
89 | ||
bd7d38db | 90 | #ifndef CONFIG_DYNAMIC_FTRACE |
819e50e2 AT |
91 | /* |
92 | * void _mcount(unsigned long return_address) | |
93 | * @return_address: return address to instrumented function | |
94 | * | |
95 | * This function makes calls, if enabled, to: | |
96 | * - tracer function to probe instrumented function's entry, | |
97 | * - ftrace_graph_caller to set up an exit hook | |
98 | */ | |
99 | ENTRY(_mcount) | |
819e50e2 AT |
100 | mcount_enter |
101 | ||
829d2bd1 | 102 | ldr_l x2, ftrace_trace_function |
819e50e2 AT |
103 | adr x0, ftrace_stub |
104 | cmp x0, x2 // if (ftrace_trace_function | |
105 | b.eq skip_ftrace_call // != ftrace_stub) { | |
106 | ||
107 | mcount_get_pc x0 // function's pc | |
108 | mcount_get_lr x1 // function's lr (= parent's pc) | |
109 | blr x2 // (*ftrace_trace_function)(pc, lr); | |
110 | ||
d125bffc JT |
111 | skip_ftrace_call: // } |
112 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
829d2bd1 | 113 | ldr_l x2, ftrace_graph_return |
f1ba46ee AB |
114 | cmp x0, x2 // if ((ftrace_graph_return |
115 | b.ne ftrace_graph_caller // != ftrace_stub) | |
116 | ||
829d2bd1 MR |
117 | ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry |
118 | adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) | |
819e50e2 AT |
119 | cmp x0, x2 |
120 | b.ne ftrace_graph_caller // ftrace_graph_caller(); | |
819e50e2 | 121 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
d125bffc | 122 | mcount_exit |
819e50e2 AT |
123 | ENDPROC(_mcount) |
124 | ||
bd7d38db AT |
125 | #else /* CONFIG_DYNAMIC_FTRACE */ |
126 | /* | |
127 | * _mcount() is used to build the kernel with -pg option, but all the branch | |
128 | * instructions to _mcount() are replaced to NOP initially at kernel start up, | |
129 | * and later on, NOP to branch to ftrace_caller() when enabled or branch to | |
130 | * NOP when disabled per-function base. | |
131 | */ | |
132 | ENTRY(_mcount) | |
133 | ret | |
134 | ENDPROC(_mcount) | |
135 | ||
136 | /* | |
137 | * void ftrace_caller(unsigned long return_address) | |
138 | * @return_address: return address to instrumented function | |
139 | * | |
140 | * This function is a counterpart of _mcount() in 'static' ftrace, and | |
141 | * makes calls to: | |
142 | * - tracer function to probe instrumented function's entry, | |
143 | * - ftrace_graph_caller to set up an exit hook | |
144 | */ | |
145 | ENTRY(ftrace_caller) | |
146 | mcount_enter | |
147 | ||
148 | mcount_get_pc0 x0 // function's pc | |
149 | mcount_get_lr x1 // function's lr | |
150 | ||
e4fe1966 | 151 | GLOBAL(ftrace_call) // tracer(pc, lr); |
bd7d38db AT |
152 | nop // This will be replaced with "bl xxx" |
153 | // where xxx can be any kind of tracer. | |
154 | ||
155 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
e4fe1966 | 156 | GLOBAL(ftrace_graph_call) // ftrace_graph_caller(); |
bd7d38db AT |
157 | nop // If enabled, this will be replaced |
158 | // "b ftrace_graph_caller" | |
159 | #endif | |
160 | ||
161 | mcount_exit | |
162 | ENDPROC(ftrace_caller) | |
163 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
164 | ||
819e50e2 AT |
165 | ENTRY(ftrace_stub) |
166 | ret | |
167 | ENDPROC(ftrace_stub) | |
168 | ||
169 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
ee556d00 LB |
170 | /* save return value regs*/ |
171 | .macro save_return_regs | |
172 | sub sp, sp, #64 | |
173 | stp x0, x1, [sp] | |
174 | stp x2, x3, [sp, #16] | |
175 | stp x4, x5, [sp, #32] | |
176 | stp x6, x7, [sp, #48] | |
177 | .endm | |
178 | ||
179 | /* restore return value regs*/ | |
180 | .macro restore_return_regs | |
181 | ldp x0, x1, [sp] | |
182 | ldp x2, x3, [sp, #16] | |
183 | ldp x4, x5, [sp, #32] | |
184 | ldp x6, x7, [sp, #48] | |
185 | add sp, sp, #64 | |
186 | .endm | |
187 | ||
819e50e2 AT |
188 | /* |
189 | * void ftrace_graph_caller(void) | |
190 | * | |
191 | * Called from _mcount() or ftrace_caller() when function_graph tracer is | |
192 | * selected. | |
193 | * This function w/ prepare_ftrace_return() fakes link register's value on | |
194 | * the call stack in order to intercept instrumented function's return path | |
195 | * and run return_to_handler() later on its exit. | |
196 | */ | |
197 | ENTRY(ftrace_graph_caller) | |
198 | mcount_get_lr_addr x0 // pointer to function's saved lr | |
199 | mcount_get_pc x1 // function's pc | |
200 | mcount_get_parent_fp x2 // parent's fp | |
201 | bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) | |
202 | ||
203 | mcount_exit | |
204 | ENDPROC(ftrace_graph_caller) | |
205 | ||
206 | /* | |
207 | * void return_to_handler(void) | |
208 | * | |
209 | * Run ftrace_return_to_handler() before going back to parent. | |
5c176aff | 210 | * @fp is checked against the value passed by ftrace_graph_caller(). |
819e50e2 AT |
211 | */ |
212 | ENTRY(return_to_handler) | |
ee556d00 | 213 | save_return_regs |
819e50e2 AT |
214 | mov x0, x29 // parent's fp |
215 | bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); | |
216 | mov x30, x0 // restore the original return address | |
ee556d00 | 217 | restore_return_regs |
819e50e2 AT |
218 | ret |
219 | END(return_to_handler) | |
220 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |