Commit | Line | Data |
---|---|---|
e18eead3 SR |
1 | /* |
2 | * linux/arch/x86_64/mcount_64.S | |
3 | * | |
4 | * Copyright (C) 2014 Steven Rostedt, Red Hat Inc | |
5 | */ | |
6 | ||
7 | #include <linux/linkage.h> | |
8 | #include <asm/ptrace.h> | |
9 | #include <asm/ftrace.h> | |
784d5699 | 10 | #include <asm/export.h> |
e18eead3 SR |
11 | |
12 | ||
13 | .code64 | |
14 | .section .entry.text, "ax" | |
15 | ||
16 | ||
17 | #ifdef CONFIG_FUNCTION_TRACER | |
18 | ||
19 | #ifdef CC_USING_FENTRY | |
20 | # define function_hook __fentry__ | |
5de0a8c0 | 21 | EXPORT_SYMBOL(__fentry__) |
e18eead3 SR |
22 | #else |
23 | # define function_hook mcount | |
5de0a8c0 | 24 | EXPORT_SYMBOL(mcount) |
e18eead3 SR |
25 | #endif |
26 | ||
0687c36e SRRH |
27 | /* All cases save the original rbp (8 bytes) */ |
28 | #ifdef CONFIG_FRAME_POINTER | |
29 | # ifdef CC_USING_FENTRY | |
30 | /* Save parent and function stack frames (rip and rbp) */ | |
31 | # define MCOUNT_FRAME_SIZE (8+16*2) | |
32 | # else | |
33 | /* Save just function stack frame (rip and rbp) */ | |
34 | # define MCOUNT_FRAME_SIZE (8+16) | |
35 | # endif | |
36 | #else | |
37 | /* No need to save a stack frame */ | |
38 | # define MCOUNT_FRAME_SIZE 8 | |
39 | #endif /* CONFIG_FRAME_POINTER */ | |
40 | ||
85f6f029 | 41 | /* Size of stack used to save mcount regs in save_mcount_regs */ |
0687c36e | 42 | #define MCOUNT_REG_SIZE (SS+8 + MCOUNT_FRAME_SIZE) |
85f6f029 | 43 | |
05df710e SRRH |
44 | /* |
45 | * gcc -pg option adds a call to 'mcount' in most functions. | |
46 | * When -mfentry is used, the call is to 'fentry' and not 'mcount' | |
47 | * and is done before the function's stack frame is set up. | |
48 | * They both require a set of regs to be saved before calling | |
49 | * any C code and restored before returning back to the function. | |
50 | * | |
51 | * On boot up, all these calls are converted into nops. When tracing | |
52 | * is enabled, the call can jump to either ftrace_caller or | |
53 | * ftrace_regs_caller. Callbacks (tracing functions) that require | |
54 | * ftrace_regs_caller (like kprobes) need to have pt_regs passed to | |
55 | * it. For this reason, the size of the pt_regs structure will be | |
56 | * allocated on the stack and the required mcount registers will | |
57 | * be saved in the locations that pt_regs has them in. | |
58 | */ | |
59 | ||
f1ab00af SRRH |
60 | /* |
61 | * @added: the amount of stack added before calling this | |
62 | * | |
63 | * After this is called, the following registers contain: | |
64 | * | |
65 | * %rdi - holds the address that called the trampoline | |
66 | * %rsi - holds the parent function (traced function's return address) | |
67 | * %rdx - holds the original %rbp | |
68 | */ | |
527aa75b | 69 | .macro save_mcount_regs added=0 |
0687c36e SRRH |
70 | |
71 | /* Always save the original rbp */ | |
72 | pushq %rbp | |
73 | ||
74 | #ifdef CONFIG_FRAME_POINTER | |
75 | /* | |
76 | * Stack traces will stop at the ftrace trampoline if the frame pointer | |
77 | * is not set up properly. If fentry is used, we need to save a frame | |
78 | * pointer for the parent as well as the function traced, because the | |
79 | * fentry is called before the stack frame is set up, where as mcount | |
80 | * is called afterward. | |
81 | */ | |
82 | #ifdef CC_USING_FENTRY | |
83 | /* Save the parent pointer (skip orig rbp and our return address) */ | |
84 | pushq \added+8*2(%rsp) | |
85 | pushq %rbp | |
86 | movq %rsp, %rbp | |
87 | /* Save the return address (now skip orig rbp, rbp and parent) */ | |
88 | pushq \added+8*3(%rsp) | |
89 | #else | |
90 | /* Can't assume that rip is before this (unless added was zero) */ | |
91 | pushq \added+8(%rsp) | |
92 | #endif | |
93 | pushq %rbp | |
94 | movq %rsp, %rbp | |
95 | #endif /* CONFIG_FRAME_POINTER */ | |
96 | ||
97 | /* | |
98 | * We add enough stack to save all regs. | |
99 | */ | |
100 | subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp | |
4bcdf152 SRRH |
101 | movq %rax, RAX(%rsp) |
102 | movq %rcx, RCX(%rsp) | |
103 | movq %rdx, RDX(%rsp) | |
104 | movq %rsi, RSI(%rsp) | |
105 | movq %rdi, RDI(%rsp) | |
106 | movq %r8, R8(%rsp) | |
107 | movq %r9, R9(%rsp) | |
0687c36e SRRH |
108 | /* |
109 | * Save the original RBP. Even though the mcount ABI does not | |
110 | * require this, it helps out callers. | |
111 | */ | |
112 | movq MCOUNT_REG_SIZE-8(%rsp), %rdx | |
113 | movq %rdx, RBP(%rsp) | |
114 | ||
f1ab00af SRRH |
115 | /* Copy the parent address into %rsi (second parameter) */ |
116 | #ifdef CC_USING_FENTRY | |
117 | movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi | |
118 | #else | |
119 | /* %rdx contains original %rbp */ | |
120 | movq 8(%rdx), %rsi | |
121 | #endif | |
122 | ||
4bcdf152 | 123 | /* Move RIP to its proper location */ |
85f6f029 | 124 | movq MCOUNT_REG_SIZE+\added(%rsp), %rdi |
094dfc54 | 125 | movq %rdi, RIP(%rsp) |
f1ab00af SRRH |
126 | |
127 | /* | |
128 | * Now %rdi (the first parameter) has the return address of | |
129 | * where ftrace_call returns. But the callbacks expect the | |
6a06bdbf | 130 | * address of the call itself. |
f1ab00af SRRH |
131 | */ |
132 | subq $MCOUNT_INSN_SIZE, %rdi | |
4bcdf152 SRRH |
133 | .endm |
134 | ||
527aa75b | 135 | .macro restore_mcount_regs |
4bcdf152 SRRH |
136 | movq R9(%rsp), %r9 |
137 | movq R8(%rsp), %r8 | |
138 | movq RDI(%rsp), %rdi | |
139 | movq RSI(%rsp), %rsi | |
140 | movq RDX(%rsp), %rdx | |
141 | movq RCX(%rsp), %rcx | |
142 | movq RAX(%rsp), %rax | |
0687c36e SRRH |
143 | |
144 | /* ftrace_regs_caller can modify %rbp */ | |
145 | movq RBP(%rsp), %rbp | |
146 | ||
85f6f029 | 147 | addq $MCOUNT_REG_SIZE, %rsp |
0687c36e | 148 | |
4bcdf152 SRRH |
149 | .endm |
150 | ||
76c2f13c SRRH |
151 | #ifdef CONFIG_DYNAMIC_FTRACE |
152 | ||
153 | ENTRY(function_hook) | |
154 | retq | |
155 | END(function_hook) | |
156 | ||
e18eead3 | 157 | ENTRY(ftrace_caller) |
f1ab00af SRRH |
158 | /* save_mcount_regs fills in first two parameters */ |
159 | save_mcount_regs | |
160 | ||
161 | GLOBAL(ftrace_caller_op_ptr) | |
162 | /* Load the ftrace_ops into the 3rd parameter */ | |
163 | movq function_trace_op(%rip), %rdx | |
164 | ||
e18eead3 SR |
165 | /* regs go into 4th parameter (but make it NULL) */ |
166 | movq $0, %rcx | |
167 | ||
168 | GLOBAL(ftrace_call) | |
169 | call ftrace_stub | |
170 | ||
05df710e | 171 | restore_mcount_regs |
f3bea491 SRRH |
172 | |
173 | /* | |
f1b92bb6 | 174 | * The copied trampoline must call ftrace_epilogue as it |
f3bea491 | 175 | * still may need to call the function graph tracer. |
f1b92bb6 BP |
176 | * |
177 | * The code up to this label is copied into trampolines so | |
178 | * think twice before adding any new code or changing the | |
179 | * layout here. | |
f3bea491 | 180 | */ |
f1b92bb6 | 181 | GLOBAL(ftrace_epilogue) |
e18eead3 SR |
182 | |
183 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
184 | GLOBAL(ftrace_graph_call) | |
185 | jmp ftrace_stub | |
186 | #endif | |
187 | ||
8329e818 SR |
188 | /* This is weak to keep gas from relaxing the jumps */ |
189 | WEAK(ftrace_stub) | |
e18eead3 SR |
190 | retq |
191 | END(ftrace_caller) | |
192 | ||
193 | ENTRY(ftrace_regs_caller) | |
527aa75b | 194 | /* Save the current flags before any operations that can change them */ |
e18eead3 SR |
195 | pushfq |
196 | ||
527aa75b | 197 | /* added 8 bytes to save flags */ |
f1ab00af SRRH |
198 | save_mcount_regs 8 |
199 | /* save_mcount_regs fills in first two parameters */ | |
200 | ||
201 | GLOBAL(ftrace_regs_caller_op_ptr) | |
202 | /* Load the ftrace_ops into the 3rd parameter */ | |
203 | movq function_trace_op(%rip), %rdx | |
e18eead3 SR |
204 | |
205 | /* Save the rest of pt_regs */ | |
206 | movq %r15, R15(%rsp) | |
207 | movq %r14, R14(%rsp) | |
208 | movq %r13, R13(%rsp) | |
209 | movq %r12, R12(%rsp) | |
210 | movq %r11, R11(%rsp) | |
211 | movq %r10, R10(%rsp) | |
e18eead3 SR |
212 | movq %rbx, RBX(%rsp) |
213 | /* Copy saved flags */ | |
85f6f029 | 214 | movq MCOUNT_REG_SIZE(%rsp), %rcx |
e18eead3 SR |
215 | movq %rcx, EFLAGS(%rsp) |
216 | /* Kernel segments */ | |
217 | movq $__KERNEL_DS, %rcx | |
218 | movq %rcx, SS(%rsp) | |
219 | movq $__KERNEL_CS, %rcx | |
220 | movq %rcx, CS(%rsp) | |
527aa75b | 221 | /* Stack - skipping return address and flags */ |
85f6f029 | 222 | leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx |
e18eead3 SR |
223 | movq %rcx, RSP(%rsp) |
224 | ||
225 | /* regs go into 4th parameter */ | |
226 | leaq (%rsp), %rcx | |
227 | ||
228 | GLOBAL(ftrace_regs_call) | |
229 | call ftrace_stub | |
230 | ||
231 | /* Copy flags back to SS, to restore them */ | |
232 | movq EFLAGS(%rsp), %rax | |
85f6f029 | 233 | movq %rax, MCOUNT_REG_SIZE(%rsp) |
e18eead3 SR |
234 | |
235 | /* Handlers can change the RIP */ | |
236 | movq RIP(%rsp), %rax | |
85f6f029 | 237 | movq %rax, MCOUNT_REG_SIZE+8(%rsp) |
e18eead3 SR |
238 | |
239 | /* restore the rest of pt_regs */ | |
240 | movq R15(%rsp), %r15 | |
241 | movq R14(%rsp), %r14 | |
242 | movq R13(%rsp), %r13 | |
243 | movq R12(%rsp), %r12 | |
244 | movq R10(%rsp), %r10 | |
e18eead3 SR |
245 | movq RBX(%rsp), %rbx |
246 | ||
527aa75b | 247 | restore_mcount_regs |
e18eead3 SR |
248 | |
249 | /* Restore flags */ | |
250 | popfq | |
251 | ||
f3bea491 | 252 | /* |
f1b92bb6 | 253 | * As this jmp to ftrace_epilogue can be a short jump |
f3bea491 SRRH |
254 | * it must not be copied into the trampoline. |
255 | * The trampoline will add the code to jump | |
256 | * to the return. | |
257 | */ | |
258 | GLOBAL(ftrace_regs_caller_end) | |
259 | ||
f1b92bb6 | 260 | jmp ftrace_epilogue |
fdc841b5 | 261 | |
e18eead3 SR |
262 | END(ftrace_regs_caller) |
263 | ||
264 | ||
265 | #else /* ! CONFIG_DYNAMIC_FTRACE */ | |
266 | ||
267 | ENTRY(function_hook) | |
e18eead3 SR |
268 | cmpq $ftrace_stub, ftrace_trace_function |
269 | jnz trace | |
270 | ||
62a207d7 | 271 | fgraph_trace: |
e18eead3 SR |
272 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
273 | cmpq $ftrace_stub, ftrace_graph_return | |
274 | jnz ftrace_graph_caller | |
275 | ||
276 | cmpq $ftrace_graph_entry_stub, ftrace_graph_entry | |
277 | jnz ftrace_graph_caller | |
278 | #endif | |
279 | ||
280 | GLOBAL(ftrace_stub) | |
281 | retq | |
282 | ||
283 | trace: | |
f1ab00af SRRH |
284 | /* save_mcount_regs fills in first two parameters */ |
285 | save_mcount_regs | |
e18eead3 | 286 | |
112677d6 NK |
287 | /* |
288 | * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not | |
289 | * set (see include/asm/ftrace.h and include/linux/ftrace.h). Only the | |
290 | * ip and parent ip are used and the list function is called when | |
291 | * function tracing is enabled. | |
292 | */ | |
e18eead3 SR |
293 | call *ftrace_trace_function |
294 | ||
05df710e | 295 | restore_mcount_regs |
e18eead3 | 296 | |
62a207d7 | 297 | jmp fgraph_trace |
e18eead3 SR |
298 | END(function_hook) |
299 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
300 | #endif /* CONFIG_FUNCTION_TRACER */ | |
301 | ||
302 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
303 | ENTRY(ftrace_graph_caller) | |
6a06bdbf | 304 | /* Saves rbp into %rdx and fills first parameter */ |
05df710e | 305 | save_mcount_regs |
e18eead3 SR |
306 | |
307 | #ifdef CC_USING_FENTRY | |
6a06bdbf | 308 | leaq MCOUNT_REG_SIZE+8(%rsp), %rsi |
e18eead3 SR |
309 | movq $0, %rdx /* No framepointers needed */ |
310 | #else | |
f1ab00af | 311 | /* Save address of the return address of traced function */ |
6a06bdbf | 312 | leaq 8(%rdx), %rsi |
f1ab00af | 313 | /* ftrace does sanity checks against frame pointers */ |
0687c36e | 314 | movq (%rdx), %rdx |
e18eead3 | 315 | #endif |
e18eead3 SR |
316 | call prepare_ftrace_return |
317 | ||
05df710e | 318 | restore_mcount_regs |
e18eead3 SR |
319 | |
320 | retq | |
321 | END(ftrace_graph_caller) | |
322 | ||
323 | GLOBAL(return_to_handler) | |
324 | subq $24, %rsp | |
325 | ||
326 | /* Save the return values */ | |
327 | movq %rax, (%rsp) | |
328 | movq %rdx, 8(%rsp) | |
329 | movq %rbp, %rdi | |
330 | ||
331 | call ftrace_return_to_handler | |
332 | ||
333 | movq %rax, %rdi | |
334 | movq 8(%rsp), %rdx | |
335 | movq (%rsp), %rax | |
336 | addq $24, %rsp | |
337 | jmp *%rdi | |
338 | #endif |