Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
3d82c59c SRV |
2 | /* |
3 | * Copyright (C) 2017 Steven Rostedt, VMware Inc. | |
4 | */ | |
5 | ||
6 | #include <linux/linkage.h> | |
7 | #include <asm/page_types.h> | |
8 | #include <asm/segment.h> | |
9 | #include <asm/export.h> | |
10 | #include <asm/ftrace.h> | |
9351803b | 11 | #include <asm/nospec-branch.h> |
ea1ed38d | 12 | #include <asm/frame.h> |
3c88c692 | 13 | #include <asm/asm-offsets.h> |
3d82c59c | 14 | |
644e0e8d SRV |
15 | # define function_hook __fentry__ |
16 | EXPORT_SYMBOL(__fentry__) | |
644e0e8d | 17 | |
562e14f7 | 18 | #ifdef CONFIG_FRAME_POINTER |
644e0e8d SRV |
19 | # define MCOUNT_FRAME 1 /* using frame = true */ |
20 | #else | |
21 | # define MCOUNT_FRAME 0 /* using frame = false */ | |
22 | #endif | |
23 | ||
24 | ENTRY(function_hook) | |
3d82c59c | 25 | ret |
644e0e8d | 26 | END(function_hook) |
3d82c59c SRV |
27 | |
28 | ENTRY(ftrace_caller) | |
e6928e58 | 29 | |
562e14f7 | 30 | #ifdef CONFIG_FRAME_POINTER |
644e0e8d SRV |
31 | /* |
32 | * Frame pointers are of ip followed by bp. | |
33 | * Since fentry is an immediate jump, we are left with | |
34 | * parent-ip, function-ip. We need to add a frame with | |
35 | * parent-ip followed by ebp. | |
36 | */ | |
37 | pushl 4(%esp) /* parent ip */ | |
e6928e58 SRV |
38 | pushl %ebp |
39 | movl %esp, %ebp | |
644e0e8d | 40 | pushl 2*4(%esp) /* function ip */ |
562e14f7 | 41 | |
644e0e8d SRV |
42 | /* For mcount, the function ip is directly above */ |
43 | pushl %ebp | |
44 | movl %esp, %ebp | |
45 | #endif | |
3d82c59c SRV |
46 | pushl %eax |
47 | pushl %ecx | |
48 | pushl %edx | |
49 | pushl $0 /* Pass NULL as regs pointer */ | |
644e0e8d | 50 | |
562e14f7 | 51 | #ifdef CONFIG_FRAME_POINTER |
644e0e8d | 52 | /* Load parent ebp into edx */ |
e6928e58 | 53 | movl 4*4(%esp), %edx |
644e0e8d SRV |
54 | #else |
55 | /* There's no frame pointer, load the appropriate stack addr instead */ | |
56 | lea 4*4(%esp), %edx | |
57 | #endif | |
58 | ||
59 | movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ | |
e6928e58 | 60 | /* Get the parent ip */ |
644e0e8d SRV |
61 | movl 4(%edx), %edx /* edx has ebp */ |
62 | ||
3d82c59c SRV |
63 | movl function_trace_op, %ecx |
64 | subl $MCOUNT_INSN_SIZE, %eax | |
65 | ||
66 | .globl ftrace_call | |
67 | ftrace_call: | |
68 | call ftrace_stub | |
69 | ||
70 | addl $4, %esp /* skip NULL pointer */ | |
71 | popl %edx | |
72 | popl %ecx | |
73 | popl %eax | |
562e14f7 | 74 | #ifdef CONFIG_FRAME_POINTER |
e6928e58 | 75 | popl %ebp |
644e0e8d SRV |
76 | addl $4,%esp /* skip function ip */ |
77 | popl %ebp /* this is the orig bp */ | |
78 | addl $4, %esp /* skip parent ip */ | |
644e0e8d | 79 | #endif |
3d82c59c SRV |
80 | .Lftrace_ret: |
81 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
82 | .globl ftrace_graph_call | |
83 | ftrace_graph_call: | |
84 | jmp ftrace_stub | |
85 | #endif | |
86 | ||
87 | /* This is weak to keep gas from relaxing the jumps */ | |
88 | WEAK(ftrace_stub) | |
89 | ret | |
90 | END(ftrace_caller) | |
91 | ||
92 | ENTRY(ftrace_regs_caller) | |
3d82c59c | 93 | /* |
3c88c692 PZ |
94 | * We're here from an mcount/fentry CALL, and the stack frame looks like: |
95 | * | |
96 | * <previous context> | |
97 | * RET-IP | |
98 | * | |
99 | * The purpose of this function is to call out in an emulated INT3 | |
100 | * environment with a stack frame like: | |
101 | * | |
102 | * <previous context> | |
103 | * gap / RET-IP | |
104 | * gap | |
105 | * gap | |
106 | * gap | |
107 | * pt_regs | |
108 | * | |
109 | * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds | |
3d82c59c | 110 | */ |
3c88c692 PZ |
111 | subl $3*4, %esp # RET-IP + 3 gaps |
112 | pushl %ss # ss | |
113 | pushl %esp # points at ss | |
114 | addl $5*4, (%esp) # make it point at <previous context> | |
115 | pushfl # flags | |
116 | pushl $__KERNEL_CS # cs | |
117 | pushl 7*4(%esp) # ip <- RET-IP | |
118 | pushl $0 # orig_eax | |
119 | ||
3d82c59c SRV |
120 | pushl %gs |
121 | pushl %fs | |
122 | pushl %es | |
123 | pushl %ds | |
ff04b440 | 124 | |
3c88c692 | 125 | pushl %eax |
3d82c59c SRV |
126 | pushl %ebp |
127 | pushl %edi | |
128 | pushl %esi | |
129 | pushl %edx | |
130 | pushl %ecx | |
131 | pushl %ebx | |
132 | ||
ea1ed38d PZ |
133 | ENCODE_FRAME_POINTER |
134 | ||
3c88c692 PZ |
135 | movl PT_EIP(%esp), %eax # 1st argument: IP |
136 | subl $MCOUNT_INSN_SIZE, %eax | |
137 | movl 21*4(%esp), %edx # 2nd argument: parent ip | |
138 | movl function_trace_op, %ecx # 3rd argument: ftrace_pos | |
139 | pushl %esp # 4th argument: pt_regs | |
3d82c59c | 140 | |
26ba4e57 | 141 | SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) |
3d82c59c SRV |
142 | call ftrace_stub |
143 | ||
3c88c692 | 144 | addl $4, %esp # skip 4th argument |
ff04b440 | 145 | |
3c88c692 PZ |
146 | /* place IP below the new SP */ |
147 | movl PT_OLDESP(%esp), %eax | |
148 | movl PT_EIP(%esp), %ecx | |
149 | movl %ecx, -4(%eax) | |
ff04b440 | 150 | |
3c88c692 PZ |
151 | /* place EAX below that */ |
152 | movl PT_EAX(%esp), %ecx | |
153 | movl %ecx, -8(%eax) | |
3d82c59c SRV |
154 | |
155 | popl %ebx | |
156 | popl %ecx | |
157 | popl %edx | |
158 | popl %esi | |
159 | popl %edi | |
160 | popl %ebp | |
3d82c59c | 161 | |
3c88c692 PZ |
162 | lea -8(%eax), %esp |
163 | popl %eax | |
ff04b440 SRV |
164 | |
165 | jmp .Lftrace_ret | |
3d82c59c SRV |
166 | |
167 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
168 | ENTRY(ftrace_graph_caller) | |
169 | pushl %eax | |
170 | pushl %ecx | |
171 | pushl %edx | |
644e0e8d SRV |
172 | movl 3*4(%esp), %eax |
173 | /* Even with frame pointers, fentry doesn't have one here */ | |
644e0e8d SRV |
174 | lea 4*4(%esp), %edx |
175 | movl $0, %ecx | |
3d82c59c SRV |
176 | subl $MCOUNT_INSN_SIZE, %eax |
177 | call prepare_ftrace_return | |
178 | popl %edx | |
179 | popl %ecx | |
180 | popl %eax | |
181 | ret | |
182 | END(ftrace_graph_caller) | |
183 | ||
184 | .globl return_to_handler | |
185 | return_to_handler: | |
186 | pushl %eax | |
187 | pushl %edx | |
644e0e8d | 188 | movl $0, %eax |
3d82c59c SRV |
189 | call ftrace_return_to_handler |
190 | movl %eax, %ecx | |
191 | popl %edx | |
192 | popl %eax | |
9351803b | 193 | JMP_NOSPEC %ecx |
3d82c59c | 194 | #endif |