[linux-2.6-block.git] / arch / arm / kernel / entry-ftrace.S

/* SPDX-License-Identifier: GPL-2.0-only */

#include <asm/assembler.h>
#include <asm/ftrace.h>
#include <asm/unwind.h>

#include "entry-header.S"

/*
 * When compiling with -pg, gcc inserts a call to the mcount routine at the
 * start of every function.  In mcount, apart from the function's address (in
 * lr), we need to get hold of the function's caller's address.
 *
 * Newer GCCs (4.4+) solve this problem by using a version of mcount with call
 * sites like:
 *
 *	push	{lr}
 *	bl	__gnu_mcount_nc
 *
 * With these compilers, frame pointers are not necessary.
 *
 * mcount can be thought of as a function called in the middle of a subroutine
 * call.  As such, it needs to be transparent for both the caller and the
 * callee: the original lr needs to be restored when leaving mcount, and no
 * registers should be clobbered.  (In the __gnu_mcount_nc implementation, we
 * clobber the ip register.  This is OK because the ARM calling convention
 * allows it to be clobbered in subroutines and doesn't use it to hold
 * parameters.)
 *
 * When using dynamic ftrace, we patch out the mcount call by a "pop {lr}"
 * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c).
 */

.macro mcount_adjust_addr rd, rn
	bic	\rd, \rn, #1		@ clear the Thumb bit if present
	sub	\rd, \rd, #MCOUNT_INSN_SIZE
.endm

.macro __mcount suffix
	mcount_enter
	ldr	r0, =ftrace_trace_function
	ldr	r2, [r0]
	adr	r0, .Lftrace_stub
	cmp	r0, r2
	bne	1f

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	ldr     r1, =ftrace_graph_return
	ldr     r2, [r1]
	cmp     r0, r2
	bne     ftrace_graph_caller\suffix

	ldr     r1, =ftrace_graph_entry
	ldr     r2, [r1]
	ldr     r0, =ftrace_graph_entry_stub
	cmp     r0, r2
	bne     ftrace_graph_caller\suffix
#endif

	mcount_exit

1: 	mcount_get_lr	r1			@ lr of instrumented func
	mcount_adjust_addr	r0, lr		@ instrumented function
	badr	lr, 2f
	mov	pc, r2
2:	mcount_exit
.endm

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS

.macro __ftrace_regs_caller

	sub	sp, sp, #8	@ space for PC and CPSR OLD_R0,
				@ OLD_R0 will overwrite previous LR

	add 	ip, sp, #12	@ move in IP the value of SP as it was
				@ before the push {lr} of the mcount mechanism

	str     lr, [sp, #0]    @ store LR instead of PC

	ldr     lr, [sp, #8]    @ get previous LR

	str	r0, [sp, #8]	@ write r0 as OLD_R0 over previous LR

	stmdb   sp!, {ip, lr}
	stmdb   sp!, {r0-r11, lr}

	@ stack content at this point:
	@ 0  4          48   52       56            60   64    68       72
	@ R0 | R1 | ... | LR | SP + 4 | previous LR | LR | PSR | OLD_R0 |

	mov r3, sp				@ struct pt_regs*

	ldr r2, =function_trace_op
	ldr r2, [r2]				@ pointer to the current
						@ function tracing op

	ldr	r1, [sp, #S_LR]			@ lr of instrumented func

	ldr	lr, [sp, #S_PC]			@ get LR

	mcount_adjust_addr	r0, lr		@ instrumented function

	.globl ftrace_regs_call
ftrace_regs_call:
	bl	ftrace_stub

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	.globl ftrace_graph_regs_call
ftrace_graph_regs_call:
	mov	r0, r0
#endif

	@ pop saved regs
	ldmia   sp!, {r0-r12}			@ restore r0 through r12
	ldr	ip, [sp, #8]			@ restore PC
	ldr	lr, [sp, #4]			@ restore LR
	ldr	sp, [sp, #0]			@ restore SP
	mov	pc, ip				@ return
.endm

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.macro __ftrace_graph_regs_caller

	sub     r0, fp, #4              @ lr of instrumented routine (parent)

	@ called from __ftrace_regs_caller
	ldr     r1, [sp, #S_PC]		@ instrumented routine (func)
	mcount_adjust_addr	r1, r1

	mov	r2, fp			@ frame pointer
	bl	prepare_ftrace_return

	@ pop registers saved in ftrace_regs_caller
	ldmia   sp!, {r0-r12}			@ restore r0 through r12
	ldr	ip, [sp, #8]			@ restore PC
	ldr	lr, [sp, #4]			@ restore LR
	ldr	sp, [sp, #0]			@ restore SP
	mov	pc, ip				@ return

.endm
#endif
#endif

.macro __ftrace_caller suffix
	mcount_enter

	mcount_get_lr	r1			@ lr of instrumented func
	mcount_adjust_addr	r0, lr		@ instrumented function

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	ldr r2, =function_trace_op
	ldr r2, [r2]				@ pointer to the current
						@ function tracing op
	mov r3, #0				@ regs is NULL
#endif

	.globl ftrace_call\suffix
ftrace_call\suffix:
	bl	ftrace_stub

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	.globl ftrace_graph_call\suffix
ftrace_graph_call\suffix:
	mov	r0, r0
#endif

	mcount_exit
.endm

.macro __ftrace_graph_caller
	sub	r0, fp, #4		@ &lr of instrumented routine (&parent)
#ifdef CONFIG_DYNAMIC_FTRACE
	@ called from __ftrace_caller, saved in mcount_enter
	ldr	r1, [sp, #16]		@ instrumented routine (func)
	mcount_adjust_addr	r1, r1
#else
	@ called from __mcount, untouched in lr
	mcount_adjust_addr	r1, lr	@ instrumented routine (func)
#endif
	mov	r2, fp			@ frame pointer
	bl	prepare_ftrace_return
	mcount_exit
.endm

/*
 * __gnu_mcount_nc
 */

.macro mcount_enter
/*
 * This pad compensates for the push {lr} at the call site.  Note that we are
 * unable to unwind through a function which does not otherwise save its lr.
 */
 UNWIND(.pad	#4)
	stmdb	sp!, {r0-r3, lr}
 UNWIND(.save	{r0-r3, lr})
.endm

.macro mcount_get_lr reg
	ldr	\reg, [sp, #20]
.endm

.macro mcount_exit
	ldmia	sp!, {r0-r3, ip, lr}
	ret	ip
.endm

ENTRY(__gnu_mcount_nc)
UNWIND(.fnstart)
#ifdef CONFIG_DYNAMIC_FTRACE
	mov	ip, lr
	ldmia	sp!, {lr}
	ret	ip
#else
	__mcount
#endif
UNWIND(.fnend)
ENDPROC(__gnu_mcount_nc)

#ifdef CONFIG_DYNAMIC_FTRACE
ENTRY(ftrace_caller)
UNWIND(.fnstart)
	__ftrace_caller
UNWIND(.fnend)
ENDPROC(ftrace_caller)

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
ENTRY(ftrace_regs_caller)
UNWIND(.fnstart)
	__ftrace_regs_caller
UNWIND(.fnend)
ENDPROC(ftrace_regs_caller)
#endif

#endif

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
ENTRY(ftrace_graph_caller)
UNWIND(.fnstart)
	__ftrace_graph_caller
UNWIND(.fnend)
ENDPROC(ftrace_graph_caller)

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
ENTRY(ftrace_graph_regs_caller)
UNWIND(.fnstart)
	__ftrace_graph_regs_caller
UNWIND(.fnend)
ENDPROC(ftrace_graph_regs_caller)
#endif
#endif

.purgem mcount_enter
.purgem mcount_get_lr
.purgem mcount_exit

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	.globl return_to_handler
return_to_handler:
	stmdb	sp!, {r0-r3}
	mov	r0, fp			@ frame pointer
	bl	ftrace_return_to_handler
	mov	lr, r0			@ r0 has real ret addr
	ldmia	sp!, {r0-r3}
	ret	lr
#endif

ENTRY(ftrace_stub)
.Lftrace_stub:
	ret	lr
ENDPROC(ftrace_stub)
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
82112379 RK	2
	3	#include <asm/assembler.h>
	4	#include <asm/ftrace.h>
	5	#include <asm/unwind.h>
	6
	7	#include "entry-header.S"
	8
	9	/*
	10	* When compiling with -pg, gcc inserts a call to the mcount routine at the
	11	* start of every function. In mcount, apart from the function's address (in
	12	* lr), we need to get hold of the function's caller's address.
	13	*
d3c61619 SA	14	* Newer GCCs (4.4+) solve this problem by using a version of mcount with call
d3c61619 SA	15	* sites like:
82112379 RK	16	*
	17	* push {lr}
	18	* bl __gnu_mcount_nc
	19	*
	20	* With these compilers, frame pointers are not necessary.
	21	*
	22	* mcount can be thought of as a function called in the middle of a subroutine
	23	* call. As such, it needs to be transparent for both the caller and the
	24	* callee: the original lr needs to be restored when leaving mcount, and no
	25	* registers should be clobbered. (In the __gnu_mcount_nc implementation, we
	26	* clobber the ip register. This is OK because the ARM calling convention
	27	* allows it to be clobbered in subroutines and doesn't use it to hold
	28	* parameters.)
	29	*
d3c61619 SA	30	* When using dynamic ftrace, we patch out the mcount call by a "pop {lr}"
d3c61619 SA	31	* instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c).
82112379 RK	32	*/
82112379 RK	33
82112379 RK	34	.macro mcount_adjust_addr rd, rn
	35	bic \rd, \rn, #1 @ clear the Thumb bit if present
	36	sub \rd, \rd, #MCOUNT_INSN_SIZE
	37	.endm
	38
	39	.macro __mcount suffix
	40	mcount_enter
	41	ldr r0, =ftrace_trace_function
	42	ldr r2, [r0]
	43	adr r0, .Lftrace_stub
	44	cmp r0, r2
	45	bne 1f
	46
	47	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	48	ldr r1, =ftrace_graph_return
	49	ldr r2, [r1]
	50	cmp r0, r2
	51	bne ftrace_graph_caller\suffix
	52
	53	ldr r1, =ftrace_graph_entry
	54	ldr r2, [r1]
	55	ldr r0, =ftrace_graph_entry_stub
	56	cmp r0, r2
	57	bne ftrace_graph_caller\suffix
	58	#endif
	59
	60	mcount_exit
	61
	62	1: mcount_get_lr r1 @ lr of instrumented func
	63	mcount_adjust_addr r0, lr @ instrumented function
14327c66	64	badr lr, 2f
82112379 RK	65	mov pc, r2
	66	2: mcount_exit
	67	.endm
	68
620176f3 AV	69	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	70
	71	.macro __ftrace_regs_caller
	72
	73	sub sp, sp, #8 @ space for PC and CPSR OLD_R0,
	74	@ OLD_R0 will overwrite previous LR
	75
	76	add ip, sp, #12 @ move in IP the value of SP as it was
	77	@ before the push {lr} of the mcount mechanism
	78
	79	str lr, [sp, #0] @ store LR instead of PC
	80
	81	ldr lr, [sp, #8] @ get previous LR
	82
	83	str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR
	84
	85	stmdb sp!, {ip, lr}
	86	stmdb sp!, {r0-r11, lr}
	87
	88	@ stack content at this point:
	89	@ 0 4 48 52 56 60 64 68 72
	90	@ R0 \| R1 \| ... \| LR \| SP + 4 \| previous LR \| LR \| PSR \| OLD_R0 \|
	91
	92	mov r3, sp @ struct pt_regs*
	93
	94	ldr r2, =function_trace_op
	95	ldr r2, [r2] @ pointer to the current
	96	@ function tracing op
	97
	98	ldr r1, [sp, #S_LR] @ lr of instrumented func
	99
	100	ldr lr, [sp, #S_PC] @ get LR
	101
	102	mcount_adjust_addr r0, lr @ instrumented function
	103
	104	.globl ftrace_regs_call
	105	ftrace_regs_call:
	106	bl ftrace_stub
	107
	108	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	109	.globl ftrace_graph_regs_call
	110	ftrace_graph_regs_call:
	111	mov r0, r0
	112	#endif
	113
	114	@ pop saved regs
	115	ldmia sp!, {r0-r12} @ restore r0 through r12
	116	ldr ip, [sp, #8] @ restore PC
	117	ldr lr, [sp, #4] @ restore LR
	118	ldr sp, [sp, #0] @ restore SP
	119	mov pc, ip @ return
	120	.endm
	121
	122	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	123	.macro __ftrace_graph_regs_caller
	124
	125	sub r0, fp, #4 @ lr of instrumented routine (parent)
	126
	127	@ called from __ftrace_regs_caller
	128	ldr r1, [sp, #S_PC] @ instrumented routine (func)
	129	mcount_adjust_addr r1, r1
	130
	131	mov r2, fp @ frame pointer
	132	bl prepare_ftrace_return
133
134	@ pop registers saved in ftrace_regs_caller
135	ldmia sp!, {r0-r12} @ restore r0 through r12
136	ldr ip, [sp, #8] @ restore PC
137	ldr lr, [sp, #4] @ restore LR
138	ldr sp, [sp, #0] @ restore SP
139	mov pc, ip @ return
140
141	.endm
142	#endif
143	#endif
144
82112379 RK	145	.macro __ftrace_caller suffix
	146	mcount_enter
	147
	148	mcount_get_lr r1 @ lr of instrumented func
	149	mcount_adjust_addr r0, lr @ instrumented function
	150
620176f3 AV	151	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	152	ldr r2, =function_trace_op
	153	ldr r2, [r2] @ pointer to the current
	154	@ function tracing op
	155	mov r3, #0 @ regs is NULL
	156	#endif
	157
82112379 RK	158	.globl ftrace_call\suffix
	159	ftrace_call\suffix:
	160	bl ftrace_stub
	161
	162	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	163	.globl ftrace_graph_call\suffix
	164	ftrace_graph_call\suffix:
	165	mov r0, r0
	166	#endif
	167
	168	mcount_exit
	169	.endm
	170
	171	.macro __ftrace_graph_caller
	172	sub r0, fp, #4 @ &lr of instrumented routine (&parent)
	173	#ifdef CONFIG_DYNAMIC_FTRACE
	174	@ called from __ftrace_caller, saved in mcount_enter
	175	ldr r1, [sp, #16] @ instrumented routine (func)
	176	mcount_adjust_addr r1, r1
	177	#else
	178	@ called from __mcount, untouched in lr
	179	mcount_adjust_addr r1, lr @ instrumented routine (func)
	180	#endif
	181	mov r2, fp @ frame pointer
	182	bl prepare_ftrace_return
	183	mcount_exit
	184	.endm
	185
82112379 RK	186	/*
	187	* __gnu_mcount_nc
	188	*/
	189
	190	.macro mcount_enter
	191	/*
	192	* This pad compensates for the push {lr} at the call site. Note that we are
	193	* unable to unwind through a function which does not otherwise save its lr.
	194	*/
	195	UNWIND(.pad #4)
	196	stmdb sp!, {r0-r3, lr}
	197	UNWIND(.save {r0-r3, lr})
	198	.endm
	199
	200	.macro mcount_get_lr reg
	201	ldr \reg, [sp, #20]
	202	.endm
	203
	204	.macro mcount_exit
	205	ldmia sp!, {r0-r3, ip, lr}
	206	ret ip
	207	.endm
	208
	209	ENTRY(__gnu_mcount_nc)
	210	UNWIND(.fnstart)
	211	#ifdef CONFIG_DYNAMIC_FTRACE
	212	mov ip, lr
	213	ldmia sp!, {lr}
	214	ret ip
	215	#else
	216	__mcount
	217	#endif
	218	UNWIND(.fnend)
	219	ENDPROC(__gnu_mcount_nc)
	220
	221	#ifdef CONFIG_DYNAMIC_FTRACE
	222	ENTRY(ftrace_caller)
	223	UNWIND(.fnstart)
	224	__ftrace_caller
	225	UNWIND(.fnend)
	226	ENDPROC(ftrace_caller)
620176f3 AV	227
	228	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	229	ENTRY(ftrace_regs_caller)
	230	UNWIND(.fnstart)
	231	__ftrace_regs_caller
	232	UNWIND(.fnend)
	233	ENDPROC(ftrace_regs_caller)
	234	#endif
	235
82112379 RK	236	#endif
	237
	238	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	239	ENTRY(ftrace_graph_caller)
	240	UNWIND(.fnstart)
	241	__ftrace_graph_caller
	242	UNWIND(.fnend)
	243	ENDPROC(ftrace_graph_caller)
620176f3 AV	244
	245	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	246	ENTRY(ftrace_graph_regs_caller)
	247	UNWIND(.fnstart)
	248	__ftrace_graph_regs_caller
	249	UNWIND(.fnend)
	250	ENDPROC(ftrace_graph_regs_caller)
	251	#endif
82112379 RK	252	#endif
	253
	254	.purgem mcount_enter
	255	.purgem mcount_get_lr
	256	.purgem mcount_exit
	257
	258	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	259	.globl return_to_handler
	260	return_to_handler:
	261	stmdb sp!, {r0-r3}
	262	mov r0, fp @ frame pointer
	263	bl ftrace_return_to_handler
	264	mov lr, r0 @ r0 has real ret addr
	265	ldmia sp!, {r0-r3}
	266	ret lr
	267	#endif
	268
	269	ENTRY(ftrace_stub)
	270	.Lftrace_stub:
	271	ret lr
	272	ENDPROC(ftrace_stub)