[linux-block.git] / arch / x86 / kernel / relocate_kernel_64.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * relocate_kernel.S - put the kernel image in place to boot
 * Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
 */

#include <linux/linkage.h>
#include <asm/page_types.h>
#include <asm/kexec.h>
#include <asm/processor-flags.h>
#include <asm/pgtable_types.h>
#include <asm/nospec-branch.h>
#include <asm/unwind_hints.h>

/*
 * Must be relocatable PIC code callable as a C function, in particular
 * there must be a plain RET and not jump to return thunk.
 */

#define PTR(x) (x << 3)
#define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)

/*
 * control_page + KEXEC_CONTROL_CODE_MAX_SIZE
 * ~ control_page + PAGE_SIZE are used as data storage and stack for
 * jumping back
 */
#define DATA(offset)		(KEXEC_CONTROL_CODE_MAX_SIZE+(offset))

/* Minimal CPU state */
#define RSP			DATA(0x0)
#define CR0			DATA(0x8)
#define CR3			DATA(0x10)
#define CR4			DATA(0x18)

/* other data */
#define CP_PA_TABLE_PAGE	DATA(0x20)
#define CP_PA_SWAP_PAGE		DATA(0x28)
#define CP_PA_BACKUP_PAGES_MAP	DATA(0x30)

	.text
	.align PAGE_SIZE
	.code64
SYM_CODE_START_NOALIGN(relocate_range)
SYM_CODE_START_NOALIGN(relocate_kernel)
	UNWIND_HINT_END_OF_STACK
	ANNOTATE_NOENDBR
	/*
	 * %rdi indirection_page
	 * %rsi page_list
	 * %rdx start address
	 * %rcx preserve_context
	 * %r8  host_mem_enc_active
	 */

	/* Save the CPU context, used for jumping back */
	pushq %rbx
	pushq %rbp
	pushq %r12
	pushq %r13
	pushq %r14
	pushq %r15
	pushf

	movq	PTR(VA_CONTROL_PAGE)(%rsi), %r11
	movq	%rsp, RSP(%r11)
	movq	%cr0, %rax
	movq	%rax, CR0(%r11)
	movq	%cr3, %rax
	movq	%rax, CR3(%r11)
	movq	%cr4, %rax
	movq	%rax, CR4(%r11)

	/* Save CR4. Required to enable the right paging mode later. */
	movq	%rax, %r13

	/* zero out flags, and disable interrupts */
	pushq $0
	popfq

	/* Save SME active flag */
	movq	%r8, %r12

	/*
	 * get physical address of control page now
	 * this is impossible after page table switch
	 */
	movq	PTR(PA_CONTROL_PAGE)(%rsi), %r8

	/* get physical address of page table now too */
	movq	PTR(PA_TABLE_PAGE)(%rsi), %r9

	/* get physical address of swap page now */
	movq	PTR(PA_SWAP_PAGE)(%rsi), %r10

	/* save some information for jumping back */
	movq	%r9, CP_PA_TABLE_PAGE(%r11)
	movq	%r10, CP_PA_SWAP_PAGE(%r11)
	movq	%rdi, CP_PA_BACKUP_PAGES_MAP(%r11)

	/* Switch to the identity mapped page tables */
	movq	%r9, %cr3

	/* setup a new stack at the end of the physical control page */
	lea	PAGE_SIZE(%r8), %rsp

	/* jump to identity mapped page */
	addq	$(identity_mapped - relocate_kernel), %r8
	pushq	%r8
	ANNOTATE_UNRET_SAFE
	ret
	int3
SYM_CODE_END(relocate_kernel)

SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
	UNWIND_HINT_END_OF_STACK
	/* set return address to 0 if not preserving context */
	pushq	$0
	/* store the start address on the stack */
	pushq   %rdx

	/*
	 * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP
	 * below.
	 */
	movq	%cr4, %rax
	andq	$~(X86_CR4_CET), %rax
	movq	%rax, %cr4

	/*
	 * Set cr0 to a known state:
	 *  - Paging enabled
	 *  - Alignment check disabled
	 *  - Write protect disabled
	 *  - No task switch
	 *  - Don't do FP software emulation.
	 *  - Protected mode enabled
	 */
	movq	%cr0, %rax
	andq	$~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax
	orl	$(X86_CR0_PG | X86_CR0_PE), %eax
	movq	%rax, %cr0

	/*
	 * Set cr4 to a known state:
	 *  - physical address extension enabled
	 *  - 5-level paging, if it was enabled before
	 */
	movl	$X86_CR4_PAE, %eax
	testq	$X86_CR4_LA57, %r13
	jz	1f
	orl	$X86_CR4_LA57, %eax
1:
	movq	%rax, %cr4

	jmp 1f
1:

	/* Flush the TLB (needed?) */
	movq	%r9, %cr3

	/*
	 * If SME is active, there could be old encrypted cache line
	 * entries that will conflict with the now unencrypted memory
	 * used by kexec. Flush the caches before copying the kernel.
	 */
	testq	%r12, %r12
	jz 1f
	wbinvd
1:

	movq	%rcx, %r11
	call	swap_pages

	/*
	 * To be certain of avoiding problems with self-modifying code
	 * I need to execute a serializing instruction here.
	 * So I flush the TLB by reloading %cr3 here, it's handy,
	 * and not processor dependent.
	 */
	movq	%cr3, %rax
	movq	%rax, %cr3

	/*
	 * set all of the registers to known values
	 * leave %rsp alone
	 */

	testq	%r11, %r11
	jnz 1f
	xorl	%eax, %eax
	xorl	%ebx, %ebx
	xorl    %ecx, %ecx
	xorl    %edx, %edx
	xorl    %esi, %esi
	xorl    %edi, %edi
	xorl    %ebp, %ebp
	xorl	%r8d, %r8d
	xorl	%r9d, %r9d
	xorl	%r10d, %r10d
	xorl	%r11d, %r11d
	xorl	%r12d, %r12d
	xorl	%r13d, %r13d
	xorl	%r14d, %r14d
	xorl	%r15d, %r15d

	ANNOTATE_UNRET_SAFE
	ret
	int3

1:
	popq	%rdx
	leaq	PAGE_SIZE(%r10), %rsp
	ANNOTATE_RETPOLINE_SAFE
	call	*%rdx

	/* get the re-entry point of the peer system */
	movq	0(%rsp), %rbp
	leaq	relocate_kernel(%rip), %r8
	movq	CP_PA_SWAP_PAGE(%r8), %r10
	movq	CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
	movq	CP_PA_TABLE_PAGE(%r8), %rax
	movq	%rax, %cr3
	lea	PAGE_SIZE(%r8), %rsp
	call	swap_pages
	movq	$virtual_mapped, %rax
	pushq	%rax
	ANNOTATE_UNRET_SAFE
	ret
	int3
SYM_CODE_END(identity_mapped)

SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
	UNWIND_HINT_END_OF_STACK
	ANNOTATE_NOENDBR // RET target, above
	movq	RSP(%r8), %rsp
	movq	CR4(%r8), %rax
	movq	%rax, %cr4
	movq	CR3(%r8), %rax
	movq	CR0(%r8), %r8
	movq	%rax, %cr3
	movq	%r8, %cr0
	movq	%rbp, %rax

	popf
	popq	%r15
	popq	%r14
	popq	%r13
	popq	%r12
	popq	%rbp
	popq	%rbx
	ANNOTATE_UNRET_SAFE
	ret
	int3
SYM_CODE_END(virtual_mapped)

	/* Do the copies */
SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
	UNWIND_HINT_END_OF_STACK
	movq	%rdi, %rcx	/* Put the page_list in %rcx */
	xorl	%edi, %edi
	xorl	%esi, %esi
	jmp	1f

0:	/* top, read another word for the indirection page */

	movq	(%rbx), %rcx
	addq	$8,	%rbx
1:
	testb	$0x1,	%cl   /* is it a destination page? */
	jz	2f
	movq	%rcx,	%rdi
	andq	$0xfffffffffffff000, %rdi
	jmp	0b
2:
	testb	$0x2,	%cl   /* is it an indirection page? */
	jz	2f
	movq	%rcx,   %rbx
	andq	$0xfffffffffffff000, %rbx
	jmp	0b
2:
	testb	$0x4,	%cl   /* is it the done indicator? */
	jz	2f
	jmp	3f
2:
	testb	$0x8,	%cl   /* is it the source indicator? */
	jz	0b	      /* Ignore it otherwise */
	movq	%rcx,   %rsi  /* For ever source page do a copy */
	andq	$0xfffffffffffff000, %rsi

	movq	%rdi, %rdx
	movq	%rsi, %rax

	movq	%r10, %rdi
	movl	$512, %ecx
	rep ; movsq

	movq	%rax, %rdi
	movq	%rdx, %rsi
	movl	$512, %ecx
	rep ; movsq

	movq	%rdx, %rdi
	movq	%r10, %rsi
	movl	$512, %ecx
	rep ; movsq

	lea	PAGE_SIZE(%rax), %rsi
	jmp	0b
3:
	ANNOTATE_UNRET_SAFE
	ret
	int3
SYM_CODE_END(swap_pages)

	.skip KEXEC_CONTROL_CODE_MAX_SIZE - (. - relocate_kernel), 0xcc
SYM_CODE_END(relocate_range);
Commit	Line	Data
40b0b3f8	1	/* SPDX-License-Identifier: GPL-2.0-only */
5234f5eb EB	2	/*
	3	* relocate_kernel.S - put the kernel image in place to boot
	4	* Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
5234f5eb EB	5	*/
	6
	7	#include <linux/linkage.h>
0341c14d	8	#include <asm/page_types.h>
4bfaaef0	9	#include <asm/kexec.h>
fd3af531	10	#include <asm/processor-flags.h>
0341c14d	11	#include <asm/pgtable_types.h>
36cc5520 PZ	12	#include <asm/nospec-branch.h>
36cc5520 PZ	13	#include <asm/unwind_hints.h>
5234f5eb	14
4bfaaef0	15	/*
697977d8 KRW	16	* Must be relocatable PIC code callable as a C function, in particular
697977d8 KRW	17	* there must be a plain RET and not jump to return thunk.
4bfaaef0 MD	18	*/
	19
	20	#define PTR(x) (x << 3)
366932de	21	#define PAGE_ATTR (_PAGE_PRESENT \| _PAGE_RW \| _PAGE_ACCESSED \| _PAGE_DIRTY)
4bfaaef0	22
fee7b0d8 HY	23	/*
	24	* control_page + KEXEC_CONTROL_CODE_MAX_SIZE
	25	* ~ control_page + PAGE_SIZE are used as data storage and stack for
	26	* jumping back
	27	*/
	28	#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset))
	29
	30	/* Minimal CPU state */
	31	#define RSP DATA(0x0)
	32	#define CR0 DATA(0x8)
	33	#define CR3 DATA(0x10)
	34	#define CR4 DATA(0x18)
	35
	36	/* other data */
	37	#define CP_PA_TABLE_PAGE DATA(0x20)
	38	#define CP_PA_SWAP_PAGE DATA(0x28)
	39	#define CP_PA_BACKUP_PAGES_MAP DATA(0x30)
	40
4bfaaef0	41	.text
288621e3	42	.align PAGE_SIZE
5234f5eb	43	.code64
e81dc127	44	SYM_CODE_START_NOALIGN(relocate_range)
6ec2a968	45	SYM_CODE_START_NOALIGN(relocate_kernel)
fb799447	46	UNWIND_HINT_END_OF_STACK
3e3f0695	47	ANNOTATE_NOENDBR
fef3a7a1 HY	48	/*
fef3a7a1 HY	49	* %rdi indirection_page
4bfaaef0 MD	50	* %rsi page_list
4bfaaef0 MD	51	* %rdx start address
fee7b0d8	52	* %rcx preserve_context
32cb4d02	53	* %r8 host_mem_enc_active
4bfaaef0 MD	54	*/
4bfaaef0 MD	55
fee7b0d8 HY	56	/* Save the CPU context, used for jumping back */
	57	pushq %rbx
	58	pushq %rbp
	59	pushq %r12
	60	pushq %r13
	61	pushq %r14
	62	pushq %r15
	63	pushf
	64
	65	movq PTR(VA_CONTROL_PAGE)(%rsi), %r11
	66	movq %rsp, RSP(%r11)
	67	movq %cr0, %rax
	68	movq %rax, CR0(%r11)
	69	movq %cr3, %rax
	70	movq %rax, CR3(%r11)
	71	movq %cr4, %rax
	72	movq %rax, CR4(%r11)
	73
5bf30316 KS	74	/* Save CR4. Required to enable the right paging mode later. */
	75	movq %rax, %r13
	76
5234f5eb EB	77	/* zero out flags, and disable interrupts */
	78	pushq $0
	79	popfq
	80
4e237903 TL	81	/* Save SME active flag */
	82	movq %r8, %r12
	83
fef3a7a1 HY	84	/*
	85	* get physical address of control page now
	86	* this is impossible after page table switch
	87	*/
4bfaaef0 MD	88	movq PTR(PA_CONTROL_PAGE)(%rsi), %r8
	89
	90	/* get physical address of page table now too */
fee7b0d8 HY	91	movq PTR(PA_TABLE_PAGE)(%rsi), %r9
	92
	93	/* get physical address of swap page now */
	94	movq PTR(PA_SWAP_PAGE)(%rsi), %r10
	95
	96	/* save some information for jumping back */
	97	movq %r9, CP_PA_TABLE_PAGE(%r11)
	98	movq %r10, CP_PA_SWAP_PAGE(%r11)
	99	movq %rdi, CP_PA_BACKUP_PAGES_MAP(%r11)
5234f5eb	100
f5deb796	101	/* Switch to the identity mapped page tables */
fee7b0d8	102	movq %r9, %cr3
4bfaaef0 MD	103
4bfaaef0 MD	104	/* setup a new stack at the end of the physical control page */
a7bba17b	105	lea PAGE_SIZE(%r8), %rsp
4bfaaef0 MD	106
	107	/* jump to identity mapped page */
	108	addq $(identity_mapped - relocate_kernel), %r8
	109	pushq %r8
697977d8 KRW	110	ANNOTATE_UNRET_SAFE
	111	ret
	112	int3
6ec2a968	113	SYM_CODE_END(relocate_kernel)
4bfaaef0	114
6ec2a968	115	SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
fb799447	116	UNWIND_HINT_END_OF_STACK
050438ed HY	117	/* set return address to 0 if not preserving context */
050438ed HY	118	pushq $0
4bfaaef0 MD	119	/* store the start address on the stack */
4bfaaef0 MD	120	pushq %rdx
5234f5eb	121
af227003 PZ	122	/*
	123	* Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP
	124	* below.
	125	*/
	126	movq %cr4, %rax
	127	andq $~(X86_CR4_CET), %rax
	128	movq %rax, %cr4
	129
fef3a7a1 HY	130	/*
fef3a7a1 HY	131	* Set cr0 to a known state:
fd3af531	132	* - Paging enabled
	133	* - Alignment check disabled
	134	* - Write protect disabled
	135	* - No task switch
	136	* - Don't do FP software emulation.
163b0991	137	* - Protected mode enabled
5234f5eb EB	138	*/
5234f5eb EB	139	movq %cr0, %rax
fd3af531	140	andq $~(X86_CR0_AM \| X86_CR0_WP \| X86_CR0_TS \| X86_CR0_EM), %rax
fd3af531	141	orl $(X86_CR0_PG \| X86_CR0_PE), %eax
5234f5eb EB	142	movq %rax, %cr0
5234f5eb EB	143
fef3a7a1 HY	144	/*
fef3a7a1 HY	145	* Set cr4 to a known state:
fd3af531	146	* - physical address extension enabled
5bf30316	147	* - 5-level paging, if it was enabled before
5234f5eb	148	*/
a734b4a2	149	movl $X86_CR4_PAE, %eax
5bf30316 KS	150	testq $X86_CR4_LA57, %r13
	151	jz 1f
	152	orl $X86_CR4_LA57, %eax
	153	1:
5234f5eb EB	154	movq %rax, %cr4
	155
	156	jmp 1f
	157	1:
	158
f5deb796	159	/* Flush the TLB (needed?) */
fee7b0d8 HY	160	movq %r9, %cr3
fee7b0d8 HY	161
4e237903 TL	162	/*
	163	* If SME is active, there could be old encrypted cache line
	164	* entries that will conflict with the now unencrypted memory
	165	* used by kexec. Flush the caches before copying the kernel.
	166	*/
	167	testq %r12, %r12
	168	jz 1f
	169	wbinvd
	170	1:
	171
fee7b0d8 HY	172	movq %rcx, %r11
	173	call swap_pages
	174
	175	/*
	176	* To be certain of avoiding problems with self-modifying code
	177	* I need to execute a serializing instruction here.
	178	* So I flush the TLB by reloading %cr3 here, it's handy,
	179	* and not processor dependent.
	180	*/
	181	movq %cr3, %rax
	182	movq %rax, %cr3
	183
	184	/*
	185	* set all of the registers to known values
	186	* leave %rsp alone
	187	*/
	188
	189	testq %r11, %r11
	190	jnz 1f
f037e416 PA	191	xorl %eax, %eax
	192	xorl %ebx, %ebx
	193	xorl %ecx, %ecx
	194	xorl %edx, %edx
	195	xorl %esi, %esi
	196	xorl %edi, %edi
	197	xorl %ebp, %ebp
	198	xorl %r8d, %r8d
	199	xorl %r9d, %r9d
	200	xorl %r10d, %r10d
	201	xorl %r11d, %r11d
	202	xorl %r12d, %r12d
	203	xorl %r13d, %r13d
	204	xorl %r14d, %r14d
	205	xorl %r15d, %r15d
fee7b0d8	206
697977d8 KRW	207	ANNOTATE_UNRET_SAFE
	208	ret
	209	int3
fee7b0d8 HY	210
	211	1:
	212	popq %rdx
	213	leaq PAGE_SIZE(%r10), %rsp
36cc5520	214	ANNOTATE_RETPOLINE_SAFE
fee7b0d8 HY	215	call *%rdx
	216
	217	/* get the re-entry point of the peer system */
	218	movq 0(%rsp), %rbp
fc8bd77d	219	leaq relocate_kernel(%rip), %r8
fee7b0d8 HY	220	movq CP_PA_SWAP_PAGE(%r8), %r10
	221	movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
	222	movq CP_PA_TABLE_PAGE(%r8), %rax
	223	movq %rax, %cr3
	224	lea PAGE_SIZE(%r8), %rsp
	225	call swap_pages
	226	movq $virtual_mapped, %rax
	227	pushq %rax
697977d8 KRW	228	ANNOTATE_UNRET_SAFE
	229	ret
	230	int3
6ec2a968	231	SYM_CODE_END(identity_mapped)
fee7b0d8	232
6ec2a968	233	SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
fb799447	234	UNWIND_HINT_END_OF_STACK
3e3f0695	235	ANNOTATE_NOENDBR // RET target, above
fee7b0d8 HY	236	movq RSP(%r8), %rsp
	237	movq CR4(%r8), %rax
	238	movq %rax, %cr4
	239	movq CR3(%r8), %rax
	240	movq CR0(%r8), %r8
	241	movq %rax, %cr3
	242	movq %r8, %cr0
	243	movq %rbp, %rax
	244
	245	popf
	246	popq %r15
	247	popq %r14
	248	popq %r13
	249	popq %r12
	250	popq %rbp
	251	popq %rbx
697977d8 KRW	252	ANNOTATE_UNRET_SAFE
	253	ret
	254	int3
6ec2a968	255	SYM_CODE_END(virtual_mapped)
5234f5eb EB	256
5234f5eb EB	257	/* Do the copies */
6ec2a968	258	SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
fb799447 JP	259	UNWIND_HINT_END_OF_STACK
fb799447 JP	260	movq %rdi, %rcx /* Put the page_list in %rcx */
f037e416 PA	261	xorl %edi, %edi
f037e416 PA	262	xorl %esi, %esi
5234f5eb EB	263	jmp 1f
	264
	265	0: /* top, read another word for the indirection page */
	266
	267	movq (%rbx), %rcx
	268	addq $8, %rbx
	269	1:
3e1aa7cb	270	testb $0x1, %cl /* is it a destination page? */
5234f5eb EB	271	jz 2f
	272	movq %rcx, %rdi
	273	andq $0xfffffffffffff000, %rdi
	274	jmp 0b
	275	2:
3e1aa7cb	276	testb $0x2, %cl /* is it an indirection page? */
5234f5eb EB	277	jz 2f
	278	movq %rcx, %rbx
	279	andq $0xfffffffffffff000, %rbx
	280	jmp 0b
	281	2:
3e1aa7cb	282	testb $0x4, %cl /* is it the done indicator? */
5234f5eb EB	283	jz 2f
	284	jmp 3f
	285	2:
3e1aa7cb	286	testb $0x8, %cl /* is it the source indicator? */
5234f5eb EB	287	jz 0b /* Ignore it otherwise */
	288	movq %rcx, %rsi /* For ever source page do a copy */
	289	andq $0xfffffffffffff000, %rsi
	290
fee7b0d8 HY	291	movq %rdi, %rdx
	292	movq %rsi, %rax
	293
	294	movq %r10, %rdi
a734b4a2	295	movl $512, %ecx
5234f5eb	296	rep ; movsq
5234f5eb	297
fee7b0d8 HY	298	movq %rax, %rdi
fee7b0d8 HY	299	movq %rdx, %rsi
a734b4a2	300	movl $512, %ecx
fee7b0d8	301	rep ; movsq
5234f5eb	302
fee7b0d8 HY	303	movq %rdx, %rdi
fee7b0d8 HY	304	movq %r10, %rsi
a734b4a2	305	movl $512, %ecx
fee7b0d8	306	rep ; movsq
5234f5eb	307
fee7b0d8 HY	308	lea PAGE_SIZE(%rax), %rsi
	309	jmp 0b
	310	3:
697977d8 KRW	311	ANNOTATE_UNRET_SAFE
	312	ret
	313	int3
6ec2a968	314	SYM_CODE_END(swap_pages)
fee7b0d8	315
e81dc127 TG	316	.skip KEXEC_CONTROL_CODE_MAX_SIZE - (. - relocate_kernel), 0xcc
e81dc127 TG	317	SYM_CODE_END(relocate_range);