| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | /* |
| 3 | * relocate_kernel.S - put the kernel image in place to boot |
| 4 | * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com> |
| 5 | */ |
| 6 | |
| 7 | #include <linux/linkage.h> |
| 8 | #include <asm/page_types.h> |
| 9 | #include <asm/kexec.h> |
| 10 | #include <asm/nospec-branch.h> |
| 11 | #include <asm/processor-flags.h> |
| 12 | |
| 13 | /* |
| 14 | * Must be relocatable PIC code callable as a C function, in particular |
| 15 | * there must be a plain RET and not jump to return thunk. |
| 16 | */ |
| 17 | |
| 18 | #define PTR(x) (x << 2) |
| 19 | |
| 20 | /* |
| 21 | * control_page + KEXEC_CONTROL_CODE_MAX_SIZE |
| 22 | * ~ control_page + PAGE_SIZE are used as data storage and stack for |
| 23 | * jumping back |
| 24 | */ |
| 25 | #define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) |
| 26 | |
| 27 | /* Minimal CPU state */ |
| 28 | #define ESP DATA(0x0) |
| 29 | #define CR0 DATA(0x4) |
| 30 | #define CR3 DATA(0x8) |
| 31 | #define CR4 DATA(0xc) |
| 32 | |
| 33 | /* other data */ |
| 34 | #define CP_VA_CONTROL_PAGE DATA(0x10) |
| 35 | #define CP_PA_PGD DATA(0x14) |
| 36 | #define CP_PA_SWAP_PAGE DATA(0x18) |
| 37 | #define CP_PA_BACKUP_PAGES_MAP DATA(0x1c) |
| 38 | |
| 39 | .text |
| 40 | SYM_CODE_START_NOALIGN(relocate_kernel) |
| 41 | /* Save the CPU context, used for jumping back */ |
| 42 | |
| 43 | pushl %ebx |
| 44 | pushl %esi |
| 45 | pushl %edi |
| 46 | pushl %ebp |
| 47 | pushf |
| 48 | |
| 49 | movl 20+8(%esp), %ebp /* list of pages */ |
| 50 | movl PTR(VA_CONTROL_PAGE)(%ebp), %edi |
| 51 | movl %esp, ESP(%edi) |
| 52 | movl %cr0, %eax |
| 53 | movl %eax, CR0(%edi) |
| 54 | movl %cr3, %eax |
| 55 | movl %eax, CR3(%edi) |
| 56 | movl %cr4, %eax |
| 57 | movl %eax, CR4(%edi) |
| 58 | |
| 59 | /* read the arguments and say goodbye to the stack */ |
| 60 | movl 20+4(%esp), %ebx /* page_list */ |
| 61 | movl 20+8(%esp), %ebp /* list of pages */ |
| 62 | movl 20+12(%esp), %edx /* start address */ |
| 63 | movl 20+16(%esp), %ecx /* cpu_has_pae */ |
| 64 | movl 20+20(%esp), %esi /* preserve_context */ |
| 65 | |
| 66 | /* zero out flags, and disable interrupts */ |
| 67 | pushl $0 |
| 68 | popfl |
| 69 | |
| 70 | /* save some information for jumping back */ |
| 71 | movl PTR(VA_CONTROL_PAGE)(%ebp), %edi |
| 72 | movl %edi, CP_VA_CONTROL_PAGE(%edi) |
| 73 | movl PTR(PA_PGD)(%ebp), %eax |
| 74 | movl %eax, CP_PA_PGD(%edi) |
| 75 | movl PTR(PA_SWAP_PAGE)(%ebp), %eax |
| 76 | movl %eax, CP_PA_SWAP_PAGE(%edi) |
| 77 | movl %ebx, CP_PA_BACKUP_PAGES_MAP(%edi) |
| 78 | |
| 79 | /* |
| 80 | * get physical address of control page now |
| 81 | * this is impossible after page table switch |
| 82 | */ |
| 83 | movl PTR(PA_CONTROL_PAGE)(%ebp), %edi |
| 84 | |
| 85 | /* switch to new set of page tables */ |
| 86 | movl PTR(PA_PGD)(%ebp), %eax |
| 87 | movl %eax, %cr3 |
| 88 | |
| 89 | /* setup a new stack at the end of the physical control page */ |
| 90 | lea PAGE_SIZE(%edi), %esp |
| 91 | |
| 92 | /* jump to identity mapped page */ |
| 93 | movl %edi, %eax |
| 94 | addl $(identity_mapped - relocate_kernel), %eax |
| 95 | pushl %eax |
| 96 | ANNOTATE_UNRET_SAFE |
| 97 | ret |
| 98 | int3 |
| 99 | SYM_CODE_END(relocate_kernel) |
| 100 | |
| 101 | SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) |
| 102 | /* set return address to 0 if not preserving context */ |
| 103 | pushl $0 |
| 104 | /* store the start address on the stack */ |
| 105 | pushl %edx |
| 106 | |
| 107 | /* |
| 108 | * Set cr0 to a known state: |
| 109 | * - Paging disabled |
| 110 | * - Alignment check disabled |
| 111 | * - Write protect disabled |
| 112 | * - No task switch |
| 113 | * - Don't do FP software emulation. |
| 114 | * - Protected mode enabled |
| 115 | */ |
| 116 | movl %cr0, %eax |
| 117 | andl $~(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax |
| 118 | orl $(X86_CR0_PE), %eax |
| 119 | movl %eax, %cr0 |
| 120 | |
| 121 | /* clear cr4 if applicable */ |
| 122 | testl %ecx, %ecx |
| 123 | jz 1f |
| 124 | /* |
| 125 | * Set cr4 to a known state: |
| 126 | * Setting everything to zero seems safe. |
| 127 | */ |
| 128 | xorl %eax, %eax |
| 129 | movl %eax, %cr4 |
| 130 | |
| 131 | jmp 1f |
| 132 | 1: |
| 133 | |
| 134 | /* Flush the TLB (needed?) */ |
| 135 | xorl %eax, %eax |
| 136 | movl %eax, %cr3 |
| 137 | |
| 138 | movl CP_PA_SWAP_PAGE(%edi), %eax |
| 139 | pushl %eax |
| 140 | pushl %ebx |
| 141 | call swap_pages |
| 142 | addl $8, %esp |
| 143 | |
| 144 | /* |
| 145 | * To be certain of avoiding problems with self-modifying code |
| 146 | * I need to execute a serializing instruction here. |
| 147 | * So I flush the TLB, it's handy, and not processor dependent. |
| 148 | */ |
| 149 | xorl %eax, %eax |
| 150 | movl %eax, %cr3 |
| 151 | |
| 152 | /* |
| 153 | * set all of the registers to known values |
| 154 | * leave %esp alone |
| 155 | */ |
| 156 | |
| 157 | testl %esi, %esi |
| 158 | jnz 1f |
| 159 | xorl %edi, %edi |
| 160 | xorl %eax, %eax |
| 161 | xorl %ebx, %ebx |
| 162 | xorl %ecx, %ecx |
| 163 | xorl %edx, %edx |
| 164 | xorl %esi, %esi |
| 165 | xorl %ebp, %ebp |
| 166 | ANNOTATE_UNRET_SAFE |
| 167 | ret |
| 168 | int3 |
| 169 | 1: |
| 170 | popl %edx |
| 171 | movl CP_PA_SWAP_PAGE(%edi), %esp |
| 172 | addl $PAGE_SIZE, %esp |
| 173 | 2: |
| 174 | ANNOTATE_RETPOLINE_SAFE |
| 175 | call *%edx |
| 176 | |
| 177 | /* get the re-entry point of the peer system */ |
| 178 | movl 0(%esp), %ebp |
| 179 | call 1f |
| 180 | 1: |
| 181 | popl %ebx |
| 182 | subl $(1b - relocate_kernel), %ebx |
| 183 | movl CP_VA_CONTROL_PAGE(%ebx), %edi |
| 184 | lea PAGE_SIZE(%ebx), %esp |
| 185 | movl CP_PA_SWAP_PAGE(%ebx), %eax |
| 186 | movl CP_PA_BACKUP_PAGES_MAP(%ebx), %edx |
| 187 | pushl %eax |
| 188 | pushl %edx |
| 189 | call swap_pages |
| 190 | addl $8, %esp |
| 191 | movl CP_PA_PGD(%ebx), %eax |
| 192 | movl %eax, %cr3 |
| 193 | movl %cr0, %eax |
| 194 | orl $X86_CR0_PG, %eax |
| 195 | movl %eax, %cr0 |
| 196 | lea PAGE_SIZE(%edi), %esp |
| 197 | movl %edi, %eax |
| 198 | addl $(virtual_mapped - relocate_kernel), %eax |
| 199 | pushl %eax |
| 200 | ANNOTATE_UNRET_SAFE |
| 201 | ret |
| 202 | int3 |
| 203 | SYM_CODE_END(identity_mapped) |
| 204 | |
| 205 | SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) |
| 206 | movl CR4(%edi), %eax |
| 207 | movl %eax, %cr4 |
| 208 | movl CR3(%edi), %eax |
| 209 | movl %eax, %cr3 |
| 210 | movl CR0(%edi), %eax |
| 211 | movl %eax, %cr0 |
| 212 | movl ESP(%edi), %esp |
| 213 | movl %ebp, %eax |
| 214 | |
| 215 | popf |
| 216 | popl %ebp |
| 217 | popl %edi |
| 218 | popl %esi |
| 219 | popl %ebx |
| 220 | ANNOTATE_UNRET_SAFE |
| 221 | ret |
| 222 | int3 |
| 223 | SYM_CODE_END(virtual_mapped) |
| 224 | |
| 225 | /* Do the copies */ |
| 226 | SYM_CODE_START_LOCAL_NOALIGN(swap_pages) |
| 227 | movl 8(%esp), %edx |
| 228 | movl 4(%esp), %ecx |
| 229 | pushl %ebp |
| 230 | pushl %ebx |
| 231 | pushl %edi |
| 232 | pushl %esi |
| 233 | movl %ecx, %ebx |
| 234 | jmp 1f |
| 235 | |
| 236 | 0: /* top, read another word from the indirection page */ |
| 237 | movl (%ebx), %ecx |
| 238 | addl $4, %ebx |
| 239 | 1: |
| 240 | testb $0x1, %cl /* is it a destination page */ |
| 241 | jz 2f |
| 242 | movl %ecx, %edi |
| 243 | andl $0xfffff000, %edi |
| 244 | jmp 0b |
| 245 | 2: |
| 246 | testb $0x2, %cl /* is it an indirection page */ |
| 247 | jz 2f |
| 248 | movl %ecx, %ebx |
| 249 | andl $0xfffff000, %ebx |
| 250 | jmp 0b |
| 251 | 2: |
| 252 | testb $0x4, %cl /* is it the done indicator */ |
| 253 | jz 2f |
| 254 | jmp 3f |
| 255 | 2: |
| 256 | testb $0x8, %cl /* is it the source indicator */ |
| 257 | jz 0b /* Ignore it otherwise */ |
| 258 | movl %ecx, %esi /* For every source page do a copy */ |
| 259 | andl $0xfffff000, %esi |
| 260 | |
| 261 | movl %edi, %eax |
| 262 | movl %esi, %ebp |
| 263 | |
| 264 | movl %edx, %edi |
| 265 | movl $1024, %ecx |
| 266 | rep ; movsl |
| 267 | |
| 268 | movl %ebp, %edi |
| 269 | movl %eax, %esi |
| 270 | movl $1024, %ecx |
| 271 | rep ; movsl |
| 272 | |
| 273 | movl %eax, %edi |
| 274 | movl %edx, %esi |
| 275 | movl $1024, %ecx |
| 276 | rep ; movsl |
| 277 | |
| 278 | lea PAGE_SIZE(%ebp), %esi |
| 279 | jmp 0b |
| 280 | 3: |
| 281 | popl %esi |
| 282 | popl %edi |
| 283 | popl %ebx |
| 284 | popl %ebp |
| 285 | ANNOTATE_UNRET_SAFE |
| 286 | ret |
| 287 | int3 |
| 288 | SYM_CODE_END(swap_pages) |
| 289 | |
| 290 | .globl kexec_control_code_size |
| 291 | .set kexec_control_code_size, . - relocate_kernel |