Commit | Line | Data |
---|---|---|
cfecea6e KC |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include "misc.h" | |
103bf75f | 3 | #include <asm/bootparam.h> |
1b3a6264 | 4 | #include <asm/e820/types.h> |
08529078 | 5 | #include <asm/processor.h> |
3548e131 | 6 | #include "pgtable.h" |
fb526835 | 7 | #include "../string.h" |
5dc91f2d | 8 | #include "efi.h" |
08529078 | 9 | |
3548e131 KS |
10 | #define BIOS_START_MIN 0x20000U /* 128K, less than this is insane */ |
11 | #define BIOS_START_MAX 0x9f000U /* 640K, absolute maximum */ | |
12 | ||
e5ceb9a0 AS |
13 | #ifdef CONFIG_X86_5LEVEL |
14 | /* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */ | |
33def849 JP |
15 | unsigned int __section(".data") __pgtable_l5_enabled; |
16 | unsigned int __section(".data") pgdir_shift = 39; | |
17 | unsigned int __section(".data") ptrs_per_p4d = 1; | |
e5ceb9a0 AS |
18 | #endif |
19 | ||
fb526835 KS |
20 | /* Buffer to preserve trampoline memory */ |
21 | static char trampoline_save[TRAMPOLINE_32BIT_SIZE]; | |
22 | ||
3548e131 KS |
23 | /* |
24 | * Trampoline address will be printed by extract_kernel() for debugging | |
25 | * purposes. | |
26 | * | |
27 | * Avoid putting the pointer into .bss as it will be cleared between | |
64ef578b | 28 | * configure_5level_paging() and extract_kernel(). |
3548e131 | 29 | */ |
33def849 | 30 | unsigned long *trampoline_32bit __section(".data"); |
3548e131 | 31 | |
372fddf7 KS |
32 | int cmdline_find_option_bool(const char *option); |
33 | ||
1b3a6264 KS |
34 | static unsigned long find_trampoline_placement(void) |
35 | { | |
6f913de3 | 36 | unsigned long bios_start = 0, ebda_start = 0; |
1b3a6264 | 37 | struct boot_e820_entry *entry; |
6f913de3 | 38 | char *signature; |
1b3a6264 KS |
39 | int i; |
40 | ||
41 | /* | |
42 | * Find a suitable spot for the trampoline. | |
43 | * This code is based on reserve_bios_regions(). | |
44 | */ | |
45 | ||
6f913de3 KS |
46 | /* |
47 | * EFI systems may not provide legacy ROM. The memory may not be mapped | |
48 | * at all. | |
49 | * | |
50 | * Only look for values in the legacy ROM for non-EFI system. | |
51 | */ | |
d55d5bc5 | 52 | signature = (char *)&boot_params_ptr->efi_info.efi_loader_signature; |
6f913de3 KS |
53 | if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) && |
54 | strncmp(signature, EFI64_LOADER_SIGNATURE, 4)) { | |
55 | ebda_start = *(unsigned short *)0x40e << 4; | |
56 | bios_start = *(unsigned short *)0x413 << 10; | |
57 | } | |
1b3a6264 KS |
58 | |
59 | if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) | |
60 | bios_start = BIOS_START_MAX; | |
61 | ||
62 | if (ebda_start > BIOS_START_MIN && ebda_start < bios_start) | |
63 | bios_start = ebda_start; | |
64 | ||
65 | bios_start = round_down(bios_start, PAGE_SIZE); | |
66 | ||
67 | /* Find the first usable memory region under bios_start. */ | |
d55d5bc5 | 68 | for (i = boot_params_ptr->e820_entries - 1; i >= 0; i--) { |
c96e8483 | 69 | unsigned long new = bios_start; |
0a46fff2 | 70 | |
d55d5bc5 | 71 | entry = &boot_params_ptr->e820_table[i]; |
1b3a6264 KS |
72 | |
73 | /* Skip all entries above bios_start. */ | |
74 | if (bios_start <= entry->addr) | |
75 | continue; | |
76 | ||
77 | /* Skip non-RAM entries. */ | |
78 | if (entry->type != E820_TYPE_RAM) | |
79 | continue; | |
80 | ||
81 | /* Adjust bios_start to the end of the entry if needed. */ | |
82 | if (bios_start > entry->addr + entry->size) | |
0a46fff2 | 83 | new = entry->addr + entry->size; |
1b3a6264 KS |
84 | |
85 | /* Keep bios_start page-aligned. */ | |
0a46fff2 | 86 | new = round_down(new, PAGE_SIZE); |
1b3a6264 KS |
87 | |
88 | /* Skip the entry if it's too small. */ | |
0a46fff2 | 89 | if (new - TRAMPOLINE_32BIT_SIZE < entry->addr) |
1b3a6264 KS |
90 | continue; |
91 | ||
0a46fff2 KS |
92 | /* Protect against underflow. */ |
93 | if (new - TRAMPOLINE_32BIT_SIZE > bios_start) | |
94 | break; | |
95 | ||
96 | bios_start = new; | |
1b3a6264 KS |
97 | break; |
98 | } | |
99 | ||
100 | /* Place the trampoline just below the end of low memory */ | |
101 | return bios_start - TRAMPOLINE_32BIT_SIZE; | |
102 | } | |
103 | ||
03dda951 | 104 | asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) |
4440977b | 105 | { |
cb83cece | 106 | void (*toggle_la57)(void *cr3); |
64ef578b | 107 | bool l5_required = false; |
08529078 | 108 | |
372fddf7 | 109 | /* Initialize boot_params. Required for cmdline_find_option_bool(). */ |
d55d5bc5 | 110 | boot_params_ptr = bp; |
372fddf7 | 111 | |
a403d798 KS |
112 | /* |
113 | * Check if LA57 is desired and supported. | |
114 | * | |
372fddf7 | 115 | * There are several parts to the check: |
a403d798 | 116 | * - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y |
372fddf7 | 117 | * - if user asked to disable 5-level paging: no5lvl in cmdline |
a403d798 KS |
118 | * - if the machine supports 5-level paging: |
119 | * + CPUID leaf 7 is supported | |
120 | * + the leaf has the feature bit set | |
121 | * | |
122 | * That's substitute for boot_cpu_has() in early boot code. | |
123 | */ | |
124 | if (IS_ENABLED(CONFIG_X86_5LEVEL) && | |
372fddf7 | 125 | !cmdline_find_option_bool("no5lvl") && |
a403d798 KS |
126 | native_cpuid_eax(0) >= 7 && |
127 | (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) { | |
64ef578b | 128 | l5_required = true; |
00c6b097 AB |
129 | |
130 | /* Initialize variables for 5-level paging */ | |
131 | __pgtable_l5_enabled = 1; | |
132 | pgdir_shift = 48; | |
133 | ptrs_per_p4d = 512; | |
a403d798 | 134 | } |
08529078 | 135 | |
f97b67a7 AB |
136 | /* |
137 | * The trampoline will not be used if the paging mode is already set to | |
138 | * the desired one. | |
139 | */ | |
140 | if (l5_required == !!(native_read_cr4() & X86_CR4_LA57)) | |
141 | return; | |
142 | ||
64ef578b | 143 | trampoline_32bit = (unsigned long *)find_trampoline_placement(); |
3548e131 | 144 | |
fb526835 KS |
145 | /* Preserve trampoline memory */ |
146 | memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE); | |
147 | ||
32fcefa2 KS |
148 | /* Clear trampoline memory first */ |
149 | memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE); | |
150 | ||
151 | /* Copy trampoline code in place */ | |
64ef578b | 152 | toggle_la57 = memcpy(trampoline_32bit + |
bd328aa0 | 153 | TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long), |
32fcefa2 KS |
154 | &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE); |
155 | ||
bd328aa0 AB |
156 | /* |
157 | * Avoid the need for a stack in the 32-bit trampoline code, by using | |
158 | * LJMP rather than LRET to return back to long mode. LJMP takes an | |
159 | * immediate absolute address, which needs to be adjusted based on the | |
160 | * placement of the trampoline. | |
161 | */ | |
64ef578b AB |
162 | *(u32 *)((u8 *)toggle_la57 + trampoline_ljmp_imm_offset) += |
163 | (unsigned long)toggle_la57; | |
bd328aa0 | 164 | |
e9d0e633 KS |
165 | /* |
166 | * The code below prepares page table in trampoline memory. | |
167 | * | |
168 | * The new page table will be used by trampoline code for switching | |
169 | * from 4- to 5-level paging or vice versa. | |
e9d0e633 KS |
170 | */ |
171 | ||
64ef578b | 172 | if (l5_required) { |
e9d0e633 KS |
173 | /* |
174 | * For 4- to 5-level paging transition, set up current CR3 as | |
175 | * the first and the only entry in a new top-level page table. | |
176 | */ | |
cb83cece | 177 | *trampoline_32bit = __native_read_cr3() | _PAGE_TABLE_NOENC; |
e9d0e633 KS |
178 | } else { |
179 | unsigned long src; | |
180 | ||
181 | /* | |
182 | * For 5- to 4-level paging transition, copy page table pointed | |
183 | * by first entry in the current top-level page table as our | |
184 | * new top-level page table. | |
185 | * | |
186 | * We cannot just point to the page table from trampoline as it | |
187 | * may be above 4G. | |
188 | */ | |
189 | src = *(unsigned long *)__native_read_cr3() & PAGE_MASK; | |
cb83cece | 190 | memcpy(trampoline_32bit, (void *)src, PAGE_SIZE); |
e9d0e633 KS |
191 | } |
192 | ||
f97b67a7 | 193 | toggle_la57(trampoline_32bit); |
e9d0e633 KS |
194 | |
195 | /* | |
03dda951 | 196 | * Move the top level page table out of trampoline memory. |
e9d0e633 | 197 | */ |
03dda951 AB |
198 | memcpy(pgtable, trampoline_32bit, PAGE_SIZE); |
199 | native_write_cr3((unsigned long)pgtable); | |
e9d0e633 | 200 | |
fb526835 KS |
201 | /* Restore trampoline memory */ |
202 | memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); | |
203 | } |