Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
c1cc1552 CM |
2 | /* |
3 | * Based on arch/arm/mm/init.c | |
4 | * | |
5 | * Copyright (C) 1995-2005 Russell King | |
6 | * Copyright (C) 2012 ARM Ltd. | |
c1cc1552 CM |
7 | */ |
8 | ||
9 | #include <linux/kernel.h> | |
10 | #include <linux/export.h> | |
11 | #include <linux/errno.h> | |
12 | #include <linux/swap.h> | |
13 | #include <linux/init.h> | |
5a9e3e15 | 14 | #include <linux/cache.h> |
c1cc1552 CM |
15 | #include <linux/mman.h> |
16 | #include <linux/nodemask.h> | |
17 | #include <linux/initrd.h> | |
18 | #include <linux/gfp.h> | |
65033574 | 19 | #include <linux/math.h> |
c1cc1552 CM |
20 | #include <linux/memblock.h> |
21 | #include <linux/sort.h> | |
764b51ea | 22 | #include <linux/of.h> |
c1cc1552 | 23 | #include <linux/of_fdt.h> |
8b5369ea | 24 | #include <linux/dma-direct.h> |
0b1abd1f | 25 | #include <linux/dma-map-ops.h> |
86c8b27a | 26 | #include <linux/efi.h> |
a1e50a82 | 27 | #include <linux/swiotlb.h> |
dae8c235 | 28 | #include <linux/vmalloc.h> |
2077be67 | 29 | #include <linux/mm.h> |
764b51ea | 30 | #include <linux/kexec.h> |
e62aaeac | 31 | #include <linux/crash_dump.h> |
cf11e85f | 32 | #include <linux/hugetlb.h> |
2b865293 | 33 | #include <linux/acpi_iort.h> |
85f58eb1 | 34 | #include <linux/kmemleak.h> |
0cc2dc49 | 35 | #include <linux/execmem.h> |
c1cc1552 | 36 | |
a7f8de16 | 37 | #include <asm/boot.h> |
08375198 | 38 | #include <asm/fixmap.h> |
f9040773 | 39 | #include <asm/kasan.h> |
a7f8de16 | 40 | #include <asm/kernel-pgtable.h> |
f320bc74 | 41 | #include <asm/kvm_host.h> |
aa03c428 | 42 | #include <asm/memory.h> |
1a2db300 | 43 | #include <asm/numa.h> |
c1cc1552 CM |
44 | #include <asm/sections.h> |
45 | #include <asm/setup.h> | |
87dfb311 | 46 | #include <linux/sizes.h> |
c1cc1552 | 47 | #include <asm/tlb.h> |
e039ee4e | 48 | #include <asm/alternative.h> |
687842ec | 49 | #include <asm/xen/swiotlb-xen.h> |
c1cc1552 | 50 | |
a7f8de16 AB |
51 | /* |
52 | * We need to be able to catch inadvertent references to memstart_addr | |
53 | * that occur (potentially in generic code) before arm64_memblock_init() | |
54 | * executes, which assigns it its actual value. So use a default value | |
55 | * that cannot be mistaken for a real physical address. | |
56 | */ | |
5a9e3e15 | 57 | s64 memstart_addr __ro_after_init = -1; |
03ef055f MR |
58 | EXPORT_SYMBOL(memstart_addr); |
59 | ||
1a8e1cef | 60 | /* |
d78050ee CM |
61 | * If the corresponding config options are enabled, we create both ZONE_DMA |
62 | * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory | |
63 | * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4). | |
64 | * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory, | |
65 | * otherwise it is empty. | |
1a8e1cef | 66 | */ |
03149563 | 67 | phys_addr_t __ro_after_init arm64_dma_phys_limit; |
c1cc1552 | 68 | |
4e0bacd6 ZJ |
69 | /* |
70 | * To make optimal use of block mappings when laying out the linear | |
71 | * mapping, round down the base of physical memory to a size that can | |
72 | * be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE | |
73 | * (64k granule), or a multiple that can be mapped using contiguous bits | |
74 | * in the page tables: 32 * PMD_SIZE (16k granule) | |
75 | */ | |
76 | #if defined(CONFIG_ARM64_4K_PAGES) | |
77 | #define ARM64_MEMSTART_SHIFT PUD_SHIFT | |
78 | #elif defined(CONFIG_ARM64_16K_PAGES) | |
79 | #define ARM64_MEMSTART_SHIFT CONT_PMD_SHIFT | |
80 | #else | |
81 | #define ARM64_MEMSTART_SHIFT PMD_SHIFT | |
82 | #endif | |
83 | ||
84 | /* | |
85 | * sparsemem vmemmap imposes an additional requirement on the alignment of | |
86 | * memstart_addr, due to the fact that the base of the vmemmap region | |
87 | * has a direct correspondence, and needs to appear sufficiently aligned | |
88 | * in the virtual address space. | |
89 | */ | |
90 | #if ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS | |
91 | #define ARM64_MEMSTART_ALIGN (1UL << SECTION_SIZE_BITS) | |
92 | #else | |
93 | #define ARM64_MEMSTART_ALIGN (1UL << ARM64_MEMSTART_SHIFT) | |
94 | #endif | |
95 | ||
fdc26823 | 96 | static void __init arch_reserve_crashkernel(void) |
764b51ea | 97 | { |
fdc26823 | 98 | unsigned long long low_size = 0; |
6c4dcadd | 99 | unsigned long long crash_base, crash_size; |
944a45ab | 100 | char *cmdline = boot_command_line; |
6c4dcadd BH |
101 | bool high = false; |
102 | int ret; | |
764b51ea | 103 | |
40254101 | 104 | if (!IS_ENABLED(CONFIG_CRASH_RESERVE)) |
d339f158 JZ |
105 | return; |
106 | ||
944a45ab | 107 | ret = parse_crashkernel(cmdline, memblock_phys_mem_size(), |
fdc26823 BH |
108 | &crash_size, &crash_base, |
109 | &low_size, &high); | |
110 | if (ret) | |
764b51ea | 111 | return; |
944a45ab | 112 | |
fdc26823 BH |
113 | reserve_crashkernel_generic(cmdline, crash_size, crash_base, |
114 | low_size, high); | |
764b51ea | 115 | } |
764b51ea | 116 | |
d50314a6 | 117 | /* |
791ab8b2 CM |
118 | * Return the maximum physical address for a zone accessible by the given bits |
119 | * limit. If DRAM starts above 32-bit, expand the zone to the maximum | |
120 | * available memory, otherwise cap it at 32-bit. | |
d50314a6 | 121 | */ |
1a8e1cef | 122 | static phys_addr_t __init max_zone_phys(unsigned int zone_bits) |
d50314a6 | 123 | { |
791ab8b2 CM |
124 | phys_addr_t zone_mask = DMA_BIT_MASK(zone_bits); |
125 | phys_addr_t phys_start = memblock_start_of_DRAM(); | |
126 | ||
127 | if (phys_start > U32_MAX) | |
128 | zone_mask = PHYS_ADDR_MAX; | |
129 | else if (phys_start > zone_mask) | |
130 | zone_mask = U32_MAX; | |
131 | ||
132 | return min(zone_mask, memblock_end_of_DRAM() - 1) + 1; | |
d50314a6 CM |
133 | } |
134 | ||
f41ef4c2 | 135 | static void __init zone_sizes_init(void) |
1a2db300 GK |
136 | { |
137 | unsigned long max_zone_pfns[MAX_NR_ZONES] = {0}; | |
2b865293 | 138 | unsigned int __maybe_unused acpi_zone_dma_bits; |
8424ecdd | 139 | unsigned int __maybe_unused dt_zone_dma_bits; |
d78050ee | 140 | phys_addr_t __maybe_unused dma32_phys_limit = max_zone_phys(32); |
1a2db300 | 141 | |
1a8e1cef | 142 | #ifdef CONFIG_ZONE_DMA |
2b865293 | 143 | acpi_zone_dma_bits = fls64(acpi_iort_dma_get_max_cpu_address()); |
8424ecdd | 144 | dt_zone_dma_bits = fls64(of_dma_get_max_cpu_address(NULL)); |
2b865293 | 145 | zone_dma_bits = min3(32U, dt_zone_dma_bits, acpi_zone_dma_bits); |
9804f8c6 | 146 | arm64_dma_phys_limit = max_zone_phys(zone_dma_bits); |
1a8e1cef NSJ |
147 | max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit); |
148 | #endif | |
0c1f14ed | 149 | #ifdef CONFIG_ZONE_DMA32 |
d78050ee CM |
150 | max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); |
151 | if (!arm64_dma_phys_limit) | |
152 | arm64_dma_phys_limit = dma32_phys_limit; | |
0c1f14ed | 153 | #endif |
504cae45 BH |
154 | if (!arm64_dma_phys_limit) |
155 | arm64_dma_phys_limit = PHYS_MASK + 1; | |
f41ef4c2 | 156 | max_zone_pfns[ZONE_NORMAL] = max_pfn; |
1a2db300 | 157 | |
9691a071 | 158 | free_area_init(max_zone_pfns); |
1a2db300 GK |
159 | } |
160 | ||
873ba463 | 161 | int pfn_is_map_memory(unsigned long pfn) |
c1cc1552 | 162 | { |
093bbe21 | 163 | phys_addr_t addr = PFN_PHYS(pfn); |
4ab21506 | 164 | |
873ba463 MR |
165 | /* avoid false positives for bogus PFNs, see comment in pfn_valid() */ |
166 | if (PHYS_PFN(addr) != pfn) | |
4ab21506 | 167 | return 0; |
eeb0753b | 168 | |
5ad356ea | 169 | return memblock_is_map_memory(addr); |
c1cc1552 | 170 | } |
873ba463 | 171 | EXPORT_SYMBOL(pfn_is_map_memory); |
c1cc1552 | 172 | |
bb425a75 | 173 | static phys_addr_t memory_limit __ro_after_init = PHYS_ADDR_MAX; |
6083fe74 MR |
174 | |
175 | /* | |
176 | * Limit the memory size that was specified via FDT. | |
177 | */ | |
178 | static int __init early_mem(char *p) | |
179 | { | |
180 | if (!p) | |
181 | return 1; | |
182 | ||
183 | memory_limit = memparse(p, &p) & PAGE_MASK; | |
184 | pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); | |
185 | ||
186 | return 0; | |
187 | } | |
188 | early_param("mem", early_mem); | |
189 | ||
c1cc1552 CM |
190 | void __init arm64_memblock_init(void) |
191 | { | |
88053ec8 AB |
192 | s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); |
193 | ||
194 | /* | |
195 | * Corner case: 52-bit VA capable systems running KVM in nVHE mode may | |
196 | * be limited in their ability to support a linear map that exceeds 51 | |
197 | * bits of VA space, depending on the placement of the ID map. Given | |
198 | * that the placement of the ID map may be randomized, let's simply | |
199 | * limit the kernel's linear map to 51 bits as well if we detect this | |
200 | * configuration. | |
201 | */ | |
202 | if (IS_ENABLED(CONFIG_KVM) && vabits_actual == 52 && | |
203 | is_hyp_mode_available() && !is_kernel_in_hyp_mode()) { | |
204 | pr_info("Capping linear region to 51 bits for KVM in nVHE mode on LVA capable hardware.\n"); | |
205 | linear_region_size = min_t(u64, linear_region_size, BIT(51)); | |
206 | } | |
a7f8de16 | 207 | |
e9eaa805 KM |
208 | /* Remove memory above our supported physical address size */ |
209 | memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX); | |
210 | ||
a7f8de16 AB |
211 | /* |
212 | * Select a suitable value for the base of physical memory. | |
213 | */ | |
214 | memstart_addr = round_down(memblock_start_of_DRAM(), | |
215 | ARM64_MEMSTART_ALIGN); | |
216 | ||
31f80a4e MZ |
217 | if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size) |
218 | pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n"); | |
219 | ||
a7f8de16 AB |
220 | /* |
221 | * Remove the memory that we will not be able to cover with the | |
222 | * linear mapping. Take care not to clip the kernel which may be | |
223 | * high in memory. | |
224 | */ | |
2077be67 LA |
225 | memblock_remove(max_t(u64, memstart_addr + linear_region_size, |
226 | __pa_symbol(_end)), ULLONG_MAX); | |
2958987f AB |
227 | if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) { |
228 | /* ensure that memstart_addr remains sufficiently aligned */ | |
229 | memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size, | |
230 | ARM64_MEMSTART_ALIGN); | |
231 | memblock_remove(0, memstart_addr); | |
232 | } | |
a7f8de16 | 233 | |
7bc1a0f9 AB |
234 | /* |
235 | * If we are running with a 52-bit kernel VA config on a system that | |
236 | * does not support it, we have to place the available physical | |
237 | * memory in the 48-bit addressable part of the linear region, i.e., | |
238 | * we have to move it upward. Since memstart_addr represents the | |
239 | * physical address of PAGE_OFFSET, we have to *subtract* from it. | |
240 | */ | |
241 | if (IS_ENABLED(CONFIG_ARM64_VA_BITS_52) && (vabits_actual != 52)) | |
9684ec18 | 242 | memstart_addr -= _PAGE_OFFSET(vabits_actual) - _PAGE_OFFSET(52); |
7bc1a0f9 | 243 | |
a7f8de16 AB |
244 | /* |
245 | * Apply the memory limit if it was set. Since the kernel may be loaded | |
246 | * high up in memory, add back the kernel region that must be accessible | |
247 | * via the linear mapping. | |
248 | */ | |
d7dc899a | 249 | if (memory_limit != PHYS_ADDR_MAX) { |
cb0a6502 | 250 | memblock_mem_limit_remove_map(memory_limit); |
2077be67 | 251 | memblock_add(__pa_symbol(_text), (u64)(_end - _text)); |
a7f8de16 | 252 | } |
6083fe74 | 253 | |
c756c592 | 254 | if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { |
177e15f0 AB |
255 | /* |
256 | * Add back the memory we just removed if it results in the | |
257 | * initrd to become inaccessible via the linear mapping. | |
258 | * Otherwise, this is a no-op | |
259 | */ | |
c756c592 | 260 | u64 base = phys_initrd_start & PAGE_MASK; |
d4d18e3e | 261 | u64 size = PAGE_ALIGN(phys_initrd_start + phys_initrd_size) - base; |
177e15f0 AB |
262 | |
263 | /* | |
264 | * We can only add back the initrd memory if we don't end up | |
265 | * with more memory than we can address via the linear mapping. | |
266 | * It is up to the bootloader to position the kernel and the | |
267 | * initrd reasonably close to each other (i.e., within 32 GB of | |
268 | * each other) so that all granule/#levels combinations can | |
269 | * always access both. | |
270 | */ | |
271 | if (WARN(base < memblock_start_of_DRAM() || | |
272 | base + size > memblock_start_of_DRAM() + | |
273 | linear_region_size, | |
274 | "initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) { | |
70b3d237 | 275 | phys_initrd_size = 0; |
177e15f0 | 276 | } else { |
177e15f0 | 277 | memblock_add(base, size); |
c0b978fe | 278 | memblock_clear_nomap(base, size); |
177e15f0 AB |
279 | memblock_reserve(base, size); |
280 | } | |
281 | } | |
282 | ||
c031a421 AB |
283 | if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { |
284 | extern u16 memstart_offset_seed; | |
97d6786e AB |
285 | u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); |
286 | int parange = cpuid_feature_extract_unsigned_field( | |
2d987e64 | 287 | mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT); |
97d6786e AB |
288 | s64 range = linear_region_size - |
289 | BIT(id_aa64mmfr0_parange_to_phys_shift(parange)); | |
c031a421 AB |
290 | |
291 | /* | |
292 | * If the size of the linear region exceeds, by a sufficient | |
97d6786e AB |
293 | * margin, the size of the region that the physical memory can |
294 | * span, randomize the linear region as well. | |
c031a421 | 295 | */ |
97d6786e | 296 | if (memstart_offset_seed > 0 && range >= (s64)ARM64_MEMSTART_ALIGN) { |
c8a43c18 | 297 | range /= ARM64_MEMSTART_ALIGN; |
c031a421 AB |
298 | memstart_addr -= ARM64_MEMSTART_ALIGN * |
299 | ((range * memstart_offset_seed) >> 16); | |
300 | } | |
301 | } | |
6083fe74 | 302 | |
bd00cd5f MR |
303 | /* |
304 | * Register the kernel text, kernel data, initrd, and initial | |
305 | * pagetables with memblock. | |
306 | */ | |
e2a073dd | 307 | memblock_reserve(__pa_symbol(_stext), _end - _stext); |
c756c592 | 308 | if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { |
a89dea58 | 309 | /* the generic initrd code expects virtual addresses */ |
c756c592 FF |
310 | initrd_start = __phys_to_virt(phys_initrd_start); |
311 | initrd_end = initrd_start + phys_initrd_size; | |
a89dea58 | 312 | } |
c1cc1552 | 313 | |
0ceac9e0 | 314 | early_init_fdt_scan_reserved_mem(); |
2d5a5612 | 315 | |
f24e5834 | 316 | high_memory = __va(memblock_end_of_DRAM() - 1) + 1; |
c1cc1552 CM |
317 | } |
318 | ||
319 | void __init bootmem_init(void) | |
320 | { | |
321 | unsigned long min, max; | |
322 | ||
323 | min = PFN_UP(memblock_start_of_DRAM()); | |
324 | max = PFN_DOWN(memblock_end_of_DRAM()); | |
325 | ||
36dd9086 VM |
326 | early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT); |
327 | ||
1a2db300 | 328 | max_pfn = max_low_pfn = max; |
19d6242e | 329 | min_low_pfn = min; |
1a2db300 | 330 | |
eb75541f | 331 | arch_numa_init(); |
618e0786 BS |
332 | |
333 | /* | |
eb75541f | 334 | * must be done after arch_numa_init() which calls numa_init() to |
618e0786 BS |
335 | * initialize node_online_map that gets used in hugetlb_cma_reserve() |
336 | * while allocating required CMA size across online nodes. | |
337 | */ | |
abb7962a AK |
338 | #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA) |
339 | arm64_hugetlb_cma_reserve(); | |
618e0786 BS |
340 | #endif |
341 | ||
f320bc74 QP |
342 | kvm_hyp_reserve(); |
343 | ||
c1cc1552 | 344 | /* |
c89ab04f MR |
345 | * sparse_init() tries to allocate memory from memblock, so must be |
346 | * done after the fixed reservations | |
c1cc1552 | 347 | */ |
c1cc1552 | 348 | sparse_init(); |
f41ef4c2 | 349 | zone_sizes_init(); |
c1cc1552 | 350 | |
d78050ee CM |
351 | /* |
352 | * Reserve the CMA area after arm64_dma_phys_limit was initialised. | |
353 | */ | |
354 | dma_contiguous_reserve(arm64_dma_phys_limit); | |
355 | ||
0a30c535 NSJ |
356 | /* |
357 | * request_standard_resources() depends on crashkernel's memory being | |
358 | * reserved, so do it here. | |
359 | */ | |
fdc26823 | 360 | arch_reserve_crashkernel(); |
0a30c535 | 361 | |
1a2db300 | 362 | memblock_dump_all(); |
c1cc1552 CM |
363 | } |
364 | ||
c1cc1552 CM |
365 | /* |
366 | * mem_init() marks the free areas in the mem_map and tells us how much memory | |
367 | * is free. This is done after various parts of the system have claimed their | |
368 | * memory after the kernel image. | |
369 | */ | |
370 | void __init mem_init(void) | |
371 | { | |
1c1a429e CM |
372 | bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit); |
373 | ||
65033574 CM |
374 | if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) { |
375 | /* | |
376 | * If no bouncing needed for ZONE_DMA, reduce the swiotlb | |
377 | * buffer for kmalloc() bouncing to 1MB per 1GB of RAM. | |
378 | */ | |
379 | unsigned long size = | |
380 | DIV_ROUND_UP(memblock_phys_mem_size(), 1024); | |
381 | swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); | |
1c1a429e | 382 | swiotlb = true; |
65033574 | 383 | } |
1c1a429e CM |
384 | |
385 | swiotlb_init(swiotlb, SWIOTLB_VERBOSE); | |
a1e50a82 | 386 | |
bee4ebd1 | 387 | /* this will put all unused low memory onto the freelists */ |
c6ffc5ca | 388 | memblock_free_all(); |
c1cc1552 | 389 | |
c1cc1552 CM |
390 | /* |
391 | * Check boundaries twice: Some fundamental inconsistencies can be | |
392 | * detected at build time already. | |
393 | */ | |
394 | #ifdef CONFIG_COMPAT | |
363524d2 | 395 | BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64); |
c1cc1552 | 396 | #endif |
c1cc1552 | 397 | |
7e04cc91 AK |
398 | /* |
399 | * Selected page table levels should match when derived from | |
400 | * scratch using the virtual address range and page size. | |
401 | */ | |
402 | BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != | |
403 | CONFIG_PGTABLE_LEVELS); | |
404 | ||
bee4ebd1 | 405 | if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { |
c1cc1552 CM |
406 | extern int sysctl_overcommit_memory; |
407 | /* | |
408 | * On a machine this small we won't get anywhere without | |
409 | * overcommit, so turn it on by default. | |
410 | */ | |
411 | sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; | |
412 | } | |
413 | } | |
414 | ||
415 | void free_initmem(void) | |
416 | { | |
2077be67 LA |
417 | free_reserved_area(lm_alias(__init_begin), |
418 | lm_alias(__init_end), | |
6ec939f8 | 419 | POISON_FREE_INITMEM, "unused kernel"); |
dae8c235 KW |
420 | /* |
421 | * Unmap the __init region but leave the VM area in place. This | |
422 | * prevents the region from being reused for kernel modules, which | |
423 | * is not supported by kallsyms. | |
424 | */ | |
4ad0ae8c | 425 | vunmap_range((u64)__init_begin, (u64)__init_end); |
c1cc1552 CM |
426 | } |
427 | ||
638d5031 | 428 | void dump_mem_limit(void) |
a7f8de16 | 429 | { |
d7dc899a | 430 | if (memory_limit != PHYS_ADDR_MAX) { |
a7f8de16 AB |
431 | pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20); |
432 | } else { | |
433 | pr_emerg("Memory Limit: none\n"); | |
434 | } | |
a7f8de16 | 435 | } |
0cc2dc49 MRI |
436 | |
437 | #ifdef CONFIG_EXECMEM | |
438 | static u64 module_direct_base __ro_after_init = 0; | |
439 | static u64 module_plt_base __ro_after_init = 0; | |
440 | ||
441 | /* | |
442 | * Choose a random page-aligned base address for a window of 'size' bytes which | |
443 | * entirely contains the interval [start, end - 1]. | |
444 | */ | |
445 | static u64 __init random_bounding_box(u64 size, u64 start, u64 end) | |
446 | { | |
447 | u64 max_pgoff, pgoff; | |
448 | ||
449 | if ((end - start) >= size) | |
450 | return 0; | |
451 | ||
452 | max_pgoff = (size - (end - start)) / PAGE_SIZE; | |
453 | pgoff = get_random_u32_inclusive(0, max_pgoff); | |
454 | ||
455 | return start - pgoff * PAGE_SIZE; | |
456 | } | |
457 | ||
458 | /* | |
459 | * Modules may directly reference data and text anywhere within the kernel | |
460 | * image and other modules. References using PREL32 relocations have a +/-2G | |
461 | * range, and so we need to ensure that the entire kernel image and all modules | |
462 | * fall within a 2G window such that these are always within range. | |
463 | * | |
464 | * Modules may directly branch to functions and code within the kernel text, | |
465 | * and to functions and code within other modules. These branches will use | |
466 | * CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure | |
467 | * that the entire kernel text and all module text falls within a 128M window | |
468 | * such that these are always within range. With PLTs, we can expand this to a | |
469 | * 2G window. | |
470 | * | |
471 | * We chose the 128M region to surround the entire kernel image (rather than | |
472 | * just the text) as using the same bounds for the 128M and 2G regions ensures | |
473 | * by construction that we never select a 128M region that is not a subset of | |
474 | * the 2G region. For very large and unusual kernel configurations this means | |
475 | * we may fall back to PLTs where they could have been avoided, but this keeps | |
476 | * the logic significantly simpler. | |
477 | */ | |
478 | static int __init module_init_limits(void) | |
479 | { | |
480 | u64 kernel_end = (u64)_end; | |
481 | u64 kernel_start = (u64)_text; | |
482 | u64 kernel_size = kernel_end - kernel_start; | |
483 | ||
484 | /* | |
485 | * The default modules region is placed immediately below the kernel | |
486 | * image, and is large enough to use the full 2G relocation range. | |
487 | */ | |
488 | BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END); | |
489 | BUILD_BUG_ON(MODULES_VSIZE < SZ_2G); | |
490 | ||
491 | if (!kaslr_enabled()) { | |
492 | if (kernel_size < SZ_128M) | |
493 | module_direct_base = kernel_end - SZ_128M; | |
494 | if (kernel_size < SZ_2G) | |
495 | module_plt_base = kernel_end - SZ_2G; | |
496 | } else { | |
497 | u64 min = kernel_start; | |
498 | u64 max = kernel_end; | |
499 | ||
500 | if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { | |
501 | pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n"); | |
502 | } else { | |
503 | module_direct_base = random_bounding_box(SZ_128M, min, max); | |
504 | if (module_direct_base) { | |
505 | min = module_direct_base; | |
506 | max = module_direct_base + SZ_128M; | |
507 | } | |
508 | } | |
509 | ||
510 | module_plt_base = random_bounding_box(SZ_2G, min, max); | |
511 | } | |
512 | ||
513 | pr_info("%llu pages in range for non-PLT usage", | |
514 | module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0); | |
515 | pr_info("%llu pages in range for PLT usage", | |
516 | module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0); | |
517 | ||
518 | return 0; | |
519 | } | |
520 | ||
521 | static struct execmem_info execmem_info __ro_after_init; | |
522 | ||
523 | struct execmem_info __init *execmem_arch_setup(void) | |
524 | { | |
525 | unsigned long fallback_start = 0, fallback_end = 0; | |
526 | unsigned long start = 0, end = 0; | |
527 | ||
528 | module_init_limits(); | |
529 | ||
530 | /* | |
531 | * Where possible, prefer to allocate within direct branch range of the | |
532 | * kernel such that no PLTs are necessary. | |
533 | */ | |
534 | if (module_direct_base) { | |
535 | start = module_direct_base; | |
536 | end = module_direct_base + SZ_128M; | |
537 | ||
538 | if (module_plt_base) { | |
539 | fallback_start = module_plt_base; | |
540 | fallback_end = module_plt_base + SZ_2G; | |
541 | } | |
542 | } else if (module_plt_base) { | |
543 | start = module_plt_base; | |
544 | end = module_plt_base + SZ_2G; | |
545 | } | |
546 | ||
547 | execmem_info = (struct execmem_info){ | |
548 | .ranges = { | |
549 | [EXECMEM_DEFAULT] = { | |
550 | .start = start, | |
551 | .end = end, | |
552 | .pgprot = PAGE_KERNEL, | |
553 | .alignment = 1, | |
554 | .fallback_start = fallback_start, | |
555 | .fallback_end = fallback_end, | |
556 | }, | |
557 | [EXECMEM_KPROBES] = { | |
558 | .start = VMALLOC_START, | |
559 | .end = VMALLOC_END, | |
560 | .pgprot = PAGE_KERNEL_ROX, | |
561 | .alignment = 1, | |
562 | }, | |
563 | [EXECMEM_BPF] = { | |
564 | .start = VMALLOC_START, | |
565 | .end = VMALLOC_END, | |
566 | .pgprot = PAGE_KERNEL, | |
567 | .alignment = 1, | |
568 | }, | |
569 | }, | |
570 | }; | |
571 | ||
572 | return &execmem_info; | |
573 | } | |
574 | #endif /* CONFIG_EXECMEM */ |