Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
c1cc1552 CM |
2 | /* |
3 | * Based on arch/arm/mm/init.c | |
4 | * | |
5 | * Copyright (C) 1995-2005 Russell King | |
6 | * Copyright (C) 2012 ARM Ltd. | |
c1cc1552 CM |
7 | */ |
8 | ||
9 | #include <linux/kernel.h> | |
10 | #include <linux/export.h> | |
11 | #include <linux/errno.h> | |
12 | #include <linux/swap.h> | |
13 | #include <linux/init.h> | |
5a9e3e15 | 14 | #include <linux/cache.h> |
c1cc1552 CM |
15 | #include <linux/mman.h> |
16 | #include <linux/nodemask.h> | |
17 | #include <linux/initrd.h> | |
18 | #include <linux/gfp.h> | |
65033574 | 19 | #include <linux/math.h> |
c1cc1552 CM |
20 | #include <linux/memblock.h> |
21 | #include <linux/sort.h> | |
764b51ea | 22 | #include <linux/of.h> |
c1cc1552 | 23 | #include <linux/of_fdt.h> |
8b5369ea | 24 | #include <linux/dma-direct.h> |
0b1abd1f | 25 | #include <linux/dma-map-ops.h> |
86c8b27a | 26 | #include <linux/efi.h> |
a1e50a82 | 27 | #include <linux/swiotlb.h> |
dae8c235 | 28 | #include <linux/vmalloc.h> |
2077be67 | 29 | #include <linux/mm.h> |
764b51ea | 30 | #include <linux/kexec.h> |
e62aaeac | 31 | #include <linux/crash_dump.h> |
cf11e85f | 32 | #include <linux/hugetlb.h> |
2b865293 | 33 | #include <linux/acpi_iort.h> |
85f58eb1 | 34 | #include <linux/kmemleak.h> |
0cc2dc49 | 35 | #include <linux/execmem.h> |
c1cc1552 | 36 | |
a7f8de16 | 37 | #include <asm/boot.h> |
08375198 | 38 | #include <asm/fixmap.h> |
f9040773 | 39 | #include <asm/kasan.h> |
a7f8de16 | 40 | #include <asm/kernel-pgtable.h> |
f320bc74 | 41 | #include <asm/kvm_host.h> |
aa03c428 | 42 | #include <asm/memory.h> |
1a2db300 | 43 | #include <asm/numa.h> |
c1cc1552 CM |
44 | #include <asm/sections.h> |
45 | #include <asm/setup.h> | |
87dfb311 | 46 | #include <linux/sizes.h> |
c1cc1552 | 47 | #include <asm/tlb.h> |
e039ee4e | 48 | #include <asm/alternative.h> |
687842ec | 49 | #include <asm/xen/swiotlb-xen.h> |
c1cc1552 | 50 | |
a7f8de16 AB |
51 | /* |
52 | * We need to be able to catch inadvertent references to memstart_addr | |
53 | * that occur (potentially in generic code) before arm64_memblock_init() | |
54 | * executes, which assigns it its actual value. So use a default value | |
55 | * that cannot be mistaken for a real physical address. | |
56 | */ | |
5a9e3e15 | 57 | s64 memstart_addr __ro_after_init = -1; |
03ef055f MR |
58 | EXPORT_SYMBOL(memstart_addr); |
59 | ||
1a8e1cef | 60 | /* |
d78050ee CM |
61 | * If the corresponding config options are enabled, we create both ZONE_DMA |
62 | * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory | |
63 | * unless restricted on specific platforms (e.g. 30-bit on Raspberry Pi 4). | |
64 | * In such case, ZONE_DMA32 covers the rest of the 32-bit addressable memory, | |
65 | * otherwise it is empty. | |
1a8e1cef | 66 | */ |
03149563 | 67 | phys_addr_t __ro_after_init arm64_dma_phys_limit; |
c1cc1552 | 68 | |
4e0bacd6 ZJ |
69 | /* |
70 | * To make optimal use of block mappings when laying out the linear | |
71 | * mapping, round down the base of physical memory to a size that can | |
72 | * be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE | |
73 | * (64k granule), or a multiple that can be mapped using contiguous bits | |
74 | * in the page tables: 32 * PMD_SIZE (16k granule) | |
75 | */ | |
76 | #if defined(CONFIG_ARM64_4K_PAGES) | |
77 | #define ARM64_MEMSTART_SHIFT PUD_SHIFT | |
78 | #elif defined(CONFIG_ARM64_16K_PAGES) | |
79 | #define ARM64_MEMSTART_SHIFT CONT_PMD_SHIFT | |
80 | #else | |
81 | #define ARM64_MEMSTART_SHIFT PMD_SHIFT | |
82 | #endif | |
83 | ||
84 | /* | |
85 | * sparsemem vmemmap imposes an additional requirement on the alignment of | |
86 | * memstart_addr, due to the fact that the base of the vmemmap region | |
87 | * has a direct correspondence, and needs to appear sufficiently aligned | |
88 | * in the virtual address space. | |
89 | */ | |
90 | #if ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS | |
91 | #define ARM64_MEMSTART_ALIGN (1UL << SECTION_SIZE_BITS) | |
92 | #else | |
93 | #define ARM64_MEMSTART_ALIGN (1UL << ARM64_MEMSTART_SHIFT) | |
94 | #endif | |
95 | ||
fdc26823 | 96 | static void __init arch_reserve_crashkernel(void) |
764b51ea | 97 | { |
fdc26823 | 98 | unsigned long long low_size = 0; |
6c4dcadd | 99 | unsigned long long crash_base, crash_size; |
944a45ab | 100 | char *cmdline = boot_command_line; |
6c4dcadd BH |
101 | bool high = false; |
102 | int ret; | |
764b51ea | 103 | |
40254101 | 104 | if (!IS_ENABLED(CONFIG_CRASH_RESERVE)) |
d339f158 JZ |
105 | return; |
106 | ||
944a45ab | 107 | ret = parse_crashkernel(cmdline, memblock_phys_mem_size(), |
fdc26823 BH |
108 | &crash_size, &crash_base, |
109 | &low_size, &high); | |
110 | if (ret) | |
764b51ea | 111 | return; |
944a45ab | 112 | |
fdc26823 BH |
113 | reserve_crashkernel_generic(cmdline, crash_size, crash_base, |
114 | low_size, high); | |
764b51ea | 115 | } |
764b51ea | 116 | |
ba0fb44a | 117 | static phys_addr_t __init max_zone_phys(phys_addr_t zone_limit) |
d50314a6 | 118 | { |
122c234e BS |
119 | /** |
120 | * Information we get from firmware (e.g. DT dma-ranges) describe DMA | |
121 | * bus constraints. Devices using DMA might have their own limitations. | |
122 | * Some of them rely on DMA zone in low 32-bit memory. Keep low RAM | |
123 | * DMA zone on platforms that have RAM there. | |
124 | */ | |
125 | if (memblock_start_of_DRAM() < U32_MAX) | |
126 | zone_limit = min(zone_limit, U32_MAX); | |
833bd284 | 127 | |
ba0fb44a | 128 | return min(zone_limit, memblock_end_of_DRAM() - 1) + 1; |
d50314a6 CM |
129 | } |
130 | ||
f41ef4c2 | 131 | static void __init zone_sizes_init(void) |
1a2db300 GK |
132 | { |
133 | unsigned long max_zone_pfns[MAX_NR_ZONES] = {0}; | |
ba0fb44a CM |
134 | phys_addr_t __maybe_unused acpi_zone_dma_limit; |
135 | phys_addr_t __maybe_unused dt_zone_dma_limit; | |
136 | phys_addr_t __maybe_unused dma32_phys_limit = | |
137 | max_zone_phys(DMA_BIT_MASK(32)); | |
1a2db300 | 138 | |
1a8e1cef | 139 | #ifdef CONFIG_ZONE_DMA |
ba0fb44a CM |
140 | acpi_zone_dma_limit = acpi_iort_dma_get_max_cpu_address(); |
141 | dt_zone_dma_limit = of_dma_get_max_cpu_address(NULL); | |
142 | zone_dma_limit = min(dt_zone_dma_limit, acpi_zone_dma_limit); | |
143 | arm64_dma_phys_limit = max_zone_phys(zone_dma_limit); | |
1a8e1cef NSJ |
144 | max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit); |
145 | #endif | |
0c1f14ed | 146 | #ifdef CONFIG_ZONE_DMA32 |
d78050ee CM |
147 | max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); |
148 | if (!arm64_dma_phys_limit) | |
149 | arm64_dma_phys_limit = dma32_phys_limit; | |
0c1f14ed | 150 | #endif |
504cae45 BH |
151 | if (!arm64_dma_phys_limit) |
152 | arm64_dma_phys_limit = PHYS_MASK + 1; | |
f41ef4c2 | 153 | max_zone_pfns[ZONE_NORMAL] = max_pfn; |
1a2db300 | 154 | |
9691a071 | 155 | free_area_init(max_zone_pfns); |
1a2db300 GK |
156 | } |
157 | ||
873ba463 | 158 | int pfn_is_map_memory(unsigned long pfn) |
c1cc1552 | 159 | { |
093bbe21 | 160 | phys_addr_t addr = PFN_PHYS(pfn); |
4ab21506 | 161 | |
873ba463 MR |
162 | /* avoid false positives for bogus PFNs, see comment in pfn_valid() */ |
163 | if (PHYS_PFN(addr) != pfn) | |
4ab21506 | 164 | return 0; |
eeb0753b | 165 | |
5ad356ea | 166 | return memblock_is_map_memory(addr); |
c1cc1552 | 167 | } |
873ba463 | 168 | EXPORT_SYMBOL(pfn_is_map_memory); |
c1cc1552 | 169 | |
bb425a75 | 170 | static phys_addr_t memory_limit __ro_after_init = PHYS_ADDR_MAX; |
6083fe74 MR |
171 | |
172 | /* | |
173 | * Limit the memory size that was specified via FDT. | |
174 | */ | |
175 | static int __init early_mem(char *p) | |
176 | { | |
177 | if (!p) | |
178 | return 1; | |
179 | ||
180 | memory_limit = memparse(p, &p) & PAGE_MASK; | |
181 | pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); | |
182 | ||
183 | return 0; | |
184 | } | |
185 | early_param("mem", early_mem); | |
186 | ||
c1cc1552 CM |
187 | void __init arm64_memblock_init(void) |
188 | { | |
88053ec8 AB |
189 | s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); |
190 | ||
191 | /* | |
192 | * Corner case: 52-bit VA capable systems running KVM in nVHE mode may | |
193 | * be limited in their ability to support a linear map that exceeds 51 | |
194 | * bits of VA space, depending on the placement of the ID map. Given | |
195 | * that the placement of the ID map may be randomized, let's simply | |
196 | * limit the kernel's linear map to 51 bits as well if we detect this | |
197 | * configuration. | |
198 | */ | |
199 | if (IS_ENABLED(CONFIG_KVM) && vabits_actual == 52 && | |
200 | is_hyp_mode_available() && !is_kernel_in_hyp_mode()) { | |
201 | pr_info("Capping linear region to 51 bits for KVM in nVHE mode on LVA capable hardware.\n"); | |
202 | linear_region_size = min_t(u64, linear_region_size, BIT(51)); | |
203 | } | |
a7f8de16 | 204 | |
e9eaa805 KM |
205 | /* Remove memory above our supported physical address size */ |
206 | memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX); | |
207 | ||
a7f8de16 AB |
208 | /* |
209 | * Select a suitable value for the base of physical memory. | |
210 | */ | |
211 | memstart_addr = round_down(memblock_start_of_DRAM(), | |
212 | ARM64_MEMSTART_ALIGN); | |
213 | ||
31f80a4e MZ |
214 | if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size) |
215 | pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n"); | |
216 | ||
a7f8de16 AB |
217 | /* |
218 | * Remove the memory that we will not be able to cover with the | |
219 | * linear mapping. Take care not to clip the kernel which may be | |
220 | * high in memory. | |
221 | */ | |
2077be67 LA |
222 | memblock_remove(max_t(u64, memstart_addr + linear_region_size, |
223 | __pa_symbol(_end)), ULLONG_MAX); | |
2958987f AB |
224 | if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) { |
225 | /* ensure that memstart_addr remains sufficiently aligned */ | |
226 | memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size, | |
227 | ARM64_MEMSTART_ALIGN); | |
228 | memblock_remove(0, memstart_addr); | |
229 | } | |
a7f8de16 | 230 | |
7bc1a0f9 AB |
231 | /* |
232 | * If we are running with a 52-bit kernel VA config on a system that | |
233 | * does not support it, we have to place the available physical | |
234 | * memory in the 48-bit addressable part of the linear region, i.e., | |
235 | * we have to move it upward. Since memstart_addr represents the | |
236 | * physical address of PAGE_OFFSET, we have to *subtract* from it. | |
237 | */ | |
238 | if (IS_ENABLED(CONFIG_ARM64_VA_BITS_52) && (vabits_actual != 52)) | |
9684ec18 | 239 | memstart_addr -= _PAGE_OFFSET(vabits_actual) - _PAGE_OFFSET(52); |
7bc1a0f9 | 240 | |
a7f8de16 AB |
241 | /* |
242 | * Apply the memory limit if it was set. Since the kernel may be loaded | |
243 | * high up in memory, add back the kernel region that must be accessible | |
244 | * via the linear mapping. | |
245 | */ | |
d7dc899a | 246 | if (memory_limit != PHYS_ADDR_MAX) { |
cb0a6502 | 247 | memblock_mem_limit_remove_map(memory_limit); |
2077be67 | 248 | memblock_add(__pa_symbol(_text), (u64)(_end - _text)); |
a7f8de16 | 249 | } |
6083fe74 | 250 | |
c756c592 | 251 | if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { |
177e15f0 AB |
252 | /* |
253 | * Add back the memory we just removed if it results in the | |
254 | * initrd to become inaccessible via the linear mapping. | |
255 | * Otherwise, this is a no-op | |
256 | */ | |
c756c592 | 257 | u64 base = phys_initrd_start & PAGE_MASK; |
d4d18e3e | 258 | u64 size = PAGE_ALIGN(phys_initrd_start + phys_initrd_size) - base; |
177e15f0 AB |
259 | |
260 | /* | |
261 | * We can only add back the initrd memory if we don't end up | |
262 | * with more memory than we can address via the linear mapping. | |
263 | * It is up to the bootloader to position the kernel and the | |
264 | * initrd reasonably close to each other (i.e., within 32 GB of | |
265 | * each other) so that all granule/#levels combinations can | |
266 | * always access both. | |
267 | */ | |
268 | if (WARN(base < memblock_start_of_DRAM() || | |
269 | base + size > memblock_start_of_DRAM() + | |
270 | linear_region_size, | |
271 | "initrd not fully accessible via the linear mapping -- please check your bootloader ...\n")) { | |
70b3d237 | 272 | phys_initrd_size = 0; |
177e15f0 | 273 | } else { |
177e15f0 | 274 | memblock_add(base, size); |
c0b978fe | 275 | memblock_clear_nomap(base, size); |
177e15f0 AB |
276 | memblock_reserve(base, size); |
277 | } | |
278 | } | |
279 | ||
c031a421 AB |
280 | if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { |
281 | extern u16 memstart_offset_seed; | |
97d6786e AB |
282 | u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); |
283 | int parange = cpuid_feature_extract_unsigned_field( | |
2d987e64 | 284 | mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT); |
97d6786e AB |
285 | s64 range = linear_region_size - |
286 | BIT(id_aa64mmfr0_parange_to_phys_shift(parange)); | |
c031a421 AB |
287 | |
288 | /* | |
289 | * If the size of the linear region exceeds, by a sufficient | |
97d6786e AB |
290 | * margin, the size of the region that the physical memory can |
291 | * span, randomize the linear region as well. | |
c031a421 | 292 | */ |
97d6786e | 293 | if (memstart_offset_seed > 0 && range >= (s64)ARM64_MEMSTART_ALIGN) { |
c8a43c18 | 294 | range /= ARM64_MEMSTART_ALIGN; |
c031a421 AB |
295 | memstart_addr -= ARM64_MEMSTART_ALIGN * |
296 | ((range * memstart_offset_seed) >> 16); | |
297 | } | |
298 | } | |
6083fe74 | 299 | |
bd00cd5f MR |
300 | /* |
301 | * Register the kernel text, kernel data, initrd, and initial | |
302 | * pagetables with memblock. | |
303 | */ | |
e2a073dd | 304 | memblock_reserve(__pa_symbol(_stext), _end - _stext); |
c756c592 | 305 | if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) { |
a89dea58 | 306 | /* the generic initrd code expects virtual addresses */ |
c756c592 FF |
307 | initrd_start = __phys_to_virt(phys_initrd_start); |
308 | initrd_end = initrd_start + phys_initrd_size; | |
a89dea58 | 309 | } |
c1cc1552 | 310 | |
0ceac9e0 | 311 | early_init_fdt_scan_reserved_mem(); |
2d5a5612 | 312 | |
f24e5834 | 313 | high_memory = __va(memblock_end_of_DRAM() - 1) + 1; |
c1cc1552 CM |
314 | } |
315 | ||
316 | void __init bootmem_init(void) | |
317 | { | |
318 | unsigned long min, max; | |
319 | ||
320 | min = PFN_UP(memblock_start_of_DRAM()); | |
321 | max = PFN_DOWN(memblock_end_of_DRAM()); | |
322 | ||
36dd9086 VM |
323 | early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT); |
324 | ||
1a2db300 | 325 | max_pfn = max_low_pfn = max; |
19d6242e | 326 | min_low_pfn = min; |
1a2db300 | 327 | |
eb75541f | 328 | arch_numa_init(); |
618e0786 BS |
329 | |
330 | /* | |
eb75541f | 331 | * must be done after arch_numa_init() which calls numa_init() to |
618e0786 BS |
332 | * initialize node_online_map that gets used in hugetlb_cma_reserve() |
333 | * while allocating required CMA size across online nodes. | |
334 | */ | |
abb7962a AK |
335 | #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA) |
336 | arm64_hugetlb_cma_reserve(); | |
618e0786 BS |
337 | #endif |
338 | ||
f320bc74 QP |
339 | kvm_hyp_reserve(); |
340 | ||
c1cc1552 | 341 | /* |
c89ab04f MR |
342 | * sparse_init() tries to allocate memory from memblock, so must be |
343 | * done after the fixed reservations | |
c1cc1552 | 344 | */ |
c1cc1552 | 345 | sparse_init(); |
f41ef4c2 | 346 | zone_sizes_init(); |
c1cc1552 | 347 | |
d78050ee CM |
348 | /* |
349 | * Reserve the CMA area after arm64_dma_phys_limit was initialised. | |
350 | */ | |
351 | dma_contiguous_reserve(arm64_dma_phys_limit); | |
352 | ||
0a30c535 NSJ |
353 | /* |
354 | * request_standard_resources() depends on crashkernel's memory being | |
355 | * reserved, so do it here. | |
356 | */ | |
fdc26823 | 357 | arch_reserve_crashkernel(); |
0a30c535 | 358 | |
1a2db300 | 359 | memblock_dump_all(); |
c1cc1552 CM |
360 | } |
361 | ||
c1cc1552 CM |
362 | /* |
363 | * mem_init() marks the free areas in the mem_map and tells us how much memory | |
364 | * is free. This is done after various parts of the system have claimed their | |
365 | * memory after the kernel image. | |
366 | */ | |
367 | void __init mem_init(void) | |
368 | { | |
1c1a429e CM |
369 | bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit); |
370 | ||
65033574 CM |
371 | if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) { |
372 | /* | |
373 | * If no bouncing needed for ZONE_DMA, reduce the swiotlb | |
374 | * buffer for kmalloc() bouncing to 1MB per 1GB of RAM. | |
375 | */ | |
376 | unsigned long size = | |
377 | DIV_ROUND_UP(memblock_phys_mem_size(), 1024); | |
378 | swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); | |
1c1a429e | 379 | swiotlb = true; |
65033574 | 380 | } |
1c1a429e CM |
381 | |
382 | swiotlb_init(swiotlb, SWIOTLB_VERBOSE); | |
a1e50a82 | 383 | |
bee4ebd1 | 384 | /* this will put all unused low memory onto the freelists */ |
c6ffc5ca | 385 | memblock_free_all(); |
c1cc1552 | 386 | |
c1cc1552 CM |
387 | /* |
388 | * Check boundaries twice: Some fundamental inconsistencies can be | |
389 | * detected at build time already. | |
390 | */ | |
391 | #ifdef CONFIG_COMPAT | |
363524d2 | 392 | BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64); |
c1cc1552 | 393 | #endif |
c1cc1552 | 394 | |
7e04cc91 AK |
395 | /* |
396 | * Selected page table levels should match when derived from | |
397 | * scratch using the virtual address range and page size. | |
398 | */ | |
399 | BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != | |
400 | CONFIG_PGTABLE_LEVELS); | |
401 | ||
bee4ebd1 | 402 | if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { |
c1cc1552 CM |
403 | extern int sysctl_overcommit_memory; |
404 | /* | |
405 | * On a machine this small we won't get anywhere without | |
406 | * overcommit, so turn it on by default. | |
407 | */ | |
408 | sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; | |
409 | } | |
410 | } | |
411 | ||
412 | void free_initmem(void) | |
413 | { | |
c02e7c5c JG |
414 | void *lm_init_begin = lm_alias(__init_begin); |
415 | void *lm_init_end = lm_alias(__init_end); | |
416 | ||
417 | WARN_ON(!IS_ALIGNED((unsigned long)lm_init_begin, PAGE_SIZE)); | |
418 | WARN_ON(!IS_ALIGNED((unsigned long)lm_init_end, PAGE_SIZE)); | |
1db9716d RQ |
419 | |
420 | /* Delete __init region from memblock.reserved. */ | |
c02e7c5c | 421 | memblock_free(lm_init_begin, lm_init_end - lm_init_begin); |
1db9716d | 422 | |
c02e7c5c | 423 | free_reserved_area(lm_init_begin, lm_init_end, |
6ec939f8 | 424 | POISON_FREE_INITMEM, "unused kernel"); |
dae8c235 KW |
425 | /* |
426 | * Unmap the __init region but leave the VM area in place. This | |
427 | * prevents the region from being reused for kernel modules, which | |
428 | * is not supported by kallsyms. | |
429 | */ | |
4ad0ae8c | 430 | vunmap_range((u64)__init_begin, (u64)__init_end); |
c1cc1552 CM |
431 | } |
432 | ||
638d5031 | 433 | void dump_mem_limit(void) |
a7f8de16 | 434 | { |
d7dc899a | 435 | if (memory_limit != PHYS_ADDR_MAX) { |
a7f8de16 AB |
436 | pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20); |
437 | } else { | |
438 | pr_emerg("Memory Limit: none\n"); | |
439 | } | |
a7f8de16 | 440 | } |
0cc2dc49 MRI |
441 | |
442 | #ifdef CONFIG_EXECMEM | |
443 | static u64 module_direct_base __ro_after_init = 0; | |
444 | static u64 module_plt_base __ro_after_init = 0; | |
445 | ||
446 | /* | |
447 | * Choose a random page-aligned base address for a window of 'size' bytes which | |
448 | * entirely contains the interval [start, end - 1]. | |
449 | */ | |
450 | static u64 __init random_bounding_box(u64 size, u64 start, u64 end) | |
451 | { | |
452 | u64 max_pgoff, pgoff; | |
453 | ||
454 | if ((end - start) >= size) | |
455 | return 0; | |
456 | ||
457 | max_pgoff = (size - (end - start)) / PAGE_SIZE; | |
458 | pgoff = get_random_u32_inclusive(0, max_pgoff); | |
459 | ||
460 | return start - pgoff * PAGE_SIZE; | |
461 | } | |
462 | ||
463 | /* | |
464 | * Modules may directly reference data and text anywhere within the kernel | |
465 | * image and other modules. References using PREL32 relocations have a +/-2G | |
466 | * range, and so we need to ensure that the entire kernel image and all modules | |
467 | * fall within a 2G window such that these are always within range. | |
468 | * | |
469 | * Modules may directly branch to functions and code within the kernel text, | |
470 | * and to functions and code within other modules. These branches will use | |
471 | * CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure | |
472 | * that the entire kernel text and all module text falls within a 128M window | |
473 | * such that these are always within range. With PLTs, we can expand this to a | |
474 | * 2G window. | |
475 | * | |
476 | * We chose the 128M region to surround the entire kernel image (rather than | |
477 | * just the text) as using the same bounds for the 128M and 2G regions ensures | |
478 | * by construction that we never select a 128M region that is not a subset of | |
479 | * the 2G region. For very large and unusual kernel configurations this means | |
480 | * we may fall back to PLTs where they could have been avoided, but this keeps | |
481 | * the logic significantly simpler. | |
482 | */ | |
483 | static int __init module_init_limits(void) | |
484 | { | |
485 | u64 kernel_end = (u64)_end; | |
486 | u64 kernel_start = (u64)_text; | |
487 | u64 kernel_size = kernel_end - kernel_start; | |
488 | ||
489 | /* | |
490 | * The default modules region is placed immediately below the kernel | |
491 | * image, and is large enough to use the full 2G relocation range. | |
492 | */ | |
493 | BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END); | |
494 | BUILD_BUG_ON(MODULES_VSIZE < SZ_2G); | |
495 | ||
496 | if (!kaslr_enabled()) { | |
497 | if (kernel_size < SZ_128M) | |
498 | module_direct_base = kernel_end - SZ_128M; | |
499 | if (kernel_size < SZ_2G) | |
500 | module_plt_base = kernel_end - SZ_2G; | |
501 | } else { | |
502 | u64 min = kernel_start; | |
503 | u64 max = kernel_end; | |
504 | ||
505 | if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { | |
506 | pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n"); | |
507 | } else { | |
508 | module_direct_base = random_bounding_box(SZ_128M, min, max); | |
509 | if (module_direct_base) { | |
510 | min = module_direct_base; | |
511 | max = module_direct_base + SZ_128M; | |
512 | } | |
513 | } | |
514 | ||
515 | module_plt_base = random_bounding_box(SZ_2G, min, max); | |
516 | } | |
517 | ||
518 | pr_info("%llu pages in range for non-PLT usage", | |
519 | module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0); | |
520 | pr_info("%llu pages in range for PLT usage", | |
521 | module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0); | |
522 | ||
523 | return 0; | |
524 | } | |
525 | ||
526 | static struct execmem_info execmem_info __ro_after_init; | |
527 | ||
528 | struct execmem_info __init *execmem_arch_setup(void) | |
529 | { | |
530 | unsigned long fallback_start = 0, fallback_end = 0; | |
531 | unsigned long start = 0, end = 0; | |
532 | ||
533 | module_init_limits(); | |
534 | ||
535 | /* | |
536 | * Where possible, prefer to allocate within direct branch range of the | |
537 | * kernel such that no PLTs are necessary. | |
538 | */ | |
539 | if (module_direct_base) { | |
540 | start = module_direct_base; | |
541 | end = module_direct_base + SZ_128M; | |
542 | ||
543 | if (module_plt_base) { | |
544 | fallback_start = module_plt_base; | |
545 | fallback_end = module_plt_base + SZ_2G; | |
546 | } | |
547 | } else if (module_plt_base) { | |
548 | start = module_plt_base; | |
549 | end = module_plt_base + SZ_2G; | |
550 | } | |
551 | ||
552 | execmem_info = (struct execmem_info){ | |
553 | .ranges = { | |
554 | [EXECMEM_DEFAULT] = { | |
555 | .start = start, | |
556 | .end = end, | |
557 | .pgprot = PAGE_KERNEL, | |
558 | .alignment = 1, | |
559 | .fallback_start = fallback_start, | |
560 | .fallback_end = fallback_end, | |
561 | }, | |
562 | [EXECMEM_KPROBES] = { | |
563 | .start = VMALLOC_START, | |
564 | .end = VMALLOC_END, | |
565 | .pgprot = PAGE_KERNEL_ROX, | |
566 | .alignment = 1, | |
567 | }, | |
568 | [EXECMEM_BPF] = { | |
569 | .start = VMALLOC_START, | |
570 | .end = VMALLOC_END, | |
571 | .pgprot = PAGE_KERNEL, | |
572 | .alignment = 1, | |
573 | }, | |
574 | }, | |
575 | }; | |
576 | ||
577 | return &execmem_info; | |
578 | } | |
579 | #endif /* CONFIG_EXECMEM */ |