Commit | Line | Data |
---|---|---|
50acfb2b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
76d2a049 PD |
2 | /* |
3 | * Copyright (C) 2012 Regents of the University of California | |
671f9a3e | 4 | * Copyright (C) 2019 Western Digital Corporation or its affiliates. |
76d2a049 PD |
5 | */ |
6 | ||
7 | #include <linux/init.h> | |
8 | #include <linux/mm.h> | |
76d2a049 | 9 | #include <linux/memblock.h> |
57c8a661 | 10 | #include <linux/initrd.h> |
76d2a049 | 11 | #include <linux/swap.h> |
5ec9c4ff | 12 | #include <linux/sizes.h> |
0651c263 | 13 | #include <linux/of_fdt.h> |
922b0375 | 14 | #include <linux/libfdt.h> |
d27c3c90 | 15 | #include <linux/set_memory.h> |
76d2a049 | 16 | |
f2c17aab | 17 | #include <asm/fixmap.h> |
76d2a049 PD |
18 | #include <asm/tlbflush.h> |
19 | #include <asm/sections.h> | |
2d268251 | 20 | #include <asm/soc.h> |
76d2a049 | 21 | #include <asm/io.h> |
b422d28b | 22 | #include <asm/ptdump.h> |
76d2a049 | 23 | |
ffaee272 PW |
24 | #include "../kernel/head.h" |
25 | ||
387181dc AP |
26 | unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] |
27 | __page_aligned_bss; | |
28 | EXPORT_SYMBOL(empty_zero_page); | |
29 | ||
d90d45d7 | 30 | extern char _start[]; |
6bd33e1e | 31 | void *dtb_early_va; |
d90d45d7 | 32 | |
76d2a049 PD |
33 | static void __init zone_sizes_init(void) |
34 | { | |
5ec9c4ff | 35 | unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; |
76d2a049 | 36 | |
d5fad48c | 37 | #ifdef CONFIG_ZONE_DMA32 |
28198c46 GR |
38 | max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, |
39 | (unsigned long) PFN_PHYS(max_low_pfn))); | |
d5fad48c | 40 | #endif |
5ec9c4ff CH |
41 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
42 | ||
9691a071 | 43 | free_area_init(max_zone_pfns); |
76d2a049 PD |
44 | } |
45 | ||
6bd33e1e | 46 | static void setup_zero_page(void) |
76d2a049 PD |
47 | { |
48 | memset((void *)empty_zero_page, 0, PAGE_SIZE); | |
49 | } | |
50 | ||
8fa3cdff | 51 | #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) |
2cc6c4a0 YS |
52 | static inline void print_mlk(char *name, unsigned long b, unsigned long t) |
53 | { | |
54 | pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, | |
55 | (((t) - (b)) >> 10)); | |
56 | } | |
57 | ||
58 | static inline void print_mlm(char *name, unsigned long b, unsigned long t) | |
59 | { | |
60 | pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, | |
61 | (((t) - (b)) >> 20)); | |
62 | } | |
63 | ||
64 | static void print_vm_layout(void) | |
65 | { | |
66 | pr_notice("Virtual kernel memory layout:\n"); | |
67 | print_mlk("fixmap", (unsigned long)FIXADDR_START, | |
68 | (unsigned long)FIXADDR_TOP); | |
69 | print_mlm("pci io", (unsigned long)PCI_IO_START, | |
70 | (unsigned long)PCI_IO_END); | |
71 | print_mlm("vmemmap", (unsigned long)VMEMMAP_START, | |
72 | (unsigned long)VMEMMAP_END); | |
73 | print_mlm("vmalloc", (unsigned long)VMALLOC_START, | |
74 | (unsigned long)VMALLOC_END); | |
75 | print_mlm("lowmem", (unsigned long)PAGE_OFFSET, | |
76 | (unsigned long)high_memory); | |
77 | } | |
78 | #else | |
79 | static void print_vm_layout(void) { } | |
80 | #endif /* CONFIG_DEBUG_VM */ | |
81 | ||
76d2a049 PD |
82 | void __init mem_init(void) |
83 | { | |
84 | #ifdef CONFIG_FLATMEM | |
85 | BUG_ON(!mem_map); | |
86 | #endif /* CONFIG_FLATMEM */ | |
87 | ||
88 | high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); | |
c6ffc5ca | 89 | memblock_free_all(); |
76d2a049 PD |
90 | |
91 | mem_init_print_info(NULL); | |
2cc6c4a0 | 92 | print_vm_layout(); |
76d2a049 PD |
93 | } |
94 | ||
76d2a049 | 95 | #ifdef CONFIG_BLK_DEV_INITRD |
0651c263 AP |
96 | static void __init setup_initrd(void) |
97 | { | |
4400231c | 98 | phys_addr_t start; |
0651c263 AP |
99 | unsigned long size; |
100 | ||
4400231c AP |
101 | /* Ignore the virtul address computed during device tree parsing */ |
102 | initrd_start = initrd_end = 0; | |
103 | ||
104 | if (!phys_initrd_size) | |
105 | return; | |
106 | /* | |
107 | * Round the memory region to page boundaries as per free_initrd_mem() | |
108 | * This allows us to detect whether the pages overlapping the initrd | |
109 | * are in use, but more importantly, reserves the entire set of pages | |
110 | * as we don't want these pages allocated for other purposes. | |
111 | */ | |
112 | start = round_down(phys_initrd_start, PAGE_SIZE); | |
113 | size = phys_initrd_size + (phys_initrd_start - start); | |
114 | size = round_up(size, PAGE_SIZE); | |
115 | ||
116 | if (!memblock_is_region_memory(start, size)) { | |
117 | pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region", | |
118 | (u64)start, size); | |
0651c263 AP |
119 | goto disable; |
120 | } | |
4400231c AP |
121 | |
122 | if (memblock_is_region_reserved(start, size)) { | |
123 | pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region\n", | |
124 | (u64)start, size); | |
0651c263 AP |
125 | goto disable; |
126 | } | |
127 | ||
4400231c AP |
128 | memblock_reserve(start, size); |
129 | /* Now convert initrd to virtual addresses */ | |
130 | initrd_start = (unsigned long)__va(phys_initrd_start); | |
131 | initrd_end = initrd_start + phys_initrd_size; | |
0651c263 AP |
132 | initrd_below_start_ok = 1; |
133 | ||
134 | pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", | |
135 | (void *)(initrd_start), size); | |
136 | return; | |
137 | disable: | |
138 | pr_cont(" - disabling initrd\n"); | |
139 | initrd_start = 0; | |
140 | initrd_end = 0; | |
141 | } | |
76d2a049 | 142 | #endif /* CONFIG_BLK_DEV_INITRD */ |
0651c263 | 143 | |
922b0375 AO |
144 | static phys_addr_t dtb_early_pa __initdata; |
145 | ||
0651c263 AP |
146 | void __init setup_bootmem(void) |
147 | { | |
0651c263 | 148 | phys_addr_t mem_size = 0; |
fa5a1983 | 149 | phys_addr_t total_mem = 0; |
b10d6bca | 150 | phys_addr_t mem_start, start, end = 0; |
ac51e005 ZL |
151 | phys_addr_t vmlinux_end = __pa_symbol(&_end); |
152 | phys_addr_t vmlinux_start = __pa_symbol(&_start); | |
b10d6bca | 153 | u64 i; |
0651c263 AP |
154 | |
155 | /* Find the memory region containing the kernel */ | |
b10d6bca MR |
156 | for_each_mem_range(i, &start, &end) { |
157 | phys_addr_t size = end - start; | |
fa5a1983 | 158 | if (!total_mem) |
b10d6bca MR |
159 | mem_start = start; |
160 | if (start <= vmlinux_start && vmlinux_end <= end) | |
161 | BUG_ON(size == 0); | |
162 | total_mem = total_mem + size; | |
0651c263 | 163 | } |
fa5a1983 AP |
164 | |
165 | /* | |
166 | * Remove memblock from the end of usable area to the | |
167 | * end of region | |
168 | */ | |
169 | mem_size = min(total_mem, (phys_addr_t)-PAGE_OFFSET); | |
170 | if (mem_start + mem_size < end) | |
171 | memblock_remove(mem_start + mem_size, | |
172 | end - mem_start - mem_size); | |
0651c263 | 173 | |
d90d45d7 AP |
174 | /* Reserve from the start of the kernel to the end of the kernel */ |
175 | memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); | |
176 | ||
c749bb2d VC |
177 | max_pfn = PFN_DOWN(memblock_end_of_DRAM()); |
178 | max_low_pfn = max_pfn; | |
d0d8aae6 | 179 | set_max_mapnr(max_low_pfn); |
0651c263 AP |
180 | |
181 | #ifdef CONFIG_BLK_DEV_INITRD | |
182 | setup_initrd(); | |
183 | #endif /* CONFIG_BLK_DEV_INITRD */ | |
184 | ||
922b0375 AO |
185 | /* |
186 | * Avoid using early_init_fdt_reserve_self() since __pa() does | |
187 | * not work for DTB pointers that are fixmap addresses | |
188 | */ | |
189 | memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); | |
190 | ||
0651c263 AP |
191 | early_init_fdt_scan_reserved_mem(); |
192 | memblock_allow_resize(); | |
193 | memblock_dump_all(); | |
0651c263 | 194 | } |
6f1e9e94 | 195 | |
6bd33e1e | 196 | #ifdef CONFIG_MMU |
387181dc AP |
197 | unsigned long va_pa_offset; |
198 | EXPORT_SYMBOL(va_pa_offset); | |
199 | unsigned long pfn_base; | |
200 | EXPORT_SYMBOL(pfn_base); | |
201 | ||
6f1e9e94 | 202 | pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
671f9a3e AP |
203 | pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
204 | pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; | |
205 | static bool mmu_enabled; | |
6f1e9e94 | 206 | |
671f9a3e | 207 | #define MAX_EARLY_MAPPING_SIZE SZ_128M |
6f1e9e94 | 208 | |
671f9a3e | 209 | pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); |
f2c17aab AP |
210 | |
211 | void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) | |
212 | { | |
213 | unsigned long addr = __fix_to_virt(idx); | |
214 | pte_t *ptep; | |
215 | ||
216 | BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); | |
217 | ||
218 | ptep = &fixmap_pte[pte_index(addr)]; | |
219 | ||
21190b74 | 220 | if (pgprot_val(prot)) |
f2c17aab | 221 | set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); |
21190b74 | 222 | else |
f2c17aab | 223 | pte_clear(&init_mm, addr, ptep); |
21190b74 | 224 | local_flush_tlb_page(addr); |
f2c17aab AP |
225 | } |
226 | ||
671f9a3e AP |
227 | static pte_t *__init get_pte_virt(phys_addr_t pa) |
228 | { | |
229 | if (mmu_enabled) { | |
230 | clear_fixmap(FIX_PTE); | |
231 | return (pte_t *)set_fixmap_offset(FIX_PTE, pa); | |
232 | } else { | |
233 | return (pte_t *)((uintptr_t)pa); | |
234 | } | |
235 | } | |
236 | ||
237 | static phys_addr_t __init alloc_pte(uintptr_t va) | |
238 | { | |
239 | /* | |
240 | * We only create PMD or PGD early mappings so we | |
241 | * should never reach here with MMU disabled. | |
242 | */ | |
243 | BUG_ON(!mmu_enabled); | |
244 | ||
245 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); | |
246 | } | |
247 | ||
248 | static void __init create_pte_mapping(pte_t *ptep, | |
249 | uintptr_t va, phys_addr_t pa, | |
250 | phys_addr_t sz, pgprot_t prot) | |
251 | { | |
974b9b2c | 252 | uintptr_t pte_idx = pte_index(va); |
671f9a3e AP |
253 | |
254 | BUG_ON(sz != PAGE_SIZE); | |
255 | ||
974b9b2c MR |
256 | if (pte_none(ptep[pte_idx])) |
257 | ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); | |
671f9a3e AP |
258 | } |
259 | ||
260 | #ifndef __PAGETABLE_PMD_FOLDED | |
261 | ||
262 | pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
263 | pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
264 | ||
265 | #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE | |
266 | #define NUM_EARLY_PMDS 1UL | |
267 | #else | |
268 | #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) | |
269 | #endif | |
270 | pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); | |
271 | ||
272 | static pmd_t *__init get_pmd_virt(phys_addr_t pa) | |
273 | { | |
274 | if (mmu_enabled) { | |
275 | clear_fixmap(FIX_PMD); | |
276 | return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); | |
277 | } else { | |
278 | return (pmd_t *)((uintptr_t)pa); | |
279 | } | |
280 | } | |
281 | ||
282 | static phys_addr_t __init alloc_pmd(uintptr_t va) | |
283 | { | |
284 | uintptr_t pmd_num; | |
285 | ||
286 | if (mmu_enabled) | |
287 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); | |
288 | ||
289 | pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; | |
290 | BUG_ON(pmd_num >= NUM_EARLY_PMDS); | |
291 | return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; | |
292 | } | |
293 | ||
294 | static void __init create_pmd_mapping(pmd_t *pmdp, | |
295 | uintptr_t va, phys_addr_t pa, | |
296 | phys_addr_t sz, pgprot_t prot) | |
297 | { | |
298 | pte_t *ptep; | |
299 | phys_addr_t pte_phys; | |
974b9b2c | 300 | uintptr_t pmd_idx = pmd_index(va); |
671f9a3e AP |
301 | |
302 | if (sz == PMD_SIZE) { | |
974b9b2c MR |
303 | if (pmd_none(pmdp[pmd_idx])) |
304 | pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); | |
671f9a3e AP |
305 | return; |
306 | } | |
307 | ||
974b9b2c | 308 | if (pmd_none(pmdp[pmd_idx])) { |
671f9a3e | 309 | pte_phys = alloc_pte(va); |
974b9b2c | 310 | pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); |
671f9a3e AP |
311 | ptep = get_pte_virt(pte_phys); |
312 | memset(ptep, 0, PAGE_SIZE); | |
313 | } else { | |
974b9b2c | 314 | pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); |
671f9a3e AP |
315 | ptep = get_pte_virt(pte_phys); |
316 | } | |
317 | ||
318 | create_pte_mapping(ptep, va, pa, sz, prot); | |
319 | } | |
320 | ||
321 | #define pgd_next_t pmd_t | |
322 | #define alloc_pgd_next(__va) alloc_pmd(__va) | |
323 | #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) | |
324 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ | |
325 | create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) | |
671f9a3e AP |
326 | #define fixmap_pgd_next fixmap_pmd |
327 | #else | |
328 | #define pgd_next_t pte_t | |
329 | #define alloc_pgd_next(__va) alloc_pte(__va) | |
330 | #define get_pgd_next_virt(__pa) get_pte_virt(__pa) | |
331 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ | |
332 | create_pte_mapping(__nextp, __va, __pa, __sz, __prot) | |
671f9a3e AP |
333 | #define fixmap_pgd_next fixmap_pte |
334 | #endif | |
335 | ||
336 | static void __init create_pgd_mapping(pgd_t *pgdp, | |
337 | uintptr_t va, phys_addr_t pa, | |
338 | phys_addr_t sz, pgprot_t prot) | |
339 | { | |
340 | pgd_next_t *nextp; | |
341 | phys_addr_t next_phys; | |
974b9b2c | 342 | uintptr_t pgd_idx = pgd_index(va); |
671f9a3e AP |
343 | |
344 | if (sz == PGDIR_SIZE) { | |
974b9b2c MR |
345 | if (pgd_val(pgdp[pgd_idx]) == 0) |
346 | pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); | |
671f9a3e AP |
347 | return; |
348 | } | |
349 | ||
974b9b2c | 350 | if (pgd_val(pgdp[pgd_idx]) == 0) { |
671f9a3e | 351 | next_phys = alloc_pgd_next(va); |
974b9b2c | 352 | pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); |
671f9a3e AP |
353 | nextp = get_pgd_next_virt(next_phys); |
354 | memset(nextp, 0, PAGE_SIZE); | |
355 | } else { | |
974b9b2c | 356 | next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); |
671f9a3e AP |
357 | nextp = get_pgd_next_virt(next_phys); |
358 | } | |
359 | ||
360 | create_pgd_next_mapping(nextp, va, pa, sz, prot); | |
361 | } | |
362 | ||
363 | static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) | |
364 | { | |
0fdc636c ZL |
365 | /* Upgrade to PMD_SIZE mappings whenever possible */ |
366 | if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) | |
367 | return PAGE_SIZE; | |
671f9a3e | 368 | |
0fdc636c | 369 | return PMD_SIZE; |
671f9a3e AP |
370 | } |
371 | ||
387181dc AP |
372 | /* |
373 | * setup_vm() is called from head.S with MMU-off. | |
374 | * | |
375 | * Following requirements should be honoured for setup_vm() to work | |
376 | * correctly: | |
377 | * 1) It should use PC-relative addressing for accessing kernel symbols. | |
378 | * To achieve this we always use GCC cmodel=medany. | |
379 | * 2) The compiler instrumentation for FTRACE will not work for setup_vm() | |
380 | * so disable compiler instrumentation when FTRACE is enabled. | |
381 | * | |
382 | * Currently, the above requirements are honoured by using custom CFLAGS | |
383 | * for init.o in mm/Makefile. | |
384 | */ | |
385 | ||
386 | #ifndef __riscv_cmodel_medany | |
6a527b67 | 387 | #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." |
387181dc AP |
388 | #endif |
389 | ||
671f9a3e | 390 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) |
6f1e9e94 | 391 | { |
671f9a3e AP |
392 | uintptr_t va, end_va; |
393 | uintptr_t load_pa = (uintptr_t)(&_start); | |
394 | uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; | |
395 | uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); | |
396 | ||
397 | va_pa_offset = PAGE_OFFSET - load_pa; | |
398 | pfn_base = PFN_DOWN(load_pa); | |
6f1e9e94 | 399 | |
671f9a3e AP |
400 | /* |
401 | * Enforce boot alignment requirements of RV32 and | |
402 | * RV64 by only allowing PMD or PGD mappings. | |
403 | */ | |
404 | BUG_ON(map_size == PAGE_SIZE); | |
6f1e9e94 AP |
405 | |
406 | /* Sanity check alignment and size */ | |
407 | BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); | |
671f9a3e AP |
408 | BUG_ON((load_pa % map_size) != 0); |
409 | BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); | |
410 | ||
411 | /* Setup early PGD for fixmap */ | |
412 | create_pgd_mapping(early_pg_dir, FIXADDR_START, | |
413 | (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); | |
6f1e9e94 AP |
414 | |
415 | #ifndef __PAGETABLE_PMD_FOLDED | |
671f9a3e AP |
416 | /* Setup fixmap PMD */ |
417 | create_pmd_mapping(fixmap_pmd, FIXADDR_START, | |
418 | (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); | |
419 | /* Setup trampoline PGD and PMD */ | |
420 | create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, | |
421 | (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); | |
422 | create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, | |
423 | load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); | |
424 | #else | |
425 | /* Setup trampoline PGD */ | |
426 | create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, | |
427 | load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); | |
428 | #endif | |
6f1e9e94 | 429 | |
671f9a3e AP |
430 | /* |
431 | * Setup early PGD covering entire kernel which will allows | |
432 | * us to reach paging_init(). We map all memory banks later | |
433 | * in setup_vm_final() below. | |
434 | */ | |
435 | end_va = PAGE_OFFSET + load_sz; | |
436 | for (va = PAGE_OFFSET; va < end_va; va += map_size) | |
437 | create_pgd_mapping(early_pg_dir, va, | |
438 | load_pa + (va - PAGE_OFFSET), | |
439 | map_size, PAGE_KERNEL_EXEC); | |
440 | ||
441 | /* Create fixed mapping for early FDT parsing */ | |
442 | end_va = __fix_to_virt(FIX_FDT) + FIX_FDT_SIZE; | |
443 | for (va = __fix_to_virt(FIX_FDT); va < end_va; va += PAGE_SIZE) | |
444 | create_pte_mapping(fixmap_pte, va, | |
445 | dtb_pa + (va - __fix_to_virt(FIX_FDT)), | |
446 | PAGE_SIZE, PAGE_KERNEL); | |
447 | ||
448 | /* Save pointer to DTB for early FDT parsing */ | |
449 | dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK); | |
922b0375 AO |
450 | /* Save physical address for memblock reservation */ |
451 | dtb_early_pa = dtb_pa; | |
671f9a3e | 452 | } |
f2c17aab | 453 | |
671f9a3e AP |
454 | static void __init setup_vm_final(void) |
455 | { | |
456 | uintptr_t va, map_size; | |
457 | phys_addr_t pa, start, end; | |
b10d6bca | 458 | u64 i; |
671f9a3e AP |
459 | |
460 | /* Set mmu_enabled flag */ | |
461 | mmu_enabled = true; | |
6f1e9e94 | 462 | |
671f9a3e AP |
463 | /* Setup swapper PGD for fixmap */ |
464 | create_pgd_mapping(swapper_pg_dir, FIXADDR_START, | |
ac51e005 | 465 | __pa_symbol(fixmap_pgd_next), |
671f9a3e | 466 | PGDIR_SIZE, PAGE_TABLE); |
f2c17aab | 467 | |
671f9a3e | 468 | /* Map all memory banks */ |
b10d6bca | 469 | for_each_mem_range(i, &start, &end) { |
671f9a3e AP |
470 | if (start >= end) |
471 | break; | |
671f9a3e AP |
472 | if (start <= __pa(PAGE_OFFSET) && |
473 | __pa(PAGE_OFFSET) < end) | |
474 | start = __pa(PAGE_OFFSET); | |
475 | ||
476 | map_size = best_map_size(start, end - start); | |
477 | for (pa = start; pa < end; pa += map_size) { | |
478 | va = (uintptr_t)__va(pa); | |
479 | create_pgd_mapping(swapper_pg_dir, va, pa, | |
480 | map_size, PAGE_KERNEL_EXEC); | |
481 | } | |
6f1e9e94 | 482 | } |
f2c17aab | 483 | |
671f9a3e AP |
484 | /* Clear fixmap PTE and PMD mappings */ |
485 | clear_fixmap(FIX_PTE); | |
486 | clear_fixmap(FIX_PMD); | |
487 | ||
488 | /* Move to swapper page table */ | |
ac51e005 | 489 | csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); |
671f9a3e AP |
490 | local_flush_tlb_all(); |
491 | } | |
6bd33e1e CH |
492 | #else |
493 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) | |
494 | { | |
2d268251 PD |
495 | #ifdef CONFIG_BUILTIN_DTB |
496 | dtb_early_va = soc_lookup_builtin_dtb(); | |
497 | if (!dtb_early_va) { | |
498 | /* Fallback to first available DTS */ | |
499 | dtb_early_va = (void *) __dtb_start; | |
500 | } | |
501 | #else | |
6bd33e1e | 502 | dtb_early_va = (void *)dtb_pa; |
2d268251 | 503 | #endif |
a78c6f59 | 504 | dtb_early_pa = dtb_pa; |
6bd33e1e CH |
505 | } |
506 | ||
507 | static inline void setup_vm_final(void) | |
508 | { | |
509 | } | |
510 | #endif /* CONFIG_MMU */ | |
671f9a3e | 511 | |
d27c3c90 | 512 | #ifdef CONFIG_STRICT_KERNEL_RWX |
d27c3c90 ZL |
513 | void mark_rodata_ro(void) |
514 | { | |
515 | unsigned long text_start = (unsigned long)_text; | |
516 | unsigned long text_end = (unsigned long)_etext; | |
517 | unsigned long rodata_start = (unsigned long)__start_rodata; | |
518 | unsigned long data_start = (unsigned long)_data; | |
519 | unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn))); | |
520 | ||
521 | set_memory_ro(text_start, (text_end - text_start) >> PAGE_SHIFT); | |
522 | set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); | |
523 | set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT); | |
524 | set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT); | |
b422d28b ZL |
525 | |
526 | debug_checkwx(); | |
d27c3c90 ZL |
527 | } |
528 | #endif | |
529 | ||
e3ef4d69 | 530 | static void __init resource_init(void) |
526fbaed ZL |
531 | { |
532 | struct memblock_region *region; | |
533 | ||
534 | for_each_memblock(memory, region) { | |
535 | struct resource *res; | |
536 | ||
537 | res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); | |
538 | if (!res) | |
539 | panic("%s: Failed to allocate %zu bytes\n", __func__, | |
540 | sizeof(struct resource)); | |
541 | ||
542 | if (memblock_is_nomap(region)) { | |
543 | res->name = "reserved"; | |
544 | res->flags = IORESOURCE_MEM; | |
545 | } else { | |
546 | res->name = "System RAM"; | |
547 | res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; | |
548 | } | |
549 | res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); | |
550 | res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; | |
551 | ||
552 | request_resource(&iomem_resource, res); | |
553 | } | |
554 | } | |
555 | ||
671f9a3e AP |
556 | void __init paging_init(void) |
557 | { | |
558 | setup_vm_final(); | |
d95f1a54 | 559 | sparse_init(); |
671f9a3e AP |
560 | setup_zero_page(); |
561 | zone_sizes_init(); | |
526fbaed | 562 | resource_init(); |
6f1e9e94 | 563 | } |
d95f1a54 | 564 | |
9fe57d8c | 565 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
d95f1a54 LG |
566 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, |
567 | struct vmem_altmap *altmap) | |
568 | { | |
1d9cfee7 | 569 | return vmemmap_populate_basepages(start, end, node, NULL); |
d95f1a54 LG |
570 | } |
571 | #endif |