Commit | Line | Data |
---|---|---|
50acfb2b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
76d2a049 PD |
2 | /* |
3 | * Copyright (C) 2012 Regents of the University of California | |
671f9a3e | 4 | * Copyright (C) 2019 Western Digital Corporation or its affiliates. |
76d2a049 PD |
5 | */ |
6 | ||
7 | #include <linux/init.h> | |
8 | #include <linux/mm.h> | |
76d2a049 | 9 | #include <linux/memblock.h> |
57c8a661 | 10 | #include <linux/initrd.h> |
76d2a049 | 11 | #include <linux/swap.h> |
5ec9c4ff | 12 | #include <linux/sizes.h> |
0651c263 | 13 | #include <linux/of_fdt.h> |
76d2a049 | 14 | |
f2c17aab | 15 | #include <asm/fixmap.h> |
76d2a049 PD |
16 | #include <asm/tlbflush.h> |
17 | #include <asm/sections.h> | |
18 | #include <asm/pgtable.h> | |
19 | #include <asm/io.h> | |
20 | ||
387181dc AP |
21 | unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] |
22 | __page_aligned_bss; | |
23 | EXPORT_SYMBOL(empty_zero_page); | |
24 | ||
d90d45d7 AP |
25 | extern char _start[]; |
26 | ||
76d2a049 PD |
27 | static void __init zone_sizes_init(void) |
28 | { | |
5ec9c4ff | 29 | unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; |
76d2a049 | 30 | |
d5fad48c | 31 | #ifdef CONFIG_ZONE_DMA32 |
28198c46 GR |
32 | max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, |
33 | (unsigned long) PFN_PHYS(max_low_pfn))); | |
d5fad48c | 34 | #endif |
5ec9c4ff CH |
35 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; |
36 | ||
37 | free_area_init_nodes(max_zone_pfns); | |
76d2a049 PD |
38 | } |
39 | ||
40 | void setup_zero_page(void) | |
41 | { | |
42 | memset((void *)empty_zero_page, 0, PAGE_SIZE); | |
43 | } | |
44 | ||
76d2a049 PD |
45 | void __init mem_init(void) |
46 | { | |
47 | #ifdef CONFIG_FLATMEM | |
48 | BUG_ON(!mem_map); | |
49 | #endif /* CONFIG_FLATMEM */ | |
50 | ||
51 | high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); | |
c6ffc5ca | 52 | memblock_free_all(); |
76d2a049 PD |
53 | |
54 | mem_init_print_info(NULL); | |
55 | } | |
56 | ||
76d2a049 | 57 | #ifdef CONFIG_BLK_DEV_INITRD |
0651c263 AP |
58 | static void __init setup_initrd(void) |
59 | { | |
60 | unsigned long size; | |
61 | ||
62 | if (initrd_start >= initrd_end) { | |
63 | pr_info("initrd not found or empty"); | |
64 | goto disable; | |
65 | } | |
66 | if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) { | |
67 | pr_err("initrd extends beyond end of memory"); | |
68 | goto disable; | |
69 | } | |
70 | ||
71 | size = initrd_end - initrd_start; | |
72 | memblock_reserve(__pa(initrd_start), size); | |
73 | initrd_below_start_ok = 1; | |
74 | ||
75 | pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", | |
76 | (void *)(initrd_start), size); | |
77 | return; | |
78 | disable: | |
79 | pr_cont(" - disabling initrd\n"); | |
80 | initrd_start = 0; | |
81 | initrd_end = 0; | |
82 | } | |
76d2a049 | 83 | #endif /* CONFIG_BLK_DEV_INITRD */ |
0651c263 AP |
84 | |
85 | void __init setup_bootmem(void) | |
86 | { | |
87 | struct memblock_region *reg; | |
88 | phys_addr_t mem_size = 0; | |
d90d45d7 AP |
89 | phys_addr_t vmlinux_end = __pa(&_end); |
90 | phys_addr_t vmlinux_start = __pa(&_start); | |
0651c263 AP |
91 | |
92 | /* Find the memory region containing the kernel */ | |
93 | for_each_memblock(memory, reg) { | |
0651c263 AP |
94 | phys_addr_t end = reg->base + reg->size; |
95 | ||
96 | if (reg->base <= vmlinux_end && vmlinux_end <= end) { | |
0651c263 | 97 | mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); |
f05badde AP |
98 | |
99 | /* | |
100 | * Remove memblock from the end of usable area to the | |
101 | * end of region | |
102 | */ | |
103 | if (reg->base + mem_size < end) | |
104 | memblock_remove(reg->base + mem_size, | |
105 | end - reg->base - mem_size); | |
0651c263 AP |
106 | } |
107 | } | |
108 | BUG_ON(mem_size == 0); | |
109 | ||
d90d45d7 AP |
110 | /* Reserve from the start of the kernel to the end of the kernel */ |
111 | memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); | |
112 | ||
0651c263 AP |
113 | set_max_mapnr(PFN_DOWN(mem_size)); |
114 | max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); | |
115 | ||
116 | #ifdef CONFIG_BLK_DEV_INITRD | |
117 | setup_initrd(); | |
118 | #endif /* CONFIG_BLK_DEV_INITRD */ | |
119 | ||
120 | early_init_fdt_reserve_self(); | |
121 | early_init_fdt_scan_reserved_mem(); | |
122 | memblock_allow_resize(); | |
123 | memblock_dump_all(); | |
124 | ||
125 | for_each_memblock(memory, reg) { | |
126 | unsigned long start_pfn = memblock_region_memory_base_pfn(reg); | |
127 | unsigned long end_pfn = memblock_region_memory_end_pfn(reg); | |
128 | ||
129 | memblock_set_node(PFN_PHYS(start_pfn), | |
130 | PFN_PHYS(end_pfn - start_pfn), | |
131 | &memblock.memory, 0); | |
132 | } | |
133 | } | |
6f1e9e94 | 134 | |
387181dc AP |
135 | unsigned long va_pa_offset; |
136 | EXPORT_SYMBOL(va_pa_offset); | |
137 | unsigned long pfn_base; | |
138 | EXPORT_SYMBOL(pfn_base); | |
139 | ||
671f9a3e | 140 | void *dtb_early_va; |
6f1e9e94 | 141 | pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
671f9a3e AP |
142 | pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; |
143 | pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; | |
144 | static bool mmu_enabled; | |
6f1e9e94 | 145 | |
671f9a3e | 146 | #define MAX_EARLY_MAPPING_SIZE SZ_128M |
6f1e9e94 | 147 | |
671f9a3e | 148 | pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); |
f2c17aab AP |
149 | |
150 | void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) | |
151 | { | |
152 | unsigned long addr = __fix_to_virt(idx); | |
153 | pte_t *ptep; | |
154 | ||
155 | BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); | |
156 | ||
157 | ptep = &fixmap_pte[pte_index(addr)]; | |
158 | ||
159 | if (pgprot_val(prot)) { | |
160 | set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); | |
161 | } else { | |
162 | pte_clear(&init_mm, addr, ptep); | |
163 | local_flush_tlb_page(addr); | |
164 | } | |
165 | } | |
166 | ||
671f9a3e AP |
167 | static pte_t *__init get_pte_virt(phys_addr_t pa) |
168 | { | |
169 | if (mmu_enabled) { | |
170 | clear_fixmap(FIX_PTE); | |
171 | return (pte_t *)set_fixmap_offset(FIX_PTE, pa); | |
172 | } else { | |
173 | return (pte_t *)((uintptr_t)pa); | |
174 | } | |
175 | } | |
176 | ||
177 | static phys_addr_t __init alloc_pte(uintptr_t va) | |
178 | { | |
179 | /* | |
180 | * We only create PMD or PGD early mappings so we | |
181 | * should never reach here with MMU disabled. | |
182 | */ | |
183 | BUG_ON(!mmu_enabled); | |
184 | ||
185 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); | |
186 | } | |
187 | ||
188 | static void __init create_pte_mapping(pte_t *ptep, | |
189 | uintptr_t va, phys_addr_t pa, | |
190 | phys_addr_t sz, pgprot_t prot) | |
191 | { | |
192 | uintptr_t pte_index = pte_index(va); | |
193 | ||
194 | BUG_ON(sz != PAGE_SIZE); | |
195 | ||
196 | if (pte_none(ptep[pte_index])) | |
197 | ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot); | |
198 | } | |
199 | ||
200 | #ifndef __PAGETABLE_PMD_FOLDED | |
201 | ||
202 | pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
203 | pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
204 | ||
205 | #if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE | |
206 | #define NUM_EARLY_PMDS 1UL | |
207 | #else | |
208 | #define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) | |
209 | #endif | |
210 | pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); | |
211 | ||
212 | static pmd_t *__init get_pmd_virt(phys_addr_t pa) | |
213 | { | |
214 | if (mmu_enabled) { | |
215 | clear_fixmap(FIX_PMD); | |
216 | return (pmd_t *)set_fixmap_offset(FIX_PMD, pa); | |
217 | } else { | |
218 | return (pmd_t *)((uintptr_t)pa); | |
219 | } | |
220 | } | |
221 | ||
222 | static phys_addr_t __init alloc_pmd(uintptr_t va) | |
223 | { | |
224 | uintptr_t pmd_num; | |
225 | ||
226 | if (mmu_enabled) | |
227 | return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); | |
228 | ||
229 | pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; | |
230 | BUG_ON(pmd_num >= NUM_EARLY_PMDS); | |
231 | return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; | |
232 | } | |
233 | ||
234 | static void __init create_pmd_mapping(pmd_t *pmdp, | |
235 | uintptr_t va, phys_addr_t pa, | |
236 | phys_addr_t sz, pgprot_t prot) | |
237 | { | |
238 | pte_t *ptep; | |
239 | phys_addr_t pte_phys; | |
240 | uintptr_t pmd_index = pmd_index(va); | |
241 | ||
242 | if (sz == PMD_SIZE) { | |
243 | if (pmd_none(pmdp[pmd_index])) | |
244 | pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot); | |
245 | return; | |
246 | } | |
247 | ||
248 | if (pmd_none(pmdp[pmd_index])) { | |
249 | pte_phys = alloc_pte(va); | |
250 | pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); | |
251 | ptep = get_pte_virt(pte_phys); | |
252 | memset(ptep, 0, PAGE_SIZE); | |
253 | } else { | |
254 | pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index])); | |
255 | ptep = get_pte_virt(pte_phys); | |
256 | } | |
257 | ||
258 | create_pte_mapping(ptep, va, pa, sz, prot); | |
259 | } | |
260 | ||
261 | #define pgd_next_t pmd_t | |
262 | #define alloc_pgd_next(__va) alloc_pmd(__va) | |
263 | #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) | |
264 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ | |
265 | create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) | |
266 | #define PTE_PARENT_SIZE PMD_SIZE | |
267 | #define fixmap_pgd_next fixmap_pmd | |
268 | #else | |
269 | #define pgd_next_t pte_t | |
270 | #define alloc_pgd_next(__va) alloc_pte(__va) | |
271 | #define get_pgd_next_virt(__pa) get_pte_virt(__pa) | |
272 | #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ | |
273 | create_pte_mapping(__nextp, __va, __pa, __sz, __prot) | |
274 | #define PTE_PARENT_SIZE PGDIR_SIZE | |
275 | #define fixmap_pgd_next fixmap_pte | |
276 | #endif | |
277 | ||
278 | static void __init create_pgd_mapping(pgd_t *pgdp, | |
279 | uintptr_t va, phys_addr_t pa, | |
280 | phys_addr_t sz, pgprot_t prot) | |
281 | { | |
282 | pgd_next_t *nextp; | |
283 | phys_addr_t next_phys; | |
284 | uintptr_t pgd_index = pgd_index(va); | |
285 | ||
286 | if (sz == PGDIR_SIZE) { | |
287 | if (pgd_val(pgdp[pgd_index]) == 0) | |
288 | pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); | |
289 | return; | |
290 | } | |
291 | ||
292 | if (pgd_val(pgdp[pgd_index]) == 0) { | |
293 | next_phys = alloc_pgd_next(va); | |
294 | pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); | |
295 | nextp = get_pgd_next_virt(next_phys); | |
296 | memset(nextp, 0, PAGE_SIZE); | |
297 | } else { | |
298 | next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); | |
299 | nextp = get_pgd_next_virt(next_phys); | |
300 | } | |
301 | ||
302 | create_pgd_next_mapping(nextp, va, pa, sz, prot); | |
303 | } | |
304 | ||
305 | static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) | |
306 | { | |
307 | uintptr_t map_size = PAGE_SIZE; | |
308 | ||
309 | /* Upgrade to PMD/PGDIR mappings whenever possible */ | |
310 | if (!(base & (PTE_PARENT_SIZE - 1)) && | |
311 | !(size & (PTE_PARENT_SIZE - 1))) | |
312 | map_size = PTE_PARENT_SIZE; | |
313 | ||
314 | return map_size; | |
315 | } | |
316 | ||
387181dc AP |
317 | /* |
318 | * setup_vm() is called from head.S with MMU-off. | |
319 | * | |
320 | * Following requirements should be honoured for setup_vm() to work | |
321 | * correctly: | |
322 | * 1) It should use PC-relative addressing for accessing kernel symbols. | |
323 | * To achieve this we always use GCC cmodel=medany. | |
324 | * 2) The compiler instrumentation for FTRACE will not work for setup_vm() | |
325 | * so disable compiler instrumentation when FTRACE is enabled. | |
326 | * | |
327 | * Currently, the above requirements are honoured by using custom CFLAGS | |
328 | * for init.o in mm/Makefile. | |
329 | */ | |
330 | ||
331 | #ifndef __riscv_cmodel_medany | |
332 | #error "setup_vm() is called from head.S before relocate so it should " | |
333 | "not use absolute addressing." | |
334 | #endif | |
335 | ||
671f9a3e | 336 | asmlinkage void __init setup_vm(uintptr_t dtb_pa) |
6f1e9e94 | 337 | { |
671f9a3e AP |
338 | uintptr_t va, end_va; |
339 | uintptr_t load_pa = (uintptr_t)(&_start); | |
340 | uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; | |
341 | uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); | |
342 | ||
343 | va_pa_offset = PAGE_OFFSET - load_pa; | |
344 | pfn_base = PFN_DOWN(load_pa); | |
6f1e9e94 | 345 | |
671f9a3e AP |
346 | /* |
347 | * Enforce boot alignment requirements of RV32 and | |
348 | * RV64 by only allowing PMD or PGD mappings. | |
349 | */ | |
350 | BUG_ON(map_size == PAGE_SIZE); | |
6f1e9e94 AP |
351 | |
352 | /* Sanity check alignment and size */ | |
353 | BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); | |
671f9a3e AP |
354 | BUG_ON((load_pa % map_size) != 0); |
355 | BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); | |
356 | ||
357 | /* Setup early PGD for fixmap */ | |
358 | create_pgd_mapping(early_pg_dir, FIXADDR_START, | |
359 | (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE); | |
6f1e9e94 AP |
360 | |
361 | #ifndef __PAGETABLE_PMD_FOLDED | |
671f9a3e AP |
362 | /* Setup fixmap PMD */ |
363 | create_pmd_mapping(fixmap_pmd, FIXADDR_START, | |
364 | (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); | |
365 | /* Setup trampoline PGD and PMD */ | |
366 | create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, | |
367 | (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); | |
368 | create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, | |
369 | load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); | |
370 | #else | |
371 | /* Setup trampoline PGD */ | |
372 | create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, | |
373 | load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); | |
374 | #endif | |
6f1e9e94 | 375 | |
671f9a3e AP |
376 | /* |
377 | * Setup early PGD covering entire kernel which will allows | |
378 | * us to reach paging_init(). We map all memory banks later | |
379 | * in setup_vm_final() below. | |
380 | */ | |
381 | end_va = PAGE_OFFSET + load_sz; | |
382 | for (va = PAGE_OFFSET; va < end_va; va += map_size) | |
383 | create_pgd_mapping(early_pg_dir, va, | |
384 | load_pa + (va - PAGE_OFFSET), | |
385 | map_size, PAGE_KERNEL_EXEC); | |
386 | ||
387 | /* Create fixed mapping for early FDT parsing */ | |
388 | end_va = __fix_to_virt(FIX_FDT) + FIX_FDT_SIZE; | |
389 | for (va = __fix_to_virt(FIX_FDT); va < end_va; va += PAGE_SIZE) | |
390 | create_pte_mapping(fixmap_pte, va, | |
391 | dtb_pa + (va - __fix_to_virt(FIX_FDT)), | |
392 | PAGE_SIZE, PAGE_KERNEL); | |
393 | ||
394 | /* Save pointer to DTB for early FDT parsing */ | |
395 | dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK); | |
396 | } | |
f2c17aab | 397 | |
671f9a3e AP |
398 | static void __init setup_vm_final(void) |
399 | { | |
400 | uintptr_t va, map_size; | |
401 | phys_addr_t pa, start, end; | |
402 | struct memblock_region *reg; | |
403 | ||
404 | /* Set mmu_enabled flag */ | |
405 | mmu_enabled = true; | |
6f1e9e94 | 406 | |
671f9a3e AP |
407 | /* Setup swapper PGD for fixmap */ |
408 | create_pgd_mapping(swapper_pg_dir, FIXADDR_START, | |
409 | __pa(fixmap_pgd_next), | |
410 | PGDIR_SIZE, PAGE_TABLE); | |
f2c17aab | 411 | |
671f9a3e AP |
412 | /* Map all memory banks */ |
413 | for_each_memblock(memory, reg) { | |
414 | start = reg->base; | |
415 | end = start + reg->size; | |
416 | ||
417 | if (start >= end) | |
418 | break; | |
419 | if (memblock_is_nomap(reg)) | |
420 | continue; | |
421 | if (start <= __pa(PAGE_OFFSET) && | |
422 | __pa(PAGE_OFFSET) < end) | |
423 | start = __pa(PAGE_OFFSET); | |
424 | ||
425 | map_size = best_map_size(start, end - start); | |
426 | for (pa = start; pa < end; pa += map_size) { | |
427 | va = (uintptr_t)__va(pa); | |
428 | create_pgd_mapping(swapper_pg_dir, va, pa, | |
429 | map_size, PAGE_KERNEL_EXEC); | |
430 | } | |
6f1e9e94 | 431 | } |
f2c17aab | 432 | |
671f9a3e AP |
433 | /* Clear fixmap PTE and PMD mappings */ |
434 | clear_fixmap(FIX_PTE); | |
435 | clear_fixmap(FIX_PMD); | |
436 | ||
437 | /* Move to swapper page table */ | |
4f3f9008 | 438 | csr_write(CSR_SATP, PFN_DOWN(__pa(swapper_pg_dir)) | SATP_MODE); |
671f9a3e AP |
439 | local_flush_tlb_all(); |
440 | } | |
441 | ||
442 | void __init paging_init(void) | |
443 | { | |
444 | setup_vm_final(); | |
d95f1a54 LG |
445 | memblocks_present(); |
446 | sparse_init(); | |
671f9a3e AP |
447 | setup_zero_page(); |
448 | zone_sizes_init(); | |
6f1e9e94 | 449 | } |
d95f1a54 LG |
450 | |
451 | #ifdef CONFIG_SPARSEMEM | |
452 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, | |
453 | struct vmem_altmap *altmap) | |
454 | { | |
455 | return vmemmap_populate_basepages(start, end, node); | |
456 | } | |
457 | #endif |