arch, mm: introduce arch_mm_preinit

Currently, implementation of mem_init() in every architecture consists of
one or more of the following:

* initializations that must run before page allocator is active, for
  instance swiotlb_init()
* a call to memblock_free_all() to release all the memory to the buddy
  allocator
* initializations that must run after page allocator is ready and there is
  no arch-specific hook other than mem_init() for that, like for example
  register_page_bootmem_info() in x86 and sparc64 or simple setting of
  mem_init_done = 1 in several architectures
* a bunch of semi-related stuff that apparently had no better place to
  live, for example a ton of BUILD_BUG_ON()s in parisc.

Introduce arch_mm_preinit() that will be the first thing called from
mm_core_init(). On architectures that have initializations that must happen
before the page allocator is ready, move those into arch_mm_preinit() along
with the code that does not depend on ordering with page allocator setup.

On several architectures this results in reduction of mem_init() to a
single call to memblock_free_all() that allows its consolidation next.

Link: https://lkml.kernel.org/r/20250313135003.836600-13-rppt@kernel.org
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>	[x86]
Tested-by: Mark Brown <broonie@kernel.org>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Andreas Larsson <andreas@gaisler.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Betkov <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Guo Ren (csky) <guoren@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jiaxun Yang <jiaxun.yang@flygoat.com>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Russel King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Thomas Gleinxer <tglx@linutronix.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vineet Gupta <vgupta@kernel.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/arc/mm/init.c b/arch/arc/mm/init.c
index 11ce638731c90f0144220bf1ffed7f0ce7a0810e..90715b4a0bfaee7b5ddc4dae4e1c468b5defc620 100644 (file)
--- a/arch/arc/mm/init.c
+++ b/arch/arc/mm/init.c
@@ -157,11 +157,16 @@ void __init setup_arch_memory(void)
        free_area_init(max_zone_pfn);
 }
 
-static void __init highmem_init(void)
+void __init arch_mm_preinit(void)
 {
 #ifdef CONFIG_HIGHMEM
        memblock_phys_free(high_mem_start, high_mem_sz);
 #endif
+
+       BUILD_BUG_ON((PTRS_PER_PGD * sizeof(pgd_t)) > PAGE_SIZE);
+       BUILD_BUG_ON((PTRS_PER_PUD * sizeof(pud_t)) > PAGE_SIZE);
+       BUILD_BUG_ON((PTRS_PER_PMD * sizeof(pmd_t)) > PAGE_SIZE);
+       BUILD_BUG_ON((PTRS_PER_PTE * sizeof(pte_t)) > PAGE_SIZE);
 }
 
 /*
@@ -172,13 +177,7 @@ static void __init highmem_init(void)
  */
 void __init mem_init(void)
 {
-       highmem_init();
        memblock_free_all();
-
-       BUILD_BUG_ON((PTRS_PER_PGD * sizeof(pgd_t)) > PAGE_SIZE);
-       BUILD_BUG_ON((PTRS_PER_PUD * sizeof(pud_t)) > PAGE_SIZE);
-       BUILD_BUG_ON((PTRS_PER_PMD * sizeof(pmd_t)) > PAGE_SIZE);
-       BUILD_BUG_ON((PTRS_PER_PTE * sizeof(pte_t)) > PAGE_SIZE);
 }
 
 #ifdef CONFIG_HIGHMEM

diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 7bb5ce02b9b51aff8f1432c2625c3f2c2a2922a8..7222100b063148e54e488cf36d5664a9b5e611cd 100644 (file)
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -237,12 +237,7 @@ static inline void poison_init_mem(void *s, size_t count)
                *p++ = 0xe7fddef0;
 }
 
-/*
- * mem_init() marks the free areas in the mem_map and tells us how much
- * memory is free.  This is done after various parts of the system have
- * claimed their memory after the kernel image.
- */
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
 #ifdef CONFIG_ARM_LPAE
        swiotlb_init(max_pfn > arm_dma_pfn_limit, SWIOTLB_VERBOSE);
@@ -253,9 +248,6 @@ void __init mem_init(void)
        memblock_phys_free(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
 #endif
 
-       /* this will put all unused low memory onto the freelists */
-       memblock_free_all();
-
        /*
         * Check boundaries twice: Some fundamental inconsistencies can
         * be detected at build time already.
@@ -271,6 +263,17 @@ void __init mem_init(void)
 #endif
 }
 
+/*
+ * mem_init() marks the free areas in the mem_map and tells us how much
+ * memory is free.  This is done after various parts of the system have
+ * claimed their memory after the kernel image.
+ */
+void __init mem_init(void)
+{
+       /* this will put all unused low memory onto the freelists */
+       memblock_free_all();
+}
+
 #ifdef CONFIG_STRICT_KERNEL_RWX
 struct section_perm {
        const char *name;

diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 53a0b105890bf430593f0992b8ca4cfb3a57351a..2312e3812043ea891868f8776be999b1df84d082 100644 (file)
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -357,12 +357,7 @@ void __init bootmem_init(void)
        memblock_dump_all();
 }
 
-/*
- * mem_init() marks the free areas in the mem_map and tells us how much memory
- * is free.  This is done after various parts of the system have claimed their
- * memory after the kernel image.
- */
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        unsigned int flags = SWIOTLB_VERBOSE;
        bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit);
@@ -386,9 +381,6 @@ void __init mem_init(void)
        swiotlb_init(swiotlb, flags);
        swiotlb_update_mem_attributes();
 
-       /* this will put all unused low memory onto the freelists */
-       memblock_free_all();
-
        /*
         * Check boundaries twice: Some fundamental inconsistencies can be
         * detected at build time already.
@@ -414,6 +406,17 @@ void __init mem_init(void)
        }
 }
 
+/*
+ * mem_init() marks the free areas in the mem_map and tells us how much memory
+ * is free.  This is done after various parts of the system have claimed their
+ * memory after the kernel image.
+ */
+void __init mem_init(void)
+{
+       /* this will put all unused low memory onto the freelists */
+       memblock_free_all();
+}
+
 void free_initmem(void)
 {
        void *lm_init_begin = lm_alias(__init_begin);

diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 075177e817ac2ad53da4ca29782e44098fc76856..eec38e7735ddc2e4c4f2e6555debc6142a8dc939 100644 (file)
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -425,7 +425,7 @@ void __init paging_init(void)
 static struct kcore_list kcore_kseg0;
 #endif
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        /*
         * When PFN_PTE_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
@@ -435,7 +435,6 @@ void __init mem_init(void)
 
        maar_init();
        setup_zero_pages();     /* Setup zeroed pages.  */
-       memblock_free_all();
 
 #ifdef CONFIG_64BIT
        if ((unsigned long) &_text > (unsigned long) CKSEG0)
@@ -446,13 +445,17 @@ void __init mem_init(void)
 #endif
 }
 #else  /* CONFIG_NUMA */
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        setup_zero_pages();     /* This comes from node 0 */
-       memblock_free_all();
 }
 #endif /* !CONFIG_NUMA */
 
+void __init mem_init(void)
+{
+       memblock_free_all();
+}
+
 void free_init_pages(const char *what, unsigned long begin, unsigned long end)
 {
        unsigned long pfn;

diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 1bc94bca99441d4f4e239effdd232075e8205134..68efdaf14e58141daa674958698a8e0bfcfe0f3f 100644 (file)
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -273,7 +273,7 @@ void __init paging_init(void)
        mark_nonram_nosave();
 }
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        /*
         * book3s is limited to 16 page sizes due to encoding this in
@@ -295,8 +295,6 @@ void __init mem_init(void)
 
        kasan_late_init();
 
-       memblock_free_all();
-
 #if defined(CONFIG_PPC_E500) && !defined(CONFIG_SMP)
        /*
         * If smp is enabled, next_tlbcam_idx is initialized in the cpu up
@@ -329,6 +327,11 @@ void __init mem_init(void)
 #endif /* CONFIG_PPC32 */
 }
 
+void __init mem_init(void)
+{
+       memblock_free_all();
+}
+
 void free_initmem(void)
 {
        ppc_md.progress = ppc_printk_progress;

diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index ac6d41e86243f7d3c2653da4f6db86b2b201f238..9efadabf6be199fd7e59e4b9ba92d3422fcca28c 100644 (file)
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -171,7 +171,7 @@ static void __init print_vm_layout(void)
 static void print_vm_layout(void) { }
 #endif /* CONFIG_DEBUG_VM */
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit);
 #ifdef CONFIG_FLATMEM
@@ -192,11 +192,15 @@ void __init mem_init(void)
        }
 
        swiotlb_init(swiotlb, SWIOTLB_VERBOSE);
-       memblock_free_all();
 
        print_vm_layout();
 }
 
+void __init mem_init(void)
+{
+       memblock_free_all();
+}
+
 /* Limit the memory size via mem. */
 static phys_addr_t memory_limit;
 #ifdef CONFIG_XIP_KERNEL

diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 4bd6f316d71f7cb0a4949e624fe84c376151362c..e771b7458d8b2248b502d0d34ab9c2855c065b3a 100644 (file)
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -154,7 +154,7 @@ static void pv_init(void)
        swiotlb_update_mem_attributes();
 }
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask);
        cpumask_set_cpu(0, mm_cpumask(&init_mm));
@@ -163,7 +163,10 @@ void __init mem_init(void)
        kfence_split_mapping();
 
        setup_zero_pages();     /* Setup zeroed pages. */
+}
 
+void __init mem_init(void)
+{
        /* this will put all low memory onto the freelists */
        memblock_free_all();
 }

diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
index 043e9b6fadd00d267ae85d944a76c332acff6f2b..e16c32c5728f815559a1dad1d5077c63774eb484 100644 (file)
--- a/arch/sparc/mm/init_32.c
+++ b/arch/sparc/mm/init_32.c
@@ -232,7 +232,7 @@ static void __init taint_real_pages(void)
        }
 }
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        int i;
 
@@ -262,7 +262,10 @@ void __init mem_init(void)
        memset(sparc_valid_addr_bitmap, 0, i << 2);
 
        taint_real_pages();
+}
 
+void __init mem_init(void)
+{
        memblock_free_all();
 }
 

diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index befed230aac28e96fe54b3b41cd68f46fa2cfac3..cce387438e60fdd58934625441ac87bde5b35276 100644 (file)
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -54,7 +54,7 @@ int kmalloc_ok = 0;
 /* Used during early boot */
 static unsigned long brk_end;
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        /* clear the zero-page */
        memset(empty_zero_page, 0, PAGE_SIZE);
@@ -66,10 +66,13 @@ void __init mem_init(void)
        map_memory(brk_end, __pa(brk_end), uml_reserved - brk_end, 1, 1, 0);
        memblock_free((void *)brk_end, uml_reserved - brk_end);
        uml_reserved = brk_end;
+       max_pfn = max_low_pfn;
+}
 
+void __init mem_init(void)
+{
        /* this will put all low memory onto the freelists */
        memblock_free_all();
-       max_pfn = max_low_pfn;
        kmalloc_ok = 1;
 }
 

diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 9ee8ec2bc5d1dc67e1ca0a8fdc281a9e55502d02..16664c5464b5d998fc1f3e90d8a78ccd210523ae 100644 (file)
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -691,13 +691,17 @@ static void __init test_wp_bit(void)
        panic("Linux doesn't support CPUs with broken WP.");
 }
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        pci_iommu_alloc();
 
 #ifdef CONFIG_FLATMEM
        BUG_ON(!mem_map);
 #endif
+}
+
+void __init mem_init(void)
+{
        /* this will put all low memory onto the freelists */
        memblock_free_all();
 

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 6e8e4ef5312ae5322709a95f8fb2033a277ad228..a88f7db8089ebb717089534be78e0f2cdc3fb836 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1348,10 +1348,13 @@ failed:
        panic("Failed to pre-allocate %s pages for vmalloc area\n", lvl);
 }
 
-void __init mem_init(void)
+void __init arch_mm_preinit(void)
 {
        pci_iommu_alloc();
+}
 
+void __init mem_init(void)
+{
        /* clear_bss() already clear the empty_zero_page */
 
        /* this will put all memory onto the freelists */

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 6c519a5098d438563d3712b602872cd9fe7e7071..c417e5634a589a214733cf5e3bec4f9354ed985e 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -43,6 +43,7 @@ struct folio_batch;
 
 extern int sysctl_page_lock_unfairness;
 
+void arch_mm_preinit(void);
 void mm_core_init(void);
 void init_mm_internals(void);
 

diff --git a/mm/mm_init.c b/mm/mm_init.c
index bd7071c32a44a002000b9a2cfefa402b83847789..6844de516a50d2e0c521ce49a8bbfa18e3d6b19b 100644 (file)
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2734,11 +2734,16 @@ static void __init mem_init_print_info(void)
                );
 }
 
+void __init __weak arch_mm_preinit(void)
+{
+}
+
 /*
  * Set up kernel memory allocators
  */
 void __init mm_core_init(void)
 {
+       arch_mm_preinit();
        hugetlb_bootmem_alloc();
 
        /* Initializations relying on SMP setup */