mm/sparse-vmemmap: generalise vmemmap_populate_hugepages()

Generalise vmemmap_populate_hugepages() so ARM64 & X86 & LoongArch can
share its implementation.

Link: https://lkml.kernel.org/r/20221027125253.3458989-4-chenhuacai@loongson.cn
Signed-off-by: Feiyang Chen <chenfeiyang@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
Acked-by: Will Deacon <will@kernel.org>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Guo Ren <guoren@kernel.org>
Cc: Jiaxun Yang <jiaxun.yang@flygoat.com>
Cc: Min Zhou <zhoumin@loongson.cn>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Philippe Mathieu-Daudé <philmd@linaro.org>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Xuefeng Li <lixuefeng@loongson.cn>
Cc: Xuerui Wang <kernel@xen0n.name>
Cc: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 556154d821bf43de4f5343697155d8bc062c7722..27217ba12e5769a7cee4d917eb09245b8e32f860 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1137,53 +1137,28 @@ static void free_empty_tables(unsigned long addr, unsigned long end,
 }
 #endif
 
+void __meminit vmemmap_set_pmd(pmd_t *pmdp, void *p, int node,
+                              unsigned long addr, unsigned long next)
+{
+       pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
+}
+
+int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
+                               unsigned long addr, unsigned long next)
+{
+       vmemmap_verify((pte_t *)pmdp, node, addr, next);
+       return 1;
+}
+
 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
                struct vmem_altmap *altmap)
 {
-       unsigned long addr = start;
-       unsigned long next;
-       pgd_t *pgdp;
-       p4d_t *p4dp;
-       pud_t *pudp;
-       pmd_t *pmdp;
-
        WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
 
        if (!ARM64_KERNEL_USES_PMD_MAPS)
                return vmemmap_populate_basepages(start, end, node, altmap);
-
-       do {
-               next = pmd_addr_end(addr, end);
-
-               pgdp = vmemmap_pgd_populate(addr, node);
-               if (!pgdp)
-                       return -ENOMEM;
-
-               p4dp = vmemmap_p4d_populate(pgdp, addr, node);
-               if (!p4dp)
-                       return -ENOMEM;
-
-               pudp = vmemmap_pud_populate(p4dp, addr, node);
-               if (!pudp)
-                       return -ENOMEM;
-
-               pmdp = pmd_offset(pudp, addr);
-               if (pmd_none(READ_ONCE(*pmdp))) {
-                       void *p = NULL;
-
-                       p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
-                       if (!p) {
-                               if (vmemmap_populate_basepages(addr, next, node, altmap))
-                                       return -ENOMEM;
-                               continue;
-                       }
-
-                       pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
-               } else
-                       vmemmap_verify((pte_t *)pmdp, node, addr, next);
-       } while (addr = next, addr != end);
-
-       return 0;
+       else
+               return vmemmap_populate_hugepages(start, end, node, altmap);
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG

diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 451d93667bccecfecd1114f78ae96dce14ef013c..e018aed345866010c0822c82bfc0c5459ef2e41e 100644 (file)
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -153,52 +153,25 @@ EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
 #endif
 
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
-static int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
-                                               int node, struct vmem_altmap *altmap)
+void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
+                              unsigned long addr, unsigned long next)
 {
-       unsigned long addr = start;
-       unsigned long next;
-       pgd_t *pgd;
-       p4d_t *p4d;
-       pud_t *pud;
-       pmd_t *pmd;
+       pmd_t entry;
 
-       for (addr = start; addr < end; addr = next) {
-               next = pmd_addr_end(addr, end);
-
-               pgd = vmemmap_pgd_populate(addr, node);
-               if (!pgd)
-                       return -ENOMEM;
-               p4d = vmemmap_p4d_populate(pgd, addr, node);
-               if (!p4d)
-                       return -ENOMEM;
-               pud = vmemmap_pud_populate(p4d, addr, node);
-               if (!pud)
-                       return -ENOMEM;
-
-               pmd = pmd_offset(pud, addr);
-               if (pmd_none(*pmd)) {
-                       void *p = NULL;
-
-                       p = vmemmap_alloc_block_buf(PMD_SIZE, node, NULL);
-                       if (p) {
-                               pmd_t entry;
-
-                               entry = pfn_pmd(virt_to_pfn(p), PAGE_KERNEL);
-                               pmd_val(entry) |= _PAGE_HUGE | _PAGE_HGLOBAL;
-                               set_pmd_at(&init_mm, addr, pmd, entry);
-
-                               continue;
-                       }
-               } else if (pmd_val(*pmd) & _PAGE_HUGE) {
-                       vmemmap_verify((pte_t *)pmd, node, addr, next);
-                       continue;
-               }
-               if (vmemmap_populate_basepages(addr, next, node, NULL))
-                       return -ENOMEM;
-       }
+       entry = pfn_pmd(virt_to_pfn(p), PAGE_KERNEL);
+       pmd_val(entry) |= _PAGE_HUGE | _PAGE_HGLOBAL;
+       set_pmd_at(&init_mm, addr, pmd, entry);
+}
+
+int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
+                               unsigned long addr, unsigned long next)
+{
+       int huge = pmd_val(*pmd) & _PAGE_HUGE;
+
+       if (huge)
+               vmemmap_verify((pte_t *)pmd, node, addr, next);
 
-       return 0;
+       return huge;
 }
 
 int __meminit vmemmap_populate(unsigned long start, unsigned long end,

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index e8db4edd7cc9b39f96afad568285092d814039b8..a190aae8ceaf70c4070c216348a307a2d28535c9 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1492,72 +1492,44 @@ static long __meminitdata addr_start, addr_end;
 static void __meminitdata *p_start, *p_end;
 static int __meminitdata node_start;
 
-static int __meminit vmemmap_populate_hugepages(unsigned long start,
-               unsigned long end, int node, struct vmem_altmap *altmap)
+void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
+                              unsigned long addr, unsigned long next)
 {
-       unsigned long addr;
-       unsigned long next;
-       pgd_t *pgd;
-       p4d_t *p4d;
-       pud_t *pud;
-       pmd_t *pmd;
-
-       for (addr = start; addr < end; addr = next) {
-               next = pmd_addr_end(addr, end);
-
-               pgd = vmemmap_pgd_populate(addr, node);
-               if (!pgd)
-                       return -ENOMEM;
-
-               p4d = vmemmap_p4d_populate(pgd, addr, node);
-               if (!p4d)
-                       return -ENOMEM;
-
-               pud = vmemmap_pud_populate(p4d, addr, node);
-               if (!pud)
-                       return -ENOMEM;
-
-               pmd = pmd_offset(pud, addr);
-               if (pmd_none(*pmd)) {
-                       void *p;
-
-                       p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
-                       if (p) {
-                               pte_t entry;
-
-                               entry = pfn_pte(__pa(p) >> PAGE_SHIFT,
-                                               PAGE_KERNEL_LARGE);
-                               set_pmd(pmd, __pmd(pte_val(entry)));
+       pte_t entry;
+
+       entry = pfn_pte(__pa(p) >> PAGE_SHIFT,
+                       PAGE_KERNEL_LARGE);
+       set_pmd(pmd, __pmd(pte_val(entry)));
+
+       /* check to see if we have contiguous blocks */
+       if (p_end != p || node_start != node) {
+               if (p_start)
+                       pr_debug(" [%lx-%lx] PMD -> [%p-%p] on node %d\n",
+                               addr_start, addr_end-1, p_start, p_end-1, node_start);
+               addr_start = addr;
+               node_start = node;
+               p_start = p;
+       }
 
-                               /* check to see if we have contiguous blocks */
-                               if (p_end != p || node_start != node) {
-                                       if (p_start)
-                                               pr_debug(" [%lx-%lx] PMD -> [%p-%p] on node %d\n",
-                                                      addr_start, addr_end-1, p_start, p_end-1, node_start);
-                                       addr_start = addr;
-                                       node_start = node;
-                                       p_start = p;
-                               }
+       addr_end = addr + PMD_SIZE;
+       p_end = p + PMD_SIZE;
 
-                               addr_end = addr + PMD_SIZE;
-                               p_end = p + PMD_SIZE;
+       if (!IS_ALIGNED(addr, PMD_SIZE) ||
+               !IS_ALIGNED(next, PMD_SIZE))
+               vmemmap_use_new_sub_pmd(addr, next);
+}
 
-                               if (!IS_ALIGNED(addr, PMD_SIZE) ||
-                                   !IS_ALIGNED(next, PMD_SIZE))
-                                       vmemmap_use_new_sub_pmd(addr, next);
+int __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
+                               unsigned long addr, unsigned long next)
+{
+       int large = pmd_large(*pmd);
 
-                               continue;
-                       } else if (altmap)
-                               return -ENOMEM; /* no fallback */
-               } else if (pmd_large(*pmd)) {
-                       vmemmap_verify((pte_t *)pmd, node, addr, next);
-                       vmemmap_use_sub_pmd(addr, next);
-                       continue;
-               }
-               if (vmemmap_populate_basepages(addr, next, node, NULL))
-                       return -ENOMEM;
+       if (pmd_large(*pmd)) {
+               vmemmap_verify((pte_t *)pmd, node, addr, next);
+               vmemmap_use_sub_pmd(addr, next);
        }
-       return 0;
+
+       return large;
 }
 
 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 7c31f898337cee52e0050c2bc6ce8704b93503fe..472cb60ace07565e5be05cac049720bfec433fe5 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3373,8 +3373,14 @@ struct vmem_altmap;
 void *vmemmap_alloc_block_buf(unsigned long size, int node,
                              struct vmem_altmap *altmap);
 void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
+void vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
+                    unsigned long addr, unsigned long next);
+int vmemmap_check_pmd(pmd_t *pmd, int node,
+                     unsigned long addr, unsigned long next);
 int vmemmap_populate_basepages(unsigned long start, unsigned long end,
                               int node, struct vmem_altmap *altmap);
+int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
+                              int node, struct vmem_altmap *altmap);
 int vmemmap_populate(unsigned long start, unsigned long end, int node,
                struct vmem_altmap *altmap);
 void vmemmap_populate_print_last(void);

diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 797b30e9050c5ff3a72757fb930b2b53066eb936..c5398a5960d05be06b3584649fe9b904c495df60 100644 (file)
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -295,6 +295,69 @@ int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
        return vmemmap_populate_range(start, end, node, altmap, NULL);
 }
 
+void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
+                                     unsigned long addr, unsigned long next)
+{
+}
+
+int __weak __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
+                                      unsigned long addr, unsigned long next)
+{
+       return 0;
+}
+
+int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
+                                        int node, struct vmem_altmap *altmap)
+{
+       unsigned long addr;
+       unsigned long next;
+       pgd_t *pgd;
+       p4d_t *p4d;
+       pud_t *pud;
+       pmd_t *pmd;
+
+       for (addr = start; addr < end; addr = next) {
+               next = pmd_addr_end(addr, end);
+
+               pgd = vmemmap_pgd_populate(addr, node);
+               if (!pgd)
+                       return -ENOMEM;
+
+               p4d = vmemmap_p4d_populate(pgd, addr, node);
+               if (!p4d)
+                       return -ENOMEM;
+
+               pud = vmemmap_pud_populate(p4d, addr, node);
+               if (!pud)
+                       return -ENOMEM;
+
+               pmd = pmd_offset(pud, addr);
+               if (pmd_none(READ_ONCE(*pmd))) {
+                       void *p;
+
+                       p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
+                       if (p) {
+                               vmemmap_set_pmd(pmd, p, node, addr, next);
+                               continue;
+                       } else if (altmap) {
+                               /*
+                                * No fallback: In any case we care about, the
+                                * altmap should be reasonably sized and aligned
+                                * such that vmemmap_alloc_block_buf() will always
+                                * succeed. For consistency with the PTE case,
+                                * return an error here as failure could indicate
+                                * a configuration issue with the size of the altmap.
+                                */
+                               return -ENOMEM;
+                       }
+               } else if (vmemmap_check_pmd(pmd, node, addr, next))
+                       continue;
+               if (vmemmap_populate_basepages(addr, next, node, altmap))
+                       return -ENOMEM;
+       }
+       return 0;
+}
+
 /*
  * For compound pages bigger than section size (e.g. x86 1G compound
  * pages with 2M subsection size) fill the rest of sections as tail