arm64: mm: Add 5 level paging support to fixmap and swapper handling

Add support for using 5 levels of paging in the fixmap, as well as in
the kernel page table handling code which uses fixmaps internally.
This also handles the case where a 5 level build runs on hardware that
only supports 4 levels of paging.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-79-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 8aabd45e9a13f2f9f4b282fa8aad80144c7f063b..87e307804b99c39148f0c58e03ef1fdd62a5a538 100644 (file)
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -87,6 +87,7 @@ enum fixed_addresses {
        FIX_PTE,
        FIX_PMD,
        FIX_PUD,
+       FIX_P4D,
        FIX_PGD,
 
        __end_of_fixed_addresses

diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 7eb2b933ed3c369a2619bfcc2689013113876a27..3d7fb3cde83d3fb158b535c0122a39266c89151a 100644 (file)
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -621,12 +621,12 @@ static inline bool pud_table(pud_t pud) { return true; }
                                 PUD_TYPE_TABLE)
 #endif
 
-extern pgd_t init_pg_dir[PTRS_PER_PGD];
+extern pgd_t init_pg_dir[];
 extern pgd_t init_pg_end[];
-extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
-extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
-extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
-extern pgd_t reserved_pg_dir[PTRS_PER_PGD];
+extern pgd_t swapper_pg_dir[];
+extern pgd_t idmap_pg_dir[];
+extern pgd_t tramp_pg_dir[];
+extern pgd_t reserved_pg_dir[];
 
 extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
 
@@ -891,12 +891,47 @@ static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr)
        return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr);
 }
 
+static inline p4d_t *p4d_set_fixmap(unsigned long addr)
+{
+       if (!pgtable_l5_enabled())
+               return NULL;
+       return (p4d_t *)set_fixmap_offset(FIX_P4D, addr);
+}
+
+static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr)
+{
+       if (!pgtable_l5_enabled())
+               return pgd_to_folded_p4d(pgdp, addr);
+       return p4d_set_fixmap(p4d_offset_phys(pgdp, addr));
+}
+
+static inline void p4d_clear_fixmap(void)
+{
+       if (pgtable_l5_enabled())
+               clear_fixmap(FIX_P4D);
+}
+
+/* use ONLY for statically allocated translation tables */
+static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr)
+{
+       if (!pgtable_l5_enabled())
+               return pgd_to_folded_p4d(pgdp, addr);
+       return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr));
+}
+
 #define pgd_page(pgd)          pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
 
 #else
 
 static inline bool pgtable_l5_enabled(void) { return false; }
 
+/* Match p4d_offset folding in <asm/generic/pgtable-nop4d.h> */
+#define p4d_set_fixmap(addr)           NULL
+#define p4d_set_fixmap_offset(p4dp, addr)      ((p4d_t *)p4dp)
+#define p4d_clear_fixmap()
+
+#define p4d_offset_kimg(dir,addr)      ((p4d_t *)dir)
+
 #endif  /* CONFIG_PGTABLE_LEVELS > 4 */
 
 #define pgd_ERROR(e)   \

diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index 9404f282f82983867397d322d1e27ae596f4c51c..d22506e9c7fdaf2150455146160ad699dfd909d4 100644 (file)
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -104,7 +104,7 @@ void __init early_fixmap_init(void)
        unsigned long end = FIXADDR_TOP;
 
        pgd_t *pgdp = pgd_offset_k(addr);
-       p4d_t *p4dp = p4d_offset(pgdp, addr);
+       p4d_t *p4dp = p4d_offset_kimg(pgdp, addr);
 
        early_fixmap_init_pud(p4dp, addr, end);
 }

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index d30ae4d3fdd9e15c6da980e0b7568755cb3fff85..8e5b3a7c5afd584ef15dd357e6b5917afc2922cd 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -313,15 +313,14 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
        } while (addr = next, addr != end);
 }
 
-static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
                           phys_addr_t phys, pgprot_t prot,
                           phys_addr_t (*pgtable_alloc)(int),
                           int flags)
 {
        unsigned long next;
-       pud_t *pudp;
-       p4d_t *p4dp = p4d_offset(pgdp, addr);
        p4d_t p4d = READ_ONCE(*p4dp);
+       pud_t *pudp;
 
        if (p4d_none(p4d)) {
                p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
@@ -369,6 +368,46 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
        pud_clear_fixmap();
 }
 
+static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
+                          phys_addr_t phys, pgprot_t prot,
+                          phys_addr_t (*pgtable_alloc)(int),
+                          int flags)
+{
+       unsigned long next;
+       pgd_t pgd = READ_ONCE(*pgdp);
+       p4d_t *p4dp;
+
+       if (pgd_none(pgd)) {
+               pgdval_t pgdval = PGD_TYPE_TABLE | PGD_TABLE_UXN;
+               phys_addr_t p4d_phys;
+
+               if (flags & NO_EXEC_MAPPINGS)
+                       pgdval |= PGD_TABLE_PXN;
+               BUG_ON(!pgtable_alloc);
+               p4d_phys = pgtable_alloc(P4D_SHIFT);
+               __pgd_populate(pgdp, p4d_phys, pgdval);
+               pgd = READ_ONCE(*pgdp);
+       }
+       BUG_ON(pgd_bad(pgd));
+
+       p4dp = p4d_set_fixmap_offset(pgdp, addr);
+       do {
+               p4d_t old_p4d = READ_ONCE(*p4dp);
+
+               next = p4d_addr_end(addr, end);
+
+               alloc_init_pud(p4dp, addr, next, phys, prot,
+                              pgtable_alloc, flags);
+
+               BUG_ON(p4d_val(old_p4d) != 0 &&
+                      p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp)));
+
+               phys += next - addr;
+       } while (p4dp++, addr = next, addr != end);
+
+       p4d_clear_fixmap();
+}
+
 static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
                                        unsigned long virt, phys_addr_t size,
                                        pgprot_t prot,
@@ -391,7 +430,7 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
 
        do {
                next = pgd_addr_end(addr, end);
-               alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
+               alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc,
                               flags);
                phys += next - addr;
        } while (pgdp++, addr = next, addr != end);