arm64: mmu: Make __cpu_replace_ttbr1() out of line

__cpu_replace_ttbr1() is a static inline, and so it gets instantiated
wherever it is used. This is not really necessary, as it is never called
on a hot path. It also has the unfortunate side effect that the symbol
idmap_cpu_replace_ttbr1 may never be referenced from kCFI enabled C
code, and this means the type id symbol may not exist either.  This will
result in a build error once we start referring to this symbol from asm
code as well. (Note that this problem only occurs when CnP, KAsan and
suspend/resume are all disabled in the Kconfig but that is a valid
config, if unusual).

So let's just move it out of line so all callers will share the same
implementation, which will reference idmap_cpu_replace_ttbr1
unconditionally.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-62-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 9ce4200508b12b7015ba0d522fe4714f03b5c851..926fbbcecbe0419e16b14359ea8b37dac2066221 100644 (file)
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -148,37 +148,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz)
        isb();
 }
 
-/*
- * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
- * avoiding the possibility of conflicting TLB entries being allocated.
- */
-static inline void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp)
-{
-       typedef void (ttbr_replace_func)(phys_addr_t);
-       extern ttbr_replace_func idmap_cpu_replace_ttbr1;
-       ttbr_replace_func *replace_phys;
-       unsigned long daif;
-
-       /* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */
-       phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
-
-       if (cnp)
-               ttbr1 |= TTBR_CNP_BIT;
-
-       replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
-
-       __cpu_install_idmap(idmap);
-
-       /*
-        * We really don't want to take *any* exceptions while TTBR1 is
-        * in the process of being replaced so mask everything.
-        */
-       daif = local_daif_save();
-       replace_phys(ttbr1);
-       local_daif_restore(daif);
-
-       cpu_uninstall_idmap();
-}
+void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp);
 
 static inline void cpu_enable_swapper_cnp(void)
 {

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 1ac7467d34c9c333ace3df5259e40cc860bb4740..f9332eea318f9a05a6c0f709892125c0f3b8a407 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -1486,3 +1486,35 @@ void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte
 {
        set_pte_at(vma->vm_mm, addr, ptep, pte);
 }
+
+/*
+ * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
+ * avoiding the possibility of conflicting TLB entries being allocated.
+ */
+void __cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap, bool cnp)
+{
+       typedef void (ttbr_replace_func)(phys_addr_t);
+       extern ttbr_replace_func idmap_cpu_replace_ttbr1;
+       ttbr_replace_func *replace_phys;
+       unsigned long daif;
+
+       /* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */
+       phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
+
+       if (cnp)
+               ttbr1 |= TTBR_CNP_BIT;
+
+       replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
+
+       __cpu_install_idmap(idmap);
+
+       /*
+        * We really don't want to take *any* exceptions while TTBR1 is
+        * in the process of being replaced so mask everything.
+        */
+       daif = local_daif_save();
+       replace_phys(ttbr1);
+       local_daif_restore(daif);
+
+       cpu_uninstall_idmap();
+}