LoongArch: Introduce hardware page table walker

Loongson-3A6000 and newer processors have hardware page table walker
(PTW) support. PTW can handle all fastpaths of TLBI/TLBL/TLBS/TLBM
exceptions by hardware, software only need to handle slowpaths (page
faults).

BTW, PTW doesn't append _PAGE_MODIFIED for page table entries, so we
change pmd_dirty() and pte_dirty() to also check _PAGE_DIRTY for the
"dirty" attribute.

Signed-off-by: Liang Gao <gaoliang@loongson.cn>
Signed-off-by: Jun Yi <yijun@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h
index f6177f133477670edc2b5f955d379101e8147032..2eafe6a6aca8189c88617c18da45c373294a6aa8 100644 (file)
--- a/arch/loongarch/include/asm/cpu-features.h
+++ b/arch/loongarch/include/asm/cpu-features.h
@@ -64,6 +64,6 @@
 #define cpu_has_eiodecode      cpu_opt(LOONGARCH_CPU_EIODECODE)
 #define cpu_has_guestid                cpu_opt(LOONGARCH_CPU_GUESTID)
 #define cpu_has_hypervisor     cpu_opt(LOONGARCH_CPU_HYPERVISOR)
-
+#define cpu_has_ptw            cpu_opt(LOONGARCH_CPU_PTW)
 
 #endif /* __ASM_CPU_FEATURES_H */

diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h
index 88773d849e332f08047460efecf97fb429976505..48b9f7168bcca03f92a63f891a346f88055b7bfc 100644 (file)
--- a/arch/loongarch/include/asm/cpu.h
+++ b/arch/loongarch/include/asm/cpu.h
@@ -98,6 +98,7 @@ enum cpu_type_enum {
 #define CPU_FEATURE_EIODECODE          23      /* CPU has EXTIOI interrupt pin decode mode */
 #define CPU_FEATURE_GUESTID            24      /* CPU has GuestID feature */
 #define CPU_FEATURE_HYPERVISOR         25      /* CPU has hypervisor (running in VM) */
+#define CPU_FEATURE_PTW                        26      /* CPU has hardware page table walker */
 
 #define LOONGARCH_CPU_CPUCFG           BIT_ULL(CPU_FEATURE_CPUCFG)
 #define LOONGARCH_CPU_LAM              BIT_ULL(CPU_FEATURE_LAM)
@@ -125,5 +126,6 @@ enum cpu_type_enum {
 #define LOONGARCH_CPU_EIODECODE                BIT_ULL(CPU_FEATURE_EIODECODE)
 #define LOONGARCH_CPU_GUESTID          BIT_ULL(CPU_FEATURE_GUESTID)
 #define LOONGARCH_CPU_HYPERVISOR       BIT_ULL(CPU_FEATURE_HYPERVISOR)
+#define LOONGARCH_CPU_PTW              BIT_ULL(CPU_FEATURE_PTW)
 
 #endif /* _ASM_CPU_H */

diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h
index 08c77d065a11a333431a5c0373797b435a66c81c..1ab1ed28d770e87d48038abd46bb4836187b1db3 100644 (file)
--- a/arch/loongarch/include/asm/loongarch.h
+++ b/arch/loongarch/include/asm/loongarch.h
@@ -135,6 +135,7 @@ __asm__(".macro     parse_r var r\n\t"
 #define  CPUCFG2_MIPSBT                        BIT(20)
 #define  CPUCFG2_LSPW                  BIT(21)
 #define  CPUCFG2_LAM                   BIT(22)
+#define  CPUCFG2_PTW                   BIT(24)
 
 #define LOONGARCH_CPUCFG3              0x3
 #define  CPUCFG3_CCDMA                 BIT(0)
@@ -412,6 +413,9 @@ __asm__(".macro     parse_r var r\n\t"
 #define  CSR_PWCTL0_PTBASE             (_ULCAST_(0x1f) << CSR_PWCTL0_PTBASE_SHIFT)
 
 #define LOONGARCH_CSR_PWCTL1           0x1d    /* PWCtl1 */
+#define  CSR_PWCTL1_PTW_SHIFT          24
+#define  CSR_PWCTL1_PTW_WIDTH          1
+#define  CSR_PWCTL1_PTW                        (_ULCAST_(0x1) << CSR_PWCTL1_PTW_SHIFT)
 #define  CSR_PWCTL1_DIR3WIDTH_SHIFT    18
 #define  CSR_PWCTL1_DIR3WIDTH_WIDTH    5
 #define  CSR_PWCTL1_DIR3WIDTH          (_ULCAST_(0x1f) << CSR_PWCTL1_DIR3WIDTH_SHIFT)

diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index 9a9f9ff9b7098000002c3ed029e54c62a7ba05b1..38afeb7dd58b6b084b7194eff69d215241d9f663 100644 (file)
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -362,7 +362,7 @@ extern pgd_t invalid_pg_dir[];
  */
 static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; }
 static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
-static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; }
+static inline int pte_dirty(pte_t pte) { return pte_val(pte) & (_PAGE_DIRTY | _PAGE_MODIFIED); }
 
 static inline pte_t pte_mkold(pte_t pte)
 {
@@ -506,7 +506,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd)
 
 static inline int pmd_dirty(pmd_t pmd)
 {
-       return !!(pmd_val(pmd) & _PAGE_MODIFIED);
+       return !!(pmd_val(pmd) & (_PAGE_DIRTY | _PAGE_MODIFIED));
 }
 
 static inline pmd_t pmd_mkclean(pmd_t pmd)

diff --git a/arch/loongarch/include/asm/tlb.h b/arch/loongarch/include/asm/tlb.h
index 0ad53f1ad25d50069292bbabd87de37867ec3117..da7a3b5b9374aeaf8bc1009d49d3ee0265938e9e 100644 (file)
--- a/arch/loongarch/include/asm/tlb.h
+++ b/arch/loongarch/include/asm/tlb.h
@@ -158,6 +158,9 @@ extern void handle_tlb_store(void);
 extern void handle_tlb_modify(void);
 extern void handle_tlb_refill(void);
 extern void handle_tlb_protect(void);
+extern void handle_tlb_load_ptw(void);
+extern void handle_tlb_store_ptw(void);
+extern void handle_tlb_modify_ptw(void);
 
 extern void dump_tlb_all(void);
 extern void dump_tlb_regs(void);

diff --git a/arch/loongarch/include/uapi/asm/hwcap.h b/arch/loongarch/include/uapi/asm/hwcap.h
index 8840b72fa8e8c58b449bab2ff221ffe678982ad5..6955a7cb2c65d1b575dd9ef9af3b1616fdd3cbab 100644 (file)
--- a/arch/loongarch/include/uapi/asm/hwcap.h
+++ b/arch/loongarch/include/uapi/asm/hwcap.h
@@ -16,5 +16,6 @@
 #define HWCAP_LOONGARCH_LBT_X86                (1 << 10)
 #define HWCAP_LOONGARCH_LBT_ARM                (1 << 11)
 #define HWCAP_LOONGARCH_LBT_MIPS       (1 << 12)
+#define HWCAP_LOONGARCH_PTW            (1 << 13)
 
 #endif /* _UAPI_ASM_HWCAP_H */

diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c
index f42acc6c8df6eefe0ad3031f535a439ea4ed9e31..e925579c7a71eab8822ff683dd8444f8982e42d6 100644 (file)
--- a/arch/loongarch/kernel/cpu-probe.c
+++ b/arch/loongarch/kernel/cpu-probe.c
@@ -136,6 +136,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c)
                c->options |= LOONGARCH_CPU_CRYPTO;
                elf_hwcap |= HWCAP_LOONGARCH_CRYPTO;
        }
+       if (config & CPUCFG2_PTW) {
+               c->options |= LOONGARCH_CPU_PTW;
+               elf_hwcap |= HWCAP_LOONGARCH_PTW;
+       }
        if (config & CPUCFG2_LVZP) {
                c->options |= LOONGARCH_CPU_LVZ;
                elf_hwcap |= HWCAP_LOONGARCH_LVZ;

diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c
index d4b270630bb5d0d93528c86fb8fe70851da9729d..0d33cbc47e511093abbea31f205577875bef0c30 100644 (file)
--- a/arch/loongarch/kernel/proc.c
+++ b/arch/loongarch/kernel/proc.c
@@ -80,6 +80,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
        if (cpu_has_crc32)      seq_printf(m, " crc32");
        if (cpu_has_complex)    seq_printf(m, " complex");
        if (cpu_has_crypto)     seq_printf(m, " crypto");
+       if (cpu_has_ptw)        seq_printf(m, " ptw");
        if (cpu_has_lvz)        seq_printf(m, " lvz");
        if (cpu_has_lbt_x86)    seq_printf(m, " lbt_x86");
        if (cpu_has_lbt_arm)    seq_printf(m, " lbt_arm");

diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c
index 8bad6b0cff59b96b2ebdda34214fdc1e4758e27e..00bb563e3c89473f460113df9c55f8c860c48e8a 100644 (file)
--- a/arch/loongarch/mm/tlb.c
+++ b/arch/loongarch/mm/tlb.c
@@ -167,6 +167,9 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep
        int idx;
        unsigned long flags;
 
+       if (cpu_has_ptw)
+               return;
+
        /*
         * Handle debugger faulting in for debugee.
         */
@@ -222,6 +225,9 @@ static void setup_ptwalker(void)
        pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25;
        pwctl1 = pgd_i | pgd_w << 6;
 
+       if (cpu_has_ptw)
+               pwctl1 |= CSR_PWCTL1_PTW;
+
        csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
        csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
        csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
@@ -264,10 +270,17 @@ void setup_tlb_handler(int cpu)
        if (cpu == 0) {
                memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
                local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
-               set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
-               set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
-               set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
-               set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
+               if (!cpu_has_ptw) {
+                       set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
+                       set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
+                       set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
+                       set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
+               } else {
+                       set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE);
+                       set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE);
+                       set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE);
+                       set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE);
+               }
                set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
                set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
                set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);

diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
index 240ced55586e2c145ca9b22e007d71b8ee97f6e9..4ad78703de6f4c4a1a55bd9d73aa89181cbef9ee 100644 (file)
--- a/arch/loongarch/mm/tlbex.S
+++ b/arch/loongarch/mm/tlbex.S
@@ -190,6 +190,13 @@ nopage_tlb_load:
        jr              t0
 SYM_FUNC_END(handle_tlb_load)
 
+SYM_FUNC_START(handle_tlb_load_ptw)
+       csrwr           t0, LOONGARCH_CSR_KS0
+       csrwr           t1, LOONGARCH_CSR_KS1
+       la_abs          t0, tlb_do_page_fault_0
+       jr              t0
+SYM_FUNC_END(handle_tlb_load_ptw)
+
 SYM_FUNC_START(handle_tlb_store)
        csrwr           t0, EXCEPTION_KS0
        csrwr           t1, EXCEPTION_KS1
@@ -339,6 +346,13 @@ nopage_tlb_store:
        jr              t0
 SYM_FUNC_END(handle_tlb_store)
 
+SYM_FUNC_START(handle_tlb_store_ptw)
+       csrwr           t0, LOONGARCH_CSR_KS0
+       csrwr           t1, LOONGARCH_CSR_KS1
+       la_abs          t0, tlb_do_page_fault_1
+       jr              t0
+SYM_FUNC_END(handle_tlb_store_ptw)
+
 SYM_FUNC_START(handle_tlb_modify)
        csrwr           t0, EXCEPTION_KS0
        csrwr           t1, EXCEPTION_KS1
@@ -486,6 +500,13 @@ nopage_tlb_modify:
        jr              t0
 SYM_FUNC_END(handle_tlb_modify)
 
+SYM_FUNC_START(handle_tlb_modify_ptw)
+       csrwr           t0, LOONGARCH_CSR_KS0
+       csrwr           t1, LOONGARCH_CSR_KS1
+       la_abs          t0, tlb_do_page_fault_1
+       jr              t0
+SYM_FUNC_END(handle_tlb_modify_ptw)
+
 SYM_FUNC_START(handle_tlb_refill)
        csrwr           t0, LOONGARCH_CSR_TLBRSAVE
        csrrd           t0, LOONGARCH_CSR_PGD