arch/riscv/include/asm/pgalloc.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com>
   4  * Copyright (C) 2012 Regents of the University of California
   5  */
   6
   7 #ifndef _ASM_RISCV_PGALLOC_H
   8 #define _ASM_RISCV_PGALLOC_H
   9
  10 #include <linux/mm.h>
  11 #include <asm/tlb.h>
  12
  13 #ifdef CONFIG_MMU
  14 #define __HAVE_ARCH_PUD_ALLOC_ONE
  15 #define __HAVE_ARCH_PUD_FREE
  16 #include <asm-generic/pgalloc.h>
  17
  18 static inline void pmd_populate_kernel(struct mm_struct *mm,
  19         pmd_t *pmd, pte_t *pte)
  20 {
  21         unsigned long pfn = virt_to_pfn(pte);
  22
  23         set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  24 }
  25
  26 static inline void pmd_populate(struct mm_struct *mm,
  27         pmd_t *pmd, pgtable_t pte)
  28 {
  29         unsigned long pfn = virt_to_pfn(page_address(pte));
  30
  31         set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  32 }
  33
  34 #ifndef __PAGETABLE_PMD_FOLDED
  35 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
  36 {
  37         unsigned long pfn = virt_to_pfn(pmd);
  38
  39         set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  40 }
  41
  42 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
  43 {
  44         if (pgtable_l4_enabled) {
  45                 unsigned long pfn = virt_to_pfn(pud);
  46
  47                 set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  48         }
  49 }
  50
  51 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,
  52                                      pud_t *pud)
  53 {
  54         if (pgtable_l4_enabled) {
  55                 unsigned long pfn = virt_to_pfn(pud);
  56
  57                 set_p4d_safe(p4d,
  58                              __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  59         }
  60 }
  61
  62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
  63 {
  64         if (pgtable_l5_enabled) {
  65                 unsigned long pfn = virt_to_pfn(p4d);
  66
  67                 set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  68         }
  69 }
  70
  71 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,
  72                                      p4d_t *p4d)
  73 {
  74         if (pgtable_l5_enabled) {
  75                 unsigned long pfn = virt_to_pfn(p4d);
  76
  77                 set_pgd_safe(pgd,
  78                              __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
  79         }
  80 }
  81
  82 #define pud_alloc_one pud_alloc_one
  83 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
  84 {
  85         if (pgtable_l4_enabled)
  86                 return __pud_alloc_one(mm, addr);
  87
  88         return NULL;
  89 }
  90
  91 #define pud_free pud_free
  92 static inline void pud_free(struct mm_struct *mm, pud_t *pud)
  93 {
  94         if (pgtable_l4_enabled)
  95                 __pud_free(mm, pud);
  96 }
  97
  98 #define __pud_free_tlb(tlb, pud, addr)                                  \
  99 do {                                                                    \
 100         if (pgtable_l4_enabled) {                                       \
 101                 pagetable_pud_dtor(virt_to_ptdesc(pud));                \
 102                 tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pud));     \
 103         }                                                               \
 104 } while (0)
 105
 106 #define p4d_alloc_one p4d_alloc_one
 107 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
 108 {
 109         if (pgtable_l5_enabled) {
 110                 gfp_t gfp = GFP_PGTABLE_USER;
 111
 112                 if (mm == &init_mm)
 113                         gfp = GFP_PGTABLE_KERNEL;
 114                 return (p4d_t *)get_zeroed_page(gfp);
 115         }
 116
 117         return NULL;
 118 }
 119
 120 static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)
 121 {
 122         BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
 123         free_page((unsigned long)p4d);
 124 }
 125
 126 #define p4d_free p4d_free
 127 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
 128 {
 129         if (pgtable_l5_enabled)
 130                 __p4d_free(mm, p4d);
 131 }
 132
 133 #define __p4d_free_tlb(tlb, p4d, addr)                                  \
 134 do {                                                                    \
 135         if (pgtable_l5_enabled)                                         \
 136                 tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(p4d));     \
 137 } while (0)
 138 #endif /* __PAGETABLE_PMD_FOLDED */
 139
 140 static inline void sync_kernel_mappings(pgd_t *pgd)
 141 {
 142         memcpy(pgd + USER_PTRS_PER_PGD,
 143                init_mm.pgd + USER_PTRS_PER_PGD,
 144                (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
 145 }
 146
 147 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 148 {
 149         pgd_t *pgd;
 150
 151         pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
 152         if (likely(pgd != NULL)) {
 153                 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
 154                 /* Copy kernel mappings */
 155                 sync_kernel_mappings(pgd);
 156         }
 157         return pgd;
 158 }
 159
 160 #ifndef __PAGETABLE_PMD_FOLDED
 161
 162 #define __pmd_free_tlb(tlb, pmd, addr)                          \
 163 do {                                                            \
 164         pagetable_pmd_dtor(virt_to_ptdesc(pmd));                \
 165         tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pmd));     \
 166 } while (0)
 167
 168 #endif /* __PAGETABLE_PMD_FOLDED */
 169
 170 #define __pte_free_tlb(tlb, pte, buf)                   \
 171 do {                                                    \
 172         pagetable_pte_dtor(page_ptdesc(pte));           \
 173         tlb_remove_page_ptdesc((tlb), page_ptdesc(pte));\
 174 } while (0)
 175 #endif /* CONFIG_MMU */
 176
 177 #endif /* _ASM_RISCV_PGALLOC_H */