mm: move the powerpc hugepd code to mm/gup.c

While only powerpc supports the hugepd case, the code is pretty generic
and I'd like to keep all GUP internals in one place.

Link: http://lkml.kernel.org/r/20190625143715.1689-15-hch@lst.de
Signed-off-by: Christoph Hellwig <hch@lst.de>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Miller <davem@davemloft.net>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@mellanox.com>
Cc: Khalid Aziz <khalid.aziz@oracle.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 959866c156ded29ae50896251dca3b7689478d37..24a41f919309ecb54fbeb951815e667560658072 100644 (file)
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -125,6 +125,7 @@ config PPC
        select ARCH_HAS_FORTIFY_SOURCE
        select ARCH_HAS_GCOV_PROFILE_ALL
        select ARCH_HAS_KCOV
+       select ARCH_HAS_HUGEPD                  if HUGETLB_PAGE
        select ARCH_HAS_MMIOWB                  if PPC64
        select ARCH_HAS_PHYS_TO_DMA
        select ARCH_HAS_PMEM_API                if PPC64

diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index b5d92dc32844bbde507d6a8d4a89915c205aad85..51716c11d0fb35c4edf3729ce96cb7f029578aa3 100644 (file)
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -511,13 +511,6 @@ retry:
        return page;
 }
 
-static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
-                                     unsigned long sz)
-{
-       unsigned long __boundary = (addr + sz) & ~(sz-1);
-       return (__boundary - 1 < end - 1) ? __boundary : end;
-}
-
 #ifdef CONFIG_PPC_MM_SLICES
 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
                                        unsigned long len, unsigned long pgoff,
@@ -665,68 +658,3 @@ void flush_dcache_icache_hugepage(struct page *page)
                }
        }
 }
-
-static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
-                      unsigned long end, int write, struct page **pages, int *nr)
-{
-       unsigned long pte_end;
-       struct page *head, *page;
-       pte_t pte;
-       int refs;
-
-       pte_end = (addr + sz) & ~(sz-1);
-       if (pte_end < end)
-               end = pte_end;
-
-       pte = READ_ONCE(*ptep);
-
-       if (!pte_access_permitted(pte, write))
-               return 0;
-
-       /* hugepages are never "special" */
-       VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
-
-       refs = 0;
-       head = pte_page(pte);
-
-       page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
-       do {
-               VM_BUG_ON(compound_head(page) != head);
-               pages[*nr] = page;
-               (*nr)++;
-               page++;
-               refs++;
-       } while (addr += PAGE_SIZE, addr != end);
-
-       if (!page_cache_add_speculative(head, refs)) {
-               *nr -= refs;
-               return 0;
-       }
-
-       if (unlikely(pte_val(pte) != pte_val(*ptep))) {
-               /* Could be optimized better */
-               *nr -= refs;
-               while (refs--)
-                       put_page(head);
-               return 0;
-       }
-
-       return 1;
-}
-
-int gup_huge_pd(hugepd_t hugepd, unsigned long addr, unsigned int pdshift,
-               unsigned long end, int write, struct page **pages, int *nr)
-{
-       pte_t *ptep;
-       unsigned long sz = 1UL << hugepd_shift(hugepd);
-       unsigned long next;
-
-       ptep = hugepte_offset(hugepd, addr, pdshift);
-       do {
-               next = hugepte_addr_end(addr, end, sz);
-               if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
-                       return 0;
-       } while (ptep++, addr = next, addr != end);
-
-       return 1;
-}

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index f895a79c6f5cb4fee5d8f1f46920a2fc1bb00224..edfca427831928abc6a072c4e0d94daee6d7811a 100644 (file)
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -16,29 +16,11 @@ struct user_struct;
 struct mmu_gather;
 
 #ifndef is_hugepd
-/*
- * Some architectures requires a hugepage directory format that is
- * required to support multiple hugepage sizes. For example
- * a4fe3ce76 "powerpc/mm: Allow more flexible layouts for hugepage pagetables"
- * introduced the same on powerpc. This allows for a more flexible hugepage
- * pagetable layout.
- */
 typedef struct { unsigned long pd; } hugepd_t;
 #define is_hugepd(hugepd) (0)
 #define __hugepd(x) ((hugepd_t) { (x) })
-static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
-                             unsigned pdshift, unsigned long end,
-                             int write, struct page **pages, int *nr)
-{
-       return 0;
-}
-#else
-extern int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
-                      unsigned pdshift, unsigned long end,
-                      int write, struct page **pages, int *nr);
 #endif
 
-
 #ifdef CONFIG_HUGETLB_PAGE
 
 #include <linux/mempolicy.h>

diff --git a/mm/Kconfig b/mm/Kconfig
index 48840b28482bc2d656d7276de3c301722b850467..0b4352557dd54643c05ccc52e6c48de6b42a5a06 100644 (file)
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -769,4 +769,14 @@ config GUP_GET_PTE_LOW_HIGH
 config ARCH_HAS_PTE_SPECIAL
        bool
 
+#
+# Some architectures require a special hugepage directory format that is
+# required to support multiple hugepage sizes. For example a4fe3ce76
+# "powerpc/mm: Allow more flexible layouts for hugepage pagetables"
+# introduced it on powerpc.  This allows for a more flexible hugepage
+# pagetable layouts.
+#
+config ARCH_HAS_HUGEPD
+       bool
+
 endmenu

diff --git a/mm/gup.c b/mm/gup.c
index 9d68cef2fa908cec86a6884fefbd963520a229ce..2f8bf7a71c745a3e64fd058a0b7574d9603c26c8 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1966,6 +1966,88 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
 }
 #endif
 
+#ifdef CONFIG_ARCH_HAS_HUGEPD
+static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
+                                     unsigned long sz)
+{
+       unsigned long __boundary = (addr + sz) & ~(sz-1);
+       return (__boundary - 1 < end - 1) ? __boundary : end;
+}
+
+static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
+                      unsigned long end, int write, struct page **pages, int *nr)
+{
+       unsigned long pte_end;
+       struct page *head, *page;
+       pte_t pte;
+       int refs;
+
+       pte_end = (addr + sz) & ~(sz-1);
+       if (pte_end < end)
+               end = pte_end;
+
+       pte = READ_ONCE(*ptep);
+
+       if (!pte_access_permitted(pte, write))
+               return 0;
+
+       /* hugepages are never "special" */
+       VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
+
+       refs = 0;
+       head = pte_page(pte);
+
+       page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
+       do {
+               VM_BUG_ON(compound_head(page) != head);
+               pages[*nr] = page;
+               (*nr)++;
+               page++;
+               refs++;
+       } while (addr += PAGE_SIZE, addr != end);
+
+       if (!page_cache_add_speculative(head, refs)) {
+               *nr -= refs;
+               return 0;
+       }
+
+       if (unlikely(pte_val(pte) != pte_val(*ptep))) {
+               /* Could be optimized better */
+               *nr -= refs;
+               while (refs--)
+                       put_page(head);
+               return 0;
+       }
+
+       return 1;
+}
+
+static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
+               unsigned int pdshift, unsigned long end, int write,
+               struct page **pages, int *nr)
+{
+       pte_t *ptep;
+       unsigned long sz = 1UL << hugepd_shift(hugepd);
+       unsigned long next;
+
+       ptep = hugepte_offset(hugepd, addr, pdshift);
+       do {
+               next = hugepte_addr_end(addr, end, sz);
+               if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
+                       return 0;
+       } while (ptep++, addr = next, addr != end);
+
+       return 1;
+}
+#else
+static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
+               unsigned pdshift, unsigned long end, int write,
+               struct page **pages, int *nr)
+{
+       return 0;
+}
+#endif /* CONFIG_ARCH_HAS_HUGEPD */
+
 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
                unsigned long end, unsigned int flags, struct page **pages, int *nr)
 {