mm: pagewalk: add 'depth' parameter to pte_hole

[linux-2.6-block.git] / mm / pagewalk.c

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 6732fc7ac4c8cbb38a4ba2971e9355b39c6eb73a..5895ce4f1a8563eeffb71aa67f86addf58a9b74f 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -4,6 +4,22 @@
 #include <linux/sched.h>
 #include <linux/hugetlb.h>
 
+/*
+ * We want to know the real level where a entry is located ignoring any
+ * folding of levels which may be happening. For example if p4d is folded then
+ * a missing entry found at level 1 (p4d) is actually at level 0 (pgd).
+ */
+static int real_depth(int depth)
+{
+       if (depth == 3 && PTRS_PER_PMD == 1)
+               depth = 2;
+       if (depth == 2 && PTRS_PER_PUD == 1)
+               depth = 1;
+       if (depth == 1 && PTRS_PER_P4D == 1)
+               depth = 0;
+       return depth;
+}
+
 static int walk_pte_range_inner(pte_t *pte, unsigned long addr,
                                unsigned long end, struct mm_walk *walk)
 {
@@ -49,6 +65,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
        unsigned long next;
        const struct mm_walk_ops *ops = walk->ops;
        int err = 0;
+       int depth = real_depth(3);
 
        pmd = pmd_offset(pud, addr);
        do {
@@ -56,7 +73,7 @@ again:
                next = pmd_addr_end(addr, end);
                if (pmd_none(*pmd) || (!walk->vma && !walk->no_vma)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, depth, walk);
                        if (err)
                                break;
                        continue;
@@ -106,6 +123,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
        unsigned long next;
        const struct mm_walk_ops *ops = walk->ops;
        int err = 0;
+       int depth = real_depth(2);
 
        pud = pud_offset(p4d, addr);
        do {
@@ -113,7 +131,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
                next = pud_addr_end(addr, end);
                if (pud_none(*pud) || (!walk->vma && !walk->no_vma)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, depth, walk);
                        if (err)
                                break;
                        continue;
@@ -154,13 +172,14 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
        unsigned long next;
        const struct mm_walk_ops *ops = walk->ops;
        int err = 0;
+       int depth = real_depth(1);
 
        p4d = p4d_offset(pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(p4d)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, depth, walk);
                        if (err)
                                break;
                        continue;
@@ -192,7 +211,7 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, 0, walk);
                        if (err)
                                break;
                        continue;
@@ -239,7 +258,7 @@ static int walk_hugetlb_range(unsigned long addr, unsigned long end,
                if (pte)
                        err = ops->hugetlb_entry(pte, hmask, addr, next, walk);
                else if (ops->pte_hole)
-                       err = ops->pte_hole(addr, next, walk);
+                       err = ops->pte_hole(addr, next, -1, walk);
 
                if (err)
                        break;
@@ -283,7 +302,7 @@ static int walk_page_test(unsigned long start, unsigned long end,
        if (vma->vm_flags & VM_PFNMAP) {
                int err = 1;
                if (ops->pte_hole)
-                       err = ops->pte_hole(start, end, walk);
+                       err = ops->pte_hole(start, end, -1, walk);
                return err ? err : 1;
        }
        return 0;