mm/hugetlb: fix races when looking up a CONT-PTE/PMD size hugetlb page

[linux-2.6-block.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 2480ba627aa5e3e1f4a465897f28a509a2aed32a..9564bf817e6a8d92999d2653304abf406a3f96d2 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3420,6 +3420,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
 {
        int i, nid = page_to_nid(page);
        struct hstate *target_hstate;
+       struct page *subpage;
        int rc = 0;
 
        target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);
@@ -3453,15 +3454,16 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
        mutex_lock(&target_hstate->resize_lock);
        for (i = 0; i < pages_per_huge_page(h);
                                i += pages_per_huge_page(target_hstate)) {
+               subpage = nth_page(page, i);
                if (hstate_is_gigantic(target_hstate))
-                       prep_compound_gigantic_page_for_demote(page + i,
+                       prep_compound_gigantic_page_for_demote(subpage,
                                                        target_hstate->order);
                else
-                       prep_compound_page(page + i, target_hstate->order);
-               set_page_private(page + i, 0);
-               set_page_refcounted(page + i);
-               prep_new_huge_page(target_hstate, page + i, nid);
-               put_page(page + i);
+                       prep_compound_page(subpage, target_hstate->order);
+               set_page_private(subpage, 0);
+               set_page_refcounted(subpage);
+               prep_new_huge_page(target_hstate, subpage, nid);
+               put_page(subpage);
        }
        mutex_unlock(&target_hstate->resize_lock);
 
@@ -6041,7 +6043,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
        if (!huge_pte_none_mostly(huge_ptep_get(dst_pte)))
                goto out_release_unlock;
 
-       if (vm_shared) {
+       if (page_in_pagecache) {
                page_dup_file_rmap(page, true);
        } else {
                ClearHPageRestoreReserve(page);
@@ -6944,12 +6946,13 @@ follow_huge_pd(struct vm_area_struct *vma,
 }
 
 struct page * __weak
-follow_huge_pmd(struct mm_struct *mm, unsigned long address,
-               pmd_t *pmd, int flags)
+follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address, int flags)
 {
+       struct hstate *h = hstate_vma(vma);
+       struct mm_struct *mm = vma->vm_mm;
        struct page *page = NULL;
        spinlock_t *ptl;
-       pte_t pte;
+       pte_t *ptep, pte;
 
        /*
         * FOLL_PIN is not supported for follow_page(). Ordinary GUP goes via
@@ -6959,17 +6962,15 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address,
                return NULL;
 
 retry:
-       ptl = pmd_lockptr(mm, pmd);
-       spin_lock(ptl);
-       /*
-        * make sure that the address range covered by this pmd is not
-        * unmapped from other threads.
-        */
-       if (!pmd_huge(*pmd))
-               goto out;
-       pte = huge_ptep_get((pte_t *)pmd);
+       ptep = huge_pte_offset(mm, address, huge_page_size(h));
+       if (!ptep)
+               return NULL;
+
+       ptl = huge_pte_lock(h, mm, ptep);
+       pte = huge_ptep_get(ptep);
        if (pte_present(pte)) {
-               page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
+               page = pte_page(pte) +
+                       ((address & ~huge_page_mask(h)) >> PAGE_SHIFT);
                /*
                 * try_grab_page() should always succeed here, because: a) we
                 * hold the pmd (ptl) lock, and b) we've just checked that the
@@ -6985,7 +6986,7 @@ retry:
        } else {
                if (is_hugetlb_entry_migration(pte)) {
                        spin_unlock(ptl);
-                       __migration_entry_wait_huge((pte_t *)pmd, ptl);
+                       __migration_entry_wait_huge(ptep, ptl);
                        goto retry;
                }
                /*