mm, hugetlb: fix racy resv_huge_pages underflow on UFFDIO_COPY

[linux-block.git] / mm / migrate.c

diff --git a/mm/migrate.c b/mm/migrate.c
index 6b37d00890ca59d7859f5af77a81e5abf551405f..8fc766e52e527e127d17f0e1c1d6a8f54d0b15ee 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -226,8 +226,10 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
 
 #ifdef CONFIG_HUGETLB_PAGE
                if (PageHuge(new)) {
+                       unsigned int shift = huge_page_shift(hstate_vma(vma));
+
                        pte = pte_mkhuge(pte);
-                       pte = arch_make_huge_pte(pte, vma, new, 0);
+                       pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
                        set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
                        if (PageAnon(new))
                                hugepage_add_anon_rmap(new, vma, pvmw.address);
@@ -295,6 +297,7 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
                goto out;
 
        page = migration_entry_to_page(entry);
+       page = compound_head(page);
 
        /*
         * Once page cache replacement of page migration started, page_count
@@ -550,7 +553,7 @@ static void __copy_gigantic_page(struct page *dst, struct page *src,
        }
 }
 
-static void copy_huge_page(struct page *dst, struct page *src)
+void copy_huge_page(struct page *dst, struct page *src)
 {
        int i;
        int nr_pages;
@@ -625,7 +628,10 @@ void migrate_page_states(struct page *newpage, struct page *page)
        if (PageSwapCache(page))
                ClearPageSwapCache(page);
        ClearPagePrivate(page);
-       set_page_private(page, 0);
+
+       /* page->private contains hugetlb specific flags */
+       if (!PageHuge(page))
+               set_page_private(page, 0);
 
        /*
         * If any waiters have accumulated on the new page then
@@ -1833,8 +1839,8 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
                struct page *page;
                int err = -EFAULT;
 
-               vma = find_vma(mm, addr);
-               if (!vma || addr < vma->vm_start)
+               vma = vma_lookup(mm, addr);
+               if (!vma)
                        goto set_status;
 
                /* FOLL_DUMP to ignore special (like zero) pages */
@@ -2779,11 +2785,11 @@ restore:
  *
  * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
  * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
- * allowing the caller to allocate device memory for those unback virtual
- * address.  For this the caller simply has to allocate device memory and
+ * allowing the caller to allocate device memory for those unbacked virtual
+ * addresses.  For this the caller simply has to allocate device memory and
  * properly set the destination entry like for regular migration.  Note that
- * this can still fails and thus inside the device driver must check if the
- * migration was successful for those entries after calling migrate_vma_pages()
+ * this can still fail, and thus inside the device driver you must check if the
+ * migration was successful for those entries after calling migrate_vma_pages(),
  * just like for regular migration.
  *
  * After that, the callers must call migrate_vma_pages() to go over each entry