Merge tag 'libnvdimm-for-4.19_dax-memory-failure' of gitolite.kernel.org:pub/scm...

[linux-block.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index feba371169ca4c516f71ef7ee9d240f22d32deac..c3bc7e9c9a2acc550ea8aeb68720a8c5a611c610 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -541,18 +541,18 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
 }
 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
 
-static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
-               gfp_t gfp)
+static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
+                       struct page *page, gfp_t gfp)
 {
        struct vm_area_struct *vma = vmf->vma;
        struct mem_cgroup *memcg;
        pgtable_t pgtable;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
-       int ret = 0;
+       vm_fault_t ret = 0;
 
        VM_BUG_ON_PAGE(!PageCompound(page), page);
 
-       if (mem_cgroup_try_charge(page, vma->vm_mm, gfp, &memcg, true)) {
+       if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) {
                put_page(page);
                count_vm_event(THP_FAULT_FALLBACK);
                return VM_FAULT_FALLBACK;
@@ -584,15 +584,15 @@ static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
 
                /* Deliver the page fault to userland */
                if (userfaultfd_missing(vma)) {
-                       int ret;
+                       vm_fault_t ret2;
 
                        spin_unlock(vmf->ptl);
                        mem_cgroup_cancel_charge(page, memcg, true);
                        put_page(page);
                        pte_free(vma->vm_mm, pgtable);
-                       ret = handle_userfault(vmf, VM_UFFD_MISSING);
-                       VM_BUG_ON(ret & VM_FAULT_FALLBACK);
-                       return ret;
+                       ret2 = handle_userfault(vmf, VM_UFFD_MISSING);
+                       VM_BUG_ON(ret2 & VM_FAULT_FALLBACK);
+                       return ret2;
                }
 
                entry = mk_huge_pmd(page, vma->vm_page_prot);
@@ -663,7 +663,7 @@ static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
        return true;
 }
 
-int do_huge_pmd_anonymous_page(struct vm_fault *vmf)
+vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
 {
        struct vm_area_struct *vma = vmf->vma;
        gfp_t gfp;
@@ -682,7 +682,7 @@ int do_huge_pmd_anonymous_page(struct vm_fault *vmf)
                pgtable_t pgtable;
                struct page *zero_page;
                bool set;
-               int ret;
+               vm_fault_t ret;
                pgtable = pte_alloc_one(vma->vm_mm, haddr);
                if (unlikely(!pgtable))
                        return VM_FAULT_OOM;
@@ -762,11 +762,11 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
         * but we need to be consistent with PTEs and architectures that
         * can't support a 'special' bit.
         */
-       BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
+       BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
+                       !pfn_t_devmap(pfn));
        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
                                                (VM_PFNMAP|VM_MIXEDMAP));
        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
-       BUG_ON(!pfn_t_devmap(pfn));
 
        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;
@@ -1118,15 +1118,16 @@ unlock:
        spin_unlock(vmf->ptl);
 }
 
-static int do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, pmd_t orig_pmd,
-               struct page *page)
+static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
+                       pmd_t orig_pmd, struct page *page)
 {
        struct vm_area_struct *vma = vmf->vma;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        struct mem_cgroup *memcg;
        pgtable_t pgtable;
        pmd_t _pmd;
-       int ret = 0, i;
+       int i;
+       vm_fault_t ret = 0;
        struct page **pages;
        unsigned long mmun_start;       /* For mmu_notifiers */
        unsigned long mmun_end;         /* For mmu_notifiers */
@@ -1142,7 +1143,7 @@ static int do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, pmd_t orig_pmd,
                pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma,
                                               vmf->address, page_to_nid(page));
                if (unlikely(!pages[i] ||
-                            mem_cgroup_try_charge(pages[i], vma->vm_mm,
+                            mem_cgroup_try_charge_delay(pages[i], vma->vm_mm,
                                     GFP_KERNEL, &memcg, false))) {
                        if (pages[i])
                                put_page(pages[i]);
@@ -1236,7 +1237,7 @@ out_free_pages:
        goto out;
 }
 
-int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
+vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
 {
        struct vm_area_struct *vma = vmf->vma;
        struct page *page = NULL, *new_page;
@@ -1245,7 +1246,7 @@ int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
        unsigned long mmun_start;       /* For mmu_notifiers */
        unsigned long mmun_end;         /* For mmu_notifiers */
        gfp_t huge_gfp;                 /* for allocation and charge */
-       int ret = 0;
+       vm_fault_t ret = 0;
 
        vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
        VM_BUG_ON_VMA(!vma->anon_vma, vma);
@@ -1312,7 +1313,7 @@ alloc:
                goto out;
        }
 
-       if (unlikely(mem_cgroup_try_charge(new_page, vma->vm_mm,
+       if (unlikely(mem_cgroup_try_charge_delay(new_page, vma->vm_mm,
                                        huge_gfp, &memcg, true))) {
                put_page(new_page);
                split_huge_pmd(vma, vmf->pmd, vmf->address);
@@ -1328,7 +1329,8 @@ alloc:
        if (!page)
                clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR);
        else
-               copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
+               copy_user_huge_page(new_page, page, vmf->address,
+                                   vma, HPAGE_PMD_NR);
        __SetPageUptodate(new_page);
 
        mmun_start = haddr;
@@ -1456,7 +1458,7 @@ out:
 }
 
 /* NUMA hinting page fault entry point for trans huge pmds */
-int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
+vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
 {
        struct vm_area_struct *vma = vmf->vma;
        struct anon_vma *anon_vma = NULL;
@@ -1740,7 +1742,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                } else {
                        if (arch_needs_pgtable_deposit())
                                zap_deposited_table(tlb->mm, pmd);
-                       add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PMD_NR);
+                       add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR);
                }
 
                spin_unlock(ptl);
@@ -2084,11 +2086,13 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
                if (vma_is_dax(vma))
                        return;
                page = pmd_page(_pmd);
+               if (!PageDirty(page) && pmd_dirty(_pmd))
+                       set_page_dirty(page);
                if (!PageReferenced(page) && pmd_young(_pmd))
                        SetPageReferenced(page);
                page_remove_rmap(page, true);
                put_page(page);
-               add_mm_counter(mm, MM_FILEPAGES, -HPAGE_PMD_NR);
+               add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
                return;
        } else if (is_huge_zero_pmd(*pmd)) {
                /*