thp: transparent hugepage core

[linux-2.6-block.git] / mm / memory.c

diff --git a/mm/memory.c b/mm/memory.c
index 60e1c68d821845f4bf75fe23e8d070ccc80eb360..c50a195041ecb1545a49732c724f4720c5f0bcb9 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -726,9 +726,9 @@ out_set_pte:
        return 0;
 }
 
-static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-               pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
-               unsigned long addr, unsigned long end)
+int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+                  pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
+                  unsigned long addr, unsigned long end)
 {
        pte_t *orig_src_pte, *orig_dst_pte;
        pte_t *src_pte, *dst_pte;
@@ -802,6 +802,16 @@ static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src
        src_pmd = pmd_offset(src_pud, addr);
        do {
                next = pmd_addr_end(addr, end);
+               if (pmd_trans_huge(*src_pmd)) {
+                       int err;
+                       err = copy_huge_pmd(dst_mm, src_mm,
+                                           dst_pmd, src_pmd, addr, vma);
+                       if (err == -ENOMEM)
+                               return -ENOMEM;
+                       if (!err)
+                               continue;
+                       /* fall through */
+               }
                if (pmd_none_or_clear_bad(src_pmd))
                        continue;
                if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
@@ -1004,6 +1014,15 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
+               if (pmd_trans_huge(*pmd)) {
+                       if (next-addr != HPAGE_PMD_SIZE)
+                               split_huge_page_pmd(vma->vm_mm, pmd);
+                       else if (zap_huge_pmd(tlb, vma, pmd)) {
+                               (*zap_work)--;
+                               continue;
+                       }
+                       /* fall through */
+               }
                if (pmd_none_or_clear_bad(pmd)) {
                        (*zap_work)--;
                        continue;
@@ -1280,11 +1299,27 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
        pmd = pmd_offset(pud, address);
        if (pmd_none(*pmd))
                goto no_page_table;
-       if (pmd_huge(*pmd)) {
+       if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) {
                BUG_ON(flags & FOLL_GET);
                page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
                goto out;
        }
+       if (pmd_trans_huge(*pmd)) {
+               spin_lock(&mm->page_table_lock);
+               if (likely(pmd_trans_huge(*pmd))) {
+                       if (unlikely(pmd_trans_splitting(*pmd))) {
+                               spin_unlock(&mm->page_table_lock);
+                               wait_split_huge_page(vma->anon_vma, pmd);
+                       } else {
+                               page = follow_trans_huge_pmd(mm, address,
+                                                            pmd, flags);
+                               spin_unlock(&mm->page_table_lock);
+                               goto out;
+                       }
+               } else
+                       spin_unlock(&mm->page_table_lock);
+               /* fall through */
+       }
        if (unlikely(pmd_bad(*pmd)))
                goto no_page_table;
 
@@ -3179,9 +3214,9 @@ static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  * but allow concurrent faults), and pte mapped but not yet locked.
  * We return with mmap_sem still held, but pte unmapped and unlocked.
  */
-static inline int handle_pte_fault(struct mm_struct *mm,
-               struct vm_area_struct *vma, unsigned long address,
-               pte_t *pte, pmd_t *pmd, unsigned int flags)
+int handle_pte_fault(struct mm_struct *mm,
+                    struct vm_area_struct *vma, unsigned long address,
+                    pte_t *pte, pmd_t *pmd, unsigned int flags)
 {
        pte_t entry;
        spinlock_t *ptl;
@@ -3260,9 +3295,40 @@ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        pmd = pmd_alloc(mm, pud, address);
        if (!pmd)
                return VM_FAULT_OOM;
-       pte = pte_alloc_map(mm, vma, pmd, address);
-       if (!pte)
+       if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) {
+               if (!vma->vm_ops)
+                       return do_huge_pmd_anonymous_page(mm, vma, address,
+                                                         pmd, flags);
+       } else {
+               pmd_t orig_pmd = *pmd;
+               barrier();
+               if (pmd_trans_huge(orig_pmd)) {
+                       if (flags & FAULT_FLAG_WRITE &&
+                           !pmd_write(orig_pmd) &&
+                           !pmd_trans_splitting(orig_pmd))
+                               return do_huge_pmd_wp_page(mm, vma, address,
+                                                          pmd, orig_pmd);
+                       return 0;
+               }
+       }
+
+       /*
+        * Use __pte_alloc instead of pte_alloc_map, because we can't
+        * run pte_offset_map on the pmd, if an huge pmd could
+        * materialize from under us from a different thread.
+        */
+       if (unlikely(__pte_alloc(mm, vma, pmd, address)))
                return VM_FAULT_OOM;
+       /* if an huge pmd materialized from under us just retry later */
+       if (unlikely(pmd_trans_huge(*pmd)))
+               return 0;
+       /*
+        * A regular pmd is established and it can't morph into a huge pmd
+        * from under us anymore at this point because we hold the mmap_sem
+        * read mode and khugepaged takes it in write mode. So now it's
+        * safe to run pte_offset_map().
+        */
+       pte = pte_offset_map(pmd, address);
 
        return handle_pte_fault(mm, vma, address, pte, pmd, flags);
 }