mm: make vma_merge() and split_vma() internal
[linux-2.6-block.git] / mm / mmap.c
index b56a7f0c9f856509a7a17eac92c71f2ce30b659c..a516f2412f79739ee2c22863d3214f474b656aef 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -583,11 +583,12 @@ again:
  * dup_anon_vma() - Helper function to duplicate anon_vma
  * @dst: The destination VMA
  * @src: The source VMA
+ * @dup: Pointer to the destination VMA when successful.
  *
  * Returns: 0 on success.
  */
 static inline int dup_anon_vma(struct vm_area_struct *dst,
-                              struct vm_area_struct *src)
+               struct vm_area_struct *src, struct vm_area_struct **dup)
 {
        /*
         * Easily overlooked: when mprotect shifts the boundary, make sure the
@@ -595,9 +596,15 @@ static inline int dup_anon_vma(struct vm_area_struct *dst,
         * anon pages imported.
         */
        if (src->anon_vma && !dst->anon_vma) {
+               int ret;
+
                vma_assert_write_locked(dst);
                dst->anon_vma = src->anon_vma;
-               return anon_vma_clone(dst, src);
+               ret = anon_vma_clone(dst, src);
+               if (ret)
+                       return ret;
+
+               *dup = dst;
        }
 
        return 0;
@@ -624,6 +631,7 @@ int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
               unsigned long start, unsigned long end, pgoff_t pgoff,
               struct vm_area_struct *next)
 {
+       struct vm_area_struct *anon_dup = NULL;
        bool remove_next = false;
        struct vma_prepare vp;
 
@@ -633,7 +641,7 @@ int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
 
                remove_next = true;
                vma_start_write(next);
-               ret = dup_anon_vma(vma, next);
+               ret = dup_anon_vma(vma, next, &anon_dup);
                if (ret)
                        return ret;
        }
@@ -661,6 +669,8 @@ int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
        return 0;
 
 nomem:
+       if (anon_dup)
+               unlink_anon_vmas(anon_dup);
        return -ENOMEM;
 }
 
@@ -860,6 +870,7 @@ struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
 {
        struct vm_area_struct *curr, *next, *res;
        struct vm_area_struct *vma, *adjust, *remove, *remove2;
+       struct vm_area_struct *anon_dup = NULL;
        struct vma_prepare vp;
        pgoff_t vma_pgoff;
        int err = 0;
@@ -927,18 +938,23 @@ struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
                vma_start_write(next);
                remove = next;                          /* case 1 */
                vma_end = next->vm_end;
-               err = dup_anon_vma(prev, next);
+               err = dup_anon_vma(prev, next, &anon_dup);
                if (curr) {                             /* case 6 */
                        vma_start_write(curr);
                        remove = curr;
                        remove2 = next;
+                       /*
+                        * Note that the dup_anon_vma below cannot overwrite err
+                        * since the first caller would do nothing unless next
+                        * has an anon_vma.
+                        */
                        if (!next->anon_vma)
-                               err = dup_anon_vma(prev, curr);
+                               err = dup_anon_vma(prev, curr, &anon_dup);
                }
        } else if (merge_prev) {                        /* case 2 */
                if (curr) {
                        vma_start_write(curr);
-                       err = dup_anon_vma(prev, curr);
+                       err = dup_anon_vma(prev, curr, &anon_dup);
                        if (end == curr->vm_end) {      /* case 7 */
                                remove = curr;
                        } else {                        /* case 5 */
@@ -954,7 +970,7 @@ struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
                        vma_end = addr;
                        adjust = next;
                        adj_start = -(prev->vm_end - addr);
-                       err = dup_anon_vma(next, prev);
+                       err = dup_anon_vma(next, prev, &anon_dup);
                } else {
                        /*
                         * Note that cases 3 and 8 are the ONLY ones where prev
@@ -968,14 +984,14 @@ struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
                                vma_pgoff = curr->vm_pgoff;
                                vma_start_write(curr);
                                remove = curr;
-                               err = dup_anon_vma(next, curr);
+                               err = dup_anon_vma(next, curr, &anon_dup);
                        }
                }
        }
 
        /* Error in anon_vma clone. */
        if (err)
-               return NULL;
+               goto anon_vma_fail;
 
        if (vma_start < vma->vm_start || vma_end > vma->vm_end)
                vma_expanded = true;
@@ -988,7 +1004,7 @@ struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
        }
 
        if (vma_iter_prealloc(vmi, vma))
-               return NULL;
+               goto prealloc_fail;
 
        init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
        VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
@@ -1016,6 +1032,15 @@ struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
        vma_complete(&vp, vmi, mm);
        khugepaged_enter_vma(res, vm_flags);
        return res;
+
+prealloc_fail:
+       if (anon_dup)
+               unlink_anon_vmas(anon_dup);
+
+anon_vma_fail:
+       vma_iter_set(vmi, addr);
+       vma_iter_load(vmi);
+       return NULL;
 }
 
 /*
@@ -2159,8 +2184,6 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon
 #else
 int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
 {
-       if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
-               return -EINVAL;
        return expand_downwards(vma, address);
 }
 
@@ -2323,8 +2346,8 @@ static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
  * has already been checked or doesn't make sense to fail.
  * VMA Iterator will point to the end VMA.
  */
-int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
-               unsigned long addr, int new_below)
+static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
+                      unsigned long addr, int new_below)
 {
        struct vma_prepare vp;
        struct vm_area_struct *new;
@@ -2405,8 +2428,8 @@ out_free_vma:
  * Split a vma into two pieces at address 'addr', a new vma is allocated
  * either for the first part or the tail.
  */
-int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
-             unsigned long addr, int new_below)
+static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
+                    unsigned long addr, int new_below)
 {
        if (vma->vm_mm->map_count >= sysctl_max_map_count)
                return -ENOMEM;
@@ -2414,6 +2437,54 @@ int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
        return __split_vma(vmi, vma, addr, new_below);
 }
 
+/*
+ * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
+ * context and anonymous VMA name within the range [start, end).
+ *
+ * As a result, we might be able to merge the newly modified VMA range with an
+ * adjacent VMA with identical properties.
+ *
+ * If no merge is possible and the range does not span the entirety of the VMA,
+ * we then need to split the VMA to accommodate the change.
+ *
+ * The function returns either the merged VMA, the original VMA if a split was
+ * required instead, or an error if the split failed.
+ */
+struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
+                                 struct vm_area_struct *prev,
+                                 struct vm_area_struct *vma,
+                                 unsigned long start, unsigned long end,
+                                 unsigned long vm_flags,
+                                 struct mempolicy *policy,
+                                 struct vm_userfaultfd_ctx uffd_ctx,
+                                 struct anon_vma_name *anon_name)
+{
+       pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
+       struct vm_area_struct *merged;
+
+       merged = vma_merge(vmi, vma->vm_mm, prev, start, end, vm_flags,
+                          vma->anon_vma, vma->vm_file, pgoff, policy,
+                          uffd_ctx, anon_name);
+       if (merged)
+               return merged;
+
+       if (vma->vm_start < start) {
+               int err = split_vma(vmi, vma, start, 1);
+
+               if (err)
+                       return ERR_PTR(err);
+       }
+
+       if (vma->vm_end > end) {
+               int err = split_vma(vmi, vma, end, 0);
+
+               if (err)
+                       return ERR_PTR(err);
+       }
+
+       return vma;
+}
+
 /*
  * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
  * @vmi: The vma iterator
@@ -3143,13 +3214,13 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
        if (!len)
                return 0;
 
-       if (mmap_write_lock_killable(mm))
-               return -EINTR;
-
        /* Until we need other flags, refuse anything except VM_EXEC. */
        if ((flags & (~VM_EXEC)) != 0)
                return -EINVAL;
 
+       if (mmap_write_lock_killable(mm))
+               return -EINTR;
+
        ret = check_brk_limits(addr, len);
        if (ret)
                goto limits_failed;
@@ -3278,7 +3349,8 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
        }
 
        if (vma_link(mm, vma)) {
-               vm_unacct_memory(charged);
+               if (vma->vm_flags & VM_ACCOUNT)
+                       vm_unacct_memory(charged);
                return -ENOMEM;
        }