Merge tag 'vfio-v5.3-rc1' of git://github.com/awilliam/linux-vfio
[linux-2.6-block.git] / mm / migrate.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Memory Migration functionality - linux/mm/migrate.c
4  *
5  * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
6  *
7  * Page migration was first developed in the context of the memory hotplug
8  * project. The main authors of the migration code are:
9  *
10  * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
11  * Hirokazu Takahashi <taka@valinux.co.jp>
12  * Dave Hansen <haveblue@us.ibm.com>
13  * Christoph Lameter
14  */
15
16 #include <linux/migrate.h>
17 #include <linux/export.h>
18 #include <linux/swap.h>
19 #include <linux/swapops.h>
20 #include <linux/pagemap.h>
21 #include <linux/buffer_head.h>
22 #include <linux/mm_inline.h>
23 #include <linux/nsproxy.h>
24 #include <linux/pagevec.h>
25 #include <linux/ksm.h>
26 #include <linux/rmap.h>
27 #include <linux/topology.h>
28 #include <linux/cpu.h>
29 #include <linux/cpuset.h>
30 #include <linux/writeback.h>
31 #include <linux/mempolicy.h>
32 #include <linux/vmalloc.h>
33 #include <linux/security.h>
34 #include <linux/backing-dev.h>
35 #include <linux/compaction.h>
36 #include <linux/syscalls.h>
37 #include <linux/compat.h>
38 #include <linux/hugetlb.h>
39 #include <linux/hugetlb_cgroup.h>
40 #include <linux/gfp.h>
41 #include <linux/pfn_t.h>
42 #include <linux/memremap.h>
43 #include <linux/userfaultfd_k.h>
44 #include <linux/balloon_compaction.h>
45 #include <linux/mmu_notifier.h>
46 #include <linux/page_idle.h>
47 #include <linux/page_owner.h>
48 #include <linux/sched/mm.h>
49 #include <linux/ptrace.h>
50
51 #include <asm/tlbflush.h>
52
53 #define CREATE_TRACE_POINTS
54 #include <trace/events/migrate.h>
55
56 #include "internal.h"
57
58 /*
59  * migrate_prep() needs to be called before we start compiling a list of pages
60  * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
61  * undesirable, use migrate_prep_local()
62  */
63 int migrate_prep(void)
64 {
65         /*
66          * Clear the LRU lists so pages can be isolated.
67          * Note that pages may be moved off the LRU after we have
68          * drained them. Those pages will fail to migrate like other
69          * pages that may be busy.
70          */
71         lru_add_drain_all();
72
73         return 0;
74 }
75
76 /* Do the necessary work of migrate_prep but not if it involves other CPUs */
77 int migrate_prep_local(void)
78 {
79         lru_add_drain();
80
81         return 0;
82 }
83
84 int isolate_movable_page(struct page *page, isolate_mode_t mode)
85 {
86         struct address_space *mapping;
87
88         /*
89          * Avoid burning cycles with pages that are yet under __free_pages(),
90          * or just got freed under us.
91          *
92          * In case we 'win' a race for a movable page being freed under us and
93          * raise its refcount preventing __free_pages() from doing its job
94          * the put_page() at the end of this block will take care of
95          * release this page, thus avoiding a nasty leakage.
96          */
97         if (unlikely(!get_page_unless_zero(page)))
98                 goto out;
99
100         /*
101          * Check PageMovable before holding a PG_lock because page's owner
102          * assumes anybody doesn't touch PG_lock of newly allocated page
103          * so unconditionally grabbing the lock ruins page's owner side.
104          */
105         if (unlikely(!__PageMovable(page)))
106                 goto out_putpage;
107         /*
108          * As movable pages are not isolated from LRU lists, concurrent
109          * compaction threads can race against page migration functions
110          * as well as race against the releasing a page.
111          *
112          * In order to avoid having an already isolated movable page
113          * being (wrongly) re-isolated while it is under migration,
114          * or to avoid attempting to isolate pages being released,
115          * lets be sure we have the page lock
116          * before proceeding with the movable page isolation steps.
117          */
118         if (unlikely(!trylock_page(page)))
119                 goto out_putpage;
120
121         if (!PageMovable(page) || PageIsolated(page))
122                 goto out_no_isolated;
123
124         mapping = page_mapping(page);
125         VM_BUG_ON_PAGE(!mapping, page);
126
127         if (!mapping->a_ops->isolate_page(page, mode))
128                 goto out_no_isolated;
129
130         /* Driver shouldn't use PG_isolated bit of page->flags */
131         WARN_ON_ONCE(PageIsolated(page));
132         __SetPageIsolated(page);
133         unlock_page(page);
134
135         return 0;
136
137 out_no_isolated:
138         unlock_page(page);
139 out_putpage:
140         put_page(page);
141 out:
142         return -EBUSY;
143 }
144
145 /* It should be called on page which is PG_movable */
146 void putback_movable_page(struct page *page)
147 {
148         struct address_space *mapping;
149
150         VM_BUG_ON_PAGE(!PageLocked(page), page);
151         VM_BUG_ON_PAGE(!PageMovable(page), page);
152         VM_BUG_ON_PAGE(!PageIsolated(page), page);
153
154         mapping = page_mapping(page);
155         mapping->a_ops->putback_page(page);
156         __ClearPageIsolated(page);
157 }
158
159 /*
160  * Put previously isolated pages back onto the appropriate lists
161  * from where they were once taken off for compaction/migration.
162  *
163  * This function shall be used whenever the isolated pageset has been
164  * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
165  * and isolate_huge_page().
166  */
167 void putback_movable_pages(struct list_head *l)
168 {
169         struct page *page;
170         struct page *page2;
171
172         list_for_each_entry_safe(page, page2, l, lru) {
173                 if (unlikely(PageHuge(page))) {
174                         putback_active_hugepage(page);
175                         continue;
176                 }
177                 list_del(&page->lru);
178                 /*
179                  * We isolated non-lru movable page so here we can use
180                  * __PageMovable because LRU page's mapping cannot have
181                  * PAGE_MAPPING_MOVABLE.
182                  */
183                 if (unlikely(__PageMovable(page))) {
184                         VM_BUG_ON_PAGE(!PageIsolated(page), page);
185                         lock_page(page);
186                         if (PageMovable(page))
187                                 putback_movable_page(page);
188                         else
189                                 __ClearPageIsolated(page);
190                         unlock_page(page);
191                         put_page(page);
192                 } else {
193                         mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
194                                         page_is_file_cache(page), -hpage_nr_pages(page));
195                         putback_lru_page(page);
196                 }
197         }
198 }
199
200 /*
201  * Restore a potential migration pte to a working pte entry
202  */
203 static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
204                                  unsigned long addr, void *old)
205 {
206         struct page_vma_mapped_walk pvmw = {
207                 .page = old,
208                 .vma = vma,
209                 .address = addr,
210                 .flags = PVMW_SYNC | PVMW_MIGRATION,
211         };
212         struct page *new;
213         pte_t pte;
214         swp_entry_t entry;
215
216         VM_BUG_ON_PAGE(PageTail(page), page);
217         while (page_vma_mapped_walk(&pvmw)) {
218                 if (PageKsm(page))
219                         new = page;
220                 else
221                         new = page - pvmw.page->index +
222                                 linear_page_index(vma, pvmw.address);
223
224 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
225                 /* PMD-mapped THP migration entry */
226                 if (!pvmw.pte) {
227                         VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
228                         remove_migration_pmd(&pvmw, new);
229                         continue;
230                 }
231 #endif
232
233                 get_page(new);
234                 pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
235                 if (pte_swp_soft_dirty(*pvmw.pte))
236                         pte = pte_mksoft_dirty(pte);
237
238                 /*
239                  * Recheck VMA as permissions can change since migration started
240                  */
241                 entry = pte_to_swp_entry(*pvmw.pte);
242                 if (is_write_migration_entry(entry))
243                         pte = maybe_mkwrite(pte, vma);
244
245                 if (unlikely(is_zone_device_page(new))) {
246                         if (is_device_private_page(new)) {
247                                 entry = make_device_private_entry(new, pte_write(pte));
248                                 pte = swp_entry_to_pte(entry);
249                         }
250                 }
251
252 #ifdef CONFIG_HUGETLB_PAGE
253                 if (PageHuge(new)) {
254                         pte = pte_mkhuge(pte);
255                         pte = arch_make_huge_pte(pte, vma, new, 0);
256                         set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
257                         if (PageAnon(new))
258                                 hugepage_add_anon_rmap(new, vma, pvmw.address);
259                         else
260                                 page_dup_rmap(new, true);
261                 } else
262 #endif
263                 {
264                         set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
265
266                         if (PageAnon(new))
267                                 page_add_anon_rmap(new, vma, pvmw.address, false);
268                         else
269                                 page_add_file_rmap(new, false);
270                 }
271                 if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
272                         mlock_vma_page(new);
273
274                 if (PageTransHuge(page) && PageMlocked(page))
275                         clear_page_mlock(page);
276
277                 /* No need to invalidate - it was non-present before */
278                 update_mmu_cache(vma, pvmw.address, pvmw.pte);
279         }
280
281         return true;
282 }
283
284 /*
285  * Get rid of all migration entries and replace them by
286  * references to the indicated page.
287  */
288 void remove_migration_ptes(struct page *old, struct page *new, bool locked)
289 {
290         struct rmap_walk_control rwc = {
291                 .rmap_one = remove_migration_pte,
292                 .arg = old,
293         };
294
295         if (locked)
296                 rmap_walk_locked(new, &rwc);
297         else
298                 rmap_walk(new, &rwc);
299 }
300
301 /*
302  * Something used the pte of a page under migration. We need to
303  * get to the page and wait until migration is finished.
304  * When we return from this function the fault will be retried.
305  */
306 void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
307                                 spinlock_t *ptl)
308 {
309         pte_t pte;
310         swp_entry_t entry;
311         struct page *page;
312
313         spin_lock(ptl);
314         pte = *ptep;
315         if (!is_swap_pte(pte))
316                 goto out;
317
318         entry = pte_to_swp_entry(pte);
319         if (!is_migration_entry(entry))
320                 goto out;
321
322         page = migration_entry_to_page(entry);
323
324         /*
325          * Once page cache replacement of page migration started, page_count
326          * is zero; but we must not call put_and_wait_on_page_locked() without
327          * a ref. Use get_page_unless_zero(), and just fault again if it fails.
328          */
329         if (!get_page_unless_zero(page))
330                 goto out;
331         pte_unmap_unlock(ptep, ptl);
332         put_and_wait_on_page_locked(page);
333         return;
334 out:
335         pte_unmap_unlock(ptep, ptl);
336 }
337
338 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
339                                 unsigned long address)
340 {
341         spinlock_t *ptl = pte_lockptr(mm, pmd);
342         pte_t *ptep = pte_offset_map(pmd, address);
343         __migration_entry_wait(mm, ptep, ptl);
344 }
345
346 void migration_entry_wait_huge(struct vm_area_struct *vma,
347                 struct mm_struct *mm, pte_t *pte)
348 {
349         spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
350         __migration_entry_wait(mm, pte, ptl);
351 }
352
353 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
354 void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
355 {
356         spinlock_t *ptl;
357         struct page *page;
358
359         ptl = pmd_lock(mm, pmd);
360         if (!is_pmd_migration_entry(*pmd))
361                 goto unlock;
362         page = migration_entry_to_page(pmd_to_swp_entry(*pmd));
363         if (!get_page_unless_zero(page))
364                 goto unlock;
365         spin_unlock(ptl);
366         put_and_wait_on_page_locked(page);
367         return;
368 unlock:
369         spin_unlock(ptl);
370 }
371 #endif
372
373 static int expected_page_refs(struct address_space *mapping, struct page *page)
374 {
375         int expected_count = 1;
376
377         /*
378          * Device public or private pages have an extra refcount as they are
379          * ZONE_DEVICE pages.
380          */
381         expected_count += is_device_private_page(page);
382         if (mapping)
383                 expected_count += hpage_nr_pages(page) + page_has_private(page);
384
385         return expected_count;
386 }
387
388 /*
389  * Replace the page in the mapping.
390  *
391  * The number of remaining references must be:
392  * 1 for anonymous pages without a mapping
393  * 2 for pages with a mapping
394  * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
395  */
396 int migrate_page_move_mapping(struct address_space *mapping,
397                 struct page *newpage, struct page *page, enum migrate_mode mode,
398                 int extra_count)
399 {
400         XA_STATE(xas, &mapping->i_pages, page_index(page));
401         struct zone *oldzone, *newzone;
402         int dirty;
403         int expected_count = expected_page_refs(mapping, page) + extra_count;
404
405         if (!mapping) {
406                 /* Anonymous page without mapping */
407                 if (page_count(page) != expected_count)
408                         return -EAGAIN;
409
410                 /* No turning back from here */
411                 newpage->index = page->index;
412                 newpage->mapping = page->mapping;
413                 if (PageSwapBacked(page))
414                         __SetPageSwapBacked(newpage);
415
416                 return MIGRATEPAGE_SUCCESS;
417         }
418
419         oldzone = page_zone(page);
420         newzone = page_zone(newpage);
421
422         xas_lock_irq(&xas);
423         if (page_count(page) != expected_count || xas_load(&xas) != page) {
424                 xas_unlock_irq(&xas);
425                 return -EAGAIN;
426         }
427
428         if (!page_ref_freeze(page, expected_count)) {
429                 xas_unlock_irq(&xas);
430                 return -EAGAIN;
431         }
432
433         /*
434          * Now we know that no one else is looking at the page:
435          * no turning back from here.
436          */
437         newpage->index = page->index;
438         newpage->mapping = page->mapping;
439         page_ref_add(newpage, hpage_nr_pages(page)); /* add cache reference */
440         if (PageSwapBacked(page)) {
441                 __SetPageSwapBacked(newpage);
442                 if (PageSwapCache(page)) {
443                         SetPageSwapCache(newpage);
444                         set_page_private(newpage, page_private(page));
445                 }
446         } else {
447                 VM_BUG_ON_PAGE(PageSwapCache(page), page);
448         }
449
450         /* Move dirty while page refs frozen and newpage not yet exposed */
451         dirty = PageDirty(page);
452         if (dirty) {
453                 ClearPageDirty(page);
454                 SetPageDirty(newpage);
455         }
456
457         xas_store(&xas, newpage);
458         if (PageTransHuge(page)) {
459                 int i;
460
461                 for (i = 1; i < HPAGE_PMD_NR; i++) {
462                         xas_next(&xas);
463                         xas_store(&xas, newpage + i);
464                 }
465         }
466
467         /*
468          * Drop cache reference from old page by unfreezing
469          * to one less reference.
470          * We know this isn't the last reference.
471          */
472         page_ref_unfreeze(page, expected_count - hpage_nr_pages(page));
473
474         xas_unlock(&xas);
475         /* Leave irq disabled to prevent preemption while updating stats */
476
477         /*
478          * If moved to a different zone then also account
479          * the page for that zone. Other VM counters will be
480          * taken care of when we establish references to the
481          * new page and drop references to the old page.
482          *
483          * Note that anonymous pages are accounted for
484          * via NR_FILE_PAGES and NR_ANON_MAPPED if they
485          * are mapped to swap space.
486          */
487         if (newzone != oldzone) {
488                 __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
489                 __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
490                 if (PageSwapBacked(page) && !PageSwapCache(page)) {
491                         __dec_node_state(oldzone->zone_pgdat, NR_SHMEM);
492                         __inc_node_state(newzone->zone_pgdat, NR_SHMEM);
493                 }
494                 if (dirty && mapping_cap_account_dirty(mapping)) {
495                         __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
496                         __dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
497                         __inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);
498                         __inc_zone_state(newzone, NR_ZONE_WRITE_PENDING);
499                 }
500         }
501         local_irq_enable();
502
503         return MIGRATEPAGE_SUCCESS;
504 }
505 EXPORT_SYMBOL(migrate_page_move_mapping);
506
507 /*
508  * The expected number of remaining references is the same as that
509  * of migrate_page_move_mapping().
510  */
511 int migrate_huge_page_move_mapping(struct address_space *mapping,
512                                    struct page *newpage, struct page *page)
513 {
514         XA_STATE(xas, &mapping->i_pages, page_index(page));
515         int expected_count;
516
517         xas_lock_irq(&xas);
518         expected_count = 2 + page_has_private(page);
519         if (page_count(page) != expected_count || xas_load(&xas) != page) {
520                 xas_unlock_irq(&xas);
521                 return -EAGAIN;
522         }
523
524         if (!page_ref_freeze(page, expected_count)) {
525                 xas_unlock_irq(&xas);
526                 return -EAGAIN;
527         }
528
529         newpage->index = page->index;
530         newpage->mapping = page->mapping;
531
532         get_page(newpage);
533
534         xas_store(&xas, newpage);
535
536         page_ref_unfreeze(page, expected_count - 1);
537
538         xas_unlock_irq(&xas);
539
540         return MIGRATEPAGE_SUCCESS;
541 }
542
543 /*
544  * Gigantic pages are so large that we do not guarantee that page++ pointer
545  * arithmetic will work across the entire page.  We need something more
546  * specialized.
547  */
548 static void __copy_gigantic_page(struct page *dst, struct page *src,
549                                 int nr_pages)
550 {
551         int i;
552         struct page *dst_base = dst;
553         struct page *src_base = src;
554
555         for (i = 0; i < nr_pages; ) {
556                 cond_resched();
557                 copy_highpage(dst, src);
558
559                 i++;
560                 dst = mem_map_next(dst, dst_base, i);
561                 src = mem_map_next(src, src_base, i);
562         }
563 }
564
565 static void copy_huge_page(struct page *dst, struct page *src)
566 {
567         int i;
568         int nr_pages;
569
570         if (PageHuge(src)) {
571                 /* hugetlbfs page */
572                 struct hstate *h = page_hstate(src);
573                 nr_pages = pages_per_huge_page(h);
574
575                 if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
576                         __copy_gigantic_page(dst, src, nr_pages);
577                         return;
578                 }
579         } else {
580                 /* thp page */
581                 BUG_ON(!PageTransHuge(src));
582                 nr_pages = hpage_nr_pages(src);
583         }
584
585         for (i = 0; i < nr_pages; i++) {
586                 cond_resched();
587                 copy_highpage(dst + i, src + i);
588         }
589 }
590
591 /*
592  * Copy the page to its new location
593  */
594 void migrate_page_states(struct page *newpage, struct page *page)
595 {
596         int cpupid;
597
598         if (PageError(page))
599                 SetPageError(newpage);
600         if (PageReferenced(page))
601                 SetPageReferenced(newpage);
602         if (PageUptodate(page))
603                 SetPageUptodate(newpage);
604         if (TestClearPageActive(page)) {
605                 VM_BUG_ON_PAGE(PageUnevictable(page), page);
606                 SetPageActive(newpage);
607         } else if (TestClearPageUnevictable(page))
608                 SetPageUnevictable(newpage);
609         if (PageWorkingset(page))
610                 SetPageWorkingset(newpage);
611         if (PageChecked(page))
612                 SetPageChecked(newpage);
613         if (PageMappedToDisk(page))
614                 SetPageMappedToDisk(newpage);
615
616         /* Move dirty on pages not done by migrate_page_move_mapping() */
617         if (PageDirty(page))
618                 SetPageDirty(newpage);
619
620         if (page_is_young(page))
621                 set_page_young(newpage);
622         if (page_is_idle(page))
623                 set_page_idle(newpage);
624
625         /*
626          * Copy NUMA information to the new page, to prevent over-eager
627          * future migrations of this same page.
628          */
629         cpupid = page_cpupid_xchg_last(page, -1);
630         page_cpupid_xchg_last(newpage, cpupid);
631
632         ksm_migrate_page(newpage, page);
633         /*
634          * Please do not reorder this without considering how mm/ksm.c's
635          * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
636          */
637         if (PageSwapCache(page))
638                 ClearPageSwapCache(page);
639         ClearPagePrivate(page);
640         set_page_private(page, 0);
641
642         /*
643          * If any waiters have accumulated on the new page then
644          * wake them up.
645          */
646         if (PageWriteback(newpage))
647                 end_page_writeback(newpage);
648
649         copy_page_owner(page, newpage);
650
651         mem_cgroup_migrate(page, newpage);
652 }
653 EXPORT_SYMBOL(migrate_page_states);
654
655 void migrate_page_copy(struct page *newpage, struct page *page)
656 {
657         if (PageHuge(page) || PageTransHuge(page))
658                 copy_huge_page(newpage, page);
659         else
660                 copy_highpage(newpage, page);
661
662         migrate_page_states(newpage, page);
663 }
664 EXPORT_SYMBOL(migrate_page_copy);
665
666 /************************************************************
667  *                    Migration functions
668  ***********************************************************/
669
670 /*
671  * Common logic to directly migrate a single LRU page suitable for
672  * pages that do not use PagePrivate/PagePrivate2.
673  *
674  * Pages are locked upon entry and exit.
675  */
676 int migrate_page(struct address_space *mapping,
677                 struct page *newpage, struct page *page,
678                 enum migrate_mode mode)
679 {
680         int rc;
681
682         BUG_ON(PageWriteback(page));    /* Writeback must be complete */
683
684         rc = migrate_page_move_mapping(mapping, newpage, page, mode, 0);
685
686         if (rc != MIGRATEPAGE_SUCCESS)
687                 return rc;
688
689         if (mode != MIGRATE_SYNC_NO_COPY)
690                 migrate_page_copy(newpage, page);
691         else
692                 migrate_page_states(newpage, page);
693         return MIGRATEPAGE_SUCCESS;
694 }
695 EXPORT_SYMBOL(migrate_page);
696
697 #ifdef CONFIG_BLOCK
698 /* Returns true if all buffers are successfully locked */
699 static bool buffer_migrate_lock_buffers(struct buffer_head *head,
700                                                         enum migrate_mode mode)
701 {
702         struct buffer_head *bh = head;
703
704         /* Simple case, sync compaction */
705         if (mode != MIGRATE_ASYNC) {
706                 do {
707                         lock_buffer(bh);
708                         bh = bh->b_this_page;
709
710                 } while (bh != head);
711
712                 return true;
713         }
714
715         /* async case, we cannot block on lock_buffer so use trylock_buffer */
716         do {
717                 if (!trylock_buffer(bh)) {
718                         /*
719                          * We failed to lock the buffer and cannot stall in
720                          * async migration. Release the taken locks
721                          */
722                         struct buffer_head *failed_bh = bh;
723                         bh = head;
724                         while (bh != failed_bh) {
725                                 unlock_buffer(bh);
726                                 bh = bh->b_this_page;
727                         }
728                         return false;
729                 }
730
731                 bh = bh->b_this_page;
732         } while (bh != head);
733         return true;
734 }
735
736 static int __buffer_migrate_page(struct address_space *mapping,
737                 struct page *newpage, struct page *page, enum migrate_mode mode,
738                 bool check_refs)
739 {
740         struct buffer_head *bh, *head;
741         int rc;
742         int expected_count;
743
744         if (!page_has_buffers(page))
745                 return migrate_page(mapping, newpage, page, mode);
746
747         /* Check whether page does not have extra refs before we do more work */
748         expected_count = expected_page_refs(mapping, page);
749         if (page_count(page) != expected_count)
750                 return -EAGAIN;
751
752         head = page_buffers(page);
753         if (!buffer_migrate_lock_buffers(head, mode))
754                 return -EAGAIN;
755
756         if (check_refs) {
757                 bool busy;
758                 bool invalidated = false;
759
760 recheck_buffers:
761                 busy = false;
762                 spin_lock(&mapping->private_lock);
763                 bh = head;
764                 do {
765                         if (atomic_read(&bh->b_count)) {
766                                 busy = true;
767                                 break;
768                         }
769                         bh = bh->b_this_page;
770                 } while (bh != head);
771                 spin_unlock(&mapping->private_lock);
772                 if (busy) {
773                         if (invalidated) {
774                                 rc = -EAGAIN;
775                                 goto unlock_buffers;
776                         }
777                         invalidate_bh_lrus();
778                         invalidated = true;
779                         goto recheck_buffers;
780                 }
781         }
782
783         rc = migrate_page_move_mapping(mapping, newpage, page, mode, 0);
784         if (rc != MIGRATEPAGE_SUCCESS)
785                 goto unlock_buffers;
786
787         ClearPagePrivate(page);
788         set_page_private(newpage, page_private(page));
789         set_page_private(page, 0);
790         put_page(page);
791         get_page(newpage);
792
793         bh = head;
794         do {
795                 set_bh_page(bh, newpage, bh_offset(bh));
796                 bh = bh->b_this_page;
797
798         } while (bh != head);
799
800         SetPagePrivate(newpage);
801
802         if (mode != MIGRATE_SYNC_NO_COPY)
803                 migrate_page_copy(newpage, page);
804         else
805                 migrate_page_states(newpage, page);
806
807         rc = MIGRATEPAGE_SUCCESS;
808 unlock_buffers:
809         bh = head;
810         do {
811                 unlock_buffer(bh);
812                 bh = bh->b_this_page;
813
814         } while (bh != head);
815
816         return rc;
817 }
818
819 /*
820  * Migration function for pages with buffers. This function can only be used
821  * if the underlying filesystem guarantees that no other references to "page"
822  * exist. For example attached buffer heads are accessed only under page lock.
823  */
824 int buffer_migrate_page(struct address_space *mapping,
825                 struct page *newpage, struct page *page, enum migrate_mode mode)
826 {
827         return __buffer_migrate_page(mapping, newpage, page, mode, false);
828 }
829 EXPORT_SYMBOL(buffer_migrate_page);
830
831 /*
832  * Same as above except that this variant is more careful and checks that there
833  * are also no buffer head references. This function is the right one for
834  * mappings where buffer heads are directly looked up and referenced (such as
835  * block device mappings).
836  */
837 int buffer_migrate_page_norefs(struct address_space *mapping,
838                 struct page *newpage, struct page *page, enum migrate_mode mode)
839 {
840         return __buffer_migrate_page(mapping, newpage, page, mode, true);
841 }
842 #endif
843
844 /*
845  * Writeback a page to clean the dirty state
846  */
847 static int writeout(struct address_space *mapping, struct page *page)
848 {
849         struct writeback_control wbc = {
850                 .sync_mode = WB_SYNC_NONE,
851                 .nr_to_write = 1,
852                 .range_start = 0,
853                 .range_end = LLONG_MAX,
854                 .for_reclaim = 1
855         };
856         int rc;
857
858         if (!mapping->a_ops->writepage)
859                 /* No write method for the address space */
860                 return -EINVAL;
861
862         if (!clear_page_dirty_for_io(page))
863                 /* Someone else already triggered a write */
864                 return -EAGAIN;
865
866         /*
867          * A dirty page may imply that the underlying filesystem has
868          * the page on some queue. So the page must be clean for
869          * migration. Writeout may mean we loose the lock and the
870          * page state is no longer what we checked for earlier.
871          * At this point we know that the migration attempt cannot
872          * be successful.
873          */
874         remove_migration_ptes(page, page, false);
875
876         rc = mapping->a_ops->writepage(page, &wbc);
877
878         if (rc != AOP_WRITEPAGE_ACTIVATE)
879                 /* unlocked. Relock */
880                 lock_page(page);
881
882         return (rc < 0) ? -EIO : -EAGAIN;
883 }
884
885 /*
886  * Default handling if a filesystem does not provide a migration function.
887  */
888 static int fallback_migrate_page(struct address_space *mapping,
889         struct page *newpage, struct page *page, enum migrate_mode mode)
890 {
891         if (PageDirty(page)) {
892                 /* Only writeback pages in full synchronous migration */
893                 switch (mode) {
894                 case MIGRATE_SYNC:
895                 case MIGRATE_SYNC_NO_COPY:
896                         break;
897                 default:
898                         return -EBUSY;
899                 }
900                 return writeout(mapping, page);
901         }
902
903         /*
904          * Buffers may be managed in a filesystem specific way.
905          * We must have no buffers or drop them.
906          */
907         if (page_has_private(page) &&
908             !try_to_release_page(page, GFP_KERNEL))
909                 return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
910
911         return migrate_page(mapping, newpage, page, mode);
912 }
913
914 /*
915  * Move a page to a newly allocated page
916  * The page is locked and all ptes have been successfully removed.
917  *
918  * The new page will have replaced the old page if this function
919  * is successful.
920  *
921  * Return value:
922  *   < 0 - error code
923  *  MIGRATEPAGE_SUCCESS - success
924  */
925 static int move_to_new_page(struct page *newpage, struct page *page,
926                                 enum migrate_mode mode)
927 {
928         struct address_space *mapping;
929         int rc = -EAGAIN;
930         bool is_lru = !__PageMovable(page);
931
932         VM_BUG_ON_PAGE(!PageLocked(page), page);
933         VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
934
935         mapping = page_mapping(page);
936
937         if (likely(is_lru)) {
938                 if (!mapping)
939                         rc = migrate_page(mapping, newpage, page, mode);
940                 else if (mapping->a_ops->migratepage)
941                         /*
942                          * Most pages have a mapping and most filesystems
943                          * provide a migratepage callback. Anonymous pages
944                          * are part of swap space which also has its own
945                          * migratepage callback. This is the most common path
946                          * for page migration.
947                          */
948                         rc = mapping->a_ops->migratepage(mapping, newpage,
949                                                         page, mode);
950                 else
951                         rc = fallback_migrate_page(mapping, newpage,
952                                                         page, mode);
953         } else {
954                 /*
955                  * In case of non-lru page, it could be released after
956                  * isolation step. In that case, we shouldn't try migration.
957                  */
958                 VM_BUG_ON_PAGE(!PageIsolated(page), page);
959                 if (!PageMovable(page)) {
960                         rc = MIGRATEPAGE_SUCCESS;
961                         __ClearPageIsolated(page);
962                         goto out;
963                 }
964
965                 rc = mapping->a_ops->migratepage(mapping, newpage,
966                                                 page, mode);
967                 WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
968                         !PageIsolated(page));
969         }
970
971         /*
972          * When successful, old pagecache page->mapping must be cleared before
973          * page is freed; but stats require that PageAnon be left as PageAnon.
974          */
975         if (rc == MIGRATEPAGE_SUCCESS) {
976                 if (__PageMovable(page)) {
977                         VM_BUG_ON_PAGE(!PageIsolated(page), page);
978
979                         /*
980                          * We clear PG_movable under page_lock so any compactor
981                          * cannot try to migrate this page.
982                          */
983                         __ClearPageIsolated(page);
984                 }
985
986                 /*
987                  * Anonymous and movable page->mapping will be cleard by
988                  * free_pages_prepare so don't reset it here for keeping
989                  * the type to work PageAnon, for example.
990                  */
991                 if (!PageMappingFlags(page))
992                         page->mapping = NULL;
993
994                 if (likely(!is_zone_device_page(newpage)))
995                         flush_dcache_page(newpage);
996
997         }
998 out:
999         return rc;
1000 }
1001
1002 static int __unmap_and_move(struct page *page, struct page *newpage,
1003                                 int force, enum migrate_mode mode)
1004 {
1005         int rc = -EAGAIN;
1006         int page_was_mapped = 0;
1007         struct anon_vma *anon_vma = NULL;
1008         bool is_lru = !__PageMovable(page);
1009
1010         if (!trylock_page(page)) {
1011                 if (!force || mode == MIGRATE_ASYNC)
1012                         goto out;
1013
1014                 /*
1015                  * It's not safe for direct compaction to call lock_page.
1016                  * For example, during page readahead pages are added locked
1017                  * to the LRU. Later, when the IO completes the pages are
1018                  * marked uptodate and unlocked. However, the queueing
1019                  * could be merging multiple pages for one bio (e.g.
1020                  * mpage_readpages). If an allocation happens for the
1021                  * second or third page, the process can end up locking
1022                  * the same page twice and deadlocking. Rather than
1023                  * trying to be clever about what pages can be locked,
1024                  * avoid the use of lock_page for direct compaction
1025                  * altogether.
1026                  */
1027                 if (current->flags & PF_MEMALLOC)
1028                         goto out;
1029
1030                 lock_page(page);
1031         }
1032
1033         if (PageWriteback(page)) {
1034                 /*
1035                  * Only in the case of a full synchronous migration is it
1036                  * necessary to wait for PageWriteback. In the async case,
1037                  * the retry loop is too short and in the sync-light case,
1038                  * the overhead of stalling is too much
1039                  */
1040                 switch (mode) {
1041                 case MIGRATE_SYNC:
1042                 case MIGRATE_SYNC_NO_COPY:
1043                         break;
1044                 default:
1045                         rc = -EBUSY;
1046                         goto out_unlock;
1047                 }
1048                 if (!force)
1049                         goto out_unlock;
1050                 wait_on_page_writeback(page);
1051         }
1052
1053         /*
1054          * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
1055          * we cannot notice that anon_vma is freed while we migrates a page.
1056          * This get_anon_vma() delays freeing anon_vma pointer until the end
1057          * of migration. File cache pages are no problem because of page_lock()
1058          * File Caches may use write_page() or lock_page() in migration, then,
1059          * just care Anon page here.
1060          *
1061          * Only page_get_anon_vma() understands the subtleties of
1062          * getting a hold on an anon_vma from outside one of its mms.
1063          * But if we cannot get anon_vma, then we won't need it anyway,
1064          * because that implies that the anon page is no longer mapped
1065          * (and cannot be remapped so long as we hold the page lock).
1066          */
1067         if (PageAnon(page) && !PageKsm(page))
1068                 anon_vma = page_get_anon_vma(page);
1069
1070         /*
1071          * Block others from accessing the new page when we get around to
1072          * establishing additional references. We are usually the only one
1073          * holding a reference to newpage at this point. We used to have a BUG
1074          * here if trylock_page(newpage) fails, but would like to allow for
1075          * cases where there might be a race with the previous use of newpage.
1076          * This is much like races on refcount of oldpage: just don't BUG().
1077          */
1078         if (unlikely(!trylock_page(newpage)))
1079                 goto out_unlock;
1080
1081         if (unlikely(!is_lru)) {
1082                 rc = move_to_new_page(newpage, page, mode);
1083                 goto out_unlock_both;
1084         }
1085
1086         /*
1087          * Corner case handling:
1088          * 1. When a new swap-cache page is read into, it is added to the LRU
1089          * and treated as swapcache but it has no rmap yet.
1090          * Calling try_to_unmap() against a page->mapping==NULL page will
1091          * trigger a BUG.  So handle it here.
1092          * 2. An orphaned page (see truncate_complete_page) might have
1093          * fs-private metadata. The page can be picked up due to memory
1094          * offlining.  Everywhere else except page reclaim, the page is
1095          * invisible to the vm, so the page can not be migrated.  So try to
1096          * free the metadata, so the page can be freed.
1097          */
1098         if (!page->mapping) {
1099                 VM_BUG_ON_PAGE(PageAnon(page), page);
1100                 if (page_has_private(page)) {
1101                         try_to_free_buffers(page);
1102                         goto out_unlock_both;
1103                 }
1104         } else if (page_mapped(page)) {
1105                 /* Establish migration ptes */
1106                 VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
1107                                 page);
1108                 try_to_unmap(page,
1109                         TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1110                 page_was_mapped = 1;
1111         }
1112
1113         if (!page_mapped(page))
1114                 rc = move_to_new_page(newpage, page, mode);
1115
1116         if (page_was_mapped)
1117                 remove_migration_ptes(page,
1118                         rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1119
1120 out_unlock_both:
1121         unlock_page(newpage);
1122 out_unlock:
1123         /* Drop an anon_vma reference if we took one */
1124         if (anon_vma)
1125                 put_anon_vma(anon_vma);
1126         unlock_page(page);
1127 out:
1128         /*
1129          * If migration is successful, decrease refcount of the newpage
1130          * which will not free the page because new page owner increased
1131          * refcounter. As well, if it is LRU page, add the page to LRU
1132          * list in here. Use the old state of the isolated source page to
1133          * determine if we migrated a LRU page. newpage was already unlocked
1134          * and possibly modified by its owner - don't rely on the page
1135          * state.
1136          */
1137         if (rc == MIGRATEPAGE_SUCCESS) {
1138                 if (unlikely(!is_lru))
1139                         put_page(newpage);
1140                 else
1141                         putback_lru_page(newpage);
1142         }
1143
1144         return rc;
1145 }
1146
1147 /*
1148  * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move().  Work
1149  * around it.
1150  */
1151 #if defined(CONFIG_ARM) && \
1152         defined(GCC_VERSION) && GCC_VERSION < 40900 && GCC_VERSION >= 40700
1153 #define ICE_noinline noinline
1154 #else
1155 #define ICE_noinline
1156 #endif
1157
1158 /*
1159  * Obtain the lock on page, remove all ptes and migrate the page
1160  * to the newly allocated page in newpage.
1161  */
1162 static ICE_noinline int unmap_and_move(new_page_t get_new_page,
1163                                    free_page_t put_new_page,
1164                                    unsigned long private, struct page *page,
1165                                    int force, enum migrate_mode mode,
1166                                    enum migrate_reason reason)
1167 {
1168         int rc = MIGRATEPAGE_SUCCESS;
1169         struct page *newpage;
1170
1171         if (!thp_migration_supported() && PageTransHuge(page))
1172                 return -ENOMEM;
1173
1174         newpage = get_new_page(page, private);
1175         if (!newpage)
1176                 return -ENOMEM;
1177
1178         if (page_count(page) == 1) {
1179                 /* page was freed from under us. So we are done. */
1180                 ClearPageActive(page);
1181                 ClearPageUnevictable(page);
1182                 if (unlikely(__PageMovable(page))) {
1183                         lock_page(page);
1184                         if (!PageMovable(page))
1185                                 __ClearPageIsolated(page);
1186                         unlock_page(page);
1187                 }
1188                 if (put_new_page)
1189                         put_new_page(newpage, private);
1190                 else
1191                         put_page(newpage);
1192                 goto out;
1193         }
1194
1195         rc = __unmap_and_move(page, newpage, force, mode);
1196         if (rc == MIGRATEPAGE_SUCCESS)
1197                 set_page_owner_migrate_reason(newpage, reason);
1198
1199 out:
1200         if (rc != -EAGAIN) {
1201                 /*
1202                  * A page that has been migrated has all references
1203                  * removed and will be freed. A page that has not been
1204                  * migrated will have kepts its references and be
1205                  * restored.
1206                  */
1207                 list_del(&page->lru);
1208
1209                 /*
1210                  * Compaction can migrate also non-LRU pages which are
1211                  * not accounted to NR_ISOLATED_*. They can be recognized
1212                  * as __PageMovable
1213                  */
1214                 if (likely(!__PageMovable(page)))
1215                         mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1216                                         page_is_file_cache(page), -hpage_nr_pages(page));
1217         }
1218
1219         /*
1220          * If migration is successful, releases reference grabbed during
1221          * isolation. Otherwise, restore the page to right list unless
1222          * we want to retry.
1223          */
1224         if (rc == MIGRATEPAGE_SUCCESS) {
1225                 put_page(page);
1226                 if (reason == MR_MEMORY_FAILURE) {
1227                         /*
1228                          * Set PG_HWPoison on just freed page
1229                          * intentionally. Although it's rather weird,
1230                          * it's how HWPoison flag works at the moment.
1231                          */
1232                         if (set_hwpoison_free_buddy_page(page))
1233                                 num_poisoned_pages_inc();
1234                 }
1235         } else {
1236                 if (rc != -EAGAIN) {
1237                         if (likely(!__PageMovable(page))) {
1238                                 putback_lru_page(page);
1239                                 goto put_new;
1240                         }
1241
1242                         lock_page(page);
1243                         if (PageMovable(page))
1244                                 putback_movable_page(page);
1245                         else
1246                                 __ClearPageIsolated(page);
1247                         unlock_page(page);
1248                         put_page(page);
1249                 }
1250 put_new:
1251                 if (put_new_page)
1252                         put_new_page(newpage, private);
1253                 else
1254                         put_page(newpage);
1255         }
1256
1257         return rc;
1258 }
1259
1260 /*
1261  * Counterpart of unmap_and_move_page() for hugepage migration.
1262  *
1263  * This function doesn't wait the completion of hugepage I/O
1264  * because there is no race between I/O and migration for hugepage.
1265  * Note that currently hugepage I/O occurs only in direct I/O
1266  * where no lock is held and PG_writeback is irrelevant,
1267  * and writeback status of all subpages are counted in the reference
1268  * count of the head page (i.e. if all subpages of a 2MB hugepage are
1269  * under direct I/O, the reference of the head page is 512 and a bit more.)
1270  * This means that when we try to migrate hugepage whose subpages are
1271  * doing direct I/O, some references remain after try_to_unmap() and
1272  * hugepage migration fails without data corruption.
1273  *
1274  * There is also no race when direct I/O is issued on the page under migration,
1275  * because then pte is replaced with migration swap entry and direct I/O code
1276  * will wait in the page fault for migration to complete.
1277  */
1278 static int unmap_and_move_huge_page(new_page_t get_new_page,
1279                                 free_page_t put_new_page, unsigned long private,
1280                                 struct page *hpage, int force,
1281                                 enum migrate_mode mode, int reason)
1282 {
1283         int rc = -EAGAIN;
1284         int page_was_mapped = 0;
1285         struct page *new_hpage;
1286         struct anon_vma *anon_vma = NULL;
1287
1288         /*
1289          * Migratability of hugepages depends on architectures and their size.
1290          * This check is necessary because some callers of hugepage migration
1291          * like soft offline and memory hotremove don't walk through page
1292          * tables or check whether the hugepage is pmd-based or not before
1293          * kicking migration.
1294          */
1295         if (!hugepage_migration_supported(page_hstate(hpage))) {
1296                 putback_active_hugepage(hpage);
1297                 return -ENOSYS;
1298         }
1299
1300         new_hpage = get_new_page(hpage, private);
1301         if (!new_hpage)
1302                 return -ENOMEM;
1303
1304         if (!trylock_page(hpage)) {
1305                 if (!force)
1306                         goto out;
1307                 switch (mode) {
1308                 case MIGRATE_SYNC:
1309                 case MIGRATE_SYNC_NO_COPY:
1310                         break;
1311                 default:
1312                         goto out;
1313                 }
1314                 lock_page(hpage);
1315         }
1316
1317         /*
1318          * Check for pages which are in the process of being freed.  Without
1319          * page_mapping() set, hugetlbfs specific move page routine will not
1320          * be called and we could leak usage counts for subpools.
1321          */
1322         if (page_private(hpage) && !page_mapping(hpage)) {
1323                 rc = -EBUSY;
1324                 goto out_unlock;
1325         }
1326
1327         if (PageAnon(hpage))
1328                 anon_vma = page_get_anon_vma(hpage);
1329
1330         if (unlikely(!trylock_page(new_hpage)))
1331                 goto put_anon;
1332
1333         if (page_mapped(hpage)) {
1334                 try_to_unmap(hpage,
1335                         TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1336                 page_was_mapped = 1;
1337         }
1338
1339         if (!page_mapped(hpage))
1340                 rc = move_to_new_page(new_hpage, hpage, mode);
1341
1342         if (page_was_mapped)
1343                 remove_migration_ptes(hpage,
1344                         rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1345
1346         unlock_page(new_hpage);
1347
1348 put_anon:
1349         if (anon_vma)
1350                 put_anon_vma(anon_vma);
1351
1352         if (rc == MIGRATEPAGE_SUCCESS) {
1353                 move_hugetlb_state(hpage, new_hpage, reason);
1354                 put_new_page = NULL;
1355         }
1356
1357 out_unlock:
1358         unlock_page(hpage);
1359 out:
1360         if (rc != -EAGAIN)
1361                 putback_active_hugepage(hpage);
1362
1363         /*
1364          * If migration was not successful and there's a freeing callback, use
1365          * it.  Otherwise, put_page() will drop the reference grabbed during
1366          * isolation.
1367          */
1368         if (put_new_page)
1369                 put_new_page(new_hpage, private);
1370         else
1371                 putback_active_hugepage(new_hpage);
1372
1373         return rc;
1374 }
1375
1376 /*
1377  * migrate_pages - migrate the pages specified in a list, to the free pages
1378  *                 supplied as the target for the page migration
1379  *
1380  * @from:               The list of pages to be migrated.
1381  * @get_new_page:       The function used to allocate free pages to be used
1382  *                      as the target of the page migration.
1383  * @put_new_page:       The function used to free target pages if migration
1384  *                      fails, or NULL if no special handling is necessary.
1385  * @private:            Private data to be passed on to get_new_page()
1386  * @mode:               The migration mode that specifies the constraints for
1387  *                      page migration, if any.
1388  * @reason:             The reason for page migration.
1389  *
1390  * The function returns after 10 attempts or if no pages are movable any more
1391  * because the list has become empty or no retryable pages exist any more.
1392  * The caller should call putback_movable_pages() to return pages to the LRU
1393  * or free list only if ret != 0.
1394  *
1395  * Returns the number of pages that were not migrated, or an error code.
1396  */
1397 int migrate_pages(struct list_head *from, new_page_t get_new_page,
1398                 free_page_t put_new_page, unsigned long private,
1399                 enum migrate_mode mode, int reason)
1400 {
1401         int retry = 1;
1402         int nr_failed = 0;
1403         int nr_succeeded = 0;
1404         int pass = 0;
1405         struct page *page;
1406         struct page *page2;
1407         int swapwrite = current->flags & PF_SWAPWRITE;
1408         int rc;
1409
1410         if (!swapwrite)
1411                 current->flags |= PF_SWAPWRITE;
1412
1413         for(pass = 0; pass < 10 && retry; pass++) {
1414                 retry = 0;
1415
1416                 list_for_each_entry_safe(page, page2, from, lru) {
1417 retry:
1418                         cond_resched();
1419
1420                         if (PageHuge(page))
1421                                 rc = unmap_and_move_huge_page(get_new_page,
1422                                                 put_new_page, private, page,
1423                                                 pass > 2, mode, reason);
1424                         else
1425                                 rc = unmap_and_move(get_new_page, put_new_page,
1426                                                 private, page, pass > 2, mode,
1427                                                 reason);
1428
1429                         switch(rc) {
1430                         case -ENOMEM:
1431                                 /*
1432                                  * THP migration might be unsupported or the
1433                                  * allocation could've failed so we should
1434                                  * retry on the same page with the THP split
1435                                  * to base pages.
1436                                  *
1437                                  * Head page is retried immediately and tail
1438                                  * pages are added to the tail of the list so
1439                                  * we encounter them after the rest of the list
1440                                  * is processed.
1441                                  */
1442                                 if (PageTransHuge(page) && !PageHuge(page)) {
1443                                         lock_page(page);
1444                                         rc = split_huge_page_to_list(page, from);
1445                                         unlock_page(page);
1446                                         if (!rc) {
1447                                                 list_safe_reset_next(page, page2, lru);
1448                                                 goto retry;
1449                                         }
1450                                 }
1451                                 nr_failed++;
1452                                 goto out;
1453                         case -EAGAIN:
1454                                 retry++;
1455                                 break;
1456                         case MIGRATEPAGE_SUCCESS:
1457                                 nr_succeeded++;
1458                                 break;
1459                         default:
1460                                 /*
1461                                  * Permanent failure (-EBUSY, -ENOSYS, etc.):
1462                                  * unlike -EAGAIN case, the failed page is
1463                                  * removed from migration page list and not
1464                                  * retried in the next outer loop.
1465                                  */
1466                                 nr_failed++;
1467                                 break;
1468                         }
1469                 }
1470         }
1471         nr_failed += retry;
1472         rc = nr_failed;
1473 out:
1474         if (nr_succeeded)
1475                 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1476         if (nr_failed)
1477                 count_vm_events(PGMIGRATE_FAIL, nr_failed);
1478         trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
1479
1480         if (!swapwrite)
1481                 current->flags &= ~PF_SWAPWRITE;
1482
1483         return rc;
1484 }
1485
1486 #ifdef CONFIG_NUMA
1487
1488 static int store_status(int __user *status, int start, int value, int nr)
1489 {
1490         while (nr-- > 0) {
1491                 if (put_user(value, status + start))
1492                         return -EFAULT;
1493                 start++;
1494         }
1495
1496         return 0;
1497 }
1498
1499 static int do_move_pages_to_node(struct mm_struct *mm,
1500                 struct list_head *pagelist, int node)
1501 {
1502         int err;
1503
1504         if (list_empty(pagelist))
1505                 return 0;
1506
1507         err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
1508                         MIGRATE_SYNC, MR_SYSCALL);
1509         if (err)
1510                 putback_movable_pages(pagelist);
1511         return err;
1512 }
1513
1514 /*
1515  * Resolves the given address to a struct page, isolates it from the LRU and
1516  * puts it to the given pagelist.
1517  * Returns -errno if the page cannot be found/isolated or 0 when it has been
1518  * queued or the page doesn't need to be migrated because it is already on
1519  * the target node
1520  */
1521 static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
1522                 int node, struct list_head *pagelist, bool migrate_all)
1523 {
1524         struct vm_area_struct *vma;
1525         struct page *page;
1526         unsigned int follflags;
1527         int err;
1528
1529         down_read(&mm->mmap_sem);
1530         err = -EFAULT;
1531         vma = find_vma(mm, addr);
1532         if (!vma || addr < vma->vm_start || !vma_migratable(vma))
1533                 goto out;
1534
1535         /* FOLL_DUMP to ignore special (like zero) pages */
1536         follflags = FOLL_GET | FOLL_DUMP;
1537         page = follow_page(vma, addr, follflags);
1538
1539         err = PTR_ERR(page);
1540         if (IS_ERR(page))
1541                 goto out;
1542
1543         err = -ENOENT;
1544         if (!page)
1545                 goto out;
1546
1547         err = 0;
1548         if (page_to_nid(page) == node)
1549                 goto out_putpage;
1550
1551         err = -EACCES;
1552         if (page_mapcount(page) > 1 && !migrate_all)
1553                 goto out_putpage;
1554
1555         if (PageHuge(page)) {
1556                 if (PageHead(page)) {
1557                         isolate_huge_page(page, pagelist);
1558                         err = 0;
1559                 }
1560         } else {
1561                 struct page *head;
1562
1563                 head = compound_head(page);
1564                 err = isolate_lru_page(head);
1565                 if (err)
1566                         goto out_putpage;
1567
1568                 err = 0;
1569                 list_add_tail(&head->lru, pagelist);
1570                 mod_node_page_state(page_pgdat(head),
1571                         NR_ISOLATED_ANON + page_is_file_cache(head),
1572                         hpage_nr_pages(head));
1573         }
1574 out_putpage:
1575         /*
1576          * Either remove the duplicate refcount from
1577          * isolate_lru_page() or drop the page ref if it was
1578          * not isolated.
1579          */
1580         put_page(page);
1581 out:
1582         up_read(&mm->mmap_sem);
1583         return err;
1584 }
1585
1586 /*
1587  * Migrate an array of page address onto an array of nodes and fill
1588  * the corresponding array of status.
1589  */
1590 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1591                          unsigned long nr_pages,
1592                          const void __user * __user *pages,
1593                          const int __user *nodes,
1594                          int __user *status, int flags)
1595 {
1596         int current_node = NUMA_NO_NODE;
1597         LIST_HEAD(pagelist);
1598         int start, i;
1599         int err = 0, err1;
1600
1601         migrate_prep();
1602
1603         for (i = start = 0; i < nr_pages; i++) {
1604                 const void __user *p;
1605                 unsigned long addr;
1606                 int node;
1607
1608                 err = -EFAULT;
1609                 if (get_user(p, pages + i))
1610                         goto out_flush;
1611                 if (get_user(node, nodes + i))
1612                         goto out_flush;
1613                 addr = (unsigned long)p;
1614
1615                 err = -ENODEV;
1616                 if (node < 0 || node >= MAX_NUMNODES)
1617                         goto out_flush;
1618                 if (!node_state(node, N_MEMORY))
1619                         goto out_flush;
1620
1621                 err = -EACCES;
1622                 if (!node_isset(node, task_nodes))
1623                         goto out_flush;
1624
1625                 if (current_node == NUMA_NO_NODE) {
1626                         current_node = node;
1627                         start = i;
1628                 } else if (node != current_node) {
1629                         err = do_move_pages_to_node(mm, &pagelist, current_node);
1630                         if (err)
1631                                 goto out;
1632                         err = store_status(status, start, current_node, i - start);
1633                         if (err)
1634                                 goto out;
1635                         start = i;
1636                         current_node = node;
1637                 }
1638
1639                 /*
1640                  * Errors in the page lookup or isolation are not fatal and we simply
1641                  * report them via status
1642                  */
1643                 err = add_page_for_migration(mm, addr, current_node,
1644                                 &pagelist, flags & MPOL_MF_MOVE_ALL);
1645                 if (!err)
1646                         continue;
1647
1648                 err = store_status(status, i, err, 1);
1649                 if (err)
1650                         goto out_flush;
1651
1652                 err = do_move_pages_to_node(mm, &pagelist, current_node);
1653                 if (err)
1654                         goto out;
1655                 if (i > start) {
1656                         err = store_status(status, start, current_node, i - start);
1657                         if (err)
1658                                 goto out;
1659                 }
1660                 current_node = NUMA_NO_NODE;
1661         }
1662 out_flush:
1663         if (list_empty(&pagelist))
1664                 return err;
1665
1666         /* Make sure we do not overwrite the existing error */
1667         err1 = do_move_pages_to_node(mm, &pagelist, current_node);
1668         if (!err1)
1669                 err1 = store_status(status, start, current_node, i - start);
1670         if (!err)
1671                 err = err1;
1672 out:
1673         return err;
1674 }
1675
1676 /*
1677  * Determine the nodes of an array of pages and store it in an array of status.
1678  */
1679 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1680                                 const void __user **pages, int *status)
1681 {
1682         unsigned long i;
1683
1684         down_read(&mm->mmap_sem);
1685
1686         for (i = 0; i < nr_pages; i++) {
1687                 unsigned long addr = (unsigned long)(*pages);
1688                 struct vm_area_struct *vma;
1689                 struct page *page;
1690                 int err = -EFAULT;
1691
1692                 vma = find_vma(mm, addr);
1693                 if (!vma || addr < vma->vm_start)
1694                         goto set_status;
1695
1696                 /* FOLL_DUMP to ignore special (like zero) pages */
1697                 page = follow_page(vma, addr, FOLL_DUMP);
1698
1699                 err = PTR_ERR(page);
1700                 if (IS_ERR(page))
1701                         goto set_status;
1702
1703                 err = page ? page_to_nid(page) : -ENOENT;
1704 set_status:
1705                 *status = err;
1706
1707                 pages++;
1708                 status++;
1709         }
1710
1711         up_read(&mm->mmap_sem);
1712 }
1713
1714 /*
1715  * Determine the nodes of a user array of pages and store it in
1716  * a user array of status.
1717  */
1718 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1719                          const void __user * __user *pages,
1720                          int __user *status)
1721 {
1722 #define DO_PAGES_STAT_CHUNK_NR 16
1723         const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1724         int chunk_status[DO_PAGES_STAT_CHUNK_NR];
1725
1726         while (nr_pages) {
1727                 unsigned long chunk_nr;
1728
1729                 chunk_nr = nr_pages;
1730                 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1731                         chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1732
1733                 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1734                         break;
1735
1736                 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1737
1738                 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1739                         break;
1740
1741                 pages += chunk_nr;
1742                 status += chunk_nr;
1743                 nr_pages -= chunk_nr;
1744         }
1745         return nr_pages ? -EFAULT : 0;
1746 }
1747
1748 /*
1749  * Move a list of pages in the address space of the currently executing
1750  * process.
1751  */
1752 static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
1753                              const void __user * __user *pages,
1754                              const int __user *nodes,
1755                              int __user *status, int flags)
1756 {
1757         struct task_struct *task;
1758         struct mm_struct *mm;
1759         int err;
1760         nodemask_t task_nodes;
1761
1762         /* Check flags */
1763         if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1764                 return -EINVAL;
1765
1766         if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1767                 return -EPERM;
1768
1769         /* Find the mm_struct */
1770         rcu_read_lock();
1771         task = pid ? find_task_by_vpid(pid) : current;
1772         if (!task) {
1773                 rcu_read_unlock();
1774                 return -ESRCH;
1775         }
1776         get_task_struct(task);
1777
1778         /*
1779          * Check if this process has the right to modify the specified
1780          * process. Use the regular "ptrace_may_access()" checks.
1781          */
1782         if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1783                 rcu_read_unlock();
1784                 err = -EPERM;
1785                 goto out;
1786         }
1787         rcu_read_unlock();
1788
1789         err = security_task_movememory(task);
1790         if (err)
1791                 goto out;
1792
1793         task_nodes = cpuset_mems_allowed(task);
1794         mm = get_task_mm(task);
1795         put_task_struct(task);
1796
1797         if (!mm)
1798                 return -EINVAL;
1799
1800         if (nodes)
1801                 err = do_pages_move(mm, task_nodes, nr_pages, pages,
1802                                     nodes, status, flags);
1803         else
1804                 err = do_pages_stat(mm, nr_pages, pages, status);
1805
1806         mmput(mm);
1807         return err;
1808
1809 out:
1810         put_task_struct(task);
1811         return err;
1812 }
1813
1814 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1815                 const void __user * __user *, pages,
1816                 const int __user *, nodes,
1817                 int __user *, status, int, flags)
1818 {
1819         return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1820 }
1821
1822 #ifdef CONFIG_COMPAT
1823 COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
1824                        compat_uptr_t __user *, pages32,
1825                        const int __user *, nodes,
1826                        int __user *, status,
1827                        int, flags)
1828 {
1829         const void __user * __user *pages;
1830         int i;
1831
1832         pages = compat_alloc_user_space(nr_pages * sizeof(void *));
1833         for (i = 0; i < nr_pages; i++) {
1834                 compat_uptr_t p;
1835
1836                 if (get_user(p, pages32 + i) ||
1837                         put_user(compat_ptr(p), pages + i))
1838                         return -EFAULT;
1839         }
1840         return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1841 }
1842 #endif /* CONFIG_COMPAT */
1843
1844 #ifdef CONFIG_NUMA_BALANCING
1845 /*
1846  * Returns true if this is a safe migration target node for misplaced NUMA
1847  * pages. Currently it only checks the watermarks which crude
1848  */
1849 static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1850                                    unsigned long nr_migrate_pages)
1851 {
1852         int z;
1853
1854         for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1855                 struct zone *zone = pgdat->node_zones + z;
1856
1857                 if (!populated_zone(zone))
1858                         continue;
1859
1860                 /* Avoid waking kswapd by allocating pages_to_migrate pages. */
1861                 if (!zone_watermark_ok(zone, 0,
1862                                        high_wmark_pages(zone) +
1863                                        nr_migrate_pages,
1864                                        0, 0))
1865                         continue;
1866                 return true;
1867         }
1868         return false;
1869 }
1870
1871 static struct page *alloc_misplaced_dst_page(struct page *page,
1872                                            unsigned long data)
1873 {
1874         int nid = (int) data;
1875         struct page *newpage;
1876
1877         newpage = __alloc_pages_node(nid,
1878                                          (GFP_HIGHUSER_MOVABLE |
1879                                           __GFP_THISNODE | __GFP_NOMEMALLOC |
1880                                           __GFP_NORETRY | __GFP_NOWARN) &
1881                                          ~__GFP_RECLAIM, 0);
1882
1883         return newpage;
1884 }
1885
1886 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1887 {
1888         int page_lru;
1889
1890         VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
1891
1892         /* Avoid migrating to a node that is nearly full */
1893         if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page)))
1894                 return 0;
1895
1896         if (isolate_lru_page(page))
1897                 return 0;
1898
1899         /*
1900          * migrate_misplaced_transhuge_page() skips page migration's usual
1901          * check on page_count(), so we must do it here, now that the page
1902          * has been isolated: a GUP pin, or any other pin, prevents migration.
1903          * The expected page count is 3: 1 for page's mapcount and 1 for the
1904          * caller's pin and 1 for the reference taken by isolate_lru_page().
1905          */
1906         if (PageTransHuge(page) && page_count(page) != 3) {
1907                 putback_lru_page(page);
1908                 return 0;
1909         }
1910
1911         page_lru = page_is_file_cache(page);
1912         mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
1913                                 hpage_nr_pages(page));
1914
1915         /*
1916          * Isolating the page has taken another reference, so the
1917          * caller's reference can be safely dropped without the page
1918          * disappearing underneath us during migration.
1919          */
1920         put_page(page);
1921         return 1;
1922 }
1923
1924 bool pmd_trans_migrating(pmd_t pmd)
1925 {
1926         struct page *page = pmd_page(pmd);
1927         return PageLocked(page);
1928 }
1929
1930 /*
1931  * Attempt to migrate a misplaced page to the specified destination
1932  * node. Caller is expected to have an elevated reference count on
1933  * the page that will be dropped by this function before returning.
1934  */
1935 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
1936                            int node)
1937 {
1938         pg_data_t *pgdat = NODE_DATA(node);
1939         int isolated;
1940         int nr_remaining;
1941         LIST_HEAD(migratepages);
1942
1943         /*
1944          * Don't migrate file pages that are mapped in multiple processes
1945          * with execute permissions as they are probably shared libraries.
1946          */
1947         if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
1948             (vma->vm_flags & VM_EXEC))
1949                 goto out;
1950
1951         /*
1952          * Also do not migrate dirty pages as not all filesystems can move
1953          * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
1954          */
1955         if (page_is_file_cache(page) && PageDirty(page))
1956                 goto out;
1957
1958         isolated = numamigrate_isolate_page(pgdat, page);
1959         if (!isolated)
1960                 goto out;
1961
1962         list_add(&page->lru, &migratepages);
1963         nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
1964                                      NULL, node, MIGRATE_ASYNC,
1965                                      MR_NUMA_MISPLACED);
1966         if (nr_remaining) {
1967                 if (!list_empty(&migratepages)) {
1968                         list_del(&page->lru);
1969                         dec_node_page_state(page, NR_ISOLATED_ANON +
1970                                         page_is_file_cache(page));
1971                         putback_lru_page(page);
1972                 }
1973                 isolated = 0;
1974         } else
1975                 count_vm_numa_event(NUMA_PAGE_MIGRATE);
1976         BUG_ON(!list_empty(&migratepages));
1977         return isolated;
1978
1979 out:
1980         put_page(page);
1981         return 0;
1982 }
1983 #endif /* CONFIG_NUMA_BALANCING */
1984
1985 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1986 /*
1987  * Migrates a THP to a given target node. page must be locked and is unlocked
1988  * before returning.
1989  */
1990 int migrate_misplaced_transhuge_page(struct mm_struct *mm,
1991                                 struct vm_area_struct *vma,
1992                                 pmd_t *pmd, pmd_t entry,
1993                                 unsigned long address,
1994                                 struct page *page, int node)
1995 {
1996         spinlock_t *ptl;
1997         pg_data_t *pgdat = NODE_DATA(node);
1998         int isolated = 0;
1999         struct page *new_page = NULL;
2000         int page_lru = page_is_file_cache(page);
2001         unsigned long start = address & HPAGE_PMD_MASK;
2002
2003         new_page = alloc_pages_node(node,
2004                 (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
2005                 HPAGE_PMD_ORDER);
2006         if (!new_page)
2007                 goto out_fail;
2008         prep_transhuge_page(new_page);
2009
2010         isolated = numamigrate_isolate_page(pgdat, page);
2011         if (!isolated) {
2012                 put_page(new_page);
2013                 goto out_fail;
2014         }
2015
2016         /* Prepare a page as a migration target */
2017         __SetPageLocked(new_page);
2018         if (PageSwapBacked(page))
2019                 __SetPageSwapBacked(new_page);
2020
2021         /* anon mapping, we can simply copy page->mapping to the new page: */
2022         new_page->mapping = page->mapping;
2023         new_page->index = page->index;
2024         /* flush the cache before copying using the kernel virtual address */
2025         flush_cache_range(vma, start, start + HPAGE_PMD_SIZE);
2026         migrate_page_copy(new_page, page);
2027         WARN_ON(PageLRU(new_page));
2028
2029         /* Recheck the target PMD */
2030         ptl = pmd_lock(mm, pmd);
2031         if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) {
2032                 spin_unlock(ptl);
2033
2034                 /* Reverse changes made by migrate_page_copy() */
2035                 if (TestClearPageActive(new_page))
2036                         SetPageActive(page);
2037                 if (TestClearPageUnevictable(new_page))
2038                         SetPageUnevictable(page);
2039
2040                 unlock_page(new_page);
2041                 put_page(new_page);             /* Free it */
2042
2043                 /* Retake the callers reference and putback on LRU */
2044                 get_page(page);
2045                 putback_lru_page(page);
2046                 mod_node_page_state(page_pgdat(page),
2047                          NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
2048
2049                 goto out_unlock;
2050         }
2051
2052         entry = mk_huge_pmd(new_page, vma->vm_page_prot);
2053         entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
2054
2055         /*
2056          * Overwrite the old entry under pagetable lock and establish
2057          * the new PTE. Any parallel GUP will either observe the old
2058          * page blocking on the page lock, block on the page table
2059          * lock or observe the new page. The SetPageUptodate on the
2060          * new page and page_add_new_anon_rmap guarantee the copy is
2061          * visible before the pagetable update.
2062          */
2063         page_add_anon_rmap(new_page, vma, start, true);
2064         /*
2065          * At this point the pmd is numa/protnone (i.e. non present) and the TLB
2066          * has already been flushed globally.  So no TLB can be currently
2067          * caching this non present pmd mapping.  There's no need to clear the
2068          * pmd before doing set_pmd_at(), nor to flush the TLB after
2069          * set_pmd_at().  Clearing the pmd here would introduce a race
2070          * condition against MADV_DONTNEED, because MADV_DONTNEED only holds the
2071          * mmap_sem for reading.  If the pmd is set to NULL at any given time,
2072          * MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this
2073          * pmd.
2074          */
2075         set_pmd_at(mm, start, pmd, entry);
2076         update_mmu_cache_pmd(vma, address, &entry);
2077
2078         page_ref_unfreeze(page, 2);
2079         mlock_migrate_page(new_page, page);
2080         page_remove_rmap(page, true);
2081         set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
2082
2083         spin_unlock(ptl);
2084
2085         /* Take an "isolate" reference and put new page on the LRU. */
2086         get_page(new_page);
2087         putback_lru_page(new_page);
2088
2089         unlock_page(new_page);
2090         unlock_page(page);
2091         put_page(page);                 /* Drop the rmap reference */
2092         put_page(page);                 /* Drop the LRU isolation reference */
2093
2094         count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
2095         count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
2096
2097         mod_node_page_state(page_pgdat(page),
2098                         NR_ISOLATED_ANON + page_lru,
2099                         -HPAGE_PMD_NR);
2100         return isolated;
2101
2102 out_fail:
2103         count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
2104         ptl = pmd_lock(mm, pmd);
2105         if (pmd_same(*pmd, entry)) {
2106                 entry = pmd_modify(entry, vma->vm_page_prot);
2107                 set_pmd_at(mm, start, pmd, entry);
2108                 update_mmu_cache_pmd(vma, address, &entry);
2109         }
2110         spin_unlock(ptl);
2111
2112 out_unlock:
2113         unlock_page(page);
2114         put_page(page);
2115         return 0;
2116 }
2117 #endif /* CONFIG_NUMA_BALANCING */
2118
2119 #endif /* CONFIG_NUMA */
2120
2121 #if defined(CONFIG_MIGRATE_VMA_HELPER)
2122 struct migrate_vma {
2123         struct vm_area_struct   *vma;
2124         unsigned long           *dst;
2125         unsigned long           *src;
2126         unsigned long           cpages;
2127         unsigned long           npages;
2128         unsigned long           start;
2129         unsigned long           end;
2130 };
2131
2132 static int migrate_vma_collect_hole(unsigned long start,
2133                                     unsigned long end,
2134                                     struct mm_walk *walk)
2135 {
2136         struct migrate_vma *migrate = walk->private;
2137         unsigned long addr;
2138
2139         for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2140                 migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2141                 migrate->dst[migrate->npages] = 0;
2142                 migrate->npages++;
2143                 migrate->cpages++;
2144         }
2145
2146         return 0;
2147 }
2148
2149 static int migrate_vma_collect_skip(unsigned long start,
2150                                     unsigned long end,
2151                                     struct mm_walk *walk)
2152 {
2153         struct migrate_vma *migrate = walk->private;
2154         unsigned long addr;
2155
2156         for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2157                 migrate->dst[migrate->npages] = 0;
2158                 migrate->src[migrate->npages++] = 0;
2159         }
2160
2161         return 0;
2162 }
2163
2164 static int migrate_vma_collect_pmd(pmd_t *pmdp,
2165                                    unsigned long start,
2166                                    unsigned long end,
2167                                    struct mm_walk *walk)
2168 {
2169         struct migrate_vma *migrate = walk->private;
2170         struct vm_area_struct *vma = walk->vma;
2171         struct mm_struct *mm = vma->vm_mm;
2172         unsigned long addr = start, unmapped = 0;
2173         spinlock_t *ptl;
2174         pte_t *ptep;
2175
2176 again:
2177         if (pmd_none(*pmdp))
2178                 return migrate_vma_collect_hole(start, end, walk);
2179
2180         if (pmd_trans_huge(*pmdp)) {
2181                 struct page *page;
2182
2183                 ptl = pmd_lock(mm, pmdp);
2184                 if (unlikely(!pmd_trans_huge(*pmdp))) {
2185                         spin_unlock(ptl);
2186                         goto again;
2187                 }
2188
2189                 page = pmd_page(*pmdp);
2190                 if (is_huge_zero_page(page)) {
2191                         spin_unlock(ptl);
2192                         split_huge_pmd(vma, pmdp, addr);
2193                         if (pmd_trans_unstable(pmdp))
2194                                 return migrate_vma_collect_skip(start, end,
2195                                                                 walk);
2196                 } else {
2197                         int ret;
2198
2199                         get_page(page);
2200                         spin_unlock(ptl);
2201                         if (unlikely(!trylock_page(page)))
2202                                 return migrate_vma_collect_skip(start, end,
2203                                                                 walk);
2204                         ret = split_huge_page(page);
2205                         unlock_page(page);
2206                         put_page(page);
2207                         if (ret)
2208                                 return migrate_vma_collect_skip(start, end,
2209                                                                 walk);
2210                         if (pmd_none(*pmdp))
2211                                 return migrate_vma_collect_hole(start, end,
2212                                                                 walk);
2213                 }
2214         }
2215
2216         if (unlikely(pmd_bad(*pmdp)))
2217                 return migrate_vma_collect_skip(start, end, walk);
2218
2219         ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2220         arch_enter_lazy_mmu_mode();
2221
2222         for (; addr < end; addr += PAGE_SIZE, ptep++) {
2223                 unsigned long mpfn, pfn;
2224                 struct page *page;
2225                 swp_entry_t entry;
2226                 pte_t pte;
2227
2228                 pte = *ptep;
2229                 pfn = pte_pfn(pte);
2230
2231                 if (pte_none(pte)) {
2232                         mpfn = MIGRATE_PFN_MIGRATE;
2233                         migrate->cpages++;
2234                         pfn = 0;
2235                         goto next;
2236                 }
2237
2238                 if (!pte_present(pte)) {
2239                         mpfn = pfn = 0;
2240
2241                         /*
2242                          * Only care about unaddressable device page special
2243                          * page table entry. Other special swap entries are not
2244                          * migratable, and we ignore regular swapped page.
2245                          */
2246                         entry = pte_to_swp_entry(pte);
2247                         if (!is_device_private_entry(entry))
2248                                 goto next;
2249
2250                         page = device_private_entry_to_page(entry);
2251                         mpfn = migrate_pfn(page_to_pfn(page))|
2252                                 MIGRATE_PFN_DEVICE | MIGRATE_PFN_MIGRATE;
2253                         if (is_write_device_private_entry(entry))
2254                                 mpfn |= MIGRATE_PFN_WRITE;
2255                 } else {
2256                         if (is_zero_pfn(pfn)) {
2257                                 mpfn = MIGRATE_PFN_MIGRATE;
2258                                 migrate->cpages++;
2259                                 pfn = 0;
2260                                 goto next;
2261                         }
2262                         page = vm_normal_page(migrate->vma, addr, pte);
2263                         mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
2264                         mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2265                 }
2266
2267                 /* FIXME support THP */
2268                 if (!page || !page->mapping || PageTransCompound(page)) {
2269                         mpfn = pfn = 0;
2270                         goto next;
2271                 }
2272                 pfn = page_to_pfn(page);
2273
2274                 /*
2275                  * By getting a reference on the page we pin it and that blocks
2276                  * any kind of migration. Side effect is that it "freezes" the
2277                  * pte.
2278                  *
2279                  * We drop this reference after isolating the page from the lru
2280                  * for non device page (device page are not on the lru and thus
2281                  * can't be dropped from it).
2282                  */
2283                 get_page(page);
2284                 migrate->cpages++;
2285
2286                 /*
2287                  * Optimize for the common case where page is only mapped once
2288                  * in one process. If we can lock the page, then we can safely
2289                  * set up a special migration page table entry now.
2290                  */
2291                 if (trylock_page(page)) {
2292                         pte_t swp_pte;
2293
2294                         mpfn |= MIGRATE_PFN_LOCKED;
2295                         ptep_get_and_clear(mm, addr, ptep);
2296
2297                         /* Setup special migration page table entry */
2298                         entry = make_migration_entry(page, mpfn &
2299                                                      MIGRATE_PFN_WRITE);
2300                         swp_pte = swp_entry_to_pte(entry);
2301                         if (pte_soft_dirty(pte))
2302                                 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2303                         set_pte_at(mm, addr, ptep, swp_pte);
2304
2305                         /*
2306                          * This is like regular unmap: we remove the rmap and
2307                          * drop page refcount. Page won't be freed, as we took
2308                          * a reference just above.
2309                          */
2310                         page_remove_rmap(page, false);
2311                         put_page(page);
2312
2313                         if (pte_present(pte))
2314                                 unmapped++;
2315                 }
2316
2317 next:
2318                 migrate->dst[migrate->npages] = 0;
2319                 migrate->src[migrate->npages++] = mpfn;
2320         }
2321         arch_leave_lazy_mmu_mode();
2322         pte_unmap_unlock(ptep - 1, ptl);
2323
2324         /* Only flush the TLB if we actually modified any entries */
2325         if (unmapped)
2326                 flush_tlb_range(walk->vma, start, end);
2327
2328         return 0;
2329 }
2330
2331 /*
2332  * migrate_vma_collect() - collect pages over a range of virtual addresses
2333  * @migrate: migrate struct containing all migration information
2334  *
2335  * This will walk the CPU page table. For each virtual address backed by a
2336  * valid page, it updates the src array and takes a reference on the page, in
2337  * order to pin the page until we lock it and unmap it.
2338  */
2339 static void migrate_vma_collect(struct migrate_vma *migrate)
2340 {
2341         struct mmu_notifier_range range;
2342         struct mm_walk mm_walk;
2343
2344         mm_walk.pmd_entry = migrate_vma_collect_pmd;
2345         mm_walk.pte_entry = NULL;
2346         mm_walk.pte_hole = migrate_vma_collect_hole;
2347         mm_walk.hugetlb_entry = NULL;
2348         mm_walk.test_walk = NULL;
2349         mm_walk.vma = migrate->vma;
2350         mm_walk.mm = migrate->vma->vm_mm;
2351         mm_walk.private = migrate;
2352
2353         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm_walk.mm,
2354                                 migrate->start,
2355                                 migrate->end);
2356         mmu_notifier_invalidate_range_start(&range);
2357         walk_page_range(migrate->start, migrate->end, &mm_walk);
2358         mmu_notifier_invalidate_range_end(&range);
2359
2360         migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2361 }
2362
2363 /*
2364  * migrate_vma_check_page() - check if page is pinned or not
2365  * @page: struct page to check
2366  *
2367  * Pinned pages cannot be migrated. This is the same test as in
2368  * migrate_page_move_mapping(), except that here we allow migration of a
2369  * ZONE_DEVICE page.
2370  */
2371 static bool migrate_vma_check_page(struct page *page)
2372 {
2373         /*
2374          * One extra ref because caller holds an extra reference, either from
2375          * isolate_lru_page() for a regular page, or migrate_vma_collect() for
2376          * a device page.
2377          */
2378         int extra = 1;
2379
2380         /*
2381          * FIXME support THP (transparent huge page), it is bit more complex to
2382          * check them than regular pages, because they can be mapped with a pmd
2383          * or with a pte (split pte mapping).
2384          */
2385         if (PageCompound(page))
2386                 return false;
2387
2388         /* Page from ZONE_DEVICE have one extra reference */
2389         if (is_zone_device_page(page)) {
2390                 /*
2391                  * Private page can never be pin as they have no valid pte and
2392                  * GUP will fail for those. Yet if there is a pending migration
2393                  * a thread might try to wait on the pte migration entry and
2394                  * will bump the page reference count. Sadly there is no way to
2395                  * differentiate a regular pin from migration wait. Hence to
2396                  * avoid 2 racing thread trying to migrate back to CPU to enter
2397                  * infinite loop (one stoping migration because the other is
2398                  * waiting on pte migration entry). We always return true here.
2399                  *
2400                  * FIXME proper solution is to rework migration_entry_wait() so
2401                  * it does not need to take a reference on page.
2402                  */
2403                 return is_device_private_page(page);
2404         }
2405
2406         /* For file back page */
2407         if (page_mapping(page))
2408                 extra += 1 + page_has_private(page);
2409
2410         if ((page_count(page) - extra) > page_mapcount(page))
2411                 return false;
2412
2413         return true;
2414 }
2415
2416 /*
2417  * migrate_vma_prepare() - lock pages and isolate them from the lru
2418  * @migrate: migrate struct containing all migration information
2419  *
2420  * This locks pages that have been collected by migrate_vma_collect(). Once each
2421  * page is locked it is isolated from the lru (for non-device pages). Finally,
2422  * the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
2423  * migrated by concurrent kernel threads.
2424  */
2425 static void migrate_vma_prepare(struct migrate_vma *migrate)
2426 {
2427         const unsigned long npages = migrate->npages;
2428         const unsigned long start = migrate->start;
2429         unsigned long addr, i, restore = 0;
2430         bool allow_drain = true;
2431
2432         lru_add_drain();
2433
2434         for (i = 0; (i < npages) && migrate->cpages; i++) {
2435                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2436                 bool remap = true;
2437
2438                 if (!page)
2439                         continue;
2440
2441                 if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
2442                         /*
2443                          * Because we are migrating several pages there can be
2444                          * a deadlock between 2 concurrent migration where each
2445                          * are waiting on each other page lock.
2446                          *
2447                          * Make migrate_vma() a best effort thing and backoff
2448                          * for any page we can not lock right away.
2449                          */
2450                         if (!trylock_page(page)) {
2451                                 migrate->src[i] = 0;
2452                                 migrate->cpages--;
2453                                 put_page(page);
2454                                 continue;
2455                         }
2456                         remap = false;
2457                         migrate->src[i] |= MIGRATE_PFN_LOCKED;
2458                 }
2459
2460                 /* ZONE_DEVICE pages are not on LRU */
2461                 if (!is_zone_device_page(page)) {
2462                         if (!PageLRU(page) && allow_drain) {
2463                                 /* Drain CPU's pagevec */
2464                                 lru_add_drain_all();
2465                                 allow_drain = false;
2466                         }
2467
2468                         if (isolate_lru_page(page)) {
2469                                 if (remap) {
2470                                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2471                                         migrate->cpages--;
2472                                         restore++;
2473                                 } else {
2474                                         migrate->src[i] = 0;
2475                                         unlock_page(page);
2476                                         migrate->cpages--;
2477                                         put_page(page);
2478                                 }
2479                                 continue;
2480                         }
2481
2482                         /* Drop the reference we took in collect */
2483                         put_page(page);
2484                 }
2485
2486                 if (!migrate_vma_check_page(page)) {
2487                         if (remap) {
2488                                 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2489                                 migrate->cpages--;
2490                                 restore++;
2491
2492                                 if (!is_zone_device_page(page)) {
2493                                         get_page(page);
2494                                         putback_lru_page(page);
2495                                 }
2496                         } else {
2497                                 migrate->src[i] = 0;
2498                                 unlock_page(page);
2499                                 migrate->cpages--;
2500
2501                                 if (!is_zone_device_page(page))
2502                                         putback_lru_page(page);
2503                                 else
2504                                         put_page(page);
2505                         }
2506                 }
2507         }
2508
2509         for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
2510                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2511
2512                 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2513                         continue;
2514
2515                 remove_migration_pte(page, migrate->vma, addr, page);
2516
2517                 migrate->src[i] = 0;
2518                 unlock_page(page);
2519                 put_page(page);
2520                 restore--;
2521         }
2522 }
2523
2524 /*
2525  * migrate_vma_unmap() - replace page mapping with special migration pte entry
2526  * @migrate: migrate struct containing all migration information
2527  *
2528  * Replace page mapping (CPU page table pte) with a special migration pte entry
2529  * and check again if it has been pinned. Pinned pages are restored because we
2530  * cannot migrate them.
2531  *
2532  * This is the last step before we call the device driver callback to allocate
2533  * destination memory and copy contents of original page over to new page.
2534  */
2535 static void migrate_vma_unmap(struct migrate_vma *migrate)
2536 {
2537         int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
2538         const unsigned long npages = migrate->npages;
2539         const unsigned long start = migrate->start;
2540         unsigned long addr, i, restore = 0;
2541
2542         for (i = 0; i < npages; i++) {
2543                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2544
2545                 if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2546                         continue;
2547
2548                 if (page_mapped(page)) {
2549                         try_to_unmap(page, flags);
2550                         if (page_mapped(page))
2551                                 goto restore;
2552                 }
2553
2554                 if (migrate_vma_check_page(page))
2555                         continue;
2556
2557 restore:
2558                 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2559                 migrate->cpages--;
2560                 restore++;
2561         }
2562
2563         for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
2564                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2565
2566                 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2567                         continue;
2568
2569                 remove_migration_ptes(page, page, false);
2570
2571                 migrate->src[i] = 0;
2572                 unlock_page(page);
2573                 restore--;
2574
2575                 if (is_zone_device_page(page))
2576                         put_page(page);
2577                 else
2578                         putback_lru_page(page);
2579         }
2580 }
2581
2582 static void migrate_vma_insert_page(struct migrate_vma *migrate,
2583                                     unsigned long addr,
2584                                     struct page *page,
2585                                     unsigned long *src,
2586                                     unsigned long *dst)
2587 {
2588         struct vm_area_struct *vma = migrate->vma;
2589         struct mm_struct *mm = vma->vm_mm;
2590         struct mem_cgroup *memcg;
2591         bool flush = false;
2592         spinlock_t *ptl;
2593         pte_t entry;
2594         pgd_t *pgdp;
2595         p4d_t *p4dp;
2596         pud_t *pudp;
2597         pmd_t *pmdp;
2598         pte_t *ptep;
2599
2600         /* Only allow populating anonymous memory */
2601         if (!vma_is_anonymous(vma))
2602                 goto abort;
2603
2604         pgdp = pgd_offset(mm, addr);
2605         p4dp = p4d_alloc(mm, pgdp, addr);
2606         if (!p4dp)
2607                 goto abort;
2608         pudp = pud_alloc(mm, p4dp, addr);
2609         if (!pudp)
2610                 goto abort;
2611         pmdp = pmd_alloc(mm, pudp, addr);
2612         if (!pmdp)
2613                 goto abort;
2614
2615         if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
2616                 goto abort;
2617
2618         /*
2619          * Use pte_alloc() instead of pte_alloc_map().  We can't run
2620          * pte_offset_map() on pmds where a huge pmd might be created
2621          * from a different thread.
2622          *
2623          * pte_alloc_map() is safe to use under down_write(mmap_sem) or when
2624          * parallel threads are excluded by other means.
2625          *
2626          * Here we only have down_read(mmap_sem).
2627          */
2628         if (pte_alloc(mm, pmdp))
2629                 goto abort;
2630
2631         /* See the comment in pte_alloc_one_map() */
2632         if (unlikely(pmd_trans_unstable(pmdp)))
2633                 goto abort;
2634
2635         if (unlikely(anon_vma_prepare(vma)))
2636                 goto abort;
2637         if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
2638                 goto abort;
2639
2640         /*
2641          * The memory barrier inside __SetPageUptodate makes sure that
2642          * preceding stores to the page contents become visible before
2643          * the set_pte_at() write.
2644          */
2645         __SetPageUptodate(page);
2646
2647         if (is_zone_device_page(page)) {
2648                 if (is_device_private_page(page)) {
2649                         swp_entry_t swp_entry;
2650
2651                         swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
2652                         entry = swp_entry_to_pte(swp_entry);
2653                 }
2654         } else {
2655                 entry = mk_pte(page, vma->vm_page_prot);
2656                 if (vma->vm_flags & VM_WRITE)
2657                         entry = pte_mkwrite(pte_mkdirty(entry));
2658         }
2659
2660         ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2661
2662         if (pte_present(*ptep)) {
2663                 unsigned long pfn = pte_pfn(*ptep);
2664
2665                 if (!is_zero_pfn(pfn)) {
2666                         pte_unmap_unlock(ptep, ptl);
2667                         mem_cgroup_cancel_charge(page, memcg, false);
2668                         goto abort;
2669                 }
2670                 flush = true;
2671         } else if (!pte_none(*ptep)) {
2672                 pte_unmap_unlock(ptep, ptl);
2673                 mem_cgroup_cancel_charge(page, memcg, false);
2674                 goto abort;
2675         }
2676
2677         /*
2678          * Check for usefaultfd but do not deliver the fault. Instead,
2679          * just back off.
2680          */
2681         if (userfaultfd_missing(vma)) {
2682                 pte_unmap_unlock(ptep, ptl);
2683                 mem_cgroup_cancel_charge(page, memcg, false);
2684                 goto abort;
2685         }
2686
2687         inc_mm_counter(mm, MM_ANONPAGES);
2688         page_add_new_anon_rmap(page, vma, addr, false);
2689         mem_cgroup_commit_charge(page, memcg, false, false);
2690         if (!is_zone_device_page(page))
2691                 lru_cache_add_active_or_unevictable(page, vma);
2692         get_page(page);
2693
2694         if (flush) {
2695                 flush_cache_page(vma, addr, pte_pfn(*ptep));
2696                 ptep_clear_flush_notify(vma, addr, ptep);
2697                 set_pte_at_notify(mm, addr, ptep, entry);
2698                 update_mmu_cache(vma, addr, ptep);
2699         } else {
2700                 /* No need to invalidate - it was non-present before */
2701                 set_pte_at(mm, addr, ptep, entry);
2702                 update_mmu_cache(vma, addr, ptep);
2703         }
2704
2705         pte_unmap_unlock(ptep, ptl);
2706         *src = MIGRATE_PFN_MIGRATE;
2707         return;
2708
2709 abort:
2710         *src &= ~MIGRATE_PFN_MIGRATE;
2711 }
2712
2713 /*
2714  * migrate_vma_pages() - migrate meta-data from src page to dst page
2715  * @migrate: migrate struct containing all migration information
2716  *
2717  * This migrates struct page meta-data from source struct page to destination
2718  * struct page. This effectively finishes the migration from source page to the
2719  * destination page.
2720  */
2721 static void migrate_vma_pages(struct migrate_vma *migrate)
2722 {
2723         const unsigned long npages = migrate->npages;
2724         const unsigned long start = migrate->start;
2725         struct mmu_notifier_range range;
2726         unsigned long addr, i;
2727         bool notified = false;
2728
2729         for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2730                 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2731                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2732                 struct address_space *mapping;
2733                 int r;
2734
2735                 if (!newpage) {
2736                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2737                         continue;
2738                 }
2739
2740                 if (!page) {
2741                         if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
2742                                 continue;
2743                         }
2744                         if (!notified) {
2745                                 notified = true;
2746
2747                                 mmu_notifier_range_init(&range,
2748                                                         MMU_NOTIFY_CLEAR, 0,
2749                                                         NULL,
2750                                                         migrate->vma->vm_mm,
2751                                                         addr, migrate->end);
2752                                 mmu_notifier_invalidate_range_start(&range);
2753                         }
2754                         migrate_vma_insert_page(migrate, addr, newpage,
2755                                                 &migrate->src[i],
2756                                                 &migrate->dst[i]);
2757                         continue;
2758                 }
2759
2760                 mapping = page_mapping(page);
2761
2762                 if (is_zone_device_page(newpage)) {
2763                         if (is_device_private_page(newpage)) {
2764                                 /*
2765                                  * For now only support private anonymous when
2766                                  * migrating to un-addressable device memory.
2767                                  */
2768                                 if (mapping) {
2769                                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2770                                         continue;
2771                                 }
2772                         } else {
2773                                 /*
2774                                  * Other types of ZONE_DEVICE page are not
2775                                  * supported.
2776                                  */
2777                                 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2778                                 continue;
2779                         }
2780                 }
2781
2782                 r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2783                 if (r != MIGRATEPAGE_SUCCESS)
2784                         migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2785         }
2786
2787         /*
2788          * No need to double call mmu_notifier->invalidate_range() callback as
2789          * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
2790          * did already call it.
2791          */
2792         if (notified)
2793                 mmu_notifier_invalidate_range_only_end(&range);
2794 }
2795
2796 /*
2797  * migrate_vma_finalize() - restore CPU page table entry
2798  * @migrate: migrate struct containing all migration information
2799  *
2800  * This replaces the special migration pte entry with either a mapping to the
2801  * new page if migration was successful for that page, or to the original page
2802  * otherwise.
2803  *
2804  * This also unlocks the pages and puts them back on the lru, or drops the extra
2805  * refcount, for device pages.
2806  */
2807 static void migrate_vma_finalize(struct migrate_vma *migrate)
2808 {
2809         const unsigned long npages = migrate->npages;
2810         unsigned long i;
2811
2812         for (i = 0; i < npages; i++) {
2813                 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2814                 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2815
2816                 if (!page) {
2817                         if (newpage) {
2818                                 unlock_page(newpage);
2819                                 put_page(newpage);
2820                         }
2821                         continue;
2822                 }
2823
2824                 if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
2825                         if (newpage) {
2826                                 unlock_page(newpage);
2827                                 put_page(newpage);
2828                         }
2829                         newpage = page;
2830                 }
2831
2832                 remove_migration_ptes(page, newpage, false);
2833                 unlock_page(page);
2834                 migrate->cpages--;
2835
2836                 if (is_zone_device_page(page))
2837                         put_page(page);
2838                 else
2839                         putback_lru_page(page);
2840
2841                 if (newpage != page) {
2842                         unlock_page(newpage);
2843                         if (is_zone_device_page(newpage))
2844                                 put_page(newpage);
2845                         else
2846                                 putback_lru_page(newpage);
2847                 }
2848         }
2849 }
2850
2851 /*
2852  * migrate_vma() - migrate a range of memory inside vma
2853  *
2854  * @ops: migration callback for allocating destination memory and copying
2855  * @vma: virtual memory area containing the range to be migrated
2856  * @start: start address of the range to migrate (inclusive)
2857  * @end: end address of the range to migrate (exclusive)
2858  * @src: array of hmm_pfn_t containing source pfns
2859  * @dst: array of hmm_pfn_t containing destination pfns
2860  * @private: pointer passed back to each of the callback
2861  * Returns: 0 on success, error code otherwise
2862  *
2863  * This function tries to migrate a range of memory virtual address range, using
2864  * callbacks to allocate and copy memory from source to destination. First it
2865  * collects all the pages backing each virtual address in the range, saving this
2866  * inside the src array. Then it locks those pages and unmaps them. Once the pages
2867  * are locked and unmapped, it checks whether each page is pinned or not. Pages
2868  * that aren't pinned have the MIGRATE_PFN_MIGRATE flag set (by this function)
2869  * in the corresponding src array entry. It then restores any pages that are
2870  * pinned, by remapping and unlocking those pages.
2871  *
2872  * At this point it calls the alloc_and_copy() callback. For documentation on
2873  * what is expected from that callback, see struct migrate_vma_ops comments in
2874  * include/linux/migrate.h
2875  *
2876  * After the alloc_and_copy() callback, this function goes over each entry in
2877  * the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
2878  * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
2879  * then the function tries to migrate struct page information from the source
2880  * struct page to the destination struct page. If it fails to migrate the struct
2881  * page information, then it clears the MIGRATE_PFN_MIGRATE flag in the src
2882  * array.
2883  *
2884  * At this point all successfully migrated pages have an entry in the src
2885  * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
2886  * array entry with MIGRATE_PFN_VALID flag set.
2887  *
2888  * It then calls the finalize_and_map() callback. See comments for "struct
2889  * migrate_vma_ops", in include/linux/migrate.h for details about
2890  * finalize_and_map() behavior.
2891  *
2892  * After the finalize_and_map() callback, for successfully migrated pages, this
2893  * function updates the CPU page table to point to new pages, otherwise it
2894  * restores the CPU page table to point to the original source pages.
2895  *
2896  * Function returns 0 after the above steps, even if no pages were migrated
2897  * (The function only returns an error if any of the arguments are invalid.)
2898  *
2899  * Both src and dst array must be big enough for (end - start) >> PAGE_SHIFT
2900  * unsigned long entries.
2901  */
2902 int migrate_vma(const struct migrate_vma_ops *ops,
2903                 struct vm_area_struct *vma,
2904                 unsigned long start,
2905                 unsigned long end,
2906                 unsigned long *src,
2907                 unsigned long *dst,
2908                 void *private)
2909 {
2910         struct migrate_vma migrate;
2911
2912         /* Sanity check the arguments */
2913         start &= PAGE_MASK;
2914         end &= PAGE_MASK;
2915         if (!vma || is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) ||
2916                         vma_is_dax(vma))
2917                 return -EINVAL;
2918         if (start < vma->vm_start || start >= vma->vm_end)
2919                 return -EINVAL;
2920         if (end <= vma->vm_start || end > vma->vm_end)
2921                 return -EINVAL;
2922         if (!ops || !src || !dst || start >= end)
2923                 return -EINVAL;
2924
2925         memset(src, 0, sizeof(*src) * ((end - start) >> PAGE_SHIFT));
2926         migrate.src = src;
2927         migrate.dst = dst;
2928         migrate.start = start;
2929         migrate.npages = 0;
2930         migrate.cpages = 0;
2931         migrate.end = end;
2932         migrate.vma = vma;
2933
2934         /* Collect, and try to unmap source pages */
2935         migrate_vma_collect(&migrate);
2936         if (!migrate.cpages)
2937                 return 0;
2938
2939         /* Lock and isolate page */
2940         migrate_vma_prepare(&migrate);
2941         if (!migrate.cpages)
2942                 return 0;
2943
2944         /* Unmap pages */
2945         migrate_vma_unmap(&migrate);
2946         if (!migrate.cpages)
2947                 return 0;
2948
2949         /*
2950          * At this point pages are locked and unmapped, and thus they have
2951          * stable content and can safely be copied to destination memory that
2952          * is allocated by the callback.
2953          *
2954          * Note that migration can fail in migrate_vma_struct_page() for each
2955          * individual page.
2956          */
2957         ops->alloc_and_copy(vma, src, dst, start, end, private);
2958
2959         /* This does the real migration of struct page */
2960         migrate_vma_pages(&migrate);
2961
2962         ops->finalize_and_map(vma, src, dst, start, end, private);
2963
2964         /* Unlock and remap pages */
2965         migrate_vma_finalize(&migrate);
2966
2967         return 0;
2968 }
2969 EXPORT_SYMBOL(migrate_vma);
2970 #endif /* defined(MIGRATE_VMA_HELPER) */