net: ipv6 bind to device issue
[linux-2.6-block.git] / mm / migrate.c
CommitLineData
b20a3503
CL
1/*
2 * Memory Migration functionality - linux/mm/migration.c
3 *
4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
5 *
6 * Page migration was first developed in the context of the memory hotplug
7 * project. The main authors of the migration code are:
8 *
9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
10 * Hirokazu Takahashi <taka@valinux.co.jp>
11 * Dave Hansen <haveblue@us.ibm.com>
cde53535 12 * Christoph Lameter
b20a3503
CL
13 */
14
15#include <linux/migrate.h>
16#include <linux/module.h>
17#include <linux/swap.h>
0697212a 18#include <linux/swapops.h>
b20a3503 19#include <linux/pagemap.h>
e23ca00b 20#include <linux/buffer_head.h>
b20a3503 21#include <linux/mm_inline.h>
b488893a 22#include <linux/nsproxy.h>
b20a3503 23#include <linux/pagevec.h>
e9995ef9 24#include <linux/ksm.h>
b20a3503
CL
25#include <linux/rmap.h>
26#include <linux/topology.h>
27#include <linux/cpu.h>
28#include <linux/cpuset.h>
04e62a29 29#include <linux/writeback.h>
742755a1
CL
30#include <linux/mempolicy.h>
31#include <linux/vmalloc.h>
86c3a764 32#include <linux/security.h>
8a9f3ccd 33#include <linux/memcontrol.h>
4f5ca265 34#include <linux/syscalls.h>
5a0e3ad6 35#include <linux/gfp.h>
b20a3503
CL
36
37#include "internal.h"
38
b20a3503
CL
39#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
40
b20a3503 41/*
742755a1
CL
42 * migrate_prep() needs to be called before we start compiling a list of pages
43 * to be migrated using isolate_lru_page().
b20a3503
CL
44 */
45int migrate_prep(void)
46{
b20a3503
CL
47 /*
48 * Clear the LRU lists so pages can be isolated.
49 * Note that pages may be moved off the LRU after we have
50 * drained them. Those pages will fail to migrate like other
51 * pages that may be busy.
52 */
53 lru_add_drain_all();
54
55 return 0;
56}
57
b20a3503 58/*
894bc310
LS
59 * Add isolated pages on the list back to the LRU under page lock
60 * to avoid leaking evictable pages back onto unevictable list.
b20a3503
CL
61 *
62 * returns the number of pages put back.
63 */
64int putback_lru_pages(struct list_head *l)
65{
66 struct page *page;
67 struct page *page2;
68 int count = 0;
69
70 list_for_each_entry_safe(page, page2, l, lru) {
e24f0b8f 71 list_del(&page->lru);
a731286d 72 dec_zone_page_state(page, NR_ISOLATED_ANON +
6c0b1351 73 page_is_file_cache(page));
894bc310 74 putback_lru_page(page);
b20a3503
CL
75 count++;
76 }
77 return count;
78}
79
0697212a
CL
80/*
81 * Restore a potential migration pte to a working pte entry
82 */
e9995ef9
HD
83static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
84 unsigned long addr, void *old)
0697212a
CL
85{
86 struct mm_struct *mm = vma->vm_mm;
87 swp_entry_t entry;
88 pgd_t *pgd;
89 pud_t *pud;
90 pmd_t *pmd;
91 pte_t *ptep, pte;
92 spinlock_t *ptl;
93
94 pgd = pgd_offset(mm, addr);
95 if (!pgd_present(*pgd))
e9995ef9 96 goto out;
0697212a
CL
97
98 pud = pud_offset(pgd, addr);
99 if (!pud_present(*pud))
e9995ef9 100 goto out;
0697212a
CL
101
102 pmd = pmd_offset(pud, addr);
103 if (!pmd_present(*pmd))
e9995ef9 104 goto out;
0697212a
CL
105
106 ptep = pte_offset_map(pmd, addr);
107
108 if (!is_swap_pte(*ptep)) {
109 pte_unmap(ptep);
e9995ef9 110 goto out;
0697212a
CL
111 }
112
113 ptl = pte_lockptr(mm, pmd);
114 spin_lock(ptl);
115 pte = *ptep;
116 if (!is_swap_pte(pte))
e9995ef9 117 goto unlock;
0697212a
CL
118
119 entry = pte_to_swp_entry(pte);
120
e9995ef9
HD
121 if (!is_migration_entry(entry) ||
122 migration_entry_to_page(entry) != old)
123 goto unlock;
0697212a 124
0697212a
CL
125 get_page(new);
126 pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
127 if (is_write_migration_entry(entry))
128 pte = pte_mkwrite(pte);
97ee0524 129 flush_cache_page(vma, addr, pte_pfn(pte));
0697212a 130 set_pte_at(mm, addr, ptep, pte);
04e62a29
CL
131
132 if (PageAnon(new))
133 page_add_anon_rmap(new, vma, addr);
134 else
135 page_add_file_rmap(new);
136
137 /* No need to invalidate - it was non-present before */
4b3073e1 138 update_mmu_cache(vma, addr, ptep);
e9995ef9 139unlock:
0697212a 140 pte_unmap_unlock(ptep, ptl);
e9995ef9
HD
141out:
142 return SWAP_AGAIN;
0697212a
CL
143}
144
04e62a29
CL
145/*
146 * Get rid of all migration entries and replace them by
147 * references to the indicated page.
148 */
149static void remove_migration_ptes(struct page *old, struct page *new)
150{
e9995ef9 151 rmap_walk(new, remove_migration_pte, old);
04e62a29
CL
152}
153
0697212a
CL
154/*
155 * Something used the pte of a page under migration. We need to
156 * get to the page and wait until migration is finished.
157 * When we return from this function the fault will be retried.
158 *
159 * This function is called from do_swap_page().
160 */
161void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
162 unsigned long address)
163{
164 pte_t *ptep, pte;
165 spinlock_t *ptl;
166 swp_entry_t entry;
167 struct page *page;
168
169 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
170 pte = *ptep;
171 if (!is_swap_pte(pte))
172 goto out;
173
174 entry = pte_to_swp_entry(pte);
175 if (!is_migration_entry(entry))
176 goto out;
177
178 page = migration_entry_to_page(entry);
179
e286781d
NP
180 /*
181 * Once radix-tree replacement of page migration started, page_count
182 * *must* be zero. And, we don't want to call wait_on_page_locked()
183 * against a page without get_page().
184 * So, we use get_page_unless_zero(), here. Even failed, page fault
185 * will occur again.
186 */
187 if (!get_page_unless_zero(page))
188 goto out;
0697212a
CL
189 pte_unmap_unlock(ptep, ptl);
190 wait_on_page_locked(page);
191 put_page(page);
192 return;
193out:
194 pte_unmap_unlock(ptep, ptl);
195}
196
b20a3503 197/*
c3fcf8a5 198 * Replace the page in the mapping.
5b5c7120
CL
199 *
200 * The number of remaining references must be:
201 * 1 for anonymous pages without a mapping
202 * 2 for pages with a mapping
266cf658 203 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
b20a3503 204 */
2d1db3b1
CL
205static int migrate_page_move_mapping(struct address_space *mapping,
206 struct page *newpage, struct page *page)
b20a3503 207{
e286781d 208 int expected_count;
7cf9c2c7 209 void **pslot;
b20a3503 210
6c5240ae 211 if (!mapping) {
0e8c7d0f 212 /* Anonymous page without mapping */
6c5240ae
CL
213 if (page_count(page) != 1)
214 return -EAGAIN;
215 return 0;
216 }
217
19fd6231 218 spin_lock_irq(&mapping->tree_lock);
b20a3503 219
7cf9c2c7
NP
220 pslot = radix_tree_lookup_slot(&mapping->page_tree,
221 page_index(page));
b20a3503 222
edcf4748 223 expected_count = 2 + page_has_private(page);
e286781d 224 if (page_count(page) != expected_count ||
7cf9c2c7 225 (struct page *)radix_tree_deref_slot(pslot) != page) {
19fd6231 226 spin_unlock_irq(&mapping->tree_lock);
e23ca00b 227 return -EAGAIN;
b20a3503
CL
228 }
229
e286781d 230 if (!page_freeze_refs(page, expected_count)) {
19fd6231 231 spin_unlock_irq(&mapping->tree_lock);
e286781d
NP
232 return -EAGAIN;
233 }
234
b20a3503
CL
235 /*
236 * Now we know that no one else is looking at the page.
b20a3503 237 */
7cf9c2c7 238 get_page(newpage); /* add cache reference */
b20a3503
CL
239 if (PageSwapCache(page)) {
240 SetPageSwapCache(newpage);
241 set_page_private(newpage, page_private(page));
242 }
243
7cf9c2c7
NP
244 radix_tree_replace_slot(pslot, newpage);
245
e286781d 246 page_unfreeze_refs(page, expected_count);
7cf9c2c7
NP
247 /*
248 * Drop cache reference from old page.
249 * We know this isn't the last reference.
250 */
b20a3503 251 __put_page(page);
7cf9c2c7 252
0e8c7d0f
CL
253 /*
254 * If moved to a different zone then also account
255 * the page for that zone. Other VM counters will be
256 * taken care of when we establish references to the
257 * new page and drop references to the old page.
258 *
259 * Note that anonymous pages are accounted for
260 * via NR_FILE_PAGES and NR_ANON_PAGES if they
261 * are mapped to swap space.
262 */
263 __dec_zone_page_state(page, NR_FILE_PAGES);
264 __inc_zone_page_state(newpage, NR_FILE_PAGES);
4b02108a
KM
265 if (PageSwapBacked(page)) {
266 __dec_zone_page_state(page, NR_SHMEM);
267 __inc_zone_page_state(newpage, NR_SHMEM);
268 }
19fd6231 269 spin_unlock_irq(&mapping->tree_lock);
b20a3503
CL
270
271 return 0;
272}
b20a3503
CL
273
274/*
275 * Copy the page to its new location
276 */
e7340f73 277static void migrate_page_copy(struct page *newpage, struct page *page)
b20a3503
CL
278{
279 copy_highpage(newpage, page);
280
281 if (PageError(page))
282 SetPageError(newpage);
283 if (PageReferenced(page))
284 SetPageReferenced(newpage);
285 if (PageUptodate(page))
286 SetPageUptodate(newpage);
894bc310
LS
287 if (TestClearPageActive(page)) {
288 VM_BUG_ON(PageUnevictable(page));
b20a3503 289 SetPageActive(newpage);
418b27ef
LS
290 } else if (TestClearPageUnevictable(page))
291 SetPageUnevictable(newpage);
b20a3503
CL
292 if (PageChecked(page))
293 SetPageChecked(newpage);
294 if (PageMappedToDisk(page))
295 SetPageMappedToDisk(newpage);
296
297 if (PageDirty(page)) {
298 clear_page_dirty_for_io(page);
3a902c5f
NP
299 /*
300 * Want to mark the page and the radix tree as dirty, and
301 * redo the accounting that clear_page_dirty_for_io undid,
302 * but we can't use set_page_dirty because that function
303 * is actually a signal that all of the page has become dirty.
304 * Wheras only part of our page may be dirty.
305 */
306 __set_page_dirty_nobuffers(newpage);
b20a3503
CL
307 }
308
b291f000 309 mlock_migrate_page(newpage, page);
e9995ef9 310 ksm_migrate_page(newpage, page);
b291f000 311
b20a3503 312 ClearPageSwapCache(page);
b20a3503
CL
313 ClearPagePrivate(page);
314 set_page_private(page, 0);
315 page->mapping = NULL;
316
317 /*
318 * If any waiters have accumulated on the new page then
319 * wake them up.
320 */
321 if (PageWriteback(newpage))
322 end_page_writeback(newpage);
323}
b20a3503 324
1d8b85cc
CL
325/************************************************************
326 * Migration functions
327 ***********************************************************/
328
329/* Always fail migration. Used for mappings that are not movable */
2d1db3b1
CL
330int fail_migrate_page(struct address_space *mapping,
331 struct page *newpage, struct page *page)
1d8b85cc
CL
332{
333 return -EIO;
334}
335EXPORT_SYMBOL(fail_migrate_page);
336
b20a3503
CL
337/*
338 * Common logic to directly migrate a single page suitable for
266cf658 339 * pages that do not use PagePrivate/PagePrivate2.
b20a3503
CL
340 *
341 * Pages are locked upon entry and exit.
342 */
2d1db3b1
CL
343int migrate_page(struct address_space *mapping,
344 struct page *newpage, struct page *page)
b20a3503
CL
345{
346 int rc;
347
348 BUG_ON(PageWriteback(page)); /* Writeback must be complete */
349
2d1db3b1 350 rc = migrate_page_move_mapping(mapping, newpage, page);
b20a3503
CL
351
352 if (rc)
353 return rc;
354
355 migrate_page_copy(newpage, page);
b20a3503
CL
356 return 0;
357}
358EXPORT_SYMBOL(migrate_page);
359
9361401e 360#ifdef CONFIG_BLOCK
1d8b85cc
CL
361/*
362 * Migration function for pages with buffers. This function can only be used
363 * if the underlying filesystem guarantees that no other references to "page"
364 * exist.
365 */
2d1db3b1
CL
366int buffer_migrate_page(struct address_space *mapping,
367 struct page *newpage, struct page *page)
1d8b85cc 368{
1d8b85cc
CL
369 struct buffer_head *bh, *head;
370 int rc;
371
1d8b85cc 372 if (!page_has_buffers(page))
2d1db3b1 373 return migrate_page(mapping, newpage, page);
1d8b85cc
CL
374
375 head = page_buffers(page);
376
2d1db3b1 377 rc = migrate_page_move_mapping(mapping, newpage, page);
1d8b85cc
CL
378
379 if (rc)
380 return rc;
381
382 bh = head;
383 do {
384 get_bh(bh);
385 lock_buffer(bh);
386 bh = bh->b_this_page;
387
388 } while (bh != head);
389
390 ClearPagePrivate(page);
391 set_page_private(newpage, page_private(page));
392 set_page_private(page, 0);
393 put_page(page);
394 get_page(newpage);
395
396 bh = head;
397 do {
398 set_bh_page(bh, newpage, bh_offset(bh));
399 bh = bh->b_this_page;
400
401 } while (bh != head);
402
403 SetPagePrivate(newpage);
404
405 migrate_page_copy(newpage, page);
406
407 bh = head;
408 do {
409 unlock_buffer(bh);
410 put_bh(bh);
411 bh = bh->b_this_page;
412
413 } while (bh != head);
414
415 return 0;
416}
417EXPORT_SYMBOL(buffer_migrate_page);
9361401e 418#endif
1d8b85cc 419
04e62a29
CL
420/*
421 * Writeback a page to clean the dirty state
422 */
423static int writeout(struct address_space *mapping, struct page *page)
8351a6e4 424{
04e62a29
CL
425 struct writeback_control wbc = {
426 .sync_mode = WB_SYNC_NONE,
427 .nr_to_write = 1,
428 .range_start = 0,
429 .range_end = LLONG_MAX,
430 .nonblocking = 1,
431 .for_reclaim = 1
432 };
433 int rc;
434
435 if (!mapping->a_ops->writepage)
436 /* No write method for the address space */
437 return -EINVAL;
438
439 if (!clear_page_dirty_for_io(page))
440 /* Someone else already triggered a write */
441 return -EAGAIN;
442
8351a6e4 443 /*
04e62a29
CL
444 * A dirty page may imply that the underlying filesystem has
445 * the page on some queue. So the page must be clean for
446 * migration. Writeout may mean we loose the lock and the
447 * page state is no longer what we checked for earlier.
448 * At this point we know that the migration attempt cannot
449 * be successful.
8351a6e4 450 */
04e62a29 451 remove_migration_ptes(page, page);
8351a6e4 452
04e62a29 453 rc = mapping->a_ops->writepage(page, &wbc);
8351a6e4 454
04e62a29
CL
455 if (rc != AOP_WRITEPAGE_ACTIVATE)
456 /* unlocked. Relock */
457 lock_page(page);
458
bda8550d 459 return (rc < 0) ? -EIO : -EAGAIN;
04e62a29
CL
460}
461
462/*
463 * Default handling if a filesystem does not provide a migration function.
464 */
465static int fallback_migrate_page(struct address_space *mapping,
466 struct page *newpage, struct page *page)
467{
468 if (PageDirty(page))
469 return writeout(mapping, page);
8351a6e4
CL
470
471 /*
472 * Buffers may be managed in a filesystem specific way.
473 * We must have no buffers or drop them.
474 */
266cf658 475 if (page_has_private(page) &&
8351a6e4
CL
476 !try_to_release_page(page, GFP_KERNEL))
477 return -EAGAIN;
478
479 return migrate_page(mapping, newpage, page);
480}
481
e24f0b8f
CL
482/*
483 * Move a page to a newly allocated page
484 * The page is locked and all ptes have been successfully removed.
485 *
486 * The new page will have replaced the old page if this function
487 * is successful.
894bc310
LS
488 *
489 * Return value:
490 * < 0 - error code
491 * == 0 - success
e24f0b8f
CL
492 */
493static int move_to_new_page(struct page *newpage, struct page *page)
494{
495 struct address_space *mapping;
496 int rc;
497
498 /*
499 * Block others from accessing the page when we get around to
500 * establishing additional references. We are the only one
501 * holding a reference to the new page at this point.
502 */
529ae9aa 503 if (!trylock_page(newpage))
e24f0b8f
CL
504 BUG();
505
506 /* Prepare mapping for the new page.*/
507 newpage->index = page->index;
508 newpage->mapping = page->mapping;
b2e18538
RR
509 if (PageSwapBacked(page))
510 SetPageSwapBacked(newpage);
e24f0b8f
CL
511
512 mapping = page_mapping(page);
513 if (!mapping)
514 rc = migrate_page(mapping, newpage, page);
515 else if (mapping->a_ops->migratepage)
516 /*
517 * Most pages have a mapping and most filesystems
518 * should provide a migration function. Anonymous
519 * pages are part of swap space which also has its
520 * own migration function. This is the most common
521 * path for page migration.
522 */
523 rc = mapping->a_ops->migratepage(mapping,
524 newpage, page);
525 else
526 rc = fallback_migrate_page(mapping, newpage, page);
527
e9995ef9 528 if (!rc)
e24f0b8f 529 remove_migration_ptes(page, newpage);
e9995ef9 530 else
e24f0b8f
CL
531 newpage->mapping = NULL;
532
533 unlock_page(newpage);
534
535 return rc;
536}
537
538/*
539 * Obtain the lock on page, remove all ptes and migrate the page
540 * to the newly allocated page in newpage.
541 */
95a402c3 542static int unmap_and_move(new_page_t get_new_page, unsigned long private,
62b61f61 543 struct page *page, int force, int offlining)
e24f0b8f
CL
544{
545 int rc = 0;
742755a1
CL
546 int *result = NULL;
547 struct page *newpage = get_new_page(page, private, &result);
989f89c5 548 int rcu_locked = 0;
ae41be37 549 int charge = 0;
e00e4316 550 struct mem_cgroup *mem = NULL;
95a402c3
CL
551
552 if (!newpage)
553 return -ENOMEM;
e24f0b8f 554
894bc310 555 if (page_count(page) == 1) {
e24f0b8f 556 /* page was freed from under us. So we are done. */
95a402c3 557 goto move_newpage;
894bc310 558 }
e24f0b8f 559
e8589cc1 560 /* prepare cgroup just returns 0 or -ENOMEM */
e24f0b8f 561 rc = -EAGAIN;
01b1ae63 562
529ae9aa 563 if (!trylock_page(page)) {
e24f0b8f 564 if (!force)
95a402c3 565 goto move_newpage;
e24f0b8f
CL
566 lock_page(page);
567 }
568
62b61f61
HD
569 /*
570 * Only memory hotplug's offline_pages() caller has locked out KSM,
571 * and can safely migrate a KSM page. The other cases have skipped
572 * PageKsm along with PageReserved - but it is only now when we have
573 * the page lock that we can be certain it will not go KSM beneath us
574 * (KSM will not upgrade a page from PageAnon to PageKsm when it sees
575 * its pagecount raised, but only here do we take the page lock which
576 * serializes that).
577 */
578 if (PageKsm(page) && !offlining) {
579 rc = -EBUSY;
580 goto unlock;
581 }
582
01b1ae63
KH
583 /* charge against new page */
584 charge = mem_cgroup_prepare_migration(page, &mem);
585 if (charge == -ENOMEM) {
586 rc = -ENOMEM;
587 goto unlock;
588 }
589 BUG_ON(charge);
590
e24f0b8f
CL
591 if (PageWriteback(page)) {
592 if (!force)
01b1ae63 593 goto uncharge;
e24f0b8f
CL
594 wait_on_page_writeback(page);
595 }
e24f0b8f 596 /*
dc386d4d
KH
597 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
598 * we cannot notice that anon_vma is freed while we migrates a page.
599 * This rcu_read_lock() delays freeing anon_vma pointer until the end
600 * of migration. File cache pages are no problem because of page_lock()
989f89c5
KH
601 * File Caches may use write_page() or lock_page() in migration, then,
602 * just care Anon page here.
dc386d4d 603 */
989f89c5
KH
604 if (PageAnon(page)) {
605 rcu_read_lock();
606 rcu_locked = 1;
607 }
62e1c553 608
dc386d4d 609 /*
62e1c553
SL
610 * Corner case handling:
611 * 1. When a new swap-cache page is read into, it is added to the LRU
612 * and treated as swapcache but it has no rmap yet.
613 * Calling try_to_unmap() against a page->mapping==NULL page will
614 * trigger a BUG. So handle it here.
615 * 2. An orphaned page (see truncate_complete_page) might have
616 * fs-private metadata. The page can be picked up due to memory
617 * offlining. Everywhere else except page reclaim, the page is
618 * invisible to the vm, so the page can not be migrated. So try to
619 * free the metadata, so the page can be freed.
e24f0b8f 620 */
62e1c553 621 if (!page->mapping) {
266cf658 622 if (!PageAnon(page) && page_has_private(page)) {
62e1c553
SL
623 /*
624 * Go direct to try_to_free_buffers() here because
625 * a) that's what try_to_release_page() would do anyway
626 * b) we may be under rcu_read_lock() here, so we can't
627 * use GFP_KERNEL which is what try_to_release_page()
628 * needs to be effective.
629 */
630 try_to_free_buffers(page);
abfc3488 631 goto rcu_unlock;
62e1c553 632 }
abfc3488 633 goto skip_unmap;
62e1c553
SL
634 }
635
dc386d4d 636 /* Establish migration ptes or remove ptes */
14fa31b8 637 try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
dc386d4d 638
abfc3488 639skip_unmap:
e6a1530d
CL
640 if (!page_mapped(page))
641 rc = move_to_new_page(newpage, page);
e24f0b8f 642
e8589cc1 643 if (rc)
e24f0b8f 644 remove_migration_ptes(page, page);
dc386d4d 645rcu_unlock:
989f89c5
KH
646 if (rcu_locked)
647 rcu_read_unlock();
01b1ae63
KH
648uncharge:
649 if (!charge)
650 mem_cgroup_end_migration(mem, page, newpage);
e24f0b8f
CL
651unlock:
652 unlock_page(page);
95a402c3 653
e24f0b8f 654 if (rc != -EAGAIN) {
aaa994b3
CL
655 /*
656 * A page that has been migrated has all references
657 * removed and will be freed. A page that has not been
658 * migrated will have kepts its references and be
659 * restored.
660 */
661 list_del(&page->lru);
a731286d 662 dec_zone_page_state(page, NR_ISOLATED_ANON +
6c0b1351 663 page_is_file_cache(page));
894bc310 664 putback_lru_page(page);
e24f0b8f 665 }
95a402c3
CL
666
667move_newpage:
894bc310 668
95a402c3
CL
669 /*
670 * Move the new page to the LRU. If migration was not successful
671 * then this will free the page.
672 */
894bc310
LS
673 putback_lru_page(newpage);
674
742755a1
CL
675 if (result) {
676 if (rc)
677 *result = rc;
678 else
679 *result = page_to_nid(newpage);
680 }
e24f0b8f
CL
681 return rc;
682}
683
b20a3503
CL
684/*
685 * migrate_pages
686 *
95a402c3
CL
687 * The function takes one list of pages to migrate and a function
688 * that determines from the page to be migrated and the private data
689 * the target of the move and allocates the page.
b20a3503
CL
690 *
691 * The function returns after 10 attempts or if no pages
692 * are movable anymore because to has become empty
aaa994b3 693 * or no retryable pages exist anymore. All pages will be
e9534b3f 694 * returned to the LRU or freed.
b20a3503 695 *
95a402c3 696 * Return: Number of pages not migrated or error code.
b20a3503 697 */
95a402c3 698int migrate_pages(struct list_head *from,
62b61f61 699 new_page_t get_new_page, unsigned long private, int offlining)
b20a3503 700{
e24f0b8f 701 int retry = 1;
b20a3503
CL
702 int nr_failed = 0;
703 int pass = 0;
704 struct page *page;
705 struct page *page2;
706 int swapwrite = current->flags & PF_SWAPWRITE;
707 int rc;
708
709 if (!swapwrite)
710 current->flags |= PF_SWAPWRITE;
711
e24f0b8f
CL
712 for(pass = 0; pass < 10 && retry; pass++) {
713 retry = 0;
b20a3503 714
e24f0b8f 715 list_for_each_entry_safe(page, page2, from, lru) {
e24f0b8f 716 cond_resched();
2d1db3b1 717
95a402c3 718 rc = unmap_and_move(get_new_page, private,
62b61f61 719 page, pass > 2, offlining);
2d1db3b1 720
e24f0b8f 721 switch(rc) {
95a402c3
CL
722 case -ENOMEM:
723 goto out;
e24f0b8f 724 case -EAGAIN:
2d1db3b1 725 retry++;
e24f0b8f
CL
726 break;
727 case 0:
e24f0b8f
CL
728 break;
729 default:
2d1db3b1 730 /* Permanent failure */
2d1db3b1 731 nr_failed++;
e24f0b8f 732 break;
2d1db3b1 733 }
b20a3503
CL
734 }
735 }
95a402c3
CL
736 rc = 0;
737out:
b20a3503
CL
738 if (!swapwrite)
739 current->flags &= ~PF_SWAPWRITE;
740
aaa994b3 741 putback_lru_pages(from);
b20a3503 742
95a402c3
CL
743 if (rc)
744 return rc;
b20a3503 745
95a402c3 746 return nr_failed + retry;
b20a3503 747}
95a402c3 748
742755a1
CL
749#ifdef CONFIG_NUMA
750/*
751 * Move a list of individual pages
752 */
753struct page_to_node {
754 unsigned long addr;
755 struct page *page;
756 int node;
757 int status;
758};
759
760static struct page *new_page_node(struct page *p, unsigned long private,
761 int **result)
762{
763 struct page_to_node *pm = (struct page_to_node *)private;
764
765 while (pm->node != MAX_NUMNODES && pm->page != p)
766 pm++;
767
768 if (pm->node == MAX_NUMNODES)
769 return NULL;
770
771 *result = &pm->status;
772
6484eb3e 773 return alloc_pages_exact_node(pm->node,
769848c0 774 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
742755a1
CL
775}
776
777/*
778 * Move a set of pages as indicated in the pm array. The addr
779 * field must be set to the virtual address of the page to be moved
780 * and the node number must contain a valid target node.
5e9a0f02 781 * The pm array ends with node = MAX_NUMNODES.
742755a1 782 */
5e9a0f02
BG
783static int do_move_page_to_node_array(struct mm_struct *mm,
784 struct page_to_node *pm,
785 int migrate_all)
742755a1
CL
786{
787 int err;
788 struct page_to_node *pp;
789 LIST_HEAD(pagelist);
790
791 down_read(&mm->mmap_sem);
792
793 /*
794 * Build a list of pages to migrate
795 */
742755a1
CL
796 for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
797 struct vm_area_struct *vma;
798 struct page *page;
799
742755a1
CL
800 err = -EFAULT;
801 vma = find_vma(mm, pp->addr);
0dc952dc 802 if (!vma || !vma_migratable(vma))
742755a1
CL
803 goto set_status;
804
805 page = follow_page(vma, pp->addr, FOLL_GET);
89f5b7da
LT
806
807 err = PTR_ERR(page);
808 if (IS_ERR(page))
809 goto set_status;
810
742755a1
CL
811 err = -ENOENT;
812 if (!page)
813 goto set_status;
814
62b61f61
HD
815 /* Use PageReserved to check for zero page */
816 if (PageReserved(page) || PageKsm(page))
742755a1
CL
817 goto put_and_set;
818
819 pp->page = page;
820 err = page_to_nid(page);
821
822 if (err == pp->node)
823 /*
824 * Node already in the right place
825 */
826 goto put_and_set;
827
828 err = -EACCES;
829 if (page_mapcount(page) > 1 &&
830 !migrate_all)
831 goto put_and_set;
832
62695a84 833 err = isolate_lru_page(page);
6d9c285a 834 if (!err) {
62695a84 835 list_add_tail(&page->lru, &pagelist);
6d9c285a
KM
836 inc_zone_page_state(page, NR_ISOLATED_ANON +
837 page_is_file_cache(page));
838 }
742755a1
CL
839put_and_set:
840 /*
841 * Either remove the duplicate refcount from
842 * isolate_lru_page() or drop the page ref if it was
843 * not isolated.
844 */
845 put_page(page);
846set_status:
847 pp->status = err;
848 }
849
e78bbfa8 850 err = 0;
742755a1
CL
851 if (!list_empty(&pagelist))
852 err = migrate_pages(&pagelist, new_page_node,
62b61f61 853 (unsigned long)pm, 0);
742755a1
CL
854
855 up_read(&mm->mmap_sem);
856 return err;
857}
858
5e9a0f02
BG
859/*
860 * Migrate an array of page address onto an array of nodes and fill
861 * the corresponding array of status.
862 */
863static int do_pages_move(struct mm_struct *mm, struct task_struct *task,
864 unsigned long nr_pages,
865 const void __user * __user *pages,
866 const int __user *nodes,
867 int __user *status, int flags)
868{
3140a227 869 struct page_to_node *pm;
5e9a0f02 870 nodemask_t task_nodes;
3140a227
BG
871 unsigned long chunk_nr_pages;
872 unsigned long chunk_start;
873 int err;
5e9a0f02
BG
874
875 task_nodes = cpuset_mems_allowed(task);
876
3140a227
BG
877 err = -ENOMEM;
878 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
879 if (!pm)
5e9a0f02 880 goto out;
35282a2d
BG
881
882 migrate_prep();
883
5e9a0f02 884 /*
3140a227
BG
885 * Store a chunk of page_to_node array in a page,
886 * but keep the last one as a marker
5e9a0f02 887 */
3140a227 888 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
5e9a0f02 889
3140a227
BG
890 for (chunk_start = 0;
891 chunk_start < nr_pages;
892 chunk_start += chunk_nr_pages) {
893 int j;
5e9a0f02 894
3140a227
BG
895 if (chunk_start + chunk_nr_pages > nr_pages)
896 chunk_nr_pages = nr_pages - chunk_start;
897
898 /* fill the chunk pm with addrs and nodes from user-space */
899 for (j = 0; j < chunk_nr_pages; j++) {
900 const void __user *p;
5e9a0f02
BG
901 int node;
902
3140a227
BG
903 err = -EFAULT;
904 if (get_user(p, pages + j + chunk_start))
905 goto out_pm;
906 pm[j].addr = (unsigned long) p;
907
908 if (get_user(node, nodes + j + chunk_start))
5e9a0f02
BG
909 goto out_pm;
910
911 err = -ENODEV;
6f5a55f1
LT
912 if (node < 0 || node >= MAX_NUMNODES)
913 goto out_pm;
914
5e9a0f02
BG
915 if (!node_state(node, N_HIGH_MEMORY))
916 goto out_pm;
917
918 err = -EACCES;
919 if (!node_isset(node, task_nodes))
920 goto out_pm;
921
3140a227
BG
922 pm[j].node = node;
923 }
924
925 /* End marker for this chunk */
926 pm[chunk_nr_pages].node = MAX_NUMNODES;
927
928 /* Migrate this chunk */
929 err = do_move_page_to_node_array(mm, pm,
930 flags & MPOL_MF_MOVE_ALL);
931 if (err < 0)
932 goto out_pm;
5e9a0f02 933
5e9a0f02 934 /* Return status information */
3140a227
BG
935 for (j = 0; j < chunk_nr_pages; j++)
936 if (put_user(pm[j].status, status + j + chunk_start)) {
5e9a0f02 937 err = -EFAULT;
3140a227
BG
938 goto out_pm;
939 }
940 }
941 err = 0;
5e9a0f02
BG
942
943out_pm:
3140a227 944 free_page((unsigned long)pm);
5e9a0f02
BG
945out:
946 return err;
947}
948
742755a1 949/*
2f007e74 950 * Determine the nodes of an array of pages and store it in an array of status.
742755a1 951 */
80bba129
BG
952static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
953 const void __user **pages, int *status)
742755a1 954{
2f007e74 955 unsigned long i;
2f007e74 956
742755a1
CL
957 down_read(&mm->mmap_sem);
958
2f007e74 959 for (i = 0; i < nr_pages; i++) {
80bba129 960 unsigned long addr = (unsigned long)(*pages);
742755a1
CL
961 struct vm_area_struct *vma;
962 struct page *page;
c095adbc 963 int err = -EFAULT;
2f007e74
BG
964
965 vma = find_vma(mm, addr);
742755a1
CL
966 if (!vma)
967 goto set_status;
968
2f007e74 969 page = follow_page(vma, addr, 0);
89f5b7da
LT
970
971 err = PTR_ERR(page);
972 if (IS_ERR(page))
973 goto set_status;
974
742755a1
CL
975 err = -ENOENT;
976 /* Use PageReserved to check for zero page */
62b61f61 977 if (!page || PageReserved(page) || PageKsm(page))
742755a1
CL
978 goto set_status;
979
980 err = page_to_nid(page);
981set_status:
80bba129
BG
982 *status = err;
983
984 pages++;
985 status++;
986 }
987
988 up_read(&mm->mmap_sem);
989}
990
991/*
992 * Determine the nodes of a user array of pages and store it in
993 * a user array of status.
994 */
995static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
996 const void __user * __user *pages,
997 int __user *status)
998{
999#define DO_PAGES_STAT_CHUNK_NR 16
1000 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1001 int chunk_status[DO_PAGES_STAT_CHUNK_NR];
80bba129 1002
87b8d1ad
PA
1003 while (nr_pages) {
1004 unsigned long chunk_nr;
80bba129 1005
87b8d1ad
PA
1006 chunk_nr = nr_pages;
1007 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1008 chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1009
1010 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1011 break;
80bba129
BG
1012
1013 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1014
87b8d1ad
PA
1015 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1016 break;
742755a1 1017
87b8d1ad
PA
1018 pages += chunk_nr;
1019 status += chunk_nr;
1020 nr_pages -= chunk_nr;
1021 }
1022 return nr_pages ? -EFAULT : 0;
742755a1
CL
1023}
1024
1025/*
1026 * Move a list of pages in the address space of the currently executing
1027 * process.
1028 */
938bb9f5
HC
1029SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1030 const void __user * __user *, pages,
1031 const int __user *, nodes,
1032 int __user *, status, int, flags)
742755a1 1033{
c69e8d9c 1034 const struct cred *cred = current_cred(), *tcred;
742755a1 1035 struct task_struct *task;
742755a1 1036 struct mm_struct *mm;
5e9a0f02 1037 int err;
742755a1
CL
1038
1039 /* Check flags */
1040 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1041 return -EINVAL;
1042
1043 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1044 return -EPERM;
1045
1046 /* Find the mm_struct */
1047 read_lock(&tasklist_lock);
228ebcbe 1048 task = pid ? find_task_by_vpid(pid) : current;
742755a1
CL
1049 if (!task) {
1050 read_unlock(&tasklist_lock);
1051 return -ESRCH;
1052 }
1053 mm = get_task_mm(task);
1054 read_unlock(&tasklist_lock);
1055
1056 if (!mm)
1057 return -EINVAL;
1058
1059 /*
1060 * Check if this process has the right to modify the specified
1061 * process. The right exists if the process has administrative
1062 * capabilities, superuser privileges or the same
1063 * userid as the target process.
1064 */
c69e8d9c
DH
1065 rcu_read_lock();
1066 tcred = __task_cred(task);
b6dff3ec
DH
1067 if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
1068 cred->uid != tcred->suid && cred->uid != tcred->uid &&
742755a1 1069 !capable(CAP_SYS_NICE)) {
c69e8d9c 1070 rcu_read_unlock();
742755a1 1071 err = -EPERM;
5e9a0f02 1072 goto out;
742755a1 1073 }
c69e8d9c 1074 rcu_read_unlock();
742755a1 1075
86c3a764
DQ
1076 err = security_task_movememory(task);
1077 if (err)
5e9a0f02 1078 goto out;
86c3a764 1079
5e9a0f02
BG
1080 if (nodes) {
1081 err = do_pages_move(mm, task, nr_pages, pages, nodes, status,
1082 flags);
1083 } else {
2f007e74 1084 err = do_pages_stat(mm, nr_pages, pages, status);
742755a1
CL
1085 }
1086
742755a1 1087out:
742755a1
CL
1088 mmput(mm);
1089 return err;
1090}
742755a1 1091
7b2259b3
CL
1092/*
1093 * Call migration functions in the vma_ops that may prepare
1094 * memory in a vm for migration. migration functions may perform
1095 * the migration for vmas that do not have an underlying page struct.
1096 */
1097int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
1098 const nodemask_t *from, unsigned long flags)
1099{
1100 struct vm_area_struct *vma;
1101 int err = 0;
1102
1001c9fb 1103 for (vma = mm->mmap; vma && !err; vma = vma->vm_next) {
7b2259b3
CL
1104 if (vma->vm_ops && vma->vm_ops->migrate) {
1105 err = vma->vm_ops->migrate(vma, to, from, flags);
1106 if (err)
1107 break;
1108 }
1109 }
1110 return err;
1111}
83d1674a 1112#endif