Unevictable LRU Infrastructure

[linux-2.6-block.git] / mm / migrate.c

diff --git a/mm/migrate.c b/mm/migrate.c
index 153572fb60b8debb24359e195e7a53b50f8e1c37..b10237d8b459823b84f1b2a070a823532d585d34 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,36 +36,6 @@
 
 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
 
-/*
- * Isolate one page from the LRU lists. If successful put it onto
- * the indicated list with elevated page count.
- *
- * Result:
- *  -EBUSY: page not on LRU list
- *  0: page removed from LRU list and added to the specified list.
- */
-int isolate_lru_page(struct page *page, struct list_head *pagelist)
-{
-       int ret = -EBUSY;
-
-       if (PageLRU(page)) {
-               struct zone *zone = page_zone(page);
-
-               spin_lock_irq(&zone->lru_lock);
-               if (PageLRU(page) && get_page_unless_zero(page)) {
-                       ret = 0;
-                       ClearPageLRU(page);
-                       if (PageActive(page))
-                               del_page_from_active_list(zone, page);
-                       else
-                               del_page_from_inactive_list(zone, page);
-                       list_add_tail(&page->lru, pagelist);
-               }
-               spin_unlock_irq(&zone->lru_lock);
-       }
-       return ret;
-}
-
 /*
  * migrate_prep() needs to be called before we start compiling a list of pages
  * to be migrated using isolate_lru_page().
@@ -83,23 +53,9 @@ int migrate_prep(void)
        return 0;
 }
 
-static inline void move_to_lru(struct page *page)
-{
-       if (PageActive(page)) {
-               /*
-                * lru_cache_add_active checks that
-                * the PG_active bit is off.
-                */
-               ClearPageActive(page);
-               lru_cache_add_active(page);
-       } else {
-               lru_cache_add(page);
-       }
-       put_page(page);
-}
-
 /*
- * Add isolated pages on the list back to the LRU.
+ * Add isolated pages on the list back to the LRU under page lock
+ * to avoid leaking evictable pages back onto unevictable list.
  *
  * returns the number of pages put back.
  */
@@ -111,7 +67,7 @@ int putback_lru_pages(struct list_head *l)
 
        list_for_each_entry_safe(page, page2, l, lru) {
                list_del(&page->lru);
-               move_to_lru(page);
+               putback_lru_page(page);
                count++;
        }
        return count;
@@ -393,8 +349,11 @@ static void migrate_page_copy(struct page *newpage, struct page *page)
                SetPageReferenced(newpage);
        if (PageUptodate(page))
                SetPageUptodate(newpage);
-       if (PageActive(page))
+       if (TestClearPageActive(page)) {
+               VM_BUG_ON(PageUnevictable(page));
                SetPageActive(newpage);
+       } else
+               unevictable_migrate_page(newpage, page);
        if (PageChecked(page))
                SetPageChecked(newpage);
        if (PageMappedToDisk(page))
@@ -415,7 +374,6 @@ static void migrate_page_copy(struct page *newpage, struct page *page)
 #ifdef CONFIG_SWAP
        ClearPageSwapCache(page);
 #endif
-       ClearPageActive(page);
        ClearPagePrivate(page);
        set_page_private(page, 0);
        page->mapping = NULL;
@@ -594,6 +552,10 @@ static int fallback_migrate_page(struct address_space *mapping,
  *
  * The new page will have replaced the old page if this function
  * is successful.
+ *
+ * Return value:
+ *   < 0 - error code
+ *  == 0 - success
  */
 static int move_to_new_page(struct page *newpage, struct page *page)
 {
@@ -605,12 +567,14 @@ static int move_to_new_page(struct page *newpage, struct page *page)
         * establishing additional references. We are the only one
         * holding a reference to the new page at this point.
         */
-       if (TestSetPageLocked(newpage))
+       if (!trylock_page(newpage))
                BUG();
 
        /* Prepare mapping for the new page.*/
        newpage->index = page->index;
        newpage->mapping = page->mapping;
+       if (PageSwapBacked(page))
+               SetPageSwapBacked(newpage);
 
        mapping = page_mapping(page);
        if (!mapping)
@@ -654,9 +618,10 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        if (!newpage)
                return -ENOMEM;
 
-       if (page_count(page) == 1)
+       if (page_count(page) == 1) {
                /* page was freed from under us. So we are done. */
                goto move_newpage;
+       }
 
        charge = mem_cgroup_prepare_migration(page, newpage);
        if (charge == -ENOMEM) {
@@ -667,7 +632,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        BUG_ON(charge);
 
        rc = -EAGAIN;
-       if (TestSetPageLocked(page)) {
+       if (!trylock_page(page)) {
                if (!force)
                        goto move_newpage;
                lock_page(page);
@@ -730,7 +695,6 @@ rcu_unlock:
                rcu_read_unlock();
 
 unlock:
-
        unlock_page(page);
 
        if (rc != -EAGAIN) {
@@ -741,17 +705,19 @@ unlock:
                 * restored.
                 */
                list_del(&page->lru);
-               move_to_lru(page);
+               putback_lru_page(page);
        }
 
 move_newpage:
        if (!charge)
                mem_cgroup_end_migration(newpage);
+
        /*
         * Move the new page to the LRU. If migration was not successful
         * then this will free the page.
         */
-       move_to_lru(newpage);
+       putback_lru_page(newpage);
+
        if (result) {
                if (rc)
                        *result = rc;
@@ -914,7 +880,9 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
                                !migrate_all)
                        goto put_and_set;
 
-               err = isolate_lru_page(page, &pagelist);
+               err = isolate_lru_page(page);
+               if (!err)
+                       list_add_tail(&page->lru, &pagelist);
 put_and_set:
                /*
                 * Either remove the duplicate refcount from