page allocator: move free_page_mlock() to page_alloc.c

[linux-2.6-block.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index abe26003124de3d3720a87684ee96daf7c720499..5dac5d8cb1489673b778965ef74d4da1eb74aa61 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -161,7 +161,9 @@ static unsigned long __meminitdata dma_reserve;
 
 #if MAX_NUMNODES > 1
 int nr_node_ids __read_mostly = MAX_NUMNODES;
+int nr_online_nodes __read_mostly = 1;
 EXPORT_SYMBOL(nr_node_ids);
+EXPORT_SYMBOL(nr_online_nodes);
 #endif
 
 int page_group_by_mobility_disabled __read_mostly;
@@ -456,7 +458,6 @@ static inline void __free_one_page(struct page *page,
                int migratetype)
 {
        unsigned long page_idx;
-       int order_size = 1 << order;
 
        if (unlikely(PageCompound(page)))
                if (unlikely(destroy_compound_page(page, order)))
@@ -466,10 +467,9 @@ static inline void __free_one_page(struct page *page,
 
        page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
 
-       VM_BUG_ON(page_idx & (order_size - 1));
+       VM_BUG_ON(page_idx & ((1 << order) - 1));
        VM_BUG_ON(bad_range(zone, page));
 
-       __mod_zone_page_state(zone, NR_FREE_PAGES, order_size);
        while (order < MAX_ORDER-1) {
                unsigned long combined_idx;
                struct page *buddy;
@@ -493,6 +493,22 @@ static inline void __free_one_page(struct page *page,
        zone->free_area[order].nr_free++;
 }
 
+#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT
+/*
+ * free_page_mlock() -- clean up attempts to free and mlocked() page.
+ * Page should not be on lru, so no need to fix that up.
+ * free_pages_check() will verify...
+ */
+static inline void free_page_mlock(struct page *page)
+{
+       __ClearPageMlocked(page);
+       __dec_zone_page_state(page, NR_MLOCK);
+       __count_vm_event(UNEVICTABLE_MLOCKFREED);
+}
+#else
+static void free_page_mlock(struct page *page) { }
+#endif
+
 static inline int free_pages_check(struct page *page)
 {
        if (unlikely(page_mapcount(page) |
@@ -524,6 +540,8 @@ static void free_pages_bulk(struct zone *zone, int count,
        spin_lock(&zone->lock);
        zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
        zone->pages_scanned = 0;
+
+       __mod_zone_page_state(zone, NR_FREE_PAGES, count << order);
        while (count--) {
                struct page *page;
 
@@ -542,6 +560,8 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
        spin_lock(&zone->lock);
        zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
        zone->pages_scanned = 0;
+
+       __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
        __free_one_page(page, zone, order, migratetype);
        spin_unlock(&zone->lock);
 }
@@ -686,7 +706,6 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
                list_del(&page->lru);
                rmv_page_order(page);
                area->nr_free--;
-               __mod_zone_page_state(zone, NR_FREE_PAGES, - (1UL << order));
                expand(zone, page, order, current_order, area, migratetype);
                return page;
        }
@@ -826,8 +845,6 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
                        /* Remove the page from the freelists */
                        list_del(&page->lru);
                        rmv_page_order(page);
-                       __mod_zone_page_state(zone, NR_FREE_PAGES,
-                                                       -(1UL << order));
 
                        if (current_order == pageblock_order)
                                set_pageblock_migratetype(page,
@@ -900,6 +917,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                set_page_private(page, migratetype);
                list = &page->lru;
        }
+       __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
        spin_unlock(&zone->lock);
        return i;
 }
@@ -1030,6 +1048,7 @@ static void free_hot_cold_page(struct page *page, int cold)
        kernel_map_pages(page, 1, 0);
 
        pcp = &zone_pcp(zone, get_cpu())->pcp;
+       set_page_private(page, get_pageblock_migratetype(page));
        local_irq_save(flags);
        if (unlikely(clearMlocked))
                free_page_mlock(page);
@@ -1039,7 +1058,6 @@ static void free_hot_cold_page(struct page *page, int cold)
                list_add_tail(&page->lru, &pcp->list);
        else
                list_add(&page->lru, &pcp->list);
-       set_page_private(page, get_pageblock_migratetype(page));
        pcp->count++;
        if (pcp->count >= pcp->high) {
                free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
@@ -1129,6 +1147,7 @@ again:
        } else {
                spin_lock_irqsave(&zone->lock, flags);
                page = __rmqueue(zone, order, migratetype);
+               __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
                spin_unlock(&zone->lock);
                if (!page)
                        goto failed;
@@ -1465,7 +1484,7 @@ this_zone_full:
                if (NUMA_BUILD)
                        zlc_mark_zone_full(zonelist, z);
 try_next_zone:
-               if (NUMA_BUILD && !did_zlc_setup && num_online_nodes() > 1) {
+               if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
                        /*
                         * we do zlc_setup after the first zone is tried but only
                         * if there are multiple nodes make it worthwhile
@@ -2264,7 +2283,7 @@ int numa_zonelist_order_handler(ctl_table *table, int write,
 }
 
 
-#define MAX_NODE_LOAD (num_online_nodes())
+#define MAX_NODE_LOAD (nr_online_nodes)
 static int node_load[MAX_NUMNODES];
 
 /**
@@ -2473,7 +2492,7 @@ static void build_zonelists(pg_data_t *pgdat)
 
        /* NUMA-aware ordering of nodes */
        local_node = pgdat->node_id;
-       load = num_online_nodes();
+       load = nr_online_nodes;
        prev_node = local_node;
        nodes_clear(used_mask);
 
@@ -2624,7 +2643,7 @@ void build_all_zonelists(void)
 
        printk("Built %i zonelists in %s order, mobility grouping %s.  "
                "Total pages: %ld\n",
-                       num_online_nodes(),
+                       nr_online_nodes,
                        zonelist_order_name[current_zonelist_order],
                        page_group_by_mobility_disabled ? "off" : "on",
                        vm_total_pages);