mm: memcontrol: flush percpu vmevents before releasing memcg

[linux-2.6-block.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6f5c0c517c497dbddad9016ef2d1993e4e1bcf05..26e2999af608d3776e609b213d522345fc793792 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3260,6 +3260,60 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
        }
 }
 
+static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
+{
+       unsigned long stat[MEMCG_NR_STAT];
+       struct mem_cgroup *mi;
+       int node, cpu, i;
+
+       for (i = 0; i < MEMCG_NR_STAT; i++)
+               stat[i] = 0;
+
+       for_each_online_cpu(cpu)
+               for (i = 0; i < MEMCG_NR_STAT; i++)
+                       stat[i] += raw_cpu_read(memcg->vmstats_percpu->stat[i]);
+
+       for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+               for (i = 0; i < MEMCG_NR_STAT; i++)
+                       atomic_long_add(stat[i], &mi->vmstats[i]);
+
+       for_each_node(node) {
+               struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
+               struct mem_cgroup_per_node *pi;
+
+               for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                       stat[i] = 0;
+
+               for_each_online_cpu(cpu)
+                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                               stat[i] += raw_cpu_read(
+                                       pn->lruvec_stat_cpu->count[i]);
+
+               for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
+                       for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                               atomic_long_add(stat[i], &pi->lruvec_stat[i]);
+       }
+}
+
+static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
+{
+       unsigned long events[NR_VM_EVENT_ITEMS];
+       struct mem_cgroup *mi;
+       int cpu, i;
+
+       for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+               events[i] = 0;
+
+       for_each_online_cpu(cpu)
+               for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                       events[i] += raw_cpu_read(
+                               memcg->vmstats_percpu->events[i]);
+
+       for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+               for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                       atomic_long_add(events[i], &mi->vmevents[i]);
+}
+
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_online_kmem(struct mem_cgroup *memcg)
 {
@@ -4682,6 +4736,12 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
        int node;
 
+       /*
+        * Flush percpu vmstats and vmevents to guarantee the value correctness
+        * on parent's and all ancestor levels.
+        */
+       memcg_flush_percpu_vmstats(memcg);
+       memcg_flush_percpu_vmevents(memcg);
        for_each_node(node)
                free_mem_cgroup_per_node_info(memcg, node);
        free_percpu(memcg->vmstats_percpu);