blk-cgroup: Flush stats before releasing blkcg_gq

[linux-block.git] / block / blk-cgroup.c

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 0ce64dd73cfe71489cce5dba5672a66b0052d4e0..f0b5c9c41cdee1c40cab795beca2e5e932ef0d00 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -34,6 +34,8 @@
 #include "blk-ioprio.h"
 #include "blk-throttle.h"
 
+static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
+
 /*
  * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
  * blkcg_pol_register_mutex nests outside of it and synchronizes entire
@@ -56,6 +58,8 @@ static LIST_HEAD(all_blkcgs);         /* protected by blkcg_pol_mutex */
 
 bool blkcg_debug_stats = false;
 
+static DEFINE_RAW_SPINLOCK(blkg_stat_lock);
+
 #define BLKG_DESTROY_BATCH_SIZE  64
 
 /*
@@ -163,10 +167,20 @@ static void blkg_free(struct blkcg_gq *blkg)
 static void __blkg_release(struct rcu_head *rcu)
 {
        struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
+       struct blkcg *blkcg = blkg->blkcg;
+       int cpu;
 
 #ifdef CONFIG_BLK_CGROUP_PUNT_BIO
        WARN_ON(!bio_list_empty(&blkg->async_bios));
 #endif
+       /*
+        * Flush all the non-empty percpu lockless lists before releasing
+        * us, given these stat belongs to us.
+        *
+        * blkg_stat_lock is for serializing blkg stat update
+        */
+       for_each_possible_cpu(cpu)
+               __blkcg_rstat_flush(blkcg, cpu);
 
        /* release the blkcg and parent blkg refs this blkg has been holding */
        css_put(&blkg->blkcg->css);
@@ -951,23 +965,26 @@ static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
        u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
 }
 
-static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
+static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
 {
-       struct blkcg *blkcg = css_to_blkcg(css);
        struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
        struct llist_node *lnode;
        struct blkg_iostat_set *bisc, *next_bisc;
 
-       /* Root-level stats are sourced from system-wide IO stats */
-       if (!cgroup_parent(css->cgroup))
-               return;
-
        rcu_read_lock();
 
        lnode = llist_del_all(lhead);
        if (!lnode)
                goto out;
 
+       /*
+        * For covering concurrent parent blkg update from blkg_release().
+        *
+        * When flushing from cgroup, cgroup_rstat_lock is always held, so
+        * this lock won't cause contention most of time.
+        */
+       raw_spin_lock(&blkg_stat_lock);
+
        /*
         * Iterate only the iostat_cpu's queued in the lockless list.
         */
@@ -991,13 +1008,19 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
                if (parent && parent->parent)
                        blkcg_iostat_update(parent, &blkg->iostat.cur,
                                            &blkg->iostat.last);
-               percpu_ref_put(&blkg->refcnt);
        }
-
+       raw_spin_unlock(&blkg_stat_lock);
 out:
        rcu_read_unlock();
 }
 
+static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
+{
+       /* Root-level stats are sourced from system-wide IO stats */
+       if (cgroup_parent(css->cgroup))
+               __blkcg_rstat_flush(css_to_blkcg(css), cpu);
+}
+
 /*
  * We source root cgroup stats from the system-wide stats to avoid
  * tracking the same information twice and incurring overhead when no
@@ -2075,7 +2098,6 @@ void blk_cgroup_bio_start(struct bio *bio)
 
                llist_add(&bis->lnode, lhead);
                WRITE_ONCE(bis->lqueued, true);
-               percpu_ref_get(&bis->blkg->refcnt);
        }
 
        u64_stats_update_end_irqrestore(&bis->sync, flags);