blk-throttle: Fix io statistics for cgroup v1

After commit f382fb0bcef4 ("block: remove legacy IO schedulers"),
blkio.throttle.io_serviced and blkio.throttle.io_service_bytes become
the only stable io stats interface of cgroup v1, and these statistics
are done in the blk-throttle code. But the current code only counts the
bios that are actually throttled. When the user does not add the throttle
limit, the io stats for cgroup v1 has nothing. I fix it according to the
statistical method of v2, and made it count all ios accurately.

Fixes: a7b36ee6ba29 ("block: move blk-throtl fast path inline")
Tested-by: Andrea Righi <andrea.righi@canonical.com>
Signed-off-by: Jinke Han <hanjinke.666@bytedance.com>
Acked-by: Muchun Song <songmuchun@bytedance.com>
Acked-by: Tejun Heo <tj@kernel.org>
Link: https://lore.kernel.org/r/20230507170631.89607-1-hanjinke.666@bytedance.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index cab33bd4f252d9837fc326ff05310e2f34c2c1b8..c8b28ec5dde91f1e45455f3513092e820e474ffb 100644 (file)
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -2062,6 +2062,9 @@ void blk_cgroup_bio_start(struct bio *bio)
        struct blkg_iostat_set *bis;
        unsigned long flags;
 
+       if (!cgroup_subsys_on_dfl(io_cgrp_subsys))
+               return;
+
        /* Root-level stats are sourced from system-wide IO stats */
        if (!cgroup_parent(blkcg->css.cgroup))
                return;
@@ -2093,8 +2096,7 @@ void blk_cgroup_bio_start(struct bio *bio)
        }
 
        u64_stats_update_end_irqrestore(&bis->sync, flags);
-       if (cgroup_subsys_on_dfl(io_cgrp_subsys))
-               cgroup_rstat_updated(blkcg->css.cgroup, cpu);
+       cgroup_rstat_updated(blkcg->css.cgroup, cpu);
        put_cpu();
 }
 

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 9d010d867fbf4ee32d3c4aedb1962afab73304c2..7397ff199d6695e40f4e181f91982d4ec8870252 100644 (file)
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -2178,12 +2178,6 @@ bool __blk_throtl_bio(struct bio *bio)
 
        rcu_read_lock();
 
-       if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
-               blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
-                               bio->bi_iter.bi_size);
-               blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
-       }
-
        spin_lock_irq(&q->queue_lock);
 
        throtl_update_latency_buckets(td);

diff --git a/block/blk-throttle.h b/block/blk-throttle.h
index ef4b7a4de987dddfd0dc368cd36893b899d8c3f2..d1ccbfe9f797882c05256a5780fa151093174c44 100644 (file)
--- a/block/blk-throttle.h
+++ b/block/blk-throttle.h
@@ -185,6 +185,15 @@ static inline bool blk_should_throtl(struct bio *bio)
        struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg);
        int rw = bio_data_dir(bio);
 
+       if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
+               if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
+                       bio_set_flag(bio, BIO_CGROUP_ACCT);
+                       blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
+                                       bio->bi_iter.bi_size);
+               }
+               blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
+       }
+
        /* iops limit is always counted */
        if (tg->has_rules_iops[rw])
                return true;