[linux-block.git] / block / blk-throttle.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef BLK_THROTTLE_H
#define BLK_THROTTLE_H

#include "blk-cgroup-rwstat.h"

/*
 * To implement hierarchical throttling, throtl_grps form a tree and bios
 * are dispatched upwards level by level until they reach the top and get
 * issued.  When dispatching bios from the children and local group at each
 * level, if the bios are dispatched into a single bio_list, there's a risk
 * of a local or child group which can queue many bios at once filling up
 * the list starving others.
 *
 * To avoid such starvation, dispatched bios are queued separately
 * according to where they came from.  When they are again dispatched to
 * the parent, they're popped in round-robin order so that no single source
 * hogs the dispatch window.
 *
 * throtl_qnode is used to keep the queued bios separated by their sources.
 * Bios are queued to throtl_qnode which in turn is queued to
 * throtl_service_queue and then dispatched in round-robin order.
 *
 * It's also used to track the reference counts on blkg's.  A qnode always
 * belongs to a throtl_grp and gets queued on itself or the parent, so
 * incrementing the reference of the associated throtl_grp when a qnode is
 * queued and decrementing when dequeued is enough to keep the whole blkg
 * tree pinned while bios are in flight.
 */
struct throtl_qnode {
	struct list_head	node;		/* service_queue->queued[] */
	struct bio_list		bios_bps;	/* queued bios for bps limit */
	struct bio_list		bios_iops;	/* queued bios for iops limit */
	struct throtl_grp	*tg;		/* tg this qnode belongs to */
};

struct throtl_service_queue {
	struct throtl_service_queue *parent_sq;	/* the parent service_queue */

	/*
	 * Bios queued directly to this service_queue or dispatched from
	 * children throtl_grp's.
	 */
	struct list_head	queued[2];	/* throtl_qnode [READ/WRITE] */
	unsigned int		nr_queued_bps[2];	/* number of queued bps bios */
	unsigned int		nr_queued_iops[2];	/* number of queued iops bios */

	/*
	 * RB tree of active children throtl_grp's, which are sorted by
	 * their ->disptime.
	 */
	struct rb_root_cached	pending_tree;	/* RB tree of active tgs */
	unsigned int		nr_pending;	/* # queued in the tree */
	unsigned long		first_pending_disptime;	/* disptime of the first tg */
	struct timer_list	pending_timer;	/* fires on first_pending_disptime */
};

enum tg_state_flags {
	THROTL_TG_PENDING		= 1 << 0,	/* on parent's pending tree */
	THROTL_TG_WAS_EMPTY		= 1 << 1,	/* bio_lists[] became non-empty */
	/*
	 * The sq's iops queue is empty, and a bio is about to be enqueued
	 * to the first qnode's bios_iops list.
	 */
	THROTL_TG_IOPS_WAS_EMPTY	= 1 << 2,
	THROTL_TG_CANCELING		= 1 << 3,	/* starts to cancel bio */
};

struct throtl_grp {
	/* must be the first member */
	struct blkg_policy_data pd;

	/* active throtl group service_queue member */
	struct rb_node rb_node;

	/* throtl_data this group belongs to */
	struct throtl_data *td;

	/* this group's service queue */
	struct throtl_service_queue service_queue;

	/*
	 * qnode_on_self is used when bios are directly queued to this
	 * throtl_grp so that local bios compete fairly with bios
	 * dispatched from children.  qnode_on_parent is used when bios are
	 * dispatched from this throtl_grp into its parent and will compete
	 * with the sibling qnode_on_parents and the parent's
	 * qnode_on_self.
	 */
	struct throtl_qnode qnode_on_self[2];
	struct throtl_qnode qnode_on_parent[2];

	/*
	 * Dispatch time in jiffies. This is the estimated time when group
	 * will unthrottle and is ready to dispatch more bio. It is used as
	 * key to sort active groups in service tree.
	 */
	unsigned long disptime;

	unsigned int flags;

	/* are there any throtl rules between this group and td? */
	bool has_rules_bps[2];
	bool has_rules_iops[2];

	/* bytes per second rate limits */
	uint64_t bps[2];

	/* IOPS limits */
	unsigned int iops[2];

	/*
	 * Number of bytes/bio's dispatched in current slice.
	 * When new configuration is submitted while some bios are still throttled,
	 * first calculate the carryover: the amount of bytes/IOs already waited
	 * under the previous configuration. Then, [bytes/io]_disp are represented
	 * as the negative of the carryover, and they will be used to calculate the
	 * wait time under the new configuration.
	 */
	int64_t bytes_disp[2];
	int io_disp[2];

	unsigned long last_check_time;

	/* When did we start a new slice */
	unsigned long slice_start[2];
	unsigned long slice_end[2];

	struct blkg_rwstat stat_bytes;
	struct blkg_rwstat stat_ios;
};

extern struct blkcg_policy blkcg_policy_throtl;

static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
{
	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
}

static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
{
	return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
}

/*
 * Internal throttling interface
 */
#ifndef CONFIG_BLK_DEV_THROTTLING
static inline void blk_throtl_exit(struct gendisk *disk) { }
static inline bool blk_throtl_bio(struct bio *bio) { return false; }
static inline void blk_throtl_cancel_bios(struct gendisk *disk) { }
#else /* CONFIG_BLK_DEV_THROTTLING */
void blk_throtl_exit(struct gendisk *disk);
bool __blk_throtl_bio(struct bio *bio);
void blk_throtl_cancel_bios(struct gendisk *disk);

static inline bool blk_throtl_activated(struct request_queue *q)
{
	return q->td != NULL;
}

static inline bool blk_should_throtl(struct bio *bio)
{
	struct throtl_grp *tg;
	int rw = bio_data_dir(bio);

	/*
	 * This is called under bio_queue_enter(), and it's synchronized with
	 * the activation of blk-throtl, which is protected by
	 * blk_mq_freeze_queue().
	 */
	if (!blk_throtl_activated(bio->bi_bdev->bd_queue))
		return false;

	tg = blkg_to_tg(bio->bi_blkg);
	if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
		if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
			bio_set_flag(bio, BIO_CGROUP_ACCT);
			blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
					bio->bi_iter.bi_size);
		}
		blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
	}

	/* iops limit is always counted */
	if (tg->has_rules_iops[rw])
		return true;

	if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED))
		return true;

	return false;
}

static inline bool blk_throtl_bio(struct bio *bio)
{

	if (!blk_should_throtl(bio))
		return false;

	return __blk_throtl_bio(bio);
}
#endif /* CONFIG_BLK_DEV_THROTTLING */

#endif
Commit	Line	Data
1b419405	1	/* SPDX-License-Identifier: GPL-2.0 */
a7b36ee6 JA	2	#ifndef BLK_THROTTLE_H
	3	#define BLK_THROTTLE_H
	4
	5	#include "blk-cgroup-rwstat.h"
	6
	7	/*
	8	* To implement hierarchical throttling, throtl_grps form a tree and bios
	9	* are dispatched upwards level by level until they reach the top and get
	10	* issued. When dispatching bios from the children and local group at each
	11	* level, if the bios are dispatched into a single bio_list, there's a risk
	12	* of a local or child group which can queue many bios at once filling up
	13	* the list starving others.
	14	*
	15	* To avoid such starvation, dispatched bios are queued separately
	16	* according to where they came from. When they are again dispatched to
	17	* the parent, they're popped in round-robin order so that no single source
	18	* hogs the dispatch window.
	19	*
	20	* throtl_qnode is used to keep the queued bios separated by their sources.
	21	* Bios are queued to throtl_qnode which in turn is queued to
	22	* throtl_service_queue and then dispatched in round-robin order.
	23	*
	24	* It's also used to track the reference counts on blkg's. A qnode always
	25	* belongs to a throtl_grp and gets queued on itself or the parent, so
	26	* incrementing the reference of the associated throtl_grp when a qnode is
	27	* queued and decrementing when dequeued is enough to keep the whole blkg
	28	* tree pinned while bios are in flight.
	29	*/
	30	struct throtl_qnode {
	31	struct list_head node; /* service_queue->queued[] */
f2c4902b ZW	32	struct bio_list bios_bps; /* queued bios for bps limit */
f2c4902b ZW	33	struct bio_list bios_iops; /* queued bios for iops limit */
a7b36ee6 JA	34	struct throtl_grp tg; / tg this qnode belongs to */
	35	};
	36
	37	struct throtl_service_queue {
	38	struct throtl_service_queue parent_sq; / the parent service_queue */
	39
	40	/*
	41	* Bios queued directly to this service_queue or dispatched from
	42	* children throtl_grp's.
	43	*/
	44	struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */
28ad83b7 ZW	45	unsigned int nr_queued_bps[2]; /* number of queued bps bios */
28ad83b7 ZW	46	unsigned int nr_queued_iops[2]; /* number of queued iops bios */
a7b36ee6 JA	47
	48	/*
	49	* RB tree of active children throtl_grp's, which are sorted by
	50	* their ->disptime.
	51	*/
	52	struct rb_root_cached pending_tree; /* RB tree of active tgs */
	53	unsigned int nr_pending; /* # queued in the tree */
	54	unsigned long first_pending_disptime; /* disptime of the first tg */
	55	struct timer_list pending_timer; /* fires on first_pending_disptime */
	56	};
	57
5a93b602	58	enum tg_state_flags {
d1ba22ab ZW	59	THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */
	60	THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
	61	/*
	62	* The sq's iops queue is empty, and a bio is about to be enqueued
	63	* to the first qnode's bios_iops list.
	64	*/
	65	THROTL_TG_IOPS_WAS_EMPTY = 1 << 2,
	66	THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */
5a93b602 ML	67	};
5a93b602 ML	68
a7b36ee6 JA	69	struct throtl_grp {
	70	/* must be the first member */
	71	struct blkg_policy_data pd;
	72
	73	/* active throtl group service_queue member */
	74	struct rb_node rb_node;
	75
	76	/* throtl_data this group belongs to */
	77	struct throtl_data *td;
	78
	79	/* this group's service queue */
	80	struct throtl_service_queue service_queue;
	81
	82	/*
	83	* qnode_on_self is used when bios are directly queued to this
	84	* throtl_grp so that local bios compete fairly with bios
	85	* dispatched from children. qnode_on_parent is used when bios are
	86	* dispatched from this throtl_grp into its parent and will compete
	87	* with the sibling qnode_on_parents and the parent's
	88	* qnode_on_self.
	89	*/
	90	struct throtl_qnode qnode_on_self[2];
	91	struct throtl_qnode qnode_on_parent[2];
	92
	93	/*
	94	* Dispatch time in jiffies. This is the estimated time when group
	95	* will unthrottle and is ready to dispatch more bio. It is used as
	96	* key to sort active groups in service tree.
	97	*/
	98	unsigned long disptime;
	99
	100	unsigned int flags;
	101
	102	/* are there any throtl rules between this group and td? */
81c7a63a YK	103	bool has_rules_bps[2];
81c7a63a YK	104	bool has_rules_iops[2];
a7b36ee6	105
0a751df4	106	/* bytes per second rate limits */
bf20ab53	107	uint64_t bps[2];
a7b36ee6	108
0a751df4	109	/* IOPS limits */
bf20ab53	110	unsigned int iops[2];
a7b36ee6	111
a880ae93	112	/*
7b89d460 ZW	113	* Number of bytes/bio's dispatched in current slice.
	114	* When new configuration is submitted while some bios are still throttled,
	115	* first calculate the carryover: the amount of bytes/IOs already waited
	116	* under the previous configuration. Then, [bytes/io]_disp are represented
	117	* as the negative of the carryover, and they will be used to calculate the
	118	* wait time under the new configuration.
a880ae93	119	*/
7b89d460 ZW	120	int64_t bytes_disp[2];
7b89d460 ZW	121	int io_disp[2];
a880ae93	122
a7b36ee6 JA	123	unsigned long last_check_time;
a7b36ee6 JA	124
a7b36ee6 JA	125	/* When did we start a new slice */
	126	unsigned long slice_start[2];
	127	unsigned long slice_end[2];
	128
a7b36ee6 JA	129	struct blkg_rwstat stat_bytes;
	130	struct blkg_rwstat stat_ios;
	131	};
	132
	133	extern struct blkcg_policy blkcg_policy_throtl;
	134
	135	static inline struct throtl_grp pd_to_tg(struct blkg_policy_data pd)
	136	{
	137	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
	138	}
	139
	140	static inline struct throtl_grp blkg_to_tg(struct blkcg_gq blkg)
	141	{
	142	return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
	143	}
	144
	145	/*
	146	* Internal throttling interface
	147	*/
	148	#ifndef CONFIG_BLK_DEV_THROTTLING
e13793ba	149	static inline void blk_throtl_exit(struct gendisk *disk) { }
a7b36ee6	150	static inline bool blk_throtl_bio(struct bio *bio) { return false; }
cad9266a	151	static inline void blk_throtl_cancel_bios(struct gendisk *disk) { }
a7b36ee6	152	#else /* CONFIG_BLK_DEV_THROTTLING */
e13793ba	153	void blk_throtl_exit(struct gendisk *disk);
a7b36ee6	154	bool __blk_throtl_bio(struct bio *bio);
cad9266a	155	void blk_throtl_cancel_bios(struct gendisk *disk);
81c7a63a	156
a3166c51 YK	157	static inline bool blk_throtl_activated(struct request_queue *q)
	158	{
	159	return q->td != NULL;
	160	}
	161
81c7a63a	162	static inline bool blk_should_throtl(struct bio *bio)
a7b36ee6	163	{
a3166c51	164	struct throtl_grp *tg;
81c7a63a YK	165	int rw = bio_data_dir(bio);
81c7a63a YK	166
a3166c51 YK	167	/*
	168	* This is called under bio_queue_enter(), and it's synchronized with
	169	* the activation of blk-throtl, which is protected by
	170	* blk_mq_freeze_queue().
	171	*/
	172	if (!blk_throtl_activated(bio->bi_bdev->bd_queue))
	173	return false;
	174
	175	tg = blkg_to_tg(bio->bi_blkg);
ad7c3b41 JH	176	if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
	177	if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
	178	bio_set_flag(bio, BIO_CGROUP_ACCT);
	179	blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
	180	bio->bi_iter.bi_size);
	181	}
	182	blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
	183	}
	184
81c7a63a YK	185	/* iops limit is always counted */
	186	if (tg->has_rules_iops[rw])
	187	return true;
	188
	189	if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED))
	190	return true;
	191
	192	return false;
	193	}
	194
	195	static inline bool blk_throtl_bio(struct bio *bio)
	196	{
a7b36ee6	197
81c7a63a	198	if (!blk_should_throtl(bio))
a7b36ee6 JA	199	return false;
	200
	201	return __blk_throtl_bio(bio);
	202	}
	203	#endif /* CONFIG_BLK_DEV_THROTTLING */
	204
	205	#endif