[linux-block.git] / block / blk-throttle.h

#ifndef BLK_THROTTLE_H
#define BLK_THROTTLE_H

#include "blk-cgroup-rwstat.h"

/*
 * To implement hierarchical throttling, throtl_grps form a tree and bios
 * are dispatched upwards level by level until they reach the top and get
 * issued.  When dispatching bios from the children and local group at each
 * level, if the bios are dispatched into a single bio_list, there's a risk
 * of a local or child group which can queue many bios at once filling up
 * the list starving others.
 *
 * To avoid such starvation, dispatched bios are queued separately
 * according to where they came from.  When they are again dispatched to
 * the parent, they're popped in round-robin order so that no single source
 * hogs the dispatch window.
 *
 * throtl_qnode is used to keep the queued bios separated by their sources.
 * Bios are queued to throtl_qnode which in turn is queued to
 * throtl_service_queue and then dispatched in round-robin order.
 *
 * It's also used to track the reference counts on blkg's.  A qnode always
 * belongs to a throtl_grp and gets queued on itself or the parent, so
 * incrementing the reference of the associated throtl_grp when a qnode is
 * queued and decrementing when dequeued is enough to keep the whole blkg
 * tree pinned while bios are in flight.
 */
struct throtl_qnode {
	struct list_head	node;		/* service_queue->queued[] */
	struct bio_list		bios;		/* queued bios */
	struct throtl_grp	*tg;		/* tg this qnode belongs to */
};

struct throtl_service_queue {
	struct throtl_service_queue *parent_sq;	/* the parent service_queue */

	/*
	 * Bios queued directly to this service_queue or dispatched from
	 * children throtl_grp's.
	 */
	struct list_head	queued[2];	/* throtl_qnode [READ/WRITE] */
	unsigned int		nr_queued[2];	/* number of queued bios */

	/*
	 * RB tree of active children throtl_grp's, which are sorted by
	 * their ->disptime.
	 */
	struct rb_root_cached	pending_tree;	/* RB tree of active tgs */
	unsigned int		nr_pending;	/* # queued in the tree */
	unsigned long		first_pending_disptime;	/* disptime of the first tg */
	struct timer_list	pending_timer;	/* fires on first_pending_disptime */
};

enum tg_state_flags {
	THROTL_TG_PENDING	= 1 << 0,	/* on parent's pending tree */
	THROTL_TG_WAS_EMPTY	= 1 << 1,	/* bio_lists[] became non-empty */
	THROTL_TG_HAS_IOPS_LIMIT = 1 << 2,	/* tg has iops limit */
	THROTL_TG_CANCELING	= 1 << 3,	/* starts to cancel bio */
};

enum {
	LIMIT_LOW,
	LIMIT_MAX,
	LIMIT_CNT,
};

struct throtl_grp {
	/* must be the first member */
	struct blkg_policy_data pd;

	/* active throtl group service_queue member */
	struct rb_node rb_node;

	/* throtl_data this group belongs to */
	struct throtl_data *td;

	/* this group's service queue */
	struct throtl_service_queue service_queue;

	/*
	 * qnode_on_self is used when bios are directly queued to this
	 * throtl_grp so that local bios compete fairly with bios
	 * dispatched from children.  qnode_on_parent is used when bios are
	 * dispatched from this throtl_grp into its parent and will compete
	 * with the sibling qnode_on_parents and the parent's
	 * qnode_on_self.
	 */
	struct throtl_qnode qnode_on_self[2];
	struct throtl_qnode qnode_on_parent[2];

	/*
	 * Dispatch time in jiffies. This is the estimated time when group
	 * will unthrottle and is ready to dispatch more bio. It is used as
	 * key to sort active groups in service tree.
	 */
	unsigned long disptime;

	unsigned int flags;

	/* are there any throtl rules between this group and td? */
	bool has_rules[2];

	/* internally used bytes per second rate limits */
	uint64_t bps[2][LIMIT_CNT];
	/* user configured bps limits */
	uint64_t bps_conf[2][LIMIT_CNT];

	/* internally used IOPS limits */
	unsigned int iops[2][LIMIT_CNT];
	/* user configured IOPS limits */
	unsigned int iops_conf[2][LIMIT_CNT];

	/* Number of bytes dispatched in current slice */
	uint64_t bytes_disp[2];
	/* Number of bio's dispatched in current slice */
	unsigned int io_disp[2];

	unsigned long last_low_overflow_time[2];

	uint64_t last_bytes_disp[2];
	unsigned int last_io_disp[2];

	/*
	 * The following two fields are updated when new configuration is
	 * submitted while some bios are still throttled, they record how many
	 * bytes/ios are waited already in previous configuration, and they will
	 * be used to calculate wait time under new configuration.
	 */
	uint64_t carryover_bytes[2];
	unsigned int carryover_ios[2];

	unsigned long last_check_time;

	unsigned long latency_target; /* us */
	unsigned long latency_target_conf; /* us */
	/* When did we start a new slice */
	unsigned long slice_start[2];
	unsigned long slice_end[2];

	unsigned long last_finish_time; /* ns / 1024 */
	unsigned long checked_last_finish_time; /* ns / 1024 */
	unsigned long avg_idletime; /* ns / 1024 */
	unsigned long idletime_threshold; /* us */
	unsigned long idletime_threshold_conf; /* us */

	unsigned int bio_cnt; /* total bios */
	unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
	unsigned long bio_cnt_reset_time;

	struct blkg_rwstat stat_bytes;
	struct blkg_rwstat stat_ios;
};

extern struct blkcg_policy blkcg_policy_throtl;

static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
{
	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
}

static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
{
	return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
}

/*
 * Internal throttling interface
 */
#ifndef CONFIG_BLK_DEV_THROTTLING
static inline int blk_throtl_init(struct request_queue *q) { return 0; }
static inline void blk_throtl_exit(struct request_queue *q) { }
static inline void blk_throtl_register_queue(struct request_queue *q) { }
static inline bool blk_throtl_bio(struct bio *bio) { return false; }
static inline void blk_throtl_cancel_bios(struct request_queue *q) { }
#else /* CONFIG_BLK_DEV_THROTTLING */
int blk_throtl_init(struct request_queue *q);
void blk_throtl_exit(struct request_queue *q);
void blk_throtl_register_queue(struct request_queue *q);
bool __blk_throtl_bio(struct bio *bio);
void blk_throtl_cancel_bios(struct request_queue *q);
static inline bool blk_throtl_bio(struct bio *bio)
{
	struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg);

	/* no need to throttle bps any more if the bio has been throttled */
	if (bio_flagged(bio, BIO_BPS_THROTTLED) &&
	    !(tg->flags & THROTL_TG_HAS_IOPS_LIMIT))
		return false;

	if (!tg->has_rules[bio_data_dir(bio)])
		return false;

	return __blk_throtl_bio(bio);
}
#endif /* CONFIG_BLK_DEV_THROTTLING */

#endif
Commit	Line	Data
a7b36ee6 JA	1	#ifndef BLK_THROTTLE_H
	2	#define BLK_THROTTLE_H
	3
	4	#include "blk-cgroup-rwstat.h"
	5
	6	/*
	7	* To implement hierarchical throttling, throtl_grps form a tree and bios
	8	* are dispatched upwards level by level until they reach the top and get
	9	* issued. When dispatching bios from the children and local group at each
	10	* level, if the bios are dispatched into a single bio_list, there's a risk
	11	* of a local or child group which can queue many bios at once filling up
	12	* the list starving others.
	13	*
	14	* To avoid such starvation, dispatched bios are queued separately
	15	* according to where they came from. When they are again dispatched to
	16	* the parent, they're popped in round-robin order so that no single source
	17	* hogs the dispatch window.
	18	*
	19	* throtl_qnode is used to keep the queued bios separated by their sources.
	20	* Bios are queued to throtl_qnode which in turn is queued to
	21	* throtl_service_queue and then dispatched in round-robin order.
	22	*
	23	* It's also used to track the reference counts on blkg's. A qnode always
	24	* belongs to a throtl_grp and gets queued on itself or the parent, so
	25	* incrementing the reference of the associated throtl_grp when a qnode is
	26	* queued and decrementing when dequeued is enough to keep the whole blkg
	27	* tree pinned while bios are in flight.
	28	*/
	29	struct throtl_qnode {
	30	struct list_head node; /* service_queue->queued[] */
	31	struct bio_list bios; /* queued bios */
	32	struct throtl_grp tg; / tg this qnode belongs to */
	33	};
	34
	35	struct throtl_service_queue {
	36	struct throtl_service_queue parent_sq; / the parent service_queue */
	37
	38	/*
	39	* Bios queued directly to this service_queue or dispatched from
	40	* children throtl_grp's.
	41	*/
	42	struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */
	43	unsigned int nr_queued[2]; /* number of queued bios */
	44
	45	/*
	46	* RB tree of active children throtl_grp's, which are sorted by
	47	* their ->disptime.
	48	*/
	49	struct rb_root_cached pending_tree; /* RB tree of active tgs */
	50	unsigned int nr_pending; /* # queued in the tree */
	51	unsigned long first_pending_disptime; /* disptime of the first tg */
	52	struct timer_list pending_timer; /* fires on first_pending_disptime */
	53	};
	54
5a93b602 ML	55	enum tg_state_flags {
	56	THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */
	57	THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
	58	THROTL_TG_HAS_IOPS_LIMIT = 1 << 2, /* tg has iops limit */
8f9e7b65	59	THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */
5a93b602 ML	60	};
5a93b602 ML	61
a7b36ee6 JA	62	enum {
	63	LIMIT_LOW,
	64	LIMIT_MAX,
	65	LIMIT_CNT,
	66	};
	67
	68	struct throtl_grp {
	69	/* must be the first member */
	70	struct blkg_policy_data pd;
	71
	72	/* active throtl group service_queue member */
	73	struct rb_node rb_node;
	74
	75	/* throtl_data this group belongs to */
	76	struct throtl_data *td;
	77
	78	/* this group's service queue */
	79	struct throtl_service_queue service_queue;
	80
	81	/*
	82	* qnode_on_self is used when bios are directly queued to this
	83	* throtl_grp so that local bios compete fairly with bios
	84	* dispatched from children. qnode_on_parent is used when bios are
	85	* dispatched from this throtl_grp into its parent and will compete
	86	* with the sibling qnode_on_parents and the parent's
	87	* qnode_on_self.
	88	*/
	89	struct throtl_qnode qnode_on_self[2];
	90	struct throtl_qnode qnode_on_parent[2];
	91
	92	/*
	93	* Dispatch time in jiffies. This is the estimated time when group
	94	* will unthrottle and is ready to dispatch more bio. It is used as
	95	* key to sort active groups in service tree.
	96	*/
	97	unsigned long disptime;
	98
	99	unsigned int flags;
	100
	101	/* are there any throtl rules between this group and td? */
	102	bool has_rules[2];
	103
	104	/* internally used bytes per second rate limits */
	105	uint64_t bps[2][LIMIT_CNT];
	106	/* user configured bps limits */
	107	uint64_t bps_conf[2][LIMIT_CNT];
	108
	109	/* internally used IOPS limits */
	110	unsigned int iops[2][LIMIT_CNT];
	111	/* user configured IOPS limits */
	112	unsigned int iops_conf[2][LIMIT_CNT];
	113
	114	/* Number of bytes dispatched in current slice */
	115	uint64_t bytes_disp[2];
	116	/* Number of bio's dispatched in current slice */
	117	unsigned int io_disp[2];
	118
	119	unsigned long last_low_overflow_time[2];
	120
	121	uint64_t last_bytes_disp[2];
	122	unsigned int last_io_disp[2];
	123
a880ae93 YK	124	/*
	125	* The following two fields are updated when new configuration is
	126	* submitted while some bios are still throttled, they record how many
	127	* bytes/ios are waited already in previous configuration, and they will
	128	* be used to calculate wait time under new configuration.
	129	*/
	130	uint64_t carryover_bytes[2];
	131	unsigned int carryover_ios[2];
	132
a7b36ee6 JA	133	unsigned long last_check_time;
	134
	135	unsigned long latency_target; /* us */
	136	unsigned long latency_target_conf; /* us */
	137	/* When did we start a new slice */
	138	unsigned long slice_start[2];
	139	unsigned long slice_end[2];
	140
	141	unsigned long last_finish_time; /* ns / 1024 */
	142	unsigned long checked_last_finish_time; /* ns / 1024 */
	143	unsigned long avg_idletime; /* ns / 1024 */
	144	unsigned long idletime_threshold; /* us */
	145	unsigned long idletime_threshold_conf; /* us */
	146
	147	unsigned int bio_cnt; /* total bios */
	148	unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
	149	unsigned long bio_cnt_reset_time;
	150
a7b36ee6 JA	151	struct blkg_rwstat stat_bytes;
	152	struct blkg_rwstat stat_ios;
	153	};
	154
	155	extern struct blkcg_policy blkcg_policy_throtl;
	156
	157	static inline struct throtl_grp pd_to_tg(struct blkg_policy_data pd)
	158	{
	159	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
	160	}
	161
	162	static inline struct throtl_grp blkg_to_tg(struct blkcg_gq blkg)
	163	{
	164	return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
	165	}
	166
	167	/*
	168	* Internal throttling interface
	169	*/
	170	#ifndef CONFIG_BLK_DEV_THROTTLING
	171	static inline int blk_throtl_init(struct request_queue *q) { return 0; }
	172	static inline void blk_throtl_exit(struct request_queue *q) { }
	173	static inline void blk_throtl_register_queue(struct request_queue *q) { }
a7b36ee6	174	static inline bool blk_throtl_bio(struct bio *bio) { return false; }
8f9e7b65	175	static inline void blk_throtl_cancel_bios(struct request_queue *q) { }
a7b36ee6 JA	176	#else /* CONFIG_BLK_DEV_THROTTLING */
	177	int blk_throtl_init(struct request_queue *q);
	178	void blk_throtl_exit(struct request_queue *q);
	179	void blk_throtl_register_queue(struct request_queue *q);
a7b36ee6	180	bool __blk_throtl_bio(struct bio *bio);
8f9e7b65	181	void blk_throtl_cancel_bios(struct request_queue *q);
a7b36ee6 JA	182	static inline bool blk_throtl_bio(struct bio *bio)
	183	{
	184	struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg);
	185
5a93b602	186	/* no need to throttle bps any more if the bio has been throttled */
320fb0f9	187	if (bio_flagged(bio, BIO_BPS_THROTTLED) &&
5a93b602 ML	188	!(tg->flags & THROTL_TG_HAS_IOPS_LIMIT))
	189	return false;
	190
a7b36ee6 JA	191	if (!tg->has_rules[bio_data_dir(bio)])
	192	return false;
	193
	194	return __blk_throtl_bio(bio);
	195	}
	196	#endif /* CONFIG_BLK_DEV_THROTTLING */
	197
	198	#endif