Commit | Line | Data |
---|---|---|
556910e3 BVA |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Block rq-qos policy for assigning an I/O priority class to requests. | |
4 | * | |
5 | * Using an rq-qos policy for assigning I/O priority class has two advantages | |
6 | * over using the ioprio_set() system call: | |
7 | * | |
8 | * - This policy is cgroup based so it has all the advantages of cgroups. | |
9 | * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos | |
10 | * controller affects page cache writeback I/O for filesystems that support | |
11 | * assiociating a cgroup with writeback I/O. See also | |
12 | * Documentation/admin-guide/cgroup-v2.rst. | |
13 | */ | |
14 | ||
15 | #include <linux/blk-cgroup.h> | |
16 | #include <linux/blk-mq.h> | |
17 | #include <linux/blk_types.h> | |
18 | #include <linux/kernel.h> | |
19 | #include <linux/module.h> | |
20 | #include "blk-ioprio.h" | |
21 | #include "blk-rq-qos.h" | |
22 | ||
23 | /** | |
24 | * enum prio_policy - I/O priority class policy. | |
25 | * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class. | |
26 | * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT. | |
27 | * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into | |
28 | * IOPRIO_CLASS_BE. | |
29 | * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE. | |
30 | * | |
31 | * See also <linux/ioprio.h>. | |
32 | */ | |
33 | enum prio_policy { | |
34 | POLICY_NO_CHANGE = 0, | |
35 | POLICY_NONE_TO_RT = 1, | |
36 | POLICY_RESTRICT_TO_BE = 2, | |
37 | POLICY_ALL_TO_IDLE = 3, | |
38 | }; | |
39 | ||
40 | static const char *policy_name[] = { | |
41 | [POLICY_NO_CHANGE] = "no-change", | |
42 | [POLICY_NONE_TO_RT] = "none-to-rt", | |
43 | [POLICY_RESTRICT_TO_BE] = "restrict-to-be", | |
44 | [POLICY_ALL_TO_IDLE] = "idle", | |
45 | }; | |
46 | ||
47 | static struct blkcg_policy ioprio_policy; | |
48 | ||
49 | /** | |
50 | * struct ioprio_blkg - Per (cgroup, request queue) data. | |
51 | * @pd: blkg_policy_data structure. | |
52 | */ | |
53 | struct ioprio_blkg { | |
54 | struct blkg_policy_data pd; | |
55 | }; | |
56 | ||
57 | /** | |
58 | * struct ioprio_blkcg - Per cgroup data. | |
59 | * @cpd: blkcg_policy_data structure. | |
60 | * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>. | |
61 | */ | |
62 | struct ioprio_blkcg { | |
63 | struct blkcg_policy_data cpd; | |
64 | enum prio_policy prio_policy; | |
25c4b5e0 | 65 | bool prio_set; |
556910e3 BVA |
66 | }; |
67 | ||
68 | static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd) | |
69 | { | |
70 | return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL; | |
71 | } | |
72 | ||
73 | static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg) | |
74 | { | |
75 | return container_of(blkcg_to_cpd(blkcg, &ioprio_policy), | |
76 | struct ioprio_blkcg, cpd); | |
77 | } | |
78 | ||
79 | static struct ioprio_blkcg * | |
80 | ioprio_blkcg_from_css(struct cgroup_subsys_state *css) | |
81 | { | |
82 | return blkcg_to_ioprio_blkcg(css_to_blkcg(css)); | |
83 | } | |
84 | ||
85 | static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio) | |
86 | { | |
87 | struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy); | |
88 | ||
89 | if (!pd) | |
90 | return NULL; | |
91 | ||
92 | return blkcg_to_ioprio_blkcg(pd->blkg->blkcg); | |
93 | } | |
94 | ||
95 | static int ioprio_show_prio_policy(struct seq_file *sf, void *v) | |
96 | { | |
97 | struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf)); | |
98 | ||
99 | seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]); | |
100 | return 0; | |
101 | } | |
102 | ||
103 | static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, | |
104 | size_t nbytes, loff_t off) | |
105 | { | |
106 | struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of)); | |
107 | int ret; | |
108 | ||
109 | if (off != 0) | |
110 | return -EIO; | |
111 | /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */ | |
112 | ret = sysfs_match_string(policy_name, buf); | |
113 | if (ret < 0) | |
114 | return ret; | |
115 | blkcg->prio_policy = ret; | |
25c4b5e0 | 116 | blkcg->prio_set = true; |
556910e3 BVA |
117 | return nbytes; |
118 | } | |
119 | ||
120 | static struct blkg_policy_data * | |
121 | ioprio_alloc_pd(gfp_t gfp, struct request_queue *q, struct blkcg *blkcg) | |
122 | { | |
123 | struct ioprio_blkg *ioprio_blkg; | |
124 | ||
125 | ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp); | |
126 | if (!ioprio_blkg) | |
127 | return NULL; | |
128 | ||
129 | return &ioprio_blkg->pd; | |
130 | } | |
131 | ||
132 | static void ioprio_free_pd(struct blkg_policy_data *pd) | |
133 | { | |
134 | struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd); | |
135 | ||
136 | kfree(ioprio_blkg); | |
137 | } | |
138 | ||
139 | static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) | |
140 | { | |
141 | struct ioprio_blkcg *blkcg; | |
142 | ||
143 | blkcg = kzalloc(sizeof(*blkcg), gfp); | |
144 | if (!blkcg) | |
145 | return NULL; | |
146 | blkcg->prio_policy = POLICY_NO_CHANGE; | |
147 | return &blkcg->cpd; | |
148 | } | |
149 | ||
150 | static void ioprio_free_cpd(struct blkcg_policy_data *cpd) | |
151 | { | |
152 | struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd); | |
153 | ||
154 | kfree(blkcg); | |
155 | } | |
156 | ||
157 | #define IOPRIO_ATTRS \ | |
158 | { \ | |
159 | .name = "prio.class", \ | |
160 | .seq_show = ioprio_show_prio_policy, \ | |
161 | .write = ioprio_set_prio_policy, \ | |
162 | }, \ | |
163 | { } /* sentinel */ | |
164 | ||
165 | /* cgroup v2 attributes */ | |
166 | static struct cftype ioprio_files[] = { | |
167 | IOPRIO_ATTRS | |
168 | }; | |
169 | ||
170 | /* cgroup v1 attributes */ | |
171 | static struct cftype ioprio_legacy_files[] = { | |
172 | IOPRIO_ATTRS | |
173 | }; | |
174 | ||
175 | static struct blkcg_policy ioprio_policy = { | |
176 | .dfl_cftypes = ioprio_files, | |
177 | .legacy_cftypes = ioprio_legacy_files, | |
178 | ||
179 | .cpd_alloc_fn = ioprio_alloc_cpd, | |
180 | .cpd_free_fn = ioprio_free_cpd, | |
181 | ||
182 | .pd_alloc_fn = ioprio_alloc_pd, | |
183 | .pd_free_fn = ioprio_free_pd, | |
184 | }; | |
185 | ||
186 | struct blk_ioprio { | |
187 | struct rq_qos rqos; | |
188 | }; | |
189 | ||
190 | static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq, | |
191 | struct bio *bio) | |
192 | { | |
193 | struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio); | |
25c4b5e0 JA |
194 | u16 prio; |
195 | ||
196 | if (!blkcg->prio_set) | |
197 | return; | |
556910e3 BVA |
198 | |
199 | /* | |
200 | * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers | |
201 | * correspond to a lower priority. Hence, the max_t() below selects | |
202 | * the lower priority of bi_ioprio and the cgroup I/O priority class. | |
203 | * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the | |
204 | * bio I/O priority is not modified. If the bio I/O priority equals | |
205 | * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio. | |
206 | */ | |
25c4b5e0 JA |
207 | prio = max_t(u16, bio->bi_ioprio, |
208 | IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); | |
209 | if (prio > bio->bi_ioprio) | |
210 | bio->bi_ioprio = prio; | |
556910e3 BVA |
211 | } |
212 | ||
213 | static void blkcg_ioprio_exit(struct rq_qos *rqos) | |
214 | { | |
215 | struct blk_ioprio *blkioprio_blkg = | |
216 | container_of(rqos, typeof(*blkioprio_blkg), rqos); | |
217 | ||
218 | blkcg_deactivate_policy(rqos->q, &ioprio_policy); | |
219 | kfree(blkioprio_blkg); | |
220 | } | |
221 | ||
222 | static struct rq_qos_ops blkcg_ioprio_ops = { | |
223 | .track = blkcg_ioprio_track, | |
224 | .exit = blkcg_ioprio_exit, | |
225 | }; | |
226 | ||
227 | int blk_ioprio_init(struct request_queue *q) | |
228 | { | |
229 | struct blk_ioprio *blkioprio_blkg; | |
230 | struct rq_qos *rqos; | |
231 | int ret; | |
232 | ||
233 | blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL); | |
234 | if (!blkioprio_blkg) | |
235 | return -ENOMEM; | |
236 | ||
237 | ret = blkcg_activate_policy(q, &ioprio_policy); | |
238 | if (ret) { | |
239 | kfree(blkioprio_blkg); | |
240 | return ret; | |
241 | } | |
242 | ||
243 | rqos = &blkioprio_blkg->rqos; | |
244 | rqos->id = RQ_QOS_IOPRIO; | |
245 | rqos->ops = &blkcg_ioprio_ops; | |
246 | rqos->q = q; | |
247 | ||
248 | /* | |
249 | * Registering the rq-qos policy after activating the blk-cgroup | |
250 | * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the | |
251 | * rq-qos callbacks. | |
252 | */ | |
253 | rq_qos_add(q, rqos); | |
254 | ||
255 | return 0; | |
256 | } | |
257 | ||
258 | static int __init ioprio_init(void) | |
259 | { | |
260 | return blkcg_policy_register(&ioprio_policy); | |
261 | } | |
262 | ||
263 | static void __exit ioprio_exit(void) | |
264 | { | |
265 | blkcg_policy_unregister(&ioprio_policy); | |
266 | } | |
267 | ||
268 | module_init(ioprio_init); | |
269 | module_exit(ioprio_exit); |