| 1 | #ifndef _BLK_CGROUP_H |
| 2 | #define _BLK_CGROUP_H |
| 3 | /* |
| 4 | * Common Block IO controller cgroup interface |
| 5 | * |
| 6 | * Based on ideas and code from CFQ, CFS and BFQ: |
| 7 | * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk> |
| 8 | * |
| 9 | * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it> |
| 10 | * Paolo Valente <paolo.valente@unimore.it> |
| 11 | * |
| 12 | * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com> |
| 13 | * Nauman Rafique <nauman@google.com> |
| 14 | */ |
| 15 | |
| 16 | #include <linux/cgroup.h> |
| 17 | #include <linux/u64_stats_sync.h> |
| 18 | #include <linux/seq_file.h> |
| 19 | |
| 20 | /* Max limits for throttle policy */ |
| 21 | #define THROTL_IOPS_MAX UINT_MAX |
| 22 | |
| 23 | /* CFQ specific, out here for blkcg->cfq_weight */ |
| 24 | #define CFQ_WEIGHT_MIN 10 |
| 25 | #define CFQ_WEIGHT_MAX 1000 |
| 26 | #define CFQ_WEIGHT_DEFAULT 500 |
| 27 | |
| 28 | #ifdef CONFIG_BLK_CGROUP |
| 29 | |
| 30 | enum blkg_rwstat_type { |
| 31 | BLKG_RWSTAT_READ, |
| 32 | BLKG_RWSTAT_WRITE, |
| 33 | BLKG_RWSTAT_SYNC, |
| 34 | BLKG_RWSTAT_ASYNC, |
| 35 | |
| 36 | BLKG_RWSTAT_NR, |
| 37 | BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR, |
| 38 | }; |
| 39 | |
| 40 | struct blkcg { |
| 41 | struct cgroup_subsys_state css; |
| 42 | spinlock_t lock; |
| 43 | struct hlist_head blkg_list; |
| 44 | |
| 45 | /* for policies to test whether associated blkcg has changed */ |
| 46 | uint64_t id; |
| 47 | |
| 48 | /* TODO: per-policy storage in blkcg */ |
| 49 | unsigned int cfq_weight; /* belongs to cfq */ |
| 50 | }; |
| 51 | |
| 52 | struct blkg_stat { |
| 53 | struct u64_stats_sync syncp; |
| 54 | uint64_t cnt; |
| 55 | }; |
| 56 | |
| 57 | struct blkg_rwstat { |
| 58 | struct u64_stats_sync syncp; |
| 59 | uint64_t cnt[BLKG_RWSTAT_NR]; |
| 60 | }; |
| 61 | |
| 62 | /* |
| 63 | * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a |
| 64 | * request_queue (q). This is used by blkcg policies which need to track |
| 65 | * information per blkcg - q pair. |
| 66 | * |
| 67 | * There can be multiple active blkcg policies and each has its private |
| 68 | * data on each blkg, the size of which is determined by |
| 69 | * blkcg_policy->pd_size. blkcg core allocates and frees such areas |
| 70 | * together with blkg and invokes pd_init/exit_fn() methods. |
| 71 | * |
| 72 | * Such private data must embed struct blkg_policy_data (pd) at the |
| 73 | * beginning and pd_size can't be smaller than pd. |
| 74 | */ |
| 75 | struct blkg_policy_data { |
| 76 | /* the blkg this per-policy data belongs to */ |
| 77 | struct blkcg_gq *blkg; |
| 78 | |
| 79 | /* used during policy activation */ |
| 80 | struct list_head alloc_node; |
| 81 | }; |
| 82 | |
| 83 | /* association between a blk cgroup and a request queue */ |
| 84 | struct blkcg_gq { |
| 85 | /* Pointer to the associated request_queue */ |
| 86 | struct request_queue *q; |
| 87 | struct list_head q_node; |
| 88 | struct hlist_node blkcg_node; |
| 89 | struct blkcg *blkcg; |
| 90 | /* reference count */ |
| 91 | int refcnt; |
| 92 | |
| 93 | struct blkg_policy_data *pd[BLKCG_MAX_POLS]; |
| 94 | |
| 95 | struct rcu_head rcu_head; |
| 96 | }; |
| 97 | |
| 98 | typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg); |
| 99 | typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg); |
| 100 | typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg); |
| 101 | |
| 102 | struct blkcg_policy_ops { |
| 103 | blkcg_pol_init_pd_fn *pd_init_fn; |
| 104 | blkcg_pol_exit_pd_fn *pd_exit_fn; |
| 105 | blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn; |
| 106 | }; |
| 107 | |
| 108 | struct blkcg_policy { |
| 109 | struct blkcg_policy_ops ops; |
| 110 | int plid; |
| 111 | /* policy specific private data size */ |
| 112 | size_t pd_size; |
| 113 | /* cgroup files for the policy */ |
| 114 | struct cftype *cftypes; |
| 115 | }; |
| 116 | |
| 117 | extern struct blkcg blkcg_root; |
| 118 | |
| 119 | struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup); |
| 120 | struct blkcg *bio_blkcg(struct bio *bio); |
| 121 | struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q); |
| 122 | struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, |
| 123 | struct request_queue *q); |
| 124 | int blkcg_init_queue(struct request_queue *q); |
| 125 | void blkcg_drain_queue(struct request_queue *q); |
| 126 | void blkcg_exit_queue(struct request_queue *q); |
| 127 | |
| 128 | /* Blkio controller policy registration */ |
| 129 | int blkcg_policy_register(struct blkcg_policy *pol); |
| 130 | void blkcg_policy_unregister(struct blkcg_policy *pol); |
| 131 | int blkcg_activate_policy(struct request_queue *q, |
| 132 | const struct blkcg_policy *pol); |
| 133 | void blkcg_deactivate_policy(struct request_queue *q, |
| 134 | const struct blkcg_policy *pol); |
| 135 | |
| 136 | void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, |
| 137 | u64 (*prfill)(struct seq_file *, |
| 138 | struct blkg_policy_data *, int), |
| 139 | const struct blkcg_policy *pol, int data, |
| 140 | bool show_total); |
| 141 | u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v); |
| 142 | u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, |
| 143 | const struct blkg_rwstat *rwstat); |
| 144 | u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off); |
| 145 | u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, |
| 146 | int off); |
| 147 | |
| 148 | struct blkg_conf_ctx { |
| 149 | struct gendisk *disk; |
| 150 | struct blkcg_gq *blkg; |
| 151 | u64 v; |
| 152 | }; |
| 153 | |
| 154 | int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, |
| 155 | const char *input, struct blkg_conf_ctx *ctx); |
| 156 | void blkg_conf_finish(struct blkg_conf_ctx *ctx); |
| 157 | |
| 158 | |
| 159 | /** |
| 160 | * blkg_to_pdata - get policy private data |
| 161 | * @blkg: blkg of interest |
| 162 | * @pol: policy of interest |
| 163 | * |
| 164 | * Return pointer to private data associated with the @blkg-@pol pair. |
| 165 | */ |
| 166 | static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, |
| 167 | struct blkcg_policy *pol) |
| 168 | { |
| 169 | return blkg ? blkg->pd[pol->plid] : NULL; |
| 170 | } |
| 171 | |
| 172 | /** |
| 173 | * pdata_to_blkg - get blkg associated with policy private data |
| 174 | * @pd: policy private data of interest |
| 175 | * |
| 176 | * @pd is policy private data. Determine the blkg it's associated with. |
| 177 | */ |
| 178 | static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) |
| 179 | { |
| 180 | return pd ? pd->blkg : NULL; |
| 181 | } |
| 182 | |
| 183 | /** |
| 184 | * blkg_path - format cgroup path of blkg |
| 185 | * @blkg: blkg of interest |
| 186 | * @buf: target buffer |
| 187 | * @buflen: target buffer length |
| 188 | * |
| 189 | * Format the path of the cgroup of @blkg into @buf. |
| 190 | */ |
| 191 | static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen) |
| 192 | { |
| 193 | int ret; |
| 194 | |
| 195 | rcu_read_lock(); |
| 196 | ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen); |
| 197 | rcu_read_unlock(); |
| 198 | if (ret) |
| 199 | strncpy(buf, "<unavailable>", buflen); |
| 200 | return ret; |
| 201 | } |
| 202 | |
| 203 | /** |
| 204 | * blkg_get - get a blkg reference |
| 205 | * @blkg: blkg to get |
| 206 | * |
| 207 | * The caller should be holding queue_lock and an existing reference. |
| 208 | */ |
| 209 | static inline void blkg_get(struct blkcg_gq *blkg) |
| 210 | { |
| 211 | lockdep_assert_held(blkg->q->queue_lock); |
| 212 | WARN_ON_ONCE(!blkg->refcnt); |
| 213 | blkg->refcnt++; |
| 214 | } |
| 215 | |
| 216 | void __blkg_release(struct blkcg_gq *blkg); |
| 217 | |
| 218 | /** |
| 219 | * blkg_put - put a blkg reference |
| 220 | * @blkg: blkg to put |
| 221 | * |
| 222 | * The caller should be holding queue_lock. |
| 223 | */ |
| 224 | static inline void blkg_put(struct blkcg_gq *blkg) |
| 225 | { |
| 226 | lockdep_assert_held(blkg->q->queue_lock); |
| 227 | WARN_ON_ONCE(blkg->refcnt <= 0); |
| 228 | if (!--blkg->refcnt) |
| 229 | __blkg_release(blkg); |
| 230 | } |
| 231 | |
| 232 | /** |
| 233 | * blkg_stat_add - add a value to a blkg_stat |
| 234 | * @stat: target blkg_stat |
| 235 | * @val: value to add |
| 236 | * |
| 237 | * Add @val to @stat. The caller is responsible for synchronizing calls to |
| 238 | * this function. |
| 239 | */ |
| 240 | static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val) |
| 241 | { |
| 242 | u64_stats_update_begin(&stat->syncp); |
| 243 | stat->cnt += val; |
| 244 | u64_stats_update_end(&stat->syncp); |
| 245 | } |
| 246 | |
| 247 | /** |
| 248 | * blkg_stat_read - read the current value of a blkg_stat |
| 249 | * @stat: blkg_stat to read |
| 250 | * |
| 251 | * Read the current value of @stat. This function can be called without |
| 252 | * synchroniztion and takes care of u64 atomicity. |
| 253 | */ |
| 254 | static inline uint64_t blkg_stat_read(struct blkg_stat *stat) |
| 255 | { |
| 256 | unsigned int start; |
| 257 | uint64_t v; |
| 258 | |
| 259 | do { |
| 260 | start = u64_stats_fetch_begin(&stat->syncp); |
| 261 | v = stat->cnt; |
| 262 | } while (u64_stats_fetch_retry(&stat->syncp, start)); |
| 263 | |
| 264 | return v; |
| 265 | } |
| 266 | |
| 267 | /** |
| 268 | * blkg_stat_reset - reset a blkg_stat |
| 269 | * @stat: blkg_stat to reset |
| 270 | */ |
| 271 | static inline void blkg_stat_reset(struct blkg_stat *stat) |
| 272 | { |
| 273 | stat->cnt = 0; |
| 274 | } |
| 275 | |
| 276 | /** |
| 277 | * blkg_rwstat_add - add a value to a blkg_rwstat |
| 278 | * @rwstat: target blkg_rwstat |
| 279 | * @rw: mask of REQ_{WRITE|SYNC} |
| 280 | * @val: value to add |
| 281 | * |
| 282 | * Add @val to @rwstat. The counters are chosen according to @rw. The |
| 283 | * caller is responsible for synchronizing calls to this function. |
| 284 | */ |
| 285 | static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, |
| 286 | int rw, uint64_t val) |
| 287 | { |
| 288 | u64_stats_update_begin(&rwstat->syncp); |
| 289 | |
| 290 | if (rw & REQ_WRITE) |
| 291 | rwstat->cnt[BLKG_RWSTAT_WRITE] += val; |
| 292 | else |
| 293 | rwstat->cnt[BLKG_RWSTAT_READ] += val; |
| 294 | if (rw & REQ_SYNC) |
| 295 | rwstat->cnt[BLKG_RWSTAT_SYNC] += val; |
| 296 | else |
| 297 | rwstat->cnt[BLKG_RWSTAT_ASYNC] += val; |
| 298 | |
| 299 | u64_stats_update_end(&rwstat->syncp); |
| 300 | } |
| 301 | |
| 302 | /** |
| 303 | * blkg_rwstat_read - read the current values of a blkg_rwstat |
| 304 | * @rwstat: blkg_rwstat to read |
| 305 | * |
| 306 | * Read the current snapshot of @rwstat and return it as the return value. |
| 307 | * This function can be called without synchronization and takes care of |
| 308 | * u64 atomicity. |
| 309 | */ |
| 310 | static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat) |
| 311 | { |
| 312 | unsigned int start; |
| 313 | struct blkg_rwstat tmp; |
| 314 | |
| 315 | do { |
| 316 | start = u64_stats_fetch_begin(&rwstat->syncp); |
| 317 | tmp = *rwstat; |
| 318 | } while (u64_stats_fetch_retry(&rwstat->syncp, start)); |
| 319 | |
| 320 | return tmp; |
| 321 | } |
| 322 | |
| 323 | /** |
| 324 | * blkg_rwstat_sum - read the total count of a blkg_rwstat |
| 325 | * @rwstat: blkg_rwstat to read |
| 326 | * |
| 327 | * Return the total count of @rwstat regardless of the IO direction. This |
| 328 | * function can be called without synchronization and takes care of u64 |
| 329 | * atomicity. |
| 330 | */ |
| 331 | static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat) |
| 332 | { |
| 333 | struct blkg_rwstat tmp = blkg_rwstat_read(rwstat); |
| 334 | |
| 335 | return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]; |
| 336 | } |
| 337 | |
| 338 | /** |
| 339 | * blkg_rwstat_reset - reset a blkg_rwstat |
| 340 | * @rwstat: blkg_rwstat to reset |
| 341 | */ |
| 342 | static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat) |
| 343 | { |
| 344 | memset(rwstat->cnt, 0, sizeof(rwstat->cnt)); |
| 345 | } |
| 346 | |
| 347 | #else /* CONFIG_BLK_CGROUP */ |
| 348 | |
| 349 | struct cgroup; |
| 350 | |
| 351 | struct blkg_policy_data { |
| 352 | }; |
| 353 | |
| 354 | struct blkcg_gq { |
| 355 | }; |
| 356 | |
| 357 | struct blkcg_policy { |
| 358 | }; |
| 359 | |
| 360 | static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; } |
| 361 | static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; } |
| 362 | static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; } |
| 363 | static inline int blkcg_init_queue(struct request_queue *q) { return 0; } |
| 364 | static inline void blkcg_drain_queue(struct request_queue *q) { } |
| 365 | static inline void blkcg_exit_queue(struct request_queue *q) { } |
| 366 | static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; } |
| 367 | static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { } |
| 368 | static inline int blkcg_activate_policy(struct request_queue *q, |
| 369 | const struct blkcg_policy *pol) { return 0; } |
| 370 | static inline void blkcg_deactivate_policy(struct request_queue *q, |
| 371 | const struct blkcg_policy *pol) { } |
| 372 | |
| 373 | static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, |
| 374 | struct blkcg_policy *pol) { return NULL; } |
| 375 | static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; } |
| 376 | static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; } |
| 377 | static inline void blkg_get(struct blkcg_gq *blkg) { } |
| 378 | static inline void blkg_put(struct blkcg_gq *blkg) { } |
| 379 | |
| 380 | #endif /* CONFIG_BLK_CGROUP */ |
| 381 | #endif /* _BLK_CGROUP_H */ |