Commit | Line | Data |
---|---|---|
3dcf60bc | 1 | // SPDX-License-Identifier: GPL-2.0 |
cf43e6be JA |
2 | /* |
3 | * Block stat tracking code | |
4 | * | |
5 | * Copyright (C) 2016 Jens Axboe | |
6 | */ | |
7 | #include <linux/kernel.h> | |
34dbad5d | 8 | #include <linux/rculist.h> |
cf43e6be JA |
9 | #include <linux/blk-mq.h> |
10 | ||
11 | #include "blk-stat.h" | |
12 | #include "blk-mq.h" | |
b9147dd1 | 13 | #include "blk.h" |
cf43e6be | 14 | |
34dbad5d OS |
15 | struct blk_queue_stats { |
16 | struct list_head callbacks; | |
17 | spinlock_t lock; | |
68497092 | 18 | int accounting; |
34dbad5d OS |
19 | }; |
20 | ||
2ecbf456 | 21 | void blk_rq_stat_init(struct blk_rq_stat *stat) |
34dbad5d OS |
22 | { |
23 | stat->min = -1ULL; | |
24 | stat->max = stat->nr_samples = stat->mean = 0; | |
eca8b53a | 25 | stat->batch = 0; |
cf43e6be JA |
26 | } |
27 | ||
eca8b53a | 28 | /* src is a per-cpu stat, mean isn't initialized */ |
2ecbf456 | 29 | void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) |
cf43e6be JA |
30 | { |
31 | if (!src->nr_samples) | |
32 | return; | |
33 | ||
cf43e6be JA |
34 | dst->min = min(dst->min, src->min); |
35 | dst->max = max(dst->max, src->max); | |
36 | ||
eca8b53a SL |
37 | dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples, |
38 | dst->nr_samples + src->nr_samples); | |
39 | ||
cf43e6be JA |
40 | dst->nr_samples += src->nr_samples; |
41 | } | |
42 | ||
2ecbf456 | 43 | void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value) |
cf43e6be | 44 | { |
34dbad5d OS |
45 | stat->min = min(stat->min, value); |
46 | stat->max = max(stat->max, value); | |
34dbad5d | 47 | stat->batch += value; |
eca8b53a | 48 | stat->nr_samples++; |
cf43e6be JA |
49 | } |
50 | ||
522a7775 | 51 | void blk_stat_add(struct request *rq, u64 now) |
cf43e6be | 52 | { |
34dbad5d OS |
53 | struct request_queue *q = rq->q; |
54 | struct blk_stat_callback *cb; | |
55 | struct blk_rq_stat *stat; | |
8148f0b5 | 56 | int bucket, cpu; |
522a7775 | 57 | u64 value; |
34dbad5d | 58 | |
544ccc8d | 59 | value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0; |
34dbad5d | 60 | |
a886001c CH |
61 | if (req_op(rq) == REQ_OP_READ || req_op(rq) == REQ_OP_WRITE) |
62 | blk_throtl_stat_add(rq, value); | |
b9147dd1 | 63 | |
34dbad5d | 64 | rcu_read_lock(); |
8148f0b5 | 65 | cpu = get_cpu(); |
34dbad5d | 66 | list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { |
d3738123 JA |
67 | if (!blk_stat_is_active(cb)) |
68 | continue; | |
69 | ||
70 | bucket = cb->bucket_fn(rq); | |
71 | if (bucket < 0) | |
72 | continue; | |
73 | ||
8148f0b5 | 74 | stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket]; |
2ecbf456 | 75 | blk_rq_stat_add(stat, value); |
cf43e6be | 76 | } |
8148f0b5 | 77 | put_cpu(); |
34dbad5d | 78 | rcu_read_unlock(); |
cf43e6be JA |
79 | } |
80 | ||
e99e88a9 | 81 | static void blk_stat_timer_fn(struct timer_list *t) |
cf43e6be | 82 | { |
e99e88a9 | 83 | struct blk_stat_callback *cb = from_timer(cb, t, timer); |
34dbad5d OS |
84 | unsigned int bucket; |
85 | int cpu; | |
cf43e6be | 86 | |
34dbad5d | 87 | for (bucket = 0; bucket < cb->buckets; bucket++) |
2ecbf456 | 88 | blk_rq_stat_init(&cb->stat[bucket]); |
cf43e6be | 89 | |
34dbad5d OS |
90 | for_each_online_cpu(cpu) { |
91 | struct blk_rq_stat *cpu_stat; | |
7cd54aa8 | 92 | |
34dbad5d OS |
93 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
94 | for (bucket = 0; bucket < cb->buckets; bucket++) { | |
2ecbf456 JB |
95 | blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]); |
96 | blk_rq_stat_init(&cpu_stat[bucket]); | |
cf43e6be | 97 | } |
34dbad5d | 98 | } |
cf43e6be | 99 | |
34dbad5d | 100 | cb->timer_fn(cb); |
cf43e6be JA |
101 | } |
102 | ||
34dbad5d OS |
103 | struct blk_stat_callback * |
104 | blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), | |
a37244e4 | 105 | int (*bucket_fn)(const struct request *), |
34dbad5d | 106 | unsigned int buckets, void *data) |
cf43e6be | 107 | { |
34dbad5d | 108 | struct blk_stat_callback *cb; |
cf43e6be | 109 | |
34dbad5d OS |
110 | cb = kmalloc(sizeof(*cb), GFP_KERNEL); |
111 | if (!cb) | |
112 | return NULL; | |
cf43e6be | 113 | |
34dbad5d OS |
114 | cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat), |
115 | GFP_KERNEL); | |
116 | if (!cb->stat) { | |
117 | kfree(cb); | |
118 | return NULL; | |
119 | } | |
120 | cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat), | |
121 | __alignof__(struct blk_rq_stat)); | |
122 | if (!cb->cpu_stat) { | |
123 | kfree(cb->stat); | |
124 | kfree(cb); | |
125 | return NULL; | |
126 | } | |
cf43e6be | 127 | |
34dbad5d OS |
128 | cb->timer_fn = timer_fn; |
129 | cb->bucket_fn = bucket_fn; | |
130 | cb->data = data; | |
131 | cb->buckets = buckets; | |
e99e88a9 | 132 | timer_setup(&cb->timer, blk_stat_timer_fn, 0); |
34dbad5d OS |
133 | |
134 | return cb; | |
cf43e6be JA |
135 | } |
136 | ||
34dbad5d OS |
137 | void blk_stat_add_callback(struct request_queue *q, |
138 | struct blk_stat_callback *cb) | |
cf43e6be | 139 | { |
34dbad5d | 140 | unsigned int bucket; |
e11d80a8 | 141 | unsigned long flags; |
34dbad5d | 142 | int cpu; |
cf43e6be | 143 | |
34dbad5d OS |
144 | for_each_possible_cpu(cpu) { |
145 | struct blk_rq_stat *cpu_stat; | |
cf43e6be | 146 | |
34dbad5d OS |
147 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
148 | for (bucket = 0; bucket < cb->buckets; bucket++) | |
2ecbf456 | 149 | blk_rq_stat_init(&cpu_stat[bucket]); |
34dbad5d | 150 | } |
cf43e6be | 151 | |
e11d80a8 | 152 | spin_lock_irqsave(&q->stats->lock, flags); |
34dbad5d | 153 | list_add_tail_rcu(&cb->list, &q->stats->callbacks); |
7dfdbc73 | 154 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
e11d80a8 | 155 | spin_unlock_irqrestore(&q->stats->lock, flags); |
34dbad5d | 156 | } |
cf43e6be | 157 | |
34dbad5d OS |
158 | void blk_stat_remove_callback(struct request_queue *q, |
159 | struct blk_stat_callback *cb) | |
160 | { | |
e11d80a8 TH |
161 | unsigned long flags; |
162 | ||
163 | spin_lock_irqsave(&q->stats->lock, flags); | |
34dbad5d | 164 | list_del_rcu(&cb->list); |
68497092 | 165 | if (list_empty(&q->stats->callbacks) && !q->stats->accounting) |
7dfdbc73 | 166 | blk_queue_flag_clear(QUEUE_FLAG_STATS, q); |
e11d80a8 | 167 | spin_unlock_irqrestore(&q->stats->lock, flags); |
cf43e6be | 168 | |
34dbad5d | 169 | del_timer_sync(&cb->timer); |
cf43e6be JA |
170 | } |
171 | ||
34dbad5d | 172 | static void blk_stat_free_callback_rcu(struct rcu_head *head) |
cf43e6be | 173 | { |
34dbad5d OS |
174 | struct blk_stat_callback *cb; |
175 | ||
176 | cb = container_of(head, struct blk_stat_callback, rcu); | |
177 | free_percpu(cb->cpu_stat); | |
178 | kfree(cb->stat); | |
179 | kfree(cb); | |
cf43e6be JA |
180 | } |
181 | ||
34dbad5d | 182 | void blk_stat_free_callback(struct blk_stat_callback *cb) |
cf43e6be | 183 | { |
a83b576c JA |
184 | if (cb) |
185 | call_rcu(&cb->rcu, blk_stat_free_callback_rcu); | |
cf43e6be JA |
186 | } |
187 | ||
68497092 JA |
188 | void blk_stat_disable_accounting(struct request_queue *q) |
189 | { | |
190 | unsigned long flags; | |
191 | ||
192 | spin_lock_irqsave(&q->stats->lock, flags); | |
193 | if (!--q->stats->accounting) | |
194 | blk_queue_flag_clear(QUEUE_FLAG_STATS, q); | |
195 | spin_unlock_irqrestore(&q->stats->lock, flags); | |
196 | } | |
197 | EXPORT_SYMBOL_GPL(blk_stat_disable_accounting); | |
198 | ||
b9147dd1 SL |
199 | void blk_stat_enable_accounting(struct request_queue *q) |
200 | { | |
e11d80a8 TH |
201 | unsigned long flags; |
202 | ||
203 | spin_lock_irqsave(&q->stats->lock, flags); | |
68497092 JA |
204 | if (!q->stats->accounting++) |
205 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); | |
e11d80a8 | 206 | spin_unlock_irqrestore(&q->stats->lock, flags); |
b9147dd1 | 207 | } |
f8232f29 | 208 | EXPORT_SYMBOL_GPL(blk_stat_enable_accounting); |
b9147dd1 | 209 | |
34dbad5d | 210 | struct blk_queue_stats *blk_alloc_queue_stats(void) |
cf43e6be | 211 | { |
34dbad5d OS |
212 | struct blk_queue_stats *stats; |
213 | ||
214 | stats = kmalloc(sizeof(*stats), GFP_KERNEL); | |
215 | if (!stats) | |
216 | return NULL; | |
217 | ||
218 | INIT_LIST_HEAD(&stats->callbacks); | |
219 | spin_lock_init(&stats->lock); | |
68497092 | 220 | stats->accounting = 0; |
34dbad5d OS |
221 | |
222 | return stats; | |
223 | } | |
224 | ||
225 | void blk_free_queue_stats(struct blk_queue_stats *stats) | |
226 | { | |
227 | if (!stats) | |
228 | return; | |
229 | ||
230 | WARN_ON(!list_empty(&stats->callbacks)); | |
cf43e6be | 231 | |
34dbad5d | 232 | kfree(stats); |
cf43e6be | 233 | } |
48b5c1fb JA |
234 | |
235 | bool blk_stats_alloc_enable(struct request_queue *q) | |
236 | { | |
237 | struct blk_rq_stat *poll_stat; | |
238 | ||
239 | poll_stat = kcalloc(BLK_MQ_POLL_STATS_BKTS, sizeof(*poll_stat), | |
240 | GFP_ATOMIC); | |
241 | if (!poll_stat) | |
242 | return false; | |
243 | ||
244 | if (cmpxchg(&q->poll_stat, NULL, poll_stat) != NULL) { | |
245 | kfree(poll_stat); | |
246 | return true; | |
247 | } | |
248 | ||
249 | blk_stat_add_callback(q, q->poll_cb); | |
250 | return false; | |
251 | } |