Commit | Line | Data |
---|---|---|
cf43e6be JA |
1 | /* |
2 | * Block stat tracking code | |
3 | * | |
4 | * Copyright (C) 2016 Jens Axboe | |
5 | */ | |
6 | #include <linux/kernel.h> | |
34dbad5d | 7 | #include <linux/rculist.h> |
cf43e6be JA |
8 | #include <linux/blk-mq.h> |
9 | ||
10 | #include "blk-stat.h" | |
11 | #include "blk-mq.h" | |
b9147dd1 | 12 | #include "blk.h" |
cf43e6be | 13 | |
4875253f OS |
14 | #define BLK_RQ_STAT_BATCH 64 |
15 | ||
34dbad5d OS |
16 | struct blk_queue_stats { |
17 | struct list_head callbacks; | |
18 | spinlock_t lock; | |
b9147dd1 | 19 | bool enable_accounting; |
34dbad5d OS |
20 | }; |
21 | ||
22 | unsigned int blk_stat_rq_ddir(const struct request *rq) | |
23 | { | |
24 | return rq_data_dir(rq); | |
25 | } | |
26 | EXPORT_SYMBOL_GPL(blk_stat_rq_ddir); | |
27 | ||
28 | static void blk_stat_init(struct blk_rq_stat *stat) | |
29 | { | |
30 | stat->min = -1ULL; | |
31 | stat->max = stat->nr_samples = stat->mean = 0; | |
32 | stat->batch = stat->nr_batch = 0; | |
33 | } | |
34 | ||
cf43e6be JA |
35 | static void blk_stat_flush_batch(struct blk_rq_stat *stat) |
36 | { | |
37 | const s32 nr_batch = READ_ONCE(stat->nr_batch); | |
209200ef | 38 | const s32 nr_samples = READ_ONCE(stat->nr_samples); |
cf43e6be JA |
39 | |
40 | if (!nr_batch) | |
41 | return; | |
42 | if (!nr_samples) | |
43 | stat->mean = div64_s64(stat->batch, nr_batch); | |
44 | else { | |
45 | stat->mean = div64_s64((stat->mean * nr_samples) + | |
46 | stat->batch, | |
47 | nr_batch + nr_samples); | |
48 | } | |
49 | ||
50 | stat->nr_samples += nr_batch; | |
51 | stat->nr_batch = stat->batch = 0; | |
52 | } | |
53 | ||
54 | static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) | |
55 | { | |
7d8d0014 OS |
56 | blk_stat_flush_batch(src); |
57 | ||
cf43e6be JA |
58 | if (!src->nr_samples) |
59 | return; | |
60 | ||
cf43e6be JA |
61 | dst->min = min(dst->min, src->min); |
62 | dst->max = max(dst->max, src->max); | |
63 | ||
64 | if (!dst->nr_samples) | |
65 | dst->mean = src->mean; | |
66 | else { | |
67 | dst->mean = div64_s64((src->mean * src->nr_samples) + | |
68 | (dst->mean * dst->nr_samples), | |
69 | dst->nr_samples + src->nr_samples); | |
70 | } | |
71 | dst->nr_samples += src->nr_samples; | |
72 | } | |
73 | ||
34dbad5d | 74 | static void __blk_stat_add(struct blk_rq_stat *stat, u64 value) |
cf43e6be | 75 | { |
34dbad5d OS |
76 | stat->min = min(stat->min, value); |
77 | stat->max = max(stat->max, value); | |
cf43e6be | 78 | |
34dbad5d OS |
79 | if (stat->batch + value < stat->batch || |
80 | stat->nr_batch + 1 == BLK_RQ_STAT_BATCH) | |
81 | blk_stat_flush_batch(stat); | |
cf43e6be | 82 | |
34dbad5d OS |
83 | stat->batch += value; |
84 | stat->nr_batch++; | |
cf43e6be JA |
85 | } |
86 | ||
34dbad5d | 87 | void blk_stat_add(struct request *rq) |
cf43e6be | 88 | { |
34dbad5d OS |
89 | struct request_queue *q = rq->q; |
90 | struct blk_stat_callback *cb; | |
91 | struct blk_rq_stat *stat; | |
92 | int bucket; | |
93 | s64 now, value; | |
94 | ||
95 | now = __blk_stat_time(ktime_to_ns(ktime_get())); | |
96 | if (now < blk_stat_time(&rq->issue_stat)) | |
97 | return; | |
98 | ||
99 | value = now - blk_stat_time(&rq->issue_stat); | |
100 | ||
b9147dd1 SL |
101 | blk_throtl_stat_add(rq, value); |
102 | ||
34dbad5d OS |
103 | rcu_read_lock(); |
104 | list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { | |
105 | if (blk_stat_is_active(cb)) { | |
106 | bucket = cb->bucket_fn(rq); | |
107 | stat = &this_cpu_ptr(cb->cpu_stat)[bucket]; | |
108 | __blk_stat_add(stat, value); | |
109 | } | |
cf43e6be | 110 | } |
34dbad5d | 111 | rcu_read_unlock(); |
cf43e6be JA |
112 | } |
113 | ||
34dbad5d | 114 | static void blk_stat_timer_fn(unsigned long data) |
cf43e6be | 115 | { |
34dbad5d OS |
116 | struct blk_stat_callback *cb = (void *)data; |
117 | unsigned int bucket; | |
118 | int cpu; | |
cf43e6be | 119 | |
34dbad5d OS |
120 | for (bucket = 0; bucket < cb->buckets; bucket++) |
121 | blk_stat_init(&cb->stat[bucket]); | |
cf43e6be | 122 | |
34dbad5d OS |
123 | for_each_online_cpu(cpu) { |
124 | struct blk_rq_stat *cpu_stat; | |
7cd54aa8 | 125 | |
34dbad5d OS |
126 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
127 | for (bucket = 0; bucket < cb->buckets; bucket++) { | |
128 | blk_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]); | |
129 | blk_stat_init(&cpu_stat[bucket]); | |
cf43e6be | 130 | } |
34dbad5d | 131 | } |
cf43e6be | 132 | |
34dbad5d | 133 | cb->timer_fn(cb); |
cf43e6be JA |
134 | } |
135 | ||
34dbad5d OS |
136 | struct blk_stat_callback * |
137 | blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), | |
138 | unsigned int (*bucket_fn)(const struct request *), | |
139 | unsigned int buckets, void *data) | |
cf43e6be | 140 | { |
34dbad5d | 141 | struct blk_stat_callback *cb; |
cf43e6be | 142 | |
34dbad5d OS |
143 | cb = kmalloc(sizeof(*cb), GFP_KERNEL); |
144 | if (!cb) | |
145 | return NULL; | |
cf43e6be | 146 | |
34dbad5d OS |
147 | cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat), |
148 | GFP_KERNEL); | |
149 | if (!cb->stat) { | |
150 | kfree(cb); | |
151 | return NULL; | |
152 | } | |
153 | cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat), | |
154 | __alignof__(struct blk_rq_stat)); | |
155 | if (!cb->cpu_stat) { | |
156 | kfree(cb->stat); | |
157 | kfree(cb); | |
158 | return NULL; | |
159 | } | |
cf43e6be | 160 | |
34dbad5d OS |
161 | cb->timer_fn = timer_fn; |
162 | cb->bucket_fn = bucket_fn; | |
163 | cb->data = data; | |
164 | cb->buckets = buckets; | |
165 | setup_timer(&cb->timer, blk_stat_timer_fn, (unsigned long)cb); | |
166 | ||
167 | return cb; | |
cf43e6be | 168 | } |
34dbad5d | 169 | EXPORT_SYMBOL_GPL(blk_stat_alloc_callback); |
cf43e6be | 170 | |
34dbad5d OS |
171 | void blk_stat_add_callback(struct request_queue *q, |
172 | struct blk_stat_callback *cb) | |
cf43e6be | 173 | { |
34dbad5d OS |
174 | unsigned int bucket; |
175 | int cpu; | |
cf43e6be | 176 | |
34dbad5d OS |
177 | for_each_possible_cpu(cpu) { |
178 | struct blk_rq_stat *cpu_stat; | |
cf43e6be | 179 | |
34dbad5d OS |
180 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
181 | for (bucket = 0; bucket < cb->buckets; bucket++) | |
182 | blk_stat_init(&cpu_stat[bucket]); | |
183 | } | |
cf43e6be | 184 | |
34dbad5d OS |
185 | spin_lock(&q->stats->lock); |
186 | list_add_tail_rcu(&cb->list, &q->stats->callbacks); | |
187 | set_bit(QUEUE_FLAG_STATS, &q->queue_flags); | |
188 | spin_unlock(&q->stats->lock); | |
189 | } | |
190 | EXPORT_SYMBOL_GPL(blk_stat_add_callback); | |
cf43e6be | 191 | |
34dbad5d OS |
192 | void blk_stat_remove_callback(struct request_queue *q, |
193 | struct blk_stat_callback *cb) | |
194 | { | |
195 | spin_lock(&q->stats->lock); | |
196 | list_del_rcu(&cb->list); | |
b9147dd1 | 197 | if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting) |
34dbad5d OS |
198 | clear_bit(QUEUE_FLAG_STATS, &q->queue_flags); |
199 | spin_unlock(&q->stats->lock); | |
cf43e6be | 200 | |
34dbad5d | 201 | del_timer_sync(&cb->timer); |
cf43e6be | 202 | } |
34dbad5d | 203 | EXPORT_SYMBOL_GPL(blk_stat_remove_callback); |
cf43e6be | 204 | |
34dbad5d | 205 | static void blk_stat_free_callback_rcu(struct rcu_head *head) |
cf43e6be | 206 | { |
34dbad5d OS |
207 | struct blk_stat_callback *cb; |
208 | ||
209 | cb = container_of(head, struct blk_stat_callback, rcu); | |
210 | free_percpu(cb->cpu_stat); | |
211 | kfree(cb->stat); | |
212 | kfree(cb); | |
cf43e6be JA |
213 | } |
214 | ||
34dbad5d | 215 | void blk_stat_free_callback(struct blk_stat_callback *cb) |
cf43e6be | 216 | { |
a83b576c JA |
217 | if (cb) |
218 | call_rcu(&cb->rcu, blk_stat_free_callback_rcu); | |
cf43e6be | 219 | } |
34dbad5d | 220 | EXPORT_SYMBOL_GPL(blk_stat_free_callback); |
cf43e6be | 221 | |
b9147dd1 SL |
222 | void blk_stat_enable_accounting(struct request_queue *q) |
223 | { | |
224 | spin_lock(&q->stats->lock); | |
225 | q->stats->enable_accounting = true; | |
226 | set_bit(QUEUE_FLAG_STATS, &q->queue_flags); | |
227 | spin_unlock(&q->stats->lock); | |
228 | } | |
229 | ||
34dbad5d | 230 | struct blk_queue_stats *blk_alloc_queue_stats(void) |
cf43e6be | 231 | { |
34dbad5d OS |
232 | struct blk_queue_stats *stats; |
233 | ||
234 | stats = kmalloc(sizeof(*stats), GFP_KERNEL); | |
235 | if (!stats) | |
236 | return NULL; | |
237 | ||
238 | INIT_LIST_HEAD(&stats->callbacks); | |
239 | spin_lock_init(&stats->lock); | |
b9147dd1 | 240 | stats->enable_accounting = false; |
34dbad5d OS |
241 | |
242 | return stats; | |
243 | } | |
244 | ||
245 | void blk_free_queue_stats(struct blk_queue_stats *stats) | |
246 | { | |
247 | if (!stats) | |
248 | return; | |
249 | ||
250 | WARN_ON(!list_empty(&stats->callbacks)); | |
cf43e6be | 251 | |
34dbad5d | 252 | kfree(stats); |
cf43e6be | 253 | } |