Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* |
3 | * net/core/gen_stats.c | |
4 | * | |
1da177e4 LT |
5 | * Authors: Thomas Graf <tgraf@suug.ch> |
6 | * Jamal Hadi Salim | |
7 | * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> | |
8 | * | |
9 | * See Documentation/networking/gen_stats.txt | |
10 | */ | |
11 | ||
12 | #include <linux/types.h> | |
13 | #include <linux/kernel.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/interrupt.h> | |
16 | #include <linux/socket.h> | |
17 | #include <linux/rtnetlink.h> | |
18 | #include <linux/gen_stats.h> | |
1e90474c | 19 | #include <net/netlink.h> |
1da177e4 LT |
20 | #include <net/gen_stats.h> |
21 | ||
22 | ||
23 | static inline int | |
9854518e | 24 | gnet_stats_copy(struct gnet_dump *d, int type, void *buf, int size, int padattr) |
1da177e4 | 25 | { |
9854518e | 26 | if (nla_put_64bit(d->skb, type, size, buf, padattr)) |
14ad6647 | 27 | goto nla_put_failure; |
1da177e4 LT |
28 | return 0; |
29 | ||
1e90474c | 30 | nla_put_failure: |
edb09eb1 ED |
31 | if (d->lock) |
32 | spin_unlock_bh(d->lock); | |
1c4cff0c IG |
33 | kfree(d->xstats); |
34 | d->xstats = NULL; | |
35 | d->xstats_len = 0; | |
1da177e4 LT |
36 | return -1; |
37 | } | |
38 | ||
39 | /** | |
40 | * gnet_stats_start_copy_compat - start dumping procedure in compatibility mode | |
41 | * @skb: socket buffer to put statistics TLVs into | |
42 | * @type: TLV type for top level statistic TLV | |
43 | * @tc_stats_type: TLV type for backward compatibility struct tc_stats TLV | |
44 | * @xstats_type: TLV type for backward compatibility xstats TLV | |
45 | * @lock: statistics lock | |
46 | * @d: dumping handle | |
e0d194ad | 47 | * @padattr: padding attribute |
1da177e4 LT |
48 | * |
49 | * Initializes the dumping handle, grabs the statistic lock and appends | |
50 | * an empty TLV header to the socket buffer for use a container for all | |
51 | * other statistic TLVS. | |
52 | * | |
53 | * The dumping handle is marked to be in backward compatibility mode telling | |
54 | * all gnet_stats_copy_XXX() functions to fill a local copy of struct tc_stats. | |
55 | * | |
56 | * Returns 0 on success or -1 if the room in the socket buffer was not sufficient. | |
57 | */ | |
58 | int | |
59 | gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int tc_stats_type, | |
9854518e ND |
60 | int xstats_type, spinlock_t *lock, |
61 | struct gnet_dump *d, int padattr) | |
9a429c49 | 62 | __acquires(lock) |
1da177e4 LT |
63 | { |
64 | memset(d, 0, sizeof(*d)); | |
4ec93edb | 65 | |
1da177e4 | 66 | if (type) |
1e90474c | 67 | d->tail = (struct nlattr *)skb_tail_pointer(skb); |
1da177e4 LT |
68 | d->skb = skb; |
69 | d->compat_tc_stats = tc_stats_type; | |
70 | d->compat_xstats = xstats_type; | |
9854518e | 71 | d->padattr = padattr; |
edb09eb1 ED |
72 | if (lock) { |
73 | d->lock = lock; | |
74 | spin_lock_bh(lock); | |
75 | } | |
d5a672ac THJ |
76 | if (d->tail) { |
77 | int ret = gnet_stats_copy(d, type, NULL, 0, padattr); | |
78 | ||
79 | /* The initial attribute added in gnet_stats_copy() may be | |
80 | * preceded by a padding attribute, in which case d->tail will | |
81 | * end up pointing at the padding instead of the real attribute. | |
82 | * Fix this so gnet_stats_finish_copy() adjusts the length of | |
83 | * the right attribute. | |
84 | */ | |
85 | if (ret == 0 && d->tail->nla_type == padattr) | |
86 | d->tail = (struct nlattr *)((char *)d->tail + | |
87 | NLA_ALIGN(d->tail->nla_len)); | |
88 | return ret; | |
89 | } | |
1da177e4 LT |
90 | |
91 | return 0; | |
92 | } | |
9e34a5b5 | 93 | EXPORT_SYMBOL(gnet_stats_start_copy_compat); |
1da177e4 LT |
94 | |
95 | /** | |
9854518e | 96 | * gnet_stats_start_copy - start dumping procedure in compatibility mode |
1da177e4 LT |
97 | * @skb: socket buffer to put statistics TLVs into |
98 | * @type: TLV type for top level statistic TLV | |
99 | * @lock: statistics lock | |
100 | * @d: dumping handle | |
e0d194ad | 101 | * @padattr: padding attribute |
1da177e4 LT |
102 | * |
103 | * Initializes the dumping handle, grabs the statistic lock and appends | |
104 | * an empty TLV header to the socket buffer for use a container for all | |
105 | * other statistic TLVS. | |
106 | * | |
107 | * Returns 0 on success or -1 if the room in the socket buffer was not sufficient. | |
108 | */ | |
109 | int | |
110 | gnet_stats_start_copy(struct sk_buff *skb, int type, spinlock_t *lock, | |
9854518e | 111 | struct gnet_dump *d, int padattr) |
1da177e4 | 112 | { |
9854518e | 113 | return gnet_stats_start_copy_compat(skb, type, 0, 0, lock, d, padattr); |
1da177e4 | 114 | } |
9e34a5b5 | 115 | EXPORT_SYMBOL(gnet_stats_start_copy); |
1da177e4 | 116 | |
22e0f8b9 JF |
117 | static void |
118 | __gnet_stats_copy_basic_cpu(struct gnet_stats_basic_packed *bstats, | |
119 | struct gnet_stats_basic_cpu __percpu *cpu) | |
120 | { | |
121 | int i; | |
122 | ||
123 | for_each_possible_cpu(i) { | |
124 | struct gnet_stats_basic_cpu *bcpu = per_cpu_ptr(cpu, i); | |
125 | unsigned int start; | |
d0083d98 | 126 | u64 bytes, packets; |
22e0f8b9 JF |
127 | |
128 | do { | |
129 | start = u64_stats_fetch_begin_irq(&bcpu->syncp); | |
130 | bytes = bcpu->bstats.bytes; | |
131 | packets = bcpu->bstats.packets; | |
132 | } while (u64_stats_fetch_retry_irq(&bcpu->syncp, start)); | |
133 | ||
02c0fc1b WC |
134 | bstats->bytes += bytes; |
135 | bstats->packets += packets; | |
22e0f8b9 JF |
136 | } |
137 | } | |
138 | ||
139 | void | |
edb09eb1 ED |
140 | __gnet_stats_copy_basic(const seqcount_t *running, |
141 | struct gnet_stats_basic_packed *bstats, | |
22e0f8b9 JF |
142 | struct gnet_stats_basic_cpu __percpu *cpu, |
143 | struct gnet_stats_basic_packed *b) | |
144 | { | |
edb09eb1 ED |
145 | unsigned int seq; |
146 | ||
22e0f8b9 JF |
147 | if (cpu) { |
148 | __gnet_stats_copy_basic_cpu(bstats, cpu); | |
edb09eb1 ED |
149 | return; |
150 | } | |
151 | do { | |
152 | if (running) | |
153 | seq = read_seqcount_begin(running); | |
22e0f8b9 JF |
154 | bstats->bytes = b->bytes; |
155 | bstats->packets = b->packets; | |
edb09eb1 | 156 | } while (running && read_seqcount_retry(running, seq)); |
22e0f8b9 JF |
157 | } |
158 | EXPORT_SYMBOL(__gnet_stats_copy_basic); | |
159 | ||
5d70a670 | 160 | static int |
5e111210 EC |
161 | ___gnet_stats_copy_basic(const seqcount_t *running, |
162 | struct gnet_dump *d, | |
163 | struct gnet_stats_basic_cpu __percpu *cpu, | |
164 | struct gnet_stats_basic_packed *b, | |
165 | int type) | |
1da177e4 | 166 | { |
22e0f8b9 JF |
167 | struct gnet_stats_basic_packed bstats = {0}; |
168 | ||
edb09eb1 | 169 | __gnet_stats_copy_basic(running, &bstats, cpu, b); |
22e0f8b9 | 170 | |
5e111210 | 171 | if (d->compat_tc_stats && type == TCA_STATS_BASIC) { |
22e0f8b9 JF |
172 | d->tc_stats.bytes = bstats.bytes; |
173 | d->tc_stats.packets = bstats.packets; | |
1da177e4 LT |
174 | } |
175 | ||
c1a8f1f1 ED |
176 | if (d->tail) { |
177 | struct gnet_stats_basic sb; | |
b33e699f | 178 | int res; |
1da177e4 | 179 | |
c1a8f1f1 | 180 | memset(&sb, 0, sizeof(sb)); |
22e0f8b9 JF |
181 | sb.bytes = bstats.bytes; |
182 | sb.packets = bstats.packets; | |
b33e699f ED |
183 | res = gnet_stats_copy(d, type, &sb, sizeof(sb), TCA_STATS_PAD); |
184 | if (res < 0 || sb.packets == bstats.packets) | |
185 | return res; | |
186 | /* emit 64bit stats only if needed */ | |
187 | return gnet_stats_copy(d, TCA_STATS_PKT64, &bstats.packets, | |
188 | sizeof(bstats.packets), TCA_STATS_PAD); | |
c1a8f1f1 | 189 | } |
1da177e4 LT |
190 | return 0; |
191 | } | |
5e111210 EC |
192 | |
193 | /** | |
194 | * gnet_stats_copy_basic - copy basic statistics into statistic TLV | |
195 | * @running: seqcount_t pointer | |
196 | * @d: dumping handle | |
197 | * @cpu: copy statistic per cpu | |
198 | * @b: basic statistics | |
199 | * | |
200 | * Appends the basic statistics to the top level TLV created by | |
201 | * gnet_stats_start_copy(). | |
202 | * | |
203 | * Returns 0 on success or -1 with the statistic lock released | |
204 | * if the room in the socket buffer was not sufficient. | |
205 | */ | |
206 | int | |
207 | gnet_stats_copy_basic(const seqcount_t *running, | |
208 | struct gnet_dump *d, | |
209 | struct gnet_stats_basic_cpu __percpu *cpu, | |
210 | struct gnet_stats_basic_packed *b) | |
211 | { | |
212 | return ___gnet_stats_copy_basic(running, d, cpu, b, | |
213 | TCA_STATS_BASIC); | |
214 | } | |
9e34a5b5 | 215 | EXPORT_SYMBOL(gnet_stats_copy_basic); |
1da177e4 | 216 | |
5e111210 EC |
217 | /** |
218 | * gnet_stats_copy_basic_hw - copy basic hw statistics into statistic TLV | |
219 | * @running: seqcount_t pointer | |
220 | * @d: dumping handle | |
221 | * @cpu: copy statistic per cpu | |
222 | * @b: basic statistics | |
223 | * | |
224 | * Appends the basic statistics to the top level TLV created by | |
225 | * gnet_stats_start_copy(). | |
226 | * | |
227 | * Returns 0 on success or -1 with the statistic lock released | |
228 | * if the room in the socket buffer was not sufficient. | |
229 | */ | |
230 | int | |
231 | gnet_stats_copy_basic_hw(const seqcount_t *running, | |
232 | struct gnet_dump *d, | |
233 | struct gnet_stats_basic_cpu __percpu *cpu, | |
234 | struct gnet_stats_basic_packed *b) | |
235 | { | |
236 | return ___gnet_stats_copy_basic(running, d, cpu, b, | |
237 | TCA_STATS_BASIC_HW); | |
238 | } | |
239 | EXPORT_SYMBOL(gnet_stats_copy_basic_hw); | |
240 | ||
1da177e4 LT |
241 | /** |
242 | * gnet_stats_copy_rate_est - copy rate estimator statistics into statistics TLV | |
243 | * @d: dumping handle | |
1c0d32fd | 244 | * @rate_est: rate estimator |
1da177e4 LT |
245 | * |
246 | * Appends the rate estimator statistics to the top level TLV created by | |
247 | * gnet_stats_start_copy(). | |
248 | * | |
249 | * Returns 0 on success or -1 with the statistic lock released | |
250 | * if the room in the socket buffer was not sufficient. | |
251 | */ | |
252 | int | |
d250a5f9 | 253 | gnet_stats_copy_rate_est(struct gnet_dump *d, |
1c0d32fd | 254 | struct net_rate_estimator __rcu **rate_est) |
1da177e4 | 255 | { |
1c0d32fd | 256 | struct gnet_stats_rate_est64 sample; |
45203a3b ED |
257 | struct gnet_stats_rate_est est; |
258 | int res; | |
259 | ||
1c0d32fd | 260 | if (!gen_estimator_read(rate_est, &sample)) |
d250a5f9 | 261 | return 0; |
1c0d32fd | 262 | est.bps = min_t(u64, UINT_MAX, sample.bps); |
45203a3b | 263 | /* we have some time before reaching 2^32 packets per second */ |
1c0d32fd | 264 | est.pps = sample.pps; |
45203a3b | 265 | |
1da177e4 | 266 | if (d->compat_tc_stats) { |
45203a3b ED |
267 | d->tc_stats.bps = est.bps; |
268 | d->tc_stats.pps = est.pps; | |
1da177e4 LT |
269 | } |
270 | ||
45203a3b | 271 | if (d->tail) { |
9854518e ND |
272 | res = gnet_stats_copy(d, TCA_STATS_RATE_EST, &est, sizeof(est), |
273 | TCA_STATS_PAD); | |
1c0d32fd | 274 | if (res < 0 || est.bps == sample.bps) |
45203a3b ED |
275 | return res; |
276 | /* emit 64bit stats only if needed */ | |
1c0d32fd ED |
277 | return gnet_stats_copy(d, TCA_STATS_RATE_EST64, &sample, |
278 | sizeof(sample), TCA_STATS_PAD); | |
45203a3b | 279 | } |
1da177e4 LT |
280 | |
281 | return 0; | |
282 | } | |
9e34a5b5 | 283 | EXPORT_SYMBOL(gnet_stats_copy_rate_est); |
1da177e4 | 284 | |
b0ab6f92 JF |
285 | static void |
286 | __gnet_stats_copy_queue_cpu(struct gnet_stats_queue *qstats, | |
287 | const struct gnet_stats_queue __percpu *q) | |
288 | { | |
289 | int i; | |
290 | ||
291 | for_each_possible_cpu(i) { | |
292 | const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i); | |
293 | ||
73eb628d | 294 | qstats->qlen = 0; |
b0ab6f92 JF |
295 | qstats->backlog += qcpu->backlog; |
296 | qstats->drops += qcpu->drops; | |
297 | qstats->requeues += qcpu->requeues; | |
298 | qstats->overlimits += qcpu->overlimits; | |
299 | } | |
300 | } | |
301 | ||
b01ac095 JF |
302 | void __gnet_stats_copy_queue(struct gnet_stats_queue *qstats, |
303 | const struct gnet_stats_queue __percpu *cpu, | |
304 | const struct gnet_stats_queue *q, | |
305 | __u32 qlen) | |
b0ab6f92 JF |
306 | { |
307 | if (cpu) { | |
308 | __gnet_stats_copy_queue_cpu(qstats, cpu); | |
309 | } else { | |
73eb628d | 310 | qstats->qlen = q->qlen; |
b0ab6f92 JF |
311 | qstats->backlog = q->backlog; |
312 | qstats->drops = q->drops; | |
313 | qstats->requeues = q->requeues; | |
314 | qstats->overlimits = q->overlimits; | |
315 | } | |
316 | ||
317 | qstats->qlen = qlen; | |
318 | } | |
b01ac095 | 319 | EXPORT_SYMBOL(__gnet_stats_copy_queue); |
b0ab6f92 | 320 | |
1da177e4 LT |
321 | /** |
322 | * gnet_stats_copy_queue - copy queue statistics into statistics TLV | |
323 | * @d: dumping handle | |
b0ab6f92 | 324 | * @cpu_q: per cpu queue statistics |
1da177e4 | 325 | * @q: queue statistics |
64015853 | 326 | * @qlen: queue length statistics |
1da177e4 LT |
327 | * |
328 | * Appends the queue statistics to the top level TLV created by | |
b0ab6f92 JF |
329 | * gnet_stats_start_copy(). Using per cpu queue statistics if |
330 | * they are available. | |
1da177e4 LT |
331 | * |
332 | * Returns 0 on success or -1 with the statistic lock released | |
333 | * if the room in the socket buffer was not sufficient. | |
334 | */ | |
335 | int | |
64015853 | 336 | gnet_stats_copy_queue(struct gnet_dump *d, |
b0ab6f92 | 337 | struct gnet_stats_queue __percpu *cpu_q, |
64015853 | 338 | struct gnet_stats_queue *q, __u32 qlen) |
1da177e4 | 339 | { |
b0ab6f92 JF |
340 | struct gnet_stats_queue qstats = {0}; |
341 | ||
342 | __gnet_stats_copy_queue(&qstats, cpu_q, q, qlen); | |
64015853 | 343 | |
1da177e4 | 344 | if (d->compat_tc_stats) { |
b0ab6f92 JF |
345 | d->tc_stats.drops = qstats.drops; |
346 | d->tc_stats.qlen = qstats.qlen; | |
347 | d->tc_stats.backlog = qstats.backlog; | |
348 | d->tc_stats.overlimits = qstats.overlimits; | |
1da177e4 LT |
349 | } |
350 | ||
351 | if (d->tail) | |
b0ab6f92 | 352 | return gnet_stats_copy(d, TCA_STATS_QUEUE, |
9854518e ND |
353 | &qstats, sizeof(qstats), |
354 | TCA_STATS_PAD); | |
1da177e4 LT |
355 | |
356 | return 0; | |
357 | } | |
9e34a5b5 | 358 | EXPORT_SYMBOL(gnet_stats_copy_queue); |
1da177e4 LT |
359 | |
360 | /** | |
361 | * gnet_stats_copy_app - copy application specific statistics into statistics TLV | |
362 | * @d: dumping handle | |
363 | * @st: application specific statistics data | |
364 | * @len: length of data | |
365 | * | |
e793c0f7 | 366 | * Appends the application specific statistics to the top level TLV created by |
1da177e4 LT |
367 | * gnet_stats_start_copy() and remembers the data for XSTATS if the dumping |
368 | * handle is in backward compatibility mode. | |
369 | * | |
370 | * Returns 0 on success or -1 with the statistic lock released | |
371 | * if the room in the socket buffer was not sufficient. | |
372 | */ | |
373 | int | |
374 | gnet_stats_copy_app(struct gnet_dump *d, void *st, int len) | |
375 | { | |
376 | if (d->compat_xstats) { | |
1c4cff0c IG |
377 | d->xstats = kmemdup(st, len, GFP_ATOMIC); |
378 | if (!d->xstats) | |
379 | goto err_out; | |
1da177e4 LT |
380 | d->xstats_len = len; |
381 | } | |
382 | ||
383 | if (d->tail) | |
9854518e ND |
384 | return gnet_stats_copy(d, TCA_STATS_APP, st, len, |
385 | TCA_STATS_PAD); | |
1da177e4 LT |
386 | |
387 | return 0; | |
1c4cff0c IG |
388 | |
389 | err_out: | |
edb09eb1 ED |
390 | if (d->lock) |
391 | spin_unlock_bh(d->lock); | |
1c4cff0c | 392 | d->xstats_len = 0; |
1c4cff0c | 393 | return -1; |
1da177e4 | 394 | } |
9e34a5b5 | 395 | EXPORT_SYMBOL(gnet_stats_copy_app); |
1da177e4 LT |
396 | |
397 | /** | |
398 | * gnet_stats_finish_copy - finish dumping procedure | |
399 | * @d: dumping handle | |
400 | * | |
401 | * Corrects the length of the top level TLV to include all TLVs added | |
402 | * by gnet_stats_copy_XXX() calls. Adds the backward compatibility TLVs | |
403 | * if gnet_stats_start_copy_compat() was used and releases the statistics | |
404 | * lock. | |
405 | * | |
406 | * Returns 0 on success or -1 with the statistic lock released | |
407 | * if the room in the socket buffer was not sufficient. | |
408 | */ | |
409 | int | |
410 | gnet_stats_finish_copy(struct gnet_dump *d) | |
411 | { | |
412 | if (d->tail) | |
1e90474c | 413 | d->tail->nla_len = skb_tail_pointer(d->skb) - (u8 *)d->tail; |
1da177e4 LT |
414 | |
415 | if (d->compat_tc_stats) | |
416 | if (gnet_stats_copy(d, d->compat_tc_stats, &d->tc_stats, | |
9854518e | 417 | sizeof(d->tc_stats), d->padattr) < 0) |
1da177e4 LT |
418 | return -1; |
419 | ||
420 | if (d->compat_xstats && d->xstats) { | |
421 | if (gnet_stats_copy(d, d->compat_xstats, d->xstats, | |
9854518e | 422 | d->xstats_len, d->padattr) < 0) |
1da177e4 LT |
423 | return -1; |
424 | } | |
425 | ||
edb09eb1 ED |
426 | if (d->lock) |
427 | spin_unlock_bh(d->lock); | |
1c4cff0c IG |
428 | kfree(d->xstats); |
429 | d->xstats = NULL; | |
430 | d->xstats_len = 0; | |
1da177e4 LT |
431 | return 0; |
432 | } | |
1da177e4 | 433 | EXPORT_SYMBOL(gnet_stats_finish_copy); |