Commit | Line | Data |
---|---|---|
bcea3f96 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e7c15cd8 | 2 | /* |
0c3c86bd | 3 | * trace_hwlat.c - A simple Hardware Latency detector. |
e7c15cd8 SRRH |
4 | * |
5 | * Use this tracer to detect large system latencies induced by the behavior of | |
6 | * certain underlying system hardware or firmware, independent of Linux itself. | |
7 | * The code was developed originally to detect the presence of SMIs on Intel | |
8 | * and AMD systems, although there is no dependency upon x86 herein. | |
9 | * | |
10 | * The classical example usage of this tracer is in detecting the presence of | |
11 | * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a | |
12 | * somewhat special form of hardware interrupt spawned from earlier CPU debug | |
13 | * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge | |
14 | * LPC (or other device) to generate a special interrupt under certain | |
15 | * circumstances, for example, upon expiration of a special SMI timer device, | |
16 | * due to certain external thermal readings, on certain I/O address accesses, | |
17 | * and other situations. An SMI hits a special CPU pin, triggers a special | |
18 | * SMI mode (complete with special memory map), and the OS is unaware. | |
19 | * | |
20 | * Although certain hardware-inducing latencies are necessary (for example, | |
21 | * a modern system often requires an SMI handler for correct thermal control | |
22 | * and remote management) they can wreak havoc upon any OS-level performance | |
23 | * guarantees toward low-latency, especially when the OS is not even made | |
24 | * aware of the presence of these interrupts. For this reason, we need a | |
25 | * somewhat brute force mechanism to detect these interrupts. In this case, | |
26 | * we do it by hogging all of the CPU(s) for configurable timer intervals, | |
27 | * sampling the built-in CPU timer, looking for discontiguous readings. | |
28 | * | |
29 | * WARNING: This implementation necessarily introduces latencies. Therefore, | |
30 | * you should NEVER use this tracer while running in a production | |
31 | * environment requiring any kind of low-latency performance | |
32 | * guarantee(s). | |
33 | * | |
34 | * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com> | |
35 | * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com> | |
36 | * | |
bb1b24cf | 37 | * Includes useful feedback from Clark Williams <williams@redhat.com> |
e7c15cd8 | 38 | * |
e7c15cd8 SRRH |
39 | */ |
40 | #include <linux/kthread.h> | |
41 | #include <linux/tracefs.h> | |
42 | #include <linux/uaccess.h> | |
0330f7aa | 43 | #include <linux/cpumask.h> |
e7c15cd8 | 44 | #include <linux/delay.h> |
e6017571 | 45 | #include <linux/sched/clock.h> |
e7c15cd8 SRRH |
46 | #include "trace.h" |
47 | ||
48 | static struct trace_array *hwlat_trace; | |
49 | ||
50 | #define U64STR_SIZE 22 /* 20 digits max */ | |
51 | ||
52 | #define BANNER "hwlat_detector: " | |
53 | #define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ | |
54 | #define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ | |
55 | #define DEFAULT_LAT_THRESHOLD 10 /* 10us */ | |
56 | ||
e7c15cd8 SRRH |
57 | static struct dentry *hwlat_sample_width; /* sample width us */ |
58 | static struct dentry *hwlat_sample_window; /* sample window us */ | |
8fa826b7 DBO |
59 | static struct dentry *hwlat_thread_mode; /* hwlat thread mode */ |
60 | ||
61 | enum { | |
62 | MODE_NONE = 0, | |
63 | MODE_ROUND_ROBIN, | |
f46b1652 | 64 | MODE_PER_CPU, |
8fa826b7 DBO |
65 | MODE_MAX |
66 | }; | |
f46b1652 | 67 | static char *thread_mode_str[] = { "none", "round-robin", "per-cpu" }; |
e7c15cd8 SRRH |
68 | |
69 | /* Save the previous tracing_thresh value */ | |
70 | static unsigned long save_tracing_thresh; | |
71 | ||
f46b1652 DBO |
72 | /* runtime kthread data */ |
73 | struct hwlat_kthread_data { | |
74 | struct task_struct *kthread; | |
75 | /* NMI timestamp counters */ | |
76 | u64 nmi_ts_start; | |
77 | u64 nmi_total_ts; | |
78 | int nmi_count; | |
79 | int nmi_cpu; | |
80 | }; | |
81 | ||
1d628891 WS |
82 | static struct hwlat_kthread_data hwlat_single_cpu_data; |
83 | static DEFINE_PER_CPU(struct hwlat_kthread_data, hwlat_per_cpu_data); | |
7b2c8625 SRRH |
84 | |
85 | /* Tells NMIs to call back to the hwlat tracer to record timestamps */ | |
86 | bool trace_hwlat_callback_enabled; | |
87 | ||
e7c15cd8 SRRH |
88 | /* If the user changed threshold, remember it */ |
89 | static u64 last_tracing_thresh = DEFAULT_LAT_THRESHOLD * NSEC_PER_USEC; | |
90 | ||
91 | /* Individual latency samples are stored here when detected. */ | |
92 | struct hwlat_sample { | |
51aad0ae DD |
93 | u64 seqnum; /* unique sequence */ |
94 | u64 duration; /* delta */ | |
95 | u64 outer_duration; /* delta (outer loop) */ | |
96 | u64 nmi_total_ts; /* Total time spent in NMIs */ | |
97 | struct timespec64 timestamp; /* wall time */ | |
98 | int nmi_count; /* # NMIs during this sample */ | |
f2cc020d | 99 | int count; /* # of iterations over thresh */ |
e7c15cd8 SRRH |
100 | }; |
101 | ||
102 | /* keep the global state somewhere. */ | |
103 | static struct hwlat_data { | |
104 | ||
105 | struct mutex lock; /* protect changes */ | |
106 | ||
107 | u64 count; /* total since reset */ | |
108 | ||
109 | u64 sample_window; /* total sampling window (on+off) */ | |
110 | u64 sample_width; /* active sampling portion of window */ | |
111 | ||
8fa826b7 DBO |
112 | int thread_mode; /* thread mode */ |
113 | ||
e7c15cd8 SRRH |
114 | } hwlat_data = { |
115 | .sample_window = DEFAULT_SAMPLE_WINDOW, | |
116 | .sample_width = DEFAULT_SAMPLE_WIDTH, | |
8fa826b7 | 117 | .thread_mode = MODE_ROUND_ROBIN |
e7c15cd8 SRRH |
118 | }; |
119 | ||
f46b1652 DBO |
120 | static struct hwlat_kthread_data *get_cpu_data(void) |
121 | { | |
122 | if (hwlat_data.thread_mode == MODE_PER_CPU) | |
123 | return this_cpu_ptr(&hwlat_per_cpu_data); | |
124 | else | |
125 | return &hwlat_single_cpu_data; | |
126 | } | |
127 | ||
8fa826b7 DBO |
128 | static bool hwlat_busy; |
129 | ||
e7c15cd8 SRRH |
130 | static void trace_hwlat_sample(struct hwlat_sample *sample) |
131 | { | |
132 | struct trace_array *tr = hwlat_trace; | |
133 | struct trace_event_call *call = &event_hwlat; | |
13292494 | 134 | struct trace_buffer *buffer = tr->array_buffer.buffer; |
e7c15cd8 SRRH |
135 | struct ring_buffer_event *event; |
136 | struct hwlat_entry *entry; | |
e7c15cd8 SRRH |
137 | |
138 | event = trace_buffer_lock_reserve(buffer, TRACE_HWLAT, sizeof(*entry), | |
36590c50 | 139 | tracing_gen_ctx()); |
e7c15cd8 SRRH |
140 | if (!event) |
141 | return; | |
142 | entry = ring_buffer_event_data(event); | |
143 | entry->seqnum = sample->seqnum; | |
144 | entry->duration = sample->duration; | |
145 | entry->outer_duration = sample->outer_duration; | |
146 | entry->timestamp = sample->timestamp; | |
7b2c8625 SRRH |
147 | entry->nmi_total_ts = sample->nmi_total_ts; |
148 | entry->nmi_count = sample->nmi_count; | |
b396bfde | 149 | entry->count = sample->count; |
e7c15cd8 SRRH |
150 | |
151 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
52ffabe3 | 152 | trace_buffer_unlock_commit_nostack(buffer, event); |
e7c15cd8 SRRH |
153 | } |
154 | ||
155 | /* Macros to encapsulate the time capturing infrastructure */ | |
156 | #define time_type u64 | |
157 | #define time_get() trace_clock_local() | |
158 | #define time_to_us(x) div_u64(x, 1000) | |
159 | #define time_sub(a, b) ((a) - (b)) | |
160 | #define init_time(a, b) (a = b) | |
161 | #define time_u64(a) a | |
162 | ||
7b2c8625 SRRH |
163 | void trace_hwlat_callback(bool enter) |
164 | { | |
f46b1652 DBO |
165 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
166 | ||
167 | if (!kdata->kthread) | |
7b2c8625 SRRH |
168 | return; |
169 | ||
170 | /* | |
171 | * Currently trace_clock_local() calls sched_clock() and the | |
172 | * generic version is not NMI safe. | |
173 | */ | |
174 | if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) { | |
175 | if (enter) | |
f46b1652 | 176 | kdata->nmi_ts_start = time_get(); |
7b2c8625 | 177 | else |
f46b1652 | 178 | kdata->nmi_total_ts += time_get() - kdata->nmi_ts_start; |
7b2c8625 SRRH |
179 | } |
180 | ||
181 | if (enter) | |
f46b1652 | 182 | kdata->nmi_count++; |
7b2c8625 SRRH |
183 | } |
184 | ||
aa892f8c DBO |
185 | /* |
186 | * hwlat_err - report a hwlat error. | |
187 | */ | |
188 | #define hwlat_err(msg) ({ \ | |
189 | struct trace_array *tr = hwlat_trace; \ | |
190 | \ | |
191 | trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg); \ | |
192 | }) | |
193 | ||
e7c15cd8 SRRH |
194 | /** |
195 | * get_sample - sample the CPU TSC and look for likely hardware latencies | |
196 | * | |
197 | * Used to repeatedly capture the CPU TSC (or similar), looking for potential | |
198 | * hardware-induced latency. Called with interrupts disabled and with | |
199 | * hwlat_data.lock held. | |
200 | */ | |
201 | static int get_sample(void) | |
202 | { | |
f46b1652 | 203 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
e7c15cd8 | 204 | struct trace_array *tr = hwlat_trace; |
b396bfde | 205 | struct hwlat_sample s; |
e7c15cd8 | 206 | time_type start, t1, t2, last_t2; |
b396bfde | 207 | s64 diff, outer_diff, total, last_total = 0; |
e7c15cd8 SRRH |
208 | u64 sample = 0; |
209 | u64 thresh = tracing_thresh; | |
210 | u64 outer_sample = 0; | |
211 | int ret = -1; | |
b396bfde | 212 | unsigned int count = 0; |
e7c15cd8 SRRH |
213 | |
214 | do_div(thresh, NSEC_PER_USEC); /* modifies interval value */ | |
215 | ||
f46b1652 DBO |
216 | kdata->nmi_total_ts = 0; |
217 | kdata->nmi_count = 0; | |
7b2c8625 SRRH |
218 | /* Make sure NMIs see this first */ |
219 | barrier(); | |
220 | ||
221 | trace_hwlat_callback_enabled = true; | |
222 | ||
e7c15cd8 SRRH |
223 | init_time(last_t2, 0); |
224 | start = time_get(); /* start timestamp */ | |
b396bfde | 225 | outer_diff = 0; |
e7c15cd8 SRRH |
226 | |
227 | do { | |
228 | ||
229 | t1 = time_get(); /* we'll look for a discontinuity */ | |
230 | t2 = time_get(); | |
231 | ||
232 | if (time_u64(last_t2)) { | |
233 | /* Check the delta from outer loop (t2 to next t1) */ | |
b396bfde | 234 | outer_diff = time_to_us(time_sub(t1, last_t2)); |
e7c15cd8 | 235 | /* This shouldn't happen */ |
b396bfde | 236 | if (outer_diff < 0) { |
aa892f8c | 237 | hwlat_err(BANNER "time running backwards\n"); |
e7c15cd8 SRRH |
238 | goto out; |
239 | } | |
b396bfde SRV |
240 | if (outer_diff > outer_sample) |
241 | outer_sample = outer_diff; | |
e7c15cd8 SRRH |
242 | } |
243 | last_t2 = t2; | |
244 | ||
245 | total = time_to_us(time_sub(t2, start)); /* sample width */ | |
246 | ||
247 | /* Check for possible overflows */ | |
248 | if (total < last_total) { | |
aa892f8c | 249 | hwlat_err("Time total overflowed\n"); |
e7c15cd8 SRRH |
250 | break; |
251 | } | |
252 | last_total = total; | |
253 | ||
254 | /* This checks the inner loop (t1 to t2) */ | |
255 | diff = time_to_us(time_sub(t2, t1)); /* current diff */ | |
256 | ||
b396bfde SRV |
257 | if (diff > thresh || outer_diff > thresh) { |
258 | if (!count) | |
259 | ktime_get_real_ts64(&s.timestamp); | |
260 | count++; | |
261 | } | |
262 | ||
e7c15cd8 SRRH |
263 | /* This shouldn't happen */ |
264 | if (diff < 0) { | |
aa892f8c | 265 | hwlat_err(BANNER "time running backwards\n"); |
e7c15cd8 SRRH |
266 | goto out; |
267 | } | |
268 | ||
269 | if (diff > sample) | |
270 | sample = diff; /* only want highest value */ | |
271 | ||
272 | } while (total <= hwlat_data.sample_width); | |
273 | ||
7b2c8625 SRRH |
274 | barrier(); /* finish the above in the view for NMIs */ |
275 | trace_hwlat_callback_enabled = false; | |
276 | barrier(); /* Make sure nmi_total_ts is no longer updated */ | |
277 | ||
e7c15cd8 SRRH |
278 | ret = 0; |
279 | ||
280 | /* If we exceed the threshold value, we have found a hardware latency */ | |
281 | if (sample > thresh || outer_sample > thresh) { | |
91edde2e | 282 | u64 latency; |
e7c15cd8 SRRH |
283 | |
284 | ret = 1; | |
285 | ||
7b2c8625 | 286 | /* We read in microseconds */ |
f46b1652 DBO |
287 | if (kdata->nmi_total_ts) |
288 | do_div(kdata->nmi_total_ts, NSEC_PER_USEC); | |
7b2c8625 | 289 | |
e7c15cd8 SRRH |
290 | hwlat_data.count++; |
291 | s.seqnum = hwlat_data.count; | |
292 | s.duration = sample; | |
293 | s.outer_duration = outer_sample; | |
f46b1652 DBO |
294 | s.nmi_total_ts = kdata->nmi_total_ts; |
295 | s.nmi_count = kdata->nmi_count; | |
b396bfde | 296 | s.count = count; |
e7c15cd8 SRRH |
297 | trace_hwlat_sample(&s); |
298 | ||
91edde2e VRB |
299 | latency = max(sample, outer_sample); |
300 | ||
e7c15cd8 | 301 | /* Keep a running maximum ever recorded hardware latency */ |
91edde2e VRB |
302 | if (latency > tr->max_latency) { |
303 | tr->max_latency = latency; | |
304 | latency_fsnotify(tr); | |
305 | } | |
e7c15cd8 SRRH |
306 | } |
307 | ||
308 | out: | |
309 | return ret; | |
310 | } | |
311 | ||
0330f7aa | 312 | static struct cpumask save_cpumask; |
0330f7aa | 313 | |
f447c196 | 314 | static void move_to_next_cpu(void) |
0330f7aa | 315 | { |
f447c196 | 316 | struct cpumask *current_mask = &save_cpumask; |
96b4833b | 317 | struct trace_array *tr = hwlat_trace; |
0330f7aa SRRH |
318 | int next_cpu; |
319 | ||
0330f7aa SRRH |
320 | /* |
321 | * If for some reason the user modifies the CPU affinity | |
0c3c86bd | 322 | * of this thread, then stop migrating for the duration |
0330f7aa SRRH |
323 | * of the current test. |
324 | */ | |
3bd37062 | 325 | if (!cpumask_equal(current_mask, current->cpus_ptr)) |
7bb7d802 | 326 | goto change_mode; |
0330f7aa | 327 | |
99c37d1a | 328 | cpus_read_lock(); |
96b4833b | 329 | cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask); |
51397dc6 | 330 | next_cpu = cpumask_next(raw_smp_processor_id(), current_mask); |
99c37d1a | 331 | cpus_read_unlock(); |
0330f7aa SRRH |
332 | |
333 | if (next_cpu >= nr_cpu_ids) | |
334 | next_cpu = cpumask_first(current_mask); | |
335 | ||
0330f7aa | 336 | if (next_cpu >= nr_cpu_ids) /* Shouldn't happen! */ |
7bb7d802 | 337 | goto change_mode; |
0330f7aa SRRH |
338 | |
339 | cpumask_clear(current_mask); | |
340 | cpumask_set_cpu(next_cpu, current_mask); | |
341 | ||
71c7a304 | 342 | set_cpus_allowed_ptr(current, current_mask); |
0330f7aa SRRH |
343 | return; |
344 | ||
7bb7d802 DBO |
345 | change_mode: |
346 | hwlat_data.thread_mode = MODE_NONE; | |
347 | pr_info(BANNER "cpumask changed while in round-robin mode, switching to mode none\n"); | |
0330f7aa SRRH |
348 | } |
349 | ||
e7c15cd8 SRRH |
350 | /* |
351 | * kthread_fn - The CPU time sampling/hardware latency detection kernel thread | |
352 | * | |
353 | * Used to periodically sample the CPU TSC via a call to get_sample. We | |
354 | * disable interrupts, which does (intentionally) introduce latency since we | |
355 | * need to ensure nothing else might be running (and thus preempting). | |
356 | * Obviously this should never be used in production environments. | |
357 | * | |
8e0f1142 | 358 | * Executes one loop interaction on each CPU in tracing_cpumask sysfs file. |
e7c15cd8 SRRH |
359 | */ |
360 | static int kthread_fn(void *data) | |
361 | { | |
362 | u64 interval; | |
363 | ||
364 | while (!kthread_should_stop()) { | |
365 | ||
8fa826b7 DBO |
366 | if (hwlat_data.thread_mode == MODE_ROUND_ROBIN) |
367 | move_to_next_cpu(); | |
0330f7aa | 368 | |
e7c15cd8 SRRH |
369 | local_irq_disable(); |
370 | get_sample(); | |
371 | local_irq_enable(); | |
372 | ||
373 | mutex_lock(&hwlat_data.lock); | |
374 | interval = hwlat_data.sample_window - hwlat_data.sample_width; | |
375 | mutex_unlock(&hwlat_data.lock); | |
376 | ||
377 | do_div(interval, USEC_PER_MSEC); /* modifies interval value */ | |
378 | ||
379 | /* Always sleep for at least 1ms */ | |
380 | if (interval < 1) | |
381 | interval = 1; | |
382 | ||
383 | if (msleep_interruptible(interval)) | |
384 | break; | |
385 | } | |
386 | ||
387 | return 0; | |
388 | } | |
389 | ||
8fa826b7 | 390 | /* |
f46b1652 DBO |
391 | * stop_stop_kthread - Inform the hardware latency sampling/detector kthread to stop |
392 | * | |
393 | * This kicks the running hardware latency sampling/detector kernel thread and | |
394 | * tells it to stop sampling now. Use this on unload and at system shutdown. | |
395 | */ | |
396 | static void stop_single_kthread(void) | |
397 | { | |
398 | struct hwlat_kthread_data *kdata = get_cpu_data(); | |
039a602d DBO |
399 | struct task_struct *kthread; |
400 | ||
99c37d1a | 401 | cpus_read_lock(); |
039a602d | 402 | kthread = kdata->kthread; |
f46b1652 DBO |
403 | |
404 | if (!kthread) | |
039a602d | 405 | goto out_put_cpus; |
f46b1652 DBO |
406 | |
407 | kthread_stop(kthread); | |
408 | kdata->kthread = NULL; | |
039a602d DBO |
409 | |
410 | out_put_cpus: | |
99c37d1a | 411 | cpus_read_unlock(); |
f46b1652 DBO |
412 | } |
413 | ||
414 | ||
415 | /* | |
416 | * start_single_kthread - Kick off the hardware latency sampling/detector kthread | |
e7c15cd8 SRRH |
417 | * |
418 | * This starts the kernel thread that will sit and sample the CPU timestamp | |
419 | * counter (TSC or similar) and look for potential hardware latencies. | |
420 | */ | |
f46b1652 | 421 | static int start_single_kthread(struct trace_array *tr) |
e7c15cd8 | 422 | { |
f46b1652 | 423 | struct hwlat_kthread_data *kdata = get_cpu_data(); |
f447c196 | 424 | struct cpumask *current_mask = &save_cpumask; |
e7c15cd8 | 425 | struct task_struct *kthread; |
f447c196 SRV |
426 | int next_cpu; |
427 | ||
99c37d1a | 428 | cpus_read_lock(); |
f46b1652 | 429 | if (kdata->kthread) |
039a602d | 430 | goto out_put_cpus; |
82fbc8c4 | 431 | |
e7c15cd8 SRRH |
432 | kthread = kthread_create(kthread_fn, NULL, "hwlatd"); |
433 | if (IS_ERR(kthread)) { | |
434 | pr_err(BANNER "could not start sampling thread\n"); | |
99c37d1a | 435 | cpus_read_unlock(); |
e7c15cd8 SRRH |
436 | return -ENOMEM; |
437 | } | |
f447c196 | 438 | |
8fa826b7 | 439 | /* Just pick the first CPU on first iteration */ |
8fa826b7 | 440 | cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask); |
8fa826b7 DBO |
441 | |
442 | if (hwlat_data.thread_mode == MODE_ROUND_ROBIN) { | |
443 | next_cpu = cpumask_first(current_mask); | |
444 | cpumask_clear(current_mask); | |
445 | cpumask_set_cpu(next_cpu, current_mask); | |
446 | ||
447 | } | |
448 | ||
71c7a304 | 449 | set_cpus_allowed_ptr(kthread, current_mask); |
f447c196 | 450 | |
f46b1652 | 451 | kdata->kthread = kthread; |
e7c15cd8 SRRH |
452 | wake_up_process(kthread); |
453 | ||
039a602d | 454 | out_put_cpus: |
99c37d1a | 455 | cpus_read_unlock(); |
e7c15cd8 SRRH |
456 | return 0; |
457 | } | |
458 | ||
8fa826b7 | 459 | /* |
f46b1652 DBO |
460 | * stop_cpu_kthread - Stop a hwlat cpu kthread |
461 | */ | |
462 | static void stop_cpu_kthread(unsigned int cpu) | |
463 | { | |
464 | struct task_struct *kthread; | |
465 | ||
466 | kthread = per_cpu(hwlat_per_cpu_data, cpu).kthread; | |
467 | if (kthread) | |
468 | kthread_stop(kthread); | |
ba998f7d | 469 | per_cpu(hwlat_per_cpu_data, cpu).kthread = NULL; |
f46b1652 DBO |
470 | } |
471 | ||
472 | /* | |
473 | * stop_per_cpu_kthreads - Inform the hardware latency sampling/detector kthread to stop | |
e7c15cd8 | 474 | * |
f46b1652 | 475 | * This kicks the running hardware latency sampling/detector kernel threads and |
e7c15cd8 SRRH |
476 | * tells it to stop sampling now. Use this on unload and at system shutdown. |
477 | */ | |
f46b1652 | 478 | static void stop_per_cpu_kthreads(void) |
e7c15cd8 | 479 | { |
f46b1652 DBO |
480 | unsigned int cpu; |
481 | ||
99c37d1a | 482 | cpus_read_lock(); |
f46b1652 DBO |
483 | for_each_online_cpu(cpu) |
484 | stop_cpu_kthread(cpu); | |
99c37d1a | 485 | cpus_read_unlock(); |
f46b1652 DBO |
486 | } |
487 | ||
488 | /* | |
489 | * start_cpu_kthread - Start a hwlat cpu kthread | |
490 | */ | |
491 | static int start_cpu_kthread(unsigned int cpu) | |
492 | { | |
493 | struct task_struct *kthread; | |
f46b1652 | 494 | |
08697bca TK |
495 | /* Do not start a new hwlatd thread if it is already running */ |
496 | if (per_cpu(hwlat_per_cpu_data, cpu).kthread) | |
497 | return 0; | |
498 | ||
ff78f667 | 499 | kthread = kthread_run_on_cpu(kthread_fn, NULL, cpu, "hwlatd/%u"); |
f46b1652 DBO |
500 | if (IS_ERR(kthread)) { |
501 | pr_err(BANNER "could not start sampling thread\n"); | |
502 | return -ENOMEM; | |
503 | } | |
504 | ||
505 | per_cpu(hwlat_per_cpu_data, cpu).kthread = kthread; | |
f46b1652 DBO |
506 | |
507 | return 0; | |
508 | } | |
509 | ||
ba998f7d DBO |
510 | #ifdef CONFIG_HOTPLUG_CPU |
511 | static void hwlat_hotplug_workfn(struct work_struct *dummy) | |
512 | { | |
513 | struct trace_array *tr = hwlat_trace; | |
514 | unsigned int cpu = smp_processor_id(); | |
515 | ||
516 | mutex_lock(&trace_types_lock); | |
517 | mutex_lock(&hwlat_data.lock); | |
99c37d1a | 518 | cpus_read_lock(); |
ba998f7d DBO |
519 | |
520 | if (!hwlat_busy || hwlat_data.thread_mode != MODE_PER_CPU) | |
521 | goto out_unlock; | |
522 | ||
523 | if (!cpumask_test_cpu(cpu, tr->tracing_cpumask)) | |
524 | goto out_unlock; | |
525 | ||
526 | start_cpu_kthread(cpu); | |
527 | ||
528 | out_unlock: | |
99c37d1a | 529 | cpus_read_unlock(); |
ba998f7d DBO |
530 | mutex_unlock(&hwlat_data.lock); |
531 | mutex_unlock(&trace_types_lock); | |
532 | } | |
533 | ||
534 | static DECLARE_WORK(hwlat_hotplug_work, hwlat_hotplug_workfn); | |
535 | ||
536 | /* | |
537 | * hwlat_cpu_init - CPU hotplug online callback function | |
538 | */ | |
539 | static int hwlat_cpu_init(unsigned int cpu) | |
540 | { | |
541 | schedule_work_on(cpu, &hwlat_hotplug_work); | |
542 | return 0; | |
543 | } | |
544 | ||
545 | /* | |
546 | * hwlat_cpu_die - CPU hotplug offline callback function | |
547 | */ | |
548 | static int hwlat_cpu_die(unsigned int cpu) | |
549 | { | |
550 | stop_cpu_kthread(cpu); | |
551 | return 0; | |
552 | } | |
553 | ||
554 | static void hwlat_init_hotplug_support(void) | |
555 | { | |
556 | int ret; | |
557 | ||
558 | ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/hwlat:online", | |
559 | hwlat_cpu_init, hwlat_cpu_die); | |
560 | if (ret < 0) | |
561 | pr_warn(BANNER "Error to init cpu hotplug support\n"); | |
562 | ||
563 | return; | |
564 | } | |
565 | #else /* CONFIG_HOTPLUG_CPU */ | |
566 | static void hwlat_init_hotplug_support(void) | |
567 | { | |
568 | return; | |
569 | } | |
570 | #endif /* CONFIG_HOTPLUG_CPU */ | |
571 | ||
f46b1652 DBO |
572 | /* |
573 | * start_per_cpu_kthreads - Kick off the hardware latency sampling/detector kthreads | |
574 | * | |
575 | * This starts the kernel threads that will sit on potentially all cpus and | |
576 | * sample the CPU timestamp counter (TSC or similar) and look for potential | |
577 | * hardware latencies. | |
578 | */ | |
579 | static int start_per_cpu_kthreads(struct trace_array *tr) | |
580 | { | |
581 | struct cpumask *current_mask = &save_cpumask; | |
582 | unsigned int cpu; | |
583 | int retval; | |
584 | ||
99c37d1a | 585 | cpus_read_lock(); |
f46b1652 DBO |
586 | /* |
587 | * Run only on CPUs in which hwlat is allowed to run. | |
588 | */ | |
589 | cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask); | |
590 | ||
f46b1652 DBO |
591 | for_each_cpu(cpu, current_mask) { |
592 | retval = start_cpu_kthread(cpu); | |
593 | if (retval) | |
594 | goto out_error; | |
595 | } | |
99c37d1a | 596 | cpus_read_unlock(); |
f46b1652 DBO |
597 | |
598 | return 0; | |
599 | ||
600 | out_error: | |
99c37d1a | 601 | cpus_read_unlock(); |
f46b1652 DBO |
602 | stop_per_cpu_kthreads(); |
603 | return retval; | |
e7c15cd8 SRRH |
604 | } |
605 | ||
8fa826b7 DBO |
606 | static void *s_mode_start(struct seq_file *s, loff_t *pos) |
607 | { | |
608 | int mode = *pos; | |
609 | ||
610 | mutex_lock(&hwlat_data.lock); | |
611 | ||
612 | if (mode >= MODE_MAX) | |
613 | return NULL; | |
614 | ||
615 | return pos; | |
616 | } | |
617 | ||
618 | static void *s_mode_next(struct seq_file *s, void *v, loff_t *pos) | |
619 | { | |
620 | int mode = ++(*pos); | |
621 | ||
622 | if (mode >= MODE_MAX) | |
623 | return NULL; | |
624 | ||
625 | return pos; | |
626 | } | |
627 | ||
628 | static int s_mode_show(struct seq_file *s, void *v) | |
629 | { | |
630 | loff_t *pos = v; | |
631 | int mode = *pos; | |
632 | ||
633 | if (mode == hwlat_data.thread_mode) | |
634 | seq_printf(s, "[%s]", thread_mode_str[mode]); | |
635 | else | |
636 | seq_printf(s, "%s", thread_mode_str[mode]); | |
637 | ||
638 | if (mode != MODE_MAX) | |
639 | seq_puts(s, " "); | |
640 | ||
641 | return 0; | |
642 | } | |
643 | ||
644 | static void s_mode_stop(struct seq_file *s, void *v) | |
645 | { | |
646 | seq_puts(s, "\n"); | |
647 | mutex_unlock(&hwlat_data.lock); | |
648 | } | |
649 | ||
650 | static const struct seq_operations thread_mode_seq_ops = { | |
651 | .start = s_mode_start, | |
652 | .next = s_mode_next, | |
653 | .show = s_mode_show, | |
654 | .stop = s_mode_stop | |
655 | }; | |
656 | ||
657 | static int hwlat_mode_open(struct inode *inode, struct file *file) | |
658 | { | |
659 | return seq_open(file, &thread_mode_seq_ops); | |
660 | }; | |
661 | ||
662 | static void hwlat_tracer_start(struct trace_array *tr); | |
663 | static void hwlat_tracer_stop(struct trace_array *tr); | |
664 | ||
665 | /** | |
666 | * hwlat_mode_write - Write function for "mode" entry | |
667 | * @filp: The active open file structure | |
668 | * @ubuf: The user buffer that contains the value to write | |
669 | * @cnt: The maximum number of bytes to write to "file" | |
670 | * @ppos: The current position in @file | |
671 | * | |
672 | * This function provides a write implementation for the "mode" interface | |
673 | * to the hardware latency detector. hwlatd has different operation modes. | |
674 | * The "none" sets the allowed cpumask for a single hwlatd thread at the | |
675 | * startup and lets the scheduler handle the migration. The default mode is | |
676 | * the "round-robin" one, in which a single hwlatd thread runs, migrating | |
f46b1652 DBO |
677 | * among the allowed CPUs in a round-robin fashion. The "per-cpu" mode |
678 | * creates one hwlatd thread per allowed CPU. | |
8fa826b7 DBO |
679 | */ |
680 | static ssize_t hwlat_mode_write(struct file *filp, const char __user *ubuf, | |
681 | size_t cnt, loff_t *ppos) | |
682 | { | |
683 | struct trace_array *tr = hwlat_trace; | |
684 | const char *mode; | |
685 | char buf[64]; | |
686 | int ret, i; | |
687 | ||
688 | if (cnt >= sizeof(buf)) | |
689 | return -EINVAL; | |
690 | ||
691 | if (copy_from_user(buf, ubuf, cnt)) | |
692 | return -EFAULT; | |
693 | ||
694 | buf[cnt] = 0; | |
695 | ||
696 | mode = strstrip(buf); | |
697 | ||
698 | ret = -EINVAL; | |
699 | ||
700 | /* | |
701 | * trace_types_lock is taken to avoid concurrency on start/stop | |
702 | * and hwlat_busy. | |
703 | */ | |
704 | mutex_lock(&trace_types_lock); | |
705 | if (hwlat_busy) | |
706 | hwlat_tracer_stop(tr); | |
707 | ||
708 | mutex_lock(&hwlat_data.lock); | |
709 | ||
710 | for (i = 0; i < MODE_MAX; i++) { | |
711 | if (strcmp(mode, thread_mode_str[i]) == 0) { | |
712 | hwlat_data.thread_mode = i; | |
713 | ret = cnt; | |
714 | } | |
715 | } | |
716 | ||
717 | mutex_unlock(&hwlat_data.lock); | |
718 | ||
719 | if (hwlat_busy) | |
720 | hwlat_tracer_start(tr); | |
721 | mutex_unlock(&trace_types_lock); | |
722 | ||
723 | *ppos += cnt; | |
724 | ||
725 | ||
726 | ||
727 | return ret; | |
728 | } | |
729 | ||
f27a1c9e DBO |
730 | /* |
731 | * The width parameter is read/write using the generic trace_min_max_param | |
732 | * method. The *val is protected by the hwlat_data lock and is upper | |
733 | * bounded by the window parameter. | |
734 | */ | |
735 | static struct trace_min_max_param hwlat_width = { | |
736 | .lock = &hwlat_data.lock, | |
737 | .val = &hwlat_data.sample_width, | |
738 | .max = &hwlat_data.sample_window, | |
739 | .min = NULL, | |
e7c15cd8 SRRH |
740 | }; |
741 | ||
f27a1c9e DBO |
742 | /* |
743 | * The window parameter is read/write using the generic trace_min_max_param | |
744 | * method. The *val is protected by the hwlat_data lock and is lower | |
745 | * bounded by the width parameter. | |
746 | */ | |
747 | static struct trace_min_max_param hwlat_window = { | |
748 | .lock = &hwlat_data.lock, | |
749 | .val = &hwlat_data.sample_window, | |
750 | .max = NULL, | |
751 | .min = &hwlat_data.sample_width, | |
e7c15cd8 SRRH |
752 | }; |
753 | ||
8fa826b7 DBO |
754 | static const struct file_operations thread_mode_fops = { |
755 | .open = hwlat_mode_open, | |
756 | .read = seq_read, | |
757 | .llseek = seq_lseek, | |
758 | .release = seq_release, | |
759 | .write = hwlat_mode_write | |
760 | }; | |
e7c15cd8 SRRH |
761 | /** |
762 | * init_tracefs - A function to initialize the tracefs interface files | |
763 | * | |
764 | * This function creates entries in tracefs for "hwlat_detector". | |
765 | * It creates the hwlat_detector directory in the tracing directory, | |
766 | * and within that directory is the count, width and window files to | |
767 | * change and view those values. | |
768 | */ | |
769 | static int init_tracefs(void) | |
770 | { | |
22c36b18 | 771 | int ret; |
e7c15cd8 SRRH |
772 | struct dentry *top_dir; |
773 | ||
22c36b18 WY |
774 | ret = tracing_init_dentry(); |
775 | if (ret) | |
e7c15cd8 SRRH |
776 | return -ENOMEM; |
777 | ||
22c36b18 | 778 | top_dir = tracefs_create_dir("hwlat_detector", NULL); |
e7c15cd8 SRRH |
779 | if (!top_dir) |
780 | return -ENOMEM; | |
781 | ||
21ccc9cd | 782 | hwlat_sample_window = tracefs_create_file("window", TRACE_MODE_WRITE, |
e7c15cd8 | 783 | top_dir, |
f27a1c9e DBO |
784 | &hwlat_window, |
785 | &trace_min_max_fops); | |
e7c15cd8 SRRH |
786 | if (!hwlat_sample_window) |
787 | goto err; | |
788 | ||
21ccc9cd | 789 | hwlat_sample_width = tracefs_create_file("width", TRACE_MODE_WRITE, |
e7c15cd8 | 790 | top_dir, |
f27a1c9e DBO |
791 | &hwlat_width, |
792 | &trace_min_max_fops); | |
e7c15cd8 SRRH |
793 | if (!hwlat_sample_width) |
794 | goto err; | |
795 | ||
21ccc9cd | 796 | hwlat_thread_mode = trace_create_file("mode", TRACE_MODE_WRITE, |
8fa826b7 DBO |
797 | top_dir, |
798 | NULL, | |
799 | &thread_mode_fops); | |
800 | if (!hwlat_thread_mode) | |
801 | goto err; | |
802 | ||
e7c15cd8 SRRH |
803 | return 0; |
804 | ||
805 | err: | |
a3d1e7eb | 806 | tracefs_remove(top_dir); |
e7c15cd8 SRRH |
807 | return -ENOMEM; |
808 | } | |
809 | ||
810 | static void hwlat_tracer_start(struct trace_array *tr) | |
811 | { | |
812 | int err; | |
813 | ||
f46b1652 DBO |
814 | if (hwlat_data.thread_mode == MODE_PER_CPU) |
815 | err = start_per_cpu_kthreads(tr); | |
816 | else | |
817 | err = start_single_kthread(tr); | |
e7c15cd8 SRRH |
818 | if (err) |
819 | pr_err(BANNER "Cannot start hwlat kthread\n"); | |
820 | } | |
821 | ||
822 | static void hwlat_tracer_stop(struct trace_array *tr) | |
823 | { | |
f46b1652 DBO |
824 | if (hwlat_data.thread_mode == MODE_PER_CPU) |
825 | stop_per_cpu_kthreads(); | |
826 | else | |
827 | stop_single_kthread(); | |
e7c15cd8 SRRH |
828 | } |
829 | ||
e7c15cd8 SRRH |
830 | static int hwlat_tracer_init(struct trace_array *tr) |
831 | { | |
832 | /* Only allow one instance to enable this */ | |
833 | if (hwlat_busy) | |
834 | return -EBUSY; | |
835 | ||
836 | hwlat_trace = tr; | |
837 | ||
838 | hwlat_data.count = 0; | |
839 | tr->max_latency = 0; | |
840 | save_tracing_thresh = tracing_thresh; | |
841 | ||
842 | /* tracing_thresh is in nsecs, we speak in usecs */ | |
843 | if (!tracing_thresh) | |
844 | tracing_thresh = last_tracing_thresh; | |
845 | ||
846 | if (tracer_tracing_is_on(tr)) | |
847 | hwlat_tracer_start(tr); | |
848 | ||
849 | hwlat_busy = true; | |
850 | ||
851 | return 0; | |
852 | } | |
853 | ||
854 | static void hwlat_tracer_reset(struct trace_array *tr) | |
855 | { | |
f46b1652 | 856 | hwlat_tracer_stop(tr); |
e7c15cd8 SRRH |
857 | |
858 | /* the tracing threshold is static between runs */ | |
859 | last_tracing_thresh = tracing_thresh; | |
860 | ||
861 | tracing_thresh = save_tracing_thresh; | |
862 | hwlat_busy = false; | |
863 | } | |
864 | ||
865 | static struct tracer hwlat_tracer __read_mostly = | |
866 | { | |
867 | .name = "hwlat", | |
868 | .init = hwlat_tracer_init, | |
869 | .reset = hwlat_tracer_reset, | |
870 | .start = hwlat_tracer_start, | |
871 | .stop = hwlat_tracer_stop, | |
872 | .allow_instances = true, | |
873 | }; | |
874 | ||
875 | __init static int init_hwlat_tracer(void) | |
876 | { | |
877 | int ret; | |
878 | ||
879 | mutex_init(&hwlat_data.lock); | |
880 | ||
881 | ret = register_tracer(&hwlat_tracer); | |
882 | if (ret) | |
883 | return ret; | |
884 | ||
ba998f7d DBO |
885 | hwlat_init_hotplug_support(); |
886 | ||
e7c15cd8 SRRH |
887 | init_tracefs(); |
888 | ||
889 | return 0; | |
890 | } | |
891 | late_initcall(init_hwlat_tracer); |