Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
29c00b4a PM |
2 | #undef TRACE_SYSTEM |
3 | #define TRACE_SYSTEM rcu | |
4 | ||
5 | #if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ) | |
6 | #define _TRACE_RCU_H | |
7 | ||
8 | #include <linux/tracepoint.h> | |
9 | ||
4f5fbd78 YS |
10 | #ifdef CONFIG_RCU_TRACE |
11 | #define TRACE_EVENT_RCU TRACE_EVENT | |
12 | #else | |
13 | #define TRACE_EVENT_RCU TRACE_EVENT_NOP | |
14 | #endif | |
15 | ||
29c00b4a | 16 | /* |
300df91c PM |
17 | * Tracepoint for start/end markers used for utilization calculations. |
18 | * By convention, the string is of the following forms: | |
19 | * | |
20 | * "Start <activity>" -- Mark the start of the specified activity, | |
21 | * such as "context switch". Nesting is permitted. | |
22 | * "End <activity>" -- Mark the end of the specified activity. | |
385680a9 PM |
23 | * |
24 | * An "@" character within "<activity>" is a comment character: Data | |
25 | * reduction scripts will ignore the "@" and the remainder of the line. | |
300df91c PM |
26 | */ |
27 | TRACE_EVENT(rcu_utilization, | |
28 | ||
e66c33d5 | 29 | TP_PROTO(const char *s), |
300df91c PM |
30 | |
31 | TP_ARGS(s), | |
32 | ||
33 | TP_STRUCT__entry( | |
e66c33d5 | 34 | __field(const char *, s) |
300df91c PM |
35 | ), |
36 | ||
37 | TP_fast_assign( | |
38 | __entry->s = s; | |
39 | ), | |
40 | ||
41 | TP_printk("%s", __entry->s) | |
42 | ); | |
43 | ||
28f6569a | 44 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) |
d4c08f2a PM |
45 | |
46 | /* | |
63c4db78 PM |
47 | * Tracepoint for grace-period events. Takes a string identifying the |
48 | * RCU flavor, the grace-period number, and a string identifying the | |
49 | * grace-period-related event as follows: | |
50 | * | |
51 | * "AccReadyCB": CPU acclerates new callbacks to RCU_NEXT_READY_TAIL. | |
52 | * "AccWaitCB": CPU accelerates new callbacks to RCU_WAIT_TAIL. | |
bb311ecc | 53 | * "newreq": Request a new grace period. |
63c4db78 PM |
54 | * "start": Start a grace period. |
55 | * "cpustart": CPU first notices a grace-period start. | |
56 | * "cpuqs": CPU passes through a quiescent state. | |
57 | * "cpuonl": CPU comes online. | |
58 | * "cpuofl": CPU goes offline. | |
477351f7 | 59 | * "cpuofl-bgp": CPU goes offline while blocking a grace period. |
63c4db78 PM |
60 | * "reqwait": GP kthread sleeps waiting for grace-period request. |
61 | * "reqwaitsig": GP kthread awakened by signal from reqwait state. | |
62 | * "fqswait": GP kthread waiting until time to force quiescent states. | |
63 | * "fqsstart": GP kthread starts forcing quiescent states. | |
64 | * "fqsend": GP kthread done forcing quiescent states. | |
65 | * "fqswaitsig": GP kthread awakened by signal from fqswait state. | |
66 | * "end": End a grace period. | |
67 | * "cpuend": CPU first notices a grace-period end. | |
d4c08f2a | 68 | */ |
4f5fbd78 | 69 | TRACE_EVENT_RCU(rcu_grace_period, |
d4c08f2a | 70 | |
477351f7 | 71 | TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent), |
d4c08f2a | 72 | |
477351f7 | 73 | TP_ARGS(rcuname, gp_seq, gpevent), |
d4c08f2a PM |
74 | |
75 | TP_STRUCT__entry( | |
e66c33d5 | 76 | __field(const char *, rcuname) |
477351f7 | 77 | __field(unsigned long, gp_seq) |
e66c33d5 | 78 | __field(const char *, gpevent) |
d4c08f2a PM |
79 | ), |
80 | ||
81 | TP_fast_assign( | |
82 | __entry->rcuname = rcuname; | |
477351f7 | 83 | __entry->gp_seq = gp_seq; |
d4c08f2a PM |
84 | __entry->gpevent = gpevent; |
85 | ), | |
86 | ||
87 | TP_printk("%s %lu %s", | |
477351f7 | 88 | __entry->rcuname, __entry->gp_seq, __entry->gpevent) |
d4c08f2a PM |
89 | ); |
90 | ||
09c7b890 | 91 | /* |
c21b8b3e PM |
92 | * Tracepoint for future grace-period events. The caller should pull |
93 | * the data from the rcu_node structure, other than rcuname, which comes | |
94 | * from the rcu_state structure, and event, which is one of the following: | |
09c7b890 | 95 | * |
c21b8b3e PM |
96 | * "Startleaf": Request a grace period based on leaf-node data. |
97 | * "Prestarted": Someone beat us to the request | |
a2165e41 PM |
98 | * "Startedleaf": Leaf node marked for future GP. |
99 | * "Startedleafroot": All nodes from leaf to root marked for future GP. | |
09c7b890 | 100 | * "Startedroot": Requested a nocb grace period based on root-node data. |
c21b8b3e | 101 | * "NoGPkthread": The RCU grace-period kthread has not yet started. |
09c7b890 PM |
102 | * "StartWait": Start waiting for the requested grace period. |
103 | * "ResumeWait": Resume waiting after signal. | |
104 | * "EndWait": Complete wait. | |
105 | * "Cleanup": Clean up rcu_node structure after previous GP. | |
c21b8b3e | 106 | * "CleanupMore": Clean up, and another GP is needed. |
09c7b890 | 107 | */ |
4f5fbd78 | 108 | TRACE_EVENT_RCU(rcu_future_grace_period, |
09c7b890 | 109 | |
b73de91d JF |
110 | TP_PROTO(const char *rcuname, unsigned long gp_seq, |
111 | unsigned long gp_seq_req, u8 level, int grplo, int grphi, | |
112 | const char *gpevent), | |
09c7b890 | 113 | |
b73de91d | 114 | TP_ARGS(rcuname, gp_seq, gp_seq_req, level, grplo, grphi, gpevent), |
09c7b890 PM |
115 | |
116 | TP_STRUCT__entry( | |
e66c33d5 | 117 | __field(const char *, rcuname) |
abd13fdd | 118 | __field(unsigned long, gp_seq) |
b73de91d | 119 | __field(unsigned long, gp_seq_req) |
09c7b890 PM |
120 | __field(u8, level) |
121 | __field(int, grplo) | |
122 | __field(int, grphi) | |
e66c33d5 | 123 | __field(const char *, gpevent) |
09c7b890 PM |
124 | ), |
125 | ||
126 | TP_fast_assign( | |
127 | __entry->rcuname = rcuname; | |
abd13fdd | 128 | __entry->gp_seq = gp_seq; |
b73de91d | 129 | __entry->gp_seq_req = gp_seq_req; |
09c7b890 PM |
130 | __entry->level = level; |
131 | __entry->grplo = grplo; | |
132 | __entry->grphi = grphi; | |
133 | __entry->gpevent = gpevent; | |
134 | ), | |
135 | ||
abd13fdd | 136 | TP_printk("%s %lu %lu %u %d %d %s", |
b73de91d | 137 | __entry->rcuname, __entry->gp_seq, __entry->gp_seq_req, __entry->level, |
abd13fdd | 138 | __entry->grplo, __entry->grphi, __entry->gpevent) |
09c7b890 PM |
139 | ); |
140 | ||
d4c08f2a PM |
141 | /* |
142 | * Tracepoint for grace-period-initialization events. These are | |
143 | * distinguished by the type of RCU, the new grace-period number, the | |
144 | * rcu_node structure level, the starting and ending CPU covered by the | |
145 | * rcu_node structure, and the mask of CPUs that will be waited for. | |
146 | * All but the type of RCU are extracted from the rcu_node structure. | |
147 | */ | |
4f5fbd78 | 148 | TRACE_EVENT_RCU(rcu_grace_period_init, |
d4c08f2a | 149 | |
63d86a7e | 150 | TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level, |
d4c08f2a PM |
151 | int grplo, int grphi, unsigned long qsmask), |
152 | ||
63d86a7e | 153 | TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask), |
d4c08f2a PM |
154 | |
155 | TP_STRUCT__entry( | |
e66c33d5 | 156 | __field(const char *, rcuname) |
63d86a7e | 157 | __field(unsigned long, gp_seq) |
d4c08f2a PM |
158 | __field(u8, level) |
159 | __field(int, grplo) | |
160 | __field(int, grphi) | |
161 | __field(unsigned long, qsmask) | |
162 | ), | |
163 | ||
164 | TP_fast_assign( | |
165 | __entry->rcuname = rcuname; | |
63d86a7e | 166 | __entry->gp_seq = gp_seq; |
d4c08f2a PM |
167 | __entry->level = level; |
168 | __entry->grplo = grplo; | |
169 | __entry->grphi = grphi; | |
170 | __entry->qsmask = qsmask; | |
171 | ), | |
172 | ||
173 | TP_printk("%s %lu %u %d %d %lx", | |
63d86a7e | 174 | __entry->rcuname, __entry->gp_seq, __entry->level, |
d4c08f2a PM |
175 | __entry->grplo, __entry->grphi, __entry->qsmask) |
176 | ); | |
177 | ||
e087816d PM |
178 | /* |
179 | * Tracepoint for expedited grace-period events. Takes a string identifying | |
180 | * the RCU flavor, the expedited grace-period sequence number, and a string | |
181 | * identifying the grace-period-related event as follows: | |
182 | * | |
183 | * "snap": Captured snapshot of expedited grace period sequence number. | |
184 | * "start": Started a real expedited grace period. | |
9a414201 PM |
185 | * "reset": Started resetting the tree |
186 | * "select": Started selecting the CPUs to wait on. | |
7f5d42d0 | 187 | * "selectofl": Selected CPU partially offline. |
9a414201 | 188 | * "startwait": Started waiting on selected CPUs. |
e087816d | 189 | * "end": Ended a real expedited grace period. |
f6a12f34 | 190 | * "endwake": Woke piggybackers up. |
e087816d PM |
191 | * "done": Someone else did the expedited grace period for us. |
192 | */ | |
4f5fbd78 | 193 | TRACE_EVENT_RCU(rcu_exp_grace_period, |
e087816d PM |
194 | |
195 | TP_PROTO(const char *rcuname, unsigned long gpseq, const char *gpevent), | |
196 | ||
197 | TP_ARGS(rcuname, gpseq, gpevent), | |
198 | ||
199 | TP_STRUCT__entry( | |
200 | __field(const char *, rcuname) | |
201 | __field(unsigned long, gpseq) | |
202 | __field(const char *, gpevent) | |
203 | ), | |
204 | ||
205 | TP_fast_assign( | |
206 | __entry->rcuname = rcuname; | |
207 | __entry->gpseq = gpseq; | |
208 | __entry->gpevent = gpevent; | |
209 | ), | |
210 | ||
211 | TP_printk("%s %lu %s", | |
212 | __entry->rcuname, __entry->gpseq, __entry->gpevent) | |
213 | ); | |
214 | ||
215 | /* | |
216 | * Tracepoint for expedited grace-period funnel-locking events. Takes a | |
217 | * string identifying the RCU flavor, an integer identifying the rcu_node | |
218 | * combining-tree level, another pair of integers identifying the lowest- | |
219 | * and highest-numbered CPU associated with the current rcu_node structure, | |
220 | * and a string. identifying the grace-period-related event as follows: | |
221 | * | |
f6a12f34 PM |
222 | * "nxtlvl": Advance to next level of rcu_node funnel |
223 | * "wait": Wait for someone else to do expedited GP | |
e087816d | 224 | */ |
4f5fbd78 | 225 | TRACE_EVENT_RCU(rcu_exp_funnel_lock, |
e087816d PM |
226 | |
227 | TP_PROTO(const char *rcuname, u8 level, int grplo, int grphi, | |
228 | const char *gpevent), | |
229 | ||
230 | TP_ARGS(rcuname, level, grplo, grphi, gpevent), | |
231 | ||
232 | TP_STRUCT__entry( | |
233 | __field(const char *, rcuname) | |
234 | __field(u8, level) | |
235 | __field(int, grplo) | |
236 | __field(int, grphi) | |
237 | __field(const char *, gpevent) | |
238 | ), | |
239 | ||
240 | TP_fast_assign( | |
241 | __entry->rcuname = rcuname; | |
242 | __entry->level = level; | |
243 | __entry->grplo = grplo; | |
244 | __entry->grphi = grphi; | |
245 | __entry->gpevent = gpevent; | |
246 | ), | |
247 | ||
248 | TP_printk("%s %d %d %d %s", | |
249 | __entry->rcuname, __entry->level, __entry->grplo, | |
250 | __entry->grphi, __entry->gpevent) | |
251 | ); | |
252 | ||
9122caf9 | 253 | #ifdef CONFIG_RCU_NOCB_CPU |
9261dd0d PM |
254 | /* |
255 | * Tracepoint for RCU no-CBs CPU callback handoffs. This event is intended | |
256 | * to assist debugging of these handoffs. | |
257 | * | |
258 | * The first argument is the name of the RCU flavor, and the second is | |
259 | * the number of the offloaded CPU are extracted. The third and final | |
260 | * argument is a string as follows: | |
261 | * | |
262 | * "WakeEmpty": Wake rcuo kthread, first CB to empty list. | |
9fdd3bc9 | 263 | * "WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list. |
9261dd0d | 264 | * "WakeOvf": Wake rcuo kthread, CB list is huge. |
9fdd3bc9 | 265 | * "WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge. |
9261dd0d PM |
266 | * "WakeNot": Don't wake rcuo kthread. |
267 | * "WakeNotPoll": Don't wake rcuo kthread because it is polling. | |
9fdd3bc9 | 268 | * "DeferredWake": Carried out the "IsDeferred" wakeup. |
69a79bb1 PM |
269 | * "Poll": Start of new polling cycle for rcu_nocb_poll. |
270 | * "Sleep": Sleep waiting for CBs for !rcu_nocb_poll. | |
9261dd0d PM |
271 | * "WokeEmpty": rcuo kthread woke to find empty list. |
272 | * "WokeNonEmpty": rcuo kthread woke to find non-empty list. | |
69a79bb1 PM |
273 | * "WaitQueue": Enqueue partially done, timed wait for it to complete. |
274 | * "WokeQueue": Partial enqueue now complete. | |
9261dd0d | 275 | */ |
4f5fbd78 | 276 | TRACE_EVENT_RCU(rcu_nocb_wake, |
9261dd0d PM |
277 | |
278 | TP_PROTO(const char *rcuname, int cpu, const char *reason), | |
279 | ||
280 | TP_ARGS(rcuname, cpu, reason), | |
281 | ||
282 | TP_STRUCT__entry( | |
283 | __field(const char *, rcuname) | |
284 | __field(int, cpu) | |
285 | __field(const char *, reason) | |
286 | ), | |
287 | ||
288 | TP_fast_assign( | |
289 | __entry->rcuname = rcuname; | |
290 | __entry->cpu = cpu; | |
291 | __entry->reason = reason; | |
292 | ), | |
293 | ||
294 | TP_printk("%s %d %s", __entry->rcuname, __entry->cpu, __entry->reason) | |
295 | ); | |
9122caf9 | 296 | #endif |
9261dd0d | 297 | |
d4c08f2a PM |
298 | /* |
299 | * Tracepoint for tasks blocking within preemptible-RCU read-side | |
300 | * critical sections. Track the type of RCU (which one day might | |
301 | * include SRCU), the grace-period number that the task is blocking | |
302 | * (the current or the next), and the task's PID. | |
303 | */ | |
4f5fbd78 | 304 | TRACE_EVENT_RCU(rcu_preempt_task, |
d4c08f2a | 305 | |
598ce094 | 306 | TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq), |
d4c08f2a | 307 | |
598ce094 | 308 | TP_ARGS(rcuname, pid, gp_seq), |
d4c08f2a PM |
309 | |
310 | TP_STRUCT__entry( | |
e66c33d5 | 311 | __field(const char *, rcuname) |
598ce094 | 312 | __field(unsigned long, gp_seq) |
d4c08f2a PM |
313 | __field(int, pid) |
314 | ), | |
315 | ||
316 | TP_fast_assign( | |
317 | __entry->rcuname = rcuname; | |
598ce094 | 318 | __entry->gp_seq = gp_seq; |
d4c08f2a PM |
319 | __entry->pid = pid; |
320 | ), | |
321 | ||
322 | TP_printk("%s %lu %d", | |
598ce094 | 323 | __entry->rcuname, __entry->gp_seq, __entry->pid) |
d4c08f2a PM |
324 | ); |
325 | ||
326 | /* | |
327 | * Tracepoint for tasks that blocked within a given preemptible-RCU | |
328 | * read-side critical section exiting that critical section. Track the | |
329 | * type of RCU (which one day might include SRCU) and the task's PID. | |
330 | */ | |
4f5fbd78 | 331 | TRACE_EVENT_RCU(rcu_unlock_preempted_task, |
d4c08f2a | 332 | |
865aa1e0 | 333 | TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid), |
d4c08f2a | 334 | |
865aa1e0 | 335 | TP_ARGS(rcuname, gp_seq, pid), |
d4c08f2a PM |
336 | |
337 | TP_STRUCT__entry( | |
e66c33d5 | 338 | __field(const char *, rcuname) |
865aa1e0 | 339 | __field(unsigned long, gp_seq) |
d4c08f2a PM |
340 | __field(int, pid) |
341 | ), | |
342 | ||
343 | TP_fast_assign( | |
344 | __entry->rcuname = rcuname; | |
865aa1e0 | 345 | __entry->gp_seq = gp_seq; |
d4c08f2a PM |
346 | __entry->pid = pid; |
347 | ), | |
348 | ||
865aa1e0 | 349 | TP_printk("%s %lu %d", __entry->rcuname, __entry->gp_seq, __entry->pid) |
d4c08f2a PM |
350 | ); |
351 | ||
352 | /* | |
353 | * Tracepoint for quiescent-state-reporting events. These are | |
354 | * distinguished by the type of RCU, the grace-period number, the | |
355 | * mask of quiescent lower-level entities, the rcu_node structure level, | |
356 | * the starting and ending CPU covered by the rcu_node structure, and | |
357 | * whether there are any blocked tasks blocking the current grace period. | |
358 | * All but the type of RCU are extracted from the rcu_node structure. | |
359 | */ | |
4f5fbd78 | 360 | TRACE_EVENT_RCU(rcu_quiescent_state_report, |
d4c08f2a | 361 | |
db023296 | 362 | TP_PROTO(const char *rcuname, unsigned long gp_seq, |
d4c08f2a PM |
363 | unsigned long mask, unsigned long qsmask, |
364 | u8 level, int grplo, int grphi, int gp_tasks), | |
365 | ||
db023296 | 366 | TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks), |
d4c08f2a PM |
367 | |
368 | TP_STRUCT__entry( | |
e66c33d5 | 369 | __field(const char *, rcuname) |
db023296 | 370 | __field(unsigned long, gp_seq) |
d4c08f2a PM |
371 | __field(unsigned long, mask) |
372 | __field(unsigned long, qsmask) | |
373 | __field(u8, level) | |
374 | __field(int, grplo) | |
375 | __field(int, grphi) | |
376 | __field(u8, gp_tasks) | |
377 | ), | |
378 | ||
379 | TP_fast_assign( | |
380 | __entry->rcuname = rcuname; | |
db023296 | 381 | __entry->gp_seq = gp_seq; |
d4c08f2a PM |
382 | __entry->mask = mask; |
383 | __entry->qsmask = qsmask; | |
384 | __entry->level = level; | |
385 | __entry->grplo = grplo; | |
386 | __entry->grphi = grphi; | |
387 | __entry->gp_tasks = gp_tasks; | |
388 | ), | |
389 | ||
390 | TP_printk("%s %lu %lx>%lx %u %d %d %u", | |
db023296 | 391 | __entry->rcuname, __entry->gp_seq, |
d4c08f2a PM |
392 | __entry->mask, __entry->qsmask, __entry->level, |
393 | __entry->grplo, __entry->grphi, __entry->gp_tasks) | |
394 | ); | |
395 | ||
396 | /* | |
397 | * Tracepoint for quiescent states detected by force_quiescent_state(). | |
e05121ba PM |
398 | * These trace events include the type of RCU, the grace-period number |
399 | * that was blocked by the CPU, the CPU itself, and the type of quiescent | |
7e28c5af PM |
400 | * state, which can be "dti" for dyntick-idle mode or "kick" when kicking |
401 | * a CPU that has been in dyntick-idle mode for too long. | |
d4c08f2a | 402 | */ |
4f5fbd78 | 403 | TRACE_EVENT_RCU(rcu_fqs, |
d4c08f2a | 404 | |
fee5997c | 405 | TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent), |
d4c08f2a | 406 | |
fee5997c | 407 | TP_ARGS(rcuname, gp_seq, cpu, qsevent), |
d4c08f2a PM |
408 | |
409 | TP_STRUCT__entry( | |
e66c33d5 | 410 | __field(const char *, rcuname) |
fee5997c | 411 | __field(unsigned long, gp_seq) |
d4c08f2a | 412 | __field(int, cpu) |
e66c33d5 | 413 | __field(const char *, qsevent) |
d4c08f2a PM |
414 | ), |
415 | ||
416 | TP_fast_assign( | |
417 | __entry->rcuname = rcuname; | |
fee5997c | 418 | __entry->gp_seq = gp_seq; |
d4c08f2a PM |
419 | __entry->cpu = cpu; |
420 | __entry->qsevent = qsevent; | |
421 | ), | |
422 | ||
423 | TP_printk("%s %lu %d %s", | |
fee5997c | 424 | __entry->rcuname, __entry->gp_seq, |
d4c08f2a PM |
425 | __entry->cpu, __entry->qsevent) |
426 | ); | |
427 | ||
28f6569a | 428 | #endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) */ |
d4c08f2a PM |
429 | |
430 | /* | |
431 | * Tracepoint for dyntick-idle entry/exit events. These take a string | |
bd2b879a PM |
432 | * as argument: "Start" for entering dyntick-idle mode, "Startirq" for |
433 | * entering it from irq/NMI, "End" for leaving it, "Endirq" for leaving it | |
434 | * to irq/NMI, "--=" for events moving towards idle, and "++=" for events | |
e68bbb26 | 435 | * moving away from idle. |
045fb931 PM |
436 | * |
437 | * These events also take a pair of numbers, which indicate the nesting | |
e68bbb26 PM |
438 | * depth before and after the event of interest, and a third number that is |
439 | * the ->dynticks counter. Note that task-related and interrupt-related | |
440 | * events use two separate counters, and that the "++=" and "--=" events | |
441 | * for irq/NMI will change the counter by two, otherwise by one. | |
d4c08f2a | 442 | */ |
4f5fbd78 | 443 | TRACE_EVENT_RCU(rcu_dyntick, |
d4c08f2a | 444 | |
dec98900 | 445 | TP_PROTO(const char *polarity, long oldnesting, long newnesting, atomic_t dynticks), |
d4c08f2a | 446 | |
dec98900 | 447 | TP_ARGS(polarity, oldnesting, newnesting, dynticks), |
d4c08f2a PM |
448 | |
449 | TP_STRUCT__entry( | |
e66c33d5 | 450 | __field(const char *, polarity) |
84585aa8 PM |
451 | __field(long, oldnesting) |
452 | __field(long, newnesting) | |
dec98900 | 453 | __field(int, dynticks) |
d4c08f2a PM |
454 | ), |
455 | ||
456 | TP_fast_assign( | |
457 | __entry->polarity = polarity; | |
4145fa7f PM |
458 | __entry->oldnesting = oldnesting; |
459 | __entry->newnesting = newnesting; | |
dec98900 | 460 | __entry->dynticks = atomic_read(&dynticks); |
d4c08f2a PM |
461 | ), |
462 | ||
dec98900 PM |
463 | TP_printk("%s %lx %lx %#3x", __entry->polarity, |
464 | __entry->oldnesting, __entry->newnesting, | |
465 | __entry->dynticks & 0xfff) | |
433cdddc PM |
466 | ); |
467 | ||
d4c08f2a PM |
468 | /* |
469 | * Tracepoint for the registration of a single RCU callback function. | |
470 | * The first argument is the type of RCU, the second argument is | |
486e2593 PM |
471 | * a pointer to the RCU callback itself, the third element is the |
472 | * number of lazy callbacks queued, and the fourth element is the | |
473 | * total number of callbacks queued. | |
d4c08f2a | 474 | */ |
4f5fbd78 | 475 | TRACE_EVENT_RCU(rcu_callback, |
d4c08f2a | 476 | |
e66c33d5 | 477 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy, |
486e2593 | 478 | long qlen), |
d4c08f2a | 479 | |
486e2593 | 480 | TP_ARGS(rcuname, rhp, qlen_lazy, qlen), |
d4c08f2a PM |
481 | |
482 | TP_STRUCT__entry( | |
e66c33d5 | 483 | __field(const char *, rcuname) |
d4c08f2a PM |
484 | __field(void *, rhp) |
485 | __field(void *, func) | |
486e2593 | 486 | __field(long, qlen_lazy) |
d4c08f2a PM |
487 | __field(long, qlen) |
488 | ), | |
489 | ||
490 | TP_fast_assign( | |
491 | __entry->rcuname = rcuname; | |
492 | __entry->rhp = rhp; | |
493 | __entry->func = rhp->func; | |
486e2593 | 494 | __entry->qlen_lazy = qlen_lazy; |
d4c08f2a PM |
495 | __entry->qlen = qlen; |
496 | ), | |
497 | ||
d75f773c | 498 | TP_printk("%s rhp=%p func=%ps %ld/%ld", |
486e2593 PM |
499 | __entry->rcuname, __entry->rhp, __entry->func, |
500 | __entry->qlen_lazy, __entry->qlen) | |
d4c08f2a PM |
501 | ); |
502 | ||
503 | /* | |
504 | * Tracepoint for the registration of a single RCU callback of the special | |
505 | * kfree() form. The first argument is the RCU type, the second argument | |
506 | * is a pointer to the RCU callback, the third argument is the offset | |
507 | * of the callback within the enclosing RCU-protected data structure, | |
486e2593 PM |
508 | * the fourth argument is the number of lazy callbacks queued, and the |
509 | * fifth argument is the total number of callbacks queued. | |
d4c08f2a | 510 | */ |
4f5fbd78 | 511 | TRACE_EVENT_RCU(rcu_kfree_callback, |
d4c08f2a | 512 | |
e66c33d5 | 513 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset, |
486e2593 | 514 | long qlen_lazy, long qlen), |
d4c08f2a | 515 | |
486e2593 | 516 | TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen), |
d4c08f2a PM |
517 | |
518 | TP_STRUCT__entry( | |
e66c33d5 | 519 | __field(const char *, rcuname) |
d4c08f2a PM |
520 | __field(void *, rhp) |
521 | __field(unsigned long, offset) | |
486e2593 | 522 | __field(long, qlen_lazy) |
d4c08f2a PM |
523 | __field(long, qlen) |
524 | ), | |
525 | ||
526 | TP_fast_assign( | |
527 | __entry->rcuname = rcuname; | |
528 | __entry->rhp = rhp; | |
529 | __entry->offset = offset; | |
486e2593 | 530 | __entry->qlen_lazy = qlen_lazy; |
d4c08f2a PM |
531 | __entry->qlen = qlen; |
532 | ), | |
533 | ||
486e2593 | 534 | TP_printk("%s rhp=%p func=%ld %ld/%ld", |
d4c08f2a | 535 | __entry->rcuname, __entry->rhp, __entry->offset, |
486e2593 | 536 | __entry->qlen_lazy, __entry->qlen) |
d4c08f2a PM |
537 | ); |
538 | ||
300df91c PM |
539 | /* |
540 | * Tracepoint for marking the beginning rcu_do_batch, performed to start | |
72fe701b | 541 | * RCU callback invocation. The first argument is the RCU flavor, |
486e2593 PM |
542 | * the second is the number of lazy callbacks queued, the third is |
543 | * the total number of callbacks queued, and the fourth argument is | |
544 | * the current RCU-callback batch limit. | |
29c00b4a | 545 | */ |
4f5fbd78 | 546 | TRACE_EVENT_RCU(rcu_batch_start, |
29c00b4a | 547 | |
e66c33d5 | 548 | TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit), |
29c00b4a | 549 | |
486e2593 | 550 | TP_ARGS(rcuname, qlen_lazy, qlen, blimit), |
29c00b4a PM |
551 | |
552 | TP_STRUCT__entry( | |
e66c33d5 | 553 | __field(const char *, rcuname) |
486e2593 | 554 | __field(long, qlen_lazy) |
300df91c | 555 | __field(long, qlen) |
3aac7a8d | 556 | __field(long, blimit) |
29c00b4a PM |
557 | ), |
558 | ||
559 | TP_fast_assign( | |
72fe701b | 560 | __entry->rcuname = rcuname; |
486e2593 | 561 | __entry->qlen_lazy = qlen_lazy; |
300df91c PM |
562 | __entry->qlen = qlen; |
563 | __entry->blimit = blimit; | |
29c00b4a PM |
564 | ), |
565 | ||
3aac7a8d | 566 | TP_printk("%s CBs=%ld/%ld bl=%ld", |
486e2593 PM |
567 | __entry->rcuname, __entry->qlen_lazy, __entry->qlen, |
568 | __entry->blimit) | |
29c00b4a PM |
569 | ); |
570 | ||
571 | /* | |
300df91c | 572 | * Tracepoint for the invocation of a single RCU callback function. |
d4c08f2a PM |
573 | * The first argument is the type of RCU, and the second argument is |
574 | * a pointer to the RCU callback itself. | |
29c00b4a | 575 | */ |
4f5fbd78 | 576 | TRACE_EVENT_RCU(rcu_invoke_callback, |
29c00b4a | 577 | |
e66c33d5 | 578 | TP_PROTO(const char *rcuname, struct rcu_head *rhp), |
29c00b4a | 579 | |
d4c08f2a | 580 | TP_ARGS(rcuname, rhp), |
29c00b4a PM |
581 | |
582 | TP_STRUCT__entry( | |
e66c33d5 | 583 | __field(const char *, rcuname) |
d4c08f2a PM |
584 | __field(void *, rhp) |
585 | __field(void *, func) | |
29c00b4a PM |
586 | ), |
587 | ||
588 | TP_fast_assign( | |
d4c08f2a | 589 | __entry->rcuname = rcuname; |
300df91c PM |
590 | __entry->rhp = rhp; |
591 | __entry->func = rhp->func; | |
29c00b4a PM |
592 | ), |
593 | ||
d75f773c | 594 | TP_printk("%s rhp=%p func=%ps", |
d4c08f2a | 595 | __entry->rcuname, __entry->rhp, __entry->func) |
29c00b4a PM |
596 | ); |
597 | ||
598 | /* | |
300df91c | 599 | * Tracepoint for the invocation of a single RCU callback of the special |
d4c08f2a PM |
600 | * kfree() form. The first argument is the RCU flavor, the second |
601 | * argument is a pointer to the RCU callback, and the third argument | |
602 | * is the offset of the callback within the enclosing RCU-protected | |
603 | * data structure. | |
29c00b4a | 604 | */ |
4f5fbd78 | 605 | TRACE_EVENT_RCU(rcu_invoke_kfree_callback, |
29c00b4a | 606 | |
e66c33d5 | 607 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), |
29c00b4a | 608 | |
d4c08f2a | 609 | TP_ARGS(rcuname, rhp, offset), |
29c00b4a PM |
610 | |
611 | TP_STRUCT__entry( | |
e66c33d5 | 612 | __field(const char *, rcuname) |
d4c08f2a | 613 | __field(void *, rhp) |
300df91c | 614 | __field(unsigned long, offset) |
29c00b4a PM |
615 | ), |
616 | ||
617 | TP_fast_assign( | |
d4c08f2a | 618 | __entry->rcuname = rcuname; |
300df91c | 619 | __entry->rhp = rhp; |
29c00b4a PM |
620 | __entry->offset = offset; |
621 | ), | |
622 | ||
d4c08f2a PM |
623 | TP_printk("%s rhp=%p func=%ld", |
624 | __entry->rcuname, __entry->rhp, __entry->offset) | |
29c00b4a PM |
625 | ); |
626 | ||
627 | /* | |
300df91c | 628 | * Tracepoint for exiting rcu_do_batch after RCU callbacks have been |
4968c300 PM |
629 | * invoked. The first argument is the name of the RCU flavor, |
630 | * the second argument is number of callbacks actually invoked, | |
631 | * the third argument (cb) is whether or not any of the callbacks that | |
632 | * were ready to invoke at the beginning of this batch are still | |
633 | * queued, the fourth argument (nr) is the return value of need_resched(), | |
634 | * the fifth argument (iit) is 1 if the current task is the idle task, | |
635 | * and the sixth argument (risk) is the return value from | |
636 | * rcu_is_callbacks_kthread(). | |
29c00b4a | 637 | */ |
4f5fbd78 | 638 | TRACE_EVENT_RCU(rcu_batch_end, |
29c00b4a | 639 | |
e66c33d5 | 640 | TP_PROTO(const char *rcuname, int callbacks_invoked, |
15f5191b | 641 | char cb, char nr, char iit, char risk), |
29c00b4a | 642 | |
4968c300 | 643 | TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk), |
29c00b4a PM |
644 | |
645 | TP_STRUCT__entry( | |
e66c33d5 | 646 | __field(const char *, rcuname) |
300df91c | 647 | __field(int, callbacks_invoked) |
15f5191b PM |
648 | __field(char, cb) |
649 | __field(char, nr) | |
650 | __field(char, iit) | |
651 | __field(char, risk) | |
29c00b4a PM |
652 | ), |
653 | ||
654 | TP_fast_assign( | |
72fe701b | 655 | __entry->rcuname = rcuname; |
300df91c | 656 | __entry->callbacks_invoked = callbacks_invoked; |
4968c300 PM |
657 | __entry->cb = cb; |
658 | __entry->nr = nr; | |
659 | __entry->iit = iit; | |
660 | __entry->risk = risk; | |
661 | ), | |
662 | ||
663 | TP_printk("%s CBs-invoked=%d idle=%c%c%c%c", | |
664 | __entry->rcuname, __entry->callbacks_invoked, | |
665 | __entry->cb ? 'C' : '.', | |
666 | __entry->nr ? 'S' : '.', | |
667 | __entry->iit ? 'I' : '.', | |
668 | __entry->risk ? 'R' : '.') | |
29c00b4a PM |
669 | ); |
670 | ||
91afaf30 PM |
671 | /* |
672 | * Tracepoint for rcutorture readers. The first argument is the name | |
673 | * of the RCU flavor from rcutorture's viewpoint and the second argument | |
d0af39e8 PM |
674 | * is the callback address. The third argument is the start time in |
675 | * seconds, and the last two arguments are the grace period numbers | |
676 | * at the beginning and end of the read, respectively. Note that the | |
677 | * callback address can be NULL. | |
91afaf30 | 678 | */ |
b3c98314 | 679 | #define RCUTORTURENAME_LEN 8 |
4f5fbd78 | 680 | TRACE_EVENT_RCU(rcu_torture_read, |
91afaf30 | 681 | |
e66c33d5 | 682 | TP_PROTO(const char *rcutorturename, struct rcu_head *rhp, |
52494535 | 683 | unsigned long secs, unsigned long c_old, unsigned long c), |
91afaf30 | 684 | |
52494535 | 685 | TP_ARGS(rcutorturename, rhp, secs, c_old, c), |
91afaf30 PM |
686 | |
687 | TP_STRUCT__entry( | |
b3c98314 | 688 | __field(char, rcutorturename[RCUTORTURENAME_LEN]) |
91afaf30 | 689 | __field(struct rcu_head *, rhp) |
52494535 PM |
690 | __field(unsigned long, secs) |
691 | __field(unsigned long, c_old) | |
692 | __field(unsigned long, c) | |
91afaf30 PM |
693 | ), |
694 | ||
695 | TP_fast_assign( | |
b3c98314 PM |
696 | strncpy(__entry->rcutorturename, rcutorturename, |
697 | RCUTORTURENAME_LEN); | |
698 | __entry->rcutorturename[RCUTORTURENAME_LEN - 1] = 0; | |
91afaf30 | 699 | __entry->rhp = rhp; |
52494535 PM |
700 | __entry->secs = secs; |
701 | __entry->c_old = c_old; | |
702 | __entry->c = c; | |
91afaf30 PM |
703 | ), |
704 | ||
52494535 PM |
705 | TP_printk("%s torture read %p %luus c: %lu %lu", |
706 | __entry->rcutorturename, __entry->rhp, | |
707 | __entry->secs, __entry->c_old, __entry->c) | |
91afaf30 PM |
708 | ); |
709 | ||
a83eff0a | 710 | /* |
dd46a788 PM |
711 | * Tracepoint for rcu_barrier() execution. The string "s" describes |
712 | * the rcu_barrier phase: | |
713 | * "Begin": rcu_barrier() started. | |
714 | * "EarlyExit": rcu_barrier() piggybacked, thus early exit. | |
715 | * "Inc1": rcu_barrier() piggyback check counter incremented. | |
716 | * "OfflineNoCB": rcu_barrier() found callback on never-online CPU | |
717 | * "OnlineNoCB": rcu_barrier() found online no-CBs CPU. | |
718 | * "OnlineQ": rcu_barrier() found online CPU with callbacks. | |
719 | * "OnlineNQ": rcu_barrier() found online CPU, no callbacks. | |
a83eff0a | 720 | * "IRQ": An rcu_barrier_callback() callback posted on remote CPU. |
f92c734f | 721 | * "IRQNQ": An rcu_barrier_callback() callback found no callbacks. |
a83eff0a PM |
722 | * "CB": An rcu_barrier_callback() invoked a callback, not the last. |
723 | * "LastCB": An rcu_barrier_callback() invoked the last callback. | |
dd46a788 | 724 | * "Inc2": rcu_barrier() piggyback check counter incremented. |
a83eff0a PM |
725 | * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument |
726 | * is the count of remaining callbacks, and "done" is the piggybacking count. | |
727 | */ | |
4f5fbd78 | 728 | TRACE_EVENT_RCU(rcu_barrier, |
a83eff0a | 729 | |
e66c33d5 | 730 | TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done), |
a83eff0a PM |
731 | |
732 | TP_ARGS(rcuname, s, cpu, cnt, done), | |
733 | ||
734 | TP_STRUCT__entry( | |
e66c33d5 SRRH |
735 | __field(const char *, rcuname) |
736 | __field(const char *, s) | |
a83eff0a PM |
737 | __field(int, cpu) |
738 | __field(int, cnt) | |
739 | __field(unsigned long, done) | |
740 | ), | |
741 | ||
742 | TP_fast_assign( | |
743 | __entry->rcuname = rcuname; | |
744 | __entry->s = s; | |
745 | __entry->cpu = cpu; | |
746 | __entry->cnt = cnt; | |
747 | __entry->done = done; | |
748 | ), | |
749 | ||
750 | TP_printk("%s %s cpu %d remaining %d # %lu", | |
751 | __entry->rcuname, __entry->s, __entry->cpu, __entry->cnt, | |
752 | __entry->done) | |
753 | ); | |
754 | ||
29c00b4a PM |
755 | #endif /* _TRACE_RCU_H */ |
756 | ||
757 | /* This part must be outside protection */ | |
758 | #include <trace/define_trace.h> |