Commit | Line | Data |
---|---|---|
22e40925 | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
9f77da9f PM |
2 | /* |
3 | * Read-Copy Update mechanism for mutual exclusion (tree-based version) | |
4 | * Internal non-public definitions. | |
5 | * | |
9f77da9f PM |
6 | * Copyright IBM Corporation, 2008 |
7 | * | |
8 | * Author: Ingo Molnar <mingo@elte.hu> | |
22e40925 | 9 | * Paul E. McKenney <paulmck@linux.ibm.com> |
9f77da9f PM |
10 | */ |
11 | ||
12 | #include <linux/cache.h> | |
13 | #include <linux/spinlock.h> | |
037741a6 | 14 | #include <linux/rtmutex.h> |
9f77da9f PM |
15 | #include <linux/threads.h> |
16 | #include <linux/cpumask.h> | |
17 | #include <linux/seqlock.h> | |
abedf8e2 | 18 | #include <linux/swait.h> |
3a6d7c64 | 19 | #include <linux/stop_machine.h> |
f2425b4e | 20 | #include <linux/rcu_node_tree.h> |
f885b7f2 | 21 | |
45753c5f IM |
22 | #include "rcu_segcblist.h" |
23 | ||
25f3d7ef PM |
24 | /* Communicate arguments to a workqueue handler. */ |
25 | struct rcu_exp_work { | |
25f3d7ef PM |
26 | unsigned long rew_s; |
27 | struct work_struct rew_work; | |
28 | }; | |
29 | ||
d71df90e PM |
30 | /* RCU's kthread states for tracing. */ |
31 | #define RCU_KTHREAD_STOPPED 0 | |
32 | #define RCU_KTHREAD_RUNNING 1 | |
33 | #define RCU_KTHREAD_WAITING 2 | |
15ba0ba8 PM |
34 | #define RCU_KTHREAD_OFFCPU 3 |
35 | #define RCU_KTHREAD_YIELDING 4 | |
36 | #define RCU_KTHREAD_MAX 4 | |
d71df90e | 37 | |
9f77da9f PM |
38 | /* |
39 | * Definition for node within the RCU grace-period-detection hierarchy. | |
40 | */ | |
41 | struct rcu_node { | |
67c583a7 BF |
42 | raw_spinlock_t __private lock; /* Root rcu_node's lock protects */ |
43 | /* some rcu_state fields as well as */ | |
44 | /* following. */ | |
de30ad51 | 45 | unsigned long gp_seq; /* Track rsp->rcu_gp_seq. */ |
adbccddb | 46 | unsigned long gp_seq_needed; /* Track furthest future GP request. */ |
4bc8d555 | 47 | unsigned long completedqs; /* All QSes done for this node. */ |
9f77da9f PM |
48 | unsigned long qsmask; /* CPUs or groups that need to switch in */ |
49 | /* order for current grace period to proceed.*/ | |
1eba8f84 PM |
50 | /* In leaf rcu_node, each bit corresponds to */ |
51 | /* an rcu_data structure, otherwise, each */ | |
52 | /* bit corresponds to a child rcu_node */ | |
53 | /* structure. */ | |
f2e2df59 | 54 | unsigned long rcu_gp_init_mask; /* Mask of offline CPUs at GP init. */ |
9f77da9f | 55 | unsigned long qsmaskinit; |
b9585e94 | 56 | /* Per-GP initial value for qsmask. */ |
0aa04b05 PM |
57 | /* Initialized from ->qsmaskinitnext at the */ |
58 | /* beginning of each grace period. */ | |
59 | unsigned long qsmaskinitnext; | |
60 | /* Online CPUs for next grace period. */ | |
b9585e94 PM |
61 | unsigned long expmask; /* CPUs or groups that need to check in */ |
62 | /* to allow the current expedited GP */ | |
63 | /* to complete. */ | |
64 | unsigned long expmaskinit; | |
65 | /* Per-GP initial values for expmask. */ | |
66 | /* Initialized from ->expmaskinitnext at the */ | |
67 | /* beginning of each expedited GP. */ | |
68 | unsigned long expmaskinitnext; | |
69 | /* Online CPUs for next expedited GP. */ | |
1de6e56d PM |
70 | /* Any CPU that has ever been online will */ |
71 | /* have its bit set. */ | |
9b9500da | 72 | unsigned long ffmask; /* Fully functional CPUs. */ |
9f77da9f | 73 | unsigned long grpmask; /* Mask to apply to parent qsmask. */ |
1eba8f84 | 74 | /* Only one bit will be set in this mask. */ |
9f77da9f PM |
75 | int grplo; /* lowest-numbered CPU or group here. */ |
76 | int grphi; /* highest-numbered CPU or group here. */ | |
77 | u8 grpnum; /* CPU/group number for next level up. */ | |
78 | u8 level; /* root is at level 0. */ | |
0aa04b05 PM |
79 | bool wait_blkd_tasks;/* Necessary to wait for blocked tasks to */ |
80 | /* exit RCU read-side critical sections */ | |
81 | /* before propagating offline up the */ | |
82 | /* rcu_node tree? */ | |
9f77da9f | 83 | struct rcu_node *parent; |
12f5f524 PM |
84 | struct list_head blkd_tasks; |
85 | /* Tasks blocked in RCU read-side critical */ | |
86 | /* section. Tasks are placed at the head */ | |
87 | /* of this list and age towards the tail. */ | |
88 | struct list_head *gp_tasks; | |
89 | /* Pointer to the first task blocking the */ | |
90 | /* current grace period, or NULL if there */ | |
91 | /* is no such task. */ | |
92 | struct list_head *exp_tasks; | |
93 | /* Pointer to the first task blocking the */ | |
94 | /* current expedited grace period, or NULL */ | |
95 | /* if there is no such task. If there */ | |
96 | /* is no current expedited grace period, */ | |
97 | /* then there can cannot be any such task. */ | |
27f4d280 PM |
98 | struct list_head *boost_tasks; |
99 | /* Pointer to first task that needs to be */ | |
100 | /* priority boosted, or NULL if no priority */ | |
101 | /* boosting is needed for this rcu_node */ | |
102 | /* structure. If there are no tasks */ | |
103 | /* queued on this rcu_node structure that */ | |
104 | /* are blocking the current grace period, */ | |
105 | /* there can be no such task. */ | |
abaa93d9 PM |
106 | struct rt_mutex boost_mtx; |
107 | /* Used only for the priority-boosting */ | |
108 | /* side effect, not as a lock. */ | |
27f4d280 PM |
109 | unsigned long boost_time; |
110 | /* When to start boosting (jiffies). */ | |
111 | struct task_struct *boost_kthread_task; | |
112 | /* kthread that takes care of priority */ | |
113 | /* boosting for this rcu_node structure. */ | |
d71df90e PM |
114 | unsigned int boost_kthread_status; |
115 | /* State of boost_kthread_task for tracing. */ | |
dae6e64d | 116 | #ifdef CONFIG_RCU_NOCB_CPU |
abedf8e2 | 117 | struct swait_queue_head nocb_gp_wq[2]; |
dae6e64d | 118 | /* Place for rcu_nocb_kthread() to wait GP. */ |
dae6e64d | 119 | #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
394f2769 | 120 | raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp; |
385b73c0 | 121 | |
f6a12f34 PM |
122 | spinlock_t exp_lock ____cacheline_internodealigned_in_smp; |
123 | unsigned long exp_seq_rq; | |
3b5f668e | 124 | wait_queue_head_t exp_wq[4]; |
25f3d7ef PM |
125 | struct rcu_exp_work rew; |
126 | bool exp_need_flush; /* Need to flush workitem? */ | |
9f77da9f PM |
127 | } ____cacheline_internodealigned_in_smp; |
128 | ||
bc75e999 MR |
129 | /* |
130 | * Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and | |
131 | * are indexed relative to this interval rather than the global CPU ID space. | |
132 | * This generates the bit for a CPU in node-local masks. | |
133 | */ | |
df63fa5b | 134 | #define leaf_node_cpu_bit(rnp, cpu) (BIT((cpu) - (rnp)->grplo)) |
bc75e999 | 135 | |
5b74c458 PM |
136 | /* |
137 | * Union to allow "aggregate OR" operation on the need for a quiescent | |
138 | * state by the normal and expedited grace periods. | |
139 | */ | |
140 | union rcu_noqs { | |
141 | struct { | |
142 | u8 norm; | |
143 | u8 exp; | |
144 | } b; /* Bits. */ | |
145 | u16 s; /* Set of bits, aggregate OR here. */ | |
146 | }; | |
147 | ||
9f77da9f PM |
148 | /* Per-CPU data for read-copy update. */ |
149 | struct rcu_data { | |
150 | /* 1) quiescent-state and grace-period handling : */ | |
de30ad51 | 151 | unsigned long gp_seq; /* Track rsp->rcu_gp_seq counter. */ |
adbccddb | 152 | unsigned long gp_seq_needed; /* Track furthest future GP request. */ |
5b74c458 | 153 | union rcu_noqs cpu_no_qs; /* No QSes yet for this CPU. */ |
97c668b8 | 154 | bool core_needs_qs; /* Core waits for quiesc state. */ |
9f77da9f | 155 | bool beenonline; /* CPU online at least once. */ |
ff3bb6f4 | 156 | bool gpwrap; /* Possible ->gp_seq wrap. */ |
1bb33644 | 157 | bool exp_deferred_qs; /* This CPU awaiting a deferred QS? */ |
9f77da9f PM |
158 | struct rcu_node *mynode; /* This CPU's leaf of hierarchy */ |
159 | unsigned long grpmask; /* Mask to apply to leaf qsmask. */ | |
a858af28 PM |
160 | unsigned long ticks_this_gp; /* The number of scheduling-clock */ |
161 | /* ticks this CPU has handled */ | |
162 | /* during and after the last grace */ | |
163 | /* period it is aware of. */ | |
0864f057 PM |
164 | struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ |
165 | bool defer_qs_iw_pending; /* Scheduler attention pending? */ | |
9f77da9f PM |
166 | |
167 | /* 2) batch handling */ | |
15fecf89 PM |
168 | struct rcu_segcblist cblist; /* Segmented callback list, with */ |
169 | /* different callbacks waiting for */ | |
170 | /* different grace periods. */ | |
37c72e56 PM |
171 | long qlen_last_fqs_check; |
172 | /* qlen at last check for QS forcing */ | |
173 | unsigned long n_force_qs_snap; | |
174 | /* did other CPU force QS recently? */ | |
9f77da9f PM |
175 | long blimit; /* Upper limit on a processed batch */ |
176 | ||
9f77da9f | 177 | /* 3) dynticks interface. */ |
9f77da9f | 178 | int dynticks_snap; /* Per-GP tracking for dynticks. */ |
dc5a4f29 PM |
179 | long dynticks_nesting; /* Track process nesting level. */ |
180 | long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ | |
181 | atomic_t dynticks; /* Even value for idle, else odd. */ | |
182 | bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ | |
183 | bool rcu_urgent_qs; /* GP old need light quiescent state. */ | |
cc72046c | 184 | #ifdef CONFIG_RCU_FAST_NO_HZ |
260e1e4f | 185 | bool all_lazy; /* All CPU's CBs lazy at idle start? */ |
dc5a4f29 PM |
186 | unsigned long last_accelerate; /* Last jiffy CBs were accelerated. */ |
187 | unsigned long last_advance_all; /* Last jiffy CBs were all advanced. */ | |
188 | int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */ | |
cc72046c | 189 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ |
9f77da9f | 190 | |
8d8a9d0e | 191 | /* 4) rcu_barrier(), OOM callbacks, and expediting. */ |
06668efa | 192 | struct rcu_head barrier_head; |
0742ac3e | 193 | int exp_dynticks_snap; /* Double-check need for IPI. */ |
06668efa | 194 | |
8d8a9d0e | 195 | /* 5) Callback offloading. */ |
3fbfbf7a PM |
196 | #ifdef CONFIG_RCU_NOCB_CPU |
197 | struct rcu_head *nocb_head; /* CBs waiting for kthread. */ | |
198 | struct rcu_head **nocb_tail; | |
41050a00 PM |
199 | atomic_long_t nocb_q_count; /* # CBs waiting for nocb */ |
200 | atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */ | |
fbce7497 PM |
201 | struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ |
202 | struct rcu_head **nocb_follower_tail; | |
abedf8e2 | 203 | struct swait_queue_head nocb_wq; /* For nocb kthreads to sleep on. */ |
3fbfbf7a | 204 | struct task_struct *nocb_kthread; |
8be6e1b1 | 205 | raw_spinlock_t nocb_lock; /* Guard following pair of fields. */ |
9fdd3bc9 | 206 | int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ |
8be6e1b1 | 207 | struct timer_list nocb_timer; /* Enforce finite deferral. */ |
fbce7497 PM |
208 | |
209 | /* The following fields are used by the leader, hence own cacheline. */ | |
210 | struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp; | |
211 | /* CBs waiting for GP. */ | |
212 | struct rcu_head **nocb_gp_tail; | |
11ed7f93 | 213 | bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */ |
fbce7497 PM |
214 | struct rcu_data *nocb_next_follower; |
215 | /* Next follower in wakeup chain. */ | |
216 | ||
217 | /* The following fields are used by the follower, hence new cachline. */ | |
218 | struct rcu_data *nocb_leader ____cacheline_internodealigned_in_smp; | |
219 | /* Leader CPU takes GP-end wakeups. */ | |
3fbfbf7a PM |
220 | #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
221 | ||
37f62d7c PM |
222 | /* 6) RCU priority boosting. */ |
223 | struct task_struct *rcu_cpu_kthread_task; | |
224 | /* rcuc per-CPU kthread or NULL. */ | |
6ffdde28 | 225 | unsigned int rcu_cpu_kthread_status; |
f7e972ee | 226 | char rcu_cpu_has_work; |
37f62d7c PM |
227 | |
228 | /* 7) Diagnostic data, including RCU CPU stall warnings. */ | |
6231069b | 229 | unsigned int softirq_snap; /* Snapshot of softirq activity. */ |
9b9500da PM |
230 | /* ->rcu_iw* fields protected by leaf rcu_node ->lock. */ |
231 | struct irq_work rcu_iw; /* Check for non-irq activity. */ | |
232 | bool rcu_iw_pending; /* Is ->rcu_iw pending? */ | |
8aa670cd | 233 | unsigned long rcu_iw_gp_seq; /* ->gp_seq associated with ->rcu_iw. */ |
57738942 PM |
234 | unsigned long rcu_ofl_gp_seq; /* ->gp_seq at last offline. */ |
235 | short rcu_ofl_gp_flags; /* ->gp_flags at last offline. */ | |
236 | unsigned long rcu_onl_gp_seq; /* ->gp_seq at last online. */ | |
237 | short rcu_onl_gp_flags; /* ->gp_flags at last online. */ | |
d3052109 | 238 | unsigned long last_fqs_resched; /* Time of last rcu_resched(). */ |
6231069b | 239 | |
9f77da9f PM |
240 | int cpu; |
241 | }; | |
242 | ||
9fdd3bc9 | 243 | /* Values for nocb_defer_wakeup field in struct rcu_data. */ |
511324e4 PM |
244 | #define RCU_NOCB_WAKE_NOT 0 |
245 | #define RCU_NOCB_WAKE 1 | |
246 | #define RCU_NOCB_WAKE_FORCE 2 | |
9fdd3bc9 | 247 | |
026ad283 PM |
248 | #define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500)) |
249 | /* For jiffies_till_first_fqs and */ | |
250 | /* and jiffies_till_next_fqs. */ | |
007b0924 | 251 | |
026ad283 PM |
252 | #define RCU_JIFFIES_FQS_DIV 256 /* Very large systems need more */ |
253 | /* delay between bouts of */ | |
254 | /* quiescent-state forcing. */ | |
255 | ||
256 | #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time to take */ | |
257 | /* at least one scheduling clock */ | |
258 | /* irq before ratting on them. */ | |
9f77da9f | 259 | |
08bca60a PZ |
260 | #define rcu_wait(cond) \ |
261 | do { \ | |
262 | for (;;) { \ | |
263 | set_current_state(TASK_INTERRUPTIBLE); \ | |
264 | if (cond) \ | |
265 | break; \ | |
266 | schedule(); \ | |
267 | } \ | |
268 | __set_current_state(TASK_RUNNING); \ | |
269 | } while (0) | |
9f77da9f PM |
270 | |
271 | /* | |
272 | * RCU global state, including node hierarchy. This hierarchy is | |
273 | * represented in "heap" form in a dense array. The root (first level) | |
274 | * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second | |
275 | * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]), | |
276 | * and the third level in ->node[m+1] and following (->node[m+1] referenced | |
277 | * by ->level[2]). The number of levels is determined by the number of | |
278 | * CPUs and by CONFIG_RCU_FANOUT. Small systems will have a "hierarchy" | |
279 | * consisting of a single rcu_node. | |
280 | */ | |
281 | struct rcu_state { | |
282 | struct rcu_node node[NUM_RCU_NODES]; /* Hierarchy. */ | |
032dfc87 AG |
283 | struct rcu_node *level[RCU_NUM_LVLS + 1]; |
284 | /* Hierarchy levels (+1 to */ | |
285 | /* shut bogus gcc warning) */ | |
b9585e94 | 286 | int ncpus; /* # CPUs seen so far. */ |
9f77da9f PM |
287 | |
288 | /* The following fields are guarded by the root rcu_node's lock. */ | |
289 | ||
77f81fe0 PM |
290 | u8 boost ____cacheline_internodealigned_in_smp; |
291 | /* Subject to priority boost. */ | |
de30ad51 | 292 | unsigned long gp_seq; /* Grace-period sequence #. */ |
b3dbec76 | 293 | struct task_struct *gp_kthread; /* Task for grace periods. */ |
abedf8e2 | 294 | struct swait_queue_head gp_wq; /* Where GP task waits. */ |
afea227f PM |
295 | short gp_flags; /* Commands for GP task. */ |
296 | short gp_state; /* GP kthread sleep state. */ | |
fd897573 PM |
297 | unsigned long gp_wake_time; /* Last GP kthread wake. */ |
298 | unsigned long gp_wake_seq; /* ->gp_seq at ^^^. */ | |
1eba8f84 | 299 | |
d9a3da06 | 300 | /* End of fields guarded by root rcu_node's lock. */ |
1eba8f84 | 301 | |
7be7f0be | 302 | struct mutex barrier_mutex; /* Guards barrier fields. */ |
24ebbca8 | 303 | atomic_t barrier_cpu_count; /* # CPUs waiting on. */ |
7db74df8 | 304 | struct completion barrier_completion; /* Wake at barrier end. */ |
4f525a52 | 305 | unsigned long barrier_sequence; /* ++ at start and end of */ |
dd46a788 | 306 | /* rcu_barrier(). */ |
a4fbe35a PM |
307 | /* End of fields guarded by barrier_mutex. */ |
308 | ||
f6a12f34 | 309 | struct mutex exp_mutex; /* Serialize expedited GP. */ |
3b5f668e | 310 | struct mutex exp_wake_mutex; /* Serialize wakeup. */ |
d6ada2cf | 311 | unsigned long expedited_sequence; /* Take a ticket. */ |
3a6d7c64 | 312 | atomic_t expedited_need_qs; /* # CPUs left to check in. */ |
abedf8e2 | 313 | struct swait_queue_head expedited_wq; /* Wait for check-ins. */ |
b9585e94 | 314 | int ncpus_snap; /* # CPUs seen last time. */ |
40694d66 | 315 | |
9f77da9f PM |
316 | unsigned long jiffies_force_qs; /* Time at which to invoke */ |
317 | /* force_quiescent_state(). */ | |
8c7c4829 PM |
318 | unsigned long jiffies_kick_kthreads; /* Time at which to kick */ |
319 | /* kthreads, if configured. */ | |
9f77da9f PM |
320 | unsigned long n_force_qs; /* Number of calls to */ |
321 | /* force_quiescent_state(). */ | |
9f77da9f PM |
322 | unsigned long gp_start; /* Time at which GP started, */ |
323 | /* but in jiffies. */ | |
c51d7b5e PM |
324 | unsigned long gp_end; /* Time last GP ended, again */ |
325 | /* in jiffies. */ | |
6ccd2ecd PM |
326 | unsigned long gp_activity; /* Time of last GP kthread */ |
327 | /* activity in jiffies. */ | |
26d950a9 PM |
328 | unsigned long gp_req_activity; /* Time of last GP request */ |
329 | /* in jiffies. */ | |
9f77da9f PM |
330 | unsigned long jiffies_stall; /* Time at which to check */ |
331 | /* for CPU stalls. */ | |
6193c76a PM |
332 | unsigned long jiffies_resched; /* Time at which to resched */ |
333 | /* a reluctant CPU. */ | |
fc908ed3 PM |
334 | unsigned long n_force_qs_gpstart; /* Snapshot of n_force_qs at */ |
335 | /* GP start. */ | |
15ba0ba8 PM |
336 | unsigned long gp_max; /* Maximum GP duration in */ |
337 | /* jiffies. */ | |
e66c33d5 | 338 | const char *name; /* Name of structure. */ |
a4889858 | 339 | char abbr; /* Abbreviated name. */ |
1e64b15a | 340 | |
894d45bb | 341 | raw_spinlock_t ofl_lock ____cacheline_internodealigned_in_smp; |
1e64b15a PM |
342 | /* Synchronize offline with */ |
343 | /* GP pre-initialization. */ | |
9f77da9f PM |
344 | }; |
345 | ||
4cdfc175 PM |
346 | /* Values for rcu_state structure's gp_flags field. */ |
347 | #define RCU_GP_FLAG_INIT 0x1 /* Need grace-period initialization. */ | |
348 | #define RCU_GP_FLAG_FQS 0x2 /* Need grace-period quiescent-state forcing. */ | |
349 | ||
c34d2f41 | 350 | /* Values for rcu_state structure's gp_state field. */ |
77f81fe0 | 351 | #define RCU_GP_IDLE 0 /* Initial state and no GP in progress. */ |
afea227f | 352 | #define RCU_GP_WAIT_GPS 1 /* Wait for grace-period start. */ |
319362c9 | 353 | #define RCU_GP_DONE_GPS 2 /* Wait done for grace-period start. */ |
fea3f222 PM |
354 | #define RCU_GP_ONOFF 3 /* Grace-period initialization hotplug. */ |
355 | #define RCU_GP_INIT 4 /* Grace-period initialization. */ | |
356 | #define RCU_GP_WAIT_FQS 5 /* Wait for force-quiescent-state time. */ | |
357 | #define RCU_GP_DOING_FQS 6 /* Wait done for force-quiescent-state time. */ | |
358 | #define RCU_GP_CLEANUP 7 /* Grace-period cleanup started. */ | |
359 | #define RCU_GP_CLEANED 8 /* Grace-period cleanup complete. */ | |
afea227f | 360 | |
6b50e119 PM |
361 | static const char * const gp_state_names[] = { |
362 | "RCU_GP_IDLE", | |
363 | "RCU_GP_WAIT_GPS", | |
364 | "RCU_GP_DONE_GPS", | |
fea3f222 PM |
365 | "RCU_GP_ONOFF", |
366 | "RCU_GP_INIT", | |
6b50e119 PM |
367 | "RCU_GP_WAIT_FQS", |
368 | "RCU_GP_DOING_FQS", | |
369 | "RCU_GP_CLEANUP", | |
370 | "RCU_GP_CLEANED", | |
371 | }; | |
358be2d3 PM |
372 | |
373 | /* | |
374 | * In order to export the rcu_state name to the tracing tools, it | |
375 | * needs to be added in the __tracepoint_string section. | |
376 | * This requires defining a separate variable tp_<sname>_varname | |
377 | * that points to the string being used, and this will allow | |
378 | * the tracing userspace tools to be able to decipher the string | |
379 | * address to the matching string. | |
380 | */ | |
381 | #ifdef CONFIG_PREEMPT_RCU | |
382 | #define RCU_ABBR 'p' | |
383 | #define RCU_NAME_RAW "rcu_preempt" | |
384 | #else /* #ifdef CONFIG_PREEMPT_RCU */ | |
385 | #define RCU_ABBR 's' | |
386 | #define RCU_NAME_RAW "rcu_sched" | |
387 | #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ | |
388 | #ifndef CONFIG_TRACING | |
389 | #define RCU_NAME RCU_NAME_RAW | |
390 | #else /* #ifdef CONFIG_TRACING */ | |
391 | static char rcu_name[] = RCU_NAME_RAW; | |
392 | static const char *tp_rcu_varname __used __tracepoint_string = rcu_name; | |
393 | #define RCU_NAME rcu_name | |
394 | #endif /* #else #ifdef CONFIG_TRACING */ | |
6b50e119 | 395 | |
dc5a4f29 | 396 | int rcu_dynticks_snap(struct rcu_data *rdp); |
02a5c550 | 397 | |
32255d51 | 398 | /* Forward declarations for tree_plugin.h */ |
dbe01350 | 399 | static void rcu_bootup_announce(void); |
45975c7d | 400 | static void rcu_qs(void); |
27f4d280 | 401 | static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); |
b668c9cf | 402 | #ifdef CONFIG_HOTPLUG_CPU |
8af3a5e7 | 403 | static bool rcu_preempt_has_tasks(struct rcu_node *rnp); |
b668c9cf | 404 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ |
74611ecb | 405 | static int rcu_print_task_exp_stall(struct rcu_node *rnp); |
81ab59a3 | 406 | static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); |
c98cac60 | 407 | static void rcu_flavor_sched_clock_irq(int user); |
b6a4ae76 | 408 | void call_rcu(struct rcu_head *head, rcu_callback_t func); |
81ab59a3 | 409 | static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); |
1217ed1b | 410 | static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); |
a46e0899 | 411 | static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); |
dff1672d | 412 | static bool rcu_is_callbacks_kthread(void); |
48d07c04 | 413 | static void rcu_cpu_kthread_setup(unsigned int cpu); |
9386c0b7 | 414 | static void __init rcu_spawn_boost_kthreads(void); |
49fb4c62 | 415 | static void rcu_prepare_kthreads(int cpu); |
8fa7845d | 416 | static void rcu_cleanup_after_idle(void); |
198bbf81 | 417 | static void rcu_prepare_for_idle(void); |
0aa04b05 | 418 | static bool rcu_preempt_has_tasks(struct rcu_node *rnp); |
3e310098 PM |
419 | static bool rcu_preempt_need_deferred_qs(struct task_struct *t); |
420 | static void rcu_preempt_deferred_qs(struct task_struct *t); | |
a858af28 | 421 | static void zero_cpu_stall_ticks(struct rcu_data *rdp); |
4580b054 | 422 | static bool rcu_nocb_cpu_needs_barrier(int cpu); |
abedf8e2 PG |
423 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); |
424 | static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); | |
dae6e64d | 425 | static void rcu_init_one_nocb(struct rcu_node *rnp); |
3fbfbf7a | 426 | static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, |
96d3fd0d | 427 | bool lazy, unsigned long flags); |
b1a2d79f | 428 | static bool rcu_nocb_adopt_orphan_cbs(struct rcu_data *my_rdp, |
96d3fd0d PM |
429 | struct rcu_data *rdp, |
430 | unsigned long flags); | |
9fdd3bc9 | 431 | static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp); |
96d3fd0d | 432 | static void do_nocb_deferred_wakeup(struct rcu_data *rdp); |
3fbfbf7a | 433 | static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp); |
ad368d15 | 434 | static void rcu_spawn_cpu_nocb_kthread(int cpu); |
35ce7f29 PM |
435 | static void __init rcu_spawn_nocb_kthreads(void); |
436 | #ifdef CONFIG_RCU_NOCB_CPU | |
4580b054 | 437 | static void __init rcu_organize_nocb_kthreads(void); |
35ce7f29 | 438 | #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
34ed6246 | 439 | static bool init_nocb_callback_list(struct rcu_data *rdp); |
903ee83d | 440 | static unsigned long rcu_get_n_cbs_nocb_cpu(struct rcu_data *rdp); |
eb75767b | 441 | static void rcu_bind_gp_kthread(void); |
4580b054 | 442 | static bool rcu_nohz_full_cpu(void); |
176f8f7a PM |
443 | static void rcu_dynticks_task_enter(void); |
444 | static void rcu_dynticks_task_exit(void); | |
32255d51 PM |
445 | |
446 | /* Forward declarations for tree_stall.h */ | |
32255d51 | 447 | static void record_gp_stall_check_time(void); |
7ac1907c | 448 | static void rcu_iw_handler(struct irq_work *iwp); |
32255d51 | 449 | static void check_cpu_stall(struct rcu_data *rdp); |
b51bcbbf PM |
450 | static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, |
451 | const unsigned long gpssdelay); |