Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Read-Copy Update mechanism for mutual exclusion | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | * | |
01c1c660 | 18 | * Copyright IBM Corporation, 2001 |
1da177e4 LT |
19 | * |
20 | * Authors: Dipankar Sarma <dipankar@in.ibm.com> | |
21 | * Manfred Spraul <manfred@colorfullife.com> | |
a71fca58 | 22 | * |
1da177e4 LT |
23 | * Based on the original work by Paul McKenney <paulmck@us.ibm.com> |
24 | * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. | |
25 | * Papers: | |
26 | * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf | |
27 | * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) | |
28 | * | |
29 | * For detailed explanation of Read-Copy Update mechanism see - | |
a71fca58 | 30 | * http://lse.sourceforge.net/locking/rcupdate.html |
1da177e4 LT |
31 | * |
32 | */ | |
33 | #include <linux/types.h> | |
34 | #include <linux/kernel.h> | |
35 | #include <linux/init.h> | |
36 | #include <linux/spinlock.h> | |
37 | #include <linux/smp.h> | |
38 | #include <linux/interrupt.h> | |
39 | #include <linux/sched.h> | |
60063497 | 40 | #include <linux/atomic.h> |
1da177e4 | 41 | #include <linux/bitops.h> |
1da177e4 LT |
42 | #include <linux/percpu.h> |
43 | #include <linux/notifier.h> | |
1da177e4 | 44 | #include <linux/cpu.h> |
9331b315 | 45 | #include <linux/mutex.h> |
9984de1a | 46 | #include <linux/export.h> |
e3818b8d | 47 | #include <linux/hardirq.h> |
1da177e4 | 48 | |
29c00b4a PM |
49 | #define CREATE_TRACE_POINTS |
50 | #include <trace/events/rcu.h> | |
51 | ||
52 | #include "rcu.h" | |
53 | ||
9dd8fb16 PM |
54 | #ifdef CONFIG_PREEMPT_RCU |
55 | ||
56 | /* | |
57 | * Check for a task exiting while in a preemptible-RCU read-side | |
58 | * critical section, clean up if so. No need to issue warnings, | |
59 | * as debug_check_no_locks_held() already does this if lockdep | |
60 | * is enabled. | |
61 | */ | |
62 | void exit_rcu(void) | |
63 | { | |
64 | struct task_struct *t = current; | |
65 | ||
66 | if (likely(list_empty(¤t->rcu_node_entry))) | |
67 | return; | |
68 | t->rcu_read_lock_nesting = 1; | |
69 | barrier(); | |
70 | t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED; | |
71 | __rcu_read_unlock(); | |
72 | } | |
73 | ||
74 | #else /* #ifdef CONFIG_PREEMPT_RCU */ | |
75 | ||
76 | void exit_rcu(void) | |
77 | { | |
78 | } | |
79 | ||
80 | #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ | |
81 | ||
162cc279 PM |
82 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
83 | static struct lock_class_key rcu_lock_key; | |
84 | struct lockdep_map rcu_lock_map = | |
85 | STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key); | |
86 | EXPORT_SYMBOL_GPL(rcu_lock_map); | |
632ee200 PM |
87 | |
88 | static struct lock_class_key rcu_bh_lock_key; | |
89 | struct lockdep_map rcu_bh_lock_map = | |
90 | STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_bh", &rcu_bh_lock_key); | |
91 | EXPORT_SYMBOL_GPL(rcu_bh_lock_map); | |
92 | ||
93 | static struct lock_class_key rcu_sched_lock_key; | |
94 | struct lockdep_map rcu_sched_lock_map = | |
95 | STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_sched", &rcu_sched_lock_key); | |
96 | EXPORT_SYMBOL_GPL(rcu_sched_lock_map); | |
162cc279 PM |
97 | #endif |
98 | ||
e3818b8d PM |
99 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
100 | ||
bc293d62 PM |
101 | int debug_lockdep_rcu_enabled(void) |
102 | { | |
103 | return rcu_scheduler_active && debug_locks && | |
104 | current->lockdep_recursion == 0; | |
105 | } | |
106 | EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); | |
107 | ||
e3818b8d | 108 | /** |
ca5ecddf | 109 | * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section? |
e3818b8d PM |
110 | * |
111 | * Check for bottom half being disabled, which covers both the | |
112 | * CONFIG_PROVE_RCU and not cases. Note that if someone uses | |
113 | * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled) | |
ca5ecddf PM |
114 | * will show the situation. This is useful for debug checks in functions |
115 | * that require that they be called within an RCU read-side critical | |
116 | * section. | |
e3818b8d PM |
117 | * |
118 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot. | |
c0d6d01b PM |
119 | * |
120 | * Note that rcu_read_lock() is disallowed if the CPU is either idle or | |
121 | * offline from an RCU perspective, so check for those as well. | |
e3818b8d PM |
122 | */ |
123 | int rcu_read_lock_bh_held(void) | |
124 | { | |
125 | if (!debug_lockdep_rcu_enabled()) | |
126 | return 1; | |
e6b80a3b FW |
127 | if (rcu_is_cpu_idle()) |
128 | return 0; | |
c0d6d01b PM |
129 | if (!rcu_lockdep_current_cpu_online()) |
130 | return 0; | |
773e3f93 | 131 | return in_softirq() || irqs_disabled(); |
e3818b8d PM |
132 | } |
133 | EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); | |
134 | ||
135 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
136 | ||
2c42818e PM |
137 | struct rcu_synchronize { |
138 | struct rcu_head head; | |
139 | struct completion completion; | |
140 | }; | |
141 | ||
fbf6bfca PM |
142 | /* |
143 | * Awaken the corresponding synchronize_rcu() instance now that a | |
144 | * grace period has elapsed. | |
145 | */ | |
2c42818e | 146 | static void wakeme_after_rcu(struct rcu_head *head) |
21a1ea9e | 147 | { |
01c1c660 PM |
148 | struct rcu_synchronize *rcu; |
149 | ||
150 | rcu = container_of(head, struct rcu_synchronize, head); | |
151 | complete(&rcu->completion); | |
21a1ea9e | 152 | } |
ee84b824 | 153 | |
2c42818e PM |
154 | void wait_rcu_gp(call_rcu_func_t crf) |
155 | { | |
156 | struct rcu_synchronize rcu; | |
157 | ||
158 | init_rcu_head_on_stack(&rcu.head); | |
159 | init_completion(&rcu.completion); | |
160 | /* Will wake me after RCU finished. */ | |
161 | crf(&rcu.head, wakeme_after_rcu); | |
162 | /* Wait for it. */ | |
163 | wait_for_completion(&rcu.completion); | |
164 | destroy_rcu_head_on_stack(&rcu.head); | |
165 | } | |
166 | EXPORT_SYMBOL_GPL(wait_rcu_gp); | |
167 | ||
ee84b824 PM |
168 | #ifdef CONFIG_PROVE_RCU |
169 | /* | |
170 | * wrapper function to avoid #include problems. | |
171 | */ | |
172 | int rcu_my_thread_group_empty(void) | |
173 | { | |
174 | return thread_group_empty(current); | |
175 | } | |
176 | EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty); | |
177 | #endif /* #ifdef CONFIG_PROVE_RCU */ | |
551d55a9 MD |
178 | |
179 | #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD | |
180 | static inline void debug_init_rcu_head(struct rcu_head *head) | |
181 | { | |
182 | debug_object_init(head, &rcuhead_debug_descr); | |
183 | } | |
184 | ||
185 | static inline void debug_rcu_head_free(struct rcu_head *head) | |
186 | { | |
187 | debug_object_free(head, &rcuhead_debug_descr); | |
188 | } | |
189 | ||
190 | /* | |
191 | * fixup_init is called when: | |
192 | * - an active object is initialized | |
193 | */ | |
194 | static int rcuhead_fixup_init(void *addr, enum debug_obj_state state) | |
195 | { | |
196 | struct rcu_head *head = addr; | |
197 | ||
198 | switch (state) { | |
199 | case ODEBUG_STATE_ACTIVE: | |
200 | /* | |
201 | * Ensure that queued callbacks are all executed. | |
202 | * If we detect that we are nested in a RCU read-side critical | |
203 | * section, we should simply fail, otherwise we would deadlock. | |
fc2ecf7e MD |
204 | * In !PREEMPT configurations, there is no way to tell if we are |
205 | * in a RCU read-side critical section or not, so we never | |
206 | * attempt any fixup and just print a warning. | |
551d55a9 | 207 | */ |
fc2ecf7e | 208 | #ifndef CONFIG_PREEMPT |
108aae22 | 209 | WARN_ON_ONCE(1); |
fc2ecf7e MD |
210 | return 0; |
211 | #endif | |
551d55a9 MD |
212 | if (rcu_preempt_depth() != 0 || preempt_count() != 0 || |
213 | irqs_disabled()) { | |
108aae22 | 214 | WARN_ON_ONCE(1); |
551d55a9 MD |
215 | return 0; |
216 | } | |
217 | rcu_barrier(); | |
218 | rcu_barrier_sched(); | |
219 | rcu_barrier_bh(); | |
220 | debug_object_init(head, &rcuhead_debug_descr); | |
221 | return 1; | |
222 | default: | |
223 | return 0; | |
224 | } | |
225 | } | |
226 | ||
227 | /* | |
228 | * fixup_activate is called when: | |
229 | * - an active object is activated | |
230 | * - an unknown object is activated (might be a statically initialized object) | |
231 | * Activation is performed internally by call_rcu(). | |
232 | */ | |
233 | static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state) | |
234 | { | |
235 | struct rcu_head *head = addr; | |
236 | ||
237 | switch (state) { | |
238 | ||
239 | case ODEBUG_STATE_NOTAVAILABLE: | |
240 | /* | |
241 | * This is not really a fixup. We just make sure that it is | |
242 | * tracked in the object tracker. | |
243 | */ | |
244 | debug_object_init(head, &rcuhead_debug_descr); | |
245 | debug_object_activate(head, &rcuhead_debug_descr); | |
246 | return 0; | |
247 | ||
248 | case ODEBUG_STATE_ACTIVE: | |
249 | /* | |
250 | * Ensure that queued callbacks are all executed. | |
251 | * If we detect that we are nested in a RCU read-side critical | |
252 | * section, we should simply fail, otherwise we would deadlock. | |
fc2ecf7e MD |
253 | * In !PREEMPT configurations, there is no way to tell if we are |
254 | * in a RCU read-side critical section or not, so we never | |
255 | * attempt any fixup and just print a warning. | |
551d55a9 | 256 | */ |
fc2ecf7e | 257 | #ifndef CONFIG_PREEMPT |
108aae22 | 258 | WARN_ON_ONCE(1); |
fc2ecf7e MD |
259 | return 0; |
260 | #endif | |
551d55a9 MD |
261 | if (rcu_preempt_depth() != 0 || preempt_count() != 0 || |
262 | irqs_disabled()) { | |
108aae22 | 263 | WARN_ON_ONCE(1); |
551d55a9 MD |
264 | return 0; |
265 | } | |
266 | rcu_barrier(); | |
267 | rcu_barrier_sched(); | |
268 | rcu_barrier_bh(); | |
269 | debug_object_activate(head, &rcuhead_debug_descr); | |
270 | return 1; | |
271 | default: | |
272 | return 0; | |
273 | } | |
274 | } | |
275 | ||
276 | /* | |
277 | * fixup_free is called when: | |
278 | * - an active object is freed | |
279 | */ | |
280 | static int rcuhead_fixup_free(void *addr, enum debug_obj_state state) | |
281 | { | |
282 | struct rcu_head *head = addr; | |
283 | ||
284 | switch (state) { | |
285 | case ODEBUG_STATE_ACTIVE: | |
286 | /* | |
287 | * Ensure that queued callbacks are all executed. | |
288 | * If we detect that we are nested in a RCU read-side critical | |
289 | * section, we should simply fail, otherwise we would deadlock. | |
fc2ecf7e MD |
290 | * In !PREEMPT configurations, there is no way to tell if we are |
291 | * in a RCU read-side critical section or not, so we never | |
292 | * attempt any fixup and just print a warning. | |
551d55a9 | 293 | */ |
fc2ecf7e | 294 | #ifndef CONFIG_PREEMPT |
108aae22 | 295 | WARN_ON_ONCE(1); |
fc2ecf7e MD |
296 | return 0; |
297 | #endif | |
551d55a9 MD |
298 | if (rcu_preempt_depth() != 0 || preempt_count() != 0 || |
299 | irqs_disabled()) { | |
108aae22 | 300 | WARN_ON_ONCE(1); |
551d55a9 MD |
301 | return 0; |
302 | } | |
303 | rcu_barrier(); | |
304 | rcu_barrier_sched(); | |
305 | rcu_barrier_bh(); | |
306 | debug_object_free(head, &rcuhead_debug_descr); | |
307 | return 1; | |
551d55a9 MD |
308 | default: |
309 | return 0; | |
310 | } | |
311 | } | |
312 | ||
313 | /** | |
314 | * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects | |
315 | * @head: pointer to rcu_head structure to be initialized | |
316 | * | |
317 | * This function informs debugobjects of a new rcu_head structure that | |
318 | * has been allocated as an auto variable on the stack. This function | |
319 | * is not required for rcu_head structures that are statically defined or | |
320 | * that are dynamically allocated on the heap. This function has no | |
321 | * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds. | |
322 | */ | |
323 | void init_rcu_head_on_stack(struct rcu_head *head) | |
324 | { | |
325 | debug_object_init_on_stack(head, &rcuhead_debug_descr); | |
326 | } | |
327 | EXPORT_SYMBOL_GPL(init_rcu_head_on_stack); | |
328 | ||
329 | /** | |
330 | * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects | |
331 | * @head: pointer to rcu_head structure to be initialized | |
332 | * | |
333 | * This function informs debugobjects that an on-stack rcu_head structure | |
334 | * is about to go out of scope. As with init_rcu_head_on_stack(), this | |
335 | * function is not required for rcu_head structures that are statically | |
336 | * defined or that are dynamically allocated on the heap. Also as with | |
337 | * init_rcu_head_on_stack(), this function has no effect for | |
338 | * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds. | |
339 | */ | |
340 | void destroy_rcu_head_on_stack(struct rcu_head *head) | |
341 | { | |
342 | debug_object_free(head, &rcuhead_debug_descr); | |
343 | } | |
344 | EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack); | |
345 | ||
346 | struct debug_obj_descr rcuhead_debug_descr = { | |
347 | .name = "rcu_head", | |
348 | .fixup_init = rcuhead_fixup_init, | |
349 | .fixup_activate = rcuhead_fixup_activate, | |
350 | .fixup_free = rcuhead_fixup_free, | |
351 | }; | |
352 | EXPORT_SYMBOL_GPL(rcuhead_debug_descr); | |
353 | #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ | |
91afaf30 PM |
354 | |
355 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE) | |
356 | void do_trace_rcu_torture_read(char *rcutorturename, struct rcu_head *rhp) | |
357 | { | |
358 | trace_rcu_torture_read(rcutorturename, rhp); | |
359 | } | |
360 | EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read); | |
361 | #else | |
362 | #define do_trace_rcu_torture_read(rcutorturename, rhp) do { } while (0) | |
363 | #endif |