Commit | Line | Data |
---|---|---|
00de9d74 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
9b1d82fa PM |
2 | /* |
3 | * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. | |
4 | * | |
9b1d82fa PM |
5 | * Copyright IBM Corporation, 2008 |
6 | * | |
00de9d74 | 7 | * Author: Paul E. McKenney <paulmck@linux.ibm.com> |
9b1d82fa PM |
8 | * |
9 | * For detailed explanation of Read-Copy Update mechanism see - | |
4ce5b903 | 10 | * Documentation/RCU |
9b1d82fa | 11 | */ |
4ce5b903 IM |
12 | #include <linux/completion.h> |
13 | #include <linux/interrupt.h> | |
9b1d82fa | 14 | #include <linux/notifier.h> |
f9411ebe | 15 | #include <linux/rcupdate_wait.h> |
4ce5b903 | 16 | #include <linux/kernel.h> |
9984de1a | 17 | #include <linux/export.h> |
9b1d82fa | 18 | #include <linux/mutex.h> |
4ce5b903 IM |
19 | #include <linux/sched.h> |
20 | #include <linux/types.h> | |
21 | #include <linux/init.h> | |
9b1d82fa | 22 | #include <linux/time.h> |
4ce5b903 | 23 | #include <linux/cpu.h> |
268bb0ce | 24 | #include <linux/prefetch.h> |
77a40f97 | 25 | #include <linux/slab.h> |
64d1d06c | 26 | #include <linux/mm.h> |
9b1d82fa | 27 | |
29c00b4a PM |
28 | #include "rcu.h" |
29 | ||
6d48152e PM |
30 | /* Global control variables for rcupdate callback mechanism. */ |
31 | struct rcu_ctrlblk { | |
32 | struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ | |
33 | struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ | |
34 | struct rcu_head **curtail; /* ->next pointer of last CB. */ | |
0909fc2b | 35 | unsigned long gp_seq; /* Grace-period counter. */ |
6d48152e PM |
36 | }; |
37 | ||
38 | /* Definition for rcupdate control block. */ | |
709fdce7 PM |
39 | static struct rcu_ctrlblk rcu_ctrlblk = { |
40 | .donetail = &rcu_ctrlblk.rcucblist, | |
41 | .curtail = &rcu_ctrlblk.rcucblist, | |
0909fc2b | 42 | .gp_seq = 0 - 300UL, |
6d48152e PM |
43 | }; |
44 | ||
709fdce7 | 45 | void rcu_barrier(void) |
9b1d82fa | 46 | { |
3cb278e7 | 47 | wait_rcu_gp(call_rcu_hurry); |
9b1d82fa | 48 | } |
709fdce7 | 49 | EXPORT_SYMBOL(rcu_barrier); |
9b1d82fa | 50 | |
65cfe358 | 51 | /* Record an rcu quiescent state. */ |
709fdce7 | 52 | void rcu_qs(void) |
9b1d82fa | 53 | { |
b554d7de ED |
54 | unsigned long flags; |
55 | ||
56 | local_irq_save(flags); | |
709fdce7 PM |
57 | if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { |
58 | rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; | |
18d7e406 | 59 | raise_softirq_irqoff(RCU_SOFTIRQ); |
65cfe358 | 60 | } |
414c1238 | 61 | WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); |
b554d7de | 62 | local_irq_restore(flags); |
9b1d82fa PM |
63 | } |
64 | ||
65 | /* | |
66 | * Check to see if the scheduling-clock interrupt came from an extended | |
9b2e4f18 PM |
67 | * quiescent state, and, if so, tell RCU about it. This function must |
68 | * be called from hardirq context. It is normally called from the | |
69 | * scheduling-clock interrupt. | |
9b1d82fa | 70 | */ |
c98cac60 | 71 | void rcu_sched_clock_irq(int user) |
9b1d82fa | 72 | { |
c5bacd94 | 73 | if (user) { |
709fdce7 | 74 | rcu_qs(); |
c5bacd94 PM |
75 | } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { |
76 | set_tsk_need_resched(current); | |
77 | set_preempt_need_resched(); | |
78 | } | |
9b1d82fa PM |
79 | } |
80 | ||
77a40f97 JFG |
81 | /* |
82 | * Reclaim the specified callback, either by invoking it for non-kfree cases or | |
83 | * freeing it directly (for kfree). Return true if kfreeing, false otherwise. | |
84 | */ | |
85 | static inline bool rcu_reclaim_tiny(struct rcu_head *head) | |
86 | { | |
87 | rcu_callback_t f; | |
88 | unsigned long offset = (unsigned long)head->func; | |
89 | ||
90 | rcu_lock_acquire(&rcu_callback_map); | |
c408b215 URS |
91 | if (__is_kvfree_rcu_offset(offset)) { |
92 | trace_rcu_invoke_kvfree_callback("", head, offset); | |
64d1d06c | 93 | kvfree((void *)head - offset); |
77a40f97 JFG |
94 | rcu_lock_release(&rcu_callback_map); |
95 | return true; | |
96 | } | |
97 | ||
98 | trace_rcu_invoke_callback("", head); | |
99 | f = head->func; | |
2cbc482d | 100 | debug_rcu_head_callback(head); |
77a40f97 JFG |
101 | WRITE_ONCE(head->func, (rcu_callback_t)0L); |
102 | f(head); | |
103 | rcu_lock_release(&rcu_callback_map); | |
104 | return false; | |
105 | } | |
106 | ||
65cfe358 PM |
107 | /* Invoke the RCU callbacks whose grace period has elapsed. */ |
108 | static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) | |
9b1d82fa | 109 | { |
9b1d82fa | 110 | struct rcu_head *next, *list; |
4ce5b903 | 111 | unsigned long flags; |
9b1d82fa | 112 | |
9b1d82fa PM |
113 | /* Move the ready-to-invoke callbacks to a local list. */ |
114 | local_irq_save(flags); | |
709fdce7 | 115 | if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) { |
6e91f8cb PM |
116 | /* No callbacks ready, so just leave. */ |
117 | local_irq_restore(flags); | |
118 | return; | |
119 | } | |
709fdce7 PM |
120 | list = rcu_ctrlblk.rcucblist; |
121 | rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail; | |
122 | *rcu_ctrlblk.donetail = NULL; | |
123 | if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail) | |
124 | rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist; | |
125 | rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist; | |
9b1d82fa PM |
126 | local_irq_restore(flags); |
127 | ||
128 | /* Invoke the callbacks on the local list. */ | |
129 | while (list) { | |
130 | next = list->next; | |
131 | prefetch(next); | |
551d55a9 | 132 | debug_rcu_head_unqueue(list); |
b2c0710c | 133 | local_bh_disable(); |
77a40f97 | 134 | rcu_reclaim_tiny(list); |
b2c0710c | 135 | local_bh_enable(); |
9b1d82fa PM |
136 | list = next; |
137 | } | |
138 | } | |
139 | ||
9b1d82fa PM |
140 | /* |
141 | * Wait for a grace period to elapse. But it is illegal to invoke | |
679d3f30 | 142 | * synchronize_rcu() from within an RCU read-side critical section. |
7f453536 PM |
143 | * Therefore, any legal call to synchronize_rcu() is a quiescent state, |
144 | * and so on a UP system, synchronize_rcu() need do nothing, other than | |
145 | * let the polled APIs know that another grace period elapsed. | |
146 | * | |
65cfe358 PM |
147 | * (But Lai Jiangshan points out the benefits of doing might_sleep() |
148 | * to reduce latency.) | |
9b1d82fa PM |
149 | * |
150 | * Cool, huh? (Due to Josh Triplett.) | |
9b1d82fa | 151 | */ |
709fdce7 | 152 | void synchronize_rcu(void) |
9b1d82fa | 153 | { |
f78f5b90 PM |
154 | RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || |
155 | lock_is_held(&rcu_lock_map) || | |
156 | lock_is_held(&rcu_sched_lock_map), | |
679d3f30 | 157 | "Illegal synchronize_rcu() in RCU read-side critical section"); |
7f453536 | 158 | WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); |
9b1d82fa | 159 | } |
709fdce7 | 160 | EXPORT_SYMBOL_GPL(synchronize_rcu); |
9b1d82fa | 161 | |
6ca0292c Z |
162 | static void tiny_rcu_leak_callback(struct rcu_head *rhp) |
163 | { | |
164 | } | |
165 | ||
9b1d82fa | 166 | /* |
679d3f30 | 167 | * Post an RCU callback to be invoked after the end of an RCU grace |
65cfe358 PM |
168 | * period. But since we have but one CPU, that would be after any |
169 | * quiescent state. | |
9b1d82fa | 170 | */ |
709fdce7 | 171 | void call_rcu(struct rcu_head *head, rcu_callback_t func) |
9b1d82fa | 172 | { |
6ca0292c | 173 | static atomic_t doublefrees; |
9b1d82fa PM |
174 | unsigned long flags; |
175 | ||
6ca0292c Z |
176 | if (debug_rcu_head_queue(head)) { |
177 | if (atomic_inc_return(&doublefrees) < 4) { | |
178 | pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); | |
179 | mem_dump_obj(head); | |
180 | } | |
181 | ||
182 | if (!__is_kvfree_rcu_offset((unsigned long)head->func)) | |
183 | WRITE_ONCE(head->func, tiny_rcu_leak_callback); | |
184 | return; | |
185 | } | |
186 | ||
9b1d82fa PM |
187 | head->func = func; |
188 | head->next = NULL; | |
4ce5b903 | 189 | |
9b1d82fa | 190 | local_irq_save(flags); |
709fdce7 PM |
191 | *rcu_ctrlblk.curtail = head; |
192 | rcu_ctrlblk.curtail = &head->next; | |
9b1d82fa | 193 | local_irq_restore(flags); |
5f6130fa LJ |
194 | |
195 | if (unlikely(is_idle_task(current))) { | |
709fdce7 | 196 | /* force scheduling for rcu_qs() */ |
5f6130fa LJ |
197 | resched_cpu(0); |
198 | } | |
9b1d82fa | 199 | } |
709fdce7 | 200 | EXPORT_SYMBOL_GPL(call_rcu); |
9dc5ad32 | 201 | |
91a967fd PM |
202 | /* |
203 | * Store a grace-period-counter "cookie". For more information, | |
204 | * see the Tree RCU header comment. | |
205 | */ | |
206 | void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) | |
207 | { | |
208 | rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; | |
209 | } | |
210 | EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); | |
211 | ||
0909fc2b PM |
212 | /* |
213 | * Return a grace-period-counter "cookie". For more information, | |
214 | * see the Tree RCU header comment. | |
215 | */ | |
216 | unsigned long get_state_synchronize_rcu(void) | |
217 | { | |
218 | return READ_ONCE(rcu_ctrlblk.gp_seq); | |
219 | } | |
220 | EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); | |
221 | ||
222 | /* | |
223 | * Return a grace-period-counter "cookie" and ensure that a future grace | |
224 | * period completes. For more information, see the Tree RCU header comment. | |
225 | */ | |
226 | unsigned long start_poll_synchronize_rcu(void) | |
227 | { | |
228 | unsigned long gp_seq = get_state_synchronize_rcu(); | |
229 | ||
230 | if (unlikely(is_idle_task(current))) { | |
231 | /* force scheduling for rcu_qs() */ | |
232 | resched_cpu(0); | |
233 | } | |
234 | return gp_seq; | |
235 | } | |
236 | EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); | |
237 | ||
238 | /* | |
239 | * Return true if the grace period corresponding to oldstate has completed | |
240 | * and false otherwise. For more information, see the Tree RCU header | |
241 | * comment. | |
242 | */ | |
243 | bool poll_state_synchronize_rcu(unsigned long oldstate) | |
244 | { | |
414c1238 | 245 | return oldstate == RCU_GET_STATE_COMPLETED || READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; |
0909fc2b PM |
246 | } |
247 | EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); | |
248 | ||
800d6acf | 249 | #ifdef CONFIG_KASAN_GENERIC |
04a522b7 | 250 | void kvfree_call_rcu(struct rcu_head *head, void *ptr) |
800d6acf | 251 | { |
04a522b7 | 252 | if (head) |
800d6acf | 253 | kasan_record_aux_stack_noalloc(ptr); |
800d6acf | 254 | |
04a522b7 | 255 | __kvfree_call_rcu(head, ptr); |
800d6acf JB |
256 | } |
257 | EXPORT_SYMBOL_GPL(kvfree_call_rcu); | |
258 | #endif | |
259 | ||
aa23c6fb | 260 | void __init rcu_init(void) |
9dc5ad32 PM |
261 | { |
262 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | |
aa23c6fb | 263 | rcu_early_boot_tests(); |
9dc5ad32 | 264 | } |