Commit | Line | Data |
---|---|---|
e7ee1501 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
d8be8173 PM |
2 | /* |
3 | * Sleepable Read-Copy Update mechanism for mutual exclusion, | |
4 | * tiny version for non-preemptible single-CPU use. | |
5 | * | |
d8be8173 PM |
6 | * Copyright (C) IBM Corporation, 2017 |
7 | * | |
e7ee1501 | 8 | * Author: Paul McKenney <paulmck@linux.ibm.com> |
d8be8173 PM |
9 | */ |
10 | ||
11 | #include <linux/export.h> | |
12 | #include <linux/mutex.h> | |
13 | #include <linux/preempt.h> | |
14 | #include <linux/rcupdate_wait.h> | |
15 | #include <linux/sched.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/srcu.h> | |
18 | ||
19 | #include <linux/rcu_node_tree.h> | |
45753c5f | 20 | #include "rcu_segcblist.h" |
d8be8173 PM |
21 | #include "rcu.h" |
22 | ||
825c5bd2 | 23 | int rcu_scheduler_active __read_mostly; |
e0fcba9a PM |
24 | static LIST_HEAD(srcu_boot_list); |
25 | static bool srcu_init_done; | |
825c5bd2 | 26 | |
aacb5d91 | 27 | static int init_srcu_struct_fields(struct srcu_struct *ssp) |
d8be8173 | 28 | { |
aacb5d91 PM |
29 | ssp->srcu_lock_nesting[0] = 0; |
30 | ssp->srcu_lock_nesting[1] = 0; | |
31 | init_swait_queue_head(&ssp->srcu_wq); | |
32 | ssp->srcu_cb_head = NULL; | |
33 | ssp->srcu_cb_tail = &ssp->srcu_cb_head; | |
34 | ssp->srcu_gp_running = false; | |
35 | ssp->srcu_gp_waiting = false; | |
36 | ssp->srcu_idx = 0; | |
37 | INIT_WORK(&ssp->srcu_work, srcu_drive_gp); | |
38 | INIT_LIST_HEAD(&ssp->srcu_work.entry); | |
d8be8173 PM |
39 | return 0; |
40 | } | |
41 | ||
42 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
43 | ||
aacb5d91 | 44 | int __init_srcu_struct(struct srcu_struct *ssp, const char *name, |
d8be8173 PM |
45 | struct lock_class_key *key) |
46 | { | |
47 | /* Don't re-initialize a lock while it is held. */ | |
aacb5d91 PM |
48 | debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); |
49 | lockdep_init_map(&ssp->dep_map, name, key, 0); | |
50 | return init_srcu_struct_fields(ssp); | |
d8be8173 PM |
51 | } |
52 | EXPORT_SYMBOL_GPL(__init_srcu_struct); | |
53 | ||
54 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
55 | ||
56 | /* | |
57 | * init_srcu_struct - initialize a sleep-RCU structure | |
aacb5d91 | 58 | * @ssp: structure to initialize. |
d8be8173 PM |
59 | * |
60 | * Must invoke this on a given srcu_struct before passing that srcu_struct | |
61 | * to any other function. Each srcu_struct represents a separate domain | |
62 | * of SRCU protection. | |
63 | */ | |
aacb5d91 | 64 | int init_srcu_struct(struct srcu_struct *ssp) |
d8be8173 | 65 | { |
aacb5d91 | 66 | return init_srcu_struct_fields(ssp); |
d8be8173 PM |
67 | } |
68 | EXPORT_SYMBOL_GPL(init_srcu_struct); | |
69 | ||
70 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
71 | ||
72 | /* | |
73 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | |
aacb5d91 | 74 | * @ssp: structure to clean up. |
d8be8173 PM |
75 | * |
76 | * Must invoke this after you are finished using a given srcu_struct that | |
77 | * was initialized via init_srcu_struct(), else you leak memory. | |
78 | */ | |
f5ad3991 | 79 | void cleanup_srcu_struct(struct srcu_struct *ssp) |
d8be8173 | 80 | { |
aacb5d91 | 81 | WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); |
f5ad3991 | 82 | flush_work(&ssp->srcu_work); |
aacb5d91 PM |
83 | WARN_ON(ssp->srcu_gp_running); |
84 | WARN_ON(ssp->srcu_gp_waiting); | |
85 | WARN_ON(ssp->srcu_cb_head); | |
86 | WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); | |
d8be8173 | 87 | } |
f5ad3991 | 88 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); |
d8be8173 | 89 | |
d8be8173 PM |
90 | /* |
91 | * Removes the count for the old reader from the appropriate element of | |
cdf7abc4 | 92 | * the srcu_struct. |
d8be8173 | 93 | */ |
aacb5d91 | 94 | void __srcu_read_unlock(struct srcu_struct *ssp, int idx) |
d8be8173 | 95 | { |
aacb5d91 | 96 | int newval = ssp->srcu_lock_nesting[idx] - 1; |
d8be8173 | 97 | |
aacb5d91 PM |
98 | WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval); |
99 | if (!newval && READ_ONCE(ssp->srcu_gp_waiting)) | |
100 | swake_up_one(&ssp->srcu_wq); | |
d8be8173 PM |
101 | } |
102 | EXPORT_SYMBOL_GPL(__srcu_read_unlock); | |
103 | ||
104 | /* | |
105 | * Workqueue handler to drive one grace period and invoke any callbacks | |
106 | * that become ready as a result. Single-CPU and !PREEMPT operation | |
107 | * means that we get away with murder on synchronization. ;-) | |
108 | */ | |
109 | void srcu_drive_gp(struct work_struct *wp) | |
110 | { | |
111 | int idx; | |
2464dd94 | 112 | struct rcu_head *lh; |
d8be8173 | 113 | struct rcu_head *rhp; |
aacb5d91 | 114 | struct srcu_struct *ssp; |
d8be8173 | 115 | |
aacb5d91 PM |
116 | ssp = container_of(wp, struct srcu_struct, srcu_work); |
117 | if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head)) | |
d8be8173 PM |
118 | return; /* Already running or nothing to do. */ |
119 | ||
2464dd94 | 120 | /* Remove recently arrived callbacks and wait for readers. */ |
aacb5d91 | 121 | WRITE_ONCE(ssp->srcu_gp_running, true); |
2464dd94 | 122 | local_irq_disable(); |
aacb5d91 PM |
123 | lh = ssp->srcu_cb_head; |
124 | ssp->srcu_cb_head = NULL; | |
125 | ssp->srcu_cb_tail = &ssp->srcu_cb_head; | |
2464dd94 | 126 | local_irq_enable(); |
aacb5d91 PM |
127 | idx = ssp->srcu_idx; |
128 | WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx); | |
129 | WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ | |
130 | swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx])); | |
131 | WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ | |
2464dd94 PM |
132 | |
133 | /* Invoke the callbacks we removed above. */ | |
134 | while (lh) { | |
135 | rhp = lh; | |
136 | lh = lh->next; | |
137 | local_bh_disable(); | |
138 | rhp->func(rhp); | |
139 | local_bh_enable(); | |
d8be8173 | 140 | } |
d8be8173 PM |
141 | |
142 | /* | |
2464dd94 PM |
143 | * Enable rescheduling, and if there are more callbacks, |
144 | * reschedule ourselves. This can race with a call_srcu() | |
145 | * at interrupt level, but the ->srcu_gp_running checks will | |
146 | * straighten that out. | |
d8be8173 | 147 | */ |
aacb5d91 PM |
148 | WRITE_ONCE(ssp->srcu_gp_running, false); |
149 | if (READ_ONCE(ssp->srcu_cb_head)) | |
150 | schedule_work(&ssp->srcu_work); | |
d8be8173 PM |
151 | } |
152 | EXPORT_SYMBOL_GPL(srcu_drive_gp); | |
153 | ||
154 | /* | |
155 | * Enqueue an SRCU callback on the specified srcu_struct structure, | |
156 | * initiating grace-period processing if it is not already running. | |
157 | */ | |
aacb5d91 | 158 | void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, |
d8be8173 PM |
159 | rcu_callback_t func) |
160 | { | |
161 | unsigned long flags; | |
162 | ||
2464dd94 PM |
163 | rhp->func = func; |
164 | rhp->next = NULL; | |
d8be8173 | 165 | local_irq_save(flags); |
aacb5d91 PM |
166 | *ssp->srcu_cb_tail = rhp; |
167 | ssp->srcu_cb_tail = &rhp->next; | |
d8be8173 | 168 | local_irq_restore(flags); |
aacb5d91 | 169 | if (!READ_ONCE(ssp->srcu_gp_running)) { |
e0fcba9a | 170 | if (likely(srcu_init_done)) |
aacb5d91 PM |
171 | schedule_work(&ssp->srcu_work); |
172 | else if (list_empty(&ssp->srcu_work.entry)) | |
173 | list_add(&ssp->srcu_work.entry, &srcu_boot_list); | |
e0fcba9a | 174 | } |
d8be8173 PM |
175 | } |
176 | EXPORT_SYMBOL_GPL(call_srcu); | |
177 | ||
178 | /* | |
179 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | |
180 | */ | |
aacb5d91 | 181 | void synchronize_srcu(struct srcu_struct *ssp) |
d8be8173 PM |
182 | { |
183 | struct rcu_synchronize rs; | |
184 | ||
185 | init_rcu_head_on_stack(&rs.head); | |
186 | init_completion(&rs.completion); | |
aacb5d91 | 187 | call_srcu(ssp, &rs.head, wakeme_after_rcu); |
d8be8173 PM |
188 | wait_for_completion(&rs.completion); |
189 | destroy_rcu_head_on_stack(&rs.head); | |
190 | } | |
191 | EXPORT_SYMBOL_GPL(synchronize_srcu); | |
825c5bd2 PM |
192 | |
193 | /* Lockdep diagnostics. */ | |
194 | void __init rcu_scheduler_starting(void) | |
195 | { | |
196 | rcu_scheduler_active = RCU_SCHEDULER_RUNNING; | |
197 | } | |
e0fcba9a PM |
198 | |
199 | /* | |
200 | * Queue work for srcu_struct structures with early boot callbacks. | |
201 | * The work won't actually execute until the workqueue initialization | |
202 | * phase that takes place after the scheduler starts. | |
203 | */ | |
204 | void __init srcu_init(void) | |
205 | { | |
aacb5d91 | 206 | struct srcu_struct *ssp; |
e0fcba9a PM |
207 | |
208 | srcu_init_done = true; | |
209 | while (!list_empty(&srcu_boot_list)) { | |
aacb5d91 | 210 | ssp = list_first_entry(&srcu_boot_list, |
4e6ea4ef | 211 | struct srcu_struct, srcu_work.entry); |
aacb5d91 PM |
212 | list_del_init(&ssp->srcu_work.entry); |
213 | schedule_work(&ssp->srcu_work); | |
e0fcba9a PM |
214 | } |
215 | } |