Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | #ifndef __LINUX_SEQLOCK_H |
3 | #define __LINUX_SEQLOCK_H | |
0d24f65e | 4 | |
1da177e4 | 5 | /* |
0d24f65e AD |
6 | * seqcount_t / seqlock_t - a reader-writer consistency mechanism with |
7 | * lockless readers (read-only retry loops), and no writer starvation. | |
8 | * | |
9 | * See Documentation/locking/seqlock.rst | |
10 | * | |
11 | * Copyrights: | |
12 | * - Based on x86_64 vsyscall gettimeofday: Keith Owens, Andrea Arcangeli | |
55f3560d | 13 | * - Sequence counters with associated locks, (C) 2020 Linutronix GmbH |
1da177e4 LT |
14 | */ |
15 | ||
7fc26327 | 16 | #include <linux/compiler.h> |
b968a08f | 17 | #include <linux/kcsan-checks.h> |
55f3560d AD |
18 | #include <linux/lockdep.h> |
19 | #include <linux/mutex.h> | |
20 | #include <linux/preempt.h> | |
21 | #include <linux/spinlock.h> | |
55f3560d | 22 | |
56a21052 | 23 | #include <asm/processor.h> |
1da177e4 | 24 | |
88ecd153 | 25 | /* |
0d24f65e AD |
26 | * The seqlock seqcount_t interface does not prescribe a precise sequence of |
27 | * read begin/retry/end. For readers, typically there is a call to | |
88ecd153 ME |
28 | * read_seqcount_begin() and read_seqcount_retry(), however, there are more |
29 | * esoteric cases which do not follow this pattern. | |
30 | * | |
31 | * As a consequence, we take the following best-effort approach for raw usage | |
32 | * via seqcount_t under KCSAN: upon beginning a seq-reader critical section, | |
5cbaefe9 | 33 | * pessimistically mark the next KCSAN_SEQLOCK_REGION_MAX memory accesses as |
88ecd153 | 34 | * atomics; if there is a matching read_seqcount_retry() call, no following |
0d24f65e AD |
35 | * memory operations are considered atomic. Usage of the seqlock_t interface |
36 | * is not affected. | |
88ecd153 ME |
37 | */ |
38 | #define KCSAN_SEQLOCK_REGION_MAX 1000 | |
39 | ||
1da177e4 | 40 | /* |
0d24f65e AD |
41 | * Sequence counters (seqcount_t) |
42 | * | |
43 | * This is the raw counting mechanism, without any writer protection. | |
44 | * | |
45 | * Write side critical sections must be serialized and non-preemptible. | |
46 | * | |
47 | * If readers can be invoked from hardirq or softirq contexts, | |
48 | * interrupts or bottom halves must also be respectively disabled before | |
49 | * entering the write section. | |
50 | * | |
51 | * This mechanism can't be used if the protected data contains pointers, | |
52 | * as the writer can invalidate a pointer that a reader is following. | |
53 | * | |
55f3560d AD |
54 | * If the write serialization mechanism is one of the common kernel |
55 | * locking primitives, use a sequence counter with associated lock | |
6dd699b1 | 56 | * (seqcount_LOCKNAME_t) instead. |
55f3560d | 57 | * |
0d24f65e AD |
58 | * If it's desired to automatically handle the sequence counter writer |
59 | * serialization and non-preemptibility requirements, use a sequential | |
60 | * lock (seqlock_t) instead. | |
61 | * | |
62 | * See Documentation/locking/seqlock.rst | |
1da177e4 | 63 | */ |
1da177e4 LT |
64 | typedef struct seqcount { |
65 | unsigned sequence; | |
1ca7d67c JS |
66 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
67 | struct lockdep_map dep_map; | |
68 | #endif | |
1da177e4 LT |
69 | } seqcount_t; |
70 | ||
1ca7d67c JS |
71 | static inline void __seqcount_init(seqcount_t *s, const char *name, |
72 | struct lock_class_key *key) | |
73 | { | |
74 | /* | |
75 | * Make sure we are not reinitializing a held lock: | |
76 | */ | |
77 | lockdep_init_map(&s->dep_map, name, key, 0); | |
78 | s->sequence = 0; | |
79 | } | |
80 | ||
81 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
ec8702da AD |
82 | |
83 | # define SEQCOUNT_DEP_MAP_INIT(lockname) \ | |
84 | .dep_map = { .name = #lockname } | |
1ca7d67c | 85 | |
89b88845 AD |
86 | /** |
87 | * seqcount_init() - runtime initializer for seqcount_t | |
88 | * @s: Pointer to the seqcount_t instance | |
89 | */ | |
ec8702da AD |
90 | # define seqcount_init(s) \ |
91 | do { \ | |
92 | static struct lock_class_key __key; \ | |
93 | __seqcount_init((s), #s, &__key); \ | |
1ca7d67c JS |
94 | } while (0) |
95 | ||
96 | static inline void seqcount_lockdep_reader_access(const seqcount_t *s) | |
97 | { | |
98 | seqcount_t *l = (seqcount_t *)s; | |
99 | unsigned long flags; | |
100 | ||
101 | local_irq_save(flags); | |
102 | seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_); | |
5facae4f | 103 | seqcount_release(&l->dep_map, _RET_IP_); |
1ca7d67c JS |
104 | local_irq_restore(flags); |
105 | } | |
106 | ||
107 | #else | |
108 | # define SEQCOUNT_DEP_MAP_INIT(lockname) | |
109 | # define seqcount_init(s) __seqcount_init(s, NULL, NULL) | |
110 | # define seqcount_lockdep_reader_access(x) | |
111 | #endif | |
112 | ||
89b88845 AD |
113 | /** |
114 | * SEQCNT_ZERO() - static initializer for seqcount_t | |
115 | * @name: Name of the seqcount_t instance | |
116 | */ | |
117 | #define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) } | |
1da177e4 | 118 | |
55f3560d | 119 | /* |
6dd699b1 | 120 | * Sequence counters with associated locks (seqcount_LOCKNAME_t) |
55f3560d AD |
121 | * |
122 | * A sequence counter which associates the lock used for writer | |
123 | * serialization at initialization time. This enables lockdep to validate | |
124 | * that the write side critical section is properly serialized. | |
125 | * | |
126 | * For associated locks which do not implicitly disable preemption, | |
127 | * preemption protection is enforced in the write side function. | |
128 | * | |
129 | * Lockdep is never used in any for the raw write variants. | |
130 | * | |
131 | * See Documentation/locking/seqlock.rst | |
132 | */ | |
133 | ||
8117ab50 AD |
134 | /* |
135 | * For PREEMPT_RT, seqcount_LOCKNAME_t write side critical sections cannot | |
136 | * disable preemption. It can lead to higher latencies, and the write side | |
137 | * sections will not be able to acquire locks which become sleeping locks | |
138 | * (e.g. spinlock_t). | |
139 | * | |
140 | * To remain preemptible while avoiding a possible livelock caused by the | |
141 | * reader preempting the writer, use a different technique: let the reader | |
142 | * detect if a seqcount_LOCKNAME_t writer is in progress. If that is the | |
143 | * case, acquire then release the associated LOCKNAME writer serialization | |
144 | * lock. This will allow any possibly-preempted writer to make progress | |
145 | * until the end of its writer serialization lock critical section. | |
146 | * | |
147 | * This lock-unlock technique must be implemented for all of PREEMPT_RT | |
148 | * sleeping locks. See Documentation/locking/locktypes.rst | |
149 | */ | |
150 | #if defined(CONFIG_LOCKDEP) || defined(CONFIG_PREEMPT_RT) | |
e55687fe | 151 | #define __SEQ_LOCK(expr) expr |
55f3560d | 152 | #else |
e55687fe | 153 | #define __SEQ_LOCK(expr) |
55f3560d AD |
154 | #endif |
155 | ||
ed3e4537 | 156 | /* |
6dd699b1 | 157 | * typedef seqcount_LOCKNAME_t - sequence counter with LOCKNAME associated |
a8772dcc | 158 | * @seqcount: The real sequence counter |
6dd699b1 | 159 | * @lock: Pointer to the associated lock |
a8772dcc | 160 | * |
6dd699b1 AD |
161 | * A plain sequence counter with external writer synchronization by |
162 | * LOCKNAME @lock. The lock is associated to the sequence counter in the | |
a8772dcc PZ |
163 | * static initializer or init function. This enables lockdep to validate |
164 | * that the write side critical section is properly serialized. | |
6dd699b1 | 165 | * |
e84815cb | 166 | * LOCKNAME: raw_spinlock, spinlock, rwlock or mutex |
a8772dcc PZ |
167 | */ |
168 | ||
a28e884b | 169 | /* |
e4e9ab3f PZ |
170 | * seqcount_LOCKNAME_init() - runtime initializer for seqcount_LOCKNAME_t |
171 | * @s: Pointer to the seqcount_LOCKNAME_t instance | |
6dd699b1 | 172 | * @lock: Pointer to the associated lock |
e4e9ab3f PZ |
173 | */ |
174 | ||
267580db | 175 | #define seqcount_LOCKNAME_init(s, _lock, lockname) \ |
176 | do { \ | |
177 | seqcount_##lockname##_t *____s = (s); \ | |
178 | seqcount_init(&____s->seqcount); \ | |
179 | __SEQ_LOCK(____s->lock = (_lock)); \ | |
180 | } while (0) | |
181 | ||
182 | #define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock) | |
183 | #define seqcount_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, spinlock) | |
149876d9 HD |
184 | #define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock) |
185 | #define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex) | |
267580db | 186 | |
55f3560d | 187 | /* |
5cdd2557 AD |
188 | * SEQCOUNT_LOCKNAME() - Instantiate seqcount_LOCKNAME_t and helpers |
189 | * seqprop_LOCKNAME_*() - Property accessors for seqcount_LOCKNAME_t | |
190 | * | |
6dd699b1 AD |
191 | * @lockname: "LOCKNAME" part of seqcount_LOCKNAME_t |
192 | * @locktype: LOCKNAME canonical C data type | |
8117ab50 | 193 | * @preemptible: preemptibility of above locktype |
a8772dcc | 194 | * @lockmember: argument for lockdep_assert_held() |
8117ab50 AD |
195 | * @lockbase: associated lock release function (prefix only) |
196 | * @lock_acquire: associated lock acquisition function (full call) | |
55f3560d | 197 | */ |
8117ab50 | 198 | #define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember, lockbase, lock_acquire) \ |
a8772dcc PZ |
199 | typedef struct seqcount_##lockname { \ |
200 | seqcount_t seqcount; \ | |
201 | __SEQ_LOCK(locktype *lock); \ | |
202 | } seqcount_##lockname##_t; \ | |
203 | \ | |
204 | static __always_inline seqcount_t * \ | |
5cdd2557 | 205 | __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s) \ |
55f3560d AD |
206 | { \ |
207 | return &s->seqcount; \ | |
208 | } \ | |
209 | \ | |
52ac39e5 AD |
210 | static __always_inline unsigned \ |
211 | __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s) \ | |
212 | { \ | |
8117ab50 AD |
213 | unsigned seq = READ_ONCE(s->seqcount.sequence); \ |
214 | \ | |
215 | if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \ | |
216 | return seq; \ | |
217 | \ | |
218 | if (preemptible && unlikely(seq & 1)) { \ | |
219 | __SEQ_LOCK(lock_acquire); \ | |
220 | __SEQ_LOCK(lockbase##_unlock(s->lock)); \ | |
221 | \ | |
222 | /* \ | |
223 | * Re-read the sequence counter since the (possibly \ | |
224 | * preempted) writer made progress. \ | |
225 | */ \ | |
226 | seq = READ_ONCE(s->seqcount.sequence); \ | |
227 | } \ | |
228 | \ | |
229 | return seq; \ | |
52ac39e5 AD |
230 | } \ |
231 | \ | |
a8772dcc | 232 | static __always_inline bool \ |
5cdd2557 | 233 | __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s) \ |
55f3560d | 234 | { \ |
8117ab50 AD |
235 | if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \ |
236 | return preemptible; \ | |
237 | \ | |
238 | /* PREEMPT_RT relies on the above LOCK+UNLOCK */ \ | |
239 | return false; \ | |
55f3560d AD |
240 | } \ |
241 | \ | |
a8772dcc | 242 | static __always_inline void \ |
5cdd2557 | 243 | __seqprop_##lockname##_assert(const seqcount_##lockname##_t *s) \ |
55f3560d | 244 | { \ |
e55687fe | 245 | __SEQ_LOCK(lockdep_assert_held(lockmember)); \ |
55f3560d AD |
246 | } |
247 | ||
248 | /* | |
a8772dcc | 249 | * __seqprop() for seqcount_t |
55f3560d AD |
250 | */ |
251 | ||
5cdd2557 | 252 | static inline seqcount_t *__seqprop_ptr(seqcount_t *s) |
55f3560d AD |
253 | { |
254 | return s; | |
255 | } | |
256 | ||
52ac39e5 AD |
257 | static inline unsigned __seqprop_sequence(const seqcount_t *s) |
258 | { | |
259 | return READ_ONCE(s->sequence); | |
260 | } | |
261 | ||
5cdd2557 | 262 | static inline bool __seqprop_preemptible(const seqcount_t *s) |
55f3560d AD |
263 | { |
264 | return false; | |
265 | } | |
266 | ||
5cdd2557 | 267 | static inline void __seqprop_assert(const seqcount_t *s) |
55f3560d AD |
268 | { |
269 | lockdep_assert_preemption_disabled(); | |
270 | } | |
271 | ||
8117ab50 AD |
272 | #define __SEQ_RT IS_ENABLED(CONFIG_PREEMPT_RT) |
273 | ||
274 | SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t, false, s->lock, raw_spin, raw_spin_lock(s->lock)) | |
275 | SEQCOUNT_LOCKNAME(spinlock, spinlock_t, __SEQ_RT, s->lock, spin, spin_lock(s->lock)) | |
276 | SEQCOUNT_LOCKNAME(rwlock, rwlock_t, __SEQ_RT, s->lock, read, read_lock(s->lock)) | |
277 | SEQCOUNT_LOCKNAME(mutex, struct mutex, true, s->lock, mutex, mutex_lock(s->lock)) | |
a8772dcc | 278 | |
a28e884b | 279 | /* |
0efc94c5 PZ |
280 | * SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t |
281 | * @name: Name of the seqcount_LOCKNAME_t instance | |
6dd699b1 | 282 | * @lock: Pointer to the associated LOCKNAME |
0efc94c5 PZ |
283 | */ |
284 | ||
6dd699b1 | 285 | #define SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock) { \ |
0efc94c5 PZ |
286 | .seqcount = SEQCNT_ZERO(seq_name.seqcount), \ |
287 | __SEQ_LOCK(.lock = (assoc_lock)) \ | |
288 | } | |
289 | ||
6dd699b1 | 290 | #define SEQCNT_RAW_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) |
267580db | 291 | #define SEQCNT_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) |
6dd699b1 AD |
292 | #define SEQCNT_RWLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) |
293 | #define SEQCNT_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) | |
294 | #define SEQCNT_WW_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) | |
0efc94c5 | 295 | |
a8772dcc | 296 | #define __seqprop_case(s, lockname, prop) \ |
5cdd2557 | 297 | seqcount_##lockname##_t: __seqprop_##lockname##_##prop((void *)(s)) |
55f3560d AD |
298 | |
299 | #define __seqprop(s, prop) _Generic(*(s), \ | |
5cdd2557 | 300 | seqcount_t: __seqprop_##prop((void *)(s)), \ |
55f3560d AD |
301 | __seqprop_case((s), raw_spinlock, prop), \ |
302 | __seqprop_case((s), spinlock, prop), \ | |
303 | __seqprop_case((s), rwlock, prop), \ | |
e84815cb | 304 | __seqprop_case((s), mutex, prop)) |
55f3560d | 305 | |
ab440b2c PZ |
306 | #define seqprop_ptr(s) __seqprop(s, ptr) |
307 | #define seqprop_sequence(s) __seqprop(s, sequence) | |
308 | #define seqprop_preemptible(s) __seqprop(s, preemptible) | |
309 | #define seqprop_assert(s) __seqprop(s, assert) | |
55f3560d | 310 | |
3c22cd57 | 311 | /** |
89b88845 | 312 | * __read_seqcount_begin() - begin a seqcount_t read section w/o barrier |
6dd699b1 | 313 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
3c22cd57 NP |
314 | * |
315 | * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() | |
316 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | |
317 | * provided before actually loading any of the variables that are to be | |
318 | * protected in this critical section. | |
319 | * | |
320 | * Use carefully, only in critical code, and comment how the barrier is | |
321 | * provided. | |
89b88845 AD |
322 | * |
323 | * Return: count to be passed to read_seqcount_retry() | |
3c22cd57 | 324 | */ |
55f3560d | 325 | #define __read_seqcount_begin(s) \ |
52ac39e5 | 326 | ({ \ |
a07c4531 | 327 | unsigned __seq; \ |
52ac39e5 | 328 | \ |
ab440b2c | 329 | while ((__seq = seqprop_sequence(s)) & 1) \ |
52ac39e5 AD |
330 | cpu_relax(); \ |
331 | \ | |
332 | kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \ | |
a07c4531 | 333 | __seq; \ |
52ac39e5 | 334 | }) |
1da177e4 | 335 | |
1ca7d67c | 336 | /** |
89b88845 | 337 | * raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep |
6dd699b1 | 338 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
1ca7d67c | 339 | * |
89b88845 | 340 | * Return: count to be passed to read_seqcount_retry() |
1ca7d67c | 341 | */ |
55f3560d | 342 | #define raw_read_seqcount_begin(s) \ |
52ac39e5 | 343 | ({ \ |
a07c4531 | 344 | unsigned _seq = __read_seqcount_begin(s); \ |
52ac39e5 AD |
345 | \ |
346 | smp_rmb(); \ | |
a07c4531 | 347 | _seq; \ |
52ac39e5 | 348 | }) |
1ca7d67c | 349 | |
3c22cd57 | 350 | /** |
89b88845 | 351 | * read_seqcount_begin() - begin a seqcount_t read critical section |
6dd699b1 | 352 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
3c22cd57 | 353 | * |
89b88845 | 354 | * Return: count to be passed to read_seqcount_retry() |
3c22cd57 | 355 | */ |
55f3560d | 356 | #define read_seqcount_begin(s) \ |
52ac39e5 | 357 | ({ \ |
ab440b2c | 358 | seqcount_lockdep_reader_access(seqprop_ptr(s)); \ |
52ac39e5 AD |
359 | raw_read_seqcount_begin(s); \ |
360 | }) | |
3c22cd57 | 361 | |
f4a27cbc | 362 | /** |
89b88845 | 363 | * raw_read_seqcount() - read the raw seqcount_t counter value |
6dd699b1 | 364 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
f4a27cbc AD |
365 | * |
366 | * raw_read_seqcount opens a read critical section of the given | |
89b88845 AD |
367 | * seqcount_t, without any lockdep checking, and without checking or |
368 | * masking the sequence counter LSB. Calling code is responsible for | |
369 | * handling that. | |
370 | * | |
371 | * Return: count to be passed to read_seqcount_retry() | |
f4a27cbc | 372 | */ |
55f3560d | 373 | #define raw_read_seqcount(s) \ |
52ac39e5 | 374 | ({ \ |
ab440b2c | 375 | unsigned __seq = seqprop_sequence(s); \ |
52ac39e5 AD |
376 | \ |
377 | smp_rmb(); \ | |
378 | kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \ | |
a07c4531 | 379 | __seq; \ |
52ac39e5 | 380 | }) |
f4a27cbc | 381 | |
4f988f15 | 382 | /** |
89b88845 AD |
383 | * raw_seqcount_begin() - begin a seqcount_t read critical section w/o |
384 | * lockdep and w/o counter stabilization | |
6dd699b1 | 385 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
4f988f15 | 386 | * |
89b88845 AD |
387 | * raw_seqcount_begin opens a read critical section of the given |
388 | * seqcount_t. Unlike read_seqcount_begin(), this function will not wait | |
389 | * for the count to stabilize. If a writer is active when it begins, it | |
390 | * will fail the read_seqcount_retry() at the end of the read critical | |
391 | * section instead of stabilizing at the beginning of it. | |
4f988f15 | 392 | * |
89b88845 AD |
393 | * Use this only in special kernel hot paths where the read section is |
394 | * small and has a high probability of success through other external | |
395 | * means. It will save a single branching instruction. | |
396 | * | |
397 | * Return: count to be passed to read_seqcount_retry() | |
4f988f15 | 398 | */ |
55f3560d | 399 | #define raw_seqcount_begin(s) \ |
52ac39e5 AD |
400 | ({ \ |
401 | /* \ | |
402 | * If the counter is odd, let read_seqcount_retry() fail \ | |
403 | * by decrementing the counter. \ | |
404 | */ \ | |
405 | raw_read_seqcount(s) & ~1; \ | |
406 | }) | |
4f988f15 | 407 | |
3c22cd57 | 408 | /** |
89b88845 | 409 | * __read_seqcount_retry() - end a seqcount_t read section w/o barrier |
6dd699b1 | 410 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 411 | * @start: count, from read_seqcount_begin() |
3c22cd57 NP |
412 | * |
413 | * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() | |
414 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | |
415 | * provided before actually loading any of the variables that are to be | |
416 | * protected in this critical section. | |
417 | * | |
418 | * Use carefully, only in critical code, and comment how the barrier is | |
419 | * provided. | |
89b88845 AD |
420 | * |
421 | * Return: true if a read section retry is required, else false | |
3c22cd57 | 422 | */ |
55f3560d | 423 | #define __read_seqcount_retry(s, start) \ |
66bcfcdf | 424 | do___read_seqcount_retry(seqprop_ptr(s), start) |
55f3560d | 425 | |
66bcfcdf | 426 | static inline int do___read_seqcount_retry(const seqcount_t *s, unsigned start) |
3c22cd57 | 427 | { |
88ecd153 ME |
428 | kcsan_atomic_next(0); |
429 | return unlikely(READ_ONCE(s->sequence) != start); | |
3c22cd57 NP |
430 | } |
431 | ||
432 | /** | |
89b88845 | 433 | * read_seqcount_retry() - end a seqcount_t read critical section |
6dd699b1 | 434 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 435 | * @start: count, from read_seqcount_begin() |
3c22cd57 | 436 | * |
89b88845 AD |
437 | * read_seqcount_retry closes the read critical section of given |
438 | * seqcount_t. If the critical section was invalid, it must be ignored | |
439 | * (and typically retried). | |
440 | * | |
441 | * Return: true if a read section retry is required, else false | |
1da177e4 | 442 | */ |
55f3560d | 443 | #define read_seqcount_retry(s, start) \ |
66bcfcdf | 444 | do_read_seqcount_retry(seqprop_ptr(s), start) |
55f3560d | 445 | |
66bcfcdf | 446 | static inline int do_read_seqcount_retry(const seqcount_t *s, unsigned start) |
1da177e4 LT |
447 | { |
448 | smp_rmb(); | |
66bcfcdf | 449 | return do___read_seqcount_retry(s, start); |
1da177e4 LT |
450 | } |
451 | ||
89b88845 AD |
452 | /** |
453 | * raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep | |
6dd699b1 | 454 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
cb262935 AD |
455 | * |
456 | * Context: check write_seqcount_begin() | |
89b88845 | 457 | */ |
55f3560d AD |
458 | #define raw_write_seqcount_begin(s) \ |
459 | do { \ | |
ab440b2c | 460 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
461 | preempt_disable(); \ |
462 | \ | |
66bcfcdf | 463 | do_raw_write_seqcount_begin(seqprop_ptr(s)); \ |
55f3560d AD |
464 | } while (0) |
465 | ||
66bcfcdf | 466 | static inline void do_raw_write_seqcount_begin(seqcount_t *s) |
0c3351d4 | 467 | { |
88ecd153 | 468 | kcsan_nestable_atomic_begin(); |
0c3351d4 JS |
469 | s->sequence++; |
470 | smp_wmb(); | |
471 | } | |
472 | ||
89b88845 AD |
473 | /** |
474 | * raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep | |
6dd699b1 | 475 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
cb262935 AD |
476 | * |
477 | * Context: check write_seqcount_end() | |
89b88845 | 478 | */ |
55f3560d AD |
479 | #define raw_write_seqcount_end(s) \ |
480 | do { \ | |
66bcfcdf | 481 | do_raw_write_seqcount_end(seqprop_ptr(s)); \ |
55f3560d | 482 | \ |
ab440b2c | 483 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
484 | preempt_enable(); \ |
485 | } while (0) | |
486 | ||
66bcfcdf | 487 | static inline void do_raw_write_seqcount_end(seqcount_t *s) |
0c3351d4 JS |
488 | { |
489 | smp_wmb(); | |
490 | s->sequence++; | |
88ecd153 | 491 | kcsan_nestable_atomic_end(); |
0c3351d4 JS |
492 | } |
493 | ||
89b88845 AD |
494 | /** |
495 | * write_seqcount_begin_nested() - start a seqcount_t write section with | |
496 | * custom lockdep nesting level | |
6dd699b1 | 497 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 AD |
498 | * @subclass: lockdep nesting level |
499 | * | |
500 | * See Documentation/locking/lockdep-design.rst | |
cb262935 | 501 | * Context: check write_seqcount_begin() |
89b88845 | 502 | */ |
55f3560d AD |
503 | #define write_seqcount_begin_nested(s, subclass) \ |
504 | do { \ | |
ab440b2c | 505 | seqprop_assert(s); \ |
55f3560d | 506 | \ |
ab440b2c | 507 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
508 | preempt_disable(); \ |
509 | \ | |
66bcfcdf | 510 | do_write_seqcount_begin_nested(seqprop_ptr(s), subclass); \ |
55f3560d | 511 | } while (0) |
859247d3 | 512 | |
66bcfcdf | 513 | static inline void do_write_seqcount_begin_nested(seqcount_t *s, int subclass) |
859247d3 | 514 | { |
55f3560d | 515 | seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); |
41b43b6c | 516 | do_raw_write_seqcount_begin(s); |
f4a27cbc AD |
517 | } |
518 | ||
89b88845 AD |
519 | /** |
520 | * write_seqcount_begin() - start a seqcount_t write side critical section | |
6dd699b1 | 521 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 522 | * |
cb262935 AD |
523 | * Context: sequence counter write side sections must be serialized and |
524 | * non-preemptible. Preemption will be automatically disabled if and | |
525 | * only if the seqcount write serialization lock is associated, and | |
526 | * preemptible. If readers can be invoked from hardirq or softirq | |
89b88845 AD |
527 | * context, interrupts or bottom halves must be respectively disabled. |
528 | */ | |
55f3560d AD |
529 | #define write_seqcount_begin(s) \ |
530 | do { \ | |
ab440b2c | 531 | seqprop_assert(s); \ |
55f3560d | 532 | \ |
ab440b2c | 533 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
534 | preempt_disable(); \ |
535 | \ | |
66bcfcdf | 536 | do_write_seqcount_begin(seqprop_ptr(s)); \ |
55f3560d AD |
537 | } while (0) |
538 | ||
66bcfcdf | 539 | static inline void do_write_seqcount_begin(seqcount_t *s) |
f4a27cbc | 540 | { |
66bcfcdf | 541 | do_write_seqcount_begin_nested(s, 0); |
f4a27cbc AD |
542 | } |
543 | ||
89b88845 AD |
544 | /** |
545 | * write_seqcount_end() - end a seqcount_t write side critical section | |
6dd699b1 | 546 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 547 | * |
cb262935 AD |
548 | * Context: Preemption will be automatically re-enabled if and only if |
549 | * the seqcount write serialization lock is associated, and preemptible. | |
89b88845 | 550 | */ |
55f3560d AD |
551 | #define write_seqcount_end(s) \ |
552 | do { \ | |
66bcfcdf | 553 | do_write_seqcount_end(seqprop_ptr(s)); \ |
55f3560d | 554 | \ |
ab440b2c | 555 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
556 | preempt_enable(); \ |
557 | } while (0) | |
558 | ||
66bcfcdf | 559 | static inline void do_write_seqcount_end(seqcount_t *s) |
f4a27cbc AD |
560 | { |
561 | seqcount_release(&s->dep_map, _RET_IP_); | |
66bcfcdf | 562 | do_raw_write_seqcount_end(s); |
f4a27cbc AD |
563 | } |
564 | ||
c4bfa3f5 | 565 | /** |
89b88845 | 566 | * raw_write_seqcount_barrier() - do a seqcount_t write barrier |
6dd699b1 | 567 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
c4bfa3f5 | 568 | * |
89b88845 AD |
569 | * This can be used to provide an ordering guarantee instead of the usual |
570 | * consistency guarantee. It is one wmb cheaper, because it can collapse | |
571 | * the two back-to-back wmb()s. | |
c4bfa3f5 | 572 | * |
5cbaefe9 | 573 | * Note that writes surrounding the barrier should be declared atomic (e.g. |
bf07132f ME |
574 | * via WRITE_ONCE): a) to ensure the writes become visible to other threads |
575 | * atomically, avoiding compiler optimizations; b) to document which writes are | |
576 | * meant to propagate to the reader critical section. This is necessary because | |
577 | * neither writes before and after the barrier are enclosed in a seq-writer | |
15cbe67b | 578 | * critical section that would ensure readers are aware of ongoing writes:: |
bf07132f | 579 | * |
15cbe67b AD |
580 | * seqcount_t seq; |
581 | * bool X = true, Y = false; | |
c4bfa3f5 | 582 | * |
15cbe67b AD |
583 | * void read(void) |
584 | * { | |
585 | * bool x, y; | |
c4bfa3f5 | 586 | * |
15cbe67b AD |
587 | * do { |
588 | * int s = read_seqcount_begin(&seq); | |
c4bfa3f5 | 589 | * |
15cbe67b | 590 | * x = X; y = Y; |
c4bfa3f5 | 591 | * |
15cbe67b | 592 | * } while (read_seqcount_retry(&seq, s)); |
c4bfa3f5 | 593 | * |
15cbe67b | 594 | * BUG_ON(!x && !y); |
c4bfa3f5 PZ |
595 | * } |
596 | * | |
597 | * void write(void) | |
598 | * { | |
15cbe67b | 599 | * WRITE_ONCE(Y, true); |
c4bfa3f5 | 600 | * |
15cbe67b | 601 | * raw_write_seqcount_barrier(seq); |
c4bfa3f5 | 602 | * |
15cbe67b | 603 | * WRITE_ONCE(X, false); |
c4bfa3f5 PZ |
604 | * } |
605 | */ | |
55f3560d | 606 | #define raw_write_seqcount_barrier(s) \ |
66bcfcdf | 607 | do_raw_write_seqcount_barrier(seqprop_ptr(s)) |
55f3560d | 608 | |
66bcfcdf | 609 | static inline void do_raw_write_seqcount_barrier(seqcount_t *s) |
c4bfa3f5 | 610 | { |
88ecd153 | 611 | kcsan_nestable_atomic_begin(); |
c4bfa3f5 PZ |
612 | s->sequence++; |
613 | smp_wmb(); | |
614 | s->sequence++; | |
88ecd153 | 615 | kcsan_nestable_atomic_end(); |
c4bfa3f5 PZ |
616 | } |
617 | ||
f4a27cbc | 618 | /** |
89b88845 AD |
619 | * write_seqcount_invalidate() - invalidate in-progress seqcount_t read |
620 | * side operations | |
6dd699b1 | 621 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
f4a27cbc | 622 | * |
89b88845 AD |
623 | * After write_seqcount_invalidate, no seqcount_t read side operations |
624 | * will complete successfully and see data older than this. | |
f4a27cbc | 625 | */ |
55f3560d | 626 | #define write_seqcount_invalidate(s) \ |
66bcfcdf | 627 | do_write_seqcount_invalidate(seqprop_ptr(s)) |
55f3560d | 628 | |
66bcfcdf | 629 | static inline void do_write_seqcount_invalidate(seqcount_t *s) |
f4a27cbc AD |
630 | { |
631 | smp_wmb(); | |
632 | kcsan_nestable_atomic_begin(); | |
633 | s->sequence+=2; | |
634 | kcsan_nestable_atomic_end(); | |
635 | } | |
636 | ||
80793c34 AD |
637 | /* |
638 | * Latch sequence counters (seqcount_latch_t) | |
89b88845 | 639 | * |
80793c34 AD |
640 | * A sequence counter variant where the counter even/odd value is used to |
641 | * switch between two copies of protected data. This allows the read path, | |
642 | * typically NMIs, to safely interrupt the write side critical section. | |
89b88845 | 643 | * |
80793c34 AD |
644 | * As the write sections are fully preemptible, no special handling for |
645 | * PREEMPT_RT is needed. | |
646 | */ | |
647 | typedef struct { | |
648 | seqcount_t seqcount; | |
649 | } seqcount_latch_t; | |
650 | ||
651 | /** | |
652 | * SEQCNT_LATCH_ZERO() - static initializer for seqcount_latch_t | |
653 | * @seq_name: Name of the seqcount_latch_t instance | |
654 | */ | |
655 | #define SEQCNT_LATCH_ZERO(seq_name) { \ | |
656 | .seqcount = SEQCNT_ZERO(seq_name.seqcount), \ | |
657 | } | |
658 | ||
659 | /** | |
660 | * seqcount_latch_init() - runtime initializer for seqcount_latch_t | |
661 | * @s: Pointer to the seqcount_latch_t instance | |
662 | */ | |
4817a52b | 663 | #define seqcount_latch_init(s) seqcount_init(&(s)->seqcount) |
80793c34 AD |
664 | |
665 | /** | |
666 | * raw_read_seqcount_latch() - pick even/odd latch data copy | |
0c9794c8 | 667 | * @s: Pointer to seqcount_latch_t |
80793c34 AD |
668 | * |
669 | * See raw_write_seqcount_latch() for details and a full reader/writer | |
670 | * usage example. | |
89b88845 AD |
671 | * |
672 | * Return: sequence counter raw value. Use the lowest bit as an index for | |
80793c34 | 673 | * picking which data copy to read. The full counter must then be checked |
d16317de | 674 | * with raw_read_seqcount_latch_retry(). |
89b88845 | 675 | */ |
d16317de | 676 | static __always_inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s) |
0c9794c8 AD |
677 | { |
678 | /* | |
679 | * Pairs with the first smp_wmb() in raw_write_seqcount_latch(). | |
680 | * Due to the dependent load, a full smp_rmb() is not needed. | |
681 | */ | |
682 | return READ_ONCE(s->seqcount.sequence); | |
683 | } | |
55f3560d | 684 | |
80793c34 | 685 | /** |
d16317de | 686 | * raw_read_seqcount_latch_retry() - end a seqcount_latch_t read section |
80793c34 AD |
687 | * @s: Pointer to seqcount_latch_t |
688 | * @start: count, from raw_read_seqcount_latch() | |
689 | * | |
690 | * Return: true if a read section retry is required, else false | |
691 | */ | |
d16317de PZ |
692 | static __always_inline int |
693 | raw_read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) | |
7fc26327 | 694 | { |
d16317de PZ |
695 | smp_rmb(); |
696 | return unlikely(READ_ONCE(s->seqcount.sequence) != start); | |
7fc26327 PZ |
697 | } |
698 | ||
6695b92a | 699 | /** |
80793c34 | 700 | * raw_write_seqcount_latch() - redirect latch readers to even/odd copy |
0c9794c8 | 701 | * @s: Pointer to seqcount_latch_t |
6695b92a PZ |
702 | * |
703 | * The latch technique is a multiversion concurrency control method that allows | |
704 | * queries during non-atomic modifications. If you can guarantee queries never | |
705 | * interrupt the modification -- e.g. the concurrency is strictly between CPUs | |
706 | * -- you most likely do not need this. | |
707 | * | |
708 | * Where the traditional RCU/lockless data structures rely on atomic | |
709 | * modifications to ensure queries observe either the old or the new state the | |
710 | * latch allows the same for non-atomic updates. The trade-off is doubling the | |
711 | * cost of storage; we have to maintain two copies of the entire data | |
712 | * structure. | |
713 | * | |
714 | * Very simply put: we first modify one copy and then the other. This ensures | |
715 | * there is always one copy in a stable state, ready to give us an answer. | |
716 | * | |
15cbe67b | 717 | * The basic form is a data structure like:: |
6695b92a | 718 | * |
15cbe67b | 719 | * struct latch_struct { |
80793c34 | 720 | * seqcount_latch_t seq; |
15cbe67b AD |
721 | * struct data_struct data[2]; |
722 | * }; | |
6695b92a PZ |
723 | * |
724 | * Where a modification, which is assumed to be externally serialized, does the | |
15cbe67b | 725 | * following:: |
6695b92a | 726 | * |
15cbe67b AD |
727 | * void latch_modify(struct latch_struct *latch, ...) |
728 | * { | |
729 | * smp_wmb(); // Ensure that the last data[1] update is visible | |
80793c34 | 730 | * latch->seq.sequence++; |
15cbe67b | 731 | * smp_wmb(); // Ensure that the seqcount update is visible |
6695b92a | 732 | * |
15cbe67b | 733 | * modify(latch->data[0], ...); |
6695b92a | 734 | * |
15cbe67b | 735 | * smp_wmb(); // Ensure that the data[0] update is visible |
80793c34 | 736 | * latch->seq.sequence++; |
15cbe67b | 737 | * smp_wmb(); // Ensure that the seqcount update is visible |
6695b92a | 738 | * |
15cbe67b AD |
739 | * modify(latch->data[1], ...); |
740 | * } | |
6695b92a | 741 | * |
15cbe67b | 742 | * The query will have a form like:: |
6695b92a | 743 | * |
15cbe67b AD |
744 | * struct entry *latch_query(struct latch_struct *latch, ...) |
745 | * { | |
746 | * struct entry *entry; | |
747 | * unsigned seq, idx; | |
6695b92a | 748 | * |
15cbe67b AD |
749 | * do { |
750 | * seq = raw_read_seqcount_latch(&latch->seq); | |
6695b92a | 751 | * |
15cbe67b AD |
752 | * idx = seq & 0x01; |
753 | * entry = data_query(latch->data[idx], ...); | |
6695b92a | 754 | * |
80793c34 | 755 | * // This includes needed smp_rmb() |
d16317de | 756 | * } while (raw_read_seqcount_latch_retry(&latch->seq, seq)); |
6695b92a | 757 | * |
15cbe67b AD |
758 | * return entry; |
759 | * } | |
6695b92a PZ |
760 | * |
761 | * So during the modification, queries are first redirected to data[1]. Then we | |
762 | * modify data[0]. When that is complete, we redirect queries back to data[0] | |
763 | * and we can modify data[1]. | |
764 | * | |
15cbe67b AD |
765 | * NOTE: |
766 | * | |
767 | * The non-requirement for atomic modifications does _NOT_ include | |
768 | * the publishing of new entries in the case where data is a dynamic | |
769 | * data structure. | |
770 | * | |
771 | * An iteration might start in data[0] and get suspended long enough | |
772 | * to miss an entire modification sequence, once it resumes it might | |
773 | * observe the new entry. | |
6695b92a | 774 | * |
a28e884b | 775 | * NOTE2: |
6695b92a | 776 | * |
15cbe67b AD |
777 | * When data is a dynamic data structure; one should use regular RCU |
778 | * patterns to manage the lifetimes of the objects within. | |
9b0fd802 | 779 | */ |
0c9794c8 AD |
780 | static inline void raw_write_seqcount_latch(seqcount_latch_t *s) |
781 | { | |
782 | smp_wmb(); /* prior stores before incrementing "sequence" */ | |
783 | s->seqcount.sequence++; | |
784 | smp_wmb(); /* increment "sequence" before following stores */ | |
9b0fd802 MD |
785 | } |
786 | ||
0d24f65e AD |
787 | /* |
788 | * Sequential locks (seqlock_t) | |
789 | * | |
790 | * Sequence counters with an embedded spinlock for writer serialization | |
791 | * and non-preemptibility. | |
792 | * | |
793 | * For more info, see: | |
794 | * - Comments on top of seqcount_t | |
795 | * - Documentation/locking/seqlock.rst | |
796 | */ | |
6617feca | 797 | typedef struct { |
1909760f AD |
798 | /* |
799 | * Make sure that readers don't starve writers on PREEMPT_RT: use | |
800 | * seqcount_spinlock_t instead of seqcount_t. Check __SEQ_LOCK(). | |
801 | */ | |
802 | seqcount_spinlock_t seqcount; | |
6617feca TG |
803 | spinlock_t lock; |
804 | } seqlock_t; | |
805 | ||
ec8702da AD |
806 | #define __SEQLOCK_UNLOCKED(lockname) \ |
807 | { \ | |
1909760f | 808 | .seqcount = SEQCNT_SPINLOCK_ZERO(lockname, &(lockname).lock), \ |
ec8702da | 809 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ |
6617feca TG |
810 | } |
811 | ||
89b88845 AD |
812 | /** |
813 | * seqlock_init() - dynamic initializer for seqlock_t | |
814 | * @sl: Pointer to the seqlock_t instance | |
815 | */ | |
ec8702da AD |
816 | #define seqlock_init(sl) \ |
817 | do { \ | |
ec8702da | 818 | spin_lock_init(&(sl)->lock); \ |
1909760f | 819 | seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock); \ |
6617feca TG |
820 | } while (0) |
821 | ||
89b88845 | 822 | /** |
24a18772 | 823 | * DEFINE_SEQLOCK(sl) - Define a statically allocated seqlock_t |
89b88845 AD |
824 | * @sl: Name of the seqlock_t instance |
825 | */ | |
826 | #define DEFINE_SEQLOCK(sl) \ | |
827 | seqlock_t sl = __SEQLOCK_UNLOCKED(sl) | |
6617feca | 828 | |
89b88845 AD |
829 | /** |
830 | * read_seqbegin() - start a seqlock_t read side critical section | |
831 | * @sl: Pointer to seqlock_t | |
832 | * | |
833 | * Return: count, to be passed to read_seqretry() | |
6617feca TG |
834 | */ |
835 | static inline unsigned read_seqbegin(const seqlock_t *sl) | |
836 | { | |
88ecd153 ME |
837 | unsigned ret = read_seqcount_begin(&sl->seqcount); |
838 | ||
5cbaefe9 | 839 | kcsan_atomic_next(0); /* non-raw usage, assume closing read_seqretry() */ |
88ecd153 ME |
840 | kcsan_flat_atomic_begin(); |
841 | return ret; | |
6617feca TG |
842 | } |
843 | ||
89b88845 AD |
844 | /** |
845 | * read_seqretry() - end a seqlock_t read side section | |
846 | * @sl: Pointer to seqlock_t | |
847 | * @start: count, from read_seqbegin() | |
848 | * | |
849 | * read_seqretry closes the read side critical section of given seqlock_t. | |
850 | * If the critical section was invalid, it must be ignored (and typically | |
851 | * retried). | |
852 | * | |
853 | * Return: true if a read section retry is required, else false | |
854 | */ | |
6617feca TG |
855 | static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) |
856 | { | |
88ecd153 | 857 | /* |
5cbaefe9 | 858 | * Assume not nested: read_seqretry() may be called multiple times when |
88ecd153 ME |
859 | * completing read critical section. |
860 | */ | |
861 | kcsan_flat_atomic_end(); | |
862 | ||
6617feca TG |
863 | return read_seqcount_retry(&sl->seqcount, start); |
864 | } | |
865 | ||
1909760f | 866 | /* |
66bcfcdf AD |
867 | * For all seqlock_t write side functions, use the the internal |
868 | * do_write_seqcount_begin() instead of generic write_seqcount_begin(). | |
869 | * This way, no redundant lockdep_assert_held() checks are added. | |
1909760f AD |
870 | */ |
871 | ||
89b88845 AD |
872 | /** |
873 | * write_seqlock() - start a seqlock_t write side critical section | |
874 | * @sl: Pointer to seqlock_t | |
875 | * | |
876 | * write_seqlock opens a write side critical section for the given | |
877 | * seqlock_t. It also implicitly acquires the spinlock_t embedded inside | |
878 | * that sequential lock. All seqlock_t write side sections are thus | |
879 | * automatically serialized and non-preemptible. | |
880 | * | |
881 | * Context: if the seqlock_t read section, or other write side critical | |
882 | * sections, can be invoked from hardirq or softirq contexts, use the | |
883 | * _irqsave or _bh variants of this function instead. | |
1da177e4 | 884 | */ |
6617feca TG |
885 | static inline void write_seqlock(seqlock_t *sl) |
886 | { | |
887 | spin_lock(&sl->lock); | |
66bcfcdf | 888 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
889 | } |
890 | ||
89b88845 AD |
891 | /** |
892 | * write_sequnlock() - end a seqlock_t write side critical section | |
893 | * @sl: Pointer to seqlock_t | |
894 | * | |
895 | * write_sequnlock closes the (serialized and non-preemptible) write side | |
896 | * critical section of given seqlock_t. | |
897 | */ | |
6617feca TG |
898 | static inline void write_sequnlock(seqlock_t *sl) |
899 | { | |
66bcfcdf | 900 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
901 | spin_unlock(&sl->lock); |
902 | } | |
903 | ||
89b88845 AD |
904 | /** |
905 | * write_seqlock_bh() - start a softirqs-disabled seqlock_t write section | |
906 | * @sl: Pointer to seqlock_t | |
907 | * | |
908 | * _bh variant of write_seqlock(). Use only if the read side section, or | |
909 | * other write side sections, can be invoked from softirq contexts. | |
910 | */ | |
6617feca TG |
911 | static inline void write_seqlock_bh(seqlock_t *sl) |
912 | { | |
913 | spin_lock_bh(&sl->lock); | |
66bcfcdf | 914 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
915 | } |
916 | ||
89b88845 AD |
917 | /** |
918 | * write_sequnlock_bh() - end a softirqs-disabled seqlock_t write section | |
919 | * @sl: Pointer to seqlock_t | |
920 | * | |
921 | * write_sequnlock_bh closes the serialized, non-preemptible, and | |
922 | * softirqs-disabled, seqlock_t write side critical section opened with | |
923 | * write_seqlock_bh(). | |
924 | */ | |
6617feca TG |
925 | static inline void write_sequnlock_bh(seqlock_t *sl) |
926 | { | |
66bcfcdf | 927 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
928 | spin_unlock_bh(&sl->lock); |
929 | } | |
930 | ||
89b88845 AD |
931 | /** |
932 | * write_seqlock_irq() - start a non-interruptible seqlock_t write section | |
933 | * @sl: Pointer to seqlock_t | |
934 | * | |
935 | * _irq variant of write_seqlock(). Use only if the read side section, or | |
936 | * other write sections, can be invoked from hardirq contexts. | |
937 | */ | |
6617feca TG |
938 | static inline void write_seqlock_irq(seqlock_t *sl) |
939 | { | |
940 | spin_lock_irq(&sl->lock); | |
66bcfcdf | 941 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
942 | } |
943 | ||
89b88845 AD |
944 | /** |
945 | * write_sequnlock_irq() - end a non-interruptible seqlock_t write section | |
946 | * @sl: Pointer to seqlock_t | |
947 | * | |
948 | * write_sequnlock_irq closes the serialized and non-interruptible | |
949 | * seqlock_t write side section opened with write_seqlock_irq(). | |
950 | */ | |
6617feca TG |
951 | static inline void write_sequnlock_irq(seqlock_t *sl) |
952 | { | |
66bcfcdf | 953 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
954 | spin_unlock_irq(&sl->lock); |
955 | } | |
956 | ||
957 | static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl) | |
958 | { | |
959 | unsigned long flags; | |
960 | ||
961 | spin_lock_irqsave(&sl->lock, flags); | |
66bcfcdf | 962 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
963 | return flags; |
964 | } | |
965 | ||
89b88845 AD |
966 | /** |
967 | * write_seqlock_irqsave() - start a non-interruptible seqlock_t write | |
968 | * section | |
969 | * @lock: Pointer to seqlock_t | |
970 | * @flags: Stack-allocated storage for saving caller's local interrupt | |
971 | * state, to be passed to write_sequnlock_irqrestore(). | |
972 | * | |
973 | * _irqsave variant of write_seqlock(). Use it only if the read side | |
974 | * section, or other write sections, can be invoked from hardirq context. | |
975 | */ | |
1da177e4 | 976 | #define write_seqlock_irqsave(lock, flags) \ |
6617feca | 977 | do { flags = __write_seqlock_irqsave(lock); } while (0) |
1da177e4 | 978 | |
89b88845 AD |
979 | /** |
980 | * write_sequnlock_irqrestore() - end non-interruptible seqlock_t write | |
981 | * section | |
982 | * @sl: Pointer to seqlock_t | |
983 | * @flags: Caller's saved interrupt state, from write_seqlock_irqsave() | |
984 | * | |
985 | * write_sequnlock_irqrestore closes the serialized and non-interruptible | |
986 | * seqlock_t write section previously opened with write_seqlock_irqsave(). | |
987 | */ | |
6617feca TG |
988 | static inline void |
989 | write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) | |
990 | { | |
66bcfcdf | 991 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
992 | spin_unlock_irqrestore(&sl->lock, flags); |
993 | } | |
1da177e4 | 994 | |
89b88845 AD |
995 | /** |
996 | * read_seqlock_excl() - begin a seqlock_t locking reader section | |
55f3560d | 997 | * @sl: Pointer to seqlock_t |
89b88845 AD |
998 | * |
999 | * read_seqlock_excl opens a seqlock_t locking reader critical section. A | |
1000 | * locking reader exclusively locks out *both* other writers *and* other | |
1001 | * locking readers, but it does not update the embedded sequence number. | |
1002 | * | |
1003 | * Locking readers act like a normal spin_lock()/spin_unlock(). | |
1004 | * | |
1005 | * Context: if the seqlock_t write section, *or other read sections*, can | |
1006 | * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | |
1007 | * variant of this function instead. | |
1008 | * | |
1009 | * The opened read section must be closed with read_sequnlock_excl(). | |
1370e97b WL |
1010 | */ |
1011 | static inline void read_seqlock_excl(seqlock_t *sl) | |
1012 | { | |
1013 | spin_lock(&sl->lock); | |
1014 | } | |
1015 | ||
89b88845 AD |
1016 | /** |
1017 | * read_sequnlock_excl() - end a seqlock_t locking reader critical section | |
1018 | * @sl: Pointer to seqlock_t | |
1019 | */ | |
1370e97b WL |
1020 | static inline void read_sequnlock_excl(seqlock_t *sl) |
1021 | { | |
1022 | spin_unlock(&sl->lock); | |
1023 | } | |
1024 | ||
89b88845 AD |
1025 | /** |
1026 | * read_seqlock_excl_bh() - start a seqlock_t locking reader section with | |
1027 | * softirqs disabled | |
1028 | * @sl: Pointer to seqlock_t | |
1029 | * | |
1030 | * _bh variant of read_seqlock_excl(). Use this variant only if the | |
1031 | * seqlock_t write side section, *or other read sections*, can be invoked | |
1032 | * from softirq contexts. | |
1033 | */ | |
1370e97b WL |
1034 | static inline void read_seqlock_excl_bh(seqlock_t *sl) |
1035 | { | |
1036 | spin_lock_bh(&sl->lock); | |
1037 | } | |
1038 | ||
89b88845 AD |
1039 | /** |
1040 | * read_sequnlock_excl_bh() - stop a seqlock_t softirq-disabled locking | |
1041 | * reader section | |
1042 | * @sl: Pointer to seqlock_t | |
1043 | */ | |
1370e97b WL |
1044 | static inline void read_sequnlock_excl_bh(seqlock_t *sl) |
1045 | { | |
1046 | spin_unlock_bh(&sl->lock); | |
1047 | } | |
1048 | ||
89b88845 AD |
1049 | /** |
1050 | * read_seqlock_excl_irq() - start a non-interruptible seqlock_t locking | |
1051 | * reader section | |
1052 | * @sl: Pointer to seqlock_t | |
1053 | * | |
1054 | * _irq variant of read_seqlock_excl(). Use this only if the seqlock_t | |
1055 | * write side section, *or other read sections*, can be invoked from a | |
1056 | * hardirq context. | |
1057 | */ | |
1370e97b WL |
1058 | static inline void read_seqlock_excl_irq(seqlock_t *sl) |
1059 | { | |
1060 | spin_lock_irq(&sl->lock); | |
1061 | } | |
1062 | ||
89b88845 AD |
1063 | /** |
1064 | * read_sequnlock_excl_irq() - end an interrupts-disabled seqlock_t | |
1065 | * locking reader section | |
1066 | * @sl: Pointer to seqlock_t | |
1067 | */ | |
1370e97b WL |
1068 | static inline void read_sequnlock_excl_irq(seqlock_t *sl) |
1069 | { | |
1070 | spin_unlock_irq(&sl->lock); | |
1071 | } | |
1072 | ||
1073 | static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl) | |
1074 | { | |
1075 | unsigned long flags; | |
1076 | ||
1077 | spin_lock_irqsave(&sl->lock, flags); | |
1078 | return flags; | |
1079 | } | |
1080 | ||
89b88845 AD |
1081 | /** |
1082 | * read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t | |
1083 | * locking reader section | |
1084 | * @lock: Pointer to seqlock_t | |
1085 | * @flags: Stack-allocated storage for saving caller's local interrupt | |
1086 | * state, to be passed to read_sequnlock_excl_irqrestore(). | |
1087 | * | |
1088 | * _irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t | |
1089 | * write side section, *or other read sections*, can be invoked from a | |
1090 | * hardirq context. | |
1091 | */ | |
1370e97b WL |
1092 | #define read_seqlock_excl_irqsave(lock, flags) \ |
1093 | do { flags = __read_seqlock_excl_irqsave(lock); } while (0) | |
1094 | ||
89b88845 AD |
1095 | /** |
1096 | * read_sequnlock_excl_irqrestore() - end non-interruptible seqlock_t | |
1097 | * locking reader section | |
1098 | * @sl: Pointer to seqlock_t | |
1099 | * @flags: Caller saved interrupt state, from read_seqlock_excl_irqsave() | |
1100 | */ | |
1370e97b WL |
1101 | static inline void |
1102 | read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) | |
1103 | { | |
1104 | spin_unlock_irqrestore(&sl->lock, flags); | |
1105 | } | |
1106 | ||
f4a27cbc | 1107 | /** |
89b88845 AD |
1108 | * read_seqbegin_or_lock() - begin a seqlock_t lockless or locking reader |
1109 | * @lock: Pointer to seqlock_t | |
1110 | * @seq : Marker and return parameter. If the passed value is even, the | |
1111 | * reader will become a *lockless* seqlock_t reader as in read_seqbegin(). | |
1112 | * If the passed value is odd, the reader will become a *locking* reader | |
1113 | * as in read_seqlock_excl(). In the first call to this function, the | |
1114 | * caller *must* initialize and pass an even value to @seq; this way, a | |
1115 | * lockless read can be optimistically tried first. | |
1116 | * | |
1117 | * read_seqbegin_or_lock is an API designed to optimistically try a normal | |
1118 | * lockless seqlock_t read section first. If an odd counter is found, the | |
1119 | * lockless read trial has failed, and the next read iteration transforms | |
1120 | * itself into a full seqlock_t locking reader. | |
1121 | * | |
1122 | * This is typically used to avoid seqlock_t lockless readers starvation | |
1123 | * (too much retry loops) in the case of a sharp spike in write side | |
1124 | * activity. | |
1125 | * | |
1126 | * Context: if the seqlock_t write section, *or other read sections*, can | |
1127 | * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | |
1128 | * variant of this function instead. | |
1129 | * | |
1130 | * Check Documentation/locking/seqlock.rst for template example code. | |
1131 | * | |
1132 | * Return: the encountered sequence counter value, through the @seq | |
1133 | * parameter, which is overloaded as a return parameter. This returned | |
1134 | * value must be checked with need_seqretry(). If the read section need to | |
1135 | * be retried, this returned value must also be passed as the @seq | |
1136 | * parameter of the next read_seqbegin_or_lock() iteration. | |
f4a27cbc AD |
1137 | */ |
1138 | static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) | |
1139 | { | |
1140 | if (!(*seq & 1)) /* Even */ | |
1141 | *seq = read_seqbegin(lock); | |
1142 | else /* Odd */ | |
1143 | read_seqlock_excl(lock); | |
1144 | } | |
1145 | ||
89b88845 AD |
1146 | /** |
1147 | * need_seqretry() - validate seqlock_t "locking or lockless" read section | |
1148 | * @lock: Pointer to seqlock_t | |
1149 | * @seq: sequence count, from read_seqbegin_or_lock() | |
1150 | * | |
1151 | * Return: true if a read section retry is required, false otherwise | |
1152 | */ | |
f4a27cbc AD |
1153 | static inline int need_seqretry(seqlock_t *lock, int seq) |
1154 | { | |
1155 | return !(seq & 1) && read_seqretry(lock, seq); | |
1156 | } | |
1157 | ||
89b88845 AD |
1158 | /** |
1159 | * done_seqretry() - end seqlock_t "locking or lockless" reader section | |
1160 | * @lock: Pointer to seqlock_t | |
1161 | * @seq: count, from read_seqbegin_or_lock() | |
1162 | * | |
1163 | * done_seqretry finishes the seqlock_t read side critical section started | |
1164 | * with read_seqbegin_or_lock() and validated by need_seqretry(). | |
1165 | */ | |
f4a27cbc AD |
1166 | static inline void done_seqretry(seqlock_t *lock, int seq) |
1167 | { | |
1168 | if (seq & 1) | |
1169 | read_sequnlock_excl(lock); | |
1170 | } | |
1171 | ||
89b88845 AD |
1172 | /** |
1173 | * read_seqbegin_or_lock_irqsave() - begin a seqlock_t lockless reader, or | |
1174 | * a non-interruptible locking reader | |
1175 | * @lock: Pointer to seqlock_t | |
1176 | * @seq: Marker and return parameter. Check read_seqbegin_or_lock(). | |
1177 | * | |
1178 | * This is the _irqsave variant of read_seqbegin_or_lock(). Use it only if | |
1179 | * the seqlock_t write section, *or other read sections*, can be invoked | |
1180 | * from hardirq context. | |
1181 | * | |
1182 | * Note: Interrupts will be disabled only for "locking reader" mode. | |
1183 | * | |
1184 | * Return: | |
1185 | * | |
1186 | * 1. The saved local interrupts state in case of a locking reader, to | |
1187 | * be passed to done_seqretry_irqrestore(). | |
1188 | * | |
1189 | * 2. The encountered sequence counter value, returned through @seq | |
1190 | * overloaded as a return parameter. Check read_seqbegin_or_lock(). | |
1191 | */ | |
ef8ac063 RR |
1192 | static inline unsigned long |
1193 | read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) | |
1194 | { | |
1195 | unsigned long flags = 0; | |
1196 | ||
1197 | if (!(*seq & 1)) /* Even */ | |
1198 | *seq = read_seqbegin(lock); | |
1199 | else /* Odd */ | |
1200 | read_seqlock_excl_irqsave(lock, flags); | |
1201 | ||
1202 | return flags; | |
1203 | } | |
1204 | ||
89b88845 AD |
1205 | /** |
1206 | * done_seqretry_irqrestore() - end a seqlock_t lockless reader, or a | |
1207 | * non-interruptible locking reader section | |
1208 | * @lock: Pointer to seqlock_t | |
1209 | * @seq: Count, from read_seqbegin_or_lock_irqsave() | |
1210 | * @flags: Caller's saved local interrupt state in case of a locking | |
1211 | * reader, also from read_seqbegin_or_lock_irqsave() | |
1212 | * | |
1213 | * This is the _irqrestore variant of done_seqretry(). The read section | |
1214 | * must've been opened with read_seqbegin_or_lock_irqsave(), and validated | |
1215 | * by need_seqretry(). | |
1216 | */ | |
ef8ac063 RR |
1217 | static inline void |
1218 | done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags) | |
1219 | { | |
1220 | if (seq & 1) | |
1221 | read_sequnlock_excl_irqrestore(lock, flags); | |
1222 | } | |
1da177e4 | 1223 | #endif /* __LINUX_SEQLOCK_H */ |