Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | #ifndef __LINUX_SEQLOCK_H |
3 | #define __LINUX_SEQLOCK_H | |
0d24f65e | 4 | |
1da177e4 | 5 | /* |
0d24f65e AD |
6 | * seqcount_t / seqlock_t - a reader-writer consistency mechanism with |
7 | * lockless readers (read-only retry loops), and no writer starvation. | |
8 | * | |
9 | * See Documentation/locking/seqlock.rst | |
10 | * | |
11 | * Copyrights: | |
12 | * - Based on x86_64 vsyscall gettimeofday: Keith Owens, Andrea Arcangeli | |
55f3560d | 13 | * - Sequence counters with associated locks, (C) 2020 Linutronix GmbH |
1da177e4 LT |
14 | */ |
15 | ||
7fc26327 | 16 | #include <linux/compiler.h> |
b968a08f | 17 | #include <linux/kcsan-checks.h> |
55f3560d AD |
18 | #include <linux/lockdep.h> |
19 | #include <linux/mutex.h> | |
20 | #include <linux/preempt.h> | |
f038cc13 | 21 | #include <linux/seqlock_types.h> |
55f3560d | 22 | #include <linux/spinlock.h> |
55f3560d | 23 | |
56a21052 | 24 | #include <asm/processor.h> |
1da177e4 | 25 | |
88ecd153 | 26 | /* |
0d24f65e AD |
27 | * The seqlock seqcount_t interface does not prescribe a precise sequence of |
28 | * read begin/retry/end. For readers, typically there is a call to | |
88ecd153 ME |
29 | * read_seqcount_begin() and read_seqcount_retry(), however, there are more |
30 | * esoteric cases which do not follow this pattern. | |
31 | * | |
32 | * As a consequence, we take the following best-effort approach for raw usage | |
33 | * via seqcount_t under KCSAN: upon beginning a seq-reader critical section, | |
5cbaefe9 | 34 | * pessimistically mark the next KCSAN_SEQLOCK_REGION_MAX memory accesses as |
88ecd153 | 35 | * atomics; if there is a matching read_seqcount_retry() call, no following |
0d24f65e AD |
36 | * memory operations are considered atomic. Usage of the seqlock_t interface |
37 | * is not affected. | |
88ecd153 ME |
38 | */ |
39 | #define KCSAN_SEQLOCK_REGION_MAX 1000 | |
40 | ||
1ca7d67c JS |
41 | static inline void __seqcount_init(seqcount_t *s, const char *name, |
42 | struct lock_class_key *key) | |
43 | { | |
44 | /* | |
45 | * Make sure we are not reinitializing a held lock: | |
46 | */ | |
47 | lockdep_init_map(&s->dep_map, name, key, 0); | |
48 | s->sequence = 0; | |
49 | } | |
50 | ||
51 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
ec8702da AD |
52 | |
53 | # define SEQCOUNT_DEP_MAP_INIT(lockname) \ | |
54 | .dep_map = { .name = #lockname } | |
1ca7d67c | 55 | |
89b88845 AD |
56 | /** |
57 | * seqcount_init() - runtime initializer for seqcount_t | |
58 | * @s: Pointer to the seqcount_t instance | |
59 | */ | |
ec8702da AD |
60 | # define seqcount_init(s) \ |
61 | do { \ | |
62 | static struct lock_class_key __key; \ | |
63 | __seqcount_init((s), #s, &__key); \ | |
1ca7d67c JS |
64 | } while (0) |
65 | ||
66 | static inline void seqcount_lockdep_reader_access(const seqcount_t *s) | |
67 | { | |
68 | seqcount_t *l = (seqcount_t *)s; | |
69 | unsigned long flags; | |
70 | ||
71 | local_irq_save(flags); | |
72 | seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_); | |
5facae4f | 73 | seqcount_release(&l->dep_map, _RET_IP_); |
1ca7d67c JS |
74 | local_irq_restore(flags); |
75 | } | |
76 | ||
77 | #else | |
78 | # define SEQCOUNT_DEP_MAP_INIT(lockname) | |
79 | # define seqcount_init(s) __seqcount_init(s, NULL, NULL) | |
80 | # define seqcount_lockdep_reader_access(x) | |
81 | #endif | |
82 | ||
89b88845 AD |
83 | /** |
84 | * SEQCNT_ZERO() - static initializer for seqcount_t | |
85 | * @name: Name of the seqcount_t instance | |
86 | */ | |
87 | #define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) } | |
1da177e4 | 88 | |
55f3560d | 89 | /* |
6dd699b1 | 90 | * Sequence counters with associated locks (seqcount_LOCKNAME_t) |
55f3560d AD |
91 | * |
92 | * A sequence counter which associates the lock used for writer | |
93 | * serialization at initialization time. This enables lockdep to validate | |
94 | * that the write side critical section is properly serialized. | |
95 | * | |
96 | * For associated locks which do not implicitly disable preemption, | |
97 | * preemption protection is enforced in the write side function. | |
98 | * | |
99 | * Lockdep is never used in any for the raw write variants. | |
100 | * | |
101 | * See Documentation/locking/seqlock.rst | |
102 | */ | |
103 | ||
ed3e4537 | 104 | /* |
6dd699b1 | 105 | * typedef seqcount_LOCKNAME_t - sequence counter with LOCKNAME associated |
a8772dcc | 106 | * @seqcount: The real sequence counter |
6dd699b1 | 107 | * @lock: Pointer to the associated lock |
a8772dcc | 108 | * |
6dd699b1 AD |
109 | * A plain sequence counter with external writer synchronization by |
110 | * LOCKNAME @lock. The lock is associated to the sequence counter in the | |
a8772dcc PZ |
111 | * static initializer or init function. This enables lockdep to validate |
112 | * that the write side critical section is properly serialized. | |
6dd699b1 | 113 | * |
e84815cb | 114 | * LOCKNAME: raw_spinlock, spinlock, rwlock or mutex |
a8772dcc PZ |
115 | */ |
116 | ||
a28e884b | 117 | /* |
e4e9ab3f PZ |
118 | * seqcount_LOCKNAME_init() - runtime initializer for seqcount_LOCKNAME_t |
119 | * @s: Pointer to the seqcount_LOCKNAME_t instance | |
6dd699b1 | 120 | * @lock: Pointer to the associated lock |
e4e9ab3f PZ |
121 | */ |
122 | ||
267580db | 123 | #define seqcount_LOCKNAME_init(s, _lock, lockname) \ |
124 | do { \ | |
125 | seqcount_##lockname##_t *____s = (s); \ | |
126 | seqcount_init(&____s->seqcount); \ | |
127 | __SEQ_LOCK(____s->lock = (_lock)); \ | |
128 | } while (0) | |
129 | ||
130 | #define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock) | |
131 | #define seqcount_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, spinlock) | |
149876d9 HD |
132 | #define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock) |
133 | #define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex) | |
267580db | 134 | |
55f3560d | 135 | /* |
5cdd2557 AD |
136 | * SEQCOUNT_LOCKNAME() - Instantiate seqcount_LOCKNAME_t and helpers |
137 | * seqprop_LOCKNAME_*() - Property accessors for seqcount_LOCKNAME_t | |
138 | * | |
6dd699b1 AD |
139 | * @lockname: "LOCKNAME" part of seqcount_LOCKNAME_t |
140 | * @locktype: LOCKNAME canonical C data type | |
8117ab50 | 141 | * @preemptible: preemptibility of above locktype |
f995443f | 142 | * @lockbase: prefix for associated lock/unlock |
55f3560d | 143 | */ |
f995443f | 144 | #define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockbase) \ |
a8772dcc | 145 | static __always_inline seqcount_t * \ |
886ee55e | 146 | __seqprop_##lockname##_ptr(seqcount_##lockname##_t *s) \ |
55f3560d | 147 | { \ |
886ee55e IM |
148 | return &s->seqcount; \ |
149 | } \ | |
150 | \ | |
151 | static __always_inline const seqcount_t * \ | |
152 | __seqprop_##lockname##_const_ptr(const seqcount_##lockname##_t *s) \ | |
153 | { \ | |
154 | return &s->seqcount; \ | |
55f3560d AD |
155 | } \ |
156 | \ | |
52ac39e5 AD |
157 | static __always_inline unsigned \ |
158 | __seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s) \ | |
159 | { \ | |
8117ab50 AD |
160 | unsigned seq = READ_ONCE(s->seqcount.sequence); \ |
161 | \ | |
162 | if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \ | |
163 | return seq; \ | |
164 | \ | |
165 | if (preemptible && unlikely(seq & 1)) { \ | |
f995443f | 166 | __SEQ_LOCK(lockbase##_lock(s->lock)); \ |
8117ab50 AD |
167 | __SEQ_LOCK(lockbase##_unlock(s->lock)); \ |
168 | \ | |
169 | /* \ | |
170 | * Re-read the sequence counter since the (possibly \ | |
171 | * preempted) writer made progress. \ | |
172 | */ \ | |
173 | seq = READ_ONCE(s->seqcount.sequence); \ | |
174 | } \ | |
175 | \ | |
176 | return seq; \ | |
52ac39e5 AD |
177 | } \ |
178 | \ | |
a8772dcc | 179 | static __always_inline bool \ |
5cdd2557 | 180 | __seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s) \ |
55f3560d | 181 | { \ |
8117ab50 AD |
182 | if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \ |
183 | return preemptible; \ | |
184 | \ | |
185 | /* PREEMPT_RT relies on the above LOCK+UNLOCK */ \ | |
186 | return false; \ | |
55f3560d AD |
187 | } \ |
188 | \ | |
a8772dcc | 189 | static __always_inline void \ |
5cdd2557 | 190 | __seqprop_##lockname##_assert(const seqcount_##lockname##_t *s) \ |
55f3560d | 191 | { \ |
f995443f | 192 | __SEQ_LOCK(lockdep_assert_held(s->lock)); \ |
55f3560d AD |
193 | } |
194 | ||
195 | /* | |
a8772dcc | 196 | * __seqprop() for seqcount_t |
55f3560d AD |
197 | */ |
198 | ||
886ee55e IM |
199 | static inline seqcount_t *__seqprop_ptr(seqcount_t *s) |
200 | { | |
201 | return s; | |
202 | } | |
203 | ||
204 | static inline const seqcount_t *__seqprop_const_ptr(const seqcount_t *s) | |
55f3560d | 205 | { |
886ee55e | 206 | return s; |
55f3560d AD |
207 | } |
208 | ||
52ac39e5 AD |
209 | static inline unsigned __seqprop_sequence(const seqcount_t *s) |
210 | { | |
211 | return READ_ONCE(s->sequence); | |
212 | } | |
213 | ||
5cdd2557 | 214 | static inline bool __seqprop_preemptible(const seqcount_t *s) |
55f3560d AD |
215 | { |
216 | return false; | |
217 | } | |
218 | ||
5cdd2557 | 219 | static inline void __seqprop_assert(const seqcount_t *s) |
55f3560d AD |
220 | { |
221 | lockdep_assert_preemption_disabled(); | |
222 | } | |
223 | ||
8117ab50 AD |
224 | #define __SEQ_RT IS_ENABLED(CONFIG_PREEMPT_RT) |
225 | ||
f995443f ON |
226 | SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t, false, raw_spin) |
227 | SEQCOUNT_LOCKNAME(spinlock, spinlock_t, __SEQ_RT, spin) | |
228 | SEQCOUNT_LOCKNAME(rwlock, rwlock_t, __SEQ_RT, read) | |
229 | SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) | |
f038cc13 | 230 | #undef SEQCOUNT_LOCKNAME |
a8772dcc | 231 | |
a28e884b | 232 | /* |
0efc94c5 PZ |
233 | * SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t |
234 | * @name: Name of the seqcount_LOCKNAME_t instance | |
6dd699b1 | 235 | * @lock: Pointer to the associated LOCKNAME |
0efc94c5 PZ |
236 | */ |
237 | ||
6dd699b1 | 238 | #define SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock) { \ |
0efc94c5 PZ |
239 | .seqcount = SEQCNT_ZERO(seq_name.seqcount), \ |
240 | __SEQ_LOCK(.lock = (assoc_lock)) \ | |
241 | } | |
242 | ||
6dd699b1 | 243 | #define SEQCNT_RAW_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) |
267580db | 244 | #define SEQCNT_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) |
6dd699b1 AD |
245 | #define SEQCNT_RWLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) |
246 | #define SEQCNT_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) | |
247 | #define SEQCNT_WW_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock) | |
0efc94c5 | 248 | |
a8772dcc | 249 | #define __seqprop_case(s, lockname, prop) \ |
e6115c6f | 250 | seqcount_##lockname##_t: __seqprop_##lockname##_##prop |
55f3560d AD |
251 | |
252 | #define __seqprop(s, prop) _Generic(*(s), \ | |
e6115c6f | 253 | seqcount_t: __seqprop_##prop, \ |
55f3560d AD |
254 | __seqprop_case((s), raw_spinlock, prop), \ |
255 | __seqprop_case((s), spinlock, prop), \ | |
256 | __seqprop_case((s), rwlock, prop), \ | |
e84815cb | 257 | __seqprop_case((s), mutex, prop)) |
55f3560d | 258 | |
e6115c6f | 259 | #define seqprop_ptr(s) __seqprop(s, ptr)(s) |
886ee55e | 260 | #define seqprop_const_ptr(s) __seqprop(s, const_ptr)(s) |
e6115c6f ON |
261 | #define seqprop_sequence(s) __seqprop(s, sequence)(s) |
262 | #define seqprop_preemptible(s) __seqprop(s, preemptible)(s) | |
263 | #define seqprop_assert(s) __seqprop(s, assert)(s) | |
55f3560d | 264 | |
3c22cd57 | 265 | /** |
89b88845 | 266 | * __read_seqcount_begin() - begin a seqcount_t read section w/o barrier |
6dd699b1 | 267 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
3c22cd57 NP |
268 | * |
269 | * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() | |
270 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | |
271 | * provided before actually loading any of the variables that are to be | |
272 | * protected in this critical section. | |
273 | * | |
274 | * Use carefully, only in critical code, and comment how the barrier is | |
275 | * provided. | |
89b88845 AD |
276 | * |
277 | * Return: count to be passed to read_seqcount_retry() | |
3c22cd57 | 278 | */ |
55f3560d | 279 | #define __read_seqcount_begin(s) \ |
52ac39e5 | 280 | ({ \ |
a07c4531 | 281 | unsigned __seq; \ |
52ac39e5 | 282 | \ |
ab440b2c | 283 | while ((__seq = seqprop_sequence(s)) & 1) \ |
52ac39e5 AD |
284 | cpu_relax(); \ |
285 | \ | |
286 | kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \ | |
a07c4531 | 287 | __seq; \ |
52ac39e5 | 288 | }) |
1da177e4 | 289 | |
1ca7d67c | 290 | /** |
89b88845 | 291 | * raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep |
6dd699b1 | 292 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
1ca7d67c | 293 | * |
89b88845 | 294 | * Return: count to be passed to read_seqcount_retry() |
1ca7d67c | 295 | */ |
55f3560d | 296 | #define raw_read_seqcount_begin(s) \ |
52ac39e5 | 297 | ({ \ |
a07c4531 | 298 | unsigned _seq = __read_seqcount_begin(s); \ |
52ac39e5 AD |
299 | \ |
300 | smp_rmb(); \ | |
a07c4531 | 301 | _seq; \ |
52ac39e5 | 302 | }) |
1ca7d67c | 303 | |
3c22cd57 | 304 | /** |
89b88845 | 305 | * read_seqcount_begin() - begin a seqcount_t read critical section |
6dd699b1 | 306 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
3c22cd57 | 307 | * |
89b88845 | 308 | * Return: count to be passed to read_seqcount_retry() |
3c22cd57 | 309 | */ |
55f3560d | 310 | #define read_seqcount_begin(s) \ |
52ac39e5 | 311 | ({ \ |
886ee55e | 312 | seqcount_lockdep_reader_access(seqprop_const_ptr(s)); \ |
52ac39e5 AD |
313 | raw_read_seqcount_begin(s); \ |
314 | }) | |
3c22cd57 | 315 | |
f4a27cbc | 316 | /** |
89b88845 | 317 | * raw_read_seqcount() - read the raw seqcount_t counter value |
6dd699b1 | 318 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
f4a27cbc AD |
319 | * |
320 | * raw_read_seqcount opens a read critical section of the given | |
89b88845 AD |
321 | * seqcount_t, without any lockdep checking, and without checking or |
322 | * masking the sequence counter LSB. Calling code is responsible for | |
323 | * handling that. | |
324 | * | |
325 | * Return: count to be passed to read_seqcount_retry() | |
f4a27cbc | 326 | */ |
55f3560d | 327 | #define raw_read_seqcount(s) \ |
52ac39e5 | 328 | ({ \ |
ab440b2c | 329 | unsigned __seq = seqprop_sequence(s); \ |
52ac39e5 AD |
330 | \ |
331 | smp_rmb(); \ | |
332 | kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \ | |
a07c4531 | 333 | __seq; \ |
52ac39e5 | 334 | }) |
f4a27cbc | 335 | |
4f988f15 | 336 | /** |
89b88845 AD |
337 | * raw_seqcount_begin() - begin a seqcount_t read critical section w/o |
338 | * lockdep and w/o counter stabilization | |
6dd699b1 | 339 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
4f988f15 | 340 | * |
89b88845 AD |
341 | * raw_seqcount_begin opens a read critical section of the given |
342 | * seqcount_t. Unlike read_seqcount_begin(), this function will not wait | |
343 | * for the count to stabilize. If a writer is active when it begins, it | |
344 | * will fail the read_seqcount_retry() at the end of the read critical | |
345 | * section instead of stabilizing at the beginning of it. | |
4f988f15 | 346 | * |
89b88845 AD |
347 | * Use this only in special kernel hot paths where the read section is |
348 | * small and has a high probability of success through other external | |
349 | * means. It will save a single branching instruction. | |
350 | * | |
351 | * Return: count to be passed to read_seqcount_retry() | |
4f988f15 | 352 | */ |
55f3560d | 353 | #define raw_seqcount_begin(s) \ |
52ac39e5 AD |
354 | ({ \ |
355 | /* \ | |
356 | * If the counter is odd, let read_seqcount_retry() fail \ | |
357 | * by decrementing the counter. \ | |
358 | */ \ | |
359 | raw_read_seqcount(s) & ~1; \ | |
360 | }) | |
4f988f15 | 361 | |
3c22cd57 | 362 | /** |
89b88845 | 363 | * __read_seqcount_retry() - end a seqcount_t read section w/o barrier |
6dd699b1 | 364 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 365 | * @start: count, from read_seqcount_begin() |
3c22cd57 NP |
366 | * |
367 | * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() | |
368 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | |
369 | * provided before actually loading any of the variables that are to be | |
370 | * protected in this critical section. | |
371 | * | |
372 | * Use carefully, only in critical code, and comment how the barrier is | |
373 | * provided. | |
89b88845 AD |
374 | * |
375 | * Return: true if a read section retry is required, else false | |
3c22cd57 | 376 | */ |
55f3560d | 377 | #define __read_seqcount_retry(s, start) \ |
886ee55e | 378 | do___read_seqcount_retry(seqprop_const_ptr(s), start) |
55f3560d | 379 | |
66bcfcdf | 380 | static inline int do___read_seqcount_retry(const seqcount_t *s, unsigned start) |
3c22cd57 | 381 | { |
88ecd153 ME |
382 | kcsan_atomic_next(0); |
383 | return unlikely(READ_ONCE(s->sequence) != start); | |
3c22cd57 NP |
384 | } |
385 | ||
386 | /** | |
89b88845 | 387 | * read_seqcount_retry() - end a seqcount_t read critical section |
6dd699b1 | 388 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 389 | * @start: count, from read_seqcount_begin() |
3c22cd57 | 390 | * |
89b88845 AD |
391 | * read_seqcount_retry closes the read critical section of given |
392 | * seqcount_t. If the critical section was invalid, it must be ignored | |
393 | * (and typically retried). | |
394 | * | |
395 | * Return: true if a read section retry is required, else false | |
1da177e4 | 396 | */ |
55f3560d | 397 | #define read_seqcount_retry(s, start) \ |
886ee55e | 398 | do_read_seqcount_retry(seqprop_const_ptr(s), start) |
55f3560d | 399 | |
66bcfcdf | 400 | static inline int do_read_seqcount_retry(const seqcount_t *s, unsigned start) |
1da177e4 LT |
401 | { |
402 | smp_rmb(); | |
66bcfcdf | 403 | return do___read_seqcount_retry(s, start); |
1da177e4 LT |
404 | } |
405 | ||
89b88845 AD |
406 | /** |
407 | * raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep | |
6dd699b1 | 408 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
cb262935 AD |
409 | * |
410 | * Context: check write_seqcount_begin() | |
89b88845 | 411 | */ |
55f3560d AD |
412 | #define raw_write_seqcount_begin(s) \ |
413 | do { \ | |
ab440b2c | 414 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
415 | preempt_disable(); \ |
416 | \ | |
66bcfcdf | 417 | do_raw_write_seqcount_begin(seqprop_ptr(s)); \ |
55f3560d AD |
418 | } while (0) |
419 | ||
66bcfcdf | 420 | static inline void do_raw_write_seqcount_begin(seqcount_t *s) |
0c3351d4 | 421 | { |
88ecd153 | 422 | kcsan_nestable_atomic_begin(); |
0c3351d4 JS |
423 | s->sequence++; |
424 | smp_wmb(); | |
425 | } | |
426 | ||
89b88845 AD |
427 | /** |
428 | * raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep | |
6dd699b1 | 429 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
cb262935 AD |
430 | * |
431 | * Context: check write_seqcount_end() | |
89b88845 | 432 | */ |
55f3560d AD |
433 | #define raw_write_seqcount_end(s) \ |
434 | do { \ | |
66bcfcdf | 435 | do_raw_write_seqcount_end(seqprop_ptr(s)); \ |
55f3560d | 436 | \ |
ab440b2c | 437 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
438 | preempt_enable(); \ |
439 | } while (0) | |
440 | ||
66bcfcdf | 441 | static inline void do_raw_write_seqcount_end(seqcount_t *s) |
0c3351d4 JS |
442 | { |
443 | smp_wmb(); | |
444 | s->sequence++; | |
88ecd153 | 445 | kcsan_nestable_atomic_end(); |
0c3351d4 JS |
446 | } |
447 | ||
89b88845 AD |
448 | /** |
449 | * write_seqcount_begin_nested() - start a seqcount_t write section with | |
450 | * custom lockdep nesting level | |
6dd699b1 | 451 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 AD |
452 | * @subclass: lockdep nesting level |
453 | * | |
454 | * See Documentation/locking/lockdep-design.rst | |
cb262935 | 455 | * Context: check write_seqcount_begin() |
89b88845 | 456 | */ |
55f3560d AD |
457 | #define write_seqcount_begin_nested(s, subclass) \ |
458 | do { \ | |
ab440b2c | 459 | seqprop_assert(s); \ |
55f3560d | 460 | \ |
ab440b2c | 461 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
462 | preempt_disable(); \ |
463 | \ | |
66bcfcdf | 464 | do_write_seqcount_begin_nested(seqprop_ptr(s), subclass); \ |
55f3560d | 465 | } while (0) |
859247d3 | 466 | |
66bcfcdf | 467 | static inline void do_write_seqcount_begin_nested(seqcount_t *s, int subclass) |
859247d3 | 468 | { |
55f3560d | 469 | seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); |
41b43b6c | 470 | do_raw_write_seqcount_begin(s); |
f4a27cbc AD |
471 | } |
472 | ||
89b88845 AD |
473 | /** |
474 | * write_seqcount_begin() - start a seqcount_t write side critical section | |
6dd699b1 | 475 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 476 | * |
cb262935 AD |
477 | * Context: sequence counter write side sections must be serialized and |
478 | * non-preemptible. Preemption will be automatically disabled if and | |
479 | * only if the seqcount write serialization lock is associated, and | |
480 | * preemptible. If readers can be invoked from hardirq or softirq | |
89b88845 AD |
481 | * context, interrupts or bottom halves must be respectively disabled. |
482 | */ | |
55f3560d AD |
483 | #define write_seqcount_begin(s) \ |
484 | do { \ | |
ab440b2c | 485 | seqprop_assert(s); \ |
55f3560d | 486 | \ |
ab440b2c | 487 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
488 | preempt_disable(); \ |
489 | \ | |
66bcfcdf | 490 | do_write_seqcount_begin(seqprop_ptr(s)); \ |
55f3560d AD |
491 | } while (0) |
492 | ||
66bcfcdf | 493 | static inline void do_write_seqcount_begin(seqcount_t *s) |
f4a27cbc | 494 | { |
66bcfcdf | 495 | do_write_seqcount_begin_nested(s, 0); |
f4a27cbc AD |
496 | } |
497 | ||
89b88845 AD |
498 | /** |
499 | * write_seqcount_end() - end a seqcount_t write side critical section | |
6dd699b1 | 500 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
89b88845 | 501 | * |
cb262935 AD |
502 | * Context: Preemption will be automatically re-enabled if and only if |
503 | * the seqcount write serialization lock is associated, and preemptible. | |
89b88845 | 504 | */ |
55f3560d AD |
505 | #define write_seqcount_end(s) \ |
506 | do { \ | |
66bcfcdf | 507 | do_write_seqcount_end(seqprop_ptr(s)); \ |
55f3560d | 508 | \ |
ab440b2c | 509 | if (seqprop_preemptible(s)) \ |
55f3560d AD |
510 | preempt_enable(); \ |
511 | } while (0) | |
512 | ||
66bcfcdf | 513 | static inline void do_write_seqcount_end(seqcount_t *s) |
f4a27cbc AD |
514 | { |
515 | seqcount_release(&s->dep_map, _RET_IP_); | |
66bcfcdf | 516 | do_raw_write_seqcount_end(s); |
f4a27cbc AD |
517 | } |
518 | ||
c4bfa3f5 | 519 | /** |
89b88845 | 520 | * raw_write_seqcount_barrier() - do a seqcount_t write barrier |
6dd699b1 | 521 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
c4bfa3f5 | 522 | * |
89b88845 AD |
523 | * This can be used to provide an ordering guarantee instead of the usual |
524 | * consistency guarantee. It is one wmb cheaper, because it can collapse | |
525 | * the two back-to-back wmb()s. | |
c4bfa3f5 | 526 | * |
5cbaefe9 | 527 | * Note that writes surrounding the barrier should be declared atomic (e.g. |
bf07132f ME |
528 | * via WRITE_ONCE): a) to ensure the writes become visible to other threads |
529 | * atomically, avoiding compiler optimizations; b) to document which writes are | |
530 | * meant to propagate to the reader critical section. This is necessary because | |
184fdf9f | 531 | * neither writes before nor after the barrier are enclosed in a seq-writer |
15cbe67b | 532 | * critical section that would ensure readers are aware of ongoing writes:: |
bf07132f | 533 | * |
15cbe67b AD |
534 | * seqcount_t seq; |
535 | * bool X = true, Y = false; | |
c4bfa3f5 | 536 | * |
15cbe67b AD |
537 | * void read(void) |
538 | * { | |
539 | * bool x, y; | |
c4bfa3f5 | 540 | * |
15cbe67b AD |
541 | * do { |
542 | * int s = read_seqcount_begin(&seq); | |
c4bfa3f5 | 543 | * |
15cbe67b | 544 | * x = X; y = Y; |
c4bfa3f5 | 545 | * |
15cbe67b | 546 | * } while (read_seqcount_retry(&seq, s)); |
c4bfa3f5 | 547 | * |
15cbe67b | 548 | * BUG_ON(!x && !y); |
c4bfa3f5 PZ |
549 | * } |
550 | * | |
551 | * void write(void) | |
552 | * { | |
15cbe67b | 553 | * WRITE_ONCE(Y, true); |
c4bfa3f5 | 554 | * |
15cbe67b | 555 | * raw_write_seqcount_barrier(seq); |
c4bfa3f5 | 556 | * |
15cbe67b | 557 | * WRITE_ONCE(X, false); |
c4bfa3f5 PZ |
558 | * } |
559 | */ | |
55f3560d | 560 | #define raw_write_seqcount_barrier(s) \ |
66bcfcdf | 561 | do_raw_write_seqcount_barrier(seqprop_ptr(s)) |
55f3560d | 562 | |
66bcfcdf | 563 | static inline void do_raw_write_seqcount_barrier(seqcount_t *s) |
c4bfa3f5 | 564 | { |
88ecd153 | 565 | kcsan_nestable_atomic_begin(); |
c4bfa3f5 PZ |
566 | s->sequence++; |
567 | smp_wmb(); | |
568 | s->sequence++; | |
88ecd153 | 569 | kcsan_nestable_atomic_end(); |
c4bfa3f5 PZ |
570 | } |
571 | ||
f4a27cbc | 572 | /** |
89b88845 AD |
573 | * write_seqcount_invalidate() - invalidate in-progress seqcount_t read |
574 | * side operations | |
6dd699b1 | 575 | * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants |
f4a27cbc | 576 | * |
89b88845 AD |
577 | * After write_seqcount_invalidate, no seqcount_t read side operations |
578 | * will complete successfully and see data older than this. | |
f4a27cbc | 579 | */ |
55f3560d | 580 | #define write_seqcount_invalidate(s) \ |
66bcfcdf | 581 | do_write_seqcount_invalidate(seqprop_ptr(s)) |
55f3560d | 582 | |
66bcfcdf | 583 | static inline void do_write_seqcount_invalidate(seqcount_t *s) |
f4a27cbc AD |
584 | { |
585 | smp_wmb(); | |
586 | kcsan_nestable_atomic_begin(); | |
587 | s->sequence+=2; | |
588 | kcsan_nestable_atomic_end(); | |
589 | } | |
590 | ||
80793c34 AD |
591 | /* |
592 | * Latch sequence counters (seqcount_latch_t) | |
89b88845 | 593 | * |
80793c34 AD |
594 | * A sequence counter variant where the counter even/odd value is used to |
595 | * switch between two copies of protected data. This allows the read path, | |
596 | * typically NMIs, to safely interrupt the write side critical section. | |
89b88845 | 597 | * |
80793c34 AD |
598 | * As the write sections are fully preemptible, no special handling for |
599 | * PREEMPT_RT is needed. | |
600 | */ | |
601 | typedef struct { | |
602 | seqcount_t seqcount; | |
603 | } seqcount_latch_t; | |
604 | ||
605 | /** | |
606 | * SEQCNT_LATCH_ZERO() - static initializer for seqcount_latch_t | |
607 | * @seq_name: Name of the seqcount_latch_t instance | |
608 | */ | |
609 | #define SEQCNT_LATCH_ZERO(seq_name) { \ | |
610 | .seqcount = SEQCNT_ZERO(seq_name.seqcount), \ | |
611 | } | |
612 | ||
613 | /** | |
614 | * seqcount_latch_init() - runtime initializer for seqcount_latch_t | |
615 | * @s: Pointer to the seqcount_latch_t instance | |
616 | */ | |
4817a52b | 617 | #define seqcount_latch_init(s) seqcount_init(&(s)->seqcount) |
80793c34 AD |
618 | |
619 | /** | |
620 | * raw_read_seqcount_latch() - pick even/odd latch data copy | |
0c9794c8 | 621 | * @s: Pointer to seqcount_latch_t |
80793c34 AD |
622 | * |
623 | * See raw_write_seqcount_latch() for details and a full reader/writer | |
624 | * usage example. | |
89b88845 AD |
625 | * |
626 | * Return: sequence counter raw value. Use the lowest bit as an index for | |
80793c34 | 627 | * picking which data copy to read. The full counter must then be checked |
d16317de | 628 | * with raw_read_seqcount_latch_retry(). |
89b88845 | 629 | */ |
d16317de | 630 | static __always_inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s) |
0c9794c8 AD |
631 | { |
632 | /* | |
633 | * Pairs with the first smp_wmb() in raw_write_seqcount_latch(). | |
634 | * Due to the dependent load, a full smp_rmb() is not needed. | |
635 | */ | |
636 | return READ_ONCE(s->seqcount.sequence); | |
637 | } | |
55f3560d | 638 | |
80793c34 | 639 | /** |
d16317de | 640 | * raw_read_seqcount_latch_retry() - end a seqcount_latch_t read section |
80793c34 AD |
641 | * @s: Pointer to seqcount_latch_t |
642 | * @start: count, from raw_read_seqcount_latch() | |
643 | * | |
644 | * Return: true if a read section retry is required, else false | |
645 | */ | |
d16317de PZ |
646 | static __always_inline int |
647 | raw_read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) | |
7fc26327 | 648 | { |
d16317de PZ |
649 | smp_rmb(); |
650 | return unlikely(READ_ONCE(s->seqcount.sequence) != start); | |
7fc26327 PZ |
651 | } |
652 | ||
6695b92a | 653 | /** |
80793c34 | 654 | * raw_write_seqcount_latch() - redirect latch readers to even/odd copy |
0c9794c8 | 655 | * @s: Pointer to seqcount_latch_t |
6695b92a PZ |
656 | * |
657 | * The latch technique is a multiversion concurrency control method that allows | |
658 | * queries during non-atomic modifications. If you can guarantee queries never | |
659 | * interrupt the modification -- e.g. the concurrency is strictly between CPUs | |
660 | * -- you most likely do not need this. | |
661 | * | |
662 | * Where the traditional RCU/lockless data structures rely on atomic | |
663 | * modifications to ensure queries observe either the old or the new state the | |
664 | * latch allows the same for non-atomic updates. The trade-off is doubling the | |
665 | * cost of storage; we have to maintain two copies of the entire data | |
666 | * structure. | |
667 | * | |
668 | * Very simply put: we first modify one copy and then the other. This ensures | |
669 | * there is always one copy in a stable state, ready to give us an answer. | |
670 | * | |
15cbe67b | 671 | * The basic form is a data structure like:: |
6695b92a | 672 | * |
15cbe67b | 673 | * struct latch_struct { |
80793c34 | 674 | * seqcount_latch_t seq; |
15cbe67b AD |
675 | * struct data_struct data[2]; |
676 | * }; | |
6695b92a PZ |
677 | * |
678 | * Where a modification, which is assumed to be externally serialized, does the | |
15cbe67b | 679 | * following:: |
6695b92a | 680 | * |
15cbe67b AD |
681 | * void latch_modify(struct latch_struct *latch, ...) |
682 | * { | |
683 | * smp_wmb(); // Ensure that the last data[1] update is visible | |
80793c34 | 684 | * latch->seq.sequence++; |
15cbe67b | 685 | * smp_wmb(); // Ensure that the seqcount update is visible |
6695b92a | 686 | * |
15cbe67b | 687 | * modify(latch->data[0], ...); |
6695b92a | 688 | * |
15cbe67b | 689 | * smp_wmb(); // Ensure that the data[0] update is visible |
80793c34 | 690 | * latch->seq.sequence++; |
15cbe67b | 691 | * smp_wmb(); // Ensure that the seqcount update is visible |
6695b92a | 692 | * |
15cbe67b AD |
693 | * modify(latch->data[1], ...); |
694 | * } | |
6695b92a | 695 | * |
15cbe67b | 696 | * The query will have a form like:: |
6695b92a | 697 | * |
15cbe67b AD |
698 | * struct entry *latch_query(struct latch_struct *latch, ...) |
699 | * { | |
700 | * struct entry *entry; | |
701 | * unsigned seq, idx; | |
6695b92a | 702 | * |
15cbe67b AD |
703 | * do { |
704 | * seq = raw_read_seqcount_latch(&latch->seq); | |
6695b92a | 705 | * |
15cbe67b AD |
706 | * idx = seq & 0x01; |
707 | * entry = data_query(latch->data[idx], ...); | |
6695b92a | 708 | * |
80793c34 | 709 | * // This includes needed smp_rmb() |
d16317de | 710 | * } while (raw_read_seqcount_latch_retry(&latch->seq, seq)); |
6695b92a | 711 | * |
15cbe67b AD |
712 | * return entry; |
713 | * } | |
6695b92a PZ |
714 | * |
715 | * So during the modification, queries are first redirected to data[1]. Then we | |
716 | * modify data[0]. When that is complete, we redirect queries back to data[0] | |
717 | * and we can modify data[1]. | |
718 | * | |
15cbe67b AD |
719 | * NOTE: |
720 | * | |
721 | * The non-requirement for atomic modifications does _NOT_ include | |
722 | * the publishing of new entries in the case where data is a dynamic | |
723 | * data structure. | |
724 | * | |
725 | * An iteration might start in data[0] and get suspended long enough | |
726 | * to miss an entire modification sequence, once it resumes it might | |
727 | * observe the new entry. | |
6695b92a | 728 | * |
a28e884b | 729 | * NOTE2: |
6695b92a | 730 | * |
15cbe67b AD |
731 | * When data is a dynamic data structure; one should use regular RCU |
732 | * patterns to manage the lifetimes of the objects within. | |
9b0fd802 | 733 | */ |
0c9794c8 AD |
734 | static inline void raw_write_seqcount_latch(seqcount_latch_t *s) |
735 | { | |
736 | smp_wmb(); /* prior stores before incrementing "sequence" */ | |
737 | s->seqcount.sequence++; | |
738 | smp_wmb(); /* increment "sequence" before following stores */ | |
9b0fd802 MD |
739 | } |
740 | ||
ec8702da AD |
741 | #define __SEQLOCK_UNLOCKED(lockname) \ |
742 | { \ | |
1909760f | 743 | .seqcount = SEQCNT_SPINLOCK_ZERO(lockname, &(lockname).lock), \ |
ec8702da | 744 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ |
6617feca TG |
745 | } |
746 | ||
89b88845 AD |
747 | /** |
748 | * seqlock_init() - dynamic initializer for seqlock_t | |
749 | * @sl: Pointer to the seqlock_t instance | |
750 | */ | |
ec8702da AD |
751 | #define seqlock_init(sl) \ |
752 | do { \ | |
ec8702da | 753 | spin_lock_init(&(sl)->lock); \ |
1909760f | 754 | seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock); \ |
6617feca TG |
755 | } while (0) |
756 | ||
89b88845 | 757 | /** |
24a18772 | 758 | * DEFINE_SEQLOCK(sl) - Define a statically allocated seqlock_t |
89b88845 AD |
759 | * @sl: Name of the seqlock_t instance |
760 | */ | |
761 | #define DEFINE_SEQLOCK(sl) \ | |
762 | seqlock_t sl = __SEQLOCK_UNLOCKED(sl) | |
6617feca | 763 | |
89b88845 AD |
764 | /** |
765 | * read_seqbegin() - start a seqlock_t read side critical section | |
766 | * @sl: Pointer to seqlock_t | |
767 | * | |
768 | * Return: count, to be passed to read_seqretry() | |
6617feca TG |
769 | */ |
770 | static inline unsigned read_seqbegin(const seqlock_t *sl) | |
771 | { | |
88ecd153 ME |
772 | unsigned ret = read_seqcount_begin(&sl->seqcount); |
773 | ||
5cbaefe9 | 774 | kcsan_atomic_next(0); /* non-raw usage, assume closing read_seqretry() */ |
88ecd153 ME |
775 | kcsan_flat_atomic_begin(); |
776 | return ret; | |
6617feca TG |
777 | } |
778 | ||
89b88845 AD |
779 | /** |
780 | * read_seqretry() - end a seqlock_t read side section | |
781 | * @sl: Pointer to seqlock_t | |
782 | * @start: count, from read_seqbegin() | |
783 | * | |
784 | * read_seqretry closes the read side critical section of given seqlock_t. | |
785 | * If the critical section was invalid, it must be ignored (and typically | |
786 | * retried). | |
787 | * | |
788 | * Return: true if a read section retry is required, else false | |
789 | */ | |
6617feca TG |
790 | static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) |
791 | { | |
88ecd153 | 792 | /* |
5cbaefe9 | 793 | * Assume not nested: read_seqretry() may be called multiple times when |
88ecd153 ME |
794 | * completing read critical section. |
795 | */ | |
796 | kcsan_flat_atomic_end(); | |
797 | ||
6617feca TG |
798 | return read_seqcount_retry(&sl->seqcount, start); |
799 | } | |
800 | ||
1909760f | 801 | /* |
0cff993e | 802 | * For all seqlock_t write side functions, use the internal |
66bcfcdf AD |
803 | * do_write_seqcount_begin() instead of generic write_seqcount_begin(). |
804 | * This way, no redundant lockdep_assert_held() checks are added. | |
1909760f AD |
805 | */ |
806 | ||
89b88845 AD |
807 | /** |
808 | * write_seqlock() - start a seqlock_t write side critical section | |
809 | * @sl: Pointer to seqlock_t | |
810 | * | |
811 | * write_seqlock opens a write side critical section for the given | |
812 | * seqlock_t. It also implicitly acquires the spinlock_t embedded inside | |
813 | * that sequential lock. All seqlock_t write side sections are thus | |
814 | * automatically serialized and non-preemptible. | |
815 | * | |
816 | * Context: if the seqlock_t read section, or other write side critical | |
817 | * sections, can be invoked from hardirq or softirq contexts, use the | |
818 | * _irqsave or _bh variants of this function instead. | |
1da177e4 | 819 | */ |
6617feca TG |
820 | static inline void write_seqlock(seqlock_t *sl) |
821 | { | |
822 | spin_lock(&sl->lock); | |
66bcfcdf | 823 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
824 | } |
825 | ||
89b88845 AD |
826 | /** |
827 | * write_sequnlock() - end a seqlock_t write side critical section | |
828 | * @sl: Pointer to seqlock_t | |
829 | * | |
830 | * write_sequnlock closes the (serialized and non-preemptible) write side | |
831 | * critical section of given seqlock_t. | |
832 | */ | |
6617feca TG |
833 | static inline void write_sequnlock(seqlock_t *sl) |
834 | { | |
66bcfcdf | 835 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
836 | spin_unlock(&sl->lock); |
837 | } | |
838 | ||
89b88845 AD |
839 | /** |
840 | * write_seqlock_bh() - start a softirqs-disabled seqlock_t write section | |
841 | * @sl: Pointer to seqlock_t | |
842 | * | |
843 | * _bh variant of write_seqlock(). Use only if the read side section, or | |
844 | * other write side sections, can be invoked from softirq contexts. | |
845 | */ | |
6617feca TG |
846 | static inline void write_seqlock_bh(seqlock_t *sl) |
847 | { | |
848 | spin_lock_bh(&sl->lock); | |
66bcfcdf | 849 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
850 | } |
851 | ||
89b88845 AD |
852 | /** |
853 | * write_sequnlock_bh() - end a softirqs-disabled seqlock_t write section | |
854 | * @sl: Pointer to seqlock_t | |
855 | * | |
856 | * write_sequnlock_bh closes the serialized, non-preemptible, and | |
857 | * softirqs-disabled, seqlock_t write side critical section opened with | |
858 | * write_seqlock_bh(). | |
859 | */ | |
6617feca TG |
860 | static inline void write_sequnlock_bh(seqlock_t *sl) |
861 | { | |
66bcfcdf | 862 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
863 | spin_unlock_bh(&sl->lock); |
864 | } | |
865 | ||
89b88845 AD |
866 | /** |
867 | * write_seqlock_irq() - start a non-interruptible seqlock_t write section | |
868 | * @sl: Pointer to seqlock_t | |
869 | * | |
870 | * _irq variant of write_seqlock(). Use only if the read side section, or | |
871 | * other write sections, can be invoked from hardirq contexts. | |
872 | */ | |
6617feca TG |
873 | static inline void write_seqlock_irq(seqlock_t *sl) |
874 | { | |
875 | spin_lock_irq(&sl->lock); | |
66bcfcdf | 876 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
877 | } |
878 | ||
89b88845 AD |
879 | /** |
880 | * write_sequnlock_irq() - end a non-interruptible seqlock_t write section | |
881 | * @sl: Pointer to seqlock_t | |
882 | * | |
883 | * write_sequnlock_irq closes the serialized and non-interruptible | |
884 | * seqlock_t write side section opened with write_seqlock_irq(). | |
885 | */ | |
6617feca TG |
886 | static inline void write_sequnlock_irq(seqlock_t *sl) |
887 | { | |
66bcfcdf | 888 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
889 | spin_unlock_irq(&sl->lock); |
890 | } | |
891 | ||
892 | static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl) | |
893 | { | |
894 | unsigned long flags; | |
895 | ||
896 | spin_lock_irqsave(&sl->lock, flags); | |
66bcfcdf | 897 | do_write_seqcount_begin(&sl->seqcount.seqcount); |
6617feca TG |
898 | return flags; |
899 | } | |
900 | ||
89b88845 AD |
901 | /** |
902 | * write_seqlock_irqsave() - start a non-interruptible seqlock_t write | |
903 | * section | |
904 | * @lock: Pointer to seqlock_t | |
905 | * @flags: Stack-allocated storage for saving caller's local interrupt | |
906 | * state, to be passed to write_sequnlock_irqrestore(). | |
907 | * | |
908 | * _irqsave variant of write_seqlock(). Use it only if the read side | |
909 | * section, or other write sections, can be invoked from hardirq context. | |
910 | */ | |
1da177e4 | 911 | #define write_seqlock_irqsave(lock, flags) \ |
6617feca | 912 | do { flags = __write_seqlock_irqsave(lock); } while (0) |
1da177e4 | 913 | |
89b88845 AD |
914 | /** |
915 | * write_sequnlock_irqrestore() - end non-interruptible seqlock_t write | |
916 | * section | |
917 | * @sl: Pointer to seqlock_t | |
918 | * @flags: Caller's saved interrupt state, from write_seqlock_irqsave() | |
919 | * | |
920 | * write_sequnlock_irqrestore closes the serialized and non-interruptible | |
921 | * seqlock_t write section previously opened with write_seqlock_irqsave(). | |
922 | */ | |
6617feca TG |
923 | static inline void |
924 | write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) | |
925 | { | |
66bcfcdf | 926 | do_write_seqcount_end(&sl->seqcount.seqcount); |
6617feca TG |
927 | spin_unlock_irqrestore(&sl->lock, flags); |
928 | } | |
1da177e4 | 929 | |
89b88845 AD |
930 | /** |
931 | * read_seqlock_excl() - begin a seqlock_t locking reader section | |
55f3560d | 932 | * @sl: Pointer to seqlock_t |
89b88845 AD |
933 | * |
934 | * read_seqlock_excl opens a seqlock_t locking reader critical section. A | |
935 | * locking reader exclusively locks out *both* other writers *and* other | |
936 | * locking readers, but it does not update the embedded sequence number. | |
937 | * | |
938 | * Locking readers act like a normal spin_lock()/spin_unlock(). | |
939 | * | |
940 | * Context: if the seqlock_t write section, *or other read sections*, can | |
941 | * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | |
942 | * variant of this function instead. | |
943 | * | |
944 | * The opened read section must be closed with read_sequnlock_excl(). | |
1370e97b WL |
945 | */ |
946 | static inline void read_seqlock_excl(seqlock_t *sl) | |
947 | { | |
948 | spin_lock(&sl->lock); | |
949 | } | |
950 | ||
89b88845 AD |
951 | /** |
952 | * read_sequnlock_excl() - end a seqlock_t locking reader critical section | |
953 | * @sl: Pointer to seqlock_t | |
954 | */ | |
1370e97b WL |
955 | static inline void read_sequnlock_excl(seqlock_t *sl) |
956 | { | |
957 | spin_unlock(&sl->lock); | |
958 | } | |
959 | ||
89b88845 AD |
960 | /** |
961 | * read_seqlock_excl_bh() - start a seqlock_t locking reader section with | |
962 | * softirqs disabled | |
963 | * @sl: Pointer to seqlock_t | |
964 | * | |
965 | * _bh variant of read_seqlock_excl(). Use this variant only if the | |
966 | * seqlock_t write side section, *or other read sections*, can be invoked | |
967 | * from softirq contexts. | |
968 | */ | |
1370e97b WL |
969 | static inline void read_seqlock_excl_bh(seqlock_t *sl) |
970 | { | |
971 | spin_lock_bh(&sl->lock); | |
972 | } | |
973 | ||
89b88845 AD |
974 | /** |
975 | * read_sequnlock_excl_bh() - stop a seqlock_t softirq-disabled locking | |
976 | * reader section | |
977 | * @sl: Pointer to seqlock_t | |
978 | */ | |
1370e97b WL |
979 | static inline void read_sequnlock_excl_bh(seqlock_t *sl) |
980 | { | |
981 | spin_unlock_bh(&sl->lock); | |
982 | } | |
983 | ||
89b88845 AD |
984 | /** |
985 | * read_seqlock_excl_irq() - start a non-interruptible seqlock_t locking | |
986 | * reader section | |
987 | * @sl: Pointer to seqlock_t | |
988 | * | |
989 | * _irq variant of read_seqlock_excl(). Use this only if the seqlock_t | |
990 | * write side section, *or other read sections*, can be invoked from a | |
991 | * hardirq context. | |
992 | */ | |
1370e97b WL |
993 | static inline void read_seqlock_excl_irq(seqlock_t *sl) |
994 | { | |
995 | spin_lock_irq(&sl->lock); | |
996 | } | |
997 | ||
89b88845 AD |
998 | /** |
999 | * read_sequnlock_excl_irq() - end an interrupts-disabled seqlock_t | |
1000 | * locking reader section | |
1001 | * @sl: Pointer to seqlock_t | |
1002 | */ | |
1370e97b WL |
1003 | static inline void read_sequnlock_excl_irq(seqlock_t *sl) |
1004 | { | |
1005 | spin_unlock_irq(&sl->lock); | |
1006 | } | |
1007 | ||
1008 | static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl) | |
1009 | { | |
1010 | unsigned long flags; | |
1011 | ||
1012 | spin_lock_irqsave(&sl->lock, flags); | |
1013 | return flags; | |
1014 | } | |
1015 | ||
89b88845 AD |
1016 | /** |
1017 | * read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t | |
1018 | * locking reader section | |
1019 | * @lock: Pointer to seqlock_t | |
1020 | * @flags: Stack-allocated storage for saving caller's local interrupt | |
1021 | * state, to be passed to read_sequnlock_excl_irqrestore(). | |
1022 | * | |
1023 | * _irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t | |
1024 | * write side section, *or other read sections*, can be invoked from a | |
1025 | * hardirq context. | |
1026 | */ | |
1370e97b WL |
1027 | #define read_seqlock_excl_irqsave(lock, flags) \ |
1028 | do { flags = __read_seqlock_excl_irqsave(lock); } while (0) | |
1029 | ||
89b88845 AD |
1030 | /** |
1031 | * read_sequnlock_excl_irqrestore() - end non-interruptible seqlock_t | |
1032 | * locking reader section | |
1033 | * @sl: Pointer to seqlock_t | |
1034 | * @flags: Caller saved interrupt state, from read_seqlock_excl_irqsave() | |
1035 | */ | |
1370e97b WL |
1036 | static inline void |
1037 | read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) | |
1038 | { | |
1039 | spin_unlock_irqrestore(&sl->lock, flags); | |
1040 | } | |
1041 | ||
f4a27cbc | 1042 | /** |
89b88845 AD |
1043 | * read_seqbegin_or_lock() - begin a seqlock_t lockless or locking reader |
1044 | * @lock: Pointer to seqlock_t | |
1045 | * @seq : Marker and return parameter. If the passed value is even, the | |
1046 | * reader will become a *lockless* seqlock_t reader as in read_seqbegin(). | |
1047 | * If the passed value is odd, the reader will become a *locking* reader | |
1048 | * as in read_seqlock_excl(). In the first call to this function, the | |
1049 | * caller *must* initialize and pass an even value to @seq; this way, a | |
1050 | * lockless read can be optimistically tried first. | |
1051 | * | |
1052 | * read_seqbegin_or_lock is an API designed to optimistically try a normal | |
1053 | * lockless seqlock_t read section first. If an odd counter is found, the | |
1054 | * lockless read trial has failed, and the next read iteration transforms | |
1055 | * itself into a full seqlock_t locking reader. | |
1056 | * | |
1057 | * This is typically used to avoid seqlock_t lockless readers starvation | |
1058 | * (too much retry loops) in the case of a sharp spike in write side | |
1059 | * activity. | |
1060 | * | |
1061 | * Context: if the seqlock_t write section, *or other read sections*, can | |
1062 | * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | |
1063 | * variant of this function instead. | |
1064 | * | |
1065 | * Check Documentation/locking/seqlock.rst for template example code. | |
1066 | * | |
1067 | * Return: the encountered sequence counter value, through the @seq | |
1068 | * parameter, which is overloaded as a return parameter. This returned | |
1069 | * value must be checked with need_seqretry(). If the read section need to | |
1070 | * be retried, this returned value must also be passed as the @seq | |
1071 | * parameter of the next read_seqbegin_or_lock() iteration. | |
f4a27cbc AD |
1072 | */ |
1073 | static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) | |
1074 | { | |
1075 | if (!(*seq & 1)) /* Even */ | |
1076 | *seq = read_seqbegin(lock); | |
1077 | else /* Odd */ | |
1078 | read_seqlock_excl(lock); | |
1079 | } | |
1080 | ||
89b88845 AD |
1081 | /** |
1082 | * need_seqretry() - validate seqlock_t "locking or lockless" read section | |
1083 | * @lock: Pointer to seqlock_t | |
1084 | * @seq: sequence count, from read_seqbegin_or_lock() | |
1085 | * | |
1086 | * Return: true if a read section retry is required, false otherwise | |
1087 | */ | |
f4a27cbc AD |
1088 | static inline int need_seqretry(seqlock_t *lock, int seq) |
1089 | { | |
1090 | return !(seq & 1) && read_seqretry(lock, seq); | |
1091 | } | |
1092 | ||
89b88845 AD |
1093 | /** |
1094 | * done_seqretry() - end seqlock_t "locking or lockless" reader section | |
1095 | * @lock: Pointer to seqlock_t | |
1096 | * @seq: count, from read_seqbegin_or_lock() | |
1097 | * | |
1098 | * done_seqretry finishes the seqlock_t read side critical section started | |
1099 | * with read_seqbegin_or_lock() and validated by need_seqretry(). | |
1100 | */ | |
f4a27cbc AD |
1101 | static inline void done_seqretry(seqlock_t *lock, int seq) |
1102 | { | |
1103 | if (seq & 1) | |
1104 | read_sequnlock_excl(lock); | |
1105 | } | |
1106 | ||
89b88845 AD |
1107 | /** |
1108 | * read_seqbegin_or_lock_irqsave() - begin a seqlock_t lockless reader, or | |
1109 | * a non-interruptible locking reader | |
1110 | * @lock: Pointer to seqlock_t | |
1111 | * @seq: Marker and return parameter. Check read_seqbegin_or_lock(). | |
1112 | * | |
1113 | * This is the _irqsave variant of read_seqbegin_or_lock(). Use it only if | |
1114 | * the seqlock_t write section, *or other read sections*, can be invoked | |
1115 | * from hardirq context. | |
1116 | * | |
1117 | * Note: Interrupts will be disabled only for "locking reader" mode. | |
1118 | * | |
1119 | * Return: | |
1120 | * | |
1121 | * 1. The saved local interrupts state in case of a locking reader, to | |
1122 | * be passed to done_seqretry_irqrestore(). | |
1123 | * | |
1124 | * 2. The encountered sequence counter value, returned through @seq | |
1125 | * overloaded as a return parameter. Check read_seqbegin_or_lock(). | |
1126 | */ | |
ef8ac063 RR |
1127 | static inline unsigned long |
1128 | read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) | |
1129 | { | |
1130 | unsigned long flags = 0; | |
1131 | ||
1132 | if (!(*seq & 1)) /* Even */ | |
1133 | *seq = read_seqbegin(lock); | |
1134 | else /* Odd */ | |
1135 | read_seqlock_excl_irqsave(lock, flags); | |
1136 | ||
1137 | return flags; | |
1138 | } | |
1139 | ||
89b88845 AD |
1140 | /** |
1141 | * done_seqretry_irqrestore() - end a seqlock_t lockless reader, or a | |
1142 | * non-interruptible locking reader section | |
1143 | * @lock: Pointer to seqlock_t | |
1144 | * @seq: Count, from read_seqbegin_or_lock_irqsave() | |
1145 | * @flags: Caller's saved local interrupt state in case of a locking | |
1146 | * reader, also from read_seqbegin_or_lock_irqsave() | |
1147 | * | |
1148 | * This is the _irqrestore variant of done_seqretry(). The read section | |
1149 | * must've been opened with read_seqbegin_or_lock_irqsave(), and validated | |
1150 | * by need_seqretry(). | |
1151 | */ | |
ef8ac063 RR |
1152 | static inline void |
1153 | done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags) | |
1154 | { | |
1155 | if (seq & 1) | |
1156 | read_sequnlock_excl_irqrestore(lock, flags); | |
1157 | } | |
1da177e4 | 1158 | #endif /* __LINUX_SEQLOCK_H */ |