Commit | Line | Data |
---|---|---|
8c366db0 | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
d8be8173 PM |
2 | /* |
3 | * Sleepable Read-Copy Update mechanism for mutual exclusion, | |
4 | * tiny variant. | |
5 | * | |
d8be8173 PM |
6 | * Copyright (C) IBM Corporation, 2017 |
7 | * | |
8c366db0 | 8 | * Author: Paul McKenney <paulmck@linux.ibm.com> |
d8be8173 PM |
9 | */ |
10 | ||
11 | #ifndef _LINUX_SRCU_TINY_H | |
12 | #define _LINUX_SRCU_TINY_H | |
13 | ||
14 | #include <linux/swait.h> | |
15 | ||
16 | struct srcu_struct { | |
3ddf20c9 | 17 | short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ |
3ddf20c9 PM |
18 | u8 srcu_gp_running; /* GP workqueue running? */ |
19 | u8 srcu_gp_waiting; /* GP waiting for readers? */ | |
5fe89191 PM |
20 | unsigned long srcu_idx; /* Current reader array element in bit 0x2. */ |
21 | unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */ | |
d8be8173 PM |
22 | struct swait_queue_head srcu_wq; |
23 | /* Last srcu_read_unlock() wakes GP. */ | |
2464dd94 PM |
24 | struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ |
25 | struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ | |
d8be8173 PM |
26 | struct work_struct srcu_work; /* For driving grace periods. */ |
27 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
28 | struct lockdep_map dep_map; | |
29 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
30 | }; | |
31 | ||
32 | void srcu_drive_gp(struct work_struct *wp); | |
33 | ||
9c80172b | 34 | #define __SRCU_STRUCT_INIT(name, __ignored) \ |
d8be8173 PM |
35 | { \ |
36 | .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ | |
2464dd94 | 37 | .srcu_cb_tail = &name.srcu_cb_head, \ |
d8be8173 PM |
38 | .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ |
39 | __SRCU_DEP_MAP_INIT(name) \ | |
40 | } | |
41 | ||
42 | /* | |
43 | * This odd _STATIC_ arrangement is needed for API compatibility with | |
44 | * Tree SRCU, which needs some per-CPU data. | |
45 | */ | |
46 | #define DEFINE_SRCU(name) \ | |
9c80172b | 47 | struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) |
d8be8173 | 48 | #define DEFINE_STATIC_SRCU(name) \ |
9c80172b | 49 | static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) |
d8be8173 | 50 | |
aacb5d91 | 51 | void synchronize_srcu(struct srcu_struct *ssp); |
d8be8173 | 52 | |
d4efe6c5 PM |
53 | /* |
54 | * Counts the new reader in the appropriate per-CPU element of the | |
55 | * srcu_struct. Can be invoked from irq/bh handlers, but the matching | |
56 | * __srcu_read_unlock() must be in the same handler instance. Returns an | |
57 | * index that must be passed to the matching srcu_read_unlock(). | |
58 | */ | |
aacb5d91 | 59 | static inline int __srcu_read_lock(struct srcu_struct *ssp) |
d4efe6c5 PM |
60 | { |
61 | int idx; | |
62 | ||
74612a07 | 63 | idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1; |
65bfdd36 | 64 | WRITE_ONCE(ssp->srcu_lock_nesting[idx], READ_ONCE(ssp->srcu_lock_nesting[idx]) + 1); |
d4efe6c5 PM |
65 | return idx; |
66 | } | |
67 | ||
aacb5d91 | 68 | static inline void synchronize_srcu_expedited(struct srcu_struct *ssp) |
d8be8173 | 69 | { |
aacb5d91 | 70 | synchronize_srcu(ssp); |
d8be8173 PM |
71 | } |
72 | ||
aacb5d91 | 73 | static inline void srcu_barrier(struct srcu_struct *ssp) |
d8be8173 | 74 | { |
aacb5d91 | 75 | synchronize_srcu(ssp); |
d8be8173 PM |
76 | } |
77 | ||
115a1a52 | 78 | /* Defined here to avoid size increase for non-torture kernels. */ |
aacb5d91 | 79 | static inline void srcu_torture_stats_print(struct srcu_struct *ssp, |
115a1a52 PM |
80 | char *tt, char *tf) |
81 | { | |
82 | int idx; | |
83 | ||
65bfdd36 | 84 | idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1; |
5fe89191 | 85 | pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n", |
115a1a52 | 86 | tt, tf, idx, |
65bfdd36 | 87 | data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])), |
d66e4cf9 PM |
88 | data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])), |
89 | data_race(READ_ONCE(ssp->srcu_idx)), | |
90 | data_race(READ_ONCE(ssp->srcu_idx_max))); | |
115a1a52 PM |
91 | } |
92 | ||
d8be8173 | 93 | #endif |