Commit | Line | Data |
---|---|---|
eb7935e4 | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
45753c5f | 2 | /* |
98059b98 | 3 | * RCU segmented callback lists, internal-to-rcu header file |
45753c5f | 4 | * |
45753c5f IM |
5 | * Copyright IBM Corporation, 2017 |
6 | * | |
eb7935e4 | 7 | * Authors: Paul E. McKenney <paulmck@linux.ibm.com> |
45753c5f IM |
8 | */ |
9 | ||
10 | #include <linux/rcu_segcblist.h> | |
11 | ||
eda669a6 PM |
12 | /* Return number of callbacks in the specified callback list. */ |
13 | static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) | |
14 | { | |
15 | return READ_ONCE(rclp->len); | |
16 | } | |
17 | ||
b4e6039e JFG |
18 | /* Return number of callbacks in segmented callback list by summing seglen. */ |
19 | long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); | |
20 | ||
98059b98 | 21 | void rcu_cblist_init(struct rcu_cblist *rclp); |
d1b222c6 PM |
22 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); |
23 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, | |
24 | struct rcu_cblist *srclp, | |
25 | struct rcu_head *rhp); | |
98059b98 | 26 | struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); |
45753c5f IM |
27 | |
28 | /* | |
29 | * Is the specified rcu_segcblist structure empty? | |
30 | * | |
31 | * But careful! The fact that the ->head field is NULL does not | |
32 | * necessarily imply that there are no callbacks associated with | |
33 | * this structure. When callbacks are being invoked, they are | |
34 | * removed as a group. If callback invocation must be preempted, | |
35 | * the remaining callbacks will be added back to the list. Either | |
36 | * way, the counts are updated later. | |
37 | * | |
38 | * So it is often the case that rcu_segcblist_n_cbs() should be used | |
39 | * instead. | |
40 | */ | |
41 | static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) | |
42 | { | |
e6060b41 | 43 | return !READ_ONCE(rsclp->head); |
45753c5f IM |
44 | } |
45 | ||
46 | /* Return number of callbacks in segmented callback list. */ | |
47 | static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) | |
48 | { | |
eda669a6 PM |
49 | #ifdef CONFIG_RCU_NOCB_CPU |
50 | return atomic_long_read(&rsclp->len); | |
51 | #else | |
45753c5f | 52 | return READ_ONCE(rsclp->len); |
eda669a6 | 53 | #endif |
45753c5f IM |
54 | } |
55 | ||
65e56032 FW |
56 | static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp, |
57 | int flags) | |
58 | { | |
59 | rsclp->flags |= flags; | |
60 | } | |
61 | ||
62 | static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp, | |
63 | int flags) | |
64 | { | |
65 | rsclp->flags &= ~flags; | |
66 | } | |
67 | ||
68 | static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp, | |
69 | int flags) | |
70 | { | |
71 | return READ_ONCE(rsclp->flags) & flags; | |
72 | } | |
73 | ||
45753c5f IM |
74 | /* |
75 | * Is the specified rcu_segcblist enabled, for example, not corresponding | |
e83e73f5 | 76 | * to an offline CPU? |
45753c5f IM |
77 | */ |
78 | static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) | |
79 | { | |
65e56032 | 80 | return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED); |
45753c5f IM |
81 | } |
82 | ||
213d56bf FW |
83 | /* |
84 | * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the | |
85 | * [de]offloading process)? | |
86 | */ | |
ce5215c1 PM |
87 | static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) |
88 | { | |
f759081e | 89 | if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && |
213d56bf | 90 | rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING)) |
f759081e | 91 | return true; |
8d346d43 FW |
92 | |
93 | return false; | |
ce5215c1 PM |
94 | } |
95 | ||
32aa2f41 FW |
96 | static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp) |
97 | { | |
213d56bf FW |
98 | if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && |
99 | !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE)) | |
f759081e | 100 | return true; |
32aa2f41 FW |
101 | |
102 | return false; | |
103 | } | |
104 | ||
45753c5f IM |
105 | /* |
106 | * Are all segments following the specified segment of the specified | |
107 | * rcu_segcblist structure empty of callbacks? (The specified | |
108 | * segment might well contain callbacks.) | |
109 | */ | |
110 | static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) | |
111 | { | |
76c6927c | 112 | return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); |
45753c5f IM |
113 | } |
114 | ||
34169061 PM |
115 | /* |
116 | * Is the specified segment of the specified rcu_segcblist structure | |
117 | * empty of callbacks? | |
118 | */ | |
119 | static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg) | |
120 | { | |
121 | if (seg == RCU_DONE_TAIL) | |
122 | return &rsclp->head == rsclp->tails[RCU_DONE_TAIL]; | |
123 | return rsclp->tails[seg - 1] == rsclp->tails[seg]; | |
124 | } | |
125 | ||
d1b222c6 | 126 | void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); |
6bc33582 | 127 | void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v); |
98059b98 PM |
128 | void rcu_segcblist_init(struct rcu_segcblist *rsclp); |
129 | void rcu_segcblist_disable(struct rcu_segcblist *rsclp); | |
d97b0781 | 130 | void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload); |
98059b98 PM |
131 | bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); |
132 | bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); | |
98059b98 PM |
133 | struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); |
134 | struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); | |
5d6742b3 | 135 | bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); |
98059b98 | 136 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
77a40f97 | 137 | struct rcu_head *rhp); |
98059b98 | 138 | bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, |
77a40f97 | 139 | struct rcu_head *rhp); |
98059b98 PM |
140 | void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, |
141 | struct rcu_cblist *rclp); | |
142 | void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, | |
143 | struct rcu_cblist *rclp); | |
144 | void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, | |
145 | struct rcu_cblist *rclp); | |
146 | void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, | |
147 | struct rcu_cblist *rclp); | |
148 | void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, | |
149 | struct rcu_cblist *rclp); | |
150 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); | |
151 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); | |
f2dbe4a5 PM |
152 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, |
153 | struct rcu_segcblist *src_rsclp); |