Commit | Line | Data |
---|---|---|
eb7935e4 | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
45753c5f | 2 | /* |
98059b98 | 3 | * RCU segmented callback lists, internal-to-rcu header file |
45753c5f | 4 | * |
45753c5f IM |
5 | * Copyright IBM Corporation, 2017 |
6 | * | |
eb7935e4 | 7 | * Authors: Paul E. McKenney <paulmck@linux.ibm.com> |
45753c5f IM |
8 | */ |
9 | ||
10 | #include <linux/rcu_segcblist.h> | |
11 | ||
45753c5f IM |
12 | /* |
13 | * Account for the fact that a previously dequeued callback turned out | |
14 | * to be marked as lazy. | |
15 | */ | |
16 | static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp) | |
17 | { | |
18 | rclp->len_lazy--; | |
19 | } | |
20 | ||
98059b98 | 21 | void rcu_cblist_init(struct rcu_cblist *rclp); |
98059b98 | 22 | struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); |
45753c5f IM |
23 | |
24 | /* | |
25 | * Is the specified rcu_segcblist structure empty? | |
26 | * | |
27 | * But careful! The fact that the ->head field is NULL does not | |
28 | * necessarily imply that there are no callbacks associated with | |
29 | * this structure. When callbacks are being invoked, they are | |
30 | * removed as a group. If callback invocation must be preempted, | |
31 | * the remaining callbacks will be added back to the list. Either | |
32 | * way, the counts are updated later. | |
33 | * | |
34 | * So it is often the case that rcu_segcblist_n_cbs() should be used | |
35 | * instead. | |
36 | */ | |
37 | static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) | |
38 | { | |
39 | return !rsclp->head; | |
40 | } | |
41 | ||
42 | /* Return number of callbacks in segmented callback list. */ | |
43 | static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) | |
44 | { | |
45 | return READ_ONCE(rsclp->len); | |
46 | } | |
47 | ||
48 | /* Return number of lazy callbacks in segmented callback list. */ | |
49 | static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp) | |
50 | { | |
51 | return rsclp->len_lazy; | |
52 | } | |
53 | ||
54 | /* Return number of lazy callbacks in segmented callback list. */ | |
55 | static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp) | |
56 | { | |
57 | return rsclp->len - rsclp->len_lazy; | |
58 | } | |
59 | ||
60 | /* | |
61 | * Is the specified rcu_segcblist enabled, for example, not corresponding | |
62 | * to an offline or callback-offloaded CPU? | |
63 | */ | |
64 | static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) | |
65 | { | |
66 | return !!rsclp->tails[RCU_NEXT_TAIL]; | |
67 | } | |
68 | ||
45753c5f IM |
69 | /* |
70 | * Are all segments following the specified segment of the specified | |
71 | * rcu_segcblist structure empty of callbacks? (The specified | |
72 | * segment might well contain callbacks.) | |
73 | */ | |
74 | static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) | |
75 | { | |
76 | return !*rsclp->tails[seg]; | |
77 | } | |
78 | ||
45753c5f IM |
79 | /* |
80 | * Interim function to return rcu_segcblist head pointer. Longer term, the | |
81 | * rcu_segcblist will be used more pervasively, removing the need for this | |
82 | * function. | |
83 | */ | |
84 | static inline struct rcu_head *rcu_segcblist_head(struct rcu_segcblist *rsclp) | |
85 | { | |
86 | return rsclp->head; | |
87 | } | |
88 | ||
89 | /* | |
90 | * Interim function to return rcu_segcblist head pointer. Longer term, the | |
91 | * rcu_segcblist will be used more pervasively, removing the need for this | |
92 | * function. | |
93 | */ | |
94 | static inline struct rcu_head **rcu_segcblist_tail(struct rcu_segcblist *rsclp) | |
95 | { | |
96 | WARN_ON_ONCE(rcu_segcblist_empty(rsclp)); | |
97 | return rsclp->tails[RCU_NEXT_TAIL]; | |
98 | } | |
98059b98 PM |
99 | |
100 | void rcu_segcblist_init(struct rcu_segcblist *rsclp); | |
101 | void rcu_segcblist_disable(struct rcu_segcblist *rsclp); | |
98059b98 PM |
102 | bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); |
103 | bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); | |
98059b98 PM |
104 | struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); |
105 | struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); | |
98059b98 PM |
106 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
107 | struct rcu_head *rhp, bool lazy); | |
108 | bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, | |
109 | struct rcu_head *rhp, bool lazy); | |
110 | void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, | |
111 | struct rcu_cblist *rclp); | |
112 | void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, | |
113 | struct rcu_cblist *rclp); | |
114 | void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, | |
115 | struct rcu_cblist *rclp); | |
116 | void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, | |
117 | struct rcu_cblist *rclp); | |
118 | void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, | |
119 | struct rcu_cblist *rclp); | |
120 | void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, | |
121 | struct rcu_cblist *rclp); | |
122 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); | |
123 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); | |
f2dbe4a5 PM |
124 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, |
125 | struct rcu_segcblist *src_rsclp); |