Commit | Line | Data |
---|---|---|
ee1ee6db TG |
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | #ifndef _LINUX_RCUREF_H | |
3 | #define _LINUX_RCUREF_H | |
4 | ||
5 | #include <linux/atomic.h> | |
6 | #include <linux/bug.h> | |
7 | #include <linux/limits.h> | |
8 | #include <linux/lockdep.h> | |
9 | #include <linux/preempt.h> | |
10 | #include <linux/rcupdate.h> | |
11 | ||
12 | #define RCUREF_ONEREF 0x00000000U | |
13 | #define RCUREF_MAXREF 0x7FFFFFFFU | |
14 | #define RCUREF_SATURATED 0xA0000000U | |
15 | #define RCUREF_RELEASED 0xC0000000U | |
16 | #define RCUREF_DEAD 0xE0000000U | |
17 | #define RCUREF_NOREF 0xFFFFFFFFU | |
18 | ||
19 | /** | |
20 | * rcuref_init - Initialize a rcuref reference count with the given reference count | |
21 | * @ref: Pointer to the reference count | |
22 | * @cnt: The initial reference count typically '1' | |
23 | */ | |
24 | static inline void rcuref_init(rcuref_t *ref, unsigned int cnt) | |
25 | { | |
26 | atomic_set(&ref->refcnt, cnt - 1); | |
27 | } | |
28 | ||
29 | /** | |
30 | * rcuref_read - Read the number of held reference counts of a rcuref | |
31 | * @ref: Pointer to the reference count | |
32 | * | |
33 | * Return: The number of held references (0 ... N) | |
34 | */ | |
35 | static inline unsigned int rcuref_read(rcuref_t *ref) | |
36 | { | |
37 | unsigned int c = atomic_read(&ref->refcnt); | |
38 | ||
39 | /* Return 0 if within the DEAD zone. */ | |
40 | return c >= RCUREF_RELEASED ? 0 : c + 1; | |
41 | } | |
42 | ||
43 | extern __must_check bool rcuref_get_slowpath(rcuref_t *ref); | |
44 | ||
45 | /** | |
46 | * rcuref_get - Acquire one reference on a rcuref reference count | |
47 | * @ref: Pointer to the reference count | |
48 | * | |
49 | * Similar to atomic_inc_not_zero() but saturates at RCUREF_MAXREF. | |
50 | * | |
51 | * Provides no memory ordering, it is assumed the caller has guaranteed the | |
52 | * object memory to be stable (RCU, etc.). It does provide a control dependency | |
53 | * and thereby orders future stores. See documentation in lib/rcuref.c | |
54 | * | |
55 | * Return: | |
56 | * False if the attempt to acquire a reference failed. This happens | |
57 | * when the last reference has been put already | |
58 | * | |
59 | * True if a reference was successfully acquired | |
60 | */ | |
61 | static inline __must_check bool rcuref_get(rcuref_t *ref) | |
62 | { | |
63 | /* | |
64 | * Unconditionally increase the reference count. The saturation and | |
65 | * dead zones provide enough tolerance for this. | |
66 | */ | |
67 | if (likely(!atomic_add_negative_relaxed(1, &ref->refcnt))) | |
68 | return true; | |
69 | ||
70 | /* Handle the cases inside the saturation and dead zones */ | |
71 | return rcuref_get_slowpath(ref); | |
72 | } | |
73 | ||
74 | extern __must_check bool rcuref_put_slowpath(rcuref_t *ref); | |
75 | ||
76 | /* | |
77 | * Internal helper. Do not invoke directly. | |
78 | */ | |
79 | static __always_inline __must_check bool __rcuref_put(rcuref_t *ref) | |
80 | { | |
81 | RCU_LOCKDEP_WARN(!rcu_read_lock_held() && preemptible(), | |
82 | "suspicious rcuref_put_rcusafe() usage"); | |
83 | /* | |
84 | * Unconditionally decrease the reference count. The saturation and | |
85 | * dead zones provide enough tolerance for this. | |
86 | */ | |
87 | if (likely(!atomic_add_negative_release(-1, &ref->refcnt))) | |
88 | return false; | |
89 | ||
90 | /* | |
91 | * Handle the last reference drop and cases inside the saturation | |
92 | * and dead zones. | |
93 | */ | |
94 | return rcuref_put_slowpath(ref); | |
95 | } | |
96 | ||
97 | /** | |
98 | * rcuref_put_rcusafe -- Release one reference for a rcuref reference count RCU safe | |
99 | * @ref: Pointer to the reference count | |
100 | * | |
101 | * Provides release memory ordering, such that prior loads and stores are done | |
102 | * before, and provides an acquire ordering on success such that free() | |
103 | * must come after. | |
104 | * | |
105 | * Can be invoked from contexts, which guarantee that no grace period can | |
106 | * happen which would free the object concurrently if the decrement drops | |
107 | * the last reference and the slowpath races against a concurrent get() and | |
108 | * put() pair. rcu_read_lock()'ed and atomic contexts qualify. | |
109 | * | |
110 | * Return: | |
111 | * True if this was the last reference with no future references | |
112 | * possible. This signals the caller that it can safely release the | |
113 | * object which is protected by the reference counter. | |
114 | * | |
115 | * False if there are still active references or the put() raced | |
116 | * with a concurrent get()/put() pair. Caller is not allowed to | |
117 | * release the protected object. | |
118 | */ | |
119 | static inline __must_check bool rcuref_put_rcusafe(rcuref_t *ref) | |
120 | { | |
121 | return __rcuref_put(ref); | |
122 | } | |
123 | ||
124 | /** | |
125 | * rcuref_put -- Release one reference for a rcuref reference count | |
126 | * @ref: Pointer to the reference count | |
127 | * | |
128 | * Can be invoked from any context. | |
129 | * | |
130 | * Provides release memory ordering, such that prior loads and stores are done | |
131 | * before, and provides an acquire ordering on success such that free() | |
132 | * must come after. | |
133 | * | |
134 | * Return: | |
135 | * | |
136 | * True if this was the last reference with no future references | |
137 | * possible. This signals the caller that it can safely schedule the | |
138 | * object, which is protected by the reference counter, for | |
139 | * deconstruction. | |
140 | * | |
141 | * False if there are still active references or the put() raced | |
142 | * with a concurrent get()/put() pair. Caller is not allowed to | |
143 | * deconstruct the protected object. | |
144 | */ | |
145 | static inline __must_check bool rcuref_put(rcuref_t *ref) | |
146 | { | |
147 | bool released; | |
148 | ||
149 | preempt_disable(); | |
150 | released = __rcuref_put(ref); | |
151 | preempt_enable(); | |
152 | return released; | |
153 | } | |
154 | ||
155 | #endif |