Commit | Line | Data |
---|---|---|
439e7271 JL |
1 | /* |
2 | * shadow.c - Shadow Variables | |
3 | * | |
4 | * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com> | |
5 | * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> | |
6 | * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU General Public License | |
10 | * as published by the Free Software Foundation; either version 2 | |
11 | * of the License, or (at your option) any later version. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, | |
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | * GNU General Public License for more details. | |
17 | * | |
18 | * You should have received a copy of the GNU General Public License | |
19 | * along with this program; if not, see <http://www.gnu.org/licenses/>. | |
20 | */ | |
21 | ||
22 | /** | |
23 | * DOC: Shadow variable API concurrency notes: | |
24 | * | |
25 | * The shadow variable API provides a simple relationship between an | |
26 | * <obj, id> pair and a pointer value. It is the responsibility of the | |
27 | * caller to provide any mutual exclusion required of the shadow data. | |
28 | * | |
29 | * Once a shadow variable is attached to its parent object via the | |
30 | * klp_shadow_*alloc() API calls, it is considered live: any subsequent | |
31 | * call to klp_shadow_get() may then return the shadow variable's data | |
32 | * pointer. Callers of klp_shadow_*alloc() should prepare shadow data | |
33 | * accordingly. | |
34 | * | |
35 | * The klp_shadow_*alloc() API calls may allocate memory for new shadow | |
36 | * variable structures. Their implementation does not call kmalloc | |
37 | * inside any spinlocks, but API callers should pass GFP flags according | |
38 | * to their specific needs. | |
39 | * | |
40 | * The klp_shadow_hash is an RCU-enabled hashtable and is safe against | |
41 | * concurrent klp_shadow_free() and klp_shadow_get() operations. | |
42 | */ | |
43 | ||
44 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
45 | ||
46 | #include <linux/hashtable.h> | |
47 | #include <linux/slab.h> | |
48 | #include <linux/livepatch.h> | |
49 | ||
50 | static DEFINE_HASHTABLE(klp_shadow_hash, 12); | |
51 | ||
52 | /* | |
53 | * klp_shadow_lock provides exclusive access to the klp_shadow_hash and | |
54 | * the shadow variables it references. | |
55 | */ | |
56 | static DEFINE_SPINLOCK(klp_shadow_lock); | |
57 | ||
58 | /** | |
59 | * struct klp_shadow - shadow variable structure | |
60 | * @node: klp_shadow_hash hash table node | |
61 | * @rcu_head: RCU is used to safely free this structure | |
62 | * @obj: pointer to parent object | |
63 | * @id: data identifier | |
64 | * @data: data area | |
65 | */ | |
66 | struct klp_shadow { | |
67 | struct hlist_node node; | |
68 | struct rcu_head rcu_head; | |
69 | void *obj; | |
70 | unsigned long id; | |
71 | char data[]; | |
72 | }; | |
73 | ||
74 | /** | |
75 | * klp_shadow_match() - verify a shadow variable matches given <obj, id> | |
76 | * @shadow: shadow variable to match | |
77 | * @obj: pointer to parent object | |
78 | * @id: data identifier | |
79 | * | |
80 | * Return: true if the shadow variable matches. | |
81 | */ | |
82 | static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj, | |
83 | unsigned long id) | |
84 | { | |
85 | return shadow->obj == obj && shadow->id == id; | |
86 | } | |
87 | ||
88 | /** | |
89 | * klp_shadow_get() - retrieve a shadow variable data pointer | |
90 | * @obj: pointer to parent object | |
91 | * @id: data identifier | |
92 | * | |
93 | * Return: the shadow variable data element, NULL on failure. | |
94 | */ | |
95 | void *klp_shadow_get(void *obj, unsigned long id) | |
96 | { | |
97 | struct klp_shadow *shadow; | |
98 | ||
99 | rcu_read_lock(); | |
100 | ||
101 | hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, | |
102 | (unsigned long)obj) { | |
103 | ||
104 | if (klp_shadow_match(shadow, obj, id)) { | |
105 | rcu_read_unlock(); | |
106 | return shadow->data; | |
107 | } | |
108 | } | |
109 | ||
110 | rcu_read_unlock(); | |
111 | ||
112 | return NULL; | |
113 | } | |
114 | EXPORT_SYMBOL_GPL(klp_shadow_get); | |
115 | ||
5d9da759 | 116 | static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data, |
439e7271 JL |
117 | size_t size, gfp_t gfp_flags, bool warn_on_exist) |
118 | { | |
119 | struct klp_shadow *new_shadow; | |
120 | void *shadow_data; | |
121 | unsigned long flags; | |
122 | ||
123 | /* Check if the shadow variable already exists */ | |
124 | shadow_data = klp_shadow_get(obj, id); | |
125 | if (shadow_data) | |
126 | goto exists; | |
127 | ||
128 | /* Allocate a new shadow variable for use inside the lock below */ | |
129 | new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags); | |
130 | if (!new_shadow) | |
131 | return NULL; | |
132 | ||
133 | new_shadow->obj = obj; | |
134 | new_shadow->id = id; | |
135 | ||
136 | /* Initialize the shadow variable if data provided */ | |
137 | if (data) | |
138 | memcpy(new_shadow->data, data, size); | |
139 | ||
140 | /* Look for <obj, id> again under the lock */ | |
141 | spin_lock_irqsave(&klp_shadow_lock, flags); | |
142 | shadow_data = klp_shadow_get(obj, id); | |
143 | if (unlikely(shadow_data)) { | |
144 | /* | |
145 | * Shadow variable was found, throw away speculative | |
146 | * allocation. | |
147 | */ | |
148 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
149 | kfree(new_shadow); | |
150 | goto exists; | |
151 | } | |
152 | ||
153 | /* No <obj, id> found, so attach the newly allocated one */ | |
154 | hash_add_rcu(klp_shadow_hash, &new_shadow->node, | |
155 | (unsigned long)new_shadow->obj); | |
156 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
157 | ||
158 | return new_shadow->data; | |
159 | ||
160 | exists: | |
161 | if (warn_on_exist) { | |
162 | WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id); | |
163 | return NULL; | |
164 | } | |
165 | ||
166 | return shadow_data; | |
167 | } | |
168 | ||
169 | /** | |
170 | * klp_shadow_alloc() - allocate and add a new shadow variable | |
171 | * @obj: pointer to parent object | |
172 | * @id: data identifier | |
173 | * @data: pointer to data to attach to parent | |
174 | * @size: size of attached data | |
175 | * @gfp_flags: GFP mask for allocation | |
176 | * | |
177 | * Allocates @size bytes for new shadow variable data using @gfp_flags | |
178 | * and copies @size bytes from @data into the new shadow variable's own | |
179 | * data space. If @data is NULL, @size bytes are still allocated, but | |
180 | * no copy is performed. The new shadow variable is then added to the | |
181 | * global hashtable. | |
182 | * | |
183 | * If an existing <obj, id> shadow variable can be found, this routine | |
184 | * will issue a WARN, exit early and return NULL. | |
185 | * | |
186 | * Return: the shadow variable data element, NULL on duplicate or | |
187 | * failure. | |
188 | */ | |
189 | void *klp_shadow_alloc(void *obj, unsigned long id, void *data, | |
190 | size_t size, gfp_t gfp_flags) | |
191 | { | |
192 | return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, true); | |
193 | } | |
194 | EXPORT_SYMBOL_GPL(klp_shadow_alloc); | |
195 | ||
196 | /** | |
197 | * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable | |
198 | * @obj: pointer to parent object | |
199 | * @id: data identifier | |
200 | * @data: pointer to data to attach to parent | |
201 | * @size: size of attached data | |
202 | * @gfp_flags: GFP mask for allocation | |
203 | * | |
204 | * Returns a pointer to existing shadow data if an <obj, id> shadow | |
205 | * variable is already present. Otherwise, it creates a new shadow | |
206 | * variable like klp_shadow_alloc(). | |
207 | * | |
208 | * This function guarantees that only one shadow variable exists with | |
209 | * the given @id for the given @obj. It also guarantees that the shadow | |
210 | * variable will be initialized by the given @data only when it did not | |
211 | * exist before. | |
212 | * | |
213 | * Return: the shadow variable data element, NULL on failure. | |
214 | */ | |
215 | void *klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data, | |
216 | size_t size, gfp_t gfp_flags) | |
217 | { | |
218 | return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, false); | |
219 | } | |
220 | EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); | |
221 | ||
222 | /** | |
223 | * klp_shadow_free() - detach and free a <obj, id> shadow variable | |
224 | * @obj: pointer to parent object | |
225 | * @id: data identifier | |
226 | * | |
227 | * This function releases the memory for this <obj, id> shadow variable | |
228 | * instance, callers should stop referencing it accordingly. | |
229 | */ | |
230 | void klp_shadow_free(void *obj, unsigned long id) | |
231 | { | |
232 | struct klp_shadow *shadow; | |
233 | unsigned long flags; | |
234 | ||
235 | spin_lock_irqsave(&klp_shadow_lock, flags); | |
236 | ||
237 | /* Delete <obj, id> from hash */ | |
238 | hash_for_each_possible(klp_shadow_hash, shadow, node, | |
239 | (unsigned long)obj) { | |
240 | ||
241 | if (klp_shadow_match(shadow, obj, id)) { | |
242 | hash_del_rcu(&shadow->node); | |
243 | kfree_rcu(shadow, rcu_head); | |
244 | break; | |
245 | } | |
246 | } | |
247 | ||
248 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
249 | } | |
250 | EXPORT_SYMBOL_GPL(klp_shadow_free); | |
251 | ||
252 | /** | |
253 | * klp_shadow_free_all() - detach and free all <*, id> shadow variables | |
254 | * @id: data identifier | |
255 | * | |
256 | * This function releases the memory for all <*, id> shadow variable | |
257 | * instances, callers should stop referencing them accordingly. | |
258 | */ | |
259 | void klp_shadow_free_all(unsigned long id) | |
260 | { | |
261 | struct klp_shadow *shadow; | |
262 | unsigned long flags; | |
263 | int i; | |
264 | ||
265 | spin_lock_irqsave(&klp_shadow_lock, flags); | |
266 | ||
267 | /* Delete all <*, id> from hash */ | |
268 | hash_for_each(klp_shadow_hash, i, shadow, node) { | |
269 | if (klp_shadow_match(shadow, shadow->obj, id)) { | |
270 | hash_del_rcu(&shadow->node); | |
271 | kfree_rcu(shadow, rcu_head); | |
272 | } | |
273 | } | |
274 | ||
275 | spin_unlock_irqrestore(&klp_shadow_lock, flags); | |
276 | } | |
277 | EXPORT_SYMBOL_GPL(klp_shadow_free_all); |