Commit | Line | Data |
---|---|---|
8256e47c MD |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | */ | |
18 | #include <linux/module.h> | |
19 | #include <linux/mutex.h> | |
20 | #include <linux/types.h> | |
21 | #include <linux/jhash.h> | |
22 | #include <linux/list.h> | |
23 | #include <linux/rcupdate.h> | |
24 | #include <linux/marker.h> | |
25 | #include <linux/err.h> | |
1aeb272c | 26 | #include <linux/slab.h> |
8256e47c MD |
27 | |
28 | extern struct marker __start___markers[]; | |
29 | extern struct marker __stop___markers[]; | |
30 | ||
fb40bd78 | 31 | /* Set to 1 to enable marker debug output */ |
ab883af5 | 32 | static const int marker_debug; |
fb40bd78 | 33 | |
8256e47c | 34 | /* |
314de8a9 | 35 | * markers_mutex nests inside module_mutex. Markers mutex protects the builtin |
fb40bd78 | 36 | * and module markers and the hash table. |
8256e47c MD |
37 | */ |
38 | static DEFINE_MUTEX(markers_mutex); | |
39 | ||
8256e47c MD |
40 | /* |
41 | * Marker hash table, containing the active markers. | |
42 | * Protected by module_mutex. | |
43 | */ | |
44 | #define MARKER_HASH_BITS 6 | |
45 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
46 | ||
fb40bd78 MD |
47 | /* |
48 | * Note about RCU : | |
49 | * It is used to make sure every handler has finished using its private data | |
50 | * between two consecutive operation (add or remove) on a given marker. It is | |
51 | * also used to delay the free of multiple probes array until a quiescent state | |
52 | * is reached. | |
53 | * marker entries modifications are protected by the markers_mutex. | |
54 | */ | |
8256e47c MD |
55 | struct marker_entry { |
56 | struct hlist_node hlist; | |
57 | char *format; | |
dc102a8f MD |
58 | /* Probe wrapper */ |
59 | void (*call)(const struct marker *mdata, void *call_private, ...); | |
fb40bd78 MD |
60 | struct marker_probe_closure single; |
61 | struct marker_probe_closure *multi; | |
8256e47c | 62 | int refcount; /* Number of times armed. 0 if disarmed. */ |
fb40bd78 MD |
63 | struct rcu_head rcu; |
64 | void *oldptr; | |
de4fc64f HH |
65 | unsigned char rcu_pending:1; |
66 | unsigned char ptype:1; | |
8256e47c MD |
67 | char name[0]; /* Contains name'\0'format'\0' */ |
68 | }; | |
69 | ||
70 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; | |
71 | ||
72 | /** | |
73 | * __mark_empty_function - Empty probe callback | |
fb40bd78 MD |
74 | * @probe_private: probe private data |
75 | * @call_private: call site private data | |
8256e47c MD |
76 | * @fmt: format string |
77 | * @...: variable argument list | |
78 | * | |
79 | * Empty callback provided as a probe to the markers. By providing this to a | |
80 | * disabled marker, we make sure the execution flow is always valid even | |
81 | * though the function pointer change and the marker enabling are two distinct | |
82 | * operations that modifies the execution flow of preemptible code. | |
83 | */ | |
fb40bd78 MD |
84 | void __mark_empty_function(void *probe_private, void *call_private, |
85 | const char *fmt, va_list *args) | |
8256e47c MD |
86 | { |
87 | } | |
88 | EXPORT_SYMBOL_GPL(__mark_empty_function); | |
89 | ||
fb40bd78 MD |
90 | /* |
91 | * marker_probe_cb Callback that prepares the variable argument list for probes. | |
92 | * @mdata: pointer of type struct marker | |
93 | * @call_private: caller site private data | |
fb40bd78 MD |
94 | * @...: Variable argument list. |
95 | * | |
96 | * Since we do not use "typical" pointer based RCU in the 1 argument case, we | |
97 | * need to put a full smp_rmb() in this branch. This is why we do not use | |
98 | * rcu_dereference() for the pointer read. | |
99 | */ | |
dc102a8f | 100 | void marker_probe_cb(const struct marker *mdata, void *call_private, ...) |
fb40bd78 MD |
101 | { |
102 | va_list args; | |
103 | char ptype; | |
104 | ||
105 | /* | |
fd3c36f8 MD |
106 | * preempt_disable does two things : disabling preemption to make sure |
107 | * the teardown of the callbacks can be done correctly when they are in | |
108 | * modules and they insure RCU read coherency. | |
fb40bd78 MD |
109 | */ |
110 | preempt_disable(); | |
58336114 | 111 | ptype = mdata->ptype; |
fb40bd78 MD |
112 | if (likely(!ptype)) { |
113 | marker_probe_func *func; | |
114 | /* Must read the ptype before ptr. They are not data dependant, | |
115 | * so we put an explicit smp_rmb() here. */ | |
116 | smp_rmb(); | |
58336114 | 117 | func = mdata->single.func; |
fb40bd78 MD |
118 | /* Must read the ptr before private data. They are not data |
119 | * dependant, so we put an explicit smp_rmb() here. */ | |
120 | smp_rmb(); | |
dc102a8f MD |
121 | va_start(args, call_private); |
122 | func(mdata->single.probe_private, call_private, mdata->format, | |
123 | &args); | |
fb40bd78 MD |
124 | va_end(args); |
125 | } else { | |
126 | struct marker_probe_closure *multi; | |
127 | int i; | |
5def9a3a MD |
128 | /* |
129 | * Read mdata->ptype before mdata->multi. | |
130 | */ | |
131 | smp_rmb(); | |
132 | multi = mdata->multi; | |
fb40bd78 MD |
133 | /* |
134 | * multi points to an array, therefore accessing the array | |
135 | * depends on reading multi. However, even in this case, | |
136 | * we must insure that the pointer is read _before_ the array | |
137 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
138 | * in the fast path, so put the explicit barrier here. | |
139 | */ | |
140 | smp_read_barrier_depends(); | |
fb40bd78 | 141 | for (i = 0; multi[i].func; i++) { |
dc102a8f MD |
142 | va_start(args, call_private); |
143 | multi[i].func(multi[i].probe_private, call_private, | |
144 | mdata->format, &args); | |
fb40bd78 MD |
145 | va_end(args); |
146 | } | |
147 | } | |
148 | preempt_enable(); | |
149 | } | |
150 | EXPORT_SYMBOL_GPL(marker_probe_cb); | |
151 | ||
152 | /* | |
153 | * marker_probe_cb Callback that does not prepare the variable argument list. | |
154 | * @mdata: pointer of type struct marker | |
155 | * @call_private: caller site private data | |
fb40bd78 MD |
156 | * @...: Variable argument list. |
157 | * | |
158 | * Should be connected to markers "MARK_NOARGS". | |
159 | */ | |
dc102a8f | 160 | void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...) |
fb40bd78 MD |
161 | { |
162 | va_list args; /* not initialized */ | |
163 | char ptype; | |
164 | ||
165 | preempt_disable(); | |
58336114 | 166 | ptype = mdata->ptype; |
fb40bd78 MD |
167 | if (likely(!ptype)) { |
168 | marker_probe_func *func; | |
169 | /* Must read the ptype before ptr. They are not data dependant, | |
170 | * so we put an explicit smp_rmb() here. */ | |
171 | smp_rmb(); | |
58336114 | 172 | func = mdata->single.func; |
fb40bd78 MD |
173 | /* Must read the ptr before private data. They are not data |
174 | * dependant, so we put an explicit smp_rmb() here. */ | |
175 | smp_rmb(); | |
dc102a8f MD |
176 | func(mdata->single.probe_private, call_private, mdata->format, |
177 | &args); | |
fb40bd78 MD |
178 | } else { |
179 | struct marker_probe_closure *multi; | |
180 | int i; | |
5def9a3a MD |
181 | /* |
182 | * Read mdata->ptype before mdata->multi. | |
183 | */ | |
184 | smp_rmb(); | |
185 | multi = mdata->multi; | |
fb40bd78 MD |
186 | /* |
187 | * multi points to an array, therefore accessing the array | |
188 | * depends on reading multi. However, even in this case, | |
189 | * we must insure that the pointer is read _before_ the array | |
190 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
191 | * in the fast path, so put the explicit barrier here. | |
192 | */ | |
193 | smp_read_barrier_depends(); | |
fb40bd78 | 194 | for (i = 0; multi[i].func; i++) |
dc102a8f MD |
195 | multi[i].func(multi[i].probe_private, call_private, |
196 | mdata->format, &args); | |
fb40bd78 MD |
197 | } |
198 | preempt_enable(); | |
199 | } | |
200 | EXPORT_SYMBOL_GPL(marker_probe_cb_noarg); | |
201 | ||
202 | static void free_old_closure(struct rcu_head *head) | |
203 | { | |
204 | struct marker_entry *entry = container_of(head, | |
205 | struct marker_entry, rcu); | |
206 | kfree(entry->oldptr); | |
207 | /* Make sure we free the data before setting the pending flag to 0 */ | |
208 | smp_wmb(); | |
209 | entry->rcu_pending = 0; | |
210 | } | |
211 | ||
212 | static void debug_print_probes(struct marker_entry *entry) | |
213 | { | |
214 | int i; | |
215 | ||
216 | if (!marker_debug) | |
217 | return; | |
218 | ||
219 | if (!entry->ptype) { | |
220 | printk(KERN_DEBUG "Single probe : %p %p\n", | |
221 | entry->single.func, | |
222 | entry->single.probe_private); | |
223 | } else { | |
224 | for (i = 0; entry->multi[i].func; i++) | |
225 | printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, | |
226 | entry->multi[i].func, | |
227 | entry->multi[i].probe_private); | |
228 | } | |
229 | } | |
230 | ||
231 | static struct marker_probe_closure * | |
232 | marker_entry_add_probe(struct marker_entry *entry, | |
233 | marker_probe_func *probe, void *probe_private) | |
234 | { | |
235 | int nr_probes = 0; | |
236 | struct marker_probe_closure *old, *new; | |
237 | ||
238 | WARN_ON(!probe); | |
239 | ||
240 | debug_print_probes(entry); | |
241 | old = entry->multi; | |
242 | if (!entry->ptype) { | |
243 | if (entry->single.func == probe && | |
244 | entry->single.probe_private == probe_private) | |
245 | return ERR_PTR(-EBUSY); | |
246 | if (entry->single.func == __mark_empty_function) { | |
247 | /* 0 -> 1 probes */ | |
248 | entry->single.func = probe; | |
249 | entry->single.probe_private = probe_private; | |
250 | entry->refcount = 1; | |
251 | entry->ptype = 0; | |
252 | debug_print_probes(entry); | |
253 | return NULL; | |
254 | } else { | |
255 | /* 1 -> 2 probes */ | |
256 | nr_probes = 1; | |
257 | old = NULL; | |
258 | } | |
259 | } else { | |
260 | /* (N -> N+1), (N != 0, 1) probes */ | |
261 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) | |
262 | if (old[nr_probes].func == probe | |
263 | && old[nr_probes].probe_private | |
264 | == probe_private) | |
265 | return ERR_PTR(-EBUSY); | |
266 | } | |
267 | /* + 2 : one for new probe, one for NULL func */ | |
268 | new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), | |
269 | GFP_KERNEL); | |
270 | if (new == NULL) | |
271 | return ERR_PTR(-ENOMEM); | |
272 | if (!old) | |
273 | new[0] = entry->single; | |
274 | else | |
275 | memcpy(new, old, | |
276 | nr_probes * sizeof(struct marker_probe_closure)); | |
277 | new[nr_probes].func = probe; | |
278 | new[nr_probes].probe_private = probe_private; | |
279 | entry->refcount = nr_probes + 1; | |
280 | entry->multi = new; | |
281 | entry->ptype = 1; | |
282 | debug_print_probes(entry); | |
283 | return old; | |
284 | } | |
285 | ||
286 | static struct marker_probe_closure * | |
287 | marker_entry_remove_probe(struct marker_entry *entry, | |
288 | marker_probe_func *probe, void *probe_private) | |
289 | { | |
290 | int nr_probes = 0, nr_del = 0, i; | |
291 | struct marker_probe_closure *old, *new; | |
292 | ||
293 | old = entry->multi; | |
294 | ||
295 | debug_print_probes(entry); | |
296 | if (!entry->ptype) { | |
297 | /* 0 -> N is an error */ | |
298 | WARN_ON(entry->single.func == __mark_empty_function); | |
299 | /* 1 -> 0 probes */ | |
300 | WARN_ON(probe && entry->single.func != probe); | |
301 | WARN_ON(entry->single.probe_private != probe_private); | |
302 | entry->single.func = __mark_empty_function; | |
303 | entry->refcount = 0; | |
304 | entry->ptype = 0; | |
305 | debug_print_probes(entry); | |
306 | return NULL; | |
307 | } else { | |
308 | /* (N -> M), (N > 1, M >= 0) probes */ | |
309 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { | |
310 | if ((!probe || old[nr_probes].func == probe) | |
311 | && old[nr_probes].probe_private | |
312 | == probe_private) | |
313 | nr_del++; | |
314 | } | |
315 | } | |
316 | ||
317 | if (nr_probes - nr_del == 0) { | |
318 | /* N -> 0, (N > 1) */ | |
319 | entry->single.func = __mark_empty_function; | |
320 | entry->refcount = 0; | |
321 | entry->ptype = 0; | |
322 | } else if (nr_probes - nr_del == 1) { | |
323 | /* N -> 1, (N > 1) */ | |
324 | for (i = 0; old[i].func; i++) | |
325 | if ((probe && old[i].func != probe) || | |
326 | old[i].probe_private != probe_private) | |
327 | entry->single = old[i]; | |
328 | entry->refcount = 1; | |
329 | entry->ptype = 0; | |
330 | } else { | |
331 | int j = 0; | |
332 | /* N -> M, (N > 1, M > 1) */ | |
333 | /* + 1 for NULL */ | |
334 | new = kzalloc((nr_probes - nr_del + 1) | |
335 | * sizeof(struct marker_probe_closure), GFP_KERNEL); | |
336 | if (new == NULL) | |
337 | return ERR_PTR(-ENOMEM); | |
338 | for (i = 0; old[i].func; i++) | |
339 | if ((probe && old[i].func != probe) || | |
340 | old[i].probe_private != probe_private) | |
341 | new[j++] = old[i]; | |
342 | entry->refcount = nr_probes - nr_del; | |
343 | entry->ptype = 1; | |
344 | entry->multi = new; | |
345 | } | |
346 | debug_print_probes(entry); | |
347 | return old; | |
348 | } | |
349 | ||
8256e47c MD |
350 | /* |
351 | * Get marker if the marker is present in the marker hash table. | |
352 | * Must be called with markers_mutex held. | |
353 | * Returns NULL if not present. | |
354 | */ | |
355 | static struct marker_entry *get_marker(const char *name) | |
356 | { | |
357 | struct hlist_head *head; | |
358 | struct hlist_node *node; | |
359 | struct marker_entry *e; | |
360 | u32 hash = jhash(name, strlen(name), 0); | |
361 | ||
362 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
363 | hlist_for_each_entry(e, node, head, hlist) { | |
364 | if (!strcmp(name, e->name)) | |
365 | return e; | |
366 | } | |
367 | return NULL; | |
368 | } | |
369 | ||
370 | /* | |
371 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
372 | * held. | |
373 | */ | |
fb40bd78 | 374 | static struct marker_entry *add_marker(const char *name, const char *format) |
8256e47c MD |
375 | { |
376 | struct hlist_head *head; | |
377 | struct hlist_node *node; | |
378 | struct marker_entry *e; | |
379 | size_t name_len = strlen(name) + 1; | |
380 | size_t format_len = 0; | |
381 | u32 hash = jhash(name, name_len-1, 0); | |
382 | ||
383 | if (format) | |
384 | format_len = strlen(format) + 1; | |
385 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
386 | hlist_for_each_entry(e, node, head, hlist) { | |
387 | if (!strcmp(name, e->name)) { | |
388 | printk(KERN_NOTICE | |
fb40bd78 MD |
389 | "Marker %s busy\n", name); |
390 | return ERR_PTR(-EBUSY); /* Already there */ | |
8256e47c MD |
391 | } |
392 | } | |
393 | /* | |
394 | * Using kmalloc here to allocate a variable length element. Could | |
395 | * cause some memory fragmentation if overused. | |
396 | */ | |
397 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
398 | GFP_KERNEL); | |
399 | if (!e) | |
fb40bd78 | 400 | return ERR_PTR(-ENOMEM); |
8256e47c MD |
401 | memcpy(&e->name[0], name, name_len); |
402 | if (format) { | |
403 | e->format = &e->name[name_len]; | |
404 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
405 | if (strcmp(e->format, MARK_NOARGS) == 0) |
406 | e->call = marker_probe_cb_noarg; | |
407 | else | |
408 | e->call = marker_probe_cb; | |
8256e47c MD |
409 | trace_mark(core_marker_format, "name %s format %s", |
410 | e->name, e->format); | |
fb40bd78 | 411 | } else { |
8256e47c | 412 | e->format = NULL; |
fb40bd78 MD |
413 | e->call = marker_probe_cb; |
414 | } | |
415 | e->single.func = __mark_empty_function; | |
416 | e->single.probe_private = NULL; | |
417 | e->multi = NULL; | |
418 | e->ptype = 0; | |
8256e47c | 419 | e->refcount = 0; |
fb40bd78 | 420 | e->rcu_pending = 0; |
8256e47c | 421 | hlist_add_head(&e->hlist, head); |
fb40bd78 | 422 | return e; |
8256e47c MD |
423 | } |
424 | ||
425 | /* | |
426 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
427 | * held. | |
428 | */ | |
fb40bd78 | 429 | static int remove_marker(const char *name) |
8256e47c MD |
430 | { |
431 | struct hlist_head *head; | |
432 | struct hlist_node *node; | |
433 | struct marker_entry *e; | |
434 | int found = 0; | |
435 | size_t len = strlen(name) + 1; | |
8256e47c MD |
436 | u32 hash = jhash(name, len-1, 0); |
437 | ||
438 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
439 | hlist_for_each_entry(e, node, head, hlist) { | |
440 | if (!strcmp(name, e->name)) { | |
441 | found = 1; | |
442 | break; | |
443 | } | |
444 | } | |
fb40bd78 MD |
445 | if (!found) |
446 | return -ENOENT; | |
447 | if (e->single.func != __mark_empty_function) | |
448 | return -EBUSY; | |
449 | hlist_del(&e->hlist); | |
450 | /* Make sure the call_rcu has been executed */ | |
451 | if (e->rcu_pending) | |
28325df0 | 452 | rcu_barrier_sched(); |
fb40bd78 MD |
453 | kfree(e); |
454 | return 0; | |
8256e47c MD |
455 | } |
456 | ||
457 | /* | |
458 | * Set the mark_entry format to the format found in the element. | |
459 | */ | |
460 | static int marker_set_format(struct marker_entry **entry, const char *format) | |
461 | { | |
462 | struct marker_entry *e; | |
463 | size_t name_len = strlen((*entry)->name) + 1; | |
464 | size_t format_len = strlen(format) + 1; | |
465 | ||
fb40bd78 | 466 | |
8256e47c MD |
467 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, |
468 | GFP_KERNEL); | |
469 | if (!e) | |
470 | return -ENOMEM; | |
471 | memcpy(&e->name[0], (*entry)->name, name_len); | |
472 | e->format = &e->name[name_len]; | |
473 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
474 | if (strcmp(e->format, MARK_NOARGS) == 0) |
475 | e->call = marker_probe_cb_noarg; | |
476 | else | |
477 | e->call = marker_probe_cb; | |
478 | e->single = (*entry)->single; | |
479 | e->multi = (*entry)->multi; | |
480 | e->ptype = (*entry)->ptype; | |
8256e47c | 481 | e->refcount = (*entry)->refcount; |
fb40bd78 | 482 | e->rcu_pending = 0; |
8256e47c MD |
483 | hlist_add_before(&e->hlist, &(*entry)->hlist); |
484 | hlist_del(&(*entry)->hlist); | |
fb40bd78 MD |
485 | /* Make sure the call_rcu has been executed */ |
486 | if ((*entry)->rcu_pending) | |
28325df0 | 487 | rcu_barrier_sched(); |
8256e47c MD |
488 | kfree(*entry); |
489 | *entry = e; | |
490 | trace_mark(core_marker_format, "name %s format %s", | |
491 | e->name, e->format); | |
492 | return 0; | |
493 | } | |
494 | ||
495 | /* | |
496 | * Sets the probe callback corresponding to one marker. | |
497 | */ | |
fb40bd78 MD |
498 | static int set_marker(struct marker_entry **entry, struct marker *elem, |
499 | int active) | |
8256e47c MD |
500 | { |
501 | int ret; | |
502 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | |
503 | ||
504 | if ((*entry)->format) { | |
505 | if (strcmp((*entry)->format, elem->format) != 0) { | |
506 | printk(KERN_NOTICE | |
507 | "Format mismatch for probe %s " | |
508 | "(%s), marker (%s)\n", | |
509 | (*entry)->name, | |
510 | (*entry)->format, | |
511 | elem->format); | |
512 | return -EPERM; | |
513 | } | |
514 | } else { | |
515 | ret = marker_set_format(entry, elem->format); | |
516 | if (ret) | |
517 | return ret; | |
518 | } | |
fb40bd78 MD |
519 | |
520 | /* | |
521 | * probe_cb setup (statically known) is done here. It is | |
522 | * asynchronous with the rest of execution, therefore we only | |
523 | * pass from a "safe" callback (with argument) to an "unsafe" | |
524 | * callback (does not set arguments). | |
525 | */ | |
526 | elem->call = (*entry)->call; | |
527 | /* | |
528 | * Sanity check : | |
529 | * We only update the single probe private data when the ptr is | |
530 | * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) | |
531 | */ | |
532 | WARN_ON(elem->single.func != __mark_empty_function | |
533 | && elem->single.probe_private | |
534 | != (*entry)->single.probe_private && | |
535 | !elem->ptype); | |
536 | elem->single.probe_private = (*entry)->single.probe_private; | |
537 | /* | |
538 | * Make sure the private data is valid when we update the | |
539 | * single probe ptr. | |
540 | */ | |
541 | smp_wmb(); | |
542 | elem->single.func = (*entry)->single.func; | |
543 | /* | |
544 | * We also make sure that the new probe callbacks array is consistent | |
545 | * before setting a pointer to it. | |
546 | */ | |
547 | rcu_assign_pointer(elem->multi, (*entry)->multi); | |
548 | /* | |
549 | * Update the function or multi probe array pointer before setting the | |
550 | * ptype. | |
551 | */ | |
552 | smp_wmb(); | |
553 | elem->ptype = (*entry)->ptype; | |
554 | elem->state = active; | |
555 | ||
8256e47c MD |
556 | return 0; |
557 | } | |
558 | ||
559 | /* | |
560 | * Disable a marker and its probe callback. | |
fd3c36f8 MD |
561 | * Note: only waiting an RCU period after setting elem->call to the empty |
562 | * function insures that the original callback is not used anymore. This insured | |
563 | * by preempt_disable around the call site. | |
8256e47c MD |
564 | */ |
565 | static void disable_marker(struct marker *elem) | |
566 | { | |
fb40bd78 | 567 | /* leave "call" as is. It is known statically. */ |
8256e47c | 568 | elem->state = 0; |
fb40bd78 MD |
569 | elem->single.func = __mark_empty_function; |
570 | /* Update the function before setting the ptype */ | |
571 | smp_wmb(); | |
572 | elem->ptype = 0; /* single probe */ | |
8256e47c MD |
573 | /* |
574 | * Leave the private data and id there, because removal is racy and | |
fd3c36f8 MD |
575 | * should be done only after an RCU period. These are never used until |
576 | * the next initialization anyway. | |
8256e47c MD |
577 | */ |
578 | } | |
579 | ||
580 | /** | |
581 | * marker_update_probe_range - Update a probe range | |
582 | * @begin: beginning of the range | |
583 | * @end: end of the range | |
8256e47c MD |
584 | * |
585 | * Updates the probe callback corresponding to a range of markers. | |
8256e47c MD |
586 | */ |
587 | void marker_update_probe_range(struct marker *begin, | |
fb40bd78 | 588 | struct marker *end) |
8256e47c MD |
589 | { |
590 | struct marker *iter; | |
591 | struct marker_entry *mark_entry; | |
592 | ||
314de8a9 | 593 | mutex_lock(&markers_mutex); |
8256e47c MD |
594 | for (iter = begin; iter < end; iter++) { |
595 | mark_entry = get_marker(iter->name); | |
fb40bd78 MD |
596 | if (mark_entry) { |
597 | set_marker(&mark_entry, iter, | |
598 | !!mark_entry->refcount); | |
8256e47c MD |
599 | /* |
600 | * ignore error, continue | |
601 | */ | |
8256e47c MD |
602 | } else { |
603 | disable_marker(iter); | |
604 | } | |
605 | } | |
314de8a9 | 606 | mutex_unlock(&markers_mutex); |
8256e47c MD |
607 | } |
608 | ||
609 | /* | |
610 | * Update probes, removing the faulty probes. | |
fb40bd78 MD |
611 | * |
612 | * Internal callback only changed before the first probe is connected to it. | |
613 | * Single probe private data can only be changed on 0 -> 1 and 2 -> 1 | |
614 | * transitions. All other transitions will leave the old private data valid. | |
615 | * This makes the non-atomicity of the callback/private data updates valid. | |
616 | * | |
617 | * "special case" updates : | |
618 | * 0 -> 1 callback | |
619 | * 1 -> 0 callback | |
620 | * 1 -> 2 callbacks | |
621 | * 2 -> 1 callbacks | |
622 | * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. | |
623 | * Site effect : marker_set_format may delete the marker entry (creating a | |
624 | * replacement). | |
8256e47c | 625 | */ |
fb40bd78 | 626 | static void marker_update_probes(void) |
8256e47c | 627 | { |
8256e47c | 628 | /* Core kernel markers */ |
fb40bd78 | 629 | marker_update_probe_range(__start___markers, __stop___markers); |
8256e47c | 630 | /* Markers in modules. */ |
fb40bd78 | 631 | module_update_markers(); |
8256e47c MD |
632 | } |
633 | ||
634 | /** | |
635 | * marker_probe_register - Connect a probe to a marker | |
636 | * @name: marker name | |
637 | * @format: format string | |
638 | * @probe: probe handler | |
fb40bd78 | 639 | * @probe_private: probe private data |
8256e47c MD |
640 | * |
641 | * private data must be a valid allocated memory address, or NULL. | |
642 | * Returns 0 if ok, error value on error. | |
fb40bd78 | 643 | * The probe address must at least be aligned on the architecture pointer size. |
8256e47c MD |
644 | */ |
645 | int marker_probe_register(const char *name, const char *format, | |
fb40bd78 | 646 | marker_probe_func *probe, void *probe_private) |
8256e47c MD |
647 | { |
648 | struct marker_entry *entry; | |
314de8a9 | 649 | int ret = 0; |
fb40bd78 | 650 | struct marker_probe_closure *old; |
8256e47c MD |
651 | |
652 | mutex_lock(&markers_mutex); | |
653 | entry = get_marker(name); | |
fb40bd78 MD |
654 | if (!entry) { |
655 | entry = add_marker(name, format); | |
656 | if (IS_ERR(entry)) { | |
657 | ret = PTR_ERR(entry); | |
658 | goto end; | |
659 | } | |
8256e47c | 660 | } |
fb40bd78 MD |
661 | /* |
662 | * If we detect that a call_rcu is pending for this marker, | |
663 | * make sure it's executed now. | |
664 | */ | |
665 | if (entry->rcu_pending) | |
28325df0 | 666 | rcu_barrier_sched(); |
fb40bd78 MD |
667 | old = marker_entry_add_probe(entry, probe, probe_private); |
668 | if (IS_ERR(old)) { | |
669 | ret = PTR_ERR(old); | |
8256e47c | 670 | goto end; |
fb40bd78 | 671 | } |
314de8a9 | 672 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
673 | marker_update_probes(); /* may update entry */ |
674 | mutex_lock(&markers_mutex); | |
675 | entry = get_marker(name); | |
676 | WARN_ON(!entry); | |
677 | entry->oldptr = old; | |
678 | entry->rcu_pending = 1; | |
679 | /* write rcu_pending before calling the RCU callback */ | |
680 | smp_wmb(); | |
28325df0 | 681 | call_rcu_sched(&entry->rcu, free_old_closure); |
8256e47c MD |
682 | end: |
683 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
684 | return ret; |
685 | } | |
686 | EXPORT_SYMBOL_GPL(marker_probe_register); | |
687 | ||
688 | /** | |
689 | * marker_probe_unregister - Disconnect a probe from a marker | |
690 | * @name: marker name | |
fb40bd78 MD |
691 | * @probe: probe function pointer |
692 | * @probe_private: probe private data | |
8256e47c MD |
693 | * |
694 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
fb40bd78 MD |
695 | * We do not need to call a synchronize_sched to make sure the probes have |
696 | * finished running before doing a module unload, because the module unload | |
697 | * itself uses stop_machine(), which insures that every preempt disabled section | |
698 | * have finished. | |
8256e47c | 699 | */ |
fb40bd78 MD |
700 | int marker_probe_unregister(const char *name, |
701 | marker_probe_func *probe, void *probe_private) | |
8256e47c | 702 | { |
8256e47c | 703 | struct marker_entry *entry; |
fb40bd78 | 704 | struct marker_probe_closure *old; |
544adb41 | 705 | int ret = -ENOENT; |
8256e47c MD |
706 | |
707 | mutex_lock(&markers_mutex); | |
708 | entry = get_marker(name); | |
544adb41 | 709 | if (!entry) |
8256e47c | 710 | goto end; |
fb40bd78 | 711 | if (entry->rcu_pending) |
28325df0 | 712 | rcu_barrier_sched(); |
fb40bd78 | 713 | old = marker_entry_remove_probe(entry, probe, probe_private); |
314de8a9 | 714 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
715 | marker_update_probes(); /* may update entry */ |
716 | mutex_lock(&markers_mutex); | |
717 | entry = get_marker(name); | |
544adb41 JJ |
718 | if (!entry) |
719 | goto end; | |
fb40bd78 MD |
720 | entry->oldptr = old; |
721 | entry->rcu_pending = 1; | |
722 | /* write rcu_pending before calling the RCU callback */ | |
723 | smp_wmb(); | |
28325df0 | 724 | call_rcu_sched(&entry->rcu, free_old_closure); |
fb40bd78 | 725 | remove_marker(name); /* Ignore busy error message */ |
544adb41 | 726 | ret = 0; |
8256e47c MD |
727 | end: |
728 | mutex_unlock(&markers_mutex); | |
fb40bd78 | 729 | return ret; |
8256e47c MD |
730 | } |
731 | EXPORT_SYMBOL_GPL(marker_probe_unregister); | |
732 | ||
fb40bd78 MD |
733 | static struct marker_entry * |
734 | get_marker_from_private_data(marker_probe_func *probe, void *probe_private) | |
8256e47c | 735 | { |
8256e47c | 736 | struct marker_entry *entry; |
8256e47c | 737 | unsigned int i; |
fb40bd78 MD |
738 | struct hlist_head *head; |
739 | struct hlist_node *node; | |
8256e47c | 740 | |
8256e47c MD |
741 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { |
742 | head = &marker_table[i]; | |
743 | hlist_for_each_entry(entry, node, head, hlist) { | |
fb40bd78 MD |
744 | if (!entry->ptype) { |
745 | if (entry->single.func == probe | |
746 | && entry->single.probe_private | |
747 | == probe_private) | |
748 | return entry; | |
749 | } else { | |
750 | struct marker_probe_closure *closure; | |
751 | closure = entry->multi; | |
752 | for (i = 0; closure[i].func; i++) { | |
753 | if (closure[i].func == probe && | |
754 | closure[i].probe_private | |
755 | == probe_private) | |
756 | return entry; | |
757 | } | |
8256e47c MD |
758 | } |
759 | } | |
760 | } | |
fb40bd78 | 761 | return NULL; |
8256e47c | 762 | } |
8256e47c MD |
763 | |
764 | /** | |
fb40bd78 MD |
765 | * marker_probe_unregister_private_data - Disconnect a probe from a marker |
766 | * @probe: probe function | |
767 | * @probe_private: probe private data | |
8256e47c | 768 | * |
fb40bd78 MD |
769 | * Unregister a probe by providing the registered private data. |
770 | * Only removes the first marker found in hash table. | |
771 | * Return 0 on success or error value. | |
772 | * We do not need to call a synchronize_sched to make sure the probes have | |
773 | * finished running before doing a module unload, because the module unload | |
774 | * itself uses stop_machine(), which insures that every preempt disabled section | |
775 | * have finished. | |
8256e47c | 776 | */ |
fb40bd78 MD |
777 | int marker_probe_unregister_private_data(marker_probe_func *probe, |
778 | void *probe_private) | |
8256e47c MD |
779 | { |
780 | struct marker_entry *entry; | |
314de8a9 | 781 | int ret = 0; |
fb40bd78 | 782 | struct marker_probe_closure *old; |
8256e47c MD |
783 | |
784 | mutex_lock(&markers_mutex); | |
fb40bd78 | 785 | entry = get_marker_from_private_data(probe, probe_private); |
8256e47c MD |
786 | if (!entry) { |
787 | ret = -ENOENT; | |
788 | goto end; | |
789 | } | |
fb40bd78 | 790 | if (entry->rcu_pending) |
28325df0 | 791 | rcu_barrier_sched(); |
fb40bd78 | 792 | old = marker_entry_remove_probe(entry, NULL, probe_private); |
8256e47c | 793 | mutex_unlock(&markers_mutex); |
fb40bd78 | 794 | marker_update_probes(); /* may update entry */ |
8256e47c | 795 | mutex_lock(&markers_mutex); |
fb40bd78 MD |
796 | entry = get_marker_from_private_data(probe, probe_private); |
797 | WARN_ON(!entry); | |
798 | entry->oldptr = old; | |
799 | entry->rcu_pending = 1; | |
800 | /* write rcu_pending before calling the RCU callback */ | |
801 | smp_wmb(); | |
28325df0 | 802 | call_rcu_sched(&entry->rcu, free_old_closure); |
fb40bd78 | 803 | remove_marker(entry->name); /* Ignore busy error message */ |
8256e47c MD |
804 | end: |
805 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
806 | return ret; |
807 | } | |
fb40bd78 | 808 | EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); |
8256e47c MD |
809 | |
810 | /** | |
811 | * marker_get_private_data - Get a marker's probe private data | |
812 | * @name: marker name | |
fb40bd78 MD |
813 | * @probe: probe to match |
814 | * @num: get the nth matching probe's private data | |
8256e47c | 815 | * |
fb40bd78 MD |
816 | * Returns the nth private data pointer (starting from 0) matching, or an |
817 | * ERR_PTR. | |
8256e47c MD |
818 | * Returns the private data pointer, or an ERR_PTR. |
819 | * The private data pointer should _only_ be dereferenced if the caller is the | |
820 | * owner of the data, or its content could vanish. This is mostly used to | |
821 | * confirm that a caller is the owner of a registered probe. | |
822 | */ | |
fb40bd78 MD |
823 | void *marker_get_private_data(const char *name, marker_probe_func *probe, |
824 | int num) | |
8256e47c MD |
825 | { |
826 | struct hlist_head *head; | |
827 | struct hlist_node *node; | |
828 | struct marker_entry *e; | |
829 | size_t name_len = strlen(name) + 1; | |
830 | u32 hash = jhash(name, name_len-1, 0); | |
fb40bd78 | 831 | int i; |
8256e47c MD |
832 | |
833 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
834 | hlist_for_each_entry(e, node, head, hlist) { | |
835 | if (!strcmp(name, e->name)) { | |
fb40bd78 MD |
836 | if (!e->ptype) { |
837 | if (num == 0 && e->single.func == probe) | |
838 | return e->single.probe_private; | |
839 | else | |
840 | break; | |
841 | } else { | |
842 | struct marker_probe_closure *closure; | |
843 | int match = 0; | |
844 | closure = e->multi; | |
845 | for (i = 0; closure[i].func; i++) { | |
846 | if (closure[i].func != probe) | |
847 | continue; | |
848 | if (match++ == num) | |
849 | return closure[i].probe_private; | |
850 | } | |
851 | } | |
8256e47c MD |
852 | } |
853 | } | |
854 | return ERR_PTR(-ENOENT); | |
855 | } | |
856 | EXPORT_SYMBOL_GPL(marker_get_private_data); |