kthread: rename probe_kthread_data() to kthread_probe_data()
[linux-2.6-block.git] / kernel / jump_label.c
CommitLineData
bf5438fc
JB
1/*
2 * jump label support
3 *
4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
90eec103 5 * Copyright (C) 2011 Peter Zijlstra
bf5438fc
JB
6 *
7 */
bf5438fc
JB
8#include <linux/memory.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/list.h>
bf5438fc
JB
12#include <linux/slab.h>
13#include <linux/sort.h>
14#include <linux/err.h>
c5905afb 15#include <linux/static_key.h>
851cf6e7 16#include <linux/jump_label_ratelimit.h>
1f69bf9c 17#include <linux/bug.h>
bf5438fc
JB
18
19#ifdef HAVE_JUMP_LABEL
20
bf5438fc
JB
21/* mutex to protect coming/going of the the jump_label table */
22static DEFINE_MUTEX(jump_label_mutex);
23
91bad2f8
JB
24void jump_label_lock(void)
25{
26 mutex_lock(&jump_label_mutex);
27}
28
29void jump_label_unlock(void)
30{
31 mutex_unlock(&jump_label_mutex);
32}
33
bf5438fc
JB
34static int jump_label_cmp(const void *a, const void *b)
35{
36 const struct jump_entry *jea = a;
37 const struct jump_entry *jeb = b;
38
39 if (jea->key < jeb->key)
40 return -1;
41
42 if (jea->key > jeb->key)
43 return 1;
44
45 return 0;
46}
47
48static void
d430d3d7 49jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
bf5438fc
JB
50{
51 unsigned long size;
52
53 size = (((unsigned long)stop - (unsigned long)start)
54 / sizeof(struct jump_entry));
55 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
56}
57
706249c2 58static void jump_label_update(struct static_key *key);
a1efb01f 59
1f69bf9c
JB
60/*
61 * There are similar definitions for the !HAVE_JUMP_LABEL case in jump_label.h.
62 * The use of 'atomic_read()' requires atomic.h and its problematic for some
63 * kernel headers such as kernel.h and others. Since static_key_count() is not
64 * used in the branch statements as it is for the !HAVE_JUMP_LABEL case its ok
65 * to have it be a function here. Similarly, for 'static_key_enable()' and
66 * 'static_key_disable()', which require bug.h. This should allow jump_label.h
67 * to be included from most/all places for HAVE_JUMP_LABEL.
68 */
69int static_key_count(struct static_key *key)
70{
71 /*
72 * -1 means the first static_key_slow_inc() is in progress.
73 * static_key_enabled() must return true, so return 1 here.
74 */
75 int n = atomic_read(&key->enabled);
76
77 return n >= 0 ? n : 1;
78}
79EXPORT_SYMBOL_GPL(static_key_count);
80
81void static_key_enable(struct static_key *key)
82{
83 int count = static_key_count(key);
84
85 WARN_ON_ONCE(count < 0 || count > 1);
86
87 if (!count)
88 static_key_slow_inc(key);
89}
90EXPORT_SYMBOL_GPL(static_key_enable);
91
92void static_key_disable(struct static_key *key)
93{
94 int count = static_key_count(key);
95
96 WARN_ON_ONCE(count < 0 || count > 1);
97
98 if (count)
99 static_key_slow_dec(key);
100}
101EXPORT_SYMBOL_GPL(static_key_disable);
102
c5905afb 103void static_key_slow_inc(struct static_key *key)
bf5438fc 104{
4c5ea0a9
PB
105 int v, v1;
106
c4b2c0c5 107 STATIC_KEY_CHECK_USE();
4c5ea0a9
PB
108
109 /*
110 * Careful if we get concurrent static_key_slow_inc() calls;
111 * later calls must wait for the first one to _finish_ the
112 * jump_label_update() process. At the same time, however,
113 * the jump_label_update() call below wants to see
114 * static_key_enabled(&key) for jumps to be updated properly.
115 *
116 * So give a special meaning to negative key->enabled: it sends
117 * static_key_slow_inc() down the slow path, and it is non-zero
118 * so it counts as "enabled" in jump_label_update(). Note that
119 * atomic_inc_unless_negative() checks >= 0, so roll our own.
120 */
121 for (v = atomic_read(&key->enabled); v > 0; v = v1) {
122 v1 = atomic_cmpxchg(&key->enabled, v, v + 1);
123 if (likely(v1 == v))
124 return;
125 }
bf5438fc 126
d430d3d7 127 jump_label_lock();
4c5ea0a9
PB
128 if (atomic_read(&key->enabled) == 0) {
129 atomic_set(&key->enabled, -1);
706249c2 130 jump_label_update(key);
4c5ea0a9
PB
131 atomic_set(&key->enabled, 1);
132 } else {
133 atomic_inc(&key->enabled);
134 }
d430d3d7 135 jump_label_unlock();
bf5438fc 136}
c5905afb 137EXPORT_SYMBOL_GPL(static_key_slow_inc);
bf5438fc 138
c5905afb 139static void __static_key_slow_dec(struct static_key *key,
b2029520 140 unsigned long rate_limit, struct delayed_work *work)
bf5438fc 141{
4c5ea0a9
PB
142 /*
143 * The negative count check is valid even when a negative
144 * key->enabled is in use by static_key_slow_inc(); a
145 * __static_key_slow_dec() before the first static_key_slow_inc()
146 * returns is unbalanced, because all other static_key_slow_inc()
147 * instances block while the update is in progress.
148 */
fadf0464
JB
149 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
150 WARN(atomic_read(&key->enabled) < 0,
151 "jump label: negative count!\n");
d430d3d7 152 return;
fadf0464 153 }
bf5438fc 154
b2029520
GN
155 if (rate_limit) {
156 atomic_inc(&key->enabled);
157 schedule_delayed_work(work, rate_limit);
c5905afb 158 } else {
706249c2 159 jump_label_update(key);
c5905afb 160 }
91bad2f8 161 jump_label_unlock();
bf5438fc
JB
162}
163
b2029520
GN
164static void jump_label_update_timeout(struct work_struct *work)
165{
c5905afb
IM
166 struct static_key_deferred *key =
167 container_of(work, struct static_key_deferred, work.work);
168 __static_key_slow_dec(&key->key, 0, NULL);
b2029520
GN
169}
170
c5905afb 171void static_key_slow_dec(struct static_key *key)
b2029520 172{
c4b2c0c5 173 STATIC_KEY_CHECK_USE();
c5905afb 174 __static_key_slow_dec(key, 0, NULL);
b2029520 175}
c5905afb 176EXPORT_SYMBOL_GPL(static_key_slow_dec);
b2029520 177
c5905afb 178void static_key_slow_dec_deferred(struct static_key_deferred *key)
b2029520 179{
c4b2c0c5 180 STATIC_KEY_CHECK_USE();
c5905afb 181 __static_key_slow_dec(&key->key, key->timeout, &key->work);
b2029520 182}
c5905afb 183EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
b2029520 184
c5905afb 185void jump_label_rate_limit(struct static_key_deferred *key,
b2029520
GN
186 unsigned long rl)
187{
c4b2c0c5 188 STATIC_KEY_CHECK_USE();
b2029520
GN
189 key->timeout = rl;
190 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
191}
a181dc14 192EXPORT_SYMBOL_GPL(jump_label_rate_limit);
b2029520 193
4c3ef6d7
JB
194static int addr_conflict(struct jump_entry *entry, void *start, void *end)
195{
196 if (entry->code <= (unsigned long)end &&
197 entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
198 return 1;
199
200 return 0;
201}
202
d430d3d7
JB
203static int __jump_label_text_reserved(struct jump_entry *iter_start,
204 struct jump_entry *iter_stop, void *start, void *end)
4c3ef6d7 205{
4c3ef6d7 206 struct jump_entry *iter;
4c3ef6d7 207
4c3ef6d7
JB
208 iter = iter_start;
209 while (iter < iter_stop) {
d430d3d7
JB
210 if (addr_conflict(iter, start, end))
211 return 1;
4c3ef6d7
JB
212 iter++;
213 }
214
d430d3d7
JB
215 return 0;
216}
217
706249c2 218/*
20284aa7
JF
219 * Update code which is definitely not currently executing.
220 * Architectures which need heavyweight synchronization to modify
221 * running code can override this to make the non-live update case
222 * cheaper.
223 */
9cdbe1cb 224void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
20284aa7
JF
225 enum jump_label_type type)
226{
706249c2 227 arch_jump_label_transform(entry, type);
20284aa7
JF
228}
229
706249c2 230static inline struct jump_entry *static_key_entries(struct static_key *key)
d430d3d7 231{
706249c2 232 return (struct jump_entry *)((unsigned long)key->entries & ~JUMP_TYPE_MASK);
4c3ef6d7
JB
233}
234
706249c2 235static inline bool static_key_type(struct static_key *key)
c5905afb 236{
706249c2 237 return (unsigned long)key->entries & JUMP_TYPE_MASK;
a1efb01f 238}
c5905afb 239
7dcfd915
PZ
240static inline struct static_key *jump_entry_key(struct jump_entry *entry)
241{
11276d53
PZ
242 return (struct static_key *)((unsigned long)entry->key & ~1UL);
243}
244
245static bool jump_entry_branch(struct jump_entry *entry)
246{
247 return (unsigned long)entry->key & 1UL;
7dcfd915
PZ
248}
249
706249c2 250static enum jump_label_type jump_label_type(struct jump_entry *entry)
a1efb01f 251{
706249c2 252 struct static_key *key = jump_entry_key(entry);
a1efb01f 253 bool enabled = static_key_enabled(key);
11276d53 254 bool branch = jump_entry_branch(entry);
c5905afb 255
11276d53
PZ
256 /* See the comment in linux/jump_label.h */
257 return enabled ^ branch;
c5905afb
IM
258}
259
706249c2
PZ
260static void __jump_label_update(struct static_key *key,
261 struct jump_entry *entry,
262 struct jump_entry *stop)
263{
264 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
265 /*
266 * entry->code set to 0 invalidates module init text sections
267 * kernel_text_address() verifies we are not in core kernel
268 * init code, see jump_label_invalidate_module_init().
269 */
270 if (entry->code && kernel_text_address(entry->code))
271 arch_jump_label_transform(entry, jump_label_type(entry));
272 }
273}
274
97ce2c88 275void __init jump_label_init(void)
bf5438fc 276{
bf5438fc
JB
277 struct jump_entry *iter_start = __start___jump_table;
278 struct jump_entry *iter_stop = __stop___jump_table;
c5905afb 279 struct static_key *key = NULL;
bf5438fc
JB
280 struct jump_entry *iter;
281
1f69bf9c
JB
282 /*
283 * Since we are initializing the static_key.enabled field with
284 * with the 'raw' int values (to avoid pulling in atomic.h) in
285 * jump_label.h, let's make sure that is safe. There are only two
286 * cases to check since we initialize to 0 or 1.
287 */
288 BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
289 BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
290
e3f91083
KH
291 if (static_key_initialized)
292 return;
293
91bad2f8 294 jump_label_lock();
d430d3d7
JB
295 jump_label_sort_entries(iter_start, iter_stop);
296
297 for (iter = iter_start; iter < iter_stop; iter++) {
c5905afb 298 struct static_key *iterk;
37348804 299
11276d53
PZ
300 /* rewrite NOPs */
301 if (jump_label_type(iter) == JUMP_LABEL_NOP)
302 arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
303
7dcfd915 304 iterk = jump_entry_key(iter);
37348804 305 if (iterk == key)
d430d3d7
JB
306 continue;
307
37348804 308 key = iterk;
c5905afb
IM
309 /*
310 * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
311 */
312 *((unsigned long *)&key->entries) += (unsigned long)iter;
d430d3d7
JB
313#ifdef CONFIG_MODULES
314 key->next = NULL;
315#endif
bf5438fc 316 }
c4b2c0c5 317 static_key_initialized = true;
91bad2f8 318 jump_label_unlock();
bf5438fc 319}
bf5438fc
JB
320
321#ifdef CONFIG_MODULES
322
11276d53
PZ
323static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
324{
325 struct static_key *key = jump_entry_key(entry);
326 bool type = static_key_type(key);
327 bool branch = jump_entry_branch(entry);
328
329 /* See the comment in linux/jump_label.h */
330 return type ^ branch;
331}
332
c5905afb
IM
333struct static_key_mod {
334 struct static_key_mod *next;
d430d3d7
JB
335 struct jump_entry *entries;
336 struct module *mod;
337};
338
339static int __jump_label_mod_text_reserved(void *start, void *end)
340{
341 struct module *mod;
342
bdc9f373 343 preempt_disable();
d430d3d7 344 mod = __module_text_address((unsigned long)start);
bdc9f373
RR
345 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
346 preempt_enable();
347
d430d3d7
JB
348 if (!mod)
349 return 0;
350
d430d3d7
JB
351
352 return __jump_label_text_reserved(mod->jump_entries,
353 mod->jump_entries + mod->num_jump_entries,
354 start, end);
355}
356
706249c2 357static void __jump_label_mod_update(struct static_key *key)
d430d3d7 358{
706249c2 359 struct static_key_mod *mod;
d430d3d7 360
706249c2 361 for (mod = key->next; mod; mod = mod->next) {
7cbc5b8d
JO
362 struct module *m = mod->mod;
363
364 __jump_label_update(key, mod->entries,
706249c2 365 m->jump_entries + m->num_jump_entries);
d430d3d7
JB
366 }
367}
368
369/***
370 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
371 * @mod: module to patch
372 *
373 * Allow for run-time selection of the optimal nops. Before the module
374 * loads patch these with arch_get_jump_label_nop(), which is specified by
375 * the arch specific jump label code.
376 */
377void jump_label_apply_nops(struct module *mod)
bf5438fc 378{
d430d3d7
JB
379 struct jump_entry *iter_start = mod->jump_entries;
380 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
381 struct jump_entry *iter;
382
383 /* if the module doesn't have jump label entries, just return */
384 if (iter_start == iter_stop)
385 return;
386
11276d53
PZ
387 for (iter = iter_start; iter < iter_stop; iter++) {
388 /* Only write NOPs for arch_branch_static(). */
389 if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
390 arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
391 }
bf5438fc
JB
392}
393
d430d3d7 394static int jump_label_add_module(struct module *mod)
bf5438fc 395{
d430d3d7
JB
396 struct jump_entry *iter_start = mod->jump_entries;
397 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
398 struct jump_entry *iter;
c5905afb
IM
399 struct static_key *key = NULL;
400 struct static_key_mod *jlm;
bf5438fc
JB
401
402 /* if the module doesn't have jump label entries, just return */
d430d3d7 403 if (iter_start == iter_stop)
bf5438fc
JB
404 return 0;
405
d430d3d7
JB
406 jump_label_sort_entries(iter_start, iter_stop);
407
408 for (iter = iter_start; iter < iter_stop; iter++) {
c5905afb 409 struct static_key *iterk;
d430d3d7 410
7dcfd915 411 iterk = jump_entry_key(iter);
c5905afb
IM
412 if (iterk == key)
413 continue;
d430d3d7 414
c5905afb 415 key = iterk;
bed831f9 416 if (within_module(iter->key, mod)) {
c5905afb
IM
417 /*
418 * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
419 */
420 *((unsigned long *)&key->entries) += (unsigned long)iter;
d430d3d7
JB
421 key->next = NULL;
422 continue;
bf5438fc 423 }
c5905afb 424 jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
d430d3d7
JB
425 if (!jlm)
426 return -ENOMEM;
d430d3d7
JB
427 jlm->mod = mod;
428 jlm->entries = iter;
429 jlm->next = key->next;
430 key->next = jlm;
431
11276d53
PZ
432 /* Only update if we've changed from our initial state */
433 if (jump_label_type(iter) != jump_label_init_type(iter))
706249c2 434 __jump_label_update(key, iter, iter_stop);
bf5438fc 435 }
d430d3d7 436
bf5438fc
JB
437 return 0;
438}
439
d430d3d7 440static void jump_label_del_module(struct module *mod)
bf5438fc 441{
d430d3d7
JB
442 struct jump_entry *iter_start = mod->jump_entries;
443 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
444 struct jump_entry *iter;
c5905afb
IM
445 struct static_key *key = NULL;
446 struct static_key_mod *jlm, **prev;
bf5438fc 447
d430d3d7 448 for (iter = iter_start; iter < iter_stop; iter++) {
7dcfd915 449 if (jump_entry_key(iter) == key)
d430d3d7
JB
450 continue;
451
7dcfd915 452 key = jump_entry_key(iter);
d430d3d7 453
bed831f9 454 if (within_module(iter->key, mod))
d430d3d7
JB
455 continue;
456
457 prev = &key->next;
458 jlm = key->next;
bf5438fc 459
d430d3d7
JB
460 while (jlm && jlm->mod != mod) {
461 prev = &jlm->next;
462 jlm = jlm->next;
463 }
464
465 if (jlm) {
466 *prev = jlm->next;
467 kfree(jlm);
bf5438fc
JB
468 }
469 }
470}
471
d430d3d7 472static void jump_label_invalidate_module_init(struct module *mod)
b842f8fa 473{
d430d3d7
JB
474 struct jump_entry *iter_start = mod->jump_entries;
475 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
b842f8fa 476 struct jump_entry *iter;
b842f8fa 477
d430d3d7
JB
478 for (iter = iter_start; iter < iter_stop; iter++) {
479 if (within_module_init(iter->code, mod))
480 iter->code = 0;
b842f8fa
JB
481 }
482}
483
bf5438fc
JB
484static int
485jump_label_module_notify(struct notifier_block *self, unsigned long val,
486 void *data)
487{
488 struct module *mod = data;
489 int ret = 0;
490
491 switch (val) {
492 case MODULE_STATE_COMING:
91bad2f8 493 jump_label_lock();
d430d3d7 494 ret = jump_label_add_module(mod);
bf5438fc 495 if (ret)
d430d3d7 496 jump_label_del_module(mod);
91bad2f8 497 jump_label_unlock();
bf5438fc
JB
498 break;
499 case MODULE_STATE_GOING:
91bad2f8 500 jump_label_lock();
d430d3d7 501 jump_label_del_module(mod);
91bad2f8 502 jump_label_unlock();
bf5438fc 503 break;
b842f8fa 504 case MODULE_STATE_LIVE:
91bad2f8 505 jump_label_lock();
d430d3d7 506 jump_label_invalidate_module_init(mod);
91bad2f8 507 jump_label_unlock();
b842f8fa 508 break;
bf5438fc 509 }
bf5438fc 510
d430d3d7 511 return notifier_from_errno(ret);
bf5438fc
JB
512}
513
885885f6 514static struct notifier_block jump_label_module_nb = {
bf5438fc 515 .notifier_call = jump_label_module_notify,
d430d3d7 516 .priority = 1, /* higher than tracepoints */
bf5438fc
JB
517};
518
d430d3d7 519static __init int jump_label_init_module(void)
bf5438fc
JB
520{
521 return register_module_notifier(&jump_label_module_nb);
522}
d430d3d7 523early_initcall(jump_label_init_module);
bf5438fc
JB
524
525#endif /* CONFIG_MODULES */
526
d430d3d7
JB
527/***
528 * jump_label_text_reserved - check if addr range is reserved
529 * @start: start text addr
530 * @end: end text addr
531 *
532 * checks if the text addr located between @start and @end
533 * overlaps with any of the jump label patch addresses. Code
534 * that wants to modify kernel text should first verify that
535 * it does not overlap with any of the jump label addresses.
536 * Caller must hold jump_label_mutex.
537 *
538 * returns 1 if there is an overlap, 0 otherwise
539 */
540int jump_label_text_reserved(void *start, void *end)
541{
542 int ret = __jump_label_text_reserved(__start___jump_table,
543 __stop___jump_table, start, end);
544
545 if (ret)
546 return ret;
547
548#ifdef CONFIG_MODULES
549 ret = __jump_label_mod_text_reserved(start, end);
550#endif
551 return ret;
552}
553
706249c2 554static void jump_label_update(struct static_key *key)
d430d3d7 555{
c5905afb 556 struct jump_entry *stop = __stop___jump_table;
a1efb01f 557 struct jump_entry *entry = static_key_entries(key);
d430d3d7 558#ifdef CONFIG_MODULES
bed831f9 559 struct module *mod;
140fe3b1 560
706249c2 561 __jump_label_mod_update(key);
140fe3b1 562
bed831f9
PZ
563 preempt_disable();
564 mod = __module_address((unsigned long)key);
140fe3b1
XG
565 if (mod)
566 stop = mod->jump_entries + mod->num_jump_entries;
bed831f9 567 preempt_enable();
d430d3d7 568#endif
140fe3b1
XG
569 /* if there are no users, entry can be NULL */
570 if (entry)
706249c2 571 __jump_label_update(key, entry, stop);
d430d3d7
JB
572}
573
1987c947
PZ
574#ifdef CONFIG_STATIC_KEYS_SELFTEST
575static DEFINE_STATIC_KEY_TRUE(sk_true);
576static DEFINE_STATIC_KEY_FALSE(sk_false);
577
578static __init int jump_label_test(void)
579{
580 int i;
581
582 for (i = 0; i < 2; i++) {
583 WARN_ON(static_key_enabled(&sk_true.key) != true);
584 WARN_ON(static_key_enabled(&sk_false.key) != false);
585
586 WARN_ON(!static_branch_likely(&sk_true));
587 WARN_ON(!static_branch_unlikely(&sk_true));
588 WARN_ON(static_branch_likely(&sk_false));
589 WARN_ON(static_branch_unlikely(&sk_false));
590
591 static_branch_disable(&sk_true);
592 static_branch_enable(&sk_false);
593
594 WARN_ON(static_key_enabled(&sk_true.key) == true);
595 WARN_ON(static_key_enabled(&sk_false.key) == false);
596
597 WARN_ON(static_branch_likely(&sk_true));
598 WARN_ON(static_branch_unlikely(&sk_true));
599 WARN_ON(!static_branch_likely(&sk_false));
600 WARN_ON(!static_branch_unlikely(&sk_false));
601
602 static_branch_enable(&sk_true);
603 static_branch_disable(&sk_false);
604 }
605
606 return 0;
607}
608late_initcall(jump_label_test);
609#endif /* STATIC_KEYS_SELFTEST */
610
611#endif /* HAVE_JUMP_LABEL */