Commit | Line | Data |
---|---|---|
bf5438fc JB |
1 | /* |
2 | * jump label support | |
3 | * | |
4 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> | |
d430d3d7 | 5 | * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> |
bf5438fc JB |
6 | * |
7 | */ | |
bf5438fc JB |
8 | #include <linux/memory.h> |
9 | #include <linux/uaccess.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/list.h> | |
bf5438fc JB |
12 | #include <linux/slab.h> |
13 | #include <linux/sort.h> | |
14 | #include <linux/err.h> | |
c5905afb | 15 | #include <linux/static_key.h> |
851cf6e7 | 16 | #include <linux/jump_label_ratelimit.h> |
bf5438fc JB |
17 | |
18 | #ifdef HAVE_JUMP_LABEL | |
19 | ||
bf5438fc JB |
20 | /* mutex to protect coming/going of the the jump_label table */ |
21 | static DEFINE_MUTEX(jump_label_mutex); | |
22 | ||
91bad2f8 JB |
23 | void jump_label_lock(void) |
24 | { | |
25 | mutex_lock(&jump_label_mutex); | |
26 | } | |
27 | ||
28 | void jump_label_unlock(void) | |
29 | { | |
30 | mutex_unlock(&jump_label_mutex); | |
31 | } | |
32 | ||
bf5438fc JB |
33 | static int jump_label_cmp(const void *a, const void *b) |
34 | { | |
35 | const struct jump_entry *jea = a; | |
36 | const struct jump_entry *jeb = b; | |
37 | ||
38 | if (jea->key < jeb->key) | |
39 | return -1; | |
40 | ||
41 | if (jea->key > jeb->key) | |
42 | return 1; | |
43 | ||
44 | return 0; | |
45 | } | |
46 | ||
47 | static void | |
d430d3d7 | 48 | jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) |
bf5438fc JB |
49 | { |
50 | unsigned long size; | |
51 | ||
52 | size = (((unsigned long)stop - (unsigned long)start) | |
53 | / sizeof(struct jump_entry)); | |
54 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); | |
55 | } | |
56 | ||
706249c2 | 57 | static void jump_label_update(struct static_key *key); |
a1efb01f | 58 | |
c5905afb | 59 | void static_key_slow_inc(struct static_key *key) |
bf5438fc | 60 | { |
c4b2c0c5 | 61 | STATIC_KEY_CHECK_USE(); |
d430d3d7 JB |
62 | if (atomic_inc_not_zero(&key->enabled)) |
63 | return; | |
bf5438fc | 64 | |
d430d3d7 | 65 | jump_label_lock(); |
706249c2 PZ |
66 | if (atomic_inc_return(&key->enabled) == 1) |
67 | jump_label_update(key); | |
d430d3d7 | 68 | jump_label_unlock(); |
bf5438fc | 69 | } |
c5905afb | 70 | EXPORT_SYMBOL_GPL(static_key_slow_inc); |
bf5438fc | 71 | |
c5905afb | 72 | static void __static_key_slow_dec(struct static_key *key, |
b2029520 | 73 | unsigned long rate_limit, struct delayed_work *work) |
bf5438fc | 74 | { |
fadf0464 JB |
75 | if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) { |
76 | WARN(atomic_read(&key->enabled) < 0, | |
77 | "jump label: negative count!\n"); | |
d430d3d7 | 78 | return; |
fadf0464 | 79 | } |
bf5438fc | 80 | |
b2029520 GN |
81 | if (rate_limit) { |
82 | atomic_inc(&key->enabled); | |
83 | schedule_delayed_work(work, rate_limit); | |
c5905afb | 84 | } else { |
706249c2 | 85 | jump_label_update(key); |
c5905afb | 86 | } |
91bad2f8 | 87 | jump_label_unlock(); |
bf5438fc JB |
88 | } |
89 | ||
b2029520 GN |
90 | static void jump_label_update_timeout(struct work_struct *work) |
91 | { | |
c5905afb IM |
92 | struct static_key_deferred *key = |
93 | container_of(work, struct static_key_deferred, work.work); | |
94 | __static_key_slow_dec(&key->key, 0, NULL); | |
b2029520 GN |
95 | } |
96 | ||
c5905afb | 97 | void static_key_slow_dec(struct static_key *key) |
b2029520 | 98 | { |
c4b2c0c5 | 99 | STATIC_KEY_CHECK_USE(); |
c5905afb | 100 | __static_key_slow_dec(key, 0, NULL); |
b2029520 | 101 | } |
c5905afb | 102 | EXPORT_SYMBOL_GPL(static_key_slow_dec); |
b2029520 | 103 | |
c5905afb | 104 | void static_key_slow_dec_deferred(struct static_key_deferred *key) |
b2029520 | 105 | { |
c4b2c0c5 | 106 | STATIC_KEY_CHECK_USE(); |
c5905afb | 107 | __static_key_slow_dec(&key->key, key->timeout, &key->work); |
b2029520 | 108 | } |
c5905afb | 109 | EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred); |
b2029520 | 110 | |
c5905afb | 111 | void jump_label_rate_limit(struct static_key_deferred *key, |
b2029520 GN |
112 | unsigned long rl) |
113 | { | |
c4b2c0c5 | 114 | STATIC_KEY_CHECK_USE(); |
b2029520 GN |
115 | key->timeout = rl; |
116 | INIT_DELAYED_WORK(&key->work, jump_label_update_timeout); | |
117 | } | |
a181dc14 | 118 | EXPORT_SYMBOL_GPL(jump_label_rate_limit); |
b2029520 | 119 | |
4c3ef6d7 JB |
120 | static int addr_conflict(struct jump_entry *entry, void *start, void *end) |
121 | { | |
122 | if (entry->code <= (unsigned long)end && | |
123 | entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) | |
124 | return 1; | |
125 | ||
126 | return 0; | |
127 | } | |
128 | ||
d430d3d7 JB |
129 | static int __jump_label_text_reserved(struct jump_entry *iter_start, |
130 | struct jump_entry *iter_stop, void *start, void *end) | |
4c3ef6d7 | 131 | { |
4c3ef6d7 | 132 | struct jump_entry *iter; |
4c3ef6d7 | 133 | |
4c3ef6d7 JB |
134 | iter = iter_start; |
135 | while (iter < iter_stop) { | |
d430d3d7 JB |
136 | if (addr_conflict(iter, start, end)) |
137 | return 1; | |
4c3ef6d7 JB |
138 | iter++; |
139 | } | |
140 | ||
d430d3d7 JB |
141 | return 0; |
142 | } | |
143 | ||
706249c2 | 144 | /* |
20284aa7 JF |
145 | * Update code which is definitely not currently executing. |
146 | * Architectures which need heavyweight synchronization to modify | |
147 | * running code can override this to make the non-live update case | |
148 | * cheaper. | |
149 | */ | |
9cdbe1cb | 150 | void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry, |
20284aa7 JF |
151 | enum jump_label_type type) |
152 | { | |
706249c2 | 153 | arch_jump_label_transform(entry, type); |
20284aa7 JF |
154 | } |
155 | ||
706249c2 | 156 | static inline struct jump_entry *static_key_entries(struct static_key *key) |
d430d3d7 | 157 | { |
706249c2 | 158 | return (struct jump_entry *)((unsigned long)key->entries & ~JUMP_TYPE_MASK); |
4c3ef6d7 JB |
159 | } |
160 | ||
706249c2 | 161 | static inline bool static_key_type(struct static_key *key) |
c5905afb | 162 | { |
706249c2 | 163 | return (unsigned long)key->entries & JUMP_TYPE_MASK; |
a1efb01f | 164 | } |
c5905afb | 165 | |
7dcfd915 PZ |
166 | static inline struct static_key *jump_entry_key(struct jump_entry *entry) |
167 | { | |
168 | return (struct static_key *)((unsigned long)entry->key); | |
169 | } | |
170 | ||
706249c2 | 171 | static enum jump_label_type jump_label_type(struct jump_entry *entry) |
a1efb01f | 172 | { |
706249c2 | 173 | struct static_key *key = jump_entry_key(entry); |
a1efb01f PZ |
174 | bool enabled = static_key_enabled(key); |
175 | bool type = static_key_type(key); | |
c5905afb | 176 | |
a1efb01f | 177 | return enabled ^ type; |
c5905afb IM |
178 | } |
179 | ||
706249c2 PZ |
180 | static void __jump_label_update(struct static_key *key, |
181 | struct jump_entry *entry, | |
182 | struct jump_entry *stop) | |
183 | { | |
184 | for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { | |
185 | /* | |
186 | * entry->code set to 0 invalidates module init text sections | |
187 | * kernel_text_address() verifies we are not in core kernel | |
188 | * init code, see jump_label_invalidate_module_init(). | |
189 | */ | |
190 | if (entry->code && kernel_text_address(entry->code)) | |
191 | arch_jump_label_transform(entry, jump_label_type(entry)); | |
192 | } | |
193 | } | |
194 | ||
97ce2c88 | 195 | void __init jump_label_init(void) |
bf5438fc | 196 | { |
bf5438fc JB |
197 | struct jump_entry *iter_start = __start___jump_table; |
198 | struct jump_entry *iter_stop = __stop___jump_table; | |
c5905afb | 199 | struct static_key *key = NULL; |
bf5438fc JB |
200 | struct jump_entry *iter; |
201 | ||
91bad2f8 | 202 | jump_label_lock(); |
d430d3d7 JB |
203 | jump_label_sort_entries(iter_start, iter_stop); |
204 | ||
205 | for (iter = iter_start; iter < iter_stop; iter++) { | |
c5905afb | 206 | struct static_key *iterk; |
37348804 | 207 | |
706249c2 | 208 | arch_jump_label_transform_static(iter, jump_label_type(iter)); |
7dcfd915 | 209 | iterk = jump_entry_key(iter); |
37348804 | 210 | if (iterk == key) |
d430d3d7 JB |
211 | continue; |
212 | ||
37348804 | 213 | key = iterk; |
c5905afb IM |
214 | /* |
215 | * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. | |
216 | */ | |
217 | *((unsigned long *)&key->entries) += (unsigned long)iter; | |
d430d3d7 JB |
218 | #ifdef CONFIG_MODULES |
219 | key->next = NULL; | |
220 | #endif | |
bf5438fc | 221 | } |
c4b2c0c5 | 222 | static_key_initialized = true; |
91bad2f8 | 223 | jump_label_unlock(); |
bf5438fc | 224 | } |
bf5438fc JB |
225 | |
226 | #ifdef CONFIG_MODULES | |
227 | ||
c5905afb IM |
228 | struct static_key_mod { |
229 | struct static_key_mod *next; | |
d430d3d7 JB |
230 | struct jump_entry *entries; |
231 | struct module *mod; | |
232 | }; | |
233 | ||
234 | static int __jump_label_mod_text_reserved(void *start, void *end) | |
235 | { | |
236 | struct module *mod; | |
237 | ||
238 | mod = __module_text_address((unsigned long)start); | |
239 | if (!mod) | |
240 | return 0; | |
241 | ||
242 | WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); | |
243 | ||
244 | return __jump_label_text_reserved(mod->jump_entries, | |
245 | mod->jump_entries + mod->num_jump_entries, | |
246 | start, end); | |
247 | } | |
248 | ||
706249c2 | 249 | static void __jump_label_mod_update(struct static_key *key) |
d430d3d7 | 250 | { |
706249c2 | 251 | struct static_key_mod *mod; |
d430d3d7 | 252 | |
706249c2 | 253 | for (mod = key->next; mod; mod = mod->next) { |
7cbc5b8d JO |
254 | struct module *m = mod->mod; |
255 | ||
256 | __jump_label_update(key, mod->entries, | |
706249c2 | 257 | m->jump_entries + m->num_jump_entries); |
d430d3d7 JB |
258 | } |
259 | } | |
260 | ||
261 | /*** | |
262 | * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() | |
263 | * @mod: module to patch | |
264 | * | |
265 | * Allow for run-time selection of the optimal nops. Before the module | |
266 | * loads patch these with arch_get_jump_label_nop(), which is specified by | |
267 | * the arch specific jump label code. | |
268 | */ | |
269 | void jump_label_apply_nops(struct module *mod) | |
bf5438fc | 270 | { |
d430d3d7 JB |
271 | struct jump_entry *iter_start = mod->jump_entries; |
272 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | |
273 | struct jump_entry *iter; | |
274 | ||
275 | /* if the module doesn't have jump label entries, just return */ | |
276 | if (iter_start == iter_stop) | |
277 | return; | |
278 | ||
706249c2 | 279 | for (iter = iter_start; iter < iter_stop; iter++) |
76b235c6 | 280 | arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); |
bf5438fc JB |
281 | } |
282 | ||
d430d3d7 | 283 | static int jump_label_add_module(struct module *mod) |
bf5438fc | 284 | { |
d430d3d7 JB |
285 | struct jump_entry *iter_start = mod->jump_entries; |
286 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | |
287 | struct jump_entry *iter; | |
c5905afb IM |
288 | struct static_key *key = NULL; |
289 | struct static_key_mod *jlm; | |
bf5438fc JB |
290 | |
291 | /* if the module doesn't have jump label entries, just return */ | |
d430d3d7 | 292 | if (iter_start == iter_stop) |
bf5438fc JB |
293 | return 0; |
294 | ||
d430d3d7 JB |
295 | jump_label_sort_entries(iter_start, iter_stop); |
296 | ||
297 | for (iter = iter_start; iter < iter_stop; iter++) { | |
c5905afb | 298 | struct static_key *iterk; |
d430d3d7 | 299 | |
7dcfd915 | 300 | iterk = jump_entry_key(iter); |
c5905afb IM |
301 | if (iterk == key) |
302 | continue; | |
d430d3d7 | 303 | |
c5905afb | 304 | key = iterk; |
bed831f9 | 305 | if (within_module(iter->key, mod)) { |
c5905afb IM |
306 | /* |
307 | * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. | |
308 | */ | |
309 | *((unsigned long *)&key->entries) += (unsigned long)iter; | |
d430d3d7 JB |
310 | key->next = NULL; |
311 | continue; | |
bf5438fc | 312 | } |
c5905afb | 313 | jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL); |
d430d3d7 JB |
314 | if (!jlm) |
315 | return -ENOMEM; | |
d430d3d7 JB |
316 | jlm->mod = mod; |
317 | jlm->entries = iter; | |
318 | jlm->next = key->next; | |
319 | key->next = jlm; | |
320 | ||
706249c2 PZ |
321 | if (jump_label_type(iter) == JUMP_LABEL_JMP) |
322 | __jump_label_update(key, iter, iter_stop); | |
bf5438fc | 323 | } |
d430d3d7 | 324 | |
bf5438fc JB |
325 | return 0; |
326 | } | |
327 | ||
d430d3d7 | 328 | static void jump_label_del_module(struct module *mod) |
bf5438fc | 329 | { |
d430d3d7 JB |
330 | struct jump_entry *iter_start = mod->jump_entries; |
331 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | |
332 | struct jump_entry *iter; | |
c5905afb IM |
333 | struct static_key *key = NULL; |
334 | struct static_key_mod *jlm, **prev; | |
bf5438fc | 335 | |
d430d3d7 | 336 | for (iter = iter_start; iter < iter_stop; iter++) { |
7dcfd915 | 337 | if (jump_entry_key(iter) == key) |
d430d3d7 JB |
338 | continue; |
339 | ||
7dcfd915 | 340 | key = jump_entry_key(iter); |
d430d3d7 | 341 | |
bed831f9 | 342 | if (within_module(iter->key, mod)) |
d430d3d7 JB |
343 | continue; |
344 | ||
345 | prev = &key->next; | |
346 | jlm = key->next; | |
bf5438fc | 347 | |
d430d3d7 JB |
348 | while (jlm && jlm->mod != mod) { |
349 | prev = &jlm->next; | |
350 | jlm = jlm->next; | |
351 | } | |
352 | ||
353 | if (jlm) { | |
354 | *prev = jlm->next; | |
355 | kfree(jlm); | |
bf5438fc JB |
356 | } |
357 | } | |
358 | } | |
359 | ||
d430d3d7 | 360 | static void jump_label_invalidate_module_init(struct module *mod) |
b842f8fa | 361 | { |
d430d3d7 JB |
362 | struct jump_entry *iter_start = mod->jump_entries; |
363 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | |
b842f8fa | 364 | struct jump_entry *iter; |
b842f8fa | 365 | |
d430d3d7 JB |
366 | for (iter = iter_start; iter < iter_stop; iter++) { |
367 | if (within_module_init(iter->code, mod)) | |
368 | iter->code = 0; | |
b842f8fa JB |
369 | } |
370 | } | |
371 | ||
bf5438fc JB |
372 | static int |
373 | jump_label_module_notify(struct notifier_block *self, unsigned long val, | |
374 | void *data) | |
375 | { | |
376 | struct module *mod = data; | |
377 | int ret = 0; | |
378 | ||
379 | switch (val) { | |
380 | case MODULE_STATE_COMING: | |
91bad2f8 | 381 | jump_label_lock(); |
d430d3d7 | 382 | ret = jump_label_add_module(mod); |
bf5438fc | 383 | if (ret) |
d430d3d7 | 384 | jump_label_del_module(mod); |
91bad2f8 | 385 | jump_label_unlock(); |
bf5438fc JB |
386 | break; |
387 | case MODULE_STATE_GOING: | |
91bad2f8 | 388 | jump_label_lock(); |
d430d3d7 | 389 | jump_label_del_module(mod); |
91bad2f8 | 390 | jump_label_unlock(); |
bf5438fc | 391 | break; |
b842f8fa | 392 | case MODULE_STATE_LIVE: |
91bad2f8 | 393 | jump_label_lock(); |
d430d3d7 | 394 | jump_label_invalidate_module_init(mod); |
91bad2f8 | 395 | jump_label_unlock(); |
b842f8fa | 396 | break; |
bf5438fc | 397 | } |
bf5438fc | 398 | |
d430d3d7 | 399 | return notifier_from_errno(ret); |
bf5438fc JB |
400 | } |
401 | ||
402 | struct notifier_block jump_label_module_nb = { | |
403 | .notifier_call = jump_label_module_notify, | |
d430d3d7 | 404 | .priority = 1, /* higher than tracepoints */ |
bf5438fc JB |
405 | }; |
406 | ||
d430d3d7 | 407 | static __init int jump_label_init_module(void) |
bf5438fc JB |
408 | { |
409 | return register_module_notifier(&jump_label_module_nb); | |
410 | } | |
d430d3d7 | 411 | early_initcall(jump_label_init_module); |
bf5438fc JB |
412 | |
413 | #endif /* CONFIG_MODULES */ | |
414 | ||
d430d3d7 JB |
415 | /*** |
416 | * jump_label_text_reserved - check if addr range is reserved | |
417 | * @start: start text addr | |
418 | * @end: end text addr | |
419 | * | |
420 | * checks if the text addr located between @start and @end | |
421 | * overlaps with any of the jump label patch addresses. Code | |
422 | * that wants to modify kernel text should first verify that | |
423 | * it does not overlap with any of the jump label addresses. | |
424 | * Caller must hold jump_label_mutex. | |
425 | * | |
426 | * returns 1 if there is an overlap, 0 otherwise | |
427 | */ | |
428 | int jump_label_text_reserved(void *start, void *end) | |
429 | { | |
430 | int ret = __jump_label_text_reserved(__start___jump_table, | |
431 | __stop___jump_table, start, end); | |
432 | ||
433 | if (ret) | |
434 | return ret; | |
435 | ||
436 | #ifdef CONFIG_MODULES | |
437 | ret = __jump_label_mod_text_reserved(start, end); | |
438 | #endif | |
439 | return ret; | |
440 | } | |
441 | ||
706249c2 | 442 | static void jump_label_update(struct static_key *key) |
d430d3d7 | 443 | { |
c5905afb | 444 | struct jump_entry *stop = __stop___jump_table; |
a1efb01f | 445 | struct jump_entry *entry = static_key_entries(key); |
d430d3d7 | 446 | #ifdef CONFIG_MODULES |
bed831f9 | 447 | struct module *mod; |
140fe3b1 | 448 | |
706249c2 | 449 | __jump_label_mod_update(key); |
140fe3b1 | 450 | |
bed831f9 PZ |
451 | preempt_disable(); |
452 | mod = __module_address((unsigned long)key); | |
140fe3b1 XG |
453 | if (mod) |
454 | stop = mod->jump_entries + mod->num_jump_entries; | |
bed831f9 | 455 | preempt_enable(); |
d430d3d7 | 456 | #endif |
140fe3b1 XG |
457 | /* if there are no users, entry can be NULL */ |
458 | if (entry) | |
706249c2 | 459 | __jump_label_update(key, entry, stop); |
d430d3d7 JB |
460 | } |
461 | ||
bf5438fc | 462 | #endif |