projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
objtool: Fix memory leak in create_static_call_sections()
[linux-block.git] / tools / objtool / check.c
CommitLineData
1ccea77e 1// SPDX-License-Identifier: GPL-2.0-or-later
dcc914f4
JP		 2/*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
dcc914f4
JP		 4 */
5
6#include <string.h>
7#include <stdlib.h>
22682a07 8#include <inttypes.h>
8b946cc3 9#include <sys/mman.h>
dcc914f4 10
7786032e
VG		 11#include <arch/elf.h>
12#include <objtool/builtin.h>
13#include <objtool/cfi.h>
14#include <objtool/arch.h>
15#include <objtool/check.h>
16#include <objtool/special.h>
17#include <objtool/warn.h>
18#include <objtool/endianness.h>
dcc914f4 19
ee819aed 20#include <linux/objtool.h>
dcc914f4
JP		 21#include <linux/hashtable.h>
22#include <linux/kernel.h>
1e7e4788 23#include <linux/static_call_types.h>
dcc914f4 24
dcc914f4
JP		 25struct alternative {
26 struct list_head list;
27 struct instruction *insn;
764eef4b 28 bool skip_orig;
dcc914f4
JP		 29};
30
8b946cc3
PZ		 31static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33static struct cfi_init_state initial_func_cfi;
34static struct cfi_state init_cfi;
35static struct cfi_state func_cfi;
dcc914f4 36
627fce14
JP		 37struct instruction *find_insn(struct objtool_file *file,
38 struct section *sec, unsigned long offset)
dcc914f4
JP		 39{
40 struct instruction *insn;
41
87ecb582 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
dcc914f4
JP		 43 if (insn->sec == sec && insn->offset == offset)
44 return insn;
87ecb582 45 }
dcc914f4
JP		 46
47 return NULL;
48}
49
50static struct instruction *next_insn_same_sec(struct objtool_file *file,
51 struct instruction *insn)
52{
53 struct instruction *next = list_next_entry(insn, list);
54
baa41469 55 if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
dcc914f4
JP		 56 return NULL;
57
58 return next;
59}
60
13810435
JP		 61static struct instruction *next_insn_same_func(struct objtool_file *file,
62 struct instruction *insn)
63{
64 struct instruction *next = list_next_entry(insn, list);
dbcdbdfd 65 struct symbol *func = insn_func(insn);
13810435
JP		 66
67 if (!func)
68 return NULL;
69
dbcdbdfd 70 if (&next->list != &file->insn_list && insn_func(next) == func)
13810435
JP		 71 return next;
72
73 /* Check if we're already in the subfunction: */
74 if (func == func->cfunc)
75 return NULL;
76
77 /* Move to the subfunction: */
78 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
79}
80
1119d265
JP		 81static struct instruction *prev_insn_same_sym(struct objtool_file *file,
82 struct instruction *insn)
83{
84 struct instruction *prev = list_prev_entry(insn, list);
85
dbcdbdfd 86 if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn))
1119d265
JP		 87 return prev;
88
89 return NULL;
90}
91
f0f70adb 92#define func_for_each_insn(file, func, insn) \
13810435
JP		 93 for (insn = find_insn(file, func->sec, func->offset); \
94 insn; \
95 insn = next_insn_same_func(file, insn))
96
dbf4aeb0
PZ		 97#define sym_for_each_insn(file, sym, insn) \
98 for (insn = find_insn(file, sym->sec, sym->offset); \
dcc914f4 99 insn && &insn->list != &file->insn_list && \
dbf4aeb0
PZ		 100 insn->sec == sym->sec && \
101 insn->offset < sym->offset + sym->len; \
dcc914f4
JP		 102 insn = list_next_entry(insn, list))
103
dbf4aeb0 104#define sym_for_each_insn_continue_reverse(file, sym, insn) \
dcc914f4
JP		 105 for (insn = list_prev_entry(insn, list); \
106 &insn->list != &file->insn_list && \
dbf4aeb0 107 insn->sec == sym->sec && insn->offset >= sym->offset; \
dcc914f4
JP		 108 insn = list_prev_entry(insn, list))
109
110#define sec_for_each_insn_from(file, insn) \
111 for (; insn; insn = next_insn_same_sec(file, insn))
112
baa41469
JP		 113#define sec_for_each_insn_continue(file, insn) \
114 for (insn = next_insn_same_sec(file, insn); insn; \
115 insn = next_insn_same_sec(file, insn))
dcc914f4 116
99033461
JP		 117static bool is_jump_table_jump(struct instruction *insn)
118{
119 struct alt_group *alt_group = insn->alt_group;
120
121 if (insn->jump_table)
122 return true;
123
124 /* Retpoline alternative for a jump table? */
125 return alt_group && alt_group->orig_group &&
126 alt_group->orig_group->first_insn->jump_table;
127}
128
0c1ddd33
JP		 129static bool is_sibling_call(struct instruction *insn)
130{
ecf11ba4 131 /*
5a9c361a 132 * Assume only STT_FUNC calls have jump-tables.
ecf11ba4 133 */
5a9c361a
PZ		 134 if (insn_func(insn)) {
135 /* An indirect jump is either a sibling call or a jump to a table. */
136 if (insn->type == INSN_JUMP_DYNAMIC)
137 return !is_jump_table_jump(insn);
138 }
0c1ddd33 139
0c1ddd33 140 /* add_jump_destinations() sets insn->call_dest for sibling calls. */
ecf11ba4 141 return (is_static_jump(insn) && insn->call_dest);
0c1ddd33
JP		 142}
143
dcc914f4
JP		 144/*
145 * This checks to see if the given function is a "noreturn" function.
146 *
147 * For global functions which are outside the scope of this object file, we
148 * have to keep a manual list of them.
149 *
150 * For local functions, we have to detect them manually by simply looking for
151 * the lack of a return instruction.
dcc914f4 152 */
8e25c9f8
JP		 153static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
154 int recursion)
dcc914f4
JP		 155{
156 int i;
157 struct instruction *insn;
158 bool empty = true;
159
160 /*
161 * Unfortunately these have to be hard coded because the noreturn
c93c296f 162 * attribute isn't provided in ELF data. Keep 'em sorted.
dcc914f4
JP		 163 */
164 static const char * const global_noreturns[] = {
c93c296f
BP		 165 "__invalid_creds",
166 "__module_put_and_kthread_exit",
167 "__reiserfs_panic",
dcc914f4 168 "__stack_chk_fail",
c93c296f
BP		 169 "__ubsan_handle_builtin_unreachable",
170 "cpu_bringup_and_idle",
171 "cpu_startup_entry",
dcc914f4 172 "do_exit",
c93c296f 173 "do_group_exit",
dcc914f4 174 "do_task_dead",
c93c296f
BP		 175 "ex_handler_msr_mce",
176 "fortify_panic",
cead1855 177 "kthread_complete_and_exit",
c93c296f
BP		 178 "kthread_exit",
179 "kunit_try_catch_throw",
dcc914f4 180 "lbug_with_loc",
684fb246 181 "machine_real_restart",
c93c296f
BP		 182 "make_task_dead",
183 "panic",
1fb466df 184 "rewind_stack_and_make_dead",
c93c296f
BP		 185 "sev_es_terminate",
186 "snp_abort",
f9cdf7ca 187 "stop_this_cpu",
c93c296f
BP		 188 "usercopy_abort",
189 "xen_start_kernel",
dcc914f4
JP		 190 };
191
c9bab22b
JP		 192 if (!func)
193 return false;
194
dcc914f4 195 if (func->bind == STB_WEAK)
8e25c9f8 196 return false;
dcc914f4
JP		 197
198 if (func->bind == STB_GLOBAL)
199 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
200 if (!strcmp(func->name, global_noreturns[i]))
8e25c9f8 201 return true;
dcc914f4 202
13810435 203 if (!func->len)
8e25c9f8 204 return false;
dcc914f4 205
13810435 206 insn = find_insn(file, func->sec, func->offset);
5f6e430f 207 if (!insn || !insn_func(insn))
8e25c9f8 208 return false;
13810435 209
f0f70adb 210 func_for_each_insn(file, func, insn) {
dcc914f4
JP		 211 empty = false;
212
213 if (insn->type == INSN_RETURN)
8e25c9f8 214 return false;
dcc914f4
JP		 215 }
216
217 if (empty)
8e25c9f8 218 return false;
dcc914f4
JP		 219
220 /*
221 * A function can have a sibling call instead of a return. In that
222 * case, the function's dead-end status depends on whether the target
223 * of the sibling call returns.
224 */
f0f70adb 225 func_for_each_insn(file, func, insn) {
0c1ddd33 226 if (is_sibling_call(insn)) {
dcc914f4 227 struct instruction *dest = insn->jump_dest;
dcc914f4
JP		 228
229 if (!dest)
230 /* sibling call to another file */
8e25c9f8 231 return false;
dcc914f4 232
0c1ddd33
JP		 233 /* local sibling call */
234 if (recursion == 5) {
235 /*
236 * Infinite recursion: two functions have
237 * sibling calls to each other. This is a very
238 * rare case. It means they aren't dead ends.
239 */
240 return false;
dcc914f4 241 }
dcc914f4 242
dbcdbdfd 243 return __dead_end_function(file, insn_func(dest), recursion+1);
0c1ddd33 244 }
dcc914f4
JP		 245 }
246
8e25c9f8 247 return true;
dcc914f4
JP		 248}
249
8e25c9f8 250static bool dead_end_function(struct objtool_file *file, struct symbol *func)
dcc914f4
JP		 251{
252 return __dead_end_function(file, func, 0);
253}
254
e7c0219b 255static void init_cfi_state(struct cfi_state *cfi)
baa41469
JP		 256{
257 int i;
258
dd88a0a0 259 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 260 cfi->regs[i].base = CFI_UNDEFINED;
261 cfi->vals[i].base = CFI_UNDEFINED;
dd88a0a0 262 }
e7c0219b
PZ		 263 cfi->cfa.base = CFI_UNDEFINED;
264 cfi->drap_reg = CFI_UNDEFINED;
265 cfi->drap_offset = -1;
266}
267
753da417
JP		 268static void init_insn_state(struct objtool_file *file, struct insn_state *state,
269 struct section *sec)
e7c0219b
PZ		 270{
271 memset(state, 0, sizeof(*state));
272 init_cfi_state(&state->cfi);
932f8e98
PZ		 273
274 /*
275 * We need the full vmlinux for noinstr validation, otherwise we can
276 * not correctly determine insn->call_dest->sec (external symbols do
277 * not have a section).
278 */
753da417 279 if (opts.link && opts.noinstr && sec)
932f8e98 280 state->noinstr = sec->noinstr;
baa41469
JP		 281}
282
8b946cc3
PZ		 283static struct cfi_state *cfi_alloc(void)
284{
285 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
286 if (!cfi) {
287 WARN("calloc failed");
288 exit(1);
289 }
290 nr_cfi++;
291 return cfi;
292}
293
294static int cfi_bits;
295static struct hlist_head *cfi_hash;
296
297static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
298{
299 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
300 (void *)cfi2 + sizeof(cfi2->hash),
301 sizeof(struct cfi_state) - sizeof(struct hlist_node));
302}
303
304static inline u32 cfi_key(struct cfi_state *cfi)
305{
306 return jhash((void *)cfi + sizeof(cfi->hash),
307 sizeof(*cfi) - sizeof(cfi->hash), 0);
308}
309
310static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
311{
312 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
313 struct cfi_state *obj;
314
315 hlist_for_each_entry(obj, head, hash) {
316 if (!cficmp(cfi, obj)) {
317 nr_cfi_cache++;
318 return obj;
319 }
320 }
321
322 obj = cfi_alloc();
323 *obj = *cfi;
324 hlist_add_head(&obj->hash, head);
325
326 return obj;
327}
328
329static void cfi_hash_add(struct cfi_state *cfi)
330{
331 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
332
333 hlist_add_head(&cfi->hash, head);
334}
335
336static void *cfi_hash_alloc(unsigned long size)
337{
338 cfi_bits = max(10, ilog2(size));
339 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
340 PROT_READ|PROT_WRITE,
341 MAP_PRIVATE|MAP_ANON, -1, 0);
342 if (cfi_hash == (void *)-1L) {
343 WARN("mmap fail cfi_hash");
344 cfi_hash = NULL;
2daf7fab 345 } else if (opts.stats) {
8b946cc3
PZ		 346 printf("cfi_bits: %d\n", cfi_bits);
347 }
348
349 return cfi_hash;
350}
351
352static unsigned long nr_insns;
353static unsigned long nr_insns_visited;
354
dcc914f4
JP		 355/*
356 * Call the arch-specific instruction decoder for all the instructions and add
357 * them to the global instruction list.
358 */
359static int decode_instructions(struct objtool_file *file)
360{
361 struct section *sec;
362 struct symbol *func;
363 unsigned long offset;
364 struct instruction *insn;
365 int ret;
366
baa41469 367 for_each_sec(file, sec) {
dcc914f4
JP		 368
369 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
370 continue;
371
627fce14
JP		 372 if (strcmp(sec->name, ".altinstr_replacement") &&
373 strcmp(sec->name, ".altinstr_aux") &&
374 strncmp(sec->name, ".discard.", 9))
375 sec->text = true;
376
0cc9ac8d 377 if (!strcmp(sec->name, ".noinstr.text") ||
951ddecf
PZ		 378 !strcmp(sec->name, ".entry.text") ||
379 !strncmp(sec->name, ".text.__x86.", 12))
c4a33939
PZ		 380 sec->noinstr = true;
381
6644ee84
PZ		 382 /*
383 * .init.text code is ran before userspace and thus doesn't
384 * strictly need retpolines, except for modules which are
385 * loaded late, they very much do need retpoline in their
386 * .init.text
387 */
388 if (!strcmp(sec->name, ".init.text") && !opts.module)
389 sec->init = true;
390
fe255fe6 391 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
dcc914f4 392 insn = malloc(sizeof(*insn));
baa41469
JP		 393 if (!insn) {
394 WARN("malloc failed");
395 return -1;
396 }
dcc914f4 397 memset(insn, 0, sizeof(*insn));
dcc914f4 398 INIT_LIST_HEAD(&insn->alts);
65ea47dc 399 INIT_LIST_HEAD(&insn->stack_ops);
89bc853e 400 INIT_LIST_HEAD(&insn->call_node);
baa41469 401
dcc914f4
JP		 402 insn->sec = sec;
403 insn->offset = offset;
404
db2b0c5d 405 ret = arch_decode_instruction(file, sec, offset,
fe255fe6 406 sec->sh.sh_size - offset,
dcc914f4 407 &insn->len, &insn->type,
baa41469 408 &insn->immediate,
65ea47dc 409 &insn->stack_ops);
dcc914f4 410 if (ret)
b7037983 411 goto err;
dcc914f4 412
0e5b613b
PZ		 413 /*
414 * By default, "ud2" is a dead end unless otherwise
415 * annotated, because GCC 7 inserts it for certain
416 * divide-by-zero cases.
417 */
418 if (insn->type == INSN_BUG)
419 insn->dead_end = true;
420
87ecb582 421 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
dcc914f4 422 list_add_tail(&insn->list, &file->insn_list);
1e11f3fd 423 nr_insns++;
dcc914f4
JP		 424 }
425
426 list_for_each_entry(func, &sec->symbol_list, list) {
dbcdbdfd
PZ		 427 if (func->type != STT_NOTYPE && func->type != STT_FUNC)
428 continue;
429
430 if (func->return_thunk || func->alias != func)
dcc914f4
JP		 431 continue;
432
433 if (!find_insn(file, sec, func->offset)) {
434 WARN("%s(): can't find starting instruction",
435 func->name);
436 return -1;
437 }
438
08f87a93 439 sym_for_each_insn(file, func, insn) {
dbcdbdfd
PZ		 440 insn->sym = func;
441 if (func->type == STT_FUNC &&
442 insn->type == INSN_ENDBR &&
443 list_empty(&insn->call_node)) {
444 if (insn->offset == func->offset) {
89bc853e 445 list_add_tail(&insn->call_node, &file->endbr_list);
08f87a93
PZ		 446 file->nr_endbr++;
447 } else {
448 file->nr_endbr_int++;
449 }
450 }
451 }
dcc914f4
JP		 452 }
453 }
454
2daf7fab 455 if (opts.stats)
1e11f3fd
PZ		 456 printf("nr_insns: %lu\n", nr_insns);
457
dcc914f4 458 return 0;
b7037983
KB		 459
460err:
461 free(insn);
462 return ret;
dcc914f4
JP		 463}
464
db2b0c5d
PZ		 465/*
466 * Read the pv_ops[] .data table to find the static initialized values.
467 */
468static int add_pv_ops(struct objtool_file *file, const char *symname)
469{
470 struct symbol *sym, *func;
471 unsigned long off, end;
472 struct reloc *rel;
473 int idx;
474
475 sym = find_symbol_by_name(file->elf, symname);
476 if (!sym)
477 return 0;
478
479 off = sym->offset;
480 end = off + sym->len;
481 for (;;) {
482 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
483 if (!rel)
484 break;
485
486 func = rel->sym;
487 if (func->type == STT_SECTION)
488 func = find_symbol_by_offset(rel->sym->sec, rel->addend);
489
490 idx = (rel->offset - sym->offset) / sizeof(unsigned long);
491
492 objtool_pv_add(file, idx, func);
493
494 off = rel->offset + 1;
495 if (off > end)
496 break;
497 }
498
499 return 0;
500}
501
502/*
503 * Allocate and initialize file->pv_ops[].
504 */
505static int init_pv_ops(struct objtool_file *file)
506{
507 static const char *pv_ops_tables[] = {
508 "pv_ops",
509 "xen_cpu_ops",
510 "xen_irq_ops",
511 "xen_mmu_ops",
512 NULL,
513 };
514 const char *pv_ops;
515 struct symbol *sym;
516 int idx, nr;
517
2daf7fab 518 if (!opts.noinstr)
db2b0c5d
PZ		 519 return 0;
520
521 file->pv_ops = NULL;
522
523 sym = find_symbol_by_name(file->elf, "pv_ops");
524 if (!sym)
525 return 0;
526
527 nr = sym->len / sizeof(unsigned long);
528 file->pv_ops = calloc(sizeof(struct pv_state), nr);
529 if (!file->pv_ops)
530 return -1;
531
532 for (idx = 0; idx < nr; idx++)
533 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
534
535 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
536 add_pv_ops(file, pv_ops);
537
538 return 0;
539}
540
6b5dd716
ST		 541static struct instruction *find_last_insn(struct objtool_file *file,
542 struct section *sec)
543{
544 struct instruction *insn = NULL;
545 unsigned int offset;
fe255fe6 546 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
6b5dd716 547
fe255fe6 548 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
6b5dd716
ST		 549 insn = find_insn(file, sec, offset);
550
551 return insn;
552}
553
dcc914f4 554/*
649ea4d5 555 * Mark "ud2" instructions and manually annotated dead ends.
dcc914f4
JP		 556 */
557static int add_dead_ends(struct objtool_file *file)
558{
559 struct section *sec;
f1974222 560 struct reloc *reloc;
dcc914f4 561 struct instruction *insn;
dcc914f4 562
649ea4d5
JP		 563 /*
564 * Check for manually annotated dead ends.
565 */
dcc914f4
JP		 566 sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
567 if (!sec)
649ea4d5 568 goto reachable;
dcc914f4 569
f1974222
MH		 570 list_for_each_entry(reloc, &sec->reloc_list, list) {
571 if (reloc->sym->type != STT_SECTION) {
dcc914f4
JP		 572 WARN("unexpected relocation symbol type in %s", sec->name);
573 return -1;
574 }
f1974222 575 insn = find_insn(file, reloc->sym->sec, reloc->addend);
dcc914f4
JP		 576 if (insn)
577 insn = list_prev_entry(insn, list);
fe255fe6 578 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 579 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 580 if (!insn) {
22682a07 581 WARN("can't find unreachable insn at %s+0x%" PRIx64,
f1974222 582 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 583 return -1;
584 }
585 } else {
22682a07 586 WARN("can't find unreachable insn at %s+0x%" PRIx64,
f1974222 587 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 588 return -1;
589 }
590
591 insn->dead_end = true;
592 }
593
649ea4d5
JP		 594reachable:
595 /*
596 * These manually annotated reachable checks are needed for GCC 4.4,
597 * where the Linux unreachable() macro isn't supported. In that case
598 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
599 * not a dead end.
600 */
601 sec = find_section_by_name(file->elf, ".rela.discard.reachable");
602 if (!sec)
603 return 0;
604
f1974222
MH		 605 list_for_each_entry(reloc, &sec->reloc_list, list) {
606 if (reloc->sym->type != STT_SECTION) {
649ea4d5
JP		 607 WARN("unexpected relocation symbol type in %s", sec->name);
608 return -1;
609 }
f1974222 610 insn = find_insn(file, reloc->sym->sec, reloc->addend);
649ea4d5
JP		 611 if (insn)
612 insn = list_prev_entry(insn, list);
fe255fe6 613 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 614 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 615 if (!insn) {
22682a07 616 WARN("can't find reachable insn at %s+0x%" PRIx64,
f1974222 617 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 618 return -1;
619 }
620 } else {
22682a07 621 WARN("can't find reachable insn at %s+0x%" PRIx64,
f1974222 622 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 623 return -1;
624 }
625
626 insn->dead_end = false;
627 }
628
dcc914f4
JP		 629 return 0;
630}
631
1e7e4788
JP		 632static int create_static_call_sections(struct objtool_file *file)
633{
ef47cc01 634 struct section *sec;
1e7e4788
JP		 635 struct static_call_site *site;
636 struct instruction *insn;
637 struct symbol *key_sym;
638 char *key_name, *tmp;
639 int idx;
640
641 sec = find_section_by_name(file->elf, ".static_call_sites");
642 if (sec) {
643 INIT_LIST_HEAD(&file->static_call_list);
644 WARN("file already has .static_call_sites section, skipping");
645 return 0;
646 }
647
648 if (list_empty(&file->static_call_list))
649 return 0;
650
651 idx = 0;
43d5430a 652 list_for_each_entry(insn, &file->static_call_list, call_node)
1e7e4788
JP		 653 idx++;
654
655 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
656 sizeof(struct static_call_site), idx);
657 if (!sec)
658 return -1;
659
1e7e4788 660 idx = 0;
43d5430a 661 list_for_each_entry(insn, &file->static_call_list, call_node) {
1e7e4788
JP		 662
663 site = (struct static_call_site *)sec->data->d_buf + idx;
664 memset(site, 0, sizeof(struct static_call_site));
665
666 /* populate reloc for 'addr' */
ef47cc01
PZ		 667 if (elf_add_reloc_to_insn(file->elf, sec,
668 idx * sizeof(struct static_call_site),
669 R_X86_64_PC32,
670 insn->sec, insn->offset))
44f6a7c0 671 return -1;
1e7e4788
JP		 672
673 /* find key symbol */
674 key_name = strdup(insn->call_dest->name);
675 if (!key_name) {
676 perror("strdup");
677 return -1;
678 }
679 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
680 STATIC_CALL_TRAMP_PREFIX_LEN)) {
681 WARN("static_call: trampoline name malformed: %s", key_name);
3da73f10 682 free(key_name);
1e7e4788
JP		 683 return -1;
684 }
685 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
686 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
687
688 key_sym = find_symbol_by_name(file->elf, tmp);
689 if (!key_sym) {
2daf7fab 690 if (!opts.module) {
73f44fe1 691 WARN("static_call: can't find static_call_key symbol: %s", tmp);
3da73f10 692 free(key_name);
73f44fe1
JP		 693 return -1;
694 }
695
696 /*
697 * For modules(), the key might not be exported, which
698 * means the module can make static calls but isn't
699 * allowed to change them.
700 *
701 * In that case we temporarily set the key to be the
702 * trampoline address. This is fixed up in
703 * static_call_add_module().
704 */
705 key_sym = insn->call_dest;
1e7e4788
JP		 706 }
707 free(key_name);
708
709 /* populate reloc for 'key' */
ef47cc01
PZ		 710 if (elf_add_reloc(file->elf, sec,
711 idx * sizeof(struct static_call_site) + 4,
712 R_X86_64_PC32, key_sym,
713 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
1e7e4788 714 return -1;
1e7e4788
JP		 715
716 idx++;
717 }
718
1e7e4788
JP		 719 return 0;
720}
721
134ab5bd
PZ		 722static int create_retpoline_sites_sections(struct objtool_file *file)
723{
724 struct instruction *insn;
725 struct section *sec;
726 int idx;
727
728 sec = find_section_by_name(file->elf, ".retpoline_sites");
729 if (sec) {
730 WARN("file already has .retpoline_sites, skipping");
731 return 0;
732 }
733
734 idx = 0;
735 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
736 idx++;
737
738 if (!idx)
739 return 0;
740
741 sec = elf_create_section(file->elf, ".retpoline_sites", 0,
742 sizeof(int), idx);
743 if (!sec) {
744 WARN("elf_create_section: .retpoline_sites");
745 return -1;
746 }
747
748 idx = 0;
749 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
750
751 int *site = (int *)sec->data->d_buf + idx;
752 *site = 0;
753
754 if (elf_add_reloc_to_insn(file->elf, sec,
755 idx * sizeof(int),
756 R_X86_64_PC32,
757 insn->sec, insn->offset)) {
758 WARN("elf_add_reloc_to_insn: .retpoline_sites");
759 return -1;
760 }
761
762 idx++;
763 }
764
765 return 0;
766}
767
d9e9d230
PZ		 768static int create_return_sites_sections(struct objtool_file *file)
769{
770 struct instruction *insn;
771 struct section *sec;
772 int idx;
773
774 sec = find_section_by_name(file->elf, ".return_sites");
775 if (sec) {
776 WARN("file already has .return_sites, skipping");
777 return 0;
778 }
779
780 idx = 0;
781 list_for_each_entry(insn, &file->return_thunk_list, call_node)
782 idx++;
783
784 if (!idx)
785 return 0;
786
787 sec = elf_create_section(file->elf, ".return_sites", 0,
788 sizeof(int), idx);
789 if (!sec) {
790 WARN("elf_create_section: .return_sites");
791 return -1;
792 }
793
794 idx = 0;
795 list_for_each_entry(insn, &file->return_thunk_list, call_node) {
796
797 int *site = (int *)sec->data->d_buf + idx;
798 *site = 0;
799
800 if (elf_add_reloc_to_insn(file->elf, sec,
801 idx * sizeof(int),
802 R_X86_64_PC32,
803 insn->sec, insn->offset)) {
804 WARN("elf_add_reloc_to_insn: .return_sites");
805 return -1;
806 }
807
808 idx++;
809 }
810
811 return 0;
812}
813
89bc853e
PZ		 814static int create_ibt_endbr_seal_sections(struct objtool_file *file)
815{
816 struct instruction *insn;
817 struct section *sec;
818 int idx;
819
820 sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
821 if (sec) {
822 WARN("file already has .ibt_endbr_seal, skipping");
823 return 0;
824 }
825
826 idx = 0;
827 list_for_each_entry(insn, &file->endbr_list, call_node)
828 idx++;
829
2daf7fab 830 if (opts.stats) {
89bc853e
PZ		 831 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
832 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
833 printf("ibt: superfluous ENDBR: %d\n", idx);
834 }
835
836 if (!idx)
837 return 0;
838
839 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
840 sizeof(int), idx);
841 if (!sec) {
842 WARN("elf_create_section: .ibt_endbr_seal");
843 return -1;
844 }
845
846 idx = 0;
847 list_for_each_entry(insn, &file->endbr_list, call_node) {
848
849 int *site = (int *)sec->data->d_buf + idx;
850 *site = 0;
851
852 if (elf_add_reloc_to_insn(file->elf, sec,
853 idx * sizeof(int),
854 R_X86_64_PC32,
855 insn->sec, insn->offset)) {
856 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
857 return -1;
858 }
859
860 idx++;
861 }
862
863 return 0;
864}
865
9a479f76
PZ		 866static int create_cfi_sections(struct objtool_file *file)
867{
868 struct section *sec, *s;
869 struct symbol *sym;
870 unsigned int *loc;
871 int idx;
872
873 sec = find_section_by_name(file->elf, ".cfi_sites");
874 if (sec) {
875 INIT_LIST_HEAD(&file->call_list);
876 WARN("file already has .cfi_sites section, skipping");
877 return 0;
878 }
879
880 idx = 0;
881 for_each_sec(file, s) {
882 if (!s->text)
883 continue;
884
885 list_for_each_entry(sym, &s->symbol_list, list) {
886 if (sym->type != STT_FUNC)
887 continue;
888
889 if (strncmp(sym->name, "__cfi_", 6))
890 continue;
891
892 idx++;
893 }
894 }
895
896 sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx);
897 if (!sec)
898 return -1;
899
900 idx = 0;
901 for_each_sec(file, s) {
902 if (!s->text)
903 continue;
904
905 list_for_each_entry(sym, &s->symbol_list, list) {
906 if (sym->type != STT_FUNC)
907 continue;
908
909 if (strncmp(sym->name, "__cfi_", 6))
910 continue;
911
912 loc = (unsigned int *)sec->data->d_buf + idx;
913 memset(loc, 0, sizeof(unsigned int));
914
915 if (elf_add_reloc_to_insn(file->elf, sec,
916 idx * sizeof(unsigned int),
917 R_X86_64_PC32,
918 s, sym->offset))
919 return -1;
920
921 idx++;
922 }
923 }
924
925 return 0;
926}
927
99d00215
PZ		 928static int create_mcount_loc_sections(struct objtool_file *file)
929{
86ea7f36 930 int addrsize = elf_class_addrsize(file->elf);
99d00215 931 struct instruction *insn;
86ea7f36 932 struct section *sec;
99d00215
PZ		 933 int idx;
934
935 sec = find_section_by_name(file->elf, "__mcount_loc");
936 if (sec) {
937 INIT_LIST_HEAD(&file->mcount_loc_list);
938 WARN("file already has __mcount_loc section, skipping");
939 return 0;
940 }
941
942 if (list_empty(&file->mcount_loc_list))
943 return 0;
944
945 idx = 0;
c509331b 946 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
99d00215
PZ		 947 idx++;
948
86ea7f36 949 sec = elf_create_section(file->elf, "__mcount_loc", 0, addrsize, idx);
99d00215
PZ		 950 if (!sec)
951 return -1;
952
86ea7f36
CL		 953 sec->sh.sh_addralign = addrsize;
954
99d00215 955 idx = 0;
c509331b 956 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
86ea7f36 957 void *loc;
99d00215 958
86ea7f36
CL		 959 loc = sec->data->d_buf + idx;
960 memset(loc, 0, addrsize);
99d00215 961
86ea7f36 962 if (elf_add_reloc_to_insn(file->elf, sec, idx,
c1449735 963 addrsize == sizeof(u64) ? R_ABS64 : R_ABS32,
ef47cc01 964 insn->sec, insn->offset))
99d00215 965 return -1;
99d00215 966
86ea7f36 967 idx += addrsize;
99d00215
PZ		 968 }
969
99d00215
PZ		 970 return 0;
971}
972
00abd384
PZ		 973static int create_direct_call_sections(struct objtool_file *file)
974{
975 struct instruction *insn;
976 struct section *sec;
977 unsigned int *loc;
978 int idx;
979
980 sec = find_section_by_name(file->elf, ".call_sites");
981 if (sec) {
982 INIT_LIST_HEAD(&file->call_list);
983 WARN("file already has .call_sites section, skipping");
984 return 0;
985 }
986
987 if (list_empty(&file->call_list))
988 return 0;
989
990 idx = 0;
991 list_for_each_entry(insn, &file->call_list, call_node)
992 idx++;
993
994 sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx);
995 if (!sec)
996 return -1;
997
998 idx = 0;
999 list_for_each_entry(insn, &file->call_list, call_node) {
1000
1001 loc = (unsigned int *)sec->data->d_buf + idx;
1002 memset(loc, 0, sizeof(unsigned int));
1003
1004 if (elf_add_reloc_to_insn(file->elf, sec,
1005 idx * sizeof(unsigned int),
1006 R_X86_64_PC32,
1007 insn->sec, insn->offset))
1008 return -1;
1009
1010 idx++;
1011 }
1012
1013 return 0;
1014}
1015
dcc914f4
JP		 1016/*
1017 * Warnings shouldn't be reported for ignored functions.
1018 */
1019static void add_ignores(struct objtool_file *file)
1020{
1021 struct instruction *insn;
1022 struct section *sec;
1023 struct symbol *func;
f1974222 1024 struct reloc *reloc;
dcc914f4 1025
aaf5c623
PZ		 1026 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1027 if (!sec)
1028 return;
dcc914f4 1029
f1974222
MH		 1030 list_for_each_entry(reloc, &sec->reloc_list, list) {
1031 switch (reloc->sym->type) {
aaf5c623 1032 case STT_FUNC:
f1974222 1033 func = reloc->sym;
aaf5c623
PZ		 1034 break;
1035
1036 case STT_SECTION:
f1974222 1037 func = find_func_by_offset(reloc->sym->sec, reloc->addend);
7acfe531 1038 if (!func)
dcc914f4 1039 continue;
aaf5c623 1040 break;
dcc914f4 1041
aaf5c623 1042 default:
f1974222 1043 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
aaf5c623 1044 continue;
dcc914f4 1045 }
aaf5c623 1046
f0f70adb 1047 func_for_each_insn(file, func, insn)
aaf5c623 1048 insn->ignore = true;
dcc914f4
JP		 1049 }
1050}
1051
ea24213d
PZ		 1052/*
1053 * This is a whitelist of functions that is allowed to be called with AC set.
1054 * The list is meant to be minimal and only contains compiler instrumentation
1055 * ABI and a few functions used to implement *_{to,from}_user() functions.
1056 *
1057 * These functions must not directly change AC, but may PUSHF/POPF.
1058 */
1059static const char *uaccess_safe_builtin[] = {
1060 /* KASAN */
1061 "kasan_report",
f00748bf 1062 "kasan_check_range",
ea24213d
PZ		 1063 /* KASAN out-of-line */
1064 "__asan_loadN_noabort",
1065 "__asan_load1_noabort",
1066 "__asan_load2_noabort",
1067 "__asan_load4_noabort",
1068 "__asan_load8_noabort",
1069 "__asan_load16_noabort",
1070 "__asan_storeN_noabort",
1071 "__asan_store1_noabort",
1072 "__asan_store2_noabort",
1073 "__asan_store4_noabort",
1074 "__asan_store8_noabort",
1075 "__asan_store16_noabort",
b0b8e56b
JH		 1076 "__kasan_check_read",
1077 "__kasan_check_write",
ea24213d
PZ		 1078 /* KASAN in-line */
1079 "__asan_report_load_n_noabort",
1080 "__asan_report_load1_noabort",
1081 "__asan_report_load2_noabort",
1082 "__asan_report_load4_noabort",
1083 "__asan_report_load8_noabort",
1084 "__asan_report_load16_noabort",
1085 "__asan_report_store_n_noabort",
1086 "__asan_report_store1_noabort",
1087 "__asan_report_store2_noabort",
1088 "__asan_report_store4_noabort",
1089 "__asan_report_store8_noabort",
1090 "__asan_report_store16_noabort",
5f5c9712 1091 /* KCSAN */
9967683c 1092 "__kcsan_check_access",
0525bd82
ME		 1093 "__kcsan_mb",
1094 "__kcsan_wmb",
1095 "__kcsan_rmb",
1096 "__kcsan_release",
5f5c9712
ME		 1097 "kcsan_found_watchpoint",
1098 "kcsan_setup_watchpoint",
9967683c 1099 "kcsan_check_scoped_accesses",
50a19ad4
ME		 1100 "kcsan_disable_current",
1101 "kcsan_enable_current_nowarn",
5f5c9712
ME		 1102 /* KCSAN/TSAN */
1103 "__tsan_func_entry",
1104 "__tsan_func_exit",
1105 "__tsan_read_range",
1106 "__tsan_write_range",
1107 "__tsan_read1",
1108 "__tsan_read2",
1109 "__tsan_read4",
1110 "__tsan_read8",
1111 "__tsan_read16",
1112 "__tsan_write1",
1113 "__tsan_write2",
1114 "__tsan_write4",
1115 "__tsan_write8",
1116 "__tsan_write16",
a81b3759
ME		 1117 "__tsan_read_write1",
1118 "__tsan_read_write2",
1119 "__tsan_read_write4",
1120 "__tsan_read_write8",
1121 "__tsan_read_write16",
63646fcb
ME		 1122 "__tsan_volatile_read1",
1123 "__tsan_volatile_read2",
1124 "__tsan_volatile_read4",
1125 "__tsan_volatile_read8",
1126 "__tsan_volatile_read16",
1127 "__tsan_volatile_write1",
1128 "__tsan_volatile_write2",
1129 "__tsan_volatile_write4",
1130 "__tsan_volatile_write8",
1131 "__tsan_volatile_write16",
883957b1
ME		 1132 "__tsan_atomic8_load",
1133 "__tsan_atomic16_load",
1134 "__tsan_atomic32_load",
1135 "__tsan_atomic64_load",
1136 "__tsan_atomic8_store",
1137 "__tsan_atomic16_store",
1138 "__tsan_atomic32_store",
1139 "__tsan_atomic64_store",
1140 "__tsan_atomic8_exchange",
1141 "__tsan_atomic16_exchange",
1142 "__tsan_atomic32_exchange",
1143 "__tsan_atomic64_exchange",
1144 "__tsan_atomic8_fetch_add",
1145 "__tsan_atomic16_fetch_add",
1146 "__tsan_atomic32_fetch_add",
1147 "__tsan_atomic64_fetch_add",
1148 "__tsan_atomic8_fetch_sub",
1149 "__tsan_atomic16_fetch_sub",
1150 "__tsan_atomic32_fetch_sub",
1151 "__tsan_atomic64_fetch_sub",
1152 "__tsan_atomic8_fetch_and",
1153 "__tsan_atomic16_fetch_and",
1154 "__tsan_atomic32_fetch_and",
1155 "__tsan_atomic64_fetch_and",
1156 "__tsan_atomic8_fetch_or",
1157 "__tsan_atomic16_fetch_or",
1158 "__tsan_atomic32_fetch_or",
1159 "__tsan_atomic64_fetch_or",
1160 "__tsan_atomic8_fetch_xor",
1161 "__tsan_atomic16_fetch_xor",
1162 "__tsan_atomic32_fetch_xor",
1163 "__tsan_atomic64_fetch_xor",
1164 "__tsan_atomic8_fetch_nand",
1165 "__tsan_atomic16_fetch_nand",
1166 "__tsan_atomic32_fetch_nand",
1167 "__tsan_atomic64_fetch_nand",
1168 "__tsan_atomic8_compare_exchange_strong",
1169 "__tsan_atomic16_compare_exchange_strong",
1170 "__tsan_atomic32_compare_exchange_strong",
1171 "__tsan_atomic64_compare_exchange_strong",
1172 "__tsan_atomic8_compare_exchange_weak",
1173 "__tsan_atomic16_compare_exchange_weak",
1174 "__tsan_atomic32_compare_exchange_weak",
1175 "__tsan_atomic64_compare_exchange_weak",
1176 "__tsan_atomic8_compare_exchange_val",
1177 "__tsan_atomic16_compare_exchange_val",
1178 "__tsan_atomic32_compare_exchange_val",
1179 "__tsan_atomic64_compare_exchange_val",
1180 "__tsan_atomic_thread_fence",
1181 "__tsan_atomic_signal_fence",
ea24213d
PZ		 1182 /* KCOV */
1183 "write_comp_data",
ae033f08 1184 "check_kcov_mode",
ea24213d
PZ		 1185 "__sanitizer_cov_trace_pc",
1186 "__sanitizer_cov_trace_const_cmp1",
1187 "__sanitizer_cov_trace_const_cmp2",
1188 "__sanitizer_cov_trace_const_cmp4",
1189 "__sanitizer_cov_trace_const_cmp8",
1190 "__sanitizer_cov_trace_cmp1",
1191 "__sanitizer_cov_trace_cmp2",
1192 "__sanitizer_cov_trace_cmp4",
1193 "__sanitizer_cov_trace_cmp8",
36b1c700 1194 "__sanitizer_cov_trace_switch",
40b22c9d
AP		 1195 /* KMSAN */
1196 "kmsan_copy_to_user",
1197 "kmsan_report",
1198 "kmsan_unpoison_entry_regs",
1199 "kmsan_unpoison_memory",
1200 "__msan_chain_origin",
1201 "__msan_get_context_state",
1202 "__msan_instrument_asm_store",
1203 "__msan_metadata_ptr_for_load_1",
1204 "__msan_metadata_ptr_for_load_2",
1205 "__msan_metadata_ptr_for_load_4",
1206 "__msan_metadata_ptr_for_load_8",
1207 "__msan_metadata_ptr_for_load_n",
1208 "__msan_metadata_ptr_for_store_1",
1209 "__msan_metadata_ptr_for_store_2",
1210 "__msan_metadata_ptr_for_store_4",
1211 "__msan_metadata_ptr_for_store_8",
1212 "__msan_metadata_ptr_for_store_n",
1213 "__msan_poison_alloca",
1214 "__msan_warning",
ea24213d
PZ		 1215 /* UBSAN */
1216 "ubsan_type_mismatch_common",
1217 "__ubsan_handle_type_mismatch",
1218 "__ubsan_handle_type_mismatch_v1",
9a50dcaf 1219 "__ubsan_handle_shift_out_of_bounds",
ea24213d
PZ		 1220 /* misc */
1221 "csum_partial_copy_generic",
ec6347bb
DW		 1222 "copy_mc_fragile",
1223 "copy_mc_fragile_handle_tail",
5da8e4a6 1224 "copy_mc_enhanced_fast_string",
ea24213d 1225 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
0db7058e
BP		 1226 "clear_user_erms",
1227 "clear_user_rep_good",
1228 "clear_user_original",
ea24213d
PZ		 1229 NULL
1230};
1231
1232static void add_uaccess_safe(struct objtool_file *file)
1233{
1234 struct symbol *func;
1235 const char **name;
1236
2daf7fab 1237 if (!opts.uaccess)
ea24213d
PZ		 1238 return;
1239
1240 for (name = uaccess_safe_builtin; *name; name++) {
1241 func = find_symbol_by_name(file->elf, *name);
1242 if (!func)
1243 continue;
1244
e10cd8fe 1245 func->uaccess_safe = true;
dcc914f4
JP		 1246 }
1247}
1248
258c7605
JP		 1249/*
1250 * FIXME: For now, just ignore any alternatives which add retpolines. This is
1251 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1252 * But it at least allows objtool to understand the control flow *around* the
1253 * retpoline.
1254 */
ff05ab23 1255static int add_ignore_alternatives(struct objtool_file *file)
258c7605
JP		 1256{
1257 struct section *sec;
f1974222 1258 struct reloc *reloc;
258c7605
JP		 1259 struct instruction *insn;
1260
ff05ab23 1261 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
258c7605
JP		 1262 if (!sec)
1263 return 0;
1264
f1974222
MH		 1265 list_for_each_entry(reloc, &sec->reloc_list, list) {
1266 if (reloc->sym->type != STT_SECTION) {
258c7605
JP		 1267 WARN("unexpected relocation symbol type in %s", sec->name);
1268 return -1;
1269 }
1270
f1974222 1271 insn = find_insn(file, reloc->sym->sec, reloc->addend);
258c7605 1272 if (!insn) {
ff05ab23 1273 WARN("bad .discard.ignore_alts entry");
258c7605
JP		 1274 return -1;
1275 }
1276
1277 insn->ignore_alts = true;
1278 }
1279
1280 return 0;
1281}
1282
530b4ddd
PZ		 1283__weak bool arch_is_retpoline(struct symbol *sym)
1284{
1285 return false;
1286}
1287
d9e9d230
PZ		 1288__weak bool arch_is_rethunk(struct symbol *sym)
1289{
1290 return false;
1291}
1292
7bd2a600
PZ		 1293#define NEGATIVE_RELOC ((void *)-1L)
1294
1295static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1296{
1297 if (insn->reloc == NEGATIVE_RELOC)
1298 return NULL;
1299
1300 if (!insn->reloc) {
db2b0c5d
PZ		 1301 if (!file)
1302 return NULL;
1303
7bd2a600
PZ		 1304 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1305 insn->offset, insn->len);
1306 if (!insn->reloc) {
1307 insn->reloc = NEGATIVE_RELOC;
1308 return NULL;
1309 }
1310 }
1311
1312 return insn->reloc;
1313}
1314
f56dae88
PZ		 1315static void remove_insn_ops(struct instruction *insn)
1316{
1317 struct stack_op *op, *tmp;
1318
1319 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1320 list_del(&op->list);
1321 free(op);
1322 }
1323}
1324
dd003ede
PZ		 1325static void annotate_call_site(struct objtool_file *file,
1326 struct instruction *insn, bool sibling)
f56dae88
PZ		 1327{
1328 struct reloc *reloc = insn_reloc(file, insn);
dd003ede 1329 struct symbol *sym = insn->call_dest;
f56dae88 1330
dd003ede
PZ		 1331 if (!sym)
1332 sym = reloc->sym;
1333
1334 /*
1335 * Alternative replacement code is just template code which is
1336 * sometimes copied to the original instruction. For now, don't
1337 * annotate it. (In the future we might consider annotating the
1338 * original instruction if/when it ever makes sense to do so.)
1339 */
1340 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
f56dae88
PZ		 1341 return;
1342
dd003ede
PZ		 1343 if (sym->static_call_tramp) {
1344 list_add_tail(&insn->call_node, &file->static_call_list);
1345 return;
f56dae88
PZ		 1346 }
1347
134ab5bd
PZ		 1348 if (sym->retpoline_thunk) {
1349 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1350 return;
1351 }
1352
f56dae88 1353 /*
05098119
ME		 1354 * Many compilers cannot disable KCOV or sanitizer calls with a function
1355 * attribute so they need a little help, NOP out any such calls from
1356 * noinstr text.
f56dae88 1357 */
22102f45 1358 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
f56dae88
PZ		 1359 if (reloc) {
1360 reloc->type = R_NONE;
1361 elf_write_reloc(file->elf, reloc);
1362 }
1363
1364 elf_write_insn(file->elf, insn->sec,
1365 insn->offset, insn->len,
1366 sibling ? arch_ret_insn(insn->len)
1367 : arch_nop_insn(insn->len));
1368
1369 insn->type = sibling ? INSN_RETURN : INSN_NOP;
7a53f408
PZ		 1370
1371 if (sibling) {
1372 /*
1373 * We've replaced the tail-call JMP insn by two new
1374 * insn: RET; INT3, except we only have a single struct
1375 * insn here. Mark it retpoline_safe to avoid the SLS
1376 * warning, instead of adding another insn.
1377 */
1378 insn->retpoline_safe = true;
1379 }
1380
dd003ede 1381 return;
f56dae88
PZ		 1382 }
1383
2daf7fab 1384 if (opts.mcount && sym->fentry) {
f56dae88
PZ		 1385 if (sibling)
1386 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
280981d6
SV		 1387 if (opts.mnop) {
1388 if (reloc) {
1389 reloc->type = R_NONE;
1390 elf_write_reloc(file->elf, reloc);
1391 }
f56dae88 1392
280981d6
SV		 1393 elf_write_insn(file->elf, insn->sec,
1394 insn->offset, insn->len,
1395 arch_nop_insn(insn->len));
f56dae88 1396
280981d6
SV		 1397 insn->type = INSN_NOP;
1398 }
f56dae88 1399
c509331b 1400 list_add_tail(&insn->call_node, &file->mcount_loc_list);
dd003ede 1401 return;
f56dae88 1402 }
0e5b613b 1403
00abd384
PZ		 1404 if (insn->type == INSN_CALL && !insn->sec->init)
1405 list_add_tail(&insn->call_node, &file->call_list);
1406
0e5b613b
PZ		 1407 if (!sibling && dead_end_function(file, sym))
1408 insn->dead_end = true;
dd003ede
PZ		 1409}
1410
1411static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1412 struct symbol *dest, bool sibling)
1413{
1414 insn->call_dest = dest;
1415 if (!dest)
1416 return;
f56dae88
PZ		 1417
1418 /*
1419 * Whatever stack impact regular CALLs have, should be undone
1420 * by the RETURN of the called function.
1421 *
1422 * Annotated intra-function calls retain the stack_ops but
1423 * are converted to JUMP, see read_intra_function_calls().
1424 */
1425 remove_insn_ops(insn);
dd003ede
PZ		 1426
1427 annotate_call_site(file, insn, sibling);
f56dae88
PZ		 1428}
1429
134ab5bd
PZ		 1430static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1431{
1432 /*
1433 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1434 * so convert them accordingly.
1435 */
1436 switch (insn->type) {
1437 case INSN_CALL:
1438 insn->type = INSN_CALL_DYNAMIC;
1439 break;
1440 case INSN_JUMP_UNCONDITIONAL:
1441 insn->type = INSN_JUMP_DYNAMIC;
1442 break;
1443 case INSN_JUMP_CONDITIONAL:
1444 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1445 break;
1446 default:
1447 return;
1448 }
1449
1450 insn->retpoline_safe = true;
1451
1452 /*
1453 * Whatever stack impact regular CALLs have, should be undone
1454 * by the RETURN of the called function.
1455 *
1456 * Annotated intra-function calls retain the stack_ops but
1457 * are converted to JUMP, see read_intra_function_calls().
1458 */
1459 remove_insn_ops(insn);
1460
1461 annotate_call_site(file, insn, false);
1462}
08f87a93 1463
a149180f 1464static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
d9e9d230
PZ		 1465{
1466 /*
1467 * Return thunk tail calls are really just returns in disguise,
1468 * so convert them accordingly.
1469 */
1470 insn->type = INSN_RETURN;
1471 insn->retpoline_safe = true;
1472
a149180f
PZ		 1473 if (add)
1474 list_add_tail(&insn->call_node, &file->return_thunk_list);
d9e9d230
PZ		 1475}
1476
5a9c361a
PZ		 1477static bool is_first_func_insn(struct objtool_file *file,
1478 struct instruction *insn, struct symbol *sym)
08f87a93 1479{
5a9c361a 1480 if (insn->offset == sym->offset)
d139bca4
PZ		 1481 return true;
1482
5a9c361a 1483 /* Allow direct CALL/JMP past ENDBR */
2daf7fab 1484 if (opts.ibt) {
d139bca4
PZ		 1485 struct instruction *prev = prev_insn_same_sym(file, insn);
1486
1487 if (prev && prev->type == INSN_ENDBR &&
5a9c361a 1488 insn->offset == sym->offset + prev->len)
d139bca4
PZ		 1489 return true;
1490 }
1491
1492 return false;
08f87a93
PZ		 1493}
1494
5a9c361a
PZ		 1495/*
1496 * A sibling call is a tail-call to another symbol -- to differentiate from a
1497 * recursive tail-call which is to the same symbol.
1498 */
1499static bool jump_is_sibling_call(struct objtool_file *file,
1500 struct instruction *from, struct instruction *to)
1501{
1502 struct symbol *fs = from->sym;
1503 struct symbol *ts = to->sym;
1504
1505 /* Not a sibling call if from/to a symbol hole */
1506 if (!fs || !ts)
1507 return false;
1508
1509 /* Not a sibling call if not targeting the start of a symbol. */
1510 if (!is_first_func_insn(file, to, ts))
1511 return false;
1512
1513 /* Disallow sibling calls into STT_NOTYPE */
1514 if (ts->type == STT_NOTYPE)
1515 return false;
1516
1517 /* Must not be self to be a sibling */
1518 return fs->pfunc != ts->pfunc;
1519}
1520
dcc914f4
JP		 1521/*
1522 * Find the destination instructions for all jumps.
1523 */
1524static int add_jump_destinations(struct objtool_file *file)
1525{
26ff6041 1526 struct instruction *insn, *jump_dest;
f1974222 1527 struct reloc *reloc;
dcc914f4
JP		 1528 struct section *dest_sec;
1529 unsigned long dest_off;
1530
1531 for_each_insn(file, insn) {
34c861e8
JP		 1532 if (insn->jump_dest) {
1533 /*
1534 * handle_group_alt() may have previously set
1535 * 'jump_dest' for some alternatives.
1536 */
1537 continue;
1538 }
a2296140 1539 if (!is_static_jump(insn))
dcc914f4
JP		 1540 continue;
1541
7bd2a600 1542 reloc = insn_reloc(file, insn);
f1974222 1543 if (!reloc) {
dcc914f4 1544 dest_sec = insn->sec;
bfb08f22 1545 dest_off = arch_jump_destination(insn);
f1974222
MH		 1546 } else if (reloc->sym->type == STT_SECTION) {
1547 dest_sec = reloc->sym->sec;
1548 dest_off = arch_dest_reloc_offset(reloc->addend);
1739c66e 1549 } else if (reloc->sym->retpoline_thunk) {
134ab5bd 1550 add_retpoline_call(file, insn);
39b73533 1551 continue;
d9e9d230 1552 } else if (reloc->sym->return_thunk) {
a149180f 1553 add_return_call(file, insn, true);
d9e9d230 1554 continue;
dbcdbdfd 1555 } else if (insn_func(insn)) {
26ff6041
JP		 1556 /*
1557 * External sibling call or internal sibling call with
1558 * STT_FUNC reloc.
1559 */
f56dae88 1560 add_call_dest(file, insn, reloc->sym, true);
dcc914f4 1561 continue;
ecf11ba4
JP		 1562 } else if (reloc->sym->sec->idx) {
1563 dest_sec = reloc->sym->sec;
1564 dest_off = reloc->sym->sym.st_value +
1565 arch_dest_reloc_offset(reloc->addend);
1566 } else {
1567 /* non-func asm code jumping to another file */
1568 continue;
dcc914f4
JP		 1569 }
1570
26ff6041
JP		 1571 jump_dest = find_insn(file, dest_sec, dest_off);
1572 if (!jump_dest) {
a149180f
PZ		 1573 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1574
1575 /*
1576 * This is a special case for zen_untrain_ret().
1577 * It jumps to __x86_return_thunk(), but objtool
1578 * can't find the thunk's starting RET
1579 * instruction, because the RET is also in the
1580 * middle of another instruction. Objtool only
1581 * knows about the outer instruction.
1582 */
1583 if (sym && sym->return_thunk) {
1584 add_return_call(file, insn, false);
1585 continue;
1586 }
1587
dcc914f4
JP		 1588 WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1589 insn->sec, insn->offset, dest_sec->name,
1590 dest_off);
1591 return -1;
1592 }
cd77849a
JP		 1593
1594 /*
54262aa2 1595 * Cross-function jump.
cd77849a 1596 */
dbcdbdfd
PZ		 1597 if (insn_func(insn) && insn_func(jump_dest) &&
1598 insn_func(insn) != insn_func(jump_dest)) {
54262aa2
PZ		 1599
1600 /*
1601 * For GCC 8+, create parent/child links for any cold
1602 * subfunctions. This is _mostly_ redundant with a
1603 * similar initialization in read_symbols().
1604 *
1605 * If a function has aliases, we want the *first* such
1606 * function in the symbol table to be the subfunction's
1607 * parent. In that case we overwrite the
1608 * initialization done in read_symbols().
1609 *
1610 * However this code can't completely replace the
1611 * read_symbols() code because this doesn't detect the
1612 * case where the parent function's only reference to a
e7c2bc37 1613 * subfunction is through a jump table.
54262aa2 1614 */
dbcdbdfd
PZ		 1615 if (!strstr(insn_func(insn)->name, ".cold") &&
1616 strstr(insn_func(jump_dest)->name, ".cold")) {
1617 insn_func(insn)->cfunc = insn_func(jump_dest);
1618 insn_func(jump_dest)->pfunc = insn_func(insn);
54262aa2 1619 }
cd77849a 1620 }
26ff6041 1621
5a9c361a
PZ		 1622 if (jump_is_sibling_call(file, insn, jump_dest)) {
1623 /*
1624 * Internal sibling call without reloc or with
1625 * STT_SECTION reloc.
1626 */
1627 add_call_dest(file, insn, insn_func(jump_dest), true);
1628 continue;
1629 }
1630
26ff6041 1631 insn->jump_dest = jump_dest;
dcc914f4
JP		 1632 }
1633
1634 return 0;
1635}
1636
2b232a22
JT		 1637static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1638{
1639 struct symbol *call_dest;
1640
1641 call_dest = find_func_by_offset(sec, offset);
1642 if (!call_dest)
1643 call_dest = find_symbol_by_offset(sec, offset);
1644
1645 return call_dest;
1646}
1647
dcc914f4
JP		 1648/*
1649 * Find the destination instructions for all calls.
1650 */
1651static int add_call_destinations(struct objtool_file *file)
1652{
1653 struct instruction *insn;
1654 unsigned long dest_off;
f56dae88 1655 struct symbol *dest;
f1974222 1656 struct reloc *reloc;
dcc914f4
JP		 1657
1658 for_each_insn(file, insn) {
1659 if (insn->type != INSN_CALL)
1660 continue;
1661
7bd2a600 1662 reloc = insn_reloc(file, insn);
f1974222 1663 if (!reloc) {
bfb08f22 1664 dest_off = arch_jump_destination(insn);
f56dae88
PZ		 1665 dest = find_call_destination(insn->sec, dest_off);
1666
1667 add_call_dest(file, insn, dest, false);
a845c7cf 1668
7acfe531
JP		 1669 if (insn->ignore)
1670 continue;
1671
1672 if (!insn->call_dest) {
8aa8eb2a 1673 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
dcc914f4
JP		 1674 return -1;
1675 }
a845c7cf 1676
dbcdbdfd 1677 if (insn_func(insn) && insn->call_dest->type != STT_FUNC) {
7acfe531
JP		 1678 WARN_FUNC("unsupported call to non-function",
1679 insn->sec, insn->offset);
1680 return -1;
1681 }
1682
f1974222
MH		 1683 } else if (reloc->sym->type == STT_SECTION) {
1684 dest_off = arch_dest_reloc_offset(reloc->addend);
f56dae88
PZ		 1685 dest = find_call_destination(reloc->sym->sec, dest_off);
1686 if (!dest) {
bfb08f22 1687 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
dcc914f4 1688 insn->sec, insn->offset,
f1974222 1689 reloc->sym->sec->name,
bfb08f22 1690 dest_off);
dcc914f4
JP		 1691 return -1;
1692 }
bcb1b6ff 1693
f56dae88
PZ		 1694 add_call_dest(file, insn, dest, false);
1695
1739c66e 1696 } else if (reloc->sym->retpoline_thunk) {
134ab5bd 1697 add_retpoline_call(file, insn);
bcb1b6ff 1698
dcc914f4 1699 } else
f56dae88 1700 add_call_dest(file, insn, reloc->sym, false);
dcc914f4
JP		 1701 }
1702
1703 return 0;
1704}
1705
1706/*
c9c324dc
JP		 1707 * The .alternatives section requires some extra special care over and above
1708 * other special sections because alternatives are patched in place.
dcc914f4
JP		 1709 */
1710static int handle_group_alt(struct objtool_file *file,
1711 struct special_alt *special_alt,
1712 struct instruction *orig_insn,
1713 struct instruction **new_insn)
1714{
c9c324dc 1715 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
b23cc71c 1716 struct alt_group *orig_alt_group, *new_alt_group;
dcc914f4
JP		 1717 unsigned long dest_off;
1718
b23cc71c
JP		 1719
1720 orig_alt_group = malloc(sizeof(*orig_alt_group));
1721 if (!orig_alt_group) {
1722 WARN("malloc failed");
1723 return -1;
1724 }
c9c324dc
JP		 1725 orig_alt_group->cfi = calloc(special_alt->orig_len,
1726 sizeof(struct cfi_state *));
1727 if (!orig_alt_group->cfi) {
1728 WARN("calloc failed");
1729 return -1;
1730 }
1731
dcc914f4
JP		 1732 last_orig_insn = NULL;
1733 insn = orig_insn;
1734 sec_for_each_insn_from(file, insn) {
1735 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1736 break;
1737
b23cc71c 1738 insn->alt_group = orig_alt_group;
dcc914f4
JP		 1739 last_orig_insn = insn;
1740 }
b23cc71c
JP		 1741 orig_alt_group->orig_group = NULL;
1742 orig_alt_group->first_insn = orig_insn;
1743 orig_alt_group->last_insn = last_orig_insn;
dcc914f4 1744
17bc3391 1745
c9c324dc
JP		 1746 new_alt_group = malloc(sizeof(*new_alt_group));
1747 if (!new_alt_group) {
1748 WARN("malloc failed");
1749 return -1;
dcc914f4 1750 }
dcc914f4 1751
c9c324dc
JP		 1752 if (special_alt->new_len < special_alt->orig_len) {
1753 /*
1754 * Insert a fake nop at the end to make the replacement
1755 * alt_group the same size as the original. This is needed to
1756 * allow propagate_alt_cfi() to do its magic. When the last
1757 * instruction affects the stack, the instruction after it (the
1758 * nop) will propagate the new state to the shared CFI array.
1759 */
1760 nop = malloc(sizeof(*nop));
1761 if (!nop) {
1762 WARN("malloc failed");
17bc3391
JP		 1763 return -1;
1764 }
c9c324dc
JP		 1765 memset(nop, 0, sizeof(*nop));
1766 INIT_LIST_HEAD(&nop->alts);
1767 INIT_LIST_HEAD(&nop->stack_ops);
c9c324dc
JP		 1768
1769 nop->sec = special_alt->new_sec;
1770 nop->offset = special_alt->new_off + special_alt->new_len;
1771 nop->len = special_alt->orig_len - special_alt->new_len;
1772 nop->type = INSN_NOP;
dbcdbdfd 1773 nop->sym = orig_insn->sym;
c9c324dc
JP		 1774 nop->alt_group = new_alt_group;
1775 nop->ignore = orig_insn->ignore_alts;
dcc914f4 1776 }
17bc3391 1777
c9c324dc
JP		 1778 if (!special_alt->new_len) {
1779 *new_insn = nop;
1780 goto end;
dcc914f4
JP		 1781 }
1782
dcc914f4
JP		 1783 insn = *new_insn;
1784 sec_for_each_insn_from(file, insn) {
45245f51
JT		 1785 struct reloc *alt_reloc;
1786
dcc914f4
JP		 1787 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1788 break;
1789
1790 last_new_insn = insn;
1791
a845c7cf 1792 insn->ignore = orig_insn->ignore_alts;
dbcdbdfd 1793 insn->sym = orig_insn->sym;
b23cc71c 1794 insn->alt_group = new_alt_group;
a845c7cf 1795
dc419723
JP		 1796 /*
1797 * Since alternative replacement code is copy/pasted by the
1798 * kernel after applying relocations, generally such code can't
1799 * have relative-address relocation references to outside the
1800 * .altinstr_replacement section, unless the arch's
1801 * alternatives code can adjust the relative offsets
1802 * accordingly.
dc419723 1803 */
7bd2a600 1804 alt_reloc = insn_reloc(file, insn);
61c6065e 1805 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
45245f51 1806 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
dc419723
JP		 1807
1808 WARN_FUNC("unsupported relocation in alternatives section",
1809 insn->sec, insn->offset);
1810 return -1;
1811 }
1812
a2296140 1813 if (!is_static_jump(insn))
dcc914f4
JP		 1814 continue;
1815
1816 if (!insn->immediate)
1817 continue;
1818
bfb08f22 1819 dest_off = arch_jump_destination(insn);
34c861e8 1820 if (dest_off == special_alt->new_off + special_alt->new_len) {
c9c324dc 1821 insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
34c861e8
JP		 1822 if (!insn->jump_dest) {
1823 WARN_FUNC("can't find alternative jump destination",
1824 insn->sec, insn->offset);
1825 return -1;
1826 }
dcc914f4
JP		 1827 }
1828 }
1829
1830 if (!last_new_insn) {
1831 WARN_FUNC("can't find last new alternative instruction",
1832 special_alt->new_sec, special_alt->new_off);
1833 return -1;
1834 }
1835
c9c324dc
JP		 1836 if (nop)
1837 list_add(&nop->list, &last_new_insn->list);
1838end:
b23cc71c
JP		 1839 new_alt_group->orig_group = orig_alt_group;
1840 new_alt_group->first_insn = *new_insn;
c9c324dc
JP		 1841 new_alt_group->last_insn = nop ? : last_new_insn;
1842 new_alt_group->cfi = orig_alt_group->cfi;
dcc914f4
JP		 1843 return 0;
1844}
1845
1846/*
1847 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1848 * If the original instruction is a jump, make the alt entry an effective nop
1849 * by just skipping the original instruction.
1850 */
1851static int handle_jump_alt(struct objtool_file *file,
1852 struct special_alt *special_alt,
1853 struct instruction *orig_insn,
1854 struct instruction **new_insn)
1855{
48001d26
PZ		 1856 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1857 orig_insn->type != INSN_NOP) {
e2d9494b 1858
dcc914f4
JP		 1859 WARN_FUNC("unsupported instruction at jump label",
1860 orig_insn->sec, orig_insn->offset);
1861 return -1;
1862 }
1863
4ab7674f 1864 if (opts.hack_jump_label && special_alt->key_addend & 2) {
6d37b83c
PZ		 1865 struct reloc *reloc = insn_reloc(file, orig_insn);
1866
1867 if (reloc) {
1868 reloc->type = R_NONE;
1869 elf_write_reloc(file->elf, reloc);
1870 }
1871 elf_write_insn(file->elf, orig_insn->sec,
1872 orig_insn->offset, orig_insn->len,
1873 arch_nop_insn(orig_insn->len));
1874 orig_insn->type = INSN_NOP;
48001d26
PZ		 1875 }
1876
1877 if (orig_insn->type == INSN_NOP) {
1878 if (orig_insn->len == 2)
1879 file->jl_nop_short++;
1880 else
1881 file->jl_nop_long++;
1882
1883 return 0;
6d37b83c
PZ		 1884 }
1885
e2d9494b
PZ		 1886 if (orig_insn->len == 2)
1887 file->jl_short++;
1888 else
1889 file->jl_long++;
1890
dcc914f4
JP		 1891 *new_insn = list_next_entry(orig_insn, list);
1892 return 0;
1893}
1894
1895/*
1896 * Read all the special sections which have alternate instructions which can be
1897 * patched in or redirected to at runtime. Each instruction having alternate
1898 * instruction(s) has them added to its insn->alts list, which will be
1899 * traversed in validate_branch().
1900 */
1901static int add_special_section_alts(struct objtool_file *file)
1902{
1903 struct list_head special_alts;
1904 struct instruction *orig_insn, *new_insn;
1905 struct special_alt *special_alt, *tmp;
1906 struct alternative *alt;
1907 int ret;
1908
1909 ret = special_get_alts(file->elf, &special_alts);
1910 if (ret)
1911 return ret;
1912
1913 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
dcc914f4
JP		 1914
1915 orig_insn = find_insn(file, special_alt->orig_sec,
1916 special_alt->orig_off);
1917 if (!orig_insn) {
1918 WARN_FUNC("special: can't find orig instruction",
1919 special_alt->orig_sec, special_alt->orig_off);
1920 ret = -1;
1921 goto out;
1922 }
1923
1924 new_insn = NULL;
1925 if (!special_alt->group || special_alt->new_len) {
1926 new_insn = find_insn(file, special_alt->new_sec,
1927 special_alt->new_off);
1928 if (!new_insn) {
1929 WARN_FUNC("special: can't find new instruction",
1930 special_alt->new_sec,
1931 special_alt->new_off);
1932 ret = -1;
1933 goto out;
1934 }
1935 }
1936
1937 if (special_alt->group) {
7170cf47
JT		 1938 if (!special_alt->orig_len) {
1939 WARN_FUNC("empty alternative entry",
1940 orig_insn->sec, orig_insn->offset);
1941 continue;
1942 }
1943
dcc914f4
JP		 1944 ret = handle_group_alt(file, special_alt, orig_insn,
1945 &new_insn);
1946 if (ret)
1947 goto out;
1948 } else if (special_alt->jump_or_nop) {
1949 ret = handle_jump_alt(file, special_alt, orig_insn,
1950 &new_insn);
1951 if (ret)
1952 goto out;
1953 }
1954
258c7605
JP		 1955 alt = malloc(sizeof(*alt));
1956 if (!alt) {
1957 WARN("malloc failed");
1958 ret = -1;
1959 goto out;
1960 }
1961
dcc914f4 1962 alt->insn = new_insn;
764eef4b 1963 alt->skip_orig = special_alt->skip_orig;
ea24213d 1964 orig_insn->ignore_alts |= special_alt->skip_alt;
dcc914f4
JP		 1965 list_add_tail(&alt->list, &orig_insn->alts);
1966
1967 list_del(&special_alt->list);
1968 free(special_alt);
1969 }
1970
2daf7fab 1971 if (opts.stats) {
e2d9494b
PZ		 1972 printf("jl\\\tNOP\tJMP\n");
1973 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1974 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1975 }
1976
dcc914f4
JP		 1977out:
1978 return ret;
1979}
1980
e7c2bc37 1981static int add_jump_table(struct objtool_file *file, struct instruction *insn,
f1974222 1982 struct reloc *table)
dcc914f4 1983{
f1974222 1984 struct reloc *reloc = table;
e7c2bc37 1985 struct instruction *dest_insn;
dcc914f4 1986 struct alternative *alt;
dbcdbdfd 1987 struct symbol *pfunc = insn_func(insn)->pfunc;
fd35c88b 1988 unsigned int prev_offset = 0;
dcc914f4 1989
e7c2bc37 1990 /*
f1974222 1991 * Each @reloc is a switch table relocation which points to the target
e7c2bc37
JP		 1992 * instruction.
1993 */
f1974222 1994 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
bd98c813
JH		 1995
1996 /* Check for the end of the table: */
f1974222 1997 if (reloc != table && reloc->jump_table_start)
dcc914f4
JP		 1998 break;
1999
e7c2bc37 2000 /* Make sure the table entries are consecutive: */
f1974222 2001 if (prev_offset && reloc->offset != prev_offset + 8)
fd35c88b
JP		 2002 break;
2003
2004 /* Detect function pointers from contiguous objects: */
f1974222
MH		 2005 if (reloc->sym->sec == pfunc->sec &&
2006 reloc->addend == pfunc->offset)
fd35c88b
JP		 2007 break;
2008
f1974222 2009 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
e7c2bc37 2010 if (!dest_insn)
dcc914f4
JP		 2011 break;
2012
e7c2bc37 2013 /* Make sure the destination is in the same function: */
dbcdbdfd 2014 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
13810435 2015 break;
dcc914f4
JP		 2016
2017 alt = malloc(sizeof(*alt));
2018 if (!alt) {
2019 WARN("malloc failed");
2020 return -1;
2021 }
2022
e7c2bc37 2023 alt->insn = dest_insn;
dcc914f4 2024 list_add_tail(&alt->list, &insn->alts);
f1974222 2025 prev_offset = reloc->offset;
fd35c88b
JP		 2026 }
2027
2028 if (!prev_offset) {
2029 WARN_FUNC("can't find switch jump table",
2030 insn->sec, insn->offset);
2031 return -1;
dcc914f4
JP		 2032 }
2033
2034 return 0;
2035}
2036
2037/*
d871f7b5
RG		 2038 * find_jump_table() - Given a dynamic jump, find the switch jump table
2039 * associated with it.
dcc914f4 2040 */
f1974222 2041static struct reloc *find_jump_table(struct objtool_file *file,
dcc914f4
JP		 2042 struct symbol *func,
2043 struct instruction *insn)
2044{
d871f7b5 2045 struct reloc *table_reloc;
113d4bc9 2046 struct instruction *dest_insn, *orig_insn = insn;
dcc914f4 2047
99ce7962
PZ		 2048 /*
2049 * Backward search using the @first_jump_src links, these help avoid
2050 * much of the 'in between' code. Which avoids us getting confused by
2051 * it.
2052 */
7dec80cc 2053 for (;
dbcdbdfd 2054 insn && insn_func(insn) && insn_func(insn)->pfunc == func;
1119d265 2055 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
99ce7962 2056
7dec80cc 2057 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
dcc914f4
JP		 2058 break;
2059
2060 /* allow small jumps within the range */
2061 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2062 insn->jump_dest &&
2063 (insn->jump_dest->offset <= insn->offset ||
2064 insn->jump_dest->offset > orig_insn->offset))
2065 break;
2066
d871f7b5 2067 table_reloc = arch_find_switch_table(file, insn);
f1974222 2068 if (!table_reloc)
e7c2bc37 2069 continue;
f1974222 2070 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
dbcdbdfd 2071 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
113d4bc9 2072 continue;
7dec80cc 2073
f1974222 2074 return table_reloc;
dcc914f4
JP		 2075 }
2076
2077 return NULL;
2078}
2079
bd98c813
JH		 2080/*
2081 * First pass: Mark the head of each jump table so that in the next pass,
2082 * we know when a given jump table ends and the next one starts.
2083 */
2084static void mark_func_jump_tables(struct objtool_file *file,
2085 struct symbol *func)
dcc914f4 2086{
bd98c813 2087 struct instruction *insn, *last = NULL;
f1974222 2088 struct reloc *reloc;
dcc914f4 2089
f0f70adb 2090 func_for_each_insn(file, func, insn) {
99ce7962
PZ		 2091 if (!last)
2092 last = insn;
2093
2094 /*
2095 * Store back-pointers for unconditional forward jumps such
e7c2bc37 2096 * that find_jump_table() can back-track using those and
99ce7962
PZ		 2097 * avoid some potentially confusing code.
2098 */
2099 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2100 insn->offset > last->offset &&
2101 insn->jump_dest->offset > insn->offset &&
2102 !insn->jump_dest->first_jump_src) {
2103
2104 insn->jump_dest->first_jump_src = insn;
2105 last = insn->jump_dest;
2106 }
2107
dcc914f4
JP		 2108 if (insn->type != INSN_JUMP_DYNAMIC)
2109 continue;
2110
f1974222
MH		 2111 reloc = find_jump_table(file, func, insn);
2112 if (reloc) {
2113 reloc->jump_table_start = true;
2114 insn->jump_table = reloc;
dcc914f4 2115 }
dcc914f4 2116 }
bd98c813
JH		 2117}
2118
2119static int add_func_jump_tables(struct objtool_file *file,
2120 struct symbol *func)
2121{
2122 struct instruction *insn;
2123 int ret;
2124
f0f70adb 2125 func_for_each_insn(file, func, insn) {
bd98c813
JH		 2126 if (!insn->jump_table)
2127 continue;
dcc914f4 2128
bd98c813 2129 ret = add_jump_table(file, insn, insn->jump_table);
dcc914f4
JP		 2130 if (ret)
2131 return ret;
2132 }
2133
2134 return 0;
2135}
2136
2137/*
2138 * For some switch statements, gcc generates a jump table in the .rodata
2139 * section which contains a list of addresses within the function to jump to.
2140 * This finds these jump tables and adds them to the insn->alts lists.
2141 */
e7c2bc37 2142static int add_jump_table_alts(struct objtool_file *file)
dcc914f4
JP		 2143{
2144 struct section *sec;
2145 struct symbol *func;
2146 int ret;
2147
4a60aa05 2148 if (!file->rodata)
dcc914f4
JP		 2149 return 0;
2150
baa41469 2151 for_each_sec(file, sec) {
dcc914f4
JP		 2152 list_for_each_entry(func, &sec->symbol_list, list) {
2153 if (func->type != STT_FUNC)
2154 continue;
2155
bd98c813 2156 mark_func_jump_tables(file, func);
e7c2bc37 2157 ret = add_func_jump_tables(file, func);
dcc914f4
JP		 2158 if (ret)
2159 return ret;
2160 }
2161 }
2162
2163 return 0;
2164}
2165
b735bd3e
JP		 2166static void set_func_state(struct cfi_state *state)
2167{
2168 state->cfa = initial_func_cfi.cfa;
2169 memcpy(&state->regs, &initial_func_cfi.regs,
2170 CFI_NUM_REGS * sizeof(struct cfi_reg));
2171 state->stack_size = initial_func_cfi.cfa.offset;
2172}
2173
39358a03
JP		 2174static int read_unwind_hints(struct objtool_file *file)
2175{
8b946cc3 2176 struct cfi_state cfi = init_cfi;
f1974222 2177 struct section *sec, *relocsec;
39358a03
JP		 2178 struct unwind_hint *hint;
2179 struct instruction *insn;
8b946cc3 2180 struct reloc *reloc;
39358a03
JP		 2181 int i;
2182
2183 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2184 if (!sec)
2185 return 0;
2186
f1974222
MH		 2187 relocsec = sec->reloc;
2188 if (!relocsec) {
39358a03
JP		 2189 WARN("missing .rela.discard.unwind_hints section");
2190 return -1;
2191 }
2192
fe255fe6 2193 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
39358a03
JP		 2194 WARN("struct unwind_hint size mismatch");
2195 return -1;
2196 }
2197
2198 file->hints = true;
2199
fe255fe6 2200 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
39358a03
JP		 2201 hint = (struct unwind_hint *)sec->data->d_buf + i;
2202
f1974222
MH		 2203 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2204 if (!reloc) {
2205 WARN("can't find reloc for unwind_hints[%d]", i);
39358a03
JP		 2206 return -1;
2207 }
2208
f1974222 2209 insn = find_insn(file, reloc->sym->sec, reloc->addend);
39358a03
JP		 2210 if (!insn) {
2211 WARN("can't find insn for unwind_hints[%d]", i);
2212 return -1;
2213 }
2214
b735bd3e 2215 insn->hint = true;
39358a03 2216
8faea26e
JP		 2217 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2218 insn->hint = false;
2219 insn->save = true;
2220 continue;
2221 }
2222
2223 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2224 insn->restore = true;
2225 continue;
2226 }
2227
a09a6e23 2228 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
08f87a93
PZ		 2229 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2230
a09a6e23
PZ		 2231 if (sym && sym->bind == STB_GLOBAL) {
2232 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2233 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
2234 insn->sec, insn->offset);
2235 }
2236
2237 insn->entry = 1;
08f87a93
PZ		 2238 }
2239 }
2240
a09a6e23
PZ		 2241 if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
2242 hint->type = UNWIND_HINT_TYPE_CALL;
2243 insn->entry = 1;
2244 }
2245
b735bd3e 2246 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
8b946cc3 2247 insn->cfi = &func_cfi;
39358a03
JP		 2248 continue;
2249 }
2250
8b946cc3
PZ		 2251 if (insn->cfi)
2252 cfi = *(insn->cfi);
2253
2254 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
39358a03
JP		 2255 WARN_FUNC("unsupported unwind_hint sp base reg %d",
2256 insn->sec, insn->offset, hint->sp_reg);
2257 return -1;
2258 }
2259
0646c28b 2260 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
8b946cc3
PZ		 2261 cfi.type = hint->type;
2262 cfi.end = hint->end;
2263
2264 insn->cfi = cfi_hash_find_or_add(&cfi);
39358a03
JP		 2265 }
2266
2267 return 0;
2268}
2269
96db4a98
PZ		 2270static int read_noendbr_hints(struct objtool_file *file)
2271{
2272 struct section *sec;
2273 struct instruction *insn;
2274 struct reloc *reloc;
2275
2276 sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2277 if (!sec)
2278 return 0;
2279
2280 list_for_each_entry(reloc, &sec->reloc_list, list) {
2281 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
2282 if (!insn) {
2283 WARN("bad .discard.noendbr entry");
2284 return -1;
2285 }
2286
2287 insn->noendbr = 1;
2288 }
2289
2290 return 0;
2291}
2292
b5bc2231
PZ		 2293static int read_retpoline_hints(struct objtool_file *file)
2294{
63474dc4 2295 struct section *sec;
b5bc2231 2296 struct instruction *insn;
f1974222 2297 struct reloc *reloc;
b5bc2231 2298
63474dc4 2299 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
b5bc2231
PZ		 2300 if (!sec)
2301 return 0;
2302
f1974222
MH		 2303 list_for_each_entry(reloc, &sec->reloc_list, list) {
2304 if (reloc->sym->type != STT_SECTION) {
63474dc4 2305 WARN("unexpected relocation symbol type in %s", sec->name);
b5bc2231
PZ		 2306 return -1;
2307 }
2308
f1974222 2309 insn = find_insn(file, reloc->sym->sec, reloc->addend);
b5bc2231 2310 if (!insn) {
63474dc4 2311 WARN("bad .discard.retpoline_safe entry");
b5bc2231
PZ		 2312 return -1;
2313 }
2314
2315 if (insn->type != INSN_JUMP_DYNAMIC &&
9bb2ec60 2316 insn->type != INSN_CALL_DYNAMIC &&
a09a6e23
PZ		 2317 insn->type != INSN_RETURN &&
2318 insn->type != INSN_NOP) {
2319 WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
b5bc2231
PZ		 2320 insn->sec, insn->offset);
2321 return -1;
2322 }
2323
2324 insn->retpoline_safe = true;
2325 }
2326
2327 return 0;
2328}
2329
c4a33939
PZ		 2330static int read_instr_hints(struct objtool_file *file)
2331{
2332 struct section *sec;
2333 struct instruction *insn;
f1974222 2334 struct reloc *reloc;
c4a33939
PZ		 2335
2336 sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2337 if (!sec)
2338 return 0;
2339
f1974222
MH		 2340 list_for_each_entry(reloc, &sec->reloc_list, list) {
2341 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 2342 WARN("unexpected relocation symbol type in %s", sec->name);
2343 return -1;
2344 }
2345
f1974222 2346 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 2347 if (!insn) {
2348 WARN("bad .discard.instr_end entry");
2349 return -1;
2350 }
2351
2352 insn->instr--;
2353 }
2354
2355 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2356 if (!sec)
2357 return 0;
2358
f1974222
MH		 2359 list_for_each_entry(reloc, &sec->reloc_list, list) {
2360 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 2361 WARN("unexpected relocation symbol type in %s", sec->name);
2362 return -1;
2363 }
2364
f1974222 2365 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 2366 if (!insn) {
2367 WARN("bad .discard.instr_begin entry");
2368 return -1;
2369 }
2370
2371 insn->instr++;
2372 }
2373
2374 return 0;
2375}
2376
8aa8eb2a
AC		 2377static int read_intra_function_calls(struct objtool_file *file)
2378{
2379 struct instruction *insn;
2380 struct section *sec;
f1974222 2381 struct reloc *reloc;
8aa8eb2a
AC		 2382
2383 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2384 if (!sec)
2385 return 0;
2386
f1974222 2387 list_for_each_entry(reloc, &sec->reloc_list, list) {
8aa8eb2a
AC		 2388 unsigned long dest_off;
2389
f1974222 2390 if (reloc->sym->type != STT_SECTION) {
8aa8eb2a
AC		 2391 WARN("unexpected relocation symbol type in %s",
2392 sec->name);
2393 return -1;
2394 }
2395
f1974222 2396 insn = find_insn(file, reloc->sym->sec, reloc->addend);
8aa8eb2a
AC		 2397 if (!insn) {
2398 WARN("bad .discard.intra_function_call entry");
2399 return -1;
2400 }
2401
2402 if (insn->type != INSN_CALL) {
2403 WARN_FUNC("intra_function_call not a direct call",
2404 insn->sec, insn->offset);
2405 return -1;
2406 }
2407
2408 /*
2409 * Treat intra-function CALLs as JMPs, but with a stack_op.
2410 * See add_call_destinations(), which strips stack_ops from
2411 * normal CALLs.
2412 */
2413 insn->type = INSN_JUMP_UNCONDITIONAL;
2414
7b3e3186 2415 dest_off = arch_jump_destination(insn);
8aa8eb2a
AC		 2416 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2417 if (!insn->jump_dest) {
2418 WARN_FUNC("can't find call dest at %s+0x%lx",
2419 insn->sec, insn->offset,
2420 insn->sec->name, dest_off);
2421 return -1;
2422 }
2423 }
2424
2425 return 0;
2426}
2427
05098119
ME		 2428/*
2429 * Return true if name matches an instrumentation function, where calls to that
2430 * function from noinstr code can safely be removed, but compilers won't do so.
2431 */
2432static bool is_profiling_func(const char *name)
2433{
2434 /*
2435 * Many compilers cannot disable KCOV with a function attribute.
2436 */
2437 if (!strncmp(name, "__sanitizer_cov_", 16))
2438 return true;
2439
2440 /*
2441 * Some compilers currently do not remove __tsan_func_entry/exit nor
2442 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2443 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2444 * minimum Clang version is 14.0, this can be removed.
2445 */
2446 if (!strncmp(name, "__tsan_func_", 12) ||
2447 !strcmp(name, "__tsan_atomic_signal_fence"))
2448 return true;
2449
2450 return false;
2451}
2452
1739c66e 2453static int classify_symbols(struct objtool_file *file)
1e7e4788
JP		 2454{
2455 struct section *sec;
2456 struct symbol *func;
2457
2458 for_each_sec(file, sec) {
2459 list_for_each_entry(func, &sec->symbol_list, list) {
1739c66e
PZ		 2460 if (func->bind != STB_GLOBAL)
2461 continue;
2462
2463 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
1e7e4788
JP		 2464 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2465 func->static_call_tramp = true;
1739c66e
PZ		 2466
2467 if (arch_is_retpoline(func))
2468 func->retpoline_thunk = true;
2469
d9e9d230
PZ		 2470 if (arch_is_rethunk(func))
2471 func->return_thunk = true;
2472
4ca993d4 2473 if (arch_ftrace_match(func->name))
1739c66e
PZ		 2474 func->fentry = true;
2475
05098119
ME		 2476 if (is_profiling_func(func->name))
2477 func->profiling_func = true;
1e7e4788
JP		 2478 }
2479 }
2480
2481 return 0;
2482}
2483
4a60aa05
AX		 2484static void mark_rodata(struct objtool_file *file)
2485{
2486 struct section *sec;
2487 bool found = false;
2488
2489 /*
87b512de
JP		 2490 * Search for the following rodata sections, each of which can
2491 * potentially contain jump tables:
2492 *
2493 * - .rodata: can contain GCC switch tables
2494 * - .rodata.<func>: same, if -fdata-sections is being used
2495 * - .rodata..c_jump_table: contains C annotated jump tables
2496 *
2497 * .rodata.str1.* sections are ignored; they don't contain jump tables.
4a60aa05
AX		 2498 */
2499 for_each_sec(file, sec) {
1ee44470
MS		 2500 if (!strncmp(sec->name, ".rodata", 7) &&
2501 !strstr(sec->name, ".str1.")) {
4a60aa05
AX		 2502 sec->rodata = true;
2503 found = true;
2504 }
2505 }
2506
2507 file->rodata = found;
2508}
2509
dcc914f4
JP		 2510static int decode_sections(struct objtool_file *file)
2511{
2512 int ret;
2513
4a60aa05
AX		 2514 mark_rodata(file);
2515
db2b0c5d
PZ		 2516 ret = init_pv_ops(file);
2517 if (ret)
2518 return ret;
2519
dbcdbdfd
PZ		 2520 /*
2521 * Must be before add_{jump_call}_destination.
2522 */
2523 ret = classify_symbols(file);
2524 if (ret)
2525 return ret;
2526
dcc914f4
JP		 2527 ret = decode_instructions(file);
2528 if (ret)
2529 return ret;
2530
dcc914f4 2531 add_ignores(file);
ea24213d 2532 add_uaccess_safe(file);
dcc914f4 2533
ff05ab23 2534 ret = add_ignore_alternatives(file);
258c7605
JP		 2535 if (ret)
2536 return ret;
2537
08f87a93
PZ		 2538 /*
2539 * Must be before read_unwind_hints() since that needs insn->noendbr.
2540 */
96db4a98
PZ		 2541 ret = read_noendbr_hints(file);
2542 if (ret)
2543 return ret;
2544
43d5430a 2545 /*
34c861e8
JP		 2546 * Must be before add_jump_destinations(), which depends on 'func'
2547 * being set for alternatives, to enable proper sibling call detection.
43d5430a 2548 */
de6fbced
SV		 2549 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2550 ret = add_special_section_alts(file);
2551 if (ret)
2552 return ret;
2553 }
dcc914f4 2554
34c861e8 2555 ret = add_jump_destinations(file);
dcc914f4
JP		 2556 if (ret)
2557 return ret;
2558
a958c4fe
PZ		 2559 /*
2560 * Must be before add_call_destination(); it changes INSN_CALL to
2561 * INSN_JUMP.
2562 */
8aa8eb2a
AC		 2563 ret = read_intra_function_calls(file);
2564 if (ret)
2565 return ret;
2566
a845c7cf 2567 ret = add_call_destinations(file);
dcc914f4
JP		 2568 if (ret)
2569 return ret;
2570
0e5b613b
PZ		 2571 /*
2572 * Must be after add_call_destinations() such that it can override
2573 * dead_end_function() marks.
2574 */
2575 ret = add_dead_ends(file);
2576 if (ret)
2577 return ret;
2578
e7c2bc37 2579 ret = add_jump_table_alts(file);
dcc914f4
JP		 2580 if (ret)
2581 return ret;
2582
39358a03
JP		 2583 ret = read_unwind_hints(file);
2584 if (ret)
2585 return ret;
2586
b5bc2231
PZ		 2587 ret = read_retpoline_hints(file);
2588 if (ret)
2589 return ret;
2590
c4a33939
PZ		 2591 ret = read_instr_hints(file);
2592 if (ret)
2593 return ret;
2594
dcc914f4
JP		 2595 return 0;
2596}
2597
2598static bool is_fentry_call(struct instruction *insn)
2599{
1739c66e
PZ		 2600 if (insn->type == INSN_CALL &&
2601 insn->call_dest &&
2602 insn->call_dest->fentry)
dcc914f4
JP		 2603 return true;
2604
2605 return false;
2606}
2607
e25eea89 2608static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
dcc914f4 2609{
e7c0219b 2610 struct cfi_state *cfi = &state->cfi;
baa41469
JP		 2611 int i;
2612
e7c0219b 2613 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
e25eea89
PZ		 2614 return true;
2615
b735bd3e 2616 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
baa41469
JP		 2617 return true;
2618
b735bd3e 2619 if (cfi->stack_size != initial_func_cfi.cfa.offset)
e25eea89
PZ		 2620 return true;
2621
2622 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 2623 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2624 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
baa41469 2625 return true;
e25eea89 2626 }
baa41469
JP		 2627
2628 return false;
2629}
2630
fb084fde
JT		 2631static bool check_reg_frame_pos(const struct cfi_reg *reg,
2632 int expected_offset)
2633{
2634 return reg->base == CFI_CFA &&
2635 reg->offset == expected_offset;
2636}
2637
baa41469
JP		 2638static bool has_valid_stack_frame(struct insn_state *state)
2639{
e7c0219b
PZ		 2640 struct cfi_state *cfi = &state->cfi;
2641
fb084fde
JT		 2642 if (cfi->cfa.base == CFI_BP &&
2643 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2644 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
baa41469
JP		 2645 return true;
2646
e7c0219b 2647 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
baa41469
JP		 2648 return true;
2649
2650 return false;
dcc914f4
JP		 2651}
2652
e7c0219b
PZ		 2653static int update_cfi_state_regs(struct instruction *insn,
2654 struct cfi_state *cfi,
65ea47dc 2655 struct stack_op *op)
627fce14 2656{
e7c0219b 2657 struct cfi_reg *cfa = &cfi->cfa;
627fce14 2658
d8dd25a4 2659 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
627fce14
JP		 2660 return 0;
2661
2662 /* push */
ea24213d 2663 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
627fce14
JP		 2664 cfa->offset += 8;
2665
2666 /* pop */
ea24213d 2667 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
627fce14
JP		 2668 cfa->offset -= 8;
2669
2670 /* add immediate to sp */
2671 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2672 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2673 cfa->offset -= op->src.offset;
2674
2675 return 0;
2676}
2677
e7c0219b 2678static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
dcc914f4 2679{
bf4d1a83 2680 if (arch_callee_saved_reg(reg) &&
e7c0219b
PZ		 2681 cfi->regs[reg].base == CFI_UNDEFINED) {
2682 cfi->regs[reg].base = base;
2683 cfi->regs[reg].offset = offset;
baa41469 2684 }
dcc914f4
JP		 2685}
2686
e7c0219b 2687static void restore_reg(struct cfi_state *cfi, unsigned char reg)
dcc914f4 2688{
e7c0219b
PZ		 2689 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2690 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
baa41469
JP		 2691}
2692
2693/*
2694 * A note about DRAP stack alignment:
2695 *
2696 * GCC has the concept of a DRAP register, which is used to help keep track of
2697 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2698 * register. The typical DRAP pattern is:
2699 *
2700 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2701 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2702 * 41 ff 72 f8 pushq -0x8(%r10)
2703 * 55 push %rbp
2704 * 48 89 e5 mov %rsp,%rbp
2705 * (more pushes)
2706 * 41 52 push %r10
2707 * ...
2708 * 41 5a pop %r10
2709 * (more pops)
2710 * 5d pop %rbp
2711 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2712 * c3 retq
2713 *
2714 * There are some variations in the epilogues, like:
2715 *
2716 * 5b pop %rbx
2717 * 41 5a pop %r10
2718 * 41 5c pop %r12
2719 * 41 5d pop %r13
2720 * 41 5e pop %r14
2721 * c9 leaveq
2722 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2723 * c3 retq
2724 *
2725 * and:
2726 *
2727 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2728 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2729 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2730 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2731 * c9 leaveq
2732 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2733 * c3 retq
2734 *
2735 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2736 * restored beforehand:
2737 *
2738 * 41 55 push %r13
2739 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2740 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2741 * ...
2742 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2743 * 41 5d pop %r13
2744 * c3 retq
2745 */
d54dba41
PZ		 2746static int update_cfi_state(struct instruction *insn,
2747 struct instruction *next_insn,
2748 struct cfi_state *cfi, struct stack_op *op)
baa41469 2749{
e7c0219b
PZ		 2750 struct cfi_reg *cfa = &cfi->cfa;
2751 struct cfi_reg *regs = cfi->regs;
baa41469
JP		 2752
2753 /* stack operations don't make sense with an undefined CFA */
2754 if (cfa->base == CFI_UNDEFINED) {
dbcdbdfd 2755 if (insn_func(insn)) {
baa41469
JP		 2756 WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2757 return -1;
2758 }
2759 return 0;
2760 }
2761
ee819aed
JT		 2762 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2763 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
e7c0219b 2764 return update_cfi_state_regs(insn, cfi, op);
627fce14 2765
baa41469
JP		 2766 switch (op->dest.type) {
2767
2768 case OP_DEST_REG:
2769 switch (op->src.type) {
2770
2771 case OP_SRC_REG:
0d0970ee
JP		 2772 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2773 cfa->base == CFI_SP &&
fb084fde 2774 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
0d0970ee
JP		 2775
2776 /* mov %rsp, %rbp */
2777 cfa->base = op->dest.reg;
e7c0219b 2778 cfi->bp_scratch = false;
0d0970ee 2779 }
dd88a0a0 2780
0d0970ee 2781 else if (op->src.reg == CFI_SP &&
e7c0219b 2782 op->dest.reg == CFI_BP && cfi->drap) {
dd88a0a0 2783
0d0970ee
JP		 2784 /* drap: mov %rsp, %rbp */
2785 regs[CFI_BP].base = CFI_BP;
e7c0219b
PZ		 2786 regs[CFI_BP].offset = -cfi->stack_size;
2787 cfi->bp_scratch = false;
0d0970ee 2788 }
dd88a0a0 2789
0d0970ee
JP		 2790 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2791
2792 /*
2793 * mov %rsp, %reg
2794 *
2795 * This is needed for the rare case where GCC
2796 * does:
2797 *
2798 * mov %rsp, %rax
2799 * ...
2800 * mov %rax, %rsp
2801 */
e7c0219b
PZ		 2802 cfi->vals[op->dest.reg].base = CFI_CFA;
2803 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
dd88a0a0
JP		 2804 }
2805
3c1f0583 2806 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
ffc7e74f 2807 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
3c1f0583
JP		 2808
2809 /*
2810 * mov %rbp, %rsp
2811 *
2812 * Restore the original stack pointer (Clang).
2813 */
e7c0219b 2814 cfi->stack_size = -cfi->regs[CFI_BP].offset;
3c1f0583
JP		 2815 }
2816
dd88a0a0
JP		 2817 else if (op->dest.reg == cfa->base) {
2818
2819 /* mov %reg, %rsp */
2820 if (cfa->base == CFI_SP &&
e7c0219b 2821 cfi->vals[op->src.reg].base == CFI_CFA) {
dd88a0a0
JP		 2822
2823 /*
2824 * This is needed for the rare case
2825 * where GCC does something dumb like:
2826 *
2827 * lea 0x8(%rsp), %rcx
2828 * ...
2829 * mov %rcx, %rsp
2830 */
e7c0219b
PZ		 2831 cfa->offset = -cfi->vals[op->src.reg].offset;
2832 cfi->stack_size = cfa->offset;
dd88a0a0 2833
aafeb14e
PZ		 2834 } else if (cfa->base == CFI_SP &&
2835 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2836 cfi->vals[op->src.reg].offset == cfa->offset) {
2837
2838 /*
2839 * Stack swizzle:
2840 *
2841 * 1: mov %rsp, (%[tos])
2842 * 2: mov %[tos], %rsp
2843 * ...
2844 * 3: pop %rsp
2845 *
2846 * Where:
2847 *
2848 * 1 - places a pointer to the previous
2849 * stack at the Top-of-Stack of the
2850 * new stack.
2851 *
2852 * 2 - switches to the new stack.
2853 *
2854 * 3 - pops the Top-of-Stack to restore
2855 * the original stack.
2856 *
2857 * Note: we set base to SP_INDIRECT
2858 * here and preserve offset. Therefore
2859 * when the unwinder reaches ToS it
2860 * will dereference SP and then add the
2861 * offset to find the next frame, IOW:
2862 * (%rsp) + offset.
2863 */
2864 cfa->base = CFI_SP_INDIRECT;
2865
dd88a0a0
JP		 2866 } else {
2867 cfa->base = CFI_UNDEFINED;
2868 cfa->offset = 0;
2869 }
baa41469
JP		 2870 }
2871
724c8a23
PZ		 2872 else if (op->dest.reg == CFI_SP &&
2873 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2874 cfi->vals[op->src.reg].offset == cfa->offset) {
2875
2876 /*
2877 * The same stack swizzle case 2) as above. But
2878 * because we can't change cfa->base, case 3)
2879 * will become a regular POP. Pretend we're a
2880 * PUSH so things don't go unbalanced.
2881 */
2882 cfi->stack_size += 8;
2883 }
2884
2885
baa41469
JP		 2886 break;
2887
2888 case OP_SRC_ADD:
2889 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2890
2891 /* add imm, %rsp */
e7c0219b 2892 cfi->stack_size -= op->src.offset;
baa41469
JP		 2893 if (cfa->base == CFI_SP)
2894 cfa->offset -= op->src.offset;
2895 break;
2896 }
2897
2898 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2899
2900 /* lea disp(%rbp), %rsp */
e7c0219b 2901 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
baa41469
JP		 2902 break;
2903 }
2904
468af56a
JT		 2905 if (!cfi->drap && op->src.reg == CFI_SP &&
2906 op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2907 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2908
2909 /* lea disp(%rsp), %rbp */
2910 cfa->base = CFI_BP;
2911 cfa->offset -= op->src.offset;
2912 cfi->bp_scratch = false;
2913 break;
2914 }
2915
dd88a0a0 2916 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
baa41469
JP		 2917
2918 /* drap: lea disp(%rsp), %drap */
e7c0219b 2919 cfi->drap_reg = op->dest.reg;
dd88a0a0
JP		 2920
2921 /*
2922 * lea disp(%rsp), %reg
2923 *
2924 * This is needed for the rare case where GCC
2925 * does something dumb like:
2926 *
2927 * lea 0x8(%rsp), %rcx
2928 * ...
2929 * mov %rcx, %rsp
2930 */
e7c0219b
PZ		 2931 cfi->vals[op->dest.reg].base = CFI_CFA;
2932 cfi->vals[op->dest.reg].offset = \
2933 -cfi->stack_size + op->src.offset;
dd88a0a0 2934
baa41469
JP		 2935 break;
2936 }
2937
e7c0219b
PZ		 2938 if (cfi->drap && op->dest.reg == CFI_SP &&
2939 op->src.reg == cfi->drap_reg) {
baa41469
JP		 2940
2941 /* drap: lea disp(%drap), %rsp */
2942 cfa->base = CFI_SP;
e7c0219b
PZ		 2943 cfa->offset = cfi->stack_size = -op->src.offset;
2944 cfi->drap_reg = CFI_UNDEFINED;
2945 cfi->drap = false;
baa41469
JP		 2946 break;
2947 }
2948
d54dba41 2949 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
baa41469
JP		 2950 WARN_FUNC("unsupported stack register modification",
2951 insn->sec, insn->offset);
2952 return -1;
2953 }
2954
2955 break;
2956
2957 case OP_SRC_AND:
2958 if (op->dest.reg != CFI_SP ||
e7c0219b
PZ		 2959 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2960 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
baa41469
JP		 2961 WARN_FUNC("unsupported stack pointer realignment",
2962 insn->sec, insn->offset);
2963 return -1;
2964 }
2965
e7c0219b 2966 if (cfi->drap_reg != CFI_UNDEFINED) {
baa41469 2967 /* drap: and imm, %rsp */
e7c0219b
PZ		 2968 cfa->base = cfi->drap_reg;
2969 cfa->offset = cfi->stack_size = 0;
2970 cfi->drap = true;
baa41469
JP		 2971 }
2972
2973 /*
2974 * Older versions of GCC (4.8ish) realign the stack
2975 * without DRAP, with a frame pointer.
2976 */
2977
2978 break;
2979
2980 case OP_SRC_POP:
ea24213d 2981 case OP_SRC_POPF:
aafeb14e
PZ		 2982 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2983
2984 /* pop %rsp; # restore from a stack swizzle */
2985 cfa->base = CFI_SP;
2986 break;
2987 }
2988
e7c0219b 2989 if (!cfi->drap && op->dest.reg == cfa->base) {
baa41469
JP		 2990
2991 /* pop %rbp */
2992 cfa->base = CFI_SP;
2993 }
2994
e7c0219b
PZ		 2995 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2996 op->dest.reg == cfi->drap_reg &&
2997 cfi->drap_offset == -cfi->stack_size) {
baa41469 2998
bf4d1a83 2999 /* drap: pop %drap */
e7c0219b 3000 cfa->base = cfi->drap_reg;
bf4d1a83 3001 cfa->offset = 0;
e7c0219b 3002 cfi->drap_offset = -1;
baa41469 3003
ffc7e74f 3004 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
baa41469 3005
bf4d1a83 3006 /* pop %reg */
e7c0219b 3007 restore_reg(cfi, op->dest.reg);
baa41469
JP		 3008 }
3009
e7c0219b 3010 cfi->stack_size -= 8;
baa41469
JP		 3011 if (cfa->base == CFI_SP)
3012 cfa->offset -= 8;
3013
3014 break;
3015
3016 case OP_SRC_REG_INDIRECT:
201ef5a9
JT		 3017 if (!cfi->drap && op->dest.reg == cfa->base &&
3018 op->dest.reg == CFI_BP) {
3019
3020 /* mov disp(%rsp), %rbp */
3021 cfa->base = CFI_SP;
3022 cfa->offset = cfi->stack_size;
3023 }
3024
e7c0219b
PZ		 3025 if (cfi->drap && op->src.reg == CFI_BP &&
3026 op->src.offset == cfi->drap_offset) {
bf4d1a83
JP		 3027
3028 /* drap: mov disp(%rbp), %drap */
e7c0219b 3029 cfa->base = cfi->drap_reg;
bf4d1a83 3030 cfa->offset = 0;
e7c0219b 3031 cfi->drap_offset = -1;
bf4d1a83
JP		 3032 }
3033
e7c0219b 3034 if (cfi->drap && op->src.reg == CFI_BP &&
baa41469
JP		 3035 op->src.offset == regs[op->dest.reg].offset) {
3036
3037 /* drap: mov disp(%rbp), %reg */
e7c0219b 3038 restore_reg(cfi, op->dest.reg);
baa41469
JP		 3039
3040 } else if (op->src.reg == cfa->base &&
3041 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3042
3043 /* mov disp(%rbp), %reg */
3044 /* mov disp(%rsp), %reg */
e7c0219b 3045 restore_reg(cfi, op->dest.reg);
201ef5a9
JT		 3046
3047 } else if (op->src.reg == CFI_SP &&
3048 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3049
3050 /* mov disp(%rsp), %reg */
3051 restore_reg(cfi, op->dest.reg);
baa41469
JP		 3052 }
3053
3054 break;
3055
3056 default:
3057 WARN_FUNC("unknown stack-related instruction",
3058 insn->sec, insn->offset);
3059 return -1;
3060 }
3061
3062 break;
3063
3064 case OP_DEST_PUSH:
ea24213d 3065 case OP_DEST_PUSHF:
e7c0219b 3066 cfi->stack_size += 8;
baa41469
JP		 3067 if (cfa->base == CFI_SP)
3068 cfa->offset += 8;
3069
3070 if (op->src.type != OP_SRC_REG)
3071 break;
3072
e7c0219b
PZ		 3073 if (cfi->drap) {
3074 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 3075
3076 /* drap: push %drap */
3077 cfa->base = CFI_BP_INDIRECT;
e7c0219b 3078 cfa->offset = -cfi->stack_size;
baa41469 3079
bf4d1a83 3080 /* save drap so we know when to restore it */
e7c0219b 3081 cfi->drap_offset = -cfi->stack_size;
baa41469 3082
e7c0219b 3083 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
baa41469
JP		 3084
3085 /* drap: push %rbp */
e7c0219b 3086 cfi->stack_size = 0;
baa41469 3087
f4f80398 3088 } else {
baa41469
JP		 3089
3090 /* drap: push %reg */
e7c0219b 3091 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
baa41469
JP		 3092 }
3093
3094 } else {
3095
3096 /* push %reg */
e7c0219b 3097 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
baa41469
JP		 3098 }
3099
3100 /* detect when asm code uses rbp as a scratch register */
dbcdbdfd 3101 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
baa41469 3102 cfa->base != CFI_BP)
e7c0219b 3103 cfi->bp_scratch = true;
baa41469
JP		 3104 break;
3105
3106 case OP_DEST_REG_INDIRECT:
3107
e7c0219b
PZ		 3108 if (cfi->drap) {
3109 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 3110
3111 /* drap: mov %drap, disp(%rbp) */
3112 cfa->base = CFI_BP_INDIRECT;
3113 cfa->offset = op->dest.offset;
3114
bf4d1a83 3115 /* save drap offset so we know when to restore it */
e7c0219b 3116 cfi->drap_offset = op->dest.offset;
f4f80398 3117 } else {
baa41469
JP		 3118
3119 /* drap: mov reg, disp(%rbp) */
e7c0219b 3120 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
baa41469
JP		 3121 }
3122
3123 } else if (op->dest.reg == cfa->base) {
3124
3125 /* mov reg, disp(%rbp) */
3126 /* mov reg, disp(%rsp) */
e7c0219b
PZ		 3127 save_reg(cfi, op->src.reg, CFI_CFA,
3128 op->dest.offset - cfi->cfa.offset);
201ef5a9
JT		 3129
3130 } else if (op->dest.reg == CFI_SP) {
3131
3132 /* mov reg, disp(%rsp) */
3133 save_reg(cfi, op->src.reg, CFI_CFA,
3134 op->dest.offset - cfi->stack_size);
aafeb14e
PZ		 3135
3136 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3137
3138 /* mov %rsp, (%reg); # setup a stack swizzle. */
3139 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3140 cfi->vals[op->dest.reg].offset = cfa->offset;
baa41469
JP		 3141 }
3142
3143 break;
3144
baa41469 3145 case OP_DEST_MEM:
ea24213d 3146 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
baa41469
JP		 3147 WARN_FUNC("unknown stack-related memory operation",
3148 insn->sec, insn->offset);
3149 return -1;
3150 }
3151
3152 /* pop mem */
e7c0219b 3153 cfi->stack_size -= 8;
baa41469
JP		 3154 if (cfa->base == CFI_SP)
3155 cfa->offset -= 8;
3156
3157 break;
3158
3159 default:
3160 WARN_FUNC("unknown stack-related instruction",
3161 insn->sec, insn->offset);
3162 return -1;
3163 }
3164
3165 return 0;
3166}
3167
c9c324dc
JP		 3168/*
3169 * The stack layouts of alternatives instructions can sometimes diverge when
3170 * they have stack modifications. That's fine as long as the potential stack
3171 * layouts don't conflict at any given potential instruction boundary.
3172 *
3173 * Flatten the CFIs of the different alternative code streams (both original
3174 * and replacement) into a single shared CFI array which can be used to detect
3175 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3176 */
3177static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
65ea47dc 3178{
c9c324dc
JP		 3179 struct cfi_state **alt_cfi;
3180 int group_off;
65ea47dc 3181
c9c324dc
JP		 3182 if (!insn->alt_group)
3183 return 0;
65ea47dc 3184
8b946cc3
PZ		 3185 if (!insn->cfi) {
3186 WARN("CFI missing");
3187 return -1;
3188 }
3189
c9c324dc
JP		 3190 alt_cfi = insn->alt_group->cfi;
3191 group_off = insn->offset - insn->alt_group->first_insn->offset;
65ea47dc 3192
c9c324dc 3193 if (!alt_cfi[group_off]) {
8b946cc3 3194 alt_cfi[group_off] = insn->cfi;
c9c324dc 3195 } else {
8b946cc3 3196 if (cficmp(alt_cfi[group_off], insn->cfi)) {
c9c324dc
JP		 3197 WARN_FUNC("stack layout conflict in alternatives",
3198 insn->sec, insn->offset);
ab3852ab
PZ		 3199 return -1;
3200 }
c9c324dc
JP		 3201 }
3202
3203 return 0;
3204}
3205
d54dba41
PZ		 3206static int handle_insn_ops(struct instruction *insn,
3207 struct instruction *next_insn,
3208 struct insn_state *state)
c9c324dc
JP		 3209{
3210 struct stack_op *op;
3211
3212 list_for_each_entry(op, &insn->stack_ops, list) {
3213
d54dba41 3214 if (update_cfi_state(insn, next_insn, &state->cfi, op))
c9c324dc 3215 return 1;
ab3852ab 3216
ba08abca
PZ		 3217 if (!insn->alt_group)
3218 continue;
3219
65ea47dc
JT		 3220 if (op->dest.type == OP_DEST_PUSHF) {
3221 if (!state->uaccess_stack) {
3222 state->uaccess_stack = 1;
3223 } else if (state->uaccess_stack >> 31) {
3224 WARN_FUNC("PUSHF stack exhausted",
3225 insn->sec, insn->offset);
3226 return 1;
3227 }
3228 state->uaccess_stack <<= 1;
3229 state->uaccess_stack |= state->uaccess;
3230 }
3231
3232 if (op->src.type == OP_SRC_POPF) {
3233 if (state->uaccess_stack) {
3234 state->uaccess = state->uaccess_stack & 1;
3235 state->uaccess_stack >>= 1;
3236 if (state->uaccess_stack == 1)
3237 state->uaccess_stack = 0;
3238 }
3239 }
3240 }
3241
3242 return 0;
3243}
3244
e7c0219b 3245static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
baa41469 3246{
8b946cc3 3247 struct cfi_state *cfi1 = insn->cfi;
baa41469
JP		 3248 int i;
3249
8b946cc3
PZ		 3250 if (!cfi1) {
3251 WARN("CFI missing");
3252 return false;
3253 }
3254
e7c0219b
PZ		 3255 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3256
baa41469
JP		 3257 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3258 insn->sec, insn->offset,
e7c0219b
PZ		 3259 cfi1->cfa.base, cfi1->cfa.offset,
3260 cfi2->cfa.base, cfi2->cfa.offset);
baa41469 3261
e7c0219b 3262 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
baa41469 3263 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b 3264 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
baa41469
JP		 3265 sizeof(struct cfi_reg)))
3266 continue;
3267
3268 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3269 insn->sec, insn->offset,
e7c0219b
PZ		 3270 i, cfi1->regs[i].base, cfi1->regs[i].offset,
3271 i, cfi2->regs[i].base, cfi2->regs[i].offset);
baa41469
JP		 3272 break;
3273 }
3274
e7c0219b
PZ		 3275 } else if (cfi1->type != cfi2->type) {
3276
627fce14 3277 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
e7c0219b
PZ		 3278 insn->sec, insn->offset, cfi1->type, cfi2->type);
3279
3280 } else if (cfi1->drap != cfi2->drap ||
3281 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3282 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
627fce14 3283
bf4d1a83 3284 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
baa41469 3285 insn->sec, insn->offset,
e7c0219b
PZ		 3286 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3287 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
baa41469
JP		 3288
3289 } else
3290 return true;
3291
3292 return false;
dcc914f4
JP		 3293}
3294
ea24213d
PZ		 3295static inline bool func_uaccess_safe(struct symbol *func)
3296{
3297 if (func)
e10cd8fe 3298 return func->uaccess_safe;
ea24213d
PZ		 3299
3300 return false;
3301}
3302
0c1ddd33 3303static inline const char *call_dest_name(struct instruction *insn)
ea24213d 3304{
82880283 3305 static char pvname[19];
db2b0c5d
PZ		 3306 struct reloc *rel;
3307 int idx;
3308
ea24213d
PZ		 3309 if (insn->call_dest)
3310 return insn->call_dest->name;
3311
db2b0c5d
PZ		 3312 rel = insn_reloc(NULL, insn);
3313 if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3314 idx = (rel->addend / sizeof(void *));
3315 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3316 return pvname;
3317 }
3318
ea24213d
PZ		 3319 return "{dynamic}";
3320}
3321
db2b0c5d
PZ		 3322static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3323{
3324 struct symbol *target;
3325 struct reloc *rel;
3326 int idx;
3327
3328 rel = insn_reloc(file, insn);
3329 if (!rel || strcmp(rel->sym->name, "pv_ops"))
3330 return false;
3331
3332 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3333
3334 if (file->pv_ops[idx].clean)
3335 return true;
3336
3337 file->pv_ops[idx].clean = true;
3338
3339 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3340 if (!target->sec->noinstr) {
3341 WARN("pv_ops[%d]: %s", idx, target->name);
3342 file->pv_ops[idx].clean = false;
3343 }
3344 }
3345
3346 return file->pv_ops[idx].clean;
3347}
3348
3349static inline bool noinstr_call_dest(struct objtool_file *file,
3350 struct instruction *insn,
3351 struct symbol *func)
6b643a07
PZ		 3352{
3353 /*
3354 * We can't deal with indirect function calls at present;
3355 * assume they're instrumented.
3356 */
db2b0c5d
PZ		 3357 if (!func) {
3358 if (file->pv_ops)
3359 return pv_call_dest(file, insn);
3360
6b643a07 3361 return false;
db2b0c5d 3362 }
6b643a07
PZ		 3363
3364 /*
3365 * If the symbol is from a noinstr section; we good.
3366 */
3367 if (func->sec->noinstr)
3368 return true;
3369
3370 /*
3371 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3372 * something 'BAD' happened. At the risk of taking the machine down,
3373 * let them proceed to get the message out.
3374 */
3375 if (!strncmp(func->name, "__ubsan_handle_", 15))
3376 return true;
3377
3378 return false;
3379}
3380
db2b0c5d
PZ		 3381static int validate_call(struct objtool_file *file,
3382 struct instruction *insn,
3383 struct insn_state *state)
ea24213d 3384{
c4a33939 3385 if (state->noinstr && state->instr <= 0 &&
db2b0c5d 3386 !noinstr_call_dest(file, insn, insn->call_dest)) {
c4a33939
PZ		 3387 WARN_FUNC("call to %s() leaves .noinstr.text section",
3388 insn->sec, insn->offset, call_dest_name(insn));
3389 return 1;
3390 }
3391
ea24213d
PZ		 3392 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
3393 WARN_FUNC("call to %s() with UACCESS enabled",
0c1ddd33 3394 insn->sec, insn->offset, call_dest_name(insn));
ea24213d
PZ		 3395 return 1;
3396 }
3397
2f0f9e9a
PZ		 3398 if (state->df) {
3399 WARN_FUNC("call to %s() with DF set",
0c1ddd33 3400 insn->sec, insn->offset, call_dest_name(insn));
2f0f9e9a
PZ		 3401 return 1;
3402 }
3403
ea24213d
PZ		 3404 return 0;
3405}
3406
db2b0c5d
PZ		 3407static int validate_sibling_call(struct objtool_file *file,
3408 struct instruction *insn,
3409 struct insn_state *state)
54262aa2 3410{
5a9c361a 3411 if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
54262aa2
PZ		 3412 WARN_FUNC("sibling call from callable instruction with modified stack frame",
3413 insn->sec, insn->offset);
3414 return 1;
3415 }
3416
db2b0c5d 3417 return validate_call(file, insn, state);
54262aa2
PZ		 3418}
3419
a92e92d1
PZ		 3420static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3421{
c4a33939
PZ		 3422 if (state->noinstr && state->instr > 0) {
3423 WARN_FUNC("return with instrumentation enabled",
3424 insn->sec, insn->offset);
3425 return 1;
3426 }
3427
a92e92d1
PZ		 3428 if (state->uaccess && !func_uaccess_safe(func)) {
3429 WARN_FUNC("return with UACCESS enabled",
3430 insn->sec, insn->offset);
3431 return 1;
3432 }
3433
3434 if (!state->uaccess && func_uaccess_safe(func)) {
3435 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
3436 insn->sec, insn->offset);
3437 return 1;
3438 }
3439
3440 if (state->df) {
3441 WARN_FUNC("return with DF set",
3442 insn->sec, insn->offset);
3443 return 1;
3444 }
3445
e25eea89 3446 if (func && has_modified_stack_frame(insn, state)) {
a92e92d1
PZ		 3447 WARN_FUNC("return with modified stack frame",
3448 insn->sec, insn->offset);
3449 return 1;
3450 }
3451
e7c0219b 3452 if (state->cfi.bp_scratch) {
b2966952
JP		 3453 WARN_FUNC("BP used as a scratch register",
3454 insn->sec, insn->offset);
a92e92d1
PZ		 3455 return 1;
3456 }
3457
3458 return 0;
3459}
3460
c9c324dc
JP		 3461static struct instruction *next_insn_to_validate(struct objtool_file *file,
3462 struct instruction *insn)
7117f16b 3463{
b23cc71c 3464 struct alt_group *alt_group = insn->alt_group;
7117f16b 3465
c9c324dc
JP		 3466 /*
3467 * Simulate the fact that alternatives are patched in-place. When the
3468 * end of a replacement alt_group is reached, redirect objtool flow to
3469 * the end of the original alt_group.
3470 */
3471 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3472 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3473
3474 return next_insn_same_sec(file, insn);
7117f16b
PZ		 3475}
3476
dcc914f4
JP		 3477/*
3478 * Follow the branch starting at the given instruction, and recursively follow
3479 * any other branches (jumps). Meanwhile, track the frame pointer state at
3480 * each instruction and validate all the rules described in
d6a21f2d 3481 * tools/objtool/Documentation/objtool.txt.
dcc914f4 3482 */
c705cecc 3483static int validate_branch(struct objtool_file *file, struct symbol *func,
b7460462 3484 struct instruction *insn, struct insn_state state)
dcc914f4
JP		 3485{
3486 struct alternative *alt;
8b946cc3 3487 struct instruction *next_insn, *prev_insn = NULL;
dcc914f4 3488 struct section *sec;
882a0db9 3489 u8 visited;
dcc914f4
JP		 3490 int ret;
3491
dcc914f4 3492 sec = insn->sec;
dcc914f4 3493
dcc914f4 3494 while (1) {
c9c324dc 3495 next_insn = next_insn_to_validate(file, insn);
39358a03 3496
dbcdbdfd 3497 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3c68a92d 3498 /* Ignore KCFI type preambles, which always fall through */
9f2899fe
PZ		 3499 if (!strncmp(func->name, "__cfi_", 6) ||
3500 !strncmp(func->name, "__pfx_", 6))
3c68a92d
ST		 3501 return 0;
3502
ee97638b 3503 WARN("%s() falls through to next function %s()",
dbcdbdfd 3504 func->name, insn_func(insn)->name);
ee97638b 3505 return 1;
dcc914f4
JP		 3506 }
3507
4855022a
JP		 3508 if (func && insn->ignore) {
3509 WARN_FUNC("BUG: why am I validating an ignored function?",
3510 sec, insn->offset);
12b25729 3511 return 1;
4855022a
JP		 3512 }
3513
a09a6e23
PZ		 3514 visited = VISITED_BRANCH << state.uaccess;
3515 if (insn->visited & VISITED_BRANCH_MASK) {
e7c0219b 3516 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
dcc914f4 3517 return 1;
dcc914f4 3518
882a0db9 3519 if (insn->visited & visited)
ea24213d 3520 return 0;
8b946cc3
PZ		 3521 } else {
3522 nr_insns_visited++;
dcc914f4
JP		 3523 }
3524
c4a33939
PZ		 3525 if (state.noinstr)
3526 state.instr += insn->instr;
3527
8b946cc3 3528 if (insn->hint) {
8faea26e
JP		 3529 if (insn->restore) {
3530 struct instruction *save_insn, *i;
3531
3532 i = insn;
3533 save_insn = NULL;
3534
3535 sym_for_each_insn_continue_reverse(file, func, i) {
3536 if (i->save) {
3537 save_insn = i;
3538 break;
3539 }
3540 }
3541
3542 if (!save_insn) {
3543 WARN_FUNC("no corresponding CFI save for CFI restore",
3544 sec, insn->offset);
3545 return 1;
3546 }
3547
3548 if (!save_insn->visited) {
3549 WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
3550 sec, insn->offset);
3551 return 1;
3552 }
3553
3554 insn->cfi = save_insn->cfi;
3555 nr_cfi_reused++;
3556 }
3557
8b946cc3
PZ		 3558 state.cfi = *insn->cfi;
3559 } else {
3560 /* XXX track if we actually changed state.cfi */
3561
3562 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3563 insn->cfi = prev_insn->cfi;
3564 nr_cfi_reused++;
3565 } else {
3566 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3567 }
3568 }
dcc914f4 3569
882a0db9 3570 insn->visited |= visited;
baa41469 3571
c9c324dc
JP		 3572 if (propagate_alt_cfi(file, insn))
3573 return 1;
3574
7117f16b 3575 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
764eef4b
PZ		 3576 bool skip_orig = false;
3577
a845c7cf 3578 list_for_each_entry(alt, &insn->alts, list) {
764eef4b
PZ		 3579 if (alt->skip_orig)
3580 skip_orig = true;
3581
c705cecc 3582 ret = validate_branch(file, func, alt->insn, state);
7697eee3 3583 if (ret) {
2daf7fab 3584 if (opts.backtrace)
7697eee3
PZ		 3585 BT_FUNC("(alt)", insn);
3586 return ret;
3587 }
a845c7cf 3588 }
764eef4b
PZ		 3589
3590 if (skip_orig)
3591 return 0;
dcc914f4
JP		 3592 }
3593
d54dba41 3594 if (handle_insn_ops(insn, next_insn, &state))
60041bcd
PZ		 3595 return 1;
3596
dcc914f4
JP		 3597 switch (insn->type) {
3598
dcc914f4 3599 case INSN_RETURN:
a92e92d1 3600 return validate_return(func, insn, &state);
dcc914f4
JP		 3601
3602 case INSN_CALL:
ea24213d 3603 case INSN_CALL_DYNAMIC:
db2b0c5d 3604 ret = validate_call(file, insn, &state);
ea24213d
PZ		 3605 if (ret)
3606 return ret;
dcc914f4 3607
72064474 3608 if (opts.stackval && func && !is_fentry_call(insn) &&
c9bab22b 3609 !has_valid_stack_frame(&state)) {
dcc914f4
JP		 3610 WARN_FUNC("call without frame pointer save/setup",
3611 sec, insn->offset);
3612 return 1;
3613 }
c9bab22b 3614
0e5b613b 3615 if (insn->dead_end)
c9bab22b
JP		 3616 return 0;
3617
dcc914f4
JP		 3618 break;
3619
3620 case INSN_JUMP_CONDITIONAL:
3621 case INSN_JUMP_UNCONDITIONAL:
ecf11ba4 3622 if (is_sibling_call(insn)) {
db2b0c5d 3623 ret = validate_sibling_call(file, insn, &state);
dcc914f4 3624 if (ret)
54262aa2 3625 return ret;
4855022a 3626
0c1ddd33 3627 } else if (insn->jump_dest) {
c705cecc
JP		 3628 ret = validate_branch(file, func,
3629 insn->jump_dest, state);
7697eee3 3630 if (ret) {
2daf7fab 3631 if (opts.backtrace)
7697eee3
PZ		 3632 BT_FUNC("(branch)", insn);
3633 return ret;
3634 }
4855022a 3635 }
dcc914f4
JP		 3636
3637 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3638 return 0;
3639
3640 break;
3641
3642 case INSN_JUMP_DYNAMIC:
b68b9907 3643 case INSN_JUMP_DYNAMIC_CONDITIONAL:
ecf11ba4 3644 if (is_sibling_call(insn)) {
db2b0c5d 3645 ret = validate_sibling_call(file, insn, &state);
54262aa2
PZ		 3646 if (ret)
3647 return ret;
dcc914f4
JP		 3648 }
3649
b68b9907
JP		 3650 if (insn->type == INSN_JUMP_DYNAMIC)
3651 return 0;
3652
3653 break;
dcc914f4 3654
39358a03
JP		 3655 case INSN_CONTEXT_SWITCH:
3656 if (func && (!next_insn || !next_insn->hint)) {
3657 WARN_FUNC("unsupported instruction in callable function",
3658 sec, insn->offset);
3659 return 1;
3660 }
3661 return 0;
3662
ea24213d
PZ		 3663 case INSN_STAC:
3664 if (state.uaccess) {
3665 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3666 return 1;
3667 }
3668
3669 state.uaccess = true;
3670 break;
3671
3672 case INSN_CLAC:
c705cecc 3673 if (!state.uaccess && func) {
ea24213d
PZ		 3674 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3675 return 1;
3676 }
3677
3678 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3679 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3680 return 1;
3681 }
3682
3683 state.uaccess = false;
baa41469
JP		 3684 break;
3685
2f0f9e9a 3686 case INSN_STD:
6f567c93 3687 if (state.df) {
2f0f9e9a 3688 WARN_FUNC("recursive STD", sec, insn->offset);
6f567c93
JP		 3689 return 1;
3690 }
2f0f9e9a
PZ		 3691
3692 state.df = true;
3693 break;
3694
3695 case INSN_CLD:
6f567c93 3696 if (!state.df && func) {
2f0f9e9a 3697 WARN_FUNC("redundant CLD", sec, insn->offset);
6f567c93
JP		 3698 return 1;
3699 }
2f0f9e9a
PZ		 3700
3701 state.df = false;
baa41469
JP		 3702 break;
3703
dcc914f4
JP		 3704 default:
3705 break;
3706 }
3707
3708 if (insn->dead_end)
3709 return 0;
3710
00d96180 3711 if (!next_insn) {
e7c0219b 3712 if (state.cfi.cfa.base == CFI_UNDEFINED)
00d96180 3713 return 0;
dcc914f4
JP		 3714 WARN("%s: unexpected end of section", sec->name);
3715 return 1;
3716 }
00d96180 3717
8b946cc3 3718 prev_insn = insn;
00d96180 3719 insn = next_insn;
dcc914f4
JP		 3720 }
3721
3722 return 0;
3723}
3724
932f8e98 3725static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
39358a03
JP		 3726{
3727 struct instruction *insn;
39358a03 3728 struct insn_state state;
932f8e98 3729 int ret, warnings = 0;
39358a03
JP		 3730
3731 if (!file->hints)
3732 return 0;
3733
753da417 3734 init_insn_state(file, &state, sec);
39358a03 3735
932f8e98
PZ		 3736 if (sec) {
3737 insn = find_insn(file, sec, 0);
3738 if (!insn)
3739 return 0;
3740 } else {
3741 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3742 }
3743
3744 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
5b284b19 3745 if (insn->hint && !insn->visited && !insn->ignore) {
dbcdbdfd 3746 ret = validate_branch(file, insn_func(insn), insn, state);
2daf7fab 3747 if (ret && opts.backtrace)
7697eee3 3748 BT_FUNC("<=== (hint)", insn);
39358a03
JP		 3749 warnings += ret;
3750 }
932f8e98
PZ		 3751
3752 insn = list_next_entry(insn, list);
39358a03
JP		 3753 }
3754
3755 return warnings;
3756}
3757
a09a6e23
PZ		 3758/*
3759 * Validate rethunk entry constraint: must untrain RET before the first RET.
3760 *
3761 * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
3762 * before an actual RET instruction.
3763 */
3764static int validate_entry(struct objtool_file *file, struct instruction *insn)
3765{
3766 struct instruction *next, *dest;
3767 int ret, warnings = 0;
3768
3769 for (;;) {
3770 next = next_insn_to_validate(file, insn);
3771
3772 if (insn->visited & VISITED_ENTRY)
3773 return 0;
3774
3775 insn->visited |= VISITED_ENTRY;
3776
3777 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3778 struct alternative *alt;
3779 bool skip_orig = false;
3780
3781 list_for_each_entry(alt, &insn->alts, list) {
3782 if (alt->skip_orig)
3783 skip_orig = true;
3784
3785 ret = validate_entry(file, alt->insn);
3786 if (ret) {
3787 if (opts.backtrace)
3788 BT_FUNC("(alt)", insn);
3789 return ret;
3790 }
3791 }
3792
3793 if (skip_orig)
3794 return 0;
3795 }
3796
3797 switch (insn->type) {
3798
3799 case INSN_CALL_DYNAMIC:
3800 case INSN_JUMP_DYNAMIC:
3801 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3802 WARN_FUNC("early indirect call", insn->sec, insn->offset);
3803 return 1;
3804
3805 case INSN_JUMP_UNCONDITIONAL:
3806 case INSN_JUMP_CONDITIONAL:
3807 if (!is_sibling_call(insn)) {
3808 if (!insn->jump_dest) {
3809 WARN_FUNC("unresolved jump target after linking?!?",
3810 insn->sec, insn->offset);
3811 return -1;
3812 }
3813 ret = validate_entry(file, insn->jump_dest);
3814 if (ret) {
3815 if (opts.backtrace) {
3816 BT_FUNC("(branch%s)", insn,
3817 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3818 }
3819 return ret;
3820 }
3821
3822 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3823 return 0;
3824
3825 break;
3826 }
3827
3828 /* fallthrough */
3829 case INSN_CALL:
3830 dest = find_insn(file, insn->call_dest->sec,
3831 insn->call_dest->offset);
3832 if (!dest) {
3833 WARN("Unresolved function after linking!?: %s",
3834 insn->call_dest->name);
3835 return -1;
3836 }
3837
3838 ret = validate_entry(file, dest);
3839 if (ret) {
3840 if (opts.backtrace)
3841 BT_FUNC("(call)", insn);
3842 return ret;
3843 }
3844 /*
3845 * If a call returns without error, it must have seen UNTRAIN_RET.
3846 * Therefore any non-error return is a success.
3847 */
3848 return 0;
3849
3850 case INSN_RETURN:
3851 WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
3852 return 1;
3853
3854 case INSN_NOP:
3855 if (insn->retpoline_safe)
3856 return 0;
3857 break;
3858
3859 default:
3860 break;
3861 }
3862
3863 if (!next) {
3864 WARN_FUNC("teh end!", insn->sec, insn->offset);
3865 return -1;
3866 }
3867 insn = next;
3868 }
3869
3870 return warnings;
3871}
3872
3873/*
3874 * Validate that all branches starting at 'insn->entry' encounter UNRET_END
3875 * before RET.
3876 */
3877static int validate_unret(struct objtool_file *file)
3878{
3879 struct instruction *insn;
3880 int ret, warnings = 0;
3881
3882 for_each_insn(file, insn) {
3883 if (!insn->entry)
3884 continue;
3885
3886 ret = validate_entry(file, insn);
3887 if (ret < 0) {
3888 WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
3889 return ret;
3890 }
3891 warnings += ret;
3892 }
3893
3894 return warnings;
3895}
3896
b5bc2231
PZ		 3897static int validate_retpoline(struct objtool_file *file)
3898{
3899 struct instruction *insn;
3900 int warnings = 0;
3901
3902 for_each_insn(file, insn) {
3903 if (insn->type != INSN_JUMP_DYNAMIC &&
9bb2ec60
PZ		 3904 insn->type != INSN_CALL_DYNAMIC &&
3905 insn->type != INSN_RETURN)
b5bc2231
PZ		 3906 continue;
3907
3908 if (insn->retpoline_safe)
3909 continue;
3910
6644ee84 3911 if (insn->sec->init)
ca41b97e
PZ		 3912 continue;
3913
9bb2ec60 3914 if (insn->type == INSN_RETURN) {
f43b9876
PZ		 3915 if (opts.rethunk) {
3916 WARN_FUNC("'naked' return found in RETHUNK build",
3917 insn->sec, insn->offset);
3918 } else
3919 continue;
9bb2ec60
PZ		 3920 } else {
3921 WARN_FUNC("indirect %s found in RETPOLINE build",
3922 insn->sec, insn->offset,
3923 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3924 }
b5bc2231
PZ		 3925
3926 warnings++;
3927 }
3928
3929 return warnings;
3930}
3931
dcc914f4
JP		 3932static bool is_kasan_insn(struct instruction *insn)
3933{
3934 return (insn->type == INSN_CALL &&
3935 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3936}
3937
3938static bool is_ubsan_insn(struct instruction *insn)
3939{
3940 return (insn->type == INSN_CALL &&
3941 !strcmp(insn->call_dest->name,
3942 "__ubsan_handle_builtin_unreachable"));
3943}
3944
14db1f0a 3945static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
dcc914f4
JP		 3946{
3947 int i;
14db1f0a 3948 struct instruction *prev_insn;
dcc914f4 3949
1ffbe4e9 3950 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
baa41469
JP		 3951 return true;
3952
3953 /*
82a8954a 3954 * Ignore alternative replacement instructions. This can happen
0e2bb2bc 3955 * when a whitelisted function uses one of the ALTERNATIVE macros.
baa41469 3956 */
82a8954a 3957 if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
0e2bb2bc 3958 !strcmp(insn->sec->name, ".altinstr_aux"))
dcc914f4
JP		 3959 return true;
3960
4adb2368 3961 /*
753da417 3962 * Whole archive runs might encounter dead code from weak symbols.
4adb2368
PZ		 3963 * This is where the linker will have dropped the weak symbol in
3964 * favour of a regular symbol, but leaves the code in place.
3965 *
3966 * In this case we'll find a piece of code (whole function) that is not
3967 * covered by a !section symbol. Ignore them.
3968 */
dbcdbdfd 3969 if (opts.link && !insn_func(insn)) {
4adb2368
PZ		 3970 int size = find_symbol_hole_containing(insn->sec, insn->offset);
3971 unsigned long end = insn->offset + size;
3972
3973 if (!size) /* not a hole */
3974 return false;
3975
3976 if (size < 0) /* hole until the end */
3977 return true;
3978
3979 sec_for_each_insn_continue(file, insn) {
3980 /*
3981 * If we reach a visited instruction at or before the
3982 * end of the hole, ignore the unreachable.
3983 */
3984 if (insn->visited)
3985 return true;
3986
3987 if (insn->offset >= end)
3988 break;
3989
3990 /*
3991 * If this hole jumps to a .cold function, mark it ignore too.
3992 */
dbcdbdfd
PZ		 3993 if (insn->jump_dest && insn_func(insn->jump_dest) &&
3994 strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4adb2368 3995 struct instruction *dest = insn->jump_dest;
dbcdbdfd 3996 func_for_each_insn(file, insn_func(dest), dest)
4adb2368
PZ		 3997 dest->ignore = true;
3998 }
3999 }
4000
4001 return false;
4002 }
4003
dbcdbdfd 4004 if (!insn_func(insn))
bd841d61
JP		 4005 return false;
4006
dbcdbdfd 4007 if (insn_func(insn)->static_call_tramp)
2105a927
PZ		 4008 return true;
4009
bd841d61
JP		 4010 /*
4011 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4012 * __builtin_unreachable(). The BUG() macro has an unreachable() after
4013 * the UD2, which causes GCC's undefined trap logic to emit another UD2
4014 * (or occasionally a JMP to UD2).
14db1f0a
IH		 4015 *
4016 * It may also insert a UD2 after calling a __noreturn function.
bd841d61 4017 */
14db1f0a
IH		 4018 prev_insn = list_prev_entry(insn, list);
4019 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
bd841d61
JP		 4020 (insn->type == INSN_BUG ||
4021 (insn->type == INSN_JUMP_UNCONDITIONAL &&
4022 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4023 return true;
4024
dcc914f4
JP		 4025 /*
4026 * Check if this (or a subsequent) instruction is related to
4027 * CONFIG_UBSAN or CONFIG_KASAN.
4028 *
4029 * End the search at 5 instructions to avoid going into the weeds.
4030 */
4031 for (i = 0; i < 5; i++) {
4032
4033 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4034 return true;
4035
fe24e271
JP		 4036 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4037 if (insn->jump_dest &&
dbcdbdfd 4038 insn_func(insn->jump_dest) == insn_func(insn)) {
fe24e271
JP		 4039 insn = insn->jump_dest;
4040 continue;
4041 }
4042
4043 break;
dcc914f4
JP		 4044 }
4045
dbcdbdfd 4046 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
dcc914f4 4047 break;
fe24e271 4048
dcc914f4
JP		 4049 insn = list_next_entry(insn, list);
4050 }
4051
4052 return false;
4053}
4054
9f2899fe
PZ		 4055static int add_prefix_symbol(struct objtool_file *file, struct symbol *func,
4056 struct instruction *insn)
4057{
4058 if (!opts.prefix)
4059 return 0;
4060
4061 for (;;) {
4062 struct instruction *prev = list_prev_entry(insn, list);
4063 u64 offset;
4064
4065 if (&prev->list == &file->insn_list)
4066 break;
4067
4068 if (prev->type != INSN_NOP)
4069 break;
4070
4071 offset = func->offset - prev->offset;
4072 if (offset >= opts.prefix) {
023f2340
PZ		 4073 if (offset == opts.prefix) {
4074 /*
4075 * Since the sec->symbol_list is ordered by
4076 * offset (see elf_add_symbol()) the added
4077 * symbol will not be seen by the iteration in
4078 * validate_section().
4079 *
4080 * Hence the lack of list_for_each_entry_safe()
4081 * there.
4082 *
4083 * The direct concequence is that prefix symbols
4084 * don't get visited (because pointless), except
4085 * for the logic in ignore_unreachable_insn()
4086 * that needs the terminating insn to be visited
4087 * otherwise it will report the hole.
4088 *
4089 * Hence mark the first instruction of the
4090 * prefix symbol as visisted.
4091 */
4092 prev->visited |= VISITED_BRANCH;
9f2899fe 4093 elf_create_prefix_symbol(file->elf, func, opts.prefix);
023f2340 4094 }
9f2899fe
PZ		 4095 break;
4096 }
4097 insn = prev;
4098 }
4099
4100 return 0;
4101}
4102
4b5e2e7f
PZ		 4103static int validate_symbol(struct objtool_file *file, struct section *sec,
4104 struct symbol *sym, struct insn_state *state)
dcc914f4 4105{
dcc914f4 4106 struct instruction *insn;
4b5e2e7f
PZ		 4107 int ret;
4108
4109 if (!sym->len) {
4110 WARN("%s() is missing an ELF size annotation", sym->name);
4111 return 1;
4112 }
4113
4114 if (sym->pfunc != sym || sym->alias != sym)
4115 return 0;
4116
4117 insn = find_insn(file, sec, sym->offset);
4118 if (!insn || insn->ignore || insn->visited)
4119 return 0;
4120
9f2899fe
PZ		 4121 add_prefix_symbol(file, sym, insn);
4122
4b5e2e7f
PZ		 4123 state->uaccess = sym->uaccess_safe;
4124
dbcdbdfd 4125 ret = validate_branch(file, insn_func(insn), insn, *state);
2daf7fab 4126 if (ret && opts.backtrace)
4b5e2e7f
PZ		 4127 BT_FUNC("<=== (sym)", insn);
4128 return ret;
4129}
4130
4131static int validate_section(struct objtool_file *file, struct section *sec)
4132{
baa41469 4133 struct insn_state state;
4b5e2e7f
PZ		 4134 struct symbol *func;
4135 int warnings = 0;
dcc914f4 4136
350994bf
PZ		 4137 list_for_each_entry(func, &sec->symbol_list, list) {
4138 if (func->type != STT_FUNC)
4139 continue;
e10cd8fe 4140
753da417 4141 init_insn_state(file, &state, sec);
b735bd3e 4142 set_func_state(&state.cfi);
0699e551 4143
4b5e2e7f 4144 warnings += validate_symbol(file, sec, func, &state);
dcc914f4
JP		 4145 }
4146
dcc914f4
JP		 4147 return warnings;
4148}
4149
753da417 4150static int validate_noinstr_sections(struct objtool_file *file)
c4a33939
PZ		 4151{
4152 struct section *sec;
932f8e98 4153 int warnings = 0;
c4a33939
PZ		 4154
4155 sec = find_section_by_name(file->elf, ".noinstr.text");
0cc9ac8d
TG		 4156 if (sec) {
4157 warnings += validate_section(file, sec);
4158 warnings += validate_unwind_hints(file, sec);
4159 }
c4a33939 4160
0cc9ac8d
TG		 4161 sec = find_section_by_name(file->elf, ".entry.text");
4162 if (sec) {
4163 warnings += validate_section(file, sec);
4164 warnings += validate_unwind_hints(file, sec);
4165 }
932f8e98
PZ		 4166
4167 return warnings;
c4a33939
PZ		 4168}
4169
350994bf
PZ		 4170static int validate_functions(struct objtool_file *file)
4171{
4172 struct section *sec;
4173 int warnings = 0;
4174
da837bd6
PZ		 4175 for_each_sec(file, sec) {
4176 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4177 continue;
4178
350994bf 4179 warnings += validate_section(file, sec);
da837bd6 4180 }
350994bf
PZ		 4181
4182 return warnings;
4183}
4184
3c6f9f77 4185static void mark_endbr_used(struct instruction *insn)
08f87a93 4186{
3c6f9f77
JP		 4187 if (!list_empty(&insn->call_node))
4188 list_del_init(&insn->call_node);
4189}
4190
08ef8c40
PZ		 4191static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4192{
4193 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4194 struct instruction *first;
4195
4196 if (!sym)
4197 return false;
4198
4199 first = find_insn(file, sym->sec, sym->offset);
4200 if (!first)
4201 return false;
4202
4203 if (first->type != INSN_ENDBR && !first->noendbr)
4204 return false;
4205
4206 return insn->offset == sym->offset + sym->len;
4207}
4208
3c6f9f77
JP		 4209static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4210{
4211 struct instruction *dest;
08f87a93 4212 struct reloc *reloc;
3c6f9f77
JP		 4213 unsigned long off;
4214 int warnings = 0;
08f87a93 4215
3c6f9f77
JP		 4216 /*
4217 * Looking for function pointer load relocations. Ignore
4218 * direct/indirect branches:
4219 */
4220 switch (insn->type) {
4221 case INSN_CALL:
4222 case INSN_CALL_DYNAMIC:
4223 case INSN_JUMP_CONDITIONAL:
4224 case INSN_JUMP_UNCONDITIONAL:
4225 case INSN_JUMP_DYNAMIC:
4226 case INSN_JUMP_DYNAMIC_CONDITIONAL:
4227 case INSN_RETURN:
4228 case INSN_NOP:
4229 return 0;
4230 default:
4231 break;
4232 }
08f87a93 4233
3c6f9f77
JP		 4234 for (reloc = insn_reloc(file, insn);
4235 reloc;
4236 reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4237 reloc->offset + 1,
4238 (insn->offset + insn->len) - (reloc->offset + 1))) {
08f87a93 4239
3c6f9f77
JP		 4240 /*
4241 * static_call_update() references the trampoline, which
4242 * doesn't have (or need) ENDBR. Skip warning in that case.
4243 */
4244 if (reloc->sym->static_call_tramp)
08f87a93
PZ		 4245 continue;
4246
3c6f9f77
JP		 4247 off = reloc->sym->offset;
4248 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
4249 off += arch_dest_reloc_offset(reloc->addend);
4250 else
4251 off += reloc->addend;
4252
4253 dest = find_insn(file, reloc->sym->sec, off);
4254 if (!dest)
08f87a93
PZ		 4255 continue;
4256
3c6f9f77
JP		 4257 if (dest->type == INSN_ENDBR) {
4258 mark_endbr_used(dest);
08f87a93 4259 continue;
3c6f9f77 4260 }
08f87a93 4261
dbcdbdfd 4262 if (insn_func(dest) && insn_func(dest) == insn_func(insn)) {
3c6f9f77
JP		 4263 /*
4264 * Anything from->to self is either _THIS_IP_ or
4265 * IRET-to-self.
4266 *
4267 * There is no sane way to annotate _THIS_IP_ since the
4268 * compiler treats the relocation as a constant and is
4269 * happy to fold in offsets, skewing any annotation we
4270 * do, leading to vast amounts of false-positives.
4271 *
4272 * There's also compiler generated _THIS_IP_ through
4273 * KCOV and such which we have no hope of annotating.
4274 *
4275 * As such, blanket accept self-references without
4276 * issue.
4277 */
08f87a93 4278 continue;
3c6f9f77 4279 }
08f87a93 4280
08ef8c40
PZ		 4281 /*
4282 * Accept anything ANNOTATE_NOENDBR.
4283 */
3c6f9f77 4284 if (dest->noendbr)
08f87a93
PZ		 4285 continue;
4286
08ef8c40
PZ		 4287 /*
4288 * Accept if this is the instruction after a symbol
4289 * that is (no)endbr -- typical code-range usage.
4290 */
4291 if (noendbr_range(file, dest))
4292 continue;
4293
3c6f9f77
JP		 4294 WARN_FUNC("relocation to !ENDBR: %s",
4295 insn->sec, insn->offset,
4296 offstr(dest->sec, dest->offset));
4297
4298 warnings++;
4299 }
4300
4301 return warnings;
4302}
4303
4304static int validate_ibt_data_reloc(struct objtool_file *file,
4305 struct reloc *reloc)
4306{
4307 struct instruction *dest;
4308
4309 dest = find_insn(file, reloc->sym->sec,
4310 reloc->sym->offset + reloc->addend);
4311 if (!dest)
4312 return 0;
4313
4314 if (dest->type == INSN_ENDBR) {
4315 mark_endbr_used(dest);
4316 return 0;
4317 }
4318
4319 if (dest->noendbr)
4320 return 0;
4321
4322 WARN_FUNC("data relocation to !ENDBR: %s",
4323 reloc->sec->base, reloc->offset,
4324 offstr(dest->sec, dest->offset));
4325
4326 return 1;
4327}
4328
4329/*
4330 * Validate IBT rules and remove used ENDBR instructions from the seal list.
4331 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4332 * NOPs) later, in create_ibt_endbr_seal_sections().
4333 */
4334static int validate_ibt(struct objtool_file *file)
4335{
4336 struct section *sec;
4337 struct reloc *reloc;
4338 struct instruction *insn;
4339 int warnings = 0;
4340
4341 for_each_insn(file, insn)
4342 warnings += validate_ibt_insn(file, insn);
4343
4344 for_each_sec(file, sec) {
4345
4346 /* Already done by validate_ibt_insn() */
4347 if (sec->sh.sh_flags & SHF_EXECINSTR)
08f87a93
PZ		 4348 continue;
4349
3c6f9f77
JP		 4350 if (!sec->reloc)
4351 continue;
08f87a93 4352
3c6f9f77
JP		 4353 /*
4354 * These sections can reference text addresses, but not with
4355 * the intent to indirect branch to them.
4356 */
e27e5bea
JP		 4357 if ((!strncmp(sec->name, ".discard", 8) &&
4358 strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
3c6f9f77
JP		 4359 !strncmp(sec->name, ".debug", 6) ||
4360 !strcmp(sec->name, ".altinstructions") ||
4361 !strcmp(sec->name, ".ibt_endbr_seal") ||
4362 !strcmp(sec->name, ".orc_unwind_ip") ||
4363 !strcmp(sec->name, ".parainstructions") ||
4364 !strcmp(sec->name, ".retpoline_sites") ||
4365 !strcmp(sec->name, ".smp_locks") ||
4366 !strcmp(sec->name, ".static_call_sites") ||
4367 !strcmp(sec->name, "_error_injection_whitelist") ||
4368 !strcmp(sec->name, "_kprobe_blacklist") ||
4369 !strcmp(sec->name, "__bug_table") ||
4370 !strcmp(sec->name, "__ex_table") ||
4371 !strcmp(sec->name, "__jump_table") ||
3c68a92d 4372 !strcmp(sec->name, "__mcount_loc") ||
0326074f 4373 !strcmp(sec->name, ".kcfi_traps") ||
9440155c 4374 strstr(sec->name, "__patchable_function_entries"))
3c6f9f77 4375 continue;
08f87a93 4376
3c6f9f77
JP		 4377 list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
4378 warnings += validate_ibt_data_reloc(file, reloc);
08f87a93
PZ		 4379 }
4380
3c6f9f77 4381 return warnings;
08f87a93
PZ		 4382}
4383
c2bdd61c
JP		 4384static int validate_sls(struct objtool_file *file)
4385{
4386 struct instruction *insn, *next_insn;
4387 int warnings = 0;
4388
4389 for_each_insn(file, insn) {
4390 next_insn = next_insn_same_sec(file, insn);
4391
4392 if (insn->retpoline_safe)
4393 continue;
4394
4395 switch (insn->type) {
4396 case INSN_RETURN:
4397 if (!next_insn || next_insn->type != INSN_TRAP) {
4398 WARN_FUNC("missing int3 after ret",
4399 insn->sec, insn->offset);
4400 warnings++;
4401 }
4402
4403 break;
4404 case INSN_JUMP_DYNAMIC:
4405 if (!next_insn || next_insn->type != INSN_TRAP) {
4406 WARN_FUNC("missing int3 after indirect jump",
4407 insn->sec, insn->offset);
4408 warnings++;
4409 }
4410 break;
4411 default:
4412 break;
4413 }
4414 }
4415
4416 return warnings;
4417}
4418
baa41469 4419static int validate_reachable_instructions(struct objtool_file *file)
dcc914f4
JP		 4420{
4421 struct instruction *insn;
baa41469
JP		 4422
4423 if (file->ignore_unreachables)
4424 return 0;
dcc914f4
JP		 4425
4426 for_each_insn(file, insn) {
14db1f0a 4427 if (insn->visited || ignore_unreachable_insn(file, insn))
baa41469
JP		 4428 continue;
4429
baa41469
JP		 4430 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
4431 return 1;
dcc914f4
JP		 4432 }
4433
baa41469 4434 return 0;
dcc914f4
JP		 4435}
4436
d44becb9 4437int check(struct objtool_file *file)
dcc914f4 4438{
dcc914f4
JP		 4439 int ret, warnings = 0;
4440
baa41469 4441 arch_initial_func_cfi_state(&initial_func_cfi);
8b946cc3
PZ		 4442 init_cfi_state(&init_cfi);
4443 init_cfi_state(&func_cfi);
4444 set_func_state(&func_cfi);
4445
4446 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4447 goto out;
4448
4449 cfi_hash_add(&init_cfi);
4450 cfi_hash_add(&func_cfi);
baa41469 4451
6545eb03 4452 ret = decode_sections(file);
dcc914f4
JP		 4453 if (ret < 0)
4454 goto out;
8b946cc3 4455
dcc914f4
JP		 4456 warnings += ret;
4457
6545eb03 4458 if (list_empty(&file->insn_list))
dcc914f4 4459 goto out;
dcc914f4 4460
2daf7fab 4461 if (opts.retpoline) {
6545eb03 4462 ret = validate_retpoline(file);
b5bc2231
PZ		 4463 if (ret < 0)
4464 return ret;
4465 warnings += ret;
4466 }
4467
c2bdd61c 4468 if (opts.stackval || opts.orc || opts.uaccess) {
7dce6204
JP		 4469 ret = validate_functions(file);
4470 if (ret < 0)
4471 goto out;
4472 warnings += ret;
39358a03 4473
7dce6204 4474 ret = validate_unwind_hints(file, NULL);
08f87a93
PZ		 4475 if (ret < 0)
4476 goto out;
4477 warnings += ret;
7dce6204
JP		 4478
4479 if (!warnings) {
4480 ret = validate_reachable_instructions(file);
4481 if (ret < 0)
4482 goto out;
4483 warnings += ret;
4484 }
753da417
JP		 4485
4486 } else if (opts.noinstr) {
4487 ret = validate_noinstr_sections(file);
4488 if (ret < 0)
4489 goto out;
4490 warnings += ret;
08f87a93
PZ		 4491 }
4492
a09a6e23
PZ		 4493 if (opts.unret) {
4494 /*
4495 * Must be after validate_branch() and friends, it plays
4496 * further games with insn->visited.
4497 */
4498 ret = validate_unret(file);
4499 if (ret < 0)
4500 return ret;
4501 warnings += ret;
4502 }
4503
7dce6204
JP		 4504 if (opts.ibt) {
4505 ret = validate_ibt(file);
baa41469
JP		 4506 if (ret < 0)
4507 goto out;
c2bdd61c
JP		 4508 warnings += ret;
4509 }
4510
4511 if (opts.sls) {
4512 ret = validate_sls(file);
4513 if (ret < 0)
4514 goto out;
baa41469
JP		 4515 warnings += ret;
4516 }
4517
26e17689
JP		 4518 if (opts.static_call) {
4519 ret = create_static_call_sections(file);
4520 if (ret < 0)
4521 goto out;
4522 warnings += ret;
4523 }
1e7e4788 4524
2daf7fab 4525 if (opts.retpoline) {
134ab5bd
PZ		 4526 ret = create_retpoline_sites_sections(file);
4527 if (ret < 0)
4528 goto out;
4529 warnings += ret;
4530 }
4531
9a479f76
PZ		 4532 if (opts.cfi) {
4533 ret = create_cfi_sections(file);
4534 if (ret < 0)
4535 goto out;
4536 warnings += ret;
4537 }
4538
f43b9876 4539 if (opts.rethunk) {
d9e9d230
PZ		 4540 ret = create_return_sites_sections(file);
4541 if (ret < 0)
4542 goto out;
4543 warnings += ret;
00abd384 4544
0c0a6d89
PZ		 4545 if (opts.hack_skylake) {
4546 ret = create_direct_call_sections(file);
4547 if (ret < 0)
4548 goto out;
4549 warnings += ret;
4550 }
134ab5bd
PZ		 4551 }
4552
2daf7fab 4553 if (opts.mcount) {
99d00215
PZ		 4554 ret = create_mcount_loc_sections(file);
4555 if (ret < 0)
4556 goto out;
4557 warnings += ret;
4558 }
4559
2daf7fab 4560 if (opts.ibt) {
89bc853e
PZ		 4561 ret = create_ibt_endbr_seal_sections(file);
4562 if (ret < 0)
4563 goto out;
4564 warnings += ret;
4565 }
4566
b51277eb
JP		 4567 if (opts.orc && !list_empty(&file->insn_list)) {
4568 ret = orc_create(file);
4569 if (ret < 0)
4570 goto out;
4571 warnings += ret;
4572 }
4573
4574
2daf7fab 4575 if (opts.stats) {
8b946cc3
PZ		 4576 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4577 printf("nr_cfi: %ld\n", nr_cfi);
4578 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4579 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4580 }
4581
dcc914f4 4582out:
655cf865
JP		 4583 /*
4584 * For now, don't fail the kernel build on fatal warnings. These
4585 * errors are still fairly common due to the growing matrix of
4586 * supported toolchains and their recent pace of change.
4587 */
dcc914f4
JP		 4588 return 0;
4589}
Linux 6.x block layer and io_uring tree(s)