projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sched, kcsan: Enable memory barrier instrumentation
[linux-block.git] / tools / objtool / check.c
CommitLineData
1ccea77e 1// SPDX-License-Identifier: GPL-2.0-or-later
dcc914f4
JP		 2/*
3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
dcc914f4
JP		 4 */
5
6#include <string.h>
7#include <stdlib.h>
8b946cc3 8#include <sys/mman.h>
dcc914f4 9
7786032e
VG		 10#include <arch/elf.h>
11#include <objtool/builtin.h>
12#include <objtool/cfi.h>
13#include <objtool/arch.h>
14#include <objtool/check.h>
15#include <objtool/special.h>
16#include <objtool/warn.h>
17#include <objtool/endianness.h>
dcc914f4 18
ee819aed 19#include <linux/objtool.h>
dcc914f4
JP		 20#include <linux/hashtable.h>
21#include <linux/kernel.h>
1e7e4788 22#include <linux/static_call_types.h>
dcc914f4 23
dcc914f4
JP		 24struct alternative {
25 struct list_head list;
26 struct instruction *insn;
764eef4b 27 bool skip_orig;
dcc914f4
JP		 28};
29
8b946cc3
PZ		 30static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
31
32static struct cfi_init_state initial_func_cfi;
33static struct cfi_state init_cfi;
34static struct cfi_state func_cfi;
dcc914f4 35
627fce14
JP		 36struct instruction *find_insn(struct objtool_file *file,
37 struct section *sec, unsigned long offset)
dcc914f4
JP		 38{
39 struct instruction *insn;
40
87ecb582 41 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
dcc914f4
JP		 42 if (insn->sec == sec && insn->offset == offset)
43 return insn;
87ecb582 44 }
dcc914f4
JP		 45
46 return NULL;
47}
48
49static struct instruction *next_insn_same_sec(struct objtool_file *file,
50 struct instruction *insn)
51{
52 struct instruction *next = list_next_entry(insn, list);
53
baa41469 54 if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
dcc914f4
JP		 55 return NULL;
56
57 return next;
58}
59
13810435
JP		 60static struct instruction *next_insn_same_func(struct objtool_file *file,
61 struct instruction *insn)
62{
63 struct instruction *next = list_next_entry(insn, list);
64 struct symbol *func = insn->func;
65
66 if (!func)
67 return NULL;
68
69 if (&next->list != &file->insn_list && next->func == func)
70 return next;
71
72 /* Check if we're already in the subfunction: */
73 if (func == func->cfunc)
74 return NULL;
75
76 /* Move to the subfunction: */
77 return find_insn(file, func->cfunc->sec, func->cfunc->offset);
78}
79
1119d265
JP		 80static struct instruction *prev_insn_same_sym(struct objtool_file *file,
81 struct instruction *insn)
82{
83 struct instruction *prev = list_prev_entry(insn, list);
84
85 if (&prev->list != &file->insn_list && prev->func == insn->func)
86 return prev;
87
88 return NULL;
89}
90
f0f70adb 91#define func_for_each_insn(file, func, insn) \
13810435
JP		 92 for (insn = find_insn(file, func->sec, func->offset); \
93 insn; \
94 insn = next_insn_same_func(file, insn))
95
dbf4aeb0
PZ		 96#define sym_for_each_insn(file, sym, insn) \
97 for (insn = find_insn(file, sym->sec, sym->offset); \
dcc914f4 98 insn && &insn->list != &file->insn_list && \
dbf4aeb0
PZ		 99 insn->sec == sym->sec && \
100 insn->offset < sym->offset + sym->len; \
dcc914f4
JP		 101 insn = list_next_entry(insn, list))
102
dbf4aeb0 103#define sym_for_each_insn_continue_reverse(file, sym, insn) \
dcc914f4
JP		 104 for (insn = list_prev_entry(insn, list); \
105 &insn->list != &file->insn_list && \
dbf4aeb0 106 insn->sec == sym->sec && insn->offset >= sym->offset; \
dcc914f4
JP		 107 insn = list_prev_entry(insn, list))
108
109#define sec_for_each_insn_from(file, insn) \
110 for (; insn; insn = next_insn_same_sec(file, insn))
111
baa41469
JP		 112#define sec_for_each_insn_continue(file, insn) \
113 for (insn = next_insn_same_sec(file, insn); insn; \
114 insn = next_insn_same_sec(file, insn))
dcc914f4 115
99033461
JP		 116static bool is_jump_table_jump(struct instruction *insn)
117{
118 struct alt_group *alt_group = insn->alt_group;
119
120 if (insn->jump_table)
121 return true;
122
123 /* Retpoline alternative for a jump table? */
124 return alt_group && alt_group->orig_group &&
125 alt_group->orig_group->first_insn->jump_table;
126}
127
0c1ddd33
JP		 128static bool is_sibling_call(struct instruction *insn)
129{
ecf11ba4
JP		 130 /*
131 * Assume only ELF functions can make sibling calls. This ensures
132 * sibling call detection consistency between vmlinux.o and individual
133 * objects.
134 */
135 if (!insn->func)
136 return false;
137
0c1ddd33
JP		 138 /* An indirect jump is either a sibling call or a jump to a table. */
139 if (insn->type == INSN_JUMP_DYNAMIC)
99033461 140 return !is_jump_table_jump(insn);
0c1ddd33 141
0c1ddd33 142 /* add_jump_destinations() sets insn->call_dest for sibling calls. */
ecf11ba4 143 return (is_static_jump(insn) && insn->call_dest);
0c1ddd33
JP		 144}
145
dcc914f4
JP		 146/*
147 * This checks to see if the given function is a "noreturn" function.
148 *
149 * For global functions which are outside the scope of this object file, we
150 * have to keep a manual list of them.
151 *
152 * For local functions, we have to detect them manually by simply looking for
153 * the lack of a return instruction.
dcc914f4 154 */
8e25c9f8
JP		 155static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
156 int recursion)
dcc914f4
JP		 157{
158 int i;
159 struct instruction *insn;
160 bool empty = true;
161
162 /*
163 * Unfortunately these have to be hard coded because the noreturn
164 * attribute isn't provided in ELF data.
165 */
166 static const char * const global_noreturns[] = {
167 "__stack_chk_fail",
168 "panic",
169 "do_exit",
170 "do_task_dead",
171 "__module_put_and_exit",
172 "complete_and_exit",
dcc914f4
JP		 173 "__reiserfs_panic",
174 "lbug_with_loc",
175 "fortify_panic",
b394d468 176 "usercopy_abort",
684fb246 177 "machine_real_restart",
4fa5ecda 178 "rewind_stack_do_exit",
33adf80f 179 "kunit_try_catch_throw",
c26acfbb 180 "xen_start_kernel",
9af9dcf1 181 "cpu_bringup_and_idle",
dcc914f4
JP		 182 };
183
c9bab22b
JP		 184 if (!func)
185 return false;
186
dcc914f4 187 if (func->bind == STB_WEAK)
8e25c9f8 188 return false;
dcc914f4
JP		 189
190 if (func->bind == STB_GLOBAL)
191 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
192 if (!strcmp(func->name, global_noreturns[i]))
8e25c9f8 193 return true;
dcc914f4 194
13810435 195 if (!func->len)
8e25c9f8 196 return false;
dcc914f4 197
13810435
JP		 198 insn = find_insn(file, func->sec, func->offset);
199 if (!insn->func)
8e25c9f8 200 return false;
13810435 201
f0f70adb 202 func_for_each_insn(file, func, insn) {
dcc914f4
JP		 203 empty = false;
204
205 if (insn->type == INSN_RETURN)
8e25c9f8 206 return false;
dcc914f4
JP		 207 }
208
209 if (empty)
8e25c9f8 210 return false;
dcc914f4
JP		 211
212 /*
213 * A function can have a sibling call instead of a return. In that
214 * case, the function's dead-end status depends on whether the target
215 * of the sibling call returns.
216 */
f0f70adb 217 func_for_each_insn(file, func, insn) {
0c1ddd33 218 if (is_sibling_call(insn)) {
dcc914f4 219 struct instruction *dest = insn->jump_dest;
dcc914f4
JP		 220
221 if (!dest)
222 /* sibling call to another file */
8e25c9f8 223 return false;
dcc914f4 224
0c1ddd33
JP		 225 /* local sibling call */
226 if (recursion == 5) {
227 /*
228 * Infinite recursion: two functions have
229 * sibling calls to each other. This is a very
230 * rare case. It means they aren't dead ends.
231 */
232 return false;
dcc914f4 233 }
dcc914f4 234
0c1ddd33
JP		 235 return __dead_end_function(file, dest->func, recursion+1);
236 }
dcc914f4
JP		 237 }
238
8e25c9f8 239 return true;
dcc914f4
JP		 240}
241
8e25c9f8 242static bool dead_end_function(struct objtool_file *file, struct symbol *func)
dcc914f4
JP		 243{
244 return __dead_end_function(file, func, 0);
245}
246
e7c0219b 247static void init_cfi_state(struct cfi_state *cfi)
baa41469
JP		 248{
249 int i;
250
dd88a0a0 251 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 252 cfi->regs[i].base = CFI_UNDEFINED;
253 cfi->vals[i].base = CFI_UNDEFINED;
dd88a0a0 254 }
e7c0219b
PZ		 255 cfi->cfa.base = CFI_UNDEFINED;
256 cfi->drap_reg = CFI_UNDEFINED;
257 cfi->drap_offset = -1;
258}
259
932f8e98 260static void init_insn_state(struct insn_state *state, struct section *sec)
e7c0219b
PZ		 261{
262 memset(state, 0, sizeof(*state));
263 init_cfi_state(&state->cfi);
932f8e98
PZ		 264
265 /*
266 * We need the full vmlinux for noinstr validation, otherwise we can
267 * not correctly determine insn->call_dest->sec (external symbols do
268 * not have a section).
269 */
41425ebe 270 if (vmlinux && noinstr && sec)
932f8e98 271 state->noinstr = sec->noinstr;
baa41469
JP		 272}
273
8b946cc3
PZ		 274static struct cfi_state *cfi_alloc(void)
275{
276 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
277 if (!cfi) {
278 WARN("calloc failed");
279 exit(1);
280 }
281 nr_cfi++;
282 return cfi;
283}
284
285static int cfi_bits;
286static struct hlist_head *cfi_hash;
287
288static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
289{
290 return memcmp((void *)cfi1 + sizeof(cfi1->hash),
291 (void *)cfi2 + sizeof(cfi2->hash),
292 sizeof(struct cfi_state) - sizeof(struct hlist_node));
293}
294
295static inline u32 cfi_key(struct cfi_state *cfi)
296{
297 return jhash((void *)cfi + sizeof(cfi->hash),
298 sizeof(*cfi) - sizeof(cfi->hash), 0);
299}
300
301static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
302{
303 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
304 struct cfi_state *obj;
305
306 hlist_for_each_entry(obj, head, hash) {
307 if (!cficmp(cfi, obj)) {
308 nr_cfi_cache++;
309 return obj;
310 }
311 }
312
313 obj = cfi_alloc();
314 *obj = *cfi;
315 hlist_add_head(&obj->hash, head);
316
317 return obj;
318}
319
320static void cfi_hash_add(struct cfi_state *cfi)
321{
322 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
323
324 hlist_add_head(&cfi->hash, head);
325}
326
327static void *cfi_hash_alloc(unsigned long size)
328{
329 cfi_bits = max(10, ilog2(size));
330 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
331 PROT_READ|PROT_WRITE,
332 MAP_PRIVATE|MAP_ANON, -1, 0);
333 if (cfi_hash == (void *)-1L) {
334 WARN("mmap fail cfi_hash");
335 cfi_hash = NULL;
336 } else if (stats) {
337 printf("cfi_bits: %d\n", cfi_bits);
338 }
339
340 return cfi_hash;
341}
342
343static unsigned long nr_insns;
344static unsigned long nr_insns_visited;
345
dcc914f4
JP		 346/*
347 * Call the arch-specific instruction decoder for all the instructions and add
348 * them to the global instruction list.
349 */
350static int decode_instructions(struct objtool_file *file)
351{
352 struct section *sec;
353 struct symbol *func;
354 unsigned long offset;
355 struct instruction *insn;
356 int ret;
357
baa41469 358 for_each_sec(file, sec) {
dcc914f4
JP		 359
360 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
361 continue;
362
627fce14
JP		 363 if (strcmp(sec->name, ".altinstr_replacement") &&
364 strcmp(sec->name, ".altinstr_aux") &&
365 strncmp(sec->name, ".discard.", 9))
366 sec->text = true;
367
0cc9ac8d
TG		 368 if (!strcmp(sec->name, ".noinstr.text") ||
369 !strcmp(sec->name, ".entry.text"))
c4a33939
PZ		 370 sec->noinstr = true;
371
fe255fe6 372 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
dcc914f4 373 insn = malloc(sizeof(*insn));
baa41469
JP		 374 if (!insn) {
375 WARN("malloc failed");
376 return -1;
377 }
dcc914f4 378 memset(insn, 0, sizeof(*insn));
dcc914f4 379 INIT_LIST_HEAD(&insn->alts);
65ea47dc 380 INIT_LIST_HEAD(&insn->stack_ops);
baa41469 381
dcc914f4
JP		 382 insn->sec = sec;
383 insn->offset = offset;
384
db2b0c5d 385 ret = arch_decode_instruction(file, sec, offset,
fe255fe6 386 sec->sh.sh_size - offset,
dcc914f4 387 &insn->len, &insn->type,
baa41469 388 &insn->immediate,
65ea47dc 389 &insn->stack_ops);
dcc914f4 390 if (ret)
b7037983 391 goto err;
dcc914f4 392
87ecb582 393 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
dcc914f4 394 list_add_tail(&insn->list, &file->insn_list);
1e11f3fd 395 nr_insns++;
dcc914f4
JP		 396 }
397
398 list_for_each_entry(func, &sec->symbol_list, list) {
e10cd8fe 399 if (func->type != STT_FUNC || func->alias != func)
dcc914f4
JP		 400 continue;
401
402 if (!find_insn(file, sec, func->offset)) {
403 WARN("%s(): can't find starting instruction",
404 func->name);
405 return -1;
406 }
407
dbf4aeb0 408 sym_for_each_insn(file, func, insn)
e10cd8fe 409 insn->func = func;
dcc914f4
JP		 410 }
411 }
412
1e11f3fd
PZ		 413 if (stats)
414 printf("nr_insns: %lu\n", nr_insns);
415
dcc914f4 416 return 0;
b7037983
KB		 417
418err:
419 free(insn);
420 return ret;
dcc914f4
JP		 421}
422
db2b0c5d
PZ		 423/*
424 * Read the pv_ops[] .data table to find the static initialized values.
425 */
426static int add_pv_ops(struct objtool_file *file, const char *symname)
427{
428 struct symbol *sym, *func;
429 unsigned long off, end;
430 struct reloc *rel;
431 int idx;
432
433 sym = find_symbol_by_name(file->elf, symname);
434 if (!sym)
435 return 0;
436
437 off = sym->offset;
438 end = off + sym->len;
439 for (;;) {
440 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
441 if (!rel)
442 break;
443
444 func = rel->sym;
445 if (func->type == STT_SECTION)
446 func = find_symbol_by_offset(rel->sym->sec, rel->addend);
447
448 idx = (rel->offset - sym->offset) / sizeof(unsigned long);
449
450 objtool_pv_add(file, idx, func);
451
452 off = rel->offset + 1;
453 if (off > end)
454 break;
455 }
456
457 return 0;
458}
459
460/*
461 * Allocate and initialize file->pv_ops[].
462 */
463static int init_pv_ops(struct objtool_file *file)
464{
465 static const char *pv_ops_tables[] = {
466 "pv_ops",
467 "xen_cpu_ops",
468 "xen_irq_ops",
469 "xen_mmu_ops",
470 NULL,
471 };
472 const char *pv_ops;
473 struct symbol *sym;
474 int idx, nr;
475
476 if (!noinstr)
477 return 0;
478
479 file->pv_ops = NULL;
480
481 sym = find_symbol_by_name(file->elf, "pv_ops");
482 if (!sym)
483 return 0;
484
485 nr = sym->len / sizeof(unsigned long);
486 file->pv_ops = calloc(sizeof(struct pv_state), nr);
487 if (!file->pv_ops)
488 return -1;
489
490 for (idx = 0; idx < nr; idx++)
491 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
492
493 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
494 add_pv_ops(file, pv_ops);
495
496 return 0;
497}
498
6b5dd716
ST		 499static struct instruction *find_last_insn(struct objtool_file *file,
500 struct section *sec)
501{
502 struct instruction *insn = NULL;
503 unsigned int offset;
fe255fe6 504 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
6b5dd716 505
fe255fe6 506 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
6b5dd716
ST		 507 insn = find_insn(file, sec, offset);
508
509 return insn;
510}
511
dcc914f4 512/*
649ea4d5 513 * Mark "ud2" instructions and manually annotated dead ends.
dcc914f4
JP		 514 */
515static int add_dead_ends(struct objtool_file *file)
516{
517 struct section *sec;
f1974222 518 struct reloc *reloc;
dcc914f4 519 struct instruction *insn;
dcc914f4 520
649ea4d5
JP		 521 /*
522 * By default, "ud2" is a dead end unless otherwise annotated, because
523 * GCC 7 inserts it for certain divide-by-zero cases.
524 */
525 for_each_insn(file, insn)
526 if (insn->type == INSN_BUG)
527 insn->dead_end = true;
528
529 /*
530 * Check for manually annotated dead ends.
531 */
dcc914f4
JP		 532 sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
533 if (!sec)
649ea4d5 534 goto reachable;
dcc914f4 535
f1974222
MH		 536 list_for_each_entry(reloc, &sec->reloc_list, list) {
537 if (reloc->sym->type != STT_SECTION) {
dcc914f4
JP		 538 WARN("unexpected relocation symbol type in %s", sec->name);
539 return -1;
540 }
f1974222 541 insn = find_insn(file, reloc->sym->sec, reloc->addend);
dcc914f4
JP		 542 if (insn)
543 insn = list_prev_entry(insn, list);
fe255fe6 544 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 545 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 546 if (!insn) {
dcc914f4 547 WARN("can't find unreachable insn at %s+0x%x",
f1974222 548 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 549 return -1;
550 }
551 } else {
552 WARN("can't find unreachable insn at %s+0x%x",
f1974222 553 reloc->sym->sec->name, reloc->addend);
dcc914f4
JP		 554 return -1;
555 }
556
557 insn->dead_end = true;
558 }
559
649ea4d5
JP		 560reachable:
561 /*
562 * These manually annotated reachable checks are needed for GCC 4.4,
563 * where the Linux unreachable() macro isn't supported. In that case
564 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
565 * not a dead end.
566 */
567 sec = find_section_by_name(file->elf, ".rela.discard.reachable");
568 if (!sec)
569 return 0;
570
f1974222
MH		 571 list_for_each_entry(reloc, &sec->reloc_list, list) {
572 if (reloc->sym->type != STT_SECTION) {
649ea4d5
JP		 573 WARN("unexpected relocation symbol type in %s", sec->name);
574 return -1;
575 }
f1974222 576 insn = find_insn(file, reloc->sym->sec, reloc->addend);
649ea4d5
JP		 577 if (insn)
578 insn = list_prev_entry(insn, list);
fe255fe6 579 else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
f1974222 580 insn = find_last_insn(file, reloc->sym->sec);
6b5dd716 581 if (!insn) {
649ea4d5 582 WARN("can't find reachable insn at %s+0x%x",
f1974222 583 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 584 return -1;
585 }
586 } else {
587 WARN("can't find reachable insn at %s+0x%x",
f1974222 588 reloc->sym->sec->name, reloc->addend);
649ea4d5
JP		 589 return -1;
590 }
591
592 insn->dead_end = false;
593 }
594
dcc914f4
JP		 595 return 0;
596}
597
1e7e4788
JP		 598static int create_static_call_sections(struct objtool_file *file)
599{
ef47cc01 600 struct section *sec;
1e7e4788
JP		 601 struct static_call_site *site;
602 struct instruction *insn;
603 struct symbol *key_sym;
604 char *key_name, *tmp;
605 int idx;
606
607 sec = find_section_by_name(file->elf, ".static_call_sites");
608 if (sec) {
609 INIT_LIST_HEAD(&file->static_call_list);
610 WARN("file already has .static_call_sites section, skipping");
611 return 0;
612 }
613
614 if (list_empty(&file->static_call_list))
615 return 0;
616
617 idx = 0;
43d5430a 618 list_for_each_entry(insn, &file->static_call_list, call_node)
1e7e4788
JP		 619 idx++;
620
621 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
622 sizeof(struct static_call_site), idx);
623 if (!sec)
624 return -1;
625
1e7e4788 626 idx = 0;
43d5430a 627 list_for_each_entry(insn, &file->static_call_list, call_node) {
1e7e4788
JP		 628
629 site = (struct static_call_site *)sec->data->d_buf + idx;
630 memset(site, 0, sizeof(struct static_call_site));
631
632 /* populate reloc for 'addr' */
ef47cc01
PZ		 633 if (elf_add_reloc_to_insn(file->elf, sec,
634 idx * sizeof(struct static_call_site),
635 R_X86_64_PC32,
636 insn->sec, insn->offset))
44f6a7c0 637 return -1;
1e7e4788
JP		 638
639 /* find key symbol */
640 key_name = strdup(insn->call_dest->name);
641 if (!key_name) {
642 perror("strdup");
643 return -1;
644 }
645 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
646 STATIC_CALL_TRAMP_PREFIX_LEN)) {
647 WARN("static_call: trampoline name malformed: %s", key_name);
648 return -1;
649 }
650 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
651 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
652
653 key_sym = find_symbol_by_name(file->elf, tmp);
654 if (!key_sym) {
73f44fe1
JP		 655 if (!module) {
656 WARN("static_call: can't find static_call_key symbol: %s", tmp);
657 return -1;
658 }
659
660 /*
661 * For modules(), the key might not be exported, which
662 * means the module can make static calls but isn't
663 * allowed to change them.
664 *
665 * In that case we temporarily set the key to be the
666 * trampoline address. This is fixed up in
667 * static_call_add_module().
668 */
669 key_sym = insn->call_dest;
1e7e4788
JP		 670 }
671 free(key_name);
672
673 /* populate reloc for 'key' */
ef47cc01
PZ		 674 if (elf_add_reloc(file->elf, sec,
675 idx * sizeof(struct static_call_site) + 4,
676 R_X86_64_PC32, key_sym,
677 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
1e7e4788 678 return -1;
1e7e4788
JP		 679
680 idx++;
681 }
682
1e7e4788
JP		 683 return 0;
684}
685
134ab5bd
PZ		 686static int create_retpoline_sites_sections(struct objtool_file *file)
687{
688 struct instruction *insn;
689 struct section *sec;
690 int idx;
691
692 sec = find_section_by_name(file->elf, ".retpoline_sites");
693 if (sec) {
694 WARN("file already has .retpoline_sites, skipping");
695 return 0;
696 }
697
698 idx = 0;
699 list_for_each_entry(insn, &file->retpoline_call_list, call_node)
700 idx++;
701
702 if (!idx)
703 return 0;
704
705 sec = elf_create_section(file->elf, ".retpoline_sites", 0,
706 sizeof(int), idx);
707 if (!sec) {
708 WARN("elf_create_section: .retpoline_sites");
709 return -1;
710 }
711
712 idx = 0;
713 list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
714
715 int *site = (int *)sec->data->d_buf + idx;
716 *site = 0;
717
718 if (elf_add_reloc_to_insn(file->elf, sec,
719 idx * sizeof(int),
720 R_X86_64_PC32,
721 insn->sec, insn->offset)) {
722 WARN("elf_add_reloc_to_insn: .retpoline_sites");
723 return -1;
724 }
725
726 idx++;
727 }
728
729 return 0;
730}
731
99d00215
PZ		 732static int create_mcount_loc_sections(struct objtool_file *file)
733{
ef47cc01 734 struct section *sec;
99d00215
PZ		 735 unsigned long *loc;
736 struct instruction *insn;
737 int idx;
738
739 sec = find_section_by_name(file->elf, "__mcount_loc");
740 if (sec) {
741 INIT_LIST_HEAD(&file->mcount_loc_list);
742 WARN("file already has __mcount_loc section, skipping");
743 return 0;
744 }
745
746 if (list_empty(&file->mcount_loc_list))
747 return 0;
748
749 idx = 0;
c509331b 750 list_for_each_entry(insn, &file->mcount_loc_list, call_node)
99d00215
PZ		 751 idx++;
752
753 sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
754 if (!sec)
755 return -1;
756
99d00215 757 idx = 0;
c509331b 758 list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
99d00215
PZ		 759
760 loc = (unsigned long *)sec->data->d_buf + idx;
761 memset(loc, 0, sizeof(unsigned long));
762
ef47cc01
PZ		 763 if (elf_add_reloc_to_insn(file->elf, sec,
764 idx * sizeof(unsigned long),
765 R_X86_64_64,
766 insn->sec, insn->offset))
99d00215 767 return -1;
99d00215
PZ		 768
769 idx++;
770 }
771
99d00215
PZ		 772 return 0;
773}
774
dcc914f4
JP		 775/*
776 * Warnings shouldn't be reported for ignored functions.
777 */
778static void add_ignores(struct objtool_file *file)
779{
780 struct instruction *insn;
781 struct section *sec;
782 struct symbol *func;
f1974222 783 struct reloc *reloc;
dcc914f4 784
aaf5c623
PZ		 785 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
786 if (!sec)
787 return;
dcc914f4 788
f1974222
MH		 789 list_for_each_entry(reloc, &sec->reloc_list, list) {
790 switch (reloc->sym->type) {
aaf5c623 791 case STT_FUNC:
f1974222 792 func = reloc->sym;
aaf5c623
PZ		 793 break;
794
795 case STT_SECTION:
f1974222 796 func = find_func_by_offset(reloc->sym->sec, reloc->addend);
7acfe531 797 if (!func)
dcc914f4 798 continue;
aaf5c623 799 break;
dcc914f4 800
aaf5c623 801 default:
f1974222 802 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
aaf5c623 803 continue;
dcc914f4 804 }
aaf5c623 805
f0f70adb 806 func_for_each_insn(file, func, insn)
aaf5c623 807 insn->ignore = true;
dcc914f4
JP		 808 }
809}
810
ea24213d
PZ		 811/*
812 * This is a whitelist of functions that is allowed to be called with AC set.
813 * The list is meant to be minimal and only contains compiler instrumentation
814 * ABI and a few functions used to implement *_{to,from}_user() functions.
815 *
816 * These functions must not directly change AC, but may PUSHF/POPF.
817 */
818static const char *uaccess_safe_builtin[] = {
819 /* KASAN */
820 "kasan_report",
f00748bf 821 "kasan_check_range",
ea24213d
PZ		 822 /* KASAN out-of-line */
823 "__asan_loadN_noabort",
824 "__asan_load1_noabort",
825 "__asan_load2_noabort",
826 "__asan_load4_noabort",
827 "__asan_load8_noabort",
828 "__asan_load16_noabort",
829 "__asan_storeN_noabort",
830 "__asan_store1_noabort",
831 "__asan_store2_noabort",
832 "__asan_store4_noabort",
833 "__asan_store8_noabort",
834 "__asan_store16_noabort",
b0b8e56b
JH		 835 "__kasan_check_read",
836 "__kasan_check_write",
ea24213d
PZ		 837 /* KASAN in-line */
838 "__asan_report_load_n_noabort",
839 "__asan_report_load1_noabort",
840 "__asan_report_load2_noabort",
841 "__asan_report_load4_noabort",
842 "__asan_report_load8_noabort",
843 "__asan_report_load16_noabort",
844 "__asan_report_store_n_noabort",
845 "__asan_report_store1_noabort",
846 "__asan_report_store2_noabort",
847 "__asan_report_store4_noabort",
848 "__asan_report_store8_noabort",
849 "__asan_report_store16_noabort",
5f5c9712 850 /* KCSAN */
9967683c 851 "__kcsan_check_access",
5f5c9712
ME		 852 "kcsan_found_watchpoint",
853 "kcsan_setup_watchpoint",
9967683c 854 "kcsan_check_scoped_accesses",
50a19ad4
ME		 855 "kcsan_disable_current",
856 "kcsan_enable_current_nowarn",
5f5c9712
ME		 857 /* KCSAN/TSAN */
858 "__tsan_func_entry",
859 "__tsan_func_exit",
860 "__tsan_read_range",
861 "__tsan_write_range",
862 "__tsan_read1",
863 "__tsan_read2",
864 "__tsan_read4",
865 "__tsan_read8",
866 "__tsan_read16",
867 "__tsan_write1",
868 "__tsan_write2",
869 "__tsan_write4",
870 "__tsan_write8",
871 "__tsan_write16",
a81b3759
ME		 872 "__tsan_read_write1",
873 "__tsan_read_write2",
874 "__tsan_read_write4",
875 "__tsan_read_write8",
876 "__tsan_read_write16",
883957b1
ME		 877 "__tsan_atomic8_load",
878 "__tsan_atomic16_load",
879 "__tsan_atomic32_load",
880 "__tsan_atomic64_load",
881 "__tsan_atomic8_store",
882 "__tsan_atomic16_store",
883 "__tsan_atomic32_store",
884 "__tsan_atomic64_store",
885 "__tsan_atomic8_exchange",
886 "__tsan_atomic16_exchange",
887 "__tsan_atomic32_exchange",
888 "__tsan_atomic64_exchange",
889 "__tsan_atomic8_fetch_add",
890 "__tsan_atomic16_fetch_add",
891 "__tsan_atomic32_fetch_add",
892 "__tsan_atomic64_fetch_add",
893 "__tsan_atomic8_fetch_sub",
894 "__tsan_atomic16_fetch_sub",
895 "__tsan_atomic32_fetch_sub",
896 "__tsan_atomic64_fetch_sub",
897 "__tsan_atomic8_fetch_and",
898 "__tsan_atomic16_fetch_and",
899 "__tsan_atomic32_fetch_and",
900 "__tsan_atomic64_fetch_and",
901 "__tsan_atomic8_fetch_or",
902 "__tsan_atomic16_fetch_or",
903 "__tsan_atomic32_fetch_or",
904 "__tsan_atomic64_fetch_or",
905 "__tsan_atomic8_fetch_xor",
906 "__tsan_atomic16_fetch_xor",
907 "__tsan_atomic32_fetch_xor",
908 "__tsan_atomic64_fetch_xor",
909 "__tsan_atomic8_fetch_nand",
910 "__tsan_atomic16_fetch_nand",
911 "__tsan_atomic32_fetch_nand",
912 "__tsan_atomic64_fetch_nand",
913 "__tsan_atomic8_compare_exchange_strong",
914 "__tsan_atomic16_compare_exchange_strong",
915 "__tsan_atomic32_compare_exchange_strong",
916 "__tsan_atomic64_compare_exchange_strong",
917 "__tsan_atomic8_compare_exchange_weak",
918 "__tsan_atomic16_compare_exchange_weak",
919 "__tsan_atomic32_compare_exchange_weak",
920 "__tsan_atomic64_compare_exchange_weak",
921 "__tsan_atomic8_compare_exchange_val",
922 "__tsan_atomic16_compare_exchange_val",
923 "__tsan_atomic32_compare_exchange_val",
924 "__tsan_atomic64_compare_exchange_val",
925 "__tsan_atomic_thread_fence",
926 "__tsan_atomic_signal_fence",
ea24213d
PZ		 927 /* KCOV */
928 "write_comp_data",
ae033f08 929 "check_kcov_mode",
ea24213d
PZ		 930 "__sanitizer_cov_trace_pc",
931 "__sanitizer_cov_trace_const_cmp1",
932 "__sanitizer_cov_trace_const_cmp2",
933 "__sanitizer_cov_trace_const_cmp4",
934 "__sanitizer_cov_trace_const_cmp8",
935 "__sanitizer_cov_trace_cmp1",
936 "__sanitizer_cov_trace_cmp2",
937 "__sanitizer_cov_trace_cmp4",
938 "__sanitizer_cov_trace_cmp8",
36b1c700 939 "__sanitizer_cov_trace_switch",
ea24213d
PZ		 940 /* UBSAN */
941 "ubsan_type_mismatch_common",
942 "__ubsan_handle_type_mismatch",
943 "__ubsan_handle_type_mismatch_v1",
9a50dcaf 944 "__ubsan_handle_shift_out_of_bounds",
ea24213d
PZ		 945 /* misc */
946 "csum_partial_copy_generic",
ec6347bb
DW		 947 "copy_mc_fragile",
948 "copy_mc_fragile_handle_tail",
5da8e4a6 949 "copy_mc_enhanced_fast_string",
ea24213d
PZ		 950 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
951 NULL
952};
953
954static void add_uaccess_safe(struct objtool_file *file)
955{
956 struct symbol *func;
957 const char **name;
958
959 if (!uaccess)
960 return;
961
962 for (name = uaccess_safe_builtin; *name; name++) {
963 func = find_symbol_by_name(file->elf, *name);
964 if (!func)
965 continue;
966
e10cd8fe 967 func->uaccess_safe = true;
dcc914f4
JP		 968 }
969}
970
258c7605
JP		 971/*
972 * FIXME: For now, just ignore any alternatives which add retpolines. This is
973 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
974 * But it at least allows objtool to understand the control flow *around* the
975 * retpoline.
976 */
ff05ab23 977static int add_ignore_alternatives(struct objtool_file *file)
258c7605
JP		 978{
979 struct section *sec;
f1974222 980 struct reloc *reloc;
258c7605
JP		 981 struct instruction *insn;
982
ff05ab23 983 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
258c7605
JP		 984 if (!sec)
985 return 0;
986
f1974222
MH		 987 list_for_each_entry(reloc, &sec->reloc_list, list) {
988 if (reloc->sym->type != STT_SECTION) {
258c7605
JP		 989 WARN("unexpected relocation symbol type in %s", sec->name);
990 return -1;
991 }
992
f1974222 993 insn = find_insn(file, reloc->sym->sec, reloc->addend);
258c7605 994 if (!insn) {
ff05ab23 995 WARN("bad .discard.ignore_alts entry");
258c7605
JP		 996 return -1;
997 }
998
999 insn->ignore_alts = true;
1000 }
1001
1002 return 0;
1003}
1004
530b4ddd
PZ		 1005__weak bool arch_is_retpoline(struct symbol *sym)
1006{
1007 return false;
1008}
1009
7bd2a600
PZ		 1010#define NEGATIVE_RELOC ((void *)-1L)
1011
1012static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1013{
1014 if (insn->reloc == NEGATIVE_RELOC)
1015 return NULL;
1016
1017 if (!insn->reloc) {
db2b0c5d
PZ		 1018 if (!file)
1019 return NULL;
1020
7bd2a600
PZ		 1021 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1022 insn->offset, insn->len);
1023 if (!insn->reloc) {
1024 insn->reloc = NEGATIVE_RELOC;
1025 return NULL;
1026 }
1027 }
1028
1029 return insn->reloc;
1030}
1031
f56dae88
PZ		 1032static void remove_insn_ops(struct instruction *insn)
1033{
1034 struct stack_op *op, *tmp;
1035
1036 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1037 list_del(&op->list);
1038 free(op);
1039 }
1040}
1041
dd003ede
PZ		 1042static void annotate_call_site(struct objtool_file *file,
1043 struct instruction *insn, bool sibling)
f56dae88
PZ		 1044{
1045 struct reloc *reloc = insn_reloc(file, insn);
dd003ede 1046 struct symbol *sym = insn->call_dest;
f56dae88 1047
dd003ede
PZ		 1048 if (!sym)
1049 sym = reloc->sym;
1050
1051 /*
1052 * Alternative replacement code is just template code which is
1053 * sometimes copied to the original instruction. For now, don't
1054 * annotate it. (In the future we might consider annotating the
1055 * original instruction if/when it ever makes sense to do so.)
1056 */
1057 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
f56dae88
PZ		 1058 return;
1059
dd003ede
PZ		 1060 if (sym->static_call_tramp) {
1061 list_add_tail(&insn->call_node, &file->static_call_list);
1062 return;
f56dae88
PZ		 1063 }
1064
134ab5bd
PZ		 1065 if (sym->retpoline_thunk) {
1066 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1067 return;
1068 }
1069
f56dae88
PZ		 1070 /*
1071 * Many compilers cannot disable KCOV with a function attribute
1072 * so they need a little help, NOP out any KCOV calls from noinstr
1073 * text.
1074 */
dd003ede 1075 if (insn->sec->noinstr && sym->kcov) {
f56dae88
PZ		 1076 if (reloc) {
1077 reloc->type = R_NONE;
1078 elf_write_reloc(file->elf, reloc);
1079 }
1080
1081 elf_write_insn(file->elf, insn->sec,
1082 insn->offset, insn->len,
1083 sibling ? arch_ret_insn(insn->len)
1084 : arch_nop_insn(insn->len));
1085
1086 insn->type = sibling ? INSN_RETURN : INSN_NOP;
dd003ede 1087 return;
f56dae88
PZ		 1088 }
1089
dd003ede 1090 if (mcount && sym->fentry) {
f56dae88
PZ		 1091 if (sibling)
1092 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1093
1094 if (reloc) {
1095 reloc->type = R_NONE;
1096 elf_write_reloc(file->elf, reloc);
1097 }
1098
1099 elf_write_insn(file->elf, insn->sec,
1100 insn->offset, insn->len,
1101 arch_nop_insn(insn->len));
1102
1103 insn->type = INSN_NOP;
1104
c509331b 1105 list_add_tail(&insn->call_node, &file->mcount_loc_list);
dd003ede 1106 return;
f56dae88 1107 }
dd003ede
PZ		 1108}
1109
1110static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1111 struct symbol *dest, bool sibling)
1112{
1113 insn->call_dest = dest;
1114 if (!dest)
1115 return;
f56dae88
PZ		 1116
1117 /*
1118 * Whatever stack impact regular CALLs have, should be undone
1119 * by the RETURN of the called function.
1120 *
1121 * Annotated intra-function calls retain the stack_ops but
1122 * are converted to JUMP, see read_intra_function_calls().
1123 */
1124 remove_insn_ops(insn);
dd003ede
PZ		 1125
1126 annotate_call_site(file, insn, sibling);
f56dae88
PZ		 1127}
1128
134ab5bd
PZ		 1129static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1130{
1131 /*
1132 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1133 * so convert them accordingly.
1134 */
1135 switch (insn->type) {
1136 case INSN_CALL:
1137 insn->type = INSN_CALL_DYNAMIC;
1138 break;
1139 case INSN_JUMP_UNCONDITIONAL:
1140 insn->type = INSN_JUMP_DYNAMIC;
1141 break;
1142 case INSN_JUMP_CONDITIONAL:
1143 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1144 break;
1145 default:
1146 return;
1147 }
1148
1149 insn->retpoline_safe = true;
1150
1151 /*
1152 * Whatever stack impact regular CALLs have, should be undone
1153 * by the RETURN of the called function.
1154 *
1155 * Annotated intra-function calls retain the stack_ops but
1156 * are converted to JUMP, see read_intra_function_calls().
1157 */
1158 remove_insn_ops(insn);
1159
1160 annotate_call_site(file, insn, false);
1161}
dcc914f4
JP		 1162/*
1163 * Find the destination instructions for all jumps.
1164 */
1165static int add_jump_destinations(struct objtool_file *file)
1166{
1167 struct instruction *insn;
f1974222 1168 struct reloc *reloc;
dcc914f4
JP		 1169 struct section *dest_sec;
1170 unsigned long dest_off;
1171
1172 for_each_insn(file, insn) {
a2296140 1173 if (!is_static_jump(insn))
dcc914f4
JP		 1174 continue;
1175
7bd2a600 1176 reloc = insn_reloc(file, insn);
f1974222 1177 if (!reloc) {
dcc914f4 1178 dest_sec = insn->sec;
bfb08f22 1179 dest_off = arch_jump_destination(insn);
f1974222
MH		 1180 } else if (reloc->sym->type == STT_SECTION) {
1181 dest_sec = reloc->sym->sec;
1182 dest_off = arch_dest_reloc_offset(reloc->addend);
1739c66e 1183 } else if (reloc->sym->retpoline_thunk) {
134ab5bd 1184 add_retpoline_call(file, insn);
39b73533 1185 continue;
ecf11ba4
JP		 1186 } else if (insn->func) {
1187 /* internal or external sibling call (with reloc) */
f56dae88 1188 add_call_dest(file, insn, reloc->sym, true);
dcc914f4 1189 continue;
ecf11ba4
JP		 1190 } else if (reloc->sym->sec->idx) {
1191 dest_sec = reloc->sym->sec;
1192 dest_off = reloc->sym->sym.st_value +
1193 arch_dest_reloc_offset(reloc->addend);
1194 } else {
1195 /* non-func asm code jumping to another file */
1196 continue;
dcc914f4
JP		 1197 }
1198
1199 insn->jump_dest = find_insn(file, dest_sec, dest_off);
1200 if (!insn->jump_dest) {
1201
1202 /*
1203 * This is a special case where an alt instruction
1204 * jumps past the end of the section. These are
1205 * handled later in handle_group_alt().
1206 */
1207 if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1208 continue;
1209
1210 WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1211 insn->sec, insn->offset, dest_sec->name,
1212 dest_off);
1213 return -1;
1214 }
cd77849a
JP		 1215
1216 /*
54262aa2 1217 * Cross-function jump.
cd77849a
JP		 1218 */
1219 if (insn->func && insn->jump_dest->func &&
54262aa2
PZ		 1220 insn->func != insn->jump_dest->func) {
1221
1222 /*
1223 * For GCC 8+, create parent/child links for any cold
1224 * subfunctions. This is _mostly_ redundant with a
1225 * similar initialization in read_symbols().
1226 *
1227 * If a function has aliases, we want the *first* such
1228 * function in the symbol table to be the subfunction's
1229 * parent. In that case we overwrite the
1230 * initialization done in read_symbols().
1231 *
1232 * However this code can't completely replace the
1233 * read_symbols() code because this doesn't detect the
1234 * case where the parent function's only reference to a
e7c2bc37 1235 * subfunction is through a jump table.
54262aa2 1236 */
34ca59e1
JP		 1237 if (!strstr(insn->func->name, ".cold") &&
1238 strstr(insn->jump_dest->func->name, ".cold")) {
54262aa2
PZ		 1239 insn->func->cfunc = insn->jump_dest->func;
1240 insn->jump_dest->func->pfunc = insn->func;
1241
1242 } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
1243 insn->jump_dest->offset == insn->jump_dest->func->offset) {
ecf11ba4 1244 /* internal sibling call (without reloc) */
f56dae88 1245 add_call_dest(file, insn, insn->jump_dest->func, true);
54262aa2 1246 }
cd77849a 1247 }
dcc914f4
JP		 1248 }
1249
1250 return 0;
1251}
1252
2b232a22
JT		 1253static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1254{
1255 struct symbol *call_dest;
1256
1257 call_dest = find_func_by_offset(sec, offset);
1258 if (!call_dest)
1259 call_dest = find_symbol_by_offset(sec, offset);
1260
1261 return call_dest;
1262}
1263
dcc914f4
JP		 1264/*
1265 * Find the destination instructions for all calls.
1266 */
1267static int add_call_destinations(struct objtool_file *file)
1268{
1269 struct instruction *insn;
1270 unsigned long dest_off;
f56dae88 1271 struct symbol *dest;
f1974222 1272 struct reloc *reloc;
dcc914f4
JP		 1273
1274 for_each_insn(file, insn) {
1275 if (insn->type != INSN_CALL)
1276 continue;
1277
7bd2a600 1278 reloc = insn_reloc(file, insn);
f1974222 1279 if (!reloc) {
bfb08f22 1280 dest_off = arch_jump_destination(insn);
f56dae88
PZ		 1281 dest = find_call_destination(insn->sec, dest_off);
1282
1283 add_call_dest(file, insn, dest, false);
a845c7cf 1284
7acfe531
JP		 1285 if (insn->ignore)
1286 continue;
1287
1288 if (!insn->call_dest) {
8aa8eb2a 1289 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
dcc914f4
JP		 1290 return -1;
1291 }
a845c7cf 1292
7acfe531
JP		 1293 if (insn->func && insn->call_dest->type != STT_FUNC) {
1294 WARN_FUNC("unsupported call to non-function",
1295 insn->sec, insn->offset);
1296 return -1;
1297 }
1298
f1974222
MH		 1299 } else if (reloc->sym->type == STT_SECTION) {
1300 dest_off = arch_dest_reloc_offset(reloc->addend);
f56dae88
PZ		 1301 dest = find_call_destination(reloc->sym->sec, dest_off);
1302 if (!dest) {
bfb08f22 1303 WARN_FUNC("can't find call dest symbol at %s+0x%lx",
dcc914f4 1304 insn->sec, insn->offset,
f1974222 1305 reloc->sym->sec->name,
bfb08f22 1306 dest_off);
dcc914f4
JP		 1307 return -1;
1308 }
bcb1b6ff 1309
f56dae88
PZ		 1310 add_call_dest(file, insn, dest, false);
1311
1739c66e 1312 } else if (reloc->sym->retpoline_thunk) {
134ab5bd 1313 add_retpoline_call(file, insn);
bcb1b6ff 1314
dcc914f4 1315 } else
f56dae88 1316 add_call_dest(file, insn, reloc->sym, false);
dcc914f4
JP		 1317 }
1318
1319 return 0;
1320}
1321
1322/*
c9c324dc
JP		 1323 * The .alternatives section requires some extra special care over and above
1324 * other special sections because alternatives are patched in place.
dcc914f4
JP		 1325 */
1326static int handle_group_alt(struct objtool_file *file,
1327 struct special_alt *special_alt,
1328 struct instruction *orig_insn,
1329 struct instruction **new_insn)
1330{
c9c324dc 1331 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
b23cc71c 1332 struct alt_group *orig_alt_group, *new_alt_group;
dcc914f4
JP		 1333 unsigned long dest_off;
1334
b23cc71c
JP		 1335
1336 orig_alt_group = malloc(sizeof(*orig_alt_group));
1337 if (!orig_alt_group) {
1338 WARN("malloc failed");
1339 return -1;
1340 }
c9c324dc
JP		 1341 orig_alt_group->cfi = calloc(special_alt->orig_len,
1342 sizeof(struct cfi_state *));
1343 if (!orig_alt_group->cfi) {
1344 WARN("calloc failed");
1345 return -1;
1346 }
1347
dcc914f4
JP		 1348 last_orig_insn = NULL;
1349 insn = orig_insn;
1350 sec_for_each_insn_from(file, insn) {
1351 if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1352 break;
1353
b23cc71c 1354 insn->alt_group = orig_alt_group;
dcc914f4
JP		 1355 last_orig_insn = insn;
1356 }
b23cc71c
JP		 1357 orig_alt_group->orig_group = NULL;
1358 orig_alt_group->first_insn = orig_insn;
1359 orig_alt_group->last_insn = last_orig_insn;
dcc914f4 1360
17bc3391 1361
c9c324dc
JP		 1362 new_alt_group = malloc(sizeof(*new_alt_group));
1363 if (!new_alt_group) {
1364 WARN("malloc failed");
1365 return -1;
dcc914f4 1366 }
dcc914f4 1367
c9c324dc
JP		 1368 if (special_alt->new_len < special_alt->orig_len) {
1369 /*
1370 * Insert a fake nop at the end to make the replacement
1371 * alt_group the same size as the original. This is needed to
1372 * allow propagate_alt_cfi() to do its magic. When the last
1373 * instruction affects the stack, the instruction after it (the
1374 * nop) will propagate the new state to the shared CFI array.
1375 */
1376 nop = malloc(sizeof(*nop));
1377 if (!nop) {
1378 WARN("malloc failed");
17bc3391
JP		 1379 return -1;
1380 }
c9c324dc
JP		 1381 memset(nop, 0, sizeof(*nop));
1382 INIT_LIST_HEAD(&nop->alts);
1383 INIT_LIST_HEAD(&nop->stack_ops);
c9c324dc
JP		 1384
1385 nop->sec = special_alt->new_sec;
1386 nop->offset = special_alt->new_off + special_alt->new_len;
1387 nop->len = special_alt->orig_len - special_alt->new_len;
1388 nop->type = INSN_NOP;
1389 nop->func = orig_insn->func;
1390 nop->alt_group = new_alt_group;
1391 nop->ignore = orig_insn->ignore_alts;
dcc914f4 1392 }
17bc3391 1393
c9c324dc
JP		 1394 if (!special_alt->new_len) {
1395 *new_insn = nop;
1396 goto end;
dcc914f4
JP		 1397 }
1398
dcc914f4
JP		 1399 insn = *new_insn;
1400 sec_for_each_insn_from(file, insn) {
45245f51
JT		 1401 struct reloc *alt_reloc;
1402
dcc914f4
JP		 1403 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1404 break;
1405
1406 last_new_insn = insn;
1407
a845c7cf 1408 insn->ignore = orig_insn->ignore_alts;
a4d09dde 1409 insn->func = orig_insn->func;
b23cc71c 1410 insn->alt_group = new_alt_group;
a845c7cf 1411
dc419723
JP		 1412 /*
1413 * Since alternative replacement code is copy/pasted by the
1414 * kernel after applying relocations, generally such code can't
1415 * have relative-address relocation references to outside the
1416 * .altinstr_replacement section, unless the arch's
1417 * alternatives code can adjust the relative offsets
1418 * accordingly.
dc419723 1419 */
7bd2a600 1420 alt_reloc = insn_reloc(file, insn);
45245f51
JT		 1421 if (alt_reloc &&
1422 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
dc419723
JP		 1423
1424 WARN_FUNC("unsupported relocation in alternatives section",
1425 insn->sec, insn->offset);
1426 return -1;
1427 }
1428
a2296140 1429 if (!is_static_jump(insn))
dcc914f4
JP		 1430 continue;
1431
1432 if (!insn->immediate)
1433 continue;
1434
bfb08f22 1435 dest_off = arch_jump_destination(insn);
c9c324dc
JP		 1436 if (dest_off == special_alt->new_off + special_alt->new_len)
1437 insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
dcc914f4
JP		 1438
1439 if (!insn->jump_dest) {
1440 WARN_FUNC("can't find alternative jump destination",
1441 insn->sec, insn->offset);
1442 return -1;
1443 }
1444 }
1445
1446 if (!last_new_insn) {
1447 WARN_FUNC("can't find last new alternative instruction",
1448 special_alt->new_sec, special_alt->new_off);
1449 return -1;
1450 }
1451
c9c324dc
JP		 1452 if (nop)
1453 list_add(&nop->list, &last_new_insn->list);
1454end:
b23cc71c
JP		 1455 new_alt_group->orig_group = orig_alt_group;
1456 new_alt_group->first_insn = *new_insn;
c9c324dc
JP		 1457 new_alt_group->last_insn = nop ? : last_new_insn;
1458 new_alt_group->cfi = orig_alt_group->cfi;
dcc914f4
JP		 1459 return 0;
1460}
1461
1462/*
1463 * A jump table entry can either convert a nop to a jump or a jump to a nop.
1464 * If the original instruction is a jump, make the alt entry an effective nop
1465 * by just skipping the original instruction.
1466 */
1467static int handle_jump_alt(struct objtool_file *file,
1468 struct special_alt *special_alt,
1469 struct instruction *orig_insn,
1470 struct instruction **new_insn)
1471{
48001d26
PZ		 1472 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1473 orig_insn->type != INSN_NOP) {
e2d9494b 1474
dcc914f4
JP		 1475 WARN_FUNC("unsupported instruction at jump label",
1476 orig_insn->sec, orig_insn->offset);
1477 return -1;
1478 }
1479
6d37b83c
PZ		 1480 if (special_alt->key_addend & 2) {
1481 struct reloc *reloc = insn_reloc(file, orig_insn);
1482
1483 if (reloc) {
1484 reloc->type = R_NONE;
1485 elf_write_reloc(file->elf, reloc);
1486 }
1487 elf_write_insn(file->elf, orig_insn->sec,
1488 orig_insn->offset, orig_insn->len,
1489 arch_nop_insn(orig_insn->len));
1490 orig_insn->type = INSN_NOP;
48001d26
PZ		 1491 }
1492
1493 if (orig_insn->type == INSN_NOP) {
1494 if (orig_insn->len == 2)
1495 file->jl_nop_short++;
1496 else
1497 file->jl_nop_long++;
1498
1499 return 0;
6d37b83c
PZ		 1500 }
1501
e2d9494b
PZ		 1502 if (orig_insn->len == 2)
1503 file->jl_short++;
1504 else
1505 file->jl_long++;
1506
dcc914f4
JP		 1507 *new_insn = list_next_entry(orig_insn, list);
1508 return 0;
1509}
1510
1511/*
1512 * Read all the special sections which have alternate instructions which can be
1513 * patched in or redirected to at runtime. Each instruction having alternate
1514 * instruction(s) has them added to its insn->alts list, which will be
1515 * traversed in validate_branch().
1516 */
1517static int add_special_section_alts(struct objtool_file *file)
1518{
1519 struct list_head special_alts;
1520 struct instruction *orig_insn, *new_insn;
1521 struct special_alt *special_alt, *tmp;
1522 struct alternative *alt;
1523 int ret;
1524
1525 ret = special_get_alts(file->elf, &special_alts);
1526 if (ret)
1527 return ret;
1528
1529 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
dcc914f4
JP		 1530
1531 orig_insn = find_insn(file, special_alt->orig_sec,
1532 special_alt->orig_off);
1533 if (!orig_insn) {
1534 WARN_FUNC("special: can't find orig instruction",
1535 special_alt->orig_sec, special_alt->orig_off);
1536 ret = -1;
1537 goto out;
1538 }
1539
1540 new_insn = NULL;
1541 if (!special_alt->group || special_alt->new_len) {
1542 new_insn = find_insn(file, special_alt->new_sec,
1543 special_alt->new_off);
1544 if (!new_insn) {
1545 WARN_FUNC("special: can't find new instruction",
1546 special_alt->new_sec,
1547 special_alt->new_off);
1548 ret = -1;
1549 goto out;
1550 }
1551 }
1552
1553 if (special_alt->group) {
7170cf47
JT		 1554 if (!special_alt->orig_len) {
1555 WARN_FUNC("empty alternative entry",
1556 orig_insn->sec, orig_insn->offset);
1557 continue;
1558 }
1559
dcc914f4
JP		 1560 ret = handle_group_alt(file, special_alt, orig_insn,
1561 &new_insn);
1562 if (ret)
1563 goto out;
1564 } else if (special_alt->jump_or_nop) {
1565 ret = handle_jump_alt(file, special_alt, orig_insn,
1566 &new_insn);
1567 if (ret)
1568 goto out;
1569 }
1570
258c7605
JP		 1571 alt = malloc(sizeof(*alt));
1572 if (!alt) {
1573 WARN("malloc failed");
1574 ret = -1;
1575 goto out;
1576 }
1577
dcc914f4 1578 alt->insn = new_insn;
764eef4b 1579 alt->skip_orig = special_alt->skip_orig;
ea24213d 1580 orig_insn->ignore_alts |= special_alt->skip_alt;
dcc914f4
JP		 1581 list_add_tail(&alt->list, &orig_insn->alts);
1582
1583 list_del(&special_alt->list);
1584 free(special_alt);
1585 }
1586
e2d9494b
PZ		 1587 if (stats) {
1588 printf("jl\\\tNOP\tJMP\n");
1589 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1590 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1591 }
1592
dcc914f4
JP		 1593out:
1594 return ret;
1595}
1596
e7c2bc37 1597static int add_jump_table(struct objtool_file *file, struct instruction *insn,
f1974222 1598 struct reloc *table)
dcc914f4 1599{
f1974222 1600 struct reloc *reloc = table;
e7c2bc37 1601 struct instruction *dest_insn;
dcc914f4 1602 struct alternative *alt;
fd35c88b
JP		 1603 struct symbol *pfunc = insn->func->pfunc;
1604 unsigned int prev_offset = 0;
dcc914f4 1605
e7c2bc37 1606 /*
f1974222 1607 * Each @reloc is a switch table relocation which points to the target
e7c2bc37
JP		 1608 * instruction.
1609 */
f1974222 1610 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
bd98c813
JH		 1611
1612 /* Check for the end of the table: */
f1974222 1613 if (reloc != table && reloc->jump_table_start)
dcc914f4
JP		 1614 break;
1615
e7c2bc37 1616 /* Make sure the table entries are consecutive: */
f1974222 1617 if (prev_offset && reloc->offset != prev_offset + 8)
fd35c88b
JP		 1618 break;
1619
1620 /* Detect function pointers from contiguous objects: */
f1974222
MH		 1621 if (reloc->sym->sec == pfunc->sec &&
1622 reloc->addend == pfunc->offset)
fd35c88b
JP		 1623 break;
1624
f1974222 1625 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
e7c2bc37 1626 if (!dest_insn)
dcc914f4
JP		 1627 break;
1628
e7c2bc37 1629 /* Make sure the destination is in the same function: */
e65050b9 1630 if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
13810435 1631 break;
dcc914f4
JP		 1632
1633 alt = malloc(sizeof(*alt));
1634 if (!alt) {
1635 WARN("malloc failed");
1636 return -1;
1637 }
1638
e7c2bc37 1639 alt->insn = dest_insn;
dcc914f4 1640 list_add_tail(&alt->list, &insn->alts);
f1974222 1641 prev_offset = reloc->offset;
fd35c88b
JP		 1642 }
1643
1644 if (!prev_offset) {
1645 WARN_FUNC("can't find switch jump table",
1646 insn->sec, insn->offset);
1647 return -1;
dcc914f4
JP		 1648 }
1649
1650 return 0;
1651}
1652
1653/*
d871f7b5
RG		 1654 * find_jump_table() - Given a dynamic jump, find the switch jump table
1655 * associated with it.
dcc914f4 1656 */
f1974222 1657static struct reloc *find_jump_table(struct objtool_file *file,
dcc914f4
JP		 1658 struct symbol *func,
1659 struct instruction *insn)
1660{
d871f7b5 1661 struct reloc *table_reloc;
113d4bc9 1662 struct instruction *dest_insn, *orig_insn = insn;
dcc914f4 1663
99ce7962
PZ		 1664 /*
1665 * Backward search using the @first_jump_src links, these help avoid
1666 * much of the 'in between' code. Which avoids us getting confused by
1667 * it.
1668 */
7dec80cc 1669 for (;
1119d265
JP		 1670 insn && insn->func && insn->func->pfunc == func;
1671 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
99ce7962 1672
7dec80cc 1673 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
dcc914f4
JP		 1674 break;
1675
1676 /* allow small jumps within the range */
1677 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1678 insn->jump_dest &&
1679 (insn->jump_dest->offset <= insn->offset ||
1680 insn->jump_dest->offset > orig_insn->offset))
1681 break;
1682
d871f7b5 1683 table_reloc = arch_find_switch_table(file, insn);
f1974222 1684 if (!table_reloc)
e7c2bc37 1685 continue;
f1974222 1686 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
113d4bc9
JP		 1687 if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1688 continue;
7dec80cc 1689
f1974222 1690 return table_reloc;
dcc914f4
JP		 1691 }
1692
1693 return NULL;
1694}
1695
bd98c813
JH		 1696/*
1697 * First pass: Mark the head of each jump table so that in the next pass,
1698 * we know when a given jump table ends and the next one starts.
1699 */
1700static void mark_func_jump_tables(struct objtool_file *file,
1701 struct symbol *func)
dcc914f4 1702{
bd98c813 1703 struct instruction *insn, *last = NULL;
f1974222 1704 struct reloc *reloc;
dcc914f4 1705
f0f70adb 1706 func_for_each_insn(file, func, insn) {
99ce7962
PZ		 1707 if (!last)
1708 last = insn;
1709
1710 /*
1711 * Store back-pointers for unconditional forward jumps such
e7c2bc37 1712 * that find_jump_table() can back-track using those and
99ce7962
PZ		 1713 * avoid some potentially confusing code.
1714 */
1715 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1716 insn->offset > last->offset &&
1717 insn->jump_dest->offset > insn->offset &&
1718 !insn->jump_dest->first_jump_src) {
1719
1720 insn->jump_dest->first_jump_src = insn;
1721 last = insn->jump_dest;
1722 }
1723
dcc914f4
JP		 1724 if (insn->type != INSN_JUMP_DYNAMIC)
1725 continue;
1726
f1974222
MH		 1727 reloc = find_jump_table(file, func, insn);
1728 if (reloc) {
1729 reloc->jump_table_start = true;
1730 insn->jump_table = reloc;
dcc914f4 1731 }
dcc914f4 1732 }
bd98c813
JH		 1733}
1734
1735static int add_func_jump_tables(struct objtool_file *file,
1736 struct symbol *func)
1737{
1738 struct instruction *insn;
1739 int ret;
1740
f0f70adb 1741 func_for_each_insn(file, func, insn) {
bd98c813
JH		 1742 if (!insn->jump_table)
1743 continue;
dcc914f4 1744
bd98c813 1745 ret = add_jump_table(file, insn, insn->jump_table);
dcc914f4
JP		 1746 if (ret)
1747 return ret;
1748 }
1749
1750 return 0;
1751}
1752
1753/*
1754 * For some switch statements, gcc generates a jump table in the .rodata
1755 * section which contains a list of addresses within the function to jump to.
1756 * This finds these jump tables and adds them to the insn->alts lists.
1757 */
e7c2bc37 1758static int add_jump_table_alts(struct objtool_file *file)
dcc914f4
JP		 1759{
1760 struct section *sec;
1761 struct symbol *func;
1762 int ret;
1763
4a60aa05 1764 if (!file->rodata)
dcc914f4
JP		 1765 return 0;
1766
baa41469 1767 for_each_sec(file, sec) {
dcc914f4
JP		 1768 list_for_each_entry(func, &sec->symbol_list, list) {
1769 if (func->type != STT_FUNC)
1770 continue;
1771
bd98c813 1772 mark_func_jump_tables(file, func);
e7c2bc37 1773 ret = add_func_jump_tables(file, func);
dcc914f4
JP		 1774 if (ret)
1775 return ret;
1776 }
1777 }
1778
1779 return 0;
1780}
1781
b735bd3e
JP		 1782static void set_func_state(struct cfi_state *state)
1783{
1784 state->cfa = initial_func_cfi.cfa;
1785 memcpy(&state->regs, &initial_func_cfi.regs,
1786 CFI_NUM_REGS * sizeof(struct cfi_reg));
1787 state->stack_size = initial_func_cfi.cfa.offset;
1788}
1789
39358a03
JP		 1790static int read_unwind_hints(struct objtool_file *file)
1791{
8b946cc3 1792 struct cfi_state cfi = init_cfi;
f1974222 1793 struct section *sec, *relocsec;
39358a03
JP		 1794 struct unwind_hint *hint;
1795 struct instruction *insn;
8b946cc3 1796 struct reloc *reloc;
39358a03
JP		 1797 int i;
1798
1799 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
1800 if (!sec)
1801 return 0;
1802
f1974222
MH		 1803 relocsec = sec->reloc;
1804 if (!relocsec) {
39358a03
JP		 1805 WARN("missing .rela.discard.unwind_hints section");
1806 return -1;
1807 }
1808
fe255fe6 1809 if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
39358a03
JP		 1810 WARN("struct unwind_hint size mismatch");
1811 return -1;
1812 }
1813
1814 file->hints = true;
1815
fe255fe6 1816 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
39358a03
JP		 1817 hint = (struct unwind_hint *)sec->data->d_buf + i;
1818
f1974222
MH		 1819 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
1820 if (!reloc) {
1821 WARN("can't find reloc for unwind_hints[%d]", i);
39358a03
JP		 1822 return -1;
1823 }
1824
f1974222 1825 insn = find_insn(file, reloc->sym->sec, reloc->addend);
39358a03
JP		 1826 if (!insn) {
1827 WARN("can't find insn for unwind_hints[%d]", i);
1828 return -1;
1829 }
1830
b735bd3e 1831 insn->hint = true;
39358a03 1832
b735bd3e 1833 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
8b946cc3 1834 insn->cfi = &func_cfi;
39358a03
JP		 1835 continue;
1836 }
1837
8b946cc3
PZ		 1838 if (insn->cfi)
1839 cfi = *(insn->cfi);
1840
1841 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
39358a03
JP		 1842 WARN_FUNC("unsupported unwind_hint sp base reg %d",
1843 insn->sec, insn->offset, hint->sp_reg);
1844 return -1;
1845 }
1846
8b946cc3
PZ		 1847 cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
1848 cfi.type = hint->type;
1849 cfi.end = hint->end;
1850
1851 insn->cfi = cfi_hash_find_or_add(&cfi);
39358a03
JP		 1852 }
1853
1854 return 0;
1855}
1856
b5bc2231
PZ		 1857static int read_retpoline_hints(struct objtool_file *file)
1858{
63474dc4 1859 struct section *sec;
b5bc2231 1860 struct instruction *insn;
f1974222 1861 struct reloc *reloc;
b5bc2231 1862
63474dc4 1863 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
b5bc2231
PZ		 1864 if (!sec)
1865 return 0;
1866
f1974222
MH		 1867 list_for_each_entry(reloc, &sec->reloc_list, list) {
1868 if (reloc->sym->type != STT_SECTION) {
63474dc4 1869 WARN("unexpected relocation symbol type in %s", sec->name);
b5bc2231
PZ		 1870 return -1;
1871 }
1872
f1974222 1873 insn = find_insn(file, reloc->sym->sec, reloc->addend);
b5bc2231 1874 if (!insn) {
63474dc4 1875 WARN("bad .discard.retpoline_safe entry");
b5bc2231
PZ		 1876 return -1;
1877 }
1878
1879 if (insn->type != INSN_JUMP_DYNAMIC &&
1880 insn->type != INSN_CALL_DYNAMIC) {
63474dc4 1881 WARN_FUNC("retpoline_safe hint not an indirect jump/call",
b5bc2231
PZ		 1882 insn->sec, insn->offset);
1883 return -1;
1884 }
1885
1886 insn->retpoline_safe = true;
1887 }
1888
1889 return 0;
1890}
1891
c4a33939
PZ		 1892static int read_instr_hints(struct objtool_file *file)
1893{
1894 struct section *sec;
1895 struct instruction *insn;
f1974222 1896 struct reloc *reloc;
c4a33939
PZ		 1897
1898 sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
1899 if (!sec)
1900 return 0;
1901
f1974222
MH		 1902 list_for_each_entry(reloc, &sec->reloc_list, list) {
1903 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 1904 WARN("unexpected relocation symbol type in %s", sec->name);
1905 return -1;
1906 }
1907
f1974222 1908 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 1909 if (!insn) {
1910 WARN("bad .discard.instr_end entry");
1911 return -1;
1912 }
1913
1914 insn->instr--;
1915 }
1916
1917 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
1918 if (!sec)
1919 return 0;
1920
f1974222
MH		 1921 list_for_each_entry(reloc, &sec->reloc_list, list) {
1922 if (reloc->sym->type != STT_SECTION) {
c4a33939
PZ		 1923 WARN("unexpected relocation symbol type in %s", sec->name);
1924 return -1;
1925 }
1926
f1974222 1927 insn = find_insn(file, reloc->sym->sec, reloc->addend);
c4a33939
PZ		 1928 if (!insn) {
1929 WARN("bad .discard.instr_begin entry");
1930 return -1;
1931 }
1932
1933 insn->instr++;
1934 }
1935
1936 return 0;
1937}
1938
8aa8eb2a
AC		 1939static int read_intra_function_calls(struct objtool_file *file)
1940{
1941 struct instruction *insn;
1942 struct section *sec;
f1974222 1943 struct reloc *reloc;
8aa8eb2a
AC		 1944
1945 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
1946 if (!sec)
1947 return 0;
1948
f1974222 1949 list_for_each_entry(reloc, &sec->reloc_list, list) {
8aa8eb2a
AC		 1950 unsigned long dest_off;
1951
f1974222 1952 if (reloc->sym->type != STT_SECTION) {
8aa8eb2a
AC		 1953 WARN("unexpected relocation symbol type in %s",
1954 sec->name);
1955 return -1;
1956 }
1957
f1974222 1958 insn = find_insn(file, reloc->sym->sec, reloc->addend);
8aa8eb2a
AC		 1959 if (!insn) {
1960 WARN("bad .discard.intra_function_call entry");
1961 return -1;
1962 }
1963
1964 if (insn->type != INSN_CALL) {
1965 WARN_FUNC("intra_function_call not a direct call",
1966 insn->sec, insn->offset);
1967 return -1;
1968 }
1969
1970 /*
1971 * Treat intra-function CALLs as JMPs, but with a stack_op.
1972 * See add_call_destinations(), which strips stack_ops from
1973 * normal CALLs.
1974 */
1975 insn->type = INSN_JUMP_UNCONDITIONAL;
1976
1977 dest_off = insn->offset + insn->len + insn->immediate;
1978 insn->jump_dest = find_insn(file, insn->sec, dest_off);
1979 if (!insn->jump_dest) {
1980 WARN_FUNC("can't find call dest at %s+0x%lx",
1981 insn->sec, insn->offset,
1982 insn->sec->name, dest_off);
1983 return -1;
1984 }
1985 }
1986
1987 return 0;
1988}
1989
1739c66e 1990static int classify_symbols(struct objtool_file *file)
1e7e4788
JP		 1991{
1992 struct section *sec;
1993 struct symbol *func;
1994
1995 for_each_sec(file, sec) {
1996 list_for_each_entry(func, &sec->symbol_list, list) {
1739c66e
PZ		 1997 if (func->bind != STB_GLOBAL)
1998 continue;
1999
2000 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
1e7e4788
JP		 2001 strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2002 func->static_call_tramp = true;
1739c66e
PZ		 2003
2004 if (arch_is_retpoline(func))
2005 func->retpoline_thunk = true;
2006
2007 if (!strcmp(func->name, "__fentry__"))
2008 func->fentry = true;
2009
2010 if (!strncmp(func->name, "__sanitizer_cov_", 16))
2011 func->kcov = true;
1e7e4788
JP		 2012 }
2013 }
2014
2015 return 0;
2016}
2017
4a60aa05
AX		 2018static void mark_rodata(struct objtool_file *file)
2019{
2020 struct section *sec;
2021 bool found = false;
2022
2023 /*
87b512de
JP		 2024 * Search for the following rodata sections, each of which can
2025 * potentially contain jump tables:
2026 *
2027 * - .rodata: can contain GCC switch tables
2028 * - .rodata.<func>: same, if -fdata-sections is being used
2029 * - .rodata..c_jump_table: contains C annotated jump tables
2030 *
2031 * .rodata.str1.* sections are ignored; they don't contain jump tables.
4a60aa05
AX		 2032 */
2033 for_each_sec(file, sec) {
1ee44470
MS		 2034 if (!strncmp(sec->name, ".rodata", 7) &&
2035 !strstr(sec->name, ".str1.")) {
4a60aa05
AX		 2036 sec->rodata = true;
2037 found = true;
2038 }
2039 }
2040
2041 file->rodata = found;
2042}
2043
dcc914f4
JP		 2044static int decode_sections(struct objtool_file *file)
2045{
2046 int ret;
2047
4a60aa05
AX		 2048 mark_rodata(file);
2049
db2b0c5d
PZ		 2050 ret = init_pv_ops(file);
2051 if (ret)
2052 return ret;
2053
dcc914f4
JP		 2054 ret = decode_instructions(file);
2055 if (ret)
2056 return ret;
2057
2058 ret = add_dead_ends(file);
2059 if (ret)
2060 return ret;
2061
2062 add_ignores(file);
ea24213d 2063 add_uaccess_safe(file);
dcc914f4 2064
ff05ab23 2065 ret = add_ignore_alternatives(file);
258c7605
JP		 2066 if (ret)
2067 return ret;
2068
a958c4fe
PZ		 2069 /*
2070 * Must be before add_{jump_call}_destination.
2071 */
1739c66e 2072 ret = classify_symbols(file);
5b06fd3b
PZ		 2073 if (ret)
2074 return ret;
2075
43d5430a
PZ		 2076 /*
2077 * Must be before add_special_section_alts() as that depends on
2078 * jump_dest being set.
2079 */
dcc914f4
JP		 2080 ret = add_jump_destinations(file);
2081 if (ret)
2082 return ret;
2083
a845c7cf 2084 ret = add_special_section_alts(file);
dcc914f4
JP		 2085 if (ret)
2086 return ret;
2087
a958c4fe
PZ		 2088 /*
2089 * Must be before add_call_destination(); it changes INSN_CALL to
2090 * INSN_JUMP.
2091 */
8aa8eb2a
AC		 2092 ret = read_intra_function_calls(file);
2093 if (ret)
2094 return ret;
2095
a845c7cf 2096 ret = add_call_destinations(file);
dcc914f4
JP		 2097 if (ret)
2098 return ret;
2099
e7c2bc37 2100 ret = add_jump_table_alts(file);
dcc914f4
JP		 2101 if (ret)
2102 return ret;
2103
39358a03
JP		 2104 ret = read_unwind_hints(file);
2105 if (ret)
2106 return ret;
2107
b5bc2231
PZ		 2108 ret = read_retpoline_hints(file);
2109 if (ret)
2110 return ret;
2111
c4a33939
PZ		 2112 ret = read_instr_hints(file);
2113 if (ret)
2114 return ret;
2115
dcc914f4
JP		 2116 return 0;
2117}
2118
2119static bool is_fentry_call(struct instruction *insn)
2120{
1739c66e
PZ		 2121 if (insn->type == INSN_CALL &&
2122 insn->call_dest &&
2123 insn->call_dest->fentry)
dcc914f4
JP		 2124 return true;
2125
2126 return false;
2127}
2128
e25eea89 2129static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
dcc914f4 2130{
e7c0219b 2131 struct cfi_state *cfi = &state->cfi;
baa41469
JP		 2132 int i;
2133
e7c0219b 2134 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
e25eea89
PZ		 2135 return true;
2136
b735bd3e 2137 if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
baa41469
JP		 2138 return true;
2139
b735bd3e 2140 if (cfi->stack_size != initial_func_cfi.cfa.offset)
e25eea89
PZ		 2141 return true;
2142
2143 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b
PZ		 2144 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2145 cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
baa41469 2146 return true;
e25eea89 2147 }
baa41469
JP		 2148
2149 return false;
2150}
2151
fb084fde
JT		 2152static bool check_reg_frame_pos(const struct cfi_reg *reg,
2153 int expected_offset)
2154{
2155 return reg->base == CFI_CFA &&
2156 reg->offset == expected_offset;
2157}
2158
baa41469
JP		 2159static bool has_valid_stack_frame(struct insn_state *state)
2160{
e7c0219b
PZ		 2161 struct cfi_state *cfi = &state->cfi;
2162
fb084fde
JT		 2163 if (cfi->cfa.base == CFI_BP &&
2164 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2165 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
baa41469
JP		 2166 return true;
2167
e7c0219b 2168 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
baa41469
JP		 2169 return true;
2170
2171 return false;
dcc914f4
JP		 2172}
2173
e7c0219b
PZ		 2174static int update_cfi_state_regs(struct instruction *insn,
2175 struct cfi_state *cfi,
65ea47dc 2176 struct stack_op *op)
627fce14 2177{
e7c0219b 2178 struct cfi_reg *cfa = &cfi->cfa;
627fce14 2179
d8dd25a4 2180 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
627fce14
JP		 2181 return 0;
2182
2183 /* push */
ea24213d 2184 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
627fce14
JP		 2185 cfa->offset += 8;
2186
2187 /* pop */
ea24213d 2188 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
627fce14
JP		 2189 cfa->offset -= 8;
2190
2191 /* add immediate to sp */
2192 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2193 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2194 cfa->offset -= op->src.offset;
2195
2196 return 0;
2197}
2198
e7c0219b 2199static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
dcc914f4 2200{
bf4d1a83 2201 if (arch_callee_saved_reg(reg) &&
e7c0219b
PZ		 2202 cfi->regs[reg].base == CFI_UNDEFINED) {
2203 cfi->regs[reg].base = base;
2204 cfi->regs[reg].offset = offset;
baa41469 2205 }
dcc914f4
JP		 2206}
2207
e7c0219b 2208static void restore_reg(struct cfi_state *cfi, unsigned char reg)
dcc914f4 2209{
e7c0219b
PZ		 2210 cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2211 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
baa41469
JP		 2212}
2213
2214/*
2215 * A note about DRAP stack alignment:
2216 *
2217 * GCC has the concept of a DRAP register, which is used to help keep track of
2218 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2219 * register. The typical DRAP pattern is:
2220 *
2221 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2222 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2223 * 41 ff 72 f8 pushq -0x8(%r10)
2224 * 55 push %rbp
2225 * 48 89 e5 mov %rsp,%rbp
2226 * (more pushes)
2227 * 41 52 push %r10
2228 * ...
2229 * 41 5a pop %r10
2230 * (more pops)
2231 * 5d pop %rbp
2232 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2233 * c3 retq
2234 *
2235 * There are some variations in the epilogues, like:
2236 *
2237 * 5b pop %rbx
2238 * 41 5a pop %r10
2239 * 41 5c pop %r12
2240 * 41 5d pop %r13
2241 * 41 5e pop %r14
2242 * c9 leaveq
2243 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2244 * c3 retq
2245 *
2246 * and:
2247 *
2248 * 4c 8b 55 e8 mov -0x18(%rbp),%r10
2249 * 48 8b 5d e0 mov -0x20(%rbp),%rbx
2250 * 4c 8b 65 f0 mov -0x10(%rbp),%r12
2251 * 4c 8b 6d f8 mov -0x8(%rbp),%r13
2252 * c9 leaveq
2253 * 49 8d 62 f8 lea -0x8(%r10),%rsp
2254 * c3 retq
2255 *
2256 * Sometimes r13 is used as the DRAP register, in which case it's saved and
2257 * restored beforehand:
2258 *
2259 * 41 55 push %r13
2260 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2261 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2262 * ...
2263 * 49 8d 65 f0 lea -0x10(%r13),%rsp
2264 * 41 5d pop %r13
2265 * c3 retq
2266 */
d54dba41
PZ		 2267static int update_cfi_state(struct instruction *insn,
2268 struct instruction *next_insn,
2269 struct cfi_state *cfi, struct stack_op *op)
baa41469 2270{
e7c0219b
PZ		 2271 struct cfi_reg *cfa = &cfi->cfa;
2272 struct cfi_reg *regs = cfi->regs;
baa41469
JP		 2273
2274 /* stack operations don't make sense with an undefined CFA */
2275 if (cfa->base == CFI_UNDEFINED) {
2276 if (insn->func) {
2277 WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2278 return -1;
2279 }
2280 return 0;
2281 }
2282
ee819aed
JT		 2283 if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2284 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
e7c0219b 2285 return update_cfi_state_regs(insn, cfi, op);
627fce14 2286
baa41469
JP		 2287 switch (op->dest.type) {
2288
2289 case OP_DEST_REG:
2290 switch (op->src.type) {
2291
2292 case OP_SRC_REG:
0d0970ee
JP		 2293 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2294 cfa->base == CFI_SP &&
fb084fde 2295 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
0d0970ee
JP		 2296
2297 /* mov %rsp, %rbp */
2298 cfa->base = op->dest.reg;
e7c0219b 2299 cfi->bp_scratch = false;
0d0970ee 2300 }
dd88a0a0 2301
0d0970ee 2302 else if (op->src.reg == CFI_SP &&
e7c0219b 2303 op->dest.reg == CFI_BP && cfi->drap) {
dd88a0a0 2304
0d0970ee
JP		 2305 /* drap: mov %rsp, %rbp */
2306 regs[CFI_BP].base = CFI_BP;
e7c0219b
PZ		 2307 regs[CFI_BP].offset = -cfi->stack_size;
2308 cfi->bp_scratch = false;
0d0970ee 2309 }
dd88a0a0 2310
0d0970ee
JP		 2311 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2312
2313 /*
2314 * mov %rsp, %reg
2315 *
2316 * This is needed for the rare case where GCC
2317 * does:
2318 *
2319 * mov %rsp, %rax
2320 * ...
2321 * mov %rax, %rsp
2322 */
e7c0219b
PZ		 2323 cfi->vals[op->dest.reg].base = CFI_CFA;
2324 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
dd88a0a0
JP		 2325 }
2326
3c1f0583 2327 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
ffc7e74f 2328 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
3c1f0583
JP		 2329
2330 /*
2331 * mov %rbp, %rsp
2332 *
2333 * Restore the original stack pointer (Clang).
2334 */
e7c0219b 2335 cfi->stack_size = -cfi->regs[CFI_BP].offset;
3c1f0583
JP		 2336 }
2337
dd88a0a0
JP		 2338 else if (op->dest.reg == cfa->base) {
2339
2340 /* mov %reg, %rsp */
2341 if (cfa->base == CFI_SP &&
e7c0219b 2342 cfi->vals[op->src.reg].base == CFI_CFA) {
dd88a0a0
JP		 2343
2344 /*
2345 * This is needed for the rare case
2346 * where GCC does something dumb like:
2347 *
2348 * lea 0x8(%rsp), %rcx
2349 * ...
2350 * mov %rcx, %rsp
2351 */
e7c0219b
PZ		 2352 cfa->offset = -cfi->vals[op->src.reg].offset;
2353 cfi->stack_size = cfa->offset;
dd88a0a0 2354
aafeb14e
PZ		 2355 } else if (cfa->base == CFI_SP &&
2356 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2357 cfi->vals[op->src.reg].offset == cfa->offset) {
2358
2359 /*
2360 * Stack swizzle:
2361 *
2362 * 1: mov %rsp, (%[tos])
2363 * 2: mov %[tos], %rsp
2364 * ...
2365 * 3: pop %rsp
2366 *
2367 * Where:
2368 *
2369 * 1 - places a pointer to the previous
2370 * stack at the Top-of-Stack of the
2371 * new stack.
2372 *
2373 * 2 - switches to the new stack.
2374 *
2375 * 3 - pops the Top-of-Stack to restore
2376 * the original stack.
2377 *
2378 * Note: we set base to SP_INDIRECT
2379 * here and preserve offset. Therefore
2380 * when the unwinder reaches ToS it
2381 * will dereference SP and then add the
2382 * offset to find the next frame, IOW:
2383 * (%rsp) + offset.
2384 */
2385 cfa->base = CFI_SP_INDIRECT;
2386
dd88a0a0
JP		 2387 } else {
2388 cfa->base = CFI_UNDEFINED;
2389 cfa->offset = 0;
2390 }
baa41469
JP		 2391 }
2392
724c8a23
PZ		 2393 else if (op->dest.reg == CFI_SP &&
2394 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2395 cfi->vals[op->src.reg].offset == cfa->offset) {
2396
2397 /*
2398 * The same stack swizzle case 2) as above. But
2399 * because we can't change cfa->base, case 3)
2400 * will become a regular POP. Pretend we're a
2401 * PUSH so things don't go unbalanced.
2402 */
2403 cfi->stack_size += 8;
2404 }
2405
2406
baa41469
JP		 2407 break;
2408
2409 case OP_SRC_ADD:
2410 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2411
2412 /* add imm, %rsp */
e7c0219b 2413 cfi->stack_size -= op->src.offset;
baa41469
JP		 2414 if (cfa->base == CFI_SP)
2415 cfa->offset -= op->src.offset;
2416 break;
2417 }
2418
2419 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2420
2421 /* lea disp(%rbp), %rsp */
e7c0219b 2422 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
baa41469
JP		 2423 break;
2424 }
2425
468af56a
JT		 2426 if (!cfi->drap && op->src.reg == CFI_SP &&
2427 op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2428 check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2429
2430 /* lea disp(%rsp), %rbp */
2431 cfa->base = CFI_BP;
2432 cfa->offset -= op->src.offset;
2433 cfi->bp_scratch = false;
2434 break;
2435 }
2436
dd88a0a0 2437 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
baa41469
JP		 2438
2439 /* drap: lea disp(%rsp), %drap */
e7c0219b 2440 cfi->drap_reg = op->dest.reg;
dd88a0a0
JP		 2441
2442 /*
2443 * lea disp(%rsp), %reg
2444 *
2445 * This is needed for the rare case where GCC
2446 * does something dumb like:
2447 *
2448 * lea 0x8(%rsp), %rcx
2449 * ...
2450 * mov %rcx, %rsp
2451 */
e7c0219b
PZ		 2452 cfi->vals[op->dest.reg].base = CFI_CFA;
2453 cfi->vals[op->dest.reg].offset = \
2454 -cfi->stack_size + op->src.offset;
dd88a0a0 2455
baa41469
JP		 2456 break;
2457 }
2458
e7c0219b
PZ		 2459 if (cfi->drap && op->dest.reg == CFI_SP &&
2460 op->src.reg == cfi->drap_reg) {
baa41469
JP		 2461
2462 /* drap: lea disp(%drap), %rsp */
2463 cfa->base = CFI_SP;
e7c0219b
PZ		 2464 cfa->offset = cfi->stack_size = -op->src.offset;
2465 cfi->drap_reg = CFI_UNDEFINED;
2466 cfi->drap = false;
baa41469
JP		 2467 break;
2468 }
2469
d54dba41 2470 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
baa41469
JP		 2471 WARN_FUNC("unsupported stack register modification",
2472 insn->sec, insn->offset);
2473 return -1;
2474 }
2475
2476 break;
2477
2478 case OP_SRC_AND:
2479 if (op->dest.reg != CFI_SP ||
e7c0219b
PZ		 2480 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2481 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
baa41469
JP		 2482 WARN_FUNC("unsupported stack pointer realignment",
2483 insn->sec, insn->offset);
2484 return -1;
2485 }
2486
e7c0219b 2487 if (cfi->drap_reg != CFI_UNDEFINED) {
baa41469 2488 /* drap: and imm, %rsp */
e7c0219b
PZ		 2489 cfa->base = cfi->drap_reg;
2490 cfa->offset = cfi->stack_size = 0;
2491 cfi->drap = true;
baa41469
JP		 2492 }
2493
2494 /*
2495 * Older versions of GCC (4.8ish) realign the stack
2496 * without DRAP, with a frame pointer.
2497 */
2498
2499 break;
2500
2501 case OP_SRC_POP:
ea24213d 2502 case OP_SRC_POPF:
aafeb14e
PZ		 2503 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2504
2505 /* pop %rsp; # restore from a stack swizzle */
2506 cfa->base = CFI_SP;
2507 break;
2508 }
2509
e7c0219b 2510 if (!cfi->drap && op->dest.reg == cfa->base) {
baa41469
JP		 2511
2512 /* pop %rbp */
2513 cfa->base = CFI_SP;
2514 }
2515
e7c0219b
PZ		 2516 if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2517 op->dest.reg == cfi->drap_reg &&
2518 cfi->drap_offset == -cfi->stack_size) {
baa41469 2519
bf4d1a83 2520 /* drap: pop %drap */
e7c0219b 2521 cfa->base = cfi->drap_reg;
bf4d1a83 2522 cfa->offset = 0;
e7c0219b 2523 cfi->drap_offset = -1;
baa41469 2524
ffc7e74f 2525 } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
baa41469 2526
bf4d1a83 2527 /* pop %reg */
e7c0219b 2528 restore_reg(cfi, op->dest.reg);
baa41469
JP		 2529 }
2530
e7c0219b 2531 cfi->stack_size -= 8;
baa41469
JP		 2532 if (cfa->base == CFI_SP)
2533 cfa->offset -= 8;
2534
2535 break;
2536
2537 case OP_SRC_REG_INDIRECT:
201ef5a9
JT		 2538 if (!cfi->drap && op->dest.reg == cfa->base &&
2539 op->dest.reg == CFI_BP) {
2540
2541 /* mov disp(%rsp), %rbp */
2542 cfa->base = CFI_SP;
2543 cfa->offset = cfi->stack_size;
2544 }
2545
e7c0219b
PZ		 2546 if (cfi->drap && op->src.reg == CFI_BP &&
2547 op->src.offset == cfi->drap_offset) {
bf4d1a83
JP		 2548
2549 /* drap: mov disp(%rbp), %drap */
e7c0219b 2550 cfa->base = cfi->drap_reg;
bf4d1a83 2551 cfa->offset = 0;
e7c0219b 2552 cfi->drap_offset = -1;
bf4d1a83
JP		 2553 }
2554
e7c0219b 2555 if (cfi->drap && op->src.reg == CFI_BP &&
baa41469
JP		 2556 op->src.offset == regs[op->dest.reg].offset) {
2557
2558 /* drap: mov disp(%rbp), %reg */
e7c0219b 2559 restore_reg(cfi, op->dest.reg);
baa41469
JP		 2560
2561 } else if (op->src.reg == cfa->base &&
2562 op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2563
2564 /* mov disp(%rbp), %reg */
2565 /* mov disp(%rsp), %reg */
e7c0219b 2566 restore_reg(cfi, op->dest.reg);
201ef5a9
JT		 2567
2568 } else if (op->src.reg == CFI_SP &&
2569 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2570
2571 /* mov disp(%rsp), %reg */
2572 restore_reg(cfi, op->dest.reg);
baa41469
JP		 2573 }
2574
2575 break;
2576
2577 default:
2578 WARN_FUNC("unknown stack-related instruction",
2579 insn->sec, insn->offset);
2580 return -1;
2581 }
2582
2583 break;
2584
2585 case OP_DEST_PUSH:
ea24213d 2586 case OP_DEST_PUSHF:
e7c0219b 2587 cfi->stack_size += 8;
baa41469
JP		 2588 if (cfa->base == CFI_SP)
2589 cfa->offset += 8;
2590
2591 if (op->src.type != OP_SRC_REG)
2592 break;
2593
e7c0219b
PZ		 2594 if (cfi->drap) {
2595 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 2596
2597 /* drap: push %drap */
2598 cfa->base = CFI_BP_INDIRECT;
e7c0219b 2599 cfa->offset = -cfi->stack_size;
baa41469 2600
bf4d1a83 2601 /* save drap so we know when to restore it */
e7c0219b 2602 cfi->drap_offset = -cfi->stack_size;
baa41469 2603
e7c0219b 2604 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
baa41469
JP		 2605
2606 /* drap: push %rbp */
e7c0219b 2607 cfi->stack_size = 0;
baa41469 2608
f4f80398 2609 } else {
baa41469
JP		 2610
2611 /* drap: push %reg */
e7c0219b 2612 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
baa41469
JP		 2613 }
2614
2615 } else {
2616
2617 /* push %reg */
e7c0219b 2618 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
baa41469
JP		 2619 }
2620
2621 /* detect when asm code uses rbp as a scratch register */
867ac9d7 2622 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
baa41469 2623 cfa->base != CFI_BP)
e7c0219b 2624 cfi->bp_scratch = true;
baa41469
JP		 2625 break;
2626
2627 case OP_DEST_REG_INDIRECT:
2628
e7c0219b
PZ		 2629 if (cfi->drap) {
2630 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
baa41469
JP		 2631
2632 /* drap: mov %drap, disp(%rbp) */
2633 cfa->base = CFI_BP_INDIRECT;
2634 cfa->offset = op->dest.offset;
2635
bf4d1a83 2636 /* save drap offset so we know when to restore it */
e7c0219b 2637 cfi->drap_offset = op->dest.offset;
f4f80398 2638 } else {
baa41469
JP		 2639
2640 /* drap: mov reg, disp(%rbp) */
e7c0219b 2641 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
baa41469
JP		 2642 }
2643
2644 } else if (op->dest.reg == cfa->base) {
2645
2646 /* mov reg, disp(%rbp) */
2647 /* mov reg, disp(%rsp) */
e7c0219b
PZ		 2648 save_reg(cfi, op->src.reg, CFI_CFA,
2649 op->dest.offset - cfi->cfa.offset);
201ef5a9
JT		 2650
2651 } else if (op->dest.reg == CFI_SP) {
2652
2653 /* mov reg, disp(%rsp) */
2654 save_reg(cfi, op->src.reg, CFI_CFA,
2655 op->dest.offset - cfi->stack_size);
aafeb14e
PZ		 2656
2657 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2658
2659 /* mov %rsp, (%reg); # setup a stack swizzle. */
2660 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2661 cfi->vals[op->dest.reg].offset = cfa->offset;
baa41469
JP		 2662 }
2663
2664 break;
2665
baa41469 2666 case OP_DEST_MEM:
ea24213d 2667 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
baa41469
JP		 2668 WARN_FUNC("unknown stack-related memory operation",
2669 insn->sec, insn->offset);
2670 return -1;
2671 }
2672
2673 /* pop mem */
e7c0219b 2674 cfi->stack_size -= 8;
baa41469
JP		 2675 if (cfa->base == CFI_SP)
2676 cfa->offset -= 8;
2677
2678 break;
2679
2680 default:
2681 WARN_FUNC("unknown stack-related instruction",
2682 insn->sec, insn->offset);
2683 return -1;
2684 }
2685
2686 return 0;
2687}
2688
c9c324dc
JP		 2689/*
2690 * The stack layouts of alternatives instructions can sometimes diverge when
2691 * they have stack modifications. That's fine as long as the potential stack
2692 * layouts don't conflict at any given potential instruction boundary.
2693 *
2694 * Flatten the CFIs of the different alternative code streams (both original
2695 * and replacement) into a single shared CFI array which can be used to detect
2696 * conflicts and nicely feed a linear array of ORC entries to the unwinder.
2697 */
2698static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
65ea47dc 2699{
c9c324dc
JP		 2700 struct cfi_state **alt_cfi;
2701 int group_off;
65ea47dc 2702
c9c324dc
JP		 2703 if (!insn->alt_group)
2704 return 0;
65ea47dc 2705
8b946cc3
PZ		 2706 if (!insn->cfi) {
2707 WARN("CFI missing");
2708 return -1;
2709 }
2710
c9c324dc
JP		 2711 alt_cfi = insn->alt_group->cfi;
2712 group_off = insn->offset - insn->alt_group->first_insn->offset;
65ea47dc 2713
c9c324dc 2714 if (!alt_cfi[group_off]) {
8b946cc3 2715 alt_cfi[group_off] = insn->cfi;
c9c324dc 2716 } else {
8b946cc3 2717 if (cficmp(alt_cfi[group_off], insn->cfi)) {
c9c324dc
JP		 2718 WARN_FUNC("stack layout conflict in alternatives",
2719 insn->sec, insn->offset);
ab3852ab
PZ		 2720 return -1;
2721 }
c9c324dc
JP		 2722 }
2723
2724 return 0;
2725}
2726
d54dba41
PZ		 2727static int handle_insn_ops(struct instruction *insn,
2728 struct instruction *next_insn,
2729 struct insn_state *state)
c9c324dc
JP		 2730{
2731 struct stack_op *op;
2732
2733 list_for_each_entry(op, &insn->stack_ops, list) {
2734
d54dba41 2735 if (update_cfi_state(insn, next_insn, &state->cfi, op))
c9c324dc 2736 return 1;
ab3852ab 2737
ba08abca
PZ		 2738 if (!insn->alt_group)
2739 continue;
2740
65ea47dc
JT		 2741 if (op->dest.type == OP_DEST_PUSHF) {
2742 if (!state->uaccess_stack) {
2743 state->uaccess_stack = 1;
2744 } else if (state->uaccess_stack >> 31) {
2745 WARN_FUNC("PUSHF stack exhausted",
2746 insn->sec, insn->offset);
2747 return 1;
2748 }
2749 state->uaccess_stack <<= 1;
2750 state->uaccess_stack |= state->uaccess;
2751 }
2752
2753 if (op->src.type == OP_SRC_POPF) {
2754 if (state->uaccess_stack) {
2755 state->uaccess = state->uaccess_stack & 1;
2756 state->uaccess_stack >>= 1;
2757 if (state->uaccess_stack == 1)
2758 state->uaccess_stack = 0;
2759 }
2760 }
2761 }
2762
2763 return 0;
2764}
2765
e7c0219b 2766static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
baa41469 2767{
8b946cc3 2768 struct cfi_state *cfi1 = insn->cfi;
baa41469
JP		 2769 int i;
2770
8b946cc3
PZ		 2771 if (!cfi1) {
2772 WARN("CFI missing");
2773 return false;
2774 }
2775
e7c0219b
PZ		 2776 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
2777
baa41469
JP		 2778 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
2779 insn->sec, insn->offset,
e7c0219b
PZ		 2780 cfi1->cfa.base, cfi1->cfa.offset,
2781 cfi2->cfa.base, cfi2->cfa.offset);
baa41469 2782
e7c0219b 2783 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
baa41469 2784 for (i = 0; i < CFI_NUM_REGS; i++) {
e7c0219b 2785 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
baa41469
JP		 2786 sizeof(struct cfi_reg)))
2787 continue;
2788
2789 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
2790 insn->sec, insn->offset,
e7c0219b
PZ		 2791 i, cfi1->regs[i].base, cfi1->regs[i].offset,
2792 i, cfi2->regs[i].base, cfi2->regs[i].offset);
baa41469
JP		 2793 break;
2794 }
2795
e7c0219b
PZ		 2796 } else if (cfi1->type != cfi2->type) {
2797
627fce14 2798 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
e7c0219b
PZ		 2799 insn->sec, insn->offset, cfi1->type, cfi2->type);
2800
2801 } else if (cfi1->drap != cfi2->drap ||
2802 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
2803 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
627fce14 2804
bf4d1a83 2805 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
baa41469 2806 insn->sec, insn->offset,
e7c0219b
PZ		 2807 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
2808 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
baa41469
JP		 2809
2810 } else
2811 return true;
2812
2813 return false;
dcc914f4
JP		 2814}
2815
ea24213d
PZ		 2816static inline bool func_uaccess_safe(struct symbol *func)
2817{
2818 if (func)
e10cd8fe 2819 return func->uaccess_safe;
ea24213d
PZ		 2820
2821 return false;
2822}
2823
0c1ddd33 2824static inline const char *call_dest_name(struct instruction *insn)
ea24213d 2825{
db2b0c5d
PZ		 2826 static char pvname[16];
2827 struct reloc *rel;
2828 int idx;
2829
ea24213d
PZ		 2830 if (insn->call_dest)
2831 return insn->call_dest->name;
2832
db2b0c5d
PZ		 2833 rel = insn_reloc(NULL, insn);
2834 if (rel && !strcmp(rel->sym->name, "pv_ops")) {
2835 idx = (rel->addend / sizeof(void *));
2836 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
2837 return pvname;
2838 }
2839
ea24213d
PZ		 2840 return "{dynamic}";
2841}
2842
db2b0c5d
PZ		 2843static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
2844{
2845 struct symbol *target;
2846 struct reloc *rel;
2847 int idx;
2848
2849 rel = insn_reloc(file, insn);
2850 if (!rel || strcmp(rel->sym->name, "pv_ops"))
2851 return false;
2852
2853 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
2854
2855 if (file->pv_ops[idx].clean)
2856 return true;
2857
2858 file->pv_ops[idx].clean = true;
2859
2860 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
2861 if (!target->sec->noinstr) {
2862 WARN("pv_ops[%d]: %s", idx, target->name);
2863 file->pv_ops[idx].clean = false;
2864 }
2865 }
2866
2867 return file->pv_ops[idx].clean;
2868}
2869
2870static inline bool noinstr_call_dest(struct objtool_file *file,
2871 struct instruction *insn,
2872 struct symbol *func)
6b643a07
PZ		 2873{
2874 /*
2875 * We can't deal with indirect function calls at present;
2876 * assume they're instrumented.
2877 */
db2b0c5d
PZ		 2878 if (!func) {
2879 if (file->pv_ops)
2880 return pv_call_dest(file, insn);
2881
6b643a07 2882 return false;
db2b0c5d 2883 }
6b643a07
PZ		 2884
2885 /*
2886 * If the symbol is from a noinstr section; we good.
2887 */
2888 if (func->sec->noinstr)
2889 return true;
2890
2891 /*
2892 * The __ubsan_handle_*() calls are like WARN(), they only happen when
2893 * something 'BAD' happened. At the risk of taking the machine down,
2894 * let them proceed to get the message out.
2895 */
2896 if (!strncmp(func->name, "__ubsan_handle_", 15))
2897 return true;
2898
2899 return false;
2900}
2901
db2b0c5d
PZ		 2902static int validate_call(struct objtool_file *file,
2903 struct instruction *insn,
2904 struct insn_state *state)
ea24213d 2905{
c4a33939 2906 if (state->noinstr && state->instr <= 0 &&
db2b0c5d 2907 !noinstr_call_dest(file, insn, insn->call_dest)) {
c4a33939
PZ		 2908 WARN_FUNC("call to %s() leaves .noinstr.text section",
2909 insn->sec, insn->offset, call_dest_name(insn));
2910 return 1;
2911 }
2912
ea24213d
PZ		 2913 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
2914 WARN_FUNC("call to %s() with UACCESS enabled",
0c1ddd33 2915 insn->sec, insn->offset, call_dest_name(insn));
ea24213d
PZ		 2916 return 1;
2917 }
2918
2f0f9e9a
PZ		 2919 if (state->df) {
2920 WARN_FUNC("call to %s() with DF set",
0c1ddd33 2921 insn->sec, insn->offset, call_dest_name(insn));
2f0f9e9a
PZ		 2922 return 1;
2923 }
2924
ea24213d
PZ		 2925 return 0;
2926}
2927
db2b0c5d
PZ		 2928static int validate_sibling_call(struct objtool_file *file,
2929 struct instruction *insn,
2930 struct insn_state *state)
54262aa2 2931{
e25eea89 2932 if (has_modified_stack_frame(insn, state)) {
54262aa2
PZ		 2933 WARN_FUNC("sibling call from callable instruction with modified stack frame",
2934 insn->sec, insn->offset);
2935 return 1;
2936 }
2937
db2b0c5d 2938 return validate_call(file, insn, state);
54262aa2
PZ		 2939}
2940
a92e92d1
PZ		 2941static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
2942{
c4a33939
PZ		 2943 if (state->noinstr && state->instr > 0) {
2944 WARN_FUNC("return with instrumentation enabled",
2945 insn->sec, insn->offset);
2946 return 1;
2947 }
2948
a92e92d1
PZ		 2949 if (state->uaccess && !func_uaccess_safe(func)) {
2950 WARN_FUNC("return with UACCESS enabled",
2951 insn->sec, insn->offset);
2952 return 1;
2953 }
2954
2955 if (!state->uaccess && func_uaccess_safe(func)) {
2956 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
2957 insn->sec, insn->offset);
2958 return 1;
2959 }
2960
2961 if (state->df) {
2962 WARN_FUNC("return with DF set",
2963 insn->sec, insn->offset);
2964 return 1;
2965 }
2966
e25eea89 2967 if (func && has_modified_stack_frame(insn, state)) {
a92e92d1
PZ		 2968 WARN_FUNC("return with modified stack frame",
2969 insn->sec, insn->offset);
2970 return 1;
2971 }
2972
e7c0219b 2973 if (state->cfi.bp_scratch) {
b2966952
JP		 2974 WARN_FUNC("BP used as a scratch register",
2975 insn->sec, insn->offset);
a92e92d1
PZ		 2976 return 1;
2977 }
2978
2979 return 0;
2980}
2981
c9c324dc
JP		 2982static struct instruction *next_insn_to_validate(struct objtool_file *file,
2983 struct instruction *insn)
7117f16b 2984{
b23cc71c 2985 struct alt_group *alt_group = insn->alt_group;
7117f16b 2986
c9c324dc
JP		 2987 /*
2988 * Simulate the fact that alternatives are patched in-place. When the
2989 * end of a replacement alt_group is reached, redirect objtool flow to
2990 * the end of the original alt_group.
2991 */
2992 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
2993 return next_insn_same_sec(file, alt_group->orig_group->last_insn);
2994
2995 return next_insn_same_sec(file, insn);
7117f16b
PZ		 2996}
2997
dcc914f4
JP		 2998/*
2999 * Follow the branch starting at the given instruction, and recursively follow
3000 * any other branches (jumps). Meanwhile, track the frame pointer state at
3001 * each instruction and validate all the rules described in
3002 * tools/objtool/Documentation/stack-validation.txt.
3003 */
c705cecc 3004static int validate_branch(struct objtool_file *file, struct symbol *func,
b7460462 3005 struct instruction *insn, struct insn_state state)
dcc914f4
JP		 3006{
3007 struct alternative *alt;
8b946cc3 3008 struct instruction *next_insn, *prev_insn = NULL;
dcc914f4 3009 struct section *sec;
882a0db9 3010 u8 visited;
dcc914f4
JP		 3011 int ret;
3012
dcc914f4 3013 sec = insn->sec;
dcc914f4 3014
dcc914f4 3015 while (1) {
c9c324dc 3016 next_insn = next_insn_to_validate(file, insn);
39358a03 3017
13810435 3018 if (file->c_file && func && insn->func && func != insn->func->pfunc) {
ee97638b
JP		 3019 WARN("%s() falls through to next function %s()",
3020 func->name, insn->func->name);
3021 return 1;
dcc914f4
JP		 3022 }
3023
4855022a
JP		 3024 if (func && insn->ignore) {
3025 WARN_FUNC("BUG: why am I validating an ignored function?",
3026 sec, insn->offset);
12b25729 3027 return 1;
4855022a
JP		 3028 }
3029
882a0db9 3030 visited = 1 << state.uaccess;
dcc914f4 3031 if (insn->visited) {
e7c0219b 3032 if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
dcc914f4 3033 return 1;
dcc914f4 3034
882a0db9 3035 if (insn->visited & visited)
ea24213d 3036 return 0;
8b946cc3
PZ		 3037 } else {
3038 nr_insns_visited++;
dcc914f4
JP		 3039 }
3040
c4a33939
PZ		 3041 if (state.noinstr)
3042 state.instr += insn->instr;
3043
8b946cc3
PZ		 3044 if (insn->hint) {
3045 state.cfi = *insn->cfi;
3046 } else {
3047 /* XXX track if we actually changed state.cfi */
3048
3049 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3050 insn->cfi = prev_insn->cfi;
3051 nr_cfi_reused++;
3052 } else {
3053 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3054 }
3055 }
dcc914f4 3056
882a0db9 3057 insn->visited |= visited;
baa41469 3058
c9c324dc
JP		 3059 if (propagate_alt_cfi(file, insn))
3060 return 1;
3061
7117f16b 3062 if (!insn->ignore_alts && !list_empty(&insn->alts)) {
764eef4b
PZ		 3063 bool skip_orig = false;
3064
a845c7cf 3065 list_for_each_entry(alt, &insn->alts, list) {
764eef4b
PZ		 3066 if (alt->skip_orig)
3067 skip_orig = true;
3068
c705cecc 3069 ret = validate_branch(file, func, alt->insn, state);
7697eee3
PZ		 3070 if (ret) {
3071 if (backtrace)
3072 BT_FUNC("(alt)", insn);
3073 return ret;
3074 }
a845c7cf 3075 }
764eef4b
PZ		 3076
3077 if (skip_orig)
3078 return 0;
dcc914f4
JP		 3079 }
3080
d54dba41 3081 if (handle_insn_ops(insn, next_insn, &state))
60041bcd
PZ		 3082 return 1;
3083
dcc914f4
JP		 3084 switch (insn->type) {
3085
dcc914f4 3086 case INSN_RETURN:
a92e92d1 3087 return validate_return(func, insn, &state);
dcc914f4
JP		 3088
3089 case INSN_CALL:
ea24213d 3090 case INSN_CALL_DYNAMIC:
db2b0c5d 3091 ret = validate_call(file, insn, &state);
ea24213d
PZ		 3092 if (ret)
3093 return ret;
dcc914f4 3094
c9bab22b
JP		 3095 if (!no_fp && func && !is_fentry_call(insn) &&
3096 !has_valid_stack_frame(&state)) {
dcc914f4
JP		 3097 WARN_FUNC("call without frame pointer save/setup",
3098 sec, insn->offset);
3099 return 1;
3100 }
c9bab22b
JP		 3101
3102 if (dead_end_function(file, insn->call_dest))
3103 return 0;
3104
dcc914f4
JP		 3105 break;
3106
3107 case INSN_JUMP_CONDITIONAL:
3108 case INSN_JUMP_UNCONDITIONAL:
ecf11ba4 3109 if (is_sibling_call(insn)) {
db2b0c5d 3110 ret = validate_sibling_call(file, insn, &state);
dcc914f4 3111 if (ret)
54262aa2 3112 return ret;
4855022a 3113
0c1ddd33 3114 } else if (insn->jump_dest) {
c705cecc
JP		 3115 ret = validate_branch(file, func,
3116 insn->jump_dest, state);
7697eee3
PZ		 3117 if (ret) {
3118 if (backtrace)
3119 BT_FUNC("(branch)", insn);
3120 return ret;
3121 }
4855022a 3122 }
dcc914f4
JP		 3123
3124 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3125 return 0;
3126
3127 break;
3128
3129 case INSN_JUMP_DYNAMIC:
b68b9907 3130 case INSN_JUMP_DYNAMIC_CONDITIONAL:
ecf11ba4 3131 if (is_sibling_call(insn)) {
db2b0c5d 3132 ret = validate_sibling_call(file, insn, &state);
54262aa2
PZ		 3133 if (ret)
3134 return ret;
dcc914f4
JP		 3135 }
3136
b68b9907
JP		 3137 if (insn->type == INSN_JUMP_DYNAMIC)
3138 return 0;
3139
3140 break;
dcc914f4 3141
39358a03
JP		 3142 case INSN_CONTEXT_SWITCH:
3143 if (func && (!next_insn || !next_insn->hint)) {
3144 WARN_FUNC("unsupported instruction in callable function",
3145 sec, insn->offset);
3146 return 1;
3147 }
3148 return 0;
3149
ea24213d
PZ		 3150 case INSN_STAC:
3151 if (state.uaccess) {
3152 WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3153 return 1;
3154 }
3155
3156 state.uaccess = true;
3157 break;
3158
3159 case INSN_CLAC:
c705cecc 3160 if (!state.uaccess && func) {
ea24213d
PZ		 3161 WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3162 return 1;
3163 }
3164
3165 if (func_uaccess_safe(func) && !state.uaccess_stack) {
3166 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3167 return 1;
3168 }
3169
3170 state.uaccess = false;
baa41469
JP		 3171 break;
3172
2f0f9e9a 3173 case INSN_STD:
6f567c93 3174 if (state.df) {
2f0f9e9a 3175 WARN_FUNC("recursive STD", sec, insn->offset);
6f567c93
JP		 3176 return 1;
3177 }
2f0f9e9a
PZ		 3178
3179 state.df = true;
3180 break;
3181
3182 case INSN_CLD:
6f567c93 3183 if (!state.df && func) {
2f0f9e9a 3184 WARN_FUNC("redundant CLD", sec, insn->offset);
6f567c93
JP		 3185 return 1;
3186 }
2f0f9e9a
PZ		 3187
3188 state.df = false;
baa41469
JP		 3189 break;
3190
dcc914f4
JP		 3191 default:
3192 break;
3193 }
3194
3195 if (insn->dead_end)
3196 return 0;
3197
00d96180 3198 if (!next_insn) {
e7c0219b 3199 if (state.cfi.cfa.base == CFI_UNDEFINED)
00d96180 3200 return 0;
dcc914f4
JP		 3201 WARN("%s: unexpected end of section", sec->name);
3202 return 1;
3203 }
00d96180 3204
8b946cc3 3205 prev_insn = insn;
00d96180 3206 insn = next_insn;
dcc914f4
JP		 3207 }
3208
3209 return 0;
3210}
3211
932f8e98 3212static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
39358a03
JP		 3213{
3214 struct instruction *insn;
39358a03 3215 struct insn_state state;
932f8e98 3216 int ret, warnings = 0;
39358a03
JP		 3217
3218 if (!file->hints)
3219 return 0;
3220
932f8e98 3221 init_insn_state(&state, sec);
39358a03 3222
932f8e98
PZ		 3223 if (sec) {
3224 insn = find_insn(file, sec, 0);
3225 if (!insn)
3226 return 0;
3227 } else {
3228 insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3229 }
3230
3231 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
5b284b19 3232 if (insn->hint && !insn->visited && !insn->ignore) {
c705cecc 3233 ret = validate_branch(file, insn->func, insn, state);
7697eee3
PZ		 3234 if (ret && backtrace)
3235 BT_FUNC("<=== (hint)", insn);
39358a03
JP		 3236 warnings += ret;
3237 }
932f8e98
PZ		 3238
3239 insn = list_next_entry(insn, list);
39358a03
JP		 3240 }
3241
3242 return warnings;
3243}
3244
b5bc2231
PZ		 3245static int validate_retpoline(struct objtool_file *file)
3246{
3247 struct instruction *insn;
3248 int warnings = 0;
3249
3250 for_each_insn(file, insn) {
3251 if (insn->type != INSN_JUMP_DYNAMIC &&
3252 insn->type != INSN_CALL_DYNAMIC)
3253 continue;
3254
3255 if (insn->retpoline_safe)
3256 continue;
3257
ca41b97e
PZ		 3258 /*
3259 * .init.text code is ran before userspace and thus doesn't
3260 * strictly need retpolines, except for modules which are
3261 * loaded late, they very much do need retpoline in their
3262 * .init.text
3263 */
3264 if (!strcmp(insn->sec->name, ".init.text") && !module)
3265 continue;
3266
b5bc2231
PZ		 3267 WARN_FUNC("indirect %s found in RETPOLINE build",
3268 insn->sec, insn->offset,
3269 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3270
3271 warnings++;
3272 }
3273
3274 return warnings;
3275}
3276
dcc914f4
JP		 3277static bool is_kasan_insn(struct instruction *insn)
3278{
3279 return (insn->type == INSN_CALL &&
3280 !strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3281}
3282
3283static bool is_ubsan_insn(struct instruction *insn)
3284{
3285 return (insn->type == INSN_CALL &&
3286 !strcmp(insn->call_dest->name,
3287 "__ubsan_handle_builtin_unreachable"));
3288}
3289
14db1f0a 3290static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
dcc914f4
JP		 3291{
3292 int i;
14db1f0a 3293 struct instruction *prev_insn;
dcc914f4 3294
baa41469
JP		 3295 if (insn->ignore || insn->type == INSN_NOP)
3296 return true;
3297
3298 /*
3299 * Ignore any unused exceptions. This can happen when a whitelisted
3300 * function has an exception table entry.
0e2bb2bc
JP		 3301 *
3302 * Also ignore alternative replacement instructions. This can happen
3303 * when a whitelisted function uses one of the ALTERNATIVE macros.
baa41469 3304 */
0e2bb2bc
JP		 3305 if (!strcmp(insn->sec->name, ".fixup") ||
3306 !strcmp(insn->sec->name, ".altinstr_replacement") ||
3307 !strcmp(insn->sec->name, ".altinstr_aux"))
dcc914f4
JP		 3308 return true;
3309
bd841d61
JP		 3310 if (!insn->func)
3311 return false;
3312
2105a927
PZ		 3313 if (insn->func->static_call_tramp)
3314 return true;
3315
bd841d61
JP		 3316 /*
3317 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3318 * __builtin_unreachable(). The BUG() macro has an unreachable() after
3319 * the UD2, which causes GCC's undefined trap logic to emit another UD2
3320 * (or occasionally a JMP to UD2).
14db1f0a
IH		 3321 *
3322 * It may also insert a UD2 after calling a __noreturn function.
bd841d61 3323 */
14db1f0a
IH		 3324 prev_insn = list_prev_entry(insn, list);
3325 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
bd841d61
JP		 3326 (insn->type == INSN_BUG ||
3327 (insn->type == INSN_JUMP_UNCONDITIONAL &&
3328 insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3329 return true;
3330
dcc914f4
JP		 3331 /*
3332 * Check if this (or a subsequent) instruction is related to
3333 * CONFIG_UBSAN or CONFIG_KASAN.
3334 *
3335 * End the search at 5 instructions to avoid going into the weeds.
3336 */
3337 for (i = 0; i < 5; i++) {
3338
3339 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3340 return true;
3341
fe24e271
JP		 3342 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3343 if (insn->jump_dest &&
3344 insn->jump_dest->func == insn->func) {
3345 insn = insn->jump_dest;
3346 continue;
3347 }
3348
3349 break;
dcc914f4
JP		 3350 }
3351
baa41469 3352 if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
dcc914f4 3353 break;
fe24e271 3354
dcc914f4
JP		 3355 insn = list_next_entry(insn, list);
3356 }
3357
3358 return false;
3359}
3360
4b5e2e7f
PZ		 3361static int validate_symbol(struct objtool_file *file, struct section *sec,
3362 struct symbol *sym, struct insn_state *state)
dcc914f4 3363{
dcc914f4 3364 struct instruction *insn;
4b5e2e7f
PZ		 3365 int ret;
3366
3367 if (!sym->len) {
3368 WARN("%s() is missing an ELF size annotation", sym->name);
3369 return 1;
3370 }
3371
3372 if (sym->pfunc != sym || sym->alias != sym)
3373 return 0;
3374
3375 insn = find_insn(file, sec, sym->offset);
3376 if (!insn || insn->ignore || insn->visited)
3377 return 0;
3378
3379 state->uaccess = sym->uaccess_safe;
3380
3381 ret = validate_branch(file, insn->func, insn, *state);
3382 if (ret && backtrace)
3383 BT_FUNC("<=== (sym)", insn);
3384 return ret;
3385}
3386
3387static int validate_section(struct objtool_file *file, struct section *sec)
3388{
baa41469 3389 struct insn_state state;
4b5e2e7f
PZ		 3390 struct symbol *func;
3391 int warnings = 0;
dcc914f4 3392
350994bf
PZ		 3393 list_for_each_entry(func, &sec->symbol_list, list) {
3394 if (func->type != STT_FUNC)
3395 continue;
e10cd8fe 3396
932f8e98 3397 init_insn_state(&state, sec);
b735bd3e 3398 set_func_state(&state.cfi);
0699e551 3399
4b5e2e7f 3400 warnings += validate_symbol(file, sec, func, &state);
dcc914f4
JP		 3401 }
3402
dcc914f4
JP		 3403 return warnings;
3404}
3405
c4a33939
PZ		 3406static int validate_vmlinux_functions(struct objtool_file *file)
3407{
3408 struct section *sec;
932f8e98 3409 int warnings = 0;
c4a33939
PZ		 3410
3411 sec = find_section_by_name(file->elf, ".noinstr.text");
0cc9ac8d
TG		 3412 if (sec) {
3413 warnings += validate_section(file, sec);
3414 warnings += validate_unwind_hints(file, sec);
3415 }
c4a33939 3416
0cc9ac8d
TG		 3417 sec = find_section_by_name(file->elf, ".entry.text");
3418 if (sec) {
3419 warnings += validate_section(file, sec);
3420 warnings += validate_unwind_hints(file, sec);
3421 }
932f8e98
PZ		 3422
3423 return warnings;
c4a33939
PZ		 3424}
3425
350994bf
PZ		 3426static int validate_functions(struct objtool_file *file)
3427{
3428 struct section *sec;
3429 int warnings = 0;
3430
da837bd6
PZ		 3431 for_each_sec(file, sec) {
3432 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3433 continue;
3434
350994bf 3435 warnings += validate_section(file, sec);
da837bd6 3436 }
350994bf
PZ		 3437
3438 return warnings;
3439}
3440
baa41469 3441static int validate_reachable_instructions(struct objtool_file *file)
dcc914f4
JP		 3442{
3443 struct instruction *insn;
baa41469
JP		 3444
3445 if (file->ignore_unreachables)
3446 return 0;
dcc914f4
JP		 3447
3448 for_each_insn(file, insn) {
14db1f0a 3449 if (insn->visited || ignore_unreachable_insn(file, insn))
baa41469
JP		 3450 continue;
3451
baa41469
JP		 3452 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
3453 return 1;
dcc914f4
JP		 3454 }
3455
baa41469 3456 return 0;
dcc914f4
JP		 3457}
3458
d44becb9 3459int check(struct objtool_file *file)
dcc914f4 3460{
dcc914f4
JP		 3461 int ret, warnings = 0;
3462
baa41469 3463 arch_initial_func_cfi_state(&initial_func_cfi);
8b946cc3
PZ		 3464 init_cfi_state(&init_cfi);
3465 init_cfi_state(&func_cfi);
3466 set_func_state(&func_cfi);
3467
3468 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
3469 goto out;
3470
3471 cfi_hash_add(&init_cfi);
3472 cfi_hash_add(&func_cfi);
baa41469 3473
6545eb03 3474 ret = decode_sections(file);
dcc914f4
JP		 3475 if (ret < 0)
3476 goto out;
8b946cc3 3477
dcc914f4
JP		 3478 warnings += ret;
3479
6545eb03 3480 if (list_empty(&file->insn_list))
dcc914f4 3481 goto out;
dcc914f4 3482
c4a33939 3483 if (vmlinux && !validate_dup) {
6545eb03 3484 ret = validate_vmlinux_functions(file);
c4a33939
PZ		 3485 if (ret < 0)
3486 goto out;
3487
3488 warnings += ret;
3489 goto out;
3490 }
3491
b5bc2231 3492 if (retpoline) {
6545eb03 3493 ret = validate_retpoline(file);
b5bc2231
PZ		 3494 if (ret < 0)
3495 return ret;
3496 warnings += ret;
3497 }
3498
6545eb03 3499 ret = validate_functions(file);
dcc914f4
JP		 3500 if (ret < 0)
3501 goto out;
3502 warnings += ret;
3503
6545eb03 3504 ret = validate_unwind_hints(file, NULL);
39358a03
JP		 3505 if (ret < 0)
3506 goto out;
3507 warnings += ret;
3508
baa41469 3509 if (!warnings) {
6545eb03 3510 ret = validate_reachable_instructions(file);
baa41469
JP		 3511 if (ret < 0)
3512 goto out;
3513 warnings += ret;
3514 }
3515
6545eb03 3516 ret = create_static_call_sections(file);
1e7e4788
JP		 3517 if (ret < 0)
3518 goto out;
3519 warnings += ret;
3520
134ab5bd
PZ		 3521 if (retpoline) {
3522 ret = create_retpoline_sites_sections(file);
3523 if (ret < 0)
3524 goto out;
3525 warnings += ret;
3526 }
3527
99d00215
PZ		 3528 if (mcount) {
3529 ret = create_mcount_loc_sections(file);
3530 if (ret < 0)
3531 goto out;
3532 warnings += ret;
3533 }
3534
8b946cc3
PZ		 3535 if (stats) {
3536 printf("nr_insns_visited: %ld\n", nr_insns_visited);
3537 printf("nr_cfi: %ld\n", nr_cfi);
3538 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
3539 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
3540 }
3541
dcc914f4 3542out:
655cf865
JP		 3543 /*
3544 * For now, don't fail the kernel build on fatal warnings. These
3545 * errors are still fairly common due to the growing matrix of
3546 * supported toolchains and their recent pace of change.
3547 */
dcc914f4
JP		 3548 return 0;
3549}
Linux 6.x block layer and io_uring tree(s)