tools/objtool/check.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
   4  */
   5
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <inttypes.h>
   9 #include <sys/mman.h>
  10
  11 #include <objtool/builtin.h>
  12 #include <objtool/cfi.h>
  13 #include <objtool/arch.h>
  14 #include <objtool/check.h>
  15 #include <objtool/special.h>
  16 #include <objtool/warn.h>
  17 #include <objtool/endianness.h>
  18
  19 #include <linux/objtool_types.h>
  20 #include <linux/hashtable.h>
  21 #include <linux/kernel.h>
  22 #include <linux/static_call_types.h>
  23 #include <linux/string.h>
  24
  25 struct alternative {
  26         struct alternative *next;
  27         struct instruction *insn;
  28         bool skip_orig;
  29 };
  30
  31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
  32
  33 static struct cfi_init_state initial_func_cfi;
  34 static struct cfi_state init_cfi;
  35 static struct cfi_state func_cfi;
  36 static struct cfi_state force_undefined_cfi;
  37
  38 struct instruction *find_insn(struct objtool_file *file,
  39                               struct section *sec, unsigned long offset)
  40 {
  41         struct instruction *insn;
  42
  43         hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
  44                 if (insn->sec == sec && insn->offset == offset)
  45                         return insn;
  46         }
  47
  48         return NULL;
  49 }
  50
  51 struct instruction *next_insn_same_sec(struct objtool_file *file,
  52                                        struct instruction *insn)
  53 {
  54         if (insn->idx == INSN_CHUNK_MAX)
  55                 return find_insn(file, insn->sec, insn->offset + insn->len);
  56
  57         insn++;
  58         if (!insn->len)
  59                 return NULL;
  60
  61         return insn;
  62 }
  63
  64 static struct instruction *next_insn_same_func(struct objtool_file *file,
  65                                                struct instruction *insn)
  66 {
  67         struct instruction *next = next_insn_same_sec(file, insn);
  68         struct symbol *func = insn_func(insn);
  69
  70         if (!func)
  71                 return NULL;
  72
  73         if (next && insn_func(next) == func)
  74                 return next;
  75
  76         /* Check if we're already in the subfunction: */
  77         if (func == func->cfunc)
  78                 return NULL;
  79
  80         /* Move to the subfunction: */
  81         return find_insn(file, func->cfunc->sec, func->cfunc->offset);
  82 }
  83
  84 static struct instruction *prev_insn_same_sec(struct objtool_file *file,
  85                                               struct instruction *insn)
  86 {
  87         if (insn->idx == 0) {
  88                 if (insn->prev_len)
  89                         return find_insn(file, insn->sec, insn->offset - insn->prev_len);
  90                 return NULL;
  91         }
  92
  93         return insn - 1;
  94 }
  95
  96 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
  97                                               struct instruction *insn)
  98 {
  99         struct instruction *prev = prev_insn_same_sec(file, insn);
 100
 101         if (prev && insn_func(prev) == insn_func(insn))
 102                 return prev;
 103
 104         return NULL;
 105 }
 106
 107 #define for_each_insn(file, insn)                                       \
 108         for (struct section *__sec, *__fake = (struct section *)1;      \
 109              __fake; __fake = NULL)                                     \
 110                 for_each_sec(file, __sec)                               \
 111                         sec_for_each_insn(file, __sec, insn)
 112
 113 #define func_for_each_insn(file, func, insn)                            \
 114         for (insn = find_insn(file, func->sec, func->offset);           \
 115              insn;                                                      \
 116              insn = next_insn_same_func(file, insn))
 117
 118 #define sym_for_each_insn(file, sym, insn)                              \
 119         for (insn = find_insn(file, sym->sec, sym->offset);             \
 120              insn && insn->offset < sym->offset + sym->len;             \
 121              insn = next_insn_same_sec(file, insn))
 122
 123 #define sym_for_each_insn_continue_reverse(file, sym, insn)             \
 124         for (insn = prev_insn_same_sec(file, insn);                     \
 125              insn && insn->offset >= sym->offset;                       \
 126              insn = prev_insn_same_sec(file, insn))
 127
 128 #define sec_for_each_insn_from(file, insn)                              \
 129         for (; insn; insn = next_insn_same_sec(file, insn))
 130
 131 #define sec_for_each_insn_continue(file, insn)                          \
 132         for (insn = next_insn_same_sec(file, insn); insn;               \
 133              insn = next_insn_same_sec(file, insn))
 134
 135 static inline struct symbol *insn_call_dest(struct instruction *insn)
 136 {
 137         if (insn->type == INSN_JUMP_DYNAMIC ||
 138             insn->type == INSN_CALL_DYNAMIC)
 139                 return NULL;
 140
 141         return insn->_call_dest;
 142 }
 143
 144 static inline struct reloc *insn_jump_table(struct instruction *insn)
 145 {
 146         if (insn->type == INSN_JUMP_DYNAMIC ||
 147             insn->type == INSN_CALL_DYNAMIC)
 148                 return insn->_jump_table;
 149
 150         return NULL;
 151 }
 152
 153 static bool is_jump_table_jump(struct instruction *insn)
 154 {
 155         struct alt_group *alt_group = insn->alt_group;
 156
 157         if (insn_jump_table(insn))
 158                 return true;
 159
 160         /* Retpoline alternative for a jump table? */
 161         return alt_group && alt_group->orig_group &&
 162                insn_jump_table(alt_group->orig_group->first_insn);
 163 }
 164
 165 static bool is_sibling_call(struct instruction *insn)
 166 {
 167         /*
 168          * Assume only STT_FUNC calls have jump-tables.
 169          */
 170         if (insn_func(insn)) {
 171                 /* An indirect jump is either a sibling call or a jump to a table. */
 172                 if (insn->type == INSN_JUMP_DYNAMIC)
 173                         return !is_jump_table_jump(insn);
 174         }
 175
 176         /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
 177         return (is_static_jump(insn) && insn_call_dest(insn));
 178 }
 179
 180 /*
 181  * This checks to see if the given function is a "noreturn" function.
 182  *
 183  * For global functions which are outside the scope of this object file, we
 184  * have to keep a manual list of them.
 185  *
 186  * For local functions, we have to detect them manually by simply looking for
 187  * the lack of a return instruction.
 188  */
 189 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
 190                                 int recursion)
 191 {
 192         int i;
 193         struct instruction *insn;
 194         bool empty = true;
 195
 196 #define NORETURN(func) __stringify(func),
 197         static const char * const global_noreturns[] = {
 198 #include "noreturns.h"
 199         };
 200 #undef NORETURN
 201
 202         if (!func)
 203                 return false;
 204
 205         if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
 206                 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
 207                         if (!strcmp(func->name, global_noreturns[i]))
 208                                 return true;
 209
 210         if (func->bind == STB_WEAK)
 211                 return false;
 212
 213         if (!func->len)
 214                 return false;
 215
 216         insn = find_insn(file, func->sec, func->offset);
 217         if (!insn || !insn_func(insn))
 218                 return false;
 219
 220         func_for_each_insn(file, func, insn) {
 221                 empty = false;
 222
 223                 if (insn->type == INSN_RETURN)
 224                         return false;
 225         }
 226
 227         if (empty)
 228                 return false;
 229
 230         /*
 231          * A function can have a sibling call instead of a return.  In that
 232          * case, the function's dead-end status depends on whether the target
 233          * of the sibling call returns.
 234          */
 235         func_for_each_insn(file, func, insn) {
 236                 if (is_sibling_call(insn)) {
 237                         struct instruction *dest = insn->jump_dest;
 238
 239                         if (!dest)
 240                                 /* sibling call to another file */
 241                                 return false;
 242
 243                         /* local sibling call */
 244                         if (recursion == 5) {
 245                                 /*
 246                                  * Infinite recursion: two functions have
 247                                  * sibling calls to each other.  This is a very
 248                                  * rare case.  It means they aren't dead ends.
 249                                  */
 250                                 return false;
 251                         }
 252
 253                         return __dead_end_function(file, insn_func(dest), recursion+1);
 254                 }
 255         }
 256
 257         return true;
 258 }
 259
 260 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
 261 {
 262         return __dead_end_function(file, func, 0);
 263 }
 264
 265 static void init_cfi_state(struct cfi_state *cfi)
 266 {
 267         int i;
 268
 269         for (i = 0; i < CFI_NUM_REGS; i++) {
 270                 cfi->regs[i].base = CFI_UNDEFINED;
 271                 cfi->vals[i].base = CFI_UNDEFINED;
 272         }
 273         cfi->cfa.base = CFI_UNDEFINED;
 274         cfi->drap_reg = CFI_UNDEFINED;
 275         cfi->drap_offset = -1;
 276 }
 277
 278 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 279                             struct section *sec)
 280 {
 281         memset(state, 0, sizeof(*state));
 282         init_cfi_state(&state->cfi);
 283
 284         /*
 285          * We need the full vmlinux for noinstr validation, otherwise we can
 286          * not correctly determine insn_call_dest(insn)->sec (external symbols
 287          * do not have a section).
 288          */
 289         if (opts.link && opts.noinstr && sec)
 290                 state->noinstr = sec->noinstr;
 291 }
 292
 293 static struct cfi_state *cfi_alloc(void)
 294 {
 295         struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
 296         if (!cfi) {
 297                 WARN("calloc failed");
 298                 exit(1);
 299         }
 300         nr_cfi++;
 301         return cfi;
 302 }
 303
 304 static int cfi_bits;
 305 static struct hlist_head *cfi_hash;
 306
 307 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
 308 {
 309         return memcmp((void *)cfi1 + sizeof(cfi1->hash),
 310                       (void *)cfi2 + sizeof(cfi2->hash),
 311                       sizeof(struct cfi_state) - sizeof(struct hlist_node));
 312 }
 313
 314 static inline u32 cfi_key(struct cfi_state *cfi)
 315 {
 316         return jhash((void *)cfi + sizeof(cfi->hash),
 317                      sizeof(*cfi) - sizeof(cfi->hash), 0);
 318 }
 319
 320 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
 321 {
 322         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 323         struct cfi_state *obj;
 324
 325         hlist_for_each_entry(obj, head, hash) {
 326                 if (!cficmp(cfi, obj)) {
 327                         nr_cfi_cache++;
 328                         return obj;
 329                 }
 330         }
 331
 332         obj = cfi_alloc();
 333         *obj = *cfi;
 334         hlist_add_head(&obj->hash, head);
 335
 336         return obj;
 337 }
 338
 339 static void cfi_hash_add(struct cfi_state *cfi)
 340 {
 341         struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
 342
 343         hlist_add_head(&cfi->hash, head);
 344 }
 345
 346 static void *cfi_hash_alloc(unsigned long size)
 347 {
 348         cfi_bits = max(10, ilog2(size));
 349         cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
 350                         PROT_READ|PROT_WRITE,
 351                         MAP_PRIVATE|MAP_ANON, -1, 0);
 352         if (cfi_hash == (void *)-1L) {
 353                 WARN("mmap fail cfi_hash");
 354                 cfi_hash = NULL;
 355         }  else if (opts.stats) {
 356                 printf("cfi_bits: %d\n", cfi_bits);
 357         }
 358
 359         return cfi_hash;
 360 }
 361
 362 static unsigned long nr_insns;
 363 static unsigned long nr_insns_visited;
 364
 365 /*
 366  * Call the arch-specific instruction decoder for all the instructions and add
 367  * them to the global instruction list.
 368  */
 369 static int decode_instructions(struct objtool_file *file)
 370 {
 371         struct section *sec;
 372         struct symbol *func;
 373         unsigned long offset;
 374         struct instruction *insn;
 375         int ret;
 376
 377         for_each_sec(file, sec) {
 378                 struct instruction *insns = NULL;
 379                 u8 prev_len = 0;
 380                 u8 idx = 0;
 381
 382                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
 383                         continue;
 384
 385                 if (strcmp(sec->name, ".altinstr_replacement") &&
 386                     strcmp(sec->name, ".altinstr_aux") &&
 387                     strncmp(sec->name, ".discard.", 9))
 388                         sec->text = true;
 389
 390                 if (!strcmp(sec->name, ".noinstr.text") ||
 391                     !strcmp(sec->name, ".entry.text") ||
 392                     !strcmp(sec->name, ".cpuidle.text") ||
 393                     !strncmp(sec->name, ".text..__x86.", 13))
 394                         sec->noinstr = true;
 395
 396                 /*
 397                  * .init.text code is ran before userspace and thus doesn't
 398                  * strictly need retpolines, except for modules which are
 399                  * loaded late, they very much do need retpoline in their
 400                  * .init.text
 401                  */
 402                 if (!strcmp(sec->name, ".init.text") && !opts.module)
 403                         sec->init = true;
 404
 405                 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
 406                         if (!insns || idx == INSN_CHUNK_MAX) {
 407                                 insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
 408                                 if (!insns) {
 409                                         WARN("malloc failed");
 410                                         return -1;
 411                                 }
 412                                 idx = 0;
 413                         } else {
 414                                 idx++;
 415                         }
 416                         insn = &insns[idx];
 417                         insn->idx = idx;
 418
 419                         INIT_LIST_HEAD(&insn->call_node);
 420                         insn->sec = sec;
 421                         insn->offset = offset;
 422                         insn->prev_len = prev_len;
 423
 424                         ret = arch_decode_instruction(file, sec, offset,
 425                                                       sec->sh.sh_size - offset,
 426                                                       insn);
 427                         if (ret)
 428                                 return ret;
 429
 430                         prev_len = insn->len;
 431
 432                         /*
 433                          * By default, "ud2" is a dead end unless otherwise
 434                          * annotated, because GCC 7 inserts it for certain
 435                          * divide-by-zero cases.
 436                          */
 437                         if (insn->type == INSN_BUG)
 438                                 insn->dead_end = true;
 439
 440                         hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
 441                         nr_insns++;
 442                 }
 443
 444 //              printf("%s: last chunk used: %d\n", sec->name, (int)idx);
 445
 446                 sec_for_each_sym(sec, func) {
 447                         if (func->type != STT_NOTYPE && func->type != STT_FUNC)
 448                                 continue;
 449
 450                         if (func->offset == sec->sh.sh_size) {
 451                                 /* Heuristic: likely an "end" symbol */
 452                                 if (func->type == STT_NOTYPE)
 453                                         continue;
 454                                 WARN("%s(): STT_FUNC at end of section",
 455                                      func->name);
 456                                 return -1;
 457                         }
 458
 459                         if (func->embedded_insn || func->alias != func)
 460                                 continue;
 461
 462                         if (!find_insn(file, sec, func->offset)) {
 463                                 WARN("%s(): can't find starting instruction",
 464                                      func->name);
 465                                 return -1;
 466                         }
 467
 468                         sym_for_each_insn(file, func, insn) {
 469                                 insn->sym = func;
 470                                 if (func->type == STT_FUNC &&
 471                                     insn->type == INSN_ENDBR &&
 472                                     list_empty(&insn->call_node)) {
 473                                         if (insn->offset == func->offset) {
 474                                                 list_add_tail(&insn->call_node, &file->endbr_list);
 475                                                 file->nr_endbr++;
 476                                         } else {
 477                                                 file->nr_endbr_int++;
 478                                         }
 479                                 }
 480                         }
 481                 }
 482         }
 483
 484         if (opts.stats)
 485                 printf("nr_insns: %lu\n", nr_insns);
 486
 487         return 0;
 488 }
 489
 490 /*
 491  * Read the pv_ops[] .data table to find the static initialized values.
 492  */
 493 static int add_pv_ops(struct objtool_file *file, const char *symname)
 494 {
 495         struct symbol *sym, *func;
 496         unsigned long off, end;
 497         struct reloc *reloc;
 498         int idx;
 499
 500         sym = find_symbol_by_name(file->elf, symname);
 501         if (!sym)
 502                 return 0;
 503
 504         off = sym->offset;
 505         end = off + sym->len;
 506         for (;;) {
 507                 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
 508                 if (!reloc)
 509                         break;
 510
 511                 func = reloc->sym;
 512                 if (func->type == STT_SECTION)
 513                         func = find_symbol_by_offset(reloc->sym->sec,
 514                                                      reloc_addend(reloc));
 515
 516                 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
 517
 518                 objtool_pv_add(file, idx, func);
 519
 520                 off = reloc_offset(reloc) + 1;
 521                 if (off > end)
 522                         break;
 523         }
 524
 525         return 0;
 526 }
 527
 528 /*
 529  * Allocate and initialize file->pv_ops[].
 530  */
 531 static int init_pv_ops(struct objtool_file *file)
 532 {
 533         static const char *pv_ops_tables[] = {
 534                 "pv_ops",
 535                 "xen_cpu_ops",
 536                 "xen_irq_ops",
 537                 "xen_mmu_ops",
 538                 NULL,
 539         };
 540         const char *pv_ops;
 541         struct symbol *sym;
 542         int idx, nr;
 543
 544         if (!opts.noinstr)
 545                 return 0;
 546
 547         file->pv_ops = NULL;
 548
 549         sym = find_symbol_by_name(file->elf, "pv_ops");
 550         if (!sym)
 551                 return 0;
 552
 553         nr = sym->len / sizeof(unsigned long);
 554         file->pv_ops = calloc(sizeof(struct pv_state), nr);
 555         if (!file->pv_ops)
 556                 return -1;
 557
 558         for (idx = 0; idx < nr; idx++)
 559                 INIT_LIST_HEAD(&file->pv_ops[idx].targets);
 560
 561         for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
 562                 add_pv_ops(file, pv_ops);
 563
 564         return 0;
 565 }
 566
 567 static struct instruction *find_last_insn(struct objtool_file *file,
 568                                           struct section *sec)
 569 {
 570         struct instruction *insn = NULL;
 571         unsigned int offset;
 572         unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
 573
 574         for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
 575                 insn = find_insn(file, sec, offset);
 576
 577         return insn;
 578 }
 579
 580 /*
 581  * Mark "ud2" instructions and manually annotated dead ends.
 582  */
 583 static int add_dead_ends(struct objtool_file *file)
 584 {
 585         struct section *rsec;
 586         struct reloc *reloc;
 587         struct instruction *insn;
 588         unsigned long offset;
 589
 590         /*
 591          * Check for manually annotated dead ends.
 592          */
 593         rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
 594         if (!rsec)
 595                 goto reachable;
 596
 597         for_each_reloc(rsec, reloc) {
 598                 if (reloc->sym->type == STT_SECTION) {
 599                         offset = reloc_addend(reloc);
 600                 } else if (reloc->sym->local_label) {
 601                         offset = reloc->sym->offset;
 602                 } else {
 603                         WARN("unexpected relocation symbol type in %s", rsec->name);
 604                         return -1;
 605                 }
 606
 607                 insn = find_insn(file, reloc->sym->sec, offset);
 608                 if (insn)
 609                         insn = prev_insn_same_sec(file, insn);
 610                 else if (offset == reloc->sym->sec->sh.sh_size) {
 611                         insn = find_last_insn(file, reloc->sym->sec);
 612                         if (!insn) {
 613                                 WARN("can't find unreachable insn at %s+0x%" PRIx64,
 614                                      reloc->sym->sec->name, offset);
 615                                 return -1;
 616                         }
 617                 } else {
 618                         WARN("can't find unreachable insn at %s+0x%" PRIx64,
 619                              reloc->sym->sec->name, offset);
 620                         return -1;
 621                 }
 622
 623                 insn->dead_end = true;
 624         }
 625
 626 reachable:
 627         /*
 628          * These manually annotated reachable checks are needed for GCC 4.4,
 629          * where the Linux unreachable() macro isn't supported.  In that case
 630          * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
 631          * not a dead end.
 632          */
 633         rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
 634         if (!rsec)
 635                 return 0;
 636
 637         for_each_reloc(rsec, reloc) {
 638                 if (reloc->sym->type == STT_SECTION) {
 639                         offset = reloc_addend(reloc);
 640                 } else if (reloc->sym->local_label) {
 641                         offset = reloc->sym->offset;
 642                 } else {
 643                         WARN("unexpected relocation symbol type in %s", rsec->name);
 644                         return -1;
 645                 }
 646
 647                 insn = find_insn(file, reloc->sym->sec, offset);
 648                 if (insn)
 649                         insn = prev_insn_same_sec(file, insn);
 650                 else if (offset == reloc->sym->sec->sh.sh_size) {
 651                         insn = find_last_insn(file, reloc->sym->sec);
 652                         if (!insn) {
 653                                 WARN("can't find reachable insn at %s+0x%" PRIx64,
 654                                      reloc->sym->sec->name, offset);
 655                                 return -1;
 656                         }
 657                 } else {
 658                         WARN("can't find reachable insn at %s+0x%" PRIx64,
 659                              reloc->sym->sec->name, offset);
 660                         return -1;
 661                 }
 662
 663                 insn->dead_end = false;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static int create_static_call_sections(struct objtool_file *file)
 670 {
 671         struct static_call_site *site;
 672         struct section *sec;
 673         struct instruction *insn;
 674         struct symbol *key_sym;
 675         char *key_name, *tmp;
 676         int idx;
 677
 678         sec = find_section_by_name(file->elf, ".static_call_sites");
 679         if (sec) {
 680                 INIT_LIST_HEAD(&file->static_call_list);
 681                 WARN("file already has .static_call_sites section, skipping");
 682                 return 0;
 683         }
 684
 685         if (list_empty(&file->static_call_list))
 686                 return 0;
 687
 688         idx = 0;
 689         list_for_each_entry(insn, &file->static_call_list, call_node)
 690                 idx++;
 691
 692         sec = elf_create_section_pair(file->elf, ".static_call_sites",
 693                                       sizeof(*site), idx, idx * 2);
 694         if (!sec)
 695                 return -1;
 696
 697         /* Allow modules to modify the low bits of static_call_site::key */
 698         sec->sh.sh_flags |= SHF_WRITE;
 699
 700         idx = 0;
 701         list_for_each_entry(insn, &file->static_call_list, call_node) {
 702
 703                 /* populate reloc for 'addr' */
 704                 if (!elf_init_reloc_text_sym(file->elf, sec,
 705                                              idx * sizeof(*site), idx * 2,
 706                                              insn->sec, insn->offset))
 707                         return -1;
 708
 709                 /* find key symbol */
 710                 key_name = strdup(insn_call_dest(insn)->name);
 711                 if (!key_name) {
 712                         perror("strdup");
 713                         return -1;
 714                 }
 715                 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
 716                             STATIC_CALL_TRAMP_PREFIX_LEN)) {
 717                         WARN("static_call: trampoline name malformed: %s", key_name);
 718                         free(key_name);
 719                         return -1;
 720                 }
 721                 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
 722                 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
 723
 724                 key_sym = find_symbol_by_name(file->elf, tmp);
 725                 if (!key_sym) {
 726                         if (!opts.module) {
 727                                 WARN("static_call: can't find static_call_key symbol: %s", tmp);
 728                                 free(key_name);
 729                                 return -1;
 730                         }
 731
 732                         /*
 733                          * For modules(), the key might not be exported, which
 734                          * means the module can make static calls but isn't
 735                          * allowed to change them.
 736                          *
 737                          * In that case we temporarily set the key to be the
 738                          * trampoline address.  This is fixed up in
 739                          * static_call_add_module().
 740                          */
 741                         key_sym = insn_call_dest(insn);
 742                 }
 743                 free(key_name);
 744
 745                 /* populate reloc for 'key' */
 746                 if (!elf_init_reloc_data_sym(file->elf, sec,
 747                                              idx * sizeof(*site) + 4,
 748                                              (idx * 2) + 1, key_sym,
 749                                              is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
 750                         return -1;
 751
 752                 idx++;
 753         }
 754
 755         return 0;
 756 }
 757
 758 static int create_retpoline_sites_sections(struct objtool_file *file)
 759 {
 760         struct instruction *insn;
 761         struct section *sec;
 762         int idx;
 763
 764         sec = find_section_by_name(file->elf, ".retpoline_sites");
 765         if (sec) {
 766                 WARN("file already has .retpoline_sites, skipping");
 767                 return 0;
 768         }
 769
 770         idx = 0;
 771         list_for_each_entry(insn, &file->retpoline_call_list, call_node)
 772                 idx++;
 773
 774         if (!idx)
 775                 return 0;
 776
 777         sec = elf_create_section_pair(file->elf, ".retpoline_sites",
 778                                       sizeof(int), idx, idx);
 779         if (!sec)
 780                 return -1;
 781
 782         idx = 0;
 783         list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
 784
 785                 if (!elf_init_reloc_text_sym(file->elf, sec,
 786                                              idx * sizeof(int), idx,
 787                                              insn->sec, insn->offset))
 788                         return -1;
 789
 790                 idx++;
 791         }
 792
 793         return 0;
 794 }
 795
 796 static int create_return_sites_sections(struct objtool_file *file)
 797 {
 798         struct instruction *insn;
 799         struct section *sec;
 800         int idx;
 801
 802         sec = find_section_by_name(file->elf, ".return_sites");
 803         if (sec) {
 804                 WARN("file already has .return_sites, skipping");
 805                 return 0;
 806         }
 807
 808         idx = 0;
 809         list_for_each_entry(insn, &file->return_thunk_list, call_node)
 810                 idx++;
 811
 812         if (!idx)
 813                 return 0;
 814
 815         sec = elf_create_section_pair(file->elf, ".return_sites",
 816                                       sizeof(int), idx, idx);
 817         if (!sec)
 818                 return -1;
 819
 820         idx = 0;
 821         list_for_each_entry(insn, &file->return_thunk_list, call_node) {
 822
 823                 if (!elf_init_reloc_text_sym(file->elf, sec,
 824                                              idx * sizeof(int), idx,
 825                                              insn->sec, insn->offset))
 826                         return -1;
 827
 828                 idx++;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
 835 {
 836         struct instruction *insn;
 837         struct section *sec;
 838         int idx;
 839
 840         sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
 841         if (sec) {
 842                 WARN("file already has .ibt_endbr_seal, skipping");
 843                 return 0;
 844         }
 845
 846         idx = 0;
 847         list_for_each_entry(insn, &file->endbr_list, call_node)
 848                 idx++;
 849
 850         if (opts.stats) {
 851                 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
 852                 printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
 853                 printf("ibt: superfluous ENDBR:       %d\n", idx);
 854         }
 855
 856         if (!idx)
 857                 return 0;
 858
 859         sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
 860                                       sizeof(int), idx, idx);
 861         if (!sec)
 862                 return -1;
 863
 864         idx = 0;
 865         list_for_each_entry(insn, &file->endbr_list, call_node) {
 866
 867                 int *site = (int *)sec->data->d_buf + idx;
 868                 struct symbol *sym = insn->sym;
 869                 *site = 0;
 870
 871                 if (opts.module && sym && sym->type == STT_FUNC &&
 872                     insn->offset == sym->offset &&
 873                     (!strcmp(sym->name, "init_module") ||
 874                      !strcmp(sym->name, "cleanup_module")))
 875                         WARN("%s(): not an indirect call target", sym->name);
 876
 877                 if (!elf_init_reloc_text_sym(file->elf, sec,
 878                                              idx * sizeof(int), idx,
 879                                              insn->sec, insn->offset))
 880                         return -1;
 881
 882                 idx++;
 883         }
 884
 885         return 0;
 886 }
 887
 888 static int create_cfi_sections(struct objtool_file *file)
 889 {
 890         struct section *sec;
 891         struct symbol *sym;
 892         int idx;
 893
 894         sec = find_section_by_name(file->elf, ".cfi_sites");
 895         if (sec) {
 896                 INIT_LIST_HEAD(&file->call_list);
 897                 WARN("file already has .cfi_sites section, skipping");
 898                 return 0;
 899         }
 900
 901         idx = 0;
 902         for_each_sym(file, sym) {
 903                 if (sym->type != STT_FUNC)
 904                         continue;
 905
 906                 if (strncmp(sym->name, "__cfi_", 6))
 907                         continue;
 908
 909                 idx++;
 910         }
 911
 912         sec = elf_create_section_pair(file->elf, ".cfi_sites",
 913                                       sizeof(unsigned int), idx, idx);
 914         if (!sec)
 915                 return -1;
 916
 917         idx = 0;
 918         for_each_sym(file, sym) {
 919                 if (sym->type != STT_FUNC)
 920                         continue;
 921
 922                 if (strncmp(sym->name, "__cfi_", 6))
 923                         continue;
 924
 925                 if (!elf_init_reloc_text_sym(file->elf, sec,
 926                                              idx * sizeof(unsigned int), idx,
 927                                              sym->sec, sym->offset))
 928                         return -1;
 929
 930                 idx++;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int create_mcount_loc_sections(struct objtool_file *file)
 937 {
 938         size_t addr_size = elf_addr_size(file->elf);
 939         struct instruction *insn;
 940         struct section *sec;
 941         int idx;
 942
 943         sec = find_section_by_name(file->elf, "__mcount_loc");
 944         if (sec) {
 945                 INIT_LIST_HEAD(&file->mcount_loc_list);
 946                 WARN("file already has __mcount_loc section, skipping");
 947                 return 0;
 948         }
 949
 950         if (list_empty(&file->mcount_loc_list))
 951                 return 0;
 952
 953         idx = 0;
 954         list_for_each_entry(insn, &file->mcount_loc_list, call_node)
 955                 idx++;
 956
 957         sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
 958                                       idx, idx);
 959         if (!sec)
 960                 return -1;
 961
 962         sec->sh.sh_addralign = addr_size;
 963
 964         idx = 0;
 965         list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
 966
 967                 struct reloc *reloc;
 968
 969                 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
 970                                                insn->sec, insn->offset);
 971                 if (!reloc)
 972                         return -1;
 973
 974                 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
 975
 976                 idx++;
 977         }
 978
 979         return 0;
 980 }
 981
 982 static int create_direct_call_sections(struct objtool_file *file)
 983 {
 984         struct instruction *insn;
 985         struct section *sec;
 986         int idx;
 987
 988         sec = find_section_by_name(file->elf, ".call_sites");
 989         if (sec) {
 990                 INIT_LIST_HEAD(&file->call_list);
 991                 WARN("file already has .call_sites section, skipping");
 992                 return 0;
 993         }
 994
 995         if (list_empty(&file->call_list))
 996                 return 0;
 997
 998         idx = 0;
 999         list_for_each_entry(insn, &file->call_list, call_node)
1000                 idx++;
1001
1002         sec = elf_create_section_pair(file->elf, ".call_sites",
1003                                       sizeof(unsigned int), idx, idx);
1004         if (!sec)
1005                 return -1;
1006
1007         idx = 0;
1008         list_for_each_entry(insn, &file->call_list, call_node) {
1009
1010                 if (!elf_init_reloc_text_sym(file->elf, sec,
1011                                              idx * sizeof(unsigned int), idx,
1012                                              insn->sec, insn->offset))
1013                         return -1;
1014
1015                 idx++;
1016         }
1017
1018         return 0;
1019 }
1020
1021 /*
1022  * Warnings shouldn't be reported for ignored functions.
1023  */
1024 static void add_ignores(struct objtool_file *file)
1025 {
1026         struct instruction *insn;
1027         struct section *rsec;
1028         struct symbol *func;
1029         struct reloc *reloc;
1030
1031         rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1032         if (!rsec)
1033                 return;
1034
1035         for_each_reloc(rsec, reloc) {
1036                 switch (reloc->sym->type) {
1037                 case STT_FUNC:
1038                         func = reloc->sym;
1039                         break;
1040
1041                 case STT_SECTION:
1042                         func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1043                         if (!func)
1044                                 continue;
1045                         break;
1046
1047                 default:
1048                         WARN("unexpected relocation symbol type in %s: %d",
1049                              rsec->name, reloc->sym->type);
1050                         continue;
1051                 }
1052
1053                 func_for_each_insn(file, func, insn)
1054                         insn->ignore = true;
1055         }
1056 }
1057
1058 /*
1059  * This is a whitelist of functions that is allowed to be called with AC set.
1060  * The list is meant to be minimal and only contains compiler instrumentation
1061  * ABI and a few functions used to implement *_{to,from}_user() functions.
1062  *
1063  * These functions must not directly change AC, but may PUSHF/POPF.
1064  */
1065 static const char *uaccess_safe_builtin[] = {
1066         /* KASAN */
1067         "kasan_report",
1068         "kasan_check_range",
1069         /* KASAN out-of-line */
1070         "__asan_loadN_noabort",
1071         "__asan_load1_noabort",
1072         "__asan_load2_noabort",
1073         "__asan_load4_noabort",
1074         "__asan_load8_noabort",
1075         "__asan_load16_noabort",
1076         "__asan_storeN_noabort",
1077         "__asan_store1_noabort",
1078         "__asan_store2_noabort",
1079         "__asan_store4_noabort",
1080         "__asan_store8_noabort",
1081         "__asan_store16_noabort",
1082         "__kasan_check_read",
1083         "__kasan_check_write",
1084         /* KASAN in-line */
1085         "__asan_report_load_n_noabort",
1086         "__asan_report_load1_noabort",
1087         "__asan_report_load2_noabort",
1088         "__asan_report_load4_noabort",
1089         "__asan_report_load8_noabort",
1090         "__asan_report_load16_noabort",
1091         "__asan_report_store_n_noabort",
1092         "__asan_report_store1_noabort",
1093         "__asan_report_store2_noabort",
1094         "__asan_report_store4_noabort",
1095         "__asan_report_store8_noabort",
1096         "__asan_report_store16_noabort",
1097         /* KCSAN */
1098         "__kcsan_check_access",
1099         "__kcsan_mb",
1100         "__kcsan_wmb",
1101         "__kcsan_rmb",
1102         "__kcsan_release",
1103         "kcsan_found_watchpoint",
1104         "kcsan_setup_watchpoint",
1105         "kcsan_check_scoped_accesses",
1106         "kcsan_disable_current",
1107         "kcsan_enable_current_nowarn",
1108         /* KCSAN/TSAN */
1109         "__tsan_func_entry",
1110         "__tsan_func_exit",
1111         "__tsan_read_range",
1112         "__tsan_write_range",
1113         "__tsan_read1",
1114         "__tsan_read2",
1115         "__tsan_read4",
1116         "__tsan_read8",
1117         "__tsan_read16",
1118         "__tsan_write1",
1119         "__tsan_write2",
1120         "__tsan_write4",
1121         "__tsan_write8",
1122         "__tsan_write16",
1123         "__tsan_read_write1",
1124         "__tsan_read_write2",
1125         "__tsan_read_write4",
1126         "__tsan_read_write8",
1127         "__tsan_read_write16",
1128         "__tsan_volatile_read1",
1129         "__tsan_volatile_read2",
1130         "__tsan_volatile_read4",
1131         "__tsan_volatile_read8",
1132         "__tsan_volatile_read16",
1133         "__tsan_volatile_write1",
1134         "__tsan_volatile_write2",
1135         "__tsan_volatile_write4",
1136         "__tsan_volatile_write8",
1137         "__tsan_volatile_write16",
1138         "__tsan_atomic8_load",
1139         "__tsan_atomic16_load",
1140         "__tsan_atomic32_load",
1141         "__tsan_atomic64_load",
1142         "__tsan_atomic8_store",
1143         "__tsan_atomic16_store",
1144         "__tsan_atomic32_store",
1145         "__tsan_atomic64_store",
1146         "__tsan_atomic8_exchange",
1147         "__tsan_atomic16_exchange",
1148         "__tsan_atomic32_exchange",
1149         "__tsan_atomic64_exchange",
1150         "__tsan_atomic8_fetch_add",
1151         "__tsan_atomic16_fetch_add",
1152         "__tsan_atomic32_fetch_add",
1153         "__tsan_atomic64_fetch_add",
1154         "__tsan_atomic8_fetch_sub",
1155         "__tsan_atomic16_fetch_sub",
1156         "__tsan_atomic32_fetch_sub",
1157         "__tsan_atomic64_fetch_sub",
1158         "__tsan_atomic8_fetch_and",
1159         "__tsan_atomic16_fetch_and",
1160         "__tsan_atomic32_fetch_and",
1161         "__tsan_atomic64_fetch_and",
1162         "__tsan_atomic8_fetch_or",
1163         "__tsan_atomic16_fetch_or",
1164         "__tsan_atomic32_fetch_or",
1165         "__tsan_atomic64_fetch_or",
1166         "__tsan_atomic8_fetch_xor",
1167         "__tsan_atomic16_fetch_xor",
1168         "__tsan_atomic32_fetch_xor",
1169         "__tsan_atomic64_fetch_xor",
1170         "__tsan_atomic8_fetch_nand",
1171         "__tsan_atomic16_fetch_nand",
1172         "__tsan_atomic32_fetch_nand",
1173         "__tsan_atomic64_fetch_nand",
1174         "__tsan_atomic8_compare_exchange_strong",
1175         "__tsan_atomic16_compare_exchange_strong",
1176         "__tsan_atomic32_compare_exchange_strong",
1177         "__tsan_atomic64_compare_exchange_strong",
1178         "__tsan_atomic8_compare_exchange_weak",
1179         "__tsan_atomic16_compare_exchange_weak",
1180         "__tsan_atomic32_compare_exchange_weak",
1181         "__tsan_atomic64_compare_exchange_weak",
1182         "__tsan_atomic8_compare_exchange_val",
1183         "__tsan_atomic16_compare_exchange_val",
1184         "__tsan_atomic32_compare_exchange_val",
1185         "__tsan_atomic64_compare_exchange_val",
1186         "__tsan_atomic_thread_fence",
1187         "__tsan_atomic_signal_fence",
1188         "__tsan_unaligned_read16",
1189         "__tsan_unaligned_write16",
1190         /* KCOV */
1191         "write_comp_data",
1192         "check_kcov_mode",
1193         "__sanitizer_cov_trace_pc",
1194         "__sanitizer_cov_trace_const_cmp1",
1195         "__sanitizer_cov_trace_const_cmp2",
1196         "__sanitizer_cov_trace_const_cmp4",
1197         "__sanitizer_cov_trace_const_cmp8",
1198         "__sanitizer_cov_trace_cmp1",
1199         "__sanitizer_cov_trace_cmp2",
1200         "__sanitizer_cov_trace_cmp4",
1201         "__sanitizer_cov_trace_cmp8",
1202         "__sanitizer_cov_trace_switch",
1203         /* KMSAN */
1204         "kmsan_copy_to_user",
1205         "kmsan_report",
1206         "kmsan_unpoison_entry_regs",
1207         "kmsan_unpoison_memory",
1208         "__msan_chain_origin",
1209         "__msan_get_context_state",
1210         "__msan_instrument_asm_store",
1211         "__msan_metadata_ptr_for_load_1",
1212         "__msan_metadata_ptr_for_load_2",
1213         "__msan_metadata_ptr_for_load_4",
1214         "__msan_metadata_ptr_for_load_8",
1215         "__msan_metadata_ptr_for_load_n",
1216         "__msan_metadata_ptr_for_store_1",
1217         "__msan_metadata_ptr_for_store_2",
1218         "__msan_metadata_ptr_for_store_4",
1219         "__msan_metadata_ptr_for_store_8",
1220         "__msan_metadata_ptr_for_store_n",
1221         "__msan_poison_alloca",
1222         "__msan_warning",
1223         /* UBSAN */
1224         "ubsan_type_mismatch_common",
1225         "__ubsan_handle_type_mismatch",
1226         "__ubsan_handle_type_mismatch_v1",
1227         "__ubsan_handle_shift_out_of_bounds",
1228         "__ubsan_handle_load_invalid_value",
1229         /* STACKLEAK */
1230         "stackleak_track_stack",
1231         /* misc */
1232         "csum_partial_copy_generic",
1233         "copy_mc_fragile",
1234         "copy_mc_fragile_handle_tail",
1235         "copy_mc_enhanced_fast_string",
1236         "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1237         "rep_stos_alternative",
1238         "rep_movs_alternative",
1239         "__copy_user_nocache",
1240         NULL
1241 };
1242
1243 static void add_uaccess_safe(struct objtool_file *file)
1244 {
1245         struct symbol *func;
1246         const char **name;
1247
1248         if (!opts.uaccess)
1249                 return;
1250
1251         for (name = uaccess_safe_builtin; *name; name++) {
1252                 func = find_symbol_by_name(file->elf, *name);
1253                 if (!func)
1254                         continue;
1255
1256                 func->uaccess_safe = true;
1257         }
1258 }
1259
1260 /*
1261  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1262  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1263  * But it at least allows objtool to understand the control flow *around* the
1264  * retpoline.
1265  */
1266 static int add_ignore_alternatives(struct objtool_file *file)
1267 {
1268         struct section *rsec;
1269         struct reloc *reloc;
1270         struct instruction *insn;
1271
1272         rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1273         if (!rsec)
1274                 return 0;
1275
1276         for_each_reloc(rsec, reloc) {
1277                 if (reloc->sym->type != STT_SECTION) {
1278                         WARN("unexpected relocation symbol type in %s", rsec->name);
1279                         return -1;
1280                 }
1281
1282                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
1283                 if (!insn) {
1284                         WARN("bad .discard.ignore_alts entry");
1285                         return -1;
1286                 }
1287
1288                 insn->ignore_alts = true;
1289         }
1290
1291         return 0;
1292 }
1293
1294 /*
1295  * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1296  * will be added to the .retpoline_sites section.
1297  */
1298 __weak bool arch_is_retpoline(struct symbol *sym)
1299 {
1300         return false;
1301 }
1302
1303 /*
1304  * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1305  * will be added to the .return_sites section.
1306  */
1307 __weak bool arch_is_rethunk(struct symbol *sym)
1308 {
1309         return false;
1310 }
1311
1312 /*
1313  * Symbols that are embedded inside other instructions, because sometimes crazy
1314  * code exists. These are mostly ignored for validation purposes.
1315  */
1316 __weak bool arch_is_embedded_insn(struct symbol *sym)
1317 {
1318         return false;
1319 }
1320
1321 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1322 {
1323         struct reloc *reloc;
1324
1325         if (insn->no_reloc)
1326                 return NULL;
1327
1328         if (!file)
1329                 return NULL;
1330
1331         reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1332                                          insn->offset, insn->len);
1333         if (!reloc) {
1334                 insn->no_reloc = 1;
1335                 return NULL;
1336         }
1337
1338         return reloc;
1339 }
1340
1341 static void remove_insn_ops(struct instruction *insn)
1342 {
1343         struct stack_op *op, *next;
1344
1345         for (op = insn->stack_ops; op; op = next) {
1346                 next = op->next;
1347                 free(op);
1348         }
1349         insn->stack_ops = NULL;
1350 }
1351
1352 static void annotate_call_site(struct objtool_file *file,
1353                                struct instruction *insn, bool sibling)
1354 {
1355         struct reloc *reloc = insn_reloc(file, insn);
1356         struct symbol *sym = insn_call_dest(insn);
1357
1358         if (!sym)
1359                 sym = reloc->sym;
1360
1361         /*
1362          * Alternative replacement code is just template code which is
1363          * sometimes copied to the original instruction. For now, don't
1364          * annotate it. (In the future we might consider annotating the
1365          * original instruction if/when it ever makes sense to do so.)
1366          */
1367         if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1368                 return;
1369
1370         if (sym->static_call_tramp) {
1371                 list_add_tail(&insn->call_node, &file->static_call_list);
1372                 return;
1373         }
1374
1375         if (sym->retpoline_thunk) {
1376                 list_add_tail(&insn->call_node, &file->retpoline_call_list);
1377                 return;
1378         }
1379
1380         /*
1381          * Many compilers cannot disable KCOV or sanitizer calls with a function
1382          * attribute so they need a little help, NOP out any such calls from
1383          * noinstr text.
1384          */
1385         if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1386                 if (reloc)
1387                         set_reloc_type(file->elf, reloc, R_NONE);
1388
1389                 elf_write_insn(file->elf, insn->sec,
1390                                insn->offset, insn->len,
1391                                sibling ? arch_ret_insn(insn->len)
1392                                        : arch_nop_insn(insn->len));
1393
1394                 insn->type = sibling ? INSN_RETURN : INSN_NOP;
1395
1396                 if (sibling) {
1397                         /*
1398                          * We've replaced the tail-call JMP insn by two new
1399                          * insn: RET; INT3, except we only have a single struct
1400                          * insn here. Mark it retpoline_safe to avoid the SLS
1401                          * warning, instead of adding another insn.
1402                          */
1403                         insn->retpoline_safe = true;
1404                 }
1405
1406                 return;
1407         }
1408
1409         if (opts.mcount && sym->fentry) {
1410                 if (sibling)
1411                         WARN_INSN(insn, "tail call to __fentry__ !?!?");
1412                 if (opts.mnop) {
1413                         if (reloc)
1414                                 set_reloc_type(file->elf, reloc, R_NONE);
1415
1416                         elf_write_insn(file->elf, insn->sec,
1417                                        insn->offset, insn->len,
1418                                        arch_nop_insn(insn->len));
1419
1420                         insn->type = INSN_NOP;
1421                 }
1422
1423                 list_add_tail(&insn->call_node, &file->mcount_loc_list);
1424                 return;
1425         }
1426
1427         if (insn->type == INSN_CALL && !insn->sec->init)
1428                 list_add_tail(&insn->call_node, &file->call_list);
1429
1430         if (!sibling && dead_end_function(file, sym))
1431                 insn->dead_end = true;
1432 }
1433
1434 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1435                           struct symbol *dest, bool sibling)
1436 {
1437         insn->_call_dest = dest;
1438         if (!dest)
1439                 return;
1440
1441         /*
1442          * Whatever stack impact regular CALLs have, should be undone
1443          * by the RETURN of the called function.
1444          *
1445          * Annotated intra-function calls retain the stack_ops but
1446          * are converted to JUMP, see read_intra_function_calls().
1447          */
1448         remove_insn_ops(insn);
1449
1450         annotate_call_site(file, insn, sibling);
1451 }
1452
1453 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1454 {
1455         /*
1456          * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1457          * so convert them accordingly.
1458          */
1459         switch (insn->type) {
1460         case INSN_CALL:
1461                 insn->type = INSN_CALL_DYNAMIC;
1462                 break;
1463         case INSN_JUMP_UNCONDITIONAL:
1464                 insn->type = INSN_JUMP_DYNAMIC;
1465                 break;
1466         case INSN_JUMP_CONDITIONAL:
1467                 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1468                 break;
1469         default:
1470                 return;
1471         }
1472
1473         insn->retpoline_safe = true;
1474
1475         /*
1476          * Whatever stack impact regular CALLs have, should be undone
1477          * by the RETURN of the called function.
1478          *
1479          * Annotated intra-function calls retain the stack_ops but
1480          * are converted to JUMP, see read_intra_function_calls().
1481          */
1482         remove_insn_ops(insn);
1483
1484         annotate_call_site(file, insn, false);
1485 }
1486
1487 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1488 {
1489         /*
1490          * Return thunk tail calls are really just returns in disguise,
1491          * so convert them accordingly.
1492          */
1493         insn->type = INSN_RETURN;
1494         insn->retpoline_safe = true;
1495
1496         if (add)
1497                 list_add_tail(&insn->call_node, &file->return_thunk_list);
1498 }
1499
1500 static bool is_first_func_insn(struct objtool_file *file,
1501                                struct instruction *insn, struct symbol *sym)
1502 {
1503         if (insn->offset == sym->offset)
1504                 return true;
1505
1506         /* Allow direct CALL/JMP past ENDBR */
1507         if (opts.ibt) {
1508                 struct instruction *prev = prev_insn_same_sym(file, insn);
1509
1510                 if (prev && prev->type == INSN_ENDBR &&
1511                     insn->offset == sym->offset + prev->len)
1512                         return true;
1513         }
1514
1515         return false;
1516 }
1517
1518 /*
1519  * A sibling call is a tail-call to another symbol -- to differentiate from a
1520  * recursive tail-call which is to the same symbol.
1521  */
1522 static bool jump_is_sibling_call(struct objtool_file *file,
1523                                  struct instruction *from, struct instruction *to)
1524 {
1525         struct symbol *fs = from->sym;
1526         struct symbol *ts = to->sym;
1527
1528         /* Not a sibling call if from/to a symbol hole */
1529         if (!fs || !ts)
1530                 return false;
1531
1532         /* Not a sibling call if not targeting the start of a symbol. */
1533         if (!is_first_func_insn(file, to, ts))
1534                 return false;
1535
1536         /* Disallow sibling calls into STT_NOTYPE */
1537         if (ts->type == STT_NOTYPE)
1538                 return false;
1539
1540         /* Must not be self to be a sibling */
1541         return fs->pfunc != ts->pfunc;
1542 }
1543
1544 /*
1545  * Find the destination instructions for all jumps.
1546  */
1547 static int add_jump_destinations(struct objtool_file *file)
1548 {
1549         struct instruction *insn, *jump_dest;
1550         struct reloc *reloc;
1551         struct section *dest_sec;
1552         unsigned long dest_off;
1553
1554         for_each_insn(file, insn) {
1555                 if (insn->jump_dest) {
1556                         /*
1557                          * handle_group_alt() may have previously set
1558                          * 'jump_dest' for some alternatives.
1559                          */
1560                         continue;
1561                 }
1562                 if (!is_static_jump(insn))
1563                         continue;
1564
1565                 reloc = insn_reloc(file, insn);
1566                 if (!reloc) {
1567                         dest_sec = insn->sec;
1568                         dest_off = arch_jump_destination(insn);
1569                 } else if (reloc->sym->type == STT_SECTION) {
1570                         dest_sec = reloc->sym->sec;
1571                         dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1572                 } else if (reloc->sym->retpoline_thunk) {
1573                         add_retpoline_call(file, insn);
1574                         continue;
1575                 } else if (reloc->sym->return_thunk) {
1576                         add_return_call(file, insn, true);
1577                         continue;
1578                 } else if (insn_func(insn)) {
1579                         /*
1580                          * External sibling call or internal sibling call with
1581                          * STT_FUNC reloc.
1582                          */
1583                         add_call_dest(file, insn, reloc->sym, true);
1584                         continue;
1585                 } else if (reloc->sym->sec->idx) {
1586                         dest_sec = reloc->sym->sec;
1587                         dest_off = reloc->sym->sym.st_value +
1588                                    arch_dest_reloc_offset(reloc_addend(reloc));
1589                 } else {
1590                         /* non-func asm code jumping to another file */
1591                         continue;
1592                 }
1593
1594                 jump_dest = find_insn(file, dest_sec, dest_off);
1595                 if (!jump_dest) {
1596                         struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1597
1598                         /*
1599                          * This is a special case for retbleed_untrain_ret().
1600                          * It jumps to __x86_return_thunk(), but objtool
1601                          * can't find the thunk's starting RET
1602                          * instruction, because the RET is also in the
1603                          * middle of another instruction.  Objtool only
1604                          * knows about the outer instruction.
1605                          */
1606                         if (sym && sym->embedded_insn) {
1607                                 add_return_call(file, insn, false);
1608                                 continue;
1609                         }
1610
1611                         WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1612                                   dest_sec->name, dest_off);
1613                         return -1;
1614                 }
1615
1616                 /*
1617                  * An intra-TU jump in retpoline.o might not have a relocation
1618                  * for its jump dest, in which case the above
1619                  * add_{retpoline,return}_call() didn't happen.
1620                  */
1621                 if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1622                         if (jump_dest->sym->retpoline_thunk) {
1623                                 add_retpoline_call(file, insn);
1624                                 continue;
1625                         }
1626                         if (jump_dest->sym->return_thunk) {
1627                                 add_return_call(file, insn, true);
1628                                 continue;
1629                         }
1630                 }
1631
1632                 /*
1633                  * Cross-function jump.
1634                  */
1635                 if (insn_func(insn) && insn_func(jump_dest) &&
1636                     insn_func(insn) != insn_func(jump_dest)) {
1637
1638                         /*
1639                          * For GCC 8+, create parent/child links for any cold
1640                          * subfunctions.  This is _mostly_ redundant with a
1641                          * similar initialization in read_symbols().
1642                          *
1643                          * If a function has aliases, we want the *first* such
1644                          * function in the symbol table to be the subfunction's
1645                          * parent.  In that case we overwrite the
1646                          * initialization done in read_symbols().
1647                          *
1648                          * However this code can't completely replace the
1649                          * read_symbols() code because this doesn't detect the
1650                          * case where the parent function's only reference to a
1651                          * subfunction is through a jump table.
1652                          */
1653                         if (!strstr(insn_func(insn)->name, ".cold") &&
1654                             strstr(insn_func(jump_dest)->name, ".cold")) {
1655                                 insn_func(insn)->cfunc = insn_func(jump_dest);
1656                                 insn_func(jump_dest)->pfunc = insn_func(insn);
1657                         }
1658                 }
1659
1660                 if (jump_is_sibling_call(file, insn, jump_dest)) {
1661                         /*
1662                          * Internal sibling call without reloc or with
1663                          * STT_SECTION reloc.
1664                          */
1665                         add_call_dest(file, insn, insn_func(jump_dest), true);
1666                         continue;
1667                 }
1668
1669                 insn->jump_dest = jump_dest;
1670         }
1671
1672         return 0;
1673 }
1674
1675 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1676 {
1677         struct symbol *call_dest;
1678
1679         call_dest = find_func_by_offset(sec, offset);
1680         if (!call_dest)
1681                 call_dest = find_symbol_by_offset(sec, offset);
1682
1683         return call_dest;
1684 }
1685
1686 /*
1687  * Find the destination instructions for all calls.
1688  */
1689 static int add_call_destinations(struct objtool_file *file)
1690 {
1691         struct instruction *insn;
1692         unsigned long dest_off;
1693         struct symbol *dest;
1694         struct reloc *reloc;
1695
1696         for_each_insn(file, insn) {
1697                 if (insn->type != INSN_CALL)
1698                         continue;
1699
1700                 reloc = insn_reloc(file, insn);
1701                 if (!reloc) {
1702                         dest_off = arch_jump_destination(insn);
1703                         dest = find_call_destination(insn->sec, dest_off);
1704
1705                         add_call_dest(file, insn, dest, false);
1706
1707                         if (insn->ignore)
1708                                 continue;
1709
1710                         if (!insn_call_dest(insn)) {
1711                                 WARN_INSN(insn, "unannotated intra-function call");
1712                                 return -1;
1713                         }
1714
1715                         if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1716                                 WARN_INSN(insn, "unsupported call to non-function");
1717                                 return -1;
1718                         }
1719
1720                 } else if (reloc->sym->type == STT_SECTION) {
1721                         dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1722                         dest = find_call_destination(reloc->sym->sec, dest_off);
1723                         if (!dest) {
1724                                 WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1725                                           reloc->sym->sec->name, dest_off);
1726                                 return -1;
1727                         }
1728
1729                         add_call_dest(file, insn, dest, false);
1730
1731                 } else if (reloc->sym->retpoline_thunk) {
1732                         add_retpoline_call(file, insn);
1733
1734                 } else
1735                         add_call_dest(file, insn, reloc->sym, false);
1736         }
1737
1738         return 0;
1739 }
1740
1741 /*
1742  * The .alternatives section requires some extra special care over and above
1743  * other special sections because alternatives are patched in place.
1744  */
1745 static int handle_group_alt(struct objtool_file *file,
1746                             struct special_alt *special_alt,
1747                             struct instruction *orig_insn,
1748                             struct instruction **new_insn)
1749 {
1750         struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1751         struct alt_group *orig_alt_group, *new_alt_group;
1752         unsigned long dest_off;
1753
1754         orig_alt_group = orig_insn->alt_group;
1755         if (!orig_alt_group) {
1756                 struct instruction *last_orig_insn = NULL;
1757
1758                 orig_alt_group = malloc(sizeof(*orig_alt_group));
1759                 if (!orig_alt_group) {
1760                         WARN("malloc failed");
1761                         return -1;
1762                 }
1763                 orig_alt_group->cfi = calloc(special_alt->orig_len,
1764                                              sizeof(struct cfi_state *));
1765                 if (!orig_alt_group->cfi) {
1766                         WARN("calloc failed");
1767                         return -1;
1768                 }
1769
1770                 insn = orig_insn;
1771                 sec_for_each_insn_from(file, insn) {
1772                         if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1773                                 break;
1774
1775                         insn->alt_group = orig_alt_group;
1776                         last_orig_insn = insn;
1777                 }
1778                 orig_alt_group->orig_group = NULL;
1779                 orig_alt_group->first_insn = orig_insn;
1780                 orig_alt_group->last_insn = last_orig_insn;
1781                 orig_alt_group->nop = NULL;
1782         } else {
1783                 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1784                     orig_alt_group->first_insn->offset != special_alt->orig_len) {
1785                         WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1786                                   orig_alt_group->last_insn->offset +
1787                                   orig_alt_group->last_insn->len -
1788                                   orig_alt_group->first_insn->offset,
1789                                   special_alt->orig_len);
1790                         return -1;
1791                 }
1792         }
1793
1794         new_alt_group = malloc(sizeof(*new_alt_group));
1795         if (!new_alt_group) {
1796                 WARN("malloc failed");
1797                 return -1;
1798         }
1799
1800         if (special_alt->new_len < special_alt->orig_len) {
1801                 /*
1802                  * Insert a fake nop at the end to make the replacement
1803                  * alt_group the same size as the original.  This is needed to
1804                  * allow propagate_alt_cfi() to do its magic.  When the last
1805                  * instruction affects the stack, the instruction after it (the
1806                  * nop) will propagate the new state to the shared CFI array.
1807                  */
1808                 nop = malloc(sizeof(*nop));
1809                 if (!nop) {
1810                         WARN("malloc failed");
1811                         return -1;
1812                 }
1813                 memset(nop, 0, sizeof(*nop));
1814
1815                 nop->sec = special_alt->new_sec;
1816                 nop->offset = special_alt->new_off + special_alt->new_len;
1817                 nop->len = special_alt->orig_len - special_alt->new_len;
1818                 nop->type = INSN_NOP;
1819                 nop->sym = orig_insn->sym;
1820                 nop->alt_group = new_alt_group;
1821                 nop->ignore = orig_insn->ignore_alts;
1822         }
1823
1824         if (!special_alt->new_len) {
1825                 *new_insn = nop;
1826                 goto end;
1827         }
1828
1829         insn = *new_insn;
1830         sec_for_each_insn_from(file, insn) {
1831                 struct reloc *alt_reloc;
1832
1833                 if (insn->offset >= special_alt->new_off + special_alt->new_len)
1834                         break;
1835
1836                 last_new_insn = insn;
1837
1838                 insn->ignore = orig_insn->ignore_alts;
1839                 insn->sym = orig_insn->sym;
1840                 insn->alt_group = new_alt_group;
1841
1842                 /*
1843                  * Since alternative replacement code is copy/pasted by the
1844                  * kernel after applying relocations, generally such code can't
1845                  * have relative-address relocation references to outside the
1846                  * .altinstr_replacement section, unless the arch's
1847                  * alternatives code can adjust the relative offsets
1848                  * accordingly.
1849                  */
1850                 alt_reloc = insn_reloc(file, insn);
1851                 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1852                     !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1853
1854                         WARN_INSN(insn, "unsupported relocation in alternatives section");
1855                         return -1;
1856                 }
1857
1858                 if (!is_static_jump(insn))
1859                         continue;
1860
1861                 if (!insn->immediate)
1862                         continue;
1863
1864                 dest_off = arch_jump_destination(insn);
1865                 if (dest_off == special_alt->new_off + special_alt->new_len) {
1866                         insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn);
1867                         if (!insn->jump_dest) {
1868                                 WARN_INSN(insn, "can't find alternative jump destination");
1869                                 return -1;
1870                         }
1871                 }
1872         }
1873
1874         if (!last_new_insn) {
1875                 WARN_FUNC("can't find last new alternative instruction",
1876                           special_alt->new_sec, special_alt->new_off);
1877                 return -1;
1878         }
1879
1880 end:
1881         new_alt_group->orig_group = orig_alt_group;
1882         new_alt_group->first_insn = *new_insn;
1883         new_alt_group->last_insn = last_new_insn;
1884         new_alt_group->nop = nop;
1885         new_alt_group->cfi = orig_alt_group->cfi;
1886         return 0;
1887 }
1888
1889 /*
1890  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1891  * If the original instruction is a jump, make the alt entry an effective nop
1892  * by just skipping the original instruction.
1893  */
1894 static int handle_jump_alt(struct objtool_file *file,
1895                            struct special_alt *special_alt,
1896                            struct instruction *orig_insn,
1897                            struct instruction **new_insn)
1898 {
1899         if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1900             orig_insn->type != INSN_NOP) {
1901
1902                 WARN_INSN(orig_insn, "unsupported instruction at jump label");
1903                 return -1;
1904         }
1905
1906         if (opts.hack_jump_label && special_alt->key_addend & 2) {
1907                 struct reloc *reloc = insn_reloc(file, orig_insn);
1908
1909                 if (reloc)
1910                         set_reloc_type(file->elf, reloc, R_NONE);
1911                 elf_write_insn(file->elf, orig_insn->sec,
1912                                orig_insn->offset, orig_insn->len,
1913                                arch_nop_insn(orig_insn->len));
1914                 orig_insn->type = INSN_NOP;
1915         }
1916
1917         if (orig_insn->type == INSN_NOP) {
1918                 if (orig_insn->len == 2)
1919                         file->jl_nop_short++;
1920                 else
1921                         file->jl_nop_long++;
1922
1923                 return 0;
1924         }
1925
1926         if (orig_insn->len == 2)
1927                 file->jl_short++;
1928         else
1929                 file->jl_long++;
1930
1931         *new_insn = next_insn_same_sec(file, orig_insn);
1932         return 0;
1933 }
1934
1935 /*
1936  * Read all the special sections which have alternate instructions which can be
1937  * patched in or redirected to at runtime.  Each instruction having alternate
1938  * instruction(s) has them added to its insn->alts list, which will be
1939  * traversed in validate_branch().
1940  */
1941 static int add_special_section_alts(struct objtool_file *file)
1942 {
1943         struct list_head special_alts;
1944         struct instruction *orig_insn, *new_insn;
1945         struct special_alt *special_alt, *tmp;
1946         struct alternative *alt;
1947         int ret;
1948
1949         ret = special_get_alts(file->elf, &special_alts);
1950         if (ret)
1951                 return ret;
1952
1953         list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1954
1955                 orig_insn = find_insn(file, special_alt->orig_sec,
1956                                       special_alt->orig_off);
1957                 if (!orig_insn) {
1958                         WARN_FUNC("special: can't find orig instruction",
1959                                   special_alt->orig_sec, special_alt->orig_off);
1960                         ret = -1;
1961                         goto out;
1962                 }
1963
1964                 new_insn = NULL;
1965                 if (!special_alt->group || special_alt->new_len) {
1966                         new_insn = find_insn(file, special_alt->new_sec,
1967                                              special_alt->new_off);
1968                         if (!new_insn) {
1969                                 WARN_FUNC("special: can't find new instruction",
1970                                           special_alt->new_sec,
1971                                           special_alt->new_off);
1972                                 ret = -1;
1973                                 goto out;
1974                         }
1975                 }
1976
1977                 if (special_alt->group) {
1978                         if (!special_alt->orig_len) {
1979                                 WARN_INSN(orig_insn, "empty alternative entry");
1980                                 continue;
1981                         }
1982
1983                         ret = handle_group_alt(file, special_alt, orig_insn,
1984                                                &new_insn);
1985                         if (ret)
1986                                 goto out;
1987                 } else if (special_alt->jump_or_nop) {
1988                         ret = handle_jump_alt(file, special_alt, orig_insn,
1989                                               &new_insn);
1990                         if (ret)
1991                                 goto out;
1992                 }
1993
1994                 alt = malloc(sizeof(*alt));
1995                 if (!alt) {
1996                         WARN("malloc failed");
1997                         ret = -1;
1998                         goto out;
1999                 }
2000
2001                 alt->insn = new_insn;
2002                 alt->skip_orig = special_alt->skip_orig;
2003                 orig_insn->ignore_alts |= special_alt->skip_alt;
2004                 alt->next = orig_insn->alts;
2005                 orig_insn->alts = alt;
2006
2007                 list_del(&special_alt->list);
2008                 free(special_alt);
2009         }
2010
2011         if (opts.stats) {
2012                 printf("jl\\\tNOP\tJMP\n");
2013                 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2014                 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2015         }
2016
2017 out:
2018         return ret;
2019 }
2020
2021 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2022                           struct reloc *next_table)
2023 {
2024         struct symbol *pfunc = insn_func(insn)->pfunc;
2025         struct reloc *table = insn_jump_table(insn);
2026         struct instruction *dest_insn;
2027         unsigned int prev_offset = 0;
2028         struct reloc *reloc = table;
2029         struct alternative *alt;
2030
2031         /*
2032          * Each @reloc is a switch table relocation which points to the target
2033          * instruction.
2034          */
2035         for_each_reloc_from(table->sec, reloc) {
2036
2037                 /* Check for the end of the table: */
2038                 if (reloc != table && reloc == next_table)
2039                         break;
2040
2041                 /* Make sure the table entries are consecutive: */
2042                 if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2043                         break;
2044
2045                 /* Detect function pointers from contiguous objects: */
2046                 if (reloc->sym->sec == pfunc->sec &&
2047                     reloc_addend(reloc) == pfunc->offset)
2048                         break;
2049
2050                 dest_insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2051                 if (!dest_insn)
2052                         break;
2053
2054                 /* Make sure the destination is in the same function: */
2055                 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2056                         break;
2057
2058                 alt = malloc(sizeof(*alt));
2059                 if (!alt) {
2060                         WARN("malloc failed");
2061                         return -1;
2062                 }
2063
2064                 alt->insn = dest_insn;
2065                 alt->next = insn->alts;
2066                 insn->alts = alt;
2067                 prev_offset = reloc_offset(reloc);
2068         }
2069
2070         if (!prev_offset) {
2071                 WARN_INSN(insn, "can't find switch jump table");
2072                 return -1;
2073         }
2074
2075         return 0;
2076 }
2077
2078 /*
2079  * find_jump_table() - Given a dynamic jump, find the switch jump table
2080  * associated with it.
2081  */
2082 static struct reloc *find_jump_table(struct objtool_file *file,
2083                                       struct symbol *func,
2084                                       struct instruction *insn)
2085 {
2086         struct reloc *table_reloc;
2087         struct instruction *dest_insn, *orig_insn = insn;
2088
2089         /*
2090          * Backward search using the @first_jump_src links, these help avoid
2091          * much of the 'in between' code. Which avoids us getting confused by
2092          * it.
2093          */
2094         for (;
2095              insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2096              insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2097
2098                 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2099                         break;
2100
2101                 /* allow small jumps within the range */
2102                 if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2103                     insn->jump_dest &&
2104                     (insn->jump_dest->offset <= insn->offset ||
2105                      insn->jump_dest->offset > orig_insn->offset))
2106                     break;
2107
2108                 table_reloc = arch_find_switch_table(file, insn);
2109                 if (!table_reloc)
2110                         continue;
2111                 dest_insn = find_insn(file, table_reloc->sym->sec, reloc_addend(table_reloc));
2112                 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2113                         continue;
2114
2115                 return table_reloc;
2116         }
2117
2118         return NULL;
2119 }
2120
2121 /*
2122  * First pass: Mark the head of each jump table so that in the next pass,
2123  * we know when a given jump table ends and the next one starts.
2124  */
2125 static void mark_func_jump_tables(struct objtool_file *file,
2126                                     struct symbol *func)
2127 {
2128         struct instruction *insn, *last = NULL;
2129         struct reloc *reloc;
2130
2131         func_for_each_insn(file, func, insn) {
2132                 if (!last)
2133                         last = insn;
2134
2135                 /*
2136                  * Store back-pointers for unconditional forward jumps such
2137                  * that find_jump_table() can back-track using those and
2138                  * avoid some potentially confusing code.
2139                  */
2140                 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2141                     insn->offset > last->offset &&
2142                     insn->jump_dest->offset > insn->offset &&
2143                     !insn->jump_dest->first_jump_src) {
2144
2145                         insn->jump_dest->first_jump_src = insn;
2146                         last = insn->jump_dest;
2147                 }
2148
2149                 if (insn->type != INSN_JUMP_DYNAMIC)
2150                         continue;
2151
2152                 reloc = find_jump_table(file, func, insn);
2153                 if (reloc)
2154                         insn->_jump_table = reloc;
2155         }
2156 }
2157
2158 static int add_func_jump_tables(struct objtool_file *file,
2159                                   struct symbol *func)
2160 {
2161         struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2162         int ret = 0;
2163
2164         func_for_each_insn(file, func, insn) {
2165                 if (!insn_jump_table(insn))
2166                         continue;
2167
2168                 if (!insn_t1) {
2169                         insn_t1 = insn;
2170                         continue;
2171                 }
2172
2173                 insn_t2 = insn;
2174
2175                 ret = add_jump_table(file, insn_t1, insn_jump_table(insn_t2));
2176                 if (ret)
2177                         return ret;
2178
2179                 insn_t1 = insn_t2;
2180         }
2181
2182         if (insn_t1)
2183                 ret = add_jump_table(file, insn_t1, NULL);
2184
2185         return ret;
2186 }
2187
2188 /*
2189  * For some switch statements, gcc generates a jump table in the .rodata
2190  * section which contains a list of addresses within the function to jump to.
2191  * This finds these jump tables and adds them to the insn->alts lists.
2192  */
2193 static int add_jump_table_alts(struct objtool_file *file)
2194 {
2195         struct symbol *func;
2196         int ret;
2197
2198         if (!file->rodata)
2199                 return 0;
2200
2201         for_each_sym(file, func) {
2202                 if (func->type != STT_FUNC)
2203                         continue;
2204
2205                 mark_func_jump_tables(file, func);
2206                 ret = add_func_jump_tables(file, func);
2207                 if (ret)
2208                         return ret;
2209         }
2210
2211         return 0;
2212 }
2213
2214 static void set_func_state(struct cfi_state *state)
2215 {
2216         state->cfa = initial_func_cfi.cfa;
2217         memcpy(&state->regs, &initial_func_cfi.regs,
2218                CFI_NUM_REGS * sizeof(struct cfi_reg));
2219         state->stack_size = initial_func_cfi.cfa.offset;
2220         state->type = UNWIND_HINT_TYPE_CALL;
2221 }
2222
2223 static int read_unwind_hints(struct objtool_file *file)
2224 {
2225         struct cfi_state cfi = init_cfi;
2226         struct section *sec;
2227         struct unwind_hint *hint;
2228         struct instruction *insn;
2229         struct reloc *reloc;
2230         unsigned long offset;
2231         int i;
2232
2233         sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2234         if (!sec)
2235                 return 0;
2236
2237         if (!sec->rsec) {
2238                 WARN("missing .rela.discard.unwind_hints section");
2239                 return -1;
2240         }
2241
2242         if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2243                 WARN("struct unwind_hint size mismatch");
2244                 return -1;
2245         }
2246
2247         file->hints = true;
2248
2249         for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2250                 hint = (struct unwind_hint *)sec->data->d_buf + i;
2251
2252                 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2253                 if (!reloc) {
2254                         WARN("can't find reloc for unwind_hints[%d]", i);
2255                         return -1;
2256                 }
2257
2258                 if (reloc->sym->type == STT_SECTION) {
2259                         offset = reloc_addend(reloc);
2260                 } else if (reloc->sym->local_label) {
2261                         offset = reloc->sym->offset;
2262                 } else {
2263                         WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2264                         return -1;
2265                 }
2266
2267                 insn = find_insn(file, reloc->sym->sec, offset);
2268                 if (!insn) {
2269                         WARN("can't find insn for unwind_hints[%d]", i);
2270                         return -1;
2271                 }
2272
2273                 insn->hint = true;
2274
2275                 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2276                         insn->cfi = &force_undefined_cfi;
2277                         continue;
2278                 }
2279
2280                 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2281                         insn->hint = false;
2282                         insn->save = true;
2283                         continue;
2284                 }
2285
2286                 if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2287                         insn->restore = true;
2288                         continue;
2289                 }
2290
2291                 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2292                         struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2293
2294                         if (sym && sym->bind == STB_GLOBAL) {
2295                                 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2296                                         WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2297                                 }
2298                         }
2299                 }
2300
2301                 if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2302                         insn->cfi = &func_cfi;
2303                         continue;
2304                 }
2305
2306                 if (insn->cfi)
2307                         cfi = *(insn->cfi);
2308
2309                 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2310                         WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2311                         return -1;
2312                 }
2313
2314                 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2315                 cfi.type = hint->type;
2316                 cfi.signal = hint->signal;
2317
2318                 insn->cfi = cfi_hash_find_or_add(&cfi);
2319         }
2320
2321         return 0;
2322 }
2323
2324 static int read_noendbr_hints(struct objtool_file *file)
2325 {
2326         struct instruction *insn;
2327         struct section *rsec;
2328         struct reloc *reloc;
2329
2330         rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2331         if (!rsec)
2332                 return 0;
2333
2334         for_each_reloc(rsec, reloc) {
2335                 insn = find_insn(file, reloc->sym->sec,
2336                                  reloc->sym->offset + reloc_addend(reloc));
2337                 if (!insn) {
2338                         WARN("bad .discard.noendbr entry");
2339                         return -1;
2340                 }
2341
2342                 insn->noendbr = 1;
2343         }
2344
2345         return 0;
2346 }
2347
2348 static int read_retpoline_hints(struct objtool_file *file)
2349 {
2350         struct section *rsec;
2351         struct instruction *insn;
2352         struct reloc *reloc;
2353
2354         rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2355         if (!rsec)
2356                 return 0;
2357
2358         for_each_reloc(rsec, reloc) {
2359                 if (reloc->sym->type != STT_SECTION) {
2360                         WARN("unexpected relocation symbol type in %s", rsec->name);
2361                         return -1;
2362                 }
2363
2364                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2365                 if (!insn) {
2366                         WARN("bad .discard.retpoline_safe entry");
2367                         return -1;
2368                 }
2369
2370                 if (insn->type != INSN_JUMP_DYNAMIC &&
2371                     insn->type != INSN_CALL_DYNAMIC &&
2372                     insn->type != INSN_RETURN &&
2373                     insn->type != INSN_NOP) {
2374                         WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2375                         return -1;
2376                 }
2377
2378                 insn->retpoline_safe = true;
2379         }
2380
2381         return 0;
2382 }
2383
2384 static int read_instr_hints(struct objtool_file *file)
2385 {
2386         struct section *rsec;
2387         struct instruction *insn;
2388         struct reloc *reloc;
2389
2390         rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2391         if (!rsec)
2392                 return 0;
2393
2394         for_each_reloc(rsec, reloc) {
2395                 if (reloc->sym->type != STT_SECTION) {
2396                         WARN("unexpected relocation symbol type in %s", rsec->name);
2397                         return -1;
2398                 }
2399
2400                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2401                 if (!insn) {
2402                         WARN("bad .discard.instr_end entry");
2403                         return -1;
2404                 }
2405
2406                 insn->instr--;
2407         }
2408
2409         rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2410         if (!rsec)
2411                 return 0;
2412
2413         for_each_reloc(rsec, reloc) {
2414                 if (reloc->sym->type != STT_SECTION) {
2415                         WARN("unexpected relocation symbol type in %s", rsec->name);
2416                         return -1;
2417                 }
2418
2419                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2420                 if (!insn) {
2421                         WARN("bad .discard.instr_begin entry");
2422                         return -1;
2423                 }
2424
2425                 insn->instr++;
2426         }
2427
2428         return 0;
2429 }
2430
2431 static int read_validate_unret_hints(struct objtool_file *file)
2432 {
2433         struct section *rsec;
2434         struct instruction *insn;
2435         struct reloc *reloc;
2436
2437         rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2438         if (!rsec)
2439                 return 0;
2440
2441         for_each_reloc(rsec, reloc) {
2442                 if (reloc->sym->type != STT_SECTION) {
2443                         WARN("unexpected relocation symbol type in %s", rsec->name);
2444                         return -1;
2445                 }
2446
2447                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2448                 if (!insn) {
2449                         WARN("bad .discard.instr_end entry");
2450                         return -1;
2451                 }
2452                 insn->unret = 1;
2453         }
2454
2455         return 0;
2456 }
2457
2458
2459 static int read_intra_function_calls(struct objtool_file *file)
2460 {
2461         struct instruction *insn;
2462         struct section *rsec;
2463         struct reloc *reloc;
2464
2465         rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2466         if (!rsec)
2467                 return 0;
2468
2469         for_each_reloc(rsec, reloc) {
2470                 unsigned long dest_off;
2471
2472                 if (reloc->sym->type != STT_SECTION) {
2473                         WARN("unexpected relocation symbol type in %s",
2474                              rsec->name);
2475                         return -1;
2476                 }
2477
2478                 insn = find_insn(file, reloc->sym->sec, reloc_addend(reloc));
2479                 if (!insn) {
2480                         WARN("bad .discard.intra_function_call entry");
2481                         return -1;
2482                 }
2483
2484                 if (insn->type != INSN_CALL) {
2485                         WARN_INSN(insn, "intra_function_call not a direct call");
2486                         return -1;
2487                 }
2488
2489                 /*
2490                  * Treat intra-function CALLs as JMPs, but with a stack_op.
2491                  * See add_call_destinations(), which strips stack_ops from
2492                  * normal CALLs.
2493                  */
2494                 insn->type = INSN_JUMP_UNCONDITIONAL;
2495
2496                 dest_off = arch_jump_destination(insn);
2497                 insn->jump_dest = find_insn(file, insn->sec, dest_off);
2498                 if (!insn->jump_dest) {
2499                         WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2500                                   insn->sec->name, dest_off);
2501                         return -1;
2502                 }
2503         }
2504
2505         return 0;
2506 }
2507
2508 /*
2509  * Return true if name matches an instrumentation function, where calls to that
2510  * function from noinstr code can safely be removed, but compilers won't do so.
2511  */
2512 static bool is_profiling_func(const char *name)
2513 {
2514         /*
2515          * Many compilers cannot disable KCOV with a function attribute.
2516          */
2517         if (!strncmp(name, "__sanitizer_cov_", 16))
2518                 return true;
2519
2520         /*
2521          * Some compilers currently do not remove __tsan_func_entry/exit nor
2522          * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2523          * the __no_sanitize_thread attribute, remove them. Once the kernel's
2524          * minimum Clang version is 14.0, this can be removed.
2525          */
2526         if (!strncmp(name, "__tsan_func_", 12) ||
2527             !strcmp(name, "__tsan_atomic_signal_fence"))
2528                 return true;
2529
2530         return false;
2531 }
2532
2533 static int classify_symbols(struct objtool_file *file)
2534 {
2535         struct symbol *func;
2536
2537         for_each_sym(file, func) {
2538                 if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2539                         func->local_label = true;
2540
2541                 if (func->bind != STB_GLOBAL)
2542                         continue;
2543
2544                 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2545                              strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2546                         func->static_call_tramp = true;
2547
2548                 if (arch_is_retpoline(func))
2549                         func->retpoline_thunk = true;
2550
2551                 if (arch_is_rethunk(func))
2552                         func->return_thunk = true;
2553
2554                 if (arch_is_embedded_insn(func))
2555                         func->embedded_insn = true;
2556
2557                 if (arch_ftrace_match(func->name))
2558                         func->fentry = true;
2559
2560                 if (is_profiling_func(func->name))
2561                         func->profiling_func = true;
2562         }
2563
2564         return 0;
2565 }
2566
2567 static void mark_rodata(struct objtool_file *file)
2568 {
2569         struct section *sec;
2570         bool found = false;
2571
2572         /*
2573          * Search for the following rodata sections, each of which can
2574          * potentially contain jump tables:
2575          *
2576          * - .rodata: can contain GCC switch tables
2577          * - .rodata.<func>: same, if -fdata-sections is being used
2578          * - .rodata..c_jump_table: contains C annotated jump tables
2579          *
2580          * .rodata.str1.* sections are ignored; they don't contain jump tables.
2581          */
2582         for_each_sec(file, sec) {
2583                 if (!strncmp(sec->name, ".rodata", 7) &&
2584                     !strstr(sec->name, ".str1.")) {
2585                         sec->rodata = true;
2586                         found = true;
2587                 }
2588         }
2589
2590         file->rodata = found;
2591 }
2592
2593 static int decode_sections(struct objtool_file *file)
2594 {
2595         int ret;
2596
2597         mark_rodata(file);
2598
2599         ret = init_pv_ops(file);
2600         if (ret)
2601                 return ret;
2602
2603         /*
2604          * Must be before add_{jump_call}_destination.
2605          */
2606         ret = classify_symbols(file);
2607         if (ret)
2608                 return ret;
2609
2610         ret = decode_instructions(file);
2611         if (ret)
2612                 return ret;
2613
2614         add_ignores(file);
2615         add_uaccess_safe(file);
2616
2617         ret = add_ignore_alternatives(file);
2618         if (ret)
2619                 return ret;
2620
2621         /*
2622          * Must be before read_unwind_hints() since that needs insn->noendbr.
2623          */
2624         ret = read_noendbr_hints(file);
2625         if (ret)
2626                 return ret;
2627
2628         /*
2629          * Must be before add_jump_destinations(), which depends on 'func'
2630          * being set for alternatives, to enable proper sibling call detection.
2631          */
2632         if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2633                 ret = add_special_section_alts(file);
2634                 if (ret)
2635                         return ret;
2636         }
2637
2638         ret = add_jump_destinations(file);
2639         if (ret)
2640                 return ret;
2641
2642         /*
2643          * Must be before add_call_destination(); it changes INSN_CALL to
2644          * INSN_JUMP.
2645          */
2646         ret = read_intra_function_calls(file);
2647         if (ret)
2648                 return ret;
2649
2650         ret = add_call_destinations(file);
2651         if (ret)
2652                 return ret;
2653
2654         /*
2655          * Must be after add_call_destinations() such that it can override
2656          * dead_end_function() marks.
2657          */
2658         ret = add_dead_ends(file);
2659         if (ret)
2660                 return ret;
2661
2662         ret = add_jump_table_alts(file);
2663         if (ret)
2664                 return ret;
2665
2666         ret = read_unwind_hints(file);
2667         if (ret)
2668                 return ret;
2669
2670         ret = read_retpoline_hints(file);
2671         if (ret)
2672                 return ret;
2673
2674         ret = read_instr_hints(file);
2675         if (ret)
2676                 return ret;
2677
2678         ret = read_validate_unret_hints(file);
2679         if (ret)
2680                 return ret;
2681
2682         return 0;
2683 }
2684
2685 static bool is_special_call(struct instruction *insn)
2686 {
2687         if (insn->type == INSN_CALL) {
2688                 struct symbol *dest = insn_call_dest(insn);
2689
2690                 if (!dest)
2691                         return false;
2692
2693                 if (dest->fentry || dest->embedded_insn)
2694                         return true;
2695         }
2696
2697         return false;
2698 }
2699
2700 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2701 {
2702         struct cfi_state *cfi = &state->cfi;
2703         int i;
2704
2705         if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2706                 return true;
2707
2708         if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2709                 return true;
2710
2711         if (cfi->stack_size != initial_func_cfi.cfa.offset)
2712                 return true;
2713
2714         for (i = 0; i < CFI_NUM_REGS; i++) {
2715                 if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2716                     cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2717                         return true;
2718         }
2719
2720         return false;
2721 }
2722
2723 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2724                                 int expected_offset)
2725 {
2726         return reg->base == CFI_CFA &&
2727                reg->offset == expected_offset;
2728 }
2729
2730 static bool has_valid_stack_frame(struct insn_state *state)
2731 {
2732         struct cfi_state *cfi = &state->cfi;
2733
2734         if (cfi->cfa.base == CFI_BP &&
2735             check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2736             check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2737                 return true;
2738
2739         if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2740                 return true;
2741
2742         return false;
2743 }
2744
2745 static int update_cfi_state_regs(struct instruction *insn,
2746                                   struct cfi_state *cfi,
2747                                   struct stack_op *op)
2748 {
2749         struct cfi_reg *cfa = &cfi->cfa;
2750
2751         if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2752                 return 0;
2753
2754         /* push */
2755         if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2756                 cfa->offset += 8;
2757
2758         /* pop */
2759         if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2760                 cfa->offset -= 8;
2761
2762         /* add immediate to sp */
2763         if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2764             op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2765                 cfa->offset -= op->src.offset;
2766
2767         return 0;
2768 }
2769
2770 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2771 {
2772         if (arch_callee_saved_reg(reg) &&
2773             cfi->regs[reg].base == CFI_UNDEFINED) {
2774                 cfi->regs[reg].base = base;
2775                 cfi->regs[reg].offset = offset;
2776         }
2777 }
2778
2779 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2780 {
2781         cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2782         cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2783 }
2784
2785 /*
2786  * A note about DRAP stack alignment:
2787  *
2788  * GCC has the concept of a DRAP register, which is used to help keep track of
2789  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2790  * register.  The typical DRAP pattern is:
2791  *
2792  *   4c 8d 54 24 08             lea    0x8(%rsp),%r10
2793  *   48 83 e4 c0                and    $0xffffffffffffffc0,%rsp
2794  *   41 ff 72 f8                pushq  -0x8(%r10)
2795  *   55                         push   %rbp
2796  *   48 89 e5                   mov    %rsp,%rbp
2797  *                              (more pushes)
2798  *   41 52                      push   %r10
2799  *                              ...
2800  *   41 5a                      pop    %r10
2801  *                              (more pops)
2802  *   5d                         pop    %rbp
2803  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2804  *   c3                         retq
2805  *
2806  * There are some variations in the epilogues, like:
2807  *
2808  *   5b                         pop    %rbx
2809  *   41 5a                      pop    %r10
2810  *   41 5c                      pop    %r12
2811  *   41 5d                      pop    %r13
2812  *   41 5e                      pop    %r14
2813  *   c9                         leaveq
2814  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2815  *   c3                         retq
2816  *
2817  * and:
2818  *
2819  *   4c 8b 55 e8                mov    -0x18(%rbp),%r10
2820  *   48 8b 5d e0                mov    -0x20(%rbp),%rbx
2821  *   4c 8b 65 f0                mov    -0x10(%rbp),%r12
2822  *   4c 8b 6d f8                mov    -0x8(%rbp),%r13
2823  *   c9                         leaveq
2824  *   49 8d 62 f8                lea    -0x8(%r10),%rsp
2825  *   c3                         retq
2826  *
2827  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2828  * restored beforehand:
2829  *
2830  *   41 55                      push   %r13
2831  *   4c 8d 6c 24 10             lea    0x10(%rsp),%r13
2832  *   48 83 e4 f0                and    $0xfffffffffffffff0,%rsp
2833  *                              ...
2834  *   49 8d 65 f0                lea    -0x10(%r13),%rsp
2835  *   41 5d                      pop    %r13
2836  *   c3                         retq
2837  */
2838 static int update_cfi_state(struct instruction *insn,
2839                             struct instruction *next_insn,
2840                             struct cfi_state *cfi, struct stack_op *op)
2841 {
2842         struct cfi_reg *cfa = &cfi->cfa;
2843         struct cfi_reg *regs = cfi->regs;
2844
2845         /* ignore UNWIND_HINT_UNDEFINED regions */
2846         if (cfi->force_undefined)
2847                 return 0;
2848
2849         /* stack operations don't make sense with an undefined CFA */
2850         if (cfa->base == CFI_UNDEFINED) {
2851                 if (insn_func(insn)) {
2852                         WARN_INSN(insn, "undefined stack state");
2853                         return -1;
2854                 }
2855                 return 0;
2856         }
2857
2858         if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2859             cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2860                 return update_cfi_state_regs(insn, cfi, op);
2861
2862         switch (op->dest.type) {
2863
2864         case OP_DEST_REG:
2865                 switch (op->src.type) {
2866
2867                 case OP_SRC_REG:
2868                         if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2869                             cfa->base == CFI_SP &&
2870                             check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2871
2872                                 /* mov %rsp, %rbp */
2873                                 cfa->base = op->dest.reg;
2874                                 cfi->bp_scratch = false;
2875                         }
2876
2877                         else if (op->src.reg == CFI_SP &&
2878                                  op->dest.reg == CFI_BP && cfi->drap) {
2879
2880                                 /* drap: mov %rsp, %rbp */
2881                                 regs[CFI_BP].base = CFI_BP;
2882                                 regs[CFI_BP].offset = -cfi->stack_size;
2883                                 cfi->bp_scratch = false;
2884                         }
2885
2886                         else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2887
2888                                 /*
2889                                  * mov %rsp, %reg
2890                                  *
2891                                  * This is needed for the rare case where GCC
2892                                  * does:
2893                                  *
2894                                  *   mov    %rsp, %rax
2895                                  *   ...
2896                                  *   mov    %rax, %rsp
2897                                  */
2898                                 cfi->vals[op->dest.reg].base = CFI_CFA;
2899                                 cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2900                         }
2901
2902                         else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2903                                  (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2904
2905                                 /*
2906                                  * mov %rbp, %rsp
2907                                  *
2908                                  * Restore the original stack pointer (Clang).
2909                                  */
2910                                 cfi->stack_size = -cfi->regs[CFI_BP].offset;
2911                         }
2912
2913                         else if (op->dest.reg == cfa->base) {
2914
2915                                 /* mov %reg, %rsp */
2916                                 if (cfa->base == CFI_SP &&
2917                                     cfi->vals[op->src.reg].base == CFI_CFA) {
2918
2919                                         /*
2920                                          * This is needed for the rare case
2921                                          * where GCC does something dumb like:
2922                                          *
2923                                          *   lea    0x8(%rsp), %rcx
2924                                          *   ...
2925                                          *   mov    %rcx, %rsp
2926                                          */
2927                                         cfa->offset = -cfi->vals[op->src.reg].offset;
2928                                         cfi->stack_size = cfa->offset;
2929
2930                                 } else if (cfa->base == CFI_SP &&
2931                                            cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2932                                            cfi->vals[op->src.reg].offset == cfa->offset) {
2933
2934                                         /*
2935                                          * Stack swizzle:
2936                                          *
2937                                          * 1: mov %rsp, (%[tos])
2938                                          * 2: mov %[tos], %rsp
2939                                          *    ...
2940                                          * 3: pop %rsp
2941                                          *
2942                                          * Where:
2943                                          *
2944                                          * 1 - places a pointer to the previous
2945                                          *     stack at the Top-of-Stack of the
2946                                          *     new stack.
2947                                          *
2948                                          * 2 - switches to the new stack.
2949                                          *
2950                                          * 3 - pops the Top-of-Stack to restore
2951                                          *     the original stack.
2952                                          *
2953                                          * Note: we set base to SP_INDIRECT
2954                                          * here and preserve offset. Therefore
2955                                          * when the unwinder reaches ToS it
2956                                          * will dereference SP and then add the
2957                                          * offset to find the next frame, IOW:
2958                                          * (%rsp) + offset.
2959                                          */
2960                                         cfa->base = CFI_SP_INDIRECT;
2961
2962                                 } else {
2963                                         cfa->base = CFI_UNDEFINED;
2964                                         cfa->offset = 0;
2965                                 }
2966                         }
2967
2968                         else if (op->dest.reg == CFI_SP &&
2969                                  cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2970                                  cfi->vals[op->src.reg].offset == cfa->offset) {
2971
2972                                 /*
2973                                  * The same stack swizzle case 2) as above. But
2974                                  * because we can't change cfa->base, case 3)
2975                                  * will become a regular POP. Pretend we're a
2976                                  * PUSH so things don't go unbalanced.
2977                                  */
2978                                 cfi->stack_size += 8;
2979                         }
2980
2981
2982                         break;
2983
2984                 case OP_SRC_ADD:
2985                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2986
2987                                 /* add imm, %rsp */
2988                                 cfi->stack_size -= op->src.offset;
2989                                 if (cfa->base == CFI_SP)
2990                                         cfa->offset -= op->src.offset;
2991                                 break;
2992                         }
2993
2994                         if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2995
2996                                 /* lea disp(%rbp), %rsp */
2997                                 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2998                                 break;
2999                         }
3000
3001                         if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3002
3003                                 /* drap: lea disp(%rsp), %drap */
3004                                 cfi->drap_reg = op->dest.reg;
3005
3006                                 /*
3007                                  * lea disp(%rsp), %reg
3008                                  *
3009                                  * This is needed for the rare case where GCC
3010                                  * does something dumb like:
3011                                  *
3012                                  *   lea    0x8(%rsp), %rcx
3013                                  *   ...
3014                                  *   mov    %rcx, %rsp
3015                                  */
3016                                 cfi->vals[op->dest.reg].base = CFI_CFA;
3017                                 cfi->vals[op->dest.reg].offset = \
3018                                         -cfi->stack_size + op->src.offset;
3019
3020                                 break;
3021                         }
3022
3023                         if (cfi->drap && op->dest.reg == CFI_SP &&
3024                             op->src.reg == cfi->drap_reg) {
3025
3026                                  /* drap: lea disp(%drap), %rsp */
3027                                 cfa->base = CFI_SP;
3028                                 cfa->offset = cfi->stack_size = -op->src.offset;
3029                                 cfi->drap_reg = CFI_UNDEFINED;
3030                                 cfi->drap = false;
3031                                 break;
3032                         }
3033
3034                         if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3035                                 WARN_INSN(insn, "unsupported stack register modification");
3036                                 return -1;
3037                         }
3038
3039                         break;
3040
3041                 case OP_SRC_AND:
3042                         if (op->dest.reg != CFI_SP ||
3043                             (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3044                             (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3045                                 WARN_INSN(insn, "unsupported stack pointer realignment");
3046                                 return -1;
3047                         }
3048
3049                         if (cfi->drap_reg != CFI_UNDEFINED) {
3050                                 /* drap: and imm, %rsp */
3051                                 cfa->base = cfi->drap_reg;
3052                                 cfa->offset = cfi->stack_size = 0;
3053                                 cfi->drap = true;
3054                         }
3055
3056                         /*
3057                          * Older versions of GCC (4.8ish) realign the stack
3058                          * without DRAP, with a frame pointer.
3059                          */
3060
3061                         break;
3062
3063                 case OP_SRC_POP:
3064                 case OP_SRC_POPF:
3065                         if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3066
3067                                 /* pop %rsp; # restore from a stack swizzle */
3068                                 cfa->base = CFI_SP;
3069                                 break;
3070                         }
3071
3072                         if (!cfi->drap && op->dest.reg == cfa->base) {
3073
3074                                 /* pop %rbp */
3075                                 cfa->base = CFI_SP;
3076                         }
3077
3078                         if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3079                             op->dest.reg == cfi->drap_reg &&
3080                             cfi->drap_offset == -cfi->stack_size) {
3081
3082                                 /* drap: pop %drap */
3083                                 cfa->base = cfi->drap_reg;
3084                                 cfa->offset = 0;
3085                                 cfi->drap_offset = -1;
3086
3087                         } else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3088
3089                                 /* pop %reg */
3090                                 restore_reg(cfi, op->dest.reg);
3091                         }
3092
3093                         cfi->stack_size -= 8;
3094                         if (cfa->base == CFI_SP)
3095                                 cfa->offset -= 8;
3096
3097                         break;
3098
3099                 case OP_SRC_REG_INDIRECT:
3100                         if (!cfi->drap && op->dest.reg == cfa->base &&
3101                             op->dest.reg == CFI_BP) {
3102
3103                                 /* mov disp(%rsp), %rbp */
3104                                 cfa->base = CFI_SP;
3105                                 cfa->offset = cfi->stack_size;
3106                         }
3107
3108                         if (cfi->drap && op->src.reg == CFI_BP &&
3109                             op->src.offset == cfi->drap_offset) {
3110
3111                                 /* drap: mov disp(%rbp), %drap */
3112                                 cfa->base = cfi->drap_reg;
3113                                 cfa->offset = 0;
3114                                 cfi->drap_offset = -1;
3115                         }
3116
3117                         if (cfi->drap && op->src.reg == CFI_BP &&
3118                             op->src.offset == regs[op->dest.reg].offset) {
3119
3120                                 /* drap: mov disp(%rbp), %reg */
3121                                 restore_reg(cfi, op->dest.reg);
3122
3123                         } else if (op->src.reg == cfa->base &&
3124                             op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3125
3126                                 /* mov disp(%rbp), %reg */
3127                                 /* mov disp(%rsp), %reg */
3128                                 restore_reg(cfi, op->dest.reg);
3129
3130                         } else if (op->src.reg == CFI_SP &&
3131                                    op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3132
3133                                 /* mov disp(%rsp), %reg */
3134                                 restore_reg(cfi, op->dest.reg);
3135                         }
3136
3137                         break;
3138
3139                 default:
3140                         WARN_INSN(insn, "unknown stack-related instruction");
3141                         return -1;
3142                 }
3143
3144                 break;
3145
3146         case OP_DEST_PUSH:
3147         case OP_DEST_PUSHF:
3148                 cfi->stack_size += 8;
3149                 if (cfa->base == CFI_SP)
3150                         cfa->offset += 8;
3151
3152                 if (op->src.type != OP_SRC_REG)
3153                         break;
3154
3155                 if (cfi->drap) {
3156                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3157
3158                                 /* drap: push %drap */
3159                                 cfa->base = CFI_BP_INDIRECT;
3160                                 cfa->offset = -cfi->stack_size;
3161
3162                                 /* save drap so we know when to restore it */
3163                                 cfi->drap_offset = -cfi->stack_size;
3164
3165                         } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3166
3167                                 /* drap: push %rbp */
3168                                 cfi->stack_size = 0;
3169
3170                         } else {
3171
3172                                 /* drap: push %reg */
3173                                 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3174                         }
3175
3176                 } else {
3177
3178                         /* push %reg */
3179                         save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3180                 }
3181
3182                 /* detect when asm code uses rbp as a scratch register */
3183                 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3184                     cfa->base != CFI_BP)
3185                         cfi->bp_scratch = true;
3186                 break;
3187
3188         case OP_DEST_REG_INDIRECT:
3189
3190                 if (cfi->drap) {
3191                         if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3192
3193                                 /* drap: mov %drap, disp(%rbp) */
3194                                 cfa->base = CFI_BP_INDIRECT;
3195                                 cfa->offset = op->dest.offset;
3196
3197                                 /* save drap offset so we know when to restore it */
3198                                 cfi->drap_offset = op->dest.offset;
3199                         } else {
3200
3201                                 /* drap: mov reg, disp(%rbp) */
3202                                 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3203                         }
3204
3205                 } else if (op->dest.reg == cfa->base) {
3206
3207                         /* mov reg, disp(%rbp) */
3208                         /* mov reg, disp(%rsp) */
3209                         save_reg(cfi, op->src.reg, CFI_CFA,
3210                                  op->dest.offset - cfi->cfa.offset);
3211
3212                 } else if (op->dest.reg == CFI_SP) {
3213
3214                         /* mov reg, disp(%rsp) */
3215                         save_reg(cfi, op->src.reg, CFI_CFA,
3216                                  op->dest.offset - cfi->stack_size);
3217
3218                 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3219
3220                         /* mov %rsp, (%reg); # setup a stack swizzle. */
3221                         cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3222                         cfi->vals[op->dest.reg].offset = cfa->offset;
3223                 }
3224
3225                 break;
3226
3227         case OP_DEST_MEM:
3228                 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3229                         WARN_INSN(insn, "unknown stack-related memory operation");
3230                         return -1;
3231                 }
3232
3233                 /* pop mem */
3234                 cfi->stack_size -= 8;
3235                 if (cfa->base == CFI_SP)
3236                         cfa->offset -= 8;
3237
3238                 break;
3239
3240         default:
3241                 WARN_INSN(insn, "unknown stack-related instruction");
3242                 return -1;
3243         }
3244
3245         return 0;
3246 }
3247
3248 /*
3249  * The stack layouts of alternatives instructions can sometimes diverge when
3250  * they have stack modifications.  That's fine as long as the potential stack
3251  * layouts don't conflict at any given potential instruction boundary.
3252  *
3253  * Flatten the CFIs of the different alternative code streams (both original
3254  * and replacement) into a single shared CFI array which can be used to detect
3255  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3256  */
3257 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3258 {
3259         struct cfi_state **alt_cfi;
3260         int group_off;
3261
3262         if (!insn->alt_group)
3263                 return 0;
3264
3265         if (!insn->cfi) {
3266                 WARN("CFI missing");
3267                 return -1;
3268         }
3269
3270         alt_cfi = insn->alt_group->cfi;
3271         group_off = insn->offset - insn->alt_group->first_insn->offset;
3272
3273         if (!alt_cfi[group_off]) {
3274                 alt_cfi[group_off] = insn->cfi;
3275         } else {
3276                 if (cficmp(alt_cfi[group_off], insn->cfi)) {
3277                         struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3278                         struct instruction *orig = orig_group->first_insn;
3279                         char *where = offstr(insn->sec, insn->offset);
3280                         WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3281                         free(where);
3282                         return -1;
3283                 }
3284         }
3285
3286         return 0;
3287 }
3288
3289 static int handle_insn_ops(struct instruction *insn,
3290                            struct instruction *next_insn,
3291                            struct insn_state *state)
3292 {
3293         struct stack_op *op;
3294
3295         for (op = insn->stack_ops; op; op = op->next) {
3296
3297                 if (update_cfi_state(insn, next_insn, &state->cfi, op))
3298                         return 1;
3299
3300                 if (!insn->alt_group)
3301                         continue;
3302
3303                 if (op->dest.type == OP_DEST_PUSHF) {
3304                         if (!state->uaccess_stack) {
3305                                 state->uaccess_stack = 1;
3306                         } else if (state->uaccess_stack >> 31) {
3307                                 WARN_INSN(insn, "PUSHF stack exhausted");
3308                                 return 1;
3309                         }
3310                         state->uaccess_stack <<= 1;
3311                         state->uaccess_stack  |= state->uaccess;
3312                 }
3313
3314                 if (op->src.type == OP_SRC_POPF) {
3315                         if (state->uaccess_stack) {
3316                                 state->uaccess = state->uaccess_stack & 1;
3317                                 state->uaccess_stack >>= 1;
3318                                 if (state->uaccess_stack == 1)
3319                                         state->uaccess_stack = 0;
3320                         }
3321                 }
3322         }
3323
3324         return 0;
3325 }
3326
3327 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3328 {
3329         struct cfi_state *cfi1 = insn->cfi;
3330         int i;
3331
3332         if (!cfi1) {
3333                 WARN("CFI missing");
3334                 return false;
3335         }
3336
3337         if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3338
3339                 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3340                           cfi1->cfa.base, cfi1->cfa.offset,
3341                           cfi2->cfa.base, cfi2->cfa.offset);
3342
3343         } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3344                 for (i = 0; i < CFI_NUM_REGS; i++) {
3345                         if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3346                                     sizeof(struct cfi_reg)))
3347                                 continue;
3348
3349                         WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3350                                   i, cfi1->regs[i].base, cfi1->regs[i].offset,
3351                                   i, cfi2->regs[i].base, cfi2->regs[i].offset);
3352                         break;
3353                 }
3354
3355         } else if (cfi1->type != cfi2->type) {
3356
3357                 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3358                           cfi1->type, cfi2->type);
3359
3360         } else if (cfi1->drap != cfi2->drap ||
3361                    (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3362                    (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3363
3364                 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3365                           cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3366                           cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3367
3368         } else
3369                 return true;
3370
3371         return false;
3372 }
3373
3374 static inline bool func_uaccess_safe(struct symbol *func)
3375 {
3376         if (func)
3377                 return func->uaccess_safe;
3378
3379         return false;
3380 }
3381
3382 static inline const char *call_dest_name(struct instruction *insn)
3383 {
3384         static char pvname[19];
3385         struct reloc *reloc;
3386         int idx;
3387
3388         if (insn_call_dest(insn))
3389                 return insn_call_dest(insn)->name;
3390
3391         reloc = insn_reloc(NULL, insn);
3392         if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3393                 idx = (reloc_addend(reloc) / sizeof(void *));
3394                 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3395                 return pvname;
3396         }
3397
3398         return "{dynamic}";
3399 }
3400
3401 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3402 {
3403         struct symbol *target;
3404         struct reloc *reloc;
3405         int idx;
3406
3407         reloc = insn_reloc(file, insn);
3408         if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3409                 return false;
3410
3411         idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3412
3413         if (file->pv_ops[idx].clean)
3414                 return true;
3415
3416         file->pv_ops[idx].clean = true;
3417
3418         list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3419                 if (!target->sec->noinstr) {
3420                         WARN("pv_ops[%d]: %s", idx, target->name);
3421                         file->pv_ops[idx].clean = false;
3422                 }
3423         }
3424
3425         return file->pv_ops[idx].clean;
3426 }
3427
3428 static inline bool noinstr_call_dest(struct objtool_file *file,
3429                                      struct instruction *insn,
3430                                      struct symbol *func)
3431 {
3432         /*
3433          * We can't deal with indirect function calls at present;
3434          * assume they're instrumented.
3435          */
3436         if (!func) {
3437                 if (file->pv_ops)
3438                         return pv_call_dest(file, insn);
3439
3440                 return false;
3441         }
3442
3443         /*
3444          * If the symbol is from a noinstr section; we good.
3445          */
3446         if (func->sec->noinstr)
3447                 return true;
3448
3449         /*
3450          * If the symbol is a static_call trampoline, we can't tell.
3451          */
3452         if (func->static_call_tramp)
3453                 return true;
3454
3455         /*
3456          * The __ubsan_handle_*() calls are like WARN(), they only happen when
3457          * something 'BAD' happened. At the risk of taking the machine down,
3458          * let them proceed to get the message out.
3459          */
3460         if (!strncmp(func->name, "__ubsan_handle_", 15))
3461                 return true;
3462
3463         return false;
3464 }
3465
3466 static int validate_call(struct objtool_file *file,
3467                          struct instruction *insn,
3468                          struct insn_state *state)
3469 {
3470         if (state->noinstr && state->instr <= 0 &&
3471             !noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3472                 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3473                 return 1;
3474         }
3475
3476         if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) {
3477                 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3478                 return 1;
3479         }
3480
3481         if (state->df) {
3482                 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3483                 return 1;
3484         }
3485
3486         return 0;
3487 }
3488
3489 static int validate_sibling_call(struct objtool_file *file,
3490                                  struct instruction *insn,
3491                                  struct insn_state *state)
3492 {
3493         if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3494                 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3495                 return 1;
3496         }
3497
3498         return validate_call(file, insn, state);
3499 }
3500
3501 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3502 {
3503         if (state->noinstr && state->instr > 0) {
3504                 WARN_INSN(insn, "return with instrumentation enabled");
3505                 return 1;
3506         }
3507
3508         if (state->uaccess && !func_uaccess_safe(func)) {
3509                 WARN_INSN(insn, "return with UACCESS enabled");
3510                 return 1;
3511         }
3512
3513         if (!state->uaccess && func_uaccess_safe(func)) {
3514                 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3515                 return 1;
3516         }
3517
3518         if (state->df) {
3519                 WARN_INSN(insn, "return with DF set");
3520                 return 1;
3521         }
3522
3523         if (func && has_modified_stack_frame(insn, state)) {
3524                 WARN_INSN(insn, "return with modified stack frame");
3525                 return 1;
3526         }
3527
3528         if (state->cfi.bp_scratch) {
3529                 WARN_INSN(insn, "BP used as a scratch register");
3530                 return 1;
3531         }
3532
3533         return 0;
3534 }
3535
3536 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3537                                                  struct instruction *insn)
3538 {
3539         struct alt_group *alt_group = insn->alt_group;
3540
3541         /*
3542          * Simulate the fact that alternatives are patched in-place.  When the
3543          * end of a replacement alt_group is reached, redirect objtool flow to
3544          * the end of the original alt_group.
3545          *
3546          * insn->alts->insn -> alt_group->first_insn
3547          *                     ...
3548          *                     alt_group->last_insn
3549          *                     [alt_group->nop]      -> next(orig_group->last_insn)
3550          */
3551         if (alt_group) {
3552                 if (alt_group->nop) {
3553                         /* ->nop implies ->orig_group */
3554                         if (insn == alt_group->last_insn)
3555                                 return alt_group->nop;
3556                         if (insn == alt_group->nop)
3557                                 goto next_orig;
3558                 }
3559                 if (insn == alt_group->last_insn && alt_group->orig_group)
3560                         goto next_orig;
3561         }
3562
3563         return next_insn_same_sec(file, insn);
3564
3565 next_orig:
3566         return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3567 }
3568
3569 /*
3570  * Follow the branch starting at the given instruction, and recursively follow
3571  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3572  * each instruction and validate all the rules described in
3573  * tools/objtool/Documentation/objtool.txt.
3574  */
3575 static int validate_branch(struct objtool_file *file, struct symbol *func,
3576                            struct instruction *insn, struct insn_state state)
3577 {
3578         struct alternative *alt;
3579         struct instruction *next_insn, *prev_insn = NULL;
3580         struct section *sec;
3581         u8 visited;
3582         int ret;
3583
3584         sec = insn->sec;
3585
3586         while (1) {
3587                 next_insn = next_insn_to_validate(file, insn);
3588
3589                 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3590                         /* Ignore KCFI type preambles, which always fall through */
3591                         if (!strncmp(func->name, "__cfi_", 6) ||
3592                             !strncmp(func->name, "__pfx_", 6))
3593                                 return 0;
3594
3595                         WARN("%s() falls through to next function %s()",
3596                              func->name, insn_func(insn)->name);
3597                         return 1;
3598                 }
3599
3600                 if (func && insn->ignore) {
3601                         WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3602                         return 1;
3603                 }
3604
3605                 visited = VISITED_BRANCH << state.uaccess;
3606                 if (insn->visited & VISITED_BRANCH_MASK) {
3607                         if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3608                                 return 1;
3609
3610                         if (insn->visited & visited)
3611                                 return 0;
3612                 } else {
3613                         nr_insns_visited++;
3614                 }
3615
3616                 if (state.noinstr)
3617                         state.instr += insn->instr;
3618
3619                 if (insn->hint) {
3620                         if (insn->restore) {
3621                                 struct instruction *save_insn, *i;
3622
3623                                 i = insn;
3624                                 save_insn = NULL;
3625
3626                                 sym_for_each_insn_continue_reverse(file, func, i) {
3627                                         if (i->save) {
3628                                                 save_insn = i;
3629                                                 break;
3630                                         }
3631                                 }
3632
3633                                 if (!save_insn) {
3634                                         WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3635                                         return 1;
3636                                 }
3637
3638                                 if (!save_insn->visited) {
3639                                         /*
3640                                          * If the restore hint insn is at the
3641                                          * beginning of a basic block and was
3642                                          * branched to from elsewhere, and the
3643                                          * save insn hasn't been visited yet,
3644                                          * defer following this branch for now.
3645                                          * It will be seen later via the
3646                                          * straight-line path.
3647                                          */
3648                                         if (!prev_insn)
3649                                                 return 0;
3650
3651                                         WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3652                                         return 1;
3653                                 }
3654
3655                                 insn->cfi = save_insn->cfi;
3656                                 nr_cfi_reused++;
3657                         }
3658
3659                         state.cfi = *insn->cfi;
3660                 } else {
3661                         /* XXX track if we actually changed state.cfi */
3662
3663                         if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3664                                 insn->cfi = prev_insn->cfi;
3665                                 nr_cfi_reused++;
3666                         } else {
3667                                 insn->cfi = cfi_hash_find_or_add(&state.cfi);
3668                         }
3669                 }
3670
3671                 insn->visited |= visited;
3672
3673                 if (propagate_alt_cfi(file, insn))
3674                         return 1;
3675
3676                 if (!insn->ignore_alts && insn->alts) {
3677                         bool skip_orig = false;
3678
3679                         for (alt = insn->alts; alt; alt = alt->next) {
3680                                 if (alt->skip_orig)
3681                                         skip_orig = true;
3682
3683                                 ret = validate_branch(file, func, alt->insn, state);
3684                                 if (ret) {
3685                                         BT_INSN(insn, "(alt)");
3686                                         return ret;
3687                                 }
3688                         }
3689
3690                         if (skip_orig)
3691                                 return 0;
3692                 }
3693
3694                 if (handle_insn_ops(insn, next_insn, &state))
3695                         return 1;
3696
3697                 switch (insn->type) {
3698
3699                 case INSN_RETURN:
3700                         return validate_return(func, insn, &state);
3701
3702                 case INSN_CALL:
3703                 case INSN_CALL_DYNAMIC:
3704                         ret = validate_call(file, insn, &state);
3705                         if (ret)
3706                                 return ret;
3707
3708                         if (opts.stackval && func && !is_special_call(insn) &&
3709                             !has_valid_stack_frame(&state)) {
3710                                 WARN_INSN(insn, "call without frame pointer save/setup");
3711                                 return 1;
3712                         }
3713
3714                         if (insn->dead_end)
3715                                 return 0;
3716
3717                         break;
3718
3719                 case INSN_JUMP_CONDITIONAL:
3720                 case INSN_JUMP_UNCONDITIONAL:
3721                         if (is_sibling_call(insn)) {
3722                                 ret = validate_sibling_call(file, insn, &state);
3723                                 if (ret)
3724                                         return ret;
3725
3726                         } else if (insn->jump_dest) {
3727                                 ret = validate_branch(file, func,
3728                                                       insn->jump_dest, state);
3729                                 if (ret) {
3730                                         BT_INSN(insn, "(branch)");
3731                                         return ret;
3732                                 }
3733                         }
3734
3735                         if (insn->type == INSN_JUMP_UNCONDITIONAL)
3736                                 return 0;
3737
3738                         break;
3739
3740                 case INSN_JUMP_DYNAMIC:
3741                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3742                         if (is_sibling_call(insn)) {
3743                                 ret = validate_sibling_call(file, insn, &state);
3744                                 if (ret)
3745                                         return ret;
3746                         }
3747
3748                         if (insn->type == INSN_JUMP_DYNAMIC)
3749                                 return 0;
3750
3751                         break;
3752
3753                 case INSN_CONTEXT_SWITCH:
3754                         if (func && (!next_insn || !next_insn->hint)) {
3755                                 WARN_INSN(insn, "unsupported instruction in callable function");
3756                                 return 1;
3757                         }
3758                         return 0;
3759
3760                 case INSN_STAC:
3761                         if (state.uaccess) {
3762                                 WARN_INSN(insn, "recursive UACCESS enable");
3763                                 return 1;
3764                         }
3765
3766                         state.uaccess = true;
3767                         break;
3768
3769                 case INSN_CLAC:
3770                         if (!state.uaccess && func) {
3771                                 WARN_INSN(insn, "redundant UACCESS disable");
3772                                 return 1;
3773                         }
3774
3775                         if (func_uaccess_safe(func) && !state.uaccess_stack) {
3776                                 WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3777                                 return 1;
3778                         }
3779
3780                         state.uaccess = false;
3781                         break;
3782
3783                 case INSN_STD:
3784                         if (state.df) {
3785                                 WARN_INSN(insn, "recursive STD");
3786                                 return 1;
3787                         }
3788
3789                         state.df = true;
3790                         break;
3791
3792                 case INSN_CLD:
3793                         if (!state.df && func) {
3794                                 WARN_INSN(insn, "redundant CLD");
3795                                 return 1;
3796                         }
3797
3798                         state.df = false;
3799                         break;
3800
3801                 default:
3802                         break;
3803                 }
3804
3805                 if (insn->dead_end)
3806                         return 0;
3807
3808                 if (!next_insn) {
3809                         if (state.cfi.cfa.base == CFI_UNDEFINED)
3810                                 return 0;
3811                         WARN("%s: unexpected end of section", sec->name);
3812                         return 1;
3813                 }
3814
3815                 prev_insn = insn;
3816                 insn = next_insn;
3817         }
3818
3819         return 0;
3820 }
3821
3822 static int validate_unwind_hint(struct objtool_file *file,
3823                                   struct instruction *insn,
3824                                   struct insn_state *state)
3825 {
3826         if (insn->hint && !insn->visited && !insn->ignore) {
3827                 int ret = validate_branch(file, insn_func(insn), insn, *state);
3828                 if (ret)
3829                         BT_INSN(insn, "<=== (hint)");
3830                 return ret;
3831         }
3832
3833         return 0;
3834 }
3835
3836 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3837 {
3838         struct instruction *insn;
3839         struct insn_state state;
3840         int warnings = 0;
3841
3842         if (!file->hints)
3843                 return 0;
3844
3845         init_insn_state(file, &state, sec);
3846
3847         if (sec) {
3848                 sec_for_each_insn(file, sec, insn)
3849                         warnings += validate_unwind_hint(file, insn, &state);
3850         } else {
3851                 for_each_insn(file, insn)
3852                         warnings += validate_unwind_hint(file, insn, &state);
3853         }
3854
3855         return warnings;
3856 }
3857
3858 /*
3859  * Validate rethunk entry constraint: must untrain RET before the first RET.
3860  *
3861  * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3862  * before an actual RET instruction.
3863  */
3864 static int validate_unret(struct objtool_file *file, struct instruction *insn)
3865 {
3866         struct instruction *next, *dest;
3867         int ret;
3868
3869         for (;;) {
3870                 next = next_insn_to_validate(file, insn);
3871
3872                 if (insn->visited & VISITED_UNRET)
3873                         return 0;
3874
3875                 insn->visited |= VISITED_UNRET;
3876
3877                 if (!insn->ignore_alts && insn->alts) {
3878                         struct alternative *alt;
3879                         bool skip_orig = false;
3880
3881                         for (alt = insn->alts; alt; alt = alt->next) {
3882                                 if (alt->skip_orig)
3883                                         skip_orig = true;
3884
3885                                 ret = validate_unret(file, alt->insn);
3886                                 if (ret) {
3887                                         BT_INSN(insn, "(alt)");
3888                                         return ret;
3889                                 }
3890                         }
3891
3892                         if (skip_orig)
3893                                 return 0;
3894                 }
3895
3896                 switch (insn->type) {
3897
3898                 case INSN_CALL_DYNAMIC:
3899                 case INSN_JUMP_DYNAMIC:
3900                 case INSN_JUMP_DYNAMIC_CONDITIONAL:
3901                         WARN_INSN(insn, "early indirect call");
3902                         return 1;
3903
3904                 case INSN_JUMP_UNCONDITIONAL:
3905                 case INSN_JUMP_CONDITIONAL:
3906                         if (!is_sibling_call(insn)) {
3907                                 if (!insn->jump_dest) {
3908                                         WARN_INSN(insn, "unresolved jump target after linking?!?");
3909                                         return -1;
3910                                 }
3911                                 ret = validate_unret(file, insn->jump_dest);
3912                                 if (ret) {
3913                                         BT_INSN(insn, "(branch%s)",
3914                                                 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3915                                         return ret;
3916                                 }
3917
3918                                 if (insn->type == INSN_JUMP_UNCONDITIONAL)
3919                                         return 0;
3920
3921                                 break;
3922                         }
3923
3924                         /* fallthrough */
3925                 case INSN_CALL:
3926                         dest = find_insn(file, insn_call_dest(insn)->sec,
3927                                          insn_call_dest(insn)->offset);
3928                         if (!dest) {
3929                                 WARN("Unresolved function after linking!?: %s",
3930                                      insn_call_dest(insn)->name);
3931                                 return -1;
3932                         }
3933
3934                         ret = validate_unret(file, dest);
3935                         if (ret) {
3936                                 BT_INSN(insn, "(call)");
3937                                 return ret;
3938                         }
3939                         /*
3940                          * If a call returns without error, it must have seen UNTRAIN_RET.
3941                          * Therefore any non-error return is a success.
3942                          */
3943                         return 0;
3944
3945                 case INSN_RETURN:
3946                         WARN_INSN(insn, "RET before UNTRAIN");
3947                         return 1;
3948
3949                 case INSN_NOP:
3950                         if (insn->retpoline_safe)
3951                                 return 0;
3952                         break;
3953
3954                 default:
3955                         break;
3956                 }
3957
3958                 if (!next) {
3959                         WARN_INSN(insn, "teh end!");
3960                         return -1;
3961                 }
3962                 insn = next;
3963         }
3964
3965         return 0;
3966 }
3967
3968 /*
3969  * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3970  * VALIDATE_UNRET_END before RET.
3971  */
3972 static int validate_unrets(struct objtool_file *file)
3973 {
3974         struct instruction *insn;
3975         int ret, warnings = 0;
3976
3977         for_each_insn(file, insn) {
3978                 if (!insn->unret)
3979                         continue;
3980
3981                 ret = validate_unret(file, insn);
3982                 if (ret < 0) {
3983                         WARN_INSN(insn, "Failed UNRET validation");
3984                         return ret;
3985                 }
3986                 warnings += ret;
3987         }
3988
3989         return warnings;
3990 }
3991
3992 static int validate_retpoline(struct objtool_file *file)
3993 {
3994         struct instruction *insn;
3995         int warnings = 0;
3996
3997         for_each_insn(file, insn) {
3998                 if (insn->type != INSN_JUMP_DYNAMIC &&
3999                     insn->type != INSN_CALL_DYNAMIC &&
4000                     insn->type != INSN_RETURN)
4001                         continue;
4002
4003                 if (insn->retpoline_safe)
4004                         continue;
4005
4006                 if (insn->sec->init)
4007                         continue;
4008
4009                 if (insn->type == INSN_RETURN) {
4010                         if (opts.rethunk) {
4011                                 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4012                         } else
4013                                 continue;
4014                 } else {
4015                         WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4016                                   insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4017                 }
4018
4019                 warnings++;
4020         }
4021
4022         return warnings;
4023 }
4024
4025 static bool is_kasan_insn(struct instruction *insn)
4026 {
4027         return (insn->type == INSN_CALL &&
4028                 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4029 }
4030
4031 static bool is_ubsan_insn(struct instruction *insn)
4032 {
4033         return (insn->type == INSN_CALL &&
4034                 !strcmp(insn_call_dest(insn)->name,
4035                         "__ubsan_handle_builtin_unreachable"));
4036 }
4037
4038 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4039 {
4040         int i;
4041         struct instruction *prev_insn;
4042
4043         if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4044                 return true;
4045
4046         /*
4047          * Ignore alternative replacement instructions.  This can happen
4048          * when a whitelisted function uses one of the ALTERNATIVE macros.
4049          */
4050         if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4051             !strcmp(insn->sec->name, ".altinstr_aux"))
4052                 return true;
4053
4054         /*
4055          * Whole archive runs might encounter dead code from weak symbols.
4056          * This is where the linker will have dropped the weak symbol in
4057          * favour of a regular symbol, but leaves the code in place.
4058          *
4059          * In this case we'll find a piece of code (whole function) that is not
4060          * covered by a !section symbol. Ignore them.
4061          */
4062         if (opts.link && !insn_func(insn)) {
4063                 int size = find_symbol_hole_containing(insn->sec, insn->offset);
4064                 unsigned long end = insn->offset + size;
4065
4066                 if (!size) /* not a hole */
4067                         return false;
4068
4069                 if (size < 0) /* hole until the end */
4070                         return true;
4071
4072                 sec_for_each_insn_continue(file, insn) {
4073                         /*
4074                          * If we reach a visited instruction at or before the
4075                          * end of the hole, ignore the unreachable.
4076                          */
4077                         if (insn->visited)
4078                                 return true;
4079
4080                         if (insn->offset >= end)
4081                                 break;
4082
4083                         /*
4084                          * If this hole jumps to a .cold function, mark it ignore too.
4085                          */
4086                         if (insn->jump_dest && insn_func(insn->jump_dest) &&
4087                             strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4088                                 struct instruction *dest = insn->jump_dest;
4089                                 func_for_each_insn(file, insn_func(dest), dest)
4090                                         dest->ignore = true;
4091                         }
4092                 }
4093
4094                 return false;
4095         }
4096
4097         if (!insn_func(insn))
4098                 return false;
4099
4100         if (insn_func(insn)->static_call_tramp)
4101                 return true;
4102
4103         /*
4104          * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4105          * __builtin_unreachable().  The BUG() macro has an unreachable() after
4106          * the UD2, which causes GCC's undefined trap logic to emit another UD2
4107          * (or occasionally a JMP to UD2).
4108          *
4109          * It may also insert a UD2 after calling a __noreturn function.
4110          */
4111         prev_insn = prev_insn_same_sec(file, insn);
4112         if (prev_insn->dead_end &&
4113             (insn->type == INSN_BUG ||
4114              (insn->type == INSN_JUMP_UNCONDITIONAL &&
4115               insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4116                 return true;
4117
4118         /*
4119          * Check if this (or a subsequent) instruction is related to
4120          * CONFIG_UBSAN or CONFIG_KASAN.
4121          *
4122          * End the search at 5 instructions to avoid going into the weeds.
4123          */
4124         for (i = 0; i < 5; i++) {
4125
4126                 if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4127                         return true;
4128
4129                 if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4130                         if (insn->jump_dest &&
4131                             insn_func(insn->jump_dest) == insn_func(insn)) {
4132                                 insn = insn->jump_dest;
4133                                 continue;
4134                         }
4135
4136                         break;
4137                 }
4138
4139                 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4140                         break;
4141
4142                 insn = next_insn_same_sec(file, insn);
4143         }
4144
4145         return false;
4146 }
4147
4148 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4149 {
4150         struct instruction *insn, *prev;
4151         struct cfi_state *cfi;
4152
4153         insn = find_insn(file, func->sec, func->offset);
4154         if (!insn)
4155                 return -1;
4156
4157         for (prev = prev_insn_same_sec(file, insn);
4158              prev;
4159              prev = prev_insn_same_sec(file, prev)) {
4160                 u64 offset;
4161
4162                 if (prev->type != INSN_NOP)
4163                         return -1;
4164
4165                 offset = func->offset - prev->offset;
4166
4167                 if (offset > opts.prefix)
4168                         return -1;
4169
4170                 if (offset < opts.prefix)
4171                         continue;
4172
4173                 elf_create_prefix_symbol(file->elf, func, opts.prefix);
4174                 break;
4175         }
4176
4177         if (!prev)
4178                 return -1;
4179
4180         if (!insn->cfi) {
4181                 /*
4182                  * This can happen if stack validation isn't enabled or the
4183                  * function is annotated with STACK_FRAME_NON_STANDARD.
4184                  */
4185                 return 0;
4186         }
4187
4188         /* Propagate insn->cfi to the prefix code */
4189         cfi = cfi_hash_find_or_add(insn->cfi);
4190         for (; prev != insn; prev = next_insn_same_sec(file, prev))
4191                 prev->cfi = cfi;
4192
4193         return 0;
4194 }
4195
4196 static int add_prefix_symbols(struct objtool_file *file)
4197 {
4198         struct section *sec;
4199         struct symbol *func;
4200
4201         for_each_sec(file, sec) {
4202                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4203                         continue;
4204
4205                 sec_for_each_sym(sec, func) {
4206                         if (func->type != STT_FUNC)
4207                                 continue;
4208
4209                         add_prefix_symbol(file, func);
4210                 }
4211         }
4212
4213         return 0;
4214 }
4215
4216 static int validate_symbol(struct objtool_file *file, struct section *sec,
4217                            struct symbol *sym, struct insn_state *state)
4218 {
4219         struct instruction *insn;
4220         int ret;
4221
4222         if (!sym->len) {
4223                 WARN("%s() is missing an ELF size annotation", sym->name);
4224                 return 1;
4225         }
4226
4227         if (sym->pfunc != sym || sym->alias != sym)
4228                 return 0;
4229
4230         insn = find_insn(file, sec, sym->offset);
4231         if (!insn || insn->ignore || insn->visited)
4232                 return 0;
4233
4234         state->uaccess = sym->uaccess_safe;
4235
4236         ret = validate_branch(file, insn_func(insn), insn, *state);
4237         if (ret)
4238                 BT_INSN(insn, "<=== (sym)");
4239         return ret;
4240 }
4241
4242 static int validate_section(struct objtool_file *file, struct section *sec)
4243 {
4244         struct insn_state state;
4245         struct symbol *func;
4246         int warnings = 0;
4247
4248         sec_for_each_sym(sec, func) {
4249                 if (func->type != STT_FUNC)
4250                         continue;
4251
4252                 init_insn_state(file, &state, sec);
4253                 set_func_state(&state.cfi);
4254
4255                 warnings += validate_symbol(file, sec, func, &state);
4256         }
4257
4258         return warnings;
4259 }
4260
4261 static int validate_noinstr_sections(struct objtool_file *file)
4262 {
4263         struct section *sec;
4264         int warnings = 0;
4265
4266         sec = find_section_by_name(file->elf, ".noinstr.text");
4267         if (sec) {
4268                 warnings += validate_section(file, sec);
4269                 warnings += validate_unwind_hints(file, sec);
4270         }
4271
4272         sec = find_section_by_name(file->elf, ".entry.text");
4273         if (sec) {
4274                 warnings += validate_section(file, sec);
4275                 warnings += validate_unwind_hints(file, sec);
4276         }
4277
4278         sec = find_section_by_name(file->elf, ".cpuidle.text");
4279         if (sec) {
4280                 warnings += validate_section(file, sec);
4281                 warnings += validate_unwind_hints(file, sec);
4282         }
4283
4284         return warnings;
4285 }
4286
4287 static int validate_functions(struct objtool_file *file)
4288 {
4289         struct section *sec;
4290         int warnings = 0;
4291
4292         for_each_sec(file, sec) {
4293                 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4294                         continue;
4295
4296                 warnings += validate_section(file, sec);
4297         }
4298
4299         return warnings;
4300 }
4301
4302 static void mark_endbr_used(struct instruction *insn)
4303 {
4304         if (!list_empty(&insn->call_node))
4305                 list_del_init(&insn->call_node);
4306 }
4307
4308 static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4309 {
4310         struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4311         struct instruction *first;
4312
4313         if (!sym)
4314                 return false;
4315
4316         first = find_insn(file, sym->sec, sym->offset);
4317         if (!first)
4318                 return false;
4319
4320         if (first->type != INSN_ENDBR && !first->noendbr)
4321                 return false;
4322
4323         return insn->offset == sym->offset + sym->len;
4324 }
4325
4326 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4327 {
4328         struct instruction *dest;
4329         struct reloc *reloc;
4330         unsigned long off;
4331         int warnings = 0;
4332
4333         /*
4334          * Looking for function pointer load relocations.  Ignore
4335          * direct/indirect branches:
4336          */
4337         switch (insn->type) {
4338         case INSN_CALL:
4339         case INSN_CALL_DYNAMIC:
4340         case INSN_JUMP_CONDITIONAL:
4341         case INSN_JUMP_UNCONDITIONAL:
4342         case INSN_JUMP_DYNAMIC:
4343         case INSN_JUMP_DYNAMIC_CONDITIONAL:
4344         case INSN_RETURN:
4345         case INSN_NOP:
4346                 return 0;
4347         default:
4348                 break;
4349         }
4350
4351         for (reloc = insn_reloc(file, insn);
4352              reloc;
4353              reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4354                                               reloc_offset(reloc) + 1,
4355                                               (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4356
4357                 /*
4358                  * static_call_update() references the trampoline, which
4359                  * doesn't have (or need) ENDBR.  Skip warning in that case.
4360                  */
4361                 if (reloc->sym->static_call_tramp)
4362                         continue;
4363
4364                 off = reloc->sym->offset;
4365                 if (reloc_type(reloc) == R_X86_64_PC32 ||
4366                     reloc_type(reloc) == R_X86_64_PLT32)
4367                         off += arch_dest_reloc_offset(reloc_addend(reloc));
4368                 else
4369                         off += reloc_addend(reloc);
4370
4371                 dest = find_insn(file, reloc->sym->sec, off);
4372                 if (!dest)
4373                         continue;
4374
4375                 if (dest->type == INSN_ENDBR) {
4376                         mark_endbr_used(dest);
4377                         continue;
4378                 }
4379
4380                 if (insn_func(dest) && insn_func(insn) &&
4381                     insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4382                         /*
4383                          * Anything from->to self is either _THIS_IP_ or
4384                          * IRET-to-self.
4385                          *
4386                          * There is no sane way to annotate _THIS_IP_ since the
4387                          * compiler treats the relocation as a constant and is
4388                          * happy to fold in offsets, skewing any annotation we
4389                          * do, leading to vast amounts of false-positives.
4390                          *
4391                          * There's also compiler generated _THIS_IP_ through
4392                          * KCOV and such which we have no hope of annotating.
4393                          *
4394                          * As such, blanket accept self-references without
4395                          * issue.
4396                          */
4397                         continue;
4398                 }
4399
4400                 /*
4401                  * Accept anything ANNOTATE_NOENDBR.
4402                  */
4403                 if (dest->noendbr)
4404                         continue;
4405
4406                 /*
4407                  * Accept if this is the instruction after a symbol
4408                  * that is (no)endbr -- typical code-range usage.
4409                  */
4410                 if (noendbr_range(file, dest))
4411                         continue;
4412
4413                 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4414
4415                 warnings++;
4416         }
4417
4418         return warnings;
4419 }
4420
4421 static int validate_ibt_data_reloc(struct objtool_file *file,
4422                                    struct reloc *reloc)
4423 {
4424         struct instruction *dest;
4425
4426         dest = find_insn(file, reloc->sym->sec,
4427                          reloc->sym->offset + reloc_addend(reloc));
4428         if (!dest)
4429                 return 0;
4430
4431         if (dest->type == INSN_ENDBR) {
4432                 mark_endbr_used(dest);
4433                 return 0;
4434         }
4435
4436         if (dest->noendbr)
4437                 return 0;
4438
4439         WARN_FUNC("data relocation to !ENDBR: %s",
4440                   reloc->sec->base, reloc_offset(reloc),
4441                   offstr(dest->sec, dest->offset));
4442
4443         return 1;
4444 }
4445
4446 /*
4447  * Validate IBT rules and remove used ENDBR instructions from the seal list.
4448  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4449  * NOPs) later, in create_ibt_endbr_seal_sections().
4450  */
4451 static int validate_ibt(struct objtool_file *file)
4452 {
4453         struct section *sec;
4454         struct reloc *reloc;
4455         struct instruction *insn;
4456         int warnings = 0;
4457
4458         for_each_insn(file, insn)
4459                 warnings += validate_ibt_insn(file, insn);
4460
4461         for_each_sec(file, sec) {
4462
4463                 /* Already done by validate_ibt_insn() */
4464                 if (sec->sh.sh_flags & SHF_EXECINSTR)
4465                         continue;
4466
4467                 if (!sec->rsec)
4468                         continue;
4469
4470                 /*
4471                  * These sections can reference text addresses, but not with
4472                  * the intent to indirect branch to them.
4473                  */
4474                 if ((!strncmp(sec->name, ".discard", 8) &&
4475                      strcmp(sec->name, ".discard.ibt_endbr_noseal"))    ||
4476                     !strncmp(sec->name, ".debug", 6)                    ||
4477                     !strcmp(sec->name, ".altinstructions")              ||
4478                     !strcmp(sec->name, ".ibt_endbr_seal")               ||
4479                     !strcmp(sec->name, ".orc_unwind_ip")                ||
4480                     !strcmp(sec->name, ".parainstructions")             ||
4481                     !strcmp(sec->name, ".retpoline_sites")              ||
4482                     !strcmp(sec->name, ".smp_locks")                    ||
4483                     !strcmp(sec->name, ".static_call_sites")            ||
4484                     !strcmp(sec->name, "_error_injection_whitelist")    ||
4485                     !strcmp(sec->name, "_kprobe_blacklist")             ||
4486                     !strcmp(sec->name, "__bug_table")                   ||
4487                     !strcmp(sec->name, "__ex_table")                    ||
4488                     !strcmp(sec->name, "__jump_table")                  ||
4489                     !strcmp(sec->name, "__mcount_loc")                  ||
4490                     !strcmp(sec->name, ".kcfi_traps")                   ||
4491                     strstr(sec->name, "__patchable_function_entries"))
4492                         continue;
4493
4494                 for_each_reloc(sec->rsec, reloc)
4495                         warnings += validate_ibt_data_reloc(file, reloc);
4496         }
4497
4498         return warnings;
4499 }
4500
4501 static int validate_sls(struct objtool_file *file)
4502 {
4503         struct instruction *insn, *next_insn;
4504         int warnings = 0;
4505
4506         for_each_insn(file, insn) {
4507                 next_insn = next_insn_same_sec(file, insn);
4508
4509                 if (insn->retpoline_safe)
4510                         continue;
4511
4512                 switch (insn->type) {
4513                 case INSN_RETURN:
4514                         if (!next_insn || next_insn->type != INSN_TRAP) {
4515                                 WARN_INSN(insn, "missing int3 after ret");
4516                                 warnings++;
4517                         }
4518
4519                         break;
4520                 case INSN_JUMP_DYNAMIC:
4521                         if (!next_insn || next_insn->type != INSN_TRAP) {
4522                                 WARN_INSN(insn, "missing int3 after indirect jump");
4523                                 warnings++;
4524                         }
4525                         break;
4526                 default:
4527                         break;
4528                 }
4529         }
4530
4531         return warnings;
4532 }
4533
4534 static bool ignore_noreturn_call(struct instruction *insn)
4535 {
4536         struct symbol *call_dest = insn_call_dest(insn);
4537
4538         /*
4539          * FIXME: hack, we need a real noreturn solution
4540          *
4541          * Problem is, exc_double_fault() may or may not return, depending on
4542          * whether CONFIG_X86_ESPFIX64 is set.  But objtool has no visibility
4543          * to the kernel config.
4544          *
4545          * Other potential ways to fix it:
4546          *
4547          *   - have compiler communicate __noreturn functions somehow
4548          *   - remove CONFIG_X86_ESPFIX64
4549          *   - read the .config file
4550          *   - add a cmdline option
4551          *   - create a generic objtool annotation format (vs a bunch of custom
4552          *     formats) and annotate it
4553          */
4554         if (!strcmp(call_dest->name, "exc_double_fault")) {
4555                 /* prevent further unreachable warnings for the caller */
4556                 insn->sym->warned = 1;
4557                 return true;
4558         }
4559
4560         return false;
4561 }
4562
4563 static int validate_reachable_instructions(struct objtool_file *file)
4564 {
4565         struct instruction *insn, *prev_insn;
4566         struct symbol *call_dest;
4567         int warnings = 0;
4568
4569         if (file->ignore_unreachables)
4570                 return 0;
4571
4572         for_each_insn(file, insn) {
4573                 if (insn->visited || ignore_unreachable_insn(file, insn))
4574                         continue;
4575
4576                 prev_insn = prev_insn_same_sec(file, insn);
4577                 if (prev_insn && prev_insn->dead_end) {
4578                         call_dest = insn_call_dest(prev_insn);
4579                         if (call_dest && !ignore_noreturn_call(prev_insn)) {
4580                                 WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4581                                           call_dest->name);
4582                                 warnings++;
4583                                 continue;
4584                         }
4585                 }
4586
4587                 WARN_INSN(insn, "unreachable instruction");
4588                 warnings++;
4589         }
4590
4591         return warnings;
4592 }
4593
4594 /* 'funcs' is a space-separated list of function names */
4595 static int disas_funcs(const char *funcs)
4596 {
4597         const char *objdump_str, *cross_compile;
4598         int size, ret;
4599         char *cmd;
4600
4601         cross_compile = getenv("CROSS_COMPILE");
4602
4603         objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4604                         "BEGIN { split(_funcs, funcs); }"
4605                         "/^$/ { func_match = 0; }"
4606                         "/<.*>:/ { "
4607                                 "f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4608                                 "for (i in funcs) {"
4609                                         "if (funcs[i] == f) {"
4610                                                 "func_match = 1;"
4611                                                 "base = strtonum(\"0x\" $1);"
4612                                                 "break;"
4613                                         "}"
4614                                 "}"
4615                         "}"
4616                         "{"
4617                                 "if (func_match) {"
4618                                         "addr = strtonum(\"0x\" $1);"
4619                                         "printf(\"%%04x \", addr - base);"
4620                                         "print;"
4621                                 "}"
4622                         "}' 1>&2";
4623
4624         /* fake snprintf() to calculate the size */
4625         size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4626         if (size <= 0) {
4627                 WARN("objdump string size calculation failed");
4628                 return -1;
4629         }
4630
4631         cmd = malloc(size);
4632
4633         /* real snprintf() */
4634         snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4635         ret = system(cmd);
4636         if (ret) {
4637                 WARN("disassembly failed: %d", ret);
4638                 return -1;
4639         }
4640
4641         return 0;
4642 }
4643
4644 static int disas_warned_funcs(struct objtool_file *file)
4645 {
4646         struct symbol *sym;
4647         char *funcs = NULL, *tmp;
4648
4649         for_each_sym(file, sym) {
4650                 if (sym->warned) {
4651                         if (!funcs) {
4652                                 funcs = malloc(strlen(sym->name) + 1);
4653                                 strcpy(funcs, sym->name);
4654                         } else {
4655                                 tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4656                                 sprintf(tmp, "%s %s", funcs, sym->name);
4657                                 free(funcs);
4658                                 funcs = tmp;
4659                         }
4660                 }
4661         }
4662
4663         if (funcs)
4664                 disas_funcs(funcs);
4665
4666         return 0;
4667 }
4668
4669 struct insn_chunk {
4670         void *addr;
4671         struct insn_chunk *next;
4672 };
4673
4674 /*
4675  * Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4676  * which can trigger more allocations for .debug_* sections whose data hasn't
4677  * been read yet.
4678  */
4679 static void free_insns(struct objtool_file *file)
4680 {
4681         struct instruction *insn;
4682         struct insn_chunk *chunks = NULL, *chunk;
4683
4684         for_each_insn(file, insn) {
4685                 if (!insn->idx) {
4686                         chunk = malloc(sizeof(*chunk));
4687                         chunk->addr = insn;
4688                         chunk->next = chunks;
4689                         chunks = chunk;
4690                 }
4691         }
4692
4693         for (chunk = chunks; chunk; chunk = chunk->next)
4694                 free(chunk->addr);
4695 }
4696
4697 int check(struct objtool_file *file)
4698 {
4699         int ret, warnings = 0;
4700
4701         arch_initial_func_cfi_state(&initial_func_cfi);
4702         init_cfi_state(&init_cfi);
4703         init_cfi_state(&func_cfi);
4704         set_func_state(&func_cfi);
4705         init_cfi_state(&force_undefined_cfi);
4706         force_undefined_cfi.force_undefined = true;
4707
4708         if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4709                 goto out;
4710
4711         cfi_hash_add(&init_cfi);
4712         cfi_hash_add(&func_cfi);
4713
4714         ret = decode_sections(file);
4715         if (ret < 0)
4716                 goto out;
4717
4718         warnings += ret;
4719
4720         if (!nr_insns)
4721                 goto out;
4722
4723         if (opts.retpoline) {
4724                 ret = validate_retpoline(file);
4725                 if (ret < 0)
4726                         return ret;
4727                 warnings += ret;
4728         }
4729
4730         if (opts.stackval || opts.orc || opts.uaccess) {
4731                 ret = validate_functions(file);
4732                 if (ret < 0)
4733                         goto out;
4734                 warnings += ret;
4735
4736                 ret = validate_unwind_hints(file, NULL);
4737                 if (ret < 0)
4738                         goto out;
4739                 warnings += ret;
4740
4741                 if (!warnings) {
4742                         ret = validate_reachable_instructions(file);
4743                         if (ret < 0)
4744                                 goto out;
4745                         warnings += ret;
4746                 }
4747
4748         } else if (opts.noinstr) {
4749                 ret = validate_noinstr_sections(file);
4750                 if (ret < 0)
4751                         goto out;
4752                 warnings += ret;
4753         }
4754
4755         if (opts.unret) {
4756                 /*
4757                  * Must be after validate_branch() and friends, it plays
4758                  * further games with insn->visited.
4759                  */
4760                 ret = validate_unrets(file);
4761                 if (ret < 0)
4762                         return ret;
4763                 warnings += ret;
4764         }
4765
4766         if (opts.ibt) {
4767                 ret = validate_ibt(file);
4768                 if (ret < 0)
4769                         goto out;
4770                 warnings += ret;
4771         }
4772
4773         if (opts.sls) {
4774                 ret = validate_sls(file);
4775                 if (ret < 0)
4776                         goto out;
4777                 warnings += ret;
4778         }
4779
4780         if (opts.static_call) {
4781                 ret = create_static_call_sections(file);
4782                 if (ret < 0)
4783                         goto out;
4784                 warnings += ret;
4785         }
4786
4787         if (opts.retpoline) {
4788                 ret = create_retpoline_sites_sections(file);
4789                 if (ret < 0)
4790                         goto out;
4791                 warnings += ret;
4792         }
4793
4794         if (opts.cfi) {
4795                 ret = create_cfi_sections(file);
4796                 if (ret < 0)
4797                         goto out;
4798                 warnings += ret;
4799         }
4800
4801         if (opts.rethunk) {
4802                 ret = create_return_sites_sections(file);
4803                 if (ret < 0)
4804                         goto out;
4805                 warnings += ret;
4806
4807                 if (opts.hack_skylake) {
4808                         ret = create_direct_call_sections(file);
4809                         if (ret < 0)
4810                                 goto out;
4811                         warnings += ret;
4812                 }
4813         }
4814
4815         if (opts.mcount) {
4816                 ret = create_mcount_loc_sections(file);
4817                 if (ret < 0)
4818                         goto out;
4819                 warnings += ret;
4820         }
4821
4822         if (opts.prefix) {
4823                 ret = add_prefix_symbols(file);
4824                 if (ret < 0)
4825                         return ret;
4826                 warnings += ret;
4827         }
4828
4829         if (opts.ibt) {
4830                 ret = create_ibt_endbr_seal_sections(file);
4831                 if (ret < 0)
4832                         goto out;
4833                 warnings += ret;
4834         }
4835
4836         if (opts.orc && nr_insns) {
4837                 ret = orc_create(file);
4838                 if (ret < 0)
4839                         goto out;
4840                 warnings += ret;
4841         }
4842
4843         free_insns(file);
4844
4845         if (opts.verbose)
4846                 disas_warned_funcs(file);
4847
4848         if (opts.stats) {
4849                 printf("nr_insns_visited: %ld\n", nr_insns_visited);
4850                 printf("nr_cfi: %ld\n", nr_cfi);
4851                 printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4852                 printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4853         }
4854
4855 out:
4856         /*
4857          *  For now, don't fail the kernel build on fatal warnings.  These
4858          *  errors are still fairly common due to the growing matrix of
4859          *  supported toolchains and their recent pace of change.
4860          */
4861         return 0;
4862 }