struct instruction *insn)
{
struct instruction *next = list_next_entry(insn, list);
- struct symbol *func = insn->func;
+ struct symbol *func = insn_func(insn);
if (!func)
return NULL;
- if (&next->list != &file->insn_list && next->func == func)
+ if (&next->list != &file->insn_list && insn_func(next) == func)
return next;
/* Check if we're already in the subfunction: */
{
struct instruction *prev = list_prev_entry(insn, list);
- if (&prev->list != &file->insn_list && prev->func == insn->func)
+ if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn))
return prev;
return NULL;
static bool is_sibling_call(struct instruction *insn)
{
/*
- * Assume only ELF functions can make sibling calls. This ensures
- * sibling call detection consistency between vmlinux.o and individual
- * objects.
+ * Assume only STT_FUNC calls have jump-tables.
*/
- if (!insn->func)
- return false;
-
- /* An indirect jump is either a sibling call or a jump to a table. */
- if (insn->type == INSN_JUMP_DYNAMIC)
- return !is_jump_table_jump(insn);
+ if (insn_func(insn)) {
+ /* An indirect jump is either a sibling call or a jump to a table. */
+ if (insn->type == INSN_JUMP_DYNAMIC)
+ return !is_jump_table_jump(insn);
+ }
/* add_jump_destinations() sets insn->call_dest for sibling calls. */
return (is_static_jump(insn) && insn->call_dest);
return false;
insn = find_insn(file, func->sec, func->offset);
- if (!insn || !insn->func)
+ if (!insn || !insn_func(insn))
return false;
func_for_each_insn(file, func, insn) {
return false;
}
- return __dead_end_function(file, dest->func, recursion+1);
+ return __dead_end_function(file, insn_func(dest), recursion+1);
}
}
!strncmp(sec->name, ".text.__x86.", 12))
sec->noinstr = true;
+ /*
+ * .init.text code is ran before userspace and thus doesn't
+ * strictly need retpolines, except for modules which are
+ * loaded late, they very much do need retpoline in their
+ * .init.text
+ */
+ if (!strcmp(sec->name, ".init.text") && !opts.module)
+ sec->init = true;
+
for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
insn = malloc(sizeof(*insn));
if (!insn) {
}
list_for_each_entry(func, &sec->symbol_list, list) {
- if (func->type != STT_FUNC || func->alias != func)
+ if (func->type != STT_NOTYPE && func->type != STT_FUNC)
+ continue;
+
+ if (func->return_thunk || func->alias != func)
continue;
if (!find_insn(file, sec, func->offset)) {
}
sym_for_each_insn(file, func, insn) {
- insn->func = func;
- if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
- if (insn->offset == insn->func->offset) {
+ insn->sym = func;
+ if (func->type == STT_FUNC &&
+ insn->type == INSN_ENDBR &&
+ list_empty(&insn->call_node)) {
+ if (insn->offset == func->offset) {
list_add_tail(&insn->call_node, &file->endbr_list);
file->nr_endbr++;
} else {
return 0;
}
+static int create_cfi_sections(struct objtool_file *file)
+{
+ struct section *sec, *s;
+ struct symbol *sym;
+ unsigned int *loc;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".cfi_sites");
+ if (sec) {
+ INIT_LIST_HEAD(&file->call_list);
+ WARN("file already has .cfi_sites section, skipping");
+ return 0;
+ }
+
+ idx = 0;
+ for_each_sec(file, s) {
+ if (!s->text)
+ continue;
+
+ list_for_each_entry(sym, &s->symbol_list, list) {
+ if (sym->type != STT_FUNC)
+ continue;
+
+ if (strncmp(sym->name, "__cfi_", 6))
+ continue;
+
+ idx++;
+ }
+ }
+
+ sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx);
+ if (!sec)
+ return -1;
+
+ idx = 0;
+ for_each_sec(file, s) {
+ if (!s->text)
+ continue;
+
+ list_for_each_entry(sym, &s->symbol_list, list) {
+ if (sym->type != STT_FUNC)
+ continue;
+
+ if (strncmp(sym->name, "__cfi_", 6))
+ continue;
+
+ loc = (unsigned int *)sec->data->d_buf + idx;
+ memset(loc, 0, sizeof(unsigned int));
+
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(unsigned int),
+ R_X86_64_PC32,
+ s, sym->offset))
+ return -1;
+
+ idx++;
+ }
+ }
+
+ return 0;
+}
+
static int create_mcount_loc_sections(struct objtool_file *file)
{
int addrsize = elf_class_addrsize(file->elf);
return 0;
}
+static int create_direct_call_sections(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct section *sec;
+ unsigned int *loc;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".call_sites");
+ if (sec) {
+ INIT_LIST_HEAD(&file->call_list);
+ WARN("file already has .call_sites section, skipping");
+ return 0;
+ }
+
+ if (list_empty(&file->call_list))
+ return 0;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->call_list, call_node)
+ idx++;
+
+ sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx);
+ if (!sec)
+ return -1;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->call_list, call_node) {
+
+ loc = (unsigned int *)sec->data->d_buf + idx;
+ memset(loc, 0, sizeof(unsigned int));
+
+ if (elf_add_reloc_to_insn(file->elf, sec,
+ idx * sizeof(unsigned int),
+ R_X86_64_PC32,
+ insn->sec, insn->offset))
+ return -1;
+
+ idx++;
+ }
+
+ return 0;
+}
+
/*
* Warnings shouldn't be reported for ignored functions.
*/
"__tsan_read_write4",
"__tsan_read_write8",
"__tsan_read_write16",
+ "__tsan_volatile_read1",
+ "__tsan_volatile_read2",
+ "__tsan_volatile_read4",
+ "__tsan_volatile_read8",
+ "__tsan_volatile_read16",
+ "__tsan_volatile_write1",
+ "__tsan_volatile_write2",
+ "__tsan_volatile_write4",
+ "__tsan_volatile_write8",
+ "__tsan_volatile_write16",
"__tsan_atomic8_load",
"__tsan_atomic16_load",
"__tsan_atomic32_load",
return;
}
+ if (insn->type == INSN_CALL && !insn->sec->init)
+ list_add_tail(&insn->call_node, &file->call_list);
+
if (!sibling && dead_end_function(file, sym))
insn->dead_end = true;
}
list_add_tail(&insn->call_node, &file->return_thunk_list);
}
-static bool same_function(struct instruction *insn1, struct instruction *insn2)
+static bool is_first_func_insn(struct objtool_file *file,
+ struct instruction *insn, struct symbol *sym)
{
- return insn1->func->pfunc == insn2->func->pfunc;
-}
-
-static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn)
-{
- if (insn->offset == insn->func->offset)
+ if (insn->offset == sym->offset)
return true;
+ /* Allow direct CALL/JMP past ENDBR */
if (opts.ibt) {
struct instruction *prev = prev_insn_same_sym(file, insn);
if (prev && prev->type == INSN_ENDBR &&
- insn->offset == insn->func->offset + prev->len)
+ insn->offset == sym->offset + prev->len)
return true;
}
return false;
}
+/*
+ * A sibling call is a tail-call to another symbol -- to differentiate from a
+ * recursive tail-call which is to the same symbol.
+ */
+static bool jump_is_sibling_call(struct objtool_file *file,
+ struct instruction *from, struct instruction *to)
+{
+ struct symbol *fs = from->sym;
+ struct symbol *ts = to->sym;
+
+ /* Not a sibling call if from/to a symbol hole */
+ if (!fs || !ts)
+ return false;
+
+ /* Not a sibling call if not targeting the start of a symbol. */
+ if (!is_first_func_insn(file, to, ts))
+ return false;
+
+ /* Disallow sibling calls into STT_NOTYPE */
+ if (ts->type == STT_NOTYPE)
+ return false;
+
+ /* Must not be self to be a sibling */
+ return fs->pfunc != ts->pfunc;
+}
+
/*
* Find the destination instructions for all jumps.
*/
} else if (reloc->sym->return_thunk) {
add_return_call(file, insn, true);
continue;
- } else if (insn->func) {
+ } else if (insn_func(insn)) {
/*
* External sibling call or internal sibling call with
* STT_FUNC reloc.
/*
* Cross-function jump.
*/
- if (insn->func && jump_dest->func &&
- insn->func != jump_dest->func) {
+ if (insn_func(insn) && insn_func(jump_dest) &&
+ insn_func(insn) != insn_func(jump_dest)) {
/*
* For GCC 8+, create parent/child links for any cold
* case where the parent function's only reference to a
* subfunction is through a jump table.
*/
- if (!strstr(insn->func->name, ".cold") &&
- strstr(jump_dest->func->name, ".cold")) {
- insn->func->cfunc = jump_dest->func;
- jump_dest->func->pfunc = insn->func;
-
- } else if (!same_function(insn, jump_dest) &&
- is_first_func_insn(file, jump_dest)) {
- /*
- * Internal sibling call without reloc or with
- * STT_SECTION reloc.
- */
- add_call_dest(file, insn, jump_dest->func, true);
- continue;
+ if (!strstr(insn_func(insn)->name, ".cold") &&
+ strstr(insn_func(jump_dest)->name, ".cold")) {
+ insn_func(insn)->cfunc = insn_func(jump_dest);
+ insn_func(jump_dest)->pfunc = insn_func(insn);
}
}
+ if (jump_is_sibling_call(file, insn, jump_dest)) {
+ /*
+ * Internal sibling call without reloc or with
+ * STT_SECTION reloc.
+ */
+ add_call_dest(file, insn, insn_func(jump_dest), true);
+ continue;
+ }
+
insn->jump_dest = jump_dest;
}
return -1;
}
- if (insn->func && insn->call_dest->type != STT_FUNC) {
+ if (insn_func(insn) && insn->call_dest->type != STT_FUNC) {
WARN_FUNC("unsupported call to non-function",
insn->sec, insn->offset);
return -1;
nop->offset = special_alt->new_off + special_alt->new_len;
nop->len = special_alt->orig_len - special_alt->new_len;
nop->type = INSN_NOP;
- nop->func = orig_insn->func;
+ nop->sym = orig_insn->sym;
nop->alt_group = new_alt_group;
nop->ignore = orig_insn->ignore_alts;
}
last_new_insn = insn;
insn->ignore = orig_insn->ignore_alts;
- insn->func = orig_insn->func;
+ insn->sym = orig_insn->sym;
insn->alt_group = new_alt_group;
/*
* accordingly.
*/
alt_reloc = insn_reloc(file, insn);
- if (alt_reloc &&
+ if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
WARN_FUNC("unsupported relocation in alternatives section",
struct reloc *reloc = table;
struct instruction *dest_insn;
struct alternative *alt;
- struct symbol *pfunc = insn->func->pfunc;
+ struct symbol *pfunc = insn_func(insn)->pfunc;
unsigned int prev_offset = 0;
/*
break;
/* Make sure the destination is in the same function: */
- if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
+ if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
break;
alt = malloc(sizeof(*alt));
* it.
*/
for (;
- insn && insn->func && insn->func->pfunc == func;
+ insn && insn_func(insn) && insn_func(insn)->pfunc == func;
insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
if (!table_reloc)
continue;
dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
- if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
+ if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
continue;
return table_reloc;
if (ret)
return ret;
+ /*
+ * Must be before add_{jump_call}_destination.
+ */
+ ret = classify_symbols(file);
+ if (ret)
+ return ret;
+
ret = decode_instructions(file);
if (ret)
return ret;
if (ret)
return ret;
- /*
- * Must be before add_{jump_call}_destination.
- */
- ret = classify_symbols(file);
- if (ret)
- return ret;
-
/*
* Must be before add_jump_destinations(), which depends on 'func'
* being set for alternatives, to enable proper sibling call detection.
/* stack operations don't make sense with an undefined CFA */
if (cfa->base == CFI_UNDEFINED) {
- if (insn->func) {
+ if (insn_func(insn)) {
WARN_FUNC("undefined stack state", insn->sec, insn->offset);
return -1;
}
}
/* detect when asm code uses rbp as a scratch register */
- if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
+ if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
cfa->base != CFI_BP)
cfi->bp_scratch = true;
break;
struct instruction *insn,
struct insn_state *state)
{
- if (has_modified_stack_frame(insn, state)) {
+ if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
WARN_FUNC("sibling call from callable instruction with modified stack frame",
insn->sec, insn->offset);
return 1;
while (1) {
next_insn = next_insn_to_validate(file, insn);
- if (func && insn->func && func != insn->func->pfunc) {
+ if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
/* Ignore KCFI type preambles, which always fall through */
- if (!strncmp(func->name, "__cfi_", 6))
+ if (!strncmp(func->name, "__cfi_", 6) ||
+ !strncmp(func->name, "__pfx_", 6))
return 0;
WARN("%s() falls through to next function %s()",
- func->name, insn->func->name);
+ func->name, insn_func(insn)->name);
return 1;
}
while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
if (insn->hint && !insn->visited && !insn->ignore) {
- ret = validate_branch(file, insn->func, insn, state);
+ ret = validate_branch(file, insn_func(insn), insn, state);
if (ret && opts.backtrace)
BT_FUNC("<=== (hint)", insn);
warnings += ret;
if (insn->retpoline_safe)
continue;
- /*
- * .init.text code is ran before userspace and thus doesn't
- * strictly need retpolines, except for modules which are
- * loaded late, they very much do need retpoline in their
- * .init.text
- */
- if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
+ if (insn->sec->init)
continue;
if (insn->type == INSN_RETURN) {
* In this case we'll find a piece of code (whole function) that is not
* covered by a !section symbol. Ignore them.
*/
- if (opts.link && !insn->func) {
+ if (opts.link && !insn_func(insn)) {
int size = find_symbol_hole_containing(insn->sec, insn->offset);
unsigned long end = insn->offset + size;
/*
* If this hole jumps to a .cold function, mark it ignore too.
*/
- if (insn->jump_dest && insn->jump_dest->func &&
- strstr(insn->jump_dest->func->name, ".cold")) {
+ if (insn->jump_dest && insn_func(insn->jump_dest) &&
+ strstr(insn_func(insn->jump_dest)->name, ".cold")) {
struct instruction *dest = insn->jump_dest;
- func_for_each_insn(file, dest->func, dest)
+ func_for_each_insn(file, insn_func(dest), dest)
dest->ignore = true;
}
}
return false;
}
- if (!insn->func)
+ if (!insn_func(insn))
return false;
- if (insn->func->static_call_tramp)
+ if (insn_func(insn)->static_call_tramp)
return true;
/*
if (insn->type == INSN_JUMP_UNCONDITIONAL) {
if (insn->jump_dest &&
- insn->jump_dest->func == insn->func) {
+ insn_func(insn->jump_dest) == insn_func(insn)) {
insn = insn->jump_dest;
continue;
}
break;
}
- if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
+ if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
break;
insn = list_next_entry(insn, list);
return false;
}
+static int add_prefix_symbol(struct objtool_file *file, struct symbol *func,
+ struct instruction *insn)
+{
+ if (!opts.prefix)
+ return 0;
+
+ for (;;) {
+ struct instruction *prev = list_prev_entry(insn, list);
+ u64 offset;
+
+ if (&prev->list == &file->insn_list)
+ break;
+
+ if (prev->type != INSN_NOP)
+ break;
+
+ offset = func->offset - prev->offset;
+ if (offset >= opts.prefix) {
+ if (offset == opts.prefix) {
+ /*
+ * Since the sec->symbol_list is ordered by
+ * offset (see elf_add_symbol()) the added
+ * symbol will not be seen by the iteration in
+ * validate_section().
+ *
+ * Hence the lack of list_for_each_entry_safe()
+ * there.
+ *
+ * The direct concequence is that prefix symbols
+ * don't get visited (because pointless), except
+ * for the logic in ignore_unreachable_insn()
+ * that needs the terminating insn to be visited
+ * otherwise it will report the hole.
+ *
+ * Hence mark the first instruction of the
+ * prefix symbol as visisted.
+ */
+ prev->visited |= VISITED_BRANCH;
+ elf_create_prefix_symbol(file->elf, func, opts.prefix);
+ }
+ break;
+ }
+ insn = prev;
+ }
+
+ return 0;
+}
+
static int validate_symbol(struct objtool_file *file, struct section *sec,
struct symbol *sym, struct insn_state *state)
{
if (!insn || insn->ignore || insn->visited)
return 0;
+ add_prefix_symbol(file, sym, insn);
+
state->uaccess = sym->uaccess_safe;
- ret = validate_branch(file, insn->func, insn, *state);
+ ret = validate_branch(file, insn_func(insn), insn, *state);
if (ret && opts.backtrace)
BT_FUNC("<=== (sym)", insn);
return ret;
list_del_init(&insn->call_node);
}
+static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
+{
+ struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
+ struct instruction *first;
+
+ if (!sym)
+ return false;
+
+ first = find_insn(file, sym->sec, sym->offset);
+ if (!first)
+ return false;
+
+ if (first->type != INSN_ENDBR && !first->noendbr)
+ return false;
+
+ return insn->offset == sym->offset + sym->len;
+}
+
static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
{
struct instruction *dest;
continue;
}
- if (dest->func && dest->func == insn->func) {
+ if (insn_func(dest) && insn_func(dest) == insn_func(insn)) {
/*
* Anything from->to self is either _THIS_IP_ or
* IRET-to-self.
continue;
}
+ /*
+ * Accept anything ANNOTATE_NOENDBR.
+ */
if (dest->noendbr)
continue;
+ /*
+ * Accept if this is the instruction after a symbol
+ * that is (no)endbr -- typical code-range usage.
+ */
+ if (noendbr_range(file, dest))
+ continue;
+
WARN_FUNC("relocation to !ENDBR: %s",
insn->sec, insn->offset,
offstr(dest->sec, dest->offset));
warnings += ret;
}
+ if (opts.cfi) {
+ ret = create_cfi_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
+
if (opts.rethunk) {
ret = create_return_sites_sections(file);
if (ret < 0)
goto out;
warnings += ret;
+
+ if (opts.hack_skylake) {
+ ret = create_direct_call_sections(file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+ }
}
if (opts.mcount) {
projects / linux-block.git / blobdiff