1 // SPDX-License-Identifier: GPL-2.0-only
3 #define pr_fmt(fmt) "callthunks: " fmt
5 #include <linux/debugfs.h>
6 #include <linux/kallsyms.h>
7 #include <linux/memory.h>
8 #include <linux/moduleloader.h>
9 #include <linux/static_call.h>
11 #include <asm/alternative.h>
12 #include <asm/asm-offsets.h>
14 #include <asm/ftrace.h>
16 #include <asm/kexec.h>
17 #include <asm/nospec-branch.h>
18 #include <asm/paravirt.h>
19 #include <asm/sections.h>
20 #include <asm/switch_to.h>
21 #include <asm/sync_core.h>
22 #include <asm/text-patching.h>
23 #include <asm/xen/hypercall.h>
25 static int __initdata_or_module debug_callthunks;
27 #define MAX_PATCH_LEN (255-1)
29 #define prdbg(fmt, args...) \
31 if (debug_callthunks) \
32 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
35 static int __init debug_thunks(char *str)
40 __setup("debug-callthunks", debug_thunks);
42 #ifdef CONFIG_CALL_THUNKS_DEBUG
43 DEFINE_PER_CPU(u64, __x86_call_count);
44 DEFINE_PER_CPU(u64, __x86_ret_count);
45 DEFINE_PER_CPU(u64, __x86_stuffs_count);
46 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
47 EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count);
48 EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count);
51 extern s32 __call_sites[], __call_sites_end[];
59 static bool thunks_initialized __ro_after_init;
61 static const struct core_text builtin_coretext = {
62 .base = (unsigned long)_text,
63 .end = (unsigned long)_etext,
68 ".pushsection .rodata \n"
69 ".global skl_call_thunk_template \n"
70 "skl_call_thunk_template: \n"
71 __stringify(INCREMENT_CALL_DEPTH)" \n"
72 ".global skl_call_thunk_tail \n"
73 "skl_call_thunk_tail: \n"
77 extern u8 skl_call_thunk_template[];
78 extern u8 skl_call_thunk_tail[];
80 #define SKL_TMPL_SIZE \
81 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
83 extern void error_entry(void);
84 extern void xen_error_entry(void);
85 extern void paranoid_entry(void);
87 static inline bool within_coretext(const struct core_text *ct, void *addr)
89 unsigned long p = (unsigned long)addr;
91 return ct->base <= p && p < ct->end;
94 static inline bool within_module_coretext(void *addr)
102 mod = __module_address((unsigned long)addr);
103 if (mod && within_module_core((unsigned long)addr, mod))
110 static bool is_coretext(const struct core_text *ct, void *addr)
112 if (ct && within_coretext(ct, addr))
114 if (within_coretext(&builtin_coretext, addr))
116 return within_module_coretext(addr);
119 static bool skip_addr(void *dest)
121 if (dest == error_entry)
123 if (dest == paranoid_entry)
125 if (dest == xen_error_entry)
127 /* Does FILL_RSB... */
128 if (dest == __switch_to_asm)
130 /* Accounts directly */
131 if (dest == ret_from_fork)
133 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
134 if (dest == soft_restart_cpu)
137 #ifdef CONFIG_FUNCTION_TRACER
138 if (dest == __fentry__)
141 #ifdef CONFIG_KEXEC_CORE
142 if (dest >= (void *)relocate_kernel &&
143 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
147 if (dest >= (void *)hypercall_page &&
148 dest < (void*)hypercall_page + PAGE_SIZE)
154 static __init_or_module void *call_get_dest(void *addr)
160 ret = insn_decode_kernel(&insn, addr);
164 /* Patched out call? */
165 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
168 dest = addr + insn.length + insn.immediate.value;
174 static const u8 nops[] = {
175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
178 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
181 static void *patch_dest(void *dest, bool direct)
183 unsigned int tsize = SKL_TMPL_SIZE;
184 u8 insn_buff[MAX_PATCH_LEN];
185 u8 *pad = dest - tsize;
187 memcpy(insn_buff, skl_call_thunk_template, tsize);
188 apply_relocation(insn_buff, tsize, pad,
189 skl_call_thunk_template, tsize);
191 /* Already patched? */
192 if (!bcmp(pad, insn_buff, tsize))
195 /* Ensure there are nops */
196 if (bcmp(pad, nops, tsize)) {
197 pr_warn_once("Invalid padding area for %pS\n", dest);
202 memcpy(pad, insn_buff, tsize);
204 text_poke_copy_locked(pad, insn_buff, tsize, true);
208 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
213 if (!within_coretext(ct, addr))
216 dest = call_get_dest(addr);
217 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
220 if (!is_coretext(ct, dest))
223 pad = patch_dest(dest, within_coretext(ct, dest));
227 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
229 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
230 text_poke_early(addr, bytes, CALL_INSN_SIZE);
233 static __init_or_module void
234 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
238 for (s = start; s < end; s++)
239 patch_call((void *)s + *s, ct);
242 static __init_or_module void
243 patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end,
244 const struct core_text *ct)
248 for (a = start; a < end; a++)
249 patch_call((void *)&a->instr_offset + a->instr_offset, ct);
252 static __init_or_module void
253 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
255 prdbg("Patching call sites %s\n", ct->name);
256 patch_call_sites(cs->call_start, cs->call_end, ct);
257 patch_alt_call_sites(cs->alt_start, cs->alt_end, ct);
258 prdbg("Patching call sites done%s\n", ct->name);
261 void __init callthunks_patch_builtin_calls(void)
263 struct callthunk_sites cs = {
264 .call_start = __call_sites,
265 .call_end = __call_sites_end,
266 .alt_start = __alt_instructions,
267 .alt_end = __alt_instructions_end
270 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
273 pr_info("Setting up call depth tracking\n");
274 mutex_lock(&text_mutex);
275 callthunks_setup(&cs, &builtin_coretext);
276 thunks_initialized = true;
277 mutex_unlock(&text_mutex);
280 void *callthunks_translate_call_dest(void *dest)
284 lockdep_assert_held(&text_mutex);
286 if (!thunks_initialized || skip_addr(dest))
289 if (!is_coretext(NULL, dest))
292 target = patch_dest(dest, false);
293 return target ? : dest;
296 #ifdef CONFIG_BPF_JIT
297 static bool is_callthunk(void *addr)
299 unsigned int tmpl_size = SKL_TMPL_SIZE;
300 u8 insn_buff[MAX_PATCH_LEN];
304 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
305 if (!thunks_initialized || skip_addr((void *)dest))
308 pad = (void *)(dest - tmpl_size);
310 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
311 apply_relocation(insn_buff, tmpl_size, pad,
312 skl_call_thunk_template, tmpl_size);
314 return !bcmp(pad, insn_buff, tmpl_size);
317 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
319 unsigned int tmpl_size = SKL_TMPL_SIZE;
320 u8 insn_buff[MAX_PATCH_LEN];
322 if (!thunks_initialized)
325 /* Is function call target a thunk? */
326 if (func && is_callthunk(func))
329 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
330 apply_relocation(insn_buff, tmpl_size, *pprog,
331 skl_call_thunk_template, tmpl_size);
333 memcpy(*pprog, insn_buff, tmpl_size);
339 #ifdef CONFIG_MODULES
340 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
343 struct core_text ct = {
344 .base = (unsigned long)mod->mem[MOD_TEXT].base,
345 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
349 if (!thunks_initialized)
352 mutex_lock(&text_mutex);
353 callthunks_setup(cs, &ct);
354 mutex_unlock(&text_mutex);
356 #endif /* CONFIG_MODULES */
358 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
359 static int callthunks_debug_show(struct seq_file *m, void *p)
361 unsigned long cpu = (unsigned long)m->private;
363 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
364 per_cpu(__x86_call_count, cpu),
365 per_cpu(__x86_ret_count, cpu),
366 per_cpu(__x86_stuffs_count, cpu),
367 per_cpu(__x86_ctxsw_count, cpu));
371 static int callthunks_debug_open(struct inode *inode, struct file *file)
373 return single_open(file, callthunks_debug_show, inode->i_private);
376 static const struct file_operations dfs_ops = {
377 .open = callthunks_debug_open,
380 .release = single_release,
383 static int __init callthunks_debugfs_init(void)
388 dir = debugfs_create_dir("callthunks", NULL);
389 for_each_possible_cpu(cpu) {
390 void *arg = (void *)cpu;
393 sprintf(name, "cpu%lu", cpu);
394 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
398 __initcall(callthunks_debugfs_init);