arch/x86/kernel/callthunks.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 #define pr_fmt(fmt) "callthunks: " fmt
   4
   5 #include <linux/debugfs.h>
   6 #include <linux/kallsyms.h>
   7 #include <linux/memory.h>
   8 #include <linux/moduleloader.h>
   9 #include <linux/static_call.h>
  10
  11 #include <asm/alternative.h>
  12 #include <asm/asm-offsets.h>
  13 #include <asm/cpu.h>
  14 #include <asm/ftrace.h>
  15 #include <asm/insn.h>
  16 #include <asm/kexec.h>
  17 #include <asm/nospec-branch.h>
  18 #include <asm/paravirt.h>
  19 #include <asm/sections.h>
  20 #include <asm/switch_to.h>
  21 #include <asm/sync_core.h>
  22 #include <asm/text-patching.h>
  23 #include <asm/xen/hypercall.h>
  24
  25 static int __initdata_or_module debug_callthunks;
  26
  27 #define prdbg(fmt, args...)                                     \
  28 do {                                                            \
  29         if (debug_callthunks)                                   \
  30                 printk(KERN_DEBUG pr_fmt(fmt), ##args);         \
  31 } while(0)
  32
  33 static int __init debug_thunks(char *str)
  34 {
  35         debug_callthunks = 1;
  36         return 1;
  37 }
  38 __setup("debug-callthunks", debug_thunks);
  39
  40 #ifdef CONFIG_CALL_THUNKS_DEBUG
  41 DEFINE_PER_CPU(u64, __x86_call_count);
  42 DEFINE_PER_CPU(u64, __x86_ret_count);
  43 DEFINE_PER_CPU(u64, __x86_stuffs_count);
  44 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
  45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
  46 EXPORT_SYMBOL_GPL(__x86_call_count);
  47 #endif
  48
  49 extern s32 __call_sites[], __call_sites_end[];
  50
  51 struct thunk_desc {
  52         void            *template;
  53         unsigned int    template_size;
  54 };
  55
  56 struct core_text {
  57         unsigned long   base;
  58         unsigned long   end;
  59         const char      *name;
  60 };
  61
  62 static bool thunks_initialized __ro_after_init;
  63
  64 static const struct core_text builtin_coretext = {
  65         .base = (unsigned long)_text,
  66         .end  = (unsigned long)_etext,
  67         .name = "builtin",
  68 };
  69
  70 asm (
  71         ".pushsection .rodata                           \n"
  72         ".global skl_call_thunk_template                \n"
  73         "skl_call_thunk_template:                       \n"
  74                 __stringify(INCREMENT_CALL_DEPTH)"      \n"
  75         ".global skl_call_thunk_tail                    \n"
  76         "skl_call_thunk_tail:                           \n"
  77         ".popsection                                    \n"
  78 );
  79
  80 extern u8 skl_call_thunk_template[];
  81 extern u8 skl_call_thunk_tail[];
  82
  83 #define SKL_TMPL_SIZE \
  84         ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
  85
  86 extern void error_entry(void);
  87 extern void xen_error_entry(void);
  88 extern void paranoid_entry(void);
  89
  90 static inline bool within_coretext(const struct core_text *ct, void *addr)
  91 {
  92         unsigned long p = (unsigned long)addr;
  93
  94         return ct->base <= p && p < ct->end;
  95 }
  96
  97 static inline bool within_module_coretext(void *addr)
  98 {
  99         bool ret = false;
 100
 101 #ifdef CONFIG_MODULES
 102         struct module *mod;
 103
 104         preempt_disable();
 105         mod = __module_address((unsigned long)addr);
 106         if (mod && within_module_core((unsigned long)addr, mod))
 107                 ret = true;
 108         preempt_enable();
 109 #endif
 110         return ret;
 111 }
 112
 113 static bool is_coretext(const struct core_text *ct, void *addr)
 114 {
 115         if (ct && within_coretext(ct, addr))
 116                 return true;
 117         if (within_coretext(&builtin_coretext, addr))
 118                 return true;
 119         return within_module_coretext(addr);
 120 }
 121
 122 static bool skip_addr(void *dest)
 123 {
 124         if (dest == error_entry)
 125                 return true;
 126         if (dest == paranoid_entry)
 127                 return true;
 128         if (dest == xen_error_entry)
 129                 return true;
 130         /* Does FILL_RSB... */
 131         if (dest == __switch_to_asm)
 132                 return true;
 133         /* Accounts directly */
 134         if (dest == ret_from_fork)
 135                 return true;
 136 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
 137         if (dest == soft_restart_cpu)
 138                 return true;
 139 #endif
 140 #ifdef CONFIG_FUNCTION_TRACER
 141         if (dest == __fentry__)
 142                 return true;
 143 #endif
 144 #ifdef CONFIG_KEXEC_CORE
 145         if (dest >= (void *)relocate_kernel &&
 146             dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
 147                 return true;
 148 #endif
 149 #ifdef CONFIG_XEN
 150         if (dest >= (void *)hypercall_page &&
 151             dest < (void*)hypercall_page + PAGE_SIZE)
 152                 return true;
 153 #endif
 154         return false;
 155 }
 156
 157 static __init_or_module void *call_get_dest(void *addr)
 158 {
 159         struct insn insn;
 160         void *dest;
 161         int ret;
 162
 163         ret = insn_decode_kernel(&insn, addr);
 164         if (ret)
 165                 return ERR_PTR(ret);
 166
 167         /* Patched out call? */
 168         if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
 169                 return NULL;
 170
 171         dest = addr + insn.length + insn.immediate.value;
 172         if (skip_addr(dest))
 173                 return NULL;
 174         return dest;
 175 }
 176
 177 static const u8 nops[] = {
 178         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 179         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 180         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 181         0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
 182 };
 183
 184 static void *patch_dest(void *dest, bool direct)
 185 {
 186         unsigned int tsize = SKL_TMPL_SIZE;
 187         u8 *pad = dest - tsize;
 188
 189         /* Already patched? */
 190         if (!bcmp(pad, skl_call_thunk_template, tsize))
 191                 return pad;
 192
 193         /* Ensure there are nops */
 194         if (bcmp(pad, nops, tsize)) {
 195                 pr_warn_once("Invalid padding area for %pS\n", dest);
 196                 return NULL;
 197         }
 198
 199         if (direct)
 200                 memcpy(pad, skl_call_thunk_template, tsize);
 201         else
 202                 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
 203         return pad;
 204 }
 205
 206 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
 207 {
 208         void *pad, *dest;
 209         u8 bytes[8];
 210
 211         if (!within_coretext(ct, addr))
 212                 return;
 213
 214         dest = call_get_dest(addr);
 215         if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
 216                 return;
 217
 218         if (!is_coretext(ct, dest))
 219                 return;
 220
 221         pad = patch_dest(dest, within_coretext(ct, dest));
 222         if (!pad)
 223                 return;
 224
 225         prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
 226                 dest, dest, pad);
 227         __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
 228         text_poke_early(addr, bytes, CALL_INSN_SIZE);
 229 }
 230
 231 static __init_or_module void
 232 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
 233 {
 234         s32 *s;
 235
 236         for (s = start; s < end; s++)
 237                 patch_call((void *)s + *s, ct);
 238 }
 239
 240 static __init_or_module void
 241 patch_paravirt_call_sites(struct paravirt_patch_site *start,
 242                           struct paravirt_patch_site *end,
 243                           const struct core_text *ct)
 244 {
 245         struct paravirt_patch_site *p;
 246
 247         for (p = start; p < end; p++)
 248                 patch_call(p->instr, ct);
 249 }
 250
 251 static __init_or_module void
 252 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
 253 {
 254         prdbg("Patching call sites %s\n", ct->name);
 255         patch_call_sites(cs->call_start, cs->call_end, ct);
 256         patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
 257         prdbg("Patching call sites done%s\n", ct->name);
 258 }
 259
 260 void __init callthunks_patch_builtin_calls(void)
 261 {
 262         struct callthunk_sites cs = {
 263                 .call_start     = __call_sites,
 264                 .call_end       = __call_sites_end,
 265                 .pv_start       = __parainstructions,
 266                 .pv_end         = __parainstructions_end
 267         };
 268
 269         if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
 270                 return;
 271
 272         pr_info("Setting up call depth tracking\n");
 273         mutex_lock(&text_mutex);
 274         callthunks_setup(&cs, &builtin_coretext);
 275         thunks_initialized = true;
 276         mutex_unlock(&text_mutex);
 277 }
 278
 279 void *callthunks_translate_call_dest(void *dest)
 280 {
 281         void *target;
 282
 283         lockdep_assert_held(&text_mutex);
 284
 285         if (!thunks_initialized || skip_addr(dest))
 286                 return dest;
 287
 288         if (!is_coretext(NULL, dest))
 289                 return dest;
 290
 291         target = patch_dest(dest, false);
 292         return target ? : dest;
 293 }
 294
 295 #ifdef CONFIG_BPF_JIT
 296 static bool is_callthunk(void *addr)
 297 {
 298         unsigned int tmpl_size = SKL_TMPL_SIZE;
 299         void *tmpl = skl_call_thunk_template;
 300         unsigned long dest;
 301
 302         dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
 303         if (!thunks_initialized || skip_addr((void *)dest))
 304                 return false;
 305
 306         return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
 307 }
 308
 309 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
 310 {
 311         unsigned int tmpl_size = SKL_TMPL_SIZE;
 312         void *tmpl = skl_call_thunk_template;
 313
 314         if (!thunks_initialized)
 315                 return 0;
 316
 317         /* Is function call target a thunk? */
 318         if (func && is_callthunk(func))
 319                 return 0;
 320
 321         memcpy(*pprog, tmpl, tmpl_size);
 322         *pprog += tmpl_size;
 323         return tmpl_size;
 324 }
 325 #endif
 326
 327 #ifdef CONFIG_MODULES
 328 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
 329                                             struct module *mod)
 330 {
 331         struct core_text ct = {
 332                 .base = (unsigned long)mod->mem[MOD_TEXT].base,
 333                 .end  = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
 334                 .name = mod->name,
 335         };
 336
 337         if (!thunks_initialized)
 338                 return;
 339
 340         mutex_lock(&text_mutex);
 341         callthunks_setup(cs, &ct);
 342         mutex_unlock(&text_mutex);
 343 }
 344 #endif /* CONFIG_MODULES */
 345
 346 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
 347 static int callthunks_debug_show(struct seq_file *m, void *p)
 348 {
 349         unsigned long cpu = (unsigned long)m->private;
 350
 351         seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
 352                    per_cpu(__x86_call_count, cpu),
 353                    per_cpu(__x86_ret_count, cpu),
 354                    per_cpu(__x86_stuffs_count, cpu),
 355                    per_cpu(__x86_ctxsw_count, cpu));
 356         return 0;
 357 }
 358
 359 static int callthunks_debug_open(struct inode *inode, struct file *file)
 360 {
 361         return single_open(file, callthunks_debug_show, inode->i_private);
 362 }
 363
 364 static const struct file_operations dfs_ops = {
 365         .open           = callthunks_debug_open,
 366         .read           = seq_read,
 367         .llseek         = seq_lseek,
 368         .release        = single_release,
 369 };
 370
 371 static int __init callthunks_debugfs_init(void)
 372 {
 373         struct dentry *dir;
 374         unsigned long cpu;
 375
 376         dir = debugfs_create_dir("callthunks", NULL);
 377         for_each_possible_cpu(cpu) {
 378                 void *arg = (void *)cpu;
 379                 char name [10];
 380
 381                 sprintf(name, "cpu%lu", cpu);
 382                 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
 383         }
 384         return 0;
 385 }
 386 __initcall(callthunks_debugfs_init);
 387 #endif