arch/powerpc/lib/code-patching.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *  Copyright 2008 Michael Ellerman, IBM Corporation.
   4  */
   5
   6 #include <linux/kprobes.h>
   7 #include <linux/mmu_context.h>
   8 #include <linux/random.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/init.h>
  11 #include <linux/cpuhotplug.h>
  12 #include <linux/uaccess.h>
  13 #include <linux/jump_label.h>
  14
  15 #include <asm/debug.h>
  16 #include <asm/pgalloc.h>
  17 #include <asm/tlb.h>
  18 #include <asm/tlbflush.h>
  19 #include <asm/page.h>
  20 #include <asm/code-patching.h>
  21 #include <asm/inst.h>
  22
  23 static int __patch_instruction(u32 *exec_addr, ppc_inst_t instr, u32 *patch_addr)
  24 {
  25         if (!ppc_inst_prefixed(instr)) {
  26                 u32 val = ppc_inst_val(instr);
  27
  28                 __put_kernel_nofault(patch_addr, &val, u32, failed);
  29         } else {
  30                 u64 val = ppc_inst_as_ulong(instr);
  31
  32                 __put_kernel_nofault(patch_addr, &val, u64, failed);
  33         }
  34
  35         asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
  36                                                             "r" (exec_addr));
  37
  38         return 0;
  39
  40 failed:
  41         return -EPERM;
  42 }
  43
  44 int raw_patch_instruction(u32 *addr, ppc_inst_t instr)
  45 {
  46         return __patch_instruction(addr, instr, addr);
  47 }
  48
  49 struct patch_context {
  50         union {
  51                 struct vm_struct *area;
  52                 struct mm_struct *mm;
  53         };
  54         unsigned long addr;
  55         pte_t *pte;
  56 };
  57
  58 static DEFINE_PER_CPU(struct patch_context, cpu_patching_context);
  59
  60 static int map_patch_area(void *addr, unsigned long text_poke_addr);
  61 static void unmap_patch_area(unsigned long addr);
  62
  63 static bool mm_patch_enabled(void)
  64 {
  65         return IS_ENABLED(CONFIG_SMP) && radix_enabled();
  66 }
  67
  68 /*
  69  * The following applies for Radix MMU. Hash MMU has different requirements,
  70  * and so is not supported.
  71  *
  72  * Changing mm requires context synchronising instructions on both sides of
  73  * the context switch, as well as a hwsync between the last instruction for
  74  * which the address of an associated storage access was translated using
  75  * the current context.
  76  *
  77  * switch_mm_irqs_off() performs an isync after the context switch. It is
  78  * the responsibility of the caller to perform the CSI and hwsync before
  79  * starting/stopping the temp mm.
  80  */
  81 static struct mm_struct *start_using_temp_mm(struct mm_struct *temp_mm)
  82 {
  83         struct mm_struct *orig_mm = current->active_mm;
  84
  85         lockdep_assert_irqs_disabled();
  86         switch_mm_irqs_off(orig_mm, temp_mm, current);
  87
  88         WARN_ON(!mm_is_thread_local(temp_mm));
  89
  90         suspend_breakpoints();
  91         return orig_mm;
  92 }
  93
  94 static void stop_using_temp_mm(struct mm_struct *temp_mm,
  95                                struct mm_struct *orig_mm)
  96 {
  97         lockdep_assert_irqs_disabled();
  98         switch_mm_irqs_off(temp_mm, orig_mm, current);
  99         restore_breakpoints();
 100 }
 101
 102 static int text_area_cpu_up(unsigned int cpu)
 103 {
 104         struct vm_struct *area;
 105         unsigned long addr;
 106         int err;
 107
 108         area = get_vm_area(PAGE_SIZE, VM_ALLOC);
 109         if (!area) {
 110                 WARN_ONCE(1, "Failed to create text area for cpu %d\n",
 111                         cpu);
 112                 return -1;
 113         }
 114
 115         // Map/unmap the area to ensure all page tables are pre-allocated
 116         addr = (unsigned long)area->addr;
 117         err = map_patch_area(empty_zero_page, addr);
 118         if (err)
 119                 return err;
 120
 121         unmap_patch_area(addr);
 122
 123         this_cpu_write(cpu_patching_context.area, area);
 124         this_cpu_write(cpu_patching_context.addr, addr);
 125         this_cpu_write(cpu_patching_context.pte, virt_to_kpte(addr));
 126
 127         return 0;
 128 }
 129
 130 static int text_area_cpu_down(unsigned int cpu)
 131 {
 132         free_vm_area(this_cpu_read(cpu_patching_context.area));
 133         this_cpu_write(cpu_patching_context.area, NULL);
 134         this_cpu_write(cpu_patching_context.addr, 0);
 135         this_cpu_write(cpu_patching_context.pte, NULL);
 136         return 0;
 137 }
 138
 139 static void put_patching_mm(struct mm_struct *mm, unsigned long patching_addr)
 140 {
 141         struct mmu_gather tlb;
 142
 143         tlb_gather_mmu(&tlb, mm);
 144         free_pgd_range(&tlb, patching_addr, patching_addr + PAGE_SIZE, 0, 0);
 145         mmput(mm);
 146 }
 147
 148 static int text_area_cpu_up_mm(unsigned int cpu)
 149 {
 150         struct mm_struct *mm;
 151         unsigned long addr;
 152         pte_t *pte;
 153         spinlock_t *ptl;
 154
 155         mm = mm_alloc();
 156         if (WARN_ON(!mm))
 157                 goto fail_no_mm;
 158
 159         /*
 160          * Choose a random page-aligned address from the interval
 161          * [PAGE_SIZE .. DEFAULT_MAP_WINDOW - PAGE_SIZE].
 162          * The lower address bound is PAGE_SIZE to avoid the zero-page.
 163          */
 164         addr = (1 + (get_random_long() % (DEFAULT_MAP_WINDOW / PAGE_SIZE - 2))) << PAGE_SHIFT;
 165
 166         /*
 167          * PTE allocation uses GFP_KERNEL which means we need to
 168          * pre-allocate the PTE here because we cannot do the
 169          * allocation during patching when IRQs are disabled.
 170          *
 171          * Using get_locked_pte() to avoid open coding, the lock
 172          * is unnecessary.
 173          */
 174         pte = get_locked_pte(mm, addr, &ptl);
 175         if (!pte)
 176                 goto fail_no_pte;
 177         pte_unmap_unlock(pte, ptl);
 178
 179         this_cpu_write(cpu_patching_context.mm, mm);
 180         this_cpu_write(cpu_patching_context.addr, addr);
 181
 182         return 0;
 183
 184 fail_no_pte:
 185         put_patching_mm(mm, addr);
 186 fail_no_mm:
 187         return -ENOMEM;
 188 }
 189
 190 static int text_area_cpu_down_mm(unsigned int cpu)
 191 {
 192         put_patching_mm(this_cpu_read(cpu_patching_context.mm),
 193                         this_cpu_read(cpu_patching_context.addr));
 194
 195         this_cpu_write(cpu_patching_context.mm, NULL);
 196         this_cpu_write(cpu_patching_context.addr, 0);
 197
 198         return 0;
 199 }
 200
 201 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poking_init_done);
 202
 203 void __init poking_init(void)
 204 {
 205         int ret;
 206
 207         if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
 208                 return;
 209
 210         if (mm_patch_enabled())
 211                 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
 212                                         "powerpc/text_poke_mm:online",
 213                                         text_area_cpu_up_mm,
 214                                         text_area_cpu_down_mm);
 215         else
 216                 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
 217                                         "powerpc/text_poke:online",
 218                                         text_area_cpu_up,
 219                                         text_area_cpu_down);
 220
 221         /* cpuhp_setup_state returns >= 0 on success */
 222         if (WARN_ON(ret < 0))
 223                 return;
 224
 225         static_branch_enable(&poking_init_done);
 226 }
 227
 228 static unsigned long get_patch_pfn(void *addr)
 229 {
 230         if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr))
 231                 return vmalloc_to_pfn(addr);
 232         else
 233                 return __pa_symbol(addr) >> PAGE_SHIFT;
 234 }
 235
 236 /*
 237  * This can be called for kernel text or a module.
 238  */
 239 static int map_patch_area(void *addr, unsigned long text_poke_addr)
 240 {
 241         unsigned long pfn = get_patch_pfn(addr);
 242
 243         return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
 244 }
 245
 246 static void unmap_patch_area(unsigned long addr)
 247 {
 248         pte_t *ptep;
 249         pmd_t *pmdp;
 250         pud_t *pudp;
 251         p4d_t *p4dp;
 252         pgd_t *pgdp;
 253
 254         pgdp = pgd_offset_k(addr);
 255         if (WARN_ON(pgd_none(*pgdp)))
 256                 return;
 257
 258         p4dp = p4d_offset(pgdp, addr);
 259         if (WARN_ON(p4d_none(*p4dp)))
 260                 return;
 261
 262         pudp = pud_offset(p4dp, addr);
 263         if (WARN_ON(pud_none(*pudp)))
 264                 return;
 265
 266         pmdp = pmd_offset(pudp, addr);
 267         if (WARN_ON(pmd_none(*pmdp)))
 268                 return;
 269
 270         ptep = pte_offset_kernel(pmdp, addr);
 271         if (WARN_ON(pte_none(*ptep)))
 272                 return;
 273
 274         /*
 275          * In hash, pte_clear flushes the tlb, in radix, we have to
 276          */
 277         pte_clear(&init_mm, addr, ptep);
 278         flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
 279 }
 280
 281 static int __do_patch_instruction_mm(u32 *addr, ppc_inst_t instr)
 282 {
 283         int err;
 284         u32 *patch_addr;
 285         unsigned long text_poke_addr;
 286         pte_t *pte;
 287         unsigned long pfn = get_patch_pfn(addr);
 288         struct mm_struct *patching_mm;
 289         struct mm_struct *orig_mm;
 290         spinlock_t *ptl;
 291
 292         patching_mm = __this_cpu_read(cpu_patching_context.mm);
 293         text_poke_addr = __this_cpu_read(cpu_patching_context.addr);
 294         patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
 295
 296         pte = get_locked_pte(patching_mm, text_poke_addr, &ptl);
 297         if (!pte)
 298                 return -ENOMEM;
 299
 300         __set_pte_at(patching_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
 301
 302         /* order PTE update before use, also serves as the hwsync */
 303         asm volatile("ptesync": : :"memory");
 304
 305         /* order context switch after arbitrary prior code */
 306         isync();
 307
 308         orig_mm = start_using_temp_mm(patching_mm);
 309
 310         err = __patch_instruction(addr, instr, patch_addr);
 311
 312         /* hwsync performed by __patch_instruction (sync) if successful */
 313         if (err)
 314                 mb();  /* sync */
 315
 316         /* context synchronisation performed by __patch_instruction (isync or exception) */
 317         stop_using_temp_mm(patching_mm, orig_mm);
 318
 319         pte_clear(patching_mm, text_poke_addr, pte);
 320         /*
 321          * ptesync to order PTE update before TLB invalidation done
 322          * by radix__local_flush_tlb_page_psize (in _tlbiel_va)
 323          */
 324         local_flush_tlb_page_psize(patching_mm, text_poke_addr, mmu_virtual_psize);
 325
 326         pte_unmap_unlock(pte, ptl);
 327
 328         return err;
 329 }
 330
 331 static int __do_patch_instruction(u32 *addr, ppc_inst_t instr)
 332 {
 333         int err;
 334         u32 *patch_addr;
 335         unsigned long text_poke_addr;
 336         pte_t *pte;
 337         unsigned long pfn = get_patch_pfn(addr);
 338
 339         text_poke_addr = (unsigned long)__this_cpu_read(cpu_patching_context.addr) & PAGE_MASK;
 340         patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
 341
 342         pte = __this_cpu_read(cpu_patching_context.pte);
 343         __set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
 344         /* See ptesync comment in radix__set_pte_at() */
 345         if (radix_enabled())
 346                 asm volatile("ptesync": : :"memory");
 347
 348         err = __patch_instruction(addr, instr, patch_addr);
 349
 350         pte_clear(&init_mm, text_poke_addr, pte);
 351         flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE);
 352
 353         return err;
 354 }
 355
 356 int patch_instruction(u32 *addr, ppc_inst_t instr)
 357 {
 358         int err;
 359         unsigned long flags;
 360
 361         /*
 362          * During early early boot patch_instruction is called
 363          * when text_poke_area is not ready, but we still need
 364          * to allow patching. We just do the plain old patching
 365          */
 366         if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) ||
 367             !static_branch_likely(&poking_init_done))
 368                 return raw_patch_instruction(addr, instr);
 369
 370         local_irq_save(flags);
 371         if (mm_patch_enabled())
 372                 err = __do_patch_instruction_mm(addr, instr);
 373         else
 374                 err = __do_patch_instruction(addr, instr);
 375         local_irq_restore(flags);
 376
 377         return err;
 378 }
 379 NOKPROBE_SYMBOL(patch_instruction);
 380
 381 int patch_branch(u32 *addr, unsigned long target, int flags)
 382 {
 383         ppc_inst_t instr;
 384
 385         if (create_branch(&instr, addr, target, flags))
 386                 return -ERANGE;
 387
 388         return patch_instruction(addr, instr);
 389 }
 390
 391 /*
 392  * Helper to check if a given instruction is a conditional branch
 393  * Derived from the conditional checks in analyse_instr()
 394  */
 395 bool is_conditional_branch(ppc_inst_t instr)
 396 {
 397         unsigned int opcode = ppc_inst_primary_opcode(instr);
 398
 399         if (opcode == 16)       /* bc, bca, bcl, bcla */
 400                 return true;
 401         if (opcode == 19) {
 402                 switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
 403                 case 16:        /* bclr, bclrl */
 404                 case 528:       /* bcctr, bcctrl */
 405                 case 560:       /* bctar, bctarl */
 406                         return true;
 407                 }
 408         }
 409         return false;
 410 }
 411 NOKPROBE_SYMBOL(is_conditional_branch);
 412
 413 int create_cond_branch(ppc_inst_t *instr, const u32 *addr,
 414                        unsigned long target, int flags)
 415 {
 416         long offset;
 417
 418         offset = target;
 419         if (! (flags & BRANCH_ABSOLUTE))
 420                 offset = offset - (unsigned long)addr;
 421
 422         /* Check we can represent the target in the instruction format */
 423         if (!is_offset_in_cond_branch_range(offset))
 424                 return 1;
 425
 426         /* Mask out the flags and target, so they don't step on each other. */
 427         *instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
 428
 429         return 0;
 430 }
 431
 432 int instr_is_relative_branch(ppc_inst_t instr)
 433 {
 434         if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
 435                 return 0;
 436
 437         return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
 438 }
 439
 440 int instr_is_relative_link_branch(ppc_inst_t instr)
 441 {
 442         return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
 443 }
 444
 445 static unsigned long branch_iform_target(const u32 *instr)
 446 {
 447         signed long imm;
 448
 449         imm = ppc_inst_val(ppc_inst_read(instr)) & 0x3FFFFFC;
 450
 451         /* If the top bit of the immediate value is set this is negative */
 452         if (imm & 0x2000000)
 453                 imm -= 0x4000000;
 454
 455         if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
 456                 imm += (unsigned long)instr;
 457
 458         return (unsigned long)imm;
 459 }
 460
 461 static unsigned long branch_bform_target(const u32 *instr)
 462 {
 463         signed long imm;
 464
 465         imm = ppc_inst_val(ppc_inst_read(instr)) & 0xFFFC;
 466
 467         /* If the top bit of the immediate value is set this is negative */
 468         if (imm & 0x8000)
 469                 imm -= 0x10000;
 470
 471         if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
 472                 imm += (unsigned long)instr;
 473
 474         return (unsigned long)imm;
 475 }
 476
 477 unsigned long branch_target(const u32 *instr)
 478 {
 479         if (instr_is_branch_iform(ppc_inst_read(instr)))
 480                 return branch_iform_target(instr);
 481         else if (instr_is_branch_bform(ppc_inst_read(instr)))
 482                 return branch_bform_target(instr);
 483
 484         return 0;
 485 }
 486
 487 int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src)
 488 {
 489         unsigned long target;
 490         target = branch_target(src);
 491
 492         if (instr_is_branch_iform(ppc_inst_read(src)))
 493                 return create_branch(instr, dest, target,
 494                                      ppc_inst_val(ppc_inst_read(src)));
 495         else if (instr_is_branch_bform(ppc_inst_read(src)))
 496                 return create_cond_branch(instr, dest, target,
 497                                           ppc_inst_val(ppc_inst_read(src)));
 498
 499         return 1;
 500 }