arch/arm64/kernel/ptrace.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/ptrace.c
   4  *
   5  * By Ross Biro 1/23/92
   6  * edited by Linus Torvalds
   7  * ARM modifications Copyright (C) 2000 Russell King
   8  * Copyright (C) 2012 ARM Ltd.
   9  */
  10
  11 #include <linux/audit.h>
  12 #include <linux/compat.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched/signal.h>
  15 #include <linux/sched/task_stack.h>
  16 #include <linux/mm.h>
  17 #include <linux/nospec.h>
  18 #include <linux/smp.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/user.h>
  21 #include <linux/seccomp.h>
  22 #include <linux/security.h>
  23 #include <linux/init.h>
  24 #include <linux/signal.h>
  25 #include <linux/string.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/perf_event.h>
  28 #include <linux/hw_breakpoint.h>
  29 #include <linux/regset.h>
  30 #include <linux/elf.h>
  31 #include <linux/rseq.h>
  32
  33 #include <asm/compat.h>
  34 #include <asm/cpufeature.h>
  35 #include <asm/debug-monitors.h>
  36 #include <asm/fpsimd.h>
  37 #include <asm/mte.h>
  38 #include <asm/pointer_auth.h>
  39 #include <asm/stacktrace.h>
  40 #include <asm/syscall.h>
  41 #include <asm/traps.h>
  42 #include <asm/system_misc.h>
  43
  44 #define CREATE_TRACE_POINTS
  45 #include <trace/events/syscalls.h>
  46
  47 struct pt_regs_offset {
  48         const char *name;
  49         int offset;
  50 };
  51
  52 #define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)}
  53 #define REG_OFFSET_END {.name = NULL, .offset = 0}
  54 #define GPR_OFFSET_NAME(r) \
  55         {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])}
  56
  57 static const struct pt_regs_offset regoffset_table[] = {
  58         GPR_OFFSET_NAME(0),
  59         GPR_OFFSET_NAME(1),
  60         GPR_OFFSET_NAME(2),
  61         GPR_OFFSET_NAME(3),
  62         GPR_OFFSET_NAME(4),
  63         GPR_OFFSET_NAME(5),
  64         GPR_OFFSET_NAME(6),
  65         GPR_OFFSET_NAME(7),
  66         GPR_OFFSET_NAME(8),
  67         GPR_OFFSET_NAME(9),
  68         GPR_OFFSET_NAME(10),
  69         GPR_OFFSET_NAME(11),
  70         GPR_OFFSET_NAME(12),
  71         GPR_OFFSET_NAME(13),
  72         GPR_OFFSET_NAME(14),
  73         GPR_OFFSET_NAME(15),
  74         GPR_OFFSET_NAME(16),
  75         GPR_OFFSET_NAME(17),
  76         GPR_OFFSET_NAME(18),
  77         GPR_OFFSET_NAME(19),
  78         GPR_OFFSET_NAME(20),
  79         GPR_OFFSET_NAME(21),
  80         GPR_OFFSET_NAME(22),
  81         GPR_OFFSET_NAME(23),
  82         GPR_OFFSET_NAME(24),
  83         GPR_OFFSET_NAME(25),
  84         GPR_OFFSET_NAME(26),
  85         GPR_OFFSET_NAME(27),
  86         GPR_OFFSET_NAME(28),
  87         GPR_OFFSET_NAME(29),
  88         GPR_OFFSET_NAME(30),
  89         {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])},
  90         REG_OFFSET_NAME(sp),
  91         REG_OFFSET_NAME(pc),
  92         REG_OFFSET_NAME(pstate),
  93         REG_OFFSET_END,
  94 };
  95
  96 /**
  97  * regs_query_register_offset() - query register offset from its name
  98  * @name:       the name of a register
  99  *
 100  * regs_query_register_offset() returns the offset of a register in struct
 101  * pt_regs from its name. If the name is invalid, this returns -EINVAL;
 102  */
 103 int regs_query_register_offset(const char *name)
 104 {
 105         const struct pt_regs_offset *roff;
 106
 107         for (roff = regoffset_table; roff->name != NULL; roff++)
 108                 if (!strcmp(roff->name, name))
 109                         return roff->offset;
 110         return -EINVAL;
 111 }
 112
 113 /**
 114  * regs_within_kernel_stack() - check the address in the stack
 115  * @regs:      pt_regs which contains kernel stack pointer.
 116  * @addr:      address which is checked.
 117  *
 118  * regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
 119  * If @addr is within the kernel stack, it returns true. If not, returns false.
 120  */
 121 static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
 122 {
 123         return ((addr & ~(THREAD_SIZE - 1))  ==
 124                 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
 125                 on_irq_stack(addr, sizeof(unsigned long));
 126 }
 127
 128 /**
 129  * regs_get_kernel_stack_nth() - get Nth entry of the stack
 130  * @regs:       pt_regs which contains kernel stack pointer.
 131  * @n:          stack entry number.
 132  *
 133  * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
 134  * is specified by @regs. If the @n th entry is NOT in the kernel stack,
 135  * this returns 0.
 136  */
 137 unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
 138 {
 139         unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
 140
 141         addr += n;
 142         if (regs_within_kernel_stack(regs, (unsigned long)addr))
 143                 return *addr;
 144         else
 145                 return 0;
 146 }
 147
 148 /*
 149  * TODO: does not yet catch signals sent when the child dies.
 150  * in exit.c or in signal.c.
 151  */
 152
 153 /*
 154  * Called by kernel/ptrace.c when detaching..
 155  */
 156 void ptrace_disable(struct task_struct *child)
 157 {
 158         /*
 159          * This would be better off in core code, but PTRACE_DETACH has
 160          * grown its fair share of arch-specific worts and changing it
 161          * is likely to cause regressions on obscure architectures.
 162          */
 163         user_disable_single_step(child);
 164 }
 165
 166 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 167 /*
 168  * Handle hitting a HW-breakpoint.
 169  */
 170 static void ptrace_hbptriggered(struct perf_event *bp,
 171                                 struct perf_sample_data *data,
 172                                 struct pt_regs *regs)
 173 {
 174         struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 175         const char *desc = "Hardware breakpoint trap (ptrace)";
 176
 177 #ifdef CONFIG_COMPAT
 178         if (is_compat_task()) {
 179                 int si_errno = 0;
 180                 int i;
 181
 182                 for (i = 0; i < ARM_MAX_BRP; ++i) {
 183                         if (current->thread.debug.hbp_break[i] == bp) {
 184                                 si_errno = (i << 1) + 1;
 185                                 break;
 186                         }
 187                 }
 188
 189                 for (i = 0; i < ARM_MAX_WRP; ++i) {
 190                         if (current->thread.debug.hbp_watch[i] == bp) {
 191                                 si_errno = -((i << 1) + 1);
 192                                 break;
 193                         }
 194                 }
 195                 arm64_force_sig_ptrace_errno_trap(si_errno, bkpt->trigger,
 196                                                   desc);
 197                 return;
 198         }
 199 #endif
 200         arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc);
 201 }
 202
 203 /*
 204  * Unregister breakpoints from this task and reset the pointers in
 205  * the thread_struct.
 206  */
 207 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 208 {
 209         int i;
 210         struct thread_struct *t = &tsk->thread;
 211
 212         for (i = 0; i < ARM_MAX_BRP; i++) {
 213                 if (t->debug.hbp_break[i]) {
 214                         unregister_hw_breakpoint(t->debug.hbp_break[i]);
 215                         t->debug.hbp_break[i] = NULL;
 216                 }
 217         }
 218
 219         for (i = 0; i < ARM_MAX_WRP; i++) {
 220                 if (t->debug.hbp_watch[i]) {
 221                         unregister_hw_breakpoint(t->debug.hbp_watch[i]);
 222                         t->debug.hbp_watch[i] = NULL;
 223                 }
 224         }
 225 }
 226
 227 void ptrace_hw_copy_thread(struct task_struct *tsk)
 228 {
 229         memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
 230 }
 231
 232 static struct perf_event *ptrace_hbp_get_event(unsigned int note_type,
 233                                                struct task_struct *tsk,
 234                                                unsigned long idx)
 235 {
 236         struct perf_event *bp = ERR_PTR(-EINVAL);
 237
 238         switch (note_type) {
 239         case NT_ARM_HW_BREAK:
 240                 if (idx >= ARM_MAX_BRP)
 241                         goto out;
 242                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 243                 bp = tsk->thread.debug.hbp_break[idx];
 244                 break;
 245         case NT_ARM_HW_WATCH:
 246                 if (idx >= ARM_MAX_WRP)
 247                         goto out;
 248                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 249                 bp = tsk->thread.debug.hbp_watch[idx];
 250                 break;
 251         }
 252
 253 out:
 254         return bp;
 255 }
 256
 257 static int ptrace_hbp_set_event(unsigned int note_type,
 258                                 struct task_struct *tsk,
 259                                 unsigned long idx,
 260                                 struct perf_event *bp)
 261 {
 262         int err = -EINVAL;
 263
 264         switch (note_type) {
 265         case NT_ARM_HW_BREAK:
 266                 if (idx >= ARM_MAX_BRP)
 267                         goto out;
 268                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 269                 tsk->thread.debug.hbp_break[idx] = bp;
 270                 err = 0;
 271                 break;
 272         case NT_ARM_HW_WATCH:
 273                 if (idx >= ARM_MAX_WRP)
 274                         goto out;
 275                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 276                 tsk->thread.debug.hbp_watch[idx] = bp;
 277                 err = 0;
 278                 break;
 279         }
 280
 281 out:
 282         return err;
 283 }
 284
 285 static struct perf_event *ptrace_hbp_create(unsigned int note_type,
 286                                             struct task_struct *tsk,
 287                                             unsigned long idx)
 288 {
 289         struct perf_event *bp;
 290         struct perf_event_attr attr;
 291         int err, type;
 292
 293         switch (note_type) {
 294         case NT_ARM_HW_BREAK:
 295                 type = HW_BREAKPOINT_X;
 296                 break;
 297         case NT_ARM_HW_WATCH:
 298                 type = HW_BREAKPOINT_RW;
 299                 break;
 300         default:
 301                 return ERR_PTR(-EINVAL);
 302         }
 303
 304         ptrace_breakpoint_init(&attr);
 305
 306         /*
 307          * Initialise fields to sane defaults
 308          * (i.e. values that will pass validation).
 309          */
 310         attr.bp_addr    = 0;
 311         attr.bp_len     = HW_BREAKPOINT_LEN_4;
 312         attr.bp_type    = type;
 313         attr.disabled   = 1;
 314
 315         bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk);
 316         if (IS_ERR(bp))
 317                 return bp;
 318
 319         err = ptrace_hbp_set_event(note_type, tsk, idx, bp);
 320         if (err)
 321                 return ERR_PTR(err);
 322
 323         return bp;
 324 }
 325
 326 static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 327                                      struct arch_hw_breakpoint_ctrl ctrl,
 328                                      struct perf_event_attr *attr)
 329 {
 330         int err, len, type, offset, disabled = !ctrl.enabled;
 331
 332         attr->disabled = disabled;
 333         if (disabled)
 334                 return 0;
 335
 336         err = arch_bp_generic_fields(ctrl, &len, &type, &offset);
 337         if (err)
 338                 return err;
 339
 340         switch (note_type) {
 341         case NT_ARM_HW_BREAK:
 342                 if ((type & HW_BREAKPOINT_X) != type)
 343                         return -EINVAL;
 344                 break;
 345         case NT_ARM_HW_WATCH:
 346                 if ((type & HW_BREAKPOINT_RW) != type)
 347                         return -EINVAL;
 348                 break;
 349         default:
 350                 return -EINVAL;
 351         }
 352
 353         attr->bp_len    = len;
 354         attr->bp_type   = type;
 355         attr->bp_addr   += offset;
 356
 357         return 0;
 358 }
 359
 360 static int ptrace_hbp_get_resource_info(unsigned int note_type, u32 *info)
 361 {
 362         u8 num;
 363         u32 reg = 0;
 364
 365         switch (note_type) {
 366         case NT_ARM_HW_BREAK:
 367                 num = hw_breakpoint_slots(TYPE_INST);
 368                 break;
 369         case NT_ARM_HW_WATCH:
 370                 num = hw_breakpoint_slots(TYPE_DATA);
 371                 break;
 372         default:
 373                 return -EINVAL;
 374         }
 375
 376         reg |= debug_monitors_arch();
 377         reg <<= 8;
 378         reg |= num;
 379
 380         *info = reg;
 381         return 0;
 382 }
 383
 384 static int ptrace_hbp_get_ctrl(unsigned int note_type,
 385                                struct task_struct *tsk,
 386                                unsigned long idx,
 387                                u32 *ctrl)
 388 {
 389         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 390
 391         if (IS_ERR(bp))
 392                 return PTR_ERR(bp);
 393
 394         *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0;
 395         return 0;
 396 }
 397
 398 static int ptrace_hbp_get_addr(unsigned int note_type,
 399                                struct task_struct *tsk,
 400                                unsigned long idx,
 401                                u64 *addr)
 402 {
 403         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 404
 405         if (IS_ERR(bp))
 406                 return PTR_ERR(bp);
 407
 408         *addr = bp ? counter_arch_bp(bp)->address : 0;
 409         return 0;
 410 }
 411
 412 static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type,
 413                                                         struct task_struct *tsk,
 414                                                         unsigned long idx)
 415 {
 416         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 417
 418         if (!bp)
 419                 bp = ptrace_hbp_create(note_type, tsk, idx);
 420
 421         return bp;
 422 }
 423
 424 static int ptrace_hbp_set_ctrl(unsigned int note_type,
 425                                struct task_struct *tsk,
 426                                unsigned long idx,
 427                                u32 uctrl)
 428 {
 429         int err;
 430         struct perf_event *bp;
 431         struct perf_event_attr attr;
 432         struct arch_hw_breakpoint_ctrl ctrl;
 433
 434         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 435         if (IS_ERR(bp)) {
 436                 err = PTR_ERR(bp);
 437                 return err;
 438         }
 439
 440         attr = bp->attr;
 441         decode_ctrl_reg(uctrl, &ctrl);
 442         err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr);
 443         if (err)
 444                 return err;
 445
 446         return modify_user_hw_breakpoint(bp, &attr);
 447 }
 448
 449 static int ptrace_hbp_set_addr(unsigned int note_type,
 450                                struct task_struct *tsk,
 451                                unsigned long idx,
 452                                u64 addr)
 453 {
 454         int err;
 455         struct perf_event *bp;
 456         struct perf_event_attr attr;
 457
 458         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 459         if (IS_ERR(bp)) {
 460                 err = PTR_ERR(bp);
 461                 return err;
 462         }
 463
 464         attr = bp->attr;
 465         attr.bp_addr = addr;
 466         err = modify_user_hw_breakpoint(bp, &attr);
 467         return err;
 468 }
 469
 470 #define PTRACE_HBP_ADDR_SZ      sizeof(u64)
 471 #define PTRACE_HBP_CTRL_SZ      sizeof(u32)
 472 #define PTRACE_HBP_PAD_SZ       sizeof(u32)
 473
 474 static int hw_break_get(struct task_struct *target,
 475                         const struct user_regset *regset,
 476                         struct membuf to)
 477 {
 478         unsigned int note_type = regset->core_note_type;
 479         int ret, idx = 0;
 480         u32 info, ctrl;
 481         u64 addr;
 482
 483         /* Resource info */
 484         ret = ptrace_hbp_get_resource_info(note_type, &info);
 485         if (ret)
 486                 return ret;
 487
 488         membuf_write(&to, &info, sizeof(info));
 489         membuf_zero(&to, sizeof(u32));
 490         /* (address, ctrl) registers */
 491         while (to.left) {
 492                 ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
 493                 if (ret)
 494                         return ret;
 495                 ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl);
 496                 if (ret)
 497                         return ret;
 498                 membuf_store(&to, addr);
 499                 membuf_store(&to, ctrl);
 500                 membuf_zero(&to, sizeof(u32));
 501                 idx++;
 502         }
 503         return 0;
 504 }
 505
 506 static int hw_break_set(struct task_struct *target,
 507                         const struct user_regset *regset,
 508                         unsigned int pos, unsigned int count,
 509                         const void *kbuf, const void __user *ubuf)
 510 {
 511         unsigned int note_type = regset->core_note_type;
 512         int ret, idx = 0, offset, limit;
 513         u32 ctrl;
 514         u64 addr;
 515
 516         /* Resource info and pad */
 517         offset = offsetof(struct user_hwdebug_state, dbg_regs);
 518         user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset);
 519
 520         /* (address, ctrl) registers */
 521         limit = regset->n * regset->size;
 522         while (count && offset < limit) {
 523                 if (count < PTRACE_HBP_ADDR_SZ)
 524                         return -EINVAL;
 525                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr,
 526                                          offset, offset + PTRACE_HBP_ADDR_SZ);
 527                 if (ret)
 528                         return ret;
 529                 ret = ptrace_hbp_set_addr(note_type, target, idx, addr);
 530                 if (ret)
 531                         return ret;
 532                 offset += PTRACE_HBP_ADDR_SZ;
 533
 534                 if (!count)
 535                         break;
 536                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl,
 537                                          offset, offset + PTRACE_HBP_CTRL_SZ);
 538                 if (ret)
 539                         return ret;
 540                 ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl);
 541                 if (ret)
 542                         return ret;
 543                 offset += PTRACE_HBP_CTRL_SZ;
 544
 545                 user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 546                                           offset, offset + PTRACE_HBP_PAD_SZ);
 547                 offset += PTRACE_HBP_PAD_SZ;
 548                 idx++;
 549         }
 550
 551         return 0;
 552 }
 553 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
 554
 555 static int gpr_get(struct task_struct *target,
 556                    const struct user_regset *regset,
 557                    struct membuf to)
 558 {
 559         struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
 560         return membuf_write(&to, uregs, sizeof(*uregs));
 561 }
 562
 563 static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 564                    unsigned int pos, unsigned int count,
 565                    const void *kbuf, const void __user *ubuf)
 566 {
 567         int ret;
 568         struct user_pt_regs newregs = task_pt_regs(target)->user_regs;
 569
 570         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1);
 571         if (ret)
 572                 return ret;
 573
 574         if (!valid_user_regs(&newregs, target))
 575                 return -EINVAL;
 576
 577         task_pt_regs(target)->user_regs = newregs;
 578         return 0;
 579 }
 580
 581 static int fpr_active(struct task_struct *target, const struct user_regset *regset)
 582 {
 583         if (!system_supports_fpsimd())
 584                 return -ENODEV;
 585         return regset->n;
 586 }
 587
 588 /*
 589  * TODO: update fp accessors for lazy context switching (sync/flush hwstate)
 590  */
 591 static int __fpr_get(struct task_struct *target,
 592                      const struct user_regset *regset,
 593                      struct membuf to)
 594 {
 595         struct user_fpsimd_state *uregs;
 596
 597         sve_sync_to_fpsimd(target);
 598
 599         uregs = &target->thread.uw.fpsimd_state;
 600
 601         return membuf_write(&to, uregs, sizeof(*uregs));
 602 }
 603
 604 static int fpr_get(struct task_struct *target, const struct user_regset *regset,
 605                    struct membuf to)
 606 {
 607         if (!system_supports_fpsimd())
 608                 return -EINVAL;
 609
 610         if (target == current)
 611                 fpsimd_preserve_current_state();
 612
 613         return __fpr_get(target, regset, to);
 614 }
 615
 616 static int __fpr_set(struct task_struct *target,
 617                      const struct user_regset *regset,
 618                      unsigned int pos, unsigned int count,
 619                      const void *kbuf, const void __user *ubuf,
 620                      unsigned int start_pos)
 621 {
 622         int ret;
 623         struct user_fpsimd_state newstate;
 624
 625         /*
 626          * Ensure target->thread.uw.fpsimd_state is up to date, so that a
 627          * short copyin can't resurrect stale data.
 628          */
 629         sve_sync_to_fpsimd(target);
 630
 631         newstate = target->thread.uw.fpsimd_state;
 632
 633         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
 634                                  start_pos, start_pos + sizeof(newstate));
 635         if (ret)
 636                 return ret;
 637
 638         target->thread.uw.fpsimd_state = newstate;
 639
 640         return ret;
 641 }
 642
 643 static int fpr_set(struct task_struct *target, const struct user_regset *regset,
 644                    unsigned int pos, unsigned int count,
 645                    const void *kbuf, const void __user *ubuf)
 646 {
 647         int ret;
 648
 649         if (!system_supports_fpsimd())
 650                 return -EINVAL;
 651
 652         ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
 653         if (ret)
 654                 return ret;
 655
 656         sve_sync_from_fpsimd_zeropad(target);
 657         fpsimd_flush_task_state(target);
 658
 659         return ret;
 660 }
 661
 662 static int tls_get(struct task_struct *target, const struct user_regset *regset,
 663                    struct membuf to)
 664 {
 665         int ret;
 666
 667         if (target == current)
 668                 tls_preserve_current_state();
 669
 670         ret = membuf_store(&to, target->thread.uw.tp_value);
 671         if (system_supports_tpidr2())
 672                 ret = membuf_store(&to, target->thread.tpidr2_el0);
 673         else
 674                 ret = membuf_zero(&to, sizeof(u64));
 675
 676         return ret;
 677 }
 678
 679 static int tls_set(struct task_struct *target, const struct user_regset *regset,
 680                    unsigned int pos, unsigned int count,
 681                    const void *kbuf, const void __user *ubuf)
 682 {
 683         int ret;
 684         unsigned long tls[2];
 685
 686         tls[0] = target->thread.uw.tp_value;
 687         if (system_supports_tpidr2())
 688                 tls[1] = target->thread.tpidr2_el0;
 689
 690         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, tls, 0, count);
 691         if (ret)
 692                 return ret;
 693
 694         target->thread.uw.tp_value = tls[0];
 695         if (system_supports_tpidr2())
 696                 target->thread.tpidr2_el0 = tls[1];
 697
 698         return ret;
 699 }
 700
 701 static int system_call_get(struct task_struct *target,
 702                            const struct user_regset *regset,
 703                            struct membuf to)
 704 {
 705         return membuf_store(&to, task_pt_regs(target)->syscallno);
 706 }
 707
 708 static int system_call_set(struct task_struct *target,
 709                            const struct user_regset *regset,
 710                            unsigned int pos, unsigned int count,
 711                            const void *kbuf, const void __user *ubuf)
 712 {
 713         int syscallno = task_pt_regs(target)->syscallno;
 714         int ret;
 715
 716         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1);
 717         if (ret)
 718                 return ret;
 719
 720         task_pt_regs(target)->syscallno = syscallno;
 721         return ret;
 722 }
 723
 724 #ifdef CONFIG_ARM64_SVE
 725
 726 static void sve_init_header_from_task(struct user_sve_header *header,
 727                                       struct task_struct *target,
 728                                       enum vec_type type)
 729 {
 730         unsigned int vq;
 731         bool active;
 732         bool fpsimd_only;
 733         enum vec_type task_type;
 734
 735         memset(header, 0, sizeof(*header));
 736
 737         /* Check if the requested registers are active for the task */
 738         if (thread_sm_enabled(&target->thread))
 739                 task_type = ARM64_VEC_SME;
 740         else
 741                 task_type = ARM64_VEC_SVE;
 742         active = (task_type == type);
 743
 744         switch (type) {
 745         case ARM64_VEC_SVE:
 746                 if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
 747                         header->flags |= SVE_PT_VL_INHERIT;
 748                 fpsimd_only = !test_tsk_thread_flag(target, TIF_SVE);
 749                 break;
 750         case ARM64_VEC_SME:
 751                 if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT))
 752                         header->flags |= SVE_PT_VL_INHERIT;
 753                 fpsimd_only = false;
 754                 break;
 755         default:
 756                 WARN_ON_ONCE(1);
 757                 return;
 758         }
 759
 760         if (active) {
 761                 if (fpsimd_only) {
 762                         header->flags |= SVE_PT_REGS_FPSIMD;
 763                 } else {
 764                         header->flags |= SVE_PT_REGS_SVE;
 765                 }
 766         }
 767
 768         header->vl = task_get_vl(target, type);
 769         vq = sve_vq_from_vl(header->vl);
 770
 771         header->max_vl = vec_max_vl(type);
 772         header->size = SVE_PT_SIZE(vq, header->flags);
 773         header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
 774                                       SVE_PT_REGS_SVE);
 775 }
 776
 777 static unsigned int sve_size_from_header(struct user_sve_header const *header)
 778 {
 779         return ALIGN(header->size, SVE_VQ_BYTES);
 780 }
 781
 782 static int sve_get_common(struct task_struct *target,
 783                           const struct user_regset *regset,
 784                           struct membuf to,
 785                           enum vec_type type)
 786 {
 787         struct user_sve_header header;
 788         unsigned int vq;
 789         unsigned long start, end;
 790
 791         /* Header */
 792         sve_init_header_from_task(&header, target, type);
 793         vq = sve_vq_from_vl(header.vl);
 794
 795         membuf_write(&to, &header, sizeof(header));
 796
 797         if (target == current)
 798                 fpsimd_preserve_current_state();
 799
 800         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 801         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 802
 803         switch ((header.flags & SVE_PT_REGS_MASK)) {
 804         case SVE_PT_REGS_FPSIMD:
 805                 return __fpr_get(target, regset, to);
 806
 807         case SVE_PT_REGS_SVE:
 808                 start = SVE_PT_SVE_OFFSET;
 809                 end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 810                 membuf_write(&to, target->thread.sve_state, end - start);
 811
 812                 start = end;
 813                 end = SVE_PT_SVE_FPSR_OFFSET(vq);
 814                 membuf_zero(&to, end - start);
 815
 816                 /*
 817                  * Copy fpsr, and fpcr which must follow contiguously in
 818                  * struct fpsimd_state:
 819                  */
 820                 start = end;
 821                 end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 822                 membuf_write(&to, &target->thread.uw.fpsimd_state.fpsr,
 823                              end - start);
 824
 825                 start = end;
 826                 end = sve_size_from_header(&header);
 827                 return membuf_zero(&to, end - start);
 828
 829         default:
 830                 return 0;
 831         }
 832 }
 833
 834 static int sve_get(struct task_struct *target,
 835                    const struct user_regset *regset,
 836                    struct membuf to)
 837 {
 838         if (!system_supports_sve())
 839                 return -EINVAL;
 840
 841         return sve_get_common(target, regset, to, ARM64_VEC_SVE);
 842 }
 843
 844 static int sve_set_common(struct task_struct *target,
 845                           const struct user_regset *regset,
 846                           unsigned int pos, unsigned int count,
 847                           const void *kbuf, const void __user *ubuf,
 848                           enum vec_type type)
 849 {
 850         int ret;
 851         struct user_sve_header header;
 852         unsigned int vq;
 853         unsigned long start, end;
 854
 855         /* Header */
 856         if (count < sizeof(header))
 857                 return -EINVAL;
 858         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
 859                                  0, sizeof(header));
 860         if (ret)
 861                 goto out;
 862
 863         /*
 864          * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are consumed by
 865          * vec_set_vector_length(), which will also validate them for us:
 866          */
 867         ret = vec_set_vector_length(target, type, header.vl,
 868                 ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
 869         if (ret)
 870                 goto out;
 871
 872         /* Actual VL set may be less than the user asked for: */
 873         vq = sve_vq_from_vl(task_get_vl(target, type));
 874
 875         /* Enter/exit streaming mode */
 876         if (system_supports_sme()) {
 877                 u64 old_svcr = target->thread.svcr;
 878
 879                 switch (type) {
 880                 case ARM64_VEC_SVE:
 881                         target->thread.svcr &= ~SVCR_SM_MASK;
 882                         break;
 883                 case ARM64_VEC_SME:
 884                         target->thread.svcr |= SVCR_SM_MASK;
 885
 886                         /*
 887                          * Disable traps and ensure there is SME storage but
 888                          * preserve any currently set values in ZA/ZT.
 889                          */
 890                         sme_alloc(target, false);
 891                         set_tsk_thread_flag(target, TIF_SME);
 892                         break;
 893                 default:
 894                         WARN_ON_ONCE(1);
 895                         ret = -EINVAL;
 896                         goto out;
 897                 }
 898
 899                 /*
 900                  * If we switched then invalidate any existing SVE
 901                  * state and ensure there's storage.
 902                  */
 903                 if (target->thread.svcr != old_svcr)
 904                         sve_alloc(target, true);
 905         }
 906
 907         /* Registers: FPSIMD-only case */
 908
 909         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 910         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
 911                 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
 912                                 SVE_PT_FPSIMD_OFFSET);
 913                 clear_tsk_thread_flag(target, TIF_SVE);
 914                 target->thread.fp_type = FP_STATE_FPSIMD;
 915                 goto out;
 916         }
 917
 918         /*
 919          * Otherwise: no registers or full SVE case.  For backwards
 920          * compatibility reasons we treat empty flags as SVE registers.
 921          */
 922
 923         /*
 924          * If setting a different VL from the requested VL and there is
 925          * register data, the data layout will be wrong: don't even
 926          * try to set the registers in this case.
 927          */
 928         if (count && vq != sve_vq_from_vl(header.vl)) {
 929                 ret = -EIO;
 930                 goto out;
 931         }
 932
 933         sve_alloc(target, true);
 934         if (!target->thread.sve_state) {
 935                 ret = -ENOMEM;
 936                 clear_tsk_thread_flag(target, TIF_SVE);
 937                 target->thread.fp_type = FP_STATE_FPSIMD;
 938                 goto out;
 939         }
 940
 941         /*
 942          * Ensure target->thread.sve_state is up to date with target's
 943          * FPSIMD regs, so that a short copyin leaves trailing
 944          * registers unmodified.  Only enable SVE if we are
 945          * configuring normal SVE, a system with streaming SVE may not
 946          * have normal SVE.
 947          */
 948         fpsimd_sync_to_sve(target);
 949         if (type == ARM64_VEC_SVE)
 950                 set_tsk_thread_flag(target, TIF_SVE);
 951         target->thread.fp_type = FP_STATE_SVE;
 952
 953         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 954         start = SVE_PT_SVE_OFFSET;
 955         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 956         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 957                                  target->thread.sve_state,
 958                                  start, end);
 959         if (ret)
 960                 goto out;
 961
 962         start = end;
 963         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 964         user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, start, end);
 965
 966         /*
 967          * Copy fpsr, and fpcr which must follow contiguously in
 968          * struct fpsimd_state:
 969          */
 970         start = end;
 971         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 972         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 973                                  &target->thread.uw.fpsimd_state.fpsr,
 974                                  start, end);
 975
 976 out:
 977         fpsimd_flush_task_state(target);
 978         return ret;
 979 }
 980
 981 static int sve_set(struct task_struct *target,
 982                    const struct user_regset *regset,
 983                    unsigned int pos, unsigned int count,
 984                    const void *kbuf, const void __user *ubuf)
 985 {
 986         if (!system_supports_sve())
 987                 return -EINVAL;
 988
 989         return sve_set_common(target, regset, pos, count, kbuf, ubuf,
 990                               ARM64_VEC_SVE);
 991 }
 992
 993 #endif /* CONFIG_ARM64_SVE */
 994
 995 #ifdef CONFIG_ARM64_SME
 996
 997 static int ssve_get(struct task_struct *target,
 998                    const struct user_regset *regset,
 999                    struct membuf to)
1000 {
1001         if (!system_supports_sme())
1002                 return -EINVAL;
1003
1004         return sve_get_common(target, regset, to, ARM64_VEC_SME);
1005 }
1006
1007 static int ssve_set(struct task_struct *target,
1008                     const struct user_regset *regset,
1009                     unsigned int pos, unsigned int count,
1010                     const void *kbuf, const void __user *ubuf)
1011 {
1012         if (!system_supports_sme())
1013                 return -EINVAL;
1014
1015         return sve_set_common(target, regset, pos, count, kbuf, ubuf,
1016                               ARM64_VEC_SME);
1017 }
1018
1019 static int za_get(struct task_struct *target,
1020                   const struct user_regset *regset,
1021                   struct membuf to)
1022 {
1023         struct user_za_header header;
1024         unsigned int vq;
1025         unsigned long start, end;
1026
1027         if (!system_supports_sme())
1028                 return -EINVAL;
1029
1030         /* Header */
1031         memset(&header, 0, sizeof(header));
1032
1033         if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT))
1034                 header.flags |= ZA_PT_VL_INHERIT;
1035
1036         header.vl = task_get_sme_vl(target);
1037         vq = sve_vq_from_vl(header.vl);
1038         header.max_vl = sme_max_vl();
1039         header.max_size = ZA_PT_SIZE(vq);
1040
1041         /* If ZA is not active there is only the header */
1042         if (thread_za_enabled(&target->thread))
1043                 header.size = ZA_PT_SIZE(vq);
1044         else
1045                 header.size = ZA_PT_ZA_OFFSET;
1046
1047         membuf_write(&to, &header, sizeof(header));
1048
1049         BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header));
1050         end = ZA_PT_ZA_OFFSET;
1051
1052         if (target == current)
1053                 fpsimd_preserve_current_state();
1054
1055         /* Any register data to include? */
1056         if (thread_za_enabled(&target->thread)) {
1057                 start = end;
1058                 end = ZA_PT_SIZE(vq);
1059                 membuf_write(&to, target->thread.sme_state, end - start);
1060         }
1061
1062         /* Zero any trailing padding */
1063         start = end;
1064         end = ALIGN(header.size, SVE_VQ_BYTES);
1065         return membuf_zero(&to, end - start);
1066 }
1067
1068 static int za_set(struct task_struct *target,
1069                   const struct user_regset *regset,
1070                   unsigned int pos, unsigned int count,
1071                   const void *kbuf, const void __user *ubuf)
1072 {
1073         int ret;
1074         struct user_za_header header;
1075         unsigned int vq;
1076         unsigned long start, end;
1077
1078         if (!system_supports_sme())
1079                 return -EINVAL;
1080
1081         /* Header */
1082         if (count < sizeof(header))
1083                 return -EINVAL;
1084         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
1085                                  0, sizeof(header));
1086         if (ret)
1087                 goto out;
1088
1089         /*
1090          * All current ZA_PT_* flags are consumed by
1091          * vec_set_vector_length(), which will also validate them for
1092          * us:
1093          */
1094         ret = vec_set_vector_length(target, ARM64_VEC_SME, header.vl,
1095                 ((unsigned long)header.flags) << 16);
1096         if (ret)
1097                 goto out;
1098
1099         /* Actual VL set may be less than the user asked for: */
1100         vq = sve_vq_from_vl(task_get_sme_vl(target));
1101
1102         /* Ensure there is some SVE storage for streaming mode */
1103         if (!target->thread.sve_state) {
1104                 sve_alloc(target, false);
1105                 if (!target->thread.sve_state) {
1106                         ret = -ENOMEM;
1107                         goto out;
1108                 }
1109         }
1110
1111         /*
1112          * Only flush the storage if PSTATE.ZA was not already set,
1113          * otherwise preserve any existing data.
1114          */
1115         sme_alloc(target, !thread_za_enabled(&target->thread));
1116         if (!target->thread.sme_state)
1117                 return -ENOMEM;
1118
1119         /* If there is no data then disable ZA */
1120         if (!count) {
1121                 target->thread.svcr &= ~SVCR_ZA_MASK;
1122                 goto out;
1123         }
1124
1125         /*
1126          * If setting a different VL from the requested VL and there is
1127          * register data, the data layout will be wrong: don't even
1128          * try to set the registers in this case.
1129          */
1130         if (vq != sve_vq_from_vl(header.vl)) {
1131                 ret = -EIO;
1132                 goto out;
1133         }
1134
1135         BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header));
1136         start = ZA_PT_ZA_OFFSET;
1137         end = ZA_PT_SIZE(vq);
1138         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1139                                  target->thread.sme_state,
1140                                  start, end);
1141         if (ret)
1142                 goto out;
1143
1144         /* Mark ZA as active and let userspace use it */
1145         set_tsk_thread_flag(target, TIF_SME);
1146         target->thread.svcr |= SVCR_ZA_MASK;
1147
1148 out:
1149         fpsimd_flush_task_state(target);
1150         return ret;
1151 }
1152
1153 static int zt_get(struct task_struct *target,
1154                   const struct user_regset *regset,
1155                   struct membuf to)
1156 {
1157         if (!system_supports_sme2())
1158                 return -EINVAL;
1159
1160         /*
1161          * If PSTATE.ZA is not set then ZT will be zeroed when it is
1162          * enabled so report the current register value as zero.
1163          */
1164         if (thread_za_enabled(&target->thread))
1165                 membuf_write(&to, thread_zt_state(&target->thread),
1166                              ZT_SIG_REG_BYTES);
1167         else
1168                 membuf_zero(&to, ZT_SIG_REG_BYTES);
1169
1170         return 0;
1171 }
1172
1173 static int zt_set(struct task_struct *target,
1174                   const struct user_regset *regset,
1175                   unsigned int pos, unsigned int count,
1176                   const void *kbuf, const void __user *ubuf)
1177 {
1178         int ret;
1179
1180         if (!system_supports_sme2())
1181                 return -EINVAL;
1182
1183         /* Ensure SVE storage in case this is first use of SME */
1184         sve_alloc(target, false);
1185         if (!target->thread.sve_state)
1186                 return -ENOMEM;
1187
1188         if (!thread_za_enabled(&target->thread)) {
1189                 sme_alloc(target, true);
1190                 if (!target->thread.sme_state)
1191                         return -ENOMEM;
1192         }
1193
1194         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1195                                  thread_zt_state(&target->thread),
1196                                  0, ZT_SIG_REG_BYTES);
1197         if (ret == 0) {
1198                 target->thread.svcr |= SVCR_ZA_MASK;
1199                 set_tsk_thread_flag(target, TIF_SME);
1200         }
1201
1202         fpsimd_flush_task_state(target);
1203
1204         return ret;
1205 }
1206
1207 #endif /* CONFIG_ARM64_SME */
1208
1209 #ifdef CONFIG_ARM64_PTR_AUTH
1210 static int pac_mask_get(struct task_struct *target,
1211                         const struct user_regset *regset,
1212                         struct membuf to)
1213 {
1214         /*
1215          * The PAC bits can differ across data and instruction pointers
1216          * depending on TCR_EL1.TBID*, which we may make use of in future, so
1217          * we expose separate masks.
1218          */
1219         unsigned long mask = ptrauth_user_pac_mask();
1220         struct user_pac_mask uregs = {
1221                 .data_mask = mask,
1222                 .insn_mask = mask,
1223         };
1224
1225         if (!system_supports_address_auth())
1226                 return -EINVAL;
1227
1228         return membuf_write(&to, &uregs, sizeof(uregs));
1229 }
1230
1231 static int pac_enabled_keys_get(struct task_struct *target,
1232                                 const struct user_regset *regset,
1233                                 struct membuf to)
1234 {
1235         long enabled_keys = ptrauth_get_enabled_keys(target);
1236
1237         if (IS_ERR_VALUE(enabled_keys))
1238                 return enabled_keys;
1239
1240         return membuf_write(&to, &enabled_keys, sizeof(enabled_keys));
1241 }
1242
1243 static int pac_enabled_keys_set(struct task_struct *target,
1244                                 const struct user_regset *regset,
1245                                 unsigned int pos, unsigned int count,
1246                                 const void *kbuf, const void __user *ubuf)
1247 {
1248         int ret;
1249         long enabled_keys = ptrauth_get_enabled_keys(target);
1250
1251         if (IS_ERR_VALUE(enabled_keys))
1252                 return enabled_keys;
1253
1254         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &enabled_keys, 0,
1255                                  sizeof(long));
1256         if (ret)
1257                 return ret;
1258
1259         return ptrauth_set_enabled_keys(target, PR_PAC_ENABLED_KEYS_MASK,
1260                                         enabled_keys);
1261 }
1262
1263 #ifdef CONFIG_CHECKPOINT_RESTORE
1264 static __uint128_t pac_key_to_user(const struct ptrauth_key *key)
1265 {
1266         return (__uint128_t)key->hi << 64 | key->lo;
1267 }
1268
1269 static struct ptrauth_key pac_key_from_user(__uint128_t ukey)
1270 {
1271         struct ptrauth_key key = {
1272                 .lo = (unsigned long)ukey,
1273                 .hi = (unsigned long)(ukey >> 64),
1274         };
1275
1276         return key;
1277 }
1278
1279 static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys,
1280                                      const struct ptrauth_keys_user *keys)
1281 {
1282         ukeys->apiakey = pac_key_to_user(&keys->apia);
1283         ukeys->apibkey = pac_key_to_user(&keys->apib);
1284         ukeys->apdakey = pac_key_to_user(&keys->apda);
1285         ukeys->apdbkey = pac_key_to_user(&keys->apdb);
1286 }
1287
1288 static void pac_address_keys_from_user(struct ptrauth_keys_user *keys,
1289                                        const struct user_pac_address_keys *ukeys)
1290 {
1291         keys->apia = pac_key_from_user(ukeys->apiakey);
1292         keys->apib = pac_key_from_user(ukeys->apibkey);
1293         keys->apda = pac_key_from_user(ukeys->apdakey);
1294         keys->apdb = pac_key_from_user(ukeys->apdbkey);
1295 }
1296
1297 static int pac_address_keys_get(struct task_struct *target,
1298                                 const struct user_regset *regset,
1299                                 struct membuf to)
1300 {
1301         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1302         struct user_pac_address_keys user_keys;
1303
1304         if (!system_supports_address_auth())
1305                 return -EINVAL;
1306
1307         pac_address_keys_to_user(&user_keys, keys);
1308
1309         return membuf_write(&to, &user_keys, sizeof(user_keys));
1310 }
1311
1312 static int pac_address_keys_set(struct task_struct *target,
1313                                 const struct user_regset *regset,
1314                                 unsigned int pos, unsigned int count,
1315                                 const void *kbuf, const void __user *ubuf)
1316 {
1317         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1318         struct user_pac_address_keys user_keys;
1319         int ret;
1320
1321         if (!system_supports_address_auth())
1322                 return -EINVAL;
1323
1324         pac_address_keys_to_user(&user_keys, keys);
1325         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1326                                  &user_keys, 0, -1);
1327         if (ret)
1328                 return ret;
1329         pac_address_keys_from_user(keys, &user_keys);
1330
1331         return 0;
1332 }
1333
1334 static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys,
1335                                      const struct ptrauth_keys_user *keys)
1336 {
1337         ukeys->apgakey = pac_key_to_user(&keys->apga);
1338 }
1339
1340 static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys,
1341                                        const struct user_pac_generic_keys *ukeys)
1342 {
1343         keys->apga = pac_key_from_user(ukeys->apgakey);
1344 }
1345
1346 static int pac_generic_keys_get(struct task_struct *target,
1347                                 const struct user_regset *regset,
1348                                 struct membuf to)
1349 {
1350         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1351         struct user_pac_generic_keys user_keys;
1352
1353         if (!system_supports_generic_auth())
1354                 return -EINVAL;
1355
1356         pac_generic_keys_to_user(&user_keys, keys);
1357
1358         return membuf_write(&to, &user_keys, sizeof(user_keys));
1359 }
1360
1361 static int pac_generic_keys_set(struct task_struct *target,
1362                                 const struct user_regset *regset,
1363                                 unsigned int pos, unsigned int count,
1364                                 const void *kbuf, const void __user *ubuf)
1365 {
1366         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1367         struct user_pac_generic_keys user_keys;
1368         int ret;
1369
1370         if (!system_supports_generic_auth())
1371                 return -EINVAL;
1372
1373         pac_generic_keys_to_user(&user_keys, keys);
1374         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1375                                  &user_keys, 0, -1);
1376         if (ret)
1377                 return ret;
1378         pac_generic_keys_from_user(keys, &user_keys);
1379
1380         return 0;
1381 }
1382 #endif /* CONFIG_CHECKPOINT_RESTORE */
1383 #endif /* CONFIG_ARM64_PTR_AUTH */
1384
1385 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1386 static int tagged_addr_ctrl_get(struct task_struct *target,
1387                                 const struct user_regset *regset,
1388                                 struct membuf to)
1389 {
1390         long ctrl = get_tagged_addr_ctrl(target);
1391
1392         if (IS_ERR_VALUE(ctrl))
1393                 return ctrl;
1394
1395         return membuf_write(&to, &ctrl, sizeof(ctrl));
1396 }
1397
1398 static int tagged_addr_ctrl_set(struct task_struct *target, const struct
1399                                 user_regset *regset, unsigned int pos,
1400                                 unsigned int count, const void *kbuf, const
1401                                 void __user *ubuf)
1402 {
1403         int ret;
1404         long ctrl;
1405
1406         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
1407         if (ret)
1408                 return ret;
1409
1410         return set_tagged_addr_ctrl(target, ctrl);
1411 }
1412 #endif
1413
1414 enum aarch64_regset {
1415         REGSET_GPR,
1416         REGSET_FPR,
1417         REGSET_TLS,
1418 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1419         REGSET_HW_BREAK,
1420         REGSET_HW_WATCH,
1421 #endif
1422         REGSET_SYSTEM_CALL,
1423 #ifdef CONFIG_ARM64_SVE
1424         REGSET_SVE,
1425 #endif
1426 #ifdef CONFIG_ARM64_SME
1427         REGSET_SSVE,
1428         REGSET_ZA,
1429         REGSET_ZT,
1430 #endif
1431 #ifdef CONFIG_ARM64_PTR_AUTH
1432         REGSET_PAC_MASK,
1433         REGSET_PAC_ENABLED_KEYS,
1434 #ifdef CONFIG_CHECKPOINT_RESTORE
1435         REGSET_PACA_KEYS,
1436         REGSET_PACG_KEYS,
1437 #endif
1438 #endif
1439 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1440         REGSET_TAGGED_ADDR_CTRL,
1441 #endif
1442 };
1443
1444 static const struct user_regset aarch64_regsets[] = {
1445         [REGSET_GPR] = {
1446                 .core_note_type = NT_PRSTATUS,
1447                 .n = sizeof(struct user_pt_regs) / sizeof(u64),
1448                 .size = sizeof(u64),
1449                 .align = sizeof(u64),
1450                 .regset_get = gpr_get,
1451                 .set = gpr_set
1452         },
1453         [REGSET_FPR] = {
1454                 .core_note_type = NT_PRFPREG,
1455                 .n = sizeof(struct user_fpsimd_state) / sizeof(u32),
1456                 /*
1457                  * We pretend we have 32-bit registers because the fpsr and
1458                  * fpcr are 32-bits wide.
1459                  */
1460                 .size = sizeof(u32),
1461                 .align = sizeof(u32),
1462                 .active = fpr_active,
1463                 .regset_get = fpr_get,
1464                 .set = fpr_set
1465         },
1466         [REGSET_TLS] = {
1467                 .core_note_type = NT_ARM_TLS,
1468                 .n = 2,
1469                 .size = sizeof(void *),
1470                 .align = sizeof(void *),
1471                 .regset_get = tls_get,
1472                 .set = tls_set,
1473         },
1474 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1475         [REGSET_HW_BREAK] = {
1476                 .core_note_type = NT_ARM_HW_BREAK,
1477                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1478                 .size = sizeof(u32),
1479                 .align = sizeof(u32),
1480                 .regset_get = hw_break_get,
1481                 .set = hw_break_set,
1482         },
1483         [REGSET_HW_WATCH] = {
1484                 .core_note_type = NT_ARM_HW_WATCH,
1485                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1486                 .size = sizeof(u32),
1487                 .align = sizeof(u32),
1488                 .regset_get = hw_break_get,
1489                 .set = hw_break_set,
1490         },
1491 #endif
1492         [REGSET_SYSTEM_CALL] = {
1493                 .core_note_type = NT_ARM_SYSTEM_CALL,
1494                 .n = 1,
1495                 .size = sizeof(int),
1496                 .align = sizeof(int),
1497                 .regset_get = system_call_get,
1498                 .set = system_call_set,
1499         },
1500 #ifdef CONFIG_ARM64_SVE
1501         [REGSET_SVE] = { /* Scalable Vector Extension */
1502                 .core_note_type = NT_ARM_SVE,
1503                 .n = DIV_ROUND_UP(SVE_PT_SIZE(ARCH_SVE_VQ_MAX,
1504                                               SVE_PT_REGS_SVE),
1505                                   SVE_VQ_BYTES),
1506                 .size = SVE_VQ_BYTES,
1507                 .align = SVE_VQ_BYTES,
1508                 .regset_get = sve_get,
1509                 .set = sve_set,
1510         },
1511 #endif
1512 #ifdef CONFIG_ARM64_SME
1513         [REGSET_SSVE] = { /* Streaming mode SVE */
1514                 .core_note_type = NT_ARM_SSVE,
1515                 .n = DIV_ROUND_UP(SVE_PT_SIZE(SME_VQ_MAX, SVE_PT_REGS_SVE),
1516                                   SVE_VQ_BYTES),
1517                 .size = SVE_VQ_BYTES,
1518                 .align = SVE_VQ_BYTES,
1519                 .regset_get = ssve_get,
1520                 .set = ssve_set,
1521         },
1522         [REGSET_ZA] = { /* SME ZA */
1523                 .core_note_type = NT_ARM_ZA,
1524                 /*
1525                  * ZA is a single register but it's variably sized and
1526                  * the ptrace core requires that the size of any data
1527                  * be an exact multiple of the configured register
1528                  * size so report as though we had SVE_VQ_BYTES
1529                  * registers. These values aren't exposed to
1530                  * userspace.
1531                  */
1532                 .n = DIV_ROUND_UP(ZA_PT_SIZE(SME_VQ_MAX), SVE_VQ_BYTES),
1533                 .size = SVE_VQ_BYTES,
1534                 .align = SVE_VQ_BYTES,
1535                 .regset_get = za_get,
1536                 .set = za_set,
1537         },
1538         [REGSET_ZT] = { /* SME ZT */
1539                 .core_note_type = NT_ARM_ZT,
1540                 .n = 1,
1541                 .size = ZT_SIG_REG_BYTES,
1542                 .align = sizeof(u64),
1543                 .regset_get = zt_get,
1544                 .set = zt_set,
1545         },
1546 #endif
1547 #ifdef CONFIG_ARM64_PTR_AUTH
1548         [REGSET_PAC_MASK] = {
1549                 .core_note_type = NT_ARM_PAC_MASK,
1550                 .n = sizeof(struct user_pac_mask) / sizeof(u64),
1551                 .size = sizeof(u64),
1552                 .align = sizeof(u64),
1553                 .regset_get = pac_mask_get,
1554                 /* this cannot be set dynamically */
1555         },
1556         [REGSET_PAC_ENABLED_KEYS] = {
1557                 .core_note_type = NT_ARM_PAC_ENABLED_KEYS,
1558                 .n = 1,
1559                 .size = sizeof(long),
1560                 .align = sizeof(long),
1561                 .regset_get = pac_enabled_keys_get,
1562                 .set = pac_enabled_keys_set,
1563         },
1564 #ifdef CONFIG_CHECKPOINT_RESTORE
1565         [REGSET_PACA_KEYS] = {
1566                 .core_note_type = NT_ARM_PACA_KEYS,
1567                 .n = sizeof(struct user_pac_address_keys) / sizeof(__uint128_t),
1568                 .size = sizeof(__uint128_t),
1569                 .align = sizeof(__uint128_t),
1570                 .regset_get = pac_address_keys_get,
1571                 .set = pac_address_keys_set,
1572         },
1573         [REGSET_PACG_KEYS] = {
1574                 .core_note_type = NT_ARM_PACG_KEYS,
1575                 .n = sizeof(struct user_pac_generic_keys) / sizeof(__uint128_t),
1576                 .size = sizeof(__uint128_t),
1577                 .align = sizeof(__uint128_t),
1578                 .regset_get = pac_generic_keys_get,
1579                 .set = pac_generic_keys_set,
1580         },
1581 #endif
1582 #endif
1583 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1584         [REGSET_TAGGED_ADDR_CTRL] = {
1585                 .core_note_type = NT_ARM_TAGGED_ADDR_CTRL,
1586                 .n = 1,
1587                 .size = sizeof(long),
1588                 .align = sizeof(long),
1589                 .regset_get = tagged_addr_ctrl_get,
1590                 .set = tagged_addr_ctrl_set,
1591         },
1592 #endif
1593 };
1594
1595 static const struct user_regset_view user_aarch64_view = {
1596         .name = "aarch64", .e_machine = EM_AARCH64,
1597         .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets)
1598 };
1599
1600 #ifdef CONFIG_COMPAT
1601 enum compat_regset {
1602         REGSET_COMPAT_GPR,
1603         REGSET_COMPAT_VFP,
1604 };
1605
1606 static inline compat_ulong_t compat_get_user_reg(struct task_struct *task, int idx)
1607 {
1608         struct pt_regs *regs = task_pt_regs(task);
1609
1610         switch (idx) {
1611         case 15:
1612                 return regs->pc;
1613         case 16:
1614                 return pstate_to_compat_psr(regs->pstate);
1615         case 17:
1616                 return regs->orig_x0;
1617         default:
1618                 return regs->regs[idx];
1619         }
1620 }
1621
1622 static int compat_gpr_get(struct task_struct *target,
1623                           const struct user_regset *regset,
1624                           struct membuf to)
1625 {
1626         int i = 0;
1627
1628         while (to.left)
1629                 membuf_store(&to, compat_get_user_reg(target, i++));
1630         return 0;
1631 }
1632
1633 static int compat_gpr_set(struct task_struct *target,
1634                           const struct user_regset *regset,
1635                           unsigned int pos, unsigned int count,
1636                           const void *kbuf, const void __user *ubuf)
1637 {
1638         struct pt_regs newregs;
1639         int ret = 0;
1640         unsigned int i, start, num_regs;
1641
1642         /* Calculate the number of AArch32 registers contained in count */
1643         num_regs = count / regset->size;
1644
1645         /* Convert pos into an register number */
1646         start = pos / regset->size;
1647
1648         if (start + num_regs > regset->n)
1649                 return -EIO;
1650
1651         newregs = *task_pt_regs(target);
1652
1653         for (i = 0; i < num_regs; ++i) {
1654                 unsigned int idx = start + i;
1655                 compat_ulong_t reg;
1656
1657                 if (kbuf) {
1658                         memcpy(&reg, kbuf, sizeof(reg));
1659                         kbuf += sizeof(reg);
1660                 } else {
1661                         ret = copy_from_user(&reg, ubuf, sizeof(reg));
1662                         if (ret) {
1663                                 ret = -EFAULT;
1664                                 break;
1665                         }
1666
1667                         ubuf += sizeof(reg);
1668                 }
1669
1670                 switch (idx) {
1671                 case 15:
1672                         newregs.pc = reg;
1673                         break;
1674                 case 16:
1675                         reg = compat_psr_to_pstate(reg);
1676                         newregs.pstate = reg;
1677                         break;
1678                 case 17:
1679                         newregs.orig_x0 = reg;
1680                         break;
1681                 default:
1682                         newregs.regs[idx] = reg;
1683                 }
1684
1685         }
1686
1687         if (valid_user_regs(&newregs.user_regs, target))
1688                 *task_pt_regs(target) = newregs;
1689         else
1690                 ret = -EINVAL;
1691
1692         return ret;
1693 }
1694
1695 static int compat_vfp_get(struct task_struct *target,
1696                           const struct user_regset *regset,
1697                           struct membuf to)
1698 {
1699         struct user_fpsimd_state *uregs;
1700         compat_ulong_t fpscr;
1701
1702         if (!system_supports_fpsimd())
1703                 return -EINVAL;
1704
1705         uregs = &target->thread.uw.fpsimd_state;
1706
1707         if (target == current)
1708                 fpsimd_preserve_current_state();
1709
1710         /*
1711          * The VFP registers are packed into the fpsimd_state, so they all sit
1712          * nicely together for us. We just need to create the fpscr separately.
1713          */
1714         membuf_write(&to, uregs, VFP_STATE_SIZE - sizeof(compat_ulong_t));
1715         fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
1716                 (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
1717         return membuf_store(&to, fpscr);
1718 }
1719
1720 static int compat_vfp_set(struct task_struct *target,
1721                           const struct user_regset *regset,
1722                           unsigned int pos, unsigned int count,
1723                           const void *kbuf, const void __user *ubuf)
1724 {
1725         struct user_fpsimd_state *uregs;
1726         compat_ulong_t fpscr;
1727         int ret, vregs_end_pos;
1728
1729         if (!system_supports_fpsimd())
1730                 return -EINVAL;
1731
1732         uregs = &target->thread.uw.fpsimd_state;
1733
1734         vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
1735         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0,
1736                                  vregs_end_pos);
1737
1738         if (count && !ret) {
1739                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpscr,
1740                                          vregs_end_pos, VFP_STATE_SIZE);
1741                 if (!ret) {
1742                         uregs->fpsr = fpscr & VFP_FPSCR_STAT_MASK;
1743                         uregs->fpcr = fpscr & VFP_FPSCR_CTRL_MASK;
1744                 }
1745         }
1746
1747         fpsimd_flush_task_state(target);
1748         return ret;
1749 }
1750
1751 static int compat_tls_get(struct task_struct *target,
1752                           const struct user_regset *regset,
1753                           struct membuf to)
1754 {
1755         return membuf_store(&to, (compat_ulong_t)target->thread.uw.tp_value);
1756 }
1757
1758 static int compat_tls_set(struct task_struct *target,
1759                           const struct user_regset *regset, unsigned int pos,
1760                           unsigned int count, const void *kbuf,
1761                           const void __user *ubuf)
1762 {
1763         int ret;
1764         compat_ulong_t tls = target->thread.uw.tp_value;
1765
1766         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
1767         if (ret)
1768                 return ret;
1769
1770         target->thread.uw.tp_value = tls;
1771         return ret;
1772 }
1773
1774 static const struct user_regset aarch32_regsets[] = {
1775         [REGSET_COMPAT_GPR] = {
1776                 .core_note_type = NT_PRSTATUS,
1777                 .n = COMPAT_ELF_NGREG,
1778                 .size = sizeof(compat_elf_greg_t),
1779                 .align = sizeof(compat_elf_greg_t),
1780                 .regset_get = compat_gpr_get,
1781                 .set = compat_gpr_set
1782         },
1783         [REGSET_COMPAT_VFP] = {
1784                 .core_note_type = NT_ARM_VFP,
1785                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1786                 .size = sizeof(compat_ulong_t),
1787                 .align = sizeof(compat_ulong_t),
1788                 .active = fpr_active,
1789                 .regset_get = compat_vfp_get,
1790                 .set = compat_vfp_set
1791         },
1792 };
1793
1794 static const struct user_regset_view user_aarch32_view = {
1795         .name = "aarch32", .e_machine = EM_ARM,
1796         .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
1797 };
1798
1799 static const struct user_regset aarch32_ptrace_regsets[] = {
1800         [REGSET_GPR] = {
1801                 .core_note_type = NT_PRSTATUS,
1802                 .n = COMPAT_ELF_NGREG,
1803                 .size = sizeof(compat_elf_greg_t),
1804                 .align = sizeof(compat_elf_greg_t),
1805                 .regset_get = compat_gpr_get,
1806                 .set = compat_gpr_set
1807         },
1808         [REGSET_FPR] = {
1809                 .core_note_type = NT_ARM_VFP,
1810                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1811                 .size = sizeof(compat_ulong_t),
1812                 .align = sizeof(compat_ulong_t),
1813                 .regset_get = compat_vfp_get,
1814                 .set = compat_vfp_set
1815         },
1816         [REGSET_TLS] = {
1817                 .core_note_type = NT_ARM_TLS,
1818                 .n = 1,
1819                 .size = sizeof(compat_ulong_t),
1820                 .align = sizeof(compat_ulong_t),
1821                 .regset_get = compat_tls_get,
1822                 .set = compat_tls_set,
1823         },
1824 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1825         [REGSET_HW_BREAK] = {
1826                 .core_note_type = NT_ARM_HW_BREAK,
1827                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1828                 .size = sizeof(u32),
1829                 .align = sizeof(u32),
1830                 .regset_get = hw_break_get,
1831                 .set = hw_break_set,
1832         },
1833         [REGSET_HW_WATCH] = {
1834                 .core_note_type = NT_ARM_HW_WATCH,
1835                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1836                 .size = sizeof(u32),
1837                 .align = sizeof(u32),
1838                 .regset_get = hw_break_get,
1839                 .set = hw_break_set,
1840         },
1841 #endif
1842         [REGSET_SYSTEM_CALL] = {
1843                 .core_note_type = NT_ARM_SYSTEM_CALL,
1844                 .n = 1,
1845                 .size = sizeof(int),
1846                 .align = sizeof(int),
1847                 .regset_get = system_call_get,
1848                 .set = system_call_set,
1849         },
1850 };
1851
1852 static const struct user_regset_view user_aarch32_ptrace_view = {
1853         .name = "aarch32", .e_machine = EM_ARM,
1854         .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
1855 };
1856
1857 static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
1858                                    compat_ulong_t __user *ret)
1859 {
1860         compat_ulong_t tmp;
1861
1862         if (off & 3)
1863                 return -EIO;
1864
1865         if (off == COMPAT_PT_TEXT_ADDR)
1866                 tmp = tsk->mm->start_code;
1867         else if (off == COMPAT_PT_DATA_ADDR)
1868                 tmp = tsk->mm->start_data;
1869         else if (off == COMPAT_PT_TEXT_END_ADDR)
1870                 tmp = tsk->mm->end_code;
1871         else if (off < sizeof(compat_elf_gregset_t))
1872                 tmp = compat_get_user_reg(tsk, off >> 2);
1873         else if (off >= COMPAT_USER_SZ)
1874                 return -EIO;
1875         else
1876                 tmp = 0;
1877
1878         return put_user(tmp, ret);
1879 }
1880
1881 static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off,
1882                                     compat_ulong_t val)
1883 {
1884         struct pt_regs newregs = *task_pt_regs(tsk);
1885         unsigned int idx = off / 4;
1886
1887         if (off & 3 || off >= COMPAT_USER_SZ)
1888                 return -EIO;
1889
1890         if (off >= sizeof(compat_elf_gregset_t))
1891                 return 0;
1892
1893         switch (idx) {
1894         case 15:
1895                 newregs.pc = val;
1896                 break;
1897         case 16:
1898                 newregs.pstate = compat_psr_to_pstate(val);
1899                 break;
1900         case 17:
1901                 newregs.orig_x0 = val;
1902                 break;
1903         default:
1904                 newregs.regs[idx] = val;
1905         }
1906
1907         if (!valid_user_regs(&newregs.user_regs, tsk))
1908                 return -EINVAL;
1909
1910         *task_pt_regs(tsk) = newregs;
1911         return 0;
1912 }
1913
1914 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1915
1916 /*
1917  * Convert a virtual register number into an index for a thread_info
1918  * breakpoint array. Breakpoints are identified using positive numbers
1919  * whilst watchpoints are negative. The registers are laid out as pairs
1920  * of (address, control), each pair mapping to a unique hw_breakpoint struct.
1921  * Register 0 is reserved for describing resource information.
1922  */
1923 static int compat_ptrace_hbp_num_to_idx(compat_long_t num)
1924 {
1925         return (abs(num) - 1) >> 1;
1926 }
1927
1928 static int compat_ptrace_hbp_get_resource_info(u32 *kdata)
1929 {
1930         u8 num_brps, num_wrps, debug_arch, wp_len;
1931         u32 reg = 0;
1932
1933         num_brps        = hw_breakpoint_slots(TYPE_INST);
1934         num_wrps        = hw_breakpoint_slots(TYPE_DATA);
1935
1936         debug_arch      = debug_monitors_arch();
1937         wp_len          = 8;
1938         reg             |= debug_arch;
1939         reg             <<= 8;
1940         reg             |= wp_len;
1941         reg             <<= 8;
1942         reg             |= num_wrps;
1943         reg             <<= 8;
1944         reg             |= num_brps;
1945
1946         *kdata = reg;
1947         return 0;
1948 }
1949
1950 static int compat_ptrace_hbp_get(unsigned int note_type,
1951                                  struct task_struct *tsk,
1952                                  compat_long_t num,
1953                                  u32 *kdata)
1954 {
1955         u64 addr = 0;
1956         u32 ctrl = 0;
1957
1958         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1959
1960         if (num & 1) {
1961                 err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr);
1962                 *kdata = (u32)addr;
1963         } else {
1964                 err = ptrace_hbp_get_ctrl(note_type, tsk, idx, &ctrl);
1965                 *kdata = ctrl;
1966         }
1967
1968         return err;
1969 }
1970
1971 static int compat_ptrace_hbp_set(unsigned int note_type,
1972                                  struct task_struct *tsk,
1973                                  compat_long_t num,
1974                                  u32 *kdata)
1975 {
1976         u64 addr;
1977         u32 ctrl;
1978
1979         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1980
1981         if (num & 1) {
1982                 addr = *kdata;
1983                 err = ptrace_hbp_set_addr(note_type, tsk, idx, addr);
1984         } else {
1985                 ctrl = *kdata;
1986                 err = ptrace_hbp_set_ctrl(note_type, tsk, idx, ctrl);
1987         }
1988
1989         return err;
1990 }
1991
1992 static int compat_ptrace_gethbpregs(struct task_struct *tsk, compat_long_t num,
1993                                     compat_ulong_t __user *data)
1994 {
1995         int ret;
1996         u32 kdata;
1997
1998         /* Watchpoint */
1999         if (num < 0) {
2000                 ret = compat_ptrace_hbp_get(NT_ARM_HW_WATCH, tsk, num, &kdata);
2001         /* Resource info */
2002         } else if (num == 0) {
2003                 ret = compat_ptrace_hbp_get_resource_info(&kdata);
2004         /* Breakpoint */
2005         } else {
2006                 ret = compat_ptrace_hbp_get(NT_ARM_HW_BREAK, tsk, num, &kdata);
2007         }
2008
2009         if (!ret)
2010                 ret = put_user(kdata, data);
2011
2012         return ret;
2013 }
2014
2015 static int compat_ptrace_sethbpregs(struct task_struct *tsk, compat_long_t num,
2016                                     compat_ulong_t __user *data)
2017 {
2018         int ret;
2019         u32 kdata = 0;
2020
2021         if (num == 0)
2022                 return 0;
2023
2024         ret = get_user(kdata, data);
2025         if (ret)
2026                 return ret;
2027
2028         if (num < 0)
2029                 ret = compat_ptrace_hbp_set(NT_ARM_HW_WATCH, tsk, num, &kdata);
2030         else
2031                 ret = compat_ptrace_hbp_set(NT_ARM_HW_BREAK, tsk, num, &kdata);
2032
2033         return ret;
2034 }
2035 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
2036
2037 long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
2038                         compat_ulong_t caddr, compat_ulong_t cdata)
2039 {
2040         unsigned long addr = caddr;
2041         unsigned long data = cdata;
2042         void __user *datap = compat_ptr(data);
2043         int ret;
2044
2045         switch (request) {
2046                 case PTRACE_PEEKUSR:
2047                         ret = compat_ptrace_read_user(child, addr, datap);
2048                         break;
2049
2050                 case PTRACE_POKEUSR:
2051                         ret = compat_ptrace_write_user(child, addr, data);
2052                         break;
2053
2054                 case COMPAT_PTRACE_GETREGS:
2055                         ret = copy_regset_to_user(child,
2056                                                   &user_aarch32_view,
2057                                                   REGSET_COMPAT_GPR,
2058                                                   0, sizeof(compat_elf_gregset_t),
2059                                                   datap);
2060                         break;
2061
2062                 case COMPAT_PTRACE_SETREGS:
2063                         ret = copy_regset_from_user(child,
2064                                                     &user_aarch32_view,
2065                                                     REGSET_COMPAT_GPR,
2066                                                     0, sizeof(compat_elf_gregset_t),
2067                                                     datap);
2068                         break;
2069
2070                 case COMPAT_PTRACE_GET_THREAD_AREA:
2071                         ret = put_user((compat_ulong_t)child->thread.uw.tp_value,
2072                                        (compat_ulong_t __user *)datap);
2073                         break;
2074
2075                 case COMPAT_PTRACE_SET_SYSCALL:
2076                         task_pt_regs(child)->syscallno = data;
2077                         ret = 0;
2078                         break;
2079
2080                 case COMPAT_PTRACE_GETVFPREGS:
2081                         ret = copy_regset_to_user(child,
2082                                                   &user_aarch32_view,
2083                                                   REGSET_COMPAT_VFP,
2084                                                   0, VFP_STATE_SIZE,
2085                                                   datap);
2086                         break;
2087
2088                 case COMPAT_PTRACE_SETVFPREGS:
2089                         ret = copy_regset_from_user(child,
2090                                                     &user_aarch32_view,
2091                                                     REGSET_COMPAT_VFP,
2092                                                     0, VFP_STATE_SIZE,
2093                                                     datap);
2094                         break;
2095
2096 #ifdef CONFIG_HAVE_HW_BREAKPOINT
2097                 case COMPAT_PTRACE_GETHBPREGS:
2098                         ret = compat_ptrace_gethbpregs(child, addr, datap);
2099                         break;
2100
2101                 case COMPAT_PTRACE_SETHBPREGS:
2102                         ret = compat_ptrace_sethbpregs(child, addr, datap);
2103                         break;
2104 #endif
2105
2106                 default:
2107                         ret = compat_ptrace_request(child, request, addr,
2108                                                     data);
2109                         break;
2110         }
2111
2112         return ret;
2113 }
2114 #endif /* CONFIG_COMPAT */
2115
2116 const struct user_regset_view *task_user_regset_view(struct task_struct *task)
2117 {
2118 #ifdef CONFIG_COMPAT
2119         /*
2120          * Core dumping of 32-bit tasks or compat ptrace requests must use the
2121          * user_aarch32_view compatible with arm32. Native ptrace requests on
2122          * 32-bit children use an extended user_aarch32_ptrace_view to allow
2123          * access to the TLS register.
2124          */
2125         if (is_compat_task())
2126                 return &user_aarch32_view;
2127         else if (is_compat_thread(task_thread_info(task)))
2128                 return &user_aarch32_ptrace_view;
2129 #endif
2130         return &user_aarch64_view;
2131 }
2132
2133 long arch_ptrace(struct task_struct *child, long request,
2134                  unsigned long addr, unsigned long data)
2135 {
2136         switch (request) {
2137         case PTRACE_PEEKMTETAGS:
2138         case PTRACE_POKEMTETAGS:
2139                 return mte_ptrace_copy_tags(child, request, addr, data);
2140         }
2141
2142         return ptrace_request(child, request, addr, data);
2143 }
2144
2145 enum ptrace_syscall_dir {
2146         PTRACE_SYSCALL_ENTER = 0,
2147         PTRACE_SYSCALL_EXIT,
2148 };
2149
2150 static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir)
2151 {
2152         int regno;
2153         unsigned long saved_reg;
2154
2155         /*
2156          * We have some ABI weirdness here in the way that we handle syscall
2157          * exit stops because we indicate whether or not the stop has been
2158          * signalled from syscall entry or syscall exit by clobbering a general
2159          * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
2160          * and restoring its old value after the stop. This means that:
2161          *
2162          * - Any writes by the tracer to this register during the stop are
2163          *   ignored/discarded.
2164          *
2165          * - The actual value of the register is not available during the stop,
2166          *   so the tracer cannot save it and restore it later.
2167          *
2168          * - Syscall stops behave differently to seccomp and pseudo-step traps
2169          *   (the latter do not nobble any registers).
2170          */
2171         regno = (is_compat_task() ? 12 : 7);
2172         saved_reg = regs->regs[regno];
2173         regs->regs[regno] = dir;
2174
2175         if (dir == PTRACE_SYSCALL_ENTER) {
2176                 if (ptrace_report_syscall_entry(regs))
2177                         forget_syscall(regs);
2178                 regs->regs[regno] = saved_reg;
2179         } else if (!test_thread_flag(TIF_SINGLESTEP)) {
2180                 ptrace_report_syscall_exit(regs, 0);
2181                 regs->regs[regno] = saved_reg;
2182         } else {
2183                 regs->regs[regno] = saved_reg;
2184
2185                 /*
2186                  * Signal a pseudo-step exception since we are stepping but
2187                  * tracer modifications to the registers may have rewound the
2188                  * state machine.
2189                  */
2190                 ptrace_report_syscall_exit(regs, 1);
2191         }
2192 }
2193
2194 int syscall_trace_enter(struct pt_regs *regs)
2195 {
2196         unsigned long flags = read_thread_flags();
2197
2198         if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
2199                 report_syscall(regs, PTRACE_SYSCALL_ENTER);
2200                 if (flags & _TIF_SYSCALL_EMU)
2201                         return NO_SYSCALL;
2202         }
2203
2204         /* Do the secure computing after ptrace; failures should be fast. */
2205         if (secure_computing() == -1)
2206                 return NO_SYSCALL;
2207
2208         if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
2209                 trace_sys_enter(regs, regs->syscallno);
2210
2211         audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
2212                             regs->regs[2], regs->regs[3]);
2213
2214         return regs->syscallno;
2215 }
2216
2217 void syscall_trace_exit(struct pt_regs *regs)
2218 {
2219         unsigned long flags = read_thread_flags();
2220
2221         audit_syscall_exit(regs);
2222
2223         if (flags & _TIF_SYSCALL_TRACEPOINT)
2224                 trace_sys_exit(regs, syscall_get_return_value(current, regs));
2225
2226         if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
2227                 report_syscall(regs, PTRACE_SYSCALL_EXIT);
2228
2229         rseq_syscall(regs);
2230 }
2231
2232 /*
2233  * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
2234  * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
2235  * not described in ARM DDI 0487D.a.
2236  * We treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may
2237  * be allocated an EL0 meaning in future.
2238  * Userspace cannot use these until they have an architectural meaning.
2239  * Note that this follows the SPSR_ELx format, not the AArch32 PSR format.
2240  * We also reserve IL for the kernel; SS is handled dynamically.
2241  */
2242 #define SPSR_EL1_AARCH64_RES0_BITS \
2243         (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \
2244          GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5))
2245 #define SPSR_EL1_AARCH32_RES0_BITS \
2246         (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))
2247
2248 static int valid_compat_regs(struct user_pt_regs *regs)
2249 {
2250         regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
2251
2252         if (!system_supports_mixed_endian_el0()) {
2253                 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
2254                         regs->pstate |= PSR_AA32_E_BIT;
2255                 else
2256                         regs->pstate &= ~PSR_AA32_E_BIT;
2257         }
2258
2259         if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
2260             (regs->pstate & PSR_AA32_A_BIT) == 0 &&
2261             (regs->pstate & PSR_AA32_I_BIT) == 0 &&
2262             (regs->pstate & PSR_AA32_F_BIT) == 0) {
2263                 return 1;
2264         }
2265
2266         /*
2267          * Force PSR to a valid 32-bit EL0t, preserving the same bits as
2268          * arch/arm.
2269          */
2270         regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT |
2271                         PSR_AA32_C_BIT | PSR_AA32_V_BIT |
2272                         PSR_AA32_Q_BIT | PSR_AA32_IT_MASK |
2273                         PSR_AA32_GE_MASK | PSR_AA32_E_BIT |
2274                         PSR_AA32_T_BIT;
2275         regs->pstate |= PSR_MODE32_BIT;
2276
2277         return 0;
2278 }
2279
2280 static int valid_native_regs(struct user_pt_regs *regs)
2281 {
2282         regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS;
2283
2284         if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) &&
2285             (regs->pstate & PSR_D_BIT) == 0 &&
2286             (regs->pstate & PSR_A_BIT) == 0 &&
2287             (regs->pstate & PSR_I_BIT) == 0 &&
2288             (regs->pstate & PSR_F_BIT) == 0) {
2289                 return 1;
2290         }
2291
2292         /* Force PSR to a valid 64-bit EL0t */
2293         regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
2294
2295         return 0;
2296 }
2297
2298 /*
2299  * Are the current registers suitable for user mode? (used to maintain
2300  * security in signal handlers)
2301  */
2302 int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task)
2303 {
2304         /* https://lore.kernel.org/lkml/20191118131525.GA4180@willie-the-truck */
2305         user_regs_reset_single_step(regs, task);
2306
2307         if (is_compat_thread(task_thread_info(task)))
2308                 return valid_compat_regs(regs);
2309         else
2310                 return valid_native_regs(regs);
2311 }