arch/powerpc/kernel/exceptions-64s.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * This file contains the 64-bit "server" PowerPC variant
   4  * of the low level exception handling including exception
   5  * vectors, exception return, part of the slb and stab
   6  * handling and other fixed offset specific things.
   7  *
   8  * This file is meant to be #included from head_64.S due to
   9  * position dependent assembly.
  10  *
  11  * Most of this originates from head_64.S and thus has the same
  12  * copyright history.
  13  *
  14  */
  15
  16 #include <asm/hw_irq.h>
  17 #include <asm/exception-64s.h>
  18 #include <asm/ptrace.h>
  19 #include <asm/cpuidle.h>
  20 #include <asm/head-64.h>
  21 #include <asm/feature-fixups.h>
  22 #include <asm/kup.h>
  23
  24 /*
  25  * We're short on space and time in the exception prolog, so we can't
  26  * use the normal LOAD_REG_IMMEDIATE macro to load the address of label.
  27  * Instead we get the base of the kernel from paca->kernelbase and or in the low
  28  * part of label. This requires that the label be within 64KB of kernelbase, and
  29  * that kernelbase be 64K aligned.
  30  */
  31 #define LOAD_HANDLER(reg, label)                                        \
  32         ld      reg,PACAKBASE(r13);     /* get high part of &label */   \
  33         ori     reg,reg,FIXED_SYMBOL_ABS_ADDR(label)
  34
  35 #define __LOAD_HANDLER(reg, label)                                      \
  36         ld      reg,PACAKBASE(r13);                                     \
  37         ori     reg,reg,(ABS_ADDR(label))@l
  38
  39 /*
  40  * Branches from unrelocated code (e.g., interrupts) to labels outside
  41  * head-y require >64K offsets.
  42  */
  43 #define __LOAD_FAR_HANDLER(reg, label)                                  \
  44         ld      reg,PACAKBASE(r13);                                     \
  45         ori     reg,reg,(ABS_ADDR(label))@l;                            \
  46         addis   reg,reg,(ABS_ADDR(label))@h
  47
  48 /* Exception register prefixes */
  49 #define EXC_HV          1
  50 #define EXC_STD         0
  51
  52 #if defined(CONFIG_RELOCATABLE)
  53 /*
  54  * If we support interrupts with relocation on AND we're a relocatable kernel,
  55  * we need to use CTR to get to the 2nd level handler.  So, save/restore it
  56  * when required.
  57  */
  58 #define SAVE_CTR(reg, area)     mfctr   reg ;   std     reg,area+EX_CTR(r13)
  59 #define GET_CTR(reg, area)                      ld      reg,area+EX_CTR(r13)
  60 #define RESTORE_CTR(reg, area)  ld      reg,area+EX_CTR(r13) ; mtctr reg
  61 #else
  62 /* ...else CTR is unused and in register. */
  63 #define SAVE_CTR(reg, area)
  64 #define GET_CTR(reg, area)      mfctr   reg
  65 #define RESTORE_CTR(reg, area)
  66 #endif
  67
  68 /*
  69  * PPR save/restore macros used in exceptions-64s.S
  70  * Used for P7 or later processors
  71  */
  72 #define SAVE_PPR(area, ra)                                              \
  73 BEGIN_FTR_SECTION_NESTED(940)                                           \
  74         ld      ra,area+EX_PPR(r13);    /* Read PPR from paca */        \
  75         std     ra,_PPR(r1);                                            \
  76 END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940)
  77
  78 #define RESTORE_PPR_PACA(area, ra)                                      \
  79 BEGIN_FTR_SECTION_NESTED(941)                                           \
  80         ld      ra,area+EX_PPR(r13);                                    \
  81         mtspr   SPRN_PPR,ra;                                            \
  82 END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,941)
  83
  84 /*
  85  * Get an SPR into a register if the CPU has the given feature
  86  */
  87 #define OPT_GET_SPR(ra, spr, ftr)                                       \
  88 BEGIN_FTR_SECTION_NESTED(943)                                           \
  89         mfspr   ra,spr;                                                 \
  90 END_FTR_SECTION_NESTED(ftr,ftr,943)
  91
  92 /*
  93  * Set an SPR from a register if the CPU has the given feature
  94  */
  95 #define OPT_SET_SPR(ra, spr, ftr)                                       \
  96 BEGIN_FTR_SECTION_NESTED(943)                                           \
  97         mtspr   spr,ra;                                                 \
  98 END_FTR_SECTION_NESTED(ftr,ftr,943)
  99
 100 /*
 101  * Save a register to the PACA if the CPU has the given feature
 102  */
 103 #define OPT_SAVE_REG_TO_PACA(offset, ra, ftr)                           \
 104 BEGIN_FTR_SECTION_NESTED(943)                                           \
 105         std     ra,offset(r13);                                         \
 106 END_FTR_SECTION_NESTED(ftr,ftr,943)
 107
 108 .macro EXCEPTION_PROLOG_0 area
 109         GET_PACA(r13)
 110         std     r9,\area\()+EX_R9(r13)          /* save r9 */
 111         OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR)
 112         HMT_MEDIUM
 113         std     r10,\area\()+EX_R10(r13)        /* save r10 - r12 */
 114         OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
 115 .endm
 116
 117 .macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask
 118         OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR)
 119         OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR)
 120         INTERRUPT_TO_KERNEL
 121         SAVE_CTR(r10, \area\())
 122         mfcr    r9
 123         .if \kvm
 124                 KVMTEST \hsrr \vec
 125         .endif
 126         .if \bitmask
 127                 lbz     r10,PACAIRQSOFTMASK(r13)
 128                 andi.   r10,r10,\bitmask
 129                 /* Associate vector numbers with bits in paca->irq_happened */
 130                 .if \vec == 0x500 || \vec == 0xea0
 131                 li      r10,PACA_IRQ_EE
 132                 .elseif \vec == 0x900
 133                 li      r10,PACA_IRQ_DEC
 134                 .elseif \vec == 0xa00 || \vec == 0xe80
 135                 li      r10,PACA_IRQ_DBELL
 136                 .elseif \vec == 0xe60
 137                 li      r10,PACA_IRQ_HMI
 138                 .elseif \vec == 0xf00
 139                 li      r10,PACA_IRQ_PMI
 140                 .else
 141                 .abort "Bad maskable vector"
 142                 .endif
 143
 144                 .if \hsrr
 145                 bne     masked_Hinterrupt
 146                 .else
 147                 bne     masked_interrupt
 148                 .endif
 149         .endif
 150
 151         std     r11,\area\()+EX_R11(r13)
 152         std     r12,\area\()+EX_R12(r13)
 153         GET_SCRATCH0(r10)
 154         std     r10,\area\()+EX_R13(r13)
 155 .endm
 156
 157 .macro EXCEPTION_PROLOG_2_REAL label, hsrr, set_ri
 158         ld      r10,PACAKMSR(r13)       /* get MSR value for kernel */
 159         .if ! \set_ri
 160         xori    r10,r10,MSR_RI          /* Clear MSR_RI */
 161         .endif
 162         .if \hsrr
 163         mfspr   r11,SPRN_HSRR0          /* save HSRR0 */
 164         .else
 165         mfspr   r11,SPRN_SRR0           /* save SRR0 */
 166         .endif
 167         LOAD_HANDLER(r12, \label\())
 168         .if \hsrr
 169         mtspr   SPRN_HSRR0,r12
 170         mfspr   r12,SPRN_HSRR1          /* and HSRR1 */
 171         mtspr   SPRN_HSRR1,r10
 172         HRFI_TO_KERNEL
 173         .else
 174         mtspr   SPRN_SRR0,r12
 175         mfspr   r12,SPRN_SRR1           /* and SRR1 */
 176         mtspr   SPRN_SRR1,r10
 177         RFI_TO_KERNEL
 178         .endif
 179         b       .       /* prevent speculative execution */
 180 .endm
 181
 182 .macro EXCEPTION_PROLOG_2_VIRT label, hsrr
 183 #ifdef CONFIG_RELOCATABLE
 184         .if \hsrr
 185         mfspr   r11,SPRN_HSRR0  /* save HSRR0 */
 186         .else
 187         mfspr   r11,SPRN_SRR0   /* save SRR0 */
 188         .endif
 189         LOAD_HANDLER(r12, \label\())
 190         mtctr   r12
 191         .if \hsrr
 192         mfspr   r12,SPRN_HSRR1  /* and HSRR1 */
 193         .else
 194         mfspr   r12,SPRN_SRR1   /* and HSRR1 */
 195         .endif
 196         li      r10,MSR_RI
 197         mtmsrd  r10,1           /* Set RI (EE=0) */
 198         bctr
 199 #else
 200         .if \hsrr
 201         mfspr   r11,SPRN_HSRR0          /* save HSRR0 */
 202         mfspr   r12,SPRN_HSRR1          /* and HSRR1 */
 203         .else
 204         mfspr   r11,SPRN_SRR0           /* save SRR0 */
 205         mfspr   r12,SPRN_SRR1           /* and SRR1 */
 206         .endif
 207         li      r10,MSR_RI
 208         mtmsrd  r10,1                   /* Set RI (EE=0) */
 209         b       \label
 210 #endif
 211 .endm
 212
 213 /*
 214  * Branch to label using its 0xC000 address. This results in instruction
 215  * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
 216  * on using mtmsr rather than rfid.
 217  *
 218  * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
 219  * load KBASE for a slight optimisation.
 220  */
 221 #define BRANCH_TO_C000(reg, label)                                      \
 222         __LOAD_HANDLER(reg, label);                                     \
 223         mtctr   reg;                                                    \
 224         bctr
 225
 226 #ifdef CONFIG_RELOCATABLE
 227 #define BRANCH_TO_COMMON(reg, label)                                    \
 228         __LOAD_HANDLER(reg, label);                                     \
 229         mtctr   reg;                                                    \
 230         bctr
 231
 232 #define BRANCH_LINK_TO_FAR(label)                                       \
 233         __LOAD_FAR_HANDLER(r12, label);                                 \
 234         mtctr   r12;                                                    \
 235         bctrl
 236
 237 #else
 238 #define BRANCH_TO_COMMON(reg, label)                                    \
 239         b       label
 240
 241 #define BRANCH_LINK_TO_FAR(label)                                       \
 242         bl      label
 243 #endif
 244
 245 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 246
 247 #ifdef CONFIG_RELOCATABLE
 248 /*
 249  * KVM requires __LOAD_FAR_HANDLER.
 250  *
 251  * __BRANCH_TO_KVM_EXIT branches are also a special case because they
 252  * explicitly use r9 then reload it from PACA before branching. Hence
 253  * the double-underscore.
 254  */
 255 #define __BRANCH_TO_KVM_EXIT(area, label)                               \
 256         mfctr   r9;                                                     \
 257         std     r9,HSTATE_SCRATCH1(r13);                                \
 258         __LOAD_FAR_HANDLER(r9, label);                                  \
 259         mtctr   r9;                                                     \
 260         ld      r9,area+EX_R9(r13);                                     \
 261         bctr
 262
 263 #else
 264 #define __BRANCH_TO_KVM_EXIT(area, label)                               \
 265         ld      r9,area+EX_R9(r13);                                     \
 266         b       label
 267 #endif
 268
 269 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 270 /*
 271  * If hv is possible, interrupts come into to the hv version
 272  * of the kvmppc_interrupt code, which then jumps to the PR handler,
 273  * kvmppc_interrupt_pr, if the guest is a PR guest.
 274  */
 275 #define kvmppc_interrupt kvmppc_interrupt_hv
 276 #else
 277 #define kvmppc_interrupt kvmppc_interrupt_pr
 278 #endif
 279
 280 .macro KVMTEST hsrr, n
 281         lbz     r10,HSTATE_IN_GUEST(r13)
 282         cmpwi   r10,0
 283         .if \hsrr
 284         bne     do_kvm_H\n
 285         .else
 286         bne     do_kvm_\n
 287         .endif
 288 .endm
 289
 290 .macro KVM_HANDLER area, hsrr, n, skip
 291         .if \skip
 292         cmpwi   r10,KVM_GUEST_MODE_SKIP
 293         beq     89f
 294         .else
 295         BEGIN_FTR_SECTION_NESTED(947)
 296         ld      r10,\area+EX_CFAR(r13)
 297         std     r10,HSTATE_CFAR(r13)
 298         END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)
 299         .endif
 300
 301         BEGIN_FTR_SECTION_NESTED(948)
 302         ld      r10,\area+EX_PPR(r13)
 303         std     r10,HSTATE_PPR(r13)
 304         END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
 305         ld      r10,\area+EX_R10(r13)
 306         std     r12,HSTATE_SCRATCH0(r13)
 307         sldi    r12,r9,32
 308         /* HSRR variants have the 0x2 bit added to their trap number */
 309         .if \hsrr
 310         ori     r12,r12,(\n + 0x2)
 311         .else
 312         ori     r12,r12,(\n)
 313         .endif
 314         /* This reloads r9 before branching to kvmppc_interrupt */
 315         __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt)
 316
 317         .if \skip
 318 89:     mtocrf  0x80,r9
 319         ld      r9,\area+EX_R9(r13)
 320         ld      r10,\area+EX_R10(r13)
 321         .if \hsrr
 322         b       kvmppc_skip_Hinterrupt
 323         .else
 324         b       kvmppc_skip_interrupt
 325         .endif
 326         .endif
 327 .endm
 328
 329 #else
 330 .macro KVMTEST hsrr, n
 331 .endm
 332 .macro KVM_HANDLER area, hsrr, n, skip
 333 .endm
 334 #endif
 335
 336 #define EXCEPTION_PROLOG_COMMON_1()                                        \
 337         std     r9,_CCR(r1);            /* save CR in stackframe        */ \
 338         std     r11,_NIP(r1);           /* save SRR0 in stackframe      */ \
 339         std     r12,_MSR(r1);           /* save SRR1 in stackframe      */ \
 340         std     r10,0(r1);              /* make stack chain pointer     */ \
 341         std     r0,GPR0(r1);            /* save r0 in stackframe        */ \
 342         std     r10,GPR1(r1);           /* save r1 in stackframe        */ \
 343
 344
 345 /*
 346  * The common exception prolog is used for all except a few exceptions
 347  * such as a segment miss on a kernel address.  We have to be prepared
 348  * to take another exception from the point where we first touch the
 349  * kernel stack onwards.
 350  *
 351  * On entry r13 points to the paca, r9-r13 are saved in the paca,
 352  * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
 353  * SRR1, and relocation is on.
 354  */
 355 #define EXCEPTION_PROLOG_COMMON(n, area)                                   \
 356         andi.   r10,r12,MSR_PR;         /* See if coming from user      */ \
 357         mr      r10,r1;                 /* Save r1                      */ \
 358         subi    r1,r1,INT_FRAME_SIZE;   /* alloc frame on kernel stack  */ \
 359         beq-    1f;                                                        \
 360         ld      r1,PACAKSAVE(r13);      /* kernel stack to use          */ \
 361 1:      cmpdi   cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace  */ \
 362         blt+    cr1,3f;                 /* abort if it is               */ \
 363         li      r1,(n);                 /* will be reloaded later       */ \
 364         sth     r1,PACA_TRAP_SAVE(r13);                                    \
 365         std     r3,area+EX_R3(r13);                                        \
 366         addi    r3,r13,area;            /* r3 -> where regs are saved*/    \
 367         RESTORE_CTR(r1, area);                                             \
 368         b       bad_stack;                                                 \
 369 3:      EXCEPTION_PROLOG_COMMON_1();                                       \
 370         kuap_save_amr_and_lock r9, r10, cr1, cr0;                          \
 371         beq     4f;                     /* if from kernel mode          */ \
 372         ACCOUNT_CPU_USER_ENTRY(r13, r9, r10);                              \
 373         SAVE_PPR(area, r9);                                                \
 374 4:      EXCEPTION_PROLOG_COMMON_2(area)                                    \
 375         EXCEPTION_PROLOG_COMMON_3(n)                                       \
 376         ACCOUNT_STOLEN_TIME
 377
 378 /* Save original regs values from save area to stack frame. */
 379 #define EXCEPTION_PROLOG_COMMON_2(area)                                    \
 380         ld      r9,area+EX_R9(r13);     /* move r9, r10 to stackframe   */ \
 381         ld      r10,area+EX_R10(r13);                                      \
 382         std     r9,GPR9(r1);                                               \
 383         std     r10,GPR10(r1);                                             \
 384         ld      r9,area+EX_R11(r13);    /* move r11 - r13 to stackframe */ \
 385         ld      r10,area+EX_R12(r13);                                      \
 386         ld      r11,area+EX_R13(r13);                                      \
 387         std     r9,GPR11(r1);                                              \
 388         std     r10,GPR12(r1);                                             \
 389         std     r11,GPR13(r1);                                             \
 390         BEGIN_FTR_SECTION_NESTED(66);                                      \
 391         ld      r10,area+EX_CFAR(r13);                                     \
 392         std     r10,ORIG_GPR3(r1);                                         \
 393         END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66);            \
 394         GET_CTR(r10, area);                                                \
 395         std     r10,_CTR(r1);
 396
 397 #define EXCEPTION_PROLOG_COMMON_3(n)                                       \
 398         std     r2,GPR2(r1);            /* save r2 in stackframe        */ \
 399         SAVE_4GPRS(3, r1);              /* save r3 - r6 in stackframe   */ \
 400         SAVE_2GPRS(7, r1);              /* save r7, r8 in stackframe    */ \
 401         mflr    r9;                     /* Get LR, later save to stack  */ \
 402         ld      r2,PACATOC(r13);        /* get kernel TOC into r2       */ \
 403         std     r9,_LINK(r1);                                              \
 404         lbz     r10,PACAIRQSOFTMASK(r13);                                  \
 405         mfspr   r11,SPRN_XER;           /* save XER in stackframe       */ \
 406         std     r10,SOFTE(r1);                                             \
 407         std     r11,_XER(r1);                                              \
 408         li      r9,(n)+1;                                                  \
 409         std     r9,_TRAP(r1);           /* set trap number              */ \
 410         li      r10,0;                                                     \
 411         ld      r11,exception_marker@toc(r2);                              \
 412         std     r10,RESULT(r1);         /* clear regs->result           */ \
 413         std     r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame      */
 414
 415 #define RUNLATCH_ON                             \
 416 BEGIN_FTR_SECTION                               \
 417         ld      r3, PACA_THREAD_INFO(r13);      \
 418         ld      r4,TI_LOCAL_FLAGS(r3);          \
 419         andi.   r0,r4,_TLF_RUNLATCH;            \
 420         beql    ppc64_runlatch_on_trampoline;   \
 421 END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
 422
 423 #define EXCEPTION_COMMON(area, trap)                            \
 424         EXCEPTION_PROLOG_COMMON(trap, area);                    \
 425
 426 /*
 427  * Exception where stack is already set in r1, r1 is saved in r10
 428  */
 429 #define EXCEPTION_COMMON_STACK(area, trap)                      \
 430         EXCEPTION_PROLOG_COMMON_1();                            \
 431         kuap_save_amr_and_lock r9, r10, cr1;                    \
 432         EXCEPTION_PROLOG_COMMON_2(area);                        \
 433         EXCEPTION_PROLOG_COMMON_3(trap)
 434
 435 /*
 436  * When the idle code in power4_idle puts the CPU into NAP mode,
 437  * it has to do so in a loop, and relies on the external interrupt
 438  * and decrementer interrupt entry code to get it out of the loop.
 439  * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
 440  * to signal that it is in the loop and needs help to get out.
 441  */
 442 #ifdef CONFIG_PPC_970_NAP
 443 #define FINISH_NAP                              \
 444 BEGIN_FTR_SECTION                               \
 445         ld      r11, PACA_THREAD_INFO(r13);     \
 446         ld      r9,TI_LOCAL_FLAGS(r11);         \
 447         andi.   r10,r9,_TLF_NAPPING;            \
 448         bnel    power4_fixup_nap;               \
 449 END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
 450 #else
 451 #define FINISH_NAP
 452 #endif
 453
 454
 455 /*
 456  * There are a few constraints to be concerned with.
 457  * - Real mode exceptions code/data must be located at their physical location.
 458  * - Virtual mode exceptions must be mapped at their 0xc000... location.
 459  * - Fixed location code must not call directly beyond the __end_interrupts
 460  *   area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence
 461  *   must be used.
 462  * - LOAD_HANDLER targets must be within first 64K of physical 0 /
 463  *   virtual 0xc00...
 464  * - Conditional branch targets must be within +/-32K of caller.
 465  *
 466  * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and
 467  * therefore don't have to run in physically located code or rfid to
 468  * virtual mode kernel code. However on relocatable kernels they do have
 469  * to branch to KERNELBASE offset because the rest of the kernel (outside
 470  * the exception vectors) may be located elsewhere.
 471  *
 472  * Virtual exceptions correspond with physical, except their entry points
 473  * are offset by 0xc000000000000000 and also tend to get an added 0x4000
 474  * offset applied. Virtual exceptions are enabled with the Alternate
 475  * Interrupt Location (AIL) bit set in the LPCR. However this does not
 476  * guarantee they will be delivered virtually. Some conditions (see the ISA)
 477  * cause exceptions to be delivered in real mode.
 478  *
 479  * It's impossible to receive interrupts below 0x300 via AIL.
 480  *
 481  * KVM: None of the virtual exceptions are from the guest. Anything that
 482  * escalated to HV=1 from HV=0 is delivered via real mode handlers.
 483  *
 484  *
 485  * We layout physical memory as follows:
 486  * 0x0000 - 0x00ff : Secondary processor spin code
 487  * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
 488  * 0x1900 - 0x3fff : Real mode trampolines
 489  * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
 490  * 0x5900 - 0x6fff : Relon mode trampolines
 491  * 0x7000 - 0x7fff : FWNMI data area
 492  * 0x8000 -   .... : Common interrupt handlers, remaining early
 493  *                   setup code, rest of kernel.
 494  *
 495  * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
 496  * is necessary. Until then it's more consistent to explicitly put VIRT_NONE
 497  * vectors there.
 498  */
 499 OPEN_FIXED_SECTION(real_vectors,        0x0100, 0x1900)
 500 OPEN_FIXED_SECTION(real_trampolines,    0x1900, 0x4000)
 501 OPEN_FIXED_SECTION(virt_vectors,        0x4000, 0x5900)
 502 OPEN_FIXED_SECTION(virt_trampolines,    0x5900, 0x7000)
 503
 504 #ifdef CONFIG_PPC_POWERNV
 505         .globl start_real_trampolines
 506         .globl end_real_trampolines
 507         .globl start_virt_trampolines
 508         .globl end_virt_trampolines
 509 #endif
 510
 511 #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
 512 /*
 513  * Data area reserved for FWNMI option.
 514  * This address (0x7000) is fixed by the RPA.
 515  * pseries and powernv need to keep the whole page from
 516  * 0x7000 to 0x8000 free for use by the firmware
 517  */
 518 ZERO_FIXED_SECTION(fwnmi_page,          0x7000, 0x8000)
 519 OPEN_TEXT_SECTION(0x8000)
 520 #else
 521 OPEN_TEXT_SECTION(0x7000)
 522 #endif
 523
 524 USE_FIXED_SECTION(real_vectors)
 525
 526 /*
 527  * This is the start of the interrupt handlers for pSeries
 528  * This code runs with relocation off.
 529  * Code from here to __end_interrupts gets copied down to real
 530  * address 0x100 when we are running a relocatable kernel.
 531  * Therefore any relative branches in this section must only
 532  * branch to labels in this section.
 533  */
 534         .globl __start_interrupts
 535 __start_interrupts:
 536
 537 /* No virt vectors corresponding with 0x0..0x100 */
 538 EXC_VIRT_NONE(0x4000, 0x100)
 539
 540
 541 EXC_REAL_BEGIN(system_reset, 0x100, 0x100)
 542         SET_SCRATCH0(r13)
 543         EXCEPTION_PROLOG_0 PACA_EXNMI
 544
 545         /* This is EXCEPTION_PROLOG_1 with the idle feature section added */
 546         OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_PPR, r9, CPU_FTR_HAS_PPR)
 547         OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_CFAR, r10, CPU_FTR_CFAR)
 548         INTERRUPT_TO_KERNEL
 549         SAVE_CTR(r10, PACA_EXNMI)
 550         mfcr    r9
 551
 552 #ifdef CONFIG_PPC_P7_NAP
 553         /*
 554          * If running native on arch 2.06 or later, check if we are waking up
 555          * from nap/sleep/winkle, and branch to idle handler. This tests SRR1
 556          * bits 46:47. A non-0 value indicates that we are coming from a power
 557          * saving state. The idle wakeup handler initially runs in real mode,
 558          * but we branch to the 0xc000... address so we can turn on relocation
 559          * with mtmsr.
 560          */
 561         BEGIN_FTR_SECTION
 562         mfspr   r10,SPRN_SRR1
 563         rlwinm. r10,r10,47-31,30,31
 564         beq-    1f
 565         cmpwi   cr1,r10,2
 566         mfspr   r3,SPRN_SRR1
 567         bltlr   cr1     /* no state loss, return to idle caller */
 568         BRANCH_TO_C000(r10, system_reset_idle_common)
 569 1:
 570         END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 571 #endif
 572
 573         KVMTEST EXC_STD 0x100
 574         std     r11,PACA_EXNMI+EX_R11(r13)
 575         std     r12,PACA_EXNMI+EX_R12(r13)
 576         GET_SCRATCH0(r10)
 577         std     r10,PACA_EXNMI+EX_R13(r13)
 578
 579         EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0
 580         /*
 581          * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is
 582          * being used, so a nested NMI exception would corrupt it.
 583          */
 584
 585 EXC_REAL_END(system_reset, 0x100, 0x100)
 586 EXC_VIRT_NONE(0x4100, 0x100)
 587 TRAMP_KVM(PACA_EXNMI, 0x100)
 588
 589 #ifdef CONFIG_PPC_P7_NAP
 590 EXC_COMMON_BEGIN(system_reset_idle_common)
 591         /*
 592          * This must be a direct branch (without linker branch stub) because
 593          * we can not use TOC at this point as r2 may not be restored yet.
 594          */
 595         b       idle_return_gpr_loss
 596 #endif
 597
 598 EXC_COMMON_BEGIN(system_reset_common)
 599         /*
 600          * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able
 601          * to recover, but nested NMI will notice in_nmi and not recover
 602          * because of the use of the NMI stack. in_nmi reentrancy is tested in
 603          * system_reset_exception.
 604          */
 605         lhz     r10,PACA_IN_NMI(r13)
 606         addi    r10,r10,1
 607         sth     r10,PACA_IN_NMI(r13)
 608         li      r10,MSR_RI
 609         mtmsrd  r10,1
 610
 611         mr      r10,r1
 612         ld      r1,PACA_NMI_EMERG_SP(r13)
 613         subi    r1,r1,INT_FRAME_SIZE
 614         EXCEPTION_COMMON_STACK(PACA_EXNMI, 0x100)
 615         bl      save_nvgprs
 616         /*
 617          * Set IRQS_ALL_DISABLED unconditionally so arch_irqs_disabled does
 618          * the right thing. We do not want to reconcile because that goes
 619          * through irq tracing which we don't want in NMI.
 620          *
 621          * Save PACAIRQHAPPENED because some code will do a hard disable
 622          * (e.g., xmon). So we want to restore this back to where it was
 623          * when we return. DAR is unused in the stack, so save it there.
 624          */
 625         li      r10,IRQS_ALL_DISABLED
 626         stb     r10,PACAIRQSOFTMASK(r13)
 627         lbz     r10,PACAIRQHAPPENED(r13)
 628         std     r10,_DAR(r1)
 629
 630         addi    r3,r1,STACK_FRAME_OVERHEAD
 631         bl      system_reset_exception
 632
 633         /* This (and MCE) can be simplified with mtmsrd L=1 */
 634         /* Clear MSR_RI before setting SRR0 and SRR1. */
 635         li      r0,MSR_RI
 636         mfmsr   r9
 637         andc    r9,r9,r0
 638         mtmsrd  r9,1
 639
 640         /*
 641          * MSR_RI is clear, now we can decrement paca->in_nmi.
 642          */
 643         lhz     r10,PACA_IN_NMI(r13)
 644         subi    r10,r10,1
 645         sth     r10,PACA_IN_NMI(r13)
 646
 647         /*
 648          * Restore soft mask settings.
 649          */
 650         ld      r10,_DAR(r1)
 651         stb     r10,PACAIRQHAPPENED(r13)
 652         ld      r10,SOFTE(r1)
 653         stb     r10,PACAIRQSOFTMASK(r13)
 654
 655         /*
 656          * Keep below code in synch with MACHINE_CHECK_HANDLER_WINDUP.
 657          * Should share common bits...
 658          */
 659
 660         /* Move original SRR0 and SRR1 into the respective regs */
 661         ld      r9,_MSR(r1)
 662         mtspr   SPRN_SRR1,r9
 663         ld      r3,_NIP(r1)
 664         mtspr   SPRN_SRR0,r3
 665         ld      r9,_CTR(r1)
 666         mtctr   r9
 667         ld      r9,_XER(r1)
 668         mtxer   r9
 669         ld      r9,_LINK(r1)
 670         mtlr    r9
 671         REST_GPR(0, r1)
 672         REST_8GPRS(2, r1)
 673         REST_GPR(10, r1)
 674         ld      r11,_CCR(r1)
 675         mtcr    r11
 676         REST_GPR(11, r1)
 677         REST_2GPRS(12, r1)
 678         /* restore original r1. */
 679         ld      r1,GPR1(r1)
 680         RFI_TO_USER_OR_KERNEL
 681
 682 #ifdef CONFIG_PPC_PSERIES
 683 /*
 684  * Vectors for the FWNMI option.  Share common code.
 685  */
 686 TRAMP_REAL_BEGIN(system_reset_fwnmi)
 687         SET_SCRATCH0(r13)               /* save r13 */
 688         /* See comment at system_reset exception, don't turn on RI */
 689         EXCEPTION_PROLOG_0 PACA_EXNMI
 690         EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0
 691         EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0
 692
 693 #endif /* CONFIG_PPC_PSERIES */
 694
 695
 696 EXC_REAL_BEGIN(machine_check, 0x200, 0x100)
 697         /* This is moved out of line as it can be patched by FW, but
 698          * some code path might still want to branch into the original
 699          * vector
 700          */
 701         SET_SCRATCH0(r13)               /* save r13 */
 702         EXCEPTION_PROLOG_0 PACA_EXMC
 703 BEGIN_FTR_SECTION
 704         b       machine_check_common_early
 705 FTR_SECTION_ELSE
 706         b       machine_check_pSeries_0
 707 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
 708 EXC_REAL_END(machine_check, 0x200, 0x100)
 709 EXC_VIRT_NONE(0x4200, 0x100)
 710 TRAMP_REAL_BEGIN(machine_check_common_early)
 711         EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200, 0
 712         /*
 713          * Register contents:
 714          * R13          = PACA
 715          * R9           = CR
 716          * Original R9 to R13 is saved on PACA_EXMC
 717          *
 718          * Switch to mc_emergency stack and handle re-entrancy (we limit
 719          * the nested MCE upto level 4 to avoid stack overflow).
 720          * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
 721          *
 722          * We use paca->in_mce to check whether this is the first entry or
 723          * nested machine check. We increment paca->in_mce to track nested
 724          * machine checks.
 725          *
 726          * If this is the first entry then set stack pointer to
 727          * paca->mc_emergency_sp, otherwise r1 is already pointing to
 728          * stack frame on mc_emergency stack.
 729          *
 730          * NOTE: We are here with MSR_ME=0 (off), which means we risk a
 731          * checkstop if we get another machine check exception before we do
 732          * rfid with MSR_ME=1.
 733          *
 734          * This interrupt can wake directly from idle. If that is the case,
 735          * the machine check is handled then the idle wakeup code is called
 736          * to restore state.
 737          */
 738         mr      r11,r1                  /* Save r1 */
 739         lhz     r10,PACA_IN_MCE(r13)
 740         cmpwi   r10,0                   /* Are we in nested machine check */
 741         bne     0f                      /* Yes, we are. */
 742         /* First machine check entry */
 743         ld      r1,PACAMCEMERGSP(r13)   /* Use MC emergency stack */
 744 0:      subi    r1,r1,INT_FRAME_SIZE    /* alloc stack frame */
 745         addi    r10,r10,1               /* increment paca->in_mce */
 746         sth     r10,PACA_IN_MCE(r13)
 747         /* Limit nested MCE to level 4 to avoid stack overflow */
 748         cmpwi   r10,MAX_MCE_DEPTH
 749         bgt     2f                      /* Check if we hit limit of 4 */
 750         std     r11,GPR1(r1)            /* Save r1 on the stack. */
 751         std     r11,0(r1)               /* make stack chain pointer */
 752         mfspr   r11,SPRN_SRR0           /* Save SRR0 */
 753         std     r11,_NIP(r1)
 754         mfspr   r11,SPRN_SRR1           /* Save SRR1 */
 755         std     r11,_MSR(r1)
 756         mfspr   r11,SPRN_DAR            /* Save DAR */
 757         std     r11,_DAR(r1)
 758         mfspr   r11,SPRN_DSISR          /* Save DSISR */
 759         std     r11,_DSISR(r1)
 760         std     r9,_CCR(r1)             /* Save CR in stackframe */
 761         /* We don't touch AMR here, we never go to virtual mode */
 762         /* Save r9 through r13 from EXMC save area to stack frame. */
 763         EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
 764         mfmsr   r11                     /* get MSR value */
 765 BEGIN_FTR_SECTION
 766         ori     r11,r11,MSR_ME          /* turn on ME bit */
 767 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
 768         ori     r11,r11,MSR_RI          /* turn on RI bit */
 769         LOAD_HANDLER(r12, machine_check_handle_early)
 770 1:      mtspr   SPRN_SRR0,r12
 771         mtspr   SPRN_SRR1,r11
 772         RFI_TO_KERNEL
 773         b       .       /* prevent speculative execution */
 774 2:
 775         /* Stack overflow. Stay on emergency stack and panic.
 776          * Keep the ME bit off while panic-ing, so that if we hit
 777          * another machine check we checkstop.
 778          */
 779         addi    r1,r1,INT_FRAME_SIZE    /* go back to previous stack frame */
 780         ld      r11,PACAKMSR(r13)
 781         LOAD_HANDLER(r12, unrecover_mce)
 782         li      r10,MSR_ME
 783         andc    r11,r11,r10             /* Turn off MSR_ME */
 784         b       1b
 785         b       .       /* prevent speculative execution */
 786
 787 TRAMP_REAL_BEGIN(machine_check_pSeries)
 788         .globl machine_check_fwnmi
 789 machine_check_fwnmi:
 790         SET_SCRATCH0(r13)               /* save r13 */
 791         EXCEPTION_PROLOG_0 PACA_EXMC
 792 BEGIN_FTR_SECTION
 793         b       machine_check_common_early
 794 END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
 795 machine_check_pSeries_0:
 796         EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 0
 797         /*
 798          * MSR_RI is not enabled, because PACA_EXMC is being used, so a
 799          * nested machine check corrupts it. machine_check_common enables
 800          * MSR_RI.
 801          */
 802         EXCEPTION_PROLOG_2_REAL machine_check_common, EXC_STD, 0
 803
 804 TRAMP_KVM_SKIP(PACA_EXMC, 0x200)
 805
 806 EXC_COMMON_BEGIN(machine_check_common)
 807         /*
 808          * Machine check is different because we use a different
 809          * save area: PACA_EXMC instead of PACA_EXGEN.
 810          */
 811         mfspr   r10,SPRN_DAR
 812         std     r10,PACA_EXMC+EX_DAR(r13)
 813         mfspr   r10,SPRN_DSISR
 814         stw     r10,PACA_EXMC+EX_DSISR(r13)
 815         EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
 816         FINISH_NAP
 817         RECONCILE_IRQ_STATE(r10, r11)
 818         ld      r3,PACA_EXMC+EX_DAR(r13)
 819         lwz     r4,PACA_EXMC+EX_DSISR(r13)
 820         /* Enable MSR_RI when finished with PACA_EXMC */
 821         li      r10,MSR_RI
 822         mtmsrd  r10,1
 823         std     r3,_DAR(r1)
 824         std     r4,_DSISR(r1)
 825         bl      save_nvgprs
 826         addi    r3,r1,STACK_FRAME_OVERHEAD
 827         bl      machine_check_exception
 828         b       ret_from_except
 829
 830 #define MACHINE_CHECK_HANDLER_WINDUP                    \
 831         /* Clear MSR_RI before setting SRR0 and SRR1. */\
 832         li      r0,MSR_RI;                              \
 833         mfmsr   r9;             /* get MSR value */     \
 834         andc    r9,r9,r0;                               \
 835         mtmsrd  r9,1;           /* Clear MSR_RI */      \
 836         /* Move original SRR0 and SRR1 into the respective regs */      \
 837         ld      r9,_MSR(r1);                            \
 838         mtspr   SPRN_SRR1,r9;                           \
 839         ld      r3,_NIP(r1);                            \
 840         mtspr   SPRN_SRR0,r3;                           \
 841         ld      r9,_CTR(r1);                            \
 842         mtctr   r9;                                     \
 843         ld      r9,_XER(r1);                            \
 844         mtxer   r9;                                     \
 845         ld      r9,_LINK(r1);                           \
 846         mtlr    r9;                                     \
 847         REST_GPR(0, r1);                                \
 848         REST_8GPRS(2, r1);                              \
 849         REST_GPR(10, r1);                               \
 850         ld      r11,_CCR(r1);                           \
 851         mtcr    r11;                                    \
 852         /* Decrement paca->in_mce. */                   \
 853         lhz     r12,PACA_IN_MCE(r13);                   \
 854         subi    r12,r12,1;                              \
 855         sth     r12,PACA_IN_MCE(r13);                   \
 856         REST_GPR(11, r1);                               \
 857         REST_2GPRS(12, r1);                             \
 858         /* restore original r1. */                      \
 859         ld      r1,GPR1(r1)
 860
 861 #ifdef CONFIG_PPC_P7_NAP
 862 /*
 863  * This is an idle wakeup. Low level machine check has already been
 864  * done. Queue the event then call the idle code to do the wake up.
 865  */
 866 EXC_COMMON_BEGIN(machine_check_idle_common)
 867         bl      machine_check_queue_event
 868
 869         /*
 870          * We have not used any non-volatile GPRs here, and as a rule
 871          * most exception code including machine check does not.
 872          * Therefore PACA_NAPSTATELOST does not need to be set. Idle
 873          * wakeup will restore volatile registers.
 874          *
 875          * Load the original SRR1 into r3 for pnv_powersave_wakeup_mce.
 876          *
 877          * Then decrement MCE nesting after finishing with the stack.
 878          */
 879         ld      r3,_MSR(r1)
 880         ld      r4,_LINK(r1)
 881
 882         lhz     r11,PACA_IN_MCE(r13)
 883         subi    r11,r11,1
 884         sth     r11,PACA_IN_MCE(r13)
 885
 886         mtlr    r4
 887         rlwinm  r10,r3,47-31,30,31
 888         cmpwi   cr1,r10,2
 889         bltlr   cr1     /* no state loss, return to idle caller */
 890         b       idle_return_gpr_loss
 891 #endif
 892         /*
 893          * Handle machine check early in real mode. We come here with
 894          * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
 895          */
 896 EXC_COMMON_BEGIN(machine_check_handle_early)
 897         std     r0,GPR0(r1)     /* Save r0 */
 898         EXCEPTION_PROLOG_COMMON_3(0x200)
 899         bl      save_nvgprs
 900         addi    r3,r1,STACK_FRAME_OVERHEAD
 901         bl      machine_check_early
 902         std     r3,RESULT(r1)   /* Save result */
 903         ld      r12,_MSR(r1)
 904 BEGIN_FTR_SECTION
 905         b       4f
 906 END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
 907
 908 #ifdef  CONFIG_PPC_P7_NAP
 909         /*
 910          * Check if thread was in power saving mode. We come here when any
 911          * of the following is true:
 912          * a. thread wasn't in power saving mode
 913          * b. thread was in power saving mode with no state loss,
 914          *    supervisor state loss or hypervisor state loss.
 915          *
 916          * Go back to nap/sleep/winkle mode again if (b) is true.
 917          */
 918         BEGIN_FTR_SECTION
 919         rlwinm. r11,r12,47-31,30,31
 920         bne     machine_check_idle_common
 921         END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 922 #endif
 923
 924         /*
 925          * Check if we are coming from hypervisor userspace. If yes then we
 926          * continue in host kernel in V mode to deliver the MC event.
 927          */
 928         rldicl. r11,r12,4,63            /* See if MC hit while in HV mode. */
 929         beq     5f
 930 4:      andi.   r11,r12,MSR_PR          /* See if coming from user. */
 931         bne     9f                      /* continue in V mode if we are. */
 932
 933 5:
 934 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 935 BEGIN_FTR_SECTION
 936         /*
 937          * We are coming from kernel context. Check if we are coming from
 938          * guest. if yes, then we can continue. We will fall through
 939          * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
 940          */
 941         lbz     r11,HSTATE_IN_GUEST(r13)
 942         cmpwi   r11,0                   /* Check if coming from guest */
 943         bne     9f                      /* continue if we are. */
 944 END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
 945 #endif
 946         /*
 947          * At this point we are not sure about what context we come from.
 948          * Queue up the MCE event and return from the interrupt.
 949          * But before that, check if this is an un-recoverable exception.
 950          * If yes, then stay on emergency stack and panic.
 951          */
 952         andi.   r11,r12,MSR_RI
 953         bne     2f
 954 1:      mfspr   r11,SPRN_SRR0
 955         LOAD_HANDLER(r10,unrecover_mce)
 956         mtspr   SPRN_SRR0,r10
 957         ld      r10,PACAKMSR(r13)
 958         /*
 959          * We are going down. But there are chances that we might get hit by
 960          * another MCE during panic path and we may run into unstable state
 961          * with no way out. Hence, turn ME bit off while going down, so that
 962          * when another MCE is hit during panic path, system will checkstop
 963          * and hypervisor will get restarted cleanly by SP.
 964          */
 965         li      r3,MSR_ME
 966         andc    r10,r10,r3              /* Turn off MSR_ME */
 967         mtspr   SPRN_SRR1,r10
 968         RFI_TO_KERNEL
 969         b       .
 970 2:
 971         /*
 972          * Check if we have successfully handled/recovered from error, if not
 973          * then stay on emergency stack and panic.
 974          */
 975         ld      r3,RESULT(r1)   /* Load result */
 976         cmpdi   r3,0            /* see if we handled MCE successfully */
 977
 978         beq     1b              /* if !handled then panic */
 979 BEGIN_FTR_SECTION
 980         /*
 981          * Return from MC interrupt.
 982          * Queue up the MCE event so that we can log it later, while
 983          * returning from kernel or opal call.
 984          */
 985         bl      machine_check_queue_event
 986         MACHINE_CHECK_HANDLER_WINDUP
 987         RFI_TO_USER_OR_KERNEL
 988 FTR_SECTION_ELSE
 989         /*
 990          * pSeries: Return from MC interrupt. Before that stay on emergency
 991          * stack and call machine_check_exception to log the MCE event.
 992          */
 993         LOAD_HANDLER(r10,mce_return)
 994         mtspr   SPRN_SRR0,r10
 995         ld      r10,PACAKMSR(r13)
 996         mtspr   SPRN_SRR1,r10
 997         RFI_TO_KERNEL
 998         b       .
 999 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
1000 9:
1001         /* Deliver the machine check to host kernel in V mode. */
1002         MACHINE_CHECK_HANDLER_WINDUP
1003         SET_SCRATCH0(r13)               /* save r13 */
1004         EXCEPTION_PROLOG_0 PACA_EXMC
1005         b       machine_check_pSeries_0
1006
1007 EXC_COMMON_BEGIN(unrecover_mce)
1008         /* Invoke machine_check_exception to print MCE event and panic. */
1009         addi    r3,r1,STACK_FRAME_OVERHEAD
1010         bl      machine_check_exception
1011         /*
1012          * We will not reach here. Even if we did, there is no way out. Call
1013          * unrecoverable_exception and die.
1014          */
1015 1:      addi    r3,r1,STACK_FRAME_OVERHEAD
1016         bl      unrecoverable_exception
1017         b       1b
1018
1019 EXC_COMMON_BEGIN(mce_return)
1020         /* Invoke machine_check_exception to print MCE event and return. */
1021         addi    r3,r1,STACK_FRAME_OVERHEAD
1022         bl      machine_check_exception
1023         MACHINE_CHECK_HANDLER_WINDUP
1024         RFI_TO_KERNEL
1025         b       .
1026
1027 EXC_REAL_BEGIN(data_access, 0x300, 0x80)
1028 SET_SCRATCH0(r13)               /* save r13 */
1029 EXCEPTION_PROLOG_0 PACA_EXGEN
1030         b       tramp_real_data_access
1031 EXC_REAL_END(data_access, 0x300, 0x80)
1032
1033 TRAMP_REAL_BEGIN(tramp_real_data_access)
1034 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 0
1035         /*
1036          * DAR/DSISR must be read before setting MSR[RI], because
1037          * a d-side MCE will clobber those registers so is not
1038          * recoverable if they are live.
1039          */
1040         mfspr   r10,SPRN_DAR
1041         mfspr   r11,SPRN_DSISR
1042         std     r10,PACA_EXGEN+EX_DAR(r13)
1043         stw     r11,PACA_EXGEN+EX_DSISR(r13)
1044 EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1
1045
1046 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
1047 SET_SCRATCH0(r13)               /* save r13 */
1048 EXCEPTION_PROLOG_0 PACA_EXGEN
1049 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 0
1050         mfspr   r10,SPRN_DAR
1051         mfspr   r11,SPRN_DSISR
1052         std     r10,PACA_EXGEN+EX_DAR(r13)
1053         stw     r11,PACA_EXGEN+EX_DSISR(r13)
1054 EXCEPTION_PROLOG_2_VIRT data_access_common, EXC_STD
1055 EXC_VIRT_END(data_access, 0x4300, 0x80)
1056
1057 TRAMP_KVM_SKIP(PACA_EXGEN, 0x300)
1058
1059 EXC_COMMON_BEGIN(data_access_common)
1060         /*
1061          * Here r13 points to the paca, r9 contains the saved CR,
1062          * SRR0 and SRR1 are saved in r11 and r12,
1063          * r9 - r13 are saved in paca->exgen.
1064          * EX_DAR and EX_DSISR have saved DAR/DSISR
1065          */
1066         EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
1067         RECONCILE_IRQ_STATE(r10, r11)
1068         ld      r12,_MSR(r1)
1069         ld      r3,PACA_EXGEN+EX_DAR(r13)
1070         lwz     r4,PACA_EXGEN+EX_DSISR(r13)
1071         li      r5,0x300
1072         std     r3,_DAR(r1)
1073         std     r4,_DSISR(r1)
1074 BEGIN_MMU_FTR_SECTION
1075         b       do_hash_page            /* Try to handle as hpte fault */
1076 MMU_FTR_SECTION_ELSE
1077         b       handle_page_fault
1078 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1079
1080
1081 EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
1082 SET_SCRATCH0(r13)               /* save r13 */
1083 EXCEPTION_PROLOG_0 PACA_EXSLB
1084         b       tramp_real_data_access_slb
1085 EXC_REAL_END(data_access_slb, 0x380, 0x80)
1086
1087 TRAMP_REAL_BEGIN(tramp_real_data_access_slb)
1088 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 0
1089         mfspr   r10,SPRN_DAR
1090         std     r10,PACA_EXSLB+EX_DAR(r13)
1091 EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1
1092
1093 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
1094 SET_SCRATCH0(r13)               /* save r13 */
1095 EXCEPTION_PROLOG_0 PACA_EXSLB
1096 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 0
1097         mfspr   r10,SPRN_DAR
1098         std     r10,PACA_EXSLB+EX_DAR(r13)
1099 EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD
1100 EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
1101
1102 TRAMP_KVM_SKIP(PACA_EXSLB, 0x380)
1103
1104 EXC_COMMON_BEGIN(data_access_slb_common)
1105         EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
1106         ld      r4,PACA_EXSLB+EX_DAR(r13)
1107         std     r4,_DAR(r1)
1108         addi    r3,r1,STACK_FRAME_OVERHEAD
1109 BEGIN_MMU_FTR_SECTION
1110         /* HPT case, do SLB fault */
1111         bl      do_slb_fault
1112         cmpdi   r3,0
1113         bne-    1f
1114         b       fast_exception_return
1115 1:      /* Error case */
1116 MMU_FTR_SECTION_ELSE
1117         /* Radix case, access is outside page table range */
1118         li      r3,-EFAULT
1119 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1120         std     r3,RESULT(r1)
1121         bl      save_nvgprs
1122         RECONCILE_IRQ_STATE(r10, r11)
1123         ld      r4,_DAR(r1)
1124         ld      r5,RESULT(r1)
1125         addi    r3,r1,STACK_FRAME_OVERHEAD
1126         bl      do_bad_slb_fault
1127         b       ret_from_except
1128
1129
1130 EXC_REAL(instruction_access, 0x400, 0x80)
1131 EXC_VIRT(instruction_access, 0x4400, 0x80, 0x400)
1132 TRAMP_KVM(PACA_EXGEN, 0x400)
1133
1134 EXC_COMMON_BEGIN(instruction_access_common)
1135         EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
1136         RECONCILE_IRQ_STATE(r10, r11)
1137         ld      r12,_MSR(r1)
1138         ld      r3,_NIP(r1)
1139         andis.  r4,r12,DSISR_SRR1_MATCH_64S@h
1140         li      r5,0x400
1141         std     r3,_DAR(r1)
1142         std     r4,_DSISR(r1)
1143 BEGIN_MMU_FTR_SECTION
1144         b       do_hash_page            /* Try to handle as hpte fault */
1145 MMU_FTR_SECTION_ELSE
1146         b       handle_page_fault
1147 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1148
1149
1150 __EXC_REAL(instruction_access_slb, 0x480, 0x80, PACA_EXSLB)
1151 __EXC_VIRT(instruction_access_slb, 0x4480, 0x80, 0x480, PACA_EXSLB)
1152 TRAMP_KVM(PACA_EXSLB, 0x480)
1153
1154 EXC_COMMON_BEGIN(instruction_access_slb_common)
1155         EXCEPTION_PROLOG_COMMON(0x480, PACA_EXSLB)
1156         ld      r4,_NIP(r1)
1157         addi    r3,r1,STACK_FRAME_OVERHEAD
1158 BEGIN_MMU_FTR_SECTION
1159         /* HPT case, do SLB fault */
1160         bl      do_slb_fault
1161         cmpdi   r3,0
1162         bne-    1f
1163         b       fast_exception_return
1164 1:      /* Error case */
1165 MMU_FTR_SECTION_ELSE
1166         /* Radix case, access is outside page table range */
1167         li      r3,-EFAULT
1168 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1169         std     r3,RESULT(r1)
1170         bl      save_nvgprs
1171         RECONCILE_IRQ_STATE(r10, r11)
1172         ld      r4,_NIP(r1)
1173         ld      r5,RESULT(r1)
1174         addi    r3,r1,STACK_FRAME_OVERHEAD
1175         bl      do_bad_slb_fault
1176         b       ret_from_except
1177
1178
1179 EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
1180         SET_SCRATCH0(r13)       /* save r13 */
1181         EXCEPTION_PROLOG_0 PACA_EXGEN
1182         BEGIN_FTR_SECTION
1183                 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED
1184                 EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1
1185         FTR_SECTION_ELSE
1186                 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED
1187                 EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1
1188         ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
1189 EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
1190
1191 EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
1192         SET_SCRATCH0(r13)       /* save r13 */
1193         EXCEPTION_PROLOG_0 PACA_EXGEN
1194         BEGIN_FTR_SECTION
1195                 EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED
1196                 EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV
1197         FTR_SECTION_ELSE
1198                 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED
1199                 EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD
1200         ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
1201 EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
1202
1203 TRAMP_KVM(PACA_EXGEN, 0x500)
1204 TRAMP_KVM_HV(PACA_EXGEN, 0x500)
1205 EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ)
1206
1207
1208 EXC_REAL_BEGIN(alignment, 0x600, 0x100)
1209 SET_SCRATCH0(r13)               /* save r13 */
1210 EXCEPTION_PROLOG_0 PACA_EXGEN
1211 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 0
1212         mfspr   r10,SPRN_DAR
1213         mfspr   r11,SPRN_DSISR
1214         std     r10,PACA_EXGEN+EX_DAR(r13)
1215         stw     r11,PACA_EXGEN+EX_DSISR(r13)
1216 EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1
1217 EXC_REAL_END(alignment, 0x600, 0x100)
1218
1219 EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
1220 SET_SCRATCH0(r13)               /* save r13 */
1221 EXCEPTION_PROLOG_0 PACA_EXGEN
1222 EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 0
1223         mfspr   r10,SPRN_DAR
1224         mfspr   r11,SPRN_DSISR
1225         std     r10,PACA_EXGEN+EX_DAR(r13)
1226         stw     r11,PACA_EXGEN+EX_DSISR(r13)
1227 EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD
1228 EXC_VIRT_END(alignment, 0x4600, 0x100)
1229
1230 TRAMP_KVM(PACA_EXGEN, 0x600)
1231 EXC_COMMON_BEGIN(alignment_common)
1232         EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
1233         ld      r3,PACA_EXGEN+EX_DAR(r13)
1234         lwz     r4,PACA_EXGEN+EX_DSISR(r13)
1235         std     r3,_DAR(r1)
1236         std     r4,_DSISR(r1)
1237         bl      save_nvgprs
1238         RECONCILE_IRQ_STATE(r10, r11)
1239         addi    r3,r1,STACK_FRAME_OVERHEAD
1240         bl      alignment_exception
1241         b       ret_from_except
1242
1243
1244 EXC_REAL(program_check, 0x700, 0x100)
1245 EXC_VIRT(program_check, 0x4700, 0x100, 0x700)
1246 TRAMP_KVM(PACA_EXGEN, 0x700)
1247 EXC_COMMON_BEGIN(program_check_common)
1248         /*
1249          * It's possible to receive a TM Bad Thing type program check with
1250          * userspace register values (in particular r1), but with SRR1 reporting
1251          * that we came from the kernel. Normally that would confuse the bad
1252          * stack logic, and we would report a bad kernel stack pointer. Instead
1253          * we switch to the emergency stack if we're taking a TM Bad Thing from
1254          * the kernel.
1255          */
1256         li      r10,MSR_PR              /* Build a mask of MSR_PR ..    */
1257         oris    r10,r10,0x200000@h      /* .. and SRR1_PROGTM           */
1258         and     r10,r10,r12             /* Mask SRR1 with that.         */
1259         srdi    r10,r10,8               /* Shift it so we can compare   */
1260         cmpldi  r10,(0x200000 >> 8)     /* .. with an immediate.        */
1261         bne 1f                          /* If != go to normal path.     */
1262
1263         /* SRR1 had PR=0 and SRR1_PROGTM=1, so use the emergency stack  */
1264         andi.   r10,r12,MSR_PR;         /* Set CR0 correctly for label  */
1265                                         /* 3 in EXCEPTION_PROLOG_COMMON */
1266         mr      r10,r1                  /* Save r1                      */
1267         ld      r1,PACAEMERGSP(r13)     /* Use emergency stack          */
1268         subi    r1,r1,INT_FRAME_SIZE    /* alloc stack frame            */
1269         b 3f                            /* Jump into the macro !!       */
1270 1:      EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1271         bl      save_nvgprs
1272         RECONCILE_IRQ_STATE(r10, r11)
1273         addi    r3,r1,STACK_FRAME_OVERHEAD
1274         bl      program_check_exception
1275         b       ret_from_except
1276
1277
1278 EXC_REAL(fp_unavailable, 0x800, 0x100)
1279 EXC_VIRT(fp_unavailable, 0x4800, 0x100, 0x800)
1280 TRAMP_KVM(PACA_EXGEN, 0x800)
1281 EXC_COMMON_BEGIN(fp_unavailable_common)
1282         EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
1283         bne     1f                      /* if from user, just load it up */
1284         bl      save_nvgprs
1285         RECONCILE_IRQ_STATE(r10, r11)
1286         addi    r3,r1,STACK_FRAME_OVERHEAD
1287         bl      kernel_fp_unavailable_exception
1288         BUG_OPCODE
1289 1:
1290 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1291 BEGIN_FTR_SECTION
1292         /* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1293          * transaction), go do TM stuff
1294          */
1295         rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1296         bne-    2f
1297 END_FTR_SECTION_IFSET(CPU_FTR_TM)
1298 #endif
1299         bl      load_up_fpu
1300         b       fast_exception_return
1301 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1302 2:      /* User process was in a transaction */
1303         bl      save_nvgprs
1304         RECONCILE_IRQ_STATE(r10, r11)
1305         addi    r3,r1,STACK_FRAME_OVERHEAD
1306         bl      fp_unavailable_tm
1307         b       ret_from_except
1308 #endif
1309
1310
1311 EXC_REAL_OOL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED)
1312 EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED)
1313 TRAMP_KVM(PACA_EXGEN, 0x900)
1314 EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
1315
1316
1317 EXC_REAL_HV(hdecrementer, 0x980, 0x80)
1318 EXC_VIRT_HV(hdecrementer, 0x4980, 0x80, 0x980)
1319 TRAMP_KVM_HV(PACA_EXGEN, 0x980)
1320 EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)
1321
1322
1323 EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100, IRQS_DISABLED)
1324 EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00, IRQS_DISABLED)
1325 TRAMP_KVM(PACA_EXGEN, 0xa00)
1326 #ifdef CONFIG_PPC_DOORBELL
1327 EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception)
1328 #else
1329 EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, unknown_exception)
1330 #endif
1331
1332
1333 EXC_REAL(trap_0b, 0xb00, 0x100)
1334 EXC_VIRT(trap_0b, 0x4b00, 0x100, 0xb00)
1335 TRAMP_KVM(PACA_EXGEN, 0xb00)
1336 EXC_COMMON(trap_0b_common, 0xb00, unknown_exception)
1337
1338 /*
1339  * system call / hypercall (0xc00, 0x4c00)
1340  *
1341  * The system call exception is invoked with "sc 0" and does not alter HV bit.
1342  * There is support for kernel code to invoke system calls but there are no
1343  * in-tree users.
1344  *
1345  * The hypercall is invoked with "sc 1" and sets HV=1.
1346  *
1347  * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to
1348  * 0x4c00 virtual mode.
1349  *
1350  * Call convention:
1351  *
1352  * syscall register convention is in Documentation/powerpc/syscall64-abi.txt
1353  *
1354  * For hypercalls, the register convention is as follows:
1355  * r0 volatile
1356  * r1-2 nonvolatile
1357  * r3 volatile parameter and return value for status
1358  * r4-r10 volatile input and output value
1359  * r11 volatile hypercall number and output value
1360  * r12 volatile input and output value
1361  * r13-r31 nonvolatile
1362  * LR nonvolatile
1363  * CTR volatile
1364  * XER volatile
1365  * CR0-1 CR5-7 volatile
1366  * CR2-4 nonvolatile
1367  * Other registers nonvolatile
1368  *
1369  * The intersection of volatile registers that don't contain possible
1370  * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry
1371  * without saving, though xer is not a good idea to use, as hardware may
1372  * interpret some bits so it may be costly to change them.
1373  */
1374 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1375         /*
1376          * There is a little bit of juggling to get syscall and hcall
1377          * working well. Save r13 in ctr to avoid using SPRG scratch
1378          * register.
1379          *
1380          * Userspace syscalls have already saved the PPR, hcalls must save
1381          * it before setting HMT_MEDIUM.
1382          */
1383 #define SYSCALL_KVMTEST                                                 \
1384         mtctr   r13;                                                    \
1385         GET_PACA(r13);                                                  \
1386         std     r10,PACA_EXGEN+EX_R10(r13);                             \
1387         INTERRUPT_TO_KERNEL;                                            \
1388         KVMTEST EXC_STD 0xc00 ; /* uses r10, branch to do_kvm_0xc00_system_call */ \
1389         HMT_MEDIUM;                                                     \
1390         mfctr   r9;
1391
1392 #else
1393 #define SYSCALL_KVMTEST                                                 \
1394         HMT_MEDIUM;                                                     \
1395         mr      r9,r13;                                                 \
1396         GET_PACA(r13);                                                  \
1397         INTERRUPT_TO_KERNEL;
1398 #endif
1399
1400 #define LOAD_SYSCALL_HANDLER(reg)                                       \
1401         __LOAD_HANDLER(reg, system_call_common)
1402
1403 /*
1404  * After SYSCALL_KVMTEST, we reach here with PACA in r13, r13 in r9,
1405  * and HMT_MEDIUM.
1406  */
1407 #define SYSCALL_REAL                                            \
1408         mfspr   r11,SPRN_SRR0 ;                                 \
1409         mfspr   r12,SPRN_SRR1 ;                                 \
1410         LOAD_SYSCALL_HANDLER(r10) ;                             \
1411         mtspr   SPRN_SRR0,r10 ;                                 \
1412         ld      r10,PACAKMSR(r13) ;                             \
1413         mtspr   SPRN_SRR1,r10 ;                                 \
1414         RFI_TO_KERNEL ;                                         \
1415         b       . ;     /* prevent speculative execution */
1416
1417 #ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
1418 #define SYSCALL_FASTENDIAN_TEST                                 \
1419 BEGIN_FTR_SECTION                                               \
1420         cmpdi   r0,0x1ebe ;                                     \
1421         beq-    1f ;                                            \
1422 END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)                          \
1423
1424 #define SYSCALL_FASTENDIAN                                      \
1425         /* Fast LE/BE switch system call */                     \
1426 1:      mfspr   r12,SPRN_SRR1 ;                                 \
1427         xori    r12,r12,MSR_LE ;                                \
1428         mtspr   SPRN_SRR1,r12 ;                                 \
1429         mr      r13,r9 ;                                        \
1430         RFI_TO_USER ;   /* return to userspace */               \
1431         b       . ;     /* prevent speculative execution */
1432 #else
1433 #define SYSCALL_FASTENDIAN_TEST
1434 #define SYSCALL_FASTENDIAN
1435 #endif /* CONFIG_PPC_FAST_ENDIAN_SWITCH */
1436
1437 #if defined(CONFIG_RELOCATABLE)
1438         /*
1439          * We can't branch directly so we do it via the CTR which
1440          * is volatile across system calls.
1441          */
1442 #define SYSCALL_VIRT                                            \
1443         LOAD_SYSCALL_HANDLER(r10) ;                             \
1444         mtctr   r10 ;                                           \
1445         mfspr   r11,SPRN_SRR0 ;                                 \
1446         mfspr   r12,SPRN_SRR1 ;                                 \
1447         li      r10,MSR_RI ;                                    \
1448         mtmsrd  r10,1 ;                                         \
1449         bctr ;
1450 #else
1451         /* We can branch directly */
1452 #define SYSCALL_VIRT                                            \
1453         mfspr   r11,SPRN_SRR0 ;                                 \
1454         mfspr   r12,SPRN_SRR1 ;                                 \
1455         li      r10,MSR_RI ;                                    \
1456         mtmsrd  r10,1 ;                 /* Set RI (EE=0) */     \
1457         b       system_call_common ;
1458 #endif
1459
1460 EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
1461         SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */
1462         SYSCALL_FASTENDIAN_TEST
1463         SYSCALL_REAL
1464         SYSCALL_FASTENDIAN
1465 EXC_REAL_END(system_call, 0xc00, 0x100)
1466
1467 EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
1468         SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */
1469         SYSCALL_FASTENDIAN_TEST
1470         SYSCALL_VIRT
1471         SYSCALL_FASTENDIAN
1472 EXC_VIRT_END(system_call, 0x4c00, 0x100)
1473
1474 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1475         /*
1476          * This is a hcall, so register convention is as above, with these
1477          * differences:
1478          * r13 = PACA
1479          * ctr = orig r13
1480          * orig r10 saved in PACA
1481          */
1482 TRAMP_KVM_BEGIN(do_kvm_0xc00)
1483          /*
1484           * Save the PPR (on systems that support it) before changing to
1485           * HMT_MEDIUM. That allows the KVM code to save that value into the
1486           * guest state (it is the guest's PPR value).
1487           */
1488         OPT_GET_SPR(r10, SPRN_PPR, CPU_FTR_HAS_PPR)
1489         HMT_MEDIUM
1490         OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r10, CPU_FTR_HAS_PPR)
1491         mfctr   r10
1492         SET_SCRATCH0(r10)
1493         std     r9,PACA_EXGEN+EX_R9(r13)
1494         mfcr    r9
1495         KVM_HANDLER PACA_EXGEN, EXC_STD, 0xc00, 0
1496 #endif
1497
1498
1499 EXC_REAL(single_step, 0xd00, 0x100)
1500 EXC_VIRT(single_step, 0x4d00, 0x100, 0xd00)
1501 TRAMP_KVM(PACA_EXGEN, 0xd00)
1502 EXC_COMMON(single_step_common, 0xd00, single_step_exception)
1503
1504 EXC_REAL_OOL_HV(h_data_storage, 0xe00, 0x20)
1505 EXC_VIRT_OOL_HV(h_data_storage, 0x4e00, 0x20, 0xe00)
1506 TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0xe00)
1507 EXC_COMMON_BEGIN(h_data_storage_common)
1508         mfspr   r10,SPRN_HDAR
1509         std     r10,PACA_EXGEN+EX_DAR(r13)
1510         mfspr   r10,SPRN_HDSISR
1511         stw     r10,PACA_EXGEN+EX_DSISR(r13)
1512         EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
1513         bl      save_nvgprs
1514         RECONCILE_IRQ_STATE(r10, r11)
1515         addi    r3,r1,STACK_FRAME_OVERHEAD
1516 BEGIN_MMU_FTR_SECTION
1517         ld      r4,PACA_EXGEN+EX_DAR(r13)
1518         lwz     r5,PACA_EXGEN+EX_DSISR(r13)
1519         std     r4,_DAR(r1)
1520         std     r5,_DSISR(r1)
1521         li      r5,SIGSEGV
1522         bl      bad_page_fault
1523 MMU_FTR_SECTION_ELSE
1524         bl      unknown_exception
1525 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
1526         b       ret_from_except
1527
1528
1529 EXC_REAL_OOL_HV(h_instr_storage, 0xe20, 0x20)
1530 EXC_VIRT_OOL_HV(h_instr_storage, 0x4e20, 0x20, 0xe20)
1531 TRAMP_KVM_HV(PACA_EXGEN, 0xe20)
1532 EXC_COMMON(h_instr_storage_common, 0xe20, unknown_exception)
1533
1534
1535 EXC_REAL_OOL_HV(emulation_assist, 0xe40, 0x20)
1536 EXC_VIRT_OOL_HV(emulation_assist, 0x4e40, 0x20, 0xe40)
1537 TRAMP_KVM_HV(PACA_EXGEN, 0xe40)
1538 EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt)
1539
1540
1541 /*
1542  * hmi_exception trampoline is a special case. It jumps to hmi_exception_early
1543  * first, and then eventaully from there to the trampoline to get into virtual
1544  * mode.
1545  */
1546 __EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0x20, hmi_exception_early)
1547 __TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED)
1548 EXC_VIRT_NONE(0x4e60, 0x20)
1549 TRAMP_KVM_HV(PACA_EXGEN, 0xe60)
1550 TRAMP_REAL_BEGIN(hmi_exception_early)
1551         EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0
1552         mr      r10,r1                  /* Save r1 */
1553         ld      r1,PACAEMERGSP(r13)     /* Use emergency stack for realmode */
1554         subi    r1,r1,INT_FRAME_SIZE    /* alloc stack frame            */
1555         mfspr   r11,SPRN_HSRR0          /* Save HSRR0 */
1556         mfspr   r12,SPRN_HSRR1          /* Save HSRR1 */
1557         EXCEPTION_PROLOG_COMMON_1()
1558         /* We don't touch AMR here, we never go to virtual mode */
1559         EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
1560         EXCEPTION_PROLOG_COMMON_3(0xe60)
1561         addi    r3,r1,STACK_FRAME_OVERHEAD
1562         BRANCH_LINK_TO_FAR(DOTSYM(hmi_exception_realmode)) /* Function call ABI */
1563         cmpdi   cr0,r3,0
1564
1565         /* Windup the stack. */
1566         /* Move original HSRR0 and HSRR1 into the respective regs */
1567         ld      r9,_MSR(r1)
1568         mtspr   SPRN_HSRR1,r9
1569         ld      r3,_NIP(r1)
1570         mtspr   SPRN_HSRR0,r3
1571         ld      r9,_CTR(r1)
1572         mtctr   r9
1573         ld      r9,_XER(r1)
1574         mtxer   r9
1575         ld      r9,_LINK(r1)
1576         mtlr    r9
1577         REST_GPR(0, r1)
1578         REST_8GPRS(2, r1)
1579         REST_GPR(10, r1)
1580         ld      r11,_CCR(r1)
1581         REST_2GPRS(12, r1)
1582         bne     1f
1583         mtcr    r11
1584         REST_GPR(11, r1)
1585         ld      r1,GPR1(r1)
1586         HRFI_TO_USER_OR_KERNEL
1587
1588 1:      mtcr    r11
1589         REST_GPR(11, r1)
1590         ld      r1,GPR1(r1)
1591
1592         /*
1593          * Go to virtual mode and pull the HMI event information from
1594          * firmware.
1595          */
1596         .globl hmi_exception_after_realmode
1597 hmi_exception_after_realmode:
1598         SET_SCRATCH0(r13)
1599         EXCEPTION_PROLOG_0 PACA_EXGEN
1600         b       tramp_real_hmi_exception
1601
1602 EXC_COMMON_BEGIN(hmi_exception_common)
1603         EXCEPTION_COMMON(PACA_EXGEN, 0xe60)
1604         FINISH_NAP
1605         bl      save_nvgprs
1606         RECONCILE_IRQ_STATE(r10, r11)
1607         RUNLATCH_ON
1608         addi    r3,r1,STACK_FRAME_OVERHEAD
1609         bl      handle_hmi_exception
1610         b       ret_from_except
1611
1612 EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20, IRQS_DISABLED)
1613 EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80, IRQS_DISABLED)
1614 TRAMP_KVM_HV(PACA_EXGEN, 0xe80)
1615 #ifdef CONFIG_PPC_DOORBELL
1616 EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception)
1617 #else
1618 EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception)
1619 #endif
1620
1621
1622 EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20, IRQS_DISABLED)
1623 EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0, IRQS_DISABLED)
1624 TRAMP_KVM_HV(PACA_EXGEN, 0xea0)
1625 EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ)
1626
1627
1628 EXC_REAL_NONE(0xec0, 0x20)
1629 EXC_VIRT_NONE(0x4ec0, 0x20)
1630 EXC_REAL_NONE(0xee0, 0x20)
1631 EXC_VIRT_NONE(0x4ee0, 0x20)
1632
1633
1634 EXC_REAL_OOL_MASKABLE(performance_monitor, 0xf00, 0x20, IRQS_PMI_DISABLED)
1635 EXC_VIRT_OOL_MASKABLE(performance_monitor, 0x4f00, 0x20, 0xf00, IRQS_PMI_DISABLED)
1636 TRAMP_KVM(PACA_EXGEN, 0xf00)
1637 EXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception)
1638
1639
1640 EXC_REAL_OOL(altivec_unavailable, 0xf20, 0x20)
1641 EXC_VIRT_OOL(altivec_unavailable, 0x4f20, 0x20, 0xf20)
1642 TRAMP_KVM(PACA_EXGEN, 0xf20)
1643 EXC_COMMON_BEGIN(altivec_unavailable_common)
1644         EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
1645 #ifdef CONFIG_ALTIVEC
1646 BEGIN_FTR_SECTION
1647         beq     1f
1648 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1649   BEGIN_FTR_SECTION_NESTED(69)
1650         /* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1651          * transaction), go do TM stuff
1652          */
1653         rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1654         bne-    2f
1655   END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1656 #endif
1657         bl      load_up_altivec
1658         b       fast_exception_return
1659 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1660 2:      /* User process was in a transaction */
1661         bl      save_nvgprs
1662         RECONCILE_IRQ_STATE(r10, r11)
1663         addi    r3,r1,STACK_FRAME_OVERHEAD
1664         bl      altivec_unavailable_tm
1665         b       ret_from_except
1666 #endif
1667 1:
1668 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1669 #endif
1670         bl      save_nvgprs
1671         RECONCILE_IRQ_STATE(r10, r11)
1672         addi    r3,r1,STACK_FRAME_OVERHEAD
1673         bl      altivec_unavailable_exception
1674         b       ret_from_except
1675
1676
1677 EXC_REAL_OOL(vsx_unavailable, 0xf40, 0x20)
1678 EXC_VIRT_OOL(vsx_unavailable, 0x4f40, 0x20, 0xf40)
1679 TRAMP_KVM(PACA_EXGEN, 0xf40)
1680 EXC_COMMON_BEGIN(vsx_unavailable_common)
1681         EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
1682 #ifdef CONFIG_VSX
1683 BEGIN_FTR_SECTION
1684         beq     1f
1685 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1686   BEGIN_FTR_SECTION_NESTED(69)
1687         /* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1688          * transaction), go do TM stuff
1689          */
1690         rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
1691         bne-    2f
1692   END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
1693 #endif
1694         b       load_up_vsx
1695 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1696 2:      /* User process was in a transaction */
1697         bl      save_nvgprs
1698         RECONCILE_IRQ_STATE(r10, r11)
1699         addi    r3,r1,STACK_FRAME_OVERHEAD
1700         bl      vsx_unavailable_tm
1701         b       ret_from_except
1702 #endif
1703 1:
1704 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1705 #endif
1706         bl      save_nvgprs
1707         RECONCILE_IRQ_STATE(r10, r11)
1708         addi    r3,r1,STACK_FRAME_OVERHEAD
1709         bl      vsx_unavailable_exception
1710         b       ret_from_except
1711
1712
1713 EXC_REAL_OOL(facility_unavailable, 0xf60, 0x20)
1714 EXC_VIRT_OOL(facility_unavailable, 0x4f60, 0x20, 0xf60)
1715 TRAMP_KVM(PACA_EXGEN, 0xf60)
1716 EXC_COMMON(facility_unavailable_common, 0xf60, facility_unavailable_exception)
1717
1718
1719 EXC_REAL_OOL_HV(h_facility_unavailable, 0xf80, 0x20)
1720 EXC_VIRT_OOL_HV(h_facility_unavailable, 0x4f80, 0x20, 0xf80)
1721 TRAMP_KVM_HV(PACA_EXGEN, 0xf80)
1722 EXC_COMMON(h_facility_unavailable_common, 0xf80, facility_unavailable_exception)
1723
1724
1725 EXC_REAL_NONE(0xfa0, 0x20)
1726 EXC_VIRT_NONE(0x4fa0, 0x20)
1727 EXC_REAL_NONE(0xfc0, 0x20)
1728 EXC_VIRT_NONE(0x4fc0, 0x20)
1729 EXC_REAL_NONE(0xfe0, 0x20)
1730 EXC_VIRT_NONE(0x4fe0, 0x20)
1731
1732 EXC_REAL_NONE(0x1000, 0x100)
1733 EXC_VIRT_NONE(0x5000, 0x100)
1734 EXC_REAL_NONE(0x1100, 0x100)
1735 EXC_VIRT_NONE(0x5100, 0x100)
1736
1737 #ifdef CONFIG_CBE_RAS
1738 EXC_REAL_HV(cbe_system_error, 0x1200, 0x100)
1739 EXC_VIRT_NONE(0x5200, 0x100)
1740 TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1200)
1741 EXC_COMMON(cbe_system_error_common, 0x1200, cbe_system_error_exception)
1742 #else /* CONFIG_CBE_RAS */
1743 EXC_REAL_NONE(0x1200, 0x100)
1744 EXC_VIRT_NONE(0x5200, 0x100)
1745 #endif
1746
1747
1748 EXC_REAL(instruction_breakpoint, 0x1300, 0x100)
1749 EXC_VIRT(instruction_breakpoint, 0x5300, 0x100, 0x1300)
1750 TRAMP_KVM_SKIP(PACA_EXGEN, 0x1300)
1751 EXC_COMMON(instruction_breakpoint_common, 0x1300, instruction_breakpoint_exception)
1752
1753 EXC_REAL_NONE(0x1400, 0x100)
1754 EXC_VIRT_NONE(0x5400, 0x100)
1755
1756 EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100)
1757         mtspr   SPRN_SPRG_HSCRATCH0,r13
1758         EXCEPTION_PROLOG_0 PACA_EXGEN
1759         EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0
1760
1761 #ifdef CONFIG_PPC_DENORMALISATION
1762         mfspr   r10,SPRN_HSRR1
1763         andis.  r10,r10,(HSRR1_DENORM)@h /* denorm? */
1764         bne+    denorm_assist
1765 #endif
1766
1767         KVMTEST EXC_HV 0x1500
1768         EXCEPTION_PROLOG_2_REAL denorm_common, EXC_HV, 1
1769 EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100)
1770
1771 #ifdef CONFIG_PPC_DENORMALISATION
1772 EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100)
1773         b       exc_real_0x1500_denorm_exception_hv
1774 EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
1775 #else
1776 EXC_VIRT_NONE(0x5500, 0x100)
1777 #endif
1778
1779 TRAMP_KVM_HV(PACA_EXGEN, 0x1500)
1780
1781 #ifdef CONFIG_PPC_DENORMALISATION
1782 TRAMP_REAL_BEGIN(denorm_assist)
1783 BEGIN_FTR_SECTION
1784 /*
1785  * To denormalise we need to move a copy of the register to itself.
1786  * For POWER6 do that here for all FP regs.
1787  */
1788         mfmsr   r10
1789         ori     r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
1790         xori    r10,r10,(MSR_FE0|MSR_FE1)
1791         mtmsrd  r10
1792         sync
1793
1794 #define FMR2(n)  fmr (n), (n) ; fmr n+1, n+1
1795 #define FMR4(n)  FMR2(n) ; FMR2(n+2)
1796 #define FMR8(n)  FMR4(n) ; FMR4(n+4)
1797 #define FMR16(n) FMR8(n) ; FMR8(n+8)
1798 #define FMR32(n) FMR16(n) ; FMR16(n+16)
1799         FMR32(0)
1800
1801 FTR_SECTION_ELSE
1802 /*
1803  * To denormalise we need to move a copy of the register to itself.
1804  * For POWER7 do that here for the first 32 VSX registers only.
1805  */
1806         mfmsr   r10
1807         oris    r10,r10,MSR_VSX@h
1808         mtmsrd  r10
1809         sync
1810
1811 #define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
1812 #define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
1813 #define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
1814 #define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
1815 #define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
1816         XVCPSGNDP32(0)
1817
1818 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
1819
1820 BEGIN_FTR_SECTION
1821         b       denorm_done
1822 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1823 /*
1824  * To denormalise we need to move a copy of the register to itself.
1825  * For POWER8 we need to do that for all 64 VSX registers
1826  */
1827         XVCPSGNDP32(32)
1828 denorm_done:
1829         mfspr   r11,SPRN_HSRR0
1830         subi    r11,r11,4
1831         mtspr   SPRN_HSRR0,r11
1832         mtcrf   0x80,r9
1833         ld      r9,PACA_EXGEN+EX_R9(r13)
1834         RESTORE_PPR_PACA(PACA_EXGEN, r10)
1835 BEGIN_FTR_SECTION
1836         ld      r10,PACA_EXGEN+EX_CFAR(r13)
1837         mtspr   SPRN_CFAR,r10
1838 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1839         ld      r10,PACA_EXGEN+EX_R10(r13)
1840         ld      r11,PACA_EXGEN+EX_R11(r13)
1841         ld      r12,PACA_EXGEN+EX_R12(r13)
1842         ld      r13,PACA_EXGEN+EX_R13(r13)
1843         HRFI_TO_UNKNOWN
1844         b       .
1845 #endif
1846
1847 EXC_COMMON(denorm_common, 0x1500, unknown_exception)
1848
1849
1850 #ifdef CONFIG_CBE_RAS
1851 EXC_REAL_HV(cbe_maintenance, 0x1600, 0x100)
1852 EXC_VIRT_NONE(0x5600, 0x100)
1853 TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1600)
1854 EXC_COMMON(cbe_maintenance_common, 0x1600, cbe_maintenance_exception)
1855 #else /* CONFIG_CBE_RAS */
1856 EXC_REAL_NONE(0x1600, 0x100)
1857 EXC_VIRT_NONE(0x5600, 0x100)
1858 #endif
1859
1860
1861 EXC_REAL(altivec_assist, 0x1700, 0x100)
1862 EXC_VIRT(altivec_assist, 0x5700, 0x100, 0x1700)
1863 TRAMP_KVM(PACA_EXGEN, 0x1700)
1864 #ifdef CONFIG_ALTIVEC
1865 EXC_COMMON(altivec_assist_common, 0x1700, altivec_assist_exception)
1866 #else
1867 EXC_COMMON(altivec_assist_common, 0x1700, unknown_exception)
1868 #endif
1869
1870
1871 #ifdef CONFIG_CBE_RAS
1872 EXC_REAL_HV(cbe_thermal, 0x1800, 0x100)
1873 EXC_VIRT_NONE(0x5800, 0x100)
1874 TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1800)
1875 EXC_COMMON(cbe_thermal_common, 0x1800, cbe_thermal_exception)
1876 #else /* CONFIG_CBE_RAS */
1877 EXC_REAL_NONE(0x1800, 0x100)
1878 EXC_VIRT_NONE(0x5800, 0x100)
1879 #endif
1880
1881 #ifdef CONFIG_PPC_WATCHDOG
1882
1883 #define MASKED_DEC_HANDLER_LABEL 3f
1884
1885 #define MASKED_DEC_HANDLER(_H)                          \
1886 3: /* soft-nmi */                                       \
1887         std     r12,PACA_EXGEN+EX_R12(r13);             \
1888         GET_SCRATCH0(r10);                              \
1889         std     r10,PACA_EXGEN+EX_R13(r13);             \
1890         EXCEPTION_PROLOG_2_REAL soft_nmi_common, _H, 1
1891
1892 /*
1893  * Branch to soft_nmi_interrupt using the emergency stack. The emergency
1894  * stack is one that is usable by maskable interrupts so long as MSR_EE
1895  * remains off. It is used for recovery when something has corrupted the
1896  * normal kernel stack, for example. The "soft NMI" must not use the process
1897  * stack because we want irq disabled sections to avoid touching the stack
1898  * at all (other than PMU interrupts), so use the emergency stack for this,
1899  * and run it entirely with interrupts hard disabled.
1900  */
1901 EXC_COMMON_BEGIN(soft_nmi_common)
1902         mr      r10,r1
1903         ld      r1,PACAEMERGSP(r13)
1904         subi    r1,r1,INT_FRAME_SIZE
1905         EXCEPTION_COMMON_STACK(PACA_EXGEN, 0x900)
1906         bl      save_nvgprs
1907         RECONCILE_IRQ_STATE(r10, r11)
1908         addi    r3,r1,STACK_FRAME_OVERHEAD
1909         bl      soft_nmi_interrupt
1910         b       ret_from_except
1911
1912 #else /* CONFIG_PPC_WATCHDOG */
1913 #define MASKED_DEC_HANDLER_LABEL 2f /* normal return */
1914 #define MASKED_DEC_HANDLER(_H)
1915 #endif /* CONFIG_PPC_WATCHDOG */
1916
1917 /*
1918  * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
1919  * - If it was a decrementer interrupt, we bump the dec to max and and return.
1920  * - If it was a doorbell we return immediately since doorbells are edge
1921  *   triggered and won't automatically refire.
1922  * - If it was a HMI we return immediately since we handled it in realmode
1923  *   and it won't refire.
1924  * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
1925  * This is called with r10 containing the value to OR to the paca field.
1926  */
1927 .macro MASKED_INTERRUPT hsrr
1928         .if \hsrr
1929 masked_Hinterrupt:
1930         .else
1931 masked_interrupt:
1932         .endif
1933         std     r11,PACA_EXGEN+EX_R11(r13)
1934         lbz     r11,PACAIRQHAPPENED(r13)
1935         or      r11,r11,r10
1936         stb     r11,PACAIRQHAPPENED(r13)
1937         cmpwi   r10,PACA_IRQ_DEC
1938         bne     1f
1939         lis     r10,0x7fff
1940         ori     r10,r10,0xffff
1941         mtspr   SPRN_DEC,r10
1942         b       MASKED_DEC_HANDLER_LABEL
1943 1:      andi.   r10,r10,PACA_IRQ_MUST_HARD_MASK
1944         beq     2f
1945         .if \hsrr
1946         mfspr   r10,SPRN_HSRR1
1947         xori    r10,r10,MSR_EE  /* clear MSR_EE */
1948         mtspr   SPRN_HSRR1,r10
1949         .else
1950         mfspr   r10,SPRN_SRR1
1951         xori    r10,r10,MSR_EE  /* clear MSR_EE */
1952         mtspr   SPRN_SRR1,r10
1953         .endif
1954         ori     r11,r11,PACA_IRQ_HARD_DIS
1955         stb     r11,PACAIRQHAPPENED(r13)
1956 2:      /* done */
1957         mtcrf   0x80,r9
1958         std     r1,PACAR1(r13)
1959         ld      r9,PACA_EXGEN+EX_R9(r13)
1960         ld      r10,PACA_EXGEN+EX_R10(r13)
1961         ld      r11,PACA_EXGEN+EX_R11(r13)
1962         /* returns to kernel where r13 must be set up, so don't restore it */
1963         .if \hsrr
1964         HRFI_TO_KERNEL
1965         .else
1966         RFI_TO_KERNEL
1967         .endif
1968         b       .
1969         MASKED_DEC_HANDLER(\hsrr\())
1970 .endm
1971
1972 TRAMP_REAL_BEGIN(stf_barrier_fallback)
1973         std     r9,PACA_EXRFI+EX_R9(r13)
1974         std     r10,PACA_EXRFI+EX_R10(r13)
1975         sync
1976         ld      r9,PACA_EXRFI+EX_R9(r13)
1977         ld      r10,PACA_EXRFI+EX_R10(r13)
1978         ori     31,31,0
1979         .rept 14
1980         b       1f
1981 1:
1982         .endr
1983         blr
1984
1985 TRAMP_REAL_BEGIN(rfi_flush_fallback)
1986         SET_SCRATCH0(r13);
1987         GET_PACA(r13);
1988         std     r1,PACA_EXRFI+EX_R12(r13)
1989         ld      r1,PACAKSAVE(r13)
1990         std     r9,PACA_EXRFI+EX_R9(r13)
1991         std     r10,PACA_EXRFI+EX_R10(r13)
1992         std     r11,PACA_EXRFI+EX_R11(r13)
1993         mfctr   r9
1994         ld      r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
1995         ld      r11,PACA_L1D_FLUSH_SIZE(r13)
1996         srdi    r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
1997         mtctr   r11
1998         DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
1999
2000         /* order ld/st prior to dcbt stop all streams with flushing */
2001         sync
2002
2003         /*
2004          * The load adresses are at staggered offsets within cachelines,
2005          * which suits some pipelines better (on others it should not
2006          * hurt).
2007          */
2008 1:
2009         ld      r11,(0x80 + 8)*0(r10)
2010         ld      r11,(0x80 + 8)*1(r10)
2011         ld      r11,(0x80 + 8)*2(r10)
2012         ld      r11,(0x80 + 8)*3(r10)
2013         ld      r11,(0x80 + 8)*4(r10)
2014         ld      r11,(0x80 + 8)*5(r10)
2015         ld      r11,(0x80 + 8)*6(r10)
2016         ld      r11,(0x80 + 8)*7(r10)
2017         addi    r10,r10,0x80*8
2018         bdnz    1b
2019
2020         mtctr   r9
2021         ld      r9,PACA_EXRFI+EX_R9(r13)
2022         ld      r10,PACA_EXRFI+EX_R10(r13)
2023         ld      r11,PACA_EXRFI+EX_R11(r13)
2024         ld      r1,PACA_EXRFI+EX_R12(r13)
2025         GET_SCRATCH0(r13);
2026         rfid
2027
2028 TRAMP_REAL_BEGIN(hrfi_flush_fallback)
2029         SET_SCRATCH0(r13);
2030         GET_PACA(r13);
2031         std     r1,PACA_EXRFI+EX_R12(r13)
2032         ld      r1,PACAKSAVE(r13)
2033         std     r9,PACA_EXRFI+EX_R9(r13)
2034         std     r10,PACA_EXRFI+EX_R10(r13)
2035         std     r11,PACA_EXRFI+EX_R11(r13)
2036         mfctr   r9
2037         ld      r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
2038         ld      r11,PACA_L1D_FLUSH_SIZE(r13)
2039         srdi    r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
2040         mtctr   r11
2041         DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
2042
2043         /* order ld/st prior to dcbt stop all streams with flushing */
2044         sync
2045
2046         /*
2047          * The load adresses are at staggered offsets within cachelines,
2048          * which suits some pipelines better (on others it should not
2049          * hurt).
2050          */
2051 1:
2052         ld      r11,(0x80 + 8)*0(r10)
2053         ld      r11,(0x80 + 8)*1(r10)
2054         ld      r11,(0x80 + 8)*2(r10)
2055         ld      r11,(0x80 + 8)*3(r10)
2056         ld      r11,(0x80 + 8)*4(r10)
2057         ld      r11,(0x80 + 8)*5(r10)
2058         ld      r11,(0x80 + 8)*6(r10)
2059         ld      r11,(0x80 + 8)*7(r10)
2060         addi    r10,r10,0x80*8
2061         bdnz    1b
2062
2063         mtctr   r9
2064         ld      r9,PACA_EXRFI+EX_R9(r13)
2065         ld      r10,PACA_EXRFI+EX_R10(r13)
2066         ld      r11,PACA_EXRFI+EX_R11(r13)
2067         ld      r1,PACA_EXRFI+EX_R12(r13)
2068         GET_SCRATCH0(r13);
2069         hrfid
2070
2071 /*
2072  * Real mode exceptions actually use this too, but alternate
2073  * instruction code patches (which end up in the common .text area)
2074  * cannot reach these if they are put there.
2075  */
2076 USE_FIXED_SECTION(virt_trampolines)
2077         MASKED_INTERRUPT EXC_STD
2078         MASKED_INTERRUPT EXC_HV
2079
2080 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2081 TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
2082         /*
2083          * Here all GPRs are unchanged from when the interrupt happened
2084          * except for r13, which is saved in SPRG_SCRATCH0.
2085          */
2086         mfspr   r13, SPRN_SRR0
2087         addi    r13, r13, 4
2088         mtspr   SPRN_SRR0, r13
2089         GET_SCRATCH0(r13)
2090         RFI_TO_KERNEL
2091         b       .
2092
2093 TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
2094         /*
2095          * Here all GPRs are unchanged from when the interrupt happened
2096          * except for r13, which is saved in SPRG_SCRATCH0.
2097          */
2098         mfspr   r13, SPRN_HSRR0
2099         addi    r13, r13, 4
2100         mtspr   SPRN_HSRR0, r13
2101         GET_SCRATCH0(r13)
2102         HRFI_TO_KERNEL
2103         b       .
2104 #endif
2105
2106 /*
2107  * Ensure that any handlers that get invoked from the exception prologs
2108  * above are below the first 64KB (0x10000) of the kernel image because
2109  * the prologs assemble the addresses of these handlers using the
2110  * LOAD_HANDLER macro, which uses an ori instruction.
2111  */
2112
2113 /*** Common interrupt handlers ***/
2114
2115
2116         /*
2117          * Relocation-on interrupts: A subset of the interrupts can be delivered
2118          * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
2119          * it.  Addresses are the same as the original interrupt addresses, but
2120          * offset by 0xc000000000004000.
2121          * It's impossible to receive interrupts below 0x300 via this mechanism.
2122          * KVM: None of these traps are from the guest ; anything that escalated
2123          * to HV=1 from HV=0 is delivered via real mode handlers.
2124          */
2125
2126         /*
2127          * This uses the standard macro, since the original 0x300 vector
2128          * only has extra guff for STAB-based processors -- which never
2129          * come here.
2130          */
2131
2132 EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
2133         b       __ppc64_runlatch_on
2134
2135 USE_FIXED_SECTION(virt_trampolines)
2136         /*
2137          * The __end_interrupts marker must be past the out-of-line (OOL)
2138          * handlers, so that they are copied to real address 0x100 when running
2139          * a relocatable kernel. This ensures they can be reached from the short
2140          * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
2141          * directly, without using LOAD_HANDLER().
2142          */
2143         .align  7
2144         .globl  __end_interrupts
2145 __end_interrupts:
2146 DEFINE_FIXED_SYMBOL(__end_interrupts)
2147
2148 #ifdef CONFIG_PPC_970_NAP
2149 EXC_COMMON_BEGIN(power4_fixup_nap)
2150         andc    r9,r9,r10
2151         std     r9,TI_LOCAL_FLAGS(r11)
2152         ld      r10,_LINK(r1)           /* make idle task do the */
2153         std     r10,_NIP(r1)            /* equivalent of a blr */
2154         blr
2155 #endif
2156
2157 CLOSE_FIXED_SECTION(real_vectors);
2158 CLOSE_FIXED_SECTION(real_trampolines);
2159 CLOSE_FIXED_SECTION(virt_vectors);
2160 CLOSE_FIXED_SECTION(virt_trampolines);
2161
2162 USE_TEXT_SECTION()
2163
2164 /*
2165  * Hash table stuff
2166  */
2167         .balign IFETCH_ALIGN_BYTES
2168 do_hash_page:
2169 #ifdef CONFIG_PPC_BOOK3S_64
2170         lis     r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h
2171         ori     r0,r0,DSISR_BAD_FAULT_64S@l
2172         and.    r0,r4,r0                /* weird error? */
2173         bne-    handle_page_fault       /* if not, try to insert a HPTE */
2174         ld      r11, PACA_THREAD_INFO(r13)
2175         lwz     r0,TI_PREEMPT(r11)      /* If we're in an "NMI" */
2176         andis.  r0,r0,NMI_MASK@h        /* (i.e. an irq when soft-disabled) */
2177         bne     77f                     /* then don't call hash_page now */
2178
2179         /*
2180          * r3 contains the faulting address
2181          * r4 msr
2182          * r5 contains the trap number
2183          * r6 contains dsisr
2184          *
2185          * at return r3 = 0 for success, 1 for page fault, negative for error
2186          */
2187         mr      r4,r12
2188         ld      r6,_DSISR(r1)
2189         bl      __hash_page             /* build HPTE if possible */
2190         cmpdi   r3,0                    /* see if __hash_page succeeded */
2191
2192         /* Success */
2193         beq     fast_exc_return_irq     /* Return from exception on success */
2194
2195         /* Error */
2196         blt-    13f
2197
2198         /* Reload DSISR into r4 for the DABR check below */
2199         ld      r4,_DSISR(r1)
2200 #endif /* CONFIG_PPC_BOOK3S_64 */
2201
2202 /* Here we have a page fault that hash_page can't handle. */
2203 handle_page_fault:
2204 11:     andis.  r0,r4,DSISR_DABRMATCH@h
2205         bne-    handle_dabr_fault
2206         ld      r4,_DAR(r1)
2207         ld      r5,_DSISR(r1)
2208         addi    r3,r1,STACK_FRAME_OVERHEAD
2209         bl      do_page_fault
2210         cmpdi   r3,0
2211         beq+    ret_from_except_lite
2212         bl      save_nvgprs
2213         mr      r5,r3
2214         addi    r3,r1,STACK_FRAME_OVERHEAD
2215         lwz     r4,_DAR(r1)
2216         bl      bad_page_fault
2217         b       ret_from_except
2218
2219 /* We have a data breakpoint exception - handle it */
2220 handle_dabr_fault:
2221         bl      save_nvgprs
2222         ld      r4,_DAR(r1)
2223         ld      r5,_DSISR(r1)
2224         addi    r3,r1,STACK_FRAME_OVERHEAD
2225         bl      do_break
2226         /*
2227          * do_break() may have changed the NV GPRS while handling a breakpoint.
2228          * If so, we need to restore them with their updated values. Don't use
2229          * ret_from_except_lite here.
2230          */
2231         b       ret_from_except
2232
2233
2234 #ifdef CONFIG_PPC_BOOK3S_64
2235 /* We have a page fault that hash_page could handle but HV refused
2236  * the PTE insertion
2237  */
2238 13:     bl      save_nvgprs
2239         mr      r5,r3
2240         addi    r3,r1,STACK_FRAME_OVERHEAD
2241         ld      r4,_DAR(r1)
2242         bl      low_hash_fault
2243         b       ret_from_except
2244 #endif
2245
2246 /*
2247  * We come here as a result of a DSI at a point where we don't want
2248  * to call hash_page, such as when we are accessing memory (possibly
2249  * user memory) inside a PMU interrupt that occurred while interrupts
2250  * were soft-disabled.  We want to invoke the exception handler for
2251  * the access, or panic if there isn't a handler.
2252  */
2253 77:     bl      save_nvgprs
2254         mr      r4,r3
2255         addi    r3,r1,STACK_FRAME_OVERHEAD
2256         li      r5,SIGSEGV
2257         bl      bad_page_fault
2258         b       ret_from_except
2259
2260 /*
2261  * Here we have detected that the kernel stack pointer is bad.
2262  * R9 contains the saved CR, r13 points to the paca,
2263  * r10 contains the (bad) kernel stack pointer,
2264  * r11 and r12 contain the saved SRR0 and SRR1.
2265  * We switch to using an emergency stack, save the registers there,
2266  * and call kernel_bad_stack(), which panics.
2267  */
2268 bad_stack:
2269         ld      r1,PACAEMERGSP(r13)
2270         subi    r1,r1,64+INT_FRAME_SIZE
2271         std     r9,_CCR(r1)
2272         std     r10,GPR1(r1)
2273         std     r11,_NIP(r1)
2274         std     r12,_MSR(r1)
2275         mfspr   r11,SPRN_DAR
2276         mfspr   r12,SPRN_DSISR
2277         std     r11,_DAR(r1)
2278         std     r12,_DSISR(r1)
2279         mflr    r10
2280         mfctr   r11
2281         mfxer   r12
2282         std     r10,_LINK(r1)
2283         std     r11,_CTR(r1)
2284         std     r12,_XER(r1)
2285         SAVE_GPR(0,r1)
2286         SAVE_GPR(2,r1)
2287         ld      r10,EX_R3(r3)
2288         std     r10,GPR3(r1)
2289         SAVE_GPR(4,r1)
2290         SAVE_4GPRS(5,r1)
2291         ld      r9,EX_R9(r3)
2292         ld      r10,EX_R10(r3)
2293         SAVE_2GPRS(9,r1)
2294         ld      r9,EX_R11(r3)
2295         ld      r10,EX_R12(r3)
2296         ld      r11,EX_R13(r3)
2297         std     r9,GPR11(r1)
2298         std     r10,GPR12(r1)
2299         std     r11,GPR13(r1)
2300 BEGIN_FTR_SECTION
2301         ld      r10,EX_CFAR(r3)
2302         std     r10,ORIG_GPR3(r1)
2303 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
2304         SAVE_8GPRS(14,r1)
2305         SAVE_10GPRS(22,r1)
2306         lhz     r12,PACA_TRAP_SAVE(r13)
2307         std     r12,_TRAP(r1)
2308         addi    r11,r1,INT_FRAME_SIZE
2309         std     r11,0(r1)
2310         li      r12,0
2311         std     r12,0(r11)
2312         ld      r2,PACATOC(r13)
2313         ld      r11,exception_marker@toc(r2)
2314         std     r12,RESULT(r1)
2315         std     r11,STACK_FRAME_OVERHEAD-16(r1)
2316 1:      addi    r3,r1,STACK_FRAME_OVERHEAD
2317         bl      kernel_bad_stack
2318         b       1b
2319 _ASM_NOKPROBE_SYMBOL(bad_stack);
2320
2321 /*
2322  * When doorbell is triggered from system reset wakeup, the message is
2323  * not cleared, so it would fire again when EE is enabled.
2324  *
2325  * When coming from local_irq_enable, there may be the same problem if
2326  * we were hard disabled.
2327  *
2328  * Execute msgclr to clear pending exceptions before handling it.
2329  */
2330 h_doorbell_common_msgclr:
2331         LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36))
2332         PPC_MSGCLR(3)
2333         b       h_doorbell_common
2334
2335 doorbell_super_common_msgclr:
2336         LOAD_REG_IMMEDIATE(r3, PPC_DBELL_MSGTYPE << (63-36))
2337         PPC_MSGCLRP(3)
2338         b       doorbell_super_common
2339
2340 /*
2341  * Called from arch_local_irq_enable when an interrupt needs
2342  * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
2343  * which kind of interrupt. MSR:EE is already off. We generate a
2344  * stackframe like if a real interrupt had happened.
2345  *
2346  * Note: While MSR:EE is off, we need to make sure that _MSR
2347  * in the generated frame has EE set to 1 or the exception
2348  * handler will not properly re-enable them.
2349  *
2350  * Note that we don't specify LR as the NIP (return address) for
2351  * the interrupt because that would unbalance the return branch
2352  * predictor.
2353  */
2354 _GLOBAL(__replay_interrupt)
2355         /* We are going to jump to the exception common code which
2356          * will retrieve various register values from the PACA which
2357          * we don't give a damn about, so we don't bother storing them.
2358          */
2359         mfmsr   r12
2360         LOAD_REG_ADDR(r11, replay_interrupt_return)
2361         mfcr    r9
2362         ori     r12,r12,MSR_EE
2363         cmpwi   r3,0x900
2364         beq     decrementer_common
2365         cmpwi   r3,0x500
2366 BEGIN_FTR_SECTION
2367         beq     h_virt_irq_common
2368 FTR_SECTION_ELSE
2369         beq     hardware_interrupt_common
2370 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300)
2371         cmpwi   r3,0xf00
2372         beq     performance_monitor_common
2373 BEGIN_FTR_SECTION
2374         cmpwi   r3,0xa00
2375         beq     h_doorbell_common_msgclr
2376         cmpwi   r3,0xe60
2377         beq     hmi_exception_common
2378 FTR_SECTION_ELSE
2379         cmpwi   r3,0xa00
2380         beq     doorbell_super_common_msgclr
2381 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
2382 replay_interrupt_return:
2383         blr
2384
2385 _ASM_NOKPROBE_SYMBOL(__replay_interrupt)