1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <asm/processor.h>
3 #include <asm/ppc_asm.h>
5 #include <asm/asm-offsets.h>
6 #include <asm/cputable.h>
7 #include <asm/thread_info.h>
9 #include <asm/ptrace.h>
10 #include <asm/export.h>
11 #include <asm/asm-compat.h>
14 * Load state from memory into VMX registers including VSCR.
15 * Assumes the caller has enabled VMX in the MSR.
17 _GLOBAL(load_vr_state)
23 EXPORT_SYMBOL(load_vr_state)
24 _ASM_NOKPROBE_SYMBOL(load_vr_state); /* used by restore_math */
27 * Store VMX state into memory, including VSCR.
28 * Assumes the caller has enabled VMX in the MSR.
30 _GLOBAL(store_vr_state)
36 EXPORT_SYMBOL(store_vr_state)
39 * Disable VMX for the task which had it previously,
40 * and save its vector registers in its thread_struct.
41 * Enables the VMX for use in the kernel on return.
42 * On SMP we know the VMX is free, since we give it up every
43 * switch (ie, no lazy save of the vector registers).
45 * Note that on 32-bit this can only use registers that will be
46 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
48 _GLOBAL(load_up_altivec)
49 mfmsr r5 /* grab the current MSR */
50 #ifdef CONFIG_PPC_BOOK3S_64
51 /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
55 MTMSRD(r5) /* enable use of AltiVec now */
59 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
60 * to optimise userspace context save/restore. Whenever we take an
61 * altivec unavailable exception we must set VRSAVE to something non
62 * zero. Set it to all 1s. See also the programming note in the ISA.
70 /* enable use of VMX after return */
75 ld r4,PACACURRENT(r13)
76 addi r5,r4,THREAD /* Get THREAD */
77 oris r12,r12,MSR_VEC@h
79 #ifdef CONFIG_PPC_BOOK3S_64
81 stb r4,PACASRR_VALID(r13)
85 stb r4,THREAD_LOAD_VEC(r5)
86 addi r6,r5,THREAD_VRSTATE
88 stw r4,THREAD_USED_VR(r5)
92 /* restore registers and return */
94 _ASM_NOKPROBE_SYMBOL(load_up_altivec)
98 * Save the vector registers to its thread_struct
100 _GLOBAL(save_altivec)
101 addi r3,r3,THREAD /* want THREAD of task */
102 PPC_LL r7,THREAD_VRSAVEAREA(r3)
103 PPC_LL r5,PT_REGS(r3)
106 addi r7,r3,THREAD_VRSTATE
107 2: SAVE_32VRS(0,r4,r7)
116 #error This asm code isn't ready for 32-bit kernels
120 * load_up_vsx(unused, unused, tsk)
121 * Disable VSX for the task which had it previously,
122 * and save its vector registers in its thread_struct.
123 * Reuse the fp and vsx saves, but first check to see if they have
124 * been saved already.
127 /* Load FP and VSX registers if they haven't been done yet */
129 beql+ load_up_fpu /* skip if already loaded */
130 andis. r5,r12,MSR_VEC@h
131 beql+ load_up_altivec /* skip if already loaded */
133 #ifdef CONFIG_PPC_BOOK3S_64
134 /* interrupt doesn't set MSR[RI] and HPT can fault on current access */
139 ld r4,PACACURRENT(r13)
140 addi r4,r4,THREAD /* Get THREAD */
142 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
143 /* enable use of VSX after return */
144 oris r12,r12,MSR_VSX@h
147 stb r4,PACASRR_VALID(r13)
148 b fast_interrupt_return_srr
150 #endif /* CONFIG_VSX */
154 * The routines below are in assembler so we can closely control the
155 * usage of floating-point registers. These routines must be called
156 * with preempt disabled.
163 .long 0x3f800000 /* 1.0 in single-precision FP */
165 .long 0x3f000000 /* 0.5 in single-precision FP */
167 #define LDCONST(fr, name) \
175 .quad 0x3ff0000000000000 /* 1.0 */
177 .quad 0x3fe0000000000000 /* 0.5 */
179 #define LDCONST(fr, name) \
180 addis r11,r2,name@toc@ha; \
181 lfd fr,name@toc@l(r11)
185 * Internal routine to enable floating point and set FPSCR to 0.
186 * Don't call it from C; it doesn't use the normal calling convention.
218 * Vector add, floating point.
235 * Vector subtract, floating point.
252 * Vector multiply and add, floating point.
264 fmadds fr0,fr0,fr2,fr1
272 * Vector negative multiply and subtract, floating point.
284 fnmsubs fr0,fr0,fr2,fr1
292 * Vector reciprocal estimate. We just compute 1.0/x.
293 * r3 -> destination, r4 -> source.
310 * Vector reciprocal square-root estimate, floating point.
311 * We use the frsqrte instruction for the initial estimate followed
312 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
313 * r3 -> destination, r4 -> source.
328 frsqrte fr1,fr0 /* r = frsqrte(s) */
329 fmuls fr3,fr1,fr0 /* r * s */
330 fmuls fr2,fr1,fr5 /* r * 0.5 */
331 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
332 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
333 fmuls fr3,fr1,fr0 /* r * s */
334 fmuls fr2,fr1,fr5 /* r * 0.5 */
335 fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
336 fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */