Commit | Line | Data |
---|---|---|
0004a9df RB |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (c) 2006 Ralf Baechle (ralf@linux-mips.org) | |
7 | */ | |
4732efbe JJ |
8 | #ifndef _ASM_FUTEX_H |
9 | #define _ASM_FUTEX_H | |
10 | ||
11 | #ifdef __KERNEL__ | |
12 | ||
13 | #include <linux/futex.h> | |
730f412c | 14 | #include <linux/uaccess.h> |
a6813fe5 | 15 | #include <asm/asm-eva.h> |
0004a9df | 16 | #include <asm/barrier.h> |
b0984c43 | 17 | #include <asm/compiler.h> |
4732efbe | 18 | #include <asm/errno.h> |
3c1d3f09 | 19 | #include <asm/sync.h> |
4732efbe | 20 | |
3f2bedab AB |
21 | #define arch_futex_atomic_op_inuser arch_futex_atomic_op_inuser |
22 | #define futex_atomic_cmpxchg_inatomic futex_atomic_cmpxchg_inatomic | |
23 | #include <asm-generic/futex.h> | |
24 | ||
25 | #define __futex_atomic_op(op, insn, ret, oldval, uaddr, oparg) \ | |
ebfaebae | 26 | { \ |
256ec489 | 27 | if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { \ |
6ee1da94 RB |
28 | __asm__ __volatile__( \ |
29 | " .set push \n" \ | |
30 | " .set noat \n" \ | |
378ed6f0 | 31 | " .set push \n" \ |
a809d460 | 32 | " .set arch=r4000 \n" \ |
0307e8d0 | 33 | "1: ll %1, %4 # __futex_atomic_op \n" \ |
378ed6f0 | 34 | " .set pop \n" \ |
6ee1da94 | 35 | " " insn " \n" \ |
a809d460 | 36 | " .set arch=r4000 \n" \ |
0307e8d0 | 37 | "2: sc $1, %2 \n" \ |
6ee1da94 | 38 | " beqzl $1, 1b \n" \ |
fd7710cb | 39 | __stringify(__WEAK_LLSC_MB) " \n" \ |
6ee1da94 | 40 | "3: \n" \ |
0e525e48 | 41 | " .insn \n" \ |
6ee1da94 | 42 | " .set pop \n" \ |
6ee1da94 | 43 | " .section .fixup,\"ax\" \n" \ |
0307e8d0 | 44 | "4: li %0, %6 \n" \ |
0f67e90e | 45 | " j 3b \n" \ |
6ee1da94 RB |
46 | " .previous \n" \ |
47 | " .section __ex_table,\"a\" \n" \ | |
48 | " "__UA_ADDR "\t1b, 4b \n" \ | |
49 | " "__UA_ADDR "\t2b, 4b \n" \ | |
50 | " .previous \n" \ | |
b0984c43 | 51 | : "=r" (ret), "=&r" (oldval), \ |
94bfb75a MC |
52 | "=" GCC_OFF_SMALL_ASM() (*uaddr) \ |
53 | : "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg), \ | |
b0984c43 | 54 | "i" (-EFAULT) \ |
0307e8d0 | 55 | : "memory"); \ |
6ee1da94 RB |
56 | } else if (cpu_has_llsc) { \ |
57 | __asm__ __volatile__( \ | |
58 | " .set push \n" \ | |
59 | " .set noat \n" \ | |
378ed6f0 | 60 | " .set push \n" \ |
1922c356 | 61 | " .set "MIPS_ISA_ARCH_LEVEL" \n" \ |
3c1d3f09 | 62 | " " __SYNC(full, loongson3_war) " \n" \ |
a6813fe5 | 63 | "1: "user_ll("%1", "%4")" # __futex_atomic_op\n" \ |
378ed6f0 | 64 | " .set pop \n" \ |
6ee1da94 | 65 | " " insn " \n" \ |
1922c356 | 66 | " .set "MIPS_ISA_ARCH_LEVEL" \n" \ |
a6813fe5 | 67 | "2: "user_sc("$1", "%2")" \n" \ |
6ee1da94 | 68 | " beqz $1, 1b \n" \ |
fd7710cb | 69 | __stringify(__WEAK_LLSC_MB) " \n" \ |
6ee1da94 | 70 | "3: \n" \ |
0e525e48 | 71 | " .insn \n" \ |
6ee1da94 | 72 | " .set pop \n" \ |
6ee1da94 | 73 | " .section .fixup,\"ax\" \n" \ |
0307e8d0 | 74 | "4: li %0, %6 \n" \ |
0f67e90e | 75 | " j 3b \n" \ |
6ee1da94 RB |
76 | " .previous \n" \ |
77 | " .section __ex_table,\"a\" \n" \ | |
78 | " "__UA_ADDR "\t1b, 4b \n" \ | |
79 | " "__UA_ADDR "\t2b, 4b \n" \ | |
80 | " .previous \n" \ | |
b0984c43 | 81 | : "=r" (ret), "=&r" (oldval), \ |
94bfb75a MC |
82 | "=" GCC_OFF_SMALL_ASM() (*uaddr) \ |
83 | : "0" (0), GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oparg), \ | |
b0984c43 | 84 | "i" (-EFAULT) \ |
0307e8d0 | 85 | : "memory"); \ |
3f2bedab AB |
86 | } else { \ |
87 | /* fallback for non-SMP */ \ | |
0f09c274 AB |
88 | ret = futex_atomic_op_inuser_local(op, oparg, oval, uaddr); \ |
89 | } \ | |
90 | } | |
ebfaebae | 91 | |
4732efbe | 92 | static inline int |
30d6e0a4 | 93 | arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) |
4732efbe | 94 | { |
4732efbe | 95 | int oldval = 0, ret; |
4732efbe | 96 | |
a08971e9 AV |
97 | if (!access_ok(uaddr, sizeof(u32))) |
98 | return -EFAULT; | |
4732efbe JJ |
99 | |
100 | switch (op) { | |
101 | case FUTEX_OP_SET: | |
3f2bedab | 102 | __futex_atomic_op(op, "move $1, %z5", ret, oldval, uaddr, oparg); |
ebfaebae RB |
103 | break; |
104 | ||
4732efbe | 105 | case FUTEX_OP_ADD: |
3f2bedab | 106 | __futex_atomic_op(op, "addu $1, %1, %z5", |
70342287 | 107 | ret, oldval, uaddr, oparg); |
ebfaebae | 108 | break; |
4732efbe | 109 | case FUTEX_OP_OR: |
3f2bedab | 110 | __futex_atomic_op(op, "or $1, %1, %z5", |
70342287 | 111 | ret, oldval, uaddr, oparg); |
ebfaebae | 112 | break; |
4732efbe | 113 | case FUTEX_OP_ANDN: |
3f2bedab | 114 | __futex_atomic_op(op, "and $1, %1, %z5", |
70342287 | 115 | ret, oldval, uaddr, ~oparg); |
ebfaebae | 116 | break; |
4732efbe | 117 | case FUTEX_OP_XOR: |
3f2bedab | 118 | __futex_atomic_op(op, "xor $1, %1, %z5", |
70342287 | 119 | ret, oldval, uaddr, oparg); |
ebfaebae | 120 | break; |
4732efbe JJ |
121 | default: |
122 | ret = -ENOSYS; | |
123 | } | |
124 | ||
30d6e0a4 JS |
125 | if (!ret) |
126 | *oval = oldval; | |
127 | ||
4732efbe JJ |
128 | return ret; |
129 | } | |
130 | ||
e9056f13 | 131 | static inline int |
8d7718aa ML |
132 | futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, |
133 | u32 oldval, u32 newval) | |
e9056f13 | 134 | { |
8d7718aa ML |
135 | int ret = 0; |
136 | u32 val; | |
6ee1da94 | 137 | |
96d4f267 | 138 | if (!access_ok(uaddr, sizeof(u32))) |
6ee1da94 RB |
139 | return -EFAULT; |
140 | ||
256ec489 | 141 | if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { |
6ee1da94 RB |
142 | __asm__ __volatile__( |
143 | "# futex_atomic_cmpxchg_inatomic \n" | |
144 | " .set push \n" | |
145 | " .set noat \n" | |
378ed6f0 | 146 | " .set push \n" |
a809d460 | 147 | " .set arch=r4000 \n" |
37a9d912 ML |
148 | "1: ll %1, %3 \n" |
149 | " bne %1, %z4, 3f \n" | |
378ed6f0 | 150 | " .set pop \n" |
37a9d912 | 151 | " move $1, %z5 \n" |
a809d460 | 152 | " .set arch=r4000 \n" |
37a9d912 | 153 | "2: sc $1, %2 \n" |
6ee1da94 | 154 | " beqzl $1, 1b \n" |
fd7710cb | 155 | __stringify(__WEAK_LLSC_MB) " \n" |
6ee1da94 | 156 | "3: \n" |
0e525e48 | 157 | " .insn \n" |
6ee1da94 RB |
158 | " .set pop \n" |
159 | " .section .fixup,\"ax\" \n" | |
37a9d912 | 160 | "4: li %0, %6 \n" |
6ee1da94 RB |
161 | " j 3b \n" |
162 | " .previous \n" | |
163 | " .section __ex_table,\"a\" \n" | |
164 | " "__UA_ADDR "\t1b, 4b \n" | |
165 | " "__UA_ADDR "\t2b, 4b \n" | |
166 | " .previous \n" | |
94bfb75a MC |
167 | : "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr) |
168 | : GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval), | |
b0984c43 | 169 | "i" (-EFAULT) |
6ee1da94 RB |
170 | : "memory"); |
171 | } else if (cpu_has_llsc) { | |
172 | __asm__ __volatile__( | |
173 | "# futex_atomic_cmpxchg_inatomic \n" | |
174 | " .set push \n" | |
175 | " .set noat \n" | |
378ed6f0 | 176 | " .set push \n" |
1922c356 | 177 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
3c1d3f09 | 178 | " " __SYNC(full, loongson3_war) " \n" |
a6813fe5 | 179 | "1: "user_ll("%1", "%3")" \n" |
37a9d912 | 180 | " bne %1, %z4, 3f \n" |
378ed6f0 | 181 | " .set pop \n" |
37a9d912 | 182 | " move $1, %z5 \n" |
1922c356 | 183 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
a6813fe5 | 184 | "2: "user_sc("$1", "%2")" \n" |
6ee1da94 | 185 | " beqz $1, 1b \n" |
3c1d3f09 | 186 | "3: " __SYNC_ELSE(full, loongson3_war, __WEAK_LLSC_MB) "\n" |
0e525e48 | 187 | " .insn \n" |
6ee1da94 RB |
188 | " .set pop \n" |
189 | " .section .fixup,\"ax\" \n" | |
37a9d912 | 190 | "4: li %0, %6 \n" |
6ee1da94 RB |
191 | " j 3b \n" |
192 | " .previous \n" | |
193 | " .section __ex_table,\"a\" \n" | |
194 | " "__UA_ADDR "\t1b, 4b \n" | |
195 | " "__UA_ADDR "\t2b, 4b \n" | |
196 | " .previous \n" | |
94bfb75a MC |
197 | : "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr) |
198 | : GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval), | |
b0984c43 | 199 | "i" (-EFAULT) |
6ee1da94 | 200 | : "memory"); |
3f2bedab AB |
201 | } else { |
202 | return futex_atomic_cmpxchg_inatomic_local(uval, uaddr, oldval, newval); | |
203 | } | |
6ee1da94 | 204 | |
37a9d912 ML |
205 | *uval = val; |
206 | return ret; | |
e9056f13 IM |
207 | } |
208 | ||
4732efbe | 209 | #endif |
0f67e90e | 210 | #endif /* _ASM_FUTEX_H */ |