Commit | Line | Data |
---|---|---|
823d0f4f NP |
1 | /* |
2 | * include/asm-arm/mutex.h | |
3 | * | |
4 | * ARM optimized mutex locking primitives | |
5 | * | |
6 | * Please look into asm-generic/mutex-xchg.h for a formal definition. | |
7 | */ | |
8 | #ifndef _ASM_MUTEX_H | |
9 | #define _ASM_MUTEX_H | |
10 | ||
11 | #if __LINUX_ARM_ARCH__ < 6 | |
12 | /* On pre-ARMv6 hardware the swp based implementation is the most efficient. */ | |
13 | # include <asm-generic/mutex-xchg.h> | |
14 | #else | |
15 | ||
16 | /* | |
17 | * Attempting to lock a mutex on ARMv6+ can be done with a bastardized | |
18 | * atomic decrement (it is not a reliable atomic decrement but it satisfies | |
19 | * the defined semantics for our purpose, while being smaller and faster | |
20 | * than a real atomic decrement or atomic swap. The idea is to attempt | |
21 | * decrementing the lock value only once. If once decremented it isn't zero, | |
22 | * or if its store-back fails due to a dispute on the exclusive store, we | |
23 | * simply bail out immediately through the slow path where the lock will be | |
24 | * reattempted until it succeeds. | |
25 | */ | |
365bf8ac NP |
26 | static inline void |
27 | __mutex_fastpath_lock(atomic_t *count, fastcall void (*fail_fn)(atomic_t *)) | |
28 | { | |
29 | int __ex_flag, __res; | |
30 | ||
31 | __asm__ ( | |
32 | ||
33 | "ldrex %0, [%2] \n\t" | |
34 | "sub %0, %0, #1 \n\t" | |
35 | "strex %1, %0, [%2] " | |
36 | ||
37 | : "=&r" (__res), "=&r" (__ex_flag) | |
38 | : "r" (&(count)->counter) | |
39 | : "cc","memory" ); | |
40 | ||
41 | __res |= __ex_flag; | |
42 | if (unlikely(__res != 0)) | |
43 | fail_fn(count); | |
44 | } | |
45 | ||
46 | static inline int | |
47 | __mutex_fastpath_lock_retval(atomic_t *count, fastcall int (*fail_fn)(atomic_t *)) | |
48 | { | |
49 | int __ex_flag, __res; | |
50 | ||
51 | __asm__ ( | |
52 | ||
53 | "ldrex %0, [%2] \n\t" | |
54 | "sub %0, %0, #1 \n\t" | |
55 | "strex %1, %0, [%2] " | |
56 | ||
57 | : "=&r" (__res), "=&r" (__ex_flag) | |
58 | : "r" (&(count)->counter) | |
59 | : "cc","memory" ); | |
60 | ||
61 | __res |= __ex_flag; | |
62 | if (unlikely(__res != 0)) | |
63 | __res = fail_fn(count); | |
64 | return __res; | |
65 | } | |
823d0f4f NP |
66 | |
67 | /* | |
68 | * Same trick is used for the unlock fast path. However the original value, | |
69 | * rather than the result, is used to test for success in order to have | |
70 | * better generated assembly. | |
71 | */ | |
365bf8ac NP |
72 | static inline void |
73 | __mutex_fastpath_unlock(atomic_t *count, fastcall void (*fail_fn)(atomic_t *)) | |
74 | { | |
75 | int __ex_flag, __res, __orig; | |
76 | ||
77 | __asm__ ( | |
78 | ||
79 | "ldrex %0, [%3] \n\t" | |
80 | "add %1, %0, #1 \n\t" | |
81 | "strex %2, %1, [%3] " | |
82 | ||
83 | : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag) | |
84 | : "r" (&(count)->counter) | |
85 | : "cc","memory" ); | |
86 | ||
87 | __orig |= __ex_flag; | |
88 | if (unlikely(__orig != 0)) | |
89 | fail_fn(count); | |
90 | } | |
823d0f4f NP |
91 | |
92 | /* | |
93 | * If the unlock was done on a contended lock, or if the unlock simply fails | |
94 | * then the mutex remains locked. | |
95 | */ | |
96 | #define __mutex_slowpath_needs_to_unlock() 1 | |
97 | ||
98 | /* | |
99 | * For __mutex_fastpath_trylock we use another construct which could be | |
100 | * described as a "single value cmpxchg". | |
101 | * | |
102 | * This provides the needed trylock semantics like cmpxchg would, but it is | |
103 | * lighter and less generic than a true cmpxchg implementation. | |
104 | */ | |
105 | static inline int | |
106 | __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) | |
107 | { | |
108 | int __ex_flag, __res, __orig; | |
109 | ||
110 | __asm__ ( | |
111 | ||
365bf8ac NP |
112 | "1: ldrex %0, [%3] \n\t" |
113 | "subs %1, %0, #1 \n\t" | |
114 | "strexeq %2, %1, [%3] \n\t" | |
115 | "movlt %0, #0 \n\t" | |
116 | "cmpeq %2, #0 \n\t" | |
117 | "bgt 1b " | |
823d0f4f NP |
118 | |
119 | : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag) | |
120 | : "r" (&count->counter) | |
121 | : "cc", "memory" ); | |
122 | ||
123 | return __orig; | |
124 | } | |
125 | ||
126 | #endif | |
127 | #endif |