Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #include <linux/cache.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/uaccess.h> | |
18 | #include <linux/module.h> | |
19 | #include <linux/mm.h> | |
60063497 | 20 | #include <linux/atomic.h> |
867e359b CM |
21 | #include <arch/chip.h> |
22 | ||
867e359b | 23 | /* This page is remapped on startup to be hash-for-home. */ |
2cb82400 | 24 | int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss; |
867e359b | 25 | |
47d632f9 | 26 | int *__atomic_hashed_lock(volatile void *v) |
867e359b | 27 | { |
5fb682b0 | 28 | /* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */ |
867e359b CM |
29 | /* |
30 | * Use bits [3, 3 + ATOMIC_HASH_SHIFT) as the lock index. | |
31 | * Using mm works here because atomic_locks is page aligned. | |
32 | */ | |
33 | unsigned long ptr = __insn_mm((unsigned long)v >> 1, | |
34 | (unsigned long)atomic_locks, | |
35 | 2, (ATOMIC_HASH_SHIFT + 2) - 1); | |
36 | return (int *)ptr; | |
867e359b CM |
37 | } |
38 | ||
39 | #ifdef CONFIG_SMP | |
40 | /* Return whether the passed pointer is a valid atomic lock pointer. */ | |
41 | static int is_atomic_lock(int *p) | |
42 | { | |
867e359b | 43 | return p >= &atomic_locks[0] && p < &atomic_locks[ATOMIC_HASH_SIZE]; |
867e359b CM |
44 | } |
45 | ||
46 | void __atomic_fault_unlock(int *irqlock_word) | |
47 | { | |
48 | BUG_ON(!is_atomic_lock(irqlock_word)); | |
49 | BUG_ON(*irqlock_word != 1); | |
50 | *irqlock_word = 0; | |
51 | } | |
52 | ||
53 | #endif /* CONFIG_SMP */ | |
54 | ||
55 | static inline int *__atomic_setup(volatile void *v) | |
56 | { | |
57 | /* Issue a load to the target to bring it into cache. */ | |
58 | *(volatile int *)v; | |
59 | return __atomic_hashed_lock(v); | |
60 | } | |
61 | ||
6dc9658f | 62 | int _atomic_xchg(int *v, int n) |
867e359b | 63 | { |
b7271b9f | 64 | return __atomic32_xchg(v, __atomic_setup(v), n).val; |
867e359b CM |
65 | } |
66 | EXPORT_SYMBOL(_atomic_xchg); | |
67 | ||
6dc9658f | 68 | int _atomic_xchg_add(int *v, int i) |
867e359b | 69 | { |
b7271b9f | 70 | return __atomic32_xchg_add(v, __atomic_setup(v), i).val; |
867e359b CM |
71 | } |
72 | EXPORT_SYMBOL(_atomic_xchg_add); | |
73 | ||
6dc9658f | 74 | int _atomic_xchg_add_unless(int *v, int a, int u) |
867e359b CM |
75 | { |
76 | /* | |
77 | * Note: argument order is switched here since it is easier | |
78 | * to use the first argument consistently as the "old value" | |
79 | * in the assembly, as is done for _atomic_cmpxchg(). | |
80 | */ | |
b7271b9f | 81 | return __atomic32_xchg_add_unless(v, __atomic_setup(v), u, a).val; |
867e359b CM |
82 | } |
83 | EXPORT_SYMBOL(_atomic_xchg_add_unless); | |
84 | ||
6dc9658f | 85 | int _atomic_cmpxchg(int *v, int o, int n) |
867e359b | 86 | { |
b7271b9f | 87 | return __atomic32_cmpxchg(v, __atomic_setup(v), o, n).val; |
867e359b CM |
88 | } |
89 | EXPORT_SYMBOL(_atomic_cmpxchg); | |
90 | ||
1af5de9a | 91 | unsigned long _atomic_fetch_or(volatile unsigned long *p, unsigned long mask) |
867e359b | 92 | { |
b7271b9f | 93 | return __atomic32_fetch_or((int *)p, __atomic_setup(p), mask).val; |
867e359b | 94 | } |
1af5de9a | 95 | EXPORT_SYMBOL(_atomic_fetch_or); |
867e359b | 96 | |
1af5de9a | 97 | unsigned long _atomic_fetch_and(volatile unsigned long *p, unsigned long mask) |
2957c035 | 98 | { |
b7271b9f | 99 | return __atomic32_fetch_and((int *)p, __atomic_setup(p), mask).val; |
2957c035 | 100 | } |
1af5de9a | 101 | EXPORT_SYMBOL(_atomic_fetch_and); |
2957c035 | 102 | |
1af5de9a | 103 | unsigned long _atomic_fetch_andn(volatile unsigned long *p, unsigned long mask) |
867e359b | 104 | { |
b7271b9f | 105 | return __atomic32_fetch_andn((int *)p, __atomic_setup(p), mask).val; |
867e359b | 106 | } |
1af5de9a | 107 | EXPORT_SYMBOL(_atomic_fetch_andn); |
867e359b | 108 | |
1af5de9a | 109 | unsigned long _atomic_fetch_xor(volatile unsigned long *p, unsigned long mask) |
867e359b | 110 | { |
b7271b9f | 111 | return __atomic32_fetch_xor((int *)p, __atomic_setup(p), mask).val; |
867e359b | 112 | } |
1af5de9a | 113 | EXPORT_SYMBOL(_atomic_fetch_xor); |
867e359b CM |
114 | |
115 | ||
b924a690 | 116 | long long _atomic64_xchg(long long *v, long long n) |
867e359b | 117 | { |
6dc9658f | 118 | return __atomic64_xchg(v, __atomic_setup(v), n); |
867e359b CM |
119 | } |
120 | EXPORT_SYMBOL(_atomic64_xchg); | |
121 | ||
b924a690 | 122 | long long _atomic64_xchg_add(long long *v, long long i) |
867e359b | 123 | { |
6dc9658f | 124 | return __atomic64_xchg_add(v, __atomic_setup(v), i); |
867e359b CM |
125 | } |
126 | EXPORT_SYMBOL(_atomic64_xchg_add); | |
127 | ||
b924a690 | 128 | long long _atomic64_xchg_add_unless(long long *v, long long a, long long u) |
867e359b CM |
129 | { |
130 | /* | |
131 | * Note: argument order is switched here since it is easier | |
132 | * to use the first argument consistently as the "old value" | |
133 | * in the assembly, as is done for _atomic_cmpxchg(). | |
134 | */ | |
6dc9658f | 135 | return __atomic64_xchg_add_unless(v, __atomic_setup(v), u, a); |
867e359b CM |
136 | } |
137 | EXPORT_SYMBOL(_atomic64_xchg_add_unless); | |
138 | ||
b924a690 | 139 | long long _atomic64_cmpxchg(long long *v, long long o, long long n) |
867e359b | 140 | { |
6dc9658f | 141 | return __atomic64_cmpxchg(v, __atomic_setup(v), o, n); |
867e359b CM |
142 | } |
143 | EXPORT_SYMBOL(_atomic64_cmpxchg); | |
144 | ||
1af5de9a | 145 | long long _atomic64_fetch_and(long long *v, long long n) |
2957c035 | 146 | { |
1af5de9a | 147 | return __atomic64_fetch_and(v, __atomic_setup(v), n); |
2957c035 | 148 | } |
1af5de9a | 149 | EXPORT_SYMBOL(_atomic64_fetch_and); |
2957c035 | 150 | |
1af5de9a | 151 | long long _atomic64_fetch_or(long long *v, long long n) |
2957c035 | 152 | { |
1af5de9a | 153 | return __atomic64_fetch_or(v, __atomic_setup(v), n); |
2957c035 | 154 | } |
1af5de9a | 155 | EXPORT_SYMBOL(_atomic64_fetch_or); |
2957c035 | 156 | |
1af5de9a | 157 | long long _atomic64_fetch_xor(long long *v, long long n) |
2957c035 | 158 | { |
1af5de9a | 159 | return __atomic64_fetch_xor(v, __atomic_setup(v), n); |
2957c035 | 160 | } |
1af5de9a | 161 | EXPORT_SYMBOL(_atomic64_fetch_xor); |
867e359b | 162 | |
867e359b CM |
163 | /* |
164 | * If any of the atomic or futex routines hit a bad address (not in | |
165 | * the page tables at kernel PL) this routine is called. The futex | |
166 | * routines are never used on kernel space, and the normal atomics and | |
167 | * bitops are never used on user space. So a fault on kernel space | |
168 | * must be fatal, but a fault on userspace is a futex fault and we | |
169 | * need to return -EFAULT. Note that the context this routine is | |
170 | * invoked in is the context of the "_atomic_xxx()" routines called | |
171 | * by the functions in this file. | |
172 | */ | |
0707ad30 | 173 | struct __get_user __atomic_bad_address(int __user *addr) |
867e359b CM |
174 | { |
175 | if (unlikely(!access_ok(VERIFY_WRITE, addr, sizeof(int)))) | |
176 | panic("Bad address used for kernel atomic op: %p\n", addr); | |
177 | return (struct __get_user) { .err = -EFAULT }; | |
178 | } | |
179 | ||
180 | ||
867e359b CM |
181 | void __init __init_atomic_per_cpu(void) |
182 | { | |
867e359b | 183 | /* Validate power-of-two and "bigger than cpus" assumption */ |
de5bbad6 | 184 | BUILD_BUG_ON(ATOMIC_HASH_SIZE & (ATOMIC_HASH_SIZE-1)); |
867e359b CM |
185 | BUG_ON(ATOMIC_HASH_SIZE < nr_cpu_ids); |
186 | ||
187 | /* | |
188 | * On TILEPro we prefer to use a single hash-for-home | |
189 | * page, since this means atomic operations are less | |
190 | * likely to encounter a TLB fault and thus should | |
191 | * in general perform faster. You may wish to disable | |
192 | * this in situations where few hash-for-home tiles | |
193 | * are configured. | |
194 | */ | |
195 | BUG_ON((unsigned long)atomic_locks % PAGE_SIZE != 0); | |
196 | ||
197 | /* The locks must all fit on one page. */ | |
de5bbad6 | 198 | BUILD_BUG_ON(ATOMIC_HASH_SIZE * sizeof(int) > PAGE_SIZE); |
867e359b CM |
199 | |
200 | /* | |
201 | * We use the page offset of the atomic value's address as | |
202 | * an index into atomic_locks, excluding the low 3 bits. | |
203 | * That should not produce more indices than ATOMIC_HASH_SIZE. | |
204 | */ | |
de5bbad6 | 205 | BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE); |
867e359b | 206 | } |