Commit | Line | Data |
---|---|---|
d5de8841 JF |
1 | /* |
2 | * Split spinlock implementation out into its own file, so it can be | |
3 | * compiled in a FTRACE-compatible way. | |
4 | */ | |
5 | #include <linux/kernel_stat.h> | |
6 | #include <linux/spinlock.h> | |
994025ca JF |
7 | #include <linux/debugfs.h> |
8 | #include <linux/log2.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
d5de8841 JF |
10 | |
11 | #include <asm/paravirt.h> | |
12 | ||
13 | #include <xen/interface/xen.h> | |
14 | #include <xen/events.h> | |
15 | ||
16 | #include "xen-ops.h" | |
994025ca JF |
17 | #include "debugfs.h" |
18 | ||
19 | #ifdef CONFIG_XEN_DEBUG_FS | |
20 | static struct xen_spinlock_stats | |
21 | { | |
22 | u64 taken; | |
23 | u32 taken_slow; | |
24 | u32 taken_slow_nested; | |
25 | u32 taken_slow_pickup; | |
26 | u32 taken_slow_spurious; | |
1e696f63 | 27 | u32 taken_slow_irqenable; |
994025ca JF |
28 | |
29 | u64 released; | |
30 | u32 released_slow; | |
31 | u32 released_slow_kicked; | |
32 | ||
f8eca41f JF |
33 | #define HISTO_BUCKETS 30 |
34 | u32 histo_spin_total[HISTO_BUCKETS+1]; | |
35 | u32 histo_spin_spinning[HISTO_BUCKETS+1]; | |
36 | u32 histo_spin_blocked[HISTO_BUCKETS+1]; | |
37 | ||
38 | u64 time_total; | |
39 | u64 time_spinning; | |
40 | u64 time_blocked; | |
994025ca JF |
41 | } spinlock_stats; |
42 | ||
43 | static u8 zero_stats; | |
44 | ||
45 | static unsigned lock_timeout = 1 << 10; | |
46 | #define TIMEOUT lock_timeout | |
47 | ||
48 | static inline void check_zero(void) | |
49 | { | |
50 | if (unlikely(zero_stats)) { | |
51 | memset(&spinlock_stats, 0, sizeof(spinlock_stats)); | |
52 | zero_stats = 0; | |
53 | } | |
54 | } | |
55 | ||
56 | #define ADD_STATS(elem, val) \ | |
57 | do { check_zero(); spinlock_stats.elem += (val); } while(0) | |
58 | ||
59 | static inline u64 spin_time_start(void) | |
60 | { | |
61 | return xen_clocksource_read(); | |
62 | } | |
63 | ||
64 | static void __spin_time_accum(u64 delta, u32 *array) | |
65 | { | |
66 | unsigned index = ilog2(delta); | |
67 | ||
68 | check_zero(); | |
69 | ||
70 | if (index < HISTO_BUCKETS) | |
71 | array[index]++; | |
72 | else | |
73 | array[HISTO_BUCKETS]++; | |
74 | } | |
75 | ||
f8eca41f JF |
76 | static inline void spin_time_accum_spinning(u64 start) |
77 | { | |
78 | u32 delta = xen_clocksource_read() - start; | |
79 | ||
80 | __spin_time_accum(delta, spinlock_stats.histo_spin_spinning); | |
81 | spinlock_stats.time_spinning += delta; | |
82 | } | |
83 | ||
84 | static inline void spin_time_accum_total(u64 start) | |
994025ca JF |
85 | { |
86 | u32 delta = xen_clocksource_read() - start; | |
87 | ||
f8eca41f JF |
88 | __spin_time_accum(delta, spinlock_stats.histo_spin_total); |
89 | spinlock_stats.time_total += delta; | |
994025ca JF |
90 | } |
91 | ||
f8eca41f | 92 | static inline void spin_time_accum_blocked(u64 start) |
994025ca JF |
93 | { |
94 | u32 delta = xen_clocksource_read() - start; | |
95 | ||
f8eca41f JF |
96 | __spin_time_accum(delta, spinlock_stats.histo_spin_blocked); |
97 | spinlock_stats.time_blocked += delta; | |
994025ca JF |
98 | } |
99 | #else /* !CONFIG_XEN_DEBUG_FS */ | |
100 | #define TIMEOUT (1 << 10) | |
101 | #define ADD_STATS(elem, val) do { (void)(val); } while(0) | |
102 | ||
103 | static inline u64 spin_time_start(void) | |
104 | { | |
105 | return 0; | |
106 | } | |
107 | ||
f8eca41f JF |
108 | static inline void spin_time_accum_total(u64 start) |
109 | { | |
110 | } | |
111 | static inline void spin_time_accum_spinning(u64 start) | |
994025ca JF |
112 | { |
113 | } | |
f8eca41f | 114 | static inline void spin_time_accum_blocked(u64 start) |
994025ca JF |
115 | { |
116 | } | |
117 | #endif /* CONFIG_XEN_DEBUG_FS */ | |
d5de8841 JF |
118 | |
119 | struct xen_spinlock { | |
120 | unsigned char lock; /* 0 -> free; 1 -> locked */ | |
121 | unsigned short spinners; /* count of waiting cpus */ | |
122 | }; | |
123 | ||
445c8951 | 124 | static int xen_spin_is_locked(struct arch_spinlock *lock) |
d5de8841 JF |
125 | { |
126 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
127 | ||
128 | return xl->lock != 0; | |
129 | } | |
130 | ||
445c8951 | 131 | static int xen_spin_is_contended(struct arch_spinlock *lock) |
d5de8841 JF |
132 | { |
133 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
134 | ||
135 | /* Not strictly true; this is only the count of contended | |
136 | lock-takers entering the slow path. */ | |
137 | return xl->spinners != 0; | |
138 | } | |
139 | ||
445c8951 | 140 | static int xen_spin_trylock(struct arch_spinlock *lock) |
d5de8841 JF |
141 | { |
142 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
143 | u8 old = 1; | |
144 | ||
145 | asm("xchgb %b0,%1" | |
146 | : "+q" (old), "+m" (xl->lock) : : "memory"); | |
147 | ||
148 | return old == 0; | |
149 | } | |
150 | ||
151 | static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; | |
152 | static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners); | |
153 | ||
168d2f46 JF |
154 | /* |
155 | * Mark a cpu as interested in a lock. Returns the CPU's previous | |
156 | * lock of interest, in case we got preempted by an interrupt. | |
157 | */ | |
158 | static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl) | |
d5de8841 | 159 | { |
168d2f46 JF |
160 | struct xen_spinlock *prev; |
161 | ||
162 | prev = __get_cpu_var(lock_spinners); | |
d5de8841 | 163 | __get_cpu_var(lock_spinners) = xl; |
168d2f46 | 164 | |
d5de8841 | 165 | wmb(); /* set lock of interest before count */ |
168d2f46 | 166 | |
d5de8841 JF |
167 | asm(LOCK_PREFIX " incw %0" |
168 | : "+m" (xl->spinners) : : "memory"); | |
168d2f46 JF |
169 | |
170 | return prev; | |
d5de8841 JF |
171 | } |
172 | ||
168d2f46 JF |
173 | /* |
174 | * Mark a cpu as no longer interested in a lock. Restores previous | |
175 | * lock of interest (NULL for none). | |
176 | */ | |
177 | static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev) | |
d5de8841 JF |
178 | { |
179 | asm(LOCK_PREFIX " decw %0" | |
180 | : "+m" (xl->spinners) : : "memory"); | |
168d2f46 JF |
181 | wmb(); /* decrement count before restoring lock */ |
182 | __get_cpu_var(lock_spinners) = prev; | |
d5de8841 JF |
183 | } |
184 | ||
445c8951 | 185 | static noinline int xen_spin_lock_slow(struct arch_spinlock *lock, bool irq_enable) |
d5de8841 JF |
186 | { |
187 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
168d2f46 | 188 | struct xen_spinlock *prev; |
d5de8841 JF |
189 | int irq = __get_cpu_var(lock_kicker_irq); |
190 | int ret; | |
f8eca41f | 191 | u64 start; |
d5de8841 JF |
192 | |
193 | /* If kicker interrupts not initialized yet, just spin */ | |
194 | if (irq == -1) | |
195 | return 0; | |
196 | ||
f8eca41f JF |
197 | start = spin_time_start(); |
198 | ||
d5de8841 | 199 | /* announce we're spinning */ |
168d2f46 | 200 | prev = spinning_lock(xl); |
d5de8841 | 201 | |
994025ca JF |
202 | ADD_STATS(taken_slow, 1); |
203 | ADD_STATS(taken_slow_nested, prev != NULL); | |
204 | ||
168d2f46 | 205 | do { |
4d576b57 JF |
206 | unsigned long flags; |
207 | ||
168d2f46 JF |
208 | /* clear pending */ |
209 | xen_clear_irq_pending(irq); | |
210 | ||
211 | /* check again make sure it didn't become free while | |
212 | we weren't looking */ | |
213 | ret = xen_spin_trylock(lock); | |
214 | if (ret) { | |
994025ca JF |
215 | ADD_STATS(taken_slow_pickup, 1); |
216 | ||
168d2f46 JF |
217 | /* |
218 | * If we interrupted another spinlock while it | |
219 | * was blocking, make sure it doesn't block | |
220 | * without rechecking the lock. | |
221 | */ | |
222 | if (prev != NULL) | |
223 | xen_set_irq_pending(irq); | |
224 | goto out; | |
225 | } | |
d5de8841 | 226 | |
4d576b57 JF |
227 | flags = __raw_local_save_flags(); |
228 | if (irq_enable) { | |
229 | ADD_STATS(taken_slow_irqenable, 1); | |
230 | raw_local_irq_enable(); | |
231 | } | |
232 | ||
168d2f46 JF |
233 | /* |
234 | * Block until irq becomes pending. If we're | |
235 | * interrupted at this point (after the trylock but | |
236 | * before entering the block), then the nested lock | |
237 | * handler guarantees that the irq will be left | |
238 | * pending if there's any chance the lock became free; | |
239 | * xen_poll_irq() returns immediately if the irq is | |
240 | * pending. | |
241 | */ | |
242 | xen_poll_irq(irq); | |
4d576b57 JF |
243 | |
244 | raw_local_irq_restore(flags); | |
245 | ||
994025ca | 246 | ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq)); |
168d2f46 | 247 | } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */ |
d5de8841 | 248 | |
d6c88a50 | 249 | kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); |
d5de8841 JF |
250 | |
251 | out: | |
168d2f46 | 252 | unspinning_lock(xl, prev); |
f8eca41f JF |
253 | spin_time_accum_blocked(start); |
254 | ||
d5de8841 JF |
255 | return ret; |
256 | } | |
257 | ||
445c8951 | 258 | static inline void __xen_spin_lock(struct arch_spinlock *lock, bool irq_enable) |
d5de8841 JF |
259 | { |
260 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
994025ca | 261 | unsigned timeout; |
d5de8841 | 262 | u8 oldval; |
994025ca JF |
263 | u64 start_spin; |
264 | ||
265 | ADD_STATS(taken, 1); | |
266 | ||
267 | start_spin = spin_time_start(); | |
d5de8841 JF |
268 | |
269 | do { | |
994025ca JF |
270 | u64 start_spin_fast = spin_time_start(); |
271 | ||
272 | timeout = TIMEOUT; | |
d5de8841 JF |
273 | |
274 | asm("1: xchgb %1,%0\n" | |
275 | " testb %1,%1\n" | |
276 | " jz 3f\n" | |
277 | "2: rep;nop\n" | |
278 | " cmpb $0,%0\n" | |
279 | " je 1b\n" | |
280 | " dec %2\n" | |
281 | " jnz 2b\n" | |
282 | "3:\n" | |
283 | : "+m" (xl->lock), "=q" (oldval), "+r" (timeout) | |
284 | : "1" (1) | |
285 | : "memory"); | |
286 | ||
f8eca41f | 287 | spin_time_accum_spinning(start_spin_fast); |
1e696f63 JF |
288 | |
289 | } while (unlikely(oldval != 0 && | |
290 | (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable)))); | |
994025ca | 291 | |
f8eca41f | 292 | spin_time_accum_total(start_spin); |
d5de8841 JF |
293 | } |
294 | ||
445c8951 | 295 | static void xen_spin_lock(struct arch_spinlock *lock) |
1e696f63 JF |
296 | { |
297 | __xen_spin_lock(lock, false); | |
298 | } | |
299 | ||
445c8951 | 300 | static void xen_spin_lock_flags(struct arch_spinlock *lock, unsigned long flags) |
1e696f63 JF |
301 | { |
302 | __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags)); | |
303 | } | |
304 | ||
d5de8841 JF |
305 | static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) |
306 | { | |
307 | int cpu; | |
308 | ||
994025ca JF |
309 | ADD_STATS(released_slow, 1); |
310 | ||
d5de8841 JF |
311 | for_each_online_cpu(cpu) { |
312 | /* XXX should mix up next cpu selection */ | |
313 | if (per_cpu(lock_spinners, cpu) == xl) { | |
994025ca | 314 | ADD_STATS(released_slow_kicked, 1); |
d5de8841 JF |
315 | xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); |
316 | break; | |
317 | } | |
318 | } | |
319 | } | |
320 | ||
445c8951 | 321 | static void xen_spin_unlock(struct arch_spinlock *lock) |
d5de8841 JF |
322 | { |
323 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
324 | ||
994025ca JF |
325 | ADD_STATS(released, 1); |
326 | ||
d5de8841 JF |
327 | smp_wmb(); /* make sure no writes get moved after unlock */ |
328 | xl->lock = 0; /* release lock */ | |
329 | ||
2496afbf YX |
330 | /* |
331 | * Make sure unlock happens before checking for waiting | |
332 | * spinners. We need a strong barrier to enforce the | |
333 | * write-read ordering to different memory locations, as the | |
334 | * CPU makes no implied guarantees about their ordering. | |
335 | */ | |
336 | mb(); | |
d5de8841 JF |
337 | |
338 | if (unlikely(xl->spinners)) | |
339 | xen_spin_unlock_slow(xl); | |
340 | } | |
341 | ||
342 | static irqreturn_t dummy_handler(int irq, void *dev_id) | |
343 | { | |
344 | BUG(); | |
345 | return IRQ_HANDLED; | |
346 | } | |
347 | ||
348 | void __cpuinit xen_init_lock_cpu(int cpu) | |
349 | { | |
350 | int irq; | |
351 | const char *name; | |
352 | ||
353 | name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); | |
354 | irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, | |
355 | cpu, | |
356 | dummy_handler, | |
357 | IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, | |
358 | name, | |
359 | NULL); | |
360 | ||
361 | if (irq >= 0) { | |
362 | disable_irq(irq); /* make sure it's never delivered */ | |
363 | per_cpu(lock_kicker_irq, cpu) = irq; | |
364 | } | |
365 | ||
366 | printk("cpu %d spinlock event irq %d\n", cpu, irq); | |
367 | } | |
368 | ||
d68d82af AN |
369 | void xen_uninit_lock_cpu(int cpu) |
370 | { | |
371 | unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); | |
372 | } | |
373 | ||
d5de8841 JF |
374 | void __init xen_init_spinlocks(void) |
375 | { | |
376 | pv_lock_ops.spin_is_locked = xen_spin_is_locked; | |
377 | pv_lock_ops.spin_is_contended = xen_spin_is_contended; | |
378 | pv_lock_ops.spin_lock = xen_spin_lock; | |
1e696f63 | 379 | pv_lock_ops.spin_lock_flags = xen_spin_lock_flags; |
d5de8841 JF |
380 | pv_lock_ops.spin_trylock = xen_spin_trylock; |
381 | pv_lock_ops.spin_unlock = xen_spin_unlock; | |
382 | } | |
994025ca JF |
383 | |
384 | #ifdef CONFIG_XEN_DEBUG_FS | |
385 | ||
386 | static struct dentry *d_spin_debug; | |
387 | ||
388 | static int __init xen_spinlock_debugfs(void) | |
389 | { | |
390 | struct dentry *d_xen = xen_init_debugfs(); | |
391 | ||
392 | if (d_xen == NULL) | |
393 | return -ENOMEM; | |
394 | ||
395 | d_spin_debug = debugfs_create_dir("spinlocks", d_xen); | |
396 | ||
397 | debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats); | |
398 | ||
399 | debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout); | |
400 | ||
401 | debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken); | |
402 | debugfs_create_u32("taken_slow", 0444, d_spin_debug, | |
403 | &spinlock_stats.taken_slow); | |
404 | debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug, | |
405 | &spinlock_stats.taken_slow_nested); | |
406 | debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug, | |
407 | &spinlock_stats.taken_slow_pickup); | |
408 | debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, | |
409 | &spinlock_stats.taken_slow_spurious); | |
1e696f63 JF |
410 | debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug, |
411 | &spinlock_stats.taken_slow_irqenable); | |
994025ca JF |
412 | |
413 | debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released); | |
414 | debugfs_create_u32("released_slow", 0444, d_spin_debug, | |
415 | &spinlock_stats.released_slow); | |
416 | debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug, | |
417 | &spinlock_stats.released_slow_kicked); | |
418 | ||
419 | debugfs_create_u64("time_spinning", 0444, d_spin_debug, | |
f8eca41f JF |
420 | &spinlock_stats.time_spinning); |
421 | debugfs_create_u64("time_blocked", 0444, d_spin_debug, | |
422 | &spinlock_stats.time_blocked); | |
994025ca | 423 | debugfs_create_u64("time_total", 0444, d_spin_debug, |
f8eca41f | 424 | &spinlock_stats.time_total); |
994025ca JF |
425 | |
426 | xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug, | |
f8eca41f | 427 | spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1); |
994025ca | 428 | xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug, |
f8eca41f JF |
429 | spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1); |
430 | xen_debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug, | |
431 | spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1); | |
994025ca JF |
432 | |
433 | return 0; | |
434 | } | |
435 | fs_initcall(xen_spinlock_debugfs); | |
436 | ||
437 | #endif /* CONFIG_XEN_DEBUG_FS */ |