[linux-2.6-block.git] / arch / mips / include / asm / spinlock.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 */
#ifndef _ASM_SPINLOCK_H
#define _ASM_SPINLOCK_H

#include <linux/compiler.h>

#include <asm/barrier.h>
#include <asm/war.h>

/*
 * Your basic SMP spinlocks, allowing only a single CPU anywhere
 *
 * Simple spin lock operations.	 There are two variants, one clears IRQ's
 * on the local processor, one does not.
 *
 * These are fair FIFO ticket locks
 *
 * (the type definitions are in asm/spinlock_types.h)
 */


/*
 * Ticket locks are conceptually two parts, one indicating the current head of
 * the queue, and the other indicating the current tail. The lock is acquired
 * by atomically noting the tail and incrementing it by one (thus adding
 * ourself to the queue and noting our position), then waiting until the head
 * becomes equal to the the initial value of the tail.
 */

static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
	u32 counters = ACCESS_ONCE(lock->lock);

	return ((counters >> 16) ^ counters) & 0xffff;
}

#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
#define arch_spin_unlock_wait(x) \
	while (arch_spin_is_locked(x)) { cpu_relax(); }

static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
	u32 counters = ACCESS_ONCE(lock->lock);

	return (((counters >> 16) - counters) & 0xffff) > 1;
}
#define arch_spin_is_contended	arch_spin_is_contended

static inline void arch_spin_lock(arch_spinlock_t *lock)
{
	int my_ticket;
	int tmp;
	int inc = 0x10000;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqzl	%[my_ticket], 1b			\n"
		"	 nop						\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[ticket], %[ticket], 0xffff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0xffff		\n"
		"	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lhu	%[ticket], %[serving_now_ptr]		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	4b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [serving_now_ptr] "+m" (lock->h.serving_now),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket)
		: [inc] "r" (inc));
	} else {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_lock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	addu	%[my_ticket], %[ticket], %[inc]		\n"
		"	sc	%[my_ticket], %[ticket_ptr]		\n"
		"	beqz	%[my_ticket], 1b			\n"
		"	 srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[ticket], %[ticket], 0xffff		\n"
		"	bne	%[ticket], %[my_ticket], 4f		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"2:							\n"
		"	.subsection 2					\n"
		"4:	andi	%[ticket], %[ticket], 0x1fff		\n"
		"	sll	%[ticket], 5				\n"
		"							\n"
		"6:	bnez	%[ticket], 6b				\n"
		"	 subu	%[ticket], 1				\n"
		"							\n"
		"	lhu	%[ticket], %[serving_now_ptr]		\n"
		"	beq	%[ticket], %[my_ticket], 2b		\n"
		"	 subu	%[ticket], %[my_ticket], %[ticket]	\n"
		"	b	4b					\n"
		"	 subu	%[ticket], %[ticket], 1			\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [serving_now_ptr] "+m" (lock->h.serving_now),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (my_ticket)
		: [inc] "r" (inc));
	}

	smp_llsc_mb();
}

static inline void arch_spin_unlock(arch_spinlock_t *lock)
{
	unsigned int serving_now = lock->h.serving_now + 1;
	wmb();
	lock->h.serving_now = (u16)serving_now;
	nudge_writes();
}

static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
{
	int tmp, tmp2, tmp3;
	int inc = 0x10000;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[now_serving], %[ticket], 0xffff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addu	%[ticket], %[ticket], %[inc]		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqzl	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3)
		: [inc] "r" (inc));
	} else {
		__asm__ __volatile__ (
		"	.set push		# arch_spin_trylock	\n"
		"	.set noreorder					\n"
		"							\n"
		"1:	ll	%[ticket], %[ticket_ptr]		\n"
		"	srl	%[my_ticket], %[ticket], 16		\n"
		"	andi	%[now_serving], %[ticket], 0xffff	\n"
		"	bne	%[my_ticket], %[now_serving], 3f	\n"
		"	 addu	%[ticket], %[ticket], %[inc]		\n"
		"	sc	%[ticket], %[ticket_ptr]		\n"
		"	beqz	%[ticket], 1b				\n"
		"	 li	%[ticket], 1				\n"
		"2:							\n"
		"	.subsection 2					\n"
		"3:	b	2b					\n"
		"	 li	%[ticket], 0				\n"
		"	.previous					\n"
		"	.set pop					\n"
		: [ticket_ptr] "+m" (lock->lock),
		  [ticket] "=&r" (tmp),
		  [my_ticket] "=&r" (tmp2),
		  [now_serving] "=&r" (tmp3)
		: [inc] "r" (inc));
	}

	smp_llsc_mb();

	return tmp;
}

/*
 * Read-write spinlocks, allowing multiple readers but only one writer.
 *
 * NOTE! it is quite common to have readers in interrupts but no interrupt
 * writers. For those circumstances we can "mix" irq-safe locks - any writer
 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
 * read-locks.
 */

/*
 * read_can_lock - would read_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define arch_read_can_lock(rw)	((rw)->lock >= 0)

/*
 * write_can_lock - would write_trylock() succeed?
 * @lock: the rwlock in question.
 */
#define arch_write_can_lock(rw) (!(rw)->lock)

static inline void arch_read_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_lock	\n"
		"1:	ll	%1, %2					\n"
		"	bltz	%1, 1b					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_read_lock	\n"
			"	bltz	%1, 1b				\n"
			"	 addu	%1, 1				\n"
			"2:	sc	%1, %0				\n"
			: "=m" (rw->lock), "=&r" (tmp)
			: "m" (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}

	smp_llsc_mb();
}

/* Note the use of sub, not subu which will make the kernel die with an
   overflow exception if we ever try to unlock an rwlock that is already
   unlocked or is being held by a writer.  */
static inline void arch_read_unlock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	smp_mb__before_llsc();

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"1:	ll	%1, %2		# arch_read_unlock	\n"
		"	sub	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_read_unlock	\n"
			"	sub	%1, 1				\n"
			"	sc	%1, %0				\n"
			: "=m" (rw->lock), "=&r" (tmp)
			: "m" (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}
}

static inline void arch_write_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_write_lock	\n"
		"1:	ll	%1, %2					\n"
		"	bnez	%1, 1b					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		: "=m" (rw->lock), "=&r" (tmp)
		: "m" (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"1:	ll	%1, %2	# arch_write_lock	\n"
			"	bnez	%1, 1b				\n"
			"	 lui	%1, 0x8000			\n"
			"2:	sc	%1, %0				\n"
			: "=m" (rw->lock), "=&r" (tmp)
			: "m" (rw->lock)
			: "memory");
		} while (unlikely(!tmp));
	}

	smp_llsc_mb();
}

static inline void arch_write_unlock(arch_rwlock_t *rw)
{
	smp_mb();

	__asm__ __volatile__(
	"				# arch_write_unlock	\n"
	"	sw	$0, %0					\n"
	: "=m" (rw->lock)
	: "m" (rw->lock)
	: "memory");
}

static inline int arch_read_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bltz	%1, 2f					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	.set	reorder					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"2:							\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	} else {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_read_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bltz	%1, 2f					\n"
		"	 addu	%1, 1					\n"
		"	sc	%1, %0					\n"
		"	beqz	%1, 1b					\n"
		"	 nop						\n"
		"	.set	reorder					\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"2:							\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	}

	return ret;
}

static inline int arch_write_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;
	int ret;

	if (R10000_LLSC_WAR) {
		__asm__ __volatile__(
		"	.set	noreorder	# arch_write_trylock	\n"
		"	li	%2, 0					\n"
		"1:	ll	%1, %3					\n"
		"	bnez	%1, 2f					\n"
		"	 lui	%1, 0x8000				\n"
		"	sc	%1, %0					\n"
		"	beqzl	%1, 1b					\n"
		"	 nop						\n"
		__WEAK_LLSC_MB
		"	li	%2, 1					\n"
		"	.set	reorder					\n"
		"2:							\n"
		: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
		: "m" (rw->lock)
		: "memory");
	} else {
		do {
			__asm__ __volatile__(
			"	ll	%1, %3	# arch_write_trylock	\n"
			"	li	%2, 0				\n"
			"	bnez	%1, 2f				\n"
			"	lui	%1, 0x8000			\n"
			"	sc	%1, %0				\n"
			"	li	%2, 1				\n"
			"2:						\n"
			: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
			: "m" (rw->lock)
			: "memory");
		} while (unlikely(!tmp));

		smp_llsc_mb();
	}

	return ret;
}

#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)

#define arch_spin_relax(lock)	cpu_relax()
#define arch_read_relax(lock)	cpu_relax()
#define arch_write_relax(lock)	cpu_relax()

#endif /* _ASM_SPINLOCK_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
f65e4fa8	6	* Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
1da177e4 LT	7	* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
	8	*/
	9	#ifndef _ASM_SPINLOCK_H
	10	#define _ASM_SPINLOCK_H
	11
2a31b033 RB	12	#include <linux/compiler.h>
2a31b033 RB	13
0004a9df	14	#include <asm/barrier.h>
1da177e4 LT	15	#include <asm/war.h>
	16
	17	/*
	18	* Your basic SMP spinlocks, allowing only a single CPU anywhere
2a31b033	19	*
70342287	20	* Simple spin lock operations. There are two variants, one clears IRQ's
2a31b033 RB	21	* on the local processor, one does not.
	22	*
	23	* These are fair FIFO ticket locks
	24	*
	25	* (the type definitions are in asm/spinlock_types.h)
1da177e4 LT	26	*/
1da177e4 LT	27
1da177e4 LT	28
1da177e4 LT	29	/*
2a31b033 RB	30	* Ticket locks are conceptually two parts, one indicating the current head of
	31	* the queue, and the other indicating the current tail. The lock is acquired
	32	* by atomically noting the tail and incrementing it by one (thus adding
	33	* ourself to the queue and noting our position), then waiting until the head
	34	* becomes equal to the the initial value of the tail.
1da177e4 LT	35	*/
1da177e4 LT	36
0199c4e6	37	static inline int arch_spin_is_locked(arch_spinlock_t *lock)
2a31b033	38	{
500c2e1f	39	u32 counters = ACCESS_ONCE(lock->lock);
2a31b033	40
500c2e1f	41	return ((counters >> 16) ^ counters) & 0xffff;
2a31b033 RB	42	}
2a31b033 RB	43
0199c4e6 TG	44	#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
	45	#define arch_spin_unlock_wait(x) \
	46	while (arch_spin_is_locked(x)) { cpu_relax(); }
2a31b033	47
0199c4e6	48	static inline int arch_spin_is_contended(arch_spinlock_t *lock)
2a31b033	49	{
500c2e1f	50	u32 counters = ACCESS_ONCE(lock->lock);
2a31b033	51
500c2e1f	52	return (((counters >> 16) - counters) & 0xffff) > 1;
2a31b033	53	}
0199c4e6	54	#define arch_spin_is_contended arch_spin_is_contended
2a31b033	55
0199c4e6	56	static inline void arch_spin_lock(arch_spinlock_t *lock)
1da177e4	57	{
2a31b033 RB	58	int my_ticket;
2a31b033 RB	59	int tmp;
500c2e1f	60	int inc = 0x10000;
1da177e4 LT	61
1da177e4 LT	62	if (R10000_LLSC_WAR) {
2a31b033	63	__asm__ __volatile__ (
0199c4e6	64	" .set push # arch_spin_lock \n"
2a31b033 RB	65	" .set noreorder \n"
	66	" \n"
	67	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f	68	" addu %[my_ticket], %[ticket], %[inc] \n"
2a31b033 RB	69	" sc %[my_ticket], %[ticket_ptr] \n"
2a31b033 RB	70	" beqzl %[my_ticket], 1b \n"
1da177e4	71	" nop \n"
500c2e1f DD	72	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f DD	73	" andi %[ticket], %[ticket], 0xffff \n"
2a31b033 RB	74	" bne %[ticket], %[my_ticket], 4f \n"
	75	" subu %[ticket], %[my_ticket], %[ticket] \n"
	76	"2: \n"
	77	" .subsection 2 \n"
500c2e1f	78	"4: andi %[ticket], %[ticket], 0xffff \n"
0e6826c7	79	" sll %[ticket], 5 \n"
2a31b033 RB	80	" \n"
	81	"6: bnez %[ticket], 6b \n"
	82	" subu %[ticket], 1 \n"
	83	" \n"
500c2e1f	84	" lhu %[ticket], %[serving_now_ptr] \n"
2a31b033 RB	85	" beq %[ticket], %[my_ticket], 2b \n"
2a31b033 RB	86	" subu %[ticket], %[my_ticket], %[ticket] \n"
0e6826c7	87	" b 4b \n"
2a31b033 RB	88	" subu %[ticket], %[ticket], 1 \n"
	89	" .previous \n"
	90	" .set pop \n"
	91	: [ticket_ptr] "+m" (lock->lock),
500c2e1f	92	[serving_now_ptr] "+m" (lock->h.serving_now),
2a31b033	93	[ticket] "=&r" (tmp),
500c2e1f DD	94	[my_ticket] "=&r" (my_ticket)
500c2e1f DD	95	: [inc] "r" (inc));
1da177e4	96	} else {
2a31b033	97	__asm__ __volatile__ (
0199c4e6	98	" .set push # arch_spin_lock \n"
2a31b033 RB	99	" .set noreorder \n"
2a31b033 RB	100	" \n"
500c2e1f DD	101	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f DD	102	" addu %[my_ticket], %[ticket], %[inc] \n"
2a31b033	103	" sc %[my_ticket], %[ticket_ptr] \n"
500c2e1f DD	104	" beqz %[my_ticket], 1b \n"
	105	" srl %[my_ticket], %[ticket], 16 \n"
	106	" andi %[ticket], %[ticket], 0xffff \n"
2a31b033 RB	107	" bne %[ticket], %[my_ticket], 4f \n"
	108	" subu %[ticket], %[my_ticket], %[ticket] \n"
	109	"2: \n"
f65e4fa8	110	" .subsection 2 \n"
2a31b033	111	"4: andi %[ticket], %[ticket], 0x1fff \n"
0e6826c7	112	" sll %[ticket], 5 \n"
2a31b033 RB	113	" \n"
	114	"6: bnez %[ticket], 6b \n"
	115	" subu %[ticket], 1 \n"
	116	" \n"
500c2e1f	117	" lhu %[ticket], %[serving_now_ptr] \n"
2a31b033 RB	118	" beq %[ticket], %[my_ticket], 2b \n"
2a31b033 RB	119	" subu %[ticket], %[my_ticket], %[ticket] \n"
0e6826c7	120	" b 4b \n"
2a31b033	121	" subu %[ticket], %[ticket], 1 \n"
f65e4fa8	122	" .previous \n"
2a31b033 RB	123	" .set pop \n"
2a31b033 RB	124	: [ticket_ptr] "+m" (lock->lock),
500c2e1f	125	[serving_now_ptr] "+m" (lock->h.serving_now),
2a31b033	126	[ticket] "=&r" (tmp),
500c2e1f DD	127	[my_ticket] "=&r" (my_ticket)
500c2e1f DD	128	: [inc] "r" (inc));
1da177e4	129	}
0004a9df	130
17099b11	131	smp_llsc_mb();
1da177e4 LT	132	}
1da177e4 LT	133
0199c4e6	134	static inline void arch_spin_unlock(arch_spinlock_t *lock)
1da177e4	135	{
500c2e1f DD	136	unsigned int serving_now = lock->h.serving_now + 1;
	137	wmb();
	138	lock->h.serving_now = (u16)serving_now;
	139	nudge_writes();
1da177e4 LT	140	}
1da177e4 LT	141
0199c4e6	142	static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
1da177e4	143	{
2a31b033	144	int tmp, tmp2, tmp3;
500c2e1f	145	int inc = 0x10000;
1da177e4 LT	146
1da177e4 LT	147	if (R10000_LLSC_WAR) {
2a31b033	148	__asm__ __volatile__ (
0199c4e6	149	" .set push # arch_spin_trylock \n"
2a31b033 RB	150	" .set noreorder \n"
	151	" \n"
	152	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f	153	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f	154	" andi %[now_serving], %[ticket], 0xffff \n"
2a31b033	155	" bne %[my_ticket], %[now_serving], 3f \n"
500c2e1f	156	" addu %[ticket], %[ticket], %[inc] \n"
2a31b033 RB	157	" sc %[ticket], %[ticket_ptr] \n"
	158	" beqzl %[ticket], 1b \n"
	159	" li %[ticket], 1 \n"
	160	"2: \n"
	161	" .subsection 2 \n"
	162	"3: b 2b \n"
	163	" li %[ticket], 0 \n"
	164	" .previous \n"
	165	" .set pop \n"
	166	: [ticket_ptr] "+m" (lock->lock),
	167	[ticket] "=&r" (tmp),
	168	[my_ticket] "=&r" (tmp2),
500c2e1f DD	169	[now_serving] "=&r" (tmp3)
500c2e1f DD	170	: [inc] "r" (inc));
1da177e4	171	} else {
2a31b033	172	__asm__ __volatile__ (
0199c4e6	173	" .set push # arch_spin_trylock \n"
2a31b033 RB	174	" .set noreorder \n"
2a31b033 RB	175	" \n"
500c2e1f DD	176	"1: ll %[ticket], %[ticket_ptr] \n"
500c2e1f DD	177	" srl %[my_ticket], %[ticket], 16 \n"
500c2e1f	178	" andi %[now_serving], %[ticket], 0xffff \n"
2a31b033	179	" bne %[my_ticket], %[now_serving], 3f \n"
500c2e1f	180	" addu %[ticket], %[ticket], %[inc] \n"
2a31b033	181	" sc %[ticket], %[ticket_ptr] \n"
500c2e1f	182	" beqz %[ticket], 1b \n"
2a31b033 RB	183	" li %[ticket], 1 \n"
2a31b033 RB	184	"2: \n"
f65e4fa8	185	" .subsection 2 \n"
2a31b033 RB	186	"3: b 2b \n"
2a31b033 RB	187	" li %[ticket], 0 \n"
f65e4fa8	188	" .previous \n"
2a31b033 RB	189	" .set pop \n"
	190	: [ticket_ptr] "+m" (lock->lock),
	191	[ticket] "=&r" (tmp),
	192	[my_ticket] "=&r" (tmp2),
500c2e1f DD	193	[now_serving] "=&r" (tmp3)
500c2e1f DD	194	: [inc] "r" (inc));
1da177e4 LT	195	}
1da177e4 LT	196
17099b11	197	smp_llsc_mb();
0004a9df	198
2a31b033	199	return tmp;
1da177e4 LT	200	}
	201
	202	/*
	203	* Read-write spinlocks, allowing multiple readers but only one writer.
	204	*
	205	* NOTE! it is quite common to have readers in interrupts but no interrupt
	206	* writers. For those circumstances we can "mix" irq-safe locks - any writer
	207	* needs to get a irq-safe write-lock, but readers can get non-irqsafe
	208	* read-locks.
	209	*/
	210
e3c48078 RB	211	/*
	212	* read_can_lock - would read_trylock() succeed?
	213	* @lock: the rwlock in question.
	214	*/
e5931943	215	#define arch_read_can_lock(rw) ((rw)->lock >= 0)
e3c48078 RB	216
	217	/*
	218	* write_can_lock - would write_trylock() succeed?
	219	* @lock: the rwlock in question.
	220	*/
70342287	221	#define arch_write_can_lock(rw) (!(rw)->lock)
e3c48078	222
e5931943	223	static inline void arch_read_lock(arch_rwlock_t *rw)
1da177e4 LT	224	{
	225	unsigned int tmp;
	226
	227	if (R10000_LLSC_WAR) {
	228	__asm__ __volatile__(
e5931943	229	" .set noreorder # arch_read_lock \n"
1da177e4 LT	230	"1: ll %1, %2 \n"
	231	" bltz %1, 1b \n"
	232	" addu %1, 1 \n"
	233	" sc %1, %0 \n"
	234	" beqzl %1, 1b \n"
	235	" nop \n"
1da177e4 LT	236	" .set reorder \n"
	237	: "=m" (rw->lock), "=&r" (tmp)
	238	: "m" (rw->lock)
	239	: "memory");
	240	} else {
e01961ce RB	241	do {
	242	__asm__ __volatile__(
	243	"1: ll %1, %2 # arch_read_lock \n"
	244	" bltz %1, 1b \n"
	245	" addu %1, 1 \n"
	246	"2: sc %1, %0 \n"
	247	: "=m" (rw->lock), "=&r" (tmp)
	248	: "m" (rw->lock)
	249	: "memory");
	250	} while (unlikely(!tmp));
1da177e4	251	}
0004a9df	252
17099b11	253	smp_llsc_mb();
1da177e4 LT	254	}
	255
	256	/* Note the use of sub, not subu which will make the kernel die with an
	257	overflow exception if we ever try to unlock an rwlock that is already
	258	unlocked or is being held by a writer. */
e5931943	259	static inline void arch_read_unlock(arch_rwlock_t *rw)
1da177e4 LT	260	{
	261	unsigned int tmp;
	262
f252ffd5	263	smp_mb__before_llsc();
0004a9df	264
1da177e4 LT	265	if (R10000_LLSC_WAR) {
1da177e4 LT	266	__asm__ __volatile__(
e5931943	267	"1: ll %1, %2 # arch_read_unlock \n"
1da177e4 LT	268	" sub %1, 1 \n"
	269	" sc %1, %0 \n"
	270	" beqzl %1, 1b \n"
1da177e4 LT	271	: "=m" (rw->lock), "=&r" (tmp)
	272	: "m" (rw->lock)
	273	: "memory");
	274	} else {
e01961ce RB	275	do {
	276	__asm__ __volatile__(
	277	"1: ll %1, %2 # arch_read_unlock \n"
	278	" sub %1, 1 \n"
	279	" sc %1, %0 \n"
	280	: "=m" (rw->lock), "=&r" (tmp)
	281	: "m" (rw->lock)
	282	: "memory");
	283	} while (unlikely(!tmp));
1da177e4 LT	284	}
	285	}
	286
e5931943	287	static inline void arch_write_lock(arch_rwlock_t *rw)
1da177e4 LT	288	{
	289	unsigned int tmp;
	290
	291	if (R10000_LLSC_WAR) {
	292	__asm__ __volatile__(
e5931943	293	" .set noreorder # arch_write_lock \n"
1da177e4 LT	294	"1: ll %1, %2 \n"
	295	" bnez %1, 1b \n"
	296	" lui %1, 0x8000 \n"
	297	" sc %1, %0 \n"
	298	" beqzl %1, 1b \n"
0004a9df	299	" nop \n"
1da177e4 LT	300	" .set reorder \n"
	301	: "=m" (rw->lock), "=&r" (tmp)
	302	: "m" (rw->lock)
	303	: "memory");
	304	} else {
e01961ce RB	305	do {
	306	__asm__ __volatile__(
	307	"1: ll %1, %2 # arch_write_lock \n"
	308	" bnez %1, 1b \n"
	309	" lui %1, 0x8000 \n"
	310	"2: sc %1, %0 \n"
	311	: "=m" (rw->lock), "=&r" (tmp)
	312	: "m" (rw->lock)
	313	: "memory");
	314	} while (unlikely(!tmp));
1da177e4	315	}
0004a9df	316
17099b11	317	smp_llsc_mb();
1da177e4 LT	318	}
1da177e4 LT	319
e5931943	320	static inline void arch_write_unlock(arch_rwlock_t *rw)
1da177e4	321	{
0004a9df RB	322	smp_mb();
0004a9df RB	323
1da177e4	324	__asm__ __volatile__(
e5931943	325	" # arch_write_unlock \n"
1da177e4 LT	326	" sw $0, %0 \n"
	327	: "=m" (rw->lock)
	328	: "m" (rw->lock)
	329	: "memory");
	330	}
	331
e5931943	332	static inline int arch_read_trylock(arch_rwlock_t *rw)
65316fd1 RB	333	{
	334	unsigned int tmp;
	335	int ret;
	336
	337	if (R10000_LLSC_WAR) {
	338	__asm__ __volatile__(
e5931943	339	" .set noreorder # arch_read_trylock \n"
65316fd1 RB	340	" li %2, 0 \n"
65316fd1 RB	341	"1: ll %1, %3 \n"
d52c2d5a	342	" bltz %1, 2f \n"
65316fd1 RB	343	" addu %1, 1 \n"
65316fd1 RB	344	" sc %1, %0 \n"
65316fd1	345	" .set reorder \n"
0004a9df RB	346	" beqzl %1, 1b \n"
0004a9df RB	347	" nop \n"
17099b11	348	__WEAK_LLSC_MB
65316fd1 RB	349	" li %2, 1 \n"
	350	"2: \n"
	351	: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
	352	: "m" (rw->lock)
	353	: "memory");
	354	} else {
	355	__asm__ __volatile__(
e5931943	356	" .set noreorder # arch_read_trylock \n"
65316fd1 RB	357	" li %2, 0 \n"
65316fd1 RB	358	"1: ll %1, %3 \n"
d52c2d5a	359	" bltz %1, 2f \n"
65316fd1 RB	360	" addu %1, 1 \n"
	361	" sc %1, %0 \n"
	362	" beqz %1, 1b \n"
0004a9df	363	" nop \n"
65316fd1	364	" .set reorder \n"
17099b11	365	__WEAK_LLSC_MB
65316fd1 RB	366	" li %2, 1 \n"
	367	"2: \n"
	368	: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
	369	: "m" (rw->lock)
	370	: "memory");
	371	}
	372
	373	return ret;
	374	}
1da177e4	375
e5931943	376	static inline int arch_write_trylock(arch_rwlock_t *rw)
1da177e4 LT	377	{
	378	unsigned int tmp;
	379	int ret;
	380
	381	if (R10000_LLSC_WAR) {
	382	__asm__ __volatile__(
e5931943	383	" .set noreorder # arch_write_trylock \n"
1da177e4 LT	384	" li %2, 0 \n"
	385	"1: ll %1, %3 \n"
	386	" bnez %1, 2f \n"
	387	" lui %1, 0x8000 \n"
	388	" sc %1, %0 \n"
	389	" beqzl %1, 1b \n"
0004a9df	390	" nop \n"
17099b11	391	__WEAK_LLSC_MB
1da177e4 LT	392	" li %2, 1 \n"
	393	" .set reorder \n"
	394	"2: \n"
	395	: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
	396	: "m" (rw->lock)
	397	: "memory");
	398	} else {
e01961ce RB	399	do {
	400	__asm__ __volatile__(
	401	" ll %1, %3 # arch_write_trylock \n"
	402	" li %2, 0 \n"
	403	" bnez %1, 2f \n"
	404	" lui %1, 0x8000 \n"
	405	" sc %1, %0 \n"
	406	" li %2, 1 \n"
	407	"2: \n"
	408	: "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
	409	: "m" (rw->lock)
	410	: "memory");
	411	} while (unlikely(!tmp));
	412
	413	smp_llsc_mb();
1da177e4 LT	414	}
	415
	416	return ret;
	417	}
	418
e5931943 TG	419	#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
e5931943 TG	420	#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
65316fd1	421
0199c4e6 TG	422	#define arch_spin_relax(lock) cpu_relax()
	423	#define arch_read_relax(lock) cpu_relax()
	424	#define arch_write_relax(lock) cpu_relax()
ef6edc97	425
1da177e4	426	#endif /* _ASM_SPINLOCK_H */