[linux-2.6-block.git] / arch / x86 / include / asm / qspinlock_paravirt.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_QSPINLOCK_PARAVIRT_H
#define __ASM_QSPINLOCK_PARAVIRT_H

#include <asm/ibt.h>

void __lockfunc __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked);

/*
 * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
 * registers. For i386, however, only 1 32-bit register needs to be saved
 * and restored. So an optimized version of __pv_queued_spin_unlock() is
 * hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
 */
#ifdef CONFIG_64BIT

__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
#define __pv_queued_spin_unlock	__pv_queued_spin_unlock

/*
 * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
 * which combines the registers saving trunk and the body of the following
 * C code.  Note that it puts the code in the .spinlock.text section which
 * is equivalent to adding __lockfunc in the C code:
 *
 * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
 * {
 *	u8 lockval = _Q_LOCKED_VAL;
 *
 *	if (try_cmpxchg(&lock->locked, &lockval, 0))
 *		return;
 *	pv_queued_spin_unlock_slowpath(lock, lockval);
 * }
 *
 * For x86-64,
 *   rdi = lock              (first argument)
 *   rsi = lockval           (second argument)
 *   rdx = internal variable (set to 0)
 */
#define PV_UNLOCK_ASM							\
	FRAME_BEGIN							\
	"push  %rdx\n\t"						\
	"mov   $" __stringify(_Q_LOCKED_VAL) ",%eax\n\t"		\
	"xor   %edx,%edx\n\t"						\
	LOCK_PREFIX "cmpxchg %dl,(%rdi)\n\t"				\
	"jne   .slowpath\n\t"						\
	"pop   %rdx\n\t"						\
	FRAME_END							\
	ASM_RET								\
	".slowpath:\n\t"						\
	"push   %rsi\n\t"						\
	"movzbl %al,%esi\n\t"						\
	"call __raw_callee_save___pv_queued_spin_unlock_slowpath\n\t"	\
	"pop    %rsi\n\t"						\
	"pop    %rdx\n\t"						\
	FRAME_END

DEFINE_ASM_FUNC(__raw_callee_save___pv_queued_spin_unlock,
		PV_UNLOCK_ASM, .spinlock.text);

#else /* CONFIG_64BIT */

extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");

#endif /* CONFIG_64BIT */
#endif
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
f233f7f1 PZI	2	#ifndef __ASM_QSPINLOCK_PARAVIRT_H
	3	#define __ASM_QSPINLOCK_PARAVIRT_H
	4
c3b03791 PZ	5	#include <asm/ibt.h>
c3b03791 PZ	6
8874a414 AB	7	void __lockfunc __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked);
8874a414 AB	8
d7804530 WL	9	/*
	10	* For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
	11	* registers. For i386, however, only 1 32-bit register needs to be saved
	12	* and restored. So an optimized version of __pv_queued_spin_unlock() is
	13	* hand-coded for 64-bit, but it isn't worthwhile to do it for 32-bit.
	14	*/
	15	#ifdef CONFIG_64BIT
	16
501f7f69	17	__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
d7804530	18	#define __pv_queued_spin_unlock __pv_queued_spin_unlock
d7804530 WL	19
	20	/*
	21	* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
	22	* which combines the registers saving trunk and the body of the following
501f7f69 NK	23	* C code. Note that it puts the code in the .spinlock.text section which
501f7f69 NK	24	* is equivalent to adding __lockfunc in the C code:
d7804530	25	*
501f7f69	26	* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
d7804530	27	* {
91095666	28	* u8 lockval = _Q_LOCKED_VAL;
d7804530	29	*
91095666	30	* if (try_cmpxchg(&lock->locked, &lockval, 0))
d7804530 WL	31	* return;
	32	* pv_queued_spin_unlock_slowpath(lock, lockval);
	33	* }
	34	*
	35	* For x86-64,
	36	* rdi = lock (first argument)
	37	* rsi = lockval (second argument)
	38	* rdx = internal variable (set to 0)
	39	*/
f1a033cc JG	40	#define PV_UNLOCK_ASM \
	41	FRAME_BEGIN \
	42	"push %rdx\n\t" \
532453e7	43	"mov $" __stringify(_Q_LOCKED_VAL) ",%eax\n\t" \
f1a033cc JG	44	"xor %edx,%edx\n\t" \
f1a033cc JG	45	LOCK_PREFIX "cmpxchg %dl,(%rdi)\n\t" \
f1a033cc JG	46	"jne .slowpath\n\t" \
	47	"pop %rdx\n\t" \
	48	FRAME_END \
	49	ASM_RET \
	50	".slowpath:\n\t" \
	51	"push %rsi\n\t" \
	52	"movzbl %al,%esi\n\t" \
	53	"call __raw_callee_save___pv_queued_spin_unlock_slowpath\n\t" \
	54	"pop %rsi\n\t" \
	55	"pop %rdx\n\t" \
16df4ff8	56	FRAME_END
f1a033cc	57
9824b00c JG	58	DEFINE_ASM_FUNC(__raw_callee_save___pv_queued_spin_unlock,
9824b00c JG	59	PV_UNLOCK_ASM, .spinlock.text);
d7804530 WL	60
	61	#else /* CONFIG_64BIT */
	62
501f7f69 NK	63	extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
501f7f69 NK	64	__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
f233f7f1	65
d7804530	66	#endif /* CONFIG_64BIT */
f233f7f1	67	#endif