[linux-2.6-block.git] / arch / sparc / include / asm / barrier_64.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __SPARC64_BARRIER_H
#define __SPARC64_BARRIER_H

/* These are here in an effort to more fully work around Spitfire Errata
 * #51.  Essentially, if a memory barrier occurs soon after a mispredicted
 * branch, the chip can stop executing instructions until a trap occurs.
 * Therefore, if interrupts are disabled, the chip can hang forever.
 *
 * It used to be believed that the memory barrier had to be right in the
 * delay slot, but a case has been traced recently wherein the memory barrier
 * was one instruction after the branch delay slot and the chip still hung.
 * The offending sequence was the following in sym_wakeup_done() of the
 * sym53c8xx_2 driver:
 *
 *	call	sym_ccb_from_dsa, 0
 *	 movge	%icc, 0, %l0
 *	brz,pn	%o0, .LL1303
 *	 mov	%o0, %l2
 *	membar	#LoadLoad
 *
 * The branch has to be mispredicted for the bug to occur.  Therefore, we put
 * the memory barrier explicitly into a "branch always, predicted taken"
 * delay slot to avoid the problem case.
 */
#define membar_safe(type) \
do {	__asm__ __volatile__("ba,pt	%%xcc, 1f\n\t" \
			     " membar	" type "\n" \
			     "1:\n" \
			     : : : "memory"); \
} while (0)

/* The kernel always executes in TSO memory model these days,
 * and furthermore most sparc64 chips implement more stringent
 * memory ordering than required by the specifications.
 */
#define mb()	membar_safe("#StoreLoad")
#define rmb()	__asm__ __volatile__("":::"memory")
#define wmb()	__asm__ __volatile__("":::"memory")

#define __smp_store_release(p, v)						\
do {									\
	compiletime_assert_atomic_type(*p);				\
	barrier();							\
	WRITE_ONCE(*p, v);						\
} while (0)

#define __smp_load_acquire(p)						\
({									\
	typeof(*p) ___p1 = READ_ONCE(*p);				\
	compiletime_assert_atomic_type(*p);				\
	barrier();							\
	___p1;								\
})

#define __smp_mb__before_atomic()	barrier()
#define __smp_mb__after_atomic()	barrier()

#include <asm-generic/barrier.h>

#endif /* !(__SPARC64_BARRIER_H) */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
d550bbd4 DH	2	#ifndef __SPARC64_BARRIER_H
	3	#define __SPARC64_BARRIER_H
	4
	5	/* These are here in an effort to more fully work around Spitfire Errata
	6	* #51. Essentially, if a memory barrier occurs soon after a mispredicted
	7	* branch, the chip can stop executing instructions until a trap occurs.
	8	* Therefore, if interrupts are disabled, the chip can hang forever.
	9	*
	10	* It used to be believed that the memory barrier had to be right in the
	11	* delay slot, but a case has been traced recently wherein the memory barrier
	12	* was one instruction after the branch delay slot and the chip still hung.
	13	* The offending sequence was the following in sym_wakeup_done() of the
	14	* sym53c8xx_2 driver:
	15	*
	16	* call sym_ccb_from_dsa, 0
	17	* movge %icc, 0, %l0
	18	* brz,pn %o0, .LL1303
	19	* mov %o0, %l2
	20	* membar #LoadLoad
	21	*
	22	* The branch has to be mispredicted for the bug to occur. Therefore, we put
	23	* the memory barrier explicitly into a "branch always, predicted taken"
	24	* delay slot to avoid the problem case.
	25	*/
	26	#define membar_safe(type) \
	27	do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \
	28	" membar " type "\n" \
	29	"1:\n" \
	30	: : : "memory"); \
	31	} while (0)
	32
	33	/* The kernel always executes in TSO memory model these days,
	34	* and furthermore most sparc64 chips implement more stringent
	35	* memory ordering than required by the specifications.
	36	*/
	37	#define mb() membar_safe("#StoreLoad")
	38	#define rmb() __asm__ __volatile__("":::"memory")
	39	#define wmb() __asm__ __volatile__("":::"memory")
	40
45d9b859	41	#define __smp_store_release(p, v) \
47933ad4 PZ	42	do { \
	43	compiletime_assert_atomic_type(*p); \
	44	barrier(); \
76695af2	45	WRITE_ONCE(*p, v); \
47933ad4 PZ	46	} while (0)
47933ad4 PZ	47
45d9b859	48	#define __smp_load_acquire(p) \
47933ad4	49	({ \
76695af2	50	typeof(p) ___p1 = READ_ONCE(p); \
47933ad4 PZ	51	compiletime_assert_atomic_type(*p); \
	52	barrier(); \
	53	___p1; \
	54	})
	55
45d9b859 MT	56	#define __smp_mb__before_atomic() barrier()
45d9b859 MT	57	#define __smp_mb__after_atomic() barrier()
56d36489	58
519be043 MT	59	#include <asm-generic/barrier.h>
519be043 MT	60
d550bbd4	61	#endif /* !(__SPARC64_BARRIER_H) */