[linux-2.6-block.git] / arch / x86 / include / asm / atomic64_32.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_ATOMIC64_32_H
#define _ASM_X86_ATOMIC64_32_H

#include <linux/compiler.h>
#include <linux/types.h>
//#include <asm/cmpxchg.h>

/* An 64bit atomic type */

typedef struct {
	s64 __aligned(8) counter;
} atomic64_t;

#define ATOMIC64_INIT(val)	{ (val) }

/*
 * Read an atomic64_t non-atomically.
 *
 * This is intended to be used in cases where a subsequent atomic operation
 * will handle the torn value, and can be used to prime the first iteration
 * of unconditional try_cmpxchg() loops, e.g.:
 *
 * 	s64 val = arch_atomic64_read_nonatomic(v);
 * 	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i);
 *
 * This is NOT safe to use where the value is not always checked by a
 * subsequent atomic operation, such as in conditional try_cmpxchg() loops
 * that can break before the atomic operation, e.g.:
 *
 * 	s64 val = arch_atomic64_read_nonatomic(v);
 * 	do {
 * 		if (condition(val))
 * 			break;
 * 	} while (!arch_atomic64_try_cmpxchg(v, &val, val OP i);
 */
static __always_inline s64 arch_atomic64_read_nonatomic(const atomic64_t *v)
{
	/* See comment in arch_atomic_read(). */
	return __READ_ONCE(v->counter);
}

#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
#ifndef ATOMIC64_EXPORT
#define ATOMIC64_DECL_ONE __ATOMIC64_DECL
#else
#define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \
	ATOMIC64_EXPORT(atomic64_##sym)
#endif

#ifdef CONFIG_X86_CX8
#define __alternative_atomic64(f, g, out, in, clobbers...)		\
	asm volatile("call %c[func]"					\
		     : ALT_OUTPUT_SP(out) \
		     : [func] "i" (atomic64_##g##_cx8)			\
		       COMMA(in)					\
		     : clobbers)

#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
#else
#define __alternative_atomic64(f, g, out, in, clobbers...)		\
	alternative_call(atomic64_##f##_386, atomic64_##g##_cx8,	\
			 X86_FEATURE_CX8, ASM_OUTPUT(out),		\
			 ASM_INPUT(in), clobbers)

#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
	ATOMIC64_DECL_ONE(sym##_386)

ATOMIC64_DECL_ONE(add_386);
ATOMIC64_DECL_ONE(sub_386);
ATOMIC64_DECL_ONE(inc_386);
ATOMIC64_DECL_ONE(dec_386);
#endif

#define alternative_atomic64(f, out, in, clobbers...) \
	__alternative_atomic64(f, f, ASM_OUTPUT(out), ASM_INPUT(in), clobbers)

ATOMIC64_DECL(read);
ATOMIC64_DECL(set);
ATOMIC64_DECL(xchg);
ATOMIC64_DECL(add_return);
ATOMIC64_DECL(sub_return);
ATOMIC64_DECL(inc_return);
ATOMIC64_DECL(dec_return);
ATOMIC64_DECL(dec_if_positive);
ATOMIC64_DECL(inc_not_zero);
ATOMIC64_DECL(add_unless);

#undef ATOMIC64_DECL
#undef ATOMIC64_DECL_ONE
#undef __ATOMIC64_DECL
#undef ATOMIC64_EXPORT

static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
{
	return arch_cmpxchg64(&v->counter, old, new);
}
#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg

static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{
	return arch_try_cmpxchg64(&v->counter, old, new);
}
#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg

static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)
{
	s64 o;
	unsigned high = (unsigned)(n >> 32);
	unsigned low = (unsigned)n;
	alternative_atomic64(xchg,
			     "=&A" (o),
			     ASM_INPUT("S" (v), "b" (low), "c" (high)),
			     "memory");
	return o;
}
#define arch_atomic64_xchg arch_atomic64_xchg

static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
{
	unsigned high = (unsigned)(i >> 32);
	unsigned low = (unsigned)i;
	alternative_atomic64(set,
			     /* no output */,
			     ASM_INPUT("S" (v), "b" (low), "c" (high)),
			     "eax", "edx", "memory");
}

static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
{
	s64 r;
	alternative_atomic64(read, "=&A" (r), "c" (v), "memory");
	return r;
}

static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
{
	alternative_atomic64(add_return,
			     ASM_OUTPUT("+A" (i), "+c" (v)),
			     /* no input */,
			     "memory");
	return i;
}
#define arch_atomic64_add_return arch_atomic64_add_return

static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
{
	alternative_atomic64(sub_return,
			     ASM_OUTPUT("+A" (i), "+c" (v)),
			     /* no input */,
			     "memory");
	return i;
}
#define arch_atomic64_sub_return arch_atomic64_sub_return

static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v)
{
	s64 a;
	alternative_atomic64(inc_return,
			     "=&A" (a),
			     "S" (v),
			     "memory", "ecx");
	return a;
}
#define arch_atomic64_inc_return arch_atomic64_inc_return

static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v)
{
	s64 a;
	alternative_atomic64(dec_return,
			     "=&A" (a),
			     "S" (v),
			     "memory", "ecx");
	return a;
}
#define arch_atomic64_dec_return arch_atomic64_dec_return

static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
{
	__alternative_atomic64(add, add_return,
			       ASM_OUTPUT("+A" (i), "+c" (v)),
			       /* no input */,
			       "memory");
}

static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
{
	__alternative_atomic64(sub, sub_return,
			       ASM_OUTPUT("+A" (i), "+c" (v)),
			       /* no input */,
			       "memory");
}

static __always_inline void arch_atomic64_inc(atomic64_t *v)
{
	__alternative_atomic64(inc, inc_return,
			       /* no output */,
			       "S" (v),
			       "memory", "eax", "ecx", "edx");
}
#define arch_atomic64_inc arch_atomic64_inc

static __always_inline void arch_atomic64_dec(atomic64_t *v)
{
	__alternative_atomic64(dec, dec_return,
			       /* no output */,
			       "S" (v),
			       "memory", "eax", "ecx", "edx");
}
#define arch_atomic64_dec arch_atomic64_dec

static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
{
	unsigned low = (unsigned)u;
	unsigned high = (unsigned)(u >> 32);
	alternative_atomic64(add_unless,
			     ASM_OUTPUT("+A" (a), "+c" (low), "+D" (high)),
			     "S" (v),
			     "memory");
	return (int)a;
}
#define arch_atomic64_add_unless arch_atomic64_add_unless

static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v)
{
	int r;
	alternative_atomic64(inc_not_zero,
			     "=&a" (r),
			     "S" (v),
			     "ecx", "edx", "memory");
	return r;
}
#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero

static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
{
	s64 r;
	alternative_atomic64(dec_if_positive,
			     "=&A" (r),
			     "S" (v),
			     "ecx", "memory");
	return r;
}
#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive

#undef alternative_atomic64
#undef __alternative_atomic64

static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
}

static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));

	return val;
}
#define arch_atomic64_fetch_and arch_atomic64_fetch_and

static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
}

static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));

	return val;
}
#define arch_atomic64_fetch_or arch_atomic64_fetch_or

static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
}

static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));

	return val;
}
#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor

static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
{
	s64 val = arch_atomic64_read_nonatomic(v);

	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i));

	return val;
}
#define arch_atomic64_fetch_add arch_atomic64_fetch_add

#define arch_atomic64_fetch_sub(i, v)	arch_atomic64_fetch_add(-(i), (v))

#endif /* _ASM_X86_ATOMIC64_32_H */
Commit	Line	Data
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _ASM_X86_ATOMIC64_32_H
	3	#define _ASM_X86_ATOMIC64_32_H
	4
	5	#include <linux/compiler.h>
	6	#include <linux/types.h>
	7	//#include <asm/cmpxchg.h>
	8
	9	/* An 64bit atomic type */
	10
	11	typedef struct {
	12	s64 __aligned(8) counter;
	13	} atomic64_t;
	14
	15	#define ATOMIC64_INIT(val) { (val) }
	16
	17	/*
	18	* Read an atomic64_t non-atomically.
	19	*
	20	* This is intended to be used in cases where a subsequent atomic operation
	21	* will handle the torn value, and can be used to prime the first iteration
	22	* of unconditional try_cmpxchg() loops, e.g.:
	23	*
	24	* s64 val = arch_atomic64_read_nonatomic(v);
	25	* do { } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i);
	26	*
	27	* This is NOT safe to use where the value is not always checked by a
	28	* subsequent atomic operation, such as in conditional try_cmpxchg() loops
	29	* that can break before the atomic operation, e.g.:
	30	*
	31	* s64 val = arch_atomic64_read_nonatomic(v);
	32	* do {
	33	* if (condition(val))
	34	* break;
	35	* } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i);
	36	*/
	37	static __always_inline s64 arch_atomic64_read_nonatomic(const atomic64_t *v)
	38	{
	39	/* See comment in arch_atomic_read(). */
	40	return __READ_ONCE(v->counter);
	41	}
	42
	43	#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
	44	#ifndef ATOMIC64_EXPORT
	45	#define ATOMIC64_DECL_ONE __ATOMIC64_DECL
	46	#else
	47	#define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \
	48	ATOMIC64_EXPORT(atomic64_##sym)
	49	#endif
	50
	51	#ifdef CONFIG_X86_CX8
	52	#define __alternative_atomic64(f, g, out, in, clobbers...) \
	53	asm volatile("call %c[func]" \
	54	: ALT_OUTPUT_SP(out) \
	55	: [func] "i" (atomic64_##g##_cx8) \
	56	COMMA(in) \
	57	: clobbers)
	58
	59	#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
	60	#else
	61	#define __alternative_atomic64(f, g, out, in, clobbers...) \
	62	alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \
	63	X86_FEATURE_CX8, ASM_OUTPUT(out), \
	64	ASM_INPUT(in), clobbers)
	65
	66	#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
	67	ATOMIC64_DECL_ONE(sym##_386)
	68
	69	ATOMIC64_DECL_ONE(add_386);
	70	ATOMIC64_DECL_ONE(sub_386);
	71	ATOMIC64_DECL_ONE(inc_386);
	72	ATOMIC64_DECL_ONE(dec_386);
	73	#endif
	74
	75	#define alternative_atomic64(f, out, in, clobbers...) \
	76	__alternative_atomic64(f, f, ASM_OUTPUT(out), ASM_INPUT(in), clobbers)
	77
	78	ATOMIC64_DECL(read);
	79	ATOMIC64_DECL(set);
	80	ATOMIC64_DECL(xchg);
	81	ATOMIC64_DECL(add_return);
	82	ATOMIC64_DECL(sub_return);
	83	ATOMIC64_DECL(inc_return);
	84	ATOMIC64_DECL(dec_return);
	85	ATOMIC64_DECL(dec_if_positive);
	86	ATOMIC64_DECL(inc_not_zero);
	87	ATOMIC64_DECL(add_unless);
	88
	89	#undef ATOMIC64_DECL
	90	#undef ATOMIC64_DECL_ONE
	91	#undef __ATOMIC64_DECL
	92	#undef ATOMIC64_EXPORT
	93
	94	static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
	95	{
	96	return arch_cmpxchg64(&v->counter, old, new);
	97	}
	98	#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
	99
	100	static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t v, s64 old, s64 new)
	101	{
	102	return arch_try_cmpxchg64(&v->counter, old, new);
	103	}
	104	#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
	105
	106	static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)
	107	{
	108	s64 o;
	109	unsigned high = (unsigned)(n >> 32);
	110	unsigned low = (unsigned)n;
	111	alternative_atomic64(xchg,
	112	"=&A" (o),
	113	ASM_INPUT("S" (v), "b" (low), "c" (high)),
	114	"memory");
	115	return o;
	116	}
	117	#define arch_atomic64_xchg arch_atomic64_xchg
	118
	119	static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
	120	{
	121	unsigned high = (unsigned)(i >> 32);
	122	unsigned low = (unsigned)i;
	123	alternative_atomic64(set,
	124	/* no output */,
	125	ASM_INPUT("S" (v), "b" (low), "c" (high)),
	126	"eax", "edx", "memory");
	127	}
	128
	129	static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
	130	{
	131	s64 r;
	132	alternative_atomic64(read, "=&A" (r), "c" (v), "memory");
	133	return r;
	134	}
	135
	136	static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
	137	{
	138	alternative_atomic64(add_return,
	139	ASM_OUTPUT("+A" (i), "+c" (v)),
	140	/* no input */,
	141	"memory");
	142	return i;
	143	}
	144	#define arch_atomic64_add_return arch_atomic64_add_return
	145
	146	static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
	147	{
	148	alternative_atomic64(sub_return,
	149	ASM_OUTPUT("+A" (i), "+c" (v)),
	150	/* no input */,
	151	"memory");
	152	return i;
	153	}
	154	#define arch_atomic64_sub_return arch_atomic64_sub_return
	155
	156	static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v)
	157	{
	158	s64 a;
	159	alternative_atomic64(inc_return,
	160	"=&A" (a),
	161	"S" (v),
	162	"memory", "ecx");
	163	return a;
	164	}
	165	#define arch_atomic64_inc_return arch_atomic64_inc_return
	166
	167	static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v)
	168	{
	169	s64 a;
	170	alternative_atomic64(dec_return,
	171	"=&A" (a),
	172	"S" (v),
	173	"memory", "ecx");
	174	return a;
	175	}
	176	#define arch_atomic64_dec_return arch_atomic64_dec_return
	177
	178	static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
	179	{
	180	__alternative_atomic64(add, add_return,
	181	ASM_OUTPUT("+A" (i), "+c" (v)),
	182	/* no input */,
	183	"memory");
	184	}
	185
	186	static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
	187	{
	188	__alternative_atomic64(sub, sub_return,
	189	ASM_OUTPUT("+A" (i), "+c" (v)),
	190	/* no input */,
	191	"memory");
	192	}
	193
	194	static __always_inline void arch_atomic64_inc(atomic64_t *v)
	195	{
	196	__alternative_atomic64(inc, inc_return,
	197	/* no output */,
	198	"S" (v),
	199	"memory", "eax", "ecx", "edx");
	200	}
	201	#define arch_atomic64_inc arch_atomic64_inc
	202
	203	static __always_inline void arch_atomic64_dec(atomic64_t *v)
	204	{
	205	__alternative_atomic64(dec, dec_return,
	206	/* no output */,
	207	"S" (v),
	208	"memory", "eax", "ecx", "edx");
	209	}
	210	#define arch_atomic64_dec arch_atomic64_dec
	211
	212	static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
	213	{
	214	unsigned low = (unsigned)u;
	215	unsigned high = (unsigned)(u >> 32);
	216	alternative_atomic64(add_unless,
	217	ASM_OUTPUT("+A" (a), "+c" (low), "+D" (high)),
	218	"S" (v),
	219	"memory");
	220	return (int)a;
	221	}
	222	#define arch_atomic64_add_unless arch_atomic64_add_unless
	223
	224	static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v)
	225	{
	226	int r;
	227	alternative_atomic64(inc_not_zero,
	228	"=&a" (r),
	229	"S" (v),
	230	"ecx", "edx", "memory");
	231	return r;
	232	}
	233	#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
	234
	235	static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
	236	{
	237	s64 r;
	238	alternative_atomic64(dec_if_positive,
	239	"=&A" (r),
	240	"S" (v),
	241	"ecx", "memory");
	242	return r;
	243	}
	244	#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
	245
	246	#undef alternative_atomic64
	247	#undef __alternative_atomic64
	248
	249	static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)
	250	{
	251	s64 val = arch_atomic64_read_nonatomic(v);
	252
	253	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
	254	}
	255
	256	static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
	257	{
	258	s64 val = arch_atomic64_read_nonatomic(v);
	259
	260	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
	261
	262	return val;
	263	}
	264	#define arch_atomic64_fetch_and arch_atomic64_fetch_and
	265
	266	static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)
	267	{
	268	s64 val = arch_atomic64_read_nonatomic(v);
	269
	270	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val \| i));
	271	}
	272
	273	static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
	274	{
	275	s64 val = arch_atomic64_read_nonatomic(v);
	276
	277	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val \| i));
	278
	279	return val;
	280	}
	281	#define arch_atomic64_fetch_or arch_atomic64_fetch_or
	282
	283	static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)
	284	{
	285	s64 val = arch_atomic64_read_nonatomic(v);
	286
	287	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
	288	}
	289
	290	static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
	291	{
	292	s64 val = arch_atomic64_read_nonatomic(v);
	293
	294	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
	295
	296	return val;
	297	}
	298	#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
	299
	300	static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
	301	{
	302	s64 val = arch_atomic64_read_nonatomic(v);
	303
	304	do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i));
	305
	306	return val;
	307	}
	308	#define arch_atomic64_fetch_add arch_atomic64_fetch_add
	309
	310	#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v))
	311
	312	#endif /* _ASM_X86_ATOMIC64_32_H */