[linux-2.6-block.git] / lib / atomic64.c

/*
 * Generic implementation of 64-bit atomics using spinlocks,
 * useful on processors that don't have 64-bit atomic instructions.
 *
 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
#include <linux/types.h>
#include <linux/cache.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/atomic.h>

/*
 * We use a hashed array of spinlocks to provide exclusive access
 * to each atomic64_t variable.  Since this is expected to used on
 * systems with small numbers of CPUs (<= 4 or so), we use a
 * relatively small array of 16 spinlocks to avoid wasting too much
 * memory on the spinlock array.
 */
#define NR_LOCKS	16

/*
 * Ensure each lock is in a separate cacheline.
 */
static union {
	raw_spinlock_t lock;
	char pad[L1_CACHE_BYTES];
} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
	[0 ... (NR_LOCKS - 1)] = {
		.lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
	},
};

static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
{
	unsigned long addr = (unsigned long) v;

	addr >>= L1_CACHE_SHIFT;
	addr ^= (addr >> 8) ^ (addr >> 16);
	return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
}

long long atomic64_read(const atomic64_t *v)
{
	unsigned long flags;
	raw_spinlock_t *lock = lock_addr(v);
	long long val;

	raw_spin_lock_irqsave(lock, flags);
	val = v->counter;
	raw_spin_unlock_irqrestore(lock, flags);
	return val;
}
EXPORT_SYMBOL(atomic64_read);

void atomic64_set(atomic64_t *v, long long i)
{
	unsigned long flags;
	raw_spinlock_t *lock = lock_addr(v);

	raw_spin_lock_irqsave(lock, flags);
	v->counter = i;
	raw_spin_unlock_irqrestore(lock, flags);
}
EXPORT_SYMBOL(atomic64_set);

#define ATOMIC64_OP(op, c_op)						\
void atomic64_##op(long long a, atomic64_t *v)				\
{									\
	unsigned long flags;						\
	raw_spinlock_t *lock = lock_addr(v);				\
									\
	raw_spin_lock_irqsave(lock, flags);				\
	v->counter c_op a;						\
	raw_spin_unlock_irqrestore(lock, flags);			\
}									\
EXPORT_SYMBOL(atomic64_##op);

#define ATOMIC64_OP_RETURN(op, c_op)					\
long long atomic64_##op##_return(long long a, atomic64_t *v)		\
{									\
	unsigned long flags;						\
	raw_spinlock_t *lock = lock_addr(v);				\
	long long val;							\
									\
	raw_spin_lock_irqsave(lock, flags);				\
	val = (v->counter c_op a);					\
	raw_spin_unlock_irqrestore(lock, flags);			\
	return val;							\
}									\
EXPORT_SYMBOL(atomic64_##op##_return);

#define ATOMIC64_OPS(op, c_op)						\
	ATOMIC64_OP(op, c_op)						\
	ATOMIC64_OP_RETURN(op, c_op)

ATOMIC64_OPS(add, +=)
ATOMIC64_OPS(sub, -=)
ATOMIC64_OP(and, &=)
ATOMIC64_OP(or, |=)
ATOMIC64_OP(xor, ^=)

#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP

long long atomic64_dec_if_positive(atomic64_t *v)
{
	unsigned long flags;
	raw_spinlock_t *lock = lock_addr(v);
	long long val;

	raw_spin_lock_irqsave(lock, flags);
	val = v->counter - 1;
	if (val >= 0)
		v->counter = val;
	raw_spin_unlock_irqrestore(lock, flags);
	return val;
}
EXPORT_SYMBOL(atomic64_dec_if_positive);

long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
{
	unsigned long flags;
	raw_spinlock_t *lock = lock_addr(v);
	long long val;

	raw_spin_lock_irqsave(lock, flags);
	val = v->counter;
	if (val == o)
		v->counter = n;
	raw_spin_unlock_irqrestore(lock, flags);
	return val;
}
EXPORT_SYMBOL(atomic64_cmpxchg);

long long atomic64_xchg(atomic64_t *v, long long new)
{
	unsigned long flags;
	raw_spinlock_t *lock = lock_addr(v);
	long long val;

	raw_spin_lock_irqsave(lock, flags);
	val = v->counter;
	v->counter = new;
	raw_spin_unlock_irqrestore(lock, flags);
	return val;
}
EXPORT_SYMBOL(atomic64_xchg);

int atomic64_add_unless(atomic64_t *v, long long a, long long u)
{
	unsigned long flags;
	raw_spinlock_t *lock = lock_addr(v);
	int ret = 0;

	raw_spin_lock_irqsave(lock, flags);
	if (v->counter != u) {
		v->counter += a;
		ret = 1;
	}
	raw_spin_unlock_irqrestore(lock, flags);
	return ret;
}
EXPORT_SYMBOL(atomic64_add_unless);
Commit	Line	Data
09d4e0ed PM	1	/*
	2	* Generic implementation of 64-bit atomics using spinlocks,
	3	* useful on processors that don't have 64-bit atomic instructions.
	4	*
	5	* Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
	6	*
	7	* This program is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU General Public License
	9	* as published by the Free Software Foundation; either version
	10	* 2 of the License, or (at your option) any later version.
	11	*/
	12	#include <linux/types.h>
	13	#include <linux/cache.h>
	14	#include <linux/spinlock.h>
	15	#include <linux/init.h>
8bc3bcc9	16	#include <linux/export.h>
60063497	17	#include <linux/atomic.h>
09d4e0ed PM	18
	19	/*
	20	* We use a hashed array of spinlocks to provide exclusive access
	21	* to each atomic64_t variable. Since this is expected to used on
	22	* systems with small numbers of CPUs (<= 4 or so), we use a
	23	* relatively small array of 16 spinlocks to avoid wasting too much
	24	* memory on the spinlock array.
	25	*/
	26	#define NR_LOCKS 16
	27
	28	/*
	29	* Ensure each lock is in a separate cacheline.
	30	*/
	31	static union {
f59ca058	32	raw_spinlock_t lock;
09d4e0ed	33	char pad[L1_CACHE_BYTES];
fcc16882 SB	34	} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
	35	[0 ... (NR_LOCKS - 1)] = {
	36	.lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
	37	},
	38	};
09d4e0ed	39
cb475de3	40	static inline raw_spinlock_t lock_addr(const atomic64_t v)
09d4e0ed PM	41	{
	42	unsigned long addr = (unsigned long) v;
	43
	44	addr >>= L1_CACHE_SHIFT;
	45	addr ^= (addr >> 8) ^ (addr >> 16);
	46	return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
	47	}
	48
	49	long long atomic64_read(const atomic64_t *v)
	50	{
	51	unsigned long flags;
cb475de3	52	raw_spinlock_t *lock = lock_addr(v);
09d4e0ed PM	53	long long val;
09d4e0ed PM	54
f59ca058	55	raw_spin_lock_irqsave(lock, flags);
09d4e0ed	56	val = v->counter;
f59ca058	57	raw_spin_unlock_irqrestore(lock, flags);
09d4e0ed PM	58	return val;
09d4e0ed PM	59	}
3fc7b4b2	60	EXPORT_SYMBOL(atomic64_read);
09d4e0ed PM	61
	62	void atomic64_set(atomic64_t *v, long long i)
	63	{
	64	unsigned long flags;
cb475de3	65	raw_spinlock_t *lock = lock_addr(v);
09d4e0ed	66
f59ca058	67	raw_spin_lock_irqsave(lock, flags);
09d4e0ed	68	v->counter = i;
f59ca058	69	raw_spin_unlock_irqrestore(lock, flags);
09d4e0ed	70	}
3fc7b4b2	71	EXPORT_SYMBOL(atomic64_set);
09d4e0ed	72
560cb12a PZ	73	#define ATOMIC64_OP(op, c_op) \
	74	void atomic64_##op(long long a, atomic64_t *v) \
	75	{ \
	76	unsigned long flags; \
	77	raw_spinlock_t *lock = lock_addr(v); \
	78	\
	79	raw_spin_lock_irqsave(lock, flags); \
	80	v->counter c_op a; \
	81	raw_spin_unlock_irqrestore(lock, flags); \
	82	} \
	83	EXPORT_SYMBOL(atomic64_##op);
	84
	85	#define ATOMIC64_OP_RETURN(op, c_op) \
	86	long long atomic64_##op##_return(long long a, atomic64_t *v) \
	87	{ \
	88	unsigned long flags; \
	89	raw_spinlock_t *lock = lock_addr(v); \
	90	long long val; \
	91	\
	92	raw_spin_lock_irqsave(lock, flags); \
	93	val = (v->counter c_op a); \
	94	raw_spin_unlock_irqrestore(lock, flags); \
	95	return val; \
	96	} \
	97	EXPORT_SYMBOL(atomic64_##op##_return);
	98
	99	#define ATOMIC64_OPS(op, c_op) \
	100	ATOMIC64_OP(op, c_op) \
	101	ATOMIC64_OP_RETURN(op, c_op)
	102
	103	ATOMIC64_OPS(add, +=)
	104	ATOMIC64_OPS(sub, -=)
e6942b7d PZ	105	ATOMIC64_OP(and, &=)
	106	ATOMIC64_OP(or, \|=)
	107	ATOMIC64_OP(xor, ^=)
560cb12a PZ	108
	109	#undef ATOMIC64_OPS
	110	#undef ATOMIC64_OP_RETURN
	111	#undef ATOMIC64_OP
09d4e0ed PM	112
	113	long long atomic64_dec_if_positive(atomic64_t *v)
	114	{
	115	unsigned long flags;
cb475de3	116	raw_spinlock_t *lock = lock_addr(v);
09d4e0ed PM	117	long long val;
09d4e0ed PM	118
f59ca058	119	raw_spin_lock_irqsave(lock, flags);
09d4e0ed PM	120	val = v->counter - 1;
	121	if (val >= 0)
	122	v->counter = val;
f59ca058	123	raw_spin_unlock_irqrestore(lock, flags);
09d4e0ed PM	124	return val;
09d4e0ed PM	125	}
3fc7b4b2	126	EXPORT_SYMBOL(atomic64_dec_if_positive);
09d4e0ed PM	127
	128	long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
	129	{
	130	unsigned long flags;
cb475de3	131	raw_spinlock_t *lock = lock_addr(v);
09d4e0ed PM	132	long long val;
09d4e0ed PM	133
f59ca058	134	raw_spin_lock_irqsave(lock, flags);
09d4e0ed PM	135	val = v->counter;
	136	if (val == o)
	137	v->counter = n;
f59ca058	138	raw_spin_unlock_irqrestore(lock, flags);
09d4e0ed PM	139	return val;
09d4e0ed PM	140	}
3fc7b4b2	141	EXPORT_SYMBOL(atomic64_cmpxchg);
09d4e0ed PM	142
	143	long long atomic64_xchg(atomic64_t *v, long long new)
	144	{
	145	unsigned long flags;
cb475de3	146	raw_spinlock_t *lock = lock_addr(v);
09d4e0ed PM	147	long long val;
09d4e0ed PM	148
f59ca058	149	raw_spin_lock_irqsave(lock, flags);
09d4e0ed PM	150	val = v->counter;
09d4e0ed PM	151	v->counter = new;
f59ca058	152	raw_spin_unlock_irqrestore(lock, flags);
09d4e0ed PM	153	return val;
09d4e0ed PM	154	}
3fc7b4b2	155	EXPORT_SYMBOL(atomic64_xchg);
09d4e0ed PM	156
	157	int atomic64_add_unless(atomic64_t *v, long long a, long long u)
	158	{
	159	unsigned long flags;
cb475de3	160	raw_spinlock_t *lock = lock_addr(v);
97577896	161	int ret = 0;
09d4e0ed	162
f59ca058	163	raw_spin_lock_irqsave(lock, flags);
09d4e0ed PM	164	if (v->counter != u) {
09d4e0ed PM	165	v->counter += a;
97577896	166	ret = 1;
09d4e0ed	167	}
f59ca058	168	raw_spin_unlock_irqrestore(lock, flags);
09d4e0ed PM	169	return ret;
09d4e0ed PM	170	}
3fc7b4b2	171	EXPORT_SYMBOL(atomic64_add_unless);