[linux-2.6-block.git] / include / linux / mmap_lock.h

#ifndef _LINUX_MMAP_LOCK_H
#define _LINUX_MMAP_LOCK_H

#include <linux/lockdep.h>
#include <linux/mm_types.h>
#include <linux/mmdebug.h>
#include <linux/rwsem.h>
#include <linux/tracepoint-defs.h>
#include <linux/types.h>

#define MMAP_LOCK_INITIALIZER(name) \
	.mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),

DECLARE_TRACEPOINT(mmap_lock_start_locking);
DECLARE_TRACEPOINT(mmap_lock_acquire_returned);
DECLARE_TRACEPOINT(mmap_lock_released);

#ifdef CONFIG_TRACING

void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write);
void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
					   bool success);
void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write);

static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,
						   bool write)
{
	if (tracepoint_enabled(mmap_lock_start_locking))
		__mmap_lock_do_trace_start_locking(mm, write);
}

static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,
						      bool write, bool success)
{
	if (tracepoint_enabled(mmap_lock_acquire_returned))
		__mmap_lock_do_trace_acquire_returned(mm, write, success);
}

static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
{
	if (tracepoint_enabled(mmap_lock_released))
		__mmap_lock_do_trace_released(mm, write);
}

#else /* !CONFIG_TRACING */

static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,
						   bool write)
{
}

static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,
						      bool write, bool success)
{
}

static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
{
}

#endif /* CONFIG_TRACING */

static inline void mmap_assert_locked(const struct mm_struct *mm)
{
	rwsem_assert_held(&mm->mmap_lock);
}

static inline void mmap_assert_write_locked(const struct mm_struct *mm)
{
	rwsem_assert_held_write(&mm->mmap_lock);
}

#ifdef CONFIG_PER_VMA_LOCK
/*
 * Drop all currently-held per-VMA locks.
 * This is called from the mmap_lock implementation directly before releasing
 * a write-locked mmap_lock (or downgrading it to read-locked).
 * This should normally NOT be called manually from other places.
 * If you want to call this manually anyway, keep in mind that this will release
 * *all* VMA write locks, including ones from further up the stack.
 */
static inline void vma_end_write_all(struct mm_struct *mm)
{
	mmap_assert_write_locked(mm);
	/*
	 * Nobody can concurrently modify mm->mm_lock_seq due to exclusive
	 * mmap_lock being held.
	 * We need RELEASE semantics here to ensure that preceding stores into
	 * the VMA take effect before we unlock it with this store.
	 * Pairs with ACQUIRE semantics in vma_start_read().
	 */
	smp_store_release(&mm->mm_lock_seq, mm->mm_lock_seq + 1);
}
#else
static inline void vma_end_write_all(struct mm_struct *mm) {}
#endif

static inline void mmap_init_lock(struct mm_struct *mm)
{
	init_rwsem(&mm->mmap_lock);
}

static inline void mmap_write_lock(struct mm_struct *mm)
{
	__mmap_lock_trace_start_locking(mm, true);
	down_write(&mm->mmap_lock);
	__mmap_lock_trace_acquire_returned(mm, true, true);
}

static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
{
	__mmap_lock_trace_start_locking(mm, true);
	down_write_nested(&mm->mmap_lock, subclass);
	__mmap_lock_trace_acquire_returned(mm, true, true);
}

static inline int mmap_write_lock_killable(struct mm_struct *mm)
{
	int ret;

	__mmap_lock_trace_start_locking(mm, true);
	ret = down_write_killable(&mm->mmap_lock);
	__mmap_lock_trace_acquire_returned(mm, true, ret == 0);
	return ret;
}

static inline void mmap_write_unlock(struct mm_struct *mm)
{
	__mmap_lock_trace_released(mm, true);
	vma_end_write_all(mm);
	up_write(&mm->mmap_lock);
}

static inline void mmap_write_downgrade(struct mm_struct *mm)
{
	__mmap_lock_trace_acquire_returned(mm, false, true);
	vma_end_write_all(mm);
	downgrade_write(&mm->mmap_lock);
}

static inline void mmap_read_lock(struct mm_struct *mm)
{
	__mmap_lock_trace_start_locking(mm, false);
	down_read(&mm->mmap_lock);
	__mmap_lock_trace_acquire_returned(mm, false, true);
}

static inline int mmap_read_lock_killable(struct mm_struct *mm)
{
	int ret;

	__mmap_lock_trace_start_locking(mm, false);
	ret = down_read_killable(&mm->mmap_lock);
	__mmap_lock_trace_acquire_returned(mm, false, ret == 0);
	return ret;
}

static inline bool mmap_read_trylock(struct mm_struct *mm)
{
	bool ret;

	__mmap_lock_trace_start_locking(mm, false);
	ret = down_read_trylock(&mm->mmap_lock) != 0;
	__mmap_lock_trace_acquire_returned(mm, false, ret);
	return ret;
}

static inline void mmap_read_unlock(struct mm_struct *mm)
{
	__mmap_lock_trace_released(mm, false);
	up_read(&mm->mmap_lock);
}

static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
{
	__mmap_lock_trace_released(mm, false);
	up_read_non_owner(&mm->mmap_lock);
}

static inline int mmap_lock_is_contended(struct mm_struct *mm)
{
	return rwsem_is_contended(&mm->mmap_lock);
}

#endif /* _LINUX_MMAP_LOCK_H */
Commit	Line	Data
9740ca4e ML	1	#ifndef _LINUX_MMAP_LOCK_H
	2	#define _LINUX_MMAP_LOCK_H
	3
2b5067a8 AR	4	#include <linux/lockdep.h>
2b5067a8 AR	5	#include <linux/mm_types.h>
42fc5414	6	#include <linux/mmdebug.h>
2b5067a8 AR	7	#include <linux/rwsem.h>
	8	#include <linux/tracepoint-defs.h>
	9	#include <linux/types.h>
42fc5414	10
14c3656b	11	#define MMAP_LOCK_INITIALIZER(name) \
da1c55f1	12	.mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),
14c3656b	13
2b5067a8 AR	14	DECLARE_TRACEPOINT(mmap_lock_start_locking);
	15	DECLARE_TRACEPOINT(mmap_lock_acquire_returned);
	16	DECLARE_TRACEPOINT(mmap_lock_released);
	17
	18	#ifdef CONFIG_TRACING
	19
	20	void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write);
	21	void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
	22	bool success);
	23	void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write);
	24
	25	static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,
	26	bool write)
	27	{
	28	if (tracepoint_enabled(mmap_lock_start_locking))
	29	__mmap_lock_do_trace_start_locking(mm, write);
	30	}
	31
	32	static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,
	33	bool write, bool success)
	34	{
	35	if (tracepoint_enabled(mmap_lock_acquire_returned))
	36	__mmap_lock_do_trace_acquire_returned(mm, write, success);
	37	}
	38
	39	static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
	40	{
	41	if (tracepoint_enabled(mmap_lock_released))
	42	__mmap_lock_do_trace_released(mm, write);
	43	}
	44
	45	#else /* !CONFIG_TRACING */
	46
	47	static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,
	48	bool write)
	49	{
	50	}
	51
	52	static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,
	53	bool write, bool success)
	54	{
	55	}
	56
	57	static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
	58	{
	59	}
	60
	61	#endif /* CONFIG_TRACING */
	62
ba168b52	63	static inline void mmap_assert_locked(const struct mm_struct *mm)
438b6e12	64	{
ba168b52	65	rwsem_assert_held(&mm->mmap_lock);
438b6e12 SB	66	}
438b6e12 SB	67
ba168b52	68	static inline void mmap_assert_write_locked(const struct mm_struct *mm)
438b6e12	69	{
ba168b52	70	rwsem_assert_held_write(&mm->mmap_lock);
438b6e12 SB	71	}
438b6e12 SB	72
5e31275c	73	#ifdef CONFIG_PER_VMA_LOCK
90717566 JH	74	/*
	75	* Drop all currently-held per-VMA locks.
	76	* This is called from the mmap_lock implementation directly before releasing
	77	* a write-locked mmap_lock (or downgrading it to read-locked).
	78	* This should normally NOT be called manually from other places.
	79	* If you want to call this manually anyway, keep in mind that this will release
	80	* all VMA write locks, including ones from further up the stack.
	81	*/
5e31275c SB	82	static inline void vma_end_write_all(struct mm_struct *mm)
	83	{
	84	mmap_assert_write_locked(mm);
b1f02b95 JH	85	/*
	86	* Nobody can concurrently modify mm->mm_lock_seq due to exclusive
	87	* mmap_lock being held.
	88	* We need RELEASE semantics here to ensure that preceding stores into
	89	* the VMA take effect before we unlock it with this store.
	90	* Pairs with ACQUIRE semantics in vma_start_read().
	91	*/
	92	smp_store_release(&mm->mm_lock_seq, mm->mm_lock_seq + 1);
5e31275c SB	93	}
	94	#else
	95	static inline void vma_end_write_all(struct mm_struct *mm) {}
	96	#endif
	97
9740ca4e ML	98	static inline void mmap_init_lock(struct mm_struct *mm)
9740ca4e ML	99	{
da1c55f1	100	init_rwsem(&mm->mmap_lock);
9740ca4e ML	101	}
	102
	103	static inline void mmap_write_lock(struct mm_struct *mm)
	104	{
2b5067a8	105	__mmap_lock_trace_start_locking(mm, true);
da1c55f1	106	down_write(&mm->mmap_lock);
2b5067a8	107	__mmap_lock_trace_acquire_returned(mm, true, true);
9740ca4e ML	108	}
9740ca4e ML	109
aaa2cc56 ML	110	static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
aaa2cc56 ML	111	{
2b5067a8	112	__mmap_lock_trace_start_locking(mm, true);
da1c55f1	113	down_write_nested(&mm->mmap_lock, subclass);
2b5067a8	114	__mmap_lock_trace_acquire_returned(mm, true, true);
aaa2cc56 ML	115	}
aaa2cc56 ML	116
9740ca4e ML	117	static inline int mmap_write_lock_killable(struct mm_struct *mm)
9740ca4e ML	118	{
2b5067a8 AR	119	int ret;
	120
	121	__mmap_lock_trace_start_locking(mm, true);
	122	ret = down_write_killable(&mm->mmap_lock);
	123	__mmap_lock_trace_acquire_returned(mm, true, ret == 0);
	124	return ret;
9740ca4e ML	125	}
9740ca4e ML	126
9740ca4e ML	127	static inline void mmap_write_unlock(struct mm_struct *mm)
9740ca4e ML	128	{
2b5067a8	129	__mmap_lock_trace_released(mm, true);
5e31275c	130	vma_end_write_all(mm);
10994316	131	up_write(&mm->mmap_lock);
9740ca4e ML	132	}
	133
	134	static inline void mmap_write_downgrade(struct mm_struct *mm)
	135	{
2b5067a8	136	__mmap_lock_trace_acquire_returned(mm, false, true);
5e31275c	137	vma_end_write_all(mm);
10994316	138	downgrade_write(&mm->mmap_lock);
9740ca4e ML	139	}
	140
	141	static inline void mmap_read_lock(struct mm_struct *mm)
	142	{
2b5067a8	143	__mmap_lock_trace_start_locking(mm, false);
da1c55f1	144	down_read(&mm->mmap_lock);
2b5067a8	145	__mmap_lock_trace_acquire_returned(mm, false, true);
9740ca4e ML	146	}
	147
	148	static inline int mmap_read_lock_killable(struct mm_struct *mm)
	149	{
2b5067a8 AR	150	int ret;
	151
	152	__mmap_lock_trace_start_locking(mm, false);
	153	ret = down_read_killable(&mm->mmap_lock);
	154	__mmap_lock_trace_acquire_returned(mm, false, ret == 0);
	155	return ret;
9740ca4e ML	156	}
	157
	158	static inline bool mmap_read_trylock(struct mm_struct *mm)
	159	{
2b5067a8 AR	160	bool ret;
	161
	162	__mmap_lock_trace_start_locking(mm, false);
	163	ret = down_read_trylock(&mm->mmap_lock) != 0;
	164	__mmap_lock_trace_acquire_returned(mm, false, ret);
	165	return ret;
9740ca4e ML	166	}
	167
	168	static inline void mmap_read_unlock(struct mm_struct *mm)
	169	{
2b5067a8	170	__mmap_lock_trace_released(mm, false);
10994316	171	up_read(&mm->mmap_lock);
9740ca4e ML	172	}
9740ca4e ML	173
0cc55a02 ML	174	static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
0cc55a02 ML	175	{
2b5067a8	176	__mmap_lock_trace_released(mm, false);
10994316	177	up_read_non_owner(&mm->mmap_lock);
0cc55a02 ML	178	}
0cc55a02 ML	179
07e5bfe6 CC	180	static inline int mmap_lock_is_contended(struct mm_struct *mm)
	181	{
	182	return rwsem_is_contended(&mm->mmap_lock);
	183	}
	184
9740ca4e	185	#endif /* _LINUX_MMAP_LOCK_H */