[linux-2.6-block.git] / kernel / locking / qrwlock.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Queued read/write locks
 *
 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
 *
 * Authors: Waiman Long <waiman.long@hp.com>
 */
#include <linux/smp.h>
#include <linux/bug.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/spinlock.h>

/**
 * queued_read_lock_slowpath - acquire read lock of a queue rwlock
 * @lock: Pointer to queue rwlock structure
 */
void queued_read_lock_slowpath(struct qrwlock *lock)
{
	/*
	 * Readers come here when they cannot get the lock without waiting
	 */
	if (unlikely(in_interrupt())) {
		/*
		 * Readers in interrupt context will get the lock immediately
		 * if the writer is just waiting (not holding the lock yet),
		 * so spin with ACQUIRE semantics until the lock is available
		 * without waiting in the queue.
		 */
		atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
		return;
	}
	atomic_sub(_QR_BIAS, &lock->cnts);

	/*
	 * Put the reader into the wait queue
	 */
	arch_spin_lock(&lock->wait_lock);
	atomic_add(_QR_BIAS, &lock->cnts);

	/*
	 * The ACQUIRE semantics of the following spinning code ensure
	 * that accesses can't leak upwards out of our subsequent critical
	 * section in the case that the lock is currently held for write.
	 */
	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));

	/*
	 * Signal the next one in queue to become queue head
	 */
	arch_spin_unlock(&lock->wait_lock);
}
EXPORT_SYMBOL(queued_read_lock_slowpath);

/**
 * queued_write_lock_slowpath - acquire write lock of a queue rwlock
 * @lock : Pointer to queue rwlock structure
 */
void queued_write_lock_slowpath(struct qrwlock *lock)
{
	int cnts;

	/* Put the writer into the wait queue */
	arch_spin_lock(&lock->wait_lock);

	/* Try to acquire the lock directly if no reader is present */
	if (!(cnts = atomic_read(&lock->cnts)) &&
	    atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
		goto unlock;

	/* Set the waiting flag to notify readers that a writer is pending */
	atomic_or(_QW_WAITING, &lock->cnts);

	/* When no more readers or writers, set the locked flag */
	do {
		cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);
	} while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
unlock:
	arch_spin_unlock(&lock->wait_lock);
}
EXPORT_SYMBOL(queued_write_lock_slowpath);
Commit	Line	Data
c942fddf	1	// SPDX-License-Identifier: GPL-2.0-or-later
70af2f8a	2	/*
c7114b4e	3	* Queued read/write locks
70af2f8a	4	*
70af2f8a WL	5	* (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
	6	*
	7	* Authors: Waiman Long <waiman.long@hp.com>
	8	*/
	9	#include <linux/smp.h>
	10	#include <linux/bug.h>
	11	#include <linux/cpumask.h>
	12	#include <linux/percpu.h>
	13	#include <linux/hardirq.h>
9ab6055f	14	#include <linux/spinlock.h>
70af2f8a	15
70af2f8a	16	/**
f7d71f20	17	* queued_read_lock_slowpath - acquire read lock of a queue rwlock
70af2f8a WL	18	* @lock: Pointer to queue rwlock structure
70af2f8a WL	19	*/
b519b56e	20	void queued_read_lock_slowpath(struct qrwlock *lock)
70af2f8a	21	{
70af2f8a WL	22	/*
	23	* Readers come here when they cannot get the lock without waiting
	24	*/
	25	if (unlikely(in_interrupt())) {
	26	/*
0e06e5be	27	* Readers in interrupt context will get the lock immediately
b519b56e WD	28	* if the writer is just waiting (not holding the lock yet),
	29	* so spin with ACQUIRE semantics until the lock is available
	30	* without waiting in the queue.
70af2f8a	31	*/
d1331661	32	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
70af2f8a WL	33	return;
	34	}
	35	atomic_sub(_QR_BIAS, &lock->cnts);
	36
	37	/*
	38	* Put the reader into the wait queue
	39	*/
6e1e5196	40	arch_spin_lock(&lock->wait_lock);
b519b56e	41	atomic_add(_QR_BIAS, &lock->cnts);
70af2f8a WL	42
70af2f8a WL	43	/*
77e430e3 WD	44	* The ACQUIRE semantics of the following spinning code ensure
	45	* that accesses can't leak upwards out of our subsequent critical
	46	* section in the case that the lock is currently held for write.
70af2f8a	47	*/
d1331661	48	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
70af2f8a WL	49
	50	/*
	51	* Signal the next one in queue to become queue head
	52	*/
6e1e5196	53	arch_spin_unlock(&lock->wait_lock);
70af2f8a	54	}
f7d71f20	55	EXPORT_SYMBOL(queued_read_lock_slowpath);
70af2f8a WL	56
70af2f8a WL	57	/**
f7d71f20	58	* queued_write_lock_slowpath - acquire write lock of a queue rwlock
70af2f8a WL	59	* @lock : Pointer to queue rwlock structure
70af2f8a WL	60	*/
f7d71f20	61	void queued_write_lock_slowpath(struct qrwlock *lock)
70af2f8a	62	{
84a24bf8 AS	63	int cnts;
84a24bf8 AS	64
70af2f8a	65	/* Put the writer into the wait queue */
6e1e5196	66	arch_spin_lock(&lock->wait_lock);
70af2f8a WL	67
70af2f8a WL	68	/* Try to acquire the lock directly if no reader is present */
28ce0e70 WL	69	if (!(cnts = atomic_read(&lock->cnts)) &&
28ce0e70 WL	70	atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
70af2f8a WL	71	goto unlock;
70af2f8a WL	72
d1331661	73	/* Set the waiting flag to notify readers that a writer is pending */
28ce0e70	74	atomic_or(_QW_WAITING, &lock->cnts);
70af2f8a	75
d1331661	76	/* When no more readers or writers, set the locked flag */
b519b56e	77	do {
84a24bf8 AS	78	cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);
84a24bf8 AS	79	} while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
70af2f8a	80	unlock:
6e1e5196	81	arch_spin_unlock(&lock->wait_lock);
70af2f8a	82	}
f7d71f20	83	EXPORT_SYMBOL(queued_write_lock_slowpath);