[linux-block.git] / fs / dlm / ast.c

// SPDX-License-Identifier: GPL-2.0-only
/******************************************************************************
*******************************************************************************
**
**  Copyright (C) Sistina Software, Inc.  1997-2003  All rights reserved.
**  Copyright (C) 2004-2010 Red Hat, Inc.  All rights reserved.
**
**
*******************************************************************************
******************************************************************************/

#include <trace/events/dlm.h>

#include "dlm_internal.h"
#include "memory.h"
#include "lock.h"
#include "user.h"
#include "ast.h"

void dlm_release_callback(struct kref *ref)
{
	struct dlm_callback *cb = container_of(ref, struct dlm_callback, ref);

	dlm_free_cb(cb);
}

void dlm_callback_set_last_ptr(struct dlm_callback **from,
			       struct dlm_callback *to)
{
	if (*from)
		kref_put(&(*from)->ref, dlm_release_callback);

	if (to)
		kref_get(&to->ref);

	*from = to;
}

void dlm_purge_lkb_callbacks(struct dlm_lkb *lkb)
{
	struct dlm_callback *cb, *safe;

	list_for_each_entry_safe(cb, safe, &lkb->lkb_callbacks, list) {
		list_del(&cb->list);
		kref_put(&cb->ref, dlm_release_callback);
	}

	clear_bit(DLM_IFL_CB_PENDING_BIT, &lkb->lkb_iflags);

	/* invalidate */
	dlm_callback_set_last_ptr(&lkb->lkb_last_cast, NULL);
	dlm_callback_set_last_ptr(&lkb->lkb_last_cb, NULL);
	lkb->lkb_last_bast_mode = -1;
}

int dlm_enqueue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
			     int status, uint32_t sbflags)
{
	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
	int rv = DLM_ENQUEUE_CALLBACK_SUCCESS;
	struct dlm_callback *cb;
	int prev_mode;

	if (flags & DLM_CB_BAST) {
		/* if cb is a bast, it should be skipped if the blocking mode is
		 * compatible with the last granted mode
		 */
		if (lkb->lkb_last_cast) {
			if (dlm_modes_compat(mode, lkb->lkb_last_cast->mode)) {
				log_debug(ls, "skip %x bast mode %d for cast mode %d",
					  lkb->lkb_id, mode,
					  lkb->lkb_last_cast->mode);
				goto out;
			}
		}

		/*
		 * Suppress some redundant basts here, do more on removal.
		 * Don't even add a bast if the callback just before it
		 * is a bast for the same mode or a more restrictive mode.
		 * (the addional > PR check is needed for PR/CW inversion)
		 */
		if (lkb->lkb_last_cb && lkb->lkb_last_cb->flags & DLM_CB_BAST) {
			prev_mode = lkb->lkb_last_cb->mode;

			if ((prev_mode == mode) ||
			    (prev_mode > mode && prev_mode > DLM_LOCK_PR)) {
				log_debug(ls, "skip %x add bast mode %d for bast mode %d",
					  lkb->lkb_id, mode, prev_mode);
				goto out;
			}
		}
	}

	cb = dlm_allocate_cb();
	if (!cb) {
		rv = DLM_ENQUEUE_CALLBACK_FAILURE;
		goto out;
	}

	cb->flags = flags;
	cb->mode = mode;
	cb->sb_status = status;
	cb->sb_flags = (sbflags & 0x000000FF);
	kref_init(&cb->ref);
	if (!test_and_set_bit(DLM_IFL_CB_PENDING_BIT, &lkb->lkb_iflags))
		rv = DLM_ENQUEUE_CALLBACK_NEED_SCHED;

	list_add_tail(&cb->list, &lkb->lkb_callbacks);

	if (flags & DLM_CB_CAST)
		dlm_callback_set_last_ptr(&lkb->lkb_last_cast, cb);

	dlm_callback_set_last_ptr(&lkb->lkb_last_cb, cb);

 out:
	return rv;
}

int dlm_dequeue_lkb_callback(struct dlm_lkb *lkb, struct dlm_callback **cb)
{
	/* oldest undelivered cb is callbacks first entry */
	*cb = list_first_entry_or_null(&lkb->lkb_callbacks,
				       struct dlm_callback, list);
	if (!*cb)
		return DLM_DEQUEUE_CALLBACK_EMPTY;

	/* remove it from callbacks so shift others down */
	list_del(&(*cb)->list);
	if (list_empty(&lkb->lkb_callbacks))
		return DLM_DEQUEUE_CALLBACK_LAST;

	return DLM_DEQUEUE_CALLBACK_SUCCESS;
}

void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,
		uint32_t sbflags)
{
	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
	int rv;

	if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) {
		dlm_user_add_ast(lkb, flags, mode, status, sbflags);
		return;
	}

	spin_lock(&lkb->lkb_cb_lock);
	rv = dlm_enqueue_lkb_callback(lkb, flags, mode, status, sbflags);
	switch (rv) {
	case DLM_ENQUEUE_CALLBACK_NEED_SCHED:
		kref_get(&lkb->lkb_ref);

		spin_lock(&ls->ls_cb_lock);
		if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) {
			list_add(&lkb->lkb_cb_list, &ls->ls_cb_delay);
		} else {
			queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work);
		}
		spin_unlock(&ls->ls_cb_lock);
		break;
	case DLM_ENQUEUE_CALLBACK_FAILURE:
		WARN_ON_ONCE(1);
		break;
	case DLM_ENQUEUE_CALLBACK_SUCCESS:
		break;
	default:
		WARN_ON_ONCE(1);
		break;
	}
	spin_unlock(&lkb->lkb_cb_lock);
}

void dlm_callback_work(struct work_struct *work)
{
	struct dlm_lkb *lkb = container_of(work, struct dlm_lkb, lkb_cb_work);
	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
	void (*castfn) (void *astparam);
	void (*bastfn) (void *astparam, int mode);
	struct dlm_callback *cb;
	int rv;

	spin_lock(&lkb->lkb_cb_lock);
	rv = dlm_dequeue_lkb_callback(lkb, &cb);
	spin_unlock(&lkb->lkb_cb_lock);

	if (WARN_ON_ONCE(rv == DLM_DEQUEUE_CALLBACK_EMPTY))
		goto out;

	for (;;) {
		castfn = lkb->lkb_astfn;
		bastfn = lkb->lkb_bastfn;

		if (cb->flags & DLM_CB_BAST) {
			trace_dlm_bast(ls, lkb, cb->mode);
			lkb->lkb_last_bast_time = ktime_get();
			lkb->lkb_last_bast_mode = cb->mode;
			bastfn(lkb->lkb_astparam, cb->mode);
		} else if (cb->flags & DLM_CB_CAST) {
			lkb->lkb_lksb->sb_status = cb->sb_status;
			lkb->lkb_lksb->sb_flags = cb->sb_flags;
			trace_dlm_ast(ls, lkb);
			lkb->lkb_last_cast_time = ktime_get();
			castfn(lkb->lkb_astparam);
		}

		kref_put(&cb->ref, dlm_release_callback);

		spin_lock(&lkb->lkb_cb_lock);
		rv = dlm_dequeue_lkb_callback(lkb, &cb);
		if (rv == DLM_DEQUEUE_CALLBACK_EMPTY) {
			clear_bit(DLM_IFL_CB_PENDING_BIT, &lkb->lkb_iflags);
			spin_unlock(&lkb->lkb_cb_lock);
			break;
		}
		spin_unlock(&lkb->lkb_cb_lock);
	}

out:
	/* undo kref_get from dlm_add_callback, may cause lkb to be freed */
	dlm_put_lkb(lkb);
}

int dlm_callback_start(struct dlm_ls *ls)
{
	ls->ls_callback_wq = alloc_workqueue("dlm_callback",
					     WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
	if (!ls->ls_callback_wq) {
		log_print("can't start dlm_callback workqueue");
		return -ENOMEM;
	}
	return 0;
}

void dlm_callback_stop(struct dlm_ls *ls)
{
	if (ls->ls_callback_wq)
		destroy_workqueue(ls->ls_callback_wq);
}

void dlm_callback_suspend(struct dlm_ls *ls)
{
	if (ls->ls_callback_wq) {
		spin_lock(&ls->ls_cb_lock);
		set_bit(LSFL_CB_DELAY, &ls->ls_flags);
		spin_unlock(&ls->ls_cb_lock);

		flush_workqueue(ls->ls_callback_wq);
	}
}

#define MAX_CB_QUEUE 25

void dlm_callback_resume(struct dlm_ls *ls)
{
	struct dlm_lkb *lkb, *safe;
	int count = 0, sum = 0;
	bool empty;

	if (!ls->ls_callback_wq)
		return;

more:
	spin_lock(&ls->ls_cb_lock);
	list_for_each_entry_safe(lkb, safe, &ls->ls_cb_delay, lkb_cb_list) {
		list_del_init(&lkb->lkb_cb_list);
		queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work);
		count++;
		if (count == MAX_CB_QUEUE)
			break;
	}
	empty = list_empty(&ls->ls_cb_delay);
	if (empty)
		clear_bit(LSFL_CB_DELAY, &ls->ls_flags);
	spin_unlock(&ls->ls_cb_lock);

	sum += count;
	if (!empty) {
		count = 0;
		cond_resched();
		goto more;
	}

	if (sum)
		log_rinfo(ls, "%s %d", __func__, sum);
}
Commit	Line	Data
2522fe45	1	// SPDX-License-Identifier: GPL-2.0-only
e7fd4179 DT	2	/******************************************************************************
	3	*******************************************************************************
	4	**
	5	** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
7fe2b319	6	** Copyright (C) 2004-2010 Red Hat, Inc. All rights reserved.
e7fd4179	7	**
e7fd4179 DT	8	**
	9	*******************************************************************************
	10	******************************************************************************/
	11
f1d3b8f9 AA	12	#include <trace/events/dlm.h>
f1d3b8f9 AA	13
e7fd4179	14	#include "dlm_internal.h"
61bed0ba	15	#include "memory.h"
e7fd4179	16	#include "lock.h"
597d0cae	17	#include "user.h"
95058571	18	#include "ast.h"
e7fd4179	19
61bed0ba AA	20	void dlm_release_callback(struct kref *ref)
	21	{
	22	struct dlm_callback *cb = container_of(ref, struct dlm_callback, ref);
	23
	24	dlm_free_cb(cb);
	25	}
	26
	27	void dlm_callback_set_last_ptr(struct dlm_callback **from,
	28	struct dlm_callback *to)
	29	{
	30	if (*from)
	31	kref_put(&(*from)->ref, dlm_release_callback);
	32
	33	if (to)
	34	kref_get(&to->ref);
	35
	36	*from = to;
	37	}
e7fd4179	38
61bed0ba	39	void dlm_purge_lkb_callbacks(struct dlm_lkb *lkb)
8304d6f2	40	{
61bed0ba AA	41	struct dlm_callback cb, safe;
	42
	43	list_for_each_entry_safe(cb, safe, &lkb->lkb_callbacks, list) {
	44	list_del(&cb->list);
	45	kref_put(&cb->ref, dlm_release_callback);
8304d6f2	46	}
61bed0ba	47
a034c137	48	clear_bit(DLM_IFL_CB_PENDING_BIT, &lkb->lkb_iflags);
61bed0ba AA	49
	50	/* invalidate */
	51	dlm_callback_set_last_ptr(&lkb->lkb_last_cast, NULL);
	52	dlm_callback_set_last_ptr(&lkb->lkb_last_cb, NULL);
	53	lkb->lkb_last_bast_mode = -1;
8304d6f2 DT	54	}
8304d6f2 DT	55
61bed0ba AA	56	int dlm_enqueue_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
61bed0ba AA	57	int status, uint32_t sbflags)
8304d6f2 DT	58	{
8304d6f2 DT	59	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
61bed0ba AA	60	int rv = DLM_ENQUEUE_CALLBACK_SUCCESS;
61bed0ba AA	61	struct dlm_callback *cb;
8304d6f2	62	int prev_mode;
8304d6f2	63
61bed0ba AA	64	if (flags & DLM_CB_BAST) {
	65	/* if cb is a bast, it should be skipped if the blocking mode is
	66	* compatible with the last granted mode
	67	*/
	68	if (lkb->lkb_last_cast) {
	69	if (dlm_modes_compat(mode, lkb->lkb_last_cast->mode)) {
	70	log_debug(ls, "skip %x bast mode %d for cast mode %d",
	71	lkb->lkb_id, mode,
	72	lkb->lkb_last_cast->mode);
	73	goto out;
	74	}
	75	}
8304d6f2 DT	76
	77	/*
	78	* Suppress some redundant basts here, do more on removal.
	79	* Don't even add a bast if the callback just before it
	80	* is a bast for the same mode or a more restrictive mode.
	81	* (the addional > PR check is needed for PR/CW inversion)
	82	*/
61bed0ba AA	83	if (lkb->lkb_last_cb && lkb->lkb_last_cb->flags & DLM_CB_BAST) {
61bed0ba AA	84	prev_mode = lkb->lkb_last_cb->mode;
8304d6f2 DT	85
	86	if ((prev_mode == mode) \|\|
	87	(prev_mode > mode && prev_mode > DLM_LOCK_PR)) {
61bed0ba AA	88	log_debug(ls, "skip %x add bast mode %d for bast mode %d",
61bed0ba AA	89	lkb->lkb_id, mode, prev_mode);
23e8e1aa	90	goto out;
8304d6f2 DT	91	}
8304d6f2 DT	92	}
8304d6f2 DT	93	}
8304d6f2 DT	94
61bed0ba AA	95	cb = dlm_allocate_cb();
	96	if (!cb) {
	97	rv = DLM_ENQUEUE_CALLBACK_FAILURE;
23e8e1aa	98	goto out;
8304d6f2	99	}
8304d6f2	100
61bed0ba AA	101	cb->flags = flags;
	102	cb->mode = mode;
	103	cb->sb_status = status;
	104	cb->sb_flags = (sbflags & 0x000000FF);
	105	kref_init(&cb->ref);
a034c137	106	if (!test_and_set_bit(DLM_IFL_CB_PENDING_BIT, &lkb->lkb_iflags))
61bed0ba	107	rv = DLM_ENQUEUE_CALLBACK_NEED_SCHED;
a034c137	108
61bed0ba	109	list_add_tail(&cb->list, &lkb->lkb_callbacks);
8304d6f2	110
61bed0ba AA	111	if (flags & DLM_CB_CAST)
61bed0ba AA	112	dlm_callback_set_last_ptr(&lkb->lkb_last_cast, cb);
8304d6f2	113
61bed0ba	114	dlm_callback_set_last_ptr(&lkb->lkb_last_cb, cb);
8304d6f2	115
23e8e1aa DT	116	out:
23e8e1aa DT	117	return rv;
8304d6f2 DT	118	}
8304d6f2 DT	119
61bed0ba AA	120	int dlm_dequeue_lkb_callback(struct dlm_lkb lkb, struct dlm_callback *cb)
	121	{
	122	/* oldest undelivered cb is callbacks first entry */
	123	*cb = list_first_entry_or_null(&lkb->lkb_callbacks,
	124	struct dlm_callback, list);
	125	if (!*cb)
	126	return DLM_DEQUEUE_CALLBACK_EMPTY;
	127
	128	/* remove it from callbacks so shift others down */
	129	list_del(&(*cb)->list);
	130	if (list_empty(&lkb->lkb_callbacks))
	131	return DLM_DEQUEUE_CALLBACK_LAST;
	132
	133	return DLM_DEQUEUE_CALLBACK_SUCCESS;
	134	}
	135
23e8e1aa DT	136	void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,
23e8e1aa DT	137	uint32_t sbflags)
8304d6f2	138	{
23e8e1aa	139	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
8304d6f2 DT	140	int rv;
8304d6f2 DT	141
8a39dcd9	142	if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) {
61bed0ba	143	dlm_user_add_ast(lkb, flags, mode, status, sbflags);
597d0cae DT	144	return;
	145	}
	146
92e95733	147	spin_lock(&lkb->lkb_cb_lock);
61bed0ba AA	148	rv = dlm_enqueue_lkb_callback(lkb, flags, mode, status, sbflags);
	149	switch (rv) {
	150	case DLM_ENQUEUE_CALLBACK_NEED_SCHED:
e7fd4179	151	kref_get(&lkb->lkb_ref);
e7fd4179	152
a4c0352b	153	spin_lock(&ls->ls_cb_lock);
23e8e1aa	154	if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) {
23e8e1aa	155	list_add(&lkb->lkb_cb_list, &ls->ls_cb_delay);
23e8e1aa DT	156	} else {
	157	queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work);
	158	}
a4c0352b	159	spin_unlock(&ls->ls_cb_lock);
61bed0ba AA	160	break;
61bed0ba AA	161	case DLM_ENQUEUE_CALLBACK_FAILURE:
740bb8fc	162	WARN_ON_ONCE(1);
61bed0ba AA	163	break;
	164	case DLM_ENQUEUE_CALLBACK_SUCCESS:
	165	break;
	166	default:
740bb8fc	167	WARN_ON_ONCE(1);
61bed0ba	168	break;
23e8e1aa	169	}
92e95733	170	spin_unlock(&lkb->lkb_cb_lock);
e7fd4179 DT	171	}
e7fd4179 DT	172
23e8e1aa	173	void dlm_callback_work(struct work_struct *work)
e7fd4179	174	{
23e8e1aa DT	175	struct dlm_lkb *lkb = container_of(work, struct dlm_lkb, lkb_cb_work);
23e8e1aa DT	176	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
7fe2b319 DT	177	void (castfn) (void astparam);
7fe2b319 DT	178	void (bastfn) (void astparam, int mode);
61bed0ba AA	179	struct dlm_callback *cb;
61bed0ba AA	180	int rv;
e7fd4179	181
92e95733	182	spin_lock(&lkb->lkb_cb_lock);
61bed0ba	183	rv = dlm_dequeue_lkb_callback(lkb, &cb);
92e95733	184	spin_unlock(&lkb->lkb_cb_lock);
7fe2b319	185
740bb8fc	186	if (WARN_ON_ONCE(rv == DLM_DEQUEUE_CALLBACK_EMPTY))
9267c857	187	goto out;
7fe2b319	188
61bed0ba AA	189	for (;;) {
	190	castfn = lkb->lkb_astfn;
	191	bastfn = lkb->lkb_bastfn;
	192
	193	if (cb->flags & DLM_CB_BAST) {
	194	trace_dlm_bast(ls, lkb, cb->mode);
27d3994e	195	lkb->lkb_last_bast_time = ktime_get();
61bed0ba AA	196	lkb->lkb_last_bast_mode = cb->mode;
	197	bastfn(lkb->lkb_astparam, cb->mode);
	198	} else if (cb->flags & DLM_CB_CAST) {
	199	lkb->lkb_lksb->sb_status = cb->sb_status;
	200	lkb->lkb_lksb->sb_flags = cb->sb_flags;
0c4c516f	201	trace_dlm_ast(ls, lkb);
27d3994e	202	lkb->lkb_last_cast_time = ktime_get();
cd1e8ca9	203	castfn(lkb->lkb_astparam);
7fe2b319	204	}
61bed0ba AA	205
	206	kref_put(&cb->ref, dlm_release_callback);
	207
	208	spin_lock(&lkb->lkb_cb_lock);
	209	rv = dlm_dequeue_lkb_callback(lkb, &cb);
	210	if (rv == DLM_DEQUEUE_CALLBACK_EMPTY) {
a034c137	211	clear_bit(DLM_IFL_CB_PENDING_BIT, &lkb->lkb_iflags);
61bed0ba AA	212	spin_unlock(&lkb->lkb_cb_lock);
	213	break;
	214	}
	215	spin_unlock(&lkb->lkb_cb_lock);
e7fd4179	216	}
e7fd4179	217
9267c857	218	out:
23e8e1aa DT	219	/* undo kref_get from dlm_add_callback, may cause lkb to be freed */
23e8e1aa DT	220	dlm_put_lkb(lkb);
e7fd4179 DT	221	}
e7fd4179 DT	222
23e8e1aa	223	int dlm_callback_start(struct dlm_ls *ls)
e7fd4179	224	{
23e8e1aa	225	ls->ls_callback_wq = alloc_workqueue("dlm_callback",
aa9f1012	226	WQ_HIGHPRI \| WQ_MEM_RECLAIM, 0);
23e8e1aa DT	227	if (!ls->ls_callback_wq) {
	228	log_print("can't start dlm_callback workqueue");
	229	return -ENOMEM;
e7fd4179 DT	230	}
	231	return 0;
	232	}
	233
23e8e1aa	234	void dlm_callback_stop(struct dlm_ls *ls)
e7fd4179	235	{
23e8e1aa DT	236	if (ls->ls_callback_wq)
23e8e1aa DT	237	destroy_workqueue(ls->ls_callback_wq);
e7fd4179 DT	238	}
e7fd4179 DT	239
23e8e1aa	240	void dlm_callback_suspend(struct dlm_ls *ls)
e7fd4179	241	{
9cb16d42	242	if (ls->ls_callback_wq) {
a4c0352b	243	spin_lock(&ls->ls_cb_lock);
9cb16d42	244	set_bit(LSFL_CB_DELAY, &ls->ls_flags);
a4c0352b	245	spin_unlock(&ls->ls_cb_lock);
e7fd4179	246
23e8e1aa	247	flush_workqueue(ls->ls_callback_wq);
9cb16d42	248	}
e7fd4179 DT	249	}
e7fd4179 DT	250
216f0efd BP	251	#define MAX_CB_QUEUE 25
216f0efd BP	252
23e8e1aa	253	void dlm_callback_resume(struct dlm_ls *ls)
e7fd4179	254	{
23e8e1aa	255	struct dlm_lkb lkb, safe;
2f05ec43	256	int count = 0, sum = 0;
f70813d6	257	bool empty;
e7fd4179	258
23e8e1aa DT	259	if (!ls->ls_callback_wq)
	260	return;
	261
216f0efd	262	more:
a4c0352b	263	spin_lock(&ls->ls_cb_lock);
23e8e1aa DT	264	list_for_each_entry_safe(lkb, safe, &ls->ls_cb_delay, lkb_cb_list) {
	265	list_del_init(&lkb->lkb_cb_list);
	266	queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work);
	267	count++;
216f0efd BP	268	if (count == MAX_CB_QUEUE)
216f0efd BP	269	break;
23e8e1aa	270	}
f70813d6	271	empty = list_empty(&ls->ls_cb_delay);
85839f27 AA	272	if (empty)
85839f27 AA	273	clear_bit(LSFL_CB_DELAY, &ls->ls_flags);
a4c0352b	274	spin_unlock(&ls->ls_cb_lock);
23e8e1aa	275
2f05ec43	276	sum += count;
f70813d6	277	if (!empty) {
216f0efd BP	278	count = 0;
	279	cond_resched();
	280	goto more;
	281	}
2f05ec43 AA	282
	283	if (sum)
	284	log_rinfo(ls, "%s %d", __func__, sum);
e7fd4179 DT	285	}
e7fd4179 DT	286