[linux-block.git] / fs / xfs / xfs_rmap_item.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright (C) 2016 Oracle.  All Rights Reserved.
 * Author: Darrick J. Wong <darrick.wong@oracle.com>
 */
#include "xfs.h"
#include "xfs_fs.h"
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
#include "xfs_bit.h"
#include "xfs_shared.h"
#include "xfs_mount.h"
#include "xfs_defer.h"
#include "xfs_trans.h"
#include "xfs_trans_priv.h"
#include "xfs_rmap_item.h"
#include "xfs_log.h"
#include "xfs_rmap.h"
#include "xfs_error.h"

kmem_zone_t	*xfs_rui_zone;
kmem_zone_t	*xfs_rud_zone;

static inline struct xfs_rui_log_item *RUI_ITEM(struct xfs_log_item *lip)
{
	return container_of(lip, struct xfs_rui_log_item, rui_item);
}

void
xfs_rui_item_free(
	struct xfs_rui_log_item	*ruip)
{
	if (ruip->rui_format.rui_nextents > XFS_RUI_MAX_FAST_EXTENTS)
		kmem_free(ruip);
	else
		kmem_cache_free(xfs_rui_zone, ruip);
}

/*
 * Freeing the RUI requires that we remove it from the AIL if it has already
 * been placed there. However, the RUI may not yet have been placed in the AIL
 * when called by xfs_rui_release() from RUD processing due to the ordering of
 * committed vs unpin operations in bulk insert operations. Hence the reference
 * count to ensure only the last caller frees the RUI.
 */
void
xfs_rui_release(
	struct xfs_rui_log_item	*ruip)
{
	ASSERT(atomic_read(&ruip->rui_refcount) > 0);
	if (atomic_dec_and_test(&ruip->rui_refcount)) {
		xfs_trans_ail_remove(&ruip->rui_item, SHUTDOWN_LOG_IO_ERROR);
		xfs_rui_item_free(ruip);
	}
}

STATIC void
xfs_rui_item_size(
	struct xfs_log_item	*lip,
	int			*nvecs,
	int			*nbytes)
{
	struct xfs_rui_log_item	*ruip = RUI_ITEM(lip);

	*nvecs += 1;
	*nbytes += xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents);
}

/*
 * This is called to fill in the vector of log iovecs for the
 * given rui log item. We use only 1 iovec, and we point that
 * at the rui_log_format structure embedded in the rui item.
 * It is at this point that we assert that all of the extent
 * slots in the rui item have been filled.
 */
STATIC void
xfs_rui_item_format(
	struct xfs_log_item	*lip,
	struct xfs_log_vec	*lv)
{
	struct xfs_rui_log_item	*ruip = RUI_ITEM(lip);
	struct xfs_log_iovec	*vecp = NULL;

	ASSERT(atomic_read(&ruip->rui_next_extent) ==
			ruip->rui_format.rui_nextents);

	ruip->rui_format.rui_type = XFS_LI_RUI;
	ruip->rui_format.rui_size = 1;

	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_RUI_FORMAT, &ruip->rui_format,
			xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents));
}

/*
 * The unpin operation is the last place an RUI is manipulated in the log. It is
 * either inserted in the AIL or aborted in the event of a log I/O error. In
 * either case, the RUI transaction has been successfully committed to make it
 * this far. Therefore, we expect whoever committed the RUI to either construct
 * and commit the RUD or drop the RUD's reference in the event of error. Simply
 * drop the log's RUI reference now that the log is done with it.
 */
STATIC void
xfs_rui_item_unpin(
	struct xfs_log_item	*lip,
	int			remove)
{
	struct xfs_rui_log_item	*ruip = RUI_ITEM(lip);

	xfs_rui_release(ruip);
}

/*
 * The RUI has been either committed or aborted if the transaction has been
 * cancelled. If the transaction was cancelled, an RUD isn't going to be
 * constructed and thus we free the RUI here directly.
 */
STATIC void
xfs_rui_item_release(
	struct xfs_log_item	*lip)
{
	xfs_rui_release(RUI_ITEM(lip));
}

static const struct xfs_item_ops xfs_rui_item_ops = {
	.iop_size	= xfs_rui_item_size,
	.iop_format	= xfs_rui_item_format,
	.iop_unpin	= xfs_rui_item_unpin,
	.iop_release	= xfs_rui_item_release,
};

/*
 * Allocate and initialize an rui item with the given number of extents.
 */
struct xfs_rui_log_item *
xfs_rui_init(
	struct xfs_mount		*mp,
	uint				nextents)

{
	struct xfs_rui_log_item		*ruip;

	ASSERT(nextents > 0);
	if (nextents > XFS_RUI_MAX_FAST_EXTENTS)
		ruip = kmem_zalloc(xfs_rui_log_item_sizeof(nextents), 0);
	else
		ruip = kmem_zone_zalloc(xfs_rui_zone, 0);

	xfs_log_item_init(mp, &ruip->rui_item, XFS_LI_RUI, &xfs_rui_item_ops);
	ruip->rui_format.rui_nextents = nextents;
	ruip->rui_format.rui_id = (uintptr_t)(void *)ruip;
	atomic_set(&ruip->rui_next_extent, 0);
	atomic_set(&ruip->rui_refcount, 2);

	return ruip;
}

/*
 * Copy an RUI format buffer from the given buf, and into the destination
 * RUI format structure.  The RUI/RUD items were designed not to need any
 * special alignment handling.
 */
int
xfs_rui_copy_format(
	struct xfs_log_iovec		*buf,
	struct xfs_rui_log_format	*dst_rui_fmt)
{
	struct xfs_rui_log_format	*src_rui_fmt;
	uint				len;

	src_rui_fmt = buf->i_addr;
	len = xfs_rui_log_format_sizeof(src_rui_fmt->rui_nextents);

	if (buf->i_len != len) {
		XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL);
		return -EFSCORRUPTED;
	}

	memcpy(dst_rui_fmt, src_rui_fmt, len);
	return 0;
}

static inline struct xfs_rud_log_item *RUD_ITEM(struct xfs_log_item *lip)
{
	return container_of(lip, struct xfs_rud_log_item, rud_item);
}

STATIC void
xfs_rud_item_size(
	struct xfs_log_item	*lip,
	int			*nvecs,
	int			*nbytes)
{
	*nvecs += 1;
	*nbytes += sizeof(struct xfs_rud_log_format);
}

/*
 * This is called to fill in the vector of log iovecs for the
 * given rud log item. We use only 1 iovec, and we point that
 * at the rud_log_format structure embedded in the rud item.
 * It is at this point that we assert that all of the extent
 * slots in the rud item have been filled.
 */
STATIC void
xfs_rud_item_format(
	struct xfs_log_item	*lip,
	struct xfs_log_vec	*lv)
{
	struct xfs_rud_log_item	*rudp = RUD_ITEM(lip);
	struct xfs_log_iovec	*vecp = NULL;

	rudp->rud_format.rud_type = XFS_LI_RUD;
	rudp->rud_format.rud_size = 1;

	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_RUD_FORMAT, &rudp->rud_format,
			sizeof(struct xfs_rud_log_format));
}

/*
 * The RUD is either committed or aborted if the transaction is cancelled. If
 * the transaction is cancelled, drop our reference to the RUI and free the
 * RUD.
 */
STATIC void
xfs_rud_item_release(
	struct xfs_log_item	*lip)
{
	struct xfs_rud_log_item	*rudp = RUD_ITEM(lip);

	xfs_rui_release(rudp->rud_ruip);
	kmem_cache_free(xfs_rud_zone, rudp);
}

static const struct xfs_item_ops xfs_rud_item_ops = {
	.flags		= XFS_ITEM_RELEASE_WHEN_COMMITTED,
	.iop_size	= xfs_rud_item_size,
	.iop_format	= xfs_rud_item_format,
	.iop_release	= xfs_rud_item_release,
};

static struct xfs_rud_log_item *
xfs_trans_get_rud(
	struct xfs_trans		*tp,
	struct xfs_rui_log_item		*ruip)
{
	struct xfs_rud_log_item		*rudp;

	rudp = kmem_zone_zalloc(xfs_rud_zone, 0);
	xfs_log_item_init(tp->t_mountp, &rudp->rud_item, XFS_LI_RUD,
			  &xfs_rud_item_ops);
	rudp->rud_ruip = ruip;
	rudp->rud_format.rud_rui_id = ruip->rui_format.rui_id;

	xfs_trans_add_item(tp, &rudp->rud_item);
	return rudp;
}

/* Set the map extent flags for this reverse mapping. */
static void
xfs_trans_set_rmap_flags(
	struct xfs_map_extent		*rmap,
	enum xfs_rmap_intent_type	type,
	int				whichfork,
	xfs_exntst_t			state)
{
	rmap->me_flags = 0;
	if (state == XFS_EXT_UNWRITTEN)
		rmap->me_flags |= XFS_RMAP_EXTENT_UNWRITTEN;
	if (whichfork == XFS_ATTR_FORK)
		rmap->me_flags |= XFS_RMAP_EXTENT_ATTR_FORK;
	switch (type) {
	case XFS_RMAP_MAP:
		rmap->me_flags |= XFS_RMAP_EXTENT_MAP;
		break;
	case XFS_RMAP_MAP_SHARED:
		rmap->me_flags |= XFS_RMAP_EXTENT_MAP_SHARED;
		break;
	case XFS_RMAP_UNMAP:
		rmap->me_flags |= XFS_RMAP_EXTENT_UNMAP;
		break;
	case XFS_RMAP_UNMAP_SHARED:
		rmap->me_flags |= XFS_RMAP_EXTENT_UNMAP_SHARED;
		break;
	case XFS_RMAP_CONVERT:
		rmap->me_flags |= XFS_RMAP_EXTENT_CONVERT;
		break;
	case XFS_RMAP_CONVERT_SHARED:
		rmap->me_flags |= XFS_RMAP_EXTENT_CONVERT_SHARED;
		break;
	case XFS_RMAP_ALLOC:
		rmap->me_flags |= XFS_RMAP_EXTENT_ALLOC;
		break;
	case XFS_RMAP_FREE:
		rmap->me_flags |= XFS_RMAP_EXTENT_FREE;
		break;
	default:
		ASSERT(0);
	}
}

/*
 * Finish an rmap update and log it to the RUD. Note that the transaction is
 * marked dirty regardless of whether the rmap update succeeds or fails to
 * support the RUI/RUD lifecycle rules.
 */
static int
xfs_trans_log_finish_rmap_update(
	struct xfs_trans		*tp,
	struct xfs_rud_log_item		*rudp,
	enum xfs_rmap_intent_type	type,
	uint64_t			owner,
	int				whichfork,
	xfs_fileoff_t			startoff,
	xfs_fsblock_t			startblock,
	xfs_filblks_t			blockcount,
	xfs_exntst_t			state,
	struct xfs_btree_cur		**pcur)
{
	int				error;

	error = xfs_rmap_finish_one(tp, type, owner, whichfork, startoff,
			startblock, blockcount, state, pcur);

	/*
	 * Mark the transaction dirty, even on error. This ensures the
	 * transaction is aborted, which:
	 *
	 * 1.) releases the RUI and frees the RUD
	 * 2.) shuts down the filesystem
	 */
	tp->t_flags |= XFS_TRANS_DIRTY;
	set_bit(XFS_LI_DIRTY, &rudp->rud_item.li_flags);

	return error;
}

/* Sort rmap intents by AG. */
static int
xfs_rmap_update_diff_items(
	void				*priv,
	struct list_head		*a,
	struct list_head		*b)
{
	struct xfs_mount		*mp = priv;
	struct xfs_rmap_intent		*ra;
	struct xfs_rmap_intent		*rb;

	ra = container_of(a, struct xfs_rmap_intent, ri_list);
	rb = container_of(b, struct xfs_rmap_intent, ri_list);
	return  XFS_FSB_TO_AGNO(mp, ra->ri_bmap.br_startblock) -
		XFS_FSB_TO_AGNO(mp, rb->ri_bmap.br_startblock);
}

/* Log rmap updates in the intent item. */
STATIC void
xfs_rmap_update_log_item(
	struct xfs_trans		*tp,
	struct xfs_rui_log_item		*ruip,
	struct xfs_rmap_intent		*rmap)
{
	uint				next_extent;
	struct xfs_map_extent		*map;

	tp->t_flags |= XFS_TRANS_DIRTY;
	set_bit(XFS_LI_DIRTY, &ruip->rui_item.li_flags);

	/*
	 * atomic_inc_return gives us the value after the increment;
	 * we want to use it as an array index so we need to subtract 1 from
	 * it.
	 */
	next_extent = atomic_inc_return(&ruip->rui_next_extent) - 1;
	ASSERT(next_extent < ruip->rui_format.rui_nextents);
	map = &ruip->rui_format.rui_extents[next_extent];
	map->me_owner = rmap->ri_owner;
	map->me_startblock = rmap->ri_bmap.br_startblock;
	map->me_startoff = rmap->ri_bmap.br_startoff;
	map->me_len = rmap->ri_bmap.br_blockcount;
	xfs_trans_set_rmap_flags(map, rmap->ri_type, rmap->ri_whichfork,
			rmap->ri_bmap.br_state);
}

static struct xfs_log_item *
xfs_rmap_update_create_intent(
	struct xfs_trans		*tp,
	struct list_head		*items,
	unsigned int			count,
	bool				sort)
{
	struct xfs_mount		*mp = tp->t_mountp;
	struct xfs_rui_log_item		*ruip = xfs_rui_init(mp, count);
	struct xfs_rmap_intent		*rmap;

	ASSERT(count > 0);

	xfs_trans_add_item(tp, &ruip->rui_item);
	if (sort)
		list_sort(mp, items, xfs_rmap_update_diff_items);
	list_for_each_entry(rmap, items, ri_list)
		xfs_rmap_update_log_item(tp, ruip, rmap);
	return &ruip->rui_item;
}

/* Get an RUD so we can process all the deferred rmap updates. */
STATIC void *
xfs_rmap_update_create_done(
	struct xfs_trans		*tp,
	struct xfs_log_item		*intent,
	unsigned int			count)
{
	return xfs_trans_get_rud(tp, RUI_ITEM(intent));
}

/* Process a deferred rmap update. */
STATIC int
xfs_rmap_update_finish_item(
	struct xfs_trans		*tp,
	struct list_head		*item,
	void				*done_item,
	void				**state)
{
	struct xfs_rmap_intent		*rmap;
	int				error;

	rmap = container_of(item, struct xfs_rmap_intent, ri_list);
	error = xfs_trans_log_finish_rmap_update(tp, done_item,
			rmap->ri_type,
			rmap->ri_owner, rmap->ri_whichfork,
			rmap->ri_bmap.br_startoff,
			rmap->ri_bmap.br_startblock,
			rmap->ri_bmap.br_blockcount,
			rmap->ri_bmap.br_state,
			(struct xfs_btree_cur **)state);
	kmem_free(rmap);
	return error;
}

/* Clean up after processing deferred rmaps. */
STATIC void
xfs_rmap_update_finish_cleanup(
	struct xfs_trans	*tp,
	void			*state,
	int			error)
{
	struct xfs_btree_cur	*rcur = state;

	xfs_rmap_finish_one_cleanup(tp, rcur, error);
}

/* Abort all pending RUIs. */
STATIC void
xfs_rmap_update_abort_intent(
	struct xfs_log_item	*intent)
{
	xfs_rui_release(RUI_ITEM(intent));
}

/* Cancel a deferred rmap update. */
STATIC void
xfs_rmap_update_cancel_item(
	struct list_head		*item)
{
	struct xfs_rmap_intent		*rmap;

	rmap = container_of(item, struct xfs_rmap_intent, ri_list);
	kmem_free(rmap);
}

const struct xfs_defer_op_type xfs_rmap_update_defer_type = {
	.max_items	= XFS_RUI_MAX_FAST_EXTENTS,
	.create_intent	= xfs_rmap_update_create_intent,
	.abort_intent	= xfs_rmap_update_abort_intent,
	.create_done	= xfs_rmap_update_create_done,
	.finish_item	= xfs_rmap_update_finish_item,
	.finish_cleanup = xfs_rmap_update_finish_cleanup,
	.cancel_item	= xfs_rmap_update_cancel_item,
};

/*
 * Process an rmap update intent item that was recovered from the log.
 * We need to update the rmapbt.
 */
int
xfs_rui_recover(
	struct xfs_mount		*mp,
	struct xfs_rui_log_item		*ruip)
{
	int				i;
	int				error = 0;
	struct xfs_map_extent		*rmap;
	xfs_fsblock_t			startblock_fsb;
	bool				op_ok;
	struct xfs_rud_log_item		*rudp;
	enum xfs_rmap_intent_type	type;
	int				whichfork;
	xfs_exntst_t			state;
	struct xfs_trans		*tp;
	struct xfs_btree_cur		*rcur = NULL;

	ASSERT(!test_bit(XFS_RUI_RECOVERED, &ruip->rui_flags));

	/*
	 * First check the validity of the extents described by the
	 * RUI.  If any are bad, then assume that all are bad and
	 * just toss the RUI.
	 */
	for (i = 0; i < ruip->rui_format.rui_nextents; i++) {
		rmap = &ruip->rui_format.rui_extents[i];
		startblock_fsb = XFS_BB_TO_FSB(mp,
				   XFS_FSB_TO_DADDR(mp, rmap->me_startblock));
		switch (rmap->me_flags & XFS_RMAP_EXTENT_TYPE_MASK) {
		case XFS_RMAP_EXTENT_MAP:
		case XFS_RMAP_EXTENT_MAP_SHARED:
		case XFS_RMAP_EXTENT_UNMAP:
		case XFS_RMAP_EXTENT_UNMAP_SHARED:
		case XFS_RMAP_EXTENT_CONVERT:
		case XFS_RMAP_EXTENT_CONVERT_SHARED:
		case XFS_RMAP_EXTENT_ALLOC:
		case XFS_RMAP_EXTENT_FREE:
			op_ok = true;
			break;
		default:
			op_ok = false;
			break;
		}
		if (!op_ok || startblock_fsb == 0 ||
		    rmap->me_len == 0 ||
		    startblock_fsb >= mp->m_sb.sb_dblocks ||
		    rmap->me_len >= mp->m_sb.sb_agblocks ||
		    (rmap->me_flags & ~XFS_RMAP_EXTENT_FLAGS)) {
			/*
			 * This will pull the RUI from the AIL and
			 * free the memory associated with it.
			 */
			set_bit(XFS_RUI_RECOVERED, &ruip->rui_flags);
			xfs_rui_release(ruip);
			return -EFSCORRUPTED;
		}
	}

	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate,
			mp->m_rmap_maxlevels, 0, XFS_TRANS_RESERVE, &tp);
	if (error)
		return error;
	rudp = xfs_trans_get_rud(tp, ruip);

	for (i = 0; i < ruip->rui_format.rui_nextents; i++) {
		rmap = &ruip->rui_format.rui_extents[i];
		state = (rmap->me_flags & XFS_RMAP_EXTENT_UNWRITTEN) ?
				XFS_EXT_UNWRITTEN : XFS_EXT_NORM;
		whichfork = (rmap->me_flags & XFS_RMAP_EXTENT_ATTR_FORK) ?
				XFS_ATTR_FORK : XFS_DATA_FORK;
		switch (rmap->me_flags & XFS_RMAP_EXTENT_TYPE_MASK) {
		case XFS_RMAP_EXTENT_MAP:
			type = XFS_RMAP_MAP;
			break;
		case XFS_RMAP_EXTENT_MAP_SHARED:
			type = XFS_RMAP_MAP_SHARED;
			break;
		case XFS_RMAP_EXTENT_UNMAP:
			type = XFS_RMAP_UNMAP;
			break;
		case XFS_RMAP_EXTENT_UNMAP_SHARED:
			type = XFS_RMAP_UNMAP_SHARED;
			break;
		case XFS_RMAP_EXTENT_CONVERT:
			type = XFS_RMAP_CONVERT;
			break;
		case XFS_RMAP_EXTENT_CONVERT_SHARED:
			type = XFS_RMAP_CONVERT_SHARED;
			break;
		case XFS_RMAP_EXTENT_ALLOC:
			type = XFS_RMAP_ALLOC;
			break;
		case XFS_RMAP_EXTENT_FREE:
			type = XFS_RMAP_FREE;
			break;
		default:
			XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL);
			error = -EFSCORRUPTED;
			goto abort_error;
		}
		error = xfs_trans_log_finish_rmap_update(tp, rudp, type,
				rmap->me_owner, whichfork,
				rmap->me_startoff, rmap->me_startblock,
				rmap->me_len, state, &rcur);
		if (error)
			goto abort_error;

	}

	xfs_rmap_finish_one_cleanup(tp, rcur, error);
	set_bit(XFS_RUI_RECOVERED, &ruip->rui_flags);
	error = xfs_trans_commit(tp);
	return error;

abort_error:
	xfs_rmap_finish_one_cleanup(tp, rcur, error);
	xfs_trans_cancel(tp);
	return error;
}
Commit	Line	Data
0b61f8a4	1	// SPDX-License-Identifier: GPL-2.0+
5880f2d7 DW	2	/*
5880f2d7 DW	3	* Copyright (C) 2016 Oracle. All Rights Reserved.
5880f2d7	4	* Author: Darrick J. Wong <darrick.wong@oracle.com>
5880f2d7 DW	5	*/
	6	#include "xfs.h"
	7	#include "xfs_fs.h"
	8	#include "xfs_format.h"
	9	#include "xfs_log_format.h"
	10	#include "xfs_trans_resv.h"
9e88b5d8	11	#include "xfs_bit.h"
b31c2bdc	12	#include "xfs_shared.h"
5880f2d7	13	#include "xfs_mount.h"
9c194644	14	#include "xfs_defer.h"
5880f2d7 DW	15	#include "xfs_trans.h"
5880f2d7 DW	16	#include "xfs_trans_priv.h"
5880f2d7 DW	17	#include "xfs_rmap_item.h"
5880f2d7 DW	18	#include "xfs_log.h"
9c194644	19	#include "xfs_rmap.h"
a5155b87	20	#include "xfs_error.h"
5880f2d7 DW	21
	22	kmem_zone_t *xfs_rui_zone;
	23	kmem_zone_t *xfs_rud_zone;
	24
	25	static inline struct xfs_rui_log_item RUI_ITEM(struct xfs_log_item lip)
	26	{
	27	return container_of(lip, struct xfs_rui_log_item, rui_item);
	28	}
	29
	30	void
	31	xfs_rui_item_free(
	32	struct xfs_rui_log_item *ruip)
	33	{
	34	if (ruip->rui_format.rui_nextents > XFS_RUI_MAX_FAST_EXTENTS)
	35	kmem_free(ruip);
	36	else
377bcd5f	37	kmem_cache_free(xfs_rui_zone, ruip);
5880f2d7 DW	38	}
5880f2d7 DW	39
0612d116 DC	40	/*
	41	* Freeing the RUI requires that we remove it from the AIL if it has already
	42	* been placed there. However, the RUI may not yet have been placed in the AIL
	43	* when called by xfs_rui_release() from RUD processing due to the ordering of
	44	* committed vs unpin operations in bulk insert operations. Hence the reference
	45	* count to ensure only the last caller frees the RUI.
	46	*/
	47	void
	48	xfs_rui_release(
	49	struct xfs_rui_log_item *ruip)
	50	{
	51	ASSERT(atomic_read(&ruip->rui_refcount) > 0);
	52	if (atomic_dec_and_test(&ruip->rui_refcount)) {
	53	xfs_trans_ail_remove(&ruip->rui_item, SHUTDOWN_LOG_IO_ERROR);
	54	xfs_rui_item_free(ruip);
	55	}
	56	}
	57
5880f2d7 DW	58	STATIC void
	59	xfs_rui_item_size(
	60	struct xfs_log_item *lip,
	61	int *nvecs,
	62	int *nbytes)
	63	{
cd00158c DW	64	struct xfs_rui_log_item *ruip = RUI_ITEM(lip);
cd00158c DW	65
5880f2d7	66	*nvecs += 1;
cd00158c	67	*nbytes += xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents);
5880f2d7 DW	68	}
	69
	70	/*
	71	* This is called to fill in the vector of log iovecs for the
	72	* given rui log item. We use only 1 iovec, and we point that
	73	* at the rui_log_format structure embedded in the rui item.
	74	* It is at this point that we assert that all of the extent
	75	* slots in the rui item have been filled.
	76	*/
	77	STATIC void
	78	xfs_rui_item_format(
	79	struct xfs_log_item *lip,
	80	struct xfs_log_vec *lv)
	81	{
	82	struct xfs_rui_log_item *ruip = RUI_ITEM(lip);
	83	struct xfs_log_iovec *vecp = NULL;
	84
	85	ASSERT(atomic_read(&ruip->rui_next_extent) ==
	86	ruip->rui_format.rui_nextents);
	87
	88	ruip->rui_format.rui_type = XFS_LI_RUI;
	89	ruip->rui_format.rui_size = 1;
	90
	91	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_RUI_FORMAT, &ruip->rui_format,
cd00158c	92	xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents));
5880f2d7 DW	93	}
5880f2d7 DW	94
5880f2d7 DW	95	/*
	96	* The unpin operation is the last place an RUI is manipulated in the log. It is
	97	* either inserted in the AIL or aborted in the event of a log I/O error. In
	98	* either case, the RUI transaction has been successfully committed to make it
	99	* this far. Therefore, we expect whoever committed the RUI to either construct
	100	* and commit the RUD or drop the RUD's reference in the event of error. Simply
	101	* drop the log's RUI reference now that the log is done with it.
	102	*/
	103	STATIC void
	104	xfs_rui_item_unpin(
	105	struct xfs_log_item *lip,
	106	int remove)
	107	{
	108	struct xfs_rui_log_item *ruip = RUI_ITEM(lip);
	109
	110	xfs_rui_release(ruip);
	111	}
	112
5880f2d7 DW	113	/*
	114	* The RUI has been either committed or aborted if the transaction has been
	115	* cancelled. If the transaction was cancelled, an RUD isn't going to be
	116	* constructed and thus we free the RUI here directly.
	117	*/
	118	STATIC void
ddf92053	119	xfs_rui_item_release(
5880f2d7 DW	120	struct xfs_log_item *lip)
5880f2d7 DW	121	{
ddf92053	122	xfs_rui_release(RUI_ITEM(lip));
5880f2d7 DW	123	}
5880f2d7 DW	124
5880f2d7 DW	125	static const struct xfs_item_ops xfs_rui_item_ops = {
	126	.iop_size = xfs_rui_item_size,
	127	.iop_format = xfs_rui_item_format,
5880f2d7	128	.iop_unpin = xfs_rui_item_unpin,
ddf92053	129	.iop_release = xfs_rui_item_release,
5880f2d7 DW	130	};
	131
	132	/*
	133	* Allocate and initialize an rui item with the given number of extents.
	134	*/
	135	struct xfs_rui_log_item *
	136	xfs_rui_init(
	137	struct xfs_mount *mp,
	138	uint nextents)
	139
	140	{
	141	struct xfs_rui_log_item *ruip;
5880f2d7 DW	142
5880f2d7 DW	143	ASSERT(nextents > 0);
cd00158c	144	if (nextents > XFS_RUI_MAX_FAST_EXTENTS)
707e0dda	145	ruip = kmem_zalloc(xfs_rui_log_item_sizeof(nextents), 0);
cd00158c	146	else
707e0dda	147	ruip = kmem_zone_zalloc(xfs_rui_zone, 0);
5880f2d7 DW	148
	149	xfs_log_item_init(mp, &ruip->rui_item, XFS_LI_RUI, &xfs_rui_item_ops);
	150	ruip->rui_format.rui_nextents = nextents;
	151	ruip->rui_format.rui_id = (uintptr_t)(void *)ruip;
	152	atomic_set(&ruip->rui_next_extent, 0);
	153	atomic_set(&ruip->rui_refcount, 2);
	154
	155	return ruip;
	156	}
	157
	158	/*
	159	* Copy an RUI format buffer from the given buf, and into the destination
	160	* RUI format structure. The RUI/RUD items were designed not to need any
	161	* special alignment handling.
	162	*/
	163	int
	164	xfs_rui_copy_format(
	165	struct xfs_log_iovec *buf,
	166	struct xfs_rui_log_format *dst_rui_fmt)
	167	{
	168	struct xfs_rui_log_format *src_rui_fmt;
	169	uint len;
	170
	171	src_rui_fmt = buf->i_addr;
cd00158c	172	len = xfs_rui_log_format_sizeof(src_rui_fmt->rui_nextents);
5880f2d7	173
a5155b87 DW	174	if (buf->i_len != len) {
a5155b87 DW	175	XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL);
5880f2d7	176	return -EFSCORRUPTED;
a5155b87	177	}
5880f2d7	178
cd00158c	179	memcpy(dst_rui_fmt, src_rui_fmt, len);
5880f2d7 DW	180	return 0;
	181	}
	182
5880f2d7 DW	183	static inline struct xfs_rud_log_item RUD_ITEM(struct xfs_log_item lip)
	184	{
	185	return container_of(lip, struct xfs_rud_log_item, rud_item);
	186	}
	187
5880f2d7 DW	188	STATIC void
	189	xfs_rud_item_size(
	190	struct xfs_log_item *lip,
	191	int *nvecs,
	192	int *nbytes)
	193	{
	194	*nvecs += 1;
722e2517	195	*nbytes += sizeof(struct xfs_rud_log_format);
5880f2d7 DW	196	}
	197
	198	/*
	199	* This is called to fill in the vector of log iovecs for the
	200	* given rud log item. We use only 1 iovec, and we point that
	201	* at the rud_log_format structure embedded in the rud item.
	202	* It is at this point that we assert that all of the extent
	203	* slots in the rud item have been filled.
	204	*/
	205	STATIC void
	206	xfs_rud_item_format(
	207	struct xfs_log_item *lip,
	208	struct xfs_log_vec *lv)
	209	{
	210	struct xfs_rud_log_item *rudp = RUD_ITEM(lip);
	211	struct xfs_log_iovec *vecp = NULL;
	212
5880f2d7 DW	213	rudp->rud_format.rud_type = XFS_LI_RUD;
	214	rudp->rud_format.rud_size = 1;
	215
	216	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_RUD_FORMAT, &rudp->rud_format,
722e2517	217	sizeof(struct xfs_rud_log_format));
5880f2d7 DW	218	}
5880f2d7 DW	219
5880f2d7 DW	220	/*
	221	* The RUD is either committed or aborted if the transaction is cancelled. If
	222	* the transaction is cancelled, drop our reference to the RUI and free the
	223	* RUD.
	224	*/
	225	STATIC void
ddf92053	226	xfs_rud_item_release(
5880f2d7 DW	227	struct xfs_log_item *lip)
	228	{
	229	struct xfs_rud_log_item *rudp = RUD_ITEM(lip);
	230
ddf92053	231	xfs_rui_release(rudp->rud_ruip);
377bcd5f	232	kmem_cache_free(xfs_rud_zone, rudp);
5880f2d7 DW	233	}
5880f2d7 DW	234
5880f2d7	235	static const struct xfs_item_ops xfs_rud_item_ops = {
9ce632a2	236	.flags = XFS_ITEM_RELEASE_WHEN_COMMITTED,
5880f2d7 DW	237	.iop_size = xfs_rud_item_size,
5880f2d7 DW	238	.iop_format = xfs_rud_item_format,
ddf92053	239	.iop_release = xfs_rud_item_release,
5880f2d7 DW	240	};
5880f2d7 DW	241
3cfce1e3	242	static struct xfs_rud_log_item *
60883447 CH	243	xfs_trans_get_rud(
60883447 CH	244	struct xfs_trans *tp,
722e2517	245	struct xfs_rui_log_item *ruip)
5880f2d7	246	{
60883447	247	struct xfs_rud_log_item *rudp;
5880f2d7	248
707e0dda	249	rudp = kmem_zone_zalloc(xfs_rud_zone, 0);
60883447 CH	250	xfs_log_item_init(tp->t_mountp, &rudp->rud_item, XFS_LI_RUD,
60883447 CH	251	&xfs_rud_item_ops);
5880f2d7	252	rudp->rud_ruip = ruip;
5880f2d7 DW	253	rudp->rud_format.rud_rui_id = ruip->rui_format.rui_id;
5880f2d7 DW	254
60883447	255	xfs_trans_add_item(tp, &rudp->rud_item);
5880f2d7 DW	256	return rudp;
5880f2d7 DW	257	}
9e88b5d8	258
3cfce1e3 CH	259	/* Set the map extent flags for this reverse mapping. */
	260	static void
	261	xfs_trans_set_rmap_flags(
	262	struct xfs_map_extent *rmap,
	263	enum xfs_rmap_intent_type type,
	264	int whichfork,
	265	xfs_exntst_t state)
	266	{
	267	rmap->me_flags = 0;
	268	if (state == XFS_EXT_UNWRITTEN)
	269	rmap->me_flags \|= XFS_RMAP_EXTENT_UNWRITTEN;
	270	if (whichfork == XFS_ATTR_FORK)
	271	rmap->me_flags \|= XFS_RMAP_EXTENT_ATTR_FORK;
	272	switch (type) {
	273	case XFS_RMAP_MAP:
	274	rmap->me_flags \|= XFS_RMAP_EXTENT_MAP;
	275	break;
	276	case XFS_RMAP_MAP_SHARED:
	277	rmap->me_flags \|= XFS_RMAP_EXTENT_MAP_SHARED;
	278	break;
	279	case XFS_RMAP_UNMAP:
	280	rmap->me_flags \|= XFS_RMAP_EXTENT_UNMAP;
	281	break;
	282	case XFS_RMAP_UNMAP_SHARED:
	283	rmap->me_flags \|= XFS_RMAP_EXTENT_UNMAP_SHARED;
	284	break;
	285	case XFS_RMAP_CONVERT:
	286	rmap->me_flags \|= XFS_RMAP_EXTENT_CONVERT;
	287	break;
	288	case XFS_RMAP_CONVERT_SHARED:
	289	rmap->me_flags \|= XFS_RMAP_EXTENT_CONVERT_SHARED;
	290	break;
	291	case XFS_RMAP_ALLOC:
	292	rmap->me_flags \|= XFS_RMAP_EXTENT_ALLOC;
	293	break;
	294	case XFS_RMAP_FREE:
	295	rmap->me_flags \|= XFS_RMAP_EXTENT_FREE;
	296	break;
	297	default:
	298	ASSERT(0);
	299	}
	300	}
	301
	302	/*
	303	* Finish an rmap update and log it to the RUD. Note that the transaction is
	304	* marked dirty regardless of whether the rmap update succeeds or fails to
	305	* support the RUI/RUD lifecycle rules.
	306	*/
	307	static int
	308	xfs_trans_log_finish_rmap_update(
	309	struct xfs_trans *tp,
	310	struct xfs_rud_log_item *rudp,
	311	enum xfs_rmap_intent_type type,
	312	uint64_t owner,
	313	int whichfork,
	314	xfs_fileoff_t startoff,
	315	xfs_fsblock_t startblock,
	316	xfs_filblks_t blockcount,
	317	xfs_exntst_t state,
	318	struct xfs_btree_cur **pcur)
	319	{
	320	int error;
	321
	322	error = xfs_rmap_finish_one(tp, type, owner, whichfork, startoff,
323	startblock, blockcount, state, pcur);
324
325	/*
326	* Mark the transaction dirty, even on error. This ensures the
327	* transaction is aborted, which:
328	*
329	* 1.) releases the RUI and frees the RUD
330	* 2.) shuts down the filesystem
331	*/
332	tp->t_flags \|= XFS_TRANS_DIRTY;
333	set_bit(XFS_LI_DIRTY, &rudp->rud_item.li_flags);
334
335	return error;
336	}
337
338	/* Sort rmap intents by AG. */
339	static int
340	xfs_rmap_update_diff_items(
341	void *priv,
342	struct list_head *a,
343	struct list_head *b)
344	{
345	struct xfs_mount *mp = priv;
346	struct xfs_rmap_intent *ra;
347	struct xfs_rmap_intent *rb;
348
349	ra = container_of(a, struct xfs_rmap_intent, ri_list);
350	rb = container_of(b, struct xfs_rmap_intent, ri_list);
351	return XFS_FSB_TO_AGNO(mp, ra->ri_bmap.br_startblock) -
352	XFS_FSB_TO_AGNO(mp, rb->ri_bmap.br_startblock);
353	}
354
3cfce1e3 CH	355	/* Log rmap updates in the intent item. */
	356	STATIC void
	357	xfs_rmap_update_log_item(
	358	struct xfs_trans *tp,
c1f09188 CH	359	struct xfs_rui_log_item *ruip,
c1f09188 CH	360	struct xfs_rmap_intent *rmap)
3cfce1e3	361	{
3cfce1e3 CH	362	uint next_extent;
	363	struct xfs_map_extent *map;
	364
3cfce1e3 CH	365	tp->t_flags \|= XFS_TRANS_DIRTY;
	366	set_bit(XFS_LI_DIRTY, &ruip->rui_item.li_flags);
	367
	368	/*
	369	* atomic_inc_return gives us the value after the increment;
	370	* we want to use it as an array index so we need to subtract 1 from
	371	* it.
	372	*/
	373	next_extent = atomic_inc_return(&ruip->rui_next_extent) - 1;
	374	ASSERT(next_extent < ruip->rui_format.rui_nextents);
	375	map = &ruip->rui_format.rui_extents[next_extent];
	376	map->me_owner = rmap->ri_owner;
	377	map->me_startblock = rmap->ri_bmap.br_startblock;
	378	map->me_startoff = rmap->ri_bmap.br_startoff;
	379	map->me_len = rmap->ri_bmap.br_blockcount;
	380	xfs_trans_set_rmap_flags(map, rmap->ri_type, rmap->ri_whichfork,
	381	rmap->ri_bmap.br_state);
	382	}
	383
13a83333	384	static struct xfs_log_item *
c1f09188 CH	385	xfs_rmap_update_create_intent(
	386	struct xfs_trans *tp,
	387	struct list_head *items,
d367a868 CH	388	unsigned int count,
d367a868 CH	389	bool sort)
c1f09188 CH	390	{
	391	struct xfs_mount *mp = tp->t_mountp;
	392	struct xfs_rui_log_item *ruip = xfs_rui_init(mp, count);
	393	struct xfs_rmap_intent *rmap;
	394
	395	ASSERT(count > 0);
	396
	397	xfs_trans_add_item(tp, &ruip->rui_item);
d367a868 CH	398	if (sort)
d367a868 CH	399	list_sort(mp, items, xfs_rmap_update_diff_items);
c1f09188 CH	400	list_for_each_entry(rmap, items, ri_list)
c1f09188 CH	401	xfs_rmap_update_log_item(tp, ruip, rmap);
13a83333	402	return &ruip->rui_item;
c1f09188 CH	403	}
c1f09188 CH	404
3cfce1e3 CH	405	/* Get an RUD so we can process all the deferred rmap updates. */
	406	STATIC void *
	407	xfs_rmap_update_create_done(
	408	struct xfs_trans *tp,
13a83333	409	struct xfs_log_item *intent,
3cfce1e3 CH	410	unsigned int count)
3cfce1e3 CH	411	{
13a83333	412	return xfs_trans_get_rud(tp, RUI_ITEM(intent));
3cfce1e3 CH	413	}
	414
	415	/* Process a deferred rmap update. */
	416	STATIC int
	417	xfs_rmap_update_finish_item(
	418	struct xfs_trans *tp,
	419	struct list_head *item,
	420	void *done_item,
	421	void **state)
	422	{
	423	struct xfs_rmap_intent *rmap;
	424	int error;
	425
	426	rmap = container_of(item, struct xfs_rmap_intent, ri_list);
	427	error = xfs_trans_log_finish_rmap_update(tp, done_item,
	428	rmap->ri_type,
	429	rmap->ri_owner, rmap->ri_whichfork,
	430	rmap->ri_bmap.br_startoff,
	431	rmap->ri_bmap.br_startblock,
	432	rmap->ri_bmap.br_blockcount,
	433	rmap->ri_bmap.br_state,
	434	(struct xfs_btree_cur **)state);
	435	kmem_free(rmap);
	436	return error;
	437	}
	438
	439	/* Clean up after processing deferred rmaps. */
	440	STATIC void
	441	xfs_rmap_update_finish_cleanup(
	442	struct xfs_trans *tp,
	443	void *state,
	444	int error)
	445	{
	446	struct xfs_btree_cur *rcur = state;
	447
	448	xfs_rmap_finish_one_cleanup(tp, rcur, error);
	449	}
	450
	451	/* Abort all pending RUIs. */
	452	STATIC void
	453	xfs_rmap_update_abort_intent(
13a83333	454	struct xfs_log_item *intent)
3cfce1e3	455	{
13a83333	456	xfs_rui_release(RUI_ITEM(intent));
3cfce1e3 CH	457	}
	458
	459	/* Cancel a deferred rmap update. */
	460	STATIC void
	461	xfs_rmap_update_cancel_item(
	462	struct list_head *item)
	463	{
	464	struct xfs_rmap_intent *rmap;
	465
	466	rmap = container_of(item, struct xfs_rmap_intent, ri_list);
	467	kmem_free(rmap);
	468	}
	469
	470	const struct xfs_defer_op_type xfs_rmap_update_defer_type = {
	471	.max_items = XFS_RUI_MAX_FAST_EXTENTS,
3cfce1e3 CH	472	.create_intent = xfs_rmap_update_create_intent,
3cfce1e3 CH	473	.abort_intent = xfs_rmap_update_abort_intent,
3cfce1e3 CH	474	.create_done = xfs_rmap_update_create_done,
	475	.finish_item = xfs_rmap_update_finish_item,
	476	.finish_cleanup = xfs_rmap_update_finish_cleanup,
	477	.cancel_item = xfs_rmap_update_cancel_item,
	478	};
	479
9e88b5d8 DW	480	/*
	481	* Process an rmap update intent item that was recovered from the log.
	482	* We need to update the rmapbt.
	483	*/
	484	int
	485	xfs_rui_recover(
	486	struct xfs_mount *mp,
	487	struct xfs_rui_log_item *ruip)
	488	{
	489	int i;
	490	int error = 0;
	491	struct xfs_map_extent *rmap;
	492	xfs_fsblock_t startblock_fsb;
	493	bool op_ok;
9c194644 DW	494	struct xfs_rud_log_item *rudp;
	495	enum xfs_rmap_intent_type type;
	496	int whichfork;
	497	xfs_exntst_t state;
	498	struct xfs_trans *tp;
	499	struct xfs_btree_cur *rcur = NULL;
9e88b5d8 DW	500
	501	ASSERT(!test_bit(XFS_RUI_RECOVERED, &ruip->rui_flags));
	502
	503	/*
	504	* First check the validity of the extents described by the
	505	* RUI. If any are bad, then assume that all are bad and
	506	* just toss the RUI.
	507	*/
	508	for (i = 0; i < ruip->rui_format.rui_nextents; i++) {
e127fafd	509	rmap = &ruip->rui_format.rui_extents[i];
9e88b5d8 DW	510	startblock_fsb = XFS_BB_TO_FSB(mp,
	511	XFS_FSB_TO_DADDR(mp, rmap->me_startblock));
	512	switch (rmap->me_flags & XFS_RMAP_EXTENT_TYPE_MASK) {
	513	case XFS_RMAP_EXTENT_MAP:
0e07c039	514	case XFS_RMAP_EXTENT_MAP_SHARED:
9e88b5d8	515	case XFS_RMAP_EXTENT_UNMAP:
0e07c039	516	case XFS_RMAP_EXTENT_UNMAP_SHARED:
9e88b5d8	517	case XFS_RMAP_EXTENT_CONVERT:
0e07c039	518	case XFS_RMAP_EXTENT_CONVERT_SHARED:
9e88b5d8 DW	519	case XFS_RMAP_EXTENT_ALLOC:
	520	case XFS_RMAP_EXTENT_FREE:
	521	op_ok = true;
	522	break;
	523	default:
	524	op_ok = false;
	525	break;
	526	}
e127fafd DW	527	if (!op_ok \|\| startblock_fsb == 0 \|\|
	528	rmap->me_len == 0 \|\|
	529	startblock_fsb >= mp->m_sb.sb_dblocks \|\|
	530	rmap->me_len >= mp->m_sb.sb_agblocks \|\|
9e88b5d8 DW	531	(rmap->me_flags & ~XFS_RMAP_EXTENT_FLAGS)) {
	532	/*
	533	* This will pull the RUI from the AIL and
	534	* free the memory associated with it.
	535	*/
	536	set_bit(XFS_RUI_RECOVERED, &ruip->rui_flags);
	537	xfs_rui_release(ruip);
895e196f	538	return -EFSCORRUPTED;
9e88b5d8 DW	539	}
	540	}
	541
b31c2bdc DW	542	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate,
b31c2bdc DW	543	mp->m_rmap_maxlevels, 0, XFS_TRANS_RESERVE, &tp);
9c194644 DW	544	if (error)
9c194644 DW	545	return error;
722e2517	546	rudp = xfs_trans_get_rud(tp, ruip);
9c194644 DW	547
9c194644 DW	548	for (i = 0; i < ruip->rui_format.rui_nextents; i++) {
e127fafd	549	rmap = &ruip->rui_format.rui_extents[i];
9c194644 DW	550	state = (rmap->me_flags & XFS_RMAP_EXTENT_UNWRITTEN) ?
	551	XFS_EXT_UNWRITTEN : XFS_EXT_NORM;
	552	whichfork = (rmap->me_flags & XFS_RMAP_EXTENT_ATTR_FORK) ?
	553	XFS_ATTR_FORK : XFS_DATA_FORK;
	554	switch (rmap->me_flags & XFS_RMAP_EXTENT_TYPE_MASK) {
	555	case XFS_RMAP_EXTENT_MAP:
	556	type = XFS_RMAP_MAP;
	557	break;
ceeb9c83 DW	558	case XFS_RMAP_EXTENT_MAP_SHARED:
	559	type = XFS_RMAP_MAP_SHARED;
	560	break;
9c194644 DW	561	case XFS_RMAP_EXTENT_UNMAP:
	562	type = XFS_RMAP_UNMAP;
	563	break;
ceeb9c83 DW	564	case XFS_RMAP_EXTENT_UNMAP_SHARED:
	565	type = XFS_RMAP_UNMAP_SHARED;
	566	break;
9c194644 DW	567	case XFS_RMAP_EXTENT_CONVERT:
	568	type = XFS_RMAP_CONVERT;
	569	break;
3f165b33 DW	570	case XFS_RMAP_EXTENT_CONVERT_SHARED:
	571	type = XFS_RMAP_CONVERT_SHARED;
	572	break;
9c194644 DW	573	case XFS_RMAP_EXTENT_ALLOC:
	574	type = XFS_RMAP_ALLOC;
	575	break;
	576	case XFS_RMAP_EXTENT_FREE:
	577	type = XFS_RMAP_FREE;
	578	break;
	579	default:
a5155b87	580	XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL);
9c194644 DW	581	error = -EFSCORRUPTED;
	582	goto abort_error;
	583	}
	584	error = xfs_trans_log_finish_rmap_update(tp, rudp, type,
	585	rmap->me_owner, whichfork,
	586	rmap->me_startoff, rmap->me_startblock,
	587	rmap->me_len, state, &rcur);
	588	if (error)
	589	goto abort_error;
	590
	591	}
	592
	593	xfs_rmap_finish_one_cleanup(tp, rcur, error);
9e88b5d8	594	set_bit(XFS_RUI_RECOVERED, &ruip->rui_flags);
9c194644 DW	595	error = xfs_trans_commit(tp);
	596	return error;
	597
	598	abort_error:
	599	xfs_rmap_finish_one_cleanup(tp, rcur, error);
	600	xfs_trans_cancel(tp);
9e88b5d8 DW	601	return error;
9e88b5d8 DW	602	}