[linux-block.git] / drivers / md / dm-io-rewind.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2022 Red Hat, Inc.
 */

#include <linux/bio.h>
#include <linux/blk-crypto.h>
#include <linux/blk-integrity.h>

#include "dm-core.h"

static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv,
				       struct bvec_iter *iter,
				       unsigned int bytes)
{
	int idx;

	iter->bi_size += bytes;
	if (bytes <= iter->bi_bvec_done) {
		iter->bi_bvec_done -= bytes;
		return true;
	}

	bytes -= iter->bi_bvec_done;
	idx = iter->bi_idx - 1;

	while (idx >= 0 && bytes && bytes > bv[idx].bv_len) {
		bytes -= bv[idx].bv_len;
		idx--;
	}

	if (WARN_ONCE(idx < 0 && bytes,
		      "Attempted to rewind iter beyond bvec's boundaries\n")) {
		iter->bi_size -= bytes;
		iter->bi_bvec_done = 0;
		iter->bi_idx = 0;
		return false;
	}

	iter->bi_idx = idx;
	iter->bi_bvec_done = bv[idx].bv_len - bytes;
	return true;
}

#if defined(CONFIG_BLK_DEV_INTEGRITY)

/**
 * dm_bio_integrity_rewind - Rewind integrity vector
 * @bio:	bio whose integrity vector to update
 * @bytes_done:	number of data bytes to rewind
 *
 * Description: This function calculates how many integrity bytes the
 * number of completed data bytes correspond to and rewind the
 * integrity vector accordingly.
 */
static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done)
{
	struct bio_integrity_payload *bip = bio_integrity(bio);
	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	unsigned int bytes = bio_integrity_bytes(bi, bytes_done >> 9);

	bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9);
	dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes);
}

#else /* CONFIG_BLK_DEV_INTEGRITY */

static inline void dm_bio_integrity_rewind(struct bio *bio,
					   unsigned int bytes_done)
{
}

#endif

#if defined(CONFIG_BLK_INLINE_ENCRYPTION)

/* Decrements @dun by @dec, treating @dun as a multi-limb integer. */
static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
				       unsigned int dec)
{
	int i;

	for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
		u64 prev = dun[i];

		dun[i] -= dec;
		if (dun[i] > prev)
			dec = 1;
		else
			dec = 0;
	}
}

static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
{
	struct bio_crypt_ctx *bc = bio->bi_crypt_context;

	dm_bio_crypt_dun_decrement(bc->bc_dun,
				   bytes >> bc->bc_key->data_unit_size_bits);
}

#else /* CONFIG_BLK_INLINE_ENCRYPTION */

static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
{
}

#endif

static inline void dm_bio_rewind_iter(const struct bio *bio,
				      struct bvec_iter *iter, unsigned int bytes)
{
	iter->bi_sector -= bytes >> 9;

	/* No advance means no rewind */
	if (bio_no_advance_iter(bio))
		iter->bi_size += bytes;
	else
		dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
}

/**
 * dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes.
 * @bio: bio to rewind
 * @bytes: how many bytes to rewind
 *
 * WARNING:
 * Caller must ensure that @bio has a fixed end sector, to allow
 * rewinding from end of bio and restoring its original position.
 * Caller is also responsibile for restoring bio's size.
 */
static void dm_bio_rewind(struct bio *bio, unsigned int bytes)
{
	if (bio_integrity(bio))
		dm_bio_integrity_rewind(bio, bytes);

	if (bio_has_crypt_ctx(bio))
		dm_bio_crypt_rewind(bio, bytes);

	dm_bio_rewind_iter(bio, &bio->bi_iter, bytes);
}

void dm_io_rewind(struct dm_io *io, struct bio_set *bs)
{
	struct bio *orig = io->orig_bio;
	struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig,
					       GFP_NOIO, bs);
	/*
	 * dm_bio_rewind can restore to previous position since the
	 * end sector is fixed for original bio, but we still need
	 * to restore bio's size manually (using io->sectors).
	 */
	dm_bio_rewind(new_orig, ((io->sector_offset << 9) -
				 orig->bi_iter.bi_size));
	bio_trim(new_orig, 0, io->sectors);

	bio_chain(new_orig, orig);
	/*
	 * __bi_remaining was increased (by dm_split_and_process_bio),
	 * so must drop the one added in bio_chain.
	 */
	atomic_dec(&orig->__bi_remaining);
	io->orig_bio = new_orig;
}
Commit	Line	Data
3bd94003	1	// SPDX-License-Identifier: GPL-2.0-only
61cbe788 ML	2	/*
	3	* Copyright 2022 Red Hat, Inc.
	4	*/
	5
	6	#include <linux/bio.h>
	7	#include <linux/blk-crypto.h>
	8	#include <linux/blk-integrity.h>
	9
	10	#include "dm-core.h"
	11
	12	static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv,
	13	struct bvec_iter *iter,
	14	unsigned int bytes)
	15	{
	16	int idx;
	17
	18	iter->bi_size += bytes;
	19	if (bytes <= iter->bi_bvec_done) {
	20	iter->bi_bvec_done -= bytes;
	21	return true;
	22	}
	23
	24	bytes -= iter->bi_bvec_done;
	25	idx = iter->bi_idx - 1;
	26
	27	while (idx >= 0 && bytes && bytes > bv[idx].bv_len) {
	28	bytes -= bv[idx].bv_len;
	29	idx--;
	30	}
	31
	32	if (WARN_ONCE(idx < 0 && bytes,
	33	"Attempted to rewind iter beyond bvec's boundaries\n")) {
	34	iter->bi_size -= bytes;
	35	iter->bi_bvec_done = 0;
	36	iter->bi_idx = 0;
	37	return false;
	38	}
	39
	40	iter->bi_idx = idx;
	41	iter->bi_bvec_done = bv[idx].bv_len - bytes;
	42	return true;
	43	}
	44
	45	#if defined(CONFIG_BLK_DEV_INTEGRITY)
	46
	47	/**
	48	* dm_bio_integrity_rewind - Rewind integrity vector
	49	* @bio: bio whose integrity vector to update
	50	* @bytes_done: number of data bytes to rewind
	51	*
	52	* Description: This function calculates how many integrity bytes the
	53	* number of completed data bytes correspond to and rewind the
	54	* integrity vector accordingly.
	55	*/
	56	static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done)
	57	{
	58	struct bio_integrity_payload *bip = bio_integrity(bio);
	59	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
86a3238c	60	unsigned int bytes = bio_integrity_bytes(bi, bytes_done >> 9);
61cbe788 ML	61
	62	bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9);
	63	dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes);
	64	}
	65
	66	#else /* CONFIG_BLK_DEV_INTEGRITY */
	67
	68	static inline void dm_bio_integrity_rewind(struct bio *bio,
	69	unsigned int bytes_done)
	70	{
61cbe788 ML	71	}
	72
	73	#endif
	74
	75	#if defined(CONFIG_BLK_INLINE_ENCRYPTION)
	76
	77	/* Decrements @dun by @dec, treating @dun as a multi-limb integer. */
	78	static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
	79	unsigned int dec)
	80	{
	81	int i;
	82
	83	for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
	84	u64 prev = dun[i];
	85
	86	dun[i] -= dec;
	87	if (dun[i] > prev)
	88	dec = 1;
	89	else
	90	dec = 0;
	91	}
	92	}
	93
	94	static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
	95	{
	96	struct bio_crypt_ctx *bc = bio->bi_crypt_context;
	97
	98	dm_bio_crypt_dun_decrement(bc->bc_dun,
	99	bytes >> bc->bc_key->data_unit_size_bits);
	100	}
	101
	102	#else /* CONFIG_BLK_INLINE_ENCRYPTION */
	103
	104	static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
	105	{
61cbe788 ML	106	}
	107
	108	#endif
	109
	110	static inline void dm_bio_rewind_iter(const struct bio *bio,
	111	struct bvec_iter *iter, unsigned int bytes)
	112	{
	113	iter->bi_sector -= bytes >> 9;
	114
	115	/* No advance means no rewind */
	116	if (bio_no_advance_iter(bio))
	117	iter->bi_size += bytes;
	118	else
	119	dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
	120	}
	121
	122	/**
	123	* dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes.
	124	* @bio: bio to rewind
	125	* @bytes: how many bytes to rewind
	126	*
	127	* WARNING:
	128	* Caller must ensure that @bio has a fixed end sector, to allow
	129	* rewinding from end of bio and restoring its original position.
	130	* Caller is also responsibile for restoring bio's size.
	131	*/
86a3238c	132	static void dm_bio_rewind(struct bio *bio, unsigned int bytes)
61cbe788 ML	133	{
	134	if (bio_integrity(bio))
	135	dm_bio_integrity_rewind(bio, bytes);
	136
	137	if (bio_has_crypt_ctx(bio))
	138	dm_bio_crypt_rewind(bio, bytes);
	139
	140	dm_bio_rewind_iter(bio, &bio->bi_iter, bytes);
	141	}
8b211aac ML	142
	143	void dm_io_rewind(struct dm_io io, struct bio_set bs)
	144	{
	145	struct bio *orig = io->orig_bio;
	146	struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig,
	147	GFP_NOIO, bs);
	148	/*
	149	* dm_bio_rewind can restore to previous position since the
	150	* end sector is fixed for original bio, but we still need
	151	* to restore bio's size manually (using io->sectors).
	152	*/
	153	dm_bio_rewind(new_orig, ((io->sector_offset << 9) -
	154	orig->bi_iter.bi_size));
	155	bio_trim(new_orig, 0, io->sectors);
	156
	157	bio_chain(new_orig, orig);
	158	/*
	159	* __bi_remaining was increased (by dm_split_and_process_bio),
	160	* so must drop the one added in bio_chain.
	161	*/
	162	atomic_dec(&orig->__bi_remaining);
	163	io->orig_bio = new_orig;
	164	}