[linux-2.6-block.git] / block / t10-pi.c

// SPDX-License-Identifier: GPL-2.0
/*
 * t10_pi.c - Functions for generating and verifying T10 Protection
 *	      Information.
 */

#include <linux/t10-pi.h>
#include <linux/blk-integrity.h>
#include <linux/crc-t10dif.h>
#include <linux/crc64.h>
#include <net/checksum.h>
#include <linux/unaligned.h>
#include "blk.h"

struct blk_integrity_iter {
	void			*prot_buf;
	void			*data_buf;
	sector_t		seed;
	unsigned int		data_size;
	unsigned short		interval;
	const char		*disk_name;
};

static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len,
		unsigned char csum_type)
{
	if (csum_type == BLK_INTEGRITY_CSUM_IP)
		return (__force __be16)ip_compute_csum(data, len);
	return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len));
}

/*
 * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
 * tag.
 */
static void t10_pi_generate(struct blk_integrity_iter *iter,
		struct blk_integrity *bi)
{
	u8 offset = bi->pi_offset;
	unsigned int i;

	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
		struct t10_pi_tuple *pi = iter->prot_buf + offset;

		pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval,
				bi->csum_type);
		if (offset)
			pi->guard_tag = t10_pi_csum(pi->guard_tag,
					iter->prot_buf, offset, bi->csum_type);
		pi->app_tag = 0;

		if (bi->flags & BLK_INTEGRITY_REF_TAG)
			pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
		else
			pi->ref_tag = 0;

		iter->data_buf += iter->interval;
		iter->prot_buf += bi->tuple_size;
		iter->seed++;
	}
}

static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
		struct blk_integrity *bi)
{
	u8 offset = bi->pi_offset;
	unsigned int i;

	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
		struct t10_pi_tuple *pi = iter->prot_buf + offset;
		__be16 csum;

		if (bi->flags & BLK_INTEGRITY_REF_TAG) {
			if (pi->app_tag == T10_PI_APP_ESCAPE)
				goto next;

			if (be32_to_cpu(pi->ref_tag) !=
			    lower_32_bits(iter->seed)) {
				pr_err("%s: ref tag error at location %llu " \
				       "(rcvd %u)\n", iter->disk_name,
				       (unsigned long long)
				       iter->seed, be32_to_cpu(pi->ref_tag));
				return BLK_STS_PROTECTION;
			}
		} else {
			if (pi->app_tag == T10_PI_APP_ESCAPE &&
			    pi->ref_tag == T10_PI_REF_ESCAPE)
				goto next;
		}

		csum = t10_pi_csum(0, iter->data_buf, iter->interval,
				bi->csum_type);
		if (offset)
			csum = t10_pi_csum(csum, iter->prot_buf, offset,
					bi->csum_type);

		if (pi->guard_tag != csum) {
			pr_err("%s: guard tag error at sector %llu " \
			       "(rcvd %04x, want %04x)\n", iter->disk_name,
			       (unsigned long long)iter->seed,
			       be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
			return BLK_STS_PROTECTION;
		}

next:
		iter->data_buf += iter->interval;
		iter->prot_buf += bi->tuple_size;
		iter->seed++;
	}

	return BLK_STS_OK;
}

/**
 * t10_pi_type1_prepare - prepare PI prior submitting request to device
 * @rq:              request with PI that should be prepared
 *
 * For Type 1/Type 2, the virtual start sector is the one that was
 * originally submitted by the block layer for the ref_tag usage. Due to
 * partitioning, MD/DM cloning, etc. the actual physical start sector is
 * likely to be different. Remap protection information to match the
 * physical LBA.
 */
static void t10_pi_type1_prepare(struct request *rq)
{
	struct blk_integrity *bi = &rq->q->limits.integrity;
	const int tuple_sz = bi->tuple_size;
	u32 ref_tag = t10_pi_ref_tag(rq);
	u8 offset = bi->pi_offset;
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u32 virt = bip_get_seed(bip) & 0xffffffff;
		struct bio_vec iv;
		struct bvec_iter iter;

		/* Already remapped? */
		if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
			break;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len; j += tuple_sz) {
				struct t10_pi_tuple *pi = p + offset;

				if (be32_to_cpu(pi->ref_tag) == virt)
					pi->ref_tag = cpu_to_be32(ref_tag);
				virt++;
				ref_tag++;
				p += tuple_sz;
			}
			kunmap_local(p);
		}

		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
	}
}

/**
 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
 * @rq:              request with PI that should be prepared
 * @nr_bytes:        total bytes to prepare
 *
 * For Type 1/Type 2, the virtual start sector is the one that was
 * originally submitted by the block layer for the ref_tag usage. Due to
 * partitioning, MD/DM cloning, etc. the actual physical start sector is
 * likely to be different. Since the physical start sector was submitted
 * to the device, we should remap it back to virtual values expected by the
 * block layer.
 */
static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
{
	struct blk_integrity *bi = &rq->q->limits.integrity;
	unsigned intervals = nr_bytes >> bi->interval_exp;
	const int tuple_sz = bi->tuple_size;
	u32 ref_tag = t10_pi_ref_tag(rq);
	u8 offset = bi->pi_offset;
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u32 virt = bip_get_seed(bip) & 0xffffffff;
		struct bio_vec iv;
		struct bvec_iter iter;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
				struct t10_pi_tuple *pi = p + offset;

				if (be32_to_cpu(pi->ref_tag) == ref_tag)
					pi->ref_tag = cpu_to_be32(virt);
				virt++;
				ref_tag++;
				intervals--;
				p += tuple_sz;
			}
			kunmap_local(p);
		}
	}
}

static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
{
	return cpu_to_be64(crc64_nvme(crc, data, len));
}

static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
		struct blk_integrity *bi)
{
	u8 offset = bi->pi_offset;
	unsigned int i;

	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
		struct crc64_pi_tuple *pi = iter->prot_buf + offset;

		pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval);
		if (offset)
			pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag),
					iter->prot_buf, offset);
		pi->app_tag = 0;

		if (bi->flags & BLK_INTEGRITY_REF_TAG)
			put_unaligned_be48(iter->seed, pi->ref_tag);
		else
			put_unaligned_be48(0ULL, pi->ref_tag);

		iter->data_buf += iter->interval;
		iter->prot_buf += bi->tuple_size;
		iter->seed++;
	}
}

static bool ext_pi_ref_escape(const u8 ref_tag[6])
{
	static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };

	return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
}

static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
		struct blk_integrity *bi)
{
	u8 offset = bi->pi_offset;
	unsigned int i;

	for (i = 0; i < iter->data_size; i += iter->interval) {
		struct crc64_pi_tuple *pi = iter->prot_buf + offset;
		u64 ref, seed;
		__be64 csum;

		if (bi->flags & BLK_INTEGRITY_REF_TAG) {
			if (pi->app_tag == T10_PI_APP_ESCAPE)
				goto next;

			ref = get_unaligned_be48(pi->ref_tag);
			seed = lower_48_bits(iter->seed);
			if (ref != seed) {
				pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
					iter->disk_name, seed, ref);
				return BLK_STS_PROTECTION;
			}
		} else {
			if (pi->app_tag == T10_PI_APP_ESCAPE &&
			    ext_pi_ref_escape(pi->ref_tag))
				goto next;
		}

		csum = ext_pi_crc64(0, iter->data_buf, iter->interval);
		if (offset)
			csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf,
					    offset);

		if (pi->guard_tag != csum) {
			pr_err("%s: guard tag error at sector %llu " \
			       "(rcvd %016llx, want %016llx)\n",
				iter->disk_name, (unsigned long long)iter->seed,
				be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
			return BLK_STS_PROTECTION;
		}

next:
		iter->data_buf += iter->interval;
		iter->prot_buf += bi->tuple_size;
		iter->seed++;
	}

	return BLK_STS_OK;
}

static void ext_pi_type1_prepare(struct request *rq)
{
	struct blk_integrity *bi = &rq->q->limits.integrity;
	const int tuple_sz = bi->tuple_size;
	u64 ref_tag = ext_pi_ref_tag(rq);
	u8 offset = bi->pi_offset;
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u64 virt = lower_48_bits(bip_get_seed(bip));
		struct bio_vec iv;
		struct bvec_iter iter;

		/* Already remapped? */
		if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
			break;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len; j += tuple_sz) {
				struct crc64_pi_tuple *pi = p +  offset;
				u64 ref = get_unaligned_be48(pi->ref_tag);

				if (ref == virt)
					put_unaligned_be48(ref_tag, pi->ref_tag);
				virt++;
				ref_tag++;
				p += tuple_sz;
			}
			kunmap_local(p);
		}

		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
	}
}

static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
{
	struct blk_integrity *bi = &rq->q->limits.integrity;
	unsigned intervals = nr_bytes >> bi->interval_exp;
	const int tuple_sz = bi->tuple_size;
	u64 ref_tag = ext_pi_ref_tag(rq);
	u8 offset = bi->pi_offset;
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u64 virt = lower_48_bits(bip_get_seed(bip));
		struct bio_vec iv;
		struct bvec_iter iter;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
				struct crc64_pi_tuple *pi = p + offset;
				u64 ref = get_unaligned_be48(pi->ref_tag);

				if (ref == ref_tag)
					put_unaligned_be48(virt, pi->ref_tag);
				virt++;
				ref_tag++;
				intervals--;
				p += tuple_sz;
			}
			kunmap_local(p);
		}
	}
}

void blk_integrity_generate(struct bio *bio)
{
	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	struct bio_integrity_payload *bip = bio_integrity(bio);
	struct blk_integrity_iter iter;
	struct bvec_iter bviter;
	struct bio_vec bv;

	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
	iter.interval = 1 << bi->interval_exp;
	iter.seed = bio->bi_iter.bi_sector;
	iter.prot_buf = bvec_virt(bip->bip_vec);
	bio_for_each_segment(bv, bio, bviter) {
		void *kaddr = bvec_kmap_local(&bv);

		iter.data_buf = kaddr;
		iter.data_size = bv.bv_len;
		switch (bi->csum_type) {
		case BLK_INTEGRITY_CSUM_CRC64:
			ext_pi_crc64_generate(&iter, bi);
			break;
		case BLK_INTEGRITY_CSUM_CRC:
		case BLK_INTEGRITY_CSUM_IP:
			t10_pi_generate(&iter, bi);
			break;
		default:
			break;
		}
		kunmap_local(kaddr);
	}
}

void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter)
{
	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	struct bio_integrity_payload *bip = bio_integrity(bio);
	struct blk_integrity_iter iter;
	struct bvec_iter bviter;
	struct bio_vec bv;

	/*
	 * At the moment verify is called bi_iter has been advanced during split
	 * and completion, so use the copy created during submission here.
	 */
	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
	iter.interval = 1 << bi->interval_exp;
	iter.seed = saved_iter->bi_sector;
	iter.prot_buf = bvec_virt(bip->bip_vec);
	__bio_for_each_segment(bv, bio, bviter, *saved_iter) {
		void *kaddr = bvec_kmap_local(&bv);
		blk_status_t ret = BLK_STS_OK;

		iter.data_buf = kaddr;
		iter.data_size = bv.bv_len;
		switch (bi->csum_type) {
		case BLK_INTEGRITY_CSUM_CRC64:
			ret = ext_pi_crc64_verify(&iter, bi);
			break;
		case BLK_INTEGRITY_CSUM_CRC:
		case BLK_INTEGRITY_CSUM_IP:
			ret = t10_pi_verify(&iter, bi);
			break;
		default:
			break;
		}
		kunmap_local(kaddr);

		if (ret) {
			bio->bi_status = ret;
			return;
		}
	}
}

void blk_integrity_prepare(struct request *rq)
{
	struct blk_integrity *bi = &rq->q->limits.integrity;

	if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
		return;

	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
		ext_pi_type1_prepare(rq);
	else
		t10_pi_type1_prepare(rq);
}

void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)
{
	struct blk_integrity *bi = &rq->q->limits.integrity;

	if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
		return;

	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
		ext_pi_type1_complete(rq, nr_bytes);
	else
		t10_pi_type1_complete(rq, nr_bytes);
}
Commit	Line	Data
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* t10_pi.c - Functions for generating and verifying T10 Protection
	4	* Information.
	5	*/
	6
	7	#include <linux/t10-pi.h>
	8	#include <linux/blk-integrity.h>
	9	#include <linux/crc-t10dif.h>
	10	#include <linux/crc64.h>
	11	#include <net/checksum.h>
	12	#include <linux/unaligned.h>
	13	#include "blk.h"
	14
	15	struct blk_integrity_iter {
	16	void *prot_buf;
	17	void *data_buf;
	18	sector_t seed;
	19	unsigned int data_size;
	20	unsigned short interval;
	21	const char *disk_name;
	22	};
	23
	24	static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len,
	25	unsigned char csum_type)
	26	{
	27	if (csum_type == BLK_INTEGRITY_CSUM_IP)
	28	return (__force __be16)ip_compute_csum(data, len);
	29	return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len));
	30	}
	31
	32	/*
	33	* Type 1 and Type 2 protection use the same format: 16 bit guard tag,
	34	* 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
	35	* tag.
	36	*/
	37	static void t10_pi_generate(struct blk_integrity_iter *iter,
	38	struct blk_integrity *bi)
	39	{
	40	u8 offset = bi->pi_offset;
	41	unsigned int i;
	42
	43	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
	44	struct t10_pi_tuple *pi = iter->prot_buf + offset;
	45
	46	pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval,
	47	bi->csum_type);
	48	if (offset)
	49	pi->guard_tag = t10_pi_csum(pi->guard_tag,
	50	iter->prot_buf, offset, bi->csum_type);
	51	pi->app_tag = 0;
	52
	53	if (bi->flags & BLK_INTEGRITY_REF_TAG)
	54	pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
	55	else
	56	pi->ref_tag = 0;
	57
	58	iter->data_buf += iter->interval;
	59	iter->prot_buf += bi->tuple_size;
	60	iter->seed++;
	61	}
	62	}
	63
	64	static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
	65	struct blk_integrity *bi)
	66	{
	67	u8 offset = bi->pi_offset;
	68	unsigned int i;
	69
	70	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
	71	struct t10_pi_tuple *pi = iter->prot_buf + offset;
	72	__be16 csum;
	73
	74	if (bi->flags & BLK_INTEGRITY_REF_TAG) {
	75	if (pi->app_tag == T10_PI_APP_ESCAPE)
	76	goto next;
	77
	78	if (be32_to_cpu(pi->ref_tag) !=
	79	lower_32_bits(iter->seed)) {
	80	pr_err("%s: ref tag error at location %llu " \
	81	"(rcvd %u)\n", iter->disk_name,
	82	(unsigned long long)
	83	iter->seed, be32_to_cpu(pi->ref_tag));
	84	return BLK_STS_PROTECTION;
	85	}
	86	} else {
	87	if (pi->app_tag == T10_PI_APP_ESCAPE &&
	88	pi->ref_tag == T10_PI_REF_ESCAPE)
	89	goto next;
	90	}
	91
	92	csum = t10_pi_csum(0, iter->data_buf, iter->interval,
	93	bi->csum_type);
	94	if (offset)
	95	csum = t10_pi_csum(csum, iter->prot_buf, offset,
	96	bi->csum_type);
	97
	98	if (pi->guard_tag != csum) {
	99	pr_err("%s: guard tag error at sector %llu " \
	100	"(rcvd %04x, want %04x)\n", iter->disk_name,
	101	(unsigned long long)iter->seed,
	102	be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
	103	return BLK_STS_PROTECTION;
	104	}
	105
	106	next:
	107	iter->data_buf += iter->interval;
	108	iter->prot_buf += bi->tuple_size;
	109	iter->seed++;
	110	}
	111
	112	return BLK_STS_OK;
	113	}
	114
	115	/**
	116	* t10_pi_type1_prepare - prepare PI prior submitting request to device
	117	* @rq: request with PI that should be prepared
	118	*
	119	* For Type 1/Type 2, the virtual start sector is the one that was
	120	* originally submitted by the block layer for the ref_tag usage. Due to
	121	* partitioning, MD/DM cloning, etc. the actual physical start sector is
	122	* likely to be different. Remap protection information to match the
	123	* physical LBA.
	124	*/
	125	static void t10_pi_type1_prepare(struct request *rq)
	126	{
	127	struct blk_integrity *bi = &rq->q->limits.integrity;
	128	const int tuple_sz = bi->tuple_size;
	129	u32 ref_tag = t10_pi_ref_tag(rq);
	130	u8 offset = bi->pi_offset;
	131	struct bio *bio;
	132
	133	__rq_for_each_bio(bio, rq) {
	134	struct bio_integrity_payload *bip = bio_integrity(bio);
	135	u32 virt = bip_get_seed(bip) & 0xffffffff;
	136	struct bio_vec iv;
	137	struct bvec_iter iter;
	138
	139	/* Already remapped? */
	140	if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
	141	break;
	142
	143	bip_for_each_vec(iv, bip, iter) {
	144	unsigned int j;
	145	void *p;
	146
	147	p = bvec_kmap_local(&iv);
	148	for (j = 0; j < iv.bv_len; j += tuple_sz) {
	149	struct t10_pi_tuple *pi = p + offset;
	150
	151	if (be32_to_cpu(pi->ref_tag) == virt)
	152	pi->ref_tag = cpu_to_be32(ref_tag);
	153	virt++;
	154	ref_tag++;
	155	p += tuple_sz;
	156	}
	157	kunmap_local(p);
	158	}
	159
	160	bip->bip_flags \|= BIP_MAPPED_INTEGRITY;
	161	}
	162	}
	163
	164	/**
	165	* t10_pi_type1_complete - prepare PI prior returning request to the blk layer
	166	* @rq: request with PI that should be prepared
	167	* @nr_bytes: total bytes to prepare
	168	*
	169	* For Type 1/Type 2, the virtual start sector is the one that was
	170	* originally submitted by the block layer for the ref_tag usage. Due to
	171	* partitioning, MD/DM cloning, etc. the actual physical start sector is
	172	* likely to be different. Since the physical start sector was submitted
	173	* to the device, we should remap it back to virtual values expected by the
	174	* block layer.
	175	*/
	176	static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
	177	{
	178	struct blk_integrity *bi = &rq->q->limits.integrity;
	179	unsigned intervals = nr_bytes >> bi->interval_exp;
	180	const int tuple_sz = bi->tuple_size;
	181	u32 ref_tag = t10_pi_ref_tag(rq);
	182	u8 offset = bi->pi_offset;
	183	struct bio *bio;
	184
	185	__rq_for_each_bio(bio, rq) {
	186	struct bio_integrity_payload *bip = bio_integrity(bio);
	187	u32 virt = bip_get_seed(bip) & 0xffffffff;
	188	struct bio_vec iv;
	189	struct bvec_iter iter;
	190
	191	bip_for_each_vec(iv, bip, iter) {
	192	unsigned int j;
	193	void *p;
	194
	195	p = bvec_kmap_local(&iv);
	196	for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
	197	struct t10_pi_tuple *pi = p + offset;
	198
	199	if (be32_to_cpu(pi->ref_tag) == ref_tag)
	200	pi->ref_tag = cpu_to_be32(virt);
	201	virt++;
	202	ref_tag++;
	203	intervals--;
	204	p += tuple_sz;
	205	}
	206	kunmap_local(p);
	207	}
	208	}
	209	}
	210
	211	static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
	212	{
	213	return cpu_to_be64(crc64_nvme(crc, data, len));
	214	}
	215
	216	static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
	217	struct blk_integrity *bi)
	218	{
	219	u8 offset = bi->pi_offset;
	220	unsigned int i;
	221
	222	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
	223	struct crc64_pi_tuple *pi = iter->prot_buf + offset;
	224
	225	pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval);
	226	if (offset)
	227	pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag),
	228	iter->prot_buf, offset);
	229	pi->app_tag = 0;
	230
	231	if (bi->flags & BLK_INTEGRITY_REF_TAG)
	232	put_unaligned_be48(iter->seed, pi->ref_tag);
	233	else
	234	put_unaligned_be48(0ULL, pi->ref_tag);
	235
	236	iter->data_buf += iter->interval;
	237	iter->prot_buf += bi->tuple_size;
	238	iter->seed++;
	239	}
	240	}
	241
	242	static bool ext_pi_ref_escape(const u8 ref_tag[6])
	243	{
	244	static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
	245
	246	return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
	247	}
	248
	249	static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
	250	struct blk_integrity *bi)
	251	{
	252	u8 offset = bi->pi_offset;
	253	unsigned int i;
	254
	255	for (i = 0; i < iter->data_size; i += iter->interval) {
	256	struct crc64_pi_tuple *pi = iter->prot_buf + offset;
	257	u64 ref, seed;
	258	__be64 csum;
	259
	260	if (bi->flags & BLK_INTEGRITY_REF_TAG) {
	261	if (pi->app_tag == T10_PI_APP_ESCAPE)
	262	goto next;
	263
	264	ref = get_unaligned_be48(pi->ref_tag);
	265	seed = lower_48_bits(iter->seed);
	266	if (ref != seed) {
	267	pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
	268	iter->disk_name, seed, ref);
	269	return BLK_STS_PROTECTION;
	270	}
	271	} else {
	272	if (pi->app_tag == T10_PI_APP_ESCAPE &&
	273	ext_pi_ref_escape(pi->ref_tag))
	274	goto next;
	275	}
	276
	277	csum = ext_pi_crc64(0, iter->data_buf, iter->interval);
	278	if (offset)
	279	csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf,
	280	offset);
	281
	282	if (pi->guard_tag != csum) {
	283	pr_err("%s: guard tag error at sector %llu " \
	284	"(rcvd %016llx, want %016llx)\n",
	285	iter->disk_name, (unsigned long long)iter->seed,
	286	be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
	287	return BLK_STS_PROTECTION;
	288	}
	289
	290	next:
	291	iter->data_buf += iter->interval;
	292	iter->prot_buf += bi->tuple_size;
	293	iter->seed++;
	294	}
	295
	296	return BLK_STS_OK;
	297	}
	298
	299	static void ext_pi_type1_prepare(struct request *rq)
	300	{
	301	struct blk_integrity *bi = &rq->q->limits.integrity;
	302	const int tuple_sz = bi->tuple_size;
	303	u64 ref_tag = ext_pi_ref_tag(rq);
	304	u8 offset = bi->pi_offset;
	305	struct bio *bio;
	306
	307	__rq_for_each_bio(bio, rq) {
	308	struct bio_integrity_payload *bip = bio_integrity(bio);
	309	u64 virt = lower_48_bits(bip_get_seed(bip));
	310	struct bio_vec iv;
	311	struct bvec_iter iter;
	312
	313	/* Already remapped? */
	314	if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
	315	break;
	316
	317	bip_for_each_vec(iv, bip, iter) {
	318	unsigned int j;
	319	void *p;
	320
	321	p = bvec_kmap_local(&iv);
	322	for (j = 0; j < iv.bv_len; j += tuple_sz) {
	323	struct crc64_pi_tuple *pi = p + offset;
	324	u64 ref = get_unaligned_be48(pi->ref_tag);
	325
	326	if (ref == virt)
	327	put_unaligned_be48(ref_tag, pi->ref_tag);
	328	virt++;
	329	ref_tag++;
	330	p += tuple_sz;
	331	}
	332	kunmap_local(p);
	333	}
	334
	335	bip->bip_flags \|= BIP_MAPPED_INTEGRITY;
	336	}
	337	}
	338
	339	static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
	340	{
	341	struct blk_integrity *bi = &rq->q->limits.integrity;
	342	unsigned intervals = nr_bytes >> bi->interval_exp;
	343	const int tuple_sz = bi->tuple_size;
	344	u64 ref_tag = ext_pi_ref_tag(rq);
	345	u8 offset = bi->pi_offset;
	346	struct bio *bio;
	347
	348	__rq_for_each_bio(bio, rq) {
	349	struct bio_integrity_payload *bip = bio_integrity(bio);
	350	u64 virt = lower_48_bits(bip_get_seed(bip));
	351	struct bio_vec iv;
	352	struct bvec_iter iter;
	353
	354	bip_for_each_vec(iv, bip, iter) {
	355	unsigned int j;
	356	void *p;
	357
	358	p = bvec_kmap_local(&iv);
	359	for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
	360	struct crc64_pi_tuple *pi = p + offset;
	361	u64 ref = get_unaligned_be48(pi->ref_tag);
	362
	363	if (ref == ref_tag)
	364	put_unaligned_be48(virt, pi->ref_tag);
	365	virt++;
	366	ref_tag++;
	367	intervals--;
	368	p += tuple_sz;
	369	}
	370	kunmap_local(p);
	371	}
	372	}
	373	}
	374
	375	void blk_integrity_generate(struct bio *bio)
	376	{
	377	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	378	struct bio_integrity_payload *bip = bio_integrity(bio);
	379	struct blk_integrity_iter iter;
	380	struct bvec_iter bviter;
	381	struct bio_vec bv;
	382
	383	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
	384	iter.interval = 1 << bi->interval_exp;
	385	iter.seed = bio->bi_iter.bi_sector;
	386	iter.prot_buf = bvec_virt(bip->bip_vec);
	387	bio_for_each_segment(bv, bio, bviter) {
	388	void *kaddr = bvec_kmap_local(&bv);
	389
	390	iter.data_buf = kaddr;
	391	iter.data_size = bv.bv_len;
	392	switch (bi->csum_type) {
	393	case BLK_INTEGRITY_CSUM_CRC64:
	394	ext_pi_crc64_generate(&iter, bi);
	395	break;
	396	case BLK_INTEGRITY_CSUM_CRC:
	397	case BLK_INTEGRITY_CSUM_IP:
	398	t10_pi_generate(&iter, bi);
	399	break;
	400	default:
	401	break;
	402	}
	403	kunmap_local(kaddr);
	404	}
	405	}
	406
	407	void blk_integrity_verify_iter(struct bio bio, struct bvec_iter saved_iter)
	408	{
	409	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	410	struct bio_integrity_payload *bip = bio_integrity(bio);
	411	struct blk_integrity_iter iter;
	412	struct bvec_iter bviter;
	413	struct bio_vec bv;
	414
	415	/*
	416	* At the moment verify is called bi_iter has been advanced during split
	417	* and completion, so use the copy created during submission here.
	418	*/
	419	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
	420	iter.interval = 1 << bi->interval_exp;
	421	iter.seed = saved_iter->bi_sector;
	422	iter.prot_buf = bvec_virt(bip->bip_vec);
	423	__bio_for_each_segment(bv, bio, bviter, *saved_iter) {
	424	void *kaddr = bvec_kmap_local(&bv);
	425	blk_status_t ret = BLK_STS_OK;
	426
	427	iter.data_buf = kaddr;
	428	iter.data_size = bv.bv_len;
	429	switch (bi->csum_type) {
	430	case BLK_INTEGRITY_CSUM_CRC64:
	431	ret = ext_pi_crc64_verify(&iter, bi);
	432	break;
	433	case BLK_INTEGRITY_CSUM_CRC:
	434	case BLK_INTEGRITY_CSUM_IP:
	435	ret = t10_pi_verify(&iter, bi);
	436	break;
	437	default:
	438	break;
	439	}
	440	kunmap_local(kaddr);
	441
	442	if (ret) {
	443	bio->bi_status = ret;
	444	return;
	445	}
	446	}
	447	}
	448
	449	void blk_integrity_prepare(struct request *rq)
	450	{
	451	struct blk_integrity *bi = &rq->q->limits.integrity;
	452
	453	if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
	454	return;
	455
	456	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
	457	ext_pi_type1_prepare(rq);
	458	else
	459	t10_pi_type1_prepare(rq);
	460	}
	461
	462	void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)
	463	{
	464	struct blk_integrity *bi = &rq->q->limits.integrity;
	465
	466	if (!(bi->flags & BLK_INTEGRITY_REF_TAG))
	467	return;
	468
	469	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)
	470	ext_pi_type1_complete(rq, nr_bytes);
	471	else
	472	t10_pi_type1_complete(rq, nr_bytes);
	473	}