Commit | Line | Data |
---|---|---|
8c16567d | 1 | // SPDX-License-Identifier: GPL-2.0 |
2341c2f8 MP |
2 | /* |
3 | * t10_pi.c - Functions for generating and verifying T10 Protection | |
4 | * Information. | |
2341c2f8 MP |
5 | */ |
6 | ||
7 | #include <linux/t10-pi.h> | |
fe45e630 | 8 | #include <linux/blk-integrity.h> |
2341c2f8 | 9 | #include <linux/crc-t10dif.h> |
a7d4383f | 10 | #include <linux/crc64.h> |
2341c2f8 | 11 | #include <net/checksum.h> |
a7d4383f | 12 | #include <asm/unaligned.h> |
e9f5f44a | 13 | #include "blk.h" |
2341c2f8 | 14 | |
d19b4634 CH |
15 | struct blk_integrity_iter { |
16 | void *prot_buf; | |
17 | void *data_buf; | |
18 | sector_t seed; | |
19 | unsigned int data_size; | |
20 | unsigned short interval; | |
21 | const char *disk_name; | |
22 | }; | |
23 | ||
e9f5f44a CH |
24 | static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len, |
25 | unsigned char csum_type) | |
2341c2f8 | 26 | { |
e9f5f44a CH |
27 | if (csum_type == BLK_INTEGRITY_CSUM_IP) |
28 | return (__force __be16)ip_compute_csum(data, len); | |
29 | return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len)); | |
2341c2f8 MP |
30 | } |
31 | ||
32 | /* | |
33 | * Type 1 and Type 2 protection use the same format: 16 bit guard tag, | |
34 | * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref | |
35 | * tag. | |
36 | */ | |
e9f5f44a CH |
37 | static void t10_pi_generate(struct blk_integrity_iter *iter, |
38 | struct blk_integrity *bi) | |
2341c2f8 | 39 | { |
e9f5f44a | 40 | u8 offset = bi->pi_offset; |
2341c2f8 MP |
41 | unsigned int i; |
42 | ||
43 | for (i = 0 ; i < iter->data_size ; i += iter->interval) { | |
60d21aac | 44 | struct t10_pi_tuple *pi = iter->prot_buf + offset; |
2341c2f8 | 45 | |
e9f5f44a CH |
46 | pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval, |
47 | bi->csum_type); | |
60d21aac | 48 | if (offset) |
e9f5f44a CH |
49 | pi->guard_tag = t10_pi_csum(pi->guard_tag, |
50 | iter->prot_buf, offset, bi->csum_type); | |
2341c2f8 MP |
51 | pi->app_tag = 0; |
52 | ||
e9f5f44a | 53 | if (bi->flags & BLK_INTEGRITY_REF_TAG) |
2341c2f8 MP |
54 | pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed)); |
55 | else | |
56 | pi->ref_tag = 0; | |
57 | ||
58 | iter->data_buf += iter->interval; | |
e9f5f44a | 59 | iter->prot_buf += bi->tuple_size; |
2341c2f8 MP |
60 | iter->seed++; |
61 | } | |
2341c2f8 MP |
62 | } |
63 | ||
4e4cbee9 | 64 | static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter, |
e9f5f44a | 65 | struct blk_integrity *bi) |
2341c2f8 | 66 | { |
e9f5f44a | 67 | u8 offset = bi->pi_offset; |
2341c2f8 MP |
68 | unsigned int i; |
69 | ||
70 | for (i = 0 ; i < iter->data_size ; i += iter->interval) { | |
60d21aac | 71 | struct t10_pi_tuple *pi = iter->prot_buf + offset; |
2341c2f8 MP |
72 | __be16 csum; |
73 | ||
e9f5f44a | 74 | if (bi->flags & BLK_INTEGRITY_REF_TAG) { |
128b6f9f | 75 | if (pi->app_tag == T10_PI_APP_ESCAPE) |
2341c2f8 MP |
76 | goto next; |
77 | ||
78 | if (be32_to_cpu(pi->ref_tag) != | |
79 | lower_32_bits(iter->seed)) { | |
80 | pr_err("%s: ref tag error at location %llu " \ | |
81 | "(rcvd %u)\n", iter->disk_name, | |
82 | (unsigned long long) | |
83 | iter->seed, be32_to_cpu(pi->ref_tag)); | |
a462b950 | 84 | return BLK_STS_PROTECTION; |
2341c2f8 | 85 | } |
e9f5f44a | 86 | } else { |
128b6f9f DM |
87 | if (pi->app_tag == T10_PI_APP_ESCAPE && |
88 | pi->ref_tag == T10_PI_REF_ESCAPE) | |
2341c2f8 | 89 | goto next; |
2341c2f8 MP |
90 | } |
91 | ||
e9f5f44a CH |
92 | csum = t10_pi_csum(0, iter->data_buf, iter->interval, |
93 | bi->csum_type); | |
60d21aac | 94 | if (offset) |
e9f5f44a CH |
95 | csum = t10_pi_csum(csum, iter->prot_buf, offset, |
96 | bi->csum_type); | |
2341c2f8 MP |
97 | |
98 | if (pi->guard_tag != csum) { | |
99 | pr_err("%s: guard tag error at sector %llu " \ | |
100 | "(rcvd %04x, want %04x)\n", iter->disk_name, | |
101 | (unsigned long long)iter->seed, | |
102 | be16_to_cpu(pi->guard_tag), be16_to_cpu(csum)); | |
4e4cbee9 | 103 | return BLK_STS_PROTECTION; |
2341c2f8 MP |
104 | } |
105 | ||
106 | next: | |
107 | iter->data_buf += iter->interval; | |
e9f5f44a | 108 | iter->prot_buf += bi->tuple_size; |
2341c2f8 MP |
109 | iter->seed++; |
110 | } | |
111 | ||
4e4cbee9 | 112 | return BLK_STS_OK; |
2341c2f8 MP |
113 | } |
114 | ||
10c41ddd | 115 | /** |
54d4e6ab | 116 | * t10_pi_type1_prepare - prepare PI prior submitting request to device |
10c41ddd | 117 | * @rq: request with PI that should be prepared |
10c41ddd MG |
118 | * |
119 | * For Type 1/Type 2, the virtual start sector is the one that was | |
120 | * originally submitted by the block layer for the ref_tag usage. Due to | |
121 | * partitioning, MD/DM cloning, etc. the actual physical start sector is | |
122 | * likely to be different. Remap protection information to match the | |
123 | * physical LBA. | |
10c41ddd | 124 | */ |
54d4e6ab | 125 | static void t10_pi_type1_prepare(struct request *rq) |
10c41ddd | 126 | { |
c6e56cf6 | 127 | struct blk_integrity *bi = &rq->q->limits.integrity; |
60d21aac | 128 | const int tuple_sz = bi->tuple_size; |
10c41ddd | 129 | u32 ref_tag = t10_pi_ref_tag(rq); |
60d21aac | 130 | u8 offset = bi->pi_offset; |
10c41ddd MG |
131 | struct bio *bio; |
132 | ||
10c41ddd MG |
133 | __rq_for_each_bio(bio, rq) { |
134 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
135 | u32 virt = bip_get_seed(bip) & 0xffffffff; | |
136 | struct bio_vec iv; | |
137 | struct bvec_iter iter; | |
138 | ||
139 | /* Already remapped? */ | |
140 | if (bip->bip_flags & BIP_MAPPED_INTEGRITY) | |
141 | break; | |
142 | ||
143 | bip_for_each_vec(iv, bip, iter) { | |
10c41ddd | 144 | unsigned int j; |
8aec120a | 145 | void *p; |
10c41ddd | 146 | |
8aec120a | 147 | p = bvec_kmap_local(&iv); |
10c41ddd | 148 | for (j = 0; j < iv.bv_len; j += tuple_sz) { |
60d21aac | 149 | struct t10_pi_tuple *pi = p + offset; |
10c41ddd MG |
150 | |
151 | if (be32_to_cpu(pi->ref_tag) == virt) | |
152 | pi->ref_tag = cpu_to_be32(ref_tag); | |
153 | virt++; | |
154 | ref_tag++; | |
155 | p += tuple_sz; | |
156 | } | |
8aec120a | 157 | kunmap_local(p); |
10c41ddd MG |
158 | } |
159 | ||
160 | bip->bip_flags |= BIP_MAPPED_INTEGRITY; | |
161 | } | |
162 | } | |
10c41ddd MG |
163 | |
164 | /** | |
54d4e6ab | 165 | * t10_pi_type1_complete - prepare PI prior returning request to the blk layer |
10c41ddd | 166 | * @rq: request with PI that should be prepared |
54d4e6ab | 167 | * @nr_bytes: total bytes to prepare |
10c41ddd MG |
168 | * |
169 | * For Type 1/Type 2, the virtual start sector is the one that was | |
170 | * originally submitted by the block layer for the ref_tag usage. Due to | |
171 | * partitioning, MD/DM cloning, etc. the actual physical start sector is | |
172 | * likely to be different. Since the physical start sector was submitted | |
173 | * to the device, we should remap it back to virtual values expected by the | |
174 | * block layer. | |
10c41ddd | 175 | */ |
54d4e6ab | 176 | static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes) |
10c41ddd | 177 | { |
c6e56cf6 | 178 | struct blk_integrity *bi = &rq->q->limits.integrity; |
60d21aac KJ |
179 | unsigned intervals = nr_bytes >> bi->interval_exp; |
180 | const int tuple_sz = bi->tuple_size; | |
10c41ddd | 181 | u32 ref_tag = t10_pi_ref_tag(rq); |
60d21aac | 182 | u8 offset = bi->pi_offset; |
10c41ddd MG |
183 | struct bio *bio; |
184 | ||
10c41ddd MG |
185 | __rq_for_each_bio(bio, rq) { |
186 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
187 | u32 virt = bip_get_seed(bip) & 0xffffffff; | |
188 | struct bio_vec iv; | |
189 | struct bvec_iter iter; | |
190 | ||
191 | bip_for_each_vec(iv, bip, iter) { | |
10c41ddd | 192 | unsigned int j; |
8aec120a | 193 | void *p; |
10c41ddd | 194 | |
8aec120a | 195 | p = bvec_kmap_local(&iv); |
10c41ddd | 196 | for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { |
60d21aac | 197 | struct t10_pi_tuple *pi = p + offset; |
10c41ddd MG |
198 | |
199 | if (be32_to_cpu(pi->ref_tag) == ref_tag) | |
200 | pi->ref_tag = cpu_to_be32(virt); | |
201 | virt++; | |
202 | ref_tag++; | |
203 | intervals--; | |
204 | p += tuple_sz; | |
205 | } | |
8aec120a | 206 | kunmap_local(p); |
10c41ddd MG |
207 | } |
208 | } | |
209 | } | |
54d4e6ab | 210 | |
6b5c132a | 211 | static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len) |
a7d4383f | 212 | { |
6b5c132a | 213 | return cpu_to_be64(crc64_rocksoft_update(crc, data, len)); |
a7d4383f KB |
214 | } |
215 | ||
e9f5f44a CH |
216 | static void ext_pi_crc64_generate(struct blk_integrity_iter *iter, |
217 | struct blk_integrity *bi) | |
a7d4383f | 218 | { |
e9f5f44a | 219 | u8 offset = bi->pi_offset; |
a7d4383f KB |
220 | unsigned int i; |
221 | ||
222 | for (i = 0 ; i < iter->data_size ; i += iter->interval) { | |
60d21aac | 223 | struct crc64_pi_tuple *pi = iter->prot_buf + offset; |
a7d4383f | 224 | |
6b5c132a | 225 | pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval); |
60d21aac KJ |
226 | if (offset) |
227 | pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag), | |
228 | iter->prot_buf, offset); | |
a7d4383f KB |
229 | pi->app_tag = 0; |
230 | ||
e9f5f44a | 231 | if (bi->flags & BLK_INTEGRITY_REF_TAG) |
a7d4383f KB |
232 | put_unaligned_be48(iter->seed, pi->ref_tag); |
233 | else | |
234 | put_unaligned_be48(0ULL, pi->ref_tag); | |
235 | ||
236 | iter->data_buf += iter->interval; | |
e9f5f44a | 237 | iter->prot_buf += bi->tuple_size; |
a7d4383f KB |
238 | iter->seed++; |
239 | } | |
a7d4383f KB |
240 | } |
241 | ||
a28dc358 | 242 | static bool ext_pi_ref_escape(const u8 ref_tag[6]) |
a7d4383f | 243 | { |
a28dc358 | 244 | static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; |
a7d4383f KB |
245 | |
246 | return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0; | |
247 | } | |
248 | ||
249 | static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter, | |
e9f5f44a | 250 | struct blk_integrity *bi) |
a7d4383f | 251 | { |
e9f5f44a | 252 | u8 offset = bi->pi_offset; |
a7d4383f KB |
253 | unsigned int i; |
254 | ||
255 | for (i = 0; i < iter->data_size; i += iter->interval) { | |
60d21aac | 256 | struct crc64_pi_tuple *pi = iter->prot_buf + offset; |
a7d4383f KB |
257 | u64 ref, seed; |
258 | __be64 csum; | |
259 | ||
e9f5f44a | 260 | if (bi->flags & BLK_INTEGRITY_REF_TAG) { |
a7d4383f KB |
261 | if (pi->app_tag == T10_PI_APP_ESCAPE) |
262 | goto next; | |
263 | ||
264 | ref = get_unaligned_be48(pi->ref_tag); | |
265 | seed = lower_48_bits(iter->seed); | |
266 | if (ref != seed) { | |
267 | pr_err("%s: ref tag error at location %llu (rcvd %llu)\n", | |
268 | iter->disk_name, seed, ref); | |
269 | return BLK_STS_PROTECTION; | |
270 | } | |
e9f5f44a | 271 | } else { |
a7d4383f KB |
272 | if (pi->app_tag == T10_PI_APP_ESCAPE && |
273 | ext_pi_ref_escape(pi->ref_tag)) | |
274 | goto next; | |
275 | } | |
276 | ||
6b5c132a | 277 | csum = ext_pi_crc64(0, iter->data_buf, iter->interval); |
60d21aac KJ |
278 | if (offset) |
279 | csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf, | |
280 | offset); | |
281 | ||
a7d4383f KB |
282 | if (pi->guard_tag != csum) { |
283 | pr_err("%s: guard tag error at sector %llu " \ | |
284 | "(rcvd %016llx, want %016llx)\n", | |
285 | iter->disk_name, (unsigned long long)iter->seed, | |
286 | be64_to_cpu(pi->guard_tag), be64_to_cpu(csum)); | |
287 | return BLK_STS_PROTECTION; | |
288 | } | |
289 | ||
290 | next: | |
291 | iter->data_buf += iter->interval; | |
e9f5f44a | 292 | iter->prot_buf += bi->tuple_size; |
a7d4383f KB |
293 | iter->seed++; |
294 | } | |
295 | ||
296 | return BLK_STS_OK; | |
297 | } | |
298 | ||
a7d4383f KB |
299 | static void ext_pi_type1_prepare(struct request *rq) |
300 | { | |
c6e56cf6 | 301 | struct blk_integrity *bi = &rq->q->limits.integrity; |
60d21aac | 302 | const int tuple_sz = bi->tuple_size; |
a7d4383f | 303 | u64 ref_tag = ext_pi_ref_tag(rq); |
60d21aac | 304 | u8 offset = bi->pi_offset; |
a7d4383f KB |
305 | struct bio *bio; |
306 | ||
307 | __rq_for_each_bio(bio, rq) { | |
308 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
309 | u64 virt = lower_48_bits(bip_get_seed(bip)); | |
310 | struct bio_vec iv; | |
311 | struct bvec_iter iter; | |
312 | ||
313 | /* Already remapped? */ | |
314 | if (bip->bip_flags & BIP_MAPPED_INTEGRITY) | |
315 | break; | |
316 | ||
317 | bip_for_each_vec(iv, bip, iter) { | |
318 | unsigned int j; | |
319 | void *p; | |
320 | ||
321 | p = bvec_kmap_local(&iv); | |
322 | for (j = 0; j < iv.bv_len; j += tuple_sz) { | |
60d21aac | 323 | struct crc64_pi_tuple *pi = p + offset; |
a7d4383f KB |
324 | u64 ref = get_unaligned_be48(pi->ref_tag); |
325 | ||
326 | if (ref == virt) | |
327 | put_unaligned_be48(ref_tag, pi->ref_tag); | |
328 | virt++; | |
329 | ref_tag++; | |
330 | p += tuple_sz; | |
331 | } | |
332 | kunmap_local(p); | |
333 | } | |
334 | ||
335 | bip->bip_flags |= BIP_MAPPED_INTEGRITY; | |
336 | } | |
337 | } | |
338 | ||
339 | static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes) | |
340 | { | |
c6e56cf6 | 341 | struct blk_integrity *bi = &rq->q->limits.integrity; |
60d21aac KJ |
342 | unsigned intervals = nr_bytes >> bi->interval_exp; |
343 | const int tuple_sz = bi->tuple_size; | |
a7d4383f | 344 | u64 ref_tag = ext_pi_ref_tag(rq); |
60d21aac | 345 | u8 offset = bi->pi_offset; |
a7d4383f KB |
346 | struct bio *bio; |
347 | ||
348 | __rq_for_each_bio(bio, rq) { | |
349 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
350 | u64 virt = lower_48_bits(bip_get_seed(bip)); | |
351 | struct bio_vec iv; | |
352 | struct bvec_iter iter; | |
353 | ||
354 | bip_for_each_vec(iv, bip, iter) { | |
355 | unsigned int j; | |
356 | void *p; | |
357 | ||
358 | p = bvec_kmap_local(&iv); | |
359 | for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { | |
60d21aac | 360 | struct crc64_pi_tuple *pi = p + offset; |
a7d4383f KB |
361 | u64 ref = get_unaligned_be48(pi->ref_tag); |
362 | ||
363 | if (ref == ref_tag) | |
364 | put_unaligned_be48(virt, pi->ref_tag); | |
365 | virt++; | |
366 | ref_tag++; | |
367 | intervals--; | |
368 | p += tuple_sz; | |
369 | } | |
370 | kunmap_local(p); | |
371 | } | |
372 | } | |
373 | } | |
374 | ||
d19b4634 | 375 | void blk_integrity_generate(struct bio *bio) |
a7d4383f | 376 | { |
d19b4634 CH |
377 | struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); |
378 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
379 | struct blk_integrity_iter iter; | |
380 | struct bvec_iter bviter; | |
381 | struct bio_vec bv; | |
382 | ||
383 | iter.disk_name = bio->bi_bdev->bd_disk->disk_name; | |
384 | iter.interval = 1 << bi->interval_exp; | |
385 | iter.seed = bio->bi_iter.bi_sector; | |
386 | iter.prot_buf = bvec_virt(bip->bip_vec); | |
387 | bio_for_each_segment(bv, bio, bviter) { | |
388 | void *kaddr = bvec_kmap_local(&bv); | |
389 | ||
390 | iter.data_buf = kaddr; | |
391 | iter.data_size = bv.bv_len; | |
392 | switch (bi->csum_type) { | |
393 | case BLK_INTEGRITY_CSUM_CRC64: | |
394 | ext_pi_crc64_generate(&iter, bi); | |
395 | break; | |
396 | case BLK_INTEGRITY_CSUM_CRC: | |
397 | case BLK_INTEGRITY_CSUM_IP: | |
398 | t10_pi_generate(&iter, bi); | |
399 | break; | |
400 | default: | |
401 | break; | |
402 | } | |
403 | kunmap_local(kaddr); | |
e9f5f44a | 404 | } |
a7d4383f KB |
405 | } |
406 | ||
d19b4634 | 407 | void blk_integrity_verify(struct bio *bio) |
a7d4383f | 408 | { |
d19b4634 CH |
409 | struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); |
410 | struct bio_integrity_payload *bip = bio_integrity(bio); | |
411 | struct blk_integrity_iter iter; | |
412 | struct bvec_iter bviter; | |
413 | struct bio_vec bv; | |
414 | ||
415 | /* | |
416 | * At the moment verify is called bi_iter has been advanced during split | |
417 | * and completion, so use the copy created during submission here. | |
418 | */ | |
419 | iter.disk_name = bio->bi_bdev->bd_disk->disk_name; | |
420 | iter.interval = 1 << bi->interval_exp; | |
421 | iter.seed = bip->bio_iter.bi_sector; | |
422 | iter.prot_buf = bvec_virt(bip->bip_vec); | |
423 | __bio_for_each_segment(bv, bio, bviter, bip->bio_iter) { | |
424 | void *kaddr = bvec_kmap_local(&bv); | |
425 | blk_status_t ret = BLK_STS_OK; | |
426 | ||
427 | iter.data_buf = kaddr; | |
428 | iter.data_size = bv.bv_len; | |
429 | switch (bi->csum_type) { | |
430 | case BLK_INTEGRITY_CSUM_CRC64: | |
431 | ret = ext_pi_crc64_verify(&iter, bi); | |
432 | break; | |
433 | case BLK_INTEGRITY_CSUM_CRC: | |
434 | case BLK_INTEGRITY_CSUM_IP: | |
435 | ret = t10_pi_verify(&iter, bi); | |
436 | break; | |
437 | default: | |
438 | break; | |
439 | } | |
440 | kunmap_local(kaddr); | |
441 | ||
442 | if (ret) { | |
443 | bio->bi_status = ret; | |
444 | return; | |
445 | } | |
e9f5f44a | 446 | } |
a7d4383f KB |
447 | } |
448 | ||
e9f5f44a CH |
449 | void blk_integrity_prepare(struct request *rq) |
450 | { | |
c6e56cf6 | 451 | struct blk_integrity *bi = &rq->q->limits.integrity; |
e9f5f44a CH |
452 | |
453 | if (!(bi->flags & BLK_INTEGRITY_REF_TAG)) | |
454 | return; | |
455 | ||
456 | if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64) | |
457 | ext_pi_type1_prepare(rq); | |
458 | else | |
459 | t10_pi_type1_prepare(rq); | |
460 | } | |
461 | ||
462 | void blk_integrity_complete(struct request *rq, unsigned int nr_bytes) | |
463 | { | |
c6e56cf6 | 464 | struct blk_integrity *bi = &rq->q->limits.integrity; |
e9f5f44a CH |
465 | |
466 | if (!(bi->flags & BLK_INTEGRITY_REF_TAG)) | |
467 | return; | |
468 | ||
469 | if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64) | |
470 | ext_pi_type1_complete(rq, nr_bytes); | |
471 | else | |
472 | t10_pi_type1_complete(rq, nr_bytes); | |
473 | } |