Commit | Line | Data |
---|---|---|
8c16567d | 1 | // SPDX-License-Identifier: GPL-2.0 |
7ba1ba12 MP |
2 | /* |
3 | * blk-integrity.c - Block layer data integrity extensions | |
4 | * | |
5 | * Copyright (C) 2007, 2008 Oracle Corporation | |
6 | * Written by: Martin K. Petersen <martin.petersen@oracle.com> | |
7ba1ba12 MP |
7 | */ |
8 | ||
9 | #include <linux/blkdev.h> | |
66114cad | 10 | #include <linux/backing-dev.h> |
7ba1ba12 MP |
11 | #include <linux/mempool.h> |
12 | #include <linux/bio.h> | |
13 | #include <linux/scatterlist.h> | |
d5decd3b | 14 | #include <linux/export.h> |
5a0e3ad6 | 15 | #include <linux/slab.h> |
7ba1ba12 MP |
16 | |
17 | #include "blk.h" | |
18 | ||
7ba1ba12 MP |
19 | /** |
20 | * blk_rq_count_integrity_sg - Count number of integrity scatterlist elements | |
13f05c8d MP |
21 | * @q: request queue |
22 | * @bio: bio with integrity metadata attached | |
7ba1ba12 MP |
23 | * |
24 | * Description: Returns the number of elements required in a | |
13f05c8d | 25 | * scatterlist corresponding to the integrity metadata in a bio. |
7ba1ba12 | 26 | */ |
13f05c8d | 27 | int blk_rq_count_integrity_sg(struct request_queue *q, struct bio *bio) |
7ba1ba12 | 28 | { |
d57a5f7c | 29 | struct bio_vec iv, ivprv = { NULL }; |
13f05c8d MP |
30 | unsigned int segments = 0; |
31 | unsigned int seg_size = 0; | |
d57a5f7c KO |
32 | struct bvec_iter iter; |
33 | int prev = 0; | |
7ba1ba12 | 34 | |
d57a5f7c | 35 | bio_for_each_integrity_vec(iv, bio, iter) { |
7ba1ba12 | 36 | |
d57a5f7c | 37 | if (prev) { |
3dccdae5 | 38 | if (!biovec_phys_mergeable(q, &ivprv, &iv)) |
13f05c8d | 39 | goto new_segment; |
d57a5f7c | 40 | if (seg_size + iv.bv_len > queue_max_segment_size(q)) |
13f05c8d | 41 | goto new_segment; |
7ba1ba12 | 42 | |
d57a5f7c | 43 | seg_size += iv.bv_len; |
13f05c8d MP |
44 | } else { |
45 | new_segment: | |
7ba1ba12 | 46 | segments++; |
d57a5f7c | 47 | seg_size = iv.bv_len; |
13f05c8d | 48 | } |
7ba1ba12 | 49 | |
d57a5f7c | 50 | prev = 1; |
7ba1ba12 MP |
51 | ivprv = iv; |
52 | } | |
53 | ||
54 | return segments; | |
55 | } | |
56 | EXPORT_SYMBOL(blk_rq_count_integrity_sg); | |
57 | ||
58 | /** | |
59 | * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist | |
13f05c8d MP |
60 | * @q: request queue |
61 | * @bio: bio with integrity metadata attached | |
7ba1ba12 MP |
62 | * @sglist: target scatterlist |
63 | * | |
64 | * Description: Map the integrity vectors in request into a | |
65 | * scatterlist. The scatterlist must be big enough to hold all | |
66 | * elements. I.e. sized using blk_rq_count_integrity_sg(). | |
67 | */ | |
13f05c8d MP |
68 | int blk_rq_map_integrity_sg(struct request_queue *q, struct bio *bio, |
69 | struct scatterlist *sglist) | |
7ba1ba12 | 70 | { |
d57a5f7c | 71 | struct bio_vec iv, ivprv = { NULL }; |
13f05c8d MP |
72 | struct scatterlist *sg = NULL; |
73 | unsigned int segments = 0; | |
d57a5f7c KO |
74 | struct bvec_iter iter; |
75 | int prev = 0; | |
7ba1ba12 | 76 | |
d57a5f7c | 77 | bio_for_each_integrity_vec(iv, bio, iter) { |
7ba1ba12 | 78 | |
d57a5f7c | 79 | if (prev) { |
3dccdae5 | 80 | if (!biovec_phys_mergeable(q, &ivprv, &iv)) |
7ba1ba12 | 81 | goto new_segment; |
d57a5f7c | 82 | if (sg->length + iv.bv_len > queue_max_segment_size(q)) |
13f05c8d MP |
83 | goto new_segment; |
84 | ||
d57a5f7c | 85 | sg->length += iv.bv_len; |
7ba1ba12 MP |
86 | } else { |
87 | new_segment: | |
88 | if (!sg) | |
89 | sg = sglist; | |
90 | else { | |
c8164d89 | 91 | sg_unmark_end(sg); |
7ba1ba12 MP |
92 | sg = sg_next(sg); |
93 | } | |
94 | ||
d57a5f7c | 95 | sg_set_page(sg, iv.bv_page, iv.bv_len, iv.bv_offset); |
7ba1ba12 MP |
96 | segments++; |
97 | } | |
98 | ||
d57a5f7c | 99 | prev = 1; |
7ba1ba12 MP |
100 | ivprv = iv; |
101 | } | |
102 | ||
103 | if (sg) | |
104 | sg_mark_end(sg); | |
105 | ||
106 | return segments; | |
107 | } | |
108 | EXPORT_SYMBOL(blk_rq_map_integrity_sg); | |
109 | ||
110 | /** | |
ad7fce93 MP |
111 | * blk_integrity_compare - Compare integrity profile of two disks |
112 | * @gd1: Disk to compare | |
113 | * @gd2: Disk to compare | |
7ba1ba12 MP |
114 | * |
115 | * Description: Meta-devices like DM and MD need to verify that all | |
116 | * sub-devices use the same integrity format before advertising to | |
117 | * upper layers that they can send/receive integrity metadata. This | |
ad7fce93 | 118 | * function can be used to check whether two gendisk devices have |
7ba1ba12 MP |
119 | * compatible integrity formats. |
120 | */ | |
ad7fce93 | 121 | int blk_integrity_compare(struct gendisk *gd1, struct gendisk *gd2) |
7ba1ba12 | 122 | { |
ac6fc48c DW |
123 | struct blk_integrity *b1 = &gd1->queue->integrity; |
124 | struct blk_integrity *b2 = &gd2->queue->integrity; | |
7ba1ba12 | 125 | |
25520d55 | 126 | if (!b1->profile && !b2->profile) |
ad7fce93 | 127 | return 0; |
7ba1ba12 | 128 | |
25520d55 | 129 | if (!b1->profile || !b2->profile) |
ad7fce93 | 130 | return -1; |
7ba1ba12 | 131 | |
a48f041d | 132 | if (b1->interval_exp != b2->interval_exp) { |
3be91c4a MP |
133 | pr_err("%s: %s/%s protection interval %u != %u\n", |
134 | __func__, gd1->disk_name, gd2->disk_name, | |
a48f041d | 135 | 1 << b1->interval_exp, 1 << b2->interval_exp); |
7ba1ba12 MP |
136 | return -1; |
137 | } | |
138 | ||
139 | if (b1->tuple_size != b2->tuple_size) { | |
25520d55 | 140 | pr_err("%s: %s/%s tuple sz %u != %u\n", __func__, |
ad7fce93 | 141 | gd1->disk_name, gd2->disk_name, |
7ba1ba12 MP |
142 | b1->tuple_size, b2->tuple_size); |
143 | return -1; | |
144 | } | |
145 | ||
146 | if (b1->tag_size && b2->tag_size && (b1->tag_size != b2->tag_size)) { | |
25520d55 | 147 | pr_err("%s: %s/%s tag sz %u != %u\n", __func__, |
ad7fce93 | 148 | gd1->disk_name, gd2->disk_name, |
7ba1ba12 MP |
149 | b1->tag_size, b2->tag_size); |
150 | return -1; | |
151 | } | |
152 | ||
0f8087ec | 153 | if (b1->profile != b2->profile) { |
25520d55 | 154 | pr_err("%s: %s/%s type %s != %s\n", __func__, |
ad7fce93 | 155 | gd1->disk_name, gd2->disk_name, |
0f8087ec | 156 | b1->profile->name, b2->profile->name); |
7ba1ba12 MP |
157 | return -1; |
158 | } | |
159 | ||
160 | return 0; | |
161 | } | |
162 | EXPORT_SYMBOL(blk_integrity_compare); | |
163 | ||
4eaf99be MP |
164 | bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, |
165 | struct request *next) | |
13f05c8d | 166 | { |
4eaf99be MP |
167 | if (blk_integrity_rq(req) == 0 && blk_integrity_rq(next) == 0) |
168 | return true; | |
169 | ||
170 | if (blk_integrity_rq(req) == 0 || blk_integrity_rq(next) == 0) | |
171 | return false; | |
172 | ||
173 | if (bio_integrity(req->bio)->bip_flags != | |
174 | bio_integrity(next->bio)->bip_flags) | |
175 | return false; | |
13f05c8d MP |
176 | |
177 | if (req->nr_integrity_segments + next->nr_integrity_segments > | |
178 | q->limits.max_integrity_segments) | |
4eaf99be | 179 | return false; |
13f05c8d | 180 | |
7f39add3 SG |
181 | if (integrity_req_gap_back_merge(req, next->bio)) |
182 | return false; | |
183 | ||
4eaf99be | 184 | return true; |
13f05c8d MP |
185 | } |
186 | EXPORT_SYMBOL(blk_integrity_merge_rq); | |
187 | ||
4eaf99be MP |
188 | bool blk_integrity_merge_bio(struct request_queue *q, struct request *req, |
189 | struct bio *bio) | |
13f05c8d MP |
190 | { |
191 | int nr_integrity_segs; | |
192 | struct bio *next = bio->bi_next; | |
193 | ||
4eaf99be MP |
194 | if (blk_integrity_rq(req) == 0 && bio_integrity(bio) == NULL) |
195 | return true; | |
196 | ||
197 | if (blk_integrity_rq(req) == 0 || bio_integrity(bio) == NULL) | |
198 | return false; | |
199 | ||
200 | if (bio_integrity(req->bio)->bip_flags != bio_integrity(bio)->bip_flags) | |
201 | return false; | |
202 | ||
13f05c8d MP |
203 | bio->bi_next = NULL; |
204 | nr_integrity_segs = blk_rq_count_integrity_sg(q, bio); | |
205 | bio->bi_next = next; | |
206 | ||
207 | if (req->nr_integrity_segments + nr_integrity_segs > | |
208 | q->limits.max_integrity_segments) | |
4eaf99be | 209 | return false; |
13f05c8d MP |
210 | |
211 | req->nr_integrity_segments += nr_integrity_segs; | |
212 | ||
4eaf99be | 213 | return true; |
13f05c8d MP |
214 | } |
215 | EXPORT_SYMBOL(blk_integrity_merge_bio); | |
216 | ||
7ba1ba12 MP |
217 | struct integrity_sysfs_entry { |
218 | struct attribute attr; | |
219 | ssize_t (*show)(struct blk_integrity *, char *); | |
220 | ssize_t (*store)(struct blk_integrity *, const char *, size_t); | |
221 | }; | |
222 | ||
223 | static ssize_t integrity_attr_show(struct kobject *kobj, struct attribute *attr, | |
224 | char *page) | |
225 | { | |
aff34e19 | 226 | struct gendisk *disk = container_of(kobj, struct gendisk, integrity_kobj); |
ac6fc48c | 227 | struct blk_integrity *bi = &disk->queue->integrity; |
7ba1ba12 MP |
228 | struct integrity_sysfs_entry *entry = |
229 | container_of(attr, struct integrity_sysfs_entry, attr); | |
230 | ||
231 | return entry->show(bi, page); | |
232 | } | |
233 | ||
b984679e JA |
234 | static ssize_t integrity_attr_store(struct kobject *kobj, |
235 | struct attribute *attr, const char *page, | |
236 | size_t count) | |
7ba1ba12 | 237 | { |
aff34e19 | 238 | struct gendisk *disk = container_of(kobj, struct gendisk, integrity_kobj); |
ac6fc48c | 239 | struct blk_integrity *bi = &disk->queue->integrity; |
7ba1ba12 MP |
240 | struct integrity_sysfs_entry *entry = |
241 | container_of(attr, struct integrity_sysfs_entry, attr); | |
242 | ssize_t ret = 0; | |
243 | ||
244 | if (entry->store) | |
245 | ret = entry->store(bi, page, count); | |
246 | ||
247 | return ret; | |
248 | } | |
249 | ||
250 | static ssize_t integrity_format_show(struct blk_integrity *bi, char *page) | |
251 | { | |
25520d55 | 252 | if (bi->profile && bi->profile->name) |
0f8087ec | 253 | return sprintf(page, "%s\n", bi->profile->name); |
7ba1ba12 MP |
254 | else |
255 | return sprintf(page, "none\n"); | |
256 | } | |
257 | ||
258 | static ssize_t integrity_tag_size_show(struct blk_integrity *bi, char *page) | |
259 | { | |
25520d55 | 260 | return sprintf(page, "%u\n", bi->tag_size); |
7ba1ba12 MP |
261 | } |
262 | ||
4c241d08 MP |
263 | static ssize_t integrity_interval_show(struct blk_integrity *bi, char *page) |
264 | { | |
25520d55 MP |
265 | return sprintf(page, "%u\n", |
266 | bi->interval_exp ? 1 << bi->interval_exp : 0); | |
4c241d08 MP |
267 | } |
268 | ||
8288f496 MP |
269 | static ssize_t integrity_verify_store(struct blk_integrity *bi, |
270 | const char *page, size_t count) | |
7ba1ba12 MP |
271 | { |
272 | char *p = (char *) page; | |
273 | unsigned long val = simple_strtoul(p, &p, 10); | |
274 | ||
275 | if (val) | |
8288f496 | 276 | bi->flags |= BLK_INTEGRITY_VERIFY; |
7ba1ba12 | 277 | else |
8288f496 | 278 | bi->flags &= ~BLK_INTEGRITY_VERIFY; |
7ba1ba12 MP |
279 | |
280 | return count; | |
281 | } | |
282 | ||
8288f496 | 283 | static ssize_t integrity_verify_show(struct blk_integrity *bi, char *page) |
7ba1ba12 | 284 | { |
8288f496 | 285 | return sprintf(page, "%d\n", (bi->flags & BLK_INTEGRITY_VERIFY) != 0); |
7ba1ba12 MP |
286 | } |
287 | ||
8288f496 MP |
288 | static ssize_t integrity_generate_store(struct blk_integrity *bi, |
289 | const char *page, size_t count) | |
7ba1ba12 MP |
290 | { |
291 | char *p = (char *) page; | |
292 | unsigned long val = simple_strtoul(p, &p, 10); | |
293 | ||
294 | if (val) | |
8288f496 | 295 | bi->flags |= BLK_INTEGRITY_GENERATE; |
7ba1ba12 | 296 | else |
8288f496 | 297 | bi->flags &= ~BLK_INTEGRITY_GENERATE; |
7ba1ba12 MP |
298 | |
299 | return count; | |
300 | } | |
301 | ||
8288f496 | 302 | static ssize_t integrity_generate_show(struct blk_integrity *bi, char *page) |
7ba1ba12 | 303 | { |
8288f496 | 304 | return sprintf(page, "%d\n", (bi->flags & BLK_INTEGRITY_GENERATE) != 0); |
7ba1ba12 MP |
305 | } |
306 | ||
3aec2f41 MP |
307 | static ssize_t integrity_device_show(struct blk_integrity *bi, char *page) |
308 | { | |
309 | return sprintf(page, "%u\n", | |
310 | (bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE) != 0); | |
311 | } | |
312 | ||
7ba1ba12 | 313 | static struct integrity_sysfs_entry integrity_format_entry = { |
5657a819 | 314 | .attr = { .name = "format", .mode = 0444 }, |
7ba1ba12 MP |
315 | .show = integrity_format_show, |
316 | }; | |
317 | ||
318 | static struct integrity_sysfs_entry integrity_tag_size_entry = { | |
5657a819 | 319 | .attr = { .name = "tag_size", .mode = 0444 }, |
7ba1ba12 MP |
320 | .show = integrity_tag_size_show, |
321 | }; | |
322 | ||
4c241d08 | 323 | static struct integrity_sysfs_entry integrity_interval_entry = { |
5657a819 | 324 | .attr = { .name = "protection_interval_bytes", .mode = 0444 }, |
4c241d08 MP |
325 | .show = integrity_interval_show, |
326 | }; | |
327 | ||
8288f496 | 328 | static struct integrity_sysfs_entry integrity_verify_entry = { |
5657a819 | 329 | .attr = { .name = "read_verify", .mode = 0644 }, |
8288f496 MP |
330 | .show = integrity_verify_show, |
331 | .store = integrity_verify_store, | |
7ba1ba12 MP |
332 | }; |
333 | ||
8288f496 | 334 | static struct integrity_sysfs_entry integrity_generate_entry = { |
5657a819 | 335 | .attr = { .name = "write_generate", .mode = 0644 }, |
8288f496 MP |
336 | .show = integrity_generate_show, |
337 | .store = integrity_generate_store, | |
7ba1ba12 MP |
338 | }; |
339 | ||
3aec2f41 | 340 | static struct integrity_sysfs_entry integrity_device_entry = { |
5657a819 | 341 | .attr = { .name = "device_is_integrity_capable", .mode = 0444 }, |
3aec2f41 MP |
342 | .show = integrity_device_show, |
343 | }; | |
344 | ||
7ba1ba12 MP |
345 | static struct attribute *integrity_attrs[] = { |
346 | &integrity_format_entry.attr, | |
347 | &integrity_tag_size_entry.attr, | |
4c241d08 | 348 | &integrity_interval_entry.attr, |
8288f496 MP |
349 | &integrity_verify_entry.attr, |
350 | &integrity_generate_entry.attr, | |
3aec2f41 | 351 | &integrity_device_entry.attr, |
7ba1ba12 MP |
352 | NULL, |
353 | }; | |
800f5aa1 | 354 | ATTRIBUTE_GROUPS(integrity); |
7ba1ba12 | 355 | |
52cf25d0 | 356 | static const struct sysfs_ops integrity_ops = { |
7ba1ba12 MP |
357 | .show = &integrity_attr_show, |
358 | .store = &integrity_attr_store, | |
359 | }; | |
360 | ||
7ba1ba12 | 361 | static struct kobj_type integrity_ktype = { |
800f5aa1 | 362 | .default_groups = integrity_groups, |
7ba1ba12 | 363 | .sysfs_ops = &integrity_ops, |
7ba1ba12 MP |
364 | }; |
365 | ||
4e4cbee9 | 366 | static blk_status_t blk_integrity_nop_fn(struct blk_integrity_iter *iter) |
4125a09b | 367 | { |
4e4cbee9 | 368 | return BLK_STS_OK; |
4125a09b DW |
369 | } |
370 | ||
54d4e6ab MG |
371 | static void blk_integrity_nop_prepare(struct request *rq) |
372 | { | |
373 | } | |
374 | ||
375 | static void blk_integrity_nop_complete(struct request *rq, | |
376 | unsigned int nr_bytes) | |
377 | { | |
378 | } | |
379 | ||
869ab90f | 380 | static const struct blk_integrity_profile nop_profile = { |
4125a09b DW |
381 | .name = "nop", |
382 | .generate_fn = blk_integrity_nop_fn, | |
383 | .verify_fn = blk_integrity_nop_fn, | |
54d4e6ab MG |
384 | .prepare_fn = blk_integrity_nop_prepare, |
385 | .complete_fn = blk_integrity_nop_complete, | |
4125a09b DW |
386 | }; |
387 | ||
7ba1ba12 MP |
388 | /** |
389 | * blk_integrity_register - Register a gendisk as being integrity-capable | |
390 | * @disk: struct gendisk pointer to make integrity-aware | |
25520d55 | 391 | * @template: block integrity profile to register |
7ba1ba12 | 392 | * |
25520d55 MP |
393 | * Description: When a device needs to advertise itself as being able to |
394 | * send/receive integrity metadata it must use this function to register | |
395 | * the capability with the block layer. The template is a blk_integrity | |
396 | * struct with values appropriate for the underlying hardware. See | |
898bd37a | 397 | * Documentation/block/data-integrity.rst. |
7ba1ba12 | 398 | */ |
25520d55 | 399 | void blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) |
7ba1ba12 | 400 | { |
ac6fc48c | 401 | struct blk_integrity *bi = &disk->queue->integrity; |
7ba1ba12 | 402 | |
25520d55 MP |
403 | bi->flags = BLK_INTEGRITY_VERIFY | BLK_INTEGRITY_GENERATE | |
404 | template->flags; | |
2859323e MS |
405 | bi->interval_exp = template->interval_exp ? : |
406 | ilog2(queue_logical_block_size(disk->queue)); | |
4125a09b | 407 | bi->profile = template->profile ? template->profile : &nop_profile; |
25520d55 MP |
408 | bi->tuple_size = template->tuple_size; |
409 | bi->tag_size = template->tag_size; | |
7ba1ba12 | 410 | |
19b7ccf8 | 411 | disk->queue->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES; |
7ba1ba12 MP |
412 | } |
413 | EXPORT_SYMBOL(blk_integrity_register); | |
414 | ||
415 | /** | |
25520d55 MP |
416 | * blk_integrity_unregister - Unregister block integrity profile |
417 | * @disk: disk whose integrity profile to unregister | |
7ba1ba12 | 418 | * |
25520d55 MP |
419 | * Description: This function unregisters the integrity capability from |
420 | * a block device. | |
7ba1ba12 MP |
421 | */ |
422 | void blk_integrity_unregister(struct gendisk *disk) | |
423 | { | |
19b7ccf8 | 424 | disk->queue->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES; |
ac6fc48c | 425 | memset(&disk->queue->integrity, 0, sizeof(struct blk_integrity)); |
25520d55 MP |
426 | } |
427 | EXPORT_SYMBOL(blk_integrity_unregister); | |
7ba1ba12 | 428 | |
25520d55 MP |
429 | void blk_integrity_add(struct gendisk *disk) |
430 | { | |
431 | if (kobject_init_and_add(&disk->integrity_kobj, &integrity_ktype, | |
432 | &disk_to_dev(disk)->kobj, "%s", "integrity")) | |
433 | return; | |
7ba1ba12 | 434 | |
25520d55 MP |
435 | kobject_uevent(&disk->integrity_kobj, KOBJ_ADD); |
436 | } | |
437 | ||
438 | void blk_integrity_del(struct gendisk *disk) | |
439 | { | |
aff34e19 MP |
440 | kobject_uevent(&disk->integrity_kobj, KOBJ_REMOVE); |
441 | kobject_del(&disk->integrity_kobj); | |
442 | kobject_put(&disk->integrity_kobj); | |
7ba1ba12 | 443 | } |