Commit | Line | Data |
---|---|---|
8c16567d | 1 | // SPDX-License-Identifier: GPL-2.0 |
7ba1ba12 MP |
2 | /* |
3 | * blk-integrity.c - Block layer data integrity extensions | |
4 | * | |
5 | * Copyright (C) 2007, 2008 Oracle Corporation | |
6 | * Written by: Martin K. Petersen <martin.petersen@oracle.com> | |
7ba1ba12 MP |
7 | */ |
8 | ||
fe45e630 | 9 | #include <linux/blk-integrity.h> |
66114cad | 10 | #include <linux/backing-dev.h> |
7ba1ba12 MP |
11 | #include <linux/mempool.h> |
12 | #include <linux/bio.h> | |
13 | #include <linux/scatterlist.h> | |
d5decd3b | 14 | #include <linux/export.h> |
5a0e3ad6 | 15 | #include <linux/slab.h> |
7ba1ba12 MP |
16 | |
17 | #include "blk.h" | |
18 | ||
7ba1ba12 MP |
19 | /** |
20 | * blk_rq_count_integrity_sg - Count number of integrity scatterlist elements | |
13f05c8d MP |
21 | * @q: request queue |
22 | * @bio: bio with integrity metadata attached | |
7ba1ba12 MP |
23 | * |
24 | * Description: Returns the number of elements required in a | |
13f05c8d | 25 | * scatterlist corresponding to the integrity metadata in a bio. |
7ba1ba12 | 26 | */ |
13f05c8d | 27 | int blk_rq_count_integrity_sg(struct request_queue *q, struct bio *bio) |
7ba1ba12 | 28 | { |
d57a5f7c | 29 | struct bio_vec iv, ivprv = { NULL }; |
13f05c8d MP |
30 | unsigned int segments = 0; |
31 | unsigned int seg_size = 0; | |
d57a5f7c KO |
32 | struct bvec_iter iter; |
33 | int prev = 0; | |
7ba1ba12 | 34 | |
d57a5f7c | 35 | bio_for_each_integrity_vec(iv, bio, iter) { |
7ba1ba12 | 36 | |
d57a5f7c | 37 | if (prev) { |
3dccdae5 | 38 | if (!biovec_phys_mergeable(q, &ivprv, &iv)) |
13f05c8d | 39 | goto new_segment; |
d57a5f7c | 40 | if (seg_size + iv.bv_len > queue_max_segment_size(q)) |
13f05c8d | 41 | goto new_segment; |
7ba1ba12 | 42 | |
d57a5f7c | 43 | seg_size += iv.bv_len; |
13f05c8d MP |
44 | } else { |
45 | new_segment: | |
7ba1ba12 | 46 | segments++; |
d57a5f7c | 47 | seg_size = iv.bv_len; |
13f05c8d | 48 | } |
7ba1ba12 | 49 | |
d57a5f7c | 50 | prev = 1; |
7ba1ba12 MP |
51 | ivprv = iv; |
52 | } | |
53 | ||
54 | return segments; | |
55 | } | |
56 | EXPORT_SYMBOL(blk_rq_count_integrity_sg); | |
57 | ||
58 | /** | |
59 | * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist | |
13f05c8d MP |
60 | * @q: request queue |
61 | * @bio: bio with integrity metadata attached | |
7ba1ba12 MP |
62 | * @sglist: target scatterlist |
63 | * | |
64 | * Description: Map the integrity vectors in request into a | |
65 | * scatterlist. The scatterlist must be big enough to hold all | |
66 | * elements. I.e. sized using blk_rq_count_integrity_sg(). | |
67 | */ | |
13f05c8d MP |
68 | int blk_rq_map_integrity_sg(struct request_queue *q, struct bio *bio, |
69 | struct scatterlist *sglist) | |
7ba1ba12 | 70 | { |
d57a5f7c | 71 | struct bio_vec iv, ivprv = { NULL }; |
13f05c8d MP |
72 | struct scatterlist *sg = NULL; |
73 | unsigned int segments = 0; | |
d57a5f7c KO |
74 | struct bvec_iter iter; |
75 | int prev = 0; | |
7ba1ba12 | 76 | |
d57a5f7c | 77 | bio_for_each_integrity_vec(iv, bio, iter) { |
7ba1ba12 | 78 | |
d57a5f7c | 79 | if (prev) { |
3dccdae5 | 80 | if (!biovec_phys_mergeable(q, &ivprv, &iv)) |
7ba1ba12 | 81 | goto new_segment; |
d57a5f7c | 82 | if (sg->length + iv.bv_len > queue_max_segment_size(q)) |
13f05c8d MP |
83 | goto new_segment; |
84 | ||
d57a5f7c | 85 | sg->length += iv.bv_len; |
7ba1ba12 MP |
86 | } else { |
87 | new_segment: | |
88 | if (!sg) | |
89 | sg = sglist; | |
90 | else { | |
c8164d89 | 91 | sg_unmark_end(sg); |
7ba1ba12 MP |
92 | sg = sg_next(sg); |
93 | } | |
94 | ||
d57a5f7c | 95 | sg_set_page(sg, iv.bv_page, iv.bv_len, iv.bv_offset); |
7ba1ba12 MP |
96 | segments++; |
97 | } | |
98 | ||
d57a5f7c | 99 | prev = 1; |
7ba1ba12 MP |
100 | ivprv = iv; |
101 | } | |
102 | ||
103 | if (sg) | |
104 | sg_mark_end(sg); | |
105 | ||
106 | return segments; | |
107 | } | |
108 | EXPORT_SYMBOL(blk_rq_map_integrity_sg); | |
109 | ||
110 | /** | |
ad7fce93 MP |
111 | * blk_integrity_compare - Compare integrity profile of two disks |
112 | * @gd1: Disk to compare | |
113 | * @gd2: Disk to compare | |
7ba1ba12 MP |
114 | * |
115 | * Description: Meta-devices like DM and MD need to verify that all | |
116 | * sub-devices use the same integrity format before advertising to | |
117 | * upper layers that they can send/receive integrity metadata. This | |
ad7fce93 | 118 | * function can be used to check whether two gendisk devices have |
7ba1ba12 MP |
119 | * compatible integrity formats. |
120 | */ | |
ad7fce93 | 121 | int blk_integrity_compare(struct gendisk *gd1, struct gendisk *gd2) |
7ba1ba12 | 122 | { |
ac6fc48c DW |
123 | struct blk_integrity *b1 = &gd1->queue->integrity; |
124 | struct blk_integrity *b2 = &gd2->queue->integrity; | |
7ba1ba12 | 125 | |
25520d55 | 126 | if (!b1->profile && !b2->profile) |
ad7fce93 | 127 | return 0; |
7ba1ba12 | 128 | |
25520d55 | 129 | if (!b1->profile || !b2->profile) |
ad7fce93 | 130 | return -1; |
7ba1ba12 | 131 | |
a48f041d | 132 | if (b1->interval_exp != b2->interval_exp) { |
3be91c4a MP |
133 | pr_err("%s: %s/%s protection interval %u != %u\n", |
134 | __func__, gd1->disk_name, gd2->disk_name, | |
a48f041d | 135 | 1 << b1->interval_exp, 1 << b2->interval_exp); |
7ba1ba12 MP |
136 | return -1; |
137 | } | |
138 | ||
139 | if (b1->tuple_size != b2->tuple_size) { | |
25520d55 | 140 | pr_err("%s: %s/%s tuple sz %u != %u\n", __func__, |
ad7fce93 | 141 | gd1->disk_name, gd2->disk_name, |
7ba1ba12 MP |
142 | b1->tuple_size, b2->tuple_size); |
143 | return -1; | |
144 | } | |
145 | ||
146 | if (b1->tag_size && b2->tag_size && (b1->tag_size != b2->tag_size)) { | |
25520d55 | 147 | pr_err("%s: %s/%s tag sz %u != %u\n", __func__, |
ad7fce93 | 148 | gd1->disk_name, gd2->disk_name, |
7ba1ba12 MP |
149 | b1->tag_size, b2->tag_size); |
150 | return -1; | |
151 | } | |
152 | ||
0f8087ec | 153 | if (b1->profile != b2->profile) { |
25520d55 | 154 | pr_err("%s: %s/%s type %s != %s\n", __func__, |
ad7fce93 | 155 | gd1->disk_name, gd2->disk_name, |
0f8087ec | 156 | b1->profile->name, b2->profile->name); |
7ba1ba12 MP |
157 | return -1; |
158 | } | |
159 | ||
160 | return 0; | |
161 | } | |
162 | EXPORT_SYMBOL(blk_integrity_compare); | |
163 | ||
4eaf99be MP |
164 | bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, |
165 | struct request *next) | |
13f05c8d | 166 | { |
4eaf99be MP |
167 | if (blk_integrity_rq(req) == 0 && blk_integrity_rq(next) == 0) |
168 | return true; | |
169 | ||
170 | if (blk_integrity_rq(req) == 0 || blk_integrity_rq(next) == 0) | |
171 | return false; | |
172 | ||
173 | if (bio_integrity(req->bio)->bip_flags != | |
174 | bio_integrity(next->bio)->bip_flags) | |
175 | return false; | |
13f05c8d MP |
176 | |
177 | if (req->nr_integrity_segments + next->nr_integrity_segments > | |
178 | q->limits.max_integrity_segments) | |
4eaf99be | 179 | return false; |
13f05c8d | 180 | |
7f39add3 SG |
181 | if (integrity_req_gap_back_merge(req, next->bio)) |
182 | return false; | |
183 | ||
4eaf99be | 184 | return true; |
13f05c8d | 185 | } |
13f05c8d | 186 | |
4eaf99be MP |
187 | bool blk_integrity_merge_bio(struct request_queue *q, struct request *req, |
188 | struct bio *bio) | |
13f05c8d MP |
189 | { |
190 | int nr_integrity_segs; | |
191 | struct bio *next = bio->bi_next; | |
192 | ||
4eaf99be MP |
193 | if (blk_integrity_rq(req) == 0 && bio_integrity(bio) == NULL) |
194 | return true; | |
195 | ||
196 | if (blk_integrity_rq(req) == 0 || bio_integrity(bio) == NULL) | |
197 | return false; | |
198 | ||
199 | if (bio_integrity(req->bio)->bip_flags != bio_integrity(bio)->bip_flags) | |
200 | return false; | |
201 | ||
13f05c8d MP |
202 | bio->bi_next = NULL; |
203 | nr_integrity_segs = blk_rq_count_integrity_sg(q, bio); | |
204 | bio->bi_next = next; | |
205 | ||
206 | if (req->nr_integrity_segments + nr_integrity_segs > | |
207 | q->limits.max_integrity_segments) | |
4eaf99be | 208 | return false; |
13f05c8d MP |
209 | |
210 | req->nr_integrity_segments += nr_integrity_segs; | |
211 | ||
4eaf99be | 212 | return true; |
13f05c8d | 213 | } |
13f05c8d | 214 | |
76b8c319 TW |
215 | static inline struct blk_integrity *dev_to_bi(struct device *dev) |
216 | { | |
217 | return &dev_to_disk(dev)->queue->integrity; | |
7ba1ba12 MP |
218 | } |
219 | ||
76b8c319 TW |
220 | static ssize_t format_show(struct device *dev, struct device_attribute *attr, |
221 | char *page) | |
7ba1ba12 | 222 | { |
76b8c319 TW |
223 | struct blk_integrity *bi = dev_to_bi(dev); |
224 | ||
25520d55 | 225 | if (bi->profile && bi->profile->name) |
3315e169 TW |
226 | return sysfs_emit(page, "%s\n", bi->profile->name); |
227 | return sysfs_emit(page, "none\n"); | |
7ba1ba12 MP |
228 | } |
229 | ||
76b8c319 TW |
230 | static ssize_t tag_size_show(struct device *dev, struct device_attribute *attr, |
231 | char *page) | |
7ba1ba12 | 232 | { |
76b8c319 TW |
233 | struct blk_integrity *bi = dev_to_bi(dev); |
234 | ||
3315e169 | 235 | return sysfs_emit(page, "%u\n", bi->tag_size); |
7ba1ba12 MP |
236 | } |
237 | ||
76b8c319 TW |
238 | static ssize_t protection_interval_bytes_show(struct device *dev, |
239 | struct device_attribute *attr, | |
240 | char *page) | |
4c241d08 | 241 | { |
76b8c319 TW |
242 | struct blk_integrity *bi = dev_to_bi(dev); |
243 | ||
3315e169 TW |
244 | return sysfs_emit(page, "%u\n", |
245 | bi->interval_exp ? 1 << bi->interval_exp : 0); | |
4c241d08 MP |
246 | } |
247 | ||
76b8c319 TW |
248 | static ssize_t read_verify_store(struct device *dev, |
249 | struct device_attribute *attr, | |
250 | const char *page, size_t count) | |
7ba1ba12 | 251 | { |
76b8c319 | 252 | struct blk_integrity *bi = dev_to_bi(dev); |
7ba1ba12 MP |
253 | char *p = (char *) page; |
254 | unsigned long val = simple_strtoul(p, &p, 10); | |
255 | ||
256 | if (val) | |
8288f496 | 257 | bi->flags |= BLK_INTEGRITY_VERIFY; |
7ba1ba12 | 258 | else |
8288f496 | 259 | bi->flags &= ~BLK_INTEGRITY_VERIFY; |
7ba1ba12 MP |
260 | |
261 | return count; | |
262 | } | |
263 | ||
76b8c319 TW |
264 | static ssize_t read_verify_show(struct device *dev, |
265 | struct device_attribute *attr, char *page) | |
7ba1ba12 | 266 | { |
76b8c319 TW |
267 | struct blk_integrity *bi = dev_to_bi(dev); |
268 | ||
3315e169 | 269 | return sysfs_emit(page, "%d\n", !!(bi->flags & BLK_INTEGRITY_VERIFY)); |
7ba1ba12 MP |
270 | } |
271 | ||
76b8c319 TW |
272 | static ssize_t write_generate_store(struct device *dev, |
273 | struct device_attribute *attr, | |
274 | const char *page, size_t count) | |
7ba1ba12 | 275 | { |
76b8c319 TW |
276 | struct blk_integrity *bi = dev_to_bi(dev); |
277 | ||
7ba1ba12 MP |
278 | char *p = (char *) page; |
279 | unsigned long val = simple_strtoul(p, &p, 10); | |
280 | ||
281 | if (val) | |
8288f496 | 282 | bi->flags |= BLK_INTEGRITY_GENERATE; |
7ba1ba12 | 283 | else |
8288f496 | 284 | bi->flags &= ~BLK_INTEGRITY_GENERATE; |
7ba1ba12 MP |
285 | |
286 | return count; | |
287 | } | |
288 | ||
76b8c319 TW |
289 | static ssize_t write_generate_show(struct device *dev, |
290 | struct device_attribute *attr, char *page) | |
7ba1ba12 | 291 | { |
76b8c319 TW |
292 | struct blk_integrity *bi = dev_to_bi(dev); |
293 | ||
3315e169 | 294 | return sysfs_emit(page, "%d\n", !!(bi->flags & BLK_INTEGRITY_GENERATE)); |
7ba1ba12 MP |
295 | } |
296 | ||
76b8c319 TW |
297 | static ssize_t device_is_integrity_capable_show(struct device *dev, |
298 | struct device_attribute *attr, | |
299 | char *page) | |
3aec2f41 | 300 | { |
76b8c319 TW |
301 | struct blk_integrity *bi = dev_to_bi(dev); |
302 | ||
3315e169 TW |
303 | return sysfs_emit(page, "%u\n", |
304 | !!(bi->flags & BLK_INTEGRITY_DEVICE_CAPABLE)); | |
3aec2f41 MP |
305 | } |
306 | ||
76b8c319 TW |
307 | static DEVICE_ATTR_RO(format); |
308 | static DEVICE_ATTR_RO(tag_size); | |
309 | static DEVICE_ATTR_RO(protection_interval_bytes); | |
310 | static DEVICE_ATTR_RW(read_verify); | |
311 | static DEVICE_ATTR_RW(write_generate); | |
312 | static DEVICE_ATTR_RO(device_is_integrity_capable); | |
3aec2f41 | 313 | |
7ba1ba12 | 314 | static struct attribute *integrity_attrs[] = { |
76b8c319 TW |
315 | &dev_attr_format.attr, |
316 | &dev_attr_tag_size.attr, | |
317 | &dev_attr_protection_interval_bytes.attr, | |
318 | &dev_attr_read_verify.attr, | |
319 | &dev_attr_write_generate.attr, | |
320 | &dev_attr_device_is_integrity_capable.attr, | |
321 | NULL | |
7ba1ba12 MP |
322 | }; |
323 | ||
ff53cd52 TW |
324 | const struct attribute_group blk_integrity_attr_group = { |
325 | .name = "integrity", | |
326 | .attrs = integrity_attrs, | |
7ba1ba12 MP |
327 | }; |
328 | ||
4e4cbee9 | 329 | static blk_status_t blk_integrity_nop_fn(struct blk_integrity_iter *iter) |
4125a09b | 330 | { |
4e4cbee9 | 331 | return BLK_STS_OK; |
4125a09b DW |
332 | } |
333 | ||
54d4e6ab MG |
334 | static void blk_integrity_nop_prepare(struct request *rq) |
335 | { | |
336 | } | |
337 | ||
338 | static void blk_integrity_nop_complete(struct request *rq, | |
339 | unsigned int nr_bytes) | |
340 | { | |
341 | } | |
342 | ||
869ab90f | 343 | static const struct blk_integrity_profile nop_profile = { |
4125a09b DW |
344 | .name = "nop", |
345 | .generate_fn = blk_integrity_nop_fn, | |
346 | .verify_fn = blk_integrity_nop_fn, | |
54d4e6ab MG |
347 | .prepare_fn = blk_integrity_nop_prepare, |
348 | .complete_fn = blk_integrity_nop_complete, | |
4125a09b DW |
349 | }; |
350 | ||
7ba1ba12 MP |
351 | /** |
352 | * blk_integrity_register - Register a gendisk as being integrity-capable | |
353 | * @disk: struct gendisk pointer to make integrity-aware | |
25520d55 | 354 | * @template: block integrity profile to register |
7ba1ba12 | 355 | * |
25520d55 MP |
356 | * Description: When a device needs to advertise itself as being able to |
357 | * send/receive integrity metadata it must use this function to register | |
358 | * the capability with the block layer. The template is a blk_integrity | |
359 | * struct with values appropriate for the underlying hardware. See | |
898bd37a | 360 | * Documentation/block/data-integrity.rst. |
7ba1ba12 | 361 | */ |
25520d55 | 362 | void blk_integrity_register(struct gendisk *disk, struct blk_integrity *template) |
7ba1ba12 | 363 | { |
ac6fc48c | 364 | struct blk_integrity *bi = &disk->queue->integrity; |
7ba1ba12 | 365 | |
25520d55 MP |
366 | bi->flags = BLK_INTEGRITY_VERIFY | BLK_INTEGRITY_GENERATE | |
367 | template->flags; | |
2859323e MS |
368 | bi->interval_exp = template->interval_exp ? : |
369 | ilog2(queue_logical_block_size(disk->queue)); | |
4125a09b | 370 | bi->profile = template->profile ? template->profile : &nop_profile; |
25520d55 MP |
371 | bi->tuple_size = template->tuple_size; |
372 | bi->tag_size = template->tag_size; | |
60d21aac | 373 | bi->pi_offset = template->pi_offset; |
7ba1ba12 | 374 | |
1cb039f3 | 375 | blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, disk->queue); |
d145dc23 ST |
376 | |
377 | #ifdef CONFIG_BLK_INLINE_ENCRYPTION | |
cb77cb5a | 378 | if (disk->queue->crypto_profile) { |
d145dc23 | 379 | pr_warn("blk-integrity: Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n"); |
72cd9df2 | 380 | disk->queue->crypto_profile = NULL; |
d145dc23 ST |
381 | } |
382 | #endif | |
7ba1ba12 MP |
383 | } |
384 | EXPORT_SYMBOL(blk_integrity_register); | |
385 | ||
386 | /** | |
25520d55 MP |
387 | * blk_integrity_unregister - Unregister block integrity profile |
388 | * @disk: disk whose integrity profile to unregister | |
7ba1ba12 | 389 | * |
25520d55 MP |
390 | * Description: This function unregisters the integrity capability from |
391 | * a block device. | |
7ba1ba12 MP |
392 | */ |
393 | void blk_integrity_unregister(struct gendisk *disk) | |
394 | { | |
783a40a1 CH |
395 | struct blk_integrity *bi = &disk->queue->integrity; |
396 | ||
397 | if (!bi->profile) | |
398 | return; | |
3df49967 LK |
399 | |
400 | /* ensure all bios are off the integrity workqueue */ | |
401 | blk_flush_integrity(); | |
1cb039f3 | 402 | blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, disk->queue); |
783a40a1 | 403 | memset(bi, 0, sizeof(*bi)); |
25520d55 MP |
404 | } |
405 | EXPORT_SYMBOL(blk_integrity_unregister); |