Commit | Line | Data |
---|---|---|
55716d26 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a4ffc152 MP |
2 | /* |
3 | * Copyright (C) 2012 Red Hat, Inc. | |
4 | * | |
5 | * Author: Mikulas Patocka <mpatocka@redhat.com> | |
6 | * | |
7 | * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors | |
8 | * | |
a4ffc152 MP |
9 | * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set |
10 | * default prefetch value. Data are read in "prefetch_cluster" chunks from the | |
11 | * hash device. Setting this greatly improves performance when data and hash | |
12 | * are on the same disk on different partitions on devices with poor random | |
13 | * access behavior. | |
14 | */ | |
15 | ||
ffa39380 | 16 | #include "dm-verity.h" |
a739ff3f | 17 | #include "dm-verity-fec.h" |
88cd3e6c | 18 | #include "dm-verity-verify-sig.h" |
074c4466 | 19 | #include "dm-audit.h" |
a4ffc152 | 20 | #include <linux/module.h> |
65ff5b7d | 21 | #include <linux/reboot.h> |
24b83deb | 22 | #include <linux/scatterlist.h> |
b6c1c574 | 23 | #include <linux/string.h> |
ba2cce82 | 24 | #include <linux/jump_label.h> |
a4ffc152 MP |
25 | |
26 | #define DM_MSG_PREFIX "verity" | |
27 | ||
65ff5b7d ST |
28 | #define DM_VERITY_ENV_LENGTH 42 |
29 | #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" | |
30 | ||
a4ffc152 MP |
31 | #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 |
32 | ||
65ff5b7d ST |
33 | #define DM_VERITY_MAX_CORRUPTED_ERRS 100 |
34 | ||
35 | #define DM_VERITY_OPT_LOGGING "ignore_corruption" | |
36 | #define DM_VERITY_OPT_RESTART "restart_on_corruption" | |
e1fef0b0 | 37 | #define DM_VERITY_OPT_PANIC "panic_on_corruption" |
0cc37c2d | 38 | #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" |
843f38d3 | 39 | #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once" |
5721d4e5 | 40 | #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet" |
a4ffc152 | 41 | |
8c22816d | 42 | #define DM_VERITY_OPTS_MAX (4 + DM_VERITY_OPTS_FEC + \ |
88cd3e6c | 43 | DM_VERITY_ROOT_HASH_VERIFICATION_OPTS) |
753c1fd0 | 44 | |
86a3238c | 45 | static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; |
a4ffc152 | 46 | |
6a808034 | 47 | module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); |
a4ffc152 | 48 | |
ba2cce82 MS |
49 | static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); |
50 | ||
3b6b7813 MP |
51 | struct dm_verity_prefetch_work { |
52 | struct work_struct work; | |
53 | struct dm_verity *v; | |
54 | sector_t block; | |
86a3238c | 55 | unsigned int n_blocks; |
3b6b7813 MP |
56 | }; |
57 | ||
a4ffc152 MP |
58 | /* |
59 | * Auxiliary structure appended to each dm-bufio buffer. If the value | |
60 | * hash_verified is nonzero, hash of the block has been verified. | |
61 | * | |
62 | * The variable hash_verified is set to 0 when allocating the buffer, then | |
63 | * it can be changed to 1 and it is never reset to 0 again. | |
64 | * | |
65 | * There is no lock around this value, a race condition can at worst cause | |
66 | * that multiple processes verify the hash of the same buffer simultaneously | |
67 | * and write 1 to hash_verified simultaneously. | |
68 | * This condition is harmless, so we don't need locking. | |
69 | */ | |
70 | struct buffer_aux { | |
71 | int hash_verified; | |
72 | }; | |
73 | ||
74 | /* | |
75 | * Initialize struct buffer_aux for a freshly created buffer. | |
76 | */ | |
77 | static void dm_bufio_alloc_callback(struct dm_buffer *buf) | |
78 | { | |
79 | struct buffer_aux *aux = dm_bufio_get_aux_data(buf); | |
80 | ||
81 | aux->hash_verified = 0; | |
82 | } | |
83 | ||
84 | /* | |
85 | * Translate input sector number to the sector number on the target device. | |
86 | */ | |
87 | static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) | |
88 | { | |
89 | return v->data_start + dm_target_offset(v->ti, bi_sector); | |
90 | } | |
91 | ||
92 | /* | |
93 | * Return hash position of a specified block at a specified tree level | |
94 | * (0 is the lowest level). | |
95 | * The lowest "hash_per_block_bits"-bits of the result denote hash position | |
96 | * inside a hash block. The remaining bits denote location of the hash block. | |
97 | */ | |
98 | static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, | |
99 | int level) | |
100 | { | |
101 | return block >> (level * v->hash_per_block_bits); | |
102 | } | |
103 | ||
d1ac3ff0 GBY |
104 | static int verity_hash_update(struct dm_verity *v, struct ahash_request *req, |
105 | const u8 *data, size_t len, | |
12f1ffc4 | 106 | struct crypto_wait *wait) |
6dbeda34 | 107 | { |
d1ac3ff0 | 108 | struct scatterlist sg; |
6dbeda34 | 109 | |
e4b069e0 MP |
110 | if (likely(!is_vmalloc_addr(data))) { |
111 | sg_init_one(&sg, data, len); | |
112 | ahash_request_set_crypt(req, &sg, NULL, len); | |
113 | return crypto_wait_req(crypto_ahash_update(req), wait); | |
e4b069e0 | 114 | } |
1c3fe2fa HM |
115 | |
116 | do { | |
117 | int r; | |
118 | size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data)); | |
119 | ||
120 | flush_kernel_vmap_range((void *)data, this_step); | |
121 | sg_init_table(&sg, 1); | |
122 | sg_set_page(&sg, vmalloc_to_page(data), this_step, offset_in_page(data)); | |
123 | ahash_request_set_crypt(req, &sg, NULL, this_step); | |
124 | r = crypto_wait_req(crypto_ahash_update(req), wait); | |
125 | if (unlikely(r)) | |
126 | return r; | |
127 | data += this_step; | |
128 | len -= this_step; | |
129 | } while (len); | |
130 | ||
131 | return 0; | |
d1ac3ff0 GBY |
132 | } |
133 | ||
134 | /* | |
135 | * Wrapper for crypto_ahash_init, which handles verity salting. | |
136 | */ | |
137 | static int verity_hash_init(struct dm_verity *v, struct ahash_request *req, | |
28f07f2a | 138 | struct crypto_wait *wait, bool may_sleep) |
d1ac3ff0 GBY |
139 | { |
140 | int r; | |
141 | ||
142 | ahash_request_set_tfm(req, v->tfm); | |
28f07f2a MP |
143 | ahash_request_set_callback(req, |
144 | may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0, | |
145 | crypto_req_done, (void *)wait); | |
12f1ffc4 | 146 | crypto_init_wait(wait); |
d1ac3ff0 | 147 | |
12f1ffc4 | 148 | r = crypto_wait_req(crypto_ahash_init(req), wait); |
d1ac3ff0 GBY |
149 | |
150 | if (unlikely(r < 0)) { | |
28f07f2a MP |
151 | if (r != -ENOMEM) |
152 | DMERR("crypto_ahash_init failed: %d", r); | |
d1ac3ff0 GBY |
153 | return r; |
154 | } | |
155 | ||
f52236e0 | 156 | if (likely(v->salt_size && (v->version >= 1))) |
12f1ffc4 | 157 | r = verity_hash_update(v, req, v->salt, v->salt_size, wait); |
6dbeda34 ST |
158 | |
159 | return r; | |
160 | } | |
161 | ||
d1ac3ff0 | 162 | static int verity_hash_final(struct dm_verity *v, struct ahash_request *req, |
12f1ffc4 | 163 | u8 *digest, struct crypto_wait *wait) |
6dbeda34 ST |
164 | { |
165 | int r; | |
166 | ||
f52236e0 | 167 | if (unlikely(v->salt_size && (!v->version))) { |
12f1ffc4 | 168 | r = verity_hash_update(v, req, v->salt, v->salt_size, wait); |
6dbeda34 ST |
169 | |
170 | if (r < 0) { | |
1c131886 | 171 | DMERR("%s failed updating salt: %d", __func__, r); |
d1ac3ff0 | 172 | goto out; |
6dbeda34 ST |
173 | } |
174 | } | |
175 | ||
d1ac3ff0 | 176 | ahash_request_set_crypt(req, NULL, digest, 0); |
12f1ffc4 | 177 | r = crypto_wait_req(crypto_ahash_final(req), wait); |
d1ac3ff0 | 178 | out: |
6dbeda34 ST |
179 | return r; |
180 | } | |
181 | ||
d1ac3ff0 | 182 | int verity_hash(struct dm_verity *v, struct ahash_request *req, |
28f07f2a | 183 | const u8 *data, size_t len, u8 *digest, bool may_sleep) |
6dbeda34 ST |
184 | { |
185 | int r; | |
12f1ffc4 | 186 | struct crypto_wait wait; |
6dbeda34 | 187 | |
28f07f2a | 188 | r = verity_hash_init(v, req, &wait, may_sleep); |
6dbeda34 | 189 | if (unlikely(r < 0)) |
d1ac3ff0 | 190 | goto out; |
6dbeda34 | 191 | |
12f1ffc4 | 192 | r = verity_hash_update(v, req, data, len, &wait); |
6dbeda34 | 193 | if (unlikely(r < 0)) |
d1ac3ff0 GBY |
194 | goto out; |
195 | ||
12f1ffc4 | 196 | r = verity_hash_final(v, req, digest, &wait); |
6dbeda34 | 197 | |
d1ac3ff0 GBY |
198 | out: |
199 | return r; | |
6dbeda34 ST |
200 | } |
201 | ||
a4ffc152 | 202 | static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, |
86a3238c | 203 | sector_t *hash_block, unsigned int *offset) |
a4ffc152 MP |
204 | { |
205 | sector_t position = verity_position_at_level(v, block, level); | |
86a3238c | 206 | unsigned int idx; |
a4ffc152 MP |
207 | |
208 | *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); | |
209 | ||
210 | if (!offset) | |
211 | return; | |
212 | ||
213 | idx = position & ((1 << v->hash_per_block_bits) - 1); | |
214 | if (!v->version) | |
215 | *offset = idx * v->digest_size; | |
216 | else | |
217 | *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); | |
218 | } | |
219 | ||
65ff5b7d ST |
220 | /* |
221 | * Handle verification errors. | |
222 | */ | |
223 | static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, | |
224 | unsigned long long block) | |
225 | { | |
226 | char verity_env[DM_VERITY_ENV_LENGTH]; | |
227 | char *envp[] = { verity_env, NULL }; | |
228 | const char *type_str = ""; | |
229 | struct mapped_device *md = dm_table_get_md(v->ti->table); | |
230 | ||
231 | /* Corruption should be visible in device status in all modes */ | |
5721d4e5 | 232 | v->hash_failed = true; |
65ff5b7d ST |
233 | |
234 | if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) | |
235 | goto out; | |
236 | ||
237 | v->corrupted_errs++; | |
238 | ||
239 | switch (type) { | |
240 | case DM_VERITY_BLOCK_TYPE_DATA: | |
241 | type_str = "data"; | |
242 | break; | |
243 | case DM_VERITY_BLOCK_TYPE_METADATA: | |
244 | type_str = "metadata"; | |
245 | break; | |
246 | default: | |
247 | BUG(); | |
248 | } | |
249 | ||
2eba4e64 MB |
250 | DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, |
251 | type_str, block); | |
65ff5b7d | 252 | |
074c4466 | 253 | if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { |
65ff5b7d | 254 | DMERR("%s: reached maximum errors", v->data_dev->name); |
074c4466 MW |
255 | dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0); |
256 | } | |
65ff5b7d ST |
257 | |
258 | snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu", | |
259 | DM_VERITY_ENV_VAR_NAME, type, block); | |
260 | ||
261 | kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp); | |
262 | ||
263 | out: | |
264 | if (v->mode == DM_VERITY_MODE_LOGGING) | |
265 | return 0; | |
266 | ||
267 | if (v->mode == DM_VERITY_MODE_RESTART) | |
268 | kernel_restart("dm-verity device corrupted"); | |
269 | ||
e1fef0b0 JL |
270 | if (v->mode == DM_VERITY_MODE_PANIC) |
271 | panic("dm-verity device corrupted"); | |
272 | ||
65ff5b7d ST |
273 | return 1; |
274 | } | |
275 | ||
a4ffc152 MP |
276 | /* |
277 | * Verify hash of a metadata block pertaining to the specified data block | |
278 | * ("block" argument) at a specified level ("level" argument). | |
279 | * | |
ffa39380 ST |
280 | * On successful return, verity_io_want_digest(v, io) contains the hash value |
281 | * for a lower tree level or for the data block (if we're at the lowest level). | |
a4ffc152 MP |
282 | * |
283 | * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. | |
284 | * If "skip_unverified" is false, unverified buffer is hashed and verified | |
ffa39380 | 285 | * against current value of verity_io_want_digest(v, io). |
a4ffc152 | 286 | */ |
6dbeda34 ST |
287 | static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, |
288 | sector_t block, int level, bool skip_unverified, | |
289 | u8 *want_digest) | |
a4ffc152 | 290 | { |
a4ffc152 MP |
291 | struct dm_buffer *buf; |
292 | struct buffer_aux *aux; | |
293 | u8 *data; | |
294 | int r; | |
295 | sector_t hash_block; | |
86a3238c | 296 | unsigned int offset; |
a4ffc152 MP |
297 | |
298 | verity_hash_at_level(v, block, level, &hash_block, &offset); | |
299 | ||
ba2cce82 | 300 | if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { |
5721d4e5 NH |
301 | data = dm_bufio_get(v->bufio, hash_block, &buf); |
302 | if (data == NULL) { | |
303 | /* | |
304 | * In tasklet and the hash was not in the bufio cache. | |
305 | * Return early and resume execution from a work-queue | |
306 | * to read the hash from disk. | |
307 | */ | |
308 | return -EAGAIN; | |
309 | } | |
310 | } else | |
311 | data = dm_bufio_read(v->bufio, hash_block, &buf); | |
312 | ||
fc0a4461 | 313 | if (IS_ERR(data)) |
a4ffc152 MP |
314 | return PTR_ERR(data); |
315 | ||
316 | aux = dm_bufio_get_aux_data(buf); | |
317 | ||
318 | if (!aux->hash_verified) { | |
a4ffc152 MP |
319 | if (skip_unverified) { |
320 | r = 1; | |
321 | goto release_ret_r; | |
322 | } | |
323 | ||
d1ac3ff0 | 324 | r = verity_hash(v, verity_io_hash_req(v, io), |
6dbeda34 | 325 | data, 1 << v->hash_dev_block_bits, |
28f07f2a | 326 | verity_io_real_digest(v, io), !io->in_tasklet); |
6dbeda34 | 327 | if (unlikely(r < 0)) |
a4ffc152 | 328 | goto release_ret_r; |
a4ffc152 | 329 | |
ffa39380 | 330 | if (likely(memcmp(verity_io_real_digest(v, io), want_digest, |
6dbeda34 ST |
331 | v->digest_size) == 0)) |
332 | aux->hash_verified = 1; | |
ba2cce82 MS |
333 | else if (static_branch_unlikely(&use_tasklet_enabled) && |
334 | io->in_tasklet) { | |
5721d4e5 NH |
335 | /* |
336 | * Error handling code (FEC included) cannot be run in a | |
337 | * tasklet since it may sleep, so fallback to work-queue. | |
338 | */ | |
339 | r = -EAGAIN; | |
340 | goto release_ret_r; | |
03b18887 HM |
341 | } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA, |
342 | hash_block, data, NULL) == 0) | |
a739ff3f | 343 | aux->hash_verified = 1; |
6dbeda34 ST |
344 | else if (verity_handle_err(v, |
345 | DM_VERITY_BLOCK_TYPE_METADATA, | |
346 | hash_block)) { | |
074c4466 MW |
347 | struct bio *bio = |
348 | dm_bio_from_per_bio_data(io, | |
349 | v->ti->per_io_data_size); | |
350 | dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio, | |
351 | block, 0); | |
6dbeda34 | 352 | r = -EIO; |
a4ffc152 MP |
353 | goto release_ret_r; |
354 | } | |
a4ffc152 MP |
355 | } |
356 | ||
357 | data += offset; | |
6dbeda34 ST |
358 | memcpy(want_digest, data, v->digest_size); |
359 | r = 0; | |
a4ffc152 MP |
360 | |
361 | release_ret_r: | |
362 | dm_bufio_release(buf); | |
a4ffc152 MP |
363 | return r; |
364 | } | |
365 | ||
6dbeda34 ST |
366 | /* |
367 | * Find a hash for a given block, write it to digest and verify the integrity | |
368 | * of the hash tree if necessary. | |
369 | */ | |
ffa39380 | 370 | int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, |
0cc37c2d | 371 | sector_t block, u8 *digest, bool *is_zero) |
6dbeda34 | 372 | { |
0cc37c2d | 373 | int r = 0, i; |
6dbeda34 ST |
374 | |
375 | if (likely(v->levels)) { | |
376 | /* | |
377 | * First, we try to get the requested hash for | |
378 | * the current block. If the hash block itself is | |
379 | * verified, zero is returned. If it isn't, this | |
380 | * function returns 1 and we fall back to whole | |
381 | * chain verification. | |
382 | */ | |
383 | r = verity_verify_level(v, io, block, 0, true, digest); | |
384 | if (likely(r <= 0)) | |
0cc37c2d | 385 | goto out; |
6dbeda34 ST |
386 | } |
387 | ||
388 | memcpy(digest, v->root_digest, v->digest_size); | |
389 | ||
390 | for (i = v->levels - 1; i >= 0; i--) { | |
391 | r = verity_verify_level(v, io, block, i, false, digest); | |
392 | if (unlikely(r)) | |
0cc37c2d | 393 | goto out; |
6dbeda34 | 394 | } |
0cc37c2d ST |
395 | out: |
396 | if (!r && v->zero_digest) | |
397 | *is_zero = !memcmp(v->zero_digest, digest, v->digest_size); | |
398 | else | |
399 | *is_zero = false; | |
6dbeda34 | 400 | |
0cc37c2d | 401 | return r; |
6dbeda34 ST |
402 | } |
403 | ||
d1ac3ff0 GBY |
404 | /* |
405 | * Calculates the digest for the given bio | |
406 | */ | |
d4b1aaf5 | 407 | static int verity_for_io_block(struct dm_verity *v, struct dm_verity_io *io, |
408 | struct bvec_iter *iter, struct crypto_wait *wait) | |
d1ac3ff0 GBY |
409 | { |
410 | unsigned int todo = 1 << v->data_dev_block_bits; | |
411 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); | |
412 | struct scatterlist sg; | |
413 | struct ahash_request *req = verity_io_hash_req(v, io); | |
414 | ||
415 | do { | |
416 | int r; | |
417 | unsigned int len; | |
418 | struct bio_vec bv = bio_iter_iovec(bio, *iter); | |
419 | ||
420 | sg_init_table(&sg, 1); | |
421 | ||
422 | len = bv.bv_len; | |
423 | ||
424 | if (likely(len >= todo)) | |
425 | len = todo; | |
426 | /* | |
427 | * Operating on a single page at a time looks suboptimal | |
428 | * until you consider the typical block size is 4,096B. | |
429 | * Going through this loops twice should be very rare. | |
430 | */ | |
431 | sg_set_page(&sg, bv.bv_page, len, bv.bv_offset); | |
432 | ahash_request_set_crypt(req, &sg, NULL, len); | |
12f1ffc4 | 433 | r = crypto_wait_req(crypto_ahash_update(req), wait); |
d1ac3ff0 GBY |
434 | |
435 | if (unlikely(r < 0)) { | |
1c131886 | 436 | DMERR("%s crypto op failed: %d", __func__, r); |
d1ac3ff0 GBY |
437 | return r; |
438 | } | |
439 | ||
440 | bio_advance_iter(bio, iter, len); | |
441 | todo -= len; | |
442 | } while (todo); | |
443 | ||
444 | return 0; | |
445 | } | |
446 | ||
bb4d73ac ST |
447 | /* |
448 | * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec | |
449 | * starting from iter. | |
450 | */ | |
451 | int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, | |
452 | struct bvec_iter *iter, | |
453 | int (*process)(struct dm_verity *v, | |
454 | struct dm_verity_io *io, u8 *data, | |
455 | size_t len)) | |
456 | { | |
86a3238c | 457 | unsigned int todo = 1 << v->data_dev_block_bits; |
30187e1d | 458 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
bb4d73ac ST |
459 | |
460 | do { | |
461 | int r; | |
462 | u8 *page; | |
86a3238c | 463 | unsigned int len; |
bb4d73ac ST |
464 | struct bio_vec bv = bio_iter_iovec(bio, *iter); |
465 | ||
30495e68 | 466 | page = bvec_kmap_local(&bv); |
bb4d73ac ST |
467 | len = bv.bv_len; |
468 | ||
469 | if (likely(len >= todo)) | |
470 | len = todo; | |
471 | ||
30495e68 CH |
472 | r = process(v, io, page, len); |
473 | kunmap_local(page); | |
bb4d73ac ST |
474 | |
475 | if (r < 0) | |
476 | return r; | |
477 | ||
478 | bio_advance_iter(bio, iter, len); | |
479 | todo -= len; | |
480 | } while (todo); | |
481 | ||
482 | return 0; | |
483 | } | |
484 | ||
9177f3c0 MP |
485 | static int verity_recheck_copy(struct dm_verity *v, struct dm_verity_io *io, |
486 | u8 *data, size_t len) | |
487 | { | |
488 | memcpy(data, io->recheck_buffer, len); | |
489 | io->recheck_buffer += len; | |
490 | ||
491 | return 0; | |
492 | } | |
493 | ||
66ad2fbc AB |
494 | static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, |
495 | struct bvec_iter start, sector_t cur_block) | |
9177f3c0 MP |
496 | { |
497 | struct page *page; | |
498 | void *buffer; | |
499 | int r; | |
500 | struct dm_io_request io_req; | |
501 | struct dm_io_region io_loc; | |
502 | ||
503 | page = mempool_alloc(&v->recheck_pool, GFP_NOIO); | |
504 | buffer = page_to_virt(page); | |
505 | ||
506 | io_req.bi_opf = REQ_OP_READ; | |
507 | io_req.mem.type = DM_IO_KMEM; | |
508 | io_req.mem.ptr.addr = buffer; | |
509 | io_req.notify.fn = NULL; | |
510 | io_req.client = v->io; | |
511 | io_loc.bdev = v->data_dev->bdev; | |
512 | io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); | |
513 | io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); | |
514 | r = dm_io(&io_req, 1, &io_loc, NULL); | |
515 | if (unlikely(r)) | |
516 | goto free_ret; | |
517 | ||
518 | r = verity_hash(v, verity_io_hash_req(v, io), buffer, | |
519 | 1 << v->data_dev_block_bits, | |
520 | verity_io_real_digest(v, io), true); | |
521 | if (unlikely(r)) | |
522 | goto free_ret; | |
523 | ||
524 | if (memcmp(verity_io_real_digest(v, io), | |
525 | verity_io_want_digest(v, io), v->digest_size)) { | |
526 | r = -EIO; | |
527 | goto free_ret; | |
528 | } | |
529 | ||
530 | io->recheck_buffer = buffer; | |
531 | r = verity_for_bv_block(v, io, &start, verity_recheck_copy); | |
532 | if (unlikely(r)) | |
533 | goto free_ret; | |
534 | ||
535 | r = 0; | |
536 | free_ret: | |
537 | mempool_free(page, &v->recheck_pool); | |
538 | ||
539 | return r; | |
540 | } | |
541 | ||
0cc37c2d ST |
542 | static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io, |
543 | u8 *data, size_t len) | |
544 | { | |
545 | memset(data, 0, len); | |
546 | return 0; | |
547 | } | |
548 | ||
843f38d3 PT |
549 | /* |
550 | * Moves the bio iter one data block forward. | |
551 | */ | |
552 | static inline void verity_bv_skip_block(struct dm_verity *v, | |
553 | struct dm_verity_io *io, | |
554 | struct bvec_iter *iter) | |
555 | { | |
556 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); | |
557 | ||
558 | bio_advance_iter(bio, iter, 1 << v->data_dev_block_bits); | |
559 | } | |
560 | ||
a4ffc152 MP |
561 | /* |
562 | * Verify one "dm_verity_io" structure. | |
563 | */ | |
564 | static int verity_verify_io(struct dm_verity_io *io) | |
565 | { | |
0cc37c2d | 566 | bool is_zero; |
a4ffc152 | 567 | struct dm_verity *v = io->v; |
bb4d73ac | 568 | struct bvec_iter start; |
e9307e3d MS |
569 | struct bvec_iter iter_copy; |
570 | struct bvec_iter *iter; | |
12f1ffc4 | 571 | struct crypto_wait wait; |
2c0468e0 | 572 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
5721d4e5 | 573 | unsigned int b; |
a4ffc152 | 574 | |
e9307e3d MS |
575 | if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { |
576 | /* | |
577 | * Copy the iterator in case we need to restart | |
578 | * verification in a work-queue. | |
579 | */ | |
580 | iter_copy = io->iter; | |
581 | iter = &iter_copy; | |
582 | } else | |
583 | iter = &io->iter; | |
a4ffc152 MP |
584 | |
585 | for (b = 0; b < io->n_blocks; b++) { | |
a4ffc152 | 586 | int r; |
843f38d3 | 587 | sector_t cur_block = io->block + b; |
d1ac3ff0 | 588 | struct ahash_request *req = verity_io_hash_req(v, io); |
a4ffc152 | 589 | |
e8c5d45f | 590 | if (v->validated_blocks && bio->bi_status == BLK_STS_OK && |
843f38d3 | 591 | likely(test_bit(cur_block, v->validated_blocks))) { |
e9307e3d | 592 | verity_bv_skip_block(v, io, iter); |
843f38d3 PT |
593 | continue; |
594 | } | |
595 | ||
596 | r = verity_hash_for_block(v, io, cur_block, | |
0cc37c2d ST |
597 | verity_io_want_digest(v, io), |
598 | &is_zero); | |
6dbeda34 ST |
599 | if (unlikely(r < 0)) |
600 | return r; | |
a4ffc152 | 601 | |
0cc37c2d ST |
602 | if (is_zero) { |
603 | /* | |
604 | * If we expect a zero block, don't validate, just | |
605 | * return zeros. | |
606 | */ | |
e9307e3d | 607 | r = verity_for_bv_block(v, io, iter, |
0cc37c2d ST |
608 | verity_bv_zero); |
609 | if (unlikely(r < 0)) | |
610 | return r; | |
611 | ||
612 | continue; | |
613 | } | |
614 | ||
28f07f2a | 615 | r = verity_hash_init(v, req, &wait, !io->in_tasklet); |
6dbeda34 | 616 | if (unlikely(r < 0)) |
a4ffc152 | 617 | return r; |
a4ffc152 | 618 | |
9177f3c0 | 619 | start = *iter; |
e9307e3d | 620 | r = verity_for_io_block(v, io, iter, &wait); |
bb4d73ac ST |
621 | if (unlikely(r < 0)) |
622 | return r; | |
a4ffc152 | 623 | |
d1ac3ff0 | 624 | r = verity_hash_final(v, req, verity_io_real_digest(v, io), |
12f1ffc4 | 625 | &wait); |
6dbeda34 | 626 | if (unlikely(r < 0)) |
a4ffc152 | 627 | return r; |
6dbeda34 | 628 | |
ffa39380 | 629 | if (likely(memcmp(verity_io_real_digest(v, io), |
843f38d3 PT |
630 | verity_io_want_digest(v, io), v->digest_size) == 0)) { |
631 | if (v->validated_blocks) | |
632 | set_bit(cur_block, v->validated_blocks); | |
6dbeda34 | 633 | continue; |
ba2cce82 MS |
634 | } else if (static_branch_unlikely(&use_tasklet_enabled) && |
635 | io->in_tasklet) { | |
5721d4e5 NH |
636 | /* |
637 | * Error handling code (FEC included) cannot be run in a | |
638 | * tasklet since it may sleep, so fallback to work-queue. | |
639 | */ | |
640 | return -EAGAIN; | |
9177f3c0 MP |
641 | } else if (verity_recheck(v, io, start, cur_block) == 0) { |
642 | if (v->validated_blocks) | |
643 | set_bit(cur_block, v->validated_blocks); | |
644 | continue; | |
0a36463f | 645 | #if defined(CONFIG_DM_VERITY_FEC) |
20e6fc85 | 646 | } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, |
5721d4e5 | 647 | cur_block, NULL, &start) == 0) { |
a739ff3f | 648 | continue; |
0a36463f | 649 | #endif |
20e6fc85 | 650 | } else { |
2c0468e0 AK |
651 | if (bio->bi_status) { |
652 | /* | |
653 | * Error correction failed; Just return error | |
654 | */ | |
655 | return -EIO; | |
656 | } | |
657 | if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, | |
074c4466 MW |
658 | cur_block)) { |
659 | dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", | |
660 | bio, cur_block, 0); | |
2c0468e0 | 661 | return -EIO; |
074c4466 | 662 | } |
2c0468e0 | 663 | } |
a4ffc152 | 664 | } |
a4ffc152 MP |
665 | |
666 | return 0; | |
667 | } | |
668 | ||
252bd125 HK |
669 | /* |
670 | * Skip verity work in response to I/O error when system is shutting down. | |
671 | */ | |
672 | static inline bool verity_is_system_shutting_down(void) | |
673 | { | |
674 | return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF | |
675 | || system_state == SYSTEM_RESTART; | |
676 | } | |
677 | ||
a4ffc152 MP |
678 | /* |
679 | * End one "io" structure with a given error. | |
680 | */ | |
4e4cbee9 | 681 | static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) |
a4ffc152 | 682 | { |
a4ffc152 | 683 | struct dm_verity *v = io->v; |
30187e1d | 684 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
a4ffc152 MP |
685 | |
686 | bio->bi_end_io = io->orig_bi_end_io; | |
4e4cbee9 | 687 | bio->bi_status = status; |
a4ffc152 | 688 | |
ba2cce82 | 689 | if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet) |
5721d4e5 | 690 | verity_fec_finish_io(io); |
a739ff3f | 691 | |
4246a0b6 | 692 | bio_endio(bio); |
a4ffc152 MP |
693 | } |
694 | ||
695 | static void verity_work(struct work_struct *w) | |
696 | { | |
697 | struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); | |
698 | ||
5721d4e5 NH |
699 | io->in_tasklet = false; |
700 | ||
4e4cbee9 | 701 | verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); |
a4ffc152 MP |
702 | } |
703 | ||
4246a0b6 | 704 | static void verity_end_io(struct bio *bio) |
a4ffc152 MP |
705 | { |
706 | struct dm_verity_io *io = bio->bi_private; | |
707 | ||
252bd125 | 708 | if (bio->bi_status && |
0193e396 WB |
709 | (!verity_fec_is_enabled(io->v) || |
710 | verity_is_system_shutting_down() || | |
711 | (bio->bi_opf & REQ_RAHEAD))) { | |
4e4cbee9 | 712 | verity_finish_io(io, bio->bi_status); |
a4ffc152 MP |
713 | return; |
714 | } | |
715 | ||
0a9bab39 MP |
716 | INIT_WORK(&io->work, verity_work); |
717 | queue_work(io->v->verify_wq, &io->work); | |
a4ffc152 MP |
718 | } |
719 | ||
720 | /* | |
721 | * Prefetch buffers for the specified io. | |
722 | * The root buffer is not prefetched, it is assumed that it will be cached | |
723 | * all the time. | |
724 | */ | |
3b6b7813 | 725 | static void verity_prefetch_io(struct work_struct *work) |
a4ffc152 | 726 | { |
3b6b7813 MP |
727 | struct dm_verity_prefetch_work *pw = |
728 | container_of(work, struct dm_verity_prefetch_work, work); | |
729 | struct dm_verity *v = pw->v; | |
a4ffc152 MP |
730 | int i; |
731 | ||
732 | for (i = v->levels - 2; i >= 0; i--) { | |
733 | sector_t hash_block_start; | |
734 | sector_t hash_block_end; | |
0ef0b471 | 735 | |
3b6b7813 MP |
736 | verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL); |
737 | verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL); | |
0ef0b471 | 738 | |
a4ffc152 | 739 | if (!i) { |
86a3238c | 740 | unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster); |
a4ffc152 MP |
741 | |
742 | cluster >>= v->data_dev_block_bits; | |
743 | if (unlikely(!cluster)) | |
744 | goto no_prefetch_cluster; | |
745 | ||
746 | if (unlikely(cluster & (cluster - 1))) | |
553d8fe0 | 747 | cluster = 1 << __fls(cluster); |
a4ffc152 MP |
748 | |
749 | hash_block_start &= ~(sector_t)(cluster - 1); | |
750 | hash_block_end |= cluster - 1; | |
751 | if (unlikely(hash_block_end >= v->hash_blocks)) | |
752 | hash_block_end = v->hash_blocks - 1; | |
753 | } | |
754 | no_prefetch_cluster: | |
755 | dm_bufio_prefetch(v->bufio, hash_block_start, | |
756 | hash_block_end - hash_block_start + 1); | |
757 | } | |
3b6b7813 MP |
758 | |
759 | kfree(pw); | |
760 | } | |
761 | ||
762 | static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) | |
763 | { | |
0a531c5a | 764 | sector_t block = io->block; |
765 | unsigned int n_blocks = io->n_blocks; | |
3b6b7813 MP |
766 | struct dm_verity_prefetch_work *pw; |
767 | ||
0a531c5a | 768 | if (v->validated_blocks) { |
769 | while (n_blocks && test_bit(block, v->validated_blocks)) { | |
770 | block++; | |
771 | n_blocks--; | |
772 | } | |
773 | while (n_blocks && test_bit(block + n_blocks - 1, | |
774 | v->validated_blocks)) | |
775 | n_blocks--; | |
776 | if (!n_blocks) | |
777 | return; | |
778 | } | |
779 | ||
3b6b7813 MP |
780 | pw = kmalloc(sizeof(struct dm_verity_prefetch_work), |
781 | GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); | |
782 | ||
783 | if (!pw) | |
784 | return; | |
785 | ||
786 | INIT_WORK(&pw->work, verity_prefetch_io); | |
787 | pw->v = v; | |
0a531c5a | 788 | pw->block = block; |
789 | pw->n_blocks = n_blocks; | |
3b6b7813 | 790 | queue_work(v->verify_wq, &pw->work); |
a4ffc152 MP |
791 | } |
792 | ||
793 | /* | |
794 | * Bio map function. It allocates dm_verity_io structure and bio vector and | |
795 | * fills them. Then it issues prefetches and the I/O. | |
796 | */ | |
7de3ee57 | 797 | static int verity_map(struct dm_target *ti, struct bio *bio) |
a4ffc152 MP |
798 | { |
799 | struct dm_verity *v = ti->private; | |
800 | struct dm_verity_io *io; | |
801 | ||
74d46992 | 802 | bio_set_dev(bio, v->data_dev->bdev); |
4f024f37 | 803 | bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector); |
a4ffc152 | 804 | |
86a3238c | 805 | if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & |
a4ffc152 MP |
806 | ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { |
807 | DMERR_LIMIT("unaligned io"); | |
846785e6 | 808 | return DM_MAPIO_KILL; |
a4ffc152 MP |
809 | } |
810 | ||
f73a1c7d | 811 | if (bio_end_sector(bio) >> |
a4ffc152 MP |
812 | (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { |
813 | DMERR_LIMIT("io out of range"); | |
846785e6 | 814 | return DM_MAPIO_KILL; |
a4ffc152 MP |
815 | } |
816 | ||
817 | if (bio_data_dir(bio) == WRITE) | |
846785e6 | 818 | return DM_MAPIO_KILL; |
a4ffc152 | 819 | |
30187e1d | 820 | io = dm_per_bio_data(bio, ti->per_io_data_size); |
a4ffc152 | 821 | io->v = v; |
a4ffc152 | 822 | io->orig_bi_end_io = bio->bi_end_io; |
4f024f37 KO |
823 | io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); |
824 | io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; | |
a4ffc152 MP |
825 | |
826 | bio->bi_end_io = verity_end_io; | |
827 | bio->bi_private = io; | |
003b5c57 | 828 | io->iter = bio->bi_iter; |
a4ffc152 | 829 | |
7be05bdf WB |
830 | verity_fec_init_io(io); |
831 | ||
3b6b7813 | 832 | verity_submit_prefetch(v, io); |
a4ffc152 | 833 | |
ed00aabd | 834 | submit_bio_noacct(bio); |
a4ffc152 MP |
835 | |
836 | return DM_MAPIO_SUBMITTED; | |
837 | } | |
838 | ||
839 | /* | |
840 | * Status: V (valid) or C (corruption found) | |
841 | */ | |
fd7c092e | 842 | static void verity_status(struct dm_target *ti, status_type_t type, |
86a3238c | 843 | unsigned int status_flags, char *result, unsigned int maxlen) |
a4ffc152 MP |
844 | { |
845 | struct dm_verity *v = ti->private; | |
86a3238c HM |
846 | unsigned int args = 0; |
847 | unsigned int sz = 0; | |
848 | unsigned int x; | |
a4ffc152 MP |
849 | |
850 | switch (type) { | |
851 | case STATUSTYPE_INFO: | |
852 | DMEMIT("%c", v->hash_failed ? 'C' : 'V'); | |
853 | break; | |
854 | case STATUSTYPE_TABLE: | |
855 | DMEMIT("%u %s %s %u %u %llu %llu %s ", | |
856 | v->version, | |
857 | v->data_dev->name, | |
858 | v->hash_dev->name, | |
859 | 1 << v->data_dev_block_bits, | |
860 | 1 << v->hash_dev_block_bits, | |
861 | (unsigned long long)v->data_blocks, | |
862 | (unsigned long long)v->hash_start, | |
863 | v->alg_name | |
864 | ); | |
865 | for (x = 0; x < v->digest_size; x++) | |
866 | DMEMIT("%02x", v->root_digest[x]); | |
867 | DMEMIT(" "); | |
868 | if (!v->salt_size) | |
869 | DMEMIT("-"); | |
870 | else | |
871 | for (x = 0; x < v->salt_size; x++) | |
872 | DMEMIT("%02x", v->salt[x]); | |
a739ff3f ST |
873 | if (v->mode != DM_VERITY_MODE_EIO) |
874 | args++; | |
875 | if (verity_fec_is_enabled(v)) | |
876 | args += DM_VERITY_OPTS_FEC; | |
0cc37c2d ST |
877 | if (v->zero_digest) |
878 | args++; | |
843f38d3 PT |
879 | if (v->validated_blocks) |
880 | args++; | |
5721d4e5 NH |
881 | if (v->use_tasklet) |
882 | args++; | |
88cd3e6c JK |
883 | if (v->signature_key_desc) |
884 | args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; | |
a739ff3f ST |
885 | if (!args) |
886 | return; | |
887 | DMEMIT(" %u", args); | |
65ff5b7d | 888 | if (v->mode != DM_VERITY_MODE_EIO) { |
a739ff3f | 889 | DMEMIT(" "); |
65ff5b7d ST |
890 | switch (v->mode) { |
891 | case DM_VERITY_MODE_LOGGING: | |
892 | DMEMIT(DM_VERITY_OPT_LOGGING); | |
893 | break; | |
894 | case DM_VERITY_MODE_RESTART: | |
895 | DMEMIT(DM_VERITY_OPT_RESTART); | |
896 | break; | |
e1fef0b0 JL |
897 | case DM_VERITY_MODE_PANIC: |
898 | DMEMIT(DM_VERITY_OPT_PANIC); | |
899 | break; | |
65ff5b7d ST |
900 | default: |
901 | BUG(); | |
902 | } | |
903 | } | |
0cc37c2d ST |
904 | if (v->zero_digest) |
905 | DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); | |
843f38d3 PT |
906 | if (v->validated_blocks) |
907 | DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); | |
5721d4e5 NH |
908 | if (v->use_tasklet) |
909 | DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); | |
a739ff3f | 910 | sz = verity_fec_status_table(v, sz, result, maxlen); |
88cd3e6c JK |
911 | if (v->signature_key_desc) |
912 | DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY | |
913 | " %s", v->signature_key_desc); | |
a4ffc152 | 914 | break; |
8ec45662 TS |
915 | |
916 | case STATUSTYPE_IMA: | |
917 | DMEMIT_TARGET_NAME_VERSION(ti->type); | |
918 | DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V'); | |
919 | DMEMIT(",verity_version=%u", v->version); | |
920 | DMEMIT(",data_device_name=%s", v->data_dev->name); | |
921 | DMEMIT(",hash_device_name=%s", v->hash_dev->name); | |
922 | DMEMIT(",verity_algorithm=%s", v->alg_name); | |
923 | ||
924 | DMEMIT(",root_digest="); | |
925 | for (x = 0; x < v->digest_size; x++) | |
926 | DMEMIT("%02x", v->root_digest[x]); | |
927 | ||
928 | DMEMIT(",salt="); | |
929 | if (!v->salt_size) | |
930 | DMEMIT("-"); | |
931 | else | |
932 | for (x = 0; x < v->salt_size; x++) | |
933 | DMEMIT("%02x", v->salt[x]); | |
934 | ||
935 | DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n'); | |
936 | DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n'); | |
33ace4ca TS |
937 | if (v->signature_key_desc) |
938 | DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc); | |
8ec45662 TS |
939 | |
940 | if (v->mode != DM_VERITY_MODE_EIO) { | |
941 | DMEMIT(",verity_mode="); | |
942 | switch (v->mode) { | |
943 | case DM_VERITY_MODE_LOGGING: | |
944 | DMEMIT(DM_VERITY_OPT_LOGGING); | |
945 | break; | |
946 | case DM_VERITY_MODE_RESTART: | |
947 | DMEMIT(DM_VERITY_OPT_RESTART); | |
948 | break; | |
949 | case DM_VERITY_MODE_PANIC: | |
950 | DMEMIT(DM_VERITY_OPT_PANIC); | |
951 | break; | |
952 | default: | |
953 | DMEMIT("invalid"); | |
954 | } | |
955 | } | |
956 | DMEMIT(";"); | |
957 | break; | |
a4ffc152 | 958 | } |
a4ffc152 MP |
959 | } |
960 | ||
5bd5e8d8 | 961 | static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) |
a4ffc152 MP |
962 | { |
963 | struct dm_verity *v = ti->private; | |
e56f81e0 CH |
964 | |
965 | *bdev = v->data_dev->bdev; | |
a4ffc152 | 966 | |
6dcbb52c | 967 | if (v->data_start || ti->len != bdev_nr_sectors(v->data_dev->bdev)) |
e56f81e0 CH |
968 | return 1; |
969 | return 0; | |
a4ffc152 MP |
970 | } |
971 | ||
a4ffc152 MP |
972 | static int verity_iterate_devices(struct dm_target *ti, |
973 | iterate_devices_callout_fn fn, void *data) | |
974 | { | |
975 | struct dm_verity *v = ti->private; | |
976 | ||
977 | return fn(ti, v->data_dev, v->data_start, ti->len, data); | |
978 | } | |
979 | ||
980 | static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) | |
981 | { | |
982 | struct dm_verity *v = ti->private; | |
983 | ||
984 | if (limits->logical_block_size < 1 << v->data_dev_block_bits) | |
985 | limits->logical_block_size = 1 << v->data_dev_block_bits; | |
986 | ||
987 | if (limits->physical_block_size < 1 << v->data_dev_block_bits) | |
988 | limits->physical_block_size = 1 << v->data_dev_block_bits; | |
989 | ||
990 | blk_limits_io_min(limits, limits->logical_block_size); | |
991 | } | |
992 | ||
993 | static void verity_dtr(struct dm_target *ti) | |
994 | { | |
995 | struct dm_verity *v = ti->private; | |
996 | ||
997 | if (v->verify_wq) | |
998 | destroy_workqueue(v->verify_wq); | |
999 | ||
9177f3c0 MP |
1000 | mempool_exit(&v->recheck_pool); |
1001 | if (v->io) | |
1002 | dm_io_client_destroy(v->io); | |
1003 | ||
a4ffc152 MP |
1004 | if (v->bufio) |
1005 | dm_bufio_client_destroy(v->bufio); | |
1006 | ||
843f38d3 | 1007 | kvfree(v->validated_blocks); |
a4ffc152 MP |
1008 | kfree(v->salt); |
1009 | kfree(v->root_digest); | |
0cc37c2d | 1010 | kfree(v->zero_digest); |
a4ffc152 MP |
1011 | |
1012 | if (v->tfm) | |
d1ac3ff0 | 1013 | crypto_free_ahash(v->tfm); |
a4ffc152 MP |
1014 | |
1015 | kfree(v->alg_name); | |
1016 | ||
1017 | if (v->hash_dev) | |
1018 | dm_put_device(ti, v->hash_dev); | |
1019 | ||
1020 | if (v->data_dev) | |
1021 | dm_put_device(ti, v->data_dev); | |
1022 | ||
a739ff3f ST |
1023 | verity_fec_dtr(v); |
1024 | ||
88cd3e6c JK |
1025 | kfree(v->signature_key_desc); |
1026 | ||
ba2cce82 MS |
1027 | if (v->use_tasklet) |
1028 | static_branch_dec(&use_tasklet_enabled); | |
1029 | ||
a4ffc152 | 1030 | kfree(v); |
074c4466 MW |
1031 | |
1032 | dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); | |
a4ffc152 MP |
1033 | } |
1034 | ||
843f38d3 PT |
1035 | static int verity_alloc_most_once(struct dm_verity *v) |
1036 | { | |
1037 | struct dm_target *ti = v->ti; | |
1038 | ||
1039 | /* the bitset can only handle INT_MAX blocks */ | |
1040 | if (v->data_blocks > INT_MAX) { | |
1041 | ti->error = "device too large to use check_at_most_once"; | |
1042 | return -E2BIG; | |
1043 | } | |
1044 | ||
778e1cdd KC |
1045 | v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks), |
1046 | sizeof(unsigned long), | |
1047 | GFP_KERNEL); | |
843f38d3 PT |
1048 | if (!v->validated_blocks) { |
1049 | ti->error = "failed to allocate bitset for check_at_most_once"; | |
1050 | return -ENOMEM; | |
1051 | } | |
1052 | ||
1053 | return 0; | |
1054 | } | |
1055 | ||
0cc37c2d ST |
1056 | static int verity_alloc_zero_digest(struct dm_verity *v) |
1057 | { | |
1058 | int r = -ENOMEM; | |
d1ac3ff0 | 1059 | struct ahash_request *req; |
0cc37c2d ST |
1060 | u8 *zero_data; |
1061 | ||
1062 | v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); | |
1063 | ||
1064 | if (!v->zero_digest) | |
1065 | return r; | |
1066 | ||
d1ac3ff0 | 1067 | req = kmalloc(v->ahash_reqsize, GFP_KERNEL); |
0cc37c2d | 1068 | |
d1ac3ff0 | 1069 | if (!req) |
0cc37c2d ST |
1070 | return r; /* verity_dtr will free zero_digest */ |
1071 | ||
1072 | zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); | |
1073 | ||
1074 | if (!zero_data) | |
1075 | goto out; | |
1076 | ||
d1ac3ff0 | 1077 | r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits, |
28f07f2a | 1078 | v->zero_digest, true); |
0cc37c2d ST |
1079 | |
1080 | out: | |
d1ac3ff0 | 1081 | kfree(req); |
0cc37c2d ST |
1082 | kfree(zero_data); |
1083 | ||
1084 | return r; | |
1085 | } | |
1086 | ||
219a9b5e JL |
1087 | static inline bool verity_is_verity_mode(const char *arg_name) |
1088 | { | |
1089 | return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) || | |
1090 | !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) || | |
1091 | !strcasecmp(arg_name, DM_VERITY_OPT_PANIC)); | |
1092 | } | |
1093 | ||
1094 | static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name) | |
1095 | { | |
1096 | if (v->mode) | |
1097 | return -EINVAL; | |
1098 | ||
1099 | if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) | |
1100 | v->mode = DM_VERITY_MODE_LOGGING; | |
1101 | else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) | |
1102 | v->mode = DM_VERITY_MODE_RESTART; | |
1103 | else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC)) | |
1104 | v->mode = DM_VERITY_MODE_PANIC; | |
1105 | ||
1106 | return 0; | |
1107 | } | |
1108 | ||
88cd3e6c | 1109 | static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, |
df326e7a MS |
1110 | struct dm_verity_sig_opts *verify_args, |
1111 | bool only_modifier_opts) | |
753c1fd0 | 1112 | { |
f876df9f | 1113 | int r = 0; |
86a3238c | 1114 | unsigned int argc; |
753c1fd0 ST |
1115 | struct dm_target *ti = v->ti; |
1116 | const char *arg_name; | |
1117 | ||
5916a22b | 1118 | static const struct dm_arg _args[] = { |
753c1fd0 ST |
1119 | {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, |
1120 | }; | |
1121 | ||
1122 | r = dm_read_arg_group(_args, as, &argc, &ti->error); | |
1123 | if (r) | |
1124 | return -EINVAL; | |
1125 | ||
1126 | if (!argc) | |
1127 | return 0; | |
1128 | ||
1129 | do { | |
1130 | arg_name = dm_shift_arg(as); | |
1131 | argc--; | |
1132 | ||
219a9b5e | 1133 | if (verity_is_verity_mode(arg_name)) { |
df326e7a MS |
1134 | if (only_modifier_opts) |
1135 | continue; | |
219a9b5e JL |
1136 | r = verity_parse_verity_mode(v, arg_name); |
1137 | if (r) { | |
1138 | ti->error = "Conflicting error handling parameters"; | |
1139 | return r; | |
1140 | } | |
e1fef0b0 JL |
1141 | continue; |
1142 | ||
0cc37c2d | 1143 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) { |
df326e7a MS |
1144 | if (only_modifier_opts) |
1145 | continue; | |
0cc37c2d ST |
1146 | r = verity_alloc_zero_digest(v); |
1147 | if (r) { | |
1148 | ti->error = "Cannot allocate zero digest"; | |
1149 | return r; | |
1150 | } | |
1151 | continue; | |
1152 | ||
843f38d3 | 1153 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) { |
df326e7a MS |
1154 | if (only_modifier_opts) |
1155 | continue; | |
843f38d3 PT |
1156 | r = verity_alloc_most_once(v); |
1157 | if (r) | |
1158 | return r; | |
1159 | continue; | |
1160 | ||
5721d4e5 NH |
1161 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { |
1162 | v->use_tasklet = true; | |
ba2cce82 | 1163 | static_branch_inc(&use_tasklet_enabled); |
5721d4e5 NH |
1164 | continue; |
1165 | ||
a739ff3f | 1166 | } else if (verity_is_fec_opt_arg(arg_name)) { |
df326e7a MS |
1167 | if (only_modifier_opts) |
1168 | continue; | |
a739ff3f ST |
1169 | r = verity_fec_parse_opt_args(as, v, &argc, arg_name); |
1170 | if (r) | |
1171 | return r; | |
1172 | continue; | |
5721d4e5 | 1173 | |
88cd3e6c | 1174 | } else if (verity_verify_is_sig_opt_arg(arg_name)) { |
df326e7a MS |
1175 | if (only_modifier_opts) |
1176 | continue; | |
88cd3e6c JK |
1177 | r = verity_verify_sig_parse_opt_args(as, v, |
1178 | verify_args, | |
1179 | &argc, arg_name); | |
1180 | if (r) | |
1181 | return r; | |
1182 | continue; | |
f876df9f MS |
1183 | |
1184 | } else if (only_modifier_opts) { | |
1185 | /* | |
1186 | * Ignore unrecognized opt, could easily be an extra | |
1187 | * argument to an option whose parsing was skipped. | |
1188 | * Normal parsing (@only_modifier_opts=false) will | |
1189 | * properly parse all options (and their extra args). | |
1190 | */ | |
1191 | continue; | |
753c1fd0 ST |
1192 | } |
1193 | ||
f876df9f | 1194 | DMERR("Unrecognized verity feature request: %s", arg_name); |
753c1fd0 ST |
1195 | ti->error = "Unrecognized verity feature request"; |
1196 | return -EINVAL; | |
1197 | } while (argc && !r); | |
1198 | ||
1199 | return r; | |
1200 | } | |
1201 | ||
a4ffc152 MP |
1202 | /* |
1203 | * Target parameters: | |
1204 | * <version> The current format is version 1. | |
1205 | * Vsn 0 is compatible with original Chromium OS releases. | |
1206 | * <data device> | |
1207 | * <hash device> | |
1208 | * <data block size> | |
1209 | * <hash block size> | |
1210 | * <the number of data blocks> | |
1211 | * <hash start block> | |
1212 | * <algorithm> | |
1213 | * <digest> | |
1214 | * <salt> Hex string or "-" if no salt. | |
1215 | */ | |
86a3238c | 1216 | static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) |
a4ffc152 MP |
1217 | { |
1218 | struct dm_verity *v; | |
88cd3e6c | 1219 | struct dm_verity_sig_opts verify_args = {0}; |
65ff5b7d | 1220 | struct dm_arg_set as; |
753c1fd0 | 1221 | unsigned int num; |
a4ffc152 MP |
1222 | unsigned long long num_ll; |
1223 | int r; | |
1224 | int i; | |
1225 | sector_t hash_position; | |
1226 | char dummy; | |
88cd3e6c | 1227 | char *root_hash_digest_to_validate; |
a4ffc152 MP |
1228 | |
1229 | v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); | |
1230 | if (!v) { | |
1231 | ti->error = "Cannot allocate verity structure"; | |
1232 | return -ENOMEM; | |
1233 | } | |
1234 | ti->private = v; | |
1235 | v->ti = ti; | |
1236 | ||
a739ff3f ST |
1237 | r = verity_fec_ctr_alloc(v); |
1238 | if (r) | |
1239 | goto bad; | |
1240 | ||
05bdb996 | 1241 | if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) { |
a4ffc152 MP |
1242 | ti->error = "Device must be readonly"; |
1243 | r = -EINVAL; | |
1244 | goto bad; | |
1245 | } | |
1246 | ||
65ff5b7d ST |
1247 | if (argc < 10) { |
1248 | ti->error = "Not enough arguments"; | |
a4ffc152 MP |
1249 | r = -EINVAL; |
1250 | goto bad; | |
1251 | } | |
1252 | ||
df326e7a MS |
1253 | /* Parse optional parameters that modify primary args */ |
1254 | if (argc > 10) { | |
1255 | as.argc = argc - 10; | |
1256 | as.argv = argv + 10; | |
1257 | r = verity_parse_opt_args(&as, v, &verify_args, true); | |
1258 | if (r < 0) | |
1259 | goto bad; | |
1260 | } | |
1261 | ||
5d8be843 MP |
1262 | if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || |
1263 | num > 1) { | |
a4ffc152 MP |
1264 | ti->error = "Invalid version"; |
1265 | r = -EINVAL; | |
1266 | goto bad; | |
1267 | } | |
1268 | v->version = num; | |
1269 | ||
05bdb996 | 1270 | r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev); |
a4ffc152 MP |
1271 | if (r) { |
1272 | ti->error = "Data device lookup failed"; | |
1273 | goto bad; | |
1274 | } | |
1275 | ||
05bdb996 | 1276 | r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev); |
a4ffc152 | 1277 | if (r) { |
21ffe552 | 1278 | ti->error = "Hash device lookup failed"; |
a4ffc152 MP |
1279 | goto bad; |
1280 | } | |
1281 | ||
1282 | if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || | |
1283 | !num || (num & (num - 1)) || | |
1284 | num < bdev_logical_block_size(v->data_dev->bdev) || | |
1285 | num > PAGE_SIZE) { | |
1286 | ti->error = "Invalid data device block size"; | |
1287 | r = -EINVAL; | |
1288 | goto bad; | |
1289 | } | |
553d8fe0 | 1290 | v->data_dev_block_bits = __ffs(num); |
a4ffc152 MP |
1291 | |
1292 | if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || | |
1293 | !num || (num & (num - 1)) || | |
1294 | num < bdev_logical_block_size(v->hash_dev->bdev) || | |
1295 | num > INT_MAX) { | |
1296 | ti->error = "Invalid hash device block size"; | |
1297 | r = -EINVAL; | |
1298 | goto bad; | |
1299 | } | |
553d8fe0 | 1300 | v->hash_dev_block_bits = __ffs(num); |
a4ffc152 MP |
1301 | |
1302 | if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || | |
1d55f6bc MP |
1303 | (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) |
1304 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { | |
a4ffc152 MP |
1305 | ti->error = "Invalid data blocks"; |
1306 | r = -EINVAL; | |
1307 | goto bad; | |
1308 | } | |
1309 | v->data_blocks = num_ll; | |
1310 | ||
1311 | if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { | |
1312 | ti->error = "Data device is too small"; | |
1313 | r = -EINVAL; | |
1314 | goto bad; | |
1315 | } | |
1316 | ||
1317 | if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || | |
1d55f6bc MP |
1318 | (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) |
1319 | >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { | |
a4ffc152 MP |
1320 | ti->error = "Invalid hash start"; |
1321 | r = -EINVAL; | |
1322 | goto bad; | |
1323 | } | |
1324 | v->hash_start = num_ll; | |
1325 | ||
1326 | v->alg_name = kstrdup(argv[7], GFP_KERNEL); | |
1327 | if (!v->alg_name) { | |
1328 | ti->error = "Cannot allocate algorithm name"; | |
1329 | r = -ENOMEM; | |
1330 | goto bad; | |
1331 | } | |
1332 | ||
df326e7a MS |
1333 | v->tfm = crypto_alloc_ahash(v->alg_name, 0, |
1334 | v->use_tasklet ? CRYPTO_ALG_ASYNC : 0); | |
a4ffc152 MP |
1335 | if (IS_ERR(v->tfm)) { |
1336 | ti->error = "Cannot initialize hash function"; | |
1337 | r = PTR_ERR(v->tfm); | |
1338 | v->tfm = NULL; | |
1339 | goto bad; | |
1340 | } | |
bbf6a566 EB |
1341 | |
1342 | /* | |
1343 | * dm-verity performance can vary greatly depending on which hash | |
1344 | * algorithm implementation is used. Help people debug performance | |
1345 | * problems by logging the ->cra_driver_name. | |
1346 | */ | |
1347 | DMINFO("%s using implementation \"%s\"", v->alg_name, | |
1348 | crypto_hash_alg_common(v->tfm)->base.cra_driver_name); | |
1349 | ||
d1ac3ff0 | 1350 | v->digest_size = crypto_ahash_digestsize(v->tfm); |
a4ffc152 MP |
1351 | if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { |
1352 | ti->error = "Digest size too big"; | |
1353 | r = -EINVAL; | |
1354 | goto bad; | |
1355 | } | |
d1ac3ff0 GBY |
1356 | v->ahash_reqsize = sizeof(struct ahash_request) + |
1357 | crypto_ahash_reqsize(v->tfm); | |
a4ffc152 MP |
1358 | |
1359 | v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); | |
1360 | if (!v->root_digest) { | |
1361 | ti->error = "Cannot allocate root digest"; | |
1362 | r = -ENOMEM; | |
1363 | goto bad; | |
1364 | } | |
1365 | if (strlen(argv[8]) != v->digest_size * 2 || | |
1366 | hex2bin(v->root_digest, argv[8], v->digest_size)) { | |
1367 | ti->error = "Invalid root digest"; | |
1368 | r = -EINVAL; | |
1369 | goto bad; | |
1370 | } | |
88cd3e6c | 1371 | root_hash_digest_to_validate = argv[8]; |
a4ffc152 MP |
1372 | |
1373 | if (strcmp(argv[9], "-")) { | |
1374 | v->salt_size = strlen(argv[9]) / 2; | |
1375 | v->salt = kmalloc(v->salt_size, GFP_KERNEL); | |
1376 | if (!v->salt) { | |
1377 | ti->error = "Cannot allocate salt"; | |
1378 | r = -ENOMEM; | |
1379 | goto bad; | |
1380 | } | |
1381 | if (strlen(argv[9]) != v->salt_size * 2 || | |
1382 | hex2bin(v->salt, argv[9], v->salt_size)) { | |
1383 | ti->error = "Invalid salt"; | |
1384 | r = -EINVAL; | |
1385 | goto bad; | |
1386 | } | |
1387 | } | |
1388 | ||
65ff5b7d ST |
1389 | argv += 10; |
1390 | argc -= 10; | |
1391 | ||
1392 | /* Optional parameters */ | |
1393 | if (argc) { | |
1394 | as.argc = argc; | |
1395 | as.argv = argv; | |
df326e7a | 1396 | r = verity_parse_opt_args(&as, v, &verify_args, false); |
753c1fd0 | 1397 | if (r < 0) |
65ff5b7d | 1398 | goto bad; |
65ff5b7d ST |
1399 | } |
1400 | ||
88cd3e6c JK |
1401 | /* Root hash signature is a optional parameter*/ |
1402 | r = verity_verify_root_hash(root_hash_digest_to_validate, | |
1403 | strlen(root_hash_digest_to_validate), | |
1404 | verify_args.sig, | |
1405 | verify_args.sig_size); | |
1406 | if (r < 0) { | |
1407 | ti->error = "Root hash verification failed"; | |
1408 | goto bad; | |
1409 | } | |
a4ffc152 | 1410 | v->hash_per_block_bits = |
553d8fe0 | 1411 | __fls((1 << v->hash_dev_block_bits) / v->digest_size); |
a4ffc152 MP |
1412 | |
1413 | v->levels = 0; | |
1414 | if (v->data_blocks) | |
1415 | while (v->hash_per_block_bits * v->levels < 64 && | |
1416 | (unsigned long long)(v->data_blocks - 1) >> | |
1417 | (v->hash_per_block_bits * v->levels)) | |
1418 | v->levels++; | |
1419 | ||
1420 | if (v->levels > DM_VERITY_MAX_LEVELS) { | |
1421 | ti->error = "Too many tree levels"; | |
1422 | r = -E2BIG; | |
1423 | goto bad; | |
1424 | } | |
1425 | ||
1426 | hash_position = v->hash_start; | |
1427 | for (i = v->levels - 1; i >= 0; i--) { | |
1428 | sector_t s; | |
0ef0b471 | 1429 | |
a4ffc152 | 1430 | v->hash_level_block[i] = hash_position; |
b1bf2de0 MP |
1431 | s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) |
1432 | >> ((i + 1) * v->hash_per_block_bits); | |
a4ffc152 MP |
1433 | if (hash_position + s < hash_position) { |
1434 | ti->error = "Hash device offset overflow"; | |
1435 | r = -E2BIG; | |
1436 | goto bad; | |
1437 | } | |
1438 | hash_position += s; | |
1439 | } | |
1440 | v->hash_blocks = hash_position; | |
1441 | ||
9177f3c0 MP |
1442 | r = mempool_init_page_pool(&v->recheck_pool, 1, 0); |
1443 | if (unlikely(r)) { | |
1444 | ti->error = "Cannot allocate mempool"; | |
1445 | goto bad; | |
1446 | } | |
1447 | ||
1448 | v->io = dm_io_client_create(); | |
1449 | if (IS_ERR(v->io)) { | |
1450 | r = PTR_ERR(v->io); | |
1451 | v->io = NULL; | |
1452 | ti->error = "Cannot allocate dm io"; | |
1453 | goto bad; | |
1454 | } | |
1455 | ||
a4ffc152 MP |
1456 | v->bufio = dm_bufio_client_create(v->hash_dev->bdev, |
1457 | 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), | |
5721d4e5 NH |
1458 | dm_bufio_alloc_callback, NULL, |
1459 | v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0); | |
a4ffc152 MP |
1460 | if (IS_ERR(v->bufio)) { |
1461 | ti->error = "Cannot initialize dm-bufio"; | |
1462 | r = PTR_ERR(v->bufio); | |
1463 | v->bufio = NULL; | |
1464 | goto bad; | |
1465 | } | |
1466 | ||
1467 | if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) { | |
1468 | ti->error = "Hash device is too small"; | |
1469 | r = -E2BIG; | |
1470 | goto bad; | |
1471 | } | |
1472 | ||
afd41fff NH |
1473 | /* |
1474 | * Using WQ_HIGHPRI improves throughput and completion latency by | |
1475 | * reducing wait times when reading from a dm-verity device. | |
1476 | * | |
1477 | * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI | |
1478 | * allows verify_wq to preempt softirq since verification in tasklet | |
1479 | * will fall-back to using it for error handling (or if the bufio cache | |
1480 | * doesn't have required hashes). | |
1481 | */ | |
c25da5b7 | 1482 | v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); |
a4ffc152 MP |
1483 | if (!v->verify_wq) { |
1484 | ti->error = "Cannot allocate workqueue"; | |
1485 | r = -ENOMEM; | |
1486 | goto bad; | |
1487 | } | |
1488 | ||
30187e1d | 1489 | ti->per_io_data_size = sizeof(struct dm_verity_io) + |
d1ac3ff0 | 1490 | v->ahash_reqsize + v->digest_size * 2; |
a739ff3f ST |
1491 | |
1492 | r = verity_fec_ctr(v); | |
1493 | if (r) | |
1494 | goto bad; | |
1495 | ||
30187e1d MS |
1496 | ti->per_io_data_size = roundup(ti->per_io_data_size, |
1497 | __alignof__(struct dm_verity_io)); | |
a739ff3f | 1498 | |
88cd3e6c JK |
1499 | verity_verify_sig_opts_cleanup(&verify_args); |
1500 | ||
074c4466 MW |
1501 | dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); |
1502 | ||
a4ffc152 MP |
1503 | return 0; |
1504 | ||
1505 | bad: | |
88cd3e6c JK |
1506 | |
1507 | verity_verify_sig_opts_cleanup(&verify_args); | |
074c4466 | 1508 | dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); |
a4ffc152 MP |
1509 | verity_dtr(ti); |
1510 | ||
1511 | return r; | |
1512 | } | |
1513 | ||
b6c1c574 MK |
1514 | /* |
1515 | * Check whether a DM target is a verity target. | |
1516 | */ | |
1517 | bool dm_is_verity_target(struct dm_target *ti) | |
1518 | { | |
1519 | return ti->type->module == THIS_MODULE; | |
1520 | } | |
1521 | ||
916ef623 MK |
1522 | /* |
1523 | * Get the verity mode (error behavior) of a verity target. | |
1524 | * | |
1525 | * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity | |
1526 | * target. | |
1527 | */ | |
1528 | int dm_verity_get_mode(struct dm_target *ti) | |
1529 | { | |
1530 | struct dm_verity *v = ti->private; | |
1531 | ||
1532 | if (!dm_is_verity_target(ti)) | |
1533 | return -EINVAL; | |
1534 | ||
1535 | return v->mode; | |
1536 | } | |
1537 | ||
b6c1c574 MK |
1538 | /* |
1539 | * Get the root digest of a verity target. | |
1540 | * | |
1541 | * Returns a copy of the root digest, the caller is responsible for | |
1542 | * freeing the memory of the digest. | |
1543 | */ | |
1544 | int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) | |
1545 | { | |
1546 | struct dm_verity *v = ti->private; | |
1547 | ||
1548 | if (!dm_is_verity_target(ti)) | |
1549 | return -EINVAL; | |
1550 | ||
1551 | *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); | |
1552 | if (*root_digest == NULL) | |
1553 | return -ENOMEM; | |
1554 | ||
1555 | *digest_size = v->digest_size; | |
1556 | ||
1557 | return 0; | |
1558 | } | |
1559 | ||
a4ffc152 MP |
1560 | static struct target_type verity_target = { |
1561 | .name = "verity", | |
4caae584 | 1562 | .features = DM_TARGET_IMMUTABLE, |
0e0c50e8 | 1563 | .version = {1, 10, 0}, |
a4ffc152 MP |
1564 | .module = THIS_MODULE, |
1565 | .ctr = verity_ctr, | |
1566 | .dtr = verity_dtr, | |
1567 | .map = verity_map, | |
1568 | .status = verity_status, | |
e56f81e0 | 1569 | .prepare_ioctl = verity_prepare_ioctl, |
a4ffc152 MP |
1570 | .iterate_devices = verity_iterate_devices, |
1571 | .io_hints = verity_io_hints, | |
1572 | }; | |
3664ff82 | 1573 | module_dm(verity); |
a4ffc152 MP |
1574 | |
1575 | MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); | |
1576 | MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); | |
1577 | MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); | |
1578 | MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); | |
1579 | MODULE_LICENSE("GPL"); |