Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
a739ff3f ST |
2 | /* |
3 | * Copyright (C) 2015 Google, Inc. | |
4 | * | |
5 | * Author: Sami Tolvanen <samitolvanen@google.com> | |
a739ff3f ST |
6 | */ |
7 | ||
8 | #include "dm-verity-fec.h" | |
9 | #include <linux/math64.h> | |
10 | ||
11 | #define DM_MSG_PREFIX "verity-fec" | |
12 | ||
13 | /* | |
14 | * If error correction has been configured, returns true. | |
15 | */ | |
16 | bool verity_fec_is_enabled(struct dm_verity *v) | |
17 | { | |
18 | return v->fec && v->fec->dev; | |
19 | } | |
20 | ||
21 | /* | |
22 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable | |
23 | * length fields. | |
24 | */ | |
25 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) | |
26 | { | |
38bc1ab1 MP |
27 | return (struct dm_verity_fec_io *) |
28 | ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); | |
a739ff3f ST |
29 | } |
30 | ||
31 | /* | |
32 | * Return an interleaved offset for a byte in RS block. | |
33 | */ | |
34 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) | |
35 | { | |
36 | u32 mod; | |
37 | ||
38 | mod = do_div(offset, v->fec->rsn); | |
39 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); | |
40 | } | |
41 | ||
42 | /* | |
43 | * Decode an RS block using Reed-Solomon. | |
44 | */ | |
45 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
46 | u8 *data, u8 *fec, int neras) | |
47 | { | |
48 | int i; | |
49 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; | |
50 | ||
51 | for (i = 0; i < v->fec->roots; i++) | |
52 | par[i] = fec[i]; | |
53 | ||
54 | return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, | |
55 | fio->erasures, 0, NULL); | |
56 | } | |
57 | ||
58 | /* | |
59 | * Read error-correcting codes for the requested RS block. Returns a pointer | |
60 | * to the data block. Caller is responsible for releasing buf. | |
61 | */ | |
62 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, | |
d95e2c34 HJ |
63 | unsigned int *offset, struct dm_buffer **buf, |
64 | unsigned short ioprio) | |
a739ff3f | 65 | { |
df7b59ba | 66 | u64 position, block, rem; |
a739ff3f ST |
67 | u8 *res; |
68 | ||
69 | position = (index + rsb) * v->fec->roots; | |
8ca7cab8 | 70 | block = div64_u64_rem(position, v->fec->io_size, &rem); |
86a3238c | 71 | *offset = (unsigned int)rem; |
a739ff3f | 72 | |
d95e2c34 | 73 | res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); |
821b40da | 74 | if (IS_ERR(res)) { |
a739ff3f ST |
75 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", |
76 | v->data_dev->name, (unsigned long long)rsb, | |
df7b59ba | 77 | (unsigned long long)block, PTR_ERR(res)); |
a739ff3f ST |
78 | *buf = NULL; |
79 | } | |
80 | ||
81 | return res; | |
82 | } | |
83 | ||
84 | /* Loop over each preallocated buffer slot. */ | |
85 | #define fec_for_each_prealloc_buffer(__i) \ | |
86 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) | |
87 | ||
88 | /* Loop over each extra buffer slot. */ | |
89 | #define fec_for_each_extra_buffer(io, __i) \ | |
90 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) | |
91 | ||
92 | /* Loop over each allocated buffer. */ | |
93 | #define fec_for_each_buffer(io, __i) \ | |
94 | for (__i = 0; __i < (io)->nbufs; __i++) | |
95 | ||
96 | /* Loop over each RS block in each allocated buffer. */ | |
97 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ | |
98 | fec_for_each_buffer(io, __i) \ | |
99 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) | |
100 | ||
101 | /* | |
102 | * Return a pointer to the current RS block when called inside | |
103 | * fec_for_each_buffer_rs_block. | |
104 | */ | |
105 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, | |
106 | struct dm_verity_fec_io *fio, | |
86a3238c | 107 | unsigned int i, unsigned int j) |
a739ff3f ST |
108 | { |
109 | return &fio->bufs[i][j * v->fec->rsn]; | |
110 | } | |
111 | ||
112 | /* | |
113 | * Return an index to the current RS block when called inside | |
114 | * fec_for_each_buffer_rs_block. | |
115 | */ | |
86a3238c | 116 | static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j) |
a739ff3f ST |
117 | { |
118 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; | |
119 | } | |
120 | ||
121 | /* | |
122 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output | |
123 | * starting from block_offset. | |
124 | */ | |
d95e2c34 HJ |
125 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, |
126 | struct dm_verity_fec_io *fio, u64 rsb, int byte_index, | |
127 | unsigned int block_offset, int neras) | |
a739ff3f ST |
128 | { |
129 | int r, corrected = 0, res; | |
130 | struct dm_buffer *buf; | |
86a3238c | 131 | unsigned int n, i, offset; |
a739ff3f | 132 | u8 *par, *block; |
d95e2c34 | 133 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
a739ff3f | 134 | |
d95e2c34 | 135 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf, bio_prio(bio)); |
a739ff3f ST |
136 | if (IS_ERR(par)) |
137 | return PTR_ERR(par); | |
138 | ||
139 | /* | |
140 | * Decode the RS blocks we have in bufs. Each RS block results in | |
141 | * one corrected target byte and consumes fec->roots parity bytes. | |
142 | */ | |
143 | fec_for_each_buffer_rs_block(fio, n, i) { | |
144 | block = fec_buffer_rs_block(v, fio, n, i); | |
145 | res = fec_decode_rs8(v, fio, block, &par[offset], neras); | |
146 | if (res < 0) { | |
a739ff3f ST |
147 | r = res; |
148 | goto error; | |
149 | } | |
150 | ||
151 | corrected += res; | |
152 | fio->output[block_offset] = block[byte_index]; | |
153 | ||
154 | block_offset++; | |
155 | if (block_offset >= 1 << v->data_dev_block_bits) | |
156 | goto done; | |
157 | ||
158 | /* read the next block when we run out of parity bytes */ | |
159 | offset += v->fec->roots; | |
8ca7cab8 | 160 | if (offset >= v->fec->io_size) { |
a739ff3f ST |
161 | dm_bufio_release(buf); |
162 | ||
d95e2c34 | 163 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf, bio_prio(bio)); |
821b40da | 164 | if (IS_ERR(par)) |
a739ff3f ST |
165 | return PTR_ERR(par); |
166 | } | |
167 | } | |
168 | done: | |
169 | r = corrected; | |
170 | error: | |
86e3e83b ST |
171 | dm_bufio_release(buf); |
172 | ||
a739ff3f ST |
173 | if (r < 0 && neras) |
174 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", | |
175 | v->data_dev->name, (unsigned long long)rsb, r); | |
176 | else if (r > 0) | |
177 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", | |
178 | v->data_dev->name, (unsigned long long)rsb, r); | |
179 | ||
180 | return r; | |
181 | } | |
182 | ||
183 | /* | |
184 | * Locate data block erasures using verity hashes. | |
185 | */ | |
186 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, | |
187 | u8 *want_digest, u8 *data) | |
188 | { | |
e8f5e933 | 189 | if (unlikely(verity_hash(v, io, data, 1 << v->data_dev_block_bits, |
28f07f2a | 190 | verity_io_real_digest(v, io), true))) |
a739ff3f ST |
191 | return 0; |
192 | ||
193 | return memcmp(verity_io_real_digest(v, io), want_digest, | |
194 | v->digest_size) != 0; | |
195 | } | |
196 | ||
197 | /* | |
198 | * Read data blocks that are part of the RS block and deinterleave as much as | |
199 | * fits into buffers. Check for erasure locations if @neras is non-NULL. | |
200 | */ | |
201 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, | |
86a3238c | 202 | u64 rsb, u64 target, unsigned int block_offset, |
a739ff3f ST |
203 | int *neras) |
204 | { | |
0cc37c2d | 205 | bool is_zero; |
a739ff3f ST |
206 | int i, j, target_index = -1; |
207 | struct dm_buffer *buf; | |
208 | struct dm_bufio_client *bufio; | |
209 | struct dm_verity_fec_io *fio = fec_io(io); | |
210 | u64 block, ileaved; | |
211 | u8 *bbuf, *rs_block; | |
6d39a124 | 212 | u8 want_digest[HASH_MAX_DIGESTSIZE]; |
86a3238c | 213 | unsigned int n, k; |
d95e2c34 | 214 | struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); |
a739ff3f ST |
215 | |
216 | if (neras) | |
217 | *neras = 0; | |
218 | ||
6d39a124 KC |
219 | if (WARN_ON(v->digest_size > sizeof(want_digest))) |
220 | return -EINVAL; | |
221 | ||
a739ff3f ST |
222 | /* |
223 | * read each of the rsn data blocks that are part of the RS block, and | |
224 | * interleave contents to available bufs | |
225 | */ | |
226 | for (i = 0; i < v->fec->rsn; i++) { | |
227 | ileaved = fec_interleave(v, rsb * v->fec->rsn + i); | |
228 | ||
229 | /* | |
230 | * target is the data block we want to correct, target_index is | |
231 | * the index of this block within the rsn RS blocks | |
232 | */ | |
233 | if (ileaved == target) | |
234 | target_index = i; | |
235 | ||
236 | block = ileaved >> v->data_dev_block_bits; | |
237 | bufio = v->fec->data_bufio; | |
238 | ||
239 | if (block >= v->data_blocks) { | |
240 | block -= v->data_blocks; | |
241 | ||
242 | /* | |
243 | * blocks outside the area were assumed to contain | |
244 | * zeros when encoding data was generated | |
245 | */ | |
246 | if (unlikely(block >= v->fec->hash_blocks)) | |
247 | continue; | |
248 | ||
249 | block += v->hash_start; | |
250 | bufio = v->bufio; | |
251 | } | |
252 | ||
d95e2c34 | 253 | bbuf = dm_bufio_read_with_ioprio(bufio, block, &buf, bio_prio(bio)); |
821b40da | 254 | if (IS_ERR(bbuf)) { |
a739ff3f ST |
255 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", |
256 | v->data_dev->name, | |
257 | (unsigned long long)rsb, | |
258 | (unsigned long long)block, PTR_ERR(bbuf)); | |
259 | ||
260 | /* assume the block is corrupted */ | |
261 | if (neras && *neras <= v->fec->roots) | |
262 | fio->erasures[(*neras)++] = i; | |
263 | ||
264 | continue; | |
265 | } | |
266 | ||
267 | /* locate erasures if the block is on the data device */ | |
268 | if (bufio == v->fec->data_bufio && | |
0cc37c2d ST |
269 | verity_hash_for_block(v, io, block, want_digest, |
270 | &is_zero) == 0) { | |
271 | /* skip known zero blocks entirely */ | |
272 | if (is_zero) | |
86e3e83b | 273 | goto done; |
0cc37c2d | 274 | |
a739ff3f ST |
275 | /* |
276 | * skip if we have already found the theoretical | |
277 | * maximum number (i.e. fec->roots) of erasures | |
278 | */ | |
279 | if (neras && *neras <= v->fec->roots && | |
280 | fec_is_erasure(v, io, want_digest, bbuf)) | |
281 | fio->erasures[(*neras)++] = i; | |
282 | } | |
283 | ||
284 | /* | |
285 | * deinterleave and copy the bytes that fit into bufs, | |
286 | * starting from block_offset | |
287 | */ | |
288 | fec_for_each_buffer_rs_block(fio, n, j) { | |
289 | k = fec_buffer_rs_index(n, j) + block_offset; | |
290 | ||
291 | if (k >= 1 << v->data_dev_block_bits) | |
292 | goto done; | |
293 | ||
294 | rs_block = fec_buffer_rs_block(v, fio, n, j); | |
295 | rs_block[i] = bbuf[k]; | |
296 | } | |
297 | done: | |
298 | dm_bufio_release(buf); | |
299 | } | |
300 | ||
301 | return target_index; | |
302 | } | |
303 | ||
304 | /* | |
305 | * Allocate RS control structure and FEC buffers from preallocated mempools, | |
306 | * and attempt to allocate as many extra buffers as available. | |
307 | */ | |
308 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
309 | { | |
86a3238c | 310 | unsigned int n; |
a739ff3f | 311 | |
34c96507 | 312 | if (!fio->rs) |
6f1c819c | 313 | fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); |
a739ff3f ST |
314 | |
315 | fec_for_each_prealloc_buffer(n) { | |
316 | if (fio->bufs[n]) | |
317 | continue; | |
318 | ||
6f1c819c | 319 | fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); |
a739ff3f ST |
320 | if (unlikely(!fio->bufs[n])) { |
321 | DMERR("failed to allocate FEC buffer"); | |
322 | return -ENOMEM; | |
323 | } | |
324 | } | |
325 | ||
326 | /* try to allocate the maximum number of buffers */ | |
327 | fec_for_each_extra_buffer(fio, n) { | |
328 | if (fio->bufs[n]) | |
329 | continue; | |
330 | ||
6f1c819c | 331 | fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); |
a739ff3f ST |
332 | /* we can manage with even one buffer if necessary */ |
333 | if (unlikely(!fio->bufs[n])) | |
334 | break; | |
335 | } | |
336 | fio->nbufs = n; | |
337 | ||
34c96507 | 338 | if (!fio->output) |
6f1c819c | 339 | fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); |
a739ff3f | 340 | |
a739ff3f ST |
341 | return 0; |
342 | } | |
343 | ||
344 | /* | |
345 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are | |
346 | * zeroed before deinterleaving. | |
347 | */ | |
348 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
349 | { | |
86a3238c | 350 | unsigned int n; |
a739ff3f ST |
351 | |
352 | fec_for_each_buffer(fio, n) | |
353 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); | |
354 | ||
355 | memset(fio->erasures, 0, sizeof(fio->erasures)); | |
356 | } | |
357 | ||
358 | /* | |
359 | * Decode all RS blocks in a single data block and return the target block | |
360 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses | |
361 | * hashes to locate erasures. | |
362 | */ | |
363 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, | |
364 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, | |
365 | bool use_erasures) | |
366 | { | |
367 | int r, neras = 0; | |
86a3238c | 368 | unsigned int pos; |
a739ff3f ST |
369 | |
370 | r = fec_alloc_bufs(v, fio); | |
371 | if (unlikely(r < 0)) | |
372 | return r; | |
373 | ||
374 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { | |
375 | fec_init_bufs(v, fio); | |
376 | ||
377 | r = fec_read_bufs(v, io, rsb, offset, pos, | |
378 | use_erasures ? &neras : NULL); | |
379 | if (unlikely(r < 0)) | |
380 | return r; | |
381 | ||
d95e2c34 | 382 | r = fec_decode_bufs(v, io, fio, rsb, r, pos, neras); |
a739ff3f ST |
383 | if (r < 0) |
384 | return r; | |
385 | ||
386 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; | |
387 | } | |
388 | ||
389 | /* Always re-validate the corrected block against the expected hash */ | |
e8f5e933 | 390 | r = verity_hash(v, io, fio->output, 1 << v->data_dev_block_bits, |
28f07f2a | 391 | verity_io_real_digest(v, io), true); |
a739ff3f ST |
392 | if (unlikely(r < 0)) |
393 | return r; | |
394 | ||
395 | if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), | |
396 | v->digest_size)) { | |
397 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", | |
398 | v->data_dev->name, (unsigned long long)rsb, neras); | |
399 | return -EILSEQ; | |
400 | } | |
401 | ||
402 | return 0; | |
403 | } | |
404 | ||
cf715f4b | 405 | /* Correct errors in a block. Copies corrected block to dest. */ |
a739ff3f | 406 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, |
cf715f4b | 407 | enum verity_block_type type, sector_t block, u8 *dest) |
a739ff3f ST |
408 | { |
409 | int r; | |
410 | struct dm_verity_fec_io *fio = fec_io(io); | |
411 | u64 offset, res, rsb; | |
412 | ||
413 | if (!verity_fec_is_enabled(v)) | |
414 | return -EOPNOTSUPP; | |
415 | ||
f1a880a9 ST |
416 | if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { |
417 | DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); | |
418 | return -EIO; | |
419 | } | |
420 | ||
421 | fio->level++; | |
422 | ||
a739ff3f | 423 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) |
ad4e80a6 | 424 | block = block - v->hash_start + v->data_blocks; |
a739ff3f ST |
425 | |
426 | /* | |
427 | * For RS(M, N), the continuous FEC data is divided into blocks of N | |
428 | * bytes. Since block size may not be divisible by N, the last block | |
429 | * is zero padded when decoding. | |
430 | * | |
431 | * Each byte of the block is covered by a different RS(M, N) code, | |
432 | * and each code is interleaved over N blocks to make it less likely | |
433 | * that bursty corruption will leave us in unrecoverable state. | |
434 | */ | |
435 | ||
436 | offset = block << v->data_dev_block_bits; | |
602d1657 | 437 | res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); |
a739ff3f ST |
438 | |
439 | /* | |
440 | * The base RS block we can feed to the interleaver to find out all | |
441 | * blocks required for decoding. | |
442 | */ | |
443 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); | |
444 | ||
445 | /* | |
446 | * Locating erasures is slow, so attempt to recover the block without | |
447 | * them first. Do a second attempt with erasures if the corruption is | |
448 | * bad enough. | |
449 | */ | |
450 | r = fec_decode_rsb(v, io, fio, rsb, offset, false); | |
451 | if (r < 0) { | |
452 | r = fec_decode_rsb(v, io, fio, rsb, offset, true); | |
453 | if (r < 0) | |
f1a880a9 | 454 | goto done; |
a739ff3f ST |
455 | } |
456 | ||
cf715f4b | 457 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); |
a739ff3f | 458 | |
f1a880a9 ST |
459 | done: |
460 | fio->level--; | |
a739ff3f ST |
461 | return r; |
462 | } | |
463 | ||
464 | /* | |
465 | * Clean up per-bio data. | |
466 | */ | |
467 | void verity_fec_finish_io(struct dm_verity_io *io) | |
468 | { | |
86a3238c | 469 | unsigned int n; |
a739ff3f ST |
470 | struct dm_verity_fec *f = io->v->fec; |
471 | struct dm_verity_fec_io *fio = fec_io(io); | |
472 | ||
473 | if (!verity_fec_is_enabled(io->v)) | |
474 | return; | |
475 | ||
6f1c819c | 476 | mempool_free(fio->rs, &f->rs_pool); |
a739ff3f ST |
477 | |
478 | fec_for_each_prealloc_buffer(n) | |
6f1c819c | 479 | mempool_free(fio->bufs[n], &f->prealloc_pool); |
a739ff3f ST |
480 | |
481 | fec_for_each_extra_buffer(fio, n) | |
6f1c819c | 482 | mempool_free(fio->bufs[n], &f->extra_pool); |
a739ff3f | 483 | |
6f1c819c | 484 | mempool_free(fio->output, &f->output_pool); |
a739ff3f ST |
485 | } |
486 | ||
487 | /* | |
488 | * Initialize per-bio data. | |
489 | */ | |
490 | void verity_fec_init_io(struct dm_verity_io *io) | |
491 | { | |
492 | struct dm_verity_fec_io *fio = fec_io(io); | |
493 | ||
494 | if (!verity_fec_is_enabled(io->v)) | |
495 | return; | |
496 | ||
497 | fio->rs = NULL; | |
498 | memset(fio->bufs, 0, sizeof(fio->bufs)); | |
499 | fio->nbufs = 0; | |
500 | fio->output = NULL; | |
f1a880a9 | 501 | fio->level = 0; |
a739ff3f ST |
502 | } |
503 | ||
504 | /* | |
505 | * Append feature arguments and values to the status table. | |
506 | */ | |
86a3238c HM |
507 | unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz, |
508 | char *result, unsigned int maxlen) | |
a739ff3f ST |
509 | { |
510 | if (!verity_fec_is_enabled(v)) | |
511 | return sz; | |
512 | ||
513 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " | |
514 | DM_VERITY_OPT_FEC_BLOCKS " %llu " | |
515 | DM_VERITY_OPT_FEC_START " %llu " | |
516 | DM_VERITY_OPT_FEC_ROOTS " %d", | |
517 | v->fec->dev->name, | |
518 | (unsigned long long)v->fec->blocks, | |
519 | (unsigned long long)v->fec->start, | |
520 | v->fec->roots); | |
521 | ||
522 | return sz; | |
523 | } | |
524 | ||
525 | void verity_fec_dtr(struct dm_verity *v) | |
526 | { | |
527 | struct dm_verity_fec *f = v->fec; | |
528 | ||
529 | if (!verity_fec_is_enabled(v)) | |
530 | goto out; | |
531 | ||
6f1c819c KO |
532 | mempool_exit(&f->rs_pool); |
533 | mempool_exit(&f->prealloc_pool); | |
534 | mempool_exit(&f->extra_pool); | |
75fa6019 | 535 | mempool_exit(&f->output_pool); |
a739ff3f ST |
536 | kmem_cache_destroy(f->cache); |
537 | ||
538 | if (f->data_bufio) | |
539 | dm_bufio_client_destroy(f->data_bufio); | |
540 | if (f->bufio) | |
541 | dm_bufio_client_destroy(f->bufio); | |
542 | ||
543 | if (f->dev) | |
544 | dm_put_device(v->ti, f->dev); | |
545 | out: | |
546 | kfree(f); | |
547 | v->fec = NULL; | |
548 | } | |
549 | ||
550 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) | |
551 | { | |
26cb62a2 | 552 | struct dm_verity *v = pool_data; |
a739ff3f | 553 | |
eb366989 | 554 | return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); |
a739ff3f ST |
555 | } |
556 | ||
557 | static void fec_rs_free(void *element, void *pool_data) | |
558 | { | |
26cb62a2 | 559 | struct rs_control *rs = element; |
a739ff3f ST |
560 | |
561 | if (rs) | |
562 | free_rs(rs); | |
563 | } | |
564 | ||
565 | bool verity_is_fec_opt_arg(const char *arg_name) | |
566 | { | |
567 | return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || | |
568 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || | |
569 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || | |
570 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); | |
571 | } | |
572 | ||
573 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, | |
86a3238c | 574 | unsigned int *argc, const char *arg_name) |
a739ff3f ST |
575 | { |
576 | int r; | |
577 | struct dm_target *ti = v->ti; | |
578 | const char *arg_value; | |
579 | unsigned long long num_ll; | |
580 | unsigned char num_c; | |
581 | char dummy; | |
582 | ||
583 | if (!*argc) { | |
584 | ti->error = "FEC feature arguments require a value"; | |
585 | return -EINVAL; | |
586 | } | |
587 | ||
588 | arg_value = dm_shift_arg(as); | |
589 | (*argc)--; | |
590 | ||
591 | if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { | |
05bdb996 | 592 | r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev); |
a739ff3f ST |
593 | if (r) { |
594 | ti->error = "FEC device lookup failed"; | |
595 | return r; | |
596 | } | |
597 | ||
598 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { | |
599 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
600 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) | |
601 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
602 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
603 | return -EINVAL; | |
604 | } | |
605 | v->fec->blocks = num_ll; | |
606 | ||
607 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { | |
608 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
609 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> | |
610 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
611 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; | |
612 | return -EINVAL; | |
613 | } | |
614 | v->fec->start = num_ll; | |
615 | ||
616 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { | |
617 | if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || | |
618 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || | |
619 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { | |
620 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; | |
621 | return -EINVAL; | |
622 | } | |
623 | v->fec->roots = num_c; | |
624 | ||
625 | } else { | |
626 | ti->error = "Unrecognized verity FEC feature request"; | |
627 | return -EINVAL; | |
628 | } | |
629 | ||
630 | return 0; | |
631 | } | |
632 | ||
633 | /* | |
634 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. | |
635 | */ | |
636 | int verity_fec_ctr_alloc(struct dm_verity *v) | |
637 | { | |
638 | struct dm_verity_fec *f; | |
639 | ||
640 | f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); | |
641 | if (!f) { | |
642 | v->ti->error = "Cannot allocate FEC structure"; | |
643 | return -ENOMEM; | |
644 | } | |
645 | v->fec = f; | |
646 | ||
647 | return 0; | |
648 | } | |
649 | ||
650 | /* | |
651 | * Validate arguments and preallocate memory. Must be called after arguments | |
652 | * have been parsed using verity_fec_parse_opt_args. | |
653 | */ | |
654 | int verity_fec_ctr(struct dm_verity *v) | |
655 | { | |
656 | struct dm_verity_fec *f = v->fec; | |
657 | struct dm_target *ti = v->ti; | |
df7b59ba | 658 | u64 hash_blocks, fec_blocks; |
6f1c819c | 659 | int ret; |
a739ff3f ST |
660 | |
661 | if (!verity_fec_is_enabled(v)) { | |
662 | verity_fec_dtr(v); | |
663 | return 0; | |
664 | } | |
665 | ||
666 | /* | |
667 | * FEC is computed over data blocks, possible metadata, and | |
668 | * hash blocks. In other words, FEC covers total of fec_blocks | |
669 | * blocks consisting of the following: | |
670 | * | |
671 | * data blocks | hash blocks | metadata (optional) | |
672 | * | |
673 | * We allow metadata after hash blocks to support a use case | |
674 | * where all data is stored on the same device and FEC covers | |
675 | * the entire area. | |
676 | * | |
677 | * If metadata is included, we require it to be available on the | |
678 | * hash device after the hash blocks. | |
679 | */ | |
680 | ||
681 | hash_blocks = v->hash_blocks - v->hash_start; | |
682 | ||
683 | /* | |
684 | * Require matching block sizes for data and hash devices for | |
685 | * simplicity. | |
686 | */ | |
687 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { | |
688 | ti->error = "Block sizes must match to use FEC"; | |
689 | return -EINVAL; | |
690 | } | |
691 | ||
692 | if (!f->roots) { | |
693 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; | |
694 | return -EINVAL; | |
695 | } | |
696 | f->rsn = DM_VERITY_FEC_RSM - f->roots; | |
697 | ||
698 | if (!f->blocks) { | |
699 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; | |
700 | return -EINVAL; | |
701 | } | |
702 | ||
703 | f->rounds = f->blocks; | |
704 | if (sector_div(f->rounds, f->rsn)) | |
705 | f->rounds++; | |
706 | ||
707 | /* | |
708 | * Due to optional metadata, f->blocks can be larger than | |
709 | * data_blocks and hash_blocks combined. | |
710 | */ | |
711 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { | |
712 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
713 | return -EINVAL; | |
714 | } | |
715 | ||
716 | /* | |
717 | * Metadata is accessed through the hash device, so we require | |
718 | * it to be large enough. | |
719 | */ | |
720 | f->hash_blocks = f->blocks - v->data_blocks; | |
721 | if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { | |
722 | ti->error = "Hash device is too small for " | |
723 | DM_VERITY_OPT_FEC_BLOCKS; | |
724 | return -E2BIG; | |
725 | } | |
726 | ||
8ca7cab8 JK |
727 | if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1)) |
728 | f->io_size = 1 << v->data_dev_block_bits; | |
729 | else | |
730 | f->io_size = v->fec->roots << SECTOR_SHIFT; | |
731 | ||
a739ff3f | 732 | f->bufio = dm_bufio_client_create(f->dev->bdev, |
8ca7cab8 | 733 | f->io_size, |
0fcb100d | 734 | 1, 0, NULL, NULL, 0); |
a739ff3f ST |
735 | if (IS_ERR(f->bufio)) { |
736 | ti->error = "Cannot initialize FEC bufio client"; | |
737 | return PTR_ERR(f->bufio); | |
738 | } | |
739 | ||
df7b59ba MB |
740 | dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); |
741 | ||
742 | fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); | |
743 | if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { | |
a739ff3f ST |
744 | ti->error = "FEC device is too small"; |
745 | return -E2BIG; | |
746 | } | |
747 | ||
748 | f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, | |
749 | 1 << v->data_dev_block_bits, | |
0fcb100d | 750 | 1, 0, NULL, NULL, 0); |
a739ff3f ST |
751 | if (IS_ERR(f->data_bufio)) { |
752 | ti->error = "Cannot initialize FEC data bufio client"; | |
753 | return PTR_ERR(f->data_bufio); | |
754 | } | |
755 | ||
756 | if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { | |
757 | ti->error = "Data device is too small"; | |
758 | return -E2BIG; | |
759 | } | |
760 | ||
761 | /* Preallocate an rs_control structure for each worker thread */ | |
6f1c819c KO |
762 | ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, |
763 | fec_rs_free, (void *) v); | |
764 | if (ret) { | |
a739ff3f | 765 | ti->error = "Cannot allocate RS pool"; |
6f1c819c | 766 | return ret; |
a739ff3f ST |
767 | } |
768 | ||
769 | f->cache = kmem_cache_create("dm_verity_fec_buffers", | |
770 | f->rsn << DM_VERITY_FEC_BUF_RS_BITS, | |
771 | 0, 0, NULL); | |
772 | if (!f->cache) { | |
773 | ti->error = "Cannot create FEC buffer cache"; | |
774 | return -ENOMEM; | |
775 | } | |
776 | ||
777 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ | |
6f1c819c KO |
778 | ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * |
779 | DM_VERITY_FEC_BUF_PREALLOC, | |
780 | f->cache); | |
781 | if (ret) { | |
a739ff3f | 782 | ti->error = "Cannot allocate FEC buffer prealloc pool"; |
6f1c819c | 783 | return ret; |
a739ff3f ST |
784 | } |
785 | ||
6f1c819c KO |
786 | ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache); |
787 | if (ret) { | |
a739ff3f | 788 | ti->error = "Cannot allocate FEC buffer extra pool"; |
6f1c819c | 789 | return ret; |
a739ff3f ST |
790 | } |
791 | ||
792 | /* Preallocate an output buffer for each thread */ | |
6f1c819c KO |
793 | ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(), |
794 | 1 << v->data_dev_block_bits); | |
795 | if (ret) { | |
a739ff3f | 796 | ti->error = "Cannot allocate FEC output pool"; |
6f1c819c | 797 | return ret; |
a739ff3f ST |
798 | } |
799 | ||
800 | /* Reserve space for our per-bio data */ | |
30187e1d | 801 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
a739ff3f ST |
802 | |
803 | return 0; | |
804 | } |