Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
a739ff3f ST |
2 | /* |
3 | * Copyright (C) 2015 Google, Inc. | |
4 | * | |
5 | * Author: Sami Tolvanen <samitolvanen@google.com> | |
a739ff3f ST |
6 | */ |
7 | ||
8 | #include "dm-verity-fec.h" | |
9 | #include <linux/math64.h> | |
10 | ||
11 | #define DM_MSG_PREFIX "verity-fec" | |
12 | ||
13 | /* | |
14 | * If error correction has been configured, returns true. | |
15 | */ | |
16 | bool verity_fec_is_enabled(struct dm_verity *v) | |
17 | { | |
18 | return v->fec && v->fec->dev; | |
19 | } | |
20 | ||
21 | /* | |
22 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable | |
23 | * length fields. | |
24 | */ | |
25 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) | |
26 | { | |
27 | return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); | |
28 | } | |
29 | ||
30 | /* | |
31 | * Return an interleaved offset for a byte in RS block. | |
32 | */ | |
33 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) | |
34 | { | |
35 | u32 mod; | |
36 | ||
37 | mod = do_div(offset, v->fec->rsn); | |
38 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); | |
39 | } | |
40 | ||
41 | /* | |
42 | * Decode an RS block using Reed-Solomon. | |
43 | */ | |
44 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
45 | u8 *data, u8 *fec, int neras) | |
46 | { | |
47 | int i; | |
48 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; | |
49 | ||
50 | for (i = 0; i < v->fec->roots; i++) | |
51 | par[i] = fec[i]; | |
52 | ||
53 | return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, | |
54 | fio->erasures, 0, NULL); | |
55 | } | |
56 | ||
57 | /* | |
58 | * Read error-correcting codes for the requested RS block. Returns a pointer | |
59 | * to the data block. Caller is responsible for releasing buf. | |
60 | */ | |
61 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, | |
62 | unsigned *offset, struct dm_buffer **buf) | |
63 | { | |
df7b59ba | 64 | u64 position, block, rem; |
a739ff3f ST |
65 | u8 *res; |
66 | ||
67 | position = (index + rsb) * v->fec->roots; | |
8ca7cab8 | 68 | block = div64_u64_rem(position, v->fec->io_size, &rem); |
df7b59ba | 69 | *offset = (unsigned)rem; |
a739ff3f | 70 | |
df7b59ba | 71 | res = dm_bufio_read(v->fec->bufio, block, buf); |
821b40da | 72 | if (IS_ERR(res)) { |
a739ff3f ST |
73 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", |
74 | v->data_dev->name, (unsigned long long)rsb, | |
df7b59ba | 75 | (unsigned long long)block, PTR_ERR(res)); |
a739ff3f ST |
76 | *buf = NULL; |
77 | } | |
78 | ||
79 | return res; | |
80 | } | |
81 | ||
82 | /* Loop over each preallocated buffer slot. */ | |
83 | #define fec_for_each_prealloc_buffer(__i) \ | |
84 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) | |
85 | ||
86 | /* Loop over each extra buffer slot. */ | |
87 | #define fec_for_each_extra_buffer(io, __i) \ | |
88 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) | |
89 | ||
90 | /* Loop over each allocated buffer. */ | |
91 | #define fec_for_each_buffer(io, __i) \ | |
92 | for (__i = 0; __i < (io)->nbufs; __i++) | |
93 | ||
94 | /* Loop over each RS block in each allocated buffer. */ | |
95 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ | |
96 | fec_for_each_buffer(io, __i) \ | |
97 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) | |
98 | ||
99 | /* | |
100 | * Return a pointer to the current RS block when called inside | |
101 | * fec_for_each_buffer_rs_block. | |
102 | */ | |
103 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, | |
104 | struct dm_verity_fec_io *fio, | |
105 | unsigned i, unsigned j) | |
106 | { | |
107 | return &fio->bufs[i][j * v->fec->rsn]; | |
108 | } | |
109 | ||
110 | /* | |
111 | * Return an index to the current RS block when called inside | |
112 | * fec_for_each_buffer_rs_block. | |
113 | */ | |
114 | static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) | |
115 | { | |
116 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; | |
117 | } | |
118 | ||
119 | /* | |
120 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output | |
121 | * starting from block_offset. | |
122 | */ | |
123 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
124 | u64 rsb, int byte_index, unsigned block_offset, | |
125 | int neras) | |
126 | { | |
127 | int r, corrected = 0, res; | |
128 | struct dm_buffer *buf; | |
129 | unsigned n, i, offset; | |
130 | u8 *par, *block; | |
131 | ||
132 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
133 | if (IS_ERR(par)) | |
134 | return PTR_ERR(par); | |
135 | ||
136 | /* | |
137 | * Decode the RS blocks we have in bufs. Each RS block results in | |
138 | * one corrected target byte and consumes fec->roots parity bytes. | |
139 | */ | |
140 | fec_for_each_buffer_rs_block(fio, n, i) { | |
141 | block = fec_buffer_rs_block(v, fio, n, i); | |
142 | res = fec_decode_rs8(v, fio, block, &par[offset], neras); | |
143 | if (res < 0) { | |
a739ff3f ST |
144 | r = res; |
145 | goto error; | |
146 | } | |
147 | ||
148 | corrected += res; | |
149 | fio->output[block_offset] = block[byte_index]; | |
150 | ||
151 | block_offset++; | |
152 | if (block_offset >= 1 << v->data_dev_block_bits) | |
153 | goto done; | |
154 | ||
155 | /* read the next block when we run out of parity bytes */ | |
156 | offset += v->fec->roots; | |
8ca7cab8 | 157 | if (offset >= v->fec->io_size) { |
a739ff3f ST |
158 | dm_bufio_release(buf); |
159 | ||
160 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
821b40da | 161 | if (IS_ERR(par)) |
a739ff3f ST |
162 | return PTR_ERR(par); |
163 | } | |
164 | } | |
165 | done: | |
166 | r = corrected; | |
167 | error: | |
86e3e83b ST |
168 | dm_bufio_release(buf); |
169 | ||
a739ff3f ST |
170 | if (r < 0 && neras) |
171 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", | |
172 | v->data_dev->name, (unsigned long long)rsb, r); | |
173 | else if (r > 0) | |
174 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", | |
175 | v->data_dev->name, (unsigned long long)rsb, r); | |
176 | ||
177 | return r; | |
178 | } | |
179 | ||
180 | /* | |
181 | * Locate data block erasures using verity hashes. | |
182 | */ | |
183 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, | |
184 | u8 *want_digest, u8 *data) | |
185 | { | |
d1ac3ff0 | 186 | if (unlikely(verity_hash(v, verity_io_hash_req(v, io), |
a739ff3f ST |
187 | data, 1 << v->data_dev_block_bits, |
188 | verity_io_real_digest(v, io)))) | |
189 | return 0; | |
190 | ||
191 | return memcmp(verity_io_real_digest(v, io), want_digest, | |
192 | v->digest_size) != 0; | |
193 | } | |
194 | ||
195 | /* | |
196 | * Read data blocks that are part of the RS block and deinterleave as much as | |
197 | * fits into buffers. Check for erasure locations if @neras is non-NULL. | |
198 | */ | |
199 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, | |
200 | u64 rsb, u64 target, unsigned block_offset, | |
201 | int *neras) | |
202 | { | |
0cc37c2d | 203 | bool is_zero; |
a739ff3f ST |
204 | int i, j, target_index = -1; |
205 | struct dm_buffer *buf; | |
206 | struct dm_bufio_client *bufio; | |
207 | struct dm_verity_fec_io *fio = fec_io(io); | |
208 | u64 block, ileaved; | |
209 | u8 *bbuf, *rs_block; | |
6d39a124 | 210 | u8 want_digest[HASH_MAX_DIGESTSIZE]; |
a739ff3f ST |
211 | unsigned n, k; |
212 | ||
213 | if (neras) | |
214 | *neras = 0; | |
215 | ||
6d39a124 KC |
216 | if (WARN_ON(v->digest_size > sizeof(want_digest))) |
217 | return -EINVAL; | |
218 | ||
a739ff3f ST |
219 | /* |
220 | * read each of the rsn data blocks that are part of the RS block, and | |
221 | * interleave contents to available bufs | |
222 | */ | |
223 | for (i = 0; i < v->fec->rsn; i++) { | |
224 | ileaved = fec_interleave(v, rsb * v->fec->rsn + i); | |
225 | ||
226 | /* | |
227 | * target is the data block we want to correct, target_index is | |
228 | * the index of this block within the rsn RS blocks | |
229 | */ | |
230 | if (ileaved == target) | |
231 | target_index = i; | |
232 | ||
233 | block = ileaved >> v->data_dev_block_bits; | |
234 | bufio = v->fec->data_bufio; | |
235 | ||
236 | if (block >= v->data_blocks) { | |
237 | block -= v->data_blocks; | |
238 | ||
239 | /* | |
240 | * blocks outside the area were assumed to contain | |
241 | * zeros when encoding data was generated | |
242 | */ | |
243 | if (unlikely(block >= v->fec->hash_blocks)) | |
244 | continue; | |
245 | ||
246 | block += v->hash_start; | |
247 | bufio = v->bufio; | |
248 | } | |
249 | ||
250 | bbuf = dm_bufio_read(bufio, block, &buf); | |
821b40da | 251 | if (IS_ERR(bbuf)) { |
a739ff3f ST |
252 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", |
253 | v->data_dev->name, | |
254 | (unsigned long long)rsb, | |
255 | (unsigned long long)block, PTR_ERR(bbuf)); | |
256 | ||
257 | /* assume the block is corrupted */ | |
258 | if (neras && *neras <= v->fec->roots) | |
259 | fio->erasures[(*neras)++] = i; | |
260 | ||
261 | continue; | |
262 | } | |
263 | ||
264 | /* locate erasures if the block is on the data device */ | |
265 | if (bufio == v->fec->data_bufio && | |
0cc37c2d ST |
266 | verity_hash_for_block(v, io, block, want_digest, |
267 | &is_zero) == 0) { | |
268 | /* skip known zero blocks entirely */ | |
269 | if (is_zero) | |
86e3e83b | 270 | goto done; |
0cc37c2d | 271 | |
a739ff3f ST |
272 | /* |
273 | * skip if we have already found the theoretical | |
274 | * maximum number (i.e. fec->roots) of erasures | |
275 | */ | |
276 | if (neras && *neras <= v->fec->roots && | |
277 | fec_is_erasure(v, io, want_digest, bbuf)) | |
278 | fio->erasures[(*neras)++] = i; | |
279 | } | |
280 | ||
281 | /* | |
282 | * deinterleave and copy the bytes that fit into bufs, | |
283 | * starting from block_offset | |
284 | */ | |
285 | fec_for_each_buffer_rs_block(fio, n, j) { | |
286 | k = fec_buffer_rs_index(n, j) + block_offset; | |
287 | ||
288 | if (k >= 1 << v->data_dev_block_bits) | |
289 | goto done; | |
290 | ||
291 | rs_block = fec_buffer_rs_block(v, fio, n, j); | |
292 | rs_block[i] = bbuf[k]; | |
293 | } | |
294 | done: | |
295 | dm_bufio_release(buf); | |
296 | } | |
297 | ||
298 | return target_index; | |
299 | } | |
300 | ||
301 | /* | |
302 | * Allocate RS control structure and FEC buffers from preallocated mempools, | |
303 | * and attempt to allocate as many extra buffers as available. | |
304 | */ | |
305 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
306 | { | |
307 | unsigned n; | |
308 | ||
34c96507 | 309 | if (!fio->rs) |
6f1c819c | 310 | fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); |
a739ff3f ST |
311 | |
312 | fec_for_each_prealloc_buffer(n) { | |
313 | if (fio->bufs[n]) | |
314 | continue; | |
315 | ||
6f1c819c | 316 | fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); |
a739ff3f ST |
317 | if (unlikely(!fio->bufs[n])) { |
318 | DMERR("failed to allocate FEC buffer"); | |
319 | return -ENOMEM; | |
320 | } | |
321 | } | |
322 | ||
323 | /* try to allocate the maximum number of buffers */ | |
324 | fec_for_each_extra_buffer(fio, n) { | |
325 | if (fio->bufs[n]) | |
326 | continue; | |
327 | ||
6f1c819c | 328 | fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); |
a739ff3f ST |
329 | /* we can manage with even one buffer if necessary */ |
330 | if (unlikely(!fio->bufs[n])) | |
331 | break; | |
332 | } | |
333 | fio->nbufs = n; | |
334 | ||
34c96507 | 335 | if (!fio->output) |
6f1c819c | 336 | fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); |
a739ff3f | 337 | |
a739ff3f ST |
338 | return 0; |
339 | } | |
340 | ||
341 | /* | |
342 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are | |
343 | * zeroed before deinterleaving. | |
344 | */ | |
345 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
346 | { | |
347 | unsigned n; | |
348 | ||
349 | fec_for_each_buffer(fio, n) | |
350 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); | |
351 | ||
352 | memset(fio->erasures, 0, sizeof(fio->erasures)); | |
353 | } | |
354 | ||
355 | /* | |
356 | * Decode all RS blocks in a single data block and return the target block | |
357 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses | |
358 | * hashes to locate erasures. | |
359 | */ | |
360 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, | |
361 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, | |
362 | bool use_erasures) | |
363 | { | |
364 | int r, neras = 0; | |
365 | unsigned pos; | |
366 | ||
367 | r = fec_alloc_bufs(v, fio); | |
368 | if (unlikely(r < 0)) | |
369 | return r; | |
370 | ||
371 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { | |
372 | fec_init_bufs(v, fio); | |
373 | ||
374 | r = fec_read_bufs(v, io, rsb, offset, pos, | |
375 | use_erasures ? &neras : NULL); | |
376 | if (unlikely(r < 0)) | |
377 | return r; | |
378 | ||
379 | r = fec_decode_bufs(v, fio, rsb, r, pos, neras); | |
380 | if (r < 0) | |
381 | return r; | |
382 | ||
383 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; | |
384 | } | |
385 | ||
386 | /* Always re-validate the corrected block against the expected hash */ | |
d1ac3ff0 | 387 | r = verity_hash(v, verity_io_hash_req(v, io), fio->output, |
a739ff3f ST |
388 | 1 << v->data_dev_block_bits, |
389 | verity_io_real_digest(v, io)); | |
390 | if (unlikely(r < 0)) | |
391 | return r; | |
392 | ||
393 | if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), | |
394 | v->digest_size)) { | |
395 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", | |
396 | v->data_dev->name, (unsigned long long)rsb, neras); | |
397 | return -EILSEQ; | |
398 | } | |
399 | ||
400 | return 0; | |
401 | } | |
402 | ||
403 | static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, | |
404 | size_t len) | |
405 | { | |
406 | struct dm_verity_fec_io *fio = fec_io(io); | |
407 | ||
408 | memcpy(data, &fio->output[fio->output_pos], len); | |
409 | fio->output_pos += len; | |
410 | ||
411 | return 0; | |
412 | } | |
413 | ||
414 | /* | |
415 | * Correct errors in a block. Copies corrected block to dest if non-NULL, | |
416 | * otherwise to a bio_vec starting from iter. | |
417 | */ | |
418 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, | |
419 | enum verity_block_type type, sector_t block, u8 *dest, | |
420 | struct bvec_iter *iter) | |
421 | { | |
422 | int r; | |
423 | struct dm_verity_fec_io *fio = fec_io(io); | |
424 | u64 offset, res, rsb; | |
425 | ||
426 | if (!verity_fec_is_enabled(v)) | |
427 | return -EOPNOTSUPP; | |
428 | ||
f1a880a9 ST |
429 | if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { |
430 | DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); | |
431 | return -EIO; | |
432 | } | |
433 | ||
434 | fio->level++; | |
435 | ||
a739ff3f | 436 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) |
ad4e80a6 | 437 | block = block - v->hash_start + v->data_blocks; |
a739ff3f ST |
438 | |
439 | /* | |
440 | * For RS(M, N), the continuous FEC data is divided into blocks of N | |
441 | * bytes. Since block size may not be divisible by N, the last block | |
442 | * is zero padded when decoding. | |
443 | * | |
444 | * Each byte of the block is covered by a different RS(M, N) code, | |
445 | * and each code is interleaved over N blocks to make it less likely | |
446 | * that bursty corruption will leave us in unrecoverable state. | |
447 | */ | |
448 | ||
449 | offset = block << v->data_dev_block_bits; | |
602d1657 | 450 | res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); |
a739ff3f ST |
451 | |
452 | /* | |
453 | * The base RS block we can feed to the interleaver to find out all | |
454 | * blocks required for decoding. | |
455 | */ | |
456 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); | |
457 | ||
458 | /* | |
459 | * Locating erasures is slow, so attempt to recover the block without | |
460 | * them first. Do a second attempt with erasures if the corruption is | |
461 | * bad enough. | |
462 | */ | |
463 | r = fec_decode_rsb(v, io, fio, rsb, offset, false); | |
464 | if (r < 0) { | |
465 | r = fec_decode_rsb(v, io, fio, rsb, offset, true); | |
466 | if (r < 0) | |
f1a880a9 | 467 | goto done; |
a739ff3f ST |
468 | } |
469 | ||
470 | if (dest) | |
471 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); | |
472 | else if (iter) { | |
473 | fio->output_pos = 0; | |
474 | r = verity_for_bv_block(v, io, iter, fec_bv_copy); | |
475 | } | |
476 | ||
f1a880a9 ST |
477 | done: |
478 | fio->level--; | |
a739ff3f ST |
479 | return r; |
480 | } | |
481 | ||
482 | /* | |
483 | * Clean up per-bio data. | |
484 | */ | |
485 | void verity_fec_finish_io(struct dm_verity_io *io) | |
486 | { | |
487 | unsigned n; | |
488 | struct dm_verity_fec *f = io->v->fec; | |
489 | struct dm_verity_fec_io *fio = fec_io(io); | |
490 | ||
491 | if (!verity_fec_is_enabled(io->v)) | |
492 | return; | |
493 | ||
6f1c819c | 494 | mempool_free(fio->rs, &f->rs_pool); |
a739ff3f ST |
495 | |
496 | fec_for_each_prealloc_buffer(n) | |
6f1c819c | 497 | mempool_free(fio->bufs[n], &f->prealloc_pool); |
a739ff3f ST |
498 | |
499 | fec_for_each_extra_buffer(fio, n) | |
6f1c819c | 500 | mempool_free(fio->bufs[n], &f->extra_pool); |
a739ff3f | 501 | |
6f1c819c | 502 | mempool_free(fio->output, &f->output_pool); |
a739ff3f ST |
503 | } |
504 | ||
505 | /* | |
506 | * Initialize per-bio data. | |
507 | */ | |
508 | void verity_fec_init_io(struct dm_verity_io *io) | |
509 | { | |
510 | struct dm_verity_fec_io *fio = fec_io(io); | |
511 | ||
512 | if (!verity_fec_is_enabled(io->v)) | |
513 | return; | |
514 | ||
515 | fio->rs = NULL; | |
516 | memset(fio->bufs, 0, sizeof(fio->bufs)); | |
517 | fio->nbufs = 0; | |
518 | fio->output = NULL; | |
f1a880a9 | 519 | fio->level = 0; |
a739ff3f ST |
520 | } |
521 | ||
522 | /* | |
523 | * Append feature arguments and values to the status table. | |
524 | */ | |
525 | unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, | |
526 | char *result, unsigned maxlen) | |
527 | { | |
528 | if (!verity_fec_is_enabled(v)) | |
529 | return sz; | |
530 | ||
531 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " | |
532 | DM_VERITY_OPT_FEC_BLOCKS " %llu " | |
533 | DM_VERITY_OPT_FEC_START " %llu " | |
534 | DM_VERITY_OPT_FEC_ROOTS " %d", | |
535 | v->fec->dev->name, | |
536 | (unsigned long long)v->fec->blocks, | |
537 | (unsigned long long)v->fec->start, | |
538 | v->fec->roots); | |
539 | ||
540 | return sz; | |
541 | } | |
542 | ||
543 | void verity_fec_dtr(struct dm_verity *v) | |
544 | { | |
545 | struct dm_verity_fec *f = v->fec; | |
546 | ||
547 | if (!verity_fec_is_enabled(v)) | |
548 | goto out; | |
549 | ||
6f1c819c KO |
550 | mempool_exit(&f->rs_pool); |
551 | mempool_exit(&f->prealloc_pool); | |
552 | mempool_exit(&f->extra_pool); | |
75fa6019 | 553 | mempool_exit(&f->output_pool); |
a739ff3f ST |
554 | kmem_cache_destroy(f->cache); |
555 | ||
556 | if (f->data_bufio) | |
557 | dm_bufio_client_destroy(f->data_bufio); | |
558 | if (f->bufio) | |
559 | dm_bufio_client_destroy(f->bufio); | |
560 | ||
561 | if (f->dev) | |
562 | dm_put_device(v->ti, f->dev); | |
563 | out: | |
564 | kfree(f); | |
565 | v->fec = NULL; | |
566 | } | |
567 | ||
568 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) | |
569 | { | |
570 | struct dm_verity *v = (struct dm_verity *)pool_data; | |
571 | ||
eb366989 | 572 | return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); |
a739ff3f ST |
573 | } |
574 | ||
575 | static void fec_rs_free(void *element, void *pool_data) | |
576 | { | |
577 | struct rs_control *rs = (struct rs_control *)element; | |
578 | ||
579 | if (rs) | |
580 | free_rs(rs); | |
581 | } | |
582 | ||
583 | bool verity_is_fec_opt_arg(const char *arg_name) | |
584 | { | |
585 | return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || | |
586 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || | |
587 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || | |
588 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); | |
589 | } | |
590 | ||
591 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, | |
592 | unsigned *argc, const char *arg_name) | |
593 | { | |
594 | int r; | |
595 | struct dm_target *ti = v->ti; | |
596 | const char *arg_value; | |
597 | unsigned long long num_ll; | |
598 | unsigned char num_c; | |
599 | char dummy; | |
600 | ||
601 | if (!*argc) { | |
602 | ti->error = "FEC feature arguments require a value"; | |
603 | return -EINVAL; | |
604 | } | |
605 | ||
606 | arg_value = dm_shift_arg(as); | |
607 | (*argc)--; | |
608 | ||
609 | if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { | |
610 | r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); | |
611 | if (r) { | |
612 | ti->error = "FEC device lookup failed"; | |
613 | return r; | |
614 | } | |
615 | ||
616 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { | |
617 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
618 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) | |
619 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
620 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
621 | return -EINVAL; | |
622 | } | |
623 | v->fec->blocks = num_ll; | |
624 | ||
625 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { | |
626 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
627 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> | |
628 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
629 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; | |
630 | return -EINVAL; | |
631 | } | |
632 | v->fec->start = num_ll; | |
633 | ||
634 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { | |
635 | if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || | |
636 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || | |
637 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { | |
638 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; | |
639 | return -EINVAL; | |
640 | } | |
641 | v->fec->roots = num_c; | |
642 | ||
643 | } else { | |
644 | ti->error = "Unrecognized verity FEC feature request"; | |
645 | return -EINVAL; | |
646 | } | |
647 | ||
648 | return 0; | |
649 | } | |
650 | ||
651 | /* | |
652 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. | |
653 | */ | |
654 | int verity_fec_ctr_alloc(struct dm_verity *v) | |
655 | { | |
656 | struct dm_verity_fec *f; | |
657 | ||
658 | f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); | |
659 | if (!f) { | |
660 | v->ti->error = "Cannot allocate FEC structure"; | |
661 | return -ENOMEM; | |
662 | } | |
663 | v->fec = f; | |
664 | ||
665 | return 0; | |
666 | } | |
667 | ||
668 | /* | |
669 | * Validate arguments and preallocate memory. Must be called after arguments | |
670 | * have been parsed using verity_fec_parse_opt_args. | |
671 | */ | |
672 | int verity_fec_ctr(struct dm_verity *v) | |
673 | { | |
674 | struct dm_verity_fec *f = v->fec; | |
675 | struct dm_target *ti = v->ti; | |
df7b59ba | 676 | u64 hash_blocks, fec_blocks; |
6f1c819c | 677 | int ret; |
a739ff3f ST |
678 | |
679 | if (!verity_fec_is_enabled(v)) { | |
680 | verity_fec_dtr(v); | |
681 | return 0; | |
682 | } | |
683 | ||
684 | /* | |
685 | * FEC is computed over data blocks, possible metadata, and | |
686 | * hash blocks. In other words, FEC covers total of fec_blocks | |
687 | * blocks consisting of the following: | |
688 | * | |
689 | * data blocks | hash blocks | metadata (optional) | |
690 | * | |
691 | * We allow metadata after hash blocks to support a use case | |
692 | * where all data is stored on the same device and FEC covers | |
693 | * the entire area. | |
694 | * | |
695 | * If metadata is included, we require it to be available on the | |
696 | * hash device after the hash blocks. | |
697 | */ | |
698 | ||
699 | hash_blocks = v->hash_blocks - v->hash_start; | |
700 | ||
701 | /* | |
702 | * Require matching block sizes for data and hash devices for | |
703 | * simplicity. | |
704 | */ | |
705 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { | |
706 | ti->error = "Block sizes must match to use FEC"; | |
707 | return -EINVAL; | |
708 | } | |
709 | ||
710 | if (!f->roots) { | |
711 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; | |
712 | return -EINVAL; | |
713 | } | |
714 | f->rsn = DM_VERITY_FEC_RSM - f->roots; | |
715 | ||
716 | if (!f->blocks) { | |
717 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; | |
718 | return -EINVAL; | |
719 | } | |
720 | ||
721 | f->rounds = f->blocks; | |
722 | if (sector_div(f->rounds, f->rsn)) | |
723 | f->rounds++; | |
724 | ||
725 | /* | |
726 | * Due to optional metadata, f->blocks can be larger than | |
727 | * data_blocks and hash_blocks combined. | |
728 | */ | |
729 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { | |
730 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
731 | return -EINVAL; | |
732 | } | |
733 | ||
734 | /* | |
735 | * Metadata is accessed through the hash device, so we require | |
736 | * it to be large enough. | |
737 | */ | |
738 | f->hash_blocks = f->blocks - v->data_blocks; | |
739 | if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { | |
740 | ti->error = "Hash device is too small for " | |
741 | DM_VERITY_OPT_FEC_BLOCKS; | |
742 | return -E2BIG; | |
743 | } | |
744 | ||
8ca7cab8 JK |
745 | if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1)) |
746 | f->io_size = 1 << v->data_dev_block_bits; | |
747 | else | |
748 | f->io_size = v->fec->roots << SECTOR_SHIFT; | |
749 | ||
a739ff3f | 750 | f->bufio = dm_bufio_client_create(f->dev->bdev, |
8ca7cab8 | 751 | f->io_size, |
0fcb100d | 752 | 1, 0, NULL, NULL, 0); |
a739ff3f ST |
753 | if (IS_ERR(f->bufio)) { |
754 | ti->error = "Cannot initialize FEC bufio client"; | |
755 | return PTR_ERR(f->bufio); | |
756 | } | |
757 | ||
df7b59ba MB |
758 | dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); |
759 | ||
760 | fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); | |
761 | if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { | |
a739ff3f ST |
762 | ti->error = "FEC device is too small"; |
763 | return -E2BIG; | |
764 | } | |
765 | ||
766 | f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, | |
767 | 1 << v->data_dev_block_bits, | |
0fcb100d | 768 | 1, 0, NULL, NULL, 0); |
a739ff3f ST |
769 | if (IS_ERR(f->data_bufio)) { |
770 | ti->error = "Cannot initialize FEC data bufio client"; | |
771 | return PTR_ERR(f->data_bufio); | |
772 | } | |
773 | ||
774 | if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { | |
775 | ti->error = "Data device is too small"; | |
776 | return -E2BIG; | |
777 | } | |
778 | ||
779 | /* Preallocate an rs_control structure for each worker thread */ | |
6f1c819c KO |
780 | ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, |
781 | fec_rs_free, (void *) v); | |
782 | if (ret) { | |
a739ff3f | 783 | ti->error = "Cannot allocate RS pool"; |
6f1c819c | 784 | return ret; |
a739ff3f ST |
785 | } |
786 | ||
787 | f->cache = kmem_cache_create("dm_verity_fec_buffers", | |
788 | f->rsn << DM_VERITY_FEC_BUF_RS_BITS, | |
789 | 0, 0, NULL); | |
790 | if (!f->cache) { | |
791 | ti->error = "Cannot create FEC buffer cache"; | |
792 | return -ENOMEM; | |
793 | } | |
794 | ||
795 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ | |
6f1c819c KO |
796 | ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * |
797 | DM_VERITY_FEC_BUF_PREALLOC, | |
798 | f->cache); | |
799 | if (ret) { | |
a739ff3f | 800 | ti->error = "Cannot allocate FEC buffer prealloc pool"; |
6f1c819c | 801 | return ret; |
a739ff3f ST |
802 | } |
803 | ||
6f1c819c KO |
804 | ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache); |
805 | if (ret) { | |
a739ff3f | 806 | ti->error = "Cannot allocate FEC buffer extra pool"; |
6f1c819c | 807 | return ret; |
a739ff3f ST |
808 | } |
809 | ||
810 | /* Preallocate an output buffer for each thread */ | |
6f1c819c KO |
811 | ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(), |
812 | 1 << v->data_dev_block_bits); | |
813 | if (ret) { | |
a739ff3f | 814 | ti->error = "Cannot allocate FEC output pool"; |
6f1c819c | 815 | return ret; |
a739ff3f ST |
816 | } |
817 | ||
818 | /* Reserve space for our per-bio data */ | |
30187e1d | 819 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
a739ff3f ST |
820 | |
821 | return 0; | |
822 | } |