Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
a739ff3f ST |
2 | /* |
3 | * Copyright (C) 2015 Google, Inc. | |
4 | * | |
5 | * Author: Sami Tolvanen <samitolvanen@google.com> | |
a739ff3f ST |
6 | */ |
7 | ||
8 | #include "dm-verity-fec.h" | |
9 | #include <linux/math64.h> | |
10 | ||
11 | #define DM_MSG_PREFIX "verity-fec" | |
12 | ||
13 | /* | |
14 | * If error correction has been configured, returns true. | |
15 | */ | |
16 | bool verity_fec_is_enabled(struct dm_verity *v) | |
17 | { | |
18 | return v->fec && v->fec->dev; | |
19 | } | |
20 | ||
21 | /* | |
22 | * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable | |
23 | * length fields. | |
24 | */ | |
25 | static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) | |
26 | { | |
27 | return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); | |
28 | } | |
29 | ||
30 | /* | |
31 | * Return an interleaved offset for a byte in RS block. | |
32 | */ | |
33 | static inline u64 fec_interleave(struct dm_verity *v, u64 offset) | |
34 | { | |
35 | u32 mod; | |
36 | ||
37 | mod = do_div(offset, v->fec->rsn); | |
38 | return offset + mod * (v->fec->rounds << v->data_dev_block_bits); | |
39 | } | |
40 | ||
41 | /* | |
42 | * Decode an RS block using Reed-Solomon. | |
43 | */ | |
44 | static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
45 | u8 *data, u8 *fec, int neras) | |
46 | { | |
47 | int i; | |
48 | uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; | |
49 | ||
50 | for (i = 0; i < v->fec->roots; i++) | |
51 | par[i] = fec[i]; | |
52 | ||
53 | return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, | |
54 | fio->erasures, 0, NULL); | |
55 | } | |
56 | ||
57 | /* | |
58 | * Read error-correcting codes for the requested RS block. Returns a pointer | |
59 | * to the data block. Caller is responsible for releasing buf. | |
60 | */ | |
61 | static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, | |
62 | unsigned *offset, struct dm_buffer **buf) | |
63 | { | |
64 | u64 position, block; | |
65 | u8 *res; | |
66 | ||
67 | position = (index + rsb) * v->fec->roots; | |
68 | block = position >> v->data_dev_block_bits; | |
69 | *offset = (unsigned)(position - (block << v->data_dev_block_bits)); | |
70 | ||
71 | res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); | |
821b40da | 72 | if (IS_ERR(res)) { |
a739ff3f ST |
73 | DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", |
74 | v->data_dev->name, (unsigned long long)rsb, | |
75 | (unsigned long long)(v->fec->start + block), | |
76 | PTR_ERR(res)); | |
77 | *buf = NULL; | |
78 | } | |
79 | ||
80 | return res; | |
81 | } | |
82 | ||
83 | /* Loop over each preallocated buffer slot. */ | |
84 | #define fec_for_each_prealloc_buffer(__i) \ | |
85 | for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) | |
86 | ||
87 | /* Loop over each extra buffer slot. */ | |
88 | #define fec_for_each_extra_buffer(io, __i) \ | |
89 | for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++) | |
90 | ||
91 | /* Loop over each allocated buffer. */ | |
92 | #define fec_for_each_buffer(io, __i) \ | |
93 | for (__i = 0; __i < (io)->nbufs; __i++) | |
94 | ||
95 | /* Loop over each RS block in each allocated buffer. */ | |
96 | #define fec_for_each_buffer_rs_block(io, __i, __j) \ | |
97 | fec_for_each_buffer(io, __i) \ | |
98 | for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++) | |
99 | ||
100 | /* | |
101 | * Return a pointer to the current RS block when called inside | |
102 | * fec_for_each_buffer_rs_block. | |
103 | */ | |
104 | static inline u8 *fec_buffer_rs_block(struct dm_verity *v, | |
105 | struct dm_verity_fec_io *fio, | |
106 | unsigned i, unsigned j) | |
107 | { | |
108 | return &fio->bufs[i][j * v->fec->rsn]; | |
109 | } | |
110 | ||
111 | /* | |
112 | * Return an index to the current RS block when called inside | |
113 | * fec_for_each_buffer_rs_block. | |
114 | */ | |
115 | static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) | |
116 | { | |
117 | return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; | |
118 | } | |
119 | ||
120 | /* | |
121 | * Decode all RS blocks from buffers and copy corrected bytes into fio->output | |
122 | * starting from block_offset. | |
123 | */ | |
124 | static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, | |
125 | u64 rsb, int byte_index, unsigned block_offset, | |
126 | int neras) | |
127 | { | |
128 | int r, corrected = 0, res; | |
129 | struct dm_buffer *buf; | |
130 | unsigned n, i, offset; | |
131 | u8 *par, *block; | |
132 | ||
133 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
134 | if (IS_ERR(par)) | |
135 | return PTR_ERR(par); | |
136 | ||
137 | /* | |
138 | * Decode the RS blocks we have in bufs. Each RS block results in | |
139 | * one corrected target byte and consumes fec->roots parity bytes. | |
140 | */ | |
141 | fec_for_each_buffer_rs_block(fio, n, i) { | |
142 | block = fec_buffer_rs_block(v, fio, n, i); | |
143 | res = fec_decode_rs8(v, fio, block, &par[offset], neras); | |
144 | if (res < 0) { | |
a739ff3f ST |
145 | r = res; |
146 | goto error; | |
147 | } | |
148 | ||
149 | corrected += res; | |
150 | fio->output[block_offset] = block[byte_index]; | |
151 | ||
152 | block_offset++; | |
153 | if (block_offset >= 1 << v->data_dev_block_bits) | |
154 | goto done; | |
155 | ||
156 | /* read the next block when we run out of parity bytes */ | |
157 | offset += v->fec->roots; | |
158 | if (offset >= 1 << v->data_dev_block_bits) { | |
159 | dm_bufio_release(buf); | |
160 | ||
161 | par = fec_read_parity(v, rsb, block_offset, &offset, &buf); | |
821b40da | 162 | if (IS_ERR(par)) |
a739ff3f ST |
163 | return PTR_ERR(par); |
164 | } | |
165 | } | |
166 | done: | |
167 | r = corrected; | |
168 | error: | |
86e3e83b ST |
169 | dm_bufio_release(buf); |
170 | ||
a739ff3f ST |
171 | if (r < 0 && neras) |
172 | DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", | |
173 | v->data_dev->name, (unsigned long long)rsb, r); | |
174 | else if (r > 0) | |
175 | DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", | |
176 | v->data_dev->name, (unsigned long long)rsb, r); | |
177 | ||
178 | return r; | |
179 | } | |
180 | ||
181 | /* | |
182 | * Locate data block erasures using verity hashes. | |
183 | */ | |
184 | static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, | |
185 | u8 *want_digest, u8 *data) | |
186 | { | |
d1ac3ff0 | 187 | if (unlikely(verity_hash(v, verity_io_hash_req(v, io), |
a739ff3f ST |
188 | data, 1 << v->data_dev_block_bits, |
189 | verity_io_real_digest(v, io)))) | |
190 | return 0; | |
191 | ||
192 | return memcmp(verity_io_real_digest(v, io), want_digest, | |
193 | v->digest_size) != 0; | |
194 | } | |
195 | ||
196 | /* | |
197 | * Read data blocks that are part of the RS block and deinterleave as much as | |
198 | * fits into buffers. Check for erasure locations if @neras is non-NULL. | |
199 | */ | |
200 | static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, | |
201 | u64 rsb, u64 target, unsigned block_offset, | |
202 | int *neras) | |
203 | { | |
0cc37c2d | 204 | bool is_zero; |
a739ff3f ST |
205 | int i, j, target_index = -1; |
206 | struct dm_buffer *buf; | |
207 | struct dm_bufio_client *bufio; | |
208 | struct dm_verity_fec_io *fio = fec_io(io); | |
209 | u64 block, ileaved; | |
210 | u8 *bbuf, *rs_block; | |
6d39a124 | 211 | u8 want_digest[HASH_MAX_DIGESTSIZE]; |
a739ff3f ST |
212 | unsigned n, k; |
213 | ||
214 | if (neras) | |
215 | *neras = 0; | |
216 | ||
6d39a124 KC |
217 | if (WARN_ON(v->digest_size > sizeof(want_digest))) |
218 | return -EINVAL; | |
219 | ||
a739ff3f ST |
220 | /* |
221 | * read each of the rsn data blocks that are part of the RS block, and | |
222 | * interleave contents to available bufs | |
223 | */ | |
224 | for (i = 0; i < v->fec->rsn; i++) { | |
225 | ileaved = fec_interleave(v, rsb * v->fec->rsn + i); | |
226 | ||
227 | /* | |
228 | * target is the data block we want to correct, target_index is | |
229 | * the index of this block within the rsn RS blocks | |
230 | */ | |
231 | if (ileaved == target) | |
232 | target_index = i; | |
233 | ||
234 | block = ileaved >> v->data_dev_block_bits; | |
235 | bufio = v->fec->data_bufio; | |
236 | ||
237 | if (block >= v->data_blocks) { | |
238 | block -= v->data_blocks; | |
239 | ||
240 | /* | |
241 | * blocks outside the area were assumed to contain | |
242 | * zeros when encoding data was generated | |
243 | */ | |
244 | if (unlikely(block >= v->fec->hash_blocks)) | |
245 | continue; | |
246 | ||
247 | block += v->hash_start; | |
248 | bufio = v->bufio; | |
249 | } | |
250 | ||
251 | bbuf = dm_bufio_read(bufio, block, &buf); | |
821b40da | 252 | if (IS_ERR(bbuf)) { |
a739ff3f ST |
253 | DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", |
254 | v->data_dev->name, | |
255 | (unsigned long long)rsb, | |
256 | (unsigned long long)block, PTR_ERR(bbuf)); | |
257 | ||
258 | /* assume the block is corrupted */ | |
259 | if (neras && *neras <= v->fec->roots) | |
260 | fio->erasures[(*neras)++] = i; | |
261 | ||
262 | continue; | |
263 | } | |
264 | ||
265 | /* locate erasures if the block is on the data device */ | |
266 | if (bufio == v->fec->data_bufio && | |
0cc37c2d ST |
267 | verity_hash_for_block(v, io, block, want_digest, |
268 | &is_zero) == 0) { | |
269 | /* skip known zero blocks entirely */ | |
270 | if (is_zero) | |
86e3e83b | 271 | goto done; |
0cc37c2d | 272 | |
a739ff3f ST |
273 | /* |
274 | * skip if we have already found the theoretical | |
275 | * maximum number (i.e. fec->roots) of erasures | |
276 | */ | |
277 | if (neras && *neras <= v->fec->roots && | |
278 | fec_is_erasure(v, io, want_digest, bbuf)) | |
279 | fio->erasures[(*neras)++] = i; | |
280 | } | |
281 | ||
282 | /* | |
283 | * deinterleave and copy the bytes that fit into bufs, | |
284 | * starting from block_offset | |
285 | */ | |
286 | fec_for_each_buffer_rs_block(fio, n, j) { | |
287 | k = fec_buffer_rs_index(n, j) + block_offset; | |
288 | ||
289 | if (k >= 1 << v->data_dev_block_bits) | |
290 | goto done; | |
291 | ||
292 | rs_block = fec_buffer_rs_block(v, fio, n, j); | |
293 | rs_block[i] = bbuf[k]; | |
294 | } | |
295 | done: | |
296 | dm_bufio_release(buf); | |
297 | } | |
298 | ||
299 | return target_index; | |
300 | } | |
301 | ||
302 | /* | |
303 | * Allocate RS control structure and FEC buffers from preallocated mempools, | |
304 | * and attempt to allocate as many extra buffers as available. | |
305 | */ | |
306 | static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
307 | { | |
308 | unsigned n; | |
309 | ||
34c96507 | 310 | if (!fio->rs) |
6f1c819c | 311 | fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); |
a739ff3f ST |
312 | |
313 | fec_for_each_prealloc_buffer(n) { | |
314 | if (fio->bufs[n]) | |
315 | continue; | |
316 | ||
6f1c819c | 317 | fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); |
a739ff3f ST |
318 | if (unlikely(!fio->bufs[n])) { |
319 | DMERR("failed to allocate FEC buffer"); | |
320 | return -ENOMEM; | |
321 | } | |
322 | } | |
323 | ||
324 | /* try to allocate the maximum number of buffers */ | |
325 | fec_for_each_extra_buffer(fio, n) { | |
326 | if (fio->bufs[n]) | |
327 | continue; | |
328 | ||
6f1c819c | 329 | fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); |
a739ff3f ST |
330 | /* we can manage with even one buffer if necessary */ |
331 | if (unlikely(!fio->bufs[n])) | |
332 | break; | |
333 | } | |
334 | fio->nbufs = n; | |
335 | ||
34c96507 | 336 | if (!fio->output) |
6f1c819c | 337 | fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); |
a739ff3f | 338 | |
a739ff3f ST |
339 | return 0; |
340 | } | |
341 | ||
342 | /* | |
343 | * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are | |
344 | * zeroed before deinterleaving. | |
345 | */ | |
346 | static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) | |
347 | { | |
348 | unsigned n; | |
349 | ||
350 | fec_for_each_buffer(fio, n) | |
351 | memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); | |
352 | ||
353 | memset(fio->erasures, 0, sizeof(fio->erasures)); | |
354 | } | |
355 | ||
356 | /* | |
357 | * Decode all RS blocks in a single data block and return the target block | |
358 | * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses | |
359 | * hashes to locate erasures. | |
360 | */ | |
361 | static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, | |
362 | struct dm_verity_fec_io *fio, u64 rsb, u64 offset, | |
363 | bool use_erasures) | |
364 | { | |
365 | int r, neras = 0; | |
366 | unsigned pos; | |
367 | ||
368 | r = fec_alloc_bufs(v, fio); | |
369 | if (unlikely(r < 0)) | |
370 | return r; | |
371 | ||
372 | for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { | |
373 | fec_init_bufs(v, fio); | |
374 | ||
375 | r = fec_read_bufs(v, io, rsb, offset, pos, | |
376 | use_erasures ? &neras : NULL); | |
377 | if (unlikely(r < 0)) | |
378 | return r; | |
379 | ||
380 | r = fec_decode_bufs(v, fio, rsb, r, pos, neras); | |
381 | if (r < 0) | |
382 | return r; | |
383 | ||
384 | pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; | |
385 | } | |
386 | ||
387 | /* Always re-validate the corrected block against the expected hash */ | |
d1ac3ff0 | 388 | r = verity_hash(v, verity_io_hash_req(v, io), fio->output, |
a739ff3f ST |
389 | 1 << v->data_dev_block_bits, |
390 | verity_io_real_digest(v, io)); | |
391 | if (unlikely(r < 0)) | |
392 | return r; | |
393 | ||
394 | if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io), | |
395 | v->digest_size)) { | |
396 | DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)", | |
397 | v->data_dev->name, (unsigned long long)rsb, neras); | |
398 | return -EILSEQ; | |
399 | } | |
400 | ||
401 | return 0; | |
402 | } | |
403 | ||
404 | static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, | |
405 | size_t len) | |
406 | { | |
407 | struct dm_verity_fec_io *fio = fec_io(io); | |
408 | ||
409 | memcpy(data, &fio->output[fio->output_pos], len); | |
410 | fio->output_pos += len; | |
411 | ||
412 | return 0; | |
413 | } | |
414 | ||
415 | /* | |
416 | * Correct errors in a block. Copies corrected block to dest if non-NULL, | |
417 | * otherwise to a bio_vec starting from iter. | |
418 | */ | |
419 | int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, | |
420 | enum verity_block_type type, sector_t block, u8 *dest, | |
421 | struct bvec_iter *iter) | |
422 | { | |
423 | int r; | |
424 | struct dm_verity_fec_io *fio = fec_io(io); | |
425 | u64 offset, res, rsb; | |
426 | ||
427 | if (!verity_fec_is_enabled(v)) | |
428 | return -EOPNOTSUPP; | |
429 | ||
f1a880a9 ST |
430 | if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { |
431 | DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); | |
432 | return -EIO; | |
433 | } | |
434 | ||
435 | fio->level++; | |
436 | ||
a739ff3f | 437 | if (type == DM_VERITY_BLOCK_TYPE_METADATA) |
ad4e80a6 | 438 | block = block - v->hash_start + v->data_blocks; |
a739ff3f ST |
439 | |
440 | /* | |
441 | * For RS(M, N), the continuous FEC data is divided into blocks of N | |
442 | * bytes. Since block size may not be divisible by N, the last block | |
443 | * is zero padded when decoding. | |
444 | * | |
445 | * Each byte of the block is covered by a different RS(M, N) code, | |
446 | * and each code is interleaved over N blocks to make it less likely | |
447 | * that bursty corruption will leave us in unrecoverable state. | |
448 | */ | |
449 | ||
450 | offset = block << v->data_dev_block_bits; | |
602d1657 | 451 | res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); |
a739ff3f ST |
452 | |
453 | /* | |
454 | * The base RS block we can feed to the interleaver to find out all | |
455 | * blocks required for decoding. | |
456 | */ | |
457 | rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); | |
458 | ||
459 | /* | |
460 | * Locating erasures is slow, so attempt to recover the block without | |
461 | * them first. Do a second attempt with erasures if the corruption is | |
462 | * bad enough. | |
463 | */ | |
464 | r = fec_decode_rsb(v, io, fio, rsb, offset, false); | |
465 | if (r < 0) { | |
466 | r = fec_decode_rsb(v, io, fio, rsb, offset, true); | |
467 | if (r < 0) | |
f1a880a9 | 468 | goto done; |
a739ff3f ST |
469 | } |
470 | ||
471 | if (dest) | |
472 | memcpy(dest, fio->output, 1 << v->data_dev_block_bits); | |
473 | else if (iter) { | |
474 | fio->output_pos = 0; | |
475 | r = verity_for_bv_block(v, io, iter, fec_bv_copy); | |
476 | } | |
477 | ||
f1a880a9 ST |
478 | done: |
479 | fio->level--; | |
a739ff3f ST |
480 | return r; |
481 | } | |
482 | ||
483 | /* | |
484 | * Clean up per-bio data. | |
485 | */ | |
486 | void verity_fec_finish_io(struct dm_verity_io *io) | |
487 | { | |
488 | unsigned n; | |
489 | struct dm_verity_fec *f = io->v->fec; | |
490 | struct dm_verity_fec_io *fio = fec_io(io); | |
491 | ||
492 | if (!verity_fec_is_enabled(io->v)) | |
493 | return; | |
494 | ||
6f1c819c | 495 | mempool_free(fio->rs, &f->rs_pool); |
a739ff3f ST |
496 | |
497 | fec_for_each_prealloc_buffer(n) | |
6f1c819c | 498 | mempool_free(fio->bufs[n], &f->prealloc_pool); |
a739ff3f ST |
499 | |
500 | fec_for_each_extra_buffer(fio, n) | |
6f1c819c | 501 | mempool_free(fio->bufs[n], &f->extra_pool); |
a739ff3f | 502 | |
6f1c819c | 503 | mempool_free(fio->output, &f->output_pool); |
a739ff3f ST |
504 | } |
505 | ||
506 | /* | |
507 | * Initialize per-bio data. | |
508 | */ | |
509 | void verity_fec_init_io(struct dm_verity_io *io) | |
510 | { | |
511 | struct dm_verity_fec_io *fio = fec_io(io); | |
512 | ||
513 | if (!verity_fec_is_enabled(io->v)) | |
514 | return; | |
515 | ||
516 | fio->rs = NULL; | |
517 | memset(fio->bufs, 0, sizeof(fio->bufs)); | |
518 | fio->nbufs = 0; | |
519 | fio->output = NULL; | |
f1a880a9 | 520 | fio->level = 0; |
a739ff3f ST |
521 | } |
522 | ||
523 | /* | |
524 | * Append feature arguments and values to the status table. | |
525 | */ | |
526 | unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, | |
527 | char *result, unsigned maxlen) | |
528 | { | |
529 | if (!verity_fec_is_enabled(v)) | |
530 | return sz; | |
531 | ||
532 | DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s " | |
533 | DM_VERITY_OPT_FEC_BLOCKS " %llu " | |
534 | DM_VERITY_OPT_FEC_START " %llu " | |
535 | DM_VERITY_OPT_FEC_ROOTS " %d", | |
536 | v->fec->dev->name, | |
537 | (unsigned long long)v->fec->blocks, | |
538 | (unsigned long long)v->fec->start, | |
539 | v->fec->roots); | |
540 | ||
541 | return sz; | |
542 | } | |
543 | ||
544 | void verity_fec_dtr(struct dm_verity *v) | |
545 | { | |
546 | struct dm_verity_fec *f = v->fec; | |
547 | ||
548 | if (!verity_fec_is_enabled(v)) | |
549 | goto out; | |
550 | ||
6f1c819c KO |
551 | mempool_exit(&f->rs_pool); |
552 | mempool_exit(&f->prealloc_pool); | |
553 | mempool_exit(&f->extra_pool); | |
75fa6019 | 554 | mempool_exit(&f->output_pool); |
a739ff3f ST |
555 | kmem_cache_destroy(f->cache); |
556 | ||
557 | if (f->data_bufio) | |
558 | dm_bufio_client_destroy(f->data_bufio); | |
559 | if (f->bufio) | |
560 | dm_bufio_client_destroy(f->bufio); | |
561 | ||
562 | if (f->dev) | |
563 | dm_put_device(v->ti, f->dev); | |
564 | out: | |
565 | kfree(f); | |
566 | v->fec = NULL; | |
567 | } | |
568 | ||
569 | static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) | |
570 | { | |
571 | struct dm_verity *v = (struct dm_verity *)pool_data; | |
572 | ||
eb366989 | 573 | return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); |
a739ff3f ST |
574 | } |
575 | ||
576 | static void fec_rs_free(void *element, void *pool_data) | |
577 | { | |
578 | struct rs_control *rs = (struct rs_control *)element; | |
579 | ||
580 | if (rs) | |
581 | free_rs(rs); | |
582 | } | |
583 | ||
584 | bool verity_is_fec_opt_arg(const char *arg_name) | |
585 | { | |
586 | return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) || | |
587 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) || | |
588 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) || | |
589 | !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)); | |
590 | } | |
591 | ||
592 | int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, | |
593 | unsigned *argc, const char *arg_name) | |
594 | { | |
595 | int r; | |
596 | struct dm_target *ti = v->ti; | |
597 | const char *arg_value; | |
598 | unsigned long long num_ll; | |
599 | unsigned char num_c; | |
600 | char dummy; | |
601 | ||
602 | if (!*argc) { | |
603 | ti->error = "FEC feature arguments require a value"; | |
604 | return -EINVAL; | |
605 | } | |
606 | ||
607 | arg_value = dm_shift_arg(as); | |
608 | (*argc)--; | |
609 | ||
610 | if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { | |
611 | r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); | |
612 | if (r) { | |
613 | ti->error = "FEC device lookup failed"; | |
614 | return r; | |
615 | } | |
616 | ||
617 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) { | |
618 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
619 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) | |
620 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
621 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
622 | return -EINVAL; | |
623 | } | |
624 | v->fec->blocks = num_ll; | |
625 | ||
626 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) { | |
627 | if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 || | |
628 | ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> | |
629 | (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { | |
630 | ti->error = "Invalid " DM_VERITY_OPT_FEC_START; | |
631 | return -EINVAL; | |
632 | } | |
633 | v->fec->start = num_ll; | |
634 | ||
635 | } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { | |
636 | if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || | |
637 | num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || | |
638 | num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { | |
639 | ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; | |
640 | return -EINVAL; | |
641 | } | |
642 | v->fec->roots = num_c; | |
643 | ||
644 | } else { | |
645 | ti->error = "Unrecognized verity FEC feature request"; | |
646 | return -EINVAL; | |
647 | } | |
648 | ||
649 | return 0; | |
650 | } | |
651 | ||
652 | /* | |
653 | * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr. | |
654 | */ | |
655 | int verity_fec_ctr_alloc(struct dm_verity *v) | |
656 | { | |
657 | struct dm_verity_fec *f; | |
658 | ||
659 | f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL); | |
660 | if (!f) { | |
661 | v->ti->error = "Cannot allocate FEC structure"; | |
662 | return -ENOMEM; | |
663 | } | |
664 | v->fec = f; | |
665 | ||
666 | return 0; | |
667 | } | |
668 | ||
669 | /* | |
670 | * Validate arguments and preallocate memory. Must be called after arguments | |
671 | * have been parsed using verity_fec_parse_opt_args. | |
672 | */ | |
673 | int verity_fec_ctr(struct dm_verity *v) | |
674 | { | |
675 | struct dm_verity_fec *f = v->fec; | |
676 | struct dm_target *ti = v->ti; | |
677 | u64 hash_blocks; | |
6f1c819c | 678 | int ret; |
a739ff3f ST |
679 | |
680 | if (!verity_fec_is_enabled(v)) { | |
681 | verity_fec_dtr(v); | |
682 | return 0; | |
683 | } | |
684 | ||
685 | /* | |
686 | * FEC is computed over data blocks, possible metadata, and | |
687 | * hash blocks. In other words, FEC covers total of fec_blocks | |
688 | * blocks consisting of the following: | |
689 | * | |
690 | * data blocks | hash blocks | metadata (optional) | |
691 | * | |
692 | * We allow metadata after hash blocks to support a use case | |
693 | * where all data is stored on the same device and FEC covers | |
694 | * the entire area. | |
695 | * | |
696 | * If metadata is included, we require it to be available on the | |
697 | * hash device after the hash blocks. | |
698 | */ | |
699 | ||
700 | hash_blocks = v->hash_blocks - v->hash_start; | |
701 | ||
702 | /* | |
703 | * Require matching block sizes for data and hash devices for | |
704 | * simplicity. | |
705 | */ | |
706 | if (v->data_dev_block_bits != v->hash_dev_block_bits) { | |
707 | ti->error = "Block sizes must match to use FEC"; | |
708 | return -EINVAL; | |
709 | } | |
710 | ||
711 | if (!f->roots) { | |
712 | ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; | |
713 | return -EINVAL; | |
714 | } | |
715 | f->rsn = DM_VERITY_FEC_RSM - f->roots; | |
716 | ||
717 | if (!f->blocks) { | |
718 | ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; | |
719 | return -EINVAL; | |
720 | } | |
721 | ||
722 | f->rounds = f->blocks; | |
723 | if (sector_div(f->rounds, f->rsn)) | |
724 | f->rounds++; | |
725 | ||
726 | /* | |
727 | * Due to optional metadata, f->blocks can be larger than | |
728 | * data_blocks and hash_blocks combined. | |
729 | */ | |
730 | if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { | |
731 | ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; | |
732 | return -EINVAL; | |
733 | } | |
734 | ||
735 | /* | |
736 | * Metadata is accessed through the hash device, so we require | |
737 | * it to be large enough. | |
738 | */ | |
739 | f->hash_blocks = f->blocks - v->data_blocks; | |
740 | if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { | |
741 | ti->error = "Hash device is too small for " | |
742 | DM_VERITY_OPT_FEC_BLOCKS; | |
743 | return -E2BIG; | |
744 | } | |
745 | ||
746 | f->bufio = dm_bufio_client_create(f->dev->bdev, | |
747 | 1 << v->data_dev_block_bits, | |
748 | 1, 0, NULL, NULL); | |
749 | if (IS_ERR(f->bufio)) { | |
750 | ti->error = "Cannot initialize FEC bufio client"; | |
751 | return PTR_ERR(f->bufio); | |
752 | } | |
753 | ||
754 | if (dm_bufio_get_device_size(f->bufio) < | |
755 | ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { | |
756 | ti->error = "FEC device is too small"; | |
757 | return -E2BIG; | |
758 | } | |
759 | ||
760 | f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, | |
761 | 1 << v->data_dev_block_bits, | |
762 | 1, 0, NULL, NULL); | |
763 | if (IS_ERR(f->data_bufio)) { | |
764 | ti->error = "Cannot initialize FEC data bufio client"; | |
765 | return PTR_ERR(f->data_bufio); | |
766 | } | |
767 | ||
768 | if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { | |
769 | ti->error = "Data device is too small"; | |
770 | return -E2BIG; | |
771 | } | |
772 | ||
773 | /* Preallocate an rs_control structure for each worker thread */ | |
6f1c819c KO |
774 | ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, |
775 | fec_rs_free, (void *) v); | |
776 | if (ret) { | |
a739ff3f | 777 | ti->error = "Cannot allocate RS pool"; |
6f1c819c | 778 | return ret; |
a739ff3f ST |
779 | } |
780 | ||
781 | f->cache = kmem_cache_create("dm_verity_fec_buffers", | |
782 | f->rsn << DM_VERITY_FEC_BUF_RS_BITS, | |
783 | 0, 0, NULL); | |
784 | if (!f->cache) { | |
785 | ti->error = "Cannot create FEC buffer cache"; | |
786 | return -ENOMEM; | |
787 | } | |
788 | ||
789 | /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */ | |
6f1c819c KO |
790 | ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * |
791 | DM_VERITY_FEC_BUF_PREALLOC, | |
792 | f->cache); | |
793 | if (ret) { | |
a739ff3f | 794 | ti->error = "Cannot allocate FEC buffer prealloc pool"; |
6f1c819c | 795 | return ret; |
a739ff3f ST |
796 | } |
797 | ||
6f1c819c KO |
798 | ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache); |
799 | if (ret) { | |
a739ff3f | 800 | ti->error = "Cannot allocate FEC buffer extra pool"; |
6f1c819c | 801 | return ret; |
a739ff3f ST |
802 | } |
803 | ||
804 | /* Preallocate an output buffer for each thread */ | |
6f1c819c KO |
805 | ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(), |
806 | 1 << v->data_dev_block_bits); | |
807 | if (ret) { | |
a739ff3f | 808 | ti->error = "Cannot allocate FEC output pool"; |
6f1c819c | 809 | return ret; |
a739ff3f ST |
810 | } |
811 | ||
812 | /* Reserve space for our per-bio data */ | |
30187e1d | 813 | ti->per_io_data_size += sizeof(struct dm_verity_fec_io); |
a739ff3f ST |
814 | |
815 | return 0; | |
816 | } |