Merge tag 'driver-core-5.18-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-block.git] / crypto / async_tx / async_raid6_recov.c
CommitLineData
fd534e9b 1// SPDX-License-Identifier: GPL-2.0-or-later
0a82a623
DW
2/*
3 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
4 * Copyright(c) 2009 Intel Corporation
5 *
6 * based on raid6recov.c:
7 * Copyright 2002 H. Peter Anvin
0a82a623
DW
8 */
9#include <linux/kernel.h>
10#include <linux/interrupt.h>
4bb33cc8 11#include <linux/module.h>
0a82a623
DW
12#include <linux/dma-mapping.h>
13#include <linux/raid/pq.h>
14#include <linux/async_tx.h>
3bbdd498 15#include <linux/dmaengine.h>
0a82a623
DW
16
17static struct dma_async_tx_descriptor *
4f86ff55
YY
18async_sum_product(struct page *dest, unsigned int d_off,
19 struct page **srcs, unsigned int *src_offs, unsigned char *coef,
20 size_t len, struct async_submit_ctl *submit)
0a82a623
DW
21{
22 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
23 &dest, 1, srcs, 2, len);
24 struct dma_device *dma = chan ? chan->device : NULL;
3bbdd498 25 struct dmaengine_unmap_data *unmap = NULL;
0a82a623
DW
26 const u8 *amul, *bmul;
27 u8 ax, bx;
28 u8 *a, *b, *c;
29
3bbdd498 30 if (dma)
b02bab6b 31 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
3bbdd498
DW
32
33 if (unmap) {
0a82a623 34 struct device *dev = dma->dev;
3bbdd498 35 dma_addr_t pq[2];
0a82a623 36 struct dma_async_tx_descriptor *tx;
0776ae7b 37 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
0a82a623 38
0403e382
DW
39 if (submit->flags & ASYNC_TX_FENCE)
40 dma_flags |= DMA_PREP_FENCE;
4f86ff55
YY
41 unmap->addr[0] = dma_map_page(dev, srcs[0], src_offs[0],
42 len, DMA_TO_DEVICE);
43 unmap->addr[1] = dma_map_page(dev, srcs[1], src_offs[1],
44 len, DMA_TO_DEVICE);
3bbdd498
DW
45 unmap->to_cnt = 2;
46
4f86ff55
YY
47 unmap->addr[2] = dma_map_page(dev, dest, d_off,
48 len, DMA_BIDIRECTIONAL);
3bbdd498
DW
49 unmap->bidi_cnt = 1;
50 /* engine only looks at Q, but expects it to follow P */
51 pq[1] = unmap->addr[2];
52
53 unmap->len = len;
54 tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
0a82a623
DW
55 len, dma_flags);
56 if (tx) {
3bbdd498 57 dma_set_unmap(tx, unmap);
0a82a623 58 async_tx_submit(chan, tx, submit);
3bbdd498 59 dmaengine_unmap_put(unmap);
0a82a623
DW
60 return tx;
61 }
1f6672d4
DW
62
63 /* could not get a descriptor, unmap and fall through to
64 * the synchronous path
65 */
3bbdd498 66 dmaengine_unmap_put(unmap);
0a82a623
DW
67 }
68
69 /* run the operation synchronously */
70 async_tx_quiesce(&submit->depend_tx);
71 amul = raid6_gfmul[coef[0]];
72 bmul = raid6_gfmul[coef[1]];
4f86ff55
YY
73 a = page_address(srcs[0]) + src_offs[0];
74 b = page_address(srcs[1]) + src_offs[1];
75 c = page_address(dest) + d_off;
0a82a623
DW
76
77 while (len--) {
78 ax = amul[*a++];
79 bx = bmul[*b++];
80 *c++ = ax ^ bx;
81 }
82
83 return NULL;
84}
85
86static struct dma_async_tx_descriptor *
4f86ff55
YY
87async_mult(struct page *dest, unsigned int d_off, struct page *src,
88 unsigned int s_off, u8 coef, size_t len,
89 struct async_submit_ctl *submit)
0a82a623
DW
90{
91 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
92 &dest, 1, &src, 1, len);
93 struct dma_device *dma = chan ? chan->device : NULL;
3bbdd498 94 struct dmaengine_unmap_data *unmap = NULL;
0a82a623
DW
95 const u8 *qmul; /* Q multiplier table */
96 u8 *d, *s;
97
3bbdd498 98 if (dma)
b02bab6b 99 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
3bbdd498
DW
100
101 if (unmap) {
0a82a623 102 dma_addr_t dma_dest[2];
0a82a623
DW
103 struct device *dev = dma->dev;
104 struct dma_async_tx_descriptor *tx;
0776ae7b 105 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
0a82a623 106
0403e382
DW
107 if (submit->flags & ASYNC_TX_FENCE)
108 dma_flags |= DMA_PREP_FENCE;
4f86ff55
YY
109 unmap->addr[0] = dma_map_page(dev, src, s_off,
110 len, DMA_TO_DEVICE);
3bbdd498 111 unmap->to_cnt++;
4f86ff55
YY
112 unmap->addr[1] = dma_map_page(dev, dest, d_off,
113 len, DMA_BIDIRECTIONAL);
3bbdd498
DW
114 dma_dest[1] = unmap->addr[1];
115 unmap->bidi_cnt++;
116 unmap->len = len;
117
118 /* this looks funny, but the engine looks for Q at
119 * dma_dest[1] and ignores dma_dest[0] as a dest
120 * due to DMA_PREP_PQ_DISABLE_P
121 */
122 tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
123 1, &coef, len, dma_flags);
124
0a82a623 125 if (tx) {
3bbdd498
DW
126 dma_set_unmap(tx, unmap);
127 dmaengine_unmap_put(unmap);
0a82a623
DW
128 async_tx_submit(chan, tx, submit);
129 return tx;
130 }
1f6672d4
DW
131
132 /* could not get a descriptor, unmap and fall through to
133 * the synchronous path
134 */
3bbdd498 135 dmaengine_unmap_put(unmap);
0a82a623
DW
136 }
137
138 /* no channel available, or failed to allocate a descriptor, so
139 * perform the operation synchronously
140 */
141 async_tx_quiesce(&submit->depend_tx);
142 qmul = raid6_gfmul[coef];
4f86ff55
YY
143 d = page_address(dest) + d_off;
144 s = page_address(src) + s_off;
0a82a623
DW
145
146 while (len--)
147 *d++ = qmul[*s++];
148
149 return NULL;
150}
151
152static struct dma_async_tx_descriptor *
da17bf43 153__2data_recov_4(int disks, size_t bytes, int faila, int failb,
4f86ff55
YY
154 struct page **blocks, unsigned int *offs,
155 struct async_submit_ctl *submit)
0a82a623
DW
156{
157 struct dma_async_tx_descriptor *tx = NULL;
158 struct page *p, *q, *a, *b;
4f86ff55 159 unsigned int p_off, q_off, a_off, b_off;
0a82a623 160 struct page *srcs[2];
4f86ff55 161 unsigned int src_offs[2];
0a82a623
DW
162 unsigned char coef[2];
163 enum async_tx_flags flags = submit->flags;
164 dma_async_tx_callback cb_fn = submit->cb_fn;
165 void *cb_param = submit->cb_param;
166 void *scribble = submit->scribble;
167
da17bf43 168 p = blocks[disks-2];
4f86ff55 169 p_off = offs[disks-2];
da17bf43 170 q = blocks[disks-1];
4f86ff55 171 q_off = offs[disks-1];
0a82a623
DW
172
173 a = blocks[faila];
4f86ff55 174 a_off = offs[faila];
0a82a623 175 b = blocks[failb];
4f86ff55 176 b_off = offs[failb];
0a82a623
DW
177
178 /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
179 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
180 srcs[0] = p;
4f86ff55 181 src_offs[0] = p_off;
0a82a623 182 srcs[1] = q;
4f86ff55 183 src_offs[1] = q_off;
0a82a623
DW
184 coef[0] = raid6_gfexi[failb-faila];
185 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382 186 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55 187 tx = async_sum_product(b, b_off, srcs, src_offs, coef, bytes, submit);
0a82a623
DW
188
189 /* Dy = P+Pxy+Dx */
190 srcs[0] = p;
4f86ff55 191 src_offs[0] = p_off;
0a82a623 192 srcs[1] = b;
4f86ff55 193 src_offs[1] = b_off;
0a82a623
DW
194 init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
195 cb_param, scribble);
4f86ff55 196 tx = async_xor_offs(a, a_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
197
198 return tx;
199
200}
201
202static struct dma_async_tx_descriptor *
da17bf43 203__2data_recov_5(int disks, size_t bytes, int faila, int failb,
4f86ff55
YY
204 struct page **blocks, unsigned int *offs,
205 struct async_submit_ctl *submit)
0a82a623
DW
206{
207 struct dma_async_tx_descriptor *tx = NULL;
208 struct page *p, *q, *g, *dp, *dq;
4f86ff55 209 unsigned int p_off, q_off, g_off, dp_off, dq_off;
0a82a623 210 struct page *srcs[2];
4f86ff55 211 unsigned int src_offs[2];
0a82a623
DW
212 unsigned char coef[2];
213 enum async_tx_flags flags = submit->flags;
214 dma_async_tx_callback cb_fn = submit->cb_fn;
215 void *cb_param = submit->cb_param;
216 void *scribble = submit->scribble;
da17bf43 217 int good_srcs, good, i;
0a82a623 218
da17bf43
DW
219 good_srcs = 0;
220 good = -1;
221 for (i = 0; i < disks-2; i++) {
222 if (blocks[i] == NULL)
223 continue;
0a82a623
DW
224 if (i == faila || i == failb)
225 continue;
da17bf43
DW
226 good = i;
227 good_srcs++;
0a82a623 228 }
da17bf43 229 BUG_ON(good_srcs > 1);
0a82a623 230
da17bf43 231 p = blocks[disks-2];
4f86ff55 232 p_off = offs[disks-2];
da17bf43 233 q = blocks[disks-1];
4f86ff55 234 q_off = offs[disks-1];
0a82a623 235 g = blocks[good];
4f86ff55 236 g_off = offs[good];
0a82a623
DW
237
238 /* Compute syndrome with zero for the missing data pages
239 * Use the dead data pages as temporary storage for delta p and
240 * delta q
241 */
242 dp = blocks[faila];
4f86ff55 243 dp_off = offs[faila];
0a82a623 244 dq = blocks[failb];
4f86ff55 245 dq_off = offs[failb];
0a82a623 246
0403e382 247 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55 248 tx = async_memcpy(dp, g, dp_off, g_off, bytes, submit);
0403e382 249 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55
YY
250 tx = async_mult(dq, dq_off, g, g_off,
251 raid6_gfexp[good], bytes, submit);
0a82a623
DW
252
253 /* compute P + Pxy */
254 srcs[0] = dp;
4f86ff55 255 src_offs[0] = dp_off;
0a82a623 256 srcs[1] = p;
4f86ff55 257 src_offs[1] = p_off;
0403e382
DW
258 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
259 NULL, NULL, scribble);
4f86ff55 260 tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
261
262 /* compute Q + Qxy */
263 srcs[0] = dq;
4f86ff55 264 src_offs[0] = dq_off;
0a82a623 265 srcs[1] = q;
4f86ff55 266 src_offs[1] = q_off;
0403e382
DW
267 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
268 NULL, NULL, scribble);
4f86ff55 269 tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
270
271 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
272 srcs[0] = dp;
4f86ff55 273 src_offs[0] = dp_off;
0a82a623 274 srcs[1] = dq;
4f86ff55 275 src_offs[1] = dq_off;
0a82a623
DW
276 coef[0] = raid6_gfexi[failb-faila];
277 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382 278 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55 279 tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
0a82a623
DW
280
281 /* Dy = P+Pxy+Dx */
282 srcs[0] = dp;
4f86ff55 283 src_offs[0] = dp_off;
0a82a623 284 srcs[1] = dq;
4f86ff55 285 src_offs[1] = dq_off;
0a82a623
DW
286 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
287 cb_param, scribble);
4f86ff55 288 tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
289
290 return tx;
291}
292
293static struct dma_async_tx_descriptor *
294__2data_recov_n(int disks, size_t bytes, int faila, int failb,
4f86ff55
YY
295 struct page **blocks, unsigned int *offs,
296 struct async_submit_ctl *submit)
0a82a623
DW
297{
298 struct dma_async_tx_descriptor *tx = NULL;
299 struct page *p, *q, *dp, *dq;
4f86ff55 300 unsigned int p_off, q_off, dp_off, dq_off;
0a82a623 301 struct page *srcs[2];
4f86ff55 302 unsigned int src_offs[2];
0a82a623
DW
303 unsigned char coef[2];
304 enum async_tx_flags flags = submit->flags;
305 dma_async_tx_callback cb_fn = submit->cb_fn;
306 void *cb_param = submit->cb_param;
307 void *scribble = submit->scribble;
308
309 p = blocks[disks-2];
4f86ff55 310 p_off = offs[disks-2];
0a82a623 311 q = blocks[disks-1];
4f86ff55 312 q_off = offs[disks-1];
0a82a623
DW
313
314 /* Compute syndrome with zero for the missing data pages
315 * Use the dead data pages as temporary storage for
316 * delta p and delta q
317 */
318 dp = blocks[faila];
4f86ff55 319 dp_off = offs[faila];
5dd33c9a 320 blocks[faila] = NULL;
0a82a623 321 blocks[disks-2] = dp;
4f86ff55 322 offs[disks-2] = dp_off;
0a82a623 323 dq = blocks[failb];
4f86ff55 324 dq_off = offs[failb];
5dd33c9a 325 blocks[failb] = NULL;
0a82a623 326 blocks[disks-1] = dq;
4f86ff55 327 offs[disks-1] = dq_off;
0a82a623 328
0403e382 329 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55 330 tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
0a82a623
DW
331
332 /* Restore pointer table */
333 blocks[faila] = dp;
4f86ff55 334 offs[faila] = dp_off;
0a82a623 335 blocks[failb] = dq;
4f86ff55 336 offs[failb] = dq_off;
0a82a623 337 blocks[disks-2] = p;
4f86ff55 338 offs[disks-2] = p_off;
0a82a623 339 blocks[disks-1] = q;
4f86ff55 340 offs[disks-1] = q_off;
0a82a623
DW
341
342 /* compute P + Pxy */
343 srcs[0] = dp;
4f86ff55 344 src_offs[0] = dp_off;
0a82a623 345 srcs[1] = p;
4f86ff55 346 src_offs[1] = p_off;
0403e382
DW
347 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
348 NULL, NULL, scribble);
4f86ff55 349 tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
350
351 /* compute Q + Qxy */
352 srcs[0] = dq;
4f86ff55 353 src_offs[0] = dq_off;
0a82a623 354 srcs[1] = q;
4f86ff55 355 src_offs[1] = q_off;
0403e382
DW
356 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
357 NULL, NULL, scribble);
4f86ff55 358 tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
359
360 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
361 srcs[0] = dp;
4f86ff55 362 src_offs[0] = dp_off;
0a82a623 363 srcs[1] = dq;
4f86ff55 364 src_offs[1] = dq_off;
0a82a623
DW
365 coef[0] = raid6_gfexi[failb-faila];
366 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
0403e382 367 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55 368 tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
0a82a623
DW
369
370 /* Dy = P+Pxy+Dx */
371 srcs[0] = dp;
4f86ff55 372 src_offs[0] = dp_off;
0a82a623 373 srcs[1] = dq;
4f86ff55 374 src_offs[1] = dq_off;
0a82a623
DW
375 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
376 cb_param, scribble);
4f86ff55 377 tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
378
379 return tx;
380}
381
382/**
383 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
384 * @disks: number of disks in the RAID-6 array
385 * @bytes: block size
386 * @faila: first failed drive index
387 * @failb: second failed drive index
388 * @blocks: array of source pointers where the last two entries are p and q
4f86ff55 389 * @offs: array of offset for pages in blocks
0a82a623
DW
390 * @submit: submission/completion modifiers
391 */
392struct dma_async_tx_descriptor *
393async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
4f86ff55
YY
394 struct page **blocks, unsigned int *offs,
395 struct async_submit_ctl *submit)
0a82a623 396{
5157b4aa 397 void *scribble = submit->scribble;
da17bf43
DW
398 int non_zero_srcs, i;
399
0a82a623
DW
400 BUG_ON(faila == failb);
401 if (failb < faila)
402 swap(faila, failb);
403
404 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
405
5157b4aa
DW
406 /* if a dma resource is not available or a scribble buffer is not
407 * available punt to the synchronous path. In the 'dma not
408 * available' case be sure to use the scribble buffer to
409 * preserve the content of 'blocks' as the caller intended.
0a82a623 410 */
5157b4aa
DW
411 if (!async_dma_find_channel(DMA_PQ) || !scribble) {
412 void **ptrs = scribble ? scribble : (void **) blocks;
0a82a623
DW
413
414 async_tx_quiesce(&submit->depend_tx);
415 for (i = 0; i < disks; i++)
5dd33c9a 416 if (blocks[i] == NULL)
da17bf43 417 ptrs[i] = (void *) raid6_empty_zero_page;
5dd33c9a 418 else
4f86ff55 419 ptrs[i] = page_address(blocks[i]) + offs[i];
0a82a623
DW
420
421 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
422
423 async_tx_sync_epilog(submit);
424
425 return NULL;
426 }
427
da17bf43
DW
428 non_zero_srcs = 0;
429 for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
430 if (blocks[i])
431 non_zero_srcs++;
432 switch (non_zero_srcs) {
433 case 0:
434 case 1:
435 /* There must be at least 2 sources - the failed devices. */
436 BUG();
437
438 case 2:
0a82a623
DW
439 /* dma devices do not uniformly understand a zero source pq
440 * operation (in contrast to the synchronous case), so
da17bf43
DW
441 * explicitly handle the special case of a 4 disk array with
442 * both data disks missing.
0a82a623 443 */
4f86ff55
YY
444 return __2data_recov_4(disks, bytes, faila, failb,
445 blocks, offs, submit);
da17bf43 446 case 3:
0a82a623
DW
447 /* dma devices do not uniformly understand a single
448 * source pq operation (in contrast to the synchronous
da17bf43
DW
449 * case), so explicitly handle the special case of a 5 disk
450 * array with 2 of 3 data disks missing.
0a82a623 451 */
4f86ff55
YY
452 return __2data_recov_5(disks, bytes, faila, failb,
453 blocks, offs, submit);
0a82a623 454 default:
4f86ff55
YY
455 return __2data_recov_n(disks, bytes, faila, failb,
456 blocks, offs, submit);
0a82a623
DW
457 }
458}
459EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
460
461/**
462 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
463 * @disks: number of disks in the RAID-6 array
464 * @bytes: block size
465 * @faila: failed drive index
466 * @blocks: array of source pointers where the last two entries are p and q
4f86ff55 467 * @offs: array of offset for pages in blocks
0a82a623
DW
468 * @submit: submission/completion modifiers
469 */
470struct dma_async_tx_descriptor *
471async_raid6_datap_recov(int disks, size_t bytes, int faila,
4f86ff55
YY
472 struct page **blocks, unsigned int *offs,
473 struct async_submit_ctl *submit)
0a82a623
DW
474{
475 struct dma_async_tx_descriptor *tx = NULL;
476 struct page *p, *q, *dq;
4f86ff55 477 unsigned int p_off, q_off, dq_off;
0a82a623
DW
478 u8 coef;
479 enum async_tx_flags flags = submit->flags;
480 dma_async_tx_callback cb_fn = submit->cb_fn;
481 void *cb_param = submit->cb_param;
482 void *scribble = submit->scribble;
da17bf43 483 int good_srcs, good, i;
0a82a623 484 struct page *srcs[2];
4f86ff55 485 unsigned int src_offs[2];
0a82a623
DW
486
487 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
488
5157b4aa
DW
489 /* if a dma resource is not available or a scribble buffer is not
490 * available punt to the synchronous path. In the 'dma not
491 * available' case be sure to use the scribble buffer to
492 * preserve the content of 'blocks' as the caller intended.
0a82a623 493 */
5157b4aa
DW
494 if (!async_dma_find_channel(DMA_PQ) || !scribble) {
495 void **ptrs = scribble ? scribble : (void **) blocks;
0a82a623
DW
496
497 async_tx_quiesce(&submit->depend_tx);
498 for (i = 0; i < disks; i++)
5dd33c9a
N
499 if (blocks[i] == NULL)
500 ptrs[i] = (void*)raid6_empty_zero_page;
501 else
4f86ff55 502 ptrs[i] = page_address(blocks[i]) + offs[i];
0a82a623
DW
503
504 raid6_datap_recov(disks, bytes, faila, ptrs);
505
506 async_tx_sync_epilog(submit);
507
508 return NULL;
509 }
510
da17bf43
DW
511 good_srcs = 0;
512 good = -1;
513 for (i = 0; i < disks-2; i++) {
514 if (i == faila)
515 continue;
516 if (blocks[i]) {
517 good = i;
518 good_srcs++;
519 if (good_srcs > 1)
520 break;
521 }
522 }
523 BUG_ON(good_srcs == 0);
524
0a82a623 525 p = blocks[disks-2];
4f86ff55 526 p_off = offs[disks-2];
0a82a623 527 q = blocks[disks-1];
4f86ff55 528 q_off = offs[disks-1];
0a82a623
DW
529
530 /* Compute syndrome with zero for the missing data page
531 * Use the dead data page as temporary storage for delta q
532 */
533 dq = blocks[faila];
4f86ff55 534 dq_off = offs[faila];
5dd33c9a 535 blocks[faila] = NULL;
0a82a623 536 blocks[disks-1] = dq;
4f86ff55 537 offs[disks-1] = dq_off;
0a82a623 538
da17bf43
DW
539 /* in the 4-disk case we only need to perform a single source
540 * multiplication with the one good data block.
0a82a623 541 */
da17bf43 542 if (good_srcs == 1) {
0a82a623 543 struct page *g = blocks[good];
4f86ff55 544 unsigned int g_off = offs[good];
0a82a623 545
0403e382
DW
546 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
547 scribble);
4f86ff55 548 tx = async_memcpy(p, g, p_off, g_off, bytes, submit);
0a82a623 549
0403e382
DW
550 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
551 scribble);
4f86ff55
YY
552 tx = async_mult(dq, dq_off, g, g_off,
553 raid6_gfexp[good], bytes, submit);
0a82a623 554 } else {
0403e382
DW
555 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
556 scribble);
4f86ff55 557 tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
0a82a623
DW
558 }
559
560 /* Restore pointer table */
561 blocks[faila] = dq;
4f86ff55 562 offs[faila] = dq_off;
0a82a623 563 blocks[disks-1] = q;
4f86ff55 564 offs[disks-1] = q_off;
0a82a623
DW
565
566 /* calculate g^{-faila} */
567 coef = raid6_gfinv[raid6_gfexp[faila]];
568
569 srcs[0] = dq;
4f86ff55 570 src_offs[0] = dq_off;
0a82a623 571 srcs[1] = q;
4f86ff55 572 src_offs[1] = q_off;
0403e382
DW
573 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
574 NULL, NULL, scribble);
4f86ff55 575 tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
0a82a623 576
0403e382 577 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
4f86ff55 578 tx = async_mult(dq, dq_off, dq, dq_off, coef, bytes, submit);
0a82a623
DW
579
580 srcs[0] = p;
4f86ff55 581 src_offs[0] = p_off;
0a82a623 582 srcs[1] = dq;
4f86ff55 583 src_offs[1] = dq_off;
0a82a623
DW
584 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
585 cb_param, scribble);
4f86ff55 586 tx = async_xor_offs(p, p_off, srcs, src_offs, 2, bytes, submit);
0a82a623
DW
587
588 return tx;
589}
590EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
591
592MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
593MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
594MODULE_LICENSE("GPL");