async_raid6_recov.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
  4. * Copyright(c) 2009 Intel Corporation
  5. *
  6. * based on raid6recov.c:
  7. * Copyright 2002 H. Peter Anvin
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/interrupt.h>
  11. #include <linux/module.h>
  12. #include <linux/dma-mapping.h>
  13. #include <linux/raid/pq.h>
  14. #include <linux/async_tx.h>
  15. #include <linux/dmaengine.h>
  16. static struct dma_async_tx_descriptor *
  17. async_sum_product(struct page *dest, unsigned int d_off,
  18. struct page **srcs, unsigned int *src_offs, unsigned char *coef,
  19. size_t len, struct async_submit_ctl *submit)
  20. {
  21. struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
  22. &dest, 1, srcs, 2, len);
  23. struct dma_device *dma = chan ? chan->device : NULL;
  24. struct dmaengine_unmap_data *unmap = NULL;
  25. const u8 *amul, *bmul;
  26. u8 ax, bx;
  27. u8 *a, *b, *c;
  28. if (dma)
  29. unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
  30. if (unmap) {
  31. struct device *dev = dma->dev;
  32. dma_addr_t pq[2];
  33. struct dma_async_tx_descriptor *tx;
  34. enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
  35. if (submit->flags & ASYNC_TX_FENCE)
  36. dma_flags |= DMA_PREP_FENCE;
  37. unmap->addr[0] = dma_map_page(dev, srcs[0], src_offs[0],
  38. len, DMA_TO_DEVICE);
  39. unmap->addr[1] = dma_map_page(dev, srcs[1], src_offs[1],
  40. len, DMA_TO_DEVICE);
  41. unmap->to_cnt = 2;
  42. unmap->addr[2] = dma_map_page(dev, dest, d_off,
  43. len, DMA_BIDIRECTIONAL);
  44. unmap->bidi_cnt = 1;
  45. /* engine only looks at Q, but expects it to follow P */
  46. pq[1] = unmap->addr[2];
  47. unmap->len = len;
  48. tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
  49. len, dma_flags);
  50. if (tx) {
  51. dma_set_unmap(tx, unmap);
  52. async_tx_submit(chan, tx, submit);
  53. dmaengine_unmap_put(unmap);
  54. return tx;
  55. }
  56. /* could not get a descriptor, unmap and fall through to
  57. * the synchronous path
  58. */
  59. dmaengine_unmap_put(unmap);
  60. }
  61. /* run the operation synchronously */
  62. async_tx_quiesce(&submit->depend_tx);
  63. amul = raid6_gfmul[coef[0]];
  64. bmul = raid6_gfmul[coef[1]];
  65. a = page_address(srcs[0]) + src_offs[0];
  66. b = page_address(srcs[1]) + src_offs[1];
  67. c = page_address(dest) + d_off;
  68. while (len--) {
  69. ax = amul[*a++];
  70. bx = bmul[*b++];
  71. *c++ = ax ^ bx;
  72. }
  73. return NULL;
  74. }
  75. static struct dma_async_tx_descriptor *
  76. async_mult(struct page *dest, unsigned int d_off, struct page *src,
  77. unsigned int s_off, u8 coef, size_t len,
  78. struct async_submit_ctl *submit)
  79. {
  80. struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
  81. &dest, 1, &src, 1, len);
  82. struct dma_device *dma = chan ? chan->device : NULL;
  83. struct dmaengine_unmap_data *unmap = NULL;
  84. const u8 *qmul; /* Q multiplier table */
  85. u8 *d, *s;
  86. if (dma)
  87. unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
  88. if (unmap) {
  89. dma_addr_t dma_dest[2];
  90. struct device *dev = dma->dev;
  91. struct dma_async_tx_descriptor *tx;
  92. enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
  93. if (submit->flags & ASYNC_TX_FENCE)
  94. dma_flags |= DMA_PREP_FENCE;
  95. unmap->addr[0] = dma_map_page(dev, src, s_off,
  96. len, DMA_TO_DEVICE);
  97. unmap->to_cnt++;
  98. unmap->addr[1] = dma_map_page(dev, dest, d_off,
  99. len, DMA_BIDIRECTIONAL);
  100. dma_dest[1] = unmap->addr[1];
  101. unmap->bidi_cnt++;
  102. unmap->len = len;
  103. /* this looks funny, but the engine looks for Q at
  104. * dma_dest[1] and ignores dma_dest[0] as a dest
  105. * due to DMA_PREP_PQ_DISABLE_P
  106. */
  107. tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
  108. 1, &coef, len, dma_flags);
  109. if (tx) {
  110. dma_set_unmap(tx, unmap);
  111. dmaengine_unmap_put(unmap);
  112. async_tx_submit(chan, tx, submit);
  113. return tx;
  114. }
  115. /* could not get a descriptor, unmap and fall through to
  116. * the synchronous path
  117. */
  118. dmaengine_unmap_put(unmap);
  119. }
  120. /* no channel available, or failed to allocate a descriptor, so
  121. * perform the operation synchronously
  122. */
  123. async_tx_quiesce(&submit->depend_tx);
  124. qmul = raid6_gfmul[coef];
  125. d = page_address(dest) + d_off;
  126. s = page_address(src) + s_off;
  127. while (len--)
  128. *d++ = qmul[*s++];
  129. return NULL;
  130. }
  131. static struct dma_async_tx_descriptor *
  132. __2data_recov_4(int disks, size_t bytes, int faila, int failb,
  133. struct page **blocks, unsigned int *offs,
  134. struct async_submit_ctl *submit)
  135. {
  136. struct dma_async_tx_descriptor *tx = NULL;
  137. struct page *p, *q, *a, *b;
  138. unsigned int p_off, q_off, a_off, b_off;
  139. struct page *srcs[2];
  140. unsigned int src_offs[2];
  141. unsigned char coef[2];
  142. enum async_tx_flags flags = submit->flags;
  143. dma_async_tx_callback cb_fn = submit->cb_fn;
  144. void *cb_param = submit->cb_param;
  145. void *scribble = submit->scribble;
  146. p = blocks[disks-2];
  147. p_off = offs[disks-2];
  148. q = blocks[disks-1];
  149. q_off = offs[disks-1];
  150. a = blocks[faila];
  151. a_off = offs[faila];
  152. b = blocks[failb];
  153. b_off = offs[failb];
  154. /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
  155. /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
  156. srcs[0] = p;
  157. src_offs[0] = p_off;
  158. srcs[1] = q;
  159. src_offs[1] = q_off;
  160. coef[0] = raid6_gfexi[failb-faila];
  161. coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
  162. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  163. tx = async_sum_product(b, b_off, srcs, src_offs, coef, bytes, submit);
  164. /* Dy = P+Pxy+Dx */
  165. srcs[0] = p;
  166. src_offs[0] = p_off;
  167. srcs[1] = b;
  168. src_offs[1] = b_off;
  169. init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
  170. cb_param, scribble);
  171. tx = async_xor_offs(a, a_off, srcs, src_offs, 2, bytes, submit);
  172. return tx;
  173. }
  174. static struct dma_async_tx_descriptor *
  175. __2data_recov_5(int disks, size_t bytes, int faila, int failb,
  176. struct page **blocks, unsigned int *offs,
  177. struct async_submit_ctl *submit)
  178. {
  179. struct dma_async_tx_descriptor *tx = NULL;
  180. struct page *p, *q, *g, *dp, *dq;
  181. unsigned int p_off, q_off, g_off, dp_off, dq_off;
  182. struct page *srcs[2];
  183. unsigned int src_offs[2];
  184. unsigned char coef[2];
  185. enum async_tx_flags flags = submit->flags;
  186. dma_async_tx_callback cb_fn = submit->cb_fn;
  187. void *cb_param = submit->cb_param;
  188. void *scribble = submit->scribble;
  189. int good_srcs, good, i;
  190. good_srcs = 0;
  191. good = -1;
  192. for (i = 0; i < disks-2; i++) {
  193. if (blocks[i] == NULL)
  194. continue;
  195. if (i == faila || i == failb)
  196. continue;
  197. good = i;
  198. good_srcs++;
  199. }
  200. BUG_ON(good_srcs > 1);
  201. p = blocks[disks-2];
  202. p_off = offs[disks-2];
  203. q = blocks[disks-1];
  204. q_off = offs[disks-1];
  205. g = blocks[good];
  206. g_off = offs[good];
  207. /* Compute syndrome with zero for the missing data pages
  208. * Use the dead data pages as temporary storage for delta p and
  209. * delta q
  210. */
  211. dp = blocks[faila];
  212. dp_off = offs[faila];
  213. dq = blocks[failb];
  214. dq_off = offs[failb];
  215. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  216. tx = async_memcpy(dp, g, dp_off, g_off, bytes, submit);
  217. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  218. tx = async_mult(dq, dq_off, g, g_off,
  219. raid6_gfexp[good], bytes, submit);
  220. /* compute P + Pxy */
  221. srcs[0] = dp;
  222. src_offs[0] = dp_off;
  223. srcs[1] = p;
  224. src_offs[1] = p_off;
  225. init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  226. NULL, NULL, scribble);
  227. tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
  228. /* compute Q + Qxy */
  229. srcs[0] = dq;
  230. src_offs[0] = dq_off;
  231. srcs[1] = q;
  232. src_offs[1] = q_off;
  233. init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  234. NULL, NULL, scribble);
  235. tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
  236. /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
  237. srcs[0] = dp;
  238. src_offs[0] = dp_off;
  239. srcs[1] = dq;
  240. src_offs[1] = dq_off;
  241. coef[0] = raid6_gfexi[failb-faila];
  242. coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
  243. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  244. tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
  245. /* Dy = P+Pxy+Dx */
  246. srcs[0] = dp;
  247. src_offs[0] = dp_off;
  248. srcs[1] = dq;
  249. src_offs[1] = dq_off;
  250. init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
  251. cb_param, scribble);
  252. tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
  253. return tx;
  254. }
  255. static struct dma_async_tx_descriptor *
  256. __2data_recov_n(int disks, size_t bytes, int faila, int failb,
  257. struct page **blocks, unsigned int *offs,
  258. struct async_submit_ctl *submit)
  259. {
  260. struct dma_async_tx_descriptor *tx = NULL;
  261. struct page *p, *q, *dp, *dq;
  262. unsigned int p_off, q_off, dp_off, dq_off;
  263. struct page *srcs[2];
  264. unsigned int src_offs[2];
  265. unsigned char coef[2];
  266. enum async_tx_flags flags = submit->flags;
  267. dma_async_tx_callback cb_fn = submit->cb_fn;
  268. void *cb_param = submit->cb_param;
  269. void *scribble = submit->scribble;
  270. p = blocks[disks-2];
  271. p_off = offs[disks-2];
  272. q = blocks[disks-1];
  273. q_off = offs[disks-1];
  274. /* Compute syndrome with zero for the missing data pages
  275. * Use the dead data pages as temporary storage for
  276. * delta p and delta q
  277. */
  278. dp = blocks[faila];
  279. dp_off = offs[faila];
  280. blocks[faila] = NULL;
  281. blocks[disks-2] = dp;
  282. offs[disks-2] = dp_off;
  283. dq = blocks[failb];
  284. dq_off = offs[failb];
  285. blocks[failb] = NULL;
  286. blocks[disks-1] = dq;
  287. offs[disks-1] = dq_off;
  288. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  289. tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
  290. /* Restore pointer table */
  291. blocks[faila] = dp;
  292. offs[faila] = dp_off;
  293. blocks[failb] = dq;
  294. offs[failb] = dq_off;
  295. blocks[disks-2] = p;
  296. offs[disks-2] = p_off;
  297. blocks[disks-1] = q;
  298. offs[disks-1] = q_off;
  299. /* compute P + Pxy */
  300. srcs[0] = dp;
  301. src_offs[0] = dp_off;
  302. srcs[1] = p;
  303. src_offs[1] = p_off;
  304. init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  305. NULL, NULL, scribble);
  306. tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
  307. /* compute Q + Qxy */
  308. srcs[0] = dq;
  309. src_offs[0] = dq_off;
  310. srcs[1] = q;
  311. src_offs[1] = q_off;
  312. init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  313. NULL, NULL, scribble);
  314. tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
  315. /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
  316. srcs[0] = dp;
  317. src_offs[0] = dp_off;
  318. srcs[1] = dq;
  319. src_offs[1] = dq_off;
  320. coef[0] = raid6_gfexi[failb-faila];
  321. coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
  322. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  323. tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit);
  324. /* Dy = P+Pxy+Dx */
  325. srcs[0] = dp;
  326. src_offs[0] = dp_off;
  327. srcs[1] = dq;
  328. src_offs[1] = dq_off;
  329. init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
  330. cb_param, scribble);
  331. tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit);
  332. return tx;
  333. }
  334. /**
  335. * async_raid6_2data_recov - asynchronously calculate two missing data blocks
  336. * @disks: number of disks in the RAID-6 array
  337. * @bytes: block size
  338. * @faila: first failed drive index
  339. * @failb: second failed drive index
  340. * @blocks: array of source pointers where the last two entries are p and q
  341. * @offs: array of offset for pages in blocks
  342. * @submit: submission/completion modifiers
  343. */
  344. struct dma_async_tx_descriptor *
  345. async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
  346. struct page **blocks, unsigned int *offs,
  347. struct async_submit_ctl *submit)
  348. {
  349. void *scribble = submit->scribble;
  350. int non_zero_srcs, i;
  351. BUG_ON(faila == failb);
  352. if (failb < faila)
  353. swap(faila, failb);
  354. pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
  355. /* if a dma resource is not available or a scribble buffer is not
  356. * available punt to the synchronous path. In the 'dma not
  357. * available' case be sure to use the scribble buffer to
  358. * preserve the content of 'blocks' as the caller intended.
  359. */
  360. if (!async_dma_find_channel(DMA_PQ) || !scribble) {
  361. void **ptrs = scribble ? scribble : (void **) blocks;
  362. async_tx_quiesce(&submit->depend_tx);
  363. for (i = 0; i < disks; i++)
  364. if (blocks[i] == NULL)
  365. ptrs[i] = (void *) raid6_empty_zero_page;
  366. else
  367. ptrs[i] = page_address(blocks[i]) + offs[i];
  368. raid6_2data_recov(disks, bytes, faila, failb, ptrs);
  369. async_tx_sync_epilog(submit);
  370. return NULL;
  371. }
  372. non_zero_srcs = 0;
  373. for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
  374. if (blocks[i])
  375. non_zero_srcs++;
  376. switch (non_zero_srcs) {
  377. case 0:
  378. case 1:
  379. /* There must be at least 2 sources - the failed devices. */
  380. BUG();
  381. case 2:
  382. /* dma devices do not uniformly understand a zero source pq
  383. * operation (in contrast to the synchronous case), so
  384. * explicitly handle the special case of a 4 disk array with
  385. * both data disks missing.
  386. */
  387. return __2data_recov_4(disks, bytes, faila, failb,
  388. blocks, offs, submit);
  389. case 3:
  390. /* dma devices do not uniformly understand a single
  391. * source pq operation (in contrast to the synchronous
  392. * case), so explicitly handle the special case of a 5 disk
  393. * array with 2 of 3 data disks missing.
  394. */
  395. return __2data_recov_5(disks, bytes, faila, failb,
  396. blocks, offs, submit);
  397. default:
  398. return __2data_recov_n(disks, bytes, faila, failb,
  399. blocks, offs, submit);
  400. }
  401. }
  402. EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
  403. /**
  404. * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
  405. * @disks: number of disks in the RAID-6 array
  406. * @bytes: block size
  407. * @faila: failed drive index
  408. * @blocks: array of source pointers where the last two entries are p and q
  409. * @offs: array of offset for pages in blocks
  410. * @submit: submission/completion modifiers
  411. */
  412. struct dma_async_tx_descriptor *
  413. async_raid6_datap_recov(int disks, size_t bytes, int faila,
  414. struct page **blocks, unsigned int *offs,
  415. struct async_submit_ctl *submit)
  416. {
  417. struct dma_async_tx_descriptor *tx = NULL;
  418. struct page *p, *q, *dq;
  419. unsigned int p_off, q_off, dq_off;
  420. u8 coef;
  421. enum async_tx_flags flags = submit->flags;
  422. dma_async_tx_callback cb_fn = submit->cb_fn;
  423. void *cb_param = submit->cb_param;
  424. void *scribble = submit->scribble;
  425. int good_srcs, good, i;
  426. struct page *srcs[2];
  427. unsigned int src_offs[2];
  428. pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
  429. /* if a dma resource is not available or a scribble buffer is not
  430. * available punt to the synchronous path. In the 'dma not
  431. * available' case be sure to use the scribble buffer to
  432. * preserve the content of 'blocks' as the caller intended.
  433. */
  434. if (!async_dma_find_channel(DMA_PQ) || !scribble) {
  435. void **ptrs = scribble ? scribble : (void **) blocks;
  436. async_tx_quiesce(&submit->depend_tx);
  437. for (i = 0; i < disks; i++)
  438. if (blocks[i] == NULL)
  439. ptrs[i] = (void*)raid6_empty_zero_page;
  440. else
  441. ptrs[i] = page_address(blocks[i]) + offs[i];
  442. raid6_datap_recov(disks, bytes, faila, ptrs);
  443. async_tx_sync_epilog(submit);
  444. return NULL;
  445. }
  446. good_srcs = 0;
  447. good = -1;
  448. for (i = 0; i < disks-2; i++) {
  449. if (i == faila)
  450. continue;
  451. if (blocks[i]) {
  452. good = i;
  453. good_srcs++;
  454. if (good_srcs > 1)
  455. break;
  456. }
  457. }
  458. BUG_ON(good_srcs == 0);
  459. p = blocks[disks-2];
  460. p_off = offs[disks-2];
  461. q = blocks[disks-1];
  462. q_off = offs[disks-1];
  463. /* Compute syndrome with zero for the missing data page
  464. * Use the dead data page as temporary storage for delta q
  465. */
  466. dq = blocks[faila];
  467. dq_off = offs[faila];
  468. blocks[faila] = NULL;
  469. blocks[disks-1] = dq;
  470. offs[disks-1] = dq_off;
  471. /* in the 4-disk case we only need to perform a single source
  472. * multiplication with the one good data block.
  473. */
  474. if (good_srcs == 1) {
  475. struct page *g = blocks[good];
  476. unsigned int g_off = offs[good];
  477. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
  478. scribble);
  479. tx = async_memcpy(p, g, p_off, g_off, bytes, submit);
  480. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
  481. scribble);
  482. tx = async_mult(dq, dq_off, g, g_off,
  483. raid6_gfexp[good], bytes, submit);
  484. } else {
  485. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
  486. scribble);
  487. tx = async_gen_syndrome(blocks, offs, disks, bytes, submit);
  488. }
  489. /* Restore pointer table */
  490. blocks[faila] = dq;
  491. offs[faila] = dq_off;
  492. blocks[disks-1] = q;
  493. offs[disks-1] = q_off;
  494. /* calculate g^{-faila} */
  495. coef = raid6_gfinv[raid6_gfexp[faila]];
  496. srcs[0] = dq;
  497. src_offs[0] = dq_off;
  498. srcs[1] = q;
  499. src_offs[1] = q_off;
  500. init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  501. NULL, NULL, scribble);
  502. tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit);
  503. init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
  504. tx = async_mult(dq, dq_off, dq, dq_off, coef, bytes, submit);
  505. srcs[0] = p;
  506. src_offs[0] = p_off;
  507. srcs[1] = dq;
  508. src_offs[1] = dq_off;
  509. init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
  510. cb_param, scribble);
  511. tx = async_xor_offs(p, p_off, srcs, src_offs, 2, bytes, submit);
  512. return tx;
  513. }
  514. EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
  515. MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
  516. MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
  517. MODULE_LICENSE("GPL");