prep.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Intel I/OAT DMA Linux driver
  4. * Copyright(c) 2004 - 2015 Intel Corporation.
  5. */
  6. #include <linux/module.h>
  7. #include <linux/pci.h>
  8. #include <linux/gfp.h>
  9. #include <linux/dmaengine.h>
  10. #include <linux/dma-mapping.h>
  11. #include <linux/prefetch.h>
  12. #include "../dmaengine.h"
  13. #include "registers.h"
  14. #include "hw.h"
  15. #include "dma.h"
  16. #define MAX_SCF 256
  17. /* provide a lookup table for setting the source address in the base or
  18. * extended descriptor of an xor or pq descriptor
  19. */
  20. static const u8 xor_idx_to_desc = 0xe0;
  21. static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 };
  22. static const u8 pq_idx_to_desc = 0xf8;
  23. static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1,
  24. 2, 2, 2, 2, 2, 2, 2 };
  25. static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
  26. static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
  27. 0, 1, 2, 3, 4, 5, 6 };
  28. static void xor_set_src(struct ioat_raw_descriptor *descs[2],
  29. dma_addr_t addr, u32 offset, int idx)
  30. {
  31. struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
  32. raw->field[xor_idx_to_field[idx]] = addr + offset;
  33. }
  34. static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
  35. {
  36. struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
  37. return raw->field[pq_idx_to_field[idx]];
  38. }
  39. static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx)
  40. {
  41. struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
  42. return raw->field[pq16_idx_to_field[idx]];
  43. }
  44. static void pq_set_src(struct ioat_raw_descriptor *descs[2],
  45. dma_addr_t addr, u32 offset, u8 coef, int idx)
  46. {
  47. struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
  48. struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
  49. raw->field[pq_idx_to_field[idx]] = addr + offset;
  50. pq->coef[idx] = coef;
  51. }
  52. static void pq16_set_src(struct ioat_raw_descriptor *desc[3],
  53. dma_addr_t addr, u32 offset, u8 coef, unsigned idx)
  54. {
  55. struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0];
  56. struct ioat_pq16a_descriptor *pq16 =
  57. (struct ioat_pq16a_descriptor *)desc[1];
  58. struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
  59. raw->field[pq16_idx_to_field[idx]] = addr + offset;
  60. if (idx < 8)
  61. pq->coef[idx] = coef;
  62. else
  63. pq16->coef[idx - 8] = coef;
  64. }
  65. static struct ioat_sed_ent *
  66. ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool)
  67. {
  68. struct ioat_sed_ent *sed;
  69. gfp_t flags = __GFP_ZERO | GFP_ATOMIC;
  70. sed = kmem_cache_alloc(ioat_sed_cache, flags);
  71. if (!sed)
  72. return NULL;
  73. sed->hw_pool = hw_pool;
  74. sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool],
  75. flags, &sed->dma);
  76. if (!sed->hw) {
  77. kmem_cache_free(ioat_sed_cache, sed);
  78. return NULL;
  79. }
  80. return sed;
  81. }
  82. struct dma_async_tx_descriptor *
  83. ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
  84. dma_addr_t dma_src, size_t len, unsigned long flags)
  85. {
  86. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  87. struct ioat_dma_descriptor *hw;
  88. struct ioat_ring_ent *desc;
  89. dma_addr_t dst = dma_dest;
  90. dma_addr_t src = dma_src;
  91. size_t total_len = len;
  92. int num_descs, idx, i;
  93. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  94. return NULL;
  95. num_descs = ioat_xferlen_to_descs(ioat_chan, len);
  96. if (likely(num_descs) &&
  97. ioat_check_space_lock(ioat_chan, num_descs) == 0)
  98. idx = ioat_chan->head;
  99. else
  100. return NULL;
  101. i = 0;
  102. do {
  103. size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);
  104. desc = ioat_get_ring_ent(ioat_chan, idx + i);
  105. hw = desc->hw;
  106. hw->size = copy;
  107. hw->ctl = 0;
  108. hw->src_addr = src;
  109. hw->dst_addr = dst;
  110. len -= copy;
  111. dst += copy;
  112. src += copy;
  113. dump_desc_dbg(ioat_chan, desc);
  114. } while (++i < num_descs);
  115. desc->txd.flags = flags;
  116. desc->len = total_len;
  117. hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
  118. hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
  119. hw->ctl_f.compl_write = 1;
  120. dump_desc_dbg(ioat_chan, desc);
  121. /* we leave the channel locked to ensure in order submission */
  122. return &desc->txd;
  123. }
  124. static struct dma_async_tx_descriptor *
  125. __ioat_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
  126. dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
  127. size_t len, unsigned long flags)
  128. {
  129. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  130. struct ioat_ring_ent *compl_desc;
  131. struct ioat_ring_ent *desc;
  132. struct ioat_ring_ent *ext;
  133. size_t total_len = len;
  134. struct ioat_xor_descriptor *xor;
  135. struct ioat_xor_ext_descriptor *xor_ex = NULL;
  136. struct ioat_dma_descriptor *hw;
  137. int num_descs, with_ext, idx, i;
  138. u32 offset = 0;
  139. u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
  140. BUG_ON(src_cnt < 2);
  141. num_descs = ioat_xferlen_to_descs(ioat_chan, len);
  142. /* we need 2x the number of descriptors to cover greater than 5
  143. * sources
  144. */
  145. if (src_cnt > 5) {
  146. with_ext = 1;
  147. num_descs *= 2;
  148. } else
  149. with_ext = 0;
  150. /* completion writes from the raid engine may pass completion
  151. * writes from the legacy engine, so we need one extra null
  152. * (legacy) descriptor to ensure all completion writes arrive in
  153. * order.
  154. */
  155. if (likely(num_descs) &&
  156. ioat_check_space_lock(ioat_chan, num_descs+1) == 0)
  157. idx = ioat_chan->head;
  158. else
  159. return NULL;
  160. i = 0;
  161. do {
  162. struct ioat_raw_descriptor *descs[2];
  163. size_t xfer_size = min_t(size_t,
  164. len, 1 << ioat_chan->xfercap_log);
  165. int s;
  166. desc = ioat_get_ring_ent(ioat_chan, idx + i);
  167. xor = desc->xor;
  168. /* save a branch by unconditionally retrieving the
  169. * extended descriptor xor_set_src() knows to not write
  170. * to it in the single descriptor case
  171. */
  172. ext = ioat_get_ring_ent(ioat_chan, idx + i + 1);
  173. xor_ex = ext->xor_ex;
  174. descs[0] = (struct ioat_raw_descriptor *) xor;
  175. descs[1] = (struct ioat_raw_descriptor *) xor_ex;
  176. for (s = 0; s < src_cnt; s++)
  177. xor_set_src(descs, src[s], offset, s);
  178. xor->size = xfer_size;
  179. xor->dst_addr = dest + offset;
  180. xor->ctl = 0;
  181. xor->ctl_f.op = op;
  182. xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
  183. len -= xfer_size;
  184. offset += xfer_size;
  185. dump_desc_dbg(ioat_chan, desc);
  186. } while ((i += 1 + with_ext) < num_descs);
  187. /* last xor descriptor carries the unmap parameters and fence bit */
  188. desc->txd.flags = flags;
  189. desc->len = total_len;
  190. if (result)
  191. desc->result = result;
  192. xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
  193. /* completion descriptor carries interrupt bit */
  194. compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
  195. compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
  196. hw = compl_desc->hw;
  197. hw->ctl = 0;
  198. hw->ctl_f.null = 1;
  199. hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
  200. hw->ctl_f.compl_write = 1;
  201. hw->size = NULL_DESC_BUFFER_SIZE;
  202. dump_desc_dbg(ioat_chan, compl_desc);
  203. /* we leave the channel locked to ensure in order submission */
  204. return &compl_desc->txd;
  205. }
  206. struct dma_async_tx_descriptor *
  207. ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
  208. unsigned int src_cnt, size_t len, unsigned long flags)
  209. {
  210. struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
  211. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  212. return NULL;
  213. return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
  214. }
  215. struct dma_async_tx_descriptor *
  216. ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
  217. unsigned int src_cnt, size_t len,
  218. enum sum_check_flags *result, unsigned long flags)
  219. {
  220. struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
  221. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  222. return NULL;
  223. /* the cleanup routine only sets bits on validate failure, it
  224. * does not clear bits on validate success... so clear it here
  225. */
  226. *result = 0;
  227. return __ioat_prep_xor_lock(chan, result, src[0], &src[1],
  228. src_cnt - 1, len, flags);
  229. }
  230. static void
  231. dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc,
  232. struct ioat_ring_ent *ext)
  233. {
  234. struct device *dev = to_dev(ioat_chan);
  235. struct ioat_pq_descriptor *pq = desc->pq;
  236. struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
  237. struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
  238. int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
  239. int i;
  240. dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
  241. " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
  242. " src_cnt: %d)\n",
  243. desc_id(desc), (unsigned long long) desc->txd.phys,
  244. (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
  245. desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op,
  246. pq->ctl_f.int_en, pq->ctl_f.compl_write,
  247. pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
  248. pq->ctl_f.src_cnt);
  249. for (i = 0; i < src_cnt; i++)
  250. dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
  251. (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
  252. dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
  253. dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
  254. dev_dbg(dev, "\tNEXT: %#llx\n", pq->next);
  255. }
  256. static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan,
  257. struct ioat_ring_ent *desc)
  258. {
  259. struct device *dev = to_dev(ioat_chan);
  260. struct ioat_pq_descriptor *pq = desc->pq;
  261. struct ioat_raw_descriptor *descs[] = { (void *)pq,
  262. (void *)pq,
  263. (void *)pq };
  264. int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);
  265. int i;
  266. if (desc->sed) {
  267. descs[1] = (void *)desc->sed->hw;
  268. descs[2] = (void *)desc->sed->hw + 64;
  269. }
  270. dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
  271. " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
  272. " src_cnt: %d)\n",
  273. desc_id(desc), (unsigned long long) desc->txd.phys,
  274. (unsigned long long) pq->next,
  275. desc->txd.flags, pq->size, pq->ctl,
  276. pq->ctl_f.op, pq->ctl_f.int_en,
  277. pq->ctl_f.compl_write,
  278. pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
  279. pq->ctl_f.src_cnt);
  280. for (i = 0; i < src_cnt; i++) {
  281. dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
  282. (unsigned long long) pq16_get_src(descs, i),
  283. pq->coef[i]);
  284. }
  285. dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
  286. dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
  287. }
  288. static struct dma_async_tx_descriptor *
  289. __ioat_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
  290. const dma_addr_t *dst, const dma_addr_t *src,
  291. unsigned int src_cnt, const unsigned char *scf,
  292. size_t len, unsigned long flags)
  293. {
  294. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  295. struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
  296. struct ioat_ring_ent *compl_desc;
  297. struct ioat_ring_ent *desc;
  298. struct ioat_ring_ent *ext;
  299. size_t total_len = len;
  300. struct ioat_pq_descriptor *pq;
  301. struct ioat_pq_ext_descriptor *pq_ex = NULL;
  302. struct ioat_dma_descriptor *hw;
  303. u32 offset = 0;
  304. u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
  305. int i, s, idx, with_ext, num_descs;
  306. int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0;
  307. dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
  308. /* the engine requires at least two sources (we provide
  309. * at least 1 implied source in the DMA_PREP_CONTINUE case)
  310. */
  311. BUG_ON(src_cnt + dmaf_continue(flags) < 2);
  312. num_descs = ioat_xferlen_to_descs(ioat_chan, len);
  313. /* we need 2x the number of descriptors to cover greater than 3
  314. * sources (we need 1 extra source in the q-only continuation
  315. * case and 3 extra sources in the p+q continuation case.
  316. */
  317. if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
  318. (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
  319. with_ext = 1;
  320. num_descs *= 2;
  321. } else
  322. with_ext = 0;
  323. /* completion writes from the raid engine may pass completion
  324. * writes from the legacy engine, so we need one extra null
  325. * (legacy) descriptor to ensure all completion writes arrive in
  326. * order.
  327. */
  328. if (likely(num_descs) &&
  329. ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0)
  330. idx = ioat_chan->head;
  331. else
  332. return NULL;
  333. i = 0;
  334. do {
  335. struct ioat_raw_descriptor *descs[2];
  336. size_t xfer_size = min_t(size_t, len,
  337. 1 << ioat_chan->xfercap_log);
  338. desc = ioat_get_ring_ent(ioat_chan, idx + i);
  339. pq = desc->pq;
  340. /* save a branch by unconditionally retrieving the
  341. * extended descriptor pq_set_src() knows to not write
  342. * to it in the single descriptor case
  343. */
  344. ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext);
  345. pq_ex = ext->pq_ex;
  346. descs[0] = (struct ioat_raw_descriptor *) pq;
  347. descs[1] = (struct ioat_raw_descriptor *) pq_ex;
  348. for (s = 0; s < src_cnt; s++)
  349. pq_set_src(descs, src[s], offset, scf[s], s);
  350. /* see the comment for dma_maxpq in include/linux/dmaengine.h */
  351. if (dmaf_p_disabled_continue(flags))
  352. pq_set_src(descs, dst[1], offset, 1, s++);
  353. else if (dmaf_continue(flags)) {
  354. pq_set_src(descs, dst[0], offset, 0, s++);
  355. pq_set_src(descs, dst[1], offset, 1, s++);
  356. pq_set_src(descs, dst[1], offset, 0, s++);
  357. }
  358. pq->size = xfer_size;
  359. pq->p_addr = dst[0] + offset;
  360. pq->q_addr = dst[1] + offset;
  361. pq->ctl = 0;
  362. pq->ctl_f.op = op;
  363. /* we turn on descriptor write back error status */
  364. if (ioat_dma->cap & IOAT_CAP_DWBES)
  365. pq->ctl_f.wb_en = result ? 1 : 0;
  366. pq->ctl_f.src_cnt = src_cnt_to_hw(s);
  367. pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
  368. pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
  369. len -= xfer_size;
  370. offset += xfer_size;
  371. } while ((i += 1 + with_ext) < num_descs);
  372. /* last pq descriptor carries the unmap parameters and fence bit */
  373. desc->txd.flags = flags;
  374. desc->len = total_len;
  375. if (result)
  376. desc->result = result;
  377. pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
  378. dump_pq_desc_dbg(ioat_chan, desc, ext);
  379. if (!cb32) {
  380. pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
  381. pq->ctl_f.compl_write = 1;
  382. compl_desc = desc;
  383. } else {
  384. /* completion descriptor carries interrupt bit */
  385. compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
  386. compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
  387. hw = compl_desc->hw;
  388. hw->ctl = 0;
  389. hw->ctl_f.null = 1;
  390. hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
  391. hw->ctl_f.compl_write = 1;
  392. hw->size = NULL_DESC_BUFFER_SIZE;
  393. dump_desc_dbg(ioat_chan, compl_desc);
  394. }
  395. /* we leave the channel locked to ensure in order submission */
  396. return &compl_desc->txd;
  397. }
  398. static struct dma_async_tx_descriptor *
  399. __ioat_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
  400. const dma_addr_t *dst, const dma_addr_t *src,
  401. unsigned int src_cnt, const unsigned char *scf,
  402. size_t len, unsigned long flags)
  403. {
  404. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  405. struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
  406. struct ioat_ring_ent *desc;
  407. size_t total_len = len;
  408. struct ioat_pq_descriptor *pq;
  409. u32 offset = 0;
  410. u8 op;
  411. int i, s, idx, num_descs;
  412. /* this function is only called with 9-16 sources */
  413. op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
  414. dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
  415. num_descs = ioat_xferlen_to_descs(ioat_chan, len);
  416. /*
  417. * 16 source pq is only available on cb3.3 and has no completion
  418. * write hw bug.
  419. */
  420. if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0)
  421. idx = ioat_chan->head;
  422. else
  423. return NULL;
  424. i = 0;
  425. do {
  426. struct ioat_raw_descriptor *descs[4];
  427. size_t xfer_size = min_t(size_t, len,
  428. 1 << ioat_chan->xfercap_log);
  429. desc = ioat_get_ring_ent(ioat_chan, idx + i);
  430. pq = desc->pq;
  431. descs[0] = (struct ioat_raw_descriptor *) pq;
  432. desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3);
  433. if (!desc->sed) {
  434. dev_err(to_dev(ioat_chan),
  435. "%s: no free sed entries\n", __func__);
  436. return NULL;
  437. }
  438. pq->sed_addr = desc->sed->dma;
  439. desc->sed->parent = desc;
  440. descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw;
  441. descs[2] = (void *)descs[1] + 64;
  442. for (s = 0; s < src_cnt; s++)
  443. pq16_set_src(descs, src[s], offset, scf[s], s);
  444. /* see the comment for dma_maxpq in include/linux/dmaengine.h */
  445. if (dmaf_p_disabled_continue(flags))
  446. pq16_set_src(descs, dst[1], offset, 1, s++);
  447. else if (dmaf_continue(flags)) {
  448. pq16_set_src(descs, dst[0], offset, 0, s++);
  449. pq16_set_src(descs, dst[1], offset, 1, s++);
  450. pq16_set_src(descs, dst[1], offset, 0, s++);
  451. }
  452. pq->size = xfer_size;
  453. pq->p_addr = dst[0] + offset;
  454. pq->q_addr = dst[1] + offset;
  455. pq->ctl = 0;
  456. pq->ctl_f.op = op;
  457. pq->ctl_f.src_cnt = src16_cnt_to_hw(s);
  458. /* we turn on descriptor write back error status */
  459. if (ioat_dma->cap & IOAT_CAP_DWBES)
  460. pq->ctl_f.wb_en = result ? 1 : 0;
  461. pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
  462. pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
  463. len -= xfer_size;
  464. offset += xfer_size;
  465. } while (++i < num_descs);
  466. /* last pq descriptor carries the unmap parameters and fence bit */
  467. desc->txd.flags = flags;
  468. desc->len = total_len;
  469. if (result)
  470. desc->result = result;
  471. pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
  472. /* with cb3.3 we should be able to do completion w/o a null desc */
  473. pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
  474. pq->ctl_f.compl_write = 1;
  475. dump_pq16_desc_dbg(ioat_chan, desc);
  476. /* we leave the channel locked to ensure in order submission */
  477. return &desc->txd;
  478. }
  479. static int src_cnt_flags(unsigned int src_cnt, unsigned long flags)
  480. {
  481. if (dmaf_p_disabled_continue(flags))
  482. return src_cnt + 1;
  483. else if (dmaf_continue(flags))
  484. return src_cnt + 3;
  485. else
  486. return src_cnt;
  487. }
  488. struct dma_async_tx_descriptor *
  489. ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
  490. unsigned int src_cnt, const unsigned char *scf, size_t len,
  491. unsigned long flags)
  492. {
  493. struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
  494. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  495. return NULL;
  496. /* specify valid address for disabled result */
  497. if (flags & DMA_PREP_PQ_DISABLE_P)
  498. dst[0] = dst[1];
  499. if (flags & DMA_PREP_PQ_DISABLE_Q)
  500. dst[1] = dst[0];
  501. /* handle the single source multiply case from the raid6
  502. * recovery path
  503. */
  504. if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
  505. dma_addr_t single_source[2];
  506. unsigned char single_source_coef[2];
  507. BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
  508. single_source[0] = src[0];
  509. single_source[1] = src[0];
  510. single_source_coef[0] = scf[0];
  511. single_source_coef[1] = 0;
  512. return src_cnt_flags(src_cnt, flags) > 8 ?
  513. __ioat_prep_pq16_lock(chan, NULL, dst, single_source,
  514. 2, single_source_coef, len,
  515. flags) :
  516. __ioat_prep_pq_lock(chan, NULL, dst, single_source, 2,
  517. single_source_coef, len, flags);
  518. } else {
  519. return src_cnt_flags(src_cnt, flags) > 8 ?
  520. __ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt,
  521. scf, len, flags) :
  522. __ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt,
  523. scf, len, flags);
  524. }
  525. }
  526. struct dma_async_tx_descriptor *
  527. ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
  528. unsigned int src_cnt, const unsigned char *scf, size_t len,
  529. enum sum_check_flags *pqres, unsigned long flags)
  530. {
  531. struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
  532. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  533. return NULL;
  534. /* specify valid address for disabled result */
  535. if (flags & DMA_PREP_PQ_DISABLE_P)
  536. pq[0] = pq[1];
  537. if (flags & DMA_PREP_PQ_DISABLE_Q)
  538. pq[1] = pq[0];
  539. /* the cleanup routine only sets bits on validate failure, it
  540. * does not clear bits on validate success... so clear it here
  541. */
  542. *pqres = 0;
  543. return src_cnt_flags(src_cnt, flags) > 8 ?
  544. __ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,
  545. flags) :
  546. __ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
  547. flags);
  548. }
  549. struct dma_async_tx_descriptor *
  550. ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
  551. unsigned int src_cnt, size_t len, unsigned long flags)
  552. {
  553. unsigned char scf[MAX_SCF];
  554. dma_addr_t pq[2];
  555. struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
  556. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  557. return NULL;
  558. if (src_cnt > MAX_SCF)
  559. return NULL;
  560. memset(scf, 0, src_cnt);
  561. pq[0] = dst;
  562. flags |= DMA_PREP_PQ_DISABLE_Q;
  563. pq[1] = dst; /* specify valid address for disabled result */
  564. return src_cnt_flags(src_cnt, flags) > 8 ?
  565. __ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,
  566. flags) :
  567. __ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
  568. flags);
  569. }
  570. struct dma_async_tx_descriptor *
  571. ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
  572. unsigned int src_cnt, size_t len,
  573. enum sum_check_flags *result, unsigned long flags)
  574. {
  575. unsigned char scf[MAX_SCF];
  576. dma_addr_t pq[2];
  577. struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
  578. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  579. return NULL;
  580. if (src_cnt > MAX_SCF)
  581. return NULL;
  582. /* the cleanup routine only sets bits on validate failure, it
  583. * does not clear bits on validate success... so clear it here
  584. */
  585. *result = 0;
  586. memset(scf, 0, src_cnt);
  587. pq[0] = src[0];
  588. flags |= DMA_PREP_PQ_DISABLE_Q;
  589. pq[1] = pq[0]; /* specify valid address for disabled result */
  590. return src_cnt_flags(src_cnt, flags) > 8 ?
  591. __ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,
  592. scf, len, flags) :
  593. __ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,
  594. scf, len, flags);
  595. }
  596. struct dma_async_tx_descriptor *
  597. ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
  598. {
  599. struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
  600. struct ioat_ring_ent *desc;
  601. struct ioat_dma_descriptor *hw;
  602. if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
  603. return NULL;
  604. if (ioat_check_space_lock(ioat_chan, 1) == 0)
  605. desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
  606. else
  607. return NULL;
  608. hw = desc->hw;
  609. hw->ctl = 0;
  610. hw->ctl_f.null = 1;
  611. hw->ctl_f.int_en = 1;
  612. hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
  613. hw->ctl_f.compl_write = 1;
  614. hw->size = NULL_DESC_BUFFER_SIZE;
  615. hw->src_addr = 0;
  616. hw->dst_addr = 0;
  617. desc->txd.flags = flags;
  618. desc->len = 1;
  619. dump_desc_dbg(ioat_chan, desc);
  620. /* we leave the channel locked to ensure in order submission */
  621. return &desc->txd;
  622. }