fsl_raid.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897
  1. /*
  2. * drivers/dma/fsl_raid.c
  3. *
  4. * Freescale RAID Engine device driver
  5. *
  6. * Author:
  7. * Harninder Rai <harninder.rai@freescale.com>
  8. * Naveen Burmi <naveenburmi@freescale.com>
  9. *
  10. * Rewrite:
  11. * Xuelin Shi <xuelin.shi@freescale.com>
  12. *
  13. * Copyright (c) 2010-2014 Freescale Semiconductor, Inc.
  14. *
  15. * Redistribution and use in source and binary forms, with or without
  16. * modification, are permitted provided that the following conditions are met:
  17. * * Redistributions of source code must retain the above copyright
  18. * notice, this list of conditions and the following disclaimer.
  19. * * Redistributions in binary form must reproduce the above copyright
  20. * notice, this list of conditions and the following disclaimer in the
  21. * documentation and/or other materials provided with the distribution.
  22. * * Neither the name of Freescale Semiconductor nor the
  23. * names of its contributors may be used to endorse or promote products
  24. * derived from this software without specific prior written permission.
  25. *
  26. * ALTERNATIVELY, this software may be distributed under the terms of the
  27. * GNU General Public License ("GPL") as published by the Free Software
  28. * Foundation, either version 2 of that License or (at your option) any
  29. * later version.
  30. *
  31. * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  32. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  33. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  34. * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  35. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  36. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  38. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  39. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  40. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  41. *
  42. * Theory of operation:
  43. *
  44. * General capabilities:
  45. * RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q
  46. * calculations required in RAID5 and RAID6 operations. RE driver
  47. * registers with Linux's ASYNC layer as dma driver. RE hardware
  48. * maintains strict ordering of the requests through chained
  49. * command queueing.
  50. *
  51. * Data flow:
  52. * Software RAID layer of Linux (MD layer) maintains RAID partitions,
  53. * strips, stripes etc. It sends requests to the underlying ASYNC layer
  54. * which further passes it to RE driver. ASYNC layer decides which request
  55. * goes to which job ring of RE hardware. For every request processed by
  56. * RAID Engine, driver gets an interrupt unless coalescing is set. The
  57. * per job ring interrupt handler checks the status register for errors,
  58. * clears the interrupt and leave the post interrupt processing to the irq
  59. * thread.
  60. */
  61. #include <linux/interrupt.h>
  62. #include <linux/module.h>
  63. #include <linux/of_irq.h>
  64. #include <linux/of_address.h>
  65. #include <linux/of_platform.h>
  66. #include <linux/dma-mapping.h>
  67. #include <linux/dmapool.h>
  68. #include <linux/dmaengine.h>
  69. #include <linux/io.h>
  70. #include <linux/spinlock.h>
  71. #include <linux/slab.h>
  72. #include "dmaengine.h"
  73. #include "fsl_raid.h"
  74. #define FSL_RE_MAX_XOR_SRCS 16
  75. #define FSL_RE_MAX_PQ_SRCS 16
  76. #define FSL_RE_MIN_DESCS 256
  77. #define FSL_RE_MAX_DESCS (4 * FSL_RE_MIN_DESCS)
  78. #define FSL_RE_FRAME_FORMAT 0x1
  79. #define FSL_RE_MAX_DATA_LEN (1024*1024)
  80. #define to_fsl_re_dma_desc(tx) container_of(tx, struct fsl_re_desc, async_tx)
  81. /* Add descriptors into per chan software queue - submit_q */
  82. static dma_cookie_t fsl_re_tx_submit(struct dma_async_tx_descriptor *tx)
  83. {
  84. struct fsl_re_desc *desc;
  85. struct fsl_re_chan *re_chan;
  86. dma_cookie_t cookie;
  87. unsigned long flags;
  88. desc = to_fsl_re_dma_desc(tx);
  89. re_chan = container_of(tx->chan, struct fsl_re_chan, chan);
  90. spin_lock_irqsave(&re_chan->desc_lock, flags);
  91. cookie = dma_cookie_assign(tx);
  92. list_add_tail(&desc->node, &re_chan->submit_q);
  93. spin_unlock_irqrestore(&re_chan->desc_lock, flags);
  94. return cookie;
  95. }
  96. /* Copy descriptor from per chan software queue into hardware job ring */
  97. static void fsl_re_issue_pending(struct dma_chan *chan)
  98. {
  99. struct fsl_re_chan *re_chan;
  100. int avail;
  101. struct fsl_re_desc *desc, *_desc;
  102. unsigned long flags;
  103. re_chan = container_of(chan, struct fsl_re_chan, chan);
  104. spin_lock_irqsave(&re_chan->desc_lock, flags);
  105. avail = FSL_RE_SLOT_AVAIL(
  106. in_be32(&re_chan->jrregs->inbring_slot_avail));
  107. list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) {
  108. if (!avail)
  109. break;
  110. list_move_tail(&desc->node, &re_chan->active_q);
  111. memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count],
  112. &desc->hwdesc, sizeof(struct fsl_re_hw_desc));
  113. re_chan->inb_count = (re_chan->inb_count + 1) &
  114. FSL_RE_RING_SIZE_MASK;
  115. out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1));
  116. avail--;
  117. }
  118. spin_unlock_irqrestore(&re_chan->desc_lock, flags);
  119. }
  120. static void fsl_re_desc_done(struct fsl_re_desc *desc)
  121. {
  122. dma_cookie_complete(&desc->async_tx);
  123. dma_descriptor_unmap(&desc->async_tx);
  124. dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
  125. }
  126. static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan)
  127. {
  128. struct fsl_re_desc *desc, *_desc;
  129. unsigned long flags;
  130. spin_lock_irqsave(&re_chan->desc_lock, flags);
  131. list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) {
  132. if (async_tx_test_ack(&desc->async_tx))
  133. list_move_tail(&desc->node, &re_chan->free_q);
  134. }
  135. spin_unlock_irqrestore(&re_chan->desc_lock, flags);
  136. fsl_re_issue_pending(&re_chan->chan);
  137. }
  138. static void fsl_re_dequeue(struct tasklet_struct *t)
  139. {
  140. struct fsl_re_chan *re_chan = from_tasklet(re_chan, t, irqtask);
  141. struct fsl_re_desc *desc, *_desc;
  142. struct fsl_re_hw_desc *hwdesc;
  143. unsigned long flags;
  144. unsigned int count, oub_count;
  145. int found;
  146. fsl_re_cleanup_descs(re_chan);
  147. spin_lock_irqsave(&re_chan->desc_lock, flags);
  148. count = FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full));
  149. while (count--) {
  150. found = 0;
  151. hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count];
  152. list_for_each_entry_safe(desc, _desc, &re_chan->active_q,
  153. node) {
  154. /* compare the hw dma addr to find the completed */
  155. if (desc->hwdesc.lbea32 == hwdesc->lbea32 &&
  156. desc->hwdesc.addr_low == hwdesc->addr_low) {
  157. found = 1;
  158. break;
  159. }
  160. }
  161. if (found) {
  162. fsl_re_desc_done(desc);
  163. list_move_tail(&desc->node, &re_chan->ack_q);
  164. } else {
  165. dev_err(re_chan->dev,
  166. "found hwdesc not in sw queue, discard it\n");
  167. }
  168. oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK;
  169. re_chan->oub_count = oub_count;
  170. out_be32(&re_chan->jrregs->oubring_job_rmvd,
  171. FSL_RE_RMVD_JOB(1));
  172. }
  173. spin_unlock_irqrestore(&re_chan->desc_lock, flags);
  174. }
  175. /* Per Job Ring interrupt handler */
  176. static irqreturn_t fsl_re_isr(int irq, void *data)
  177. {
  178. struct fsl_re_chan *re_chan;
  179. u32 irqstate, status;
  180. re_chan = dev_get_drvdata((struct device *)data);
  181. irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status);
  182. if (!irqstate)
  183. return IRQ_NONE;
  184. /*
  185. * There's no way in upper layer (read MD layer) to recover from
  186. * error conditions except restart everything. In long term we
  187. * need to do something more than just crashing
  188. */
  189. if (irqstate & FSL_RE_ERROR) {
  190. status = in_be32(&re_chan->jrregs->jr_status);
  191. dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n",
  192. irqstate, status);
  193. }
  194. /* Clear interrupt */
  195. out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR);
  196. tasklet_schedule(&re_chan->irqtask);
  197. return IRQ_HANDLED;
  198. }
  199. static enum dma_status fsl_re_tx_status(struct dma_chan *chan,
  200. dma_cookie_t cookie,
  201. struct dma_tx_state *txstate)
  202. {
  203. return dma_cookie_status(chan, cookie, txstate);
  204. }
  205. static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index,
  206. size_t length, dma_addr_t addr, bool final)
  207. {
  208. u32 efrl = length & FSL_RE_CF_LENGTH_MASK;
  209. efrl |= final << FSL_RE_CF_FINAL_SHIFT;
  210. cf[index].efrl32 = efrl;
  211. cf[index].addr_high = upper_32_bits(addr);
  212. cf[index].addr_low = lower_32_bits(addr);
  213. }
  214. static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,
  215. struct fsl_re_desc *desc,
  216. void *cf, dma_addr_t paddr)
  217. {
  218. desc->re_chan = re_chan;
  219. desc->async_tx.tx_submit = fsl_re_tx_submit;
  220. dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);
  221. INIT_LIST_HEAD(&desc->node);
  222. desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT;
  223. desc->hwdesc.lbea32 = upper_32_bits(paddr);
  224. desc->hwdesc.addr_low = lower_32_bits(paddr);
  225. desc->cf_addr = cf;
  226. desc->cf_paddr = paddr;
  227. desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE);
  228. desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE;
  229. return desc;
  230. }
  231. static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan,
  232. unsigned long flags)
  233. {
  234. struct fsl_re_desc *desc = NULL;
  235. void *cf;
  236. dma_addr_t paddr;
  237. unsigned long lock_flag;
  238. fsl_re_cleanup_descs(re_chan);
  239. spin_lock_irqsave(&re_chan->desc_lock, lock_flag);
  240. if (!list_empty(&re_chan->free_q)) {
  241. /* take one desc from free_q */
  242. desc = list_first_entry(&re_chan->free_q,
  243. struct fsl_re_desc, node);
  244. list_del(&desc->node);
  245. desc->async_tx.flags = flags;
  246. }
  247. spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);
  248. if (!desc) {
  249. desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
  250. if (!desc)
  251. return NULL;
  252. cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,
  253. &paddr);
  254. if (!cf) {
  255. kfree(desc);
  256. return NULL;
  257. }
  258. desc = fsl_re_init_desc(re_chan, desc, cf, paddr);
  259. desc->async_tx.flags = flags;
  260. spin_lock_irqsave(&re_chan->desc_lock, lock_flag);
  261. re_chan->alloc_count++;
  262. spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);
  263. }
  264. return desc;
  265. }
  266. static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq(
  267. struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
  268. unsigned int src_cnt, const unsigned char *scf, size_t len,
  269. unsigned long flags)
  270. {
  271. struct fsl_re_chan *re_chan;
  272. struct fsl_re_desc *desc;
  273. struct fsl_re_xor_cdb *xor;
  274. struct fsl_re_cmpnd_frame *cf;
  275. u32 cdb;
  276. unsigned int i, j;
  277. unsigned int save_src_cnt = src_cnt;
  278. int cont_q = 0;
  279. re_chan = container_of(chan, struct fsl_re_chan, chan);
  280. if (len > FSL_RE_MAX_DATA_LEN) {
  281. dev_err(re_chan->dev, "genq tx length %zu, max length %d\n",
  282. len, FSL_RE_MAX_DATA_LEN);
  283. return NULL;
  284. }
  285. desc = fsl_re_chan_alloc_desc(re_chan, flags);
  286. if (desc <= 0)
  287. return NULL;
  288. if (scf && (flags & DMA_PREP_CONTINUE)) {
  289. cont_q = 1;
  290. src_cnt += 1;
  291. }
  292. /* Filling xor CDB */
  293. cdb = FSL_RE_XOR_OPCODE << FSL_RE_CDB_OPCODE_SHIFT;
  294. cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT;
  295. cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT;
  296. cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT;
  297. cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;
  298. xor = desc->cdb_addr;
  299. xor->cdb32 = cdb;
  300. if (scf) {
  301. /* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */
  302. for (i = 0; i < save_src_cnt; i++)
  303. xor->gfm[i] = scf[i];
  304. if (cont_q)
  305. xor->gfm[i++] = 1;
  306. } else {
  307. /* compute P, that is XOR all srcs */
  308. for (i = 0; i < src_cnt; i++)
  309. xor->gfm[i] = 1;
  310. }
  311. /* Filling frame 0 of compound frame descriptor with CDB */
  312. cf = desc->cf_addr;
  313. fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0);
  314. /* Fill CFD's 1st frame with dest buffer */
  315. fill_cfd_frame(cf, 1, len, dest, 0);
  316. /* Fill CFD's rest of the frames with source buffers */
  317. for (i = 2, j = 0; j < save_src_cnt; i++, j++)
  318. fill_cfd_frame(cf, i, len, src[j], 0);
  319. if (cont_q)
  320. fill_cfd_frame(cf, i++, len, dest, 0);
  321. /* Setting the final bit in the last source buffer frame in CFD */
  322. cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;
  323. return &desc->async_tx;
  324. }
  325. /*
  326. * Prep function for P parity calculation.In RAID Engine terminology,
  327. * XOR calculation is called GenQ calculation done through GenQ command
  328. */
  329. static struct dma_async_tx_descriptor *fsl_re_prep_dma_xor(
  330. struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
  331. unsigned int src_cnt, size_t len, unsigned long flags)
  332. {
  333. /* NULL let genq take all coef as 1 */
  334. return fsl_re_prep_dma_genq(chan, dest, src, src_cnt, NULL, len, flags);
  335. }
  336. /*
  337. * Prep function for P/Q parity calculation.In RAID Engine terminology,
  338. * P/Q calculation is called GenQQ done through GenQQ command
  339. */
  340. static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq(
  341. struct dma_chan *chan, dma_addr_t *dest, dma_addr_t *src,
  342. unsigned int src_cnt, const unsigned char *scf, size_t len,
  343. unsigned long flags)
  344. {
  345. struct fsl_re_chan *re_chan;
  346. struct fsl_re_desc *desc;
  347. struct fsl_re_pq_cdb *pq;
  348. struct fsl_re_cmpnd_frame *cf;
  349. u32 cdb;
  350. u8 *p;
  351. int gfmq_len, i, j;
  352. unsigned int save_src_cnt = src_cnt;
  353. re_chan = container_of(chan, struct fsl_re_chan, chan);
  354. if (len > FSL_RE_MAX_DATA_LEN) {
  355. dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n",
  356. len, FSL_RE_MAX_DATA_LEN);
  357. return NULL;
  358. }
  359. /*
  360. * RE requires at least 2 sources, if given only one source, we pass the
  361. * second source same as the first one.
  362. * With only one source, generating P is meaningless, only generate Q.
  363. */
  364. if (src_cnt == 1) {
  365. struct dma_async_tx_descriptor *tx;
  366. dma_addr_t dma_src[2];
  367. unsigned char coef[2];
  368. dma_src[0] = *src;
  369. coef[0] = *scf;
  370. dma_src[1] = *src;
  371. coef[1] = 0;
  372. tx = fsl_re_prep_dma_genq(chan, dest[1], dma_src, 2, coef, len,
  373. flags);
  374. if (tx)
  375. desc = to_fsl_re_dma_desc(tx);
  376. return tx;
  377. }
  378. /*
  379. * During RAID6 array creation, Linux's MD layer gets P and Q
  380. * calculated separately in two steps. But our RAID Engine has
  381. * the capability to calculate both P and Q with a single command
  382. * Hence to merge well with MD layer, we need to provide a hook
  383. * here and call re_jq_prep_dma_genq() function
  384. */
  385. if (flags & DMA_PREP_PQ_DISABLE_P)
  386. return fsl_re_prep_dma_genq(chan, dest[1], src, src_cnt,
  387. scf, len, flags);
  388. if (flags & DMA_PREP_CONTINUE)
  389. src_cnt += 3;
  390. desc = fsl_re_chan_alloc_desc(re_chan, flags);
  391. if (desc <= 0)
  392. return NULL;
  393. /* Filling GenQQ CDB */
  394. cdb = FSL_RE_PQ_OPCODE << FSL_RE_CDB_OPCODE_SHIFT;
  395. cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT;
  396. cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT;
  397. cdb |= FSL_RE_BUFFER_OUTPUT << FSL_RE_CDB_BUFFER_SHIFT;
  398. cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;
  399. pq = desc->cdb_addr;
  400. pq->cdb32 = cdb;
  401. p = pq->gfm_q1;
  402. /* Init gfm_q1[] */
  403. for (i = 0; i < src_cnt; i++)
  404. p[i] = 1;
  405. /* Align gfm[] to 32bit */
  406. gfmq_len = ALIGN(src_cnt, 4);
  407. /* Init gfm_q2[] */
  408. p += gfmq_len;
  409. for (i = 0; i < src_cnt; i++)
  410. p[i] = scf[i];
  411. /* Filling frame 0 of compound frame descriptor with CDB */
  412. cf = desc->cf_addr;
  413. fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0);
  414. /* Fill CFD's 1st & 2nd frame with dest buffers */
  415. for (i = 1, j = 0; i < 3; i++, j++)
  416. fill_cfd_frame(cf, i, len, dest[j], 0);
  417. /* Fill CFD's rest of the frames with source buffers */
  418. for (i = 3, j = 0; j < save_src_cnt; i++, j++)
  419. fill_cfd_frame(cf, i, len, src[j], 0);
  420. /* PQ computation continuation */
  421. if (flags & DMA_PREP_CONTINUE) {
  422. if (src_cnt - save_src_cnt == 3) {
  423. p[save_src_cnt] = 0;
  424. p[save_src_cnt + 1] = 0;
  425. p[save_src_cnt + 2] = 1;
  426. fill_cfd_frame(cf, i++, len, dest[0], 0);
  427. fill_cfd_frame(cf, i++, len, dest[1], 0);
  428. fill_cfd_frame(cf, i++, len, dest[1], 0);
  429. } else {
  430. dev_err(re_chan->dev, "PQ tx continuation error!\n");
  431. return NULL;
  432. }
  433. }
  434. /* Setting the final bit in the last source buffer frame in CFD */
  435. cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;
  436. return &desc->async_tx;
  437. }
  438. /*
  439. * Prep function for memcpy. In RAID Engine, memcpy is done through MOVE
  440. * command. Logic of this function will need to be modified once multipage
  441. * support is added in Linux's MD/ASYNC Layer
  442. */
  443. static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy(
  444. struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  445. size_t len, unsigned long flags)
  446. {
  447. struct fsl_re_chan *re_chan;
  448. struct fsl_re_desc *desc;
  449. size_t length;
  450. struct fsl_re_cmpnd_frame *cf;
  451. struct fsl_re_move_cdb *move;
  452. u32 cdb;
  453. re_chan = container_of(chan, struct fsl_re_chan, chan);
  454. if (len > FSL_RE_MAX_DATA_LEN) {
  455. dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n",
  456. len, FSL_RE_MAX_DATA_LEN);
  457. return NULL;
  458. }
  459. desc = fsl_re_chan_alloc_desc(re_chan, flags);
  460. if (desc <= 0)
  461. return NULL;
  462. /* Filling move CDB */
  463. cdb = FSL_RE_MOVE_OPCODE << FSL_RE_CDB_OPCODE_SHIFT;
  464. cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT;
  465. cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT;
  466. cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;
  467. move = desc->cdb_addr;
  468. move->cdb32 = cdb;
  469. /* Filling frame 0 of CFD with move CDB */
  470. cf = desc->cf_addr;
  471. fill_cfd_frame(cf, 0, sizeof(*move), desc->cdb_paddr, 0);
  472. length = min_t(size_t, len, FSL_RE_MAX_DATA_LEN);
  473. /* Fill CFD's 1st frame with dest buffer */
  474. fill_cfd_frame(cf, 1, length, dest, 0);
  475. /* Fill CFD's 2nd frame with src buffer */
  476. fill_cfd_frame(cf, 2, length, src, 1);
  477. return &desc->async_tx;
  478. }
  479. static int fsl_re_alloc_chan_resources(struct dma_chan *chan)
  480. {
  481. struct fsl_re_chan *re_chan;
  482. struct fsl_re_desc *desc;
  483. void *cf;
  484. dma_addr_t paddr;
  485. int i;
  486. re_chan = container_of(chan, struct fsl_re_chan, chan);
  487. for (i = 0; i < FSL_RE_MIN_DESCS; i++) {
  488. desc = kzalloc(sizeof(*desc), GFP_KERNEL);
  489. if (!desc)
  490. break;
  491. cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL,
  492. &paddr);
  493. if (!cf) {
  494. kfree(desc);
  495. break;
  496. }
  497. INIT_LIST_HEAD(&desc->node);
  498. fsl_re_init_desc(re_chan, desc, cf, paddr);
  499. list_add_tail(&desc->node, &re_chan->free_q);
  500. re_chan->alloc_count++;
  501. }
  502. return re_chan->alloc_count;
  503. }
  504. static void fsl_re_free_chan_resources(struct dma_chan *chan)
  505. {
  506. struct fsl_re_chan *re_chan;
  507. struct fsl_re_desc *desc;
  508. re_chan = container_of(chan, struct fsl_re_chan, chan);
  509. while (re_chan->alloc_count--) {
  510. desc = list_first_entry(&re_chan->free_q,
  511. struct fsl_re_desc,
  512. node);
  513. list_del(&desc->node);
  514. dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr,
  515. desc->cf_paddr);
  516. kfree(desc);
  517. }
  518. if (!list_empty(&re_chan->free_q))
  519. dev_err(re_chan->dev, "chan resource cannot be cleaned!\n");
  520. }
  521. static int fsl_re_chan_probe(struct platform_device *ofdev,
  522. struct device_node *np, u8 q, u32 off)
  523. {
  524. struct device *dev, *chandev;
  525. struct fsl_re_drv_private *re_priv;
  526. struct fsl_re_chan *chan;
  527. struct dma_device *dma_dev;
  528. u32 ptr;
  529. u32 status;
  530. int ret = 0, rc;
  531. struct platform_device *chan_ofdev;
  532. dev = &ofdev->dev;
  533. re_priv = dev_get_drvdata(dev);
  534. dma_dev = &re_priv->dma_dev;
  535. chan = devm_kzalloc(dev, sizeof(*chan), GFP_KERNEL);
  536. if (!chan)
  537. return -ENOMEM;
  538. /* create platform device for chan node */
  539. chan_ofdev = of_platform_device_create(np, NULL, dev);
  540. if (!chan_ofdev) {
  541. dev_err(dev, "Not able to create ofdev for jr %d\n", q);
  542. ret = -EINVAL;
  543. goto err_free;
  544. }
  545. /* read reg property from dts */
  546. rc = of_property_read_u32(np, "reg", &ptr);
  547. if (rc) {
  548. dev_err(dev, "Reg property not found in jr %d\n", q);
  549. ret = -ENODEV;
  550. goto err_free;
  551. }
  552. chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs +
  553. off + ptr);
  554. /* read irq property from dts */
  555. chan->irq = irq_of_parse_and_map(np, 0);
  556. if (!chan->irq) {
  557. dev_err(dev, "No IRQ defined for JR %d\n", q);
  558. ret = -ENODEV;
  559. goto err_free;
  560. }
  561. snprintf(chan->name, sizeof(chan->name), "re_jr%02d", q);
  562. chandev = &chan_ofdev->dev;
  563. tasklet_setup(&chan->irqtask, fsl_re_dequeue);
  564. ret = request_irq(chan->irq, fsl_re_isr, 0, chan->name, chandev);
  565. if (ret) {
  566. dev_err(dev, "Unable to register interrupt for JR %d\n", q);
  567. ret = -EINVAL;
  568. goto err_free;
  569. }
  570. re_priv->re_jrs[q] = chan;
  571. chan->chan.device = dma_dev;
  572. chan->chan.private = chan;
  573. chan->dev = chandev;
  574. chan->re_dev = re_priv;
  575. spin_lock_init(&chan->desc_lock);
  576. INIT_LIST_HEAD(&chan->ack_q);
  577. INIT_LIST_HEAD(&chan->active_q);
  578. INIT_LIST_HEAD(&chan->submit_q);
  579. INIT_LIST_HEAD(&chan->free_q);
  580. chan->inb_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool,
  581. GFP_KERNEL, &chan->inb_phys_addr);
  582. if (!chan->inb_ring_virt_addr) {
  583. dev_err(dev, "No dma memory for inb_ring_virt_addr\n");
  584. ret = -ENOMEM;
  585. goto err_free;
  586. }
  587. chan->oub_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool,
  588. GFP_KERNEL, &chan->oub_phys_addr);
  589. if (!chan->oub_ring_virt_addr) {
  590. dev_err(dev, "No dma memory for oub_ring_virt_addr\n");
  591. ret = -ENOMEM;
  592. goto err_free_1;
  593. }
  594. /* Program the Inbound/Outbound ring base addresses and size */
  595. out_be32(&chan->jrregs->inbring_base_h,
  596. chan->inb_phys_addr & FSL_RE_ADDR_BIT_MASK);
  597. out_be32(&chan->jrregs->oubring_base_h,
  598. chan->oub_phys_addr & FSL_RE_ADDR_BIT_MASK);
  599. out_be32(&chan->jrregs->inbring_base_l,
  600. chan->inb_phys_addr >> FSL_RE_ADDR_BIT_SHIFT);
  601. out_be32(&chan->jrregs->oubring_base_l,
  602. chan->oub_phys_addr >> FSL_RE_ADDR_BIT_SHIFT);
  603. out_be32(&chan->jrregs->inbring_size,
  604. FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT);
  605. out_be32(&chan->jrregs->oubring_size,
  606. FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT);
  607. /* Read LIODN value from u-boot */
  608. status = in_be32(&chan->jrregs->jr_config_1) & FSL_RE_REG_LIODN_MASK;
  609. /* Program the CFG reg */
  610. out_be32(&chan->jrregs->jr_config_1,
  611. FSL_RE_CFG1_CBSI | FSL_RE_CFG1_CBS0 | status);
  612. dev_set_drvdata(chandev, chan);
  613. /* Enable RE/CHAN */
  614. out_be32(&chan->jrregs->jr_command, FSL_RE_ENABLE);
  615. return 0;
  616. err_free_1:
  617. dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr,
  618. chan->inb_phys_addr);
  619. err_free:
  620. return ret;
  621. }
  622. /* Probe function for RAID Engine */
  623. static int fsl_re_probe(struct platform_device *ofdev)
  624. {
  625. struct fsl_re_drv_private *re_priv;
  626. struct device_node *np;
  627. struct device_node *child;
  628. u32 off;
  629. u8 ridx = 0;
  630. struct dma_device *dma_dev;
  631. struct resource *res;
  632. int rc;
  633. struct device *dev = &ofdev->dev;
  634. re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL);
  635. if (!re_priv)
  636. return -ENOMEM;
  637. res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
  638. if (!res)
  639. return -ENODEV;
  640. /* IOMAP the entire RAID Engine region */
  641. re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res));
  642. if (!re_priv->re_regs)
  643. return -EBUSY;
  644. /* Program the RE mode */
  645. out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE);
  646. /* Program Galois Field polynomial */
  647. out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY);
  648. dev_info(dev, "version %x, mode %x, gfp %x\n",
  649. in_be32(&re_priv->re_regs->re_version_id),
  650. in_be32(&re_priv->re_regs->global_config),
  651. in_be32(&re_priv->re_regs->galois_field_config));
  652. dma_dev = &re_priv->dma_dev;
  653. dma_dev->dev = dev;
  654. INIT_LIST_HEAD(&dma_dev->channels);
  655. dma_set_mask(dev, DMA_BIT_MASK(40));
  656. dma_dev->device_alloc_chan_resources = fsl_re_alloc_chan_resources;
  657. dma_dev->device_tx_status = fsl_re_tx_status;
  658. dma_dev->device_issue_pending = fsl_re_issue_pending;
  659. dma_dev->max_xor = FSL_RE_MAX_XOR_SRCS;
  660. dma_dev->device_prep_dma_xor = fsl_re_prep_dma_xor;
  661. dma_cap_set(DMA_XOR, dma_dev->cap_mask);
  662. dma_dev->max_pq = FSL_RE_MAX_PQ_SRCS;
  663. dma_dev->device_prep_dma_pq = fsl_re_prep_dma_pq;
  664. dma_cap_set(DMA_PQ, dma_dev->cap_mask);
  665. dma_dev->device_prep_dma_memcpy = fsl_re_prep_dma_memcpy;
  666. dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
  667. dma_dev->device_free_chan_resources = fsl_re_free_chan_resources;
  668. re_priv->total_chans = 0;
  669. re_priv->cf_desc_pool = dmam_pool_create("fsl_re_cf_desc_pool", dev,
  670. FSL_RE_CF_CDB_SIZE,
  671. FSL_RE_CF_CDB_ALIGN, 0);
  672. if (!re_priv->cf_desc_pool) {
  673. dev_err(dev, "No memory for fsl re_cf desc pool\n");
  674. return -ENOMEM;
  675. }
  676. re_priv->hw_desc_pool = dmam_pool_create("fsl_re_hw_desc_pool", dev,
  677. sizeof(struct fsl_re_hw_desc) * FSL_RE_RING_SIZE,
  678. FSL_RE_FRAME_ALIGN, 0);
  679. if (!re_priv->hw_desc_pool) {
  680. dev_err(dev, "No memory for fsl re_hw desc pool\n");
  681. return -ENOMEM;
  682. }
  683. dev_set_drvdata(dev, re_priv);
  684. /* Parse Device tree to find out the total number of JQs present */
  685. for_each_compatible_node(np, NULL, "fsl,raideng-v1.0-job-queue") {
  686. rc = of_property_read_u32(np, "reg", &off);
  687. if (rc) {
  688. dev_err(dev, "Reg property not found in JQ node\n");
  689. of_node_put(np);
  690. return -ENODEV;
  691. }
  692. /* Find out the Job Rings present under each JQ */
  693. for_each_child_of_node(np, child) {
  694. rc = of_device_is_compatible(child,
  695. "fsl,raideng-v1.0-job-ring");
  696. if (rc) {
  697. fsl_re_chan_probe(ofdev, child, ridx++, off);
  698. re_priv->total_chans++;
  699. }
  700. }
  701. }
  702. dma_async_device_register(dma_dev);
  703. return 0;
  704. }
  705. static void fsl_re_remove_chan(struct fsl_re_chan *chan)
  706. {
  707. tasklet_kill(&chan->irqtask);
  708. dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr,
  709. chan->inb_phys_addr);
  710. dma_pool_free(chan->re_dev->hw_desc_pool, chan->oub_ring_virt_addr,
  711. chan->oub_phys_addr);
  712. }
  713. static int fsl_re_remove(struct platform_device *ofdev)
  714. {
  715. struct fsl_re_drv_private *re_priv;
  716. struct device *dev;
  717. int i;
  718. dev = &ofdev->dev;
  719. re_priv = dev_get_drvdata(dev);
  720. /* Cleanup chan related memory areas */
  721. for (i = 0; i < re_priv->total_chans; i++)
  722. fsl_re_remove_chan(re_priv->re_jrs[i]);
  723. /* Unregister the driver */
  724. dma_async_device_unregister(&re_priv->dma_dev);
  725. return 0;
  726. }
  727. static const struct of_device_id fsl_re_ids[] = {
  728. { .compatible = "fsl,raideng-v1.0", },
  729. {}
  730. };
  731. MODULE_DEVICE_TABLE(of, fsl_re_ids);
  732. static struct platform_driver fsl_re_driver = {
  733. .driver = {
  734. .name = "fsl-raideng",
  735. .of_match_table = fsl_re_ids,
  736. },
  737. .probe = fsl_re_probe,
  738. .remove = fsl_re_remove,
  739. };
  740. module_platform_driver(fsl_re_driver);
  741. MODULE_AUTHOR("Harninder Rai <harninder.rai@freescale.com>");
  742. MODULE_LICENSE("GPL v2");
  743. MODULE_DESCRIPTION("Freescale RAID Engine Device Driver");