ep93xx_dma.c 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Driver for the Cirrus Logic EP93xx DMA Controller
  4. *
  5. * Copyright (C) 2011 Mika Westerberg
  6. *
  7. * DMA M2P implementation is based on the original
  8. * arch/arm/mach-ep93xx/dma-m2p.c which has following copyrights:
  9. *
  10. * Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org>
  11. * Copyright (C) 2006 Applied Data Systems
  12. * Copyright (C) 2009 Ryan Mallon <rmallon@gmail.com>
  13. *
  14. * This driver is based on dw_dmac and amba-pl08x drivers.
  15. */
  16. #include <linux/clk.h>
  17. #include <linux/init.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/dmaengine.h>
  20. #include <linux/module.h>
  21. #include <linux/mod_devicetable.h>
  22. #include <linux/platform_device.h>
  23. #include <linux/slab.h>
  24. #include <linux/platform_data/dma-ep93xx.h>
  25. #include "dmaengine.h"
  26. /* M2P registers */
  27. #define M2P_CONTROL 0x0000
  28. #define M2P_CONTROL_STALLINT BIT(0)
  29. #define M2P_CONTROL_NFBINT BIT(1)
  30. #define M2P_CONTROL_CH_ERROR_INT BIT(3)
  31. #define M2P_CONTROL_ENABLE BIT(4)
  32. #define M2P_CONTROL_ICE BIT(6)
  33. #define M2P_INTERRUPT 0x0004
  34. #define M2P_INTERRUPT_STALL BIT(0)
  35. #define M2P_INTERRUPT_NFB BIT(1)
  36. #define M2P_INTERRUPT_ERROR BIT(3)
  37. #define M2P_PPALLOC 0x0008
  38. #define M2P_STATUS 0x000c
  39. #define M2P_MAXCNT0 0x0020
  40. #define M2P_BASE0 0x0024
  41. #define M2P_MAXCNT1 0x0030
  42. #define M2P_BASE1 0x0034
  43. #define M2P_STATE_IDLE 0
  44. #define M2P_STATE_STALL 1
  45. #define M2P_STATE_ON 2
  46. #define M2P_STATE_NEXT 3
  47. /* M2M registers */
  48. #define M2M_CONTROL 0x0000
  49. #define M2M_CONTROL_DONEINT BIT(2)
  50. #define M2M_CONTROL_ENABLE BIT(3)
  51. #define M2M_CONTROL_START BIT(4)
  52. #define M2M_CONTROL_DAH BIT(11)
  53. #define M2M_CONTROL_SAH BIT(12)
  54. #define M2M_CONTROL_PW_SHIFT 9
  55. #define M2M_CONTROL_PW_8 (0 << M2M_CONTROL_PW_SHIFT)
  56. #define M2M_CONTROL_PW_16 (1 << M2M_CONTROL_PW_SHIFT)
  57. #define M2M_CONTROL_PW_32 (2 << M2M_CONTROL_PW_SHIFT)
  58. #define M2M_CONTROL_PW_MASK (3 << M2M_CONTROL_PW_SHIFT)
  59. #define M2M_CONTROL_TM_SHIFT 13
  60. #define M2M_CONTROL_TM_TX (1 << M2M_CONTROL_TM_SHIFT)
  61. #define M2M_CONTROL_TM_RX (2 << M2M_CONTROL_TM_SHIFT)
  62. #define M2M_CONTROL_NFBINT BIT(21)
  63. #define M2M_CONTROL_RSS_SHIFT 22
  64. #define M2M_CONTROL_RSS_SSPRX (1 << M2M_CONTROL_RSS_SHIFT)
  65. #define M2M_CONTROL_RSS_SSPTX (2 << M2M_CONTROL_RSS_SHIFT)
  66. #define M2M_CONTROL_RSS_IDE (3 << M2M_CONTROL_RSS_SHIFT)
  67. #define M2M_CONTROL_NO_HDSK BIT(24)
  68. #define M2M_CONTROL_PWSC_SHIFT 25
  69. #define M2M_INTERRUPT 0x0004
  70. #define M2M_INTERRUPT_MASK 6
  71. #define M2M_STATUS 0x000c
  72. #define M2M_STATUS_CTL_SHIFT 1
  73. #define M2M_STATUS_CTL_IDLE (0 << M2M_STATUS_CTL_SHIFT)
  74. #define M2M_STATUS_CTL_STALL (1 << M2M_STATUS_CTL_SHIFT)
  75. #define M2M_STATUS_CTL_MEMRD (2 << M2M_STATUS_CTL_SHIFT)
  76. #define M2M_STATUS_CTL_MEMWR (3 << M2M_STATUS_CTL_SHIFT)
  77. #define M2M_STATUS_CTL_BWCWAIT (4 << M2M_STATUS_CTL_SHIFT)
  78. #define M2M_STATUS_CTL_MASK (7 << M2M_STATUS_CTL_SHIFT)
  79. #define M2M_STATUS_BUF_SHIFT 4
  80. #define M2M_STATUS_BUF_NO (0 << M2M_STATUS_BUF_SHIFT)
  81. #define M2M_STATUS_BUF_ON (1 << M2M_STATUS_BUF_SHIFT)
  82. #define M2M_STATUS_BUF_NEXT (2 << M2M_STATUS_BUF_SHIFT)
  83. #define M2M_STATUS_BUF_MASK (3 << M2M_STATUS_BUF_SHIFT)
  84. #define M2M_STATUS_DONE BIT(6)
  85. #define M2M_BCR0 0x0010
  86. #define M2M_BCR1 0x0014
  87. #define M2M_SAR_BASE0 0x0018
  88. #define M2M_SAR_BASE1 0x001c
  89. #define M2M_DAR_BASE0 0x002c
  90. #define M2M_DAR_BASE1 0x0030
  91. #define DMA_MAX_CHAN_BYTES 0xffff
  92. #define DMA_MAX_CHAN_DESCRIPTORS 32
  93. struct ep93xx_dma_engine;
  94. static int ep93xx_dma_slave_config_write(struct dma_chan *chan,
  95. enum dma_transfer_direction dir,
  96. struct dma_slave_config *config);
  97. /**
  98. * struct ep93xx_dma_desc - EP93xx specific transaction descriptor
  99. * @src_addr: source address of the transaction
  100. * @dst_addr: destination address of the transaction
  101. * @size: size of the transaction (in bytes)
  102. * @complete: this descriptor is completed
  103. * @txd: dmaengine API descriptor
  104. * @tx_list: list of linked descriptors
  105. * @node: link used for putting this into a channel queue
  106. */
  107. struct ep93xx_dma_desc {
  108. u32 src_addr;
  109. u32 dst_addr;
  110. size_t size;
  111. bool complete;
  112. struct dma_async_tx_descriptor txd;
  113. struct list_head tx_list;
  114. struct list_head node;
  115. };
  116. /**
  117. * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel
  118. * @chan: dmaengine API channel
  119. * @edma: pointer to to the engine device
  120. * @regs: memory mapped registers
  121. * @irq: interrupt number of the channel
  122. * @clk: clock used by this channel
  123. * @tasklet: channel specific tasklet used for callbacks
  124. * @lock: lock protecting the fields following
  125. * @flags: flags for the channel
  126. * @buffer: which buffer to use next (0/1)
  127. * @active: flattened chain of descriptors currently being processed
  128. * @queue: pending descriptors which are handled next
  129. * @free_list: list of free descriptors which can be used
  130. * @runtime_addr: physical address currently used as dest/src (M2M only). This
  131. * is set via .device_config before slave operation is
  132. * prepared
  133. * @runtime_ctrl: M2M runtime values for the control register.
  134. * @slave_config: slave configuration
  135. *
  136. * As EP93xx DMA controller doesn't support real chained DMA descriptors we
  137. * will have slightly different scheme here: @active points to a head of
  138. * flattened DMA descriptor chain.
  139. *
  140. * @queue holds pending transactions. These are linked through the first
  141. * descriptor in the chain. When a descriptor is moved to the @active queue,
  142. * the first and chained descriptors are flattened into a single list.
  143. *
  144. * @chan.private holds pointer to &struct ep93xx_dma_data which contains
  145. * necessary channel configuration information. For memcpy channels this must
  146. * be %NULL.
  147. */
  148. struct ep93xx_dma_chan {
  149. struct dma_chan chan;
  150. const struct ep93xx_dma_engine *edma;
  151. void __iomem *regs;
  152. int irq;
  153. struct clk *clk;
  154. struct tasklet_struct tasklet;
  155. /* protects the fields following */
  156. spinlock_t lock;
  157. unsigned long flags;
  158. /* Channel is configured for cyclic transfers */
  159. #define EP93XX_DMA_IS_CYCLIC 0
  160. int buffer;
  161. struct list_head active;
  162. struct list_head queue;
  163. struct list_head free_list;
  164. u32 runtime_addr;
  165. u32 runtime_ctrl;
  166. struct dma_slave_config slave_config;
  167. };
  168. /**
  169. * struct ep93xx_dma_engine - the EP93xx DMA engine instance
  170. * @dma_dev: holds the dmaengine device
  171. * @m2m: is this an M2M or M2P device
  172. * @hw_setup: method which sets the channel up for operation
  173. * @hw_synchronize: synchronizes DMA channel termination to current context
  174. * @hw_shutdown: shuts the channel down and flushes whatever is left
  175. * @hw_submit: pushes active descriptor(s) to the hardware
  176. * @hw_interrupt: handle the interrupt
  177. * @num_channels: number of channels for this instance
  178. * @channels: array of channels
  179. *
  180. * There is one instance of this struct for the M2P channels and one for the
  181. * M2M channels. hw_xxx() methods are used to perform operations which are
  182. * different on M2M and M2P channels. These methods are called with channel
  183. * lock held and interrupts disabled so they cannot sleep.
  184. */
  185. struct ep93xx_dma_engine {
  186. struct dma_device dma_dev;
  187. bool m2m;
  188. int (*hw_setup)(struct ep93xx_dma_chan *);
  189. void (*hw_synchronize)(struct ep93xx_dma_chan *);
  190. void (*hw_shutdown)(struct ep93xx_dma_chan *);
  191. void (*hw_submit)(struct ep93xx_dma_chan *);
  192. int (*hw_interrupt)(struct ep93xx_dma_chan *);
  193. #define INTERRUPT_UNKNOWN 0
  194. #define INTERRUPT_DONE 1
  195. #define INTERRUPT_NEXT_BUFFER 2
  196. size_t num_channels;
  197. struct ep93xx_dma_chan channels[];
  198. };
  199. static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)
  200. {
  201. return &edmac->chan.dev->device;
  202. }
  203. static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan)
  204. {
  205. return container_of(chan, struct ep93xx_dma_chan, chan);
  206. }
  207. /**
  208. * ep93xx_dma_set_active - set new active descriptor chain
  209. * @edmac: channel
  210. * @desc: head of the new active descriptor chain
  211. *
  212. * Sets @desc to be the head of the new active descriptor chain. This is the
  213. * chain which is processed next. The active list must be empty before calling
  214. * this function.
  215. *
  216. * Called with @edmac->lock held and interrupts disabled.
  217. */
  218. static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,
  219. struct ep93xx_dma_desc *desc)
  220. {
  221. BUG_ON(!list_empty(&edmac->active));
  222. list_add_tail(&desc->node, &edmac->active);
  223. /* Flatten the @desc->tx_list chain into @edmac->active list */
  224. while (!list_empty(&desc->tx_list)) {
  225. struct ep93xx_dma_desc *d = list_first_entry(&desc->tx_list,
  226. struct ep93xx_dma_desc, node);
  227. /*
  228. * We copy the callback parameters from the first descriptor
  229. * to all the chained descriptors. This way we can call the
  230. * callback without having to find out the first descriptor in
  231. * the chain. Useful for cyclic transfers.
  232. */
  233. d->txd.callback = desc->txd.callback;
  234. d->txd.callback_param = desc->txd.callback_param;
  235. list_move_tail(&d->node, &edmac->active);
  236. }
  237. }
  238. /* Called with @edmac->lock held and interrupts disabled */
  239. static struct ep93xx_dma_desc *
  240. ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)
  241. {
  242. return list_first_entry_or_null(&edmac->active,
  243. struct ep93xx_dma_desc, node);
  244. }
  245. /**
  246. * ep93xx_dma_advance_active - advances to the next active descriptor
  247. * @edmac: channel
  248. *
  249. * Function advances active descriptor to the next in the @edmac->active and
  250. * returns %true if we still have descriptors in the chain to process.
  251. * Otherwise returns %false.
  252. *
  253. * When the channel is in cyclic mode always returns %true.
  254. *
  255. * Called with @edmac->lock held and interrupts disabled.
  256. */
  257. static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)
  258. {
  259. struct ep93xx_dma_desc *desc;
  260. list_rotate_left(&edmac->active);
  261. if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
  262. return true;
  263. desc = ep93xx_dma_get_active(edmac);
  264. if (!desc)
  265. return false;
  266. /*
  267. * If txd.cookie is set it means that we are back in the first
  268. * descriptor in the chain and hence done with it.
  269. */
  270. return !desc->txd.cookie;
  271. }
  272. /*
  273. * M2P DMA implementation
  274. */
  275. static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)
  276. {
  277. writel(control, edmac->regs + M2P_CONTROL);
  278. /*
  279. * EP93xx User's Guide states that we must perform a dummy read after
  280. * write to the control register.
  281. */
  282. readl(edmac->regs + M2P_CONTROL);
  283. }
  284. static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)
  285. {
  286. struct ep93xx_dma_data *data = edmac->chan.private;
  287. u32 control;
  288. writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);
  289. control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE
  290. | M2P_CONTROL_ENABLE;
  291. m2p_set_control(edmac, control);
  292. edmac->buffer = 0;
  293. return 0;
  294. }
  295. static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)
  296. {
  297. return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;
  298. }
  299. static void m2p_hw_synchronize(struct ep93xx_dma_chan *edmac)
  300. {
  301. unsigned long flags;
  302. u32 control;
  303. spin_lock_irqsave(&edmac->lock, flags);
  304. control = readl(edmac->regs + M2P_CONTROL);
  305. control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
  306. m2p_set_control(edmac, control);
  307. spin_unlock_irqrestore(&edmac->lock, flags);
  308. while (m2p_channel_state(edmac) >= M2P_STATE_ON)
  309. schedule();
  310. }
  311. static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)
  312. {
  313. m2p_set_control(edmac, 0);
  314. while (m2p_channel_state(edmac) != M2P_STATE_IDLE)
  315. dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n");
  316. }
  317. static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
  318. {
  319. struct ep93xx_dma_desc *desc;
  320. u32 bus_addr;
  321. desc = ep93xx_dma_get_active(edmac);
  322. if (!desc) {
  323. dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");
  324. return;
  325. }
  326. if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)
  327. bus_addr = desc->src_addr;
  328. else
  329. bus_addr = desc->dst_addr;
  330. if (edmac->buffer == 0) {
  331. writel(desc->size, edmac->regs + M2P_MAXCNT0);
  332. writel(bus_addr, edmac->regs + M2P_BASE0);
  333. } else {
  334. writel(desc->size, edmac->regs + M2P_MAXCNT1);
  335. writel(bus_addr, edmac->regs + M2P_BASE1);
  336. }
  337. edmac->buffer ^= 1;
  338. }
  339. static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)
  340. {
  341. u32 control = readl(edmac->regs + M2P_CONTROL);
  342. m2p_fill_desc(edmac);
  343. control |= M2P_CONTROL_STALLINT;
  344. if (ep93xx_dma_advance_active(edmac)) {
  345. m2p_fill_desc(edmac);
  346. control |= M2P_CONTROL_NFBINT;
  347. }
  348. m2p_set_control(edmac, control);
  349. }
  350. static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)
  351. {
  352. u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);
  353. u32 control;
  354. if (irq_status & M2P_INTERRUPT_ERROR) {
  355. struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);
  356. /* Clear the error interrupt */
  357. writel(1, edmac->regs + M2P_INTERRUPT);
  358. /*
  359. * It seems that there is no easy way of reporting errors back
  360. * to client so we just report the error here and continue as
  361. * usual.
  362. *
  363. * Revisit this when there is a mechanism to report back the
  364. * errors.
  365. */
  366. dev_err(chan2dev(edmac),
  367. "DMA transfer failed! Details:\n"
  368. "\tcookie : %d\n"
  369. "\tsrc_addr : 0x%08x\n"
  370. "\tdst_addr : 0x%08x\n"
  371. "\tsize : %zu\n",
  372. desc->txd.cookie, desc->src_addr, desc->dst_addr,
  373. desc->size);
  374. }
  375. /*
  376. * Even latest E2 silicon revision sometimes assert STALL interrupt
  377. * instead of NFB. Therefore we treat them equally, basing on the
  378. * amount of data we still have to transfer.
  379. */
  380. if (!(irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB)))
  381. return INTERRUPT_UNKNOWN;
  382. if (ep93xx_dma_advance_active(edmac)) {
  383. m2p_fill_desc(edmac);
  384. return INTERRUPT_NEXT_BUFFER;
  385. }
  386. /* Disable interrupts */
  387. control = readl(edmac->regs + M2P_CONTROL);
  388. control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
  389. m2p_set_control(edmac, control);
  390. return INTERRUPT_DONE;
  391. }
  392. /*
  393. * M2M DMA implementation
  394. */
  395. static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)
  396. {
  397. const struct ep93xx_dma_data *data = edmac->chan.private;
  398. u32 control = 0;
  399. if (!data) {
  400. /* This is memcpy channel, nothing to configure */
  401. writel(control, edmac->regs + M2M_CONTROL);
  402. return 0;
  403. }
  404. switch (data->port) {
  405. case EP93XX_DMA_SSP:
  406. /*
  407. * This was found via experimenting - anything less than 5
  408. * causes the channel to perform only a partial transfer which
  409. * leads to problems since we don't get DONE interrupt then.
  410. */
  411. control = (5 << M2M_CONTROL_PWSC_SHIFT);
  412. control |= M2M_CONTROL_NO_HDSK;
  413. if (data->direction == DMA_MEM_TO_DEV) {
  414. control |= M2M_CONTROL_DAH;
  415. control |= M2M_CONTROL_TM_TX;
  416. control |= M2M_CONTROL_RSS_SSPTX;
  417. } else {
  418. control |= M2M_CONTROL_SAH;
  419. control |= M2M_CONTROL_TM_RX;
  420. control |= M2M_CONTROL_RSS_SSPRX;
  421. }
  422. break;
  423. case EP93XX_DMA_IDE:
  424. /*
  425. * This IDE part is totally untested. Values below are taken
  426. * from the EP93xx Users's Guide and might not be correct.
  427. */
  428. if (data->direction == DMA_MEM_TO_DEV) {
  429. /* Worst case from the UG */
  430. control = (3 << M2M_CONTROL_PWSC_SHIFT);
  431. control |= M2M_CONTROL_DAH;
  432. control |= M2M_CONTROL_TM_TX;
  433. } else {
  434. control = (2 << M2M_CONTROL_PWSC_SHIFT);
  435. control |= M2M_CONTROL_SAH;
  436. control |= M2M_CONTROL_TM_RX;
  437. }
  438. control |= M2M_CONTROL_NO_HDSK;
  439. control |= M2M_CONTROL_RSS_IDE;
  440. control |= M2M_CONTROL_PW_16;
  441. break;
  442. default:
  443. return -EINVAL;
  444. }
  445. writel(control, edmac->regs + M2M_CONTROL);
  446. return 0;
  447. }
  448. static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)
  449. {
  450. /* Just disable the channel */
  451. writel(0, edmac->regs + M2M_CONTROL);
  452. }
  453. static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)
  454. {
  455. struct ep93xx_dma_desc *desc;
  456. desc = ep93xx_dma_get_active(edmac);
  457. if (!desc) {
  458. dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");
  459. return;
  460. }
  461. if (edmac->buffer == 0) {
  462. writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);
  463. writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);
  464. writel(desc->size, edmac->regs + M2M_BCR0);
  465. } else {
  466. writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);
  467. writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);
  468. writel(desc->size, edmac->regs + M2M_BCR1);
  469. }
  470. edmac->buffer ^= 1;
  471. }
  472. static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)
  473. {
  474. struct ep93xx_dma_data *data = edmac->chan.private;
  475. u32 control = readl(edmac->regs + M2M_CONTROL);
  476. /*
  477. * Since we allow clients to configure PW (peripheral width) we always
  478. * clear PW bits here and then set them according what is given in
  479. * the runtime configuration.
  480. */
  481. control &= ~M2M_CONTROL_PW_MASK;
  482. control |= edmac->runtime_ctrl;
  483. m2m_fill_desc(edmac);
  484. control |= M2M_CONTROL_DONEINT;
  485. if (ep93xx_dma_advance_active(edmac)) {
  486. m2m_fill_desc(edmac);
  487. control |= M2M_CONTROL_NFBINT;
  488. }
  489. /*
  490. * Now we can finally enable the channel. For M2M channel this must be
  491. * done _after_ the BCRx registers are programmed.
  492. */
  493. control |= M2M_CONTROL_ENABLE;
  494. writel(control, edmac->regs + M2M_CONTROL);
  495. if (!data) {
  496. /*
  497. * For memcpy channels the software trigger must be asserted
  498. * in order to start the memcpy operation.
  499. */
  500. control |= M2M_CONTROL_START;
  501. writel(control, edmac->regs + M2M_CONTROL);
  502. }
  503. }
  504. /*
  505. * According to EP93xx User's Guide, we should receive DONE interrupt when all
  506. * M2M DMA controller transactions complete normally. This is not always the
  507. * case - sometimes EP93xx M2M DMA asserts DONE interrupt when the DMA channel
  508. * is still running (channel Buffer FSM in DMA_BUF_ON state, and channel
  509. * Control FSM in DMA_MEM_RD state, observed at least in IDE-DMA operation).
  510. * In effect, disabling the channel when only DONE bit is set could stop
  511. * currently running DMA transfer. To avoid this, we use Buffer FSM and
  512. * Control FSM to check current state of DMA channel.
  513. */
  514. static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)
  515. {
  516. u32 status = readl(edmac->regs + M2M_STATUS);
  517. u32 ctl_fsm = status & M2M_STATUS_CTL_MASK;
  518. u32 buf_fsm = status & M2M_STATUS_BUF_MASK;
  519. bool done = status & M2M_STATUS_DONE;
  520. bool last_done;
  521. u32 control;
  522. struct ep93xx_dma_desc *desc;
  523. /* Accept only DONE and NFB interrupts */
  524. if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK))
  525. return INTERRUPT_UNKNOWN;
  526. if (done) {
  527. /* Clear the DONE bit */
  528. writel(0, edmac->regs + M2M_INTERRUPT);
  529. }
  530. /*
  531. * Check whether we are done with descriptors or not. This, together
  532. * with DMA channel state, determines action to take in interrupt.
  533. */
  534. desc = ep93xx_dma_get_active(edmac);
  535. last_done = !desc || desc->txd.cookie;
  536. /*
  537. * Use M2M DMA Buffer FSM and Control FSM to check current state of
  538. * DMA channel. Using DONE and NFB bits from channel status register
  539. * or bits from channel interrupt register is not reliable.
  540. */
  541. if (!last_done &&
  542. (buf_fsm == M2M_STATUS_BUF_NO ||
  543. buf_fsm == M2M_STATUS_BUF_ON)) {
  544. /*
  545. * Two buffers are ready for update when Buffer FSM is in
  546. * DMA_NO_BUF state. Only one buffer can be prepared without
  547. * disabling the channel or polling the DONE bit.
  548. * To simplify things, always prepare only one buffer.
  549. */
  550. if (ep93xx_dma_advance_active(edmac)) {
  551. m2m_fill_desc(edmac);
  552. if (done && !edmac->chan.private) {
  553. /* Software trigger for memcpy channel */
  554. control = readl(edmac->regs + M2M_CONTROL);
  555. control |= M2M_CONTROL_START;
  556. writel(control, edmac->regs + M2M_CONTROL);
  557. }
  558. return INTERRUPT_NEXT_BUFFER;
  559. } else {
  560. last_done = true;
  561. }
  562. }
  563. /*
  564. * Disable the channel only when Buffer FSM is in DMA_NO_BUF state
  565. * and Control FSM is in DMA_STALL state.
  566. */
  567. if (last_done &&
  568. buf_fsm == M2M_STATUS_BUF_NO &&
  569. ctl_fsm == M2M_STATUS_CTL_STALL) {
  570. /* Disable interrupts and the channel */
  571. control = readl(edmac->regs + M2M_CONTROL);
  572. control &= ~(M2M_CONTROL_DONEINT | M2M_CONTROL_NFBINT
  573. | M2M_CONTROL_ENABLE);
  574. writel(control, edmac->regs + M2M_CONTROL);
  575. return INTERRUPT_DONE;
  576. }
  577. /*
  578. * Nothing to do this time.
  579. */
  580. return INTERRUPT_NEXT_BUFFER;
  581. }
  582. /*
  583. * DMA engine API implementation
  584. */
  585. static struct ep93xx_dma_desc *
  586. ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)
  587. {
  588. struct ep93xx_dma_desc *desc, *_desc;
  589. struct ep93xx_dma_desc *ret = NULL;
  590. unsigned long flags;
  591. spin_lock_irqsave(&edmac->lock, flags);
  592. list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {
  593. if (async_tx_test_ack(&desc->txd)) {
  594. list_del_init(&desc->node);
  595. /* Re-initialize the descriptor */
  596. desc->src_addr = 0;
  597. desc->dst_addr = 0;
  598. desc->size = 0;
  599. desc->complete = false;
  600. desc->txd.cookie = 0;
  601. desc->txd.callback = NULL;
  602. desc->txd.callback_param = NULL;
  603. ret = desc;
  604. break;
  605. }
  606. }
  607. spin_unlock_irqrestore(&edmac->lock, flags);
  608. return ret;
  609. }
  610. static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,
  611. struct ep93xx_dma_desc *desc)
  612. {
  613. if (desc) {
  614. unsigned long flags;
  615. spin_lock_irqsave(&edmac->lock, flags);
  616. list_splice_init(&desc->tx_list, &edmac->free_list);
  617. list_add(&desc->node, &edmac->free_list);
  618. spin_unlock_irqrestore(&edmac->lock, flags);
  619. }
  620. }
  621. /**
  622. * ep93xx_dma_advance_work - start processing the next pending transaction
  623. * @edmac: channel
  624. *
  625. * If we have pending transactions queued and we are currently idling, this
  626. * function takes the next queued transaction from the @edmac->queue and
  627. * pushes it to the hardware for execution.
  628. */
  629. static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
  630. {
  631. struct ep93xx_dma_desc *new;
  632. unsigned long flags;
  633. spin_lock_irqsave(&edmac->lock, flags);
  634. if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {
  635. spin_unlock_irqrestore(&edmac->lock, flags);
  636. return;
  637. }
  638. /* Take the next descriptor from the pending queue */
  639. new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);
  640. list_del_init(&new->node);
  641. ep93xx_dma_set_active(edmac, new);
  642. /* Push it to the hardware */
  643. edmac->edma->hw_submit(edmac);
  644. spin_unlock_irqrestore(&edmac->lock, flags);
  645. }
  646. static void ep93xx_dma_tasklet(struct tasklet_struct *t)
  647. {
  648. struct ep93xx_dma_chan *edmac = from_tasklet(edmac, t, tasklet);
  649. struct ep93xx_dma_desc *desc, *d;
  650. struct dmaengine_desc_callback cb;
  651. LIST_HEAD(list);
  652. memset(&cb, 0, sizeof(cb));
  653. spin_lock_irq(&edmac->lock);
  654. /*
  655. * If dma_terminate_all() was called before we get to run, the active
  656. * list has become empty. If that happens we aren't supposed to do
  657. * anything more than call ep93xx_dma_advance_work().
  658. */
  659. desc = ep93xx_dma_get_active(edmac);
  660. if (desc) {
  661. if (desc->complete) {
  662. /* mark descriptor complete for non cyclic case only */
  663. if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
  664. dma_cookie_complete(&desc->txd);
  665. list_splice_init(&edmac->active, &list);
  666. }
  667. dmaengine_desc_get_callback(&desc->txd, &cb);
  668. }
  669. spin_unlock_irq(&edmac->lock);
  670. /* Pick up the next descriptor from the queue */
  671. ep93xx_dma_advance_work(edmac);
  672. /* Now we can release all the chained descriptors */
  673. list_for_each_entry_safe(desc, d, &list, node) {
  674. dma_descriptor_unmap(&desc->txd);
  675. ep93xx_dma_desc_put(edmac, desc);
  676. }
  677. dmaengine_desc_callback_invoke(&cb, NULL);
  678. }
  679. static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
  680. {
  681. struct ep93xx_dma_chan *edmac = dev_id;
  682. struct ep93xx_dma_desc *desc;
  683. irqreturn_t ret = IRQ_HANDLED;
  684. spin_lock(&edmac->lock);
  685. desc = ep93xx_dma_get_active(edmac);
  686. if (!desc) {
  687. dev_warn(chan2dev(edmac),
  688. "got interrupt while active list is empty\n");
  689. spin_unlock(&edmac->lock);
  690. return IRQ_NONE;
  691. }
  692. switch (edmac->edma->hw_interrupt(edmac)) {
  693. case INTERRUPT_DONE:
  694. desc->complete = true;
  695. tasklet_schedule(&edmac->tasklet);
  696. break;
  697. case INTERRUPT_NEXT_BUFFER:
  698. if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
  699. tasklet_schedule(&edmac->tasklet);
  700. break;
  701. default:
  702. dev_warn(chan2dev(edmac), "unknown interrupt!\n");
  703. ret = IRQ_NONE;
  704. break;
  705. }
  706. spin_unlock(&edmac->lock);
  707. return ret;
  708. }
  709. /**
  710. * ep93xx_dma_tx_submit - set the prepared descriptor(s) to be executed
  711. * @tx: descriptor to be executed
  712. *
  713. * Function will execute given descriptor on the hardware or if the hardware
  714. * is busy, queue the descriptor to be executed later on. Returns cookie which
  715. * can be used to poll the status of the descriptor.
  716. */
  717. static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx)
  718. {
  719. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);
  720. struct ep93xx_dma_desc *desc;
  721. dma_cookie_t cookie;
  722. unsigned long flags;
  723. spin_lock_irqsave(&edmac->lock, flags);
  724. cookie = dma_cookie_assign(tx);
  725. desc = container_of(tx, struct ep93xx_dma_desc, txd);
  726. /*
  727. * If nothing is currently prosessed, we push this descriptor
  728. * directly to the hardware. Otherwise we put the descriptor
  729. * to the pending queue.
  730. */
  731. if (list_empty(&edmac->active)) {
  732. ep93xx_dma_set_active(edmac, desc);
  733. edmac->edma->hw_submit(edmac);
  734. } else {
  735. list_add_tail(&desc->node, &edmac->queue);
  736. }
  737. spin_unlock_irqrestore(&edmac->lock, flags);
  738. return cookie;
  739. }
  740. /**
  741. * ep93xx_dma_alloc_chan_resources - allocate resources for the channel
  742. * @chan: channel to allocate resources
  743. *
  744. * Function allocates necessary resources for the given DMA channel and
  745. * returns number of allocated descriptors for the channel. Negative errno
  746. * is returned in case of failure.
  747. */
  748. static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
  749. {
  750. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  751. struct ep93xx_dma_data *data = chan->private;
  752. const char *name = dma_chan_name(chan);
  753. int ret, i;
  754. /* Sanity check the channel parameters */
  755. if (!edmac->edma->m2m) {
  756. if (!data)
  757. return -EINVAL;
  758. if (data->port < EP93XX_DMA_I2S1 ||
  759. data->port > EP93XX_DMA_IRDA)
  760. return -EINVAL;
  761. if (data->direction != ep93xx_dma_chan_direction(chan))
  762. return -EINVAL;
  763. } else {
  764. if (data) {
  765. switch (data->port) {
  766. case EP93XX_DMA_SSP:
  767. case EP93XX_DMA_IDE:
  768. if (!is_slave_direction(data->direction))
  769. return -EINVAL;
  770. break;
  771. default:
  772. return -EINVAL;
  773. }
  774. }
  775. }
  776. if (data && data->name)
  777. name = data->name;
  778. ret = clk_enable(edmac->clk);
  779. if (ret)
  780. return ret;
  781. ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);
  782. if (ret)
  783. goto fail_clk_disable;
  784. spin_lock_irq(&edmac->lock);
  785. dma_cookie_init(&edmac->chan);
  786. ret = edmac->edma->hw_setup(edmac);
  787. spin_unlock_irq(&edmac->lock);
  788. if (ret)
  789. goto fail_free_irq;
  790. for (i = 0; i < DMA_MAX_CHAN_DESCRIPTORS; i++) {
  791. struct ep93xx_dma_desc *desc;
  792. desc = kzalloc(sizeof(*desc), GFP_KERNEL);
  793. if (!desc) {
  794. dev_warn(chan2dev(edmac), "not enough descriptors\n");
  795. break;
  796. }
  797. INIT_LIST_HEAD(&desc->tx_list);
  798. dma_async_tx_descriptor_init(&desc->txd, chan);
  799. desc->txd.flags = DMA_CTRL_ACK;
  800. desc->txd.tx_submit = ep93xx_dma_tx_submit;
  801. ep93xx_dma_desc_put(edmac, desc);
  802. }
  803. return i;
  804. fail_free_irq:
  805. free_irq(edmac->irq, edmac);
  806. fail_clk_disable:
  807. clk_disable(edmac->clk);
  808. return ret;
  809. }
  810. /**
  811. * ep93xx_dma_free_chan_resources - release resources for the channel
  812. * @chan: channel
  813. *
  814. * Function releases all the resources allocated for the given channel.
  815. * The channel must be idle when this is called.
  816. */
  817. static void ep93xx_dma_free_chan_resources(struct dma_chan *chan)
  818. {
  819. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  820. struct ep93xx_dma_desc *desc, *d;
  821. unsigned long flags;
  822. LIST_HEAD(list);
  823. BUG_ON(!list_empty(&edmac->active));
  824. BUG_ON(!list_empty(&edmac->queue));
  825. spin_lock_irqsave(&edmac->lock, flags);
  826. edmac->edma->hw_shutdown(edmac);
  827. edmac->runtime_addr = 0;
  828. edmac->runtime_ctrl = 0;
  829. edmac->buffer = 0;
  830. list_splice_init(&edmac->free_list, &list);
  831. spin_unlock_irqrestore(&edmac->lock, flags);
  832. list_for_each_entry_safe(desc, d, &list, node)
  833. kfree(desc);
  834. clk_disable(edmac->clk);
  835. free_irq(edmac->irq, edmac);
  836. }
  837. /**
  838. * ep93xx_dma_prep_dma_memcpy - prepare a memcpy DMA operation
  839. * @chan: channel
  840. * @dest: destination bus address
  841. * @src: source bus address
  842. * @len: size of the transaction
  843. * @flags: flags for the descriptor
  844. *
  845. * Returns a valid DMA descriptor or %NULL in case of failure.
  846. */
  847. static struct dma_async_tx_descriptor *
  848. ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
  849. dma_addr_t src, size_t len, unsigned long flags)
  850. {
  851. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  852. struct ep93xx_dma_desc *desc, *first;
  853. size_t bytes, offset;
  854. first = NULL;
  855. for (offset = 0; offset < len; offset += bytes) {
  856. desc = ep93xx_dma_desc_get(edmac);
  857. if (!desc) {
  858. dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
  859. goto fail;
  860. }
  861. bytes = min_t(size_t, len - offset, DMA_MAX_CHAN_BYTES);
  862. desc->src_addr = src + offset;
  863. desc->dst_addr = dest + offset;
  864. desc->size = bytes;
  865. if (!first)
  866. first = desc;
  867. else
  868. list_add_tail(&desc->node, &first->tx_list);
  869. }
  870. first->txd.cookie = -EBUSY;
  871. first->txd.flags = flags;
  872. return &first->txd;
  873. fail:
  874. ep93xx_dma_desc_put(edmac, first);
  875. return NULL;
  876. }
  877. /**
  878. * ep93xx_dma_prep_slave_sg - prepare a slave DMA operation
  879. * @chan: channel
  880. * @sgl: list of buffers to transfer
  881. * @sg_len: number of entries in @sgl
  882. * @dir: direction of tha DMA transfer
  883. * @flags: flags for the descriptor
  884. * @context: operation context (ignored)
  885. *
  886. * Returns a valid DMA descriptor or %NULL in case of failure.
  887. */
  888. static struct dma_async_tx_descriptor *
  889. ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  890. unsigned int sg_len, enum dma_transfer_direction dir,
  891. unsigned long flags, void *context)
  892. {
  893. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  894. struct ep93xx_dma_desc *desc, *first;
  895. struct scatterlist *sg;
  896. int i;
  897. if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
  898. dev_warn(chan2dev(edmac),
  899. "channel was configured with different direction\n");
  900. return NULL;
  901. }
  902. if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
  903. dev_warn(chan2dev(edmac),
  904. "channel is already used for cyclic transfers\n");
  905. return NULL;
  906. }
  907. ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);
  908. first = NULL;
  909. for_each_sg(sgl, sg, sg_len, i) {
  910. size_t len = sg_dma_len(sg);
  911. if (len > DMA_MAX_CHAN_BYTES) {
  912. dev_warn(chan2dev(edmac), "too big transfer size %zu\n",
  913. len);
  914. goto fail;
  915. }
  916. desc = ep93xx_dma_desc_get(edmac);
  917. if (!desc) {
  918. dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
  919. goto fail;
  920. }
  921. if (dir == DMA_MEM_TO_DEV) {
  922. desc->src_addr = sg_dma_address(sg);
  923. desc->dst_addr = edmac->runtime_addr;
  924. } else {
  925. desc->src_addr = edmac->runtime_addr;
  926. desc->dst_addr = sg_dma_address(sg);
  927. }
  928. desc->size = len;
  929. if (!first)
  930. first = desc;
  931. else
  932. list_add_tail(&desc->node, &first->tx_list);
  933. }
  934. first->txd.cookie = -EBUSY;
  935. first->txd.flags = flags;
  936. return &first->txd;
  937. fail:
  938. ep93xx_dma_desc_put(edmac, first);
  939. return NULL;
  940. }
  941. /**
  942. * ep93xx_dma_prep_dma_cyclic - prepare a cyclic DMA operation
  943. * @chan: channel
  944. * @dma_addr: DMA mapped address of the buffer
  945. * @buf_len: length of the buffer (in bytes)
  946. * @period_len: length of a single period
  947. * @dir: direction of the operation
  948. * @flags: tx descriptor status flags
  949. *
  950. * Prepares a descriptor for cyclic DMA operation. This means that once the
  951. * descriptor is submitted, we will be submitting in a @period_len sized
  952. * buffers and calling callback once the period has been elapsed. Transfer
  953. * terminates only when client calls dmaengine_terminate_all() for this
  954. * channel.
  955. *
  956. * Returns a valid DMA descriptor or %NULL in case of failure.
  957. */
  958. static struct dma_async_tx_descriptor *
  959. ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
  960. size_t buf_len, size_t period_len,
  961. enum dma_transfer_direction dir, unsigned long flags)
  962. {
  963. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  964. struct ep93xx_dma_desc *desc, *first;
  965. size_t offset = 0;
  966. if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
  967. dev_warn(chan2dev(edmac),
  968. "channel was configured with different direction\n");
  969. return NULL;
  970. }
  971. if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
  972. dev_warn(chan2dev(edmac),
  973. "channel is already used for cyclic transfers\n");
  974. return NULL;
  975. }
  976. if (period_len > DMA_MAX_CHAN_BYTES) {
  977. dev_warn(chan2dev(edmac), "too big period length %zu\n",
  978. period_len);
  979. return NULL;
  980. }
  981. ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);
  982. /* Split the buffer into period size chunks */
  983. first = NULL;
  984. for (offset = 0; offset < buf_len; offset += period_len) {
  985. desc = ep93xx_dma_desc_get(edmac);
  986. if (!desc) {
  987. dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
  988. goto fail;
  989. }
  990. if (dir == DMA_MEM_TO_DEV) {
  991. desc->src_addr = dma_addr + offset;
  992. desc->dst_addr = edmac->runtime_addr;
  993. } else {
  994. desc->src_addr = edmac->runtime_addr;
  995. desc->dst_addr = dma_addr + offset;
  996. }
  997. desc->size = period_len;
  998. if (!first)
  999. first = desc;
  1000. else
  1001. list_add_tail(&desc->node, &first->tx_list);
  1002. }
  1003. first->txd.cookie = -EBUSY;
  1004. return &first->txd;
  1005. fail:
  1006. ep93xx_dma_desc_put(edmac, first);
  1007. return NULL;
  1008. }
  1009. /**
  1010. * ep93xx_dma_synchronize - Synchronizes the termination of transfers to the
  1011. * current context.
  1012. * @chan: channel
  1013. *
  1014. * Synchronizes the DMA channel termination to the current context. When this
  1015. * function returns it is guaranteed that all transfers for previously issued
  1016. * descriptors have stopped and and it is safe to free the memory associated
  1017. * with them. Furthermore it is guaranteed that all complete callback functions
  1018. * for a previously submitted descriptor have finished running and it is safe to
  1019. * free resources accessed from within the complete callbacks.
  1020. */
  1021. static void ep93xx_dma_synchronize(struct dma_chan *chan)
  1022. {
  1023. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  1024. if (edmac->edma->hw_synchronize)
  1025. edmac->edma->hw_synchronize(edmac);
  1026. }
  1027. /**
  1028. * ep93xx_dma_terminate_all - terminate all transactions
  1029. * @chan: channel
  1030. *
  1031. * Stops all DMA transactions. All descriptors are put back to the
  1032. * @edmac->free_list and callbacks are _not_ called.
  1033. */
  1034. static int ep93xx_dma_terminate_all(struct dma_chan *chan)
  1035. {
  1036. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  1037. struct ep93xx_dma_desc *desc, *_d;
  1038. unsigned long flags;
  1039. LIST_HEAD(list);
  1040. spin_lock_irqsave(&edmac->lock, flags);
  1041. /* First we disable and flush the DMA channel */
  1042. edmac->edma->hw_shutdown(edmac);
  1043. clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);
  1044. list_splice_init(&edmac->active, &list);
  1045. list_splice_init(&edmac->queue, &list);
  1046. /*
  1047. * We then re-enable the channel. This way we can continue submitting
  1048. * the descriptors by just calling ->hw_submit() again.
  1049. */
  1050. edmac->edma->hw_setup(edmac);
  1051. spin_unlock_irqrestore(&edmac->lock, flags);
  1052. list_for_each_entry_safe(desc, _d, &list, node)
  1053. ep93xx_dma_desc_put(edmac, desc);
  1054. return 0;
  1055. }
  1056. static int ep93xx_dma_slave_config(struct dma_chan *chan,
  1057. struct dma_slave_config *config)
  1058. {
  1059. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  1060. memcpy(&edmac->slave_config, config, sizeof(*config));
  1061. return 0;
  1062. }
  1063. static int ep93xx_dma_slave_config_write(struct dma_chan *chan,
  1064. enum dma_transfer_direction dir,
  1065. struct dma_slave_config *config)
  1066. {
  1067. struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
  1068. enum dma_slave_buswidth width;
  1069. unsigned long flags;
  1070. u32 addr, ctrl;
  1071. if (!edmac->edma->m2m)
  1072. return -EINVAL;
  1073. switch (dir) {
  1074. case DMA_DEV_TO_MEM:
  1075. width = config->src_addr_width;
  1076. addr = config->src_addr;
  1077. break;
  1078. case DMA_MEM_TO_DEV:
  1079. width = config->dst_addr_width;
  1080. addr = config->dst_addr;
  1081. break;
  1082. default:
  1083. return -EINVAL;
  1084. }
  1085. switch (width) {
  1086. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  1087. ctrl = 0;
  1088. break;
  1089. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  1090. ctrl = M2M_CONTROL_PW_16;
  1091. break;
  1092. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  1093. ctrl = M2M_CONTROL_PW_32;
  1094. break;
  1095. default:
  1096. return -EINVAL;
  1097. }
  1098. spin_lock_irqsave(&edmac->lock, flags);
  1099. edmac->runtime_addr = addr;
  1100. edmac->runtime_ctrl = ctrl;
  1101. spin_unlock_irqrestore(&edmac->lock, flags);
  1102. return 0;
  1103. }
  1104. /**
  1105. * ep93xx_dma_tx_status - check if a transaction is completed
  1106. * @chan: channel
  1107. * @cookie: transaction specific cookie
  1108. * @state: state of the transaction is stored here if given
  1109. *
  1110. * This function can be used to query state of a given transaction.
  1111. */
  1112. static enum dma_status ep93xx_dma_tx_status(struct dma_chan *chan,
  1113. dma_cookie_t cookie,
  1114. struct dma_tx_state *state)
  1115. {
  1116. return dma_cookie_status(chan, cookie, state);
  1117. }
  1118. /**
  1119. * ep93xx_dma_issue_pending - push pending transactions to the hardware
  1120. * @chan: channel
  1121. *
  1122. * When this function is called, all pending transactions are pushed to the
  1123. * hardware and executed.
  1124. */
  1125. static void ep93xx_dma_issue_pending(struct dma_chan *chan)
  1126. {
  1127. ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan));
  1128. }
  1129. static int __init ep93xx_dma_probe(struct platform_device *pdev)
  1130. {
  1131. struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev);
  1132. struct ep93xx_dma_engine *edma;
  1133. struct dma_device *dma_dev;
  1134. size_t edma_size;
  1135. int ret, i;
  1136. edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan);
  1137. edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL);
  1138. if (!edma)
  1139. return -ENOMEM;
  1140. dma_dev = &edma->dma_dev;
  1141. edma->m2m = platform_get_device_id(pdev)->driver_data;
  1142. edma->num_channels = pdata->num_channels;
  1143. INIT_LIST_HEAD(&dma_dev->channels);
  1144. for (i = 0; i < pdata->num_channels; i++) {
  1145. const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i];
  1146. struct ep93xx_dma_chan *edmac = &edma->channels[i];
  1147. edmac->chan.device = dma_dev;
  1148. edmac->regs = cdata->base;
  1149. edmac->irq = cdata->irq;
  1150. edmac->edma = edma;
  1151. edmac->clk = clk_get(NULL, cdata->name);
  1152. if (IS_ERR(edmac->clk)) {
  1153. dev_warn(&pdev->dev, "failed to get clock for %s\n",
  1154. cdata->name);
  1155. continue;
  1156. }
  1157. spin_lock_init(&edmac->lock);
  1158. INIT_LIST_HEAD(&edmac->active);
  1159. INIT_LIST_HEAD(&edmac->queue);
  1160. INIT_LIST_HEAD(&edmac->free_list);
  1161. tasklet_setup(&edmac->tasklet, ep93xx_dma_tasklet);
  1162. list_add_tail(&edmac->chan.device_node,
  1163. &dma_dev->channels);
  1164. }
  1165. dma_cap_zero(dma_dev->cap_mask);
  1166. dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
  1167. dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
  1168. dma_dev->dev = &pdev->dev;
  1169. dma_dev->device_alloc_chan_resources = ep93xx_dma_alloc_chan_resources;
  1170. dma_dev->device_free_chan_resources = ep93xx_dma_free_chan_resources;
  1171. dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg;
  1172. dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic;
  1173. dma_dev->device_config = ep93xx_dma_slave_config;
  1174. dma_dev->device_synchronize = ep93xx_dma_synchronize;
  1175. dma_dev->device_terminate_all = ep93xx_dma_terminate_all;
  1176. dma_dev->device_issue_pending = ep93xx_dma_issue_pending;
  1177. dma_dev->device_tx_status = ep93xx_dma_tx_status;
  1178. dma_set_max_seg_size(dma_dev->dev, DMA_MAX_CHAN_BYTES);
  1179. if (edma->m2m) {
  1180. dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
  1181. dma_dev->device_prep_dma_memcpy = ep93xx_dma_prep_dma_memcpy;
  1182. edma->hw_setup = m2m_hw_setup;
  1183. edma->hw_shutdown = m2m_hw_shutdown;
  1184. edma->hw_submit = m2m_hw_submit;
  1185. edma->hw_interrupt = m2m_hw_interrupt;
  1186. } else {
  1187. dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
  1188. edma->hw_synchronize = m2p_hw_synchronize;
  1189. edma->hw_setup = m2p_hw_setup;
  1190. edma->hw_shutdown = m2p_hw_shutdown;
  1191. edma->hw_submit = m2p_hw_submit;
  1192. edma->hw_interrupt = m2p_hw_interrupt;
  1193. }
  1194. ret = dma_async_device_register(dma_dev);
  1195. if (unlikely(ret)) {
  1196. for (i = 0; i < edma->num_channels; i++) {
  1197. struct ep93xx_dma_chan *edmac = &edma->channels[i];
  1198. if (!IS_ERR_OR_NULL(edmac->clk))
  1199. clk_put(edmac->clk);
  1200. }
  1201. kfree(edma);
  1202. } else {
  1203. dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n",
  1204. edma->m2m ? "M" : "P");
  1205. }
  1206. return ret;
  1207. }
  1208. static const struct platform_device_id ep93xx_dma_driver_ids[] = {
  1209. { "ep93xx-dma-m2p", 0 },
  1210. { "ep93xx-dma-m2m", 1 },
  1211. { },
  1212. };
  1213. static struct platform_driver ep93xx_dma_driver = {
  1214. .driver = {
  1215. .name = "ep93xx-dma",
  1216. },
  1217. .id_table = ep93xx_dma_driver_ids,
  1218. };
  1219. static int __init ep93xx_dma_module_init(void)
  1220. {
  1221. return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe);
  1222. }
  1223. subsys_initcall(ep93xx_dma_module_init);
  1224. MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>");
  1225. MODULE_DESCRIPTION("EP93xx DMA driver");
  1226. MODULE_LICENSE("GPL");