s3c24xx-dma.c 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * S3C24XX DMA handling
  4. *
  5. * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
  6. *
  7. * based on amba-pl08x.c
  8. *
  9. * Copyright (c) 2006 ARM Ltd.
  10. * Copyright (c) 2010 ST-Ericsson SA
  11. *
  12. * Author: Peter Pearse <peter.pearse@arm.com>
  13. * Author: Linus Walleij <linus.walleij@stericsson.com>
  14. *
  15. * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals
  16. * that can be routed to any of the 4 to 8 hardware-channels.
  17. *
  18. * Therefore on these DMA controllers the number of channels
  19. * and the number of incoming DMA signals are two totally different things.
  20. * It is usually not possible to theoretically handle all physical signals,
  21. * so a multiplexing scheme with possible denial of use is necessary.
  22. *
  23. * Open items:
  24. * - bursts
  25. */
  26. #include <linux/platform_device.h>
  27. #include <linux/types.h>
  28. #include <linux/dmaengine.h>
  29. #include <linux/dma-mapping.h>
  30. #include <linux/interrupt.h>
  31. #include <linux/clk.h>
  32. #include <linux/module.h>
  33. #include <linux/mod_devicetable.h>
  34. #include <linux/slab.h>
  35. #include <linux/platform_data/dma-s3c24xx.h>
  36. #include "dmaengine.h"
  37. #include "virt-dma.h"
  38. #define MAX_DMA_CHANNELS 8
  39. #define S3C24XX_DISRC 0x00
  40. #define S3C24XX_DISRCC 0x04
  41. #define S3C24XX_DISRCC_INC_INCREMENT 0
  42. #define S3C24XX_DISRCC_INC_FIXED BIT(0)
  43. #define S3C24XX_DISRCC_LOC_AHB 0
  44. #define S3C24XX_DISRCC_LOC_APB BIT(1)
  45. #define S3C24XX_DIDST 0x08
  46. #define S3C24XX_DIDSTC 0x0c
  47. #define S3C24XX_DIDSTC_INC_INCREMENT 0
  48. #define S3C24XX_DIDSTC_INC_FIXED BIT(0)
  49. #define S3C24XX_DIDSTC_LOC_AHB 0
  50. #define S3C24XX_DIDSTC_LOC_APB BIT(1)
  51. #define S3C24XX_DIDSTC_INT_TC0 0
  52. #define S3C24XX_DIDSTC_INT_RELOAD BIT(2)
  53. #define S3C24XX_DCON 0x10
  54. #define S3C24XX_DCON_TC_MASK 0xfffff
  55. #define S3C24XX_DCON_DSZ_BYTE (0 << 20)
  56. #define S3C24XX_DCON_DSZ_HALFWORD (1 << 20)
  57. #define S3C24XX_DCON_DSZ_WORD (2 << 20)
  58. #define S3C24XX_DCON_DSZ_MASK (3 << 20)
  59. #define S3C24XX_DCON_DSZ_SHIFT 20
  60. #define S3C24XX_DCON_AUTORELOAD 0
  61. #define S3C24XX_DCON_NORELOAD BIT(22)
  62. #define S3C24XX_DCON_HWTRIG BIT(23)
  63. #define S3C24XX_DCON_HWSRC_SHIFT 24
  64. #define S3C24XX_DCON_SERV_SINGLE 0
  65. #define S3C24XX_DCON_SERV_WHOLE BIT(27)
  66. #define S3C24XX_DCON_TSZ_UNIT 0
  67. #define S3C24XX_DCON_TSZ_BURST4 BIT(28)
  68. #define S3C24XX_DCON_INT BIT(29)
  69. #define S3C24XX_DCON_SYNC_PCLK 0
  70. #define S3C24XX_DCON_SYNC_HCLK BIT(30)
  71. #define S3C24XX_DCON_DEMAND 0
  72. #define S3C24XX_DCON_HANDSHAKE BIT(31)
  73. #define S3C24XX_DSTAT 0x14
  74. #define S3C24XX_DSTAT_STAT_BUSY BIT(20)
  75. #define S3C24XX_DSTAT_CURRTC_MASK 0xfffff
  76. #define S3C24XX_DMASKTRIG 0x20
  77. #define S3C24XX_DMASKTRIG_SWTRIG BIT(0)
  78. #define S3C24XX_DMASKTRIG_ON BIT(1)
  79. #define S3C24XX_DMASKTRIG_STOP BIT(2)
  80. #define S3C24XX_DMAREQSEL 0x24
  81. #define S3C24XX_DMAREQSEL_HW BIT(0)
  82. /*
  83. * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel
  84. * for a DMA source. Instead only specific channels are valid.
  85. * All of these SoCs have 4 physical channels and the number of request
  86. * source bits is 3. Additionally we also need 1 bit to mark the channel
  87. * as valid.
  88. * Therefore we separate the chansel element of the channel data into 4
  89. * parts of 4 bits each, to hold the information if the channel is valid
  90. * and the hw request source to use.
  91. *
  92. * Example:
  93. * SDI is valid on channels 0, 2 and 3 - with varying hw request sources.
  94. * For it the chansel field would look like
  95. *
  96. * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1
  97. * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2
  98. * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2
  99. */
  100. #define S3C24XX_CHANSEL_WIDTH 4
  101. #define S3C24XX_CHANSEL_VALID BIT(3)
  102. #define S3C24XX_CHANSEL_REQ_MASK 7
  103. /*
  104. * struct soc_data - vendor-specific config parameters for individual SoCs
  105. * @stride: spacing between the registers of each channel
  106. * @has_reqsel: does the controller use the newer requestselection mechanism
  107. * @has_clocks: are controllable dma-clocks present
  108. */
  109. struct soc_data {
  110. int stride;
  111. bool has_reqsel;
  112. bool has_clocks;
  113. };
  114. /*
  115. * enum s3c24xx_dma_chan_state - holds the virtual channel states
  116. * @S3C24XX_DMA_CHAN_IDLE: the channel is idle
  117. * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport
  118. * channel and is running a transfer on it
  119. * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport
  120. * channel to become available (only pertains to memcpy channels)
  121. */
  122. enum s3c24xx_dma_chan_state {
  123. S3C24XX_DMA_CHAN_IDLE,
  124. S3C24XX_DMA_CHAN_RUNNING,
  125. S3C24XX_DMA_CHAN_WAITING,
  126. };
  127. /*
  128. * struct s3c24xx_sg - structure containing data per sg
  129. * @src_addr: src address of sg
  130. * @dst_addr: dst address of sg
  131. * @len: transfer len in bytes
  132. * @node: node for txd's dsg_list
  133. */
  134. struct s3c24xx_sg {
  135. dma_addr_t src_addr;
  136. dma_addr_t dst_addr;
  137. size_t len;
  138. struct list_head node;
  139. };
  140. /*
  141. * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor
  142. * @vd: virtual DMA descriptor
  143. * @dsg_list: list of children sg's
  144. * @at: sg currently being transfered
  145. * @width: transfer width
  146. * @disrcc: value for source control register
  147. * @didstc: value for destination control register
  148. * @dcon: base value for dcon register
  149. * @cyclic: indicate cyclic transfer
  150. */
  151. struct s3c24xx_txd {
  152. struct virt_dma_desc vd;
  153. struct list_head dsg_list;
  154. struct list_head *at;
  155. u8 width;
  156. u32 disrcc;
  157. u32 didstc;
  158. u32 dcon;
  159. bool cyclic;
  160. };
  161. struct s3c24xx_dma_chan;
  162. /*
  163. * struct s3c24xx_dma_phy - holder for the physical channels
  164. * @id: physical index to this channel
  165. * @valid: does the channel have all required elements
  166. * @base: virtual memory base (remapped) for the this channel
  167. * @irq: interrupt for this channel
  168. * @clk: clock for this channel
  169. * @lock: a lock to use when altering an instance of this struct
  170. * @serving: virtual channel currently being served by this physicalchannel
  171. * @host: a pointer to the host (internal use)
  172. */
  173. struct s3c24xx_dma_phy {
  174. unsigned int id;
  175. bool valid;
  176. void __iomem *base;
  177. int irq;
  178. struct clk *clk;
  179. spinlock_t lock;
  180. struct s3c24xx_dma_chan *serving;
  181. struct s3c24xx_dma_engine *host;
  182. };
  183. /*
  184. * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel
  185. * @id: the id of the channel
  186. * @name: name of the channel
  187. * @vc: wrappped virtual channel
  188. * @phy: the physical channel utilized by this channel, if there is one
  189. * @runtime_addr: address for RX/TX according to the runtime config
  190. * @at: active transaction on this channel
  191. * @lock: a lock for this channel data
  192. * @host: a pointer to the host (internal use)
  193. * @state: whether the channel is idle, running etc
  194. * @slave: whether this channel is a device (slave) or for memcpy
  195. */
  196. struct s3c24xx_dma_chan {
  197. int id;
  198. const char *name;
  199. struct virt_dma_chan vc;
  200. struct s3c24xx_dma_phy *phy;
  201. struct dma_slave_config cfg;
  202. struct s3c24xx_txd *at;
  203. struct s3c24xx_dma_engine *host;
  204. enum s3c24xx_dma_chan_state state;
  205. bool slave;
  206. };
  207. /*
  208. * struct s3c24xx_dma_engine - the local state holder for the S3C24XX
  209. * @pdev: the corresponding platform device
  210. * @pdata: platform data passed in from the platform/machine
  211. * @base: virtual memory base (remapped)
  212. * @slave: slave engine for this instance
  213. * @memcpy: memcpy engine for this instance
  214. * @phy_chans: array of data for the physical channels
  215. */
  216. struct s3c24xx_dma_engine {
  217. struct platform_device *pdev;
  218. const struct s3c24xx_dma_platdata *pdata;
  219. struct soc_data *sdata;
  220. void __iomem *base;
  221. struct dma_device slave;
  222. struct dma_device memcpy;
  223. struct s3c24xx_dma_phy *phy_chans;
  224. };
  225. /*
  226. * Physical channel handling
  227. */
  228. /*
  229. * Check whether a certain channel is busy or not.
  230. */
  231. static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
  232. {
  233. unsigned int val = readl(phy->base + S3C24XX_DSTAT);
  234. return val & S3C24XX_DSTAT_STAT_BUSY;
  235. }
  236. static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
  237. struct s3c24xx_dma_phy *phy)
  238. {
  239. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  240. const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  241. struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
  242. int phyvalid;
  243. /* every phy is valid for memcopy channels */
  244. if (!s3cchan->slave)
  245. return true;
  246. /* On newer variants all phys can be used for all virtual channels */
  247. if (s3cdma->sdata->has_reqsel)
  248. return true;
  249. phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
  250. return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
  251. }
  252. /*
  253. * Allocate a physical channel for a virtual channel
  254. *
  255. * Try to locate a physical channel to be used for this transfer. If all
  256. * are taken return NULL and the requester will have to cope by using
  257. * some fallback PIO mode or retrying later.
  258. */
  259. static
  260. struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
  261. {
  262. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  263. struct s3c24xx_dma_phy *phy = NULL;
  264. unsigned long flags;
  265. int i;
  266. int ret;
  267. for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
  268. phy = &s3cdma->phy_chans[i];
  269. if (!phy->valid)
  270. continue;
  271. if (!s3c24xx_dma_phy_valid(s3cchan, phy))
  272. continue;
  273. spin_lock_irqsave(&phy->lock, flags);
  274. if (!phy->serving) {
  275. phy->serving = s3cchan;
  276. spin_unlock_irqrestore(&phy->lock, flags);
  277. break;
  278. }
  279. spin_unlock_irqrestore(&phy->lock, flags);
  280. }
  281. /* No physical channel available, cope with it */
  282. if (i == s3cdma->pdata->num_phy_channels) {
  283. dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
  284. return NULL;
  285. }
  286. /* start the phy clock */
  287. if (s3cdma->sdata->has_clocks) {
  288. ret = clk_enable(phy->clk);
  289. if (ret) {
  290. dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
  291. phy->id, ret);
  292. phy->serving = NULL;
  293. return NULL;
  294. }
  295. }
  296. return phy;
  297. }
  298. /*
  299. * Mark the physical channel as free.
  300. *
  301. * This drops the link between the physical and virtual channel.
  302. */
  303. static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
  304. {
  305. struct s3c24xx_dma_engine *s3cdma = phy->host;
  306. if (s3cdma->sdata->has_clocks)
  307. clk_disable(phy->clk);
  308. phy->serving = NULL;
  309. }
  310. /*
  311. * Stops the channel by writing the stop bit.
  312. * This should not be used for an on-going transfer, but as a method of
  313. * shutting down a channel (eg, when it's no longer used) or terminating a
  314. * transfer.
  315. */
  316. static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
  317. {
  318. writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
  319. }
  320. /*
  321. * Virtual channel handling
  322. */
  323. static inline
  324. struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
  325. {
  326. return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
  327. }
  328. static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
  329. {
  330. struct s3c24xx_dma_phy *phy = s3cchan->phy;
  331. struct s3c24xx_txd *txd = s3cchan->at;
  332. u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
  333. return tc * txd->width;
  334. }
  335. static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
  336. struct dma_slave_config *config)
  337. {
  338. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  339. unsigned long flags;
  340. int ret = 0;
  341. /* Reject definitely invalid configurations */
  342. if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
  343. config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
  344. return -EINVAL;
  345. spin_lock_irqsave(&s3cchan->vc.lock, flags);
  346. if (!s3cchan->slave) {
  347. ret = -EINVAL;
  348. goto out;
  349. }
  350. s3cchan->cfg = *config;
  351. out:
  352. spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  353. return ret;
  354. }
  355. /*
  356. * Transfer handling
  357. */
  358. static inline
  359. struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
  360. {
  361. return container_of(tx, struct s3c24xx_txd, vd.tx);
  362. }
  363. static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
  364. {
  365. struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
  366. if (txd) {
  367. INIT_LIST_HEAD(&txd->dsg_list);
  368. txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
  369. }
  370. return txd;
  371. }
  372. static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
  373. {
  374. struct s3c24xx_sg *dsg, *_dsg;
  375. list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
  376. list_del(&dsg->node);
  377. kfree(dsg);
  378. }
  379. kfree(txd);
  380. }
  381. static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
  382. struct s3c24xx_txd *txd)
  383. {
  384. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  385. struct s3c24xx_dma_phy *phy = s3cchan->phy;
  386. const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  387. struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
  388. u32 dcon = txd->dcon;
  389. u32 val;
  390. /* transfer-size and -count from len and width */
  391. switch (txd->width) {
  392. case 1:
  393. dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
  394. break;
  395. case 2:
  396. dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
  397. break;
  398. case 4:
  399. dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
  400. break;
  401. }
  402. if (s3cchan->slave) {
  403. struct s3c24xx_dma_channel *cdata =
  404. &pdata->channels[s3cchan->id];
  405. if (s3cdma->sdata->has_reqsel) {
  406. writel_relaxed((cdata->chansel << 1) |
  407. S3C24XX_DMAREQSEL_HW,
  408. phy->base + S3C24XX_DMAREQSEL);
  409. } else {
  410. int csel = cdata->chansel >> (phy->id *
  411. S3C24XX_CHANSEL_WIDTH);
  412. csel &= S3C24XX_CHANSEL_REQ_MASK;
  413. dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
  414. dcon |= S3C24XX_DCON_HWTRIG;
  415. }
  416. } else {
  417. if (s3cdma->sdata->has_reqsel)
  418. writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
  419. }
  420. writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
  421. writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
  422. writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
  423. writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
  424. writel_relaxed(dcon, phy->base + S3C24XX_DCON);
  425. val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
  426. val &= ~S3C24XX_DMASKTRIG_STOP;
  427. val |= S3C24XX_DMASKTRIG_ON;
  428. /* trigger the dma operation for memcpy transfers */
  429. if (!s3cchan->slave)
  430. val |= S3C24XX_DMASKTRIG_SWTRIG;
  431. writel(val, phy->base + S3C24XX_DMASKTRIG);
  432. }
  433. /*
  434. * Set the initial DMA register values and start first sg.
  435. */
  436. static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
  437. {
  438. struct s3c24xx_dma_phy *phy = s3cchan->phy;
  439. struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
  440. struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
  441. list_del(&txd->vd.node);
  442. s3cchan->at = txd;
  443. /* Wait for channel inactive */
  444. while (s3c24xx_dma_phy_busy(phy))
  445. cpu_relax();
  446. /* point to the first element of the sg list */
  447. txd->at = txd->dsg_list.next;
  448. s3c24xx_dma_start_next_sg(s3cchan, txd);
  449. }
  450. /*
  451. * Try to allocate a physical channel. When successful, assign it to
  452. * this virtual channel, and initiate the next descriptor. The
  453. * virtual channel lock must be held at this point.
  454. */
  455. static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
  456. {
  457. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  458. struct s3c24xx_dma_phy *phy;
  459. phy = s3c24xx_dma_get_phy(s3cchan);
  460. if (!phy) {
  461. dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
  462. s3cchan->name);
  463. s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
  464. return;
  465. }
  466. dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
  467. phy->id, s3cchan->name);
  468. s3cchan->phy = phy;
  469. s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
  470. s3c24xx_dma_start_next_txd(s3cchan);
  471. }
  472. static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
  473. struct s3c24xx_dma_chan *s3cchan)
  474. {
  475. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  476. dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
  477. phy->id, s3cchan->name);
  478. /*
  479. * We do this without taking the lock; we're really only concerned
  480. * about whether this pointer is NULL or not, and we're guaranteed
  481. * that this will only be called when it _already_ is non-NULL.
  482. */
  483. phy->serving = s3cchan;
  484. s3cchan->phy = phy;
  485. s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
  486. s3c24xx_dma_start_next_txd(s3cchan);
  487. }
  488. /*
  489. * Free a physical DMA channel, potentially reallocating it to another
  490. * virtual channel if we have any pending.
  491. */
  492. static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
  493. {
  494. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  495. struct s3c24xx_dma_chan *p, *next;
  496. retry:
  497. next = NULL;
  498. /* Find a waiting virtual channel for the next transfer. */
  499. list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
  500. if (p->state == S3C24XX_DMA_CHAN_WAITING) {
  501. next = p;
  502. break;
  503. }
  504. if (!next) {
  505. list_for_each_entry(p, &s3cdma->slave.channels,
  506. vc.chan.device_node)
  507. if (p->state == S3C24XX_DMA_CHAN_WAITING &&
  508. s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
  509. next = p;
  510. break;
  511. }
  512. }
  513. /* Ensure that the physical channel is stopped */
  514. s3c24xx_dma_terminate_phy(s3cchan->phy);
  515. if (next) {
  516. bool success;
  517. /*
  518. * Eww. We know this isn't going to deadlock
  519. * but lockdep probably doesn't.
  520. */
  521. spin_lock(&next->vc.lock);
  522. /* Re-check the state now that we have the lock */
  523. success = next->state == S3C24XX_DMA_CHAN_WAITING;
  524. if (success)
  525. s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
  526. spin_unlock(&next->vc.lock);
  527. /* If the state changed, try to find another channel */
  528. if (!success)
  529. goto retry;
  530. } else {
  531. /* No more jobs, so free up the physical channel */
  532. s3c24xx_dma_put_phy(s3cchan->phy);
  533. }
  534. s3cchan->phy = NULL;
  535. s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
  536. }
  537. static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
  538. {
  539. struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
  540. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
  541. if (!s3cchan->slave)
  542. dma_descriptor_unmap(&vd->tx);
  543. s3c24xx_dma_free_txd(txd);
  544. }
  545. static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
  546. {
  547. struct s3c24xx_dma_phy *phy = data;
  548. struct s3c24xx_dma_chan *s3cchan = phy->serving;
  549. struct s3c24xx_txd *txd;
  550. dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
  551. /*
  552. * Interrupts happen to notify the completion of a transfer and the
  553. * channel should have moved into its stop state already on its own.
  554. * Therefore interrupts on channels not bound to a virtual channel
  555. * should never happen. Nevertheless send a terminate command to the
  556. * channel if the unlikely case happens.
  557. */
  558. if (unlikely(!s3cchan)) {
  559. dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
  560. phy->id);
  561. s3c24xx_dma_terminate_phy(phy);
  562. return IRQ_HANDLED;
  563. }
  564. spin_lock(&s3cchan->vc.lock);
  565. txd = s3cchan->at;
  566. if (txd) {
  567. /* when more sg's are in this txd, start the next one */
  568. if (!list_is_last(txd->at, &txd->dsg_list)) {
  569. txd->at = txd->at->next;
  570. if (txd->cyclic)
  571. vchan_cyclic_callback(&txd->vd);
  572. s3c24xx_dma_start_next_sg(s3cchan, txd);
  573. } else if (!txd->cyclic) {
  574. s3cchan->at = NULL;
  575. vchan_cookie_complete(&txd->vd);
  576. /*
  577. * And start the next descriptor (if any),
  578. * otherwise free this channel.
  579. */
  580. if (vchan_next_desc(&s3cchan->vc))
  581. s3c24xx_dma_start_next_txd(s3cchan);
  582. else
  583. s3c24xx_dma_phy_free(s3cchan);
  584. } else {
  585. vchan_cyclic_callback(&txd->vd);
  586. /* Cyclic: reset at beginning */
  587. txd->at = txd->dsg_list.next;
  588. s3c24xx_dma_start_next_sg(s3cchan, txd);
  589. }
  590. }
  591. spin_unlock(&s3cchan->vc.lock);
  592. return IRQ_HANDLED;
  593. }
  594. /*
  595. * The DMA ENGINE API
  596. */
  597. static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
  598. {
  599. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  600. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  601. LIST_HEAD(head);
  602. unsigned long flags;
  603. int ret;
  604. spin_lock_irqsave(&s3cchan->vc.lock, flags);
  605. if (!s3cchan->phy && !s3cchan->at) {
  606. dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
  607. s3cchan->id);
  608. ret = -EINVAL;
  609. goto unlock;
  610. }
  611. s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
  612. /* Mark physical channel as free */
  613. if (s3cchan->phy)
  614. s3c24xx_dma_phy_free(s3cchan);
  615. /* Dequeue current job */
  616. if (s3cchan->at) {
  617. vchan_terminate_vdesc(&s3cchan->at->vd);
  618. s3cchan->at = NULL;
  619. }
  620. /* Dequeue jobs not yet fired as well */
  621. vchan_get_all_descriptors(&s3cchan->vc, &head);
  622. spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  623. vchan_dma_desc_free_list(&s3cchan->vc, &head);
  624. return 0;
  625. unlock:
  626. spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  627. return ret;
  628. }
  629. static void s3c24xx_dma_synchronize(struct dma_chan *chan)
  630. {
  631. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  632. vchan_synchronize(&s3cchan->vc);
  633. }
  634. static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
  635. {
  636. /* Ensure all queued descriptors are freed */
  637. vchan_free_chan_resources(to_virt_chan(chan));
  638. }
  639. static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
  640. dma_cookie_t cookie, struct dma_tx_state *txstate)
  641. {
  642. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  643. struct s3c24xx_txd *txd;
  644. struct s3c24xx_sg *dsg;
  645. struct virt_dma_desc *vd;
  646. unsigned long flags;
  647. enum dma_status ret;
  648. size_t bytes = 0;
  649. spin_lock_irqsave(&s3cchan->vc.lock, flags);
  650. ret = dma_cookie_status(chan, cookie, txstate);
  651. /*
  652. * There's no point calculating the residue if there's
  653. * no txstate to store the value.
  654. */
  655. if (ret == DMA_COMPLETE || !txstate) {
  656. spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  657. return ret;
  658. }
  659. vd = vchan_find_desc(&s3cchan->vc, cookie);
  660. if (vd) {
  661. /* On the issued list, so hasn't been processed yet */
  662. txd = to_s3c24xx_txd(&vd->tx);
  663. list_for_each_entry(dsg, &txd->dsg_list, node)
  664. bytes += dsg->len;
  665. } else {
  666. /*
  667. * Currently running, so sum over the pending sg's and
  668. * the currently active one.
  669. */
  670. txd = s3cchan->at;
  671. dsg = list_entry(txd->at, struct s3c24xx_sg, node);
  672. list_for_each_entry_from(dsg, &txd->dsg_list, node)
  673. bytes += dsg->len;
  674. bytes += s3c24xx_dma_getbytes_chan(s3cchan);
  675. }
  676. spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  677. /*
  678. * This cookie not complete yet
  679. * Get number of bytes left in the active transactions and queue
  680. */
  681. dma_set_residue(txstate, bytes);
  682. /* Whether waiting or running, we're in progress */
  683. return ret;
  684. }
  685. /*
  686. * Initialize a descriptor to be used by memcpy submit
  687. */
  688. static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
  689. struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  690. size_t len, unsigned long flags)
  691. {
  692. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  693. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  694. struct s3c24xx_txd *txd;
  695. struct s3c24xx_sg *dsg;
  696. int src_mod, dest_mod;
  697. dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n",
  698. len, s3cchan->name);
  699. if ((len & S3C24XX_DCON_TC_MASK) != len) {
  700. dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len);
  701. return NULL;
  702. }
  703. txd = s3c24xx_dma_get_txd();
  704. if (!txd)
  705. return NULL;
  706. dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
  707. if (!dsg) {
  708. s3c24xx_dma_free_txd(txd);
  709. return NULL;
  710. }
  711. list_add_tail(&dsg->node, &txd->dsg_list);
  712. dsg->src_addr = src;
  713. dsg->dst_addr = dest;
  714. dsg->len = len;
  715. /*
  716. * Determine a suitable transfer width.
  717. * The DMA controller cannot fetch/store information which is not
  718. * naturally aligned on the bus, i.e., a 4 byte fetch must start at
  719. * an address divisible by 4 - more generally addr % width must be 0.
  720. */
  721. src_mod = src % 4;
  722. dest_mod = dest % 4;
  723. switch (len % 4) {
  724. case 0:
  725. txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
  726. break;
  727. case 2:
  728. txd->width = ((src_mod == 2 || src_mod == 0) &&
  729. (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
  730. break;
  731. default:
  732. txd->width = 1;
  733. break;
  734. }
  735. txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
  736. txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
  737. txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
  738. S3C24XX_DCON_SERV_WHOLE;
  739. return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
  740. }
  741. static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
  742. struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
  743. enum dma_transfer_direction direction, unsigned long flags)
  744. {
  745. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  746. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  747. const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  748. struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
  749. struct s3c24xx_txd *txd;
  750. struct s3c24xx_sg *dsg;
  751. unsigned sg_len;
  752. dma_addr_t slave_addr;
  753. u32 hwcfg = 0;
  754. int i;
  755. dev_dbg(&s3cdma->pdev->dev,
  756. "prepare cyclic transaction of %zu bytes with period %zu from %s\n",
  757. size, period, s3cchan->name);
  758. if (!is_slave_direction(direction)) {
  759. dev_err(&s3cdma->pdev->dev,
  760. "direction %d unsupported\n", direction);
  761. return NULL;
  762. }
  763. txd = s3c24xx_dma_get_txd();
  764. if (!txd)
  765. return NULL;
  766. txd->cyclic = 1;
  767. if (cdata->handshake)
  768. txd->dcon |= S3C24XX_DCON_HANDSHAKE;
  769. switch (cdata->bus) {
  770. case S3C24XX_DMA_APB:
  771. txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
  772. hwcfg |= S3C24XX_DISRCC_LOC_APB;
  773. break;
  774. case S3C24XX_DMA_AHB:
  775. txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
  776. hwcfg |= S3C24XX_DISRCC_LOC_AHB;
  777. break;
  778. }
  779. /*
  780. * Always assume our peripheral desintation is a fixed
  781. * address in memory.
  782. */
  783. hwcfg |= S3C24XX_DISRCC_INC_FIXED;
  784. /*
  785. * Individual dma operations are requested by the slave,
  786. * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
  787. */
  788. txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
  789. if (direction == DMA_MEM_TO_DEV) {
  790. txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
  791. S3C24XX_DISRCC_INC_INCREMENT;
  792. txd->didstc = hwcfg;
  793. slave_addr = s3cchan->cfg.dst_addr;
  794. txd->width = s3cchan->cfg.dst_addr_width;
  795. } else {
  796. txd->disrcc = hwcfg;
  797. txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
  798. S3C24XX_DIDSTC_INC_INCREMENT;
  799. slave_addr = s3cchan->cfg.src_addr;
  800. txd->width = s3cchan->cfg.src_addr_width;
  801. }
  802. sg_len = size / period;
  803. for (i = 0; i < sg_len; i++) {
  804. dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
  805. if (!dsg) {
  806. s3c24xx_dma_free_txd(txd);
  807. return NULL;
  808. }
  809. list_add_tail(&dsg->node, &txd->dsg_list);
  810. dsg->len = period;
  811. /* Check last period length */
  812. if (i == sg_len - 1)
  813. dsg->len = size - period * i;
  814. if (direction == DMA_MEM_TO_DEV) {
  815. dsg->src_addr = addr + period * i;
  816. dsg->dst_addr = slave_addr;
  817. } else { /* DMA_DEV_TO_MEM */
  818. dsg->src_addr = slave_addr;
  819. dsg->dst_addr = addr + period * i;
  820. }
  821. }
  822. return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
  823. }
  824. static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
  825. struct dma_chan *chan, struct scatterlist *sgl,
  826. unsigned int sg_len, enum dma_transfer_direction direction,
  827. unsigned long flags, void *context)
  828. {
  829. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  830. struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
  831. const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
  832. struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
  833. struct s3c24xx_txd *txd;
  834. struct s3c24xx_sg *dsg;
  835. struct scatterlist *sg;
  836. dma_addr_t slave_addr;
  837. u32 hwcfg = 0;
  838. int tmp;
  839. dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
  840. sg_dma_len(sgl), s3cchan->name);
  841. txd = s3c24xx_dma_get_txd();
  842. if (!txd)
  843. return NULL;
  844. if (cdata->handshake)
  845. txd->dcon |= S3C24XX_DCON_HANDSHAKE;
  846. switch (cdata->bus) {
  847. case S3C24XX_DMA_APB:
  848. txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
  849. hwcfg |= S3C24XX_DISRCC_LOC_APB;
  850. break;
  851. case S3C24XX_DMA_AHB:
  852. txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
  853. hwcfg |= S3C24XX_DISRCC_LOC_AHB;
  854. break;
  855. }
  856. /*
  857. * Always assume our peripheral desintation is a fixed
  858. * address in memory.
  859. */
  860. hwcfg |= S3C24XX_DISRCC_INC_FIXED;
  861. /*
  862. * Individual dma operations are requested by the slave,
  863. * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
  864. */
  865. txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
  866. if (direction == DMA_MEM_TO_DEV) {
  867. txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
  868. S3C24XX_DISRCC_INC_INCREMENT;
  869. txd->didstc = hwcfg;
  870. slave_addr = s3cchan->cfg.dst_addr;
  871. txd->width = s3cchan->cfg.dst_addr_width;
  872. } else if (direction == DMA_DEV_TO_MEM) {
  873. txd->disrcc = hwcfg;
  874. txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
  875. S3C24XX_DIDSTC_INC_INCREMENT;
  876. slave_addr = s3cchan->cfg.src_addr;
  877. txd->width = s3cchan->cfg.src_addr_width;
  878. } else {
  879. s3c24xx_dma_free_txd(txd);
  880. dev_err(&s3cdma->pdev->dev,
  881. "direction %d unsupported\n", direction);
  882. return NULL;
  883. }
  884. for_each_sg(sgl, sg, sg_len, tmp) {
  885. dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
  886. if (!dsg) {
  887. s3c24xx_dma_free_txd(txd);
  888. return NULL;
  889. }
  890. list_add_tail(&dsg->node, &txd->dsg_list);
  891. dsg->len = sg_dma_len(sg);
  892. if (direction == DMA_MEM_TO_DEV) {
  893. dsg->src_addr = sg_dma_address(sg);
  894. dsg->dst_addr = slave_addr;
  895. } else { /* DMA_DEV_TO_MEM */
  896. dsg->src_addr = slave_addr;
  897. dsg->dst_addr = sg_dma_address(sg);
  898. }
  899. }
  900. return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
  901. }
  902. /*
  903. * Slave transactions callback to the slave device to allow
  904. * synchronization of slave DMA signals with the DMAC enable
  905. */
  906. static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
  907. {
  908. struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
  909. unsigned long flags;
  910. spin_lock_irqsave(&s3cchan->vc.lock, flags);
  911. if (vchan_issue_pending(&s3cchan->vc)) {
  912. if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
  913. s3c24xx_dma_phy_alloc_and_start(s3cchan);
  914. }
  915. spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
  916. }
  917. /*
  918. * Bringup and teardown
  919. */
  920. /*
  921. * Initialise the DMAC memcpy/slave channels.
  922. * Make a local wrapper to hold required data
  923. */
  924. static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
  925. struct dma_device *dmadev, unsigned int channels, bool slave)
  926. {
  927. struct s3c24xx_dma_chan *chan;
  928. int i;
  929. INIT_LIST_HEAD(&dmadev->channels);
  930. /*
  931. * Register as many many memcpy as we have physical channels,
  932. * we won't always be able to use all but the code will have
  933. * to cope with that situation.
  934. */
  935. for (i = 0; i < channels; i++) {
  936. chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
  937. if (!chan)
  938. return -ENOMEM;
  939. chan->id = i;
  940. chan->host = s3cdma;
  941. chan->state = S3C24XX_DMA_CHAN_IDLE;
  942. if (slave) {
  943. chan->slave = true;
  944. chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
  945. if (!chan->name)
  946. return -ENOMEM;
  947. } else {
  948. chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
  949. if (!chan->name)
  950. return -ENOMEM;
  951. }
  952. dev_dbg(dmadev->dev,
  953. "initialize virtual channel \"%s\"\n",
  954. chan->name);
  955. chan->vc.desc_free = s3c24xx_dma_desc_free;
  956. vchan_init(&chan->vc, dmadev);
  957. }
  958. dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
  959. i, slave ? "slave" : "memcpy");
  960. return i;
  961. }
  962. static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
  963. {
  964. struct s3c24xx_dma_chan *chan = NULL;
  965. struct s3c24xx_dma_chan *next;
  966. list_for_each_entry_safe(chan,
  967. next, &dmadev->channels, vc.chan.device_node) {
  968. list_del(&chan->vc.chan.device_node);
  969. tasklet_kill(&chan->vc.task);
  970. }
  971. }
  972. /* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */
  973. static struct soc_data soc_s3c2410 = {
  974. .stride = 0x40,
  975. .has_reqsel = false,
  976. .has_clocks = false,
  977. };
  978. /* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */
  979. static struct soc_data soc_s3c2412 = {
  980. .stride = 0x40,
  981. .has_reqsel = true,
  982. .has_clocks = true,
  983. };
  984. /* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */
  985. static struct soc_data soc_s3c2443 = {
  986. .stride = 0x100,
  987. .has_reqsel = true,
  988. .has_clocks = true,
  989. };
  990. static const struct platform_device_id s3c24xx_dma_driver_ids[] = {
  991. {
  992. .name = "s3c2410-dma",
  993. .driver_data = (kernel_ulong_t)&soc_s3c2410,
  994. }, {
  995. .name = "s3c2412-dma",
  996. .driver_data = (kernel_ulong_t)&soc_s3c2412,
  997. }, {
  998. .name = "s3c2443-dma",
  999. .driver_data = (kernel_ulong_t)&soc_s3c2443,
  1000. },
  1001. { },
  1002. };
  1003. static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
  1004. {
  1005. return (struct soc_data *)
  1006. platform_get_device_id(pdev)->driver_data;
  1007. }
  1008. static int s3c24xx_dma_probe(struct platform_device *pdev)
  1009. {
  1010. const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
  1011. struct s3c24xx_dma_engine *s3cdma;
  1012. struct soc_data *sdata;
  1013. struct resource *res;
  1014. int ret;
  1015. int i;
  1016. if (!pdata) {
  1017. dev_err(&pdev->dev, "platform data missing\n");
  1018. return -ENODEV;
  1019. }
  1020. /* Basic sanity check */
  1021. if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
  1022. dev_err(&pdev->dev, "too many dma channels %d, max %d\n",
  1023. pdata->num_phy_channels, MAX_DMA_CHANNELS);
  1024. return -EINVAL;
  1025. }
  1026. sdata = s3c24xx_dma_get_soc_data(pdev);
  1027. if (!sdata)
  1028. return -EINVAL;
  1029. s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
  1030. if (!s3cdma)
  1031. return -ENOMEM;
  1032. s3cdma->pdev = pdev;
  1033. s3cdma->pdata = pdata;
  1034. s3cdma->sdata = sdata;
  1035. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1036. s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
  1037. if (IS_ERR(s3cdma->base))
  1038. return PTR_ERR(s3cdma->base);
  1039. s3cdma->phy_chans = devm_kcalloc(&pdev->dev,
  1040. pdata->num_phy_channels,
  1041. sizeof(struct s3c24xx_dma_phy),
  1042. GFP_KERNEL);
  1043. if (!s3cdma->phy_chans)
  1044. return -ENOMEM;
  1045. /* acquire irqs and clocks for all physical channels */
  1046. for (i = 0; i < pdata->num_phy_channels; i++) {
  1047. struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  1048. char clk_name[6];
  1049. phy->id = i;
  1050. phy->base = s3cdma->base + (i * sdata->stride);
  1051. phy->host = s3cdma;
  1052. phy->irq = platform_get_irq(pdev, i);
  1053. if (phy->irq < 0)
  1054. continue;
  1055. ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
  1056. 0, pdev->name, phy);
  1057. if (ret) {
  1058. dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
  1059. i, ret);
  1060. continue;
  1061. }
  1062. if (sdata->has_clocks) {
  1063. sprintf(clk_name, "dma.%d", i);
  1064. phy->clk = devm_clk_get(&pdev->dev, clk_name);
  1065. if (IS_ERR(phy->clk) && sdata->has_clocks) {
  1066. dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n",
  1067. i, PTR_ERR(phy->clk));
  1068. continue;
  1069. }
  1070. ret = clk_prepare(phy->clk);
  1071. if (ret) {
  1072. dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
  1073. i, ret);
  1074. continue;
  1075. }
  1076. }
  1077. spin_lock_init(&phy->lock);
  1078. phy->valid = true;
  1079. dev_dbg(&pdev->dev, "physical channel %d is %s\n",
  1080. i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
  1081. }
  1082. /* Initialize memcpy engine */
  1083. dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
  1084. dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
  1085. s3cdma->memcpy.dev = &pdev->dev;
  1086. s3cdma->memcpy.device_free_chan_resources =
  1087. s3c24xx_dma_free_chan_resources;
  1088. s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
  1089. s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
  1090. s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
  1091. s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
  1092. s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
  1093. s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize;
  1094. /* Initialize slave engine for SoC internal dedicated peripherals */
  1095. dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
  1096. dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
  1097. dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
  1098. s3cdma->slave.dev = &pdev->dev;
  1099. s3cdma->slave.device_free_chan_resources =
  1100. s3c24xx_dma_free_chan_resources;
  1101. s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
  1102. s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
  1103. s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
  1104. s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
  1105. s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
  1106. s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
  1107. s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize;
  1108. s3cdma->slave.filter.map = pdata->slave_map;
  1109. s3cdma->slave.filter.mapcnt = pdata->slavecnt;
  1110. s3cdma->slave.filter.fn = s3c24xx_dma_filter;
  1111. /* Register as many memcpy channels as there are physical channels */
  1112. ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
  1113. pdata->num_phy_channels, false);
  1114. if (ret <= 0) {
  1115. dev_warn(&pdev->dev,
  1116. "%s failed to enumerate memcpy channels - %d\n",
  1117. __func__, ret);
  1118. goto err_memcpy;
  1119. }
  1120. /* Register slave channels */
  1121. ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
  1122. pdata->num_channels, true);
  1123. if (ret <= 0) {
  1124. dev_warn(&pdev->dev,
  1125. "%s failed to enumerate slave channels - %d\n",
  1126. __func__, ret);
  1127. goto err_slave;
  1128. }
  1129. ret = dma_async_device_register(&s3cdma->memcpy);
  1130. if (ret) {
  1131. dev_warn(&pdev->dev,
  1132. "%s failed to register memcpy as an async device - %d\n",
  1133. __func__, ret);
  1134. goto err_memcpy_reg;
  1135. }
  1136. ret = dma_async_device_register(&s3cdma->slave);
  1137. if (ret) {
  1138. dev_warn(&pdev->dev,
  1139. "%s failed to register slave as an async device - %d\n",
  1140. __func__, ret);
  1141. goto err_slave_reg;
  1142. }
  1143. platform_set_drvdata(pdev, s3cdma);
  1144. dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
  1145. pdata->num_phy_channels);
  1146. return 0;
  1147. err_slave_reg:
  1148. dma_async_device_unregister(&s3cdma->memcpy);
  1149. err_memcpy_reg:
  1150. s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
  1151. err_slave:
  1152. s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
  1153. err_memcpy:
  1154. if (sdata->has_clocks)
  1155. for (i = 0; i < pdata->num_phy_channels; i++) {
  1156. struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  1157. if (phy->valid)
  1158. clk_unprepare(phy->clk);
  1159. }
  1160. return ret;
  1161. }
  1162. static void s3c24xx_dma_free_irq(struct platform_device *pdev,
  1163. struct s3c24xx_dma_engine *s3cdma)
  1164. {
  1165. int i;
  1166. for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
  1167. struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  1168. devm_free_irq(&pdev->dev, phy->irq, phy);
  1169. }
  1170. }
  1171. static int s3c24xx_dma_remove(struct platform_device *pdev)
  1172. {
  1173. const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
  1174. struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
  1175. struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
  1176. int i;
  1177. dma_async_device_unregister(&s3cdma->slave);
  1178. dma_async_device_unregister(&s3cdma->memcpy);
  1179. s3c24xx_dma_free_irq(pdev, s3cdma);
  1180. s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
  1181. s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
  1182. if (sdata->has_clocks)
  1183. for (i = 0; i < pdata->num_phy_channels; i++) {
  1184. struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
  1185. if (phy->valid)
  1186. clk_unprepare(phy->clk);
  1187. }
  1188. return 0;
  1189. }
  1190. static struct platform_driver s3c24xx_dma_driver = {
  1191. .driver = {
  1192. .name = "s3c24xx-dma",
  1193. },
  1194. .id_table = s3c24xx_dma_driver_ids,
  1195. .probe = s3c24xx_dma_probe,
  1196. .remove = s3c24xx_dma_remove,
  1197. };
  1198. module_platform_driver(s3c24xx_dma_driver);
  1199. bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
  1200. {
  1201. struct s3c24xx_dma_chan *s3cchan;
  1202. if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
  1203. return false;
  1204. s3cchan = to_s3c24xx_dma_chan(chan);
  1205. return s3cchan->id == (uintptr_t)param;
  1206. }
  1207. EXPORT_SYMBOL(s3c24xx_dma_filter);
  1208. MODULE_DESCRIPTION("S3C24XX DMA Driver");
  1209. MODULE_AUTHOR("Heiko Stuebner");
  1210. MODULE_LICENSE("GPL v2");