pch_dma.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Topcliff PCH DMA controller driver
  4. * Copyright (c) 2010 Intel Corporation
  5. * Copyright (C) 2011 LAPIS Semiconductor Co., Ltd.
  6. */
  7. #include <linux/dmaengine.h>
  8. #include <linux/dma-mapping.h>
  9. #include <linux/init.h>
  10. #include <linux/pci.h>
  11. #include <linux/slab.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/module.h>
  14. #include <linux/pch_dma.h>
  15. #include "dmaengine.h"
  16. #define DRV_NAME "pch-dma"
  17. #define DMA_CTL0_DISABLE 0x0
  18. #define DMA_CTL0_SG 0x1
  19. #define DMA_CTL0_ONESHOT 0x2
  20. #define DMA_CTL0_MODE_MASK_BITS 0x3
  21. #define DMA_CTL0_DIR_SHIFT_BITS 2
  22. #define DMA_CTL0_BITS_PER_CH 4
  23. #define DMA_CTL2_START_SHIFT_BITS 8
  24. #define DMA_CTL2_IRQ_ENABLE_MASK ((1UL << DMA_CTL2_START_SHIFT_BITS) - 1)
  25. #define DMA_STATUS_IDLE 0x0
  26. #define DMA_STATUS_DESC_READ 0x1
  27. #define DMA_STATUS_WAIT 0x2
  28. #define DMA_STATUS_ACCESS 0x3
  29. #define DMA_STATUS_BITS_PER_CH 2
  30. #define DMA_STATUS_MASK_BITS 0x3
  31. #define DMA_STATUS_SHIFT_BITS 16
  32. #define DMA_STATUS_IRQ(x) (0x1 << (x))
  33. #define DMA_STATUS0_ERR(x) (0x1 << ((x) + 8))
  34. #define DMA_STATUS2_ERR(x) (0x1 << (x))
  35. #define DMA_DESC_WIDTH_SHIFT_BITS 12
  36. #define DMA_DESC_WIDTH_1_BYTE (0x3 << DMA_DESC_WIDTH_SHIFT_BITS)
  37. #define DMA_DESC_WIDTH_2_BYTES (0x2 << DMA_DESC_WIDTH_SHIFT_BITS)
  38. #define DMA_DESC_WIDTH_4_BYTES (0x0 << DMA_DESC_WIDTH_SHIFT_BITS)
  39. #define DMA_DESC_MAX_COUNT_1_BYTE 0x3FF
  40. #define DMA_DESC_MAX_COUNT_2_BYTES 0x3FF
  41. #define DMA_DESC_MAX_COUNT_4_BYTES 0x7FF
  42. #define DMA_DESC_END_WITHOUT_IRQ 0x0
  43. #define DMA_DESC_END_WITH_IRQ 0x1
  44. #define DMA_DESC_FOLLOW_WITHOUT_IRQ 0x2
  45. #define DMA_DESC_FOLLOW_WITH_IRQ 0x3
  46. #define MAX_CHAN_NR 12
  47. #define DMA_MASK_CTL0_MODE 0x33333333
  48. #define DMA_MASK_CTL2_MODE 0x00003333
  49. static unsigned int init_nr_desc_per_channel = 64;
  50. module_param(init_nr_desc_per_channel, uint, 0644);
  51. MODULE_PARM_DESC(init_nr_desc_per_channel,
  52. "initial descriptors per channel (default: 64)");
  53. struct pch_dma_desc_regs {
  54. u32 dev_addr;
  55. u32 mem_addr;
  56. u32 size;
  57. u32 next;
  58. };
  59. struct pch_dma_regs {
  60. u32 dma_ctl0;
  61. u32 dma_ctl1;
  62. u32 dma_ctl2;
  63. u32 dma_ctl3;
  64. u32 dma_sts0;
  65. u32 dma_sts1;
  66. u32 dma_sts2;
  67. u32 reserved3;
  68. struct pch_dma_desc_regs desc[MAX_CHAN_NR];
  69. };
  70. struct pch_dma_desc {
  71. struct pch_dma_desc_regs regs;
  72. struct dma_async_tx_descriptor txd;
  73. struct list_head desc_node;
  74. struct list_head tx_list;
  75. };
  76. struct pch_dma_chan {
  77. struct dma_chan chan;
  78. void __iomem *membase;
  79. enum dma_transfer_direction dir;
  80. struct tasklet_struct tasklet;
  81. unsigned long err_status;
  82. spinlock_t lock;
  83. struct list_head active_list;
  84. struct list_head queue;
  85. struct list_head free_list;
  86. unsigned int descs_allocated;
  87. };
  88. #define PDC_DEV_ADDR 0x00
  89. #define PDC_MEM_ADDR 0x04
  90. #define PDC_SIZE 0x08
  91. #define PDC_NEXT 0x0C
  92. #define channel_readl(pdc, name) \
  93. readl((pdc)->membase + PDC_##name)
  94. #define channel_writel(pdc, name, val) \
  95. writel((val), (pdc)->membase + PDC_##name)
  96. struct pch_dma {
  97. struct dma_device dma;
  98. void __iomem *membase;
  99. struct dma_pool *pool;
  100. struct pch_dma_regs regs;
  101. struct pch_dma_desc_regs ch_regs[MAX_CHAN_NR];
  102. struct pch_dma_chan channels[MAX_CHAN_NR];
  103. };
  104. #define PCH_DMA_CTL0 0x00
  105. #define PCH_DMA_CTL1 0x04
  106. #define PCH_DMA_CTL2 0x08
  107. #define PCH_DMA_CTL3 0x0C
  108. #define PCH_DMA_STS0 0x10
  109. #define PCH_DMA_STS1 0x14
  110. #define PCH_DMA_STS2 0x18
  111. #define dma_readl(pd, name) \
  112. readl((pd)->membase + PCH_DMA_##name)
  113. #define dma_writel(pd, name, val) \
  114. writel((val), (pd)->membase + PCH_DMA_##name)
  115. static inline
  116. struct pch_dma_desc *to_pd_desc(struct dma_async_tx_descriptor *txd)
  117. {
  118. return container_of(txd, struct pch_dma_desc, txd);
  119. }
  120. static inline struct pch_dma_chan *to_pd_chan(struct dma_chan *chan)
  121. {
  122. return container_of(chan, struct pch_dma_chan, chan);
  123. }
  124. static inline struct pch_dma *to_pd(struct dma_device *ddev)
  125. {
  126. return container_of(ddev, struct pch_dma, dma);
  127. }
  128. static inline struct device *chan2dev(struct dma_chan *chan)
  129. {
  130. return &chan->dev->device;
  131. }
  132. static inline struct device *chan2parent(struct dma_chan *chan)
  133. {
  134. return chan->dev->device.parent;
  135. }
  136. static inline
  137. struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan)
  138. {
  139. return list_first_entry(&pd_chan->active_list,
  140. struct pch_dma_desc, desc_node);
  141. }
  142. static inline
  143. struct pch_dma_desc *pdc_first_queued(struct pch_dma_chan *pd_chan)
  144. {
  145. return list_first_entry(&pd_chan->queue,
  146. struct pch_dma_desc, desc_node);
  147. }
  148. static void pdc_enable_irq(struct dma_chan *chan, int enable)
  149. {
  150. struct pch_dma *pd = to_pd(chan->device);
  151. u32 val;
  152. int pos;
  153. if (chan->chan_id < 8)
  154. pos = chan->chan_id;
  155. else
  156. pos = chan->chan_id + 8;
  157. val = dma_readl(pd, CTL2);
  158. if (enable)
  159. val |= 0x1 << pos;
  160. else
  161. val &= ~(0x1 << pos);
  162. dma_writel(pd, CTL2, val);
  163. dev_dbg(chan2dev(chan), "pdc_enable_irq: chan %d -> %x\n",
  164. chan->chan_id, val);
  165. }
  166. static void pdc_set_dir(struct dma_chan *chan)
  167. {
  168. struct pch_dma_chan *pd_chan = to_pd_chan(chan);
  169. struct pch_dma *pd = to_pd(chan->device);
  170. u32 val;
  171. u32 mask_mode;
  172. u32 mask_ctl;
  173. if (chan->chan_id < 8) {
  174. val = dma_readl(pd, CTL0);
  175. mask_mode = DMA_CTL0_MODE_MASK_BITS <<
  176. (DMA_CTL0_BITS_PER_CH * chan->chan_id);
  177. mask_ctl = DMA_MASK_CTL0_MODE & ~(DMA_CTL0_MODE_MASK_BITS <<
  178. (DMA_CTL0_BITS_PER_CH * chan->chan_id));
  179. val &= mask_mode;
  180. if (pd_chan->dir == DMA_MEM_TO_DEV)
  181. val |= 0x1 << (DMA_CTL0_BITS_PER_CH * chan->chan_id +
  182. DMA_CTL0_DIR_SHIFT_BITS);
  183. else
  184. val &= ~(0x1 << (DMA_CTL0_BITS_PER_CH * chan->chan_id +
  185. DMA_CTL0_DIR_SHIFT_BITS));
  186. val |= mask_ctl;
  187. dma_writel(pd, CTL0, val);
  188. } else {
  189. int ch = chan->chan_id - 8; /* ch8-->0 ch9-->1 ... ch11->3 */
  190. val = dma_readl(pd, CTL3);
  191. mask_mode = DMA_CTL0_MODE_MASK_BITS <<
  192. (DMA_CTL0_BITS_PER_CH * ch);
  193. mask_ctl = DMA_MASK_CTL2_MODE & ~(DMA_CTL0_MODE_MASK_BITS <<
  194. (DMA_CTL0_BITS_PER_CH * ch));
  195. val &= mask_mode;
  196. if (pd_chan->dir == DMA_MEM_TO_DEV)
  197. val |= 0x1 << (DMA_CTL0_BITS_PER_CH * ch +
  198. DMA_CTL0_DIR_SHIFT_BITS);
  199. else
  200. val &= ~(0x1 << (DMA_CTL0_BITS_PER_CH * ch +
  201. DMA_CTL0_DIR_SHIFT_BITS));
  202. val |= mask_ctl;
  203. dma_writel(pd, CTL3, val);
  204. }
  205. dev_dbg(chan2dev(chan), "pdc_set_dir: chan %d -> %x\n",
  206. chan->chan_id, val);
  207. }
  208. static void pdc_set_mode(struct dma_chan *chan, u32 mode)
  209. {
  210. struct pch_dma *pd = to_pd(chan->device);
  211. u32 val;
  212. u32 mask_ctl;
  213. u32 mask_dir;
  214. if (chan->chan_id < 8) {
  215. mask_ctl = DMA_MASK_CTL0_MODE & ~(DMA_CTL0_MODE_MASK_BITS <<
  216. (DMA_CTL0_BITS_PER_CH * chan->chan_id));
  217. mask_dir = 1 << (DMA_CTL0_BITS_PER_CH * chan->chan_id +\
  218. DMA_CTL0_DIR_SHIFT_BITS);
  219. val = dma_readl(pd, CTL0);
  220. val &= mask_dir;
  221. val |= mode << (DMA_CTL0_BITS_PER_CH * chan->chan_id);
  222. val |= mask_ctl;
  223. dma_writel(pd, CTL0, val);
  224. } else {
  225. int ch = chan->chan_id - 8; /* ch8-->0 ch9-->1 ... ch11->3 */
  226. mask_ctl = DMA_MASK_CTL2_MODE & ~(DMA_CTL0_MODE_MASK_BITS <<
  227. (DMA_CTL0_BITS_PER_CH * ch));
  228. mask_dir = 1 << (DMA_CTL0_BITS_PER_CH * ch +\
  229. DMA_CTL0_DIR_SHIFT_BITS);
  230. val = dma_readl(pd, CTL3);
  231. val &= mask_dir;
  232. val |= mode << (DMA_CTL0_BITS_PER_CH * ch);
  233. val |= mask_ctl;
  234. dma_writel(pd, CTL3, val);
  235. }
  236. dev_dbg(chan2dev(chan), "pdc_set_mode: chan %d -> %x\n",
  237. chan->chan_id, val);
  238. }
  239. static u32 pdc_get_status0(struct pch_dma_chan *pd_chan)
  240. {
  241. struct pch_dma *pd = to_pd(pd_chan->chan.device);
  242. u32 val;
  243. val = dma_readl(pd, STS0);
  244. return DMA_STATUS_MASK_BITS & (val >> (DMA_STATUS_SHIFT_BITS +
  245. DMA_STATUS_BITS_PER_CH * pd_chan->chan.chan_id));
  246. }
  247. static u32 pdc_get_status2(struct pch_dma_chan *pd_chan)
  248. {
  249. struct pch_dma *pd = to_pd(pd_chan->chan.device);
  250. u32 val;
  251. val = dma_readl(pd, STS2);
  252. return DMA_STATUS_MASK_BITS & (val >> (DMA_STATUS_SHIFT_BITS +
  253. DMA_STATUS_BITS_PER_CH * (pd_chan->chan.chan_id - 8)));
  254. }
  255. static bool pdc_is_idle(struct pch_dma_chan *pd_chan)
  256. {
  257. u32 sts;
  258. if (pd_chan->chan.chan_id < 8)
  259. sts = pdc_get_status0(pd_chan);
  260. else
  261. sts = pdc_get_status2(pd_chan);
  262. if (sts == DMA_STATUS_IDLE)
  263. return true;
  264. else
  265. return false;
  266. }
  267. static void pdc_dostart(struct pch_dma_chan *pd_chan, struct pch_dma_desc* desc)
  268. {
  269. if (!pdc_is_idle(pd_chan)) {
  270. dev_err(chan2dev(&pd_chan->chan),
  271. "BUG: Attempt to start non-idle channel\n");
  272. return;
  273. }
  274. dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> dev_addr: %x\n",
  275. pd_chan->chan.chan_id, desc->regs.dev_addr);
  276. dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> mem_addr: %x\n",
  277. pd_chan->chan.chan_id, desc->regs.mem_addr);
  278. dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> size: %x\n",
  279. pd_chan->chan.chan_id, desc->regs.size);
  280. dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> next: %x\n",
  281. pd_chan->chan.chan_id, desc->regs.next);
  282. if (list_empty(&desc->tx_list)) {
  283. channel_writel(pd_chan, DEV_ADDR, desc->regs.dev_addr);
  284. channel_writel(pd_chan, MEM_ADDR, desc->regs.mem_addr);
  285. channel_writel(pd_chan, SIZE, desc->regs.size);
  286. channel_writel(pd_chan, NEXT, desc->regs.next);
  287. pdc_set_mode(&pd_chan->chan, DMA_CTL0_ONESHOT);
  288. } else {
  289. channel_writel(pd_chan, NEXT, desc->txd.phys);
  290. pdc_set_mode(&pd_chan->chan, DMA_CTL0_SG);
  291. }
  292. }
  293. static void pdc_chain_complete(struct pch_dma_chan *pd_chan,
  294. struct pch_dma_desc *desc)
  295. {
  296. struct dma_async_tx_descriptor *txd = &desc->txd;
  297. struct dmaengine_desc_callback cb;
  298. dmaengine_desc_get_callback(txd, &cb);
  299. list_splice_init(&desc->tx_list, &pd_chan->free_list);
  300. list_move(&desc->desc_node, &pd_chan->free_list);
  301. dmaengine_desc_callback_invoke(&cb, NULL);
  302. }
  303. static void pdc_complete_all(struct pch_dma_chan *pd_chan)
  304. {
  305. struct pch_dma_desc *desc, *_d;
  306. LIST_HEAD(list);
  307. BUG_ON(!pdc_is_idle(pd_chan));
  308. if (!list_empty(&pd_chan->queue))
  309. pdc_dostart(pd_chan, pdc_first_queued(pd_chan));
  310. list_splice_init(&pd_chan->active_list, &list);
  311. list_splice_init(&pd_chan->queue, &pd_chan->active_list);
  312. list_for_each_entry_safe(desc, _d, &list, desc_node)
  313. pdc_chain_complete(pd_chan, desc);
  314. }
  315. static void pdc_handle_error(struct pch_dma_chan *pd_chan)
  316. {
  317. struct pch_dma_desc *bad_desc;
  318. bad_desc = pdc_first_active(pd_chan);
  319. list_del(&bad_desc->desc_node);
  320. list_splice_init(&pd_chan->queue, pd_chan->active_list.prev);
  321. if (!list_empty(&pd_chan->active_list))
  322. pdc_dostart(pd_chan, pdc_first_active(pd_chan));
  323. dev_crit(chan2dev(&pd_chan->chan), "Bad descriptor submitted\n");
  324. dev_crit(chan2dev(&pd_chan->chan), "descriptor cookie: %d\n",
  325. bad_desc->txd.cookie);
  326. pdc_chain_complete(pd_chan, bad_desc);
  327. }
  328. static void pdc_advance_work(struct pch_dma_chan *pd_chan)
  329. {
  330. if (list_empty(&pd_chan->active_list) ||
  331. list_is_singular(&pd_chan->active_list)) {
  332. pdc_complete_all(pd_chan);
  333. } else {
  334. pdc_chain_complete(pd_chan, pdc_first_active(pd_chan));
  335. pdc_dostart(pd_chan, pdc_first_active(pd_chan));
  336. }
  337. }
  338. static dma_cookie_t pd_tx_submit(struct dma_async_tx_descriptor *txd)
  339. {
  340. struct pch_dma_desc *desc = to_pd_desc(txd);
  341. struct pch_dma_chan *pd_chan = to_pd_chan(txd->chan);
  342. spin_lock(&pd_chan->lock);
  343. if (list_empty(&pd_chan->active_list)) {
  344. list_add_tail(&desc->desc_node, &pd_chan->active_list);
  345. pdc_dostart(pd_chan, desc);
  346. } else {
  347. list_add_tail(&desc->desc_node, &pd_chan->queue);
  348. }
  349. spin_unlock(&pd_chan->lock);
  350. return 0;
  351. }
  352. static struct pch_dma_desc *pdc_alloc_desc(struct dma_chan *chan, gfp_t flags)
  353. {
  354. struct pch_dma_desc *desc = NULL;
  355. struct pch_dma *pd = to_pd(chan->device);
  356. dma_addr_t addr;
  357. desc = dma_pool_zalloc(pd->pool, flags, &addr);
  358. if (desc) {
  359. INIT_LIST_HEAD(&desc->tx_list);
  360. dma_async_tx_descriptor_init(&desc->txd, chan);
  361. desc->txd.tx_submit = pd_tx_submit;
  362. desc->txd.flags = DMA_CTRL_ACK;
  363. desc->txd.phys = addr;
  364. }
  365. return desc;
  366. }
  367. static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan)
  368. {
  369. struct pch_dma_desc *desc, *_d;
  370. struct pch_dma_desc *ret = NULL;
  371. int i = 0;
  372. spin_lock(&pd_chan->lock);
  373. list_for_each_entry_safe(desc, _d, &pd_chan->free_list, desc_node) {
  374. i++;
  375. if (async_tx_test_ack(&desc->txd)) {
  376. list_del(&desc->desc_node);
  377. ret = desc;
  378. break;
  379. }
  380. dev_dbg(chan2dev(&pd_chan->chan), "desc %p not ACKed\n", desc);
  381. }
  382. spin_unlock(&pd_chan->lock);
  383. dev_dbg(chan2dev(&pd_chan->chan), "scanned %d descriptors\n", i);
  384. if (!ret) {
  385. ret = pdc_alloc_desc(&pd_chan->chan, GFP_ATOMIC);
  386. if (ret) {
  387. spin_lock(&pd_chan->lock);
  388. pd_chan->descs_allocated++;
  389. spin_unlock(&pd_chan->lock);
  390. } else {
  391. dev_err(chan2dev(&pd_chan->chan),
  392. "failed to alloc desc\n");
  393. }
  394. }
  395. return ret;
  396. }
  397. static void pdc_desc_put(struct pch_dma_chan *pd_chan,
  398. struct pch_dma_desc *desc)
  399. {
  400. if (desc) {
  401. spin_lock(&pd_chan->lock);
  402. list_splice_init(&desc->tx_list, &pd_chan->free_list);
  403. list_add(&desc->desc_node, &pd_chan->free_list);
  404. spin_unlock(&pd_chan->lock);
  405. }
  406. }
  407. static int pd_alloc_chan_resources(struct dma_chan *chan)
  408. {
  409. struct pch_dma_chan *pd_chan = to_pd_chan(chan);
  410. struct pch_dma_desc *desc;
  411. LIST_HEAD(tmp_list);
  412. int i;
  413. if (!pdc_is_idle(pd_chan)) {
  414. dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
  415. return -EIO;
  416. }
  417. if (!list_empty(&pd_chan->free_list))
  418. return pd_chan->descs_allocated;
  419. for (i = 0; i < init_nr_desc_per_channel; i++) {
  420. desc = pdc_alloc_desc(chan, GFP_KERNEL);
  421. if (!desc) {
  422. dev_warn(chan2dev(chan),
  423. "Only allocated %d initial descriptors\n", i);
  424. break;
  425. }
  426. list_add_tail(&desc->desc_node, &tmp_list);
  427. }
  428. spin_lock_irq(&pd_chan->lock);
  429. list_splice(&tmp_list, &pd_chan->free_list);
  430. pd_chan->descs_allocated = i;
  431. dma_cookie_init(chan);
  432. spin_unlock_irq(&pd_chan->lock);
  433. pdc_enable_irq(chan, 1);
  434. return pd_chan->descs_allocated;
  435. }
  436. static void pd_free_chan_resources(struct dma_chan *chan)
  437. {
  438. struct pch_dma_chan *pd_chan = to_pd_chan(chan);
  439. struct pch_dma *pd = to_pd(chan->device);
  440. struct pch_dma_desc *desc, *_d;
  441. LIST_HEAD(tmp_list);
  442. BUG_ON(!pdc_is_idle(pd_chan));
  443. BUG_ON(!list_empty(&pd_chan->active_list));
  444. BUG_ON(!list_empty(&pd_chan->queue));
  445. spin_lock_irq(&pd_chan->lock);
  446. list_splice_init(&pd_chan->free_list, &tmp_list);
  447. pd_chan->descs_allocated = 0;
  448. spin_unlock_irq(&pd_chan->lock);
  449. list_for_each_entry_safe(desc, _d, &tmp_list, desc_node)
  450. dma_pool_free(pd->pool, desc, desc->txd.phys);
  451. pdc_enable_irq(chan, 0);
  452. }
  453. static enum dma_status pd_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
  454. struct dma_tx_state *txstate)
  455. {
  456. return dma_cookie_status(chan, cookie, txstate);
  457. }
  458. static void pd_issue_pending(struct dma_chan *chan)
  459. {
  460. struct pch_dma_chan *pd_chan = to_pd_chan(chan);
  461. if (pdc_is_idle(pd_chan)) {
  462. spin_lock(&pd_chan->lock);
  463. pdc_advance_work(pd_chan);
  464. spin_unlock(&pd_chan->lock);
  465. }
  466. }
  467. static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan,
  468. struct scatterlist *sgl, unsigned int sg_len,
  469. enum dma_transfer_direction direction, unsigned long flags,
  470. void *context)
  471. {
  472. struct pch_dma_chan *pd_chan = to_pd_chan(chan);
  473. struct pch_dma_slave *pd_slave = chan->private;
  474. struct pch_dma_desc *first = NULL;
  475. struct pch_dma_desc *prev = NULL;
  476. struct pch_dma_desc *desc = NULL;
  477. struct scatterlist *sg;
  478. dma_addr_t reg;
  479. int i;
  480. if (unlikely(!sg_len)) {
  481. dev_info(chan2dev(chan), "prep_slave_sg: length is zero!\n");
  482. return NULL;
  483. }
  484. if (direction == DMA_DEV_TO_MEM)
  485. reg = pd_slave->rx_reg;
  486. else if (direction == DMA_MEM_TO_DEV)
  487. reg = pd_slave->tx_reg;
  488. else
  489. return NULL;
  490. pd_chan->dir = direction;
  491. pdc_set_dir(chan);
  492. for_each_sg(sgl, sg, sg_len, i) {
  493. desc = pdc_desc_get(pd_chan);
  494. if (!desc)
  495. goto err_desc_get;
  496. desc->regs.dev_addr = reg;
  497. desc->regs.mem_addr = sg_dma_address(sg);
  498. desc->regs.size = sg_dma_len(sg);
  499. desc->regs.next = DMA_DESC_FOLLOW_WITHOUT_IRQ;
  500. switch (pd_slave->width) {
  501. case PCH_DMA_WIDTH_1_BYTE:
  502. if (desc->regs.size > DMA_DESC_MAX_COUNT_1_BYTE)
  503. goto err_desc_get;
  504. desc->regs.size |= DMA_DESC_WIDTH_1_BYTE;
  505. break;
  506. case PCH_DMA_WIDTH_2_BYTES:
  507. if (desc->regs.size > DMA_DESC_MAX_COUNT_2_BYTES)
  508. goto err_desc_get;
  509. desc->regs.size |= DMA_DESC_WIDTH_2_BYTES;
  510. break;
  511. case PCH_DMA_WIDTH_4_BYTES:
  512. if (desc->regs.size > DMA_DESC_MAX_COUNT_4_BYTES)
  513. goto err_desc_get;
  514. desc->regs.size |= DMA_DESC_WIDTH_4_BYTES;
  515. break;
  516. default:
  517. goto err_desc_get;
  518. }
  519. if (!first) {
  520. first = desc;
  521. } else {
  522. prev->regs.next |= desc->txd.phys;
  523. list_add_tail(&desc->desc_node, &first->tx_list);
  524. }
  525. prev = desc;
  526. }
  527. if (flags & DMA_PREP_INTERRUPT)
  528. desc->regs.next = DMA_DESC_END_WITH_IRQ;
  529. else
  530. desc->regs.next = DMA_DESC_END_WITHOUT_IRQ;
  531. first->txd.cookie = -EBUSY;
  532. desc->txd.flags = flags;
  533. return &first->txd;
  534. err_desc_get:
  535. dev_err(chan2dev(chan), "failed to get desc or wrong parameters\n");
  536. pdc_desc_put(pd_chan, first);
  537. return NULL;
  538. }
  539. static int pd_device_terminate_all(struct dma_chan *chan)
  540. {
  541. struct pch_dma_chan *pd_chan = to_pd_chan(chan);
  542. struct pch_dma_desc *desc, *_d;
  543. LIST_HEAD(list);
  544. spin_lock_irq(&pd_chan->lock);
  545. pdc_set_mode(&pd_chan->chan, DMA_CTL0_DISABLE);
  546. list_splice_init(&pd_chan->active_list, &list);
  547. list_splice_init(&pd_chan->queue, &list);
  548. list_for_each_entry_safe(desc, _d, &list, desc_node)
  549. pdc_chain_complete(pd_chan, desc);
  550. spin_unlock_irq(&pd_chan->lock);
  551. return 0;
  552. }
  553. static void pdc_tasklet(struct tasklet_struct *t)
  554. {
  555. struct pch_dma_chan *pd_chan = from_tasklet(pd_chan, t, tasklet);
  556. unsigned long flags;
  557. if (!pdc_is_idle(pd_chan)) {
  558. dev_err(chan2dev(&pd_chan->chan),
  559. "BUG: handle non-idle channel in tasklet\n");
  560. return;
  561. }
  562. spin_lock_irqsave(&pd_chan->lock, flags);
  563. if (test_and_clear_bit(0, &pd_chan->err_status))
  564. pdc_handle_error(pd_chan);
  565. else
  566. pdc_advance_work(pd_chan);
  567. spin_unlock_irqrestore(&pd_chan->lock, flags);
  568. }
  569. static irqreturn_t pd_irq(int irq, void *devid)
  570. {
  571. struct pch_dma *pd = (struct pch_dma *)devid;
  572. struct pch_dma_chan *pd_chan;
  573. u32 sts0;
  574. u32 sts2;
  575. int i;
  576. int ret0 = IRQ_NONE;
  577. int ret2 = IRQ_NONE;
  578. sts0 = dma_readl(pd, STS0);
  579. sts2 = dma_readl(pd, STS2);
  580. dev_dbg(pd->dma.dev, "pd_irq sts0: %x\n", sts0);
  581. for (i = 0; i < pd->dma.chancnt; i++) {
  582. pd_chan = &pd->channels[i];
  583. if (i < 8) {
  584. if (sts0 & DMA_STATUS_IRQ(i)) {
  585. if (sts0 & DMA_STATUS0_ERR(i))
  586. set_bit(0, &pd_chan->err_status);
  587. tasklet_schedule(&pd_chan->tasklet);
  588. ret0 = IRQ_HANDLED;
  589. }
  590. } else {
  591. if (sts2 & DMA_STATUS_IRQ(i - 8)) {
  592. if (sts2 & DMA_STATUS2_ERR(i))
  593. set_bit(0, &pd_chan->err_status);
  594. tasklet_schedule(&pd_chan->tasklet);
  595. ret2 = IRQ_HANDLED;
  596. }
  597. }
  598. }
  599. /* clear interrupt bits in status register */
  600. if (ret0)
  601. dma_writel(pd, STS0, sts0);
  602. if (ret2)
  603. dma_writel(pd, STS2, sts2);
  604. return ret0 | ret2;
  605. }
  606. static void __maybe_unused pch_dma_save_regs(struct pch_dma *pd)
  607. {
  608. struct pch_dma_chan *pd_chan;
  609. struct dma_chan *chan, *_c;
  610. int i = 0;
  611. pd->regs.dma_ctl0 = dma_readl(pd, CTL0);
  612. pd->regs.dma_ctl1 = dma_readl(pd, CTL1);
  613. pd->regs.dma_ctl2 = dma_readl(pd, CTL2);
  614. pd->regs.dma_ctl3 = dma_readl(pd, CTL3);
  615. list_for_each_entry_safe(chan, _c, &pd->dma.channels, device_node) {
  616. pd_chan = to_pd_chan(chan);
  617. pd->ch_regs[i].dev_addr = channel_readl(pd_chan, DEV_ADDR);
  618. pd->ch_regs[i].mem_addr = channel_readl(pd_chan, MEM_ADDR);
  619. pd->ch_regs[i].size = channel_readl(pd_chan, SIZE);
  620. pd->ch_regs[i].next = channel_readl(pd_chan, NEXT);
  621. i++;
  622. }
  623. }
  624. static void __maybe_unused pch_dma_restore_regs(struct pch_dma *pd)
  625. {
  626. struct pch_dma_chan *pd_chan;
  627. struct dma_chan *chan, *_c;
  628. int i = 0;
  629. dma_writel(pd, CTL0, pd->regs.dma_ctl0);
  630. dma_writel(pd, CTL1, pd->regs.dma_ctl1);
  631. dma_writel(pd, CTL2, pd->regs.dma_ctl2);
  632. dma_writel(pd, CTL3, pd->regs.dma_ctl3);
  633. list_for_each_entry_safe(chan, _c, &pd->dma.channels, device_node) {
  634. pd_chan = to_pd_chan(chan);
  635. channel_writel(pd_chan, DEV_ADDR, pd->ch_regs[i].dev_addr);
  636. channel_writel(pd_chan, MEM_ADDR, pd->ch_regs[i].mem_addr);
  637. channel_writel(pd_chan, SIZE, pd->ch_regs[i].size);
  638. channel_writel(pd_chan, NEXT, pd->ch_regs[i].next);
  639. i++;
  640. }
  641. }
  642. static int __maybe_unused pch_dma_suspend(struct device *dev)
  643. {
  644. struct pch_dma *pd = dev_get_drvdata(dev);
  645. if (pd)
  646. pch_dma_save_regs(pd);
  647. return 0;
  648. }
  649. static int __maybe_unused pch_dma_resume(struct device *dev)
  650. {
  651. struct pch_dma *pd = dev_get_drvdata(dev);
  652. if (pd)
  653. pch_dma_restore_regs(pd);
  654. return 0;
  655. }
  656. static int pch_dma_probe(struct pci_dev *pdev,
  657. const struct pci_device_id *id)
  658. {
  659. struct pch_dma *pd;
  660. struct pch_dma_regs *regs;
  661. unsigned int nr_channels;
  662. int err;
  663. int i;
  664. nr_channels = id->driver_data;
  665. pd = kzalloc(sizeof(*pd), GFP_KERNEL);
  666. if (!pd)
  667. return -ENOMEM;
  668. pci_set_drvdata(pdev, pd);
  669. err = pci_enable_device(pdev);
  670. if (err) {
  671. dev_err(&pdev->dev, "Cannot enable PCI device\n");
  672. goto err_free_mem;
  673. }
  674. if (!(pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
  675. dev_err(&pdev->dev, "Cannot find proper base address\n");
  676. err = -ENODEV;
  677. goto err_disable_pdev;
  678. }
  679. err = pci_request_regions(pdev, DRV_NAME);
  680. if (err) {
  681. dev_err(&pdev->dev, "Cannot obtain PCI resources\n");
  682. goto err_disable_pdev;
  683. }
  684. err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  685. if (err) {
  686. dev_err(&pdev->dev, "Cannot set proper DMA config\n");
  687. goto err_free_res;
  688. }
  689. regs = pd->membase = pci_iomap(pdev, 1, 0);
  690. if (!pd->membase) {
  691. dev_err(&pdev->dev, "Cannot map MMIO registers\n");
  692. err = -ENOMEM;
  693. goto err_free_res;
  694. }
  695. pci_set_master(pdev);
  696. pd->dma.dev = &pdev->dev;
  697. err = request_irq(pdev->irq, pd_irq, IRQF_SHARED, DRV_NAME, pd);
  698. if (err) {
  699. dev_err(&pdev->dev, "Failed to request IRQ\n");
  700. goto err_iounmap;
  701. }
  702. pd->pool = dma_pool_create("pch_dma_desc_pool", &pdev->dev,
  703. sizeof(struct pch_dma_desc), 4, 0);
  704. if (!pd->pool) {
  705. dev_err(&pdev->dev, "Failed to alloc DMA descriptors\n");
  706. err = -ENOMEM;
  707. goto err_free_irq;
  708. }
  709. INIT_LIST_HEAD(&pd->dma.channels);
  710. for (i = 0; i < nr_channels; i++) {
  711. struct pch_dma_chan *pd_chan = &pd->channels[i];
  712. pd_chan->chan.device = &pd->dma;
  713. dma_cookie_init(&pd_chan->chan);
  714. pd_chan->membase = &regs->desc[i];
  715. spin_lock_init(&pd_chan->lock);
  716. INIT_LIST_HEAD(&pd_chan->active_list);
  717. INIT_LIST_HEAD(&pd_chan->queue);
  718. INIT_LIST_HEAD(&pd_chan->free_list);
  719. tasklet_setup(&pd_chan->tasklet, pdc_tasklet);
  720. list_add_tail(&pd_chan->chan.device_node, &pd->dma.channels);
  721. }
  722. dma_cap_zero(pd->dma.cap_mask);
  723. dma_cap_set(DMA_PRIVATE, pd->dma.cap_mask);
  724. dma_cap_set(DMA_SLAVE, pd->dma.cap_mask);
  725. pd->dma.device_alloc_chan_resources = pd_alloc_chan_resources;
  726. pd->dma.device_free_chan_resources = pd_free_chan_resources;
  727. pd->dma.device_tx_status = pd_tx_status;
  728. pd->dma.device_issue_pending = pd_issue_pending;
  729. pd->dma.device_prep_slave_sg = pd_prep_slave_sg;
  730. pd->dma.device_terminate_all = pd_device_terminate_all;
  731. err = dma_async_device_register(&pd->dma);
  732. if (err) {
  733. dev_err(&pdev->dev, "Failed to register DMA device\n");
  734. goto err_free_pool;
  735. }
  736. return 0;
  737. err_free_pool:
  738. dma_pool_destroy(pd->pool);
  739. err_free_irq:
  740. free_irq(pdev->irq, pd);
  741. err_iounmap:
  742. pci_iounmap(pdev, pd->membase);
  743. err_free_res:
  744. pci_release_regions(pdev);
  745. err_disable_pdev:
  746. pci_disable_device(pdev);
  747. err_free_mem:
  748. kfree(pd);
  749. return err;
  750. }
  751. static void pch_dma_remove(struct pci_dev *pdev)
  752. {
  753. struct pch_dma *pd = pci_get_drvdata(pdev);
  754. struct pch_dma_chan *pd_chan;
  755. struct dma_chan *chan, *_c;
  756. if (pd) {
  757. dma_async_device_unregister(&pd->dma);
  758. free_irq(pdev->irq, pd);
  759. list_for_each_entry_safe(chan, _c, &pd->dma.channels,
  760. device_node) {
  761. pd_chan = to_pd_chan(chan);
  762. tasklet_kill(&pd_chan->tasklet);
  763. }
  764. dma_pool_destroy(pd->pool);
  765. pci_iounmap(pdev, pd->membase);
  766. pci_release_regions(pdev);
  767. pci_disable_device(pdev);
  768. kfree(pd);
  769. }
  770. }
  771. /* PCI Device ID of DMA device */
  772. #define PCI_DEVICE_ID_EG20T_PCH_DMA_8CH 0x8810
  773. #define PCI_DEVICE_ID_EG20T_PCH_DMA_4CH 0x8815
  774. #define PCI_DEVICE_ID_ML7213_DMA1_8CH 0x8026
  775. #define PCI_DEVICE_ID_ML7213_DMA2_8CH 0x802B
  776. #define PCI_DEVICE_ID_ML7213_DMA3_4CH 0x8034
  777. #define PCI_DEVICE_ID_ML7213_DMA4_12CH 0x8032
  778. #define PCI_DEVICE_ID_ML7223_DMA1_4CH 0x800B
  779. #define PCI_DEVICE_ID_ML7223_DMA2_4CH 0x800E
  780. #define PCI_DEVICE_ID_ML7223_DMA3_4CH 0x8017
  781. #define PCI_DEVICE_ID_ML7223_DMA4_4CH 0x803B
  782. #define PCI_DEVICE_ID_ML7831_DMA1_8CH 0x8810
  783. #define PCI_DEVICE_ID_ML7831_DMA2_4CH 0x8815
  784. static const struct pci_device_id pch_dma_id_table[] = {
  785. { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_8CH), 8 },
  786. { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_4CH), 4 },
  787. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA1_8CH), 8}, /* UART Video */
  788. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA2_8CH), 8}, /* PCMIF SPI */
  789. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA3_4CH), 4}, /* FPGA */
  790. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA4_12CH), 12}, /* I2S */
  791. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7223_DMA1_4CH), 4}, /* UART */
  792. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7223_DMA2_4CH), 4}, /* Video SPI */
  793. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7223_DMA3_4CH), 4}, /* Security */
  794. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7223_DMA4_4CH), 4}, /* FPGA */
  795. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_DMA1_8CH), 8}, /* UART */
  796. { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_DMA2_4CH), 4}, /* SPI */
  797. { 0, },
  798. };
  799. static SIMPLE_DEV_PM_OPS(pch_dma_pm_ops, pch_dma_suspend, pch_dma_resume);
  800. static struct pci_driver pch_dma_driver = {
  801. .name = DRV_NAME,
  802. .id_table = pch_dma_id_table,
  803. .probe = pch_dma_probe,
  804. .remove = pch_dma_remove,
  805. .driver.pm = &pch_dma_pm_ops,
  806. };
  807. module_pci_driver(pch_dma_driver);
  808. MODULE_DESCRIPTION("Intel EG20T PCH / LAPIS Semicon ML7213/ML7223/ML7831 IOH "
  809. "DMA controller driver");
  810. MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>");
  811. MODULE_LICENSE("GPL v2");
  812. MODULE_DEVICE_TABLE(pci, pch_dma_id_table);