iop-adma.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright © 2006, Intel Corporation.
  4. */
  5. #ifndef _ADMA_H
  6. #define _ADMA_H
  7. #include <linux/types.h>
  8. #include <linux/io.h>
  9. #include <linux/platform_data/dma-iop32x.h>
  10. /* Memory copy units */
  11. #define DMA_CCR(chan) (chan->mmr_base + 0x0)
  12. #define DMA_CSR(chan) (chan->mmr_base + 0x4)
  13. #define DMA_DAR(chan) (chan->mmr_base + 0xc)
  14. #define DMA_NDAR(chan) (chan->mmr_base + 0x10)
  15. #define DMA_PADR(chan) (chan->mmr_base + 0x14)
  16. #define DMA_PUADR(chan) (chan->mmr_base + 0x18)
  17. #define DMA_LADR(chan) (chan->mmr_base + 0x1c)
  18. #define DMA_BCR(chan) (chan->mmr_base + 0x20)
  19. #define DMA_DCR(chan) (chan->mmr_base + 0x24)
  20. /* Application accelerator unit */
  21. #define AAU_ACR(chan) (chan->mmr_base + 0x0)
  22. #define AAU_ASR(chan) (chan->mmr_base + 0x4)
  23. #define AAU_ADAR(chan) (chan->mmr_base + 0x8)
  24. #define AAU_ANDAR(chan) (chan->mmr_base + 0xc)
  25. #define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2)))
  26. #define AAU_DAR(chan) (chan->mmr_base + 0x20)
  27. #define AAU_ABCR(chan) (chan->mmr_base + 0x24)
  28. #define AAU_ADCR(chan) (chan->mmr_base + 0x28)
  29. #define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2)))
  30. #define AAU_EDCR0_IDX 8
  31. #define AAU_EDCR1_IDX 17
  32. #define AAU_EDCR2_IDX 26
  33. struct iop3xx_aau_desc_ctrl {
  34. unsigned int int_en:1;
  35. unsigned int blk1_cmd_ctrl:3;
  36. unsigned int blk2_cmd_ctrl:3;
  37. unsigned int blk3_cmd_ctrl:3;
  38. unsigned int blk4_cmd_ctrl:3;
  39. unsigned int blk5_cmd_ctrl:3;
  40. unsigned int blk6_cmd_ctrl:3;
  41. unsigned int blk7_cmd_ctrl:3;
  42. unsigned int blk8_cmd_ctrl:3;
  43. unsigned int blk_ctrl:2;
  44. unsigned int dual_xor_en:1;
  45. unsigned int tx_complete:1;
  46. unsigned int zero_result_err:1;
  47. unsigned int zero_result_en:1;
  48. unsigned int dest_write_en:1;
  49. };
  50. struct iop3xx_aau_e_desc_ctrl {
  51. unsigned int reserved:1;
  52. unsigned int blk1_cmd_ctrl:3;
  53. unsigned int blk2_cmd_ctrl:3;
  54. unsigned int blk3_cmd_ctrl:3;
  55. unsigned int blk4_cmd_ctrl:3;
  56. unsigned int blk5_cmd_ctrl:3;
  57. unsigned int blk6_cmd_ctrl:3;
  58. unsigned int blk7_cmd_ctrl:3;
  59. unsigned int blk8_cmd_ctrl:3;
  60. unsigned int reserved2:7;
  61. };
  62. struct iop3xx_dma_desc_ctrl {
  63. unsigned int pci_transaction:4;
  64. unsigned int int_en:1;
  65. unsigned int dac_cycle_en:1;
  66. unsigned int mem_to_mem_en:1;
  67. unsigned int crc_data_tx_en:1;
  68. unsigned int crc_gen_en:1;
  69. unsigned int crc_seed_dis:1;
  70. unsigned int reserved:21;
  71. unsigned int crc_tx_complete:1;
  72. };
  73. struct iop3xx_desc_dma {
  74. u32 next_desc;
  75. union {
  76. u32 pci_src_addr;
  77. u32 pci_dest_addr;
  78. u32 src_addr;
  79. };
  80. union {
  81. u32 upper_pci_src_addr;
  82. u32 upper_pci_dest_addr;
  83. };
  84. union {
  85. u32 local_pci_src_addr;
  86. u32 local_pci_dest_addr;
  87. u32 dest_addr;
  88. };
  89. u32 byte_count;
  90. union {
  91. u32 desc_ctrl;
  92. struct iop3xx_dma_desc_ctrl desc_ctrl_field;
  93. };
  94. u32 crc_addr;
  95. };
  96. struct iop3xx_desc_aau {
  97. u32 next_desc;
  98. u32 src[4];
  99. u32 dest_addr;
  100. u32 byte_count;
  101. union {
  102. u32 desc_ctrl;
  103. struct iop3xx_aau_desc_ctrl desc_ctrl_field;
  104. };
  105. union {
  106. u32 src_addr;
  107. u32 e_desc_ctrl;
  108. struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
  109. } src_edc[31];
  110. };
  111. struct iop3xx_aau_gfmr {
  112. unsigned int gfmr1:8;
  113. unsigned int gfmr2:8;
  114. unsigned int gfmr3:8;
  115. unsigned int gfmr4:8;
  116. };
  117. struct iop3xx_desc_pq_xor {
  118. u32 next_desc;
  119. u32 src[3];
  120. union {
  121. u32 data_mult1;
  122. struct iop3xx_aau_gfmr data_mult1_field;
  123. };
  124. u32 dest_addr;
  125. u32 byte_count;
  126. union {
  127. u32 desc_ctrl;
  128. struct iop3xx_aau_desc_ctrl desc_ctrl_field;
  129. };
  130. union {
  131. u32 src_addr;
  132. u32 e_desc_ctrl;
  133. struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
  134. u32 data_multiplier;
  135. struct iop3xx_aau_gfmr data_mult_field;
  136. u32 reserved;
  137. } src_edc_gfmr[19];
  138. };
  139. struct iop3xx_desc_dual_xor {
  140. u32 next_desc;
  141. u32 src0_addr;
  142. u32 src1_addr;
  143. u32 h_src_addr;
  144. u32 d_src_addr;
  145. u32 h_dest_addr;
  146. u32 byte_count;
  147. union {
  148. u32 desc_ctrl;
  149. struct iop3xx_aau_desc_ctrl desc_ctrl_field;
  150. };
  151. u32 d_dest_addr;
  152. };
  153. union iop3xx_desc {
  154. struct iop3xx_desc_aau *aau;
  155. struct iop3xx_desc_dma *dma;
  156. struct iop3xx_desc_pq_xor *pq_xor;
  157. struct iop3xx_desc_dual_xor *dual_xor;
  158. void *ptr;
  159. };
  160. /* No support for p+q operations */
  161. static inline int
  162. iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op)
  163. {
  164. BUG();
  165. return 0;
  166. }
  167. static inline void
  168. iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
  169. unsigned long flags)
  170. {
  171. BUG();
  172. }
  173. static inline void
  174. iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
  175. {
  176. BUG();
  177. }
  178. static inline void
  179. iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
  180. dma_addr_t addr, unsigned char coef)
  181. {
  182. BUG();
  183. }
  184. static inline int
  185. iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op)
  186. {
  187. BUG();
  188. return 0;
  189. }
  190. static inline void
  191. iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
  192. unsigned long flags)
  193. {
  194. BUG();
  195. }
  196. static inline void
  197. iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
  198. {
  199. BUG();
  200. }
  201. #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
  202. static inline void
  203. iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
  204. dma_addr_t *src)
  205. {
  206. BUG();
  207. }
  208. static inline int iop_adma_get_max_xor(void)
  209. {
  210. return 32;
  211. }
  212. static inline int iop_adma_get_max_pq(void)
  213. {
  214. BUG();
  215. return 0;
  216. }
  217. static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
  218. {
  219. int id = chan->device->id;
  220. switch (id) {
  221. case DMA0_ID:
  222. case DMA1_ID:
  223. return __raw_readl(DMA_DAR(chan));
  224. case AAU_ID:
  225. return __raw_readl(AAU_ADAR(chan));
  226. default:
  227. BUG();
  228. }
  229. return 0;
  230. }
  231. static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan,
  232. u32 next_desc_addr)
  233. {
  234. int id = chan->device->id;
  235. switch (id) {
  236. case DMA0_ID:
  237. case DMA1_ID:
  238. __raw_writel(next_desc_addr, DMA_NDAR(chan));
  239. break;
  240. case AAU_ID:
  241. __raw_writel(next_desc_addr, AAU_ANDAR(chan));
  242. break;
  243. }
  244. }
  245. #define IOP_ADMA_STATUS_BUSY (1 << 10)
  246. #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024)
  247. #define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024)
  248. #define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
  249. static inline int iop_chan_is_busy(struct iop_adma_chan *chan)
  250. {
  251. u32 status = __raw_readl(DMA_CSR(chan));
  252. return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0;
  253. }
  254. static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc,
  255. int num_slots)
  256. {
  257. /* num_slots will only ever be 1, 2, 4, or 8 */
  258. return (desc->idx & (num_slots - 1)) ? 0 : 1;
  259. }
  260. /* to do: support large (i.e. > hw max) buffer sizes */
  261. static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op)
  262. {
  263. *slots_per_op = 1;
  264. return 1;
  265. }
  266. /* to do: support large (i.e. > hw max) buffer sizes */
  267. static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op)
  268. {
  269. *slots_per_op = 1;
  270. return 1;
  271. }
  272. static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt,
  273. int *slots_per_op)
  274. {
  275. static const char slot_count_table[] = {
  276. 1, 1, 1, 1, /* 01 - 04 */
  277. 2, 2, 2, 2, /* 05 - 08 */
  278. 4, 4, 4, 4, /* 09 - 12 */
  279. 4, 4, 4, 4, /* 13 - 16 */
  280. 8, 8, 8, 8, /* 17 - 20 */
  281. 8, 8, 8, 8, /* 21 - 24 */
  282. 8, 8, 8, 8, /* 25 - 28 */
  283. 8, 8, 8, 8, /* 29 - 32 */
  284. };
  285. *slots_per_op = slot_count_table[src_cnt - 1];
  286. return *slots_per_op;
  287. }
  288. static inline int
  289. iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan)
  290. {
  291. switch (chan->device->id) {
  292. case DMA0_ID:
  293. case DMA1_ID:
  294. return iop_chan_memcpy_slot_count(0, slots_per_op);
  295. case AAU_ID:
  296. return iop3xx_aau_xor_slot_count(0, 2, slots_per_op);
  297. default:
  298. BUG();
  299. }
  300. return 0;
  301. }
  302. static inline int iop_chan_xor_slot_count(size_t len, int src_cnt,
  303. int *slots_per_op)
  304. {
  305. int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
  306. if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT)
  307. return slot_cnt;
  308. len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
  309. while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) {
  310. len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
  311. slot_cnt += *slots_per_op;
  312. }
  313. slot_cnt += *slots_per_op;
  314. return slot_cnt;
  315. }
  316. /* zero sum on iop3xx is limited to 1k at a time so it requires multiple
  317. * descriptors
  318. */
  319. static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
  320. int *slots_per_op)
  321. {
  322. int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
  323. if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT)
  324. return slot_cnt;
  325. len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
  326. while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
  327. len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
  328. slot_cnt += *slots_per_op;
  329. }
  330. slot_cnt += *slots_per_op;
  331. return slot_cnt;
  332. }
  333. static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
  334. struct iop_adma_chan *chan)
  335. {
  336. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  337. switch (chan->device->id) {
  338. case DMA0_ID:
  339. case DMA1_ID:
  340. return hw_desc.dma->byte_count;
  341. case AAU_ID:
  342. return hw_desc.aau->byte_count;
  343. default:
  344. BUG();
  345. }
  346. return 0;
  347. }
  348. /* translate the src_idx to a descriptor word index */
  349. static inline int __desc_idx(int src_idx)
  350. {
  351. static const int desc_idx_table[] = { 0, 0, 0, 0,
  352. 0, 1, 2, 3,
  353. 5, 6, 7, 8,
  354. 9, 10, 11, 12,
  355. 14, 15, 16, 17,
  356. 18, 19, 20, 21,
  357. 23, 24, 25, 26,
  358. 27, 28, 29, 30,
  359. };
  360. return desc_idx_table[src_idx];
  361. }
  362. static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc,
  363. struct iop_adma_chan *chan,
  364. int src_idx)
  365. {
  366. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  367. switch (chan->device->id) {
  368. case DMA0_ID:
  369. case DMA1_ID:
  370. return hw_desc.dma->src_addr;
  371. case AAU_ID:
  372. break;
  373. default:
  374. BUG();
  375. }
  376. if (src_idx < 4)
  377. return hw_desc.aau->src[src_idx];
  378. else
  379. return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr;
  380. }
  381. static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc,
  382. int src_idx, dma_addr_t addr)
  383. {
  384. if (src_idx < 4)
  385. hw_desc->src[src_idx] = addr;
  386. else
  387. hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr;
  388. }
  389. static inline void
  390. iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags)
  391. {
  392. struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
  393. union {
  394. u32 value;
  395. struct iop3xx_dma_desc_ctrl field;
  396. } u_desc_ctrl;
  397. u_desc_ctrl.value = 0;
  398. u_desc_ctrl.field.mem_to_mem_en = 1;
  399. u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */
  400. u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
  401. hw_desc->desc_ctrl = u_desc_ctrl.value;
  402. hw_desc->upper_pci_src_addr = 0;
  403. hw_desc->crc_addr = 0;
  404. }
  405. static inline void
  406. iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags)
  407. {
  408. struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
  409. union {
  410. u32 value;
  411. struct iop3xx_aau_desc_ctrl field;
  412. } u_desc_ctrl;
  413. u_desc_ctrl.value = 0;
  414. u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */
  415. u_desc_ctrl.field.dest_write_en = 1;
  416. u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
  417. hw_desc->desc_ctrl = u_desc_ctrl.value;
  418. }
  419. static inline u32
  420. iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt,
  421. unsigned long flags)
  422. {
  423. int i, shift;
  424. u32 edcr;
  425. union {
  426. u32 value;
  427. struct iop3xx_aau_desc_ctrl field;
  428. } u_desc_ctrl;
  429. u_desc_ctrl.value = 0;
  430. switch (src_cnt) {
  431. case 25 ... 32:
  432. u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
  433. edcr = 0;
  434. shift = 1;
  435. for (i = 24; i < src_cnt; i++) {
  436. edcr |= (1 << shift);
  437. shift += 3;
  438. }
  439. hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr;
  440. src_cnt = 24;
  441. fallthrough;
  442. case 17 ... 24:
  443. if (!u_desc_ctrl.field.blk_ctrl) {
  444. hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
  445. u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
  446. }
  447. edcr = 0;
  448. shift = 1;
  449. for (i = 16; i < src_cnt; i++) {
  450. edcr |= (1 << shift);
  451. shift += 3;
  452. }
  453. hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr;
  454. src_cnt = 16;
  455. fallthrough;
  456. case 9 ... 16:
  457. if (!u_desc_ctrl.field.blk_ctrl)
  458. u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
  459. edcr = 0;
  460. shift = 1;
  461. for (i = 8; i < src_cnt; i++) {
  462. edcr |= (1 << shift);
  463. shift += 3;
  464. }
  465. hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr;
  466. src_cnt = 8;
  467. fallthrough;
  468. case 2 ... 8:
  469. shift = 1;
  470. for (i = 0; i < src_cnt; i++) {
  471. u_desc_ctrl.value |= (1 << shift);
  472. shift += 3;
  473. }
  474. if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
  475. u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
  476. }
  477. u_desc_ctrl.field.dest_write_en = 1;
  478. u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */
  479. u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
  480. hw_desc->desc_ctrl = u_desc_ctrl.value;
  481. return u_desc_ctrl.value;
  482. }
  483. static inline void
  484. iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt,
  485. unsigned long flags)
  486. {
  487. iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags);
  488. }
  489. /* return the number of operations */
  490. static inline int
  491. iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
  492. unsigned long flags)
  493. {
  494. int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
  495. struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter;
  496. union {
  497. u32 value;
  498. struct iop3xx_aau_desc_ctrl field;
  499. } u_desc_ctrl;
  500. int i, j;
  501. hw_desc = desc->hw_desc;
  502. for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0;
  503. i += slots_per_op, j++) {
  504. iter = iop_hw_desc_slot_idx(hw_desc, i);
  505. u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags);
  506. u_desc_ctrl.field.dest_write_en = 0;
  507. u_desc_ctrl.field.zero_result_en = 1;
  508. u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
  509. iter->desc_ctrl = u_desc_ctrl.value;
  510. /* for the subsequent descriptors preserve the store queue
  511. * and chain them together
  512. */
  513. if (i) {
  514. prev_hw_desc =
  515. iop_hw_desc_slot_idx(hw_desc, i - slots_per_op);
  516. prev_hw_desc->next_desc =
  517. (u32) (desc->async_tx.phys + (i << 5));
  518. }
  519. }
  520. return j;
  521. }
  522. static inline void
  523. iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt,
  524. unsigned long flags)
  525. {
  526. struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
  527. union {
  528. u32 value;
  529. struct iop3xx_aau_desc_ctrl field;
  530. } u_desc_ctrl;
  531. u_desc_ctrl.value = 0;
  532. switch (src_cnt) {
  533. case 25 ... 32:
  534. u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
  535. hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
  536. fallthrough;
  537. case 17 ... 24:
  538. if (!u_desc_ctrl.field.blk_ctrl) {
  539. hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
  540. u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
  541. }
  542. hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0;
  543. fallthrough;
  544. case 9 ... 16:
  545. if (!u_desc_ctrl.field.blk_ctrl)
  546. u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
  547. hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0;
  548. fallthrough;
  549. case 1 ... 8:
  550. if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
  551. u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
  552. }
  553. u_desc_ctrl.field.dest_write_en = 0;
  554. u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
  555. hw_desc->desc_ctrl = u_desc_ctrl.value;
  556. }
  557. static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
  558. struct iop_adma_chan *chan,
  559. u32 byte_count)
  560. {
  561. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  562. switch (chan->device->id) {
  563. case DMA0_ID:
  564. case DMA1_ID:
  565. hw_desc.dma->byte_count = byte_count;
  566. break;
  567. case AAU_ID:
  568. hw_desc.aau->byte_count = byte_count;
  569. break;
  570. default:
  571. BUG();
  572. }
  573. }
  574. static inline void
  575. iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
  576. struct iop_adma_chan *chan)
  577. {
  578. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  579. switch (chan->device->id) {
  580. case DMA0_ID:
  581. case DMA1_ID:
  582. iop_desc_init_memcpy(desc, 1);
  583. hw_desc.dma->byte_count = 0;
  584. hw_desc.dma->dest_addr = 0;
  585. hw_desc.dma->src_addr = 0;
  586. break;
  587. case AAU_ID:
  588. iop_desc_init_null_xor(desc, 2, 1);
  589. hw_desc.aau->byte_count = 0;
  590. hw_desc.aau->dest_addr = 0;
  591. hw_desc.aau->src[0] = 0;
  592. hw_desc.aau->src[1] = 0;
  593. break;
  594. default:
  595. BUG();
  596. }
  597. }
  598. static inline void
  599. iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
  600. {
  601. int slots_per_op = desc->slots_per_op;
  602. struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
  603. int i = 0;
  604. if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
  605. hw_desc->byte_count = len;
  606. } else {
  607. do {
  608. iter = iop_hw_desc_slot_idx(hw_desc, i);
  609. iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
  610. len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
  611. i += slots_per_op;
  612. } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT);
  613. iter = iop_hw_desc_slot_idx(hw_desc, i);
  614. iter->byte_count = len;
  615. }
  616. }
  617. static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
  618. struct iop_adma_chan *chan,
  619. dma_addr_t addr)
  620. {
  621. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  622. switch (chan->device->id) {
  623. case DMA0_ID:
  624. case DMA1_ID:
  625. hw_desc.dma->dest_addr = addr;
  626. break;
  627. case AAU_ID:
  628. hw_desc.aau->dest_addr = addr;
  629. break;
  630. default:
  631. BUG();
  632. }
  633. }
  634. static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
  635. dma_addr_t addr)
  636. {
  637. struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
  638. hw_desc->src_addr = addr;
  639. }
  640. static inline void
  641. iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
  642. dma_addr_t addr)
  643. {
  644. struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
  645. int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
  646. int i;
  647. for (i = 0; (slot_cnt -= slots_per_op) >= 0;
  648. i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
  649. iter = iop_hw_desc_slot_idx(hw_desc, i);
  650. iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
  651. }
  652. }
  653. static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
  654. int src_idx, dma_addr_t addr)
  655. {
  656. struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
  657. int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
  658. int i;
  659. for (i = 0; (slot_cnt -= slots_per_op) >= 0;
  660. i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) {
  661. iter = iop_hw_desc_slot_idx(hw_desc, i);
  662. iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
  663. }
  664. }
  665. static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
  666. u32 next_desc_addr)
  667. {
  668. /* hw_desc->next_desc is the same location for all channels */
  669. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  670. iop_paranoia(hw_desc.dma->next_desc);
  671. hw_desc.dma->next_desc = next_desc_addr;
  672. }
  673. static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc)
  674. {
  675. /* hw_desc->next_desc is the same location for all channels */
  676. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  677. return hw_desc.dma->next_desc;
  678. }
  679. static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc)
  680. {
  681. /* hw_desc->next_desc is the same location for all channels */
  682. union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
  683. hw_desc.dma->next_desc = 0;
  684. }
  685. static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
  686. u32 val)
  687. {
  688. struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
  689. hw_desc->src[0] = val;
  690. }
  691. static inline enum sum_check_flags
  692. iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
  693. {
  694. struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
  695. struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
  696. iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en));
  697. return desc_ctrl.zero_result_err << SUM_CHECK_P;
  698. }
  699. static inline void iop_chan_append(struct iop_adma_chan *chan)
  700. {
  701. u32 dma_chan_ctrl;
  702. dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
  703. dma_chan_ctrl |= 0x2;
  704. __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
  705. }
  706. static inline u32 iop_chan_get_status(struct iop_adma_chan *chan)
  707. {
  708. return __raw_readl(DMA_CSR(chan));
  709. }
  710. static inline void iop_chan_disable(struct iop_adma_chan *chan)
  711. {
  712. u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
  713. dma_chan_ctrl &= ~1;
  714. __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
  715. }
  716. static inline void iop_chan_enable(struct iop_adma_chan *chan)
  717. {
  718. u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
  719. dma_chan_ctrl |= 1;
  720. __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
  721. }
  722. static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan)
  723. {
  724. u32 status = __raw_readl(DMA_CSR(chan));
  725. status &= (1 << 9);
  726. __raw_writel(status, DMA_CSR(chan));
  727. }
  728. static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan)
  729. {
  730. u32 status = __raw_readl(DMA_CSR(chan));
  731. status &= (1 << 8);
  732. __raw_writel(status, DMA_CSR(chan));
  733. }
  734. static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan)
  735. {
  736. u32 status = __raw_readl(DMA_CSR(chan));
  737. switch (chan->device->id) {
  738. case DMA0_ID:
  739. case DMA1_ID:
  740. status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1);
  741. break;
  742. case AAU_ID:
  743. status &= (1 << 5);
  744. break;
  745. default:
  746. BUG();
  747. }
  748. __raw_writel(status, DMA_CSR(chan));
  749. }
  750. static inline int
  751. iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan)
  752. {
  753. return 0;
  754. }
  755. static inline int
  756. iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan)
  757. {
  758. return 0;
  759. }
  760. static inline int
  761. iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan)
  762. {
  763. return 0;
  764. }
  765. static inline int
  766. iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan)
  767. {
  768. return test_bit(5, &status);
  769. }
  770. static inline int
  771. iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan)
  772. {
  773. switch (chan->device->id) {
  774. case DMA0_ID:
  775. case DMA1_ID:
  776. return test_bit(2, &status);
  777. default:
  778. return 0;
  779. }
  780. }
  781. static inline int
  782. iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan)
  783. {
  784. switch (chan->device->id) {
  785. case DMA0_ID:
  786. case DMA1_ID:
  787. return test_bit(3, &status);
  788. default:
  789. return 0;
  790. }
  791. }
  792. static inline int
  793. iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan)
  794. {
  795. switch (chan->device->id) {
  796. case DMA0_ID:
  797. case DMA1_ID:
  798. return test_bit(1, &status);
  799. default:
  800. return 0;
  801. }
  802. }
  803. #endif /* _ADMA_H */