imx-sdma.c 58 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218
  1. // SPDX-License-Identifier: GPL-2.0+
  2. //
  3. // drivers/dma/imx-sdma.c
  4. //
  5. // This file contains a driver for the Freescale Smart DMA engine
  6. //
  7. // Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
  8. //
  9. // Based on code from Freescale:
  10. //
  11. // Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
  12. #include <linux/init.h>
  13. #include <linux/iopoll.h>
  14. #include <linux/module.h>
  15. #include <linux/types.h>
  16. #include <linux/bitops.h>
  17. #include <linux/mm.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/clk.h>
  20. #include <linux/delay.h>
  21. #include <linux/sched.h>
  22. #include <linux/semaphore.h>
  23. #include <linux/spinlock.h>
  24. #include <linux/device.h>
  25. #include <linux/dma-mapping.h>
  26. #include <linux/firmware.h>
  27. #include <linux/slab.h>
  28. #include <linux/platform_device.h>
  29. #include <linux/dmaengine.h>
  30. #include <linux/of.h>
  31. #include <linux/of_address.h>
  32. #include <linux/of_device.h>
  33. #include <linux/of_dma.h>
  34. #include <linux/workqueue.h>
  35. #include <asm/irq.h>
  36. #include <linux/platform_data/dma-imx-sdma.h>
  37. #include <linux/platform_data/dma-imx.h>
  38. #include <linux/regmap.h>
  39. #include <linux/mfd/syscon.h>
  40. #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
  41. #include "dmaengine.h"
  42. #include "virt-dma.h"
  43. /* SDMA registers */
  44. #define SDMA_H_C0PTR 0x000
  45. #define SDMA_H_INTR 0x004
  46. #define SDMA_H_STATSTOP 0x008
  47. #define SDMA_H_START 0x00c
  48. #define SDMA_H_EVTOVR 0x010
  49. #define SDMA_H_DSPOVR 0x014
  50. #define SDMA_H_HOSTOVR 0x018
  51. #define SDMA_H_EVTPEND 0x01c
  52. #define SDMA_H_DSPENBL 0x020
  53. #define SDMA_H_RESET 0x024
  54. #define SDMA_H_EVTERR 0x028
  55. #define SDMA_H_INTRMSK 0x02c
  56. #define SDMA_H_PSW 0x030
  57. #define SDMA_H_EVTERRDBG 0x034
  58. #define SDMA_H_CONFIG 0x038
  59. #define SDMA_ONCE_ENB 0x040
  60. #define SDMA_ONCE_DATA 0x044
  61. #define SDMA_ONCE_INSTR 0x048
  62. #define SDMA_ONCE_STAT 0x04c
  63. #define SDMA_ONCE_CMD 0x050
  64. #define SDMA_EVT_MIRROR 0x054
  65. #define SDMA_ILLINSTADDR 0x058
  66. #define SDMA_CHN0ADDR 0x05c
  67. #define SDMA_ONCE_RTB 0x060
  68. #define SDMA_XTRIG_CONF1 0x070
  69. #define SDMA_XTRIG_CONF2 0x074
  70. #define SDMA_CHNENBL0_IMX35 0x200
  71. #define SDMA_CHNENBL0_IMX31 0x080
  72. #define SDMA_CHNPRI_0 0x100
  73. /*
  74. * Buffer descriptor status values.
  75. */
  76. #define BD_DONE 0x01
  77. #define BD_WRAP 0x02
  78. #define BD_CONT 0x04
  79. #define BD_INTR 0x08
  80. #define BD_RROR 0x10
  81. #define BD_LAST 0x20
  82. #define BD_EXTD 0x80
  83. /*
  84. * Data Node descriptor status values.
  85. */
  86. #define DND_END_OF_FRAME 0x80
  87. #define DND_END_OF_XFER 0x40
  88. #define DND_DONE 0x20
  89. #define DND_UNUSED 0x01
  90. /*
  91. * IPCV2 descriptor status values.
  92. */
  93. #define BD_IPCV2_END_OF_FRAME 0x40
  94. #define IPCV2_MAX_NODES 50
  95. /*
  96. * Error bit set in the CCB status field by the SDMA,
  97. * in setbd routine, in case of a transfer error
  98. */
  99. #define DATA_ERROR 0x10000000
  100. /*
  101. * Buffer descriptor commands.
  102. */
  103. #define C0_ADDR 0x01
  104. #define C0_LOAD 0x02
  105. #define C0_DUMP 0x03
  106. #define C0_SETCTX 0x07
  107. #define C0_GETCTX 0x03
  108. #define C0_SETDM 0x01
  109. #define C0_SETPM 0x04
  110. #define C0_GETDM 0x02
  111. #define C0_GETPM 0x08
  112. /*
  113. * Change endianness indicator in the BD command field
  114. */
  115. #define CHANGE_ENDIANNESS 0x80
  116. /*
  117. * p_2_p watermark_level description
  118. * Bits Name Description
  119. * 0-7 Lower WML Lower watermark level
  120. * 8 PS 1: Pad Swallowing
  121. * 0: No Pad Swallowing
  122. * 9 PA 1: Pad Adding
  123. * 0: No Pad Adding
  124. * 10 SPDIF If this bit is set both source
  125. * and destination are on SPBA
  126. * 11 Source Bit(SP) 1: Source on SPBA
  127. * 0: Source on AIPS
  128. * 12 Destination Bit(DP) 1: Destination on SPBA
  129. * 0: Destination on AIPS
  130. * 13-15 --------- MUST BE 0
  131. * 16-23 Higher WML HWML
  132. * 24-27 N Total number of samples after
  133. * which Pad adding/Swallowing
  134. * must be done. It must be odd.
  135. * 28 Lower WML Event(LWE) SDMA events reg to check for
  136. * LWML event mask
  137. * 0: LWE in EVENTS register
  138. * 1: LWE in EVENTS2 register
  139. * 29 Higher WML Event(HWE) SDMA events reg to check for
  140. * HWML event mask
  141. * 0: HWE in EVENTS register
  142. * 1: HWE in EVENTS2 register
  143. * 30 --------- MUST BE 0
  144. * 31 CONT 1: Amount of samples to be
  145. * transferred is unknown and
  146. * script will keep on
  147. * transferring samples as long as
  148. * both events are detected and
  149. * script must be manually stopped
  150. * by the application
  151. * 0: The amount of samples to be
  152. * transferred is equal to the
  153. * count field of mode word
  154. */
  155. #define SDMA_WATERMARK_LEVEL_LWML 0xFF
  156. #define SDMA_WATERMARK_LEVEL_PS BIT(8)
  157. #define SDMA_WATERMARK_LEVEL_PA BIT(9)
  158. #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
  159. #define SDMA_WATERMARK_LEVEL_SP BIT(11)
  160. #define SDMA_WATERMARK_LEVEL_DP BIT(12)
  161. #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
  162. #define SDMA_WATERMARK_LEVEL_LWE BIT(28)
  163. #define SDMA_WATERMARK_LEVEL_HWE BIT(29)
  164. #define SDMA_WATERMARK_LEVEL_CONT BIT(31)
  165. #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  166. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  167. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
  168. #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \
  169. BIT(DMA_MEM_TO_DEV) | \
  170. BIT(DMA_DEV_TO_DEV))
  171. /*
  172. * Mode/Count of data node descriptors - IPCv2
  173. */
  174. struct sdma_mode_count {
  175. #define SDMA_BD_MAX_CNT 0xffff
  176. u32 count : 16; /* size of the buffer pointed by this BD */
  177. u32 status : 8; /* E,R,I,C,W,D status bits stored here */
  178. u32 command : 8; /* command mostly used for channel 0 */
  179. };
  180. /*
  181. * Buffer descriptor
  182. */
  183. struct sdma_buffer_descriptor {
  184. struct sdma_mode_count mode;
  185. u32 buffer_addr; /* address of the buffer described */
  186. u32 ext_buffer_addr; /* extended buffer address */
  187. } __attribute__ ((packed));
  188. /**
  189. * struct sdma_channel_control - Channel control Block
  190. *
  191. * @current_bd_ptr: current buffer descriptor processed
  192. * @base_bd_ptr: first element of buffer descriptor array
  193. * @unused: padding. The SDMA engine expects an array of 128 byte
  194. * control blocks
  195. */
  196. struct sdma_channel_control {
  197. u32 current_bd_ptr;
  198. u32 base_bd_ptr;
  199. u32 unused[2];
  200. } __attribute__ ((packed));
  201. /**
  202. * struct sdma_state_registers - SDMA context for a channel
  203. *
  204. * @pc: program counter
  205. * @unused1: unused
  206. * @t: test bit: status of arithmetic & test instruction
  207. * @rpc: return program counter
  208. * @unused0: unused
  209. * @sf: source fault while loading data
  210. * @spc: loop start program counter
  211. * @unused2: unused
  212. * @df: destination fault while storing data
  213. * @epc: loop end program counter
  214. * @lm: loop mode
  215. */
  216. struct sdma_state_registers {
  217. u32 pc :14;
  218. u32 unused1: 1;
  219. u32 t : 1;
  220. u32 rpc :14;
  221. u32 unused0: 1;
  222. u32 sf : 1;
  223. u32 spc :14;
  224. u32 unused2: 1;
  225. u32 df : 1;
  226. u32 epc :14;
  227. u32 lm : 2;
  228. } __attribute__ ((packed));
  229. /**
  230. * struct sdma_context_data - sdma context specific to a channel
  231. *
  232. * @channel_state: channel state bits
  233. * @gReg: general registers
  234. * @mda: burst dma destination address register
  235. * @msa: burst dma source address register
  236. * @ms: burst dma status register
  237. * @md: burst dma data register
  238. * @pda: peripheral dma destination address register
  239. * @psa: peripheral dma source address register
  240. * @ps: peripheral dma status register
  241. * @pd: peripheral dma data register
  242. * @ca: CRC polynomial register
  243. * @cs: CRC accumulator register
  244. * @dda: dedicated core destination address register
  245. * @dsa: dedicated core source address register
  246. * @ds: dedicated core status register
  247. * @dd: dedicated core data register
  248. * @scratch0: 1st word of dedicated ram for context switch
  249. * @scratch1: 2nd word of dedicated ram for context switch
  250. * @scratch2: 3rd word of dedicated ram for context switch
  251. * @scratch3: 4th word of dedicated ram for context switch
  252. * @scratch4: 5th word of dedicated ram for context switch
  253. * @scratch5: 6th word of dedicated ram for context switch
  254. * @scratch6: 7th word of dedicated ram for context switch
  255. * @scratch7: 8th word of dedicated ram for context switch
  256. */
  257. struct sdma_context_data {
  258. struct sdma_state_registers channel_state;
  259. u32 gReg[8];
  260. u32 mda;
  261. u32 msa;
  262. u32 ms;
  263. u32 md;
  264. u32 pda;
  265. u32 psa;
  266. u32 ps;
  267. u32 pd;
  268. u32 ca;
  269. u32 cs;
  270. u32 dda;
  271. u32 dsa;
  272. u32 ds;
  273. u32 dd;
  274. u32 scratch0;
  275. u32 scratch1;
  276. u32 scratch2;
  277. u32 scratch3;
  278. u32 scratch4;
  279. u32 scratch5;
  280. u32 scratch6;
  281. u32 scratch7;
  282. } __attribute__ ((packed));
  283. struct sdma_engine;
  284. /**
  285. * struct sdma_desc - descriptor structor for one transfer
  286. * @vd: descriptor for virt dma
  287. * @num_bd: number of descriptors currently handling
  288. * @bd_phys: physical address of bd
  289. * @buf_tail: ID of the buffer that was processed
  290. * @buf_ptail: ID of the previous buffer that was processed
  291. * @period_len: period length, used in cyclic.
  292. * @chn_real_count: the real count updated from bd->mode.count
  293. * @chn_count: the transfer count set
  294. * @sdmac: sdma_channel pointer
  295. * @bd: pointer of allocate bd
  296. */
  297. struct sdma_desc {
  298. struct virt_dma_desc vd;
  299. unsigned int num_bd;
  300. dma_addr_t bd_phys;
  301. unsigned int buf_tail;
  302. unsigned int buf_ptail;
  303. unsigned int period_len;
  304. unsigned int chn_real_count;
  305. unsigned int chn_count;
  306. struct sdma_channel *sdmac;
  307. struct sdma_buffer_descriptor *bd;
  308. };
  309. /**
  310. * struct sdma_channel - housekeeping for a SDMA channel
  311. *
  312. * @vc: virt_dma base structure
  313. * @desc: sdma description including vd and other special member
  314. * @sdma: pointer to the SDMA engine for this channel
  315. * @channel: the channel number, matches dmaengine chan_id + 1
  316. * @direction: transfer type. Needed for setting SDMA script
  317. * @slave_config: Slave configuration
  318. * @peripheral_type: Peripheral type. Needed for setting SDMA script
  319. * @event_id0: aka dma request line
  320. * @event_id1: for channels that use 2 events
  321. * @word_size: peripheral access size
  322. * @pc_from_device: script address for those device_2_memory
  323. * @pc_to_device: script address for those memory_2_device
  324. * @device_to_device: script address for those device_2_device
  325. * @pc_to_pc: script address for those memory_2_memory
  326. * @flags: loop mode or not
  327. * @per_address: peripheral source or destination address in common case
  328. * destination address in p_2_p case
  329. * @per_address2: peripheral source address in p_2_p case
  330. * @event_mask: event mask used in p_2_p script
  331. * @watermark_level: value for gReg[7], some script will extend it from
  332. * basic watermark such as p_2_p
  333. * @shp_addr: value for gReg[6]
  334. * @per_addr: value for gReg[2]
  335. * @status: status of dma channel
  336. * @context_loaded: ensure context is only loaded once
  337. * @data: specific sdma interface structure
  338. * @bd_pool: dma_pool for bd
  339. * @terminate_worker: used to call back into terminate work function
  340. */
  341. struct sdma_channel {
  342. struct virt_dma_chan vc;
  343. struct sdma_desc *desc;
  344. struct sdma_engine *sdma;
  345. unsigned int channel;
  346. enum dma_transfer_direction direction;
  347. struct dma_slave_config slave_config;
  348. enum sdma_peripheral_type peripheral_type;
  349. unsigned int event_id0;
  350. unsigned int event_id1;
  351. enum dma_slave_buswidth word_size;
  352. unsigned int pc_from_device, pc_to_device;
  353. unsigned int device_to_device;
  354. unsigned int pc_to_pc;
  355. unsigned long flags;
  356. dma_addr_t per_address, per_address2;
  357. unsigned long event_mask[2];
  358. unsigned long watermark_level;
  359. u32 shp_addr, per_addr;
  360. enum dma_status status;
  361. struct imx_dma_data data;
  362. struct work_struct terminate_worker;
  363. };
  364. #define IMX_DMA_SG_LOOP BIT(0)
  365. #define MAX_DMA_CHANNELS 32
  366. #define MXC_SDMA_DEFAULT_PRIORITY 1
  367. #define MXC_SDMA_MIN_PRIORITY 1
  368. #define MXC_SDMA_MAX_PRIORITY 7
  369. #define SDMA_FIRMWARE_MAGIC 0x414d4453
  370. /**
  371. * struct sdma_firmware_header - Layout of the firmware image
  372. *
  373. * @magic: "SDMA"
  374. * @version_major: increased whenever layout of struct
  375. * sdma_script_start_addrs changes.
  376. * @version_minor: firmware minor version (for binary compatible changes)
  377. * @script_addrs_start: offset of struct sdma_script_start_addrs in this image
  378. * @num_script_addrs: Number of script addresses in this image
  379. * @ram_code_start: offset of SDMA ram image in this firmware image
  380. * @ram_code_size: size of SDMA ram image
  381. * @script_addrs: Stores the start address of the SDMA scripts
  382. * (in SDMA memory space)
  383. */
  384. struct sdma_firmware_header {
  385. u32 magic;
  386. u32 version_major;
  387. u32 version_minor;
  388. u32 script_addrs_start;
  389. u32 num_script_addrs;
  390. u32 ram_code_start;
  391. u32 ram_code_size;
  392. };
  393. struct sdma_driver_data {
  394. int chnenbl0;
  395. int num_events;
  396. struct sdma_script_start_addrs *script_addrs;
  397. bool check_ratio;
  398. };
  399. struct sdma_engine {
  400. struct device *dev;
  401. struct sdma_channel channel[MAX_DMA_CHANNELS];
  402. struct sdma_channel_control *channel_control;
  403. void __iomem *regs;
  404. struct sdma_context_data *context;
  405. dma_addr_t context_phys;
  406. struct dma_device dma_device;
  407. struct clk *clk_ipg;
  408. struct clk *clk_ahb;
  409. spinlock_t channel_0_lock;
  410. u32 script_number;
  411. struct sdma_script_start_addrs *script_addrs;
  412. const struct sdma_driver_data *drvdata;
  413. u32 spba_start_addr;
  414. u32 spba_end_addr;
  415. unsigned int irq;
  416. dma_addr_t bd0_phys;
  417. struct sdma_buffer_descriptor *bd0;
  418. /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/
  419. bool clk_ratio;
  420. };
  421. static int sdma_config_write(struct dma_chan *chan,
  422. struct dma_slave_config *dmaengine_cfg,
  423. enum dma_transfer_direction direction);
  424. static struct sdma_driver_data sdma_imx31 = {
  425. .chnenbl0 = SDMA_CHNENBL0_IMX31,
  426. .num_events = 32,
  427. };
  428. static struct sdma_script_start_addrs sdma_script_imx25 = {
  429. .ap_2_ap_addr = 729,
  430. .uart_2_mcu_addr = 904,
  431. .per_2_app_addr = 1255,
  432. .mcu_2_app_addr = 834,
  433. .uartsh_2_mcu_addr = 1120,
  434. .per_2_shp_addr = 1329,
  435. .mcu_2_shp_addr = 1048,
  436. .ata_2_mcu_addr = 1560,
  437. .mcu_2_ata_addr = 1479,
  438. .app_2_per_addr = 1189,
  439. .app_2_mcu_addr = 770,
  440. .shp_2_per_addr = 1407,
  441. .shp_2_mcu_addr = 979,
  442. };
  443. static struct sdma_driver_data sdma_imx25 = {
  444. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  445. .num_events = 48,
  446. .script_addrs = &sdma_script_imx25,
  447. };
  448. static struct sdma_driver_data sdma_imx35 = {
  449. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  450. .num_events = 48,
  451. };
  452. static struct sdma_script_start_addrs sdma_script_imx51 = {
  453. .ap_2_ap_addr = 642,
  454. .uart_2_mcu_addr = 817,
  455. .mcu_2_app_addr = 747,
  456. .mcu_2_shp_addr = 961,
  457. .ata_2_mcu_addr = 1473,
  458. .mcu_2_ata_addr = 1392,
  459. .app_2_per_addr = 1033,
  460. .app_2_mcu_addr = 683,
  461. .shp_2_per_addr = 1251,
  462. .shp_2_mcu_addr = 892,
  463. };
  464. static struct sdma_driver_data sdma_imx51 = {
  465. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  466. .num_events = 48,
  467. .script_addrs = &sdma_script_imx51,
  468. };
  469. static struct sdma_script_start_addrs sdma_script_imx53 = {
  470. .ap_2_ap_addr = 642,
  471. .app_2_mcu_addr = 683,
  472. .mcu_2_app_addr = 747,
  473. .uart_2_mcu_addr = 817,
  474. .shp_2_mcu_addr = 891,
  475. .mcu_2_shp_addr = 960,
  476. .uartsh_2_mcu_addr = 1032,
  477. .spdif_2_mcu_addr = 1100,
  478. .mcu_2_spdif_addr = 1134,
  479. .firi_2_mcu_addr = 1193,
  480. .mcu_2_firi_addr = 1290,
  481. };
  482. static struct sdma_driver_data sdma_imx53 = {
  483. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  484. .num_events = 48,
  485. .script_addrs = &sdma_script_imx53,
  486. };
  487. static struct sdma_script_start_addrs sdma_script_imx6q = {
  488. .ap_2_ap_addr = 642,
  489. .uart_2_mcu_addr = 817,
  490. .mcu_2_app_addr = 747,
  491. .per_2_per_addr = 6331,
  492. .uartsh_2_mcu_addr = 1032,
  493. .mcu_2_shp_addr = 960,
  494. .app_2_mcu_addr = 683,
  495. .shp_2_mcu_addr = 891,
  496. .spdif_2_mcu_addr = 1100,
  497. .mcu_2_spdif_addr = 1134,
  498. };
  499. static struct sdma_driver_data sdma_imx6q = {
  500. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  501. .num_events = 48,
  502. .script_addrs = &sdma_script_imx6q,
  503. };
  504. static struct sdma_script_start_addrs sdma_script_imx7d = {
  505. .ap_2_ap_addr = 644,
  506. .uart_2_mcu_addr = 819,
  507. .mcu_2_app_addr = 749,
  508. .uartsh_2_mcu_addr = 1034,
  509. .mcu_2_shp_addr = 962,
  510. .app_2_mcu_addr = 685,
  511. .shp_2_mcu_addr = 893,
  512. .spdif_2_mcu_addr = 1102,
  513. .mcu_2_spdif_addr = 1136,
  514. };
  515. static struct sdma_driver_data sdma_imx7d = {
  516. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  517. .num_events = 48,
  518. .script_addrs = &sdma_script_imx7d,
  519. };
  520. static struct sdma_driver_data sdma_imx8mq = {
  521. .chnenbl0 = SDMA_CHNENBL0_IMX35,
  522. .num_events = 48,
  523. .script_addrs = &sdma_script_imx7d,
  524. .check_ratio = 1,
  525. };
  526. static const struct platform_device_id sdma_devtypes[] = {
  527. {
  528. .name = "imx25-sdma",
  529. .driver_data = (unsigned long)&sdma_imx25,
  530. }, {
  531. .name = "imx31-sdma",
  532. .driver_data = (unsigned long)&sdma_imx31,
  533. }, {
  534. .name = "imx35-sdma",
  535. .driver_data = (unsigned long)&sdma_imx35,
  536. }, {
  537. .name = "imx51-sdma",
  538. .driver_data = (unsigned long)&sdma_imx51,
  539. }, {
  540. .name = "imx53-sdma",
  541. .driver_data = (unsigned long)&sdma_imx53,
  542. }, {
  543. .name = "imx6q-sdma",
  544. .driver_data = (unsigned long)&sdma_imx6q,
  545. }, {
  546. .name = "imx7d-sdma",
  547. .driver_data = (unsigned long)&sdma_imx7d,
  548. }, {
  549. .name = "imx8mq-sdma",
  550. .driver_data = (unsigned long)&sdma_imx8mq,
  551. }, {
  552. /* sentinel */
  553. }
  554. };
  555. MODULE_DEVICE_TABLE(platform, sdma_devtypes);
  556. static const struct of_device_id sdma_dt_ids[] = {
  557. { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
  558. { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
  559. { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
  560. { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
  561. { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
  562. { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
  563. { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
  564. { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
  565. { /* sentinel */ }
  566. };
  567. MODULE_DEVICE_TABLE(of, sdma_dt_ids);
  568. #define SDMA_H_CONFIG_DSPDMA BIT(12) /* indicates if the DSPDMA is used */
  569. #define SDMA_H_CONFIG_RTD_PINS BIT(11) /* indicates if Real-Time Debug pins are enabled */
  570. #define SDMA_H_CONFIG_ACR BIT(4) /* indicates if AHB freq /core freq = 2 or 1 */
  571. #define SDMA_H_CONFIG_CSM (3) /* indicates which context switch mode is selected*/
  572. static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
  573. {
  574. u32 chnenbl0 = sdma->drvdata->chnenbl0;
  575. return chnenbl0 + event * 4;
  576. }
  577. static int sdma_config_ownership(struct sdma_channel *sdmac,
  578. bool event_override, bool mcu_override, bool dsp_override)
  579. {
  580. struct sdma_engine *sdma = sdmac->sdma;
  581. int channel = sdmac->channel;
  582. unsigned long evt, mcu, dsp;
  583. if (event_override && mcu_override && dsp_override)
  584. return -EINVAL;
  585. evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
  586. mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
  587. dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
  588. if (dsp_override)
  589. __clear_bit(channel, &dsp);
  590. else
  591. __set_bit(channel, &dsp);
  592. if (event_override)
  593. __clear_bit(channel, &evt);
  594. else
  595. __set_bit(channel, &evt);
  596. if (mcu_override)
  597. __clear_bit(channel, &mcu);
  598. else
  599. __set_bit(channel, &mcu);
  600. writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
  601. writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
  602. writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
  603. return 0;
  604. }
  605. static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
  606. {
  607. writel(BIT(channel), sdma->regs + SDMA_H_START);
  608. }
  609. /*
  610. * sdma_run_channel0 - run a channel and wait till it's done
  611. */
  612. static int sdma_run_channel0(struct sdma_engine *sdma)
  613. {
  614. int ret;
  615. u32 reg;
  616. sdma_enable_channel(sdma, 0);
  617. ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,
  618. reg, !(reg & 1), 1, 500);
  619. if (ret)
  620. dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
  621. /* Set bits of CONFIG register with dynamic context switching */
  622. reg = readl(sdma->regs + SDMA_H_CONFIG);
  623. if ((reg & SDMA_H_CONFIG_CSM) == 0) {
  624. reg |= SDMA_H_CONFIG_CSM;
  625. writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
  626. }
  627. return ret;
  628. }
  629. static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
  630. u32 address)
  631. {
  632. struct sdma_buffer_descriptor *bd0 = sdma->bd0;
  633. void *buf_virt;
  634. dma_addr_t buf_phys;
  635. int ret;
  636. unsigned long flags;
  637. buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL);
  638. if (!buf_virt) {
  639. return -ENOMEM;
  640. }
  641. spin_lock_irqsave(&sdma->channel_0_lock, flags);
  642. bd0->mode.command = C0_SETPM;
  643. bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
  644. bd0->mode.count = size / 2;
  645. bd0->buffer_addr = buf_phys;
  646. bd0->ext_buffer_addr = address;
  647. memcpy(buf_virt, buf, size);
  648. ret = sdma_run_channel0(sdma);
  649. spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
  650. dma_free_coherent(sdma->dev, size, buf_virt, buf_phys);
  651. return ret;
  652. }
  653. static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
  654. {
  655. struct sdma_engine *sdma = sdmac->sdma;
  656. int channel = sdmac->channel;
  657. unsigned long val;
  658. u32 chnenbl = chnenbl_ofs(sdma, event);
  659. val = readl_relaxed(sdma->regs + chnenbl);
  660. __set_bit(channel, &val);
  661. writel_relaxed(val, sdma->regs + chnenbl);
  662. }
  663. static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
  664. {
  665. struct sdma_engine *sdma = sdmac->sdma;
  666. int channel = sdmac->channel;
  667. u32 chnenbl = chnenbl_ofs(sdma, event);
  668. unsigned long val;
  669. val = readl_relaxed(sdma->regs + chnenbl);
  670. __clear_bit(channel, &val);
  671. writel_relaxed(val, sdma->regs + chnenbl);
  672. }
  673. static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
  674. {
  675. return container_of(t, struct sdma_desc, vd.tx);
  676. }
  677. static void sdma_start_desc(struct sdma_channel *sdmac)
  678. {
  679. struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
  680. struct sdma_desc *desc;
  681. struct sdma_engine *sdma = sdmac->sdma;
  682. int channel = sdmac->channel;
  683. if (!vd) {
  684. sdmac->desc = NULL;
  685. return;
  686. }
  687. sdmac->desc = desc = to_sdma_desc(&vd->tx);
  688. list_del(&vd->node);
  689. sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
  690. sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
  691. sdma_enable_channel(sdma, sdmac->channel);
  692. }
  693. static void sdma_update_channel_loop(struct sdma_channel *sdmac)
  694. {
  695. struct sdma_buffer_descriptor *bd;
  696. int error = 0;
  697. enum dma_status old_status = sdmac->status;
  698. /*
  699. * loop mode. Iterate over descriptors, re-setup them and
  700. * call callback function.
  701. */
  702. while (sdmac->desc) {
  703. struct sdma_desc *desc = sdmac->desc;
  704. bd = &desc->bd[desc->buf_tail];
  705. if (bd->mode.status & BD_DONE)
  706. break;
  707. if (bd->mode.status & BD_RROR) {
  708. bd->mode.status &= ~BD_RROR;
  709. sdmac->status = DMA_ERROR;
  710. error = -EIO;
  711. }
  712. /*
  713. * We use bd->mode.count to calculate the residue, since contains
  714. * the number of bytes present in the current buffer descriptor.
  715. */
  716. desc->chn_real_count = bd->mode.count;
  717. bd->mode.status |= BD_DONE;
  718. bd->mode.count = desc->period_len;
  719. desc->buf_ptail = desc->buf_tail;
  720. desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
  721. /*
  722. * The callback is called from the interrupt context in order
  723. * to reduce latency and to avoid the risk of altering the
  724. * SDMA transaction status by the time the client tasklet is
  725. * executed.
  726. */
  727. spin_unlock(&sdmac->vc.lock);
  728. dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
  729. spin_lock(&sdmac->vc.lock);
  730. if (error)
  731. sdmac->status = old_status;
  732. }
  733. }
  734. static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
  735. {
  736. struct sdma_channel *sdmac = (struct sdma_channel *) data;
  737. struct sdma_buffer_descriptor *bd;
  738. int i, error = 0;
  739. sdmac->desc->chn_real_count = 0;
  740. /*
  741. * non loop mode. Iterate over all descriptors, collect
  742. * errors and call callback function
  743. */
  744. for (i = 0; i < sdmac->desc->num_bd; i++) {
  745. bd = &sdmac->desc->bd[i];
  746. if (bd->mode.status & (BD_DONE | BD_RROR))
  747. error = -EIO;
  748. sdmac->desc->chn_real_count += bd->mode.count;
  749. }
  750. if (error)
  751. sdmac->status = DMA_ERROR;
  752. else
  753. sdmac->status = DMA_COMPLETE;
  754. }
  755. static irqreturn_t sdma_int_handler(int irq, void *dev_id)
  756. {
  757. struct sdma_engine *sdma = dev_id;
  758. unsigned long stat;
  759. stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
  760. writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
  761. /* channel 0 is special and not handled here, see run_channel0() */
  762. stat &= ~1;
  763. while (stat) {
  764. int channel = fls(stat) - 1;
  765. struct sdma_channel *sdmac = &sdma->channel[channel];
  766. struct sdma_desc *desc;
  767. spin_lock(&sdmac->vc.lock);
  768. desc = sdmac->desc;
  769. if (desc) {
  770. if (sdmac->flags & IMX_DMA_SG_LOOP) {
  771. sdma_update_channel_loop(sdmac);
  772. } else {
  773. mxc_sdma_handle_channel_normal(sdmac);
  774. vchan_cookie_complete(&desc->vd);
  775. sdma_start_desc(sdmac);
  776. }
  777. }
  778. spin_unlock(&sdmac->vc.lock);
  779. __clear_bit(channel, &stat);
  780. }
  781. return IRQ_HANDLED;
  782. }
  783. /*
  784. * sets the pc of SDMA script according to the peripheral type
  785. */
  786. static void sdma_get_pc(struct sdma_channel *sdmac,
  787. enum sdma_peripheral_type peripheral_type)
  788. {
  789. struct sdma_engine *sdma = sdmac->sdma;
  790. int per_2_emi = 0, emi_2_per = 0;
  791. /*
  792. * These are needed once we start to support transfers between
  793. * two peripherals or memory-to-memory transfers
  794. */
  795. int per_2_per = 0, emi_2_emi = 0;
  796. sdmac->pc_from_device = 0;
  797. sdmac->pc_to_device = 0;
  798. sdmac->device_to_device = 0;
  799. sdmac->pc_to_pc = 0;
  800. switch (peripheral_type) {
  801. case IMX_DMATYPE_MEMORY:
  802. emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
  803. break;
  804. case IMX_DMATYPE_DSP:
  805. emi_2_per = sdma->script_addrs->bp_2_ap_addr;
  806. per_2_emi = sdma->script_addrs->ap_2_bp_addr;
  807. break;
  808. case IMX_DMATYPE_FIRI:
  809. per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
  810. emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
  811. break;
  812. case IMX_DMATYPE_UART:
  813. per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
  814. emi_2_per = sdma->script_addrs->mcu_2_app_addr;
  815. break;
  816. case IMX_DMATYPE_UART_SP:
  817. per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
  818. emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
  819. break;
  820. case IMX_DMATYPE_ATA:
  821. per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
  822. emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
  823. break;
  824. case IMX_DMATYPE_CSPI:
  825. case IMX_DMATYPE_EXT:
  826. case IMX_DMATYPE_SSI:
  827. case IMX_DMATYPE_SAI:
  828. per_2_emi = sdma->script_addrs->app_2_mcu_addr;
  829. emi_2_per = sdma->script_addrs->mcu_2_app_addr;
  830. break;
  831. case IMX_DMATYPE_SSI_DUAL:
  832. per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
  833. emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
  834. break;
  835. case IMX_DMATYPE_SSI_SP:
  836. case IMX_DMATYPE_MMC:
  837. case IMX_DMATYPE_SDHC:
  838. case IMX_DMATYPE_CSPI_SP:
  839. case IMX_DMATYPE_ESAI:
  840. case IMX_DMATYPE_MSHC_SP:
  841. per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
  842. emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
  843. break;
  844. case IMX_DMATYPE_ASRC:
  845. per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
  846. emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
  847. per_2_per = sdma->script_addrs->per_2_per_addr;
  848. break;
  849. case IMX_DMATYPE_ASRC_SP:
  850. per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
  851. emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
  852. per_2_per = sdma->script_addrs->per_2_per_addr;
  853. break;
  854. case IMX_DMATYPE_MSHC:
  855. per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
  856. emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
  857. break;
  858. case IMX_DMATYPE_CCM:
  859. per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
  860. break;
  861. case IMX_DMATYPE_SPDIF:
  862. per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
  863. emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
  864. break;
  865. case IMX_DMATYPE_IPU_MEMORY:
  866. emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
  867. break;
  868. default:
  869. break;
  870. }
  871. sdmac->pc_from_device = per_2_emi;
  872. sdmac->pc_to_device = emi_2_per;
  873. sdmac->device_to_device = per_2_per;
  874. sdmac->pc_to_pc = emi_2_emi;
  875. }
  876. static int sdma_load_context(struct sdma_channel *sdmac)
  877. {
  878. struct sdma_engine *sdma = sdmac->sdma;
  879. int channel = sdmac->channel;
  880. int load_address;
  881. struct sdma_context_data *context = sdma->context;
  882. struct sdma_buffer_descriptor *bd0 = sdma->bd0;
  883. int ret;
  884. unsigned long flags;
  885. if (sdmac->direction == DMA_DEV_TO_MEM)
  886. load_address = sdmac->pc_from_device;
  887. else if (sdmac->direction == DMA_DEV_TO_DEV)
  888. load_address = sdmac->device_to_device;
  889. else if (sdmac->direction == DMA_MEM_TO_MEM)
  890. load_address = sdmac->pc_to_pc;
  891. else
  892. load_address = sdmac->pc_to_device;
  893. if (load_address < 0)
  894. return load_address;
  895. dev_dbg(sdma->dev, "load_address = %d\n", load_address);
  896. dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
  897. dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
  898. dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
  899. dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
  900. dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
  901. spin_lock_irqsave(&sdma->channel_0_lock, flags);
  902. memset(context, 0, sizeof(*context));
  903. context->channel_state.pc = load_address;
  904. /* Send by context the event mask,base address for peripheral
  905. * and watermark level
  906. */
  907. context->gReg[0] = sdmac->event_mask[1];
  908. context->gReg[1] = sdmac->event_mask[0];
  909. context->gReg[2] = sdmac->per_addr;
  910. context->gReg[6] = sdmac->shp_addr;
  911. context->gReg[7] = sdmac->watermark_level;
  912. bd0->mode.command = C0_SETDM;
  913. bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
  914. bd0->mode.count = sizeof(*context) / 4;
  915. bd0->buffer_addr = sdma->context_phys;
  916. bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
  917. ret = sdma_run_channel0(sdma);
  918. spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
  919. return ret;
  920. }
  921. static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
  922. {
  923. return container_of(chan, struct sdma_channel, vc.chan);
  924. }
  925. static int sdma_disable_channel(struct dma_chan *chan)
  926. {
  927. struct sdma_channel *sdmac = to_sdma_chan(chan);
  928. struct sdma_engine *sdma = sdmac->sdma;
  929. int channel = sdmac->channel;
  930. writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
  931. sdmac->status = DMA_ERROR;
  932. return 0;
  933. }
  934. static void sdma_channel_terminate_work(struct work_struct *work)
  935. {
  936. struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
  937. terminate_worker);
  938. unsigned long flags;
  939. LIST_HEAD(head);
  940. /*
  941. * According to NXP R&D team a delay of one BD SDMA cost time
  942. * (maximum is 1ms) should be added after disable of the channel
  943. * bit, to ensure SDMA core has really been stopped after SDMA
  944. * clients call .device_terminate_all.
  945. */
  946. usleep_range(1000, 2000);
  947. spin_lock_irqsave(&sdmac->vc.lock, flags);
  948. vchan_get_all_descriptors(&sdmac->vc, &head);
  949. spin_unlock_irqrestore(&sdmac->vc.lock, flags);
  950. vchan_dma_desc_free_list(&sdmac->vc, &head);
  951. }
  952. static int sdma_terminate_all(struct dma_chan *chan)
  953. {
  954. struct sdma_channel *sdmac = to_sdma_chan(chan);
  955. unsigned long flags;
  956. spin_lock_irqsave(&sdmac->vc.lock, flags);
  957. sdma_disable_channel(chan);
  958. if (sdmac->desc) {
  959. vchan_terminate_vdesc(&sdmac->desc->vd);
  960. sdmac->desc = NULL;
  961. schedule_work(&sdmac->terminate_worker);
  962. }
  963. spin_unlock_irqrestore(&sdmac->vc.lock, flags);
  964. return 0;
  965. }
  966. static void sdma_channel_synchronize(struct dma_chan *chan)
  967. {
  968. struct sdma_channel *sdmac = to_sdma_chan(chan);
  969. vchan_synchronize(&sdmac->vc);
  970. flush_work(&sdmac->terminate_worker);
  971. }
  972. static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
  973. {
  974. struct sdma_engine *sdma = sdmac->sdma;
  975. int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
  976. int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
  977. set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
  978. set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
  979. if (sdmac->event_id0 > 31)
  980. sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
  981. if (sdmac->event_id1 > 31)
  982. sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
  983. /*
  984. * If LWML(src_maxburst) > HWML(dst_maxburst), we need
  985. * swap LWML and HWML of INFO(A.3.2.5.1), also need swap
  986. * r0(event_mask[1]) and r1(event_mask[0]).
  987. */
  988. if (lwml > hwml) {
  989. sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
  990. SDMA_WATERMARK_LEVEL_HWML);
  991. sdmac->watermark_level |= hwml;
  992. sdmac->watermark_level |= lwml << 16;
  993. swap(sdmac->event_mask[0], sdmac->event_mask[1]);
  994. }
  995. if (sdmac->per_address2 >= sdma->spba_start_addr &&
  996. sdmac->per_address2 <= sdma->spba_end_addr)
  997. sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
  998. if (sdmac->per_address >= sdma->spba_start_addr &&
  999. sdmac->per_address <= sdma->spba_end_addr)
  1000. sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
  1001. sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
  1002. }
  1003. static int sdma_config_channel(struct dma_chan *chan)
  1004. {
  1005. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1006. sdma_disable_channel(chan);
  1007. sdmac->event_mask[0] = 0;
  1008. sdmac->event_mask[1] = 0;
  1009. sdmac->shp_addr = 0;
  1010. sdmac->per_addr = 0;
  1011. switch (sdmac->peripheral_type) {
  1012. case IMX_DMATYPE_DSP:
  1013. sdma_config_ownership(sdmac, false, true, true);
  1014. break;
  1015. case IMX_DMATYPE_MEMORY:
  1016. sdma_config_ownership(sdmac, false, true, false);
  1017. break;
  1018. default:
  1019. sdma_config_ownership(sdmac, true, true, false);
  1020. break;
  1021. }
  1022. sdma_get_pc(sdmac, sdmac->peripheral_type);
  1023. if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
  1024. (sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
  1025. /* Handle multiple event channels differently */
  1026. if (sdmac->event_id1) {
  1027. if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
  1028. sdmac->peripheral_type == IMX_DMATYPE_ASRC)
  1029. sdma_set_watermarklevel_for_p2p(sdmac);
  1030. } else
  1031. __set_bit(sdmac->event_id0, sdmac->event_mask);
  1032. /* Address */
  1033. sdmac->shp_addr = sdmac->per_address;
  1034. sdmac->per_addr = sdmac->per_address2;
  1035. } else {
  1036. sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
  1037. }
  1038. return 0;
  1039. }
  1040. static int sdma_set_channel_priority(struct sdma_channel *sdmac,
  1041. unsigned int priority)
  1042. {
  1043. struct sdma_engine *sdma = sdmac->sdma;
  1044. int channel = sdmac->channel;
  1045. if (priority < MXC_SDMA_MIN_PRIORITY
  1046. || priority > MXC_SDMA_MAX_PRIORITY) {
  1047. return -EINVAL;
  1048. }
  1049. writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
  1050. return 0;
  1051. }
  1052. static int sdma_request_channel0(struct sdma_engine *sdma)
  1053. {
  1054. int ret = -EBUSY;
  1055. sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys,
  1056. GFP_NOWAIT);
  1057. if (!sdma->bd0) {
  1058. ret = -ENOMEM;
  1059. goto out;
  1060. }
  1061. sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
  1062. sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
  1063. sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
  1064. return 0;
  1065. out:
  1066. return ret;
  1067. }
  1068. static int sdma_alloc_bd(struct sdma_desc *desc)
  1069. {
  1070. u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
  1071. int ret = 0;
  1072. desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size,
  1073. &desc->bd_phys, GFP_NOWAIT);
  1074. if (!desc->bd) {
  1075. ret = -ENOMEM;
  1076. goto out;
  1077. }
  1078. out:
  1079. return ret;
  1080. }
  1081. static void sdma_free_bd(struct sdma_desc *desc)
  1082. {
  1083. u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
  1084. dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd,
  1085. desc->bd_phys);
  1086. }
  1087. static void sdma_desc_free(struct virt_dma_desc *vd)
  1088. {
  1089. struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
  1090. sdma_free_bd(desc);
  1091. kfree(desc);
  1092. }
  1093. static int sdma_alloc_chan_resources(struct dma_chan *chan)
  1094. {
  1095. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1096. struct imx_dma_data *data = chan->private;
  1097. struct imx_dma_data mem_data;
  1098. int prio, ret;
  1099. /*
  1100. * MEMCPY may never setup chan->private by filter function such as
  1101. * dmatest, thus create 'struct imx_dma_data mem_data' for this case.
  1102. * Please note in any other slave case, you have to setup chan->private
  1103. * with 'struct imx_dma_data' in your own filter function if you want to
  1104. * request dma channel by dma_request_channel() rather than
  1105. * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear
  1106. * to warn you to correct your filter function.
  1107. */
  1108. if (!data) {
  1109. dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
  1110. mem_data.priority = 2;
  1111. mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
  1112. mem_data.dma_request = 0;
  1113. mem_data.dma_request2 = 0;
  1114. data = &mem_data;
  1115. sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
  1116. }
  1117. switch (data->priority) {
  1118. case DMA_PRIO_HIGH:
  1119. prio = 3;
  1120. break;
  1121. case DMA_PRIO_MEDIUM:
  1122. prio = 2;
  1123. break;
  1124. case DMA_PRIO_LOW:
  1125. default:
  1126. prio = 1;
  1127. break;
  1128. }
  1129. sdmac->peripheral_type = data->peripheral_type;
  1130. sdmac->event_id0 = data->dma_request;
  1131. sdmac->event_id1 = data->dma_request2;
  1132. ret = clk_enable(sdmac->sdma->clk_ipg);
  1133. if (ret)
  1134. return ret;
  1135. ret = clk_enable(sdmac->sdma->clk_ahb);
  1136. if (ret)
  1137. goto disable_clk_ipg;
  1138. ret = sdma_set_channel_priority(sdmac, prio);
  1139. if (ret)
  1140. goto disable_clk_ahb;
  1141. return 0;
  1142. disable_clk_ahb:
  1143. clk_disable(sdmac->sdma->clk_ahb);
  1144. disable_clk_ipg:
  1145. clk_disable(sdmac->sdma->clk_ipg);
  1146. return ret;
  1147. }
  1148. static void sdma_free_chan_resources(struct dma_chan *chan)
  1149. {
  1150. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1151. struct sdma_engine *sdma = sdmac->sdma;
  1152. sdma_terminate_all(chan);
  1153. sdma_channel_synchronize(chan);
  1154. sdma_event_disable(sdmac, sdmac->event_id0);
  1155. if (sdmac->event_id1)
  1156. sdma_event_disable(sdmac, sdmac->event_id1);
  1157. sdmac->event_id0 = 0;
  1158. sdmac->event_id1 = 0;
  1159. sdma_set_channel_priority(sdmac, 0);
  1160. clk_disable(sdma->clk_ipg);
  1161. clk_disable(sdma->clk_ahb);
  1162. }
  1163. static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
  1164. enum dma_transfer_direction direction, u32 bds)
  1165. {
  1166. struct sdma_desc *desc;
  1167. desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
  1168. if (!desc)
  1169. goto err_out;
  1170. sdmac->status = DMA_IN_PROGRESS;
  1171. sdmac->direction = direction;
  1172. sdmac->flags = 0;
  1173. desc->chn_count = 0;
  1174. desc->chn_real_count = 0;
  1175. desc->buf_tail = 0;
  1176. desc->buf_ptail = 0;
  1177. desc->sdmac = sdmac;
  1178. desc->num_bd = bds;
  1179. if (sdma_alloc_bd(desc))
  1180. goto err_desc_out;
  1181. /* No slave_config called in MEMCPY case, so do here */
  1182. if (direction == DMA_MEM_TO_MEM)
  1183. sdma_config_ownership(sdmac, false, true, false);
  1184. if (sdma_load_context(sdmac))
  1185. goto err_desc_out;
  1186. return desc;
  1187. err_desc_out:
  1188. kfree(desc);
  1189. err_out:
  1190. return NULL;
  1191. }
  1192. static struct dma_async_tx_descriptor *sdma_prep_memcpy(
  1193. struct dma_chan *chan, dma_addr_t dma_dst,
  1194. dma_addr_t dma_src, size_t len, unsigned long flags)
  1195. {
  1196. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1197. struct sdma_engine *sdma = sdmac->sdma;
  1198. int channel = sdmac->channel;
  1199. size_t count;
  1200. int i = 0, param;
  1201. struct sdma_buffer_descriptor *bd;
  1202. struct sdma_desc *desc;
  1203. if (!chan || !len)
  1204. return NULL;
  1205. dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
  1206. &dma_src, &dma_dst, len, channel);
  1207. desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
  1208. len / SDMA_BD_MAX_CNT + 1);
  1209. if (!desc)
  1210. return NULL;
  1211. do {
  1212. count = min_t(size_t, len, SDMA_BD_MAX_CNT);
  1213. bd = &desc->bd[i];
  1214. bd->buffer_addr = dma_src;
  1215. bd->ext_buffer_addr = dma_dst;
  1216. bd->mode.count = count;
  1217. desc->chn_count += count;
  1218. bd->mode.command = 0;
  1219. dma_src += count;
  1220. dma_dst += count;
  1221. len -= count;
  1222. i++;
  1223. param = BD_DONE | BD_EXTD | BD_CONT;
  1224. /* last bd */
  1225. if (!len) {
  1226. param |= BD_INTR;
  1227. param |= BD_LAST;
  1228. param &= ~BD_CONT;
  1229. }
  1230. dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
  1231. i, count, bd->buffer_addr,
  1232. param & BD_WRAP ? "wrap" : "",
  1233. param & BD_INTR ? " intr" : "");
  1234. bd->mode.status = param;
  1235. } while (len);
  1236. return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
  1237. }
  1238. static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
  1239. struct dma_chan *chan, struct scatterlist *sgl,
  1240. unsigned int sg_len, enum dma_transfer_direction direction,
  1241. unsigned long flags, void *context)
  1242. {
  1243. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1244. struct sdma_engine *sdma = sdmac->sdma;
  1245. int i, count;
  1246. int channel = sdmac->channel;
  1247. struct scatterlist *sg;
  1248. struct sdma_desc *desc;
  1249. sdma_config_write(chan, &sdmac->slave_config, direction);
  1250. desc = sdma_transfer_init(sdmac, direction, sg_len);
  1251. if (!desc)
  1252. goto err_out;
  1253. dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
  1254. sg_len, channel);
  1255. for_each_sg(sgl, sg, sg_len, i) {
  1256. struct sdma_buffer_descriptor *bd = &desc->bd[i];
  1257. int param;
  1258. bd->buffer_addr = sg->dma_address;
  1259. count = sg_dma_len(sg);
  1260. if (count > SDMA_BD_MAX_CNT) {
  1261. dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
  1262. channel, count, SDMA_BD_MAX_CNT);
  1263. goto err_bd_out;
  1264. }
  1265. bd->mode.count = count;
  1266. desc->chn_count += count;
  1267. if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
  1268. goto err_bd_out;
  1269. switch (sdmac->word_size) {
  1270. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  1271. bd->mode.command = 0;
  1272. if (count & 3 || sg->dma_address & 3)
  1273. goto err_bd_out;
  1274. break;
  1275. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  1276. bd->mode.command = 2;
  1277. if (count & 1 || sg->dma_address & 1)
  1278. goto err_bd_out;
  1279. break;
  1280. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  1281. bd->mode.command = 1;
  1282. break;
  1283. default:
  1284. goto err_bd_out;
  1285. }
  1286. param = BD_DONE | BD_EXTD | BD_CONT;
  1287. if (i + 1 == sg_len) {
  1288. param |= BD_INTR;
  1289. param |= BD_LAST;
  1290. param &= ~BD_CONT;
  1291. }
  1292. dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
  1293. i, count, (u64)sg->dma_address,
  1294. param & BD_WRAP ? "wrap" : "",
  1295. param & BD_INTR ? " intr" : "");
  1296. bd->mode.status = param;
  1297. }
  1298. return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
  1299. err_bd_out:
  1300. sdma_free_bd(desc);
  1301. kfree(desc);
  1302. err_out:
  1303. sdmac->status = DMA_ERROR;
  1304. return NULL;
  1305. }
  1306. static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
  1307. struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
  1308. size_t period_len, enum dma_transfer_direction direction,
  1309. unsigned long flags)
  1310. {
  1311. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1312. struct sdma_engine *sdma = sdmac->sdma;
  1313. int num_periods = buf_len / period_len;
  1314. int channel = sdmac->channel;
  1315. int i = 0, buf = 0;
  1316. struct sdma_desc *desc;
  1317. dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
  1318. sdma_config_write(chan, &sdmac->slave_config, direction);
  1319. desc = sdma_transfer_init(sdmac, direction, num_periods);
  1320. if (!desc)
  1321. goto err_out;
  1322. desc->period_len = period_len;
  1323. sdmac->flags |= IMX_DMA_SG_LOOP;
  1324. if (period_len > SDMA_BD_MAX_CNT) {
  1325. dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
  1326. channel, period_len, SDMA_BD_MAX_CNT);
  1327. goto err_bd_out;
  1328. }
  1329. while (buf < buf_len) {
  1330. struct sdma_buffer_descriptor *bd = &desc->bd[i];
  1331. int param;
  1332. bd->buffer_addr = dma_addr;
  1333. bd->mode.count = period_len;
  1334. if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
  1335. goto err_bd_out;
  1336. if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
  1337. bd->mode.command = 0;
  1338. else
  1339. bd->mode.command = sdmac->word_size;
  1340. param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
  1341. if (i + 1 == num_periods)
  1342. param |= BD_WRAP;
  1343. dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n",
  1344. i, period_len, (u64)dma_addr,
  1345. param & BD_WRAP ? "wrap" : "",
  1346. param & BD_INTR ? " intr" : "");
  1347. bd->mode.status = param;
  1348. dma_addr += period_len;
  1349. buf += period_len;
  1350. i++;
  1351. }
  1352. return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
  1353. err_bd_out:
  1354. sdma_free_bd(desc);
  1355. kfree(desc);
  1356. err_out:
  1357. sdmac->status = DMA_ERROR;
  1358. return NULL;
  1359. }
  1360. static int sdma_config_write(struct dma_chan *chan,
  1361. struct dma_slave_config *dmaengine_cfg,
  1362. enum dma_transfer_direction direction)
  1363. {
  1364. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1365. if (direction == DMA_DEV_TO_MEM) {
  1366. sdmac->per_address = dmaengine_cfg->src_addr;
  1367. sdmac->watermark_level = dmaengine_cfg->src_maxburst *
  1368. dmaengine_cfg->src_addr_width;
  1369. sdmac->word_size = dmaengine_cfg->src_addr_width;
  1370. } else if (direction == DMA_DEV_TO_DEV) {
  1371. sdmac->per_address2 = dmaengine_cfg->src_addr;
  1372. sdmac->per_address = dmaengine_cfg->dst_addr;
  1373. sdmac->watermark_level = dmaengine_cfg->src_maxburst &
  1374. SDMA_WATERMARK_LEVEL_LWML;
  1375. sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
  1376. SDMA_WATERMARK_LEVEL_HWML;
  1377. sdmac->word_size = dmaengine_cfg->dst_addr_width;
  1378. } else {
  1379. sdmac->per_address = dmaengine_cfg->dst_addr;
  1380. sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
  1381. dmaengine_cfg->dst_addr_width;
  1382. sdmac->word_size = dmaengine_cfg->dst_addr_width;
  1383. }
  1384. sdmac->direction = direction;
  1385. return sdma_config_channel(chan);
  1386. }
  1387. static int sdma_config(struct dma_chan *chan,
  1388. struct dma_slave_config *dmaengine_cfg)
  1389. {
  1390. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1391. memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
  1392. /* Set ENBLn earlier to make sure dma request triggered after that */
  1393. if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
  1394. return -EINVAL;
  1395. sdma_event_enable(sdmac, sdmac->event_id0);
  1396. if (sdmac->event_id1) {
  1397. if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
  1398. return -EINVAL;
  1399. sdma_event_enable(sdmac, sdmac->event_id1);
  1400. }
  1401. return 0;
  1402. }
  1403. static enum dma_status sdma_tx_status(struct dma_chan *chan,
  1404. dma_cookie_t cookie,
  1405. struct dma_tx_state *txstate)
  1406. {
  1407. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1408. struct sdma_desc *desc = NULL;
  1409. u32 residue;
  1410. struct virt_dma_desc *vd;
  1411. enum dma_status ret;
  1412. unsigned long flags;
  1413. ret = dma_cookie_status(chan, cookie, txstate);
  1414. if (ret == DMA_COMPLETE || !txstate)
  1415. return ret;
  1416. spin_lock_irqsave(&sdmac->vc.lock, flags);
  1417. vd = vchan_find_desc(&sdmac->vc, cookie);
  1418. if (vd)
  1419. desc = to_sdma_desc(&vd->tx);
  1420. else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
  1421. desc = sdmac->desc;
  1422. if (desc) {
  1423. if (sdmac->flags & IMX_DMA_SG_LOOP)
  1424. residue = (desc->num_bd - desc->buf_ptail) *
  1425. desc->period_len - desc->chn_real_count;
  1426. else
  1427. residue = desc->chn_count - desc->chn_real_count;
  1428. } else {
  1429. residue = 0;
  1430. }
  1431. spin_unlock_irqrestore(&sdmac->vc.lock, flags);
  1432. dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
  1433. residue);
  1434. return sdmac->status;
  1435. }
  1436. static void sdma_issue_pending(struct dma_chan *chan)
  1437. {
  1438. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1439. unsigned long flags;
  1440. spin_lock_irqsave(&sdmac->vc.lock, flags);
  1441. if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
  1442. sdma_start_desc(sdmac);
  1443. spin_unlock_irqrestore(&sdmac->vc.lock, flags);
  1444. }
  1445. #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34
  1446. #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38
  1447. #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41
  1448. #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42
  1449. static void sdma_add_scripts(struct sdma_engine *sdma,
  1450. const struct sdma_script_start_addrs *addr)
  1451. {
  1452. s32 *addr_arr = (u32 *)addr;
  1453. s32 *saddr_arr = (u32 *)sdma->script_addrs;
  1454. int i;
  1455. /* use the default firmware in ROM if missing external firmware */
  1456. if (!sdma->script_number)
  1457. sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
  1458. if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
  1459. / sizeof(s32)) {
  1460. dev_err(sdma->dev,
  1461. "SDMA script number %d not match with firmware.\n",
  1462. sdma->script_number);
  1463. return;
  1464. }
  1465. for (i = 0; i < sdma->script_number; i++)
  1466. if (addr_arr[i] > 0)
  1467. saddr_arr[i] = addr_arr[i];
  1468. }
  1469. static void sdma_load_firmware(const struct firmware *fw, void *context)
  1470. {
  1471. struct sdma_engine *sdma = context;
  1472. const struct sdma_firmware_header *header;
  1473. const struct sdma_script_start_addrs *addr;
  1474. unsigned short *ram_code;
  1475. if (!fw) {
  1476. dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
  1477. /* In this case we just use the ROM firmware. */
  1478. return;
  1479. }
  1480. if (fw->size < sizeof(*header))
  1481. goto err_firmware;
  1482. header = (struct sdma_firmware_header *)fw->data;
  1483. if (header->magic != SDMA_FIRMWARE_MAGIC)
  1484. goto err_firmware;
  1485. if (header->ram_code_start + header->ram_code_size > fw->size)
  1486. goto err_firmware;
  1487. switch (header->version_major) {
  1488. case 1:
  1489. sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
  1490. break;
  1491. case 2:
  1492. sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
  1493. break;
  1494. case 3:
  1495. sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
  1496. break;
  1497. case 4:
  1498. sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
  1499. break;
  1500. default:
  1501. dev_err(sdma->dev, "unknown firmware version\n");
  1502. goto err_firmware;
  1503. }
  1504. addr = (void *)header + header->script_addrs_start;
  1505. ram_code = (void *)header + header->ram_code_start;
  1506. clk_enable(sdma->clk_ipg);
  1507. clk_enable(sdma->clk_ahb);
  1508. /* download the RAM image for SDMA */
  1509. sdma_load_script(sdma, ram_code,
  1510. header->ram_code_size,
  1511. addr->ram_code_start_addr);
  1512. clk_disable(sdma->clk_ipg);
  1513. clk_disable(sdma->clk_ahb);
  1514. sdma_add_scripts(sdma, addr);
  1515. dev_info(sdma->dev, "loaded firmware %d.%d\n",
  1516. header->version_major,
  1517. header->version_minor);
  1518. err_firmware:
  1519. release_firmware(fw);
  1520. }
  1521. #define EVENT_REMAP_CELLS 3
  1522. static int sdma_event_remap(struct sdma_engine *sdma)
  1523. {
  1524. struct device_node *np = sdma->dev->of_node;
  1525. struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0);
  1526. struct property *event_remap;
  1527. struct regmap *gpr;
  1528. char propname[] = "fsl,sdma-event-remap";
  1529. u32 reg, val, shift, num_map, i;
  1530. int ret = 0;
  1531. if (IS_ERR(np) || !gpr_np)
  1532. goto out;
  1533. event_remap = of_find_property(np, propname, NULL);
  1534. num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
  1535. if (!num_map) {
  1536. dev_dbg(sdma->dev, "no event needs to be remapped\n");
  1537. goto out;
  1538. } else if (num_map % EVENT_REMAP_CELLS) {
  1539. dev_err(sdma->dev, "the property %s must modulo %d\n",
  1540. propname, EVENT_REMAP_CELLS);
  1541. ret = -EINVAL;
  1542. goto out;
  1543. }
  1544. gpr = syscon_node_to_regmap(gpr_np);
  1545. if (IS_ERR(gpr)) {
  1546. dev_err(sdma->dev, "failed to get gpr regmap\n");
  1547. ret = PTR_ERR(gpr);
  1548. goto out;
  1549. }
  1550. for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
  1551. ret = of_property_read_u32_index(np, propname, i, &reg);
  1552. if (ret) {
  1553. dev_err(sdma->dev, "failed to read property %s index %d\n",
  1554. propname, i);
  1555. goto out;
  1556. }
  1557. ret = of_property_read_u32_index(np, propname, i + 1, &shift);
  1558. if (ret) {
  1559. dev_err(sdma->dev, "failed to read property %s index %d\n",
  1560. propname, i + 1);
  1561. goto out;
  1562. }
  1563. ret = of_property_read_u32_index(np, propname, i + 2, &val);
  1564. if (ret) {
  1565. dev_err(sdma->dev, "failed to read property %s index %d\n",
  1566. propname, i + 2);
  1567. goto out;
  1568. }
  1569. regmap_update_bits(gpr, reg, BIT(shift), val << shift);
  1570. }
  1571. out:
  1572. if (gpr_np)
  1573. of_node_put(gpr_np);
  1574. return ret;
  1575. }
  1576. static int sdma_get_firmware(struct sdma_engine *sdma,
  1577. const char *fw_name)
  1578. {
  1579. int ret;
  1580. ret = request_firmware_nowait(THIS_MODULE,
  1581. FW_ACTION_HOTPLUG, fw_name, sdma->dev,
  1582. GFP_KERNEL, sdma, sdma_load_firmware);
  1583. return ret;
  1584. }
  1585. static int sdma_init(struct sdma_engine *sdma)
  1586. {
  1587. int i, ret;
  1588. dma_addr_t ccb_phys;
  1589. ret = clk_enable(sdma->clk_ipg);
  1590. if (ret)
  1591. return ret;
  1592. ret = clk_enable(sdma->clk_ahb);
  1593. if (ret)
  1594. goto disable_clk_ipg;
  1595. if (sdma->drvdata->check_ratio &&
  1596. (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)))
  1597. sdma->clk_ratio = 1;
  1598. /* Be sure SDMA has not started yet */
  1599. writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
  1600. sdma->channel_control = dma_alloc_coherent(sdma->dev,
  1601. MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
  1602. sizeof(struct sdma_context_data),
  1603. &ccb_phys, GFP_KERNEL);
  1604. if (!sdma->channel_control) {
  1605. ret = -ENOMEM;
  1606. goto err_dma_alloc;
  1607. }
  1608. sdma->context = (void *)sdma->channel_control +
  1609. MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
  1610. sdma->context_phys = ccb_phys +
  1611. MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
  1612. /* disable all channels */
  1613. for (i = 0; i < sdma->drvdata->num_events; i++)
  1614. writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
  1615. /* All channels have priority 0 */
  1616. for (i = 0; i < MAX_DMA_CHANNELS; i++)
  1617. writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
  1618. ret = sdma_request_channel0(sdma);
  1619. if (ret)
  1620. goto err_dma_alloc;
  1621. sdma_config_ownership(&sdma->channel[0], false, true, false);
  1622. /* Set Command Channel (Channel Zero) */
  1623. writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
  1624. /* Set bits of CONFIG register but with static context switching */
  1625. if (sdma->clk_ratio)
  1626. writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
  1627. else
  1628. writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
  1629. writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
  1630. /* Initializes channel's priorities */
  1631. sdma_set_channel_priority(&sdma->channel[0], 7);
  1632. clk_disable(sdma->clk_ipg);
  1633. clk_disable(sdma->clk_ahb);
  1634. return 0;
  1635. err_dma_alloc:
  1636. clk_disable(sdma->clk_ahb);
  1637. disable_clk_ipg:
  1638. clk_disable(sdma->clk_ipg);
  1639. dev_err(sdma->dev, "initialisation failed with %d\n", ret);
  1640. return ret;
  1641. }
  1642. static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
  1643. {
  1644. struct sdma_channel *sdmac = to_sdma_chan(chan);
  1645. struct imx_dma_data *data = fn_param;
  1646. if (!imx_dma_is_general_purpose(chan))
  1647. return false;
  1648. sdmac->data = *data;
  1649. chan->private = &sdmac->data;
  1650. return true;
  1651. }
  1652. static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
  1653. struct of_dma *ofdma)
  1654. {
  1655. struct sdma_engine *sdma = ofdma->of_dma_data;
  1656. dma_cap_mask_t mask = sdma->dma_device.cap_mask;
  1657. struct imx_dma_data data;
  1658. if (dma_spec->args_count != 3)
  1659. return NULL;
  1660. data.dma_request = dma_spec->args[0];
  1661. data.peripheral_type = dma_spec->args[1];
  1662. data.priority = dma_spec->args[2];
  1663. /*
  1664. * init dma_request2 to zero, which is not used by the dts.
  1665. * For P2P, dma_request2 is init from dma_request_channel(),
  1666. * chan->private will point to the imx_dma_data, and in
  1667. * device_alloc_chan_resources(), imx_dma_data.dma_request2 will
  1668. * be set to sdmac->event_id1.
  1669. */
  1670. data.dma_request2 = 0;
  1671. return __dma_request_channel(&mask, sdma_filter_fn, &data,
  1672. ofdma->of_node);
  1673. }
  1674. static int sdma_probe(struct platform_device *pdev)
  1675. {
  1676. const struct of_device_id *of_id =
  1677. of_match_device(sdma_dt_ids, &pdev->dev);
  1678. struct device_node *np = pdev->dev.of_node;
  1679. struct device_node *spba_bus;
  1680. const char *fw_name;
  1681. int ret;
  1682. int irq;
  1683. struct resource *iores;
  1684. struct resource spba_res;
  1685. struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev);
  1686. int i;
  1687. struct sdma_engine *sdma;
  1688. s32 *saddr_arr;
  1689. const struct sdma_driver_data *drvdata = NULL;
  1690. if (of_id)
  1691. drvdata = of_id->data;
  1692. else if (pdev->id_entry)
  1693. drvdata = (void *)pdev->id_entry->driver_data;
  1694. if (!drvdata) {
  1695. dev_err(&pdev->dev, "unable to find driver data\n");
  1696. return -EINVAL;
  1697. }
  1698. ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
  1699. if (ret)
  1700. return ret;
  1701. sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);
  1702. if (!sdma)
  1703. return -ENOMEM;
  1704. spin_lock_init(&sdma->channel_0_lock);
  1705. sdma->dev = &pdev->dev;
  1706. sdma->drvdata = drvdata;
  1707. irq = platform_get_irq(pdev, 0);
  1708. if (irq < 0)
  1709. return irq;
  1710. iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1711. sdma->regs = devm_ioremap_resource(&pdev->dev, iores);
  1712. if (IS_ERR(sdma->regs))
  1713. return PTR_ERR(sdma->regs);
  1714. sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
  1715. if (IS_ERR(sdma->clk_ipg))
  1716. return PTR_ERR(sdma->clk_ipg);
  1717. sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
  1718. if (IS_ERR(sdma->clk_ahb))
  1719. return PTR_ERR(sdma->clk_ahb);
  1720. ret = clk_prepare(sdma->clk_ipg);
  1721. if (ret)
  1722. return ret;
  1723. ret = clk_prepare(sdma->clk_ahb);
  1724. if (ret)
  1725. goto err_clk;
  1726. ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma",
  1727. sdma);
  1728. if (ret)
  1729. goto err_irq;
  1730. sdma->irq = irq;
  1731. sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
  1732. if (!sdma->script_addrs) {
  1733. ret = -ENOMEM;
  1734. goto err_irq;
  1735. }
  1736. /* initially no scripts available */
  1737. saddr_arr = (s32 *)sdma->script_addrs;
  1738. for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++)
  1739. saddr_arr[i] = -EINVAL;
  1740. dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
  1741. dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
  1742. dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
  1743. INIT_LIST_HEAD(&sdma->dma_device.channels);
  1744. /* Initialize channel parameters */
  1745. for (i = 0; i < MAX_DMA_CHANNELS; i++) {
  1746. struct sdma_channel *sdmac = &sdma->channel[i];
  1747. sdmac->sdma = sdma;
  1748. sdmac->channel = i;
  1749. sdmac->vc.desc_free = sdma_desc_free;
  1750. INIT_WORK(&sdmac->terminate_worker,
  1751. sdma_channel_terminate_work);
  1752. /*
  1753. * Add the channel to the DMAC list. Do not add channel 0 though
  1754. * because we need it internally in the SDMA driver. This also means
  1755. * that channel 0 in dmaengine counting matches sdma channel 1.
  1756. */
  1757. if (i)
  1758. vchan_init(&sdmac->vc, &sdma->dma_device);
  1759. }
  1760. ret = sdma_init(sdma);
  1761. if (ret)
  1762. goto err_init;
  1763. ret = sdma_event_remap(sdma);
  1764. if (ret)
  1765. goto err_init;
  1766. if (sdma->drvdata->script_addrs)
  1767. sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
  1768. if (pdata && pdata->script_addrs)
  1769. sdma_add_scripts(sdma, pdata->script_addrs);
  1770. sdma->dma_device.dev = &pdev->dev;
  1771. sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
  1772. sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
  1773. sdma->dma_device.device_tx_status = sdma_tx_status;
  1774. sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
  1775. sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
  1776. sdma->dma_device.device_config = sdma_config;
  1777. sdma->dma_device.device_terminate_all = sdma_terminate_all;
  1778. sdma->dma_device.device_synchronize = sdma_channel_synchronize;
  1779. sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
  1780. sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
  1781. sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
  1782. sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
  1783. sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
  1784. sdma->dma_device.device_issue_pending = sdma_issue_pending;
  1785. sdma->dma_device.copy_align = 2;
  1786. dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
  1787. platform_set_drvdata(pdev, sdma);
  1788. ret = dma_async_device_register(&sdma->dma_device);
  1789. if (ret) {
  1790. dev_err(&pdev->dev, "unable to register\n");
  1791. goto err_init;
  1792. }
  1793. if (np) {
  1794. ret = of_dma_controller_register(np, sdma_xlate, sdma);
  1795. if (ret) {
  1796. dev_err(&pdev->dev, "failed to register controller\n");
  1797. goto err_register;
  1798. }
  1799. spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
  1800. ret = of_address_to_resource(spba_bus, 0, &spba_res);
  1801. if (!ret) {
  1802. sdma->spba_start_addr = spba_res.start;
  1803. sdma->spba_end_addr = spba_res.end;
  1804. }
  1805. of_node_put(spba_bus);
  1806. }
  1807. /*
  1808. * Kick off firmware loading as the very last step:
  1809. * attempt to load firmware only if we're not on the error path, because
  1810. * the firmware callback requires a fully functional and allocated sdma
  1811. * instance.
  1812. */
  1813. if (pdata) {
  1814. ret = sdma_get_firmware(sdma, pdata->fw_name);
  1815. if (ret)
  1816. dev_warn(&pdev->dev, "failed to get firmware from platform data\n");
  1817. } else {
  1818. /*
  1819. * Because that device tree does not encode ROM script address,
  1820. * the RAM script in firmware is mandatory for device tree
  1821. * probe, otherwise it fails.
  1822. */
  1823. ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
  1824. &fw_name);
  1825. if (ret) {
  1826. dev_warn(&pdev->dev, "failed to get firmware name\n");
  1827. } else {
  1828. ret = sdma_get_firmware(sdma, fw_name);
  1829. if (ret)
  1830. dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
  1831. }
  1832. }
  1833. return 0;
  1834. err_register:
  1835. dma_async_device_unregister(&sdma->dma_device);
  1836. err_init:
  1837. kfree(sdma->script_addrs);
  1838. err_irq:
  1839. clk_unprepare(sdma->clk_ahb);
  1840. err_clk:
  1841. clk_unprepare(sdma->clk_ipg);
  1842. return ret;
  1843. }
  1844. static int sdma_remove(struct platform_device *pdev)
  1845. {
  1846. struct sdma_engine *sdma = platform_get_drvdata(pdev);
  1847. int i;
  1848. devm_free_irq(&pdev->dev, sdma->irq, sdma);
  1849. dma_async_device_unregister(&sdma->dma_device);
  1850. kfree(sdma->script_addrs);
  1851. clk_unprepare(sdma->clk_ahb);
  1852. clk_unprepare(sdma->clk_ipg);
  1853. /* Kill the tasklet */
  1854. for (i = 0; i < MAX_DMA_CHANNELS; i++) {
  1855. struct sdma_channel *sdmac = &sdma->channel[i];
  1856. tasklet_kill(&sdmac->vc.task);
  1857. sdma_free_chan_resources(&sdmac->vc.chan);
  1858. }
  1859. platform_set_drvdata(pdev, NULL);
  1860. return 0;
  1861. }
  1862. static struct platform_driver sdma_driver = {
  1863. .driver = {
  1864. .name = "imx-sdma",
  1865. .of_match_table = sdma_dt_ids,
  1866. },
  1867. .id_table = sdma_devtypes,
  1868. .remove = sdma_remove,
  1869. .probe = sdma_probe,
  1870. };
  1871. module_platform_driver(sdma_driver);
  1872. MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
  1873. MODULE_DESCRIPTION("i.MX SDMA driver");
  1874. #if IS_ENABLED(CONFIG_SOC_IMX6Q)
  1875. MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
  1876. #endif
  1877. #if IS_ENABLED(CONFIG_SOC_IMX7D)
  1878. MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
  1879. #endif
  1880. MODULE_LICENSE("GPL");