amba-pl08x.c 81 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (c) 2006 ARM Ltd.
  4. * Copyright (c) 2010 ST-Ericsson SA
  5. * Copyirght (c) 2017 Linaro Ltd.
  6. *
  7. * Author: Peter Pearse <peter.pearse@arm.com>
  8. * Author: Linus Walleij <linus.walleij@linaro.org>
  9. *
  10. * Documentation: ARM DDI 0196G == PL080
  11. * Documentation: ARM DDI 0218E == PL081
  12. * Documentation: S3C6410 User's Manual == PL080S
  13. *
  14. * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
  15. * channel.
  16. *
  17. * The PL080 has 8 channels available for simultaneous use, and the PL081
  18. * has only two channels. So on these DMA controllers the number of channels
  19. * and the number of incoming DMA signals are two totally different things.
  20. * It is usually not possible to theoretically handle all physical signals,
  21. * so a multiplexing scheme with possible denial of use is necessary.
  22. *
  23. * The PL080 has a dual bus master, PL081 has a single master.
  24. *
  25. * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
  26. * It differs in following aspects:
  27. * - CH_CONFIG register at different offset,
  28. * - separate CH_CONTROL2 register for transfer size,
  29. * - bigger maximum transfer size,
  30. * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
  31. * - no support for peripheral flow control.
  32. *
  33. * Memory to peripheral transfer may be visualized as
  34. * Get data from memory to DMAC
  35. * Until no data left
  36. * On burst request from peripheral
  37. * Destination burst from DMAC to peripheral
  38. * Clear burst request
  39. * Raise terminal count interrupt
  40. *
  41. * For peripherals with a FIFO:
  42. * Source burst size == half the depth of the peripheral FIFO
  43. * Destination burst size == the depth of the peripheral FIFO
  44. *
  45. * (Bursts are irrelevant for mem to mem transfers - there are no burst
  46. * signals, the DMA controller will simply facilitate its AHB master.)
  47. *
  48. * ASSUMES default (little) endianness for DMA transfers
  49. *
  50. * The PL08x has two flow control settings:
  51. * - DMAC flow control: the transfer size defines the number of transfers
  52. * which occur for the current LLI entry, and the DMAC raises TC at the
  53. * end of every LLI entry. Observed behaviour shows the DMAC listening
  54. * to both the BREQ and SREQ signals (contrary to documented),
  55. * transferring data if either is active. The LBREQ and LSREQ signals
  56. * are ignored.
  57. *
  58. * - Peripheral flow control: the transfer size is ignored (and should be
  59. * zero). The data is transferred from the current LLI entry, until
  60. * after the final transfer signalled by LBREQ or LSREQ. The DMAC
  61. * will then move to the next LLI entry. Unsupported by PL080S.
  62. */
  63. #include <linux/amba/bus.h>
  64. #include <linux/amba/pl08x.h>
  65. #include <linux/debugfs.h>
  66. #include <linux/delay.h>
  67. #include <linux/device.h>
  68. #include <linux/dmaengine.h>
  69. #include <linux/dmapool.h>
  70. #include <linux/dma-mapping.h>
  71. #include <linux/export.h>
  72. #include <linux/init.h>
  73. #include <linux/interrupt.h>
  74. #include <linux/module.h>
  75. #include <linux/of.h>
  76. #include <linux/of_dma.h>
  77. #include <linux/pm_runtime.h>
  78. #include <linux/seq_file.h>
  79. #include <linux/slab.h>
  80. #include <linux/amba/pl080.h>
  81. #include "dmaengine.h"
  82. #include "virt-dma.h"
  83. #define DRIVER_NAME "pl08xdmac"
  84. #define PL80X_DMA_BUSWIDTHS \
  85. BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
  86. BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  87. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  88. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
  89. static struct amba_driver pl08x_amba_driver;
  90. struct pl08x_driver_data;
  91. /**
  92. * struct vendor_data - vendor-specific config parameters for PL08x derivatives
  93. * @config_offset: offset to the configuration register
  94. * @channels: the number of channels available in this variant
  95. * @signals: the number of request signals available from the hardware
  96. * @dualmaster: whether this version supports dual AHB masters or not.
  97. * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the
  98. * channels have Nomadik security extension bits that need to be checked
  99. * for permission before use and some registers are missing
  100. * @pl080s: whether this variant is a Samsung PL080S, which has separate
  101. * register and LLI word for transfer size.
  102. * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020
  103. * @max_transfer_size: the maximum single element transfer size for this
  104. * PL08x variant.
  105. */
  106. struct vendor_data {
  107. u8 config_offset;
  108. u8 channels;
  109. u8 signals;
  110. bool dualmaster;
  111. bool nomadik;
  112. bool pl080s;
  113. bool ftdmac020;
  114. u32 max_transfer_size;
  115. };
  116. /**
  117. * struct pl08x_bus_data - information of source or destination
  118. * busses for a transfer
  119. * @addr: current address
  120. * @maxwidth: the maximum width of a transfer on this bus
  121. * @buswidth: the width of this bus in bytes: 1, 2 or 4
  122. */
  123. struct pl08x_bus_data {
  124. dma_addr_t addr;
  125. u8 maxwidth;
  126. u8 buswidth;
  127. };
  128. #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
  129. /**
  130. * struct pl08x_phy_chan - holder for the physical channels
  131. * @id: physical index to this channel
  132. * @base: memory base address for this physical channel
  133. * @reg_config: configuration address for this physical channel
  134. * @reg_control: control address for this physical channel
  135. * @reg_src: transfer source address register
  136. * @reg_dst: transfer destination address register
  137. * @reg_lli: transfer LLI address register
  138. * @reg_busy: if the variant has a special per-channel busy register,
  139. * this contains a pointer to it
  140. * @lock: a lock to use when altering an instance of this struct
  141. * @serving: the virtual channel currently being served by this physical
  142. * channel
  143. * @locked: channel unavailable for the system, e.g. dedicated to secure
  144. * world
  145. * @ftdmac020: channel is on a FTDMAC020
  146. * @pl080s: channel is on a PL08s
  147. */
  148. struct pl08x_phy_chan {
  149. unsigned int id;
  150. void __iomem *base;
  151. void __iomem *reg_config;
  152. void __iomem *reg_control;
  153. void __iomem *reg_src;
  154. void __iomem *reg_dst;
  155. void __iomem *reg_lli;
  156. void __iomem *reg_busy;
  157. spinlock_t lock;
  158. struct pl08x_dma_chan *serving;
  159. bool locked;
  160. bool ftdmac020;
  161. bool pl080s;
  162. };
  163. /**
  164. * struct pl08x_sg - structure containing data per sg
  165. * @src_addr: src address of sg
  166. * @dst_addr: dst address of sg
  167. * @len: transfer len in bytes
  168. * @node: node for txd's dsg_list
  169. */
  170. struct pl08x_sg {
  171. dma_addr_t src_addr;
  172. dma_addr_t dst_addr;
  173. size_t len;
  174. struct list_head node;
  175. };
  176. /**
  177. * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
  178. * @vd: virtual DMA descriptor
  179. * @dsg_list: list of children sg's
  180. * @llis_bus: DMA memory address (physical) start for the LLIs
  181. * @llis_va: virtual memory address start for the LLIs
  182. * @cctl: control reg values for current txd
  183. * @ccfg: config reg values for current txd
  184. * @done: this marks completed descriptors, which should not have their
  185. * mux released.
  186. * @cyclic: indicate cyclic transfers
  187. */
  188. struct pl08x_txd {
  189. struct virt_dma_desc vd;
  190. struct list_head dsg_list;
  191. dma_addr_t llis_bus;
  192. u32 *llis_va;
  193. /* Default cctl value for LLIs */
  194. u32 cctl;
  195. /*
  196. * Settings to be put into the physical channel when we
  197. * trigger this txd. Other registers are in llis_va[0].
  198. */
  199. u32 ccfg;
  200. bool done;
  201. bool cyclic;
  202. };
  203. /**
  204. * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
  205. * states
  206. * @PL08X_CHAN_IDLE: the channel is idle
  207. * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
  208. * channel and is running a transfer on it
  209. * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
  210. * channel, but the transfer is currently paused
  211. * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
  212. * channel to become available (only pertains to memcpy channels)
  213. */
  214. enum pl08x_dma_chan_state {
  215. PL08X_CHAN_IDLE,
  216. PL08X_CHAN_RUNNING,
  217. PL08X_CHAN_PAUSED,
  218. PL08X_CHAN_WAITING,
  219. };
  220. /**
  221. * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
  222. * @vc: wrappped virtual channel
  223. * @phychan: the physical channel utilized by this channel, if there is one
  224. * @name: name of channel
  225. * @cd: channel platform data
  226. * @cfg: slave configuration
  227. * @at: active transaction on this channel
  228. * @host: a pointer to the host (internal use)
  229. * @state: whether the channel is idle, paused, running etc
  230. * @slave: whether this channel is a device (slave) or for memcpy
  231. * @signal: the physical DMA request signal which this channel is using
  232. * @mux_use: count of descriptors using this DMA request signal setting
  233. * @waiting_at: time in jiffies when this channel moved to waiting state
  234. */
  235. struct pl08x_dma_chan {
  236. struct virt_dma_chan vc;
  237. struct pl08x_phy_chan *phychan;
  238. const char *name;
  239. struct pl08x_channel_data *cd;
  240. struct dma_slave_config cfg;
  241. struct pl08x_txd *at;
  242. struct pl08x_driver_data *host;
  243. enum pl08x_dma_chan_state state;
  244. bool slave;
  245. int signal;
  246. unsigned mux_use;
  247. unsigned long waiting_at;
  248. };
  249. /**
  250. * struct pl08x_driver_data - the local state holder for the PL08x
  251. * @slave: optional slave engine for this instance
  252. * @memcpy: memcpy engine for this instance
  253. * @has_slave: the PL08x has a slave engine (routed signals)
  254. * @base: virtual memory base (remapped) for the PL08x
  255. * @adev: the corresponding AMBA (PrimeCell) bus entry
  256. * @vd: vendor data for this PL08x variant
  257. * @pd: platform data passed in from the platform/machine
  258. * @phy_chans: array of data for the physical channels
  259. * @pool: a pool for the LLI descriptors
  260. * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
  261. * fetches
  262. * @mem_buses: set to indicate memory transfers on AHB2.
  263. * @lli_words: how many words are used in each LLI item for this variant
  264. */
  265. struct pl08x_driver_data {
  266. struct dma_device slave;
  267. struct dma_device memcpy;
  268. bool has_slave;
  269. void __iomem *base;
  270. struct amba_device *adev;
  271. const struct vendor_data *vd;
  272. struct pl08x_platform_data *pd;
  273. struct pl08x_phy_chan *phy_chans;
  274. struct dma_pool *pool;
  275. u8 lli_buses;
  276. u8 mem_buses;
  277. u8 lli_words;
  278. };
  279. /*
  280. * PL08X specific defines
  281. */
  282. /* The order of words in an LLI. */
  283. #define PL080_LLI_SRC 0
  284. #define PL080_LLI_DST 1
  285. #define PL080_LLI_LLI 2
  286. #define PL080_LLI_CCTL 3
  287. #define PL080S_LLI_CCTL2 4
  288. /* Total words in an LLI. */
  289. #define PL080_LLI_WORDS 4
  290. #define PL080S_LLI_WORDS 8
  291. /*
  292. * Number of LLIs in each LLI buffer allocated for one transfer
  293. * (maximum times we call dma_pool_alloc on this pool without freeing)
  294. */
  295. #define MAX_NUM_TSFR_LLIS 512
  296. #define PL08X_ALIGN 8
  297. static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
  298. {
  299. return container_of(chan, struct pl08x_dma_chan, vc.chan);
  300. }
  301. static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
  302. {
  303. return container_of(tx, struct pl08x_txd, vd.tx);
  304. }
  305. /*
  306. * Mux handling.
  307. *
  308. * This gives us the DMA request input to the PL08x primecell which the
  309. * peripheral described by the channel data will be routed to, possibly
  310. * via a board/SoC specific external MUX. One important point to note
  311. * here is that this does not depend on the physical channel.
  312. */
  313. static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
  314. {
  315. const struct pl08x_platform_data *pd = plchan->host->pd;
  316. int ret;
  317. if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
  318. ret = pd->get_xfer_signal(plchan->cd);
  319. if (ret < 0) {
  320. plchan->mux_use = 0;
  321. return ret;
  322. }
  323. plchan->signal = ret;
  324. }
  325. return 0;
  326. }
  327. static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
  328. {
  329. const struct pl08x_platform_data *pd = plchan->host->pd;
  330. if (plchan->signal >= 0) {
  331. WARN_ON(plchan->mux_use == 0);
  332. if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
  333. pd->put_xfer_signal(plchan->cd, plchan->signal);
  334. plchan->signal = -1;
  335. }
  336. }
  337. }
  338. /*
  339. * Physical channel handling
  340. */
  341. /* Whether a certain channel is busy or not */
  342. static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
  343. {
  344. unsigned int val;
  345. /* If we have a special busy register, take a shortcut */
  346. if (ch->reg_busy) {
  347. val = readl(ch->reg_busy);
  348. return !!(val & BIT(ch->id));
  349. }
  350. val = readl(ch->reg_config);
  351. return val & PL080_CONFIG_ACTIVE;
  352. }
  353. /*
  354. * pl08x_write_lli() - Write an LLI into the DMA controller.
  355. *
  356. * The PL08x derivatives support linked lists, but the first item of the
  357. * list containing the source, destination, control word and next LLI is
  358. * ignored. Instead the driver has to write those values directly into the
  359. * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE
  360. * register need to be set up for the first transfer.
  361. */
  362. static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
  363. struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
  364. {
  365. if (pl08x->vd->pl080s)
  366. dev_vdbg(&pl08x->adev->dev,
  367. "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
  368. "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
  369. phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
  370. lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
  371. lli[PL080S_LLI_CCTL2], ccfg);
  372. else
  373. dev_vdbg(&pl08x->adev->dev,
  374. "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
  375. "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
  376. phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
  377. lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
  378. writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src);
  379. writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst);
  380. writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli);
  381. /*
  382. * The FTMAC020 has a different layout in the CCTL word of the LLI
  383. * and the CCTL register which is split in CSR and SIZE registers.
  384. * Convert the LLI item CCTL into the proper values to write into
  385. * the CSR and SIZE registers.
  386. */
  387. if (phychan->ftdmac020) {
  388. u32 llictl = lli[PL080_LLI_CCTL];
  389. u32 val = 0;
  390. /* Write the transfer size (12 bits) to the size register */
  391. writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK,
  392. phychan->base + FTDMAC020_CH_SIZE);
  393. /*
  394. * Then write the control bits 28..16 to the control register
  395. * by shuffleing the bits around to where they are in the
  396. * main register. The mapping is as follows:
  397. * Bit 28: TC_MSK - mask on all except last LLI
  398. * Bit 27..25: SRC_WIDTH
  399. * Bit 24..22: DST_WIDTH
  400. * Bit 21..20: SRCAD_CTRL
  401. * Bit 19..17: DSTAD_CTRL
  402. * Bit 17: SRC_SEL
  403. * Bit 16: DST_SEL
  404. */
  405. if (llictl & FTDMAC020_LLI_TC_MSK)
  406. val |= FTDMAC020_CH_CSR_TC_MSK;
  407. val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
  408. (FTDMAC020_LLI_SRC_WIDTH_SHIFT -
  409. FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT));
  410. val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
  411. (FTDMAC020_LLI_DST_WIDTH_SHIFT -
  412. FTDMAC020_CH_CSR_DST_WIDTH_SHIFT));
  413. val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >>
  414. (FTDMAC020_LLI_SRCAD_CTL_SHIFT -
  415. FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT));
  416. val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >>
  417. (FTDMAC020_LLI_DSTAD_CTL_SHIFT -
  418. FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT));
  419. if (llictl & FTDMAC020_LLI_SRC_SEL)
  420. val |= FTDMAC020_CH_CSR_SRC_SEL;
  421. if (llictl & FTDMAC020_LLI_DST_SEL)
  422. val |= FTDMAC020_CH_CSR_DST_SEL;
  423. /*
  424. * Set up the bits that exist in the CSR but are not
  425. * part the LLI, i.e. only gets written to the control
  426. * register right here.
  427. *
  428. * FIXME: do not just handle memcpy, also handle slave DMA.
  429. */
  430. switch (pl08x->pd->memcpy_burst_size) {
  431. default:
  432. case PL08X_BURST_SZ_1:
  433. val |= PL080_BSIZE_1 <<
  434. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  435. break;
  436. case PL08X_BURST_SZ_4:
  437. val |= PL080_BSIZE_4 <<
  438. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  439. break;
  440. case PL08X_BURST_SZ_8:
  441. val |= PL080_BSIZE_8 <<
  442. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  443. break;
  444. case PL08X_BURST_SZ_16:
  445. val |= PL080_BSIZE_16 <<
  446. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  447. break;
  448. case PL08X_BURST_SZ_32:
  449. val |= PL080_BSIZE_32 <<
  450. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  451. break;
  452. case PL08X_BURST_SZ_64:
  453. val |= PL080_BSIZE_64 <<
  454. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  455. break;
  456. case PL08X_BURST_SZ_128:
  457. val |= PL080_BSIZE_128 <<
  458. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  459. break;
  460. case PL08X_BURST_SZ_256:
  461. val |= PL080_BSIZE_256 <<
  462. FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
  463. break;
  464. }
  465. /* Protection flags */
  466. if (pl08x->pd->memcpy_prot_buff)
  467. val |= FTDMAC020_CH_CSR_PROT2;
  468. if (pl08x->pd->memcpy_prot_cache)
  469. val |= FTDMAC020_CH_CSR_PROT3;
  470. /* We are the kernel, so we are in privileged mode */
  471. val |= FTDMAC020_CH_CSR_PROT1;
  472. writel_relaxed(val, phychan->reg_control);
  473. } else {
  474. /* Bits are just identical */
  475. writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control);
  476. }
  477. /* Second control word on the PL080s */
  478. if (pl08x->vd->pl080s)
  479. writel_relaxed(lli[PL080S_LLI_CCTL2],
  480. phychan->base + PL080S_CH_CONTROL2);
  481. writel(ccfg, phychan->reg_config);
  482. }
  483. /*
  484. * Set the initial DMA register values i.e. those for the first LLI
  485. * The next LLI pointer and the configuration interrupt bit have
  486. * been set when the LLIs were constructed. Poke them into the hardware
  487. * and start the transfer.
  488. */
  489. static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
  490. {
  491. struct pl08x_driver_data *pl08x = plchan->host;
  492. struct pl08x_phy_chan *phychan = plchan->phychan;
  493. struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
  494. struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
  495. u32 val;
  496. list_del(&txd->vd.node);
  497. plchan->at = txd;
  498. /* Wait for channel inactive */
  499. while (pl08x_phy_channel_busy(phychan))
  500. cpu_relax();
  501. pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
  502. /* Enable the DMA channel */
  503. /* Do not access config register until channel shows as disabled */
  504. while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))
  505. cpu_relax();
  506. /* Do not access config register until channel shows as inactive */
  507. if (phychan->ftdmac020) {
  508. val = readl(phychan->reg_config);
  509. while (val & FTDMAC020_CH_CFG_BUSY)
  510. val = readl(phychan->reg_config);
  511. val = readl(phychan->reg_control);
  512. while (val & FTDMAC020_CH_CSR_EN)
  513. val = readl(phychan->reg_control);
  514. writel(val | FTDMAC020_CH_CSR_EN,
  515. phychan->reg_control);
  516. } else {
  517. val = readl(phychan->reg_config);
  518. while ((val & PL080_CONFIG_ACTIVE) ||
  519. (val & PL080_CONFIG_ENABLE))
  520. val = readl(phychan->reg_config);
  521. writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
  522. }
  523. }
  524. /*
  525. * Pause the channel by setting the HALT bit.
  526. *
  527. * For M->P transfers, pause the DMAC first and then stop the peripheral -
  528. * the FIFO can only drain if the peripheral is still requesting data.
  529. * (note: this can still timeout if the DMAC FIFO never drains of data.)
  530. *
  531. * For P->M transfers, disable the peripheral first to stop it filling
  532. * the DMAC FIFO, and then pause the DMAC.
  533. */
  534. static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
  535. {
  536. u32 val;
  537. int timeout;
  538. if (ch->ftdmac020) {
  539. /* Use the enable bit on the FTDMAC020 */
  540. val = readl(ch->reg_control);
  541. val &= ~FTDMAC020_CH_CSR_EN;
  542. writel(val, ch->reg_control);
  543. return;
  544. }
  545. /* Set the HALT bit and wait for the FIFO to drain */
  546. val = readl(ch->reg_config);
  547. val |= PL080_CONFIG_HALT;
  548. writel(val, ch->reg_config);
  549. /* Wait for channel inactive */
  550. for (timeout = 1000; timeout; timeout--) {
  551. if (!pl08x_phy_channel_busy(ch))
  552. break;
  553. udelay(1);
  554. }
  555. if (pl08x_phy_channel_busy(ch))
  556. pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
  557. }
  558. static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
  559. {
  560. u32 val;
  561. /* Use the enable bit on the FTDMAC020 */
  562. if (ch->ftdmac020) {
  563. val = readl(ch->reg_control);
  564. val |= FTDMAC020_CH_CSR_EN;
  565. writel(val, ch->reg_control);
  566. return;
  567. }
  568. /* Clear the HALT bit */
  569. val = readl(ch->reg_config);
  570. val &= ~PL080_CONFIG_HALT;
  571. writel(val, ch->reg_config);
  572. }
  573. /*
  574. * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
  575. * clears any pending interrupt status. This should not be used for
  576. * an on-going transfer, but as a method of shutting down a channel
  577. * (eg, when it's no longer used) or terminating a transfer.
  578. */
  579. static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
  580. struct pl08x_phy_chan *ch)
  581. {
  582. u32 val;
  583. /* The layout for the FTDMAC020 is different */
  584. if (ch->ftdmac020) {
  585. /* Disable all interrupts */
  586. val = readl(ch->reg_config);
  587. val |= (FTDMAC020_CH_CFG_INT_ABT_MASK |
  588. FTDMAC020_CH_CFG_INT_ERR_MASK |
  589. FTDMAC020_CH_CFG_INT_TC_MASK);
  590. writel(val, ch->reg_config);
  591. /* Abort and disable channel */
  592. val = readl(ch->reg_control);
  593. val &= ~FTDMAC020_CH_CSR_EN;
  594. val |= FTDMAC020_CH_CSR_ABT;
  595. writel(val, ch->reg_control);
  596. /* Clear ABT and ERR interrupt flags */
  597. writel(BIT(ch->id) | BIT(ch->id + 16),
  598. pl08x->base + PL080_ERR_CLEAR);
  599. writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
  600. return;
  601. }
  602. val = readl(ch->reg_config);
  603. val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
  604. PL080_CONFIG_TC_IRQ_MASK);
  605. writel(val, ch->reg_config);
  606. writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR);
  607. writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
  608. }
  609. static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch)
  610. {
  611. u32 val;
  612. u32 bytes;
  613. if (ch->ftdmac020) {
  614. bytes = readl(ch->base + FTDMAC020_CH_SIZE);
  615. val = readl(ch->reg_control);
  616. val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK;
  617. val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT;
  618. } else if (ch->pl080s) {
  619. val = readl(ch->base + PL080S_CH_CONTROL2);
  620. bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
  621. val = readl(ch->reg_control);
  622. val &= PL080_CONTROL_SWIDTH_MASK;
  623. val >>= PL080_CONTROL_SWIDTH_SHIFT;
  624. } else {
  625. /* Plain PL08x */
  626. val = readl(ch->reg_control);
  627. bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
  628. val &= PL080_CONTROL_SWIDTH_MASK;
  629. val >>= PL080_CONTROL_SWIDTH_SHIFT;
  630. }
  631. switch (val) {
  632. case PL080_WIDTH_8BIT:
  633. break;
  634. case PL080_WIDTH_16BIT:
  635. bytes *= 2;
  636. break;
  637. case PL080_WIDTH_32BIT:
  638. bytes *= 4;
  639. break;
  640. }
  641. return bytes;
  642. }
  643. static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va)
  644. {
  645. u32 val;
  646. u32 bytes;
  647. if (ch->ftdmac020) {
  648. val = llis_va[PL080_LLI_CCTL];
  649. bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK;
  650. val = llis_va[PL080_LLI_CCTL];
  651. val &= FTDMAC020_LLI_SRC_WIDTH_MSK;
  652. val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT;
  653. } else if (ch->pl080s) {
  654. val = llis_va[PL080S_LLI_CCTL2];
  655. bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
  656. val = llis_va[PL080_LLI_CCTL];
  657. val &= PL080_CONTROL_SWIDTH_MASK;
  658. val >>= PL080_CONTROL_SWIDTH_SHIFT;
  659. } else {
  660. /* Plain PL08x */
  661. val = llis_va[PL080_LLI_CCTL];
  662. bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
  663. val &= PL080_CONTROL_SWIDTH_MASK;
  664. val >>= PL080_CONTROL_SWIDTH_SHIFT;
  665. }
  666. switch (val) {
  667. case PL080_WIDTH_8BIT:
  668. break;
  669. case PL080_WIDTH_16BIT:
  670. bytes *= 2;
  671. break;
  672. case PL080_WIDTH_32BIT:
  673. bytes *= 4;
  674. break;
  675. }
  676. return bytes;
  677. }
  678. /* The channel should be paused when calling this */
  679. static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
  680. {
  681. struct pl08x_driver_data *pl08x = plchan->host;
  682. const u32 *llis_va, *llis_va_limit;
  683. struct pl08x_phy_chan *ch;
  684. dma_addr_t llis_bus;
  685. struct pl08x_txd *txd;
  686. u32 llis_max_words;
  687. size_t bytes;
  688. u32 clli;
  689. ch = plchan->phychan;
  690. txd = plchan->at;
  691. if (!ch || !txd)
  692. return 0;
  693. /*
  694. * Follow the LLIs to get the number of remaining
  695. * bytes in the currently active transaction.
  696. */
  697. clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2;
  698. /* First get the remaining bytes in the active transfer */
  699. bytes = get_bytes_in_phy_channel(ch);
  700. if (!clli)
  701. return bytes;
  702. llis_va = txd->llis_va;
  703. llis_bus = txd->llis_bus;
  704. llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
  705. BUG_ON(clli < llis_bus || clli >= llis_bus +
  706. sizeof(u32) * llis_max_words);
  707. /*
  708. * Locate the next LLI - as this is an array,
  709. * it's simple maths to find.
  710. */
  711. llis_va += (clli - llis_bus) / sizeof(u32);
  712. llis_va_limit = llis_va + llis_max_words;
  713. for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
  714. bytes += get_bytes_in_lli(ch, llis_va);
  715. /*
  716. * A LLI pointer going backward terminates the LLI list
  717. */
  718. if (llis_va[PL080_LLI_LLI] <= clli)
  719. break;
  720. }
  721. return bytes;
  722. }
  723. /*
  724. * Allocate a physical channel for a virtual channel
  725. *
  726. * Try to locate a physical channel to be used for this transfer. If all
  727. * are taken return NULL and the requester will have to cope by using
  728. * some fallback PIO mode or retrying later.
  729. */
  730. static struct pl08x_phy_chan *
  731. pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
  732. struct pl08x_dma_chan *virt_chan)
  733. {
  734. struct pl08x_phy_chan *ch = NULL;
  735. unsigned long flags;
  736. int i;
  737. for (i = 0; i < pl08x->vd->channels; i++) {
  738. ch = &pl08x->phy_chans[i];
  739. spin_lock_irqsave(&ch->lock, flags);
  740. if (!ch->locked && !ch->serving) {
  741. ch->serving = virt_chan;
  742. spin_unlock_irqrestore(&ch->lock, flags);
  743. break;
  744. }
  745. spin_unlock_irqrestore(&ch->lock, flags);
  746. }
  747. if (i == pl08x->vd->channels) {
  748. /* No physical channel available, cope with it */
  749. return NULL;
  750. }
  751. return ch;
  752. }
  753. /* Mark the physical channel as free. Note, this write is atomic. */
  754. static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
  755. struct pl08x_phy_chan *ch)
  756. {
  757. ch->serving = NULL;
  758. }
  759. /*
  760. * Try to allocate a physical channel. When successful, assign it to
  761. * this virtual channel, and initiate the next descriptor. The
  762. * virtual channel lock must be held at this point.
  763. */
  764. static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
  765. {
  766. struct pl08x_driver_data *pl08x = plchan->host;
  767. struct pl08x_phy_chan *ch;
  768. ch = pl08x_get_phy_channel(pl08x, plchan);
  769. if (!ch) {
  770. dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
  771. plchan->state = PL08X_CHAN_WAITING;
  772. plchan->waiting_at = jiffies;
  773. return;
  774. }
  775. dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
  776. ch->id, plchan->name);
  777. plchan->phychan = ch;
  778. plchan->state = PL08X_CHAN_RUNNING;
  779. pl08x_start_next_txd(plchan);
  780. }
  781. static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
  782. struct pl08x_dma_chan *plchan)
  783. {
  784. struct pl08x_driver_data *pl08x = plchan->host;
  785. dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
  786. ch->id, plchan->name);
  787. /*
  788. * We do this without taking the lock; we're really only concerned
  789. * about whether this pointer is NULL or not, and we're guaranteed
  790. * that this will only be called when it _already_ is non-NULL.
  791. */
  792. ch->serving = plchan;
  793. plchan->phychan = ch;
  794. plchan->state = PL08X_CHAN_RUNNING;
  795. pl08x_start_next_txd(plchan);
  796. }
  797. /*
  798. * Free a physical DMA channel, potentially reallocating it to another
  799. * virtual channel if we have any pending.
  800. */
  801. static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
  802. {
  803. struct pl08x_driver_data *pl08x = plchan->host;
  804. struct pl08x_dma_chan *p, *next;
  805. unsigned long waiting_at;
  806. retry:
  807. next = NULL;
  808. waiting_at = jiffies;
  809. /*
  810. * Find a waiting virtual channel for the next transfer.
  811. * To be fair, time when each channel reached waiting state is compared
  812. * to select channel that is waiting for the longest time.
  813. */
  814. list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
  815. if (p->state == PL08X_CHAN_WAITING &&
  816. p->waiting_at <= waiting_at) {
  817. next = p;
  818. waiting_at = p->waiting_at;
  819. }
  820. if (!next && pl08x->has_slave) {
  821. list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
  822. if (p->state == PL08X_CHAN_WAITING &&
  823. p->waiting_at <= waiting_at) {
  824. next = p;
  825. waiting_at = p->waiting_at;
  826. }
  827. }
  828. /* Ensure that the physical channel is stopped */
  829. pl08x_terminate_phy_chan(pl08x, plchan->phychan);
  830. if (next) {
  831. bool success;
  832. /*
  833. * Eww. We know this isn't going to deadlock
  834. * but lockdep probably doesn't.
  835. */
  836. spin_lock(&next->vc.lock);
  837. /* Re-check the state now that we have the lock */
  838. success = next->state == PL08X_CHAN_WAITING;
  839. if (success)
  840. pl08x_phy_reassign_start(plchan->phychan, next);
  841. spin_unlock(&next->vc.lock);
  842. /* If the state changed, try to find another channel */
  843. if (!success)
  844. goto retry;
  845. } else {
  846. /* No more jobs, so free up the physical channel */
  847. pl08x_put_phy_channel(pl08x, plchan->phychan);
  848. }
  849. plchan->phychan = NULL;
  850. plchan->state = PL08X_CHAN_IDLE;
  851. }
  852. /*
  853. * LLI handling
  854. */
  855. static inline unsigned int
  856. pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x,
  857. u32 cctl,
  858. bool source)
  859. {
  860. u32 val;
  861. if (pl08x->vd->ftdmac020) {
  862. if (source)
  863. val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
  864. FTDMAC020_LLI_SRC_WIDTH_SHIFT;
  865. else
  866. val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
  867. FTDMAC020_LLI_DST_WIDTH_SHIFT;
  868. } else {
  869. if (source)
  870. val = (cctl & PL080_CONTROL_SWIDTH_MASK) >>
  871. PL080_CONTROL_SWIDTH_SHIFT;
  872. else
  873. val = (cctl & PL080_CONTROL_DWIDTH_MASK) >>
  874. PL080_CONTROL_DWIDTH_SHIFT;
  875. }
  876. switch (val) {
  877. case PL080_WIDTH_8BIT:
  878. return 1;
  879. case PL080_WIDTH_16BIT:
  880. return 2;
  881. case PL080_WIDTH_32BIT:
  882. return 4;
  883. default:
  884. break;
  885. }
  886. BUG();
  887. return 0;
  888. }
  889. static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,
  890. u32 cctl,
  891. u8 srcwidth, u8 dstwidth,
  892. size_t tsize)
  893. {
  894. u32 retbits = cctl;
  895. /*
  896. * Remove all src, dst and transfer size bits, then set the
  897. * width and size according to the parameters. The bit offsets
  898. * are different in the FTDMAC020 so we need to accound for this.
  899. */
  900. if (pl08x->vd->ftdmac020) {
  901. retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;
  902. retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK;
  903. retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK;
  904. switch (srcwidth) {
  905. case 1:
  906. retbits |= PL080_WIDTH_8BIT <<
  907. FTDMAC020_LLI_SRC_WIDTH_SHIFT;
  908. break;
  909. case 2:
  910. retbits |= PL080_WIDTH_16BIT <<
  911. FTDMAC020_LLI_SRC_WIDTH_SHIFT;
  912. break;
  913. case 4:
  914. retbits |= PL080_WIDTH_32BIT <<
  915. FTDMAC020_LLI_SRC_WIDTH_SHIFT;
  916. break;
  917. default:
  918. BUG();
  919. break;
  920. }
  921. switch (dstwidth) {
  922. case 1:
  923. retbits |= PL080_WIDTH_8BIT <<
  924. FTDMAC020_LLI_DST_WIDTH_SHIFT;
  925. break;
  926. case 2:
  927. retbits |= PL080_WIDTH_16BIT <<
  928. FTDMAC020_LLI_DST_WIDTH_SHIFT;
  929. break;
  930. case 4:
  931. retbits |= PL080_WIDTH_32BIT <<
  932. FTDMAC020_LLI_DST_WIDTH_SHIFT;
  933. break;
  934. default:
  935. BUG();
  936. break;
  937. }
  938. tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK;
  939. retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT;
  940. } else {
  941. retbits &= ~PL080_CONTROL_DWIDTH_MASK;
  942. retbits &= ~PL080_CONTROL_SWIDTH_MASK;
  943. retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
  944. switch (srcwidth) {
  945. case 1:
  946. retbits |= PL080_WIDTH_8BIT <<
  947. PL080_CONTROL_SWIDTH_SHIFT;
  948. break;
  949. case 2:
  950. retbits |= PL080_WIDTH_16BIT <<
  951. PL080_CONTROL_SWIDTH_SHIFT;
  952. break;
  953. case 4:
  954. retbits |= PL080_WIDTH_32BIT <<
  955. PL080_CONTROL_SWIDTH_SHIFT;
  956. break;
  957. default:
  958. BUG();
  959. break;
  960. }
  961. switch (dstwidth) {
  962. case 1:
  963. retbits |= PL080_WIDTH_8BIT <<
  964. PL080_CONTROL_DWIDTH_SHIFT;
  965. break;
  966. case 2:
  967. retbits |= PL080_WIDTH_16BIT <<
  968. PL080_CONTROL_DWIDTH_SHIFT;
  969. break;
  970. case 4:
  971. retbits |= PL080_WIDTH_32BIT <<
  972. PL080_CONTROL_DWIDTH_SHIFT;
  973. break;
  974. default:
  975. BUG();
  976. break;
  977. }
  978. tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
  979. retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
  980. }
  981. return retbits;
  982. }
  983. struct pl08x_lli_build_data {
  984. struct pl08x_txd *txd;
  985. struct pl08x_bus_data srcbus;
  986. struct pl08x_bus_data dstbus;
  987. size_t remainder;
  988. u32 lli_bus;
  989. };
  990. /*
  991. * Autoselect a master bus to use for the transfer. Slave will be the chosen as
  992. * victim in case src & dest are not similarly aligned. i.e. If after aligning
  993. * masters address with width requirements of transfer (by sending few byte by
  994. * byte data), slave is still not aligned, then its width will be reduced to
  995. * BYTE.
  996. * - prefers the destination bus if both available
  997. * - prefers bus with fixed address (i.e. peripheral)
  998. */
  999. static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x,
  1000. struct pl08x_lli_build_data *bd,
  1001. struct pl08x_bus_data **mbus,
  1002. struct pl08x_bus_data **sbus,
  1003. u32 cctl)
  1004. {
  1005. bool dst_incr;
  1006. bool src_incr;
  1007. /*
  1008. * The FTDMAC020 only supports memory-to-memory transfer, so
  1009. * source and destination always increase.
  1010. */
  1011. if (pl08x->vd->ftdmac020) {
  1012. dst_incr = true;
  1013. src_incr = true;
  1014. } else {
  1015. dst_incr = !!(cctl & PL080_CONTROL_DST_INCR);
  1016. src_incr = !!(cctl & PL080_CONTROL_SRC_INCR);
  1017. }
  1018. /*
  1019. * If either bus is not advancing, i.e. it is a peripheral, that
  1020. * one becomes master
  1021. */
  1022. if (!dst_incr) {
  1023. *mbus = &bd->dstbus;
  1024. *sbus = &bd->srcbus;
  1025. } else if (!src_incr) {
  1026. *mbus = &bd->srcbus;
  1027. *sbus = &bd->dstbus;
  1028. } else {
  1029. if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
  1030. *mbus = &bd->dstbus;
  1031. *sbus = &bd->srcbus;
  1032. } else {
  1033. *mbus = &bd->srcbus;
  1034. *sbus = &bd->dstbus;
  1035. }
  1036. }
  1037. }
  1038. /*
  1039. * Fills in one LLI for a certain transfer descriptor and advance the counter
  1040. */
  1041. static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
  1042. struct pl08x_lli_build_data *bd,
  1043. int num_llis, int len, u32 cctl, u32 cctl2)
  1044. {
  1045. u32 offset = num_llis * pl08x->lli_words;
  1046. u32 *llis_va = bd->txd->llis_va + offset;
  1047. dma_addr_t llis_bus = bd->txd->llis_bus;
  1048. BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
  1049. /* Advance the offset to next LLI. */
  1050. offset += pl08x->lli_words;
  1051. llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
  1052. llis_va[PL080_LLI_DST] = bd->dstbus.addr;
  1053. llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
  1054. llis_va[PL080_LLI_LLI] |= bd->lli_bus;
  1055. llis_va[PL080_LLI_CCTL] = cctl;
  1056. if (pl08x->vd->pl080s)
  1057. llis_va[PL080S_LLI_CCTL2] = cctl2;
  1058. if (pl08x->vd->ftdmac020) {
  1059. /* FIXME: only memcpy so far so both increase */
  1060. bd->srcbus.addr += len;
  1061. bd->dstbus.addr += len;
  1062. } else {
  1063. if (cctl & PL080_CONTROL_SRC_INCR)
  1064. bd->srcbus.addr += len;
  1065. if (cctl & PL080_CONTROL_DST_INCR)
  1066. bd->dstbus.addr += len;
  1067. }
  1068. BUG_ON(bd->remainder < len);
  1069. bd->remainder -= len;
  1070. }
  1071. static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
  1072. struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
  1073. int num_llis, size_t *total_bytes)
  1074. {
  1075. *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len);
  1076. pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
  1077. (*total_bytes) += len;
  1078. }
  1079. #if 1
  1080. static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
  1081. const u32 *llis_va, int num_llis)
  1082. {
  1083. int i;
  1084. if (pl08x->vd->pl080s) {
  1085. dev_vdbg(&pl08x->adev->dev,
  1086. "%-3s %-9s %-10s %-10s %-10s %-10s %s\n",
  1087. "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
  1088. for (i = 0; i < num_llis; i++) {
  1089. dev_vdbg(&pl08x->adev->dev,
  1090. "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
  1091. i, llis_va, llis_va[PL080_LLI_SRC],
  1092. llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
  1093. llis_va[PL080_LLI_CCTL],
  1094. llis_va[PL080S_LLI_CCTL2]);
  1095. llis_va += pl08x->lli_words;
  1096. }
  1097. } else {
  1098. dev_vdbg(&pl08x->adev->dev,
  1099. "%-3s %-9s %-10s %-10s %-10s %s\n",
  1100. "lli", "", "csrc", "cdst", "clli", "cctl");
  1101. for (i = 0; i < num_llis; i++) {
  1102. dev_vdbg(&pl08x->adev->dev,
  1103. "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
  1104. i, llis_va, llis_va[PL080_LLI_SRC],
  1105. llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
  1106. llis_va[PL080_LLI_CCTL]);
  1107. llis_va += pl08x->lli_words;
  1108. }
  1109. }
  1110. }
  1111. #else
  1112. static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
  1113. const u32 *llis_va, int num_llis) {}
  1114. #endif
  1115. /*
  1116. * This fills in the table of LLIs for the transfer descriptor
  1117. * Note that we assume we never have to change the burst sizes
  1118. * Return 0 for error
  1119. */
  1120. static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
  1121. struct pl08x_txd *txd)
  1122. {
  1123. struct pl08x_bus_data *mbus, *sbus;
  1124. struct pl08x_lli_build_data bd;
  1125. int num_llis = 0;
  1126. u32 cctl, early_bytes = 0;
  1127. size_t max_bytes_per_lli, total_bytes;
  1128. u32 *llis_va, *last_lli;
  1129. struct pl08x_sg *dsg;
  1130. txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
  1131. if (!txd->llis_va) {
  1132. dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
  1133. return 0;
  1134. }
  1135. bd.txd = txd;
  1136. bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
  1137. cctl = txd->cctl;
  1138. /* Find maximum width of the source bus */
  1139. bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true);
  1140. /* Find maximum width of the destination bus */
  1141. bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false);
  1142. list_for_each_entry(dsg, &txd->dsg_list, node) {
  1143. total_bytes = 0;
  1144. cctl = txd->cctl;
  1145. bd.srcbus.addr = dsg->src_addr;
  1146. bd.dstbus.addr = dsg->dst_addr;
  1147. bd.remainder = dsg->len;
  1148. bd.srcbus.buswidth = bd.srcbus.maxwidth;
  1149. bd.dstbus.buswidth = bd.dstbus.maxwidth;
  1150. pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl);
  1151. dev_vdbg(&pl08x->adev->dev,
  1152. "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
  1153. (u64)bd.srcbus.addr,
  1154. cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
  1155. bd.srcbus.buswidth,
  1156. (u64)bd.dstbus.addr,
  1157. cctl & PL080_CONTROL_DST_INCR ? "+" : "",
  1158. bd.dstbus.buswidth,
  1159. bd.remainder);
  1160. dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
  1161. mbus == &bd.srcbus ? "src" : "dst",
  1162. sbus == &bd.srcbus ? "src" : "dst");
  1163. /*
  1164. * Zero length is only allowed if all these requirements are
  1165. * met:
  1166. * - flow controller is peripheral.
  1167. * - src.addr is aligned to src.width
  1168. * - dst.addr is aligned to dst.width
  1169. *
  1170. * sg_len == 1 should be true, as there can be two cases here:
  1171. *
  1172. * - Memory addresses are contiguous and are not scattered.
  1173. * Here, Only one sg will be passed by user driver, with
  1174. * memory address and zero length. We pass this to controller
  1175. * and after the transfer it will receive the last burst
  1176. * request from peripheral and so transfer finishes.
  1177. *
  1178. * - Memory addresses are scattered and are not contiguous.
  1179. * Here, Obviously as DMA controller doesn't know when a lli's
  1180. * transfer gets over, it can't load next lli. So in this
  1181. * case, there has to be an assumption that only one lli is
  1182. * supported. Thus, we can't have scattered addresses.
  1183. */
  1184. if (!bd.remainder) {
  1185. u32 fc;
  1186. /* FTDMAC020 only does memory-to-memory */
  1187. if (pl08x->vd->ftdmac020)
  1188. fc = PL080_FLOW_MEM2MEM;
  1189. else
  1190. fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
  1191. PL080_CONFIG_FLOW_CONTROL_SHIFT;
  1192. if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
  1193. (fc <= PL080_FLOW_SRC2DST_SRC))) {
  1194. dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
  1195. __func__);
  1196. return 0;
  1197. }
  1198. if (!IS_BUS_ALIGNED(&bd.srcbus) ||
  1199. !IS_BUS_ALIGNED(&bd.dstbus)) {
  1200. dev_err(&pl08x->adev->dev,
  1201. "%s src & dst address must be aligned to src"
  1202. " & dst width if peripheral is flow controller",
  1203. __func__);
  1204. return 0;
  1205. }
  1206. cctl = pl08x_lli_control_bits(pl08x, cctl,
  1207. bd.srcbus.buswidth, bd.dstbus.buswidth,
  1208. 0);
  1209. pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
  1210. 0, cctl, 0);
  1211. break;
  1212. }
  1213. /*
  1214. * Send byte by byte for following cases
  1215. * - Less than a bus width available
  1216. * - until master bus is aligned
  1217. */
  1218. if (bd.remainder < mbus->buswidth)
  1219. early_bytes = bd.remainder;
  1220. else if (!IS_BUS_ALIGNED(mbus)) {
  1221. early_bytes = mbus->buswidth -
  1222. (mbus->addr & (mbus->buswidth - 1));
  1223. if ((bd.remainder - early_bytes) < mbus->buswidth)
  1224. early_bytes = bd.remainder;
  1225. }
  1226. if (early_bytes) {
  1227. dev_vdbg(&pl08x->adev->dev,
  1228. "%s byte width LLIs (remain 0x%08zx)\n",
  1229. __func__, bd.remainder);
  1230. prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
  1231. num_llis++, &total_bytes);
  1232. }
  1233. if (bd.remainder) {
  1234. /*
  1235. * Master now aligned
  1236. * - if slave is not then we must set its width down
  1237. */
  1238. if (!IS_BUS_ALIGNED(sbus)) {
  1239. dev_dbg(&pl08x->adev->dev,
  1240. "%s set down bus width to one byte\n",
  1241. __func__);
  1242. sbus->buswidth = 1;
  1243. }
  1244. /*
  1245. * Bytes transferred = tsize * src width, not
  1246. * MIN(buswidths)
  1247. */
  1248. max_bytes_per_lli = bd.srcbus.buswidth *
  1249. pl08x->vd->max_transfer_size;
  1250. dev_vdbg(&pl08x->adev->dev,
  1251. "%s max bytes per lli = %zu\n",
  1252. __func__, max_bytes_per_lli);
  1253. /*
  1254. * Make largest possible LLIs until less than one bus
  1255. * width left
  1256. */
  1257. while (bd.remainder > (mbus->buswidth - 1)) {
  1258. size_t lli_len, tsize, width;
  1259. /*
  1260. * If enough left try to send max possible,
  1261. * otherwise try to send the remainder
  1262. */
  1263. lli_len = min(bd.remainder, max_bytes_per_lli);
  1264. /*
  1265. * Check against maximum bus alignment:
  1266. * Calculate actual transfer size in relation to
  1267. * bus width an get a maximum remainder of the
  1268. * highest bus width - 1
  1269. */
  1270. width = max(mbus->buswidth, sbus->buswidth);
  1271. lli_len = (lli_len / width) * width;
  1272. tsize = lli_len / bd.srcbus.buswidth;
  1273. dev_vdbg(&pl08x->adev->dev,
  1274. "%s fill lli with single lli chunk of "
  1275. "size 0x%08zx (remainder 0x%08zx)\n",
  1276. __func__, lli_len, bd.remainder);
  1277. cctl = pl08x_lli_control_bits(pl08x, cctl,
  1278. bd.srcbus.buswidth, bd.dstbus.buswidth,
  1279. tsize);
  1280. pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
  1281. lli_len, cctl, tsize);
  1282. total_bytes += lli_len;
  1283. }
  1284. /*
  1285. * Send any odd bytes
  1286. */
  1287. if (bd.remainder) {
  1288. dev_vdbg(&pl08x->adev->dev,
  1289. "%s align with boundary, send odd bytes (remain %zu)\n",
  1290. __func__, bd.remainder);
  1291. prep_byte_width_lli(pl08x, &bd, &cctl,
  1292. bd.remainder, num_llis++, &total_bytes);
  1293. }
  1294. }
  1295. if (total_bytes != dsg->len) {
  1296. dev_err(&pl08x->adev->dev,
  1297. "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
  1298. __func__, total_bytes, dsg->len);
  1299. return 0;
  1300. }
  1301. if (num_llis >= MAX_NUM_TSFR_LLIS) {
  1302. dev_err(&pl08x->adev->dev,
  1303. "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
  1304. __func__, MAX_NUM_TSFR_LLIS);
  1305. return 0;
  1306. }
  1307. }
  1308. llis_va = txd->llis_va;
  1309. last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
  1310. if (txd->cyclic) {
  1311. /* Link back to the first LLI. */
  1312. last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
  1313. } else {
  1314. /* The final LLI terminates the LLI. */
  1315. last_lli[PL080_LLI_LLI] = 0;
  1316. /* The final LLI element shall also fire an interrupt. */
  1317. if (pl08x->vd->ftdmac020)
  1318. last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK;
  1319. else
  1320. last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
  1321. }
  1322. pl08x_dump_lli(pl08x, llis_va, num_llis);
  1323. return num_llis;
  1324. }
  1325. static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
  1326. struct pl08x_txd *txd)
  1327. {
  1328. struct pl08x_sg *dsg, *_dsg;
  1329. if (txd->llis_va)
  1330. dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
  1331. list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
  1332. list_del(&dsg->node);
  1333. kfree(dsg);
  1334. }
  1335. kfree(txd);
  1336. }
  1337. static void pl08x_desc_free(struct virt_dma_desc *vd)
  1338. {
  1339. struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
  1340. struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
  1341. dma_descriptor_unmap(&vd->tx);
  1342. if (!txd->done)
  1343. pl08x_release_mux(plchan);
  1344. pl08x_free_txd(plchan->host, txd);
  1345. }
  1346. static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
  1347. struct pl08x_dma_chan *plchan)
  1348. {
  1349. LIST_HEAD(head);
  1350. vchan_get_all_descriptors(&plchan->vc, &head);
  1351. vchan_dma_desc_free_list(&plchan->vc, &head);
  1352. }
  1353. /*
  1354. * The DMA ENGINE API
  1355. */
  1356. static void pl08x_free_chan_resources(struct dma_chan *chan)
  1357. {
  1358. /* Ensure all queued descriptors are freed */
  1359. vchan_free_chan_resources(to_virt_chan(chan));
  1360. }
  1361. static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
  1362. struct dma_chan *chan, unsigned long flags)
  1363. {
  1364. struct dma_async_tx_descriptor *retval = NULL;
  1365. return retval;
  1366. }
  1367. /*
  1368. * Code accessing dma_async_is_complete() in a tight loop may give problems.
  1369. * If slaves are relying on interrupts to signal completion this function
  1370. * must not be called with interrupts disabled.
  1371. */
  1372. static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
  1373. dma_cookie_t cookie, struct dma_tx_state *txstate)
  1374. {
  1375. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1376. struct virt_dma_desc *vd;
  1377. unsigned long flags;
  1378. enum dma_status ret;
  1379. size_t bytes = 0;
  1380. ret = dma_cookie_status(chan, cookie, txstate);
  1381. if (ret == DMA_COMPLETE)
  1382. return ret;
  1383. /*
  1384. * There's no point calculating the residue if there's
  1385. * no txstate to store the value.
  1386. */
  1387. if (!txstate) {
  1388. if (plchan->state == PL08X_CHAN_PAUSED)
  1389. ret = DMA_PAUSED;
  1390. return ret;
  1391. }
  1392. spin_lock_irqsave(&plchan->vc.lock, flags);
  1393. ret = dma_cookie_status(chan, cookie, txstate);
  1394. if (ret != DMA_COMPLETE) {
  1395. vd = vchan_find_desc(&plchan->vc, cookie);
  1396. if (vd) {
  1397. /* On the issued list, so hasn't been processed yet */
  1398. struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
  1399. struct pl08x_sg *dsg;
  1400. list_for_each_entry(dsg, &txd->dsg_list, node)
  1401. bytes += dsg->len;
  1402. } else {
  1403. bytes = pl08x_getbytes_chan(plchan);
  1404. }
  1405. }
  1406. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1407. /*
  1408. * This cookie not complete yet
  1409. * Get number of bytes left in the active transactions and queue
  1410. */
  1411. dma_set_residue(txstate, bytes);
  1412. if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
  1413. ret = DMA_PAUSED;
  1414. /* Whether waiting or running, we're in progress */
  1415. return ret;
  1416. }
  1417. /* PrimeCell DMA extension */
  1418. struct burst_table {
  1419. u32 burstwords;
  1420. u32 reg;
  1421. };
  1422. static const struct burst_table burst_sizes[] = {
  1423. {
  1424. .burstwords = 256,
  1425. .reg = PL080_BSIZE_256,
  1426. },
  1427. {
  1428. .burstwords = 128,
  1429. .reg = PL080_BSIZE_128,
  1430. },
  1431. {
  1432. .burstwords = 64,
  1433. .reg = PL080_BSIZE_64,
  1434. },
  1435. {
  1436. .burstwords = 32,
  1437. .reg = PL080_BSIZE_32,
  1438. },
  1439. {
  1440. .burstwords = 16,
  1441. .reg = PL080_BSIZE_16,
  1442. },
  1443. {
  1444. .burstwords = 8,
  1445. .reg = PL080_BSIZE_8,
  1446. },
  1447. {
  1448. .burstwords = 4,
  1449. .reg = PL080_BSIZE_4,
  1450. },
  1451. {
  1452. .burstwords = 0,
  1453. .reg = PL080_BSIZE_1,
  1454. },
  1455. };
  1456. /*
  1457. * Given the source and destination available bus masks, select which
  1458. * will be routed to each port. We try to have source and destination
  1459. * on separate ports, but always respect the allowable settings.
  1460. */
  1461. static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst)
  1462. {
  1463. u32 cctl = 0;
  1464. u32 dst_ahb2;
  1465. u32 src_ahb2;
  1466. /* The FTDMAC020 use different bits to indicate src/dst bus */
  1467. if (ftdmac020) {
  1468. dst_ahb2 = FTDMAC020_LLI_DST_SEL;
  1469. src_ahb2 = FTDMAC020_LLI_SRC_SEL;
  1470. } else {
  1471. dst_ahb2 = PL080_CONTROL_DST_AHB2;
  1472. src_ahb2 = PL080_CONTROL_SRC_AHB2;
  1473. }
  1474. if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
  1475. cctl |= dst_ahb2;
  1476. if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
  1477. cctl |= src_ahb2;
  1478. return cctl;
  1479. }
  1480. static u32 pl08x_cctl(u32 cctl)
  1481. {
  1482. cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
  1483. PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
  1484. PL080_CONTROL_PROT_MASK);
  1485. /* Access the cell in privileged mode, non-bufferable, non-cacheable */
  1486. return cctl | PL080_CONTROL_PROT_SYS;
  1487. }
  1488. static u32 pl08x_width(enum dma_slave_buswidth width)
  1489. {
  1490. switch (width) {
  1491. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  1492. return PL080_WIDTH_8BIT;
  1493. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  1494. return PL080_WIDTH_16BIT;
  1495. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  1496. return PL080_WIDTH_32BIT;
  1497. default:
  1498. return ~0;
  1499. }
  1500. }
  1501. static u32 pl08x_burst(u32 maxburst)
  1502. {
  1503. int i;
  1504. for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
  1505. if (burst_sizes[i].burstwords <= maxburst)
  1506. break;
  1507. return burst_sizes[i].reg;
  1508. }
  1509. static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
  1510. enum dma_slave_buswidth addr_width, u32 maxburst)
  1511. {
  1512. u32 width, burst, cctl = 0;
  1513. width = pl08x_width(addr_width);
  1514. if (width == ~0)
  1515. return ~0;
  1516. cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
  1517. cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
  1518. /*
  1519. * If this channel will only request single transfers, set this
  1520. * down to ONE element. Also select one element if no maxburst
  1521. * is specified.
  1522. */
  1523. if (plchan->cd->single)
  1524. maxburst = 1;
  1525. burst = pl08x_burst(maxburst);
  1526. cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
  1527. cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
  1528. return pl08x_cctl(cctl);
  1529. }
  1530. /*
  1531. * Slave transactions callback to the slave device to allow
  1532. * synchronization of slave DMA signals with the DMAC enable
  1533. */
  1534. static void pl08x_issue_pending(struct dma_chan *chan)
  1535. {
  1536. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1537. unsigned long flags;
  1538. spin_lock_irqsave(&plchan->vc.lock, flags);
  1539. if (vchan_issue_pending(&plchan->vc)) {
  1540. if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
  1541. pl08x_phy_alloc_and_start(plchan);
  1542. }
  1543. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1544. }
  1545. static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
  1546. {
  1547. struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
  1548. if (txd)
  1549. INIT_LIST_HEAD(&txd->dsg_list);
  1550. return txd;
  1551. }
  1552. static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x)
  1553. {
  1554. u32 cctl = 0;
  1555. /* Conjure cctl */
  1556. switch (pl08x->pd->memcpy_burst_size) {
  1557. default:
  1558. dev_err(&pl08x->adev->dev,
  1559. "illegal burst size for memcpy, set to 1\n");
  1560. fallthrough;
  1561. case PL08X_BURST_SZ_1:
  1562. cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |
  1563. PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;
  1564. break;
  1565. case PL08X_BURST_SZ_4:
  1566. cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |
  1567. PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;
  1568. break;
  1569. case PL08X_BURST_SZ_8:
  1570. cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |
  1571. PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;
  1572. break;
  1573. case PL08X_BURST_SZ_16:
  1574. cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |
  1575. PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;
  1576. break;
  1577. case PL08X_BURST_SZ_32:
  1578. cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |
  1579. PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;
  1580. break;
  1581. case PL08X_BURST_SZ_64:
  1582. cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |
  1583. PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;
  1584. break;
  1585. case PL08X_BURST_SZ_128:
  1586. cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |
  1587. PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;
  1588. break;
  1589. case PL08X_BURST_SZ_256:
  1590. cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |
  1591. PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;
  1592. break;
  1593. }
  1594. switch (pl08x->pd->memcpy_bus_width) {
  1595. default:
  1596. dev_err(&pl08x->adev->dev,
  1597. "illegal bus width for memcpy, set to 8 bits\n");
  1598. fallthrough;
  1599. case PL08X_BUS_WIDTH_8_BITS:
  1600. cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |
  1601. PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
  1602. break;
  1603. case PL08X_BUS_WIDTH_16_BITS:
  1604. cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |
  1605. PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
  1606. break;
  1607. case PL08X_BUS_WIDTH_32_BITS:
  1608. cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |
  1609. PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
  1610. break;
  1611. }
  1612. /* Protection flags */
  1613. if (pl08x->pd->memcpy_prot_buff)
  1614. cctl |= PL080_CONTROL_PROT_BUFF;
  1615. if (pl08x->pd->memcpy_prot_cache)
  1616. cctl |= PL080_CONTROL_PROT_CACHE;
  1617. /* We are the kernel, so we are in privileged mode */
  1618. cctl |= PL080_CONTROL_PROT_SYS;
  1619. /* Both to be incremented or the code will break */
  1620. cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
  1621. if (pl08x->vd->dualmaster)
  1622. cctl |= pl08x_select_bus(false,
  1623. pl08x->mem_buses,
  1624. pl08x->mem_buses);
  1625. return cctl;
  1626. }
  1627. static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x)
  1628. {
  1629. u32 cctl = 0;
  1630. /* Conjure cctl */
  1631. switch (pl08x->pd->memcpy_bus_width) {
  1632. default:
  1633. dev_err(&pl08x->adev->dev,
  1634. "illegal bus width for memcpy, set to 8 bits\n");
  1635. fallthrough;
  1636. case PL08X_BUS_WIDTH_8_BITS:
  1637. cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
  1638. PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
  1639. break;
  1640. case PL08X_BUS_WIDTH_16_BITS:
  1641. cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
  1642. PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
  1643. break;
  1644. case PL08X_BUS_WIDTH_32_BITS:
  1645. cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
  1646. PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
  1647. break;
  1648. }
  1649. /*
  1650. * By default mask the TC IRQ on all LLIs, it will be unmasked on
  1651. * the last LLI item by other code.
  1652. */
  1653. cctl |= FTDMAC020_LLI_TC_MSK;
  1654. /*
  1655. * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL
  1656. * and FTDMAC020_LLI_DSTAD_CTL as zero
  1657. */
  1658. if (pl08x->vd->dualmaster)
  1659. cctl |= pl08x_select_bus(true,
  1660. pl08x->mem_buses,
  1661. pl08x->mem_buses);
  1662. return cctl;
  1663. }
  1664. /*
  1665. * Initialize a descriptor to be used by memcpy submit
  1666. */
  1667. static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
  1668. struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  1669. size_t len, unsigned long flags)
  1670. {
  1671. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1672. struct pl08x_driver_data *pl08x = plchan->host;
  1673. struct pl08x_txd *txd;
  1674. struct pl08x_sg *dsg;
  1675. int ret;
  1676. txd = pl08x_get_txd(plchan);
  1677. if (!txd) {
  1678. dev_err(&pl08x->adev->dev,
  1679. "%s no memory for descriptor\n", __func__);
  1680. return NULL;
  1681. }
  1682. dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
  1683. if (!dsg) {
  1684. pl08x_free_txd(pl08x, txd);
  1685. return NULL;
  1686. }
  1687. list_add_tail(&dsg->node, &txd->dsg_list);
  1688. dsg->src_addr = src;
  1689. dsg->dst_addr = dest;
  1690. dsg->len = len;
  1691. if (pl08x->vd->ftdmac020) {
  1692. /* Writing CCFG zero ENABLES all interrupts */
  1693. txd->ccfg = 0;
  1694. txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x);
  1695. } else {
  1696. txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
  1697. PL080_CONFIG_TC_IRQ_MASK |
  1698. PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
  1699. txd->cctl = pl08x_memcpy_cctl(pl08x);
  1700. }
  1701. ret = pl08x_fill_llis_for_desc(plchan->host, txd);
  1702. if (!ret) {
  1703. pl08x_free_txd(pl08x, txd);
  1704. return NULL;
  1705. }
  1706. return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
  1707. }
  1708. static struct pl08x_txd *pl08x_init_txd(
  1709. struct dma_chan *chan,
  1710. enum dma_transfer_direction direction,
  1711. dma_addr_t *slave_addr)
  1712. {
  1713. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1714. struct pl08x_driver_data *pl08x = plchan->host;
  1715. struct pl08x_txd *txd;
  1716. enum dma_slave_buswidth addr_width;
  1717. int ret, tmp;
  1718. u8 src_buses, dst_buses;
  1719. u32 maxburst, cctl;
  1720. txd = pl08x_get_txd(plchan);
  1721. if (!txd) {
  1722. dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
  1723. return NULL;
  1724. }
  1725. /*
  1726. * Set up addresses, the PrimeCell configured address
  1727. * will take precedence since this may configure the
  1728. * channel target address dynamically at runtime.
  1729. */
  1730. if (direction == DMA_MEM_TO_DEV) {
  1731. cctl = PL080_CONTROL_SRC_INCR;
  1732. *slave_addr = plchan->cfg.dst_addr;
  1733. addr_width = plchan->cfg.dst_addr_width;
  1734. maxburst = plchan->cfg.dst_maxburst;
  1735. src_buses = pl08x->mem_buses;
  1736. dst_buses = plchan->cd->periph_buses;
  1737. } else if (direction == DMA_DEV_TO_MEM) {
  1738. cctl = PL080_CONTROL_DST_INCR;
  1739. *slave_addr = plchan->cfg.src_addr;
  1740. addr_width = plchan->cfg.src_addr_width;
  1741. maxburst = plchan->cfg.src_maxburst;
  1742. src_buses = plchan->cd->periph_buses;
  1743. dst_buses = pl08x->mem_buses;
  1744. } else {
  1745. pl08x_free_txd(pl08x, txd);
  1746. dev_err(&pl08x->adev->dev,
  1747. "%s direction unsupported\n", __func__);
  1748. return NULL;
  1749. }
  1750. cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
  1751. if (cctl == ~0) {
  1752. pl08x_free_txd(pl08x, txd);
  1753. dev_err(&pl08x->adev->dev,
  1754. "DMA slave configuration botched?\n");
  1755. return NULL;
  1756. }
  1757. txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses);
  1758. if (plchan->cfg.device_fc)
  1759. tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
  1760. PL080_FLOW_PER2MEM_PER;
  1761. else
  1762. tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
  1763. PL080_FLOW_PER2MEM;
  1764. txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
  1765. PL080_CONFIG_TC_IRQ_MASK |
  1766. tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
  1767. ret = pl08x_request_mux(plchan);
  1768. if (ret < 0) {
  1769. pl08x_free_txd(pl08x, txd);
  1770. dev_dbg(&pl08x->adev->dev,
  1771. "unable to mux for transfer on %s due to platform restrictions\n",
  1772. plchan->name);
  1773. return NULL;
  1774. }
  1775. dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
  1776. plchan->signal, plchan->name);
  1777. /* Assign the flow control signal to this channel */
  1778. if (direction == DMA_MEM_TO_DEV)
  1779. txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
  1780. else
  1781. txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
  1782. return txd;
  1783. }
  1784. static int pl08x_tx_add_sg(struct pl08x_txd *txd,
  1785. enum dma_transfer_direction direction,
  1786. dma_addr_t slave_addr,
  1787. dma_addr_t buf_addr,
  1788. unsigned int len)
  1789. {
  1790. struct pl08x_sg *dsg;
  1791. dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
  1792. if (!dsg)
  1793. return -ENOMEM;
  1794. list_add_tail(&dsg->node, &txd->dsg_list);
  1795. dsg->len = len;
  1796. if (direction == DMA_MEM_TO_DEV) {
  1797. dsg->src_addr = buf_addr;
  1798. dsg->dst_addr = slave_addr;
  1799. } else {
  1800. dsg->src_addr = slave_addr;
  1801. dsg->dst_addr = buf_addr;
  1802. }
  1803. return 0;
  1804. }
  1805. static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
  1806. struct dma_chan *chan, struct scatterlist *sgl,
  1807. unsigned int sg_len, enum dma_transfer_direction direction,
  1808. unsigned long flags, void *context)
  1809. {
  1810. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1811. struct pl08x_driver_data *pl08x = plchan->host;
  1812. struct pl08x_txd *txd;
  1813. struct scatterlist *sg;
  1814. int ret, tmp;
  1815. dma_addr_t slave_addr;
  1816. dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
  1817. __func__, sg_dma_len(sgl), plchan->name);
  1818. txd = pl08x_init_txd(chan, direction, &slave_addr);
  1819. if (!txd)
  1820. return NULL;
  1821. for_each_sg(sgl, sg, sg_len, tmp) {
  1822. ret = pl08x_tx_add_sg(txd, direction, slave_addr,
  1823. sg_dma_address(sg),
  1824. sg_dma_len(sg));
  1825. if (ret) {
  1826. pl08x_release_mux(plchan);
  1827. pl08x_free_txd(pl08x, txd);
  1828. dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
  1829. __func__);
  1830. return NULL;
  1831. }
  1832. }
  1833. ret = pl08x_fill_llis_for_desc(plchan->host, txd);
  1834. if (!ret) {
  1835. pl08x_release_mux(plchan);
  1836. pl08x_free_txd(pl08x, txd);
  1837. return NULL;
  1838. }
  1839. return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
  1840. }
  1841. static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
  1842. struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
  1843. size_t period_len, enum dma_transfer_direction direction,
  1844. unsigned long flags)
  1845. {
  1846. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1847. struct pl08x_driver_data *pl08x = plchan->host;
  1848. struct pl08x_txd *txd;
  1849. int ret, tmp;
  1850. dma_addr_t slave_addr;
  1851. dev_dbg(&pl08x->adev->dev,
  1852. "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
  1853. __func__, period_len, buf_len,
  1854. direction == DMA_MEM_TO_DEV ? "to" : "from",
  1855. plchan->name);
  1856. txd = pl08x_init_txd(chan, direction, &slave_addr);
  1857. if (!txd)
  1858. return NULL;
  1859. txd->cyclic = true;
  1860. txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
  1861. for (tmp = 0; tmp < buf_len; tmp += period_len) {
  1862. ret = pl08x_tx_add_sg(txd, direction, slave_addr,
  1863. buf_addr + tmp, period_len);
  1864. if (ret) {
  1865. pl08x_release_mux(plchan);
  1866. pl08x_free_txd(pl08x, txd);
  1867. return NULL;
  1868. }
  1869. }
  1870. ret = pl08x_fill_llis_for_desc(plchan->host, txd);
  1871. if (!ret) {
  1872. pl08x_release_mux(plchan);
  1873. pl08x_free_txd(pl08x, txd);
  1874. return NULL;
  1875. }
  1876. return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
  1877. }
  1878. static int pl08x_config(struct dma_chan *chan,
  1879. struct dma_slave_config *config)
  1880. {
  1881. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1882. struct pl08x_driver_data *pl08x = plchan->host;
  1883. if (!plchan->slave)
  1884. return -EINVAL;
  1885. /* Reject definitely invalid configurations */
  1886. if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
  1887. config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
  1888. return -EINVAL;
  1889. if (config->device_fc && pl08x->vd->pl080s) {
  1890. dev_err(&pl08x->adev->dev,
  1891. "%s: PL080S does not support peripheral flow control\n",
  1892. __func__);
  1893. return -EINVAL;
  1894. }
  1895. plchan->cfg = *config;
  1896. return 0;
  1897. }
  1898. static int pl08x_terminate_all(struct dma_chan *chan)
  1899. {
  1900. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1901. struct pl08x_driver_data *pl08x = plchan->host;
  1902. unsigned long flags;
  1903. spin_lock_irqsave(&plchan->vc.lock, flags);
  1904. if (!plchan->phychan && !plchan->at) {
  1905. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1906. return 0;
  1907. }
  1908. plchan->state = PL08X_CHAN_IDLE;
  1909. if (plchan->phychan) {
  1910. /*
  1911. * Mark physical channel as free and free any slave
  1912. * signal
  1913. */
  1914. pl08x_phy_free(plchan);
  1915. }
  1916. /* Dequeue jobs and free LLIs */
  1917. if (plchan->at) {
  1918. vchan_terminate_vdesc(&plchan->at->vd);
  1919. plchan->at = NULL;
  1920. }
  1921. /* Dequeue jobs not yet fired as well */
  1922. pl08x_free_txd_list(pl08x, plchan);
  1923. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1924. return 0;
  1925. }
  1926. static void pl08x_synchronize(struct dma_chan *chan)
  1927. {
  1928. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1929. vchan_synchronize(&plchan->vc);
  1930. }
  1931. static int pl08x_pause(struct dma_chan *chan)
  1932. {
  1933. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1934. unsigned long flags;
  1935. /*
  1936. * Anything succeeds on channels with no physical allocation and
  1937. * no queued transfers.
  1938. */
  1939. spin_lock_irqsave(&plchan->vc.lock, flags);
  1940. if (!plchan->phychan && !plchan->at) {
  1941. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1942. return 0;
  1943. }
  1944. pl08x_pause_phy_chan(plchan->phychan);
  1945. plchan->state = PL08X_CHAN_PAUSED;
  1946. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1947. return 0;
  1948. }
  1949. static int pl08x_resume(struct dma_chan *chan)
  1950. {
  1951. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1952. unsigned long flags;
  1953. /*
  1954. * Anything succeeds on channels with no physical allocation and
  1955. * no queued transfers.
  1956. */
  1957. spin_lock_irqsave(&plchan->vc.lock, flags);
  1958. if (!plchan->phychan && !plchan->at) {
  1959. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1960. return 0;
  1961. }
  1962. pl08x_resume_phy_chan(plchan->phychan);
  1963. plchan->state = PL08X_CHAN_RUNNING;
  1964. spin_unlock_irqrestore(&plchan->vc.lock, flags);
  1965. return 0;
  1966. }
  1967. bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
  1968. {
  1969. struct pl08x_dma_chan *plchan;
  1970. char *name = chan_id;
  1971. /* Reject channels for devices not bound to this driver */
  1972. if (chan->device->dev->driver != &pl08x_amba_driver.drv)
  1973. return false;
  1974. plchan = to_pl08x_chan(chan);
  1975. /* Check that the channel is not taken! */
  1976. if (!strcmp(plchan->name, name))
  1977. return true;
  1978. return false;
  1979. }
  1980. EXPORT_SYMBOL_GPL(pl08x_filter_id);
  1981. static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)
  1982. {
  1983. struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
  1984. return plchan->cd == chan_id;
  1985. }
  1986. /*
  1987. * Just check that the device is there and active
  1988. * TODO: turn this bit on/off depending on the number of physical channels
  1989. * actually used, if it is zero... well shut it off. That will save some
  1990. * power. Cut the clock at the same time.
  1991. */
  1992. static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
  1993. {
  1994. /* The Nomadik variant does not have the config register */
  1995. if (pl08x->vd->nomadik)
  1996. return;
  1997. /* The FTDMAC020 variant does this in another register */
  1998. if (pl08x->vd->ftdmac020) {
  1999. writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR);
  2000. return;
  2001. }
  2002. writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
  2003. }
  2004. static irqreturn_t pl08x_irq(int irq, void *dev)
  2005. {
  2006. struct pl08x_driver_data *pl08x = dev;
  2007. u32 mask = 0, err, tc, i;
  2008. /* check & clear - ERR & TC interrupts */
  2009. err = readl(pl08x->base + PL080_ERR_STATUS);
  2010. if (err) {
  2011. dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
  2012. __func__, err);
  2013. writel(err, pl08x->base + PL080_ERR_CLEAR);
  2014. }
  2015. tc = readl(pl08x->base + PL080_TC_STATUS);
  2016. if (tc)
  2017. writel(tc, pl08x->base + PL080_TC_CLEAR);
  2018. if (!err && !tc)
  2019. return IRQ_NONE;
  2020. for (i = 0; i < pl08x->vd->channels; i++) {
  2021. if ((BIT(i) & err) || (BIT(i) & tc)) {
  2022. /* Locate physical channel */
  2023. struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
  2024. struct pl08x_dma_chan *plchan = phychan->serving;
  2025. struct pl08x_txd *tx;
  2026. if (!plchan) {
  2027. dev_err(&pl08x->adev->dev,
  2028. "%s Error TC interrupt on unused channel: 0x%08x\n",
  2029. __func__, i);
  2030. continue;
  2031. }
  2032. spin_lock(&plchan->vc.lock);
  2033. tx = plchan->at;
  2034. if (tx && tx->cyclic) {
  2035. vchan_cyclic_callback(&tx->vd);
  2036. } else if (tx) {
  2037. plchan->at = NULL;
  2038. /*
  2039. * This descriptor is done, release its mux
  2040. * reservation.
  2041. */
  2042. pl08x_release_mux(plchan);
  2043. tx->done = true;
  2044. vchan_cookie_complete(&tx->vd);
  2045. /*
  2046. * And start the next descriptor (if any),
  2047. * otherwise free this channel.
  2048. */
  2049. if (vchan_next_desc(&plchan->vc))
  2050. pl08x_start_next_txd(plchan);
  2051. else
  2052. pl08x_phy_free(plchan);
  2053. }
  2054. spin_unlock(&plchan->vc.lock);
  2055. mask |= BIT(i);
  2056. }
  2057. }
  2058. return mask ? IRQ_HANDLED : IRQ_NONE;
  2059. }
  2060. static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
  2061. {
  2062. chan->slave = true;
  2063. chan->name = chan->cd->bus_id;
  2064. chan->cfg.src_addr = chan->cd->addr;
  2065. chan->cfg.dst_addr = chan->cd->addr;
  2066. }
  2067. /*
  2068. * Initialise the DMAC memcpy/slave channels.
  2069. * Make a local wrapper to hold required data
  2070. */
  2071. static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
  2072. struct dma_device *dmadev, unsigned int channels, bool slave)
  2073. {
  2074. struct pl08x_dma_chan *chan;
  2075. int i;
  2076. INIT_LIST_HEAD(&dmadev->channels);
  2077. /*
  2078. * Register as many many memcpy as we have physical channels,
  2079. * we won't always be able to use all but the code will have
  2080. * to cope with that situation.
  2081. */
  2082. for (i = 0; i < channels; i++) {
  2083. chan = kzalloc(sizeof(*chan), GFP_KERNEL);
  2084. if (!chan)
  2085. return -ENOMEM;
  2086. chan->host = pl08x;
  2087. chan->state = PL08X_CHAN_IDLE;
  2088. chan->signal = -1;
  2089. if (slave) {
  2090. chan->cd = &pl08x->pd->slave_channels[i];
  2091. /*
  2092. * Some implementations have muxed signals, whereas some
  2093. * use a mux in front of the signals and need dynamic
  2094. * assignment of signals.
  2095. */
  2096. chan->signal = i;
  2097. pl08x_dma_slave_init(chan);
  2098. } else {
  2099. chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL);
  2100. if (!chan->cd) {
  2101. kfree(chan);
  2102. return -ENOMEM;
  2103. }
  2104. chan->cd->bus_id = "memcpy";
  2105. chan->cd->periph_buses = pl08x->pd->mem_buses;
  2106. chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
  2107. if (!chan->name) {
  2108. kfree(chan->cd);
  2109. kfree(chan);
  2110. return -ENOMEM;
  2111. }
  2112. }
  2113. dev_dbg(&pl08x->adev->dev,
  2114. "initialize virtual channel \"%s\"\n",
  2115. chan->name);
  2116. chan->vc.desc_free = pl08x_desc_free;
  2117. vchan_init(&chan->vc, dmadev);
  2118. }
  2119. dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
  2120. i, slave ? "slave" : "memcpy");
  2121. return i;
  2122. }
  2123. static void pl08x_free_virtual_channels(struct dma_device *dmadev)
  2124. {
  2125. struct pl08x_dma_chan *chan = NULL;
  2126. struct pl08x_dma_chan *next;
  2127. list_for_each_entry_safe(chan,
  2128. next, &dmadev->channels, vc.chan.device_node) {
  2129. list_del(&chan->vc.chan.device_node);
  2130. kfree(chan);
  2131. }
  2132. }
  2133. #ifdef CONFIG_DEBUG_FS
  2134. static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
  2135. {
  2136. switch (state) {
  2137. case PL08X_CHAN_IDLE:
  2138. return "idle";
  2139. case PL08X_CHAN_RUNNING:
  2140. return "running";
  2141. case PL08X_CHAN_PAUSED:
  2142. return "paused";
  2143. case PL08X_CHAN_WAITING:
  2144. return "waiting";
  2145. default:
  2146. break;
  2147. }
  2148. return "UNKNOWN STATE";
  2149. }
  2150. static int pl08x_debugfs_show(struct seq_file *s, void *data)
  2151. {
  2152. struct pl08x_driver_data *pl08x = s->private;
  2153. struct pl08x_dma_chan *chan;
  2154. struct pl08x_phy_chan *ch;
  2155. unsigned long flags;
  2156. int i;
  2157. seq_printf(s, "PL08x physical channels:\n");
  2158. seq_printf(s, "CHANNEL:\tUSER:\n");
  2159. seq_printf(s, "--------\t-----\n");
  2160. for (i = 0; i < pl08x->vd->channels; i++) {
  2161. struct pl08x_dma_chan *virt_chan;
  2162. ch = &pl08x->phy_chans[i];
  2163. spin_lock_irqsave(&ch->lock, flags);
  2164. virt_chan = ch->serving;
  2165. seq_printf(s, "%d\t\t%s%s\n",
  2166. ch->id,
  2167. virt_chan ? virt_chan->name : "(none)",
  2168. ch->locked ? " LOCKED" : "");
  2169. spin_unlock_irqrestore(&ch->lock, flags);
  2170. }
  2171. seq_printf(s, "\nPL08x virtual memcpy channels:\n");
  2172. seq_printf(s, "CHANNEL:\tSTATE:\n");
  2173. seq_printf(s, "--------\t------\n");
  2174. list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
  2175. seq_printf(s, "%s\t\t%s\n", chan->name,
  2176. pl08x_state_str(chan->state));
  2177. }
  2178. if (pl08x->has_slave) {
  2179. seq_printf(s, "\nPL08x virtual slave channels:\n");
  2180. seq_printf(s, "CHANNEL:\tSTATE:\n");
  2181. seq_printf(s, "--------\t------\n");
  2182. list_for_each_entry(chan, &pl08x->slave.channels,
  2183. vc.chan.device_node) {
  2184. seq_printf(s, "%s\t\t%s\n", chan->name,
  2185. pl08x_state_str(chan->state));
  2186. }
  2187. }
  2188. return 0;
  2189. }
  2190. DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
  2191. static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
  2192. {
  2193. /* Expose a simple debugfs interface to view all clocks */
  2194. debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
  2195. NULL, pl08x, &pl08x_debugfs_fops);
  2196. }
  2197. #else
  2198. static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
  2199. {
  2200. }
  2201. #endif
  2202. #ifdef CONFIG_OF
  2203. static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,
  2204. u32 id)
  2205. {
  2206. struct pl08x_dma_chan *chan;
  2207. /* Trying to get a slave channel from something with no slave support */
  2208. if (!pl08x->has_slave)
  2209. return NULL;
  2210. list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
  2211. if (chan->signal == id)
  2212. return &chan->vc.chan;
  2213. }
  2214. return NULL;
  2215. }
  2216. static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
  2217. struct of_dma *ofdma)
  2218. {
  2219. struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
  2220. struct dma_chan *dma_chan;
  2221. struct pl08x_dma_chan *plchan;
  2222. if (!pl08x)
  2223. return NULL;
  2224. if (dma_spec->args_count != 2) {
  2225. dev_err(&pl08x->adev->dev,
  2226. "DMA channel translation requires two cells\n");
  2227. return NULL;
  2228. }
  2229. dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);
  2230. if (!dma_chan) {
  2231. dev_err(&pl08x->adev->dev,
  2232. "DMA slave channel not found\n");
  2233. return NULL;
  2234. }
  2235. plchan = to_pl08x_chan(dma_chan);
  2236. dev_dbg(&pl08x->adev->dev,
  2237. "translated channel for signal %d\n",
  2238. dma_spec->args[0]);
  2239. /* Augment channel data for applicable AHB buses */
  2240. plchan->cd->periph_buses = dma_spec->args[1];
  2241. return dma_get_slave_channel(dma_chan);
  2242. }
  2243. static int pl08x_of_probe(struct amba_device *adev,
  2244. struct pl08x_driver_data *pl08x,
  2245. struct device_node *np)
  2246. {
  2247. struct pl08x_platform_data *pd;
  2248. struct pl08x_channel_data *chanp = NULL;
  2249. u32 val;
  2250. int ret;
  2251. int i;
  2252. pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);
  2253. if (!pd)
  2254. return -ENOMEM;
  2255. /* Eligible bus masters for fetching LLIs */
  2256. if (of_property_read_bool(np, "lli-bus-interface-ahb1"))
  2257. pd->lli_buses |= PL08X_AHB1;
  2258. if (of_property_read_bool(np, "lli-bus-interface-ahb2"))
  2259. pd->lli_buses |= PL08X_AHB2;
  2260. if (!pd->lli_buses) {
  2261. dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");
  2262. pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;
  2263. }
  2264. /* Eligible bus masters for memory access */
  2265. if (of_property_read_bool(np, "mem-bus-interface-ahb1"))
  2266. pd->mem_buses |= PL08X_AHB1;
  2267. if (of_property_read_bool(np, "mem-bus-interface-ahb2"))
  2268. pd->mem_buses |= PL08X_AHB2;
  2269. if (!pd->mem_buses) {
  2270. dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");
  2271. pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;
  2272. }
  2273. /* Parse the memcpy channel properties */
  2274. ret = of_property_read_u32(np, "memcpy-burst-size", &val);
  2275. if (ret) {
  2276. dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");
  2277. val = 1;
  2278. }
  2279. switch (val) {
  2280. default:
  2281. dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");
  2282. fallthrough;
  2283. case 1:
  2284. pd->memcpy_burst_size = PL08X_BURST_SZ_1;
  2285. break;
  2286. case 4:
  2287. pd->memcpy_burst_size = PL08X_BURST_SZ_4;
  2288. break;
  2289. case 8:
  2290. pd->memcpy_burst_size = PL08X_BURST_SZ_8;
  2291. break;
  2292. case 16:
  2293. pd->memcpy_burst_size = PL08X_BURST_SZ_16;
  2294. break;
  2295. case 32:
  2296. pd->memcpy_burst_size = PL08X_BURST_SZ_32;
  2297. break;
  2298. case 64:
  2299. pd->memcpy_burst_size = PL08X_BURST_SZ_64;
  2300. break;
  2301. case 128:
  2302. pd->memcpy_burst_size = PL08X_BURST_SZ_128;
  2303. break;
  2304. case 256:
  2305. pd->memcpy_burst_size = PL08X_BURST_SZ_256;
  2306. break;
  2307. }
  2308. ret = of_property_read_u32(np, "memcpy-bus-width", &val);
  2309. if (ret) {
  2310. dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");
  2311. val = 8;
  2312. }
  2313. switch (val) {
  2314. default:
  2315. dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");
  2316. fallthrough;
  2317. case 8:
  2318. pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS;
  2319. break;
  2320. case 16:
  2321. pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS;
  2322. break;
  2323. case 32:
  2324. pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS;
  2325. break;
  2326. }
  2327. /*
  2328. * Allocate channel data for all possible slave channels (one
  2329. * for each possible signal), channels will then be allocated
  2330. * for a device and have it's AHB interfaces set up at
  2331. * translation time.
  2332. */
  2333. if (pl08x->vd->signals) {
  2334. chanp = devm_kcalloc(&adev->dev,
  2335. pl08x->vd->signals,
  2336. sizeof(struct pl08x_channel_data),
  2337. GFP_KERNEL);
  2338. if (!chanp)
  2339. return -ENOMEM;
  2340. pd->slave_channels = chanp;
  2341. for (i = 0; i < pl08x->vd->signals; i++) {
  2342. /*
  2343. * chanp->periph_buses will be assigned at translation
  2344. */
  2345. chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i);
  2346. chanp++;
  2347. }
  2348. pd->num_slave_channels = pl08x->vd->signals;
  2349. }
  2350. pl08x->pd = pd;
  2351. return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,
  2352. pl08x);
  2353. }
  2354. #else
  2355. static inline int pl08x_of_probe(struct amba_device *adev,
  2356. struct pl08x_driver_data *pl08x,
  2357. struct device_node *np)
  2358. {
  2359. return -EINVAL;
  2360. }
  2361. #endif
  2362. static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
  2363. {
  2364. struct pl08x_driver_data *pl08x;
  2365. struct vendor_data *vd = id->data;
  2366. struct device_node *np = adev->dev.of_node;
  2367. u32 tsfr_size;
  2368. int ret = 0;
  2369. int i;
  2370. ret = amba_request_regions(adev, NULL);
  2371. if (ret)
  2372. return ret;
  2373. /* Ensure that we can do DMA */
  2374. ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
  2375. if (ret)
  2376. goto out_no_pl08x;
  2377. /* Create the driver state holder */
  2378. pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
  2379. if (!pl08x) {
  2380. ret = -ENOMEM;
  2381. goto out_no_pl08x;
  2382. }
  2383. /* Assign useful pointers to the driver state */
  2384. pl08x->adev = adev;
  2385. pl08x->vd = vd;
  2386. pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
  2387. if (!pl08x->base) {
  2388. ret = -ENOMEM;
  2389. goto out_no_ioremap;
  2390. }
  2391. if (vd->ftdmac020) {
  2392. u32 val;
  2393. val = readl(pl08x->base + FTDMAC020_REVISION);
  2394. dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n",
  2395. (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);
  2396. val = readl(pl08x->base + FTDMAC020_FEATURE);
  2397. dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, "
  2398. "%s built-in bridge, %s, %s linked lists\n",
  2399. (val >> 12) & 0x0f,
  2400. (val & BIT(10)) ? "no" : "has",
  2401. (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0",
  2402. (val & BIT(8)) ? "supports" : "does not support");
  2403. /* Vendor data from feature register */
  2404. if (!(val & BIT(8)))
  2405. dev_warn(&pl08x->adev->dev,
  2406. "linked lists not supported, required\n");
  2407. vd->channels = (val >> 12) & 0x0f;
  2408. vd->dualmaster = !!(val & BIT(9));
  2409. }
  2410. /* Initialize memcpy engine */
  2411. dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
  2412. pl08x->memcpy.dev = &adev->dev;
  2413. pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
  2414. pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
  2415. pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
  2416. pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
  2417. pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
  2418. pl08x->memcpy.device_config = pl08x_config;
  2419. pl08x->memcpy.device_pause = pl08x_pause;
  2420. pl08x->memcpy.device_resume = pl08x_resume;
  2421. pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
  2422. pl08x->memcpy.device_synchronize = pl08x_synchronize;
  2423. pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;
  2424. pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
  2425. pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);
  2426. pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
  2427. if (vd->ftdmac020)
  2428. pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;
  2429. /*
  2430. * Initialize slave engine, if the block has no signals, that means
  2431. * we have no slave support.
  2432. */
  2433. if (vd->signals) {
  2434. pl08x->has_slave = true;
  2435. dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
  2436. dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
  2437. pl08x->slave.dev = &adev->dev;
  2438. pl08x->slave.device_free_chan_resources =
  2439. pl08x_free_chan_resources;
  2440. pl08x->slave.device_prep_dma_interrupt =
  2441. pl08x_prep_dma_interrupt;
  2442. pl08x->slave.device_tx_status = pl08x_dma_tx_status;
  2443. pl08x->slave.device_issue_pending = pl08x_issue_pending;
  2444. pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
  2445. pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
  2446. pl08x->slave.device_config = pl08x_config;
  2447. pl08x->slave.device_pause = pl08x_pause;
  2448. pl08x->slave.device_resume = pl08x_resume;
  2449. pl08x->slave.device_terminate_all = pl08x_terminate_all;
  2450. pl08x->slave.device_synchronize = pl08x_synchronize;
  2451. pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;
  2452. pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
  2453. pl08x->slave.directions =
  2454. BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  2455. pl08x->slave.residue_granularity =
  2456. DMA_RESIDUE_GRANULARITY_SEGMENT;
  2457. }
  2458. /* Get the platform data */
  2459. pl08x->pd = dev_get_platdata(&adev->dev);
  2460. if (!pl08x->pd) {
  2461. if (np) {
  2462. ret = pl08x_of_probe(adev, pl08x, np);
  2463. if (ret)
  2464. goto out_no_platdata;
  2465. } else {
  2466. dev_err(&adev->dev, "no platform data supplied\n");
  2467. ret = -EINVAL;
  2468. goto out_no_platdata;
  2469. }
  2470. } else {
  2471. pl08x->slave.filter.map = pl08x->pd->slave_map;
  2472. pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;
  2473. pl08x->slave.filter.fn = pl08x_filter_fn;
  2474. }
  2475. /* By default, AHB1 only. If dualmaster, from platform */
  2476. pl08x->lli_buses = PL08X_AHB1;
  2477. pl08x->mem_buses = PL08X_AHB1;
  2478. if (pl08x->vd->dualmaster) {
  2479. pl08x->lli_buses = pl08x->pd->lli_buses;
  2480. pl08x->mem_buses = pl08x->pd->mem_buses;
  2481. }
  2482. if (vd->pl080s)
  2483. pl08x->lli_words = PL080S_LLI_WORDS;
  2484. else
  2485. pl08x->lli_words = PL080_LLI_WORDS;
  2486. tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
  2487. /* A DMA memory pool for LLIs, align on 1-byte boundary */
  2488. pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
  2489. tsfr_size, PL08X_ALIGN, 0);
  2490. if (!pl08x->pool) {
  2491. ret = -ENOMEM;
  2492. goto out_no_lli_pool;
  2493. }
  2494. /* Turn on the PL08x */
  2495. pl08x_ensure_on(pl08x);
  2496. /* Clear any pending interrupts */
  2497. if (vd->ftdmac020)
  2498. /* This variant has error IRQs in bits 16-19 */
  2499. writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR);
  2500. else
  2501. writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
  2502. writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
  2503. /* Attach the interrupt handler */
  2504. ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);
  2505. if (ret) {
  2506. dev_err(&adev->dev, "%s failed to request interrupt %d\n",
  2507. __func__, adev->irq[0]);
  2508. goto out_no_irq;
  2509. }
  2510. /* Initialize physical channels */
  2511. pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
  2512. GFP_KERNEL);
  2513. if (!pl08x->phy_chans) {
  2514. ret = -ENOMEM;
  2515. goto out_no_phychans;
  2516. }
  2517. for (i = 0; i < vd->channels; i++) {
  2518. struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
  2519. ch->id = i;
  2520. ch->base = pl08x->base + PL080_Cx_BASE(i);
  2521. if (vd->ftdmac020) {
  2522. /* FTDMA020 has a special channel busy register */
  2523. ch->reg_busy = ch->base + FTDMAC020_CH_BUSY;
  2524. ch->reg_config = ch->base + FTDMAC020_CH_CFG;
  2525. ch->reg_control = ch->base + FTDMAC020_CH_CSR;
  2526. ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR;
  2527. ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR;
  2528. ch->reg_lli = ch->base + FTDMAC020_CH_LLP;
  2529. ch->ftdmac020 = true;
  2530. } else {
  2531. ch->reg_config = ch->base + vd->config_offset;
  2532. ch->reg_control = ch->base + PL080_CH_CONTROL;
  2533. ch->reg_src = ch->base + PL080_CH_SRC_ADDR;
  2534. ch->reg_dst = ch->base + PL080_CH_DST_ADDR;
  2535. ch->reg_lli = ch->base + PL080_CH_LLI;
  2536. }
  2537. if (vd->pl080s)
  2538. ch->pl080s = true;
  2539. spin_lock_init(&ch->lock);
  2540. /*
  2541. * Nomadik variants can have channels that are locked
  2542. * down for the secure world only. Lock up these channels
  2543. * by perpetually serving a dummy virtual channel.
  2544. */
  2545. if (vd->nomadik) {
  2546. u32 val;
  2547. val = readl(ch->reg_config);
  2548. if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
  2549. dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
  2550. ch->locked = true;
  2551. }
  2552. }
  2553. dev_dbg(&adev->dev, "physical channel %d is %s\n",
  2554. i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
  2555. }
  2556. /* Register as many memcpy channels as there are physical channels */
  2557. ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
  2558. pl08x->vd->channels, false);
  2559. if (ret <= 0) {
  2560. dev_warn(&pl08x->adev->dev,
  2561. "%s failed to enumerate memcpy channels - %d\n",
  2562. __func__, ret);
  2563. goto out_no_memcpy;
  2564. }
  2565. /* Register slave channels */
  2566. if (pl08x->has_slave) {
  2567. ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
  2568. pl08x->pd->num_slave_channels, true);
  2569. if (ret < 0) {
  2570. dev_warn(&pl08x->adev->dev,
  2571. "%s failed to enumerate slave channels - %d\n",
  2572. __func__, ret);
  2573. goto out_no_slave;
  2574. }
  2575. }
  2576. ret = dma_async_device_register(&pl08x->memcpy);
  2577. if (ret) {
  2578. dev_warn(&pl08x->adev->dev,
  2579. "%s failed to register memcpy as an async device - %d\n",
  2580. __func__, ret);
  2581. goto out_no_memcpy_reg;
  2582. }
  2583. if (pl08x->has_slave) {
  2584. ret = dma_async_device_register(&pl08x->slave);
  2585. if (ret) {
  2586. dev_warn(&pl08x->adev->dev,
  2587. "%s failed to register slave as an async device - %d\n",
  2588. __func__, ret);
  2589. goto out_no_slave_reg;
  2590. }
  2591. }
  2592. amba_set_drvdata(adev, pl08x);
  2593. init_pl08x_debugfs(pl08x);
  2594. dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",
  2595. amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),
  2596. (unsigned long long)adev->res.start, adev->irq[0]);
  2597. return 0;
  2598. out_no_slave_reg:
  2599. dma_async_device_unregister(&pl08x->memcpy);
  2600. out_no_memcpy_reg:
  2601. if (pl08x->has_slave)
  2602. pl08x_free_virtual_channels(&pl08x->slave);
  2603. out_no_slave:
  2604. pl08x_free_virtual_channels(&pl08x->memcpy);
  2605. out_no_memcpy:
  2606. kfree(pl08x->phy_chans);
  2607. out_no_phychans:
  2608. free_irq(adev->irq[0], pl08x);
  2609. out_no_irq:
  2610. dma_pool_destroy(pl08x->pool);
  2611. out_no_lli_pool:
  2612. out_no_platdata:
  2613. iounmap(pl08x->base);
  2614. out_no_ioremap:
  2615. kfree(pl08x);
  2616. out_no_pl08x:
  2617. amba_release_regions(adev);
  2618. return ret;
  2619. }
  2620. /* PL080 has 8 channels and the PL080 have just 2 */
  2621. static struct vendor_data vendor_pl080 = {
  2622. .config_offset = PL080_CH_CONFIG,
  2623. .channels = 8,
  2624. .signals = 16,
  2625. .dualmaster = true,
  2626. .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
  2627. };
  2628. static struct vendor_data vendor_nomadik = {
  2629. .config_offset = PL080_CH_CONFIG,
  2630. .channels = 8,
  2631. .signals = 32,
  2632. .dualmaster = true,
  2633. .nomadik = true,
  2634. .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
  2635. };
  2636. static struct vendor_data vendor_pl080s = {
  2637. .config_offset = PL080S_CH_CONFIG,
  2638. .channels = 8,
  2639. .signals = 32,
  2640. .pl080s = true,
  2641. .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
  2642. };
  2643. static struct vendor_data vendor_pl081 = {
  2644. .config_offset = PL080_CH_CONFIG,
  2645. .channels = 2,
  2646. .signals = 16,
  2647. .dualmaster = false,
  2648. .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
  2649. };
  2650. static struct vendor_data vendor_ftdmac020 = {
  2651. .config_offset = PL080_CH_CONFIG,
  2652. .ftdmac020 = true,
  2653. .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
  2654. };
  2655. static const struct amba_id pl08x_ids[] = {
  2656. /* Samsung PL080S variant */
  2657. {
  2658. .id = 0x0a141080,
  2659. .mask = 0xffffffff,
  2660. .data = &vendor_pl080s,
  2661. },
  2662. /* PL080 */
  2663. {
  2664. .id = 0x00041080,
  2665. .mask = 0x000fffff,
  2666. .data = &vendor_pl080,
  2667. },
  2668. /* PL081 */
  2669. {
  2670. .id = 0x00041081,
  2671. .mask = 0x000fffff,
  2672. .data = &vendor_pl081,
  2673. },
  2674. /* Nomadik 8815 PL080 variant */
  2675. {
  2676. .id = 0x00280080,
  2677. .mask = 0x00ffffff,
  2678. .data = &vendor_nomadik,
  2679. },
  2680. /* Faraday Technology FTDMAC020 */
  2681. {
  2682. .id = 0x0003b080,
  2683. .mask = 0x000fffff,
  2684. .data = &vendor_ftdmac020,
  2685. },
  2686. { 0, 0 },
  2687. };
  2688. MODULE_DEVICE_TABLE(amba, pl08x_ids);
  2689. static struct amba_driver pl08x_amba_driver = {
  2690. .drv.name = DRIVER_NAME,
  2691. .id_table = pl08x_ids,
  2692. .probe = pl08x_probe,
  2693. };
  2694. static int __init pl08x_init(void)
  2695. {
  2696. int retval;
  2697. retval = amba_driver_register(&pl08x_amba_driver);
  2698. if (retval)
  2699. printk(KERN_WARNING DRIVER_NAME
  2700. "failed to register as an AMBA device (%d)\n",
  2701. retval);
  2702. return retval;
  2703. }
  2704. subsys_initcall(pl08x_init);