tsi721.c 81 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
  4. *
  5. * Copyright 2011 Integrated Device Technology, Inc.
  6. * Alexandre Bounine <alexandre.bounine@idt.com>
  7. * Chul Kim <chul.kim@idt.com>
  8. */
  9. #include <linux/io.h>
  10. #include <linux/errno.h>
  11. #include <linux/init.h>
  12. #include <linux/ioport.h>
  13. #include <linux/kernel.h>
  14. #include <linux/module.h>
  15. #include <linux/pci.h>
  16. #include <linux/rio.h>
  17. #include <linux/rio_drv.h>
  18. #include <linux/dma-mapping.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/kfifo.h>
  21. #include <linux/delay.h>
  22. #include "tsi721.h"
  23. #ifdef DEBUG
  24. u32 tsi_dbg_level;
  25. module_param_named(dbg_level, tsi_dbg_level, uint, S_IWUSR | S_IRUGO);
  26. MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
  27. #endif
  28. static int pcie_mrrs = -1;
  29. module_param(pcie_mrrs, int, S_IRUGO);
  30. MODULE_PARM_DESC(pcie_mrrs, "PCIe MRRS override value (0...5)");
  31. static u8 mbox_sel = 0x0f;
  32. module_param(mbox_sel, byte, S_IRUGO);
  33. MODULE_PARM_DESC(mbox_sel,
  34. "RIO Messaging MBOX Selection Mask (default: 0x0f = all)");
  35. static DEFINE_SPINLOCK(tsi721_maint_lock);
  36. static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
  37. static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
  38. /**
  39. * tsi721_lcread - read from local SREP config space
  40. * @mport: RapidIO master port info
  41. * @index: ID of RapdiIO interface
  42. * @offset: Offset into configuration space
  43. * @len: Length (in bytes) of the maintenance transaction
  44. * @data: Value to be read into
  45. *
  46. * Generates a local SREP space read. Returns %0 on
  47. * success or %-EINVAL on failure.
  48. */
  49. static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
  50. int len, u32 *data)
  51. {
  52. struct tsi721_device *priv = mport->priv;
  53. if (len != sizeof(u32))
  54. return -EINVAL; /* only 32-bit access is supported */
  55. *data = ioread32(priv->regs + offset);
  56. return 0;
  57. }
  58. /**
  59. * tsi721_lcwrite - write into local SREP config space
  60. * @mport: RapidIO master port info
  61. * @index: ID of RapdiIO interface
  62. * @offset: Offset into configuration space
  63. * @len: Length (in bytes) of the maintenance transaction
  64. * @data: Value to be written
  65. *
  66. * Generates a local write into SREP configuration space. Returns %0 on
  67. * success or %-EINVAL on failure.
  68. */
  69. static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
  70. int len, u32 data)
  71. {
  72. struct tsi721_device *priv = mport->priv;
  73. if (len != sizeof(u32))
  74. return -EINVAL; /* only 32-bit access is supported */
  75. iowrite32(data, priv->regs + offset);
  76. return 0;
  77. }
  78. /**
  79. * tsi721_maint_dma - Helper function to generate RapidIO maintenance
  80. * transactions using designated Tsi721 DMA channel.
  81. * @priv: pointer to tsi721 private data
  82. * @sys_size: RapdiIO transport system size
  83. * @destid: Destination ID of transaction
  84. * @hopcount: Number of hops to target device
  85. * @offset: Offset into configuration space
  86. * @len: Length (in bytes) of the maintenance transaction
  87. * @data: Location to be read from or write into
  88. * @do_wr: Operation flag (1 == MAINT_WR)
  89. *
  90. * Generates a RapidIO maintenance transaction (Read or Write).
  91. * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
  92. */
  93. static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
  94. u16 destid, u8 hopcount, u32 offset, int len,
  95. u32 *data, int do_wr)
  96. {
  97. void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
  98. struct tsi721_dma_desc *bd_ptr;
  99. u32 rd_count, swr_ptr, ch_stat;
  100. unsigned long flags;
  101. int i, err = 0;
  102. u32 op = do_wr ? MAINT_WR : MAINT_RD;
  103. if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
  104. return -EINVAL;
  105. spin_lock_irqsave(&tsi721_maint_lock, flags);
  106. bd_ptr = priv->mdma.bd_base;
  107. rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
  108. /* Initialize DMA descriptor */
  109. bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
  110. bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
  111. bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
  112. bd_ptr[0].raddr_hi = 0;
  113. if (do_wr)
  114. bd_ptr[0].data[0] = cpu_to_be32p(data);
  115. else
  116. bd_ptr[0].data[0] = 0xffffffff;
  117. mb();
  118. /* Start DMA operation */
  119. iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
  120. ioread32(regs + TSI721_DMAC_DWRCNT);
  121. i = 0;
  122. /* Wait until DMA transfer is finished */
  123. while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
  124. & TSI721_DMAC_STS_RUN) {
  125. udelay(1);
  126. if (++i >= 5000000) {
  127. tsi_debug(MAINT, &priv->pdev->dev,
  128. "DMA[%d] read timeout ch_status=%x",
  129. priv->mdma.ch_id, ch_stat);
  130. if (!do_wr)
  131. *data = 0xffffffff;
  132. err = -EIO;
  133. goto err_out;
  134. }
  135. }
  136. if (ch_stat & TSI721_DMAC_STS_ABORT) {
  137. /* If DMA operation aborted due to error,
  138. * reinitialize DMA channel
  139. */
  140. tsi_debug(MAINT, &priv->pdev->dev, "DMA ABORT ch_stat=%x",
  141. ch_stat);
  142. tsi_debug(MAINT, &priv->pdev->dev,
  143. "OP=%d : destid=%x hc=%x off=%x",
  144. do_wr ? MAINT_WR : MAINT_RD,
  145. destid, hopcount, offset);
  146. iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
  147. iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
  148. udelay(10);
  149. iowrite32(0, regs + TSI721_DMAC_DWRCNT);
  150. udelay(1);
  151. if (!do_wr)
  152. *data = 0xffffffff;
  153. err = -EIO;
  154. goto err_out;
  155. }
  156. if (!do_wr)
  157. *data = be32_to_cpu(bd_ptr[0].data[0]);
  158. /*
  159. * Update descriptor status FIFO RD pointer.
  160. * NOTE: Skipping check and clear FIFO entries because we are waiting
  161. * for transfer to be completed.
  162. */
  163. swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
  164. iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
  165. err_out:
  166. spin_unlock_irqrestore(&tsi721_maint_lock, flags);
  167. return err;
  168. }
  169. /**
  170. * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
  171. * using Tsi721 BDMA engine.
  172. * @mport: RapidIO master port control structure
  173. * @index: ID of RapdiIO interface
  174. * @destid: Destination ID of transaction
  175. * @hopcount: Number of hops to target device
  176. * @offset: Offset into configuration space
  177. * @len: Length (in bytes) of the maintenance transaction
  178. * @val: Location to be read into
  179. *
  180. * Generates a RapidIO maintenance read transaction.
  181. * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
  182. */
  183. static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
  184. u8 hopcount, u32 offset, int len, u32 *data)
  185. {
  186. struct tsi721_device *priv = mport->priv;
  187. return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
  188. offset, len, data, 0);
  189. }
  190. /**
  191. * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
  192. * using Tsi721 BDMA engine
  193. * @mport: RapidIO master port control structure
  194. * @index: ID of RapdiIO interface
  195. * @destid: Destination ID of transaction
  196. * @hopcount: Number of hops to target device
  197. * @offset: Offset into configuration space
  198. * @len: Length (in bytes) of the maintenance transaction
  199. * @val: Value to be written
  200. *
  201. * Generates a RapidIO maintenance write transaction.
  202. * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
  203. */
  204. static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
  205. u8 hopcount, u32 offset, int len, u32 data)
  206. {
  207. struct tsi721_device *priv = mport->priv;
  208. u32 temp = data;
  209. return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
  210. offset, len, &temp, 1);
  211. }
  212. /**
  213. * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
  214. * @priv: tsi721 device private structure
  215. *
  216. * Handles inbound port-write interrupts. Copies PW message from an internal
  217. * buffer into PW message FIFO and schedules deferred routine to process
  218. * queued messages.
  219. */
  220. static int
  221. tsi721_pw_handler(struct tsi721_device *priv)
  222. {
  223. u32 pw_stat;
  224. u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
  225. pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
  226. if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
  227. pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
  228. pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
  229. pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
  230. pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
  231. /* Queue PW message (if there is room in FIFO),
  232. * otherwise discard it.
  233. */
  234. spin_lock(&priv->pw_fifo_lock);
  235. if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
  236. kfifo_in(&priv->pw_fifo, pw_buf,
  237. TSI721_RIO_PW_MSG_SIZE);
  238. else
  239. priv->pw_discard_count++;
  240. spin_unlock(&priv->pw_fifo_lock);
  241. }
  242. /* Clear pending PW interrupts */
  243. iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
  244. priv->regs + TSI721_RIO_PW_RX_STAT);
  245. schedule_work(&priv->pw_work);
  246. return 0;
  247. }
  248. static void tsi721_pw_dpc(struct work_struct *work)
  249. {
  250. struct tsi721_device *priv = container_of(work, struct tsi721_device,
  251. pw_work);
  252. union rio_pw_msg pwmsg;
  253. /*
  254. * Process port-write messages
  255. */
  256. while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)&pwmsg,
  257. TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
  258. /* Pass the port-write message to RIO core for processing */
  259. rio_inb_pwrite_handler(&priv->mport, &pwmsg);
  260. }
  261. }
  262. /**
  263. * tsi721_pw_enable - enable/disable port-write interface init
  264. * @mport: Master port implementing the port write unit
  265. * @enable: 1=enable; 0=disable port-write message handling
  266. */
  267. static int tsi721_pw_enable(struct rio_mport *mport, int enable)
  268. {
  269. struct tsi721_device *priv = mport->priv;
  270. u32 rval;
  271. rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
  272. if (enable)
  273. rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
  274. else
  275. rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
  276. /* Clear pending PW interrupts */
  277. iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
  278. priv->regs + TSI721_RIO_PW_RX_STAT);
  279. /* Update enable bits */
  280. iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
  281. return 0;
  282. }
  283. /**
  284. * tsi721_dsend - Send a RapidIO doorbell
  285. * @mport: RapidIO master port info
  286. * @index: ID of RapidIO interface
  287. * @destid: Destination ID of target device
  288. * @data: 16-bit info field of RapidIO doorbell
  289. *
  290. * Sends a RapidIO doorbell message. Always returns %0.
  291. */
  292. static int tsi721_dsend(struct rio_mport *mport, int index,
  293. u16 destid, u16 data)
  294. {
  295. struct tsi721_device *priv = mport->priv;
  296. u32 offset;
  297. offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
  298. (destid << 2);
  299. tsi_debug(DBELL, &priv->pdev->dev,
  300. "Send Doorbell 0x%04x to destID 0x%x", data, destid);
  301. iowrite16be(data, priv->odb_base + offset);
  302. return 0;
  303. }
  304. /**
  305. * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
  306. * @priv: tsi721 device-specific data structure
  307. *
  308. * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
  309. * buffer into DB message FIFO and schedules deferred routine to process
  310. * queued DBs.
  311. */
  312. static int
  313. tsi721_dbell_handler(struct tsi721_device *priv)
  314. {
  315. u32 regval;
  316. /* Disable IDB interrupts */
  317. regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
  318. regval &= ~TSI721_SR_CHINT_IDBQRCV;
  319. iowrite32(regval,
  320. priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
  321. schedule_work(&priv->idb_work);
  322. return 0;
  323. }
  324. static void tsi721_db_dpc(struct work_struct *work)
  325. {
  326. struct tsi721_device *priv = container_of(work, struct tsi721_device,
  327. idb_work);
  328. struct rio_mport *mport;
  329. struct rio_dbell *dbell;
  330. int found = 0;
  331. u32 wr_ptr, rd_ptr;
  332. u64 *idb_entry;
  333. u32 regval;
  334. union {
  335. u64 msg;
  336. u8 bytes[8];
  337. } idb;
  338. /*
  339. * Process queued inbound doorbells
  340. */
  341. mport = &priv->mport;
  342. wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
  343. rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
  344. while (wr_ptr != rd_ptr) {
  345. idb_entry = (u64 *)(priv->idb_base +
  346. (TSI721_IDB_ENTRY_SIZE * rd_ptr));
  347. rd_ptr++;
  348. rd_ptr %= IDB_QSIZE;
  349. idb.msg = *idb_entry;
  350. *idb_entry = 0;
  351. /* Process one doorbell */
  352. list_for_each_entry(dbell, &mport->dbells, node) {
  353. if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
  354. (dbell->res->end >= DBELL_INF(idb.bytes))) {
  355. found = 1;
  356. break;
  357. }
  358. }
  359. if (found) {
  360. dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
  361. DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
  362. } else {
  363. tsi_debug(DBELL, &priv->pdev->dev,
  364. "spurious IDB sid %2.2x tid %2.2x info %4.4x",
  365. DBELL_SID(idb.bytes), DBELL_TID(idb.bytes),
  366. DBELL_INF(idb.bytes));
  367. }
  368. wr_ptr = ioread32(priv->regs +
  369. TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
  370. }
  371. iowrite32(rd_ptr & (IDB_QSIZE - 1),
  372. priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
  373. /* Re-enable IDB interrupts */
  374. regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
  375. regval |= TSI721_SR_CHINT_IDBQRCV;
  376. iowrite32(regval,
  377. priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
  378. wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
  379. if (wr_ptr != rd_ptr)
  380. schedule_work(&priv->idb_work);
  381. }
  382. /**
  383. * tsi721_irqhandler - Tsi721 interrupt handler
  384. * @irq: Linux interrupt number
  385. * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
  386. *
  387. * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
  388. * interrupt events and calls an event-specific handler(s).
  389. */
  390. static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
  391. {
  392. struct tsi721_device *priv = (struct tsi721_device *)ptr;
  393. u32 dev_int;
  394. u32 dev_ch_int;
  395. u32 intval;
  396. u32 ch_inte;
  397. /* For MSI mode disable all device-level interrupts */
  398. if (priv->flags & TSI721_USING_MSI)
  399. iowrite32(0, priv->regs + TSI721_DEV_INTE);
  400. dev_int = ioread32(priv->regs + TSI721_DEV_INT);
  401. if (!dev_int)
  402. return IRQ_NONE;
  403. dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
  404. if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
  405. /* Service SR2PC Channel interrupts */
  406. if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
  407. /* Service Inbound Doorbell interrupt */
  408. intval = ioread32(priv->regs +
  409. TSI721_SR_CHINT(IDB_QUEUE));
  410. if (intval & TSI721_SR_CHINT_IDBQRCV)
  411. tsi721_dbell_handler(priv);
  412. else
  413. tsi_info(&priv->pdev->dev,
  414. "Unsupported SR_CH_INT %x", intval);
  415. /* Clear interrupts */
  416. iowrite32(intval,
  417. priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
  418. ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
  419. }
  420. }
  421. if (dev_int & TSI721_DEV_INT_SMSG_CH) {
  422. int ch;
  423. /*
  424. * Service channel interrupts from Messaging Engine
  425. */
  426. if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
  427. /* Disable signaled OB MSG Channel interrupts */
  428. ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  429. ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
  430. iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
  431. /*
  432. * Process Inbound Message interrupt for each MBOX
  433. */
  434. for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
  435. if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
  436. continue;
  437. tsi721_imsg_handler(priv, ch);
  438. }
  439. }
  440. if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
  441. /* Disable signaled OB MSG Channel interrupts */
  442. ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  443. ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
  444. iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
  445. /*
  446. * Process Outbound Message interrupts for each MBOX
  447. */
  448. for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
  449. if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
  450. continue;
  451. tsi721_omsg_handler(priv, ch);
  452. }
  453. }
  454. }
  455. if (dev_int & TSI721_DEV_INT_SRIO) {
  456. /* Service SRIO MAC interrupts */
  457. intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
  458. if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
  459. tsi721_pw_handler(priv);
  460. }
  461. #ifdef CONFIG_RAPIDIO_DMA_ENGINE
  462. if (dev_int & TSI721_DEV_INT_BDMA_CH) {
  463. int ch;
  464. if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
  465. tsi_debug(DMA, &priv->pdev->dev,
  466. "IRQ from DMA channel 0x%08x", dev_ch_int);
  467. for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
  468. if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
  469. continue;
  470. tsi721_bdma_handler(&priv->bdma[ch]);
  471. }
  472. }
  473. }
  474. #endif
  475. /* For MSI mode re-enable device-level interrupts */
  476. if (priv->flags & TSI721_USING_MSI) {
  477. dev_int = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
  478. TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
  479. iowrite32(dev_int, priv->regs + TSI721_DEV_INTE);
  480. }
  481. return IRQ_HANDLED;
  482. }
  483. static void tsi721_interrupts_init(struct tsi721_device *priv)
  484. {
  485. u32 intr;
  486. /* Enable IDB interrupts */
  487. iowrite32(TSI721_SR_CHINT_ALL,
  488. priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
  489. iowrite32(TSI721_SR_CHINT_IDBQRCV,
  490. priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
  491. /* Enable SRIO MAC interrupts */
  492. iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
  493. priv->regs + TSI721_RIO_EM_DEV_INT_EN);
  494. /* Enable interrupts from channels in use */
  495. #ifdef CONFIG_RAPIDIO_DMA_ENGINE
  496. intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
  497. (TSI721_INT_BDMA_CHAN_M &
  498. ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
  499. #else
  500. intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
  501. #endif
  502. iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
  503. if (priv->flags & TSI721_USING_MSIX)
  504. intr = TSI721_DEV_INT_SRIO;
  505. else
  506. intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
  507. TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
  508. iowrite32(intr, priv->regs + TSI721_DEV_INTE);
  509. ioread32(priv->regs + TSI721_DEV_INTE);
  510. }
  511. #ifdef CONFIG_PCI_MSI
  512. /**
  513. * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
  514. * @irq: Linux interrupt number
  515. * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
  516. *
  517. * Handles outbound messaging interrupts signaled using MSI-X.
  518. */
  519. static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
  520. {
  521. struct tsi721_device *priv = (struct tsi721_device *)ptr;
  522. int mbox;
  523. mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
  524. tsi721_omsg_handler(priv, mbox);
  525. return IRQ_HANDLED;
  526. }
  527. /**
  528. * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
  529. * @irq: Linux interrupt number
  530. * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
  531. *
  532. * Handles inbound messaging interrupts signaled using MSI-X.
  533. */
  534. static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
  535. {
  536. struct tsi721_device *priv = (struct tsi721_device *)ptr;
  537. int mbox;
  538. mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
  539. tsi721_imsg_handler(priv, mbox + 4);
  540. return IRQ_HANDLED;
  541. }
  542. /**
  543. * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
  544. * @irq: Linux interrupt number
  545. * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
  546. *
  547. * Handles Tsi721 interrupts from SRIO MAC.
  548. */
  549. static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
  550. {
  551. struct tsi721_device *priv = (struct tsi721_device *)ptr;
  552. u32 srio_int;
  553. /* Service SRIO MAC interrupts */
  554. srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
  555. if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
  556. tsi721_pw_handler(priv);
  557. return IRQ_HANDLED;
  558. }
  559. /**
  560. * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
  561. * @irq: Linux interrupt number
  562. * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
  563. *
  564. * Handles Tsi721 interrupts from SR2PC Channel.
  565. * NOTE: At this moment services only one SR2PC channel associated with inbound
  566. * doorbells.
  567. */
  568. static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
  569. {
  570. struct tsi721_device *priv = (struct tsi721_device *)ptr;
  571. u32 sr_ch_int;
  572. /* Service Inbound DB interrupt from SR2PC channel */
  573. sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
  574. if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
  575. tsi721_dbell_handler(priv);
  576. /* Clear interrupts */
  577. iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
  578. /* Read back to ensure that interrupt was cleared */
  579. sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
  580. return IRQ_HANDLED;
  581. }
  582. /**
  583. * tsi721_request_msix - register interrupt service for MSI-X mode.
  584. * @priv: tsi721 device-specific data structure
  585. *
  586. * Registers MSI-X interrupt service routines for interrupts that are active
  587. * immediately after mport initialization. Messaging interrupt service routines
  588. * should be registered during corresponding open requests.
  589. */
  590. static int tsi721_request_msix(struct tsi721_device *priv)
  591. {
  592. int err = 0;
  593. err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
  594. tsi721_sr2pc_ch_msix, 0,
  595. priv->msix[TSI721_VECT_IDB].irq_name, (void *)priv);
  596. if (err)
  597. return err;
  598. err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
  599. tsi721_srio_msix, 0,
  600. priv->msix[TSI721_VECT_PWRX].irq_name, (void *)priv);
  601. if (err) {
  602. free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
  603. return err;
  604. }
  605. return 0;
  606. }
  607. /**
  608. * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
  609. * @priv: pointer to tsi721 private data
  610. *
  611. * Configures MSI-X support for Tsi721. Supports only an exact number
  612. * of requested vectors.
  613. */
  614. static int tsi721_enable_msix(struct tsi721_device *priv)
  615. {
  616. struct msix_entry entries[TSI721_VECT_MAX];
  617. int err;
  618. int i;
  619. entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
  620. entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
  621. /*
  622. * Initialize MSI-X entries for Messaging Engine:
  623. * this driver supports four RIO mailboxes (inbound and outbound)
  624. * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
  625. * offset +4 is added to IB MBOX number.
  626. */
  627. for (i = 0; i < RIO_MAX_MBOX; i++) {
  628. entries[TSI721_VECT_IMB0_RCV + i].entry =
  629. TSI721_MSIX_IMSG_DQ_RCV(i + 4);
  630. entries[TSI721_VECT_IMB0_INT + i].entry =
  631. TSI721_MSIX_IMSG_INT(i + 4);
  632. entries[TSI721_VECT_OMB0_DONE + i].entry =
  633. TSI721_MSIX_OMSG_DONE(i);
  634. entries[TSI721_VECT_OMB0_INT + i].entry =
  635. TSI721_MSIX_OMSG_INT(i);
  636. }
  637. #ifdef CONFIG_RAPIDIO_DMA_ENGINE
  638. /*
  639. * Initialize MSI-X entries for Block DMA Engine:
  640. * this driver supports XXX DMA channels
  641. * (one is reserved for SRIO maintenance transactions)
  642. */
  643. for (i = 0; i < TSI721_DMA_CHNUM; i++) {
  644. entries[TSI721_VECT_DMA0_DONE + i].entry =
  645. TSI721_MSIX_DMACH_DONE(i);
  646. entries[TSI721_VECT_DMA0_INT + i].entry =
  647. TSI721_MSIX_DMACH_INT(i);
  648. }
  649. #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
  650. err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries));
  651. if (err) {
  652. tsi_err(&priv->pdev->dev,
  653. "Failed to enable MSI-X (err=%d)", err);
  654. return err;
  655. }
  656. /*
  657. * Copy MSI-X vector information into tsi721 private structure
  658. */
  659. priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
  660. snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
  661. DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
  662. priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
  663. snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
  664. DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
  665. for (i = 0; i < RIO_MAX_MBOX; i++) {
  666. priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
  667. entries[TSI721_VECT_IMB0_RCV + i].vector;
  668. snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
  669. IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
  670. i, pci_name(priv->pdev));
  671. priv->msix[TSI721_VECT_IMB0_INT + i].vector =
  672. entries[TSI721_VECT_IMB0_INT + i].vector;
  673. snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
  674. IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
  675. i, pci_name(priv->pdev));
  676. priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
  677. entries[TSI721_VECT_OMB0_DONE + i].vector;
  678. snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
  679. IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
  680. i, pci_name(priv->pdev));
  681. priv->msix[TSI721_VECT_OMB0_INT + i].vector =
  682. entries[TSI721_VECT_OMB0_INT + i].vector;
  683. snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
  684. IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
  685. i, pci_name(priv->pdev));
  686. }
  687. #ifdef CONFIG_RAPIDIO_DMA_ENGINE
  688. for (i = 0; i < TSI721_DMA_CHNUM; i++) {
  689. priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
  690. entries[TSI721_VECT_DMA0_DONE + i].vector;
  691. snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
  692. IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
  693. i, pci_name(priv->pdev));
  694. priv->msix[TSI721_VECT_DMA0_INT + i].vector =
  695. entries[TSI721_VECT_DMA0_INT + i].vector;
  696. snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
  697. IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
  698. i, pci_name(priv->pdev));
  699. }
  700. #endif /* CONFIG_RAPIDIO_DMA_ENGINE */
  701. return 0;
  702. }
  703. #endif /* CONFIG_PCI_MSI */
  704. static int tsi721_request_irq(struct tsi721_device *priv)
  705. {
  706. int err;
  707. #ifdef CONFIG_PCI_MSI
  708. if (priv->flags & TSI721_USING_MSIX)
  709. err = tsi721_request_msix(priv);
  710. else
  711. #endif
  712. err = request_irq(priv->pdev->irq, tsi721_irqhandler,
  713. (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
  714. DRV_NAME, (void *)priv);
  715. if (err)
  716. tsi_err(&priv->pdev->dev,
  717. "Unable to allocate interrupt, err=%d", err);
  718. return err;
  719. }
  720. static void tsi721_free_irq(struct tsi721_device *priv)
  721. {
  722. #ifdef CONFIG_PCI_MSI
  723. if (priv->flags & TSI721_USING_MSIX) {
  724. free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
  725. free_irq(priv->msix[TSI721_VECT_PWRX].vector, (void *)priv);
  726. } else
  727. #endif
  728. free_irq(priv->pdev->irq, (void *)priv);
  729. }
  730. static int
  731. tsi721_obw_alloc(struct tsi721_device *priv, struct tsi721_obw_bar *pbar,
  732. u32 size, int *win_id)
  733. {
  734. u64 win_base;
  735. u64 bar_base;
  736. u64 bar_end;
  737. u32 align;
  738. struct tsi721_ob_win *win;
  739. struct tsi721_ob_win *new_win = NULL;
  740. int new_win_idx = -1;
  741. int i = 0;
  742. bar_base = pbar->base;
  743. bar_end = bar_base + pbar->size;
  744. win_base = bar_base;
  745. align = size/TSI721_PC2SR_ZONES;
  746. while (i < TSI721_IBWIN_NUM) {
  747. for (i = 0; i < TSI721_IBWIN_NUM; i++) {
  748. if (!priv->ob_win[i].active) {
  749. if (new_win == NULL) {
  750. new_win = &priv->ob_win[i];
  751. new_win_idx = i;
  752. }
  753. continue;
  754. }
  755. /*
  756. * If this window belongs to the current BAR check it
  757. * for overlap
  758. */
  759. win = &priv->ob_win[i];
  760. if (win->base >= bar_base && win->base < bar_end) {
  761. if (win_base < (win->base + win->size) &&
  762. (win_base + size) > win->base) {
  763. /* Overlap detected */
  764. win_base = win->base + win->size;
  765. win_base = ALIGN(win_base, align);
  766. break;
  767. }
  768. }
  769. }
  770. }
  771. if (win_base + size > bar_end)
  772. return -ENOMEM;
  773. if (!new_win) {
  774. tsi_err(&priv->pdev->dev, "OBW count tracking failed");
  775. return -EIO;
  776. }
  777. new_win->active = true;
  778. new_win->base = win_base;
  779. new_win->size = size;
  780. new_win->pbar = pbar;
  781. priv->obwin_cnt--;
  782. pbar->free -= size;
  783. *win_id = new_win_idx;
  784. return 0;
  785. }
  786. static int tsi721_map_outb_win(struct rio_mport *mport, u16 destid, u64 rstart,
  787. u32 size, u32 flags, dma_addr_t *laddr)
  788. {
  789. struct tsi721_device *priv = mport->priv;
  790. int i;
  791. struct tsi721_obw_bar *pbar;
  792. struct tsi721_ob_win *ob_win;
  793. int obw = -1;
  794. u32 rval;
  795. u64 rio_addr;
  796. u32 zsize;
  797. int ret = -ENOMEM;
  798. tsi_debug(OBW, &priv->pdev->dev,
  799. "did=%d ra=0x%llx sz=0x%x", destid, rstart, size);
  800. if (!is_power_of_2(size) || (size < 0x8000) || (rstart & (size - 1)))
  801. return -EINVAL;
  802. if (priv->obwin_cnt == 0)
  803. return -EBUSY;
  804. for (i = 0; i < 2; i++) {
  805. if (priv->p2r_bar[i].free >= size) {
  806. pbar = &priv->p2r_bar[i];
  807. ret = tsi721_obw_alloc(priv, pbar, size, &obw);
  808. if (!ret)
  809. break;
  810. }
  811. }
  812. if (ret)
  813. return ret;
  814. WARN_ON(obw == -1);
  815. ob_win = &priv->ob_win[obw];
  816. ob_win->destid = destid;
  817. ob_win->rstart = rstart;
  818. tsi_debug(OBW, &priv->pdev->dev,
  819. "allocated OBW%d @%llx", obw, ob_win->base);
  820. /*
  821. * Configure Outbound Window
  822. */
  823. zsize = size/TSI721_PC2SR_ZONES;
  824. rio_addr = rstart;
  825. /*
  826. * Program Address Translation Zones:
  827. * This implementation uses all 8 zones associated wit window.
  828. */
  829. for (i = 0; i < TSI721_PC2SR_ZONES; i++) {
  830. while (ioread32(priv->regs + TSI721_ZONE_SEL) &
  831. TSI721_ZONE_SEL_GO) {
  832. udelay(1);
  833. }
  834. rval = (u32)(rio_addr & TSI721_LUT_DATA0_ADD) |
  835. TSI721_LUT_DATA0_NREAD | TSI721_LUT_DATA0_NWR;
  836. iowrite32(rval, priv->regs + TSI721_LUT_DATA0);
  837. rval = (u32)(rio_addr >> 32);
  838. iowrite32(rval, priv->regs + TSI721_LUT_DATA1);
  839. rval = destid;
  840. iowrite32(rval, priv->regs + TSI721_LUT_DATA2);
  841. rval = TSI721_ZONE_SEL_GO | (obw << 3) | i;
  842. iowrite32(rval, priv->regs + TSI721_ZONE_SEL);
  843. rio_addr += zsize;
  844. }
  845. iowrite32(TSI721_OBWIN_SIZE(size) << 8,
  846. priv->regs + TSI721_OBWINSZ(obw));
  847. iowrite32((u32)(ob_win->base >> 32), priv->regs + TSI721_OBWINUB(obw));
  848. iowrite32((u32)(ob_win->base & TSI721_OBWINLB_BA) | TSI721_OBWINLB_WEN,
  849. priv->regs + TSI721_OBWINLB(obw));
  850. *laddr = ob_win->base;
  851. return 0;
  852. }
  853. static void tsi721_unmap_outb_win(struct rio_mport *mport,
  854. u16 destid, u64 rstart)
  855. {
  856. struct tsi721_device *priv = mport->priv;
  857. struct tsi721_ob_win *ob_win;
  858. int i;
  859. tsi_debug(OBW, &priv->pdev->dev, "did=%d ra=0x%llx", destid, rstart);
  860. for (i = 0; i < TSI721_OBWIN_NUM; i++) {
  861. ob_win = &priv->ob_win[i];
  862. if (ob_win->active &&
  863. ob_win->destid == destid && ob_win->rstart == rstart) {
  864. tsi_debug(OBW, &priv->pdev->dev,
  865. "free OBW%d @%llx", i, ob_win->base);
  866. ob_win->active = false;
  867. iowrite32(0, priv->regs + TSI721_OBWINLB(i));
  868. ob_win->pbar->free += ob_win->size;
  869. priv->obwin_cnt++;
  870. break;
  871. }
  872. }
  873. }
  874. /**
  875. * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
  876. * translation regions.
  877. * @priv: pointer to tsi721 private data
  878. *
  879. * Disables SREP translation regions.
  880. */
  881. static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
  882. {
  883. int i, z;
  884. u32 rval;
  885. /* Disable all PC2SR translation windows */
  886. for (i = 0; i < TSI721_OBWIN_NUM; i++)
  887. iowrite32(0, priv->regs + TSI721_OBWINLB(i));
  888. /* Initialize zone lookup tables to avoid ECC errors on reads */
  889. iowrite32(0, priv->regs + TSI721_LUT_DATA0);
  890. iowrite32(0, priv->regs + TSI721_LUT_DATA1);
  891. iowrite32(0, priv->regs + TSI721_LUT_DATA2);
  892. for (i = 0; i < TSI721_OBWIN_NUM; i++) {
  893. for (z = 0; z < TSI721_PC2SR_ZONES; z++) {
  894. while (ioread32(priv->regs + TSI721_ZONE_SEL) &
  895. TSI721_ZONE_SEL_GO) {
  896. udelay(1);
  897. }
  898. rval = TSI721_ZONE_SEL_GO | (i << 3) | z;
  899. iowrite32(rval, priv->regs + TSI721_ZONE_SEL);
  900. }
  901. }
  902. if (priv->p2r_bar[0].size == 0 && priv->p2r_bar[1].size == 0) {
  903. priv->obwin_cnt = 0;
  904. return;
  905. }
  906. priv->p2r_bar[0].free = priv->p2r_bar[0].size;
  907. priv->p2r_bar[1].free = priv->p2r_bar[1].size;
  908. for (i = 0; i < TSI721_OBWIN_NUM; i++)
  909. priv->ob_win[i].active = false;
  910. priv->obwin_cnt = TSI721_OBWIN_NUM;
  911. }
  912. /**
  913. * tsi721_rio_map_inb_mem -- Mapping inbound memory region.
  914. * @mport: RapidIO master port
  915. * @lstart: Local memory space start address.
  916. * @rstart: RapidIO space start address.
  917. * @size: The mapping region size.
  918. * @flags: Flags for mapping. 0 for using default flags.
  919. *
  920. * Return: 0 -- Success.
  921. *
  922. * This function will create the inbound mapping
  923. * from rstart to lstart.
  924. */
  925. static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
  926. u64 rstart, u64 size, u32 flags)
  927. {
  928. struct tsi721_device *priv = mport->priv;
  929. int i, avail = -1;
  930. u32 regval;
  931. struct tsi721_ib_win *ib_win;
  932. bool direct = (lstart == rstart);
  933. u64 ibw_size;
  934. dma_addr_t loc_start;
  935. u64 ibw_start;
  936. struct tsi721_ib_win_mapping *map = NULL;
  937. int ret = -EBUSY;
  938. /* Max IBW size supported by HW is 16GB */
  939. if (size > 0x400000000UL)
  940. return -EINVAL;
  941. if (direct) {
  942. /* Calculate minimal acceptable window size and base address */
  943. ibw_size = roundup_pow_of_two(size);
  944. ibw_start = lstart & ~(ibw_size - 1);
  945. tsi_debug(IBW, &priv->pdev->dev,
  946. "Direct (RIO_0x%llx -> PCIe_%pad), size=0x%llx, ibw_start = 0x%llx",
  947. rstart, &lstart, size, ibw_start);
  948. while ((lstart + size) > (ibw_start + ibw_size)) {
  949. ibw_size *= 2;
  950. ibw_start = lstart & ~(ibw_size - 1);
  951. /* Check for crossing IBW max size 16GB */
  952. if (ibw_size > 0x400000000UL)
  953. return -EBUSY;
  954. }
  955. loc_start = ibw_start;
  956. map = kzalloc(sizeof(struct tsi721_ib_win_mapping), GFP_ATOMIC);
  957. if (map == NULL)
  958. return -ENOMEM;
  959. } else {
  960. tsi_debug(IBW, &priv->pdev->dev,
  961. "Translated (RIO_0x%llx -> PCIe_%pad), size=0x%llx",
  962. rstart, &lstart, size);
  963. if (!is_power_of_2(size) || size < 0x1000 ||
  964. ((u64)lstart & (size - 1)) || (rstart & (size - 1)))
  965. return -EINVAL;
  966. if (priv->ibwin_cnt == 0)
  967. return -EBUSY;
  968. ibw_start = rstart;
  969. ibw_size = size;
  970. loc_start = lstart;
  971. }
  972. /*
  973. * Scan for overlapping with active regions and mark the first available
  974. * IB window at the same time.
  975. */
  976. for (i = 0; i < TSI721_IBWIN_NUM; i++) {
  977. ib_win = &priv->ib_win[i];
  978. if (!ib_win->active) {
  979. if (avail == -1) {
  980. avail = i;
  981. ret = 0;
  982. }
  983. } else if (ibw_start < (ib_win->rstart + ib_win->size) &&
  984. (ibw_start + ibw_size) > ib_win->rstart) {
  985. /* Return error if address translation involved */
  986. if (!direct || ib_win->xlat) {
  987. ret = -EFAULT;
  988. break;
  989. }
  990. /*
  991. * Direct mappings usually are larger than originally
  992. * requested fragments - check if this new request fits
  993. * into it.
  994. */
  995. if (rstart >= ib_win->rstart &&
  996. (rstart + size) <= (ib_win->rstart +
  997. ib_win->size)) {
  998. /* We are in - no further mapping required */
  999. map->lstart = lstart;
  1000. list_add_tail(&map->node, &ib_win->mappings);
  1001. return 0;
  1002. }
  1003. ret = -EFAULT;
  1004. break;
  1005. }
  1006. }
  1007. if (ret)
  1008. goto out;
  1009. i = avail;
  1010. /* Sanity check: available IB window must be disabled at this point */
  1011. regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
  1012. if (WARN_ON(regval & TSI721_IBWIN_LB_WEN)) {
  1013. ret = -EIO;
  1014. goto out;
  1015. }
  1016. ib_win = &priv->ib_win[i];
  1017. ib_win->active = true;
  1018. ib_win->rstart = ibw_start;
  1019. ib_win->lstart = loc_start;
  1020. ib_win->size = ibw_size;
  1021. ib_win->xlat = (lstart != rstart);
  1022. INIT_LIST_HEAD(&ib_win->mappings);
  1023. /*
  1024. * When using direct IBW mapping and have larger than requested IBW size
  1025. * we can have multiple local memory blocks mapped through the same IBW
  1026. * To handle this situation we maintain list of "clients" for such IBWs.
  1027. */
  1028. if (direct) {
  1029. map->lstart = lstart;
  1030. list_add_tail(&map->node, &ib_win->mappings);
  1031. }
  1032. iowrite32(TSI721_IBWIN_SIZE(ibw_size) << 8,
  1033. priv->regs + TSI721_IBWIN_SZ(i));
  1034. iowrite32(((u64)loc_start >> 32), priv->regs + TSI721_IBWIN_TUA(i));
  1035. iowrite32(((u64)loc_start & TSI721_IBWIN_TLA_ADD),
  1036. priv->regs + TSI721_IBWIN_TLA(i));
  1037. iowrite32(ibw_start >> 32, priv->regs + TSI721_IBWIN_UB(i));
  1038. iowrite32((ibw_start & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN,
  1039. priv->regs + TSI721_IBWIN_LB(i));
  1040. priv->ibwin_cnt--;
  1041. tsi_debug(IBW, &priv->pdev->dev,
  1042. "Configured IBWIN%d (RIO_0x%llx -> PCIe_%pad), size=0x%llx",
  1043. i, ibw_start, &loc_start, ibw_size);
  1044. return 0;
  1045. out:
  1046. kfree(map);
  1047. return ret;
  1048. }
  1049. /**
  1050. * tsi721_rio_unmap_inb_mem -- Unmapping inbound memory region.
  1051. * @mport: RapidIO master port
  1052. * @lstart: Local memory space start address.
  1053. */
  1054. static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport,
  1055. dma_addr_t lstart)
  1056. {
  1057. struct tsi721_device *priv = mport->priv;
  1058. struct tsi721_ib_win *ib_win;
  1059. int i;
  1060. tsi_debug(IBW, &priv->pdev->dev,
  1061. "Unmap IBW mapped to PCIe_%pad", &lstart);
  1062. /* Search for matching active inbound translation window */
  1063. for (i = 0; i < TSI721_IBWIN_NUM; i++) {
  1064. ib_win = &priv->ib_win[i];
  1065. /* Address translating IBWs must to be an exact march */
  1066. if (!ib_win->active ||
  1067. (ib_win->xlat && lstart != ib_win->lstart))
  1068. continue;
  1069. if (lstart >= ib_win->lstart &&
  1070. lstart < (ib_win->lstart + ib_win->size)) {
  1071. if (!ib_win->xlat) {
  1072. struct tsi721_ib_win_mapping *map;
  1073. int found = 0;
  1074. list_for_each_entry(map,
  1075. &ib_win->mappings, node) {
  1076. if (map->lstart == lstart) {
  1077. list_del(&map->node);
  1078. kfree(map);
  1079. found = 1;
  1080. break;
  1081. }
  1082. }
  1083. if (!found)
  1084. continue;
  1085. if (!list_empty(&ib_win->mappings))
  1086. break;
  1087. }
  1088. tsi_debug(IBW, &priv->pdev->dev, "Disable IBWIN_%d", i);
  1089. iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
  1090. ib_win->active = false;
  1091. priv->ibwin_cnt++;
  1092. break;
  1093. }
  1094. }
  1095. if (i == TSI721_IBWIN_NUM)
  1096. tsi_debug(IBW, &priv->pdev->dev,
  1097. "IB window mapped to %pad not found", &lstart);
  1098. }
  1099. /**
  1100. * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
  1101. * translation regions.
  1102. * @priv: pointer to tsi721 private data
  1103. *
  1104. * Disables inbound windows.
  1105. */
  1106. static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
  1107. {
  1108. int i;
  1109. /* Disable all SR2PC inbound windows */
  1110. for (i = 0; i < TSI721_IBWIN_NUM; i++)
  1111. iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
  1112. priv->ibwin_cnt = TSI721_IBWIN_NUM;
  1113. }
  1114. /*
  1115. * tsi721_close_sr2pc_mapping - closes all active inbound (SRIO->PCIe)
  1116. * translation regions.
  1117. * @priv: pointer to tsi721 device private data
  1118. */
  1119. static void tsi721_close_sr2pc_mapping(struct tsi721_device *priv)
  1120. {
  1121. struct tsi721_ib_win *ib_win;
  1122. int i;
  1123. /* Disable all active SR2PC inbound windows */
  1124. for (i = 0; i < TSI721_IBWIN_NUM; i++) {
  1125. ib_win = &priv->ib_win[i];
  1126. if (ib_win->active) {
  1127. iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
  1128. ib_win->active = false;
  1129. }
  1130. }
  1131. }
  1132. /**
  1133. * tsi721_port_write_init - Inbound port write interface init
  1134. * @priv: pointer to tsi721 private data
  1135. *
  1136. * Initializes inbound port write handler.
  1137. * Returns %0 on success or %-ENOMEM on failure.
  1138. */
  1139. static int tsi721_port_write_init(struct tsi721_device *priv)
  1140. {
  1141. priv->pw_discard_count = 0;
  1142. INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
  1143. spin_lock_init(&priv->pw_fifo_lock);
  1144. if (kfifo_alloc(&priv->pw_fifo,
  1145. TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
  1146. tsi_err(&priv->pdev->dev, "PW FIFO allocation failed");
  1147. return -ENOMEM;
  1148. }
  1149. /* Use reliable port-write capture mode */
  1150. iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
  1151. return 0;
  1152. }
  1153. static void tsi721_port_write_free(struct tsi721_device *priv)
  1154. {
  1155. kfifo_free(&priv->pw_fifo);
  1156. }
  1157. static int tsi721_doorbell_init(struct tsi721_device *priv)
  1158. {
  1159. /* Outbound Doorbells do not require any setup.
  1160. * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
  1161. * That BAR1 was mapped during the probe routine.
  1162. */
  1163. /* Initialize Inbound Doorbell processing DPC and queue */
  1164. priv->db_discard_count = 0;
  1165. INIT_WORK(&priv->idb_work, tsi721_db_dpc);
  1166. /* Allocate buffer for inbound doorbells queue */
  1167. priv->idb_base = dma_alloc_coherent(&priv->pdev->dev,
  1168. IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
  1169. &priv->idb_dma, GFP_KERNEL);
  1170. if (!priv->idb_base)
  1171. return -ENOMEM;
  1172. tsi_debug(DBELL, &priv->pdev->dev,
  1173. "Allocated IDB buffer @ %p (phys = %pad)",
  1174. priv->idb_base, &priv->idb_dma);
  1175. iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
  1176. priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
  1177. iowrite32(((u64)priv->idb_dma >> 32),
  1178. priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
  1179. iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
  1180. priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
  1181. /* Enable accepting all inbound doorbells */
  1182. iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
  1183. iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
  1184. iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
  1185. return 0;
  1186. }
  1187. static void tsi721_doorbell_free(struct tsi721_device *priv)
  1188. {
  1189. if (priv->idb_base == NULL)
  1190. return;
  1191. /* Free buffer allocated for inbound doorbell queue */
  1192. dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
  1193. priv->idb_base, priv->idb_dma);
  1194. priv->idb_base = NULL;
  1195. }
  1196. /**
  1197. * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
  1198. * @priv: pointer to tsi721 private data
  1199. *
  1200. * Initialize BDMA channel allocated for RapidIO maintenance read/write
  1201. * request generation
  1202. * Returns %0 on success or %-ENOMEM on failure.
  1203. */
  1204. static int tsi721_bdma_maint_init(struct tsi721_device *priv)
  1205. {
  1206. struct tsi721_dma_desc *bd_ptr;
  1207. u64 *sts_ptr;
  1208. dma_addr_t bd_phys, sts_phys;
  1209. int sts_size;
  1210. int bd_num = 2;
  1211. void __iomem *regs;
  1212. tsi_debug(MAINT, &priv->pdev->dev,
  1213. "Init BDMA_%d Maintenance requests", TSI721_DMACH_MAINT);
  1214. /*
  1215. * Initialize DMA channel for maintenance requests
  1216. */
  1217. priv->mdma.ch_id = TSI721_DMACH_MAINT;
  1218. regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
  1219. /* Allocate space for DMA descriptors */
  1220. bd_ptr = dma_alloc_coherent(&priv->pdev->dev,
  1221. bd_num * sizeof(struct tsi721_dma_desc),
  1222. &bd_phys, GFP_KERNEL);
  1223. if (!bd_ptr)
  1224. return -ENOMEM;
  1225. priv->mdma.bd_num = bd_num;
  1226. priv->mdma.bd_phys = bd_phys;
  1227. priv->mdma.bd_base = bd_ptr;
  1228. tsi_debug(MAINT, &priv->pdev->dev, "DMA descriptors @ %p (phys = %pad)",
  1229. bd_ptr, &bd_phys);
  1230. /* Allocate space for descriptor status FIFO */
  1231. sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
  1232. bd_num : TSI721_DMA_MINSTSSZ;
  1233. sts_size = roundup_pow_of_two(sts_size);
  1234. sts_ptr = dma_alloc_coherent(&priv->pdev->dev,
  1235. sts_size * sizeof(struct tsi721_dma_sts),
  1236. &sts_phys, GFP_KERNEL);
  1237. if (!sts_ptr) {
  1238. /* Free space allocated for DMA descriptors */
  1239. dma_free_coherent(&priv->pdev->dev,
  1240. bd_num * sizeof(struct tsi721_dma_desc),
  1241. bd_ptr, bd_phys);
  1242. priv->mdma.bd_base = NULL;
  1243. return -ENOMEM;
  1244. }
  1245. priv->mdma.sts_phys = sts_phys;
  1246. priv->mdma.sts_base = sts_ptr;
  1247. priv->mdma.sts_size = sts_size;
  1248. tsi_debug(MAINT, &priv->pdev->dev,
  1249. "desc status FIFO @ %p (phys = %pad) size=0x%x",
  1250. sts_ptr, &sts_phys, sts_size);
  1251. /* Initialize DMA descriptors ring */
  1252. bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
  1253. bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
  1254. TSI721_DMAC_DPTRL_MASK);
  1255. bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
  1256. /* Setup DMA descriptor pointers */
  1257. iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
  1258. iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
  1259. regs + TSI721_DMAC_DPTRL);
  1260. /* Setup descriptor status FIFO */
  1261. iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
  1262. iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
  1263. regs + TSI721_DMAC_DSBL);
  1264. iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
  1265. regs + TSI721_DMAC_DSSZ);
  1266. /* Clear interrupt bits */
  1267. iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
  1268. ioread32(regs + TSI721_DMAC_INT);
  1269. /* Toggle DMA channel initialization */
  1270. iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
  1271. ioread32(regs + TSI721_DMAC_CTL);
  1272. udelay(10);
  1273. return 0;
  1274. }
  1275. static int tsi721_bdma_maint_free(struct tsi721_device *priv)
  1276. {
  1277. u32 ch_stat;
  1278. struct tsi721_bdma_maint *mdma = &priv->mdma;
  1279. void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
  1280. if (mdma->bd_base == NULL)
  1281. return 0;
  1282. /* Check if DMA channel still running */
  1283. ch_stat = ioread32(regs + TSI721_DMAC_STS);
  1284. if (ch_stat & TSI721_DMAC_STS_RUN)
  1285. return -EFAULT;
  1286. /* Put DMA channel into init state */
  1287. iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
  1288. /* Free space allocated for DMA descriptors */
  1289. dma_free_coherent(&priv->pdev->dev,
  1290. mdma->bd_num * sizeof(struct tsi721_dma_desc),
  1291. mdma->bd_base, mdma->bd_phys);
  1292. mdma->bd_base = NULL;
  1293. /* Free space allocated for status FIFO */
  1294. dma_free_coherent(&priv->pdev->dev,
  1295. mdma->sts_size * sizeof(struct tsi721_dma_sts),
  1296. mdma->sts_base, mdma->sts_phys);
  1297. mdma->sts_base = NULL;
  1298. return 0;
  1299. }
  1300. /* Enable Inbound Messaging Interrupts */
  1301. static void
  1302. tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
  1303. u32 inte_mask)
  1304. {
  1305. u32 rval;
  1306. if (!inte_mask)
  1307. return;
  1308. /* Clear pending Inbound Messaging interrupts */
  1309. iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
  1310. /* Enable Inbound Messaging interrupts */
  1311. rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
  1312. iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
  1313. if (priv->flags & TSI721_USING_MSIX)
  1314. return; /* Finished if we are in MSI-X mode */
  1315. /*
  1316. * For MSI and INTA interrupt signalling we need to enable next levels
  1317. */
  1318. /* Enable Device Channel Interrupt */
  1319. rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  1320. iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
  1321. priv->regs + TSI721_DEV_CHAN_INTE);
  1322. }
  1323. /* Disable Inbound Messaging Interrupts */
  1324. static void
  1325. tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
  1326. u32 inte_mask)
  1327. {
  1328. u32 rval;
  1329. if (!inte_mask)
  1330. return;
  1331. /* Clear pending Inbound Messaging interrupts */
  1332. iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
  1333. /* Disable Inbound Messaging interrupts */
  1334. rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
  1335. rval &= ~inte_mask;
  1336. iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
  1337. if (priv->flags & TSI721_USING_MSIX)
  1338. return; /* Finished if we are in MSI-X mode */
  1339. /*
  1340. * For MSI and INTA interrupt signalling we need to disable next levels
  1341. */
  1342. /* Disable Device Channel Interrupt */
  1343. rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  1344. rval &= ~TSI721_INT_IMSG_CHAN(ch);
  1345. iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
  1346. }
  1347. /* Enable Outbound Messaging interrupts */
  1348. static void
  1349. tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
  1350. u32 inte_mask)
  1351. {
  1352. u32 rval;
  1353. if (!inte_mask)
  1354. return;
  1355. /* Clear pending Outbound Messaging interrupts */
  1356. iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
  1357. /* Enable Outbound Messaging channel interrupts */
  1358. rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
  1359. iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
  1360. if (priv->flags & TSI721_USING_MSIX)
  1361. return; /* Finished if we are in MSI-X mode */
  1362. /*
  1363. * For MSI and INTA interrupt signalling we need to enable next levels
  1364. */
  1365. /* Enable Device Channel Interrupt */
  1366. rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  1367. iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
  1368. priv->regs + TSI721_DEV_CHAN_INTE);
  1369. }
  1370. /* Disable Outbound Messaging interrupts */
  1371. static void
  1372. tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
  1373. u32 inte_mask)
  1374. {
  1375. u32 rval;
  1376. if (!inte_mask)
  1377. return;
  1378. /* Clear pending Outbound Messaging interrupts */
  1379. iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
  1380. /* Disable Outbound Messaging interrupts */
  1381. rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
  1382. rval &= ~inte_mask;
  1383. iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
  1384. if (priv->flags & TSI721_USING_MSIX)
  1385. return; /* Finished if we are in MSI-X mode */
  1386. /*
  1387. * For MSI and INTA interrupt signalling we need to disable next levels
  1388. */
  1389. /* Disable Device Channel Interrupt */
  1390. rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  1391. rval &= ~TSI721_INT_OMSG_CHAN(ch);
  1392. iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
  1393. }
  1394. /**
  1395. * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
  1396. * @mport: Master port with outbound message queue
  1397. * @rdev: Target of outbound message
  1398. * @mbox: Outbound mailbox
  1399. * @buffer: Message to add to outbound queue
  1400. * @len: Length of message
  1401. */
  1402. static int
  1403. tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
  1404. void *buffer, size_t len)
  1405. {
  1406. struct tsi721_device *priv = mport->priv;
  1407. struct tsi721_omsg_desc *desc;
  1408. u32 tx_slot;
  1409. unsigned long flags;
  1410. if (!priv->omsg_init[mbox] ||
  1411. len > TSI721_MSG_MAX_SIZE || len < 8)
  1412. return -EINVAL;
  1413. spin_lock_irqsave(&priv->omsg_ring[mbox].lock, flags);
  1414. tx_slot = priv->omsg_ring[mbox].tx_slot;
  1415. /* Copy copy message into transfer buffer */
  1416. memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
  1417. if (len & 0x7)
  1418. len += 8;
  1419. /* Build descriptor associated with buffer */
  1420. desc = priv->omsg_ring[mbox].omd_base;
  1421. desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
  1422. #ifdef TSI721_OMSG_DESC_INT
  1423. /* Request IOF_DONE interrupt generation for each N-th frame in queue */
  1424. if (tx_slot % 4 == 0)
  1425. desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
  1426. #endif
  1427. desc[tx_slot].msg_info =
  1428. cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
  1429. (0xe << 12) | (len & 0xff8));
  1430. desc[tx_slot].bufptr_lo =
  1431. cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
  1432. 0xffffffff);
  1433. desc[tx_slot].bufptr_hi =
  1434. cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
  1435. priv->omsg_ring[mbox].wr_count++;
  1436. /* Go to next descriptor */
  1437. if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
  1438. priv->omsg_ring[mbox].tx_slot = 0;
  1439. /* Move through the ring link descriptor at the end */
  1440. priv->omsg_ring[mbox].wr_count++;
  1441. }
  1442. mb();
  1443. /* Set new write count value */
  1444. iowrite32(priv->omsg_ring[mbox].wr_count,
  1445. priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
  1446. ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
  1447. spin_unlock_irqrestore(&priv->omsg_ring[mbox].lock, flags);
  1448. return 0;
  1449. }
  1450. /**
  1451. * tsi721_omsg_handler - Outbound Message Interrupt Handler
  1452. * @priv: pointer to tsi721 private data
  1453. * @ch: number of OB MSG channel to service
  1454. *
  1455. * Services channel interrupts from outbound messaging engine.
  1456. */
  1457. static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
  1458. {
  1459. u32 omsg_int;
  1460. struct rio_mport *mport = &priv->mport;
  1461. void *dev_id = NULL;
  1462. u32 tx_slot = 0xffffffff;
  1463. int do_callback = 0;
  1464. spin_lock(&priv->omsg_ring[ch].lock);
  1465. omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
  1466. if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
  1467. tsi_info(&priv->pdev->dev,
  1468. "OB MBOX%d: Status FIFO is full", ch);
  1469. if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
  1470. u32 srd_ptr;
  1471. u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
  1472. int i, j;
  1473. /*
  1474. * Find last successfully processed descriptor
  1475. */
  1476. /* Check and clear descriptor status FIFO entries */
  1477. srd_ptr = priv->omsg_ring[ch].sts_rdptr;
  1478. sts_ptr = priv->omsg_ring[ch].sts_base;
  1479. j = srd_ptr * 8;
  1480. while (sts_ptr[j]) {
  1481. for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
  1482. prev_ptr = last_ptr;
  1483. last_ptr = le64_to_cpu(sts_ptr[j]);
  1484. sts_ptr[j] = 0;
  1485. }
  1486. ++srd_ptr;
  1487. srd_ptr %= priv->omsg_ring[ch].sts_size;
  1488. j = srd_ptr * 8;
  1489. }
  1490. if (last_ptr == 0)
  1491. goto no_sts_update;
  1492. priv->omsg_ring[ch].sts_rdptr = srd_ptr;
  1493. iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
  1494. if (!mport->outb_msg[ch].mcback)
  1495. goto no_sts_update;
  1496. /* Inform upper layer about transfer completion */
  1497. tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
  1498. sizeof(struct tsi721_omsg_desc);
  1499. /*
  1500. * Check if this is a Link Descriptor (LD).
  1501. * If yes, ignore LD and use descriptor processed
  1502. * before LD.
  1503. */
  1504. if (tx_slot == priv->omsg_ring[ch].size) {
  1505. if (prev_ptr)
  1506. tx_slot = (prev_ptr -
  1507. (u64)priv->omsg_ring[ch].omd_phys)/
  1508. sizeof(struct tsi721_omsg_desc);
  1509. else
  1510. goto no_sts_update;
  1511. }
  1512. if (tx_slot >= priv->omsg_ring[ch].size)
  1513. tsi_debug(OMSG, &priv->pdev->dev,
  1514. "OB_MSG tx_slot=%x > size=%x",
  1515. tx_slot, priv->omsg_ring[ch].size);
  1516. WARN_ON(tx_slot >= priv->omsg_ring[ch].size);
  1517. /* Move slot index to the next message to be sent */
  1518. ++tx_slot;
  1519. if (tx_slot == priv->omsg_ring[ch].size)
  1520. tx_slot = 0;
  1521. dev_id = priv->omsg_ring[ch].dev_id;
  1522. do_callback = 1;
  1523. }
  1524. no_sts_update:
  1525. if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
  1526. /*
  1527. * Outbound message operation aborted due to error,
  1528. * reinitialize OB MSG channel
  1529. */
  1530. tsi_debug(OMSG, &priv->pdev->dev, "OB MSG ABORT ch_stat=%x",
  1531. ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
  1532. iowrite32(TSI721_OBDMAC_INT_ERROR,
  1533. priv->regs + TSI721_OBDMAC_INT(ch));
  1534. iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
  1535. priv->regs + TSI721_OBDMAC_CTL(ch));
  1536. ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
  1537. /* Inform upper level to clear all pending tx slots */
  1538. dev_id = priv->omsg_ring[ch].dev_id;
  1539. tx_slot = priv->omsg_ring[ch].tx_slot;
  1540. do_callback = 1;
  1541. /* Synch tx_slot tracking */
  1542. iowrite32(priv->omsg_ring[ch].tx_slot,
  1543. priv->regs + TSI721_OBDMAC_DRDCNT(ch));
  1544. ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
  1545. priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
  1546. priv->omsg_ring[ch].sts_rdptr = 0;
  1547. }
  1548. /* Clear channel interrupts */
  1549. iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
  1550. if (!(priv->flags & TSI721_USING_MSIX)) {
  1551. u32 ch_inte;
  1552. /* Re-enable channel interrupts */
  1553. ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  1554. ch_inte |= TSI721_INT_OMSG_CHAN(ch);
  1555. iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
  1556. }
  1557. spin_unlock(&priv->omsg_ring[ch].lock);
  1558. if (mport->outb_msg[ch].mcback && do_callback)
  1559. mport->outb_msg[ch].mcback(mport, dev_id, ch, tx_slot);
  1560. }
  1561. /**
  1562. * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
  1563. * @mport: Master port implementing Outbound Messaging Engine
  1564. * @dev_id: Device specific pointer to pass on event
  1565. * @mbox: Mailbox to open
  1566. * @entries: Number of entries in the outbound mailbox ring
  1567. */
  1568. static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
  1569. int mbox, int entries)
  1570. {
  1571. struct tsi721_device *priv = mport->priv;
  1572. struct tsi721_omsg_desc *bd_ptr;
  1573. int i, rc = 0;
  1574. if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
  1575. (entries > (TSI721_OMSGD_RING_SIZE)) ||
  1576. (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
  1577. rc = -EINVAL;
  1578. goto out;
  1579. }
  1580. if ((mbox_sel & (1 << mbox)) == 0) {
  1581. rc = -ENODEV;
  1582. goto out;
  1583. }
  1584. priv->omsg_ring[mbox].dev_id = dev_id;
  1585. priv->omsg_ring[mbox].size = entries;
  1586. priv->omsg_ring[mbox].sts_rdptr = 0;
  1587. spin_lock_init(&priv->omsg_ring[mbox].lock);
  1588. /* Outbound Msg Buffer allocation based on
  1589. the number of maximum descriptor entries */
  1590. for (i = 0; i < entries; i++) {
  1591. priv->omsg_ring[mbox].omq_base[i] =
  1592. dma_alloc_coherent(
  1593. &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
  1594. &priv->omsg_ring[mbox].omq_phys[i],
  1595. GFP_KERNEL);
  1596. if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
  1597. tsi_debug(OMSG, &priv->pdev->dev,
  1598. "ENOMEM for OB_MSG_%d data buffer", mbox);
  1599. rc = -ENOMEM;
  1600. goto out_buf;
  1601. }
  1602. }
  1603. /* Outbound message descriptor allocation */
  1604. priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
  1605. &priv->pdev->dev,
  1606. (entries + 1) * sizeof(struct tsi721_omsg_desc),
  1607. &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
  1608. if (priv->omsg_ring[mbox].omd_base == NULL) {
  1609. tsi_debug(OMSG, &priv->pdev->dev,
  1610. "ENOMEM for OB_MSG_%d descriptor memory", mbox);
  1611. rc = -ENOMEM;
  1612. goto out_buf;
  1613. }
  1614. priv->omsg_ring[mbox].tx_slot = 0;
  1615. /* Outbound message descriptor status FIFO allocation */
  1616. priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
  1617. priv->omsg_ring[mbox].sts_base = dma_alloc_coherent(&priv->pdev->dev,
  1618. priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
  1619. &priv->omsg_ring[mbox].sts_phys,
  1620. GFP_KERNEL);
  1621. if (priv->omsg_ring[mbox].sts_base == NULL) {
  1622. tsi_debug(OMSG, &priv->pdev->dev,
  1623. "ENOMEM for OB_MSG_%d status FIFO", mbox);
  1624. rc = -ENOMEM;
  1625. goto out_desc;
  1626. }
  1627. /*
  1628. * Configure Outbound Messaging Engine
  1629. */
  1630. /* Setup Outbound Message descriptor pointer */
  1631. iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
  1632. priv->regs + TSI721_OBDMAC_DPTRH(mbox));
  1633. iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
  1634. TSI721_OBDMAC_DPTRL_MASK),
  1635. priv->regs + TSI721_OBDMAC_DPTRL(mbox));
  1636. /* Setup Outbound Message descriptor status FIFO */
  1637. iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
  1638. priv->regs + TSI721_OBDMAC_DSBH(mbox));
  1639. iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
  1640. TSI721_OBDMAC_DSBL_MASK),
  1641. priv->regs + TSI721_OBDMAC_DSBL(mbox));
  1642. iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
  1643. priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
  1644. /* Enable interrupts */
  1645. #ifdef CONFIG_PCI_MSI
  1646. if (priv->flags & TSI721_USING_MSIX) {
  1647. int idx = TSI721_VECT_OMB0_DONE + mbox;
  1648. /* Request interrupt service if we are in MSI-X mode */
  1649. rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
  1650. priv->msix[idx].irq_name, (void *)priv);
  1651. if (rc) {
  1652. tsi_debug(OMSG, &priv->pdev->dev,
  1653. "Unable to get MSI-X IRQ for OBOX%d-DONE",
  1654. mbox);
  1655. goto out_stat;
  1656. }
  1657. idx = TSI721_VECT_OMB0_INT + mbox;
  1658. rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
  1659. priv->msix[idx].irq_name, (void *)priv);
  1660. if (rc) {
  1661. tsi_debug(OMSG, &priv->pdev->dev,
  1662. "Unable to get MSI-X IRQ for MBOX%d-INT", mbox);
  1663. idx = TSI721_VECT_OMB0_DONE + mbox;
  1664. free_irq(priv->msix[idx].vector, (void *)priv);
  1665. goto out_stat;
  1666. }
  1667. }
  1668. #endif /* CONFIG_PCI_MSI */
  1669. tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
  1670. /* Initialize Outbound Message descriptors ring */
  1671. bd_ptr = priv->omsg_ring[mbox].omd_base;
  1672. bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
  1673. bd_ptr[entries].msg_info = 0;
  1674. bd_ptr[entries].next_lo =
  1675. cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
  1676. TSI721_OBDMAC_DPTRL_MASK);
  1677. bd_ptr[entries].next_hi =
  1678. cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
  1679. priv->omsg_ring[mbox].wr_count = 0;
  1680. mb();
  1681. /* Initialize Outbound Message engine */
  1682. iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
  1683. priv->regs + TSI721_OBDMAC_CTL(mbox));
  1684. ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
  1685. udelay(10);
  1686. priv->omsg_init[mbox] = 1;
  1687. return 0;
  1688. #ifdef CONFIG_PCI_MSI
  1689. out_stat:
  1690. dma_free_coherent(&priv->pdev->dev,
  1691. priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
  1692. priv->omsg_ring[mbox].sts_base,
  1693. priv->omsg_ring[mbox].sts_phys);
  1694. priv->omsg_ring[mbox].sts_base = NULL;
  1695. #endif /* CONFIG_PCI_MSI */
  1696. out_desc:
  1697. dma_free_coherent(&priv->pdev->dev,
  1698. (entries + 1) * sizeof(struct tsi721_omsg_desc),
  1699. priv->omsg_ring[mbox].omd_base,
  1700. priv->omsg_ring[mbox].omd_phys);
  1701. priv->omsg_ring[mbox].omd_base = NULL;
  1702. out_buf:
  1703. for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
  1704. if (priv->omsg_ring[mbox].omq_base[i]) {
  1705. dma_free_coherent(&priv->pdev->dev,
  1706. TSI721_MSG_BUFFER_SIZE,
  1707. priv->omsg_ring[mbox].omq_base[i],
  1708. priv->omsg_ring[mbox].omq_phys[i]);
  1709. priv->omsg_ring[mbox].omq_base[i] = NULL;
  1710. }
  1711. }
  1712. out:
  1713. return rc;
  1714. }
  1715. /**
  1716. * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
  1717. * @mport: Master port implementing the outbound message unit
  1718. * @mbox: Mailbox to close
  1719. */
  1720. static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
  1721. {
  1722. struct tsi721_device *priv = mport->priv;
  1723. u32 i;
  1724. if (!priv->omsg_init[mbox])
  1725. return;
  1726. priv->omsg_init[mbox] = 0;
  1727. /* Disable Interrupts */
  1728. tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
  1729. #ifdef CONFIG_PCI_MSI
  1730. if (priv->flags & TSI721_USING_MSIX) {
  1731. free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
  1732. (void *)priv);
  1733. free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
  1734. (void *)priv);
  1735. }
  1736. #endif /* CONFIG_PCI_MSI */
  1737. /* Free OMSG Descriptor Status FIFO */
  1738. dma_free_coherent(&priv->pdev->dev,
  1739. priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
  1740. priv->omsg_ring[mbox].sts_base,
  1741. priv->omsg_ring[mbox].sts_phys);
  1742. priv->omsg_ring[mbox].sts_base = NULL;
  1743. /* Free OMSG descriptors */
  1744. dma_free_coherent(&priv->pdev->dev,
  1745. (priv->omsg_ring[mbox].size + 1) *
  1746. sizeof(struct tsi721_omsg_desc),
  1747. priv->omsg_ring[mbox].omd_base,
  1748. priv->omsg_ring[mbox].omd_phys);
  1749. priv->omsg_ring[mbox].omd_base = NULL;
  1750. /* Free message buffers */
  1751. for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
  1752. if (priv->omsg_ring[mbox].omq_base[i]) {
  1753. dma_free_coherent(&priv->pdev->dev,
  1754. TSI721_MSG_BUFFER_SIZE,
  1755. priv->omsg_ring[mbox].omq_base[i],
  1756. priv->omsg_ring[mbox].omq_phys[i]);
  1757. priv->omsg_ring[mbox].omq_base[i] = NULL;
  1758. }
  1759. }
  1760. }
  1761. /**
  1762. * tsi721_imsg_handler - Inbound Message Interrupt Handler
  1763. * @priv: pointer to tsi721 private data
  1764. * @ch: inbound message channel number to service
  1765. *
  1766. * Services channel interrupts from inbound messaging engine.
  1767. */
  1768. static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
  1769. {
  1770. u32 mbox = ch - 4;
  1771. u32 imsg_int;
  1772. struct rio_mport *mport = &priv->mport;
  1773. spin_lock(&priv->imsg_ring[mbox].lock);
  1774. imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
  1775. if (imsg_int & TSI721_IBDMAC_INT_SRTO)
  1776. tsi_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout", mbox);
  1777. if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
  1778. tsi_info(&priv->pdev->dev, "IB MBOX%d PCIe error", mbox);
  1779. if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
  1780. tsi_info(&priv->pdev->dev, "IB MBOX%d IB free queue low", mbox);
  1781. /* Clear IB channel interrupts */
  1782. iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
  1783. /* If an IB Msg is received notify the upper layer */
  1784. if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
  1785. mport->inb_msg[mbox].mcback)
  1786. mport->inb_msg[mbox].mcback(mport,
  1787. priv->imsg_ring[mbox].dev_id, mbox, -1);
  1788. if (!(priv->flags & TSI721_USING_MSIX)) {
  1789. u32 ch_inte;
  1790. /* Re-enable channel interrupts */
  1791. ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
  1792. ch_inte |= TSI721_INT_IMSG_CHAN(ch);
  1793. iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
  1794. }
  1795. spin_unlock(&priv->imsg_ring[mbox].lock);
  1796. }
  1797. /**
  1798. * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
  1799. * @mport: Master port implementing the Inbound Messaging Engine
  1800. * @dev_id: Device specific pointer to pass on event
  1801. * @mbox: Mailbox to open
  1802. * @entries: Number of entries in the inbound mailbox ring
  1803. */
  1804. static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
  1805. int mbox, int entries)
  1806. {
  1807. struct tsi721_device *priv = mport->priv;
  1808. int ch = mbox + 4;
  1809. int i;
  1810. u64 *free_ptr;
  1811. int rc = 0;
  1812. if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
  1813. (entries > TSI721_IMSGD_RING_SIZE) ||
  1814. (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
  1815. rc = -EINVAL;
  1816. goto out;
  1817. }
  1818. if ((mbox_sel & (1 << mbox)) == 0) {
  1819. rc = -ENODEV;
  1820. goto out;
  1821. }
  1822. /* Initialize IB Messaging Ring */
  1823. priv->imsg_ring[mbox].dev_id = dev_id;
  1824. priv->imsg_ring[mbox].size = entries;
  1825. priv->imsg_ring[mbox].rx_slot = 0;
  1826. priv->imsg_ring[mbox].desc_rdptr = 0;
  1827. priv->imsg_ring[mbox].fq_wrptr = 0;
  1828. for (i = 0; i < priv->imsg_ring[mbox].size; i++)
  1829. priv->imsg_ring[mbox].imq_base[i] = NULL;
  1830. spin_lock_init(&priv->imsg_ring[mbox].lock);
  1831. /* Allocate buffers for incoming messages */
  1832. priv->imsg_ring[mbox].buf_base =
  1833. dma_alloc_coherent(&priv->pdev->dev,
  1834. entries * TSI721_MSG_BUFFER_SIZE,
  1835. &priv->imsg_ring[mbox].buf_phys,
  1836. GFP_KERNEL);
  1837. if (priv->imsg_ring[mbox].buf_base == NULL) {
  1838. tsi_err(&priv->pdev->dev,
  1839. "Failed to allocate buffers for IB MBOX%d", mbox);
  1840. rc = -ENOMEM;
  1841. goto out;
  1842. }
  1843. /* Allocate memory for circular free list */
  1844. priv->imsg_ring[mbox].imfq_base =
  1845. dma_alloc_coherent(&priv->pdev->dev,
  1846. entries * 8,
  1847. &priv->imsg_ring[mbox].imfq_phys,
  1848. GFP_KERNEL);
  1849. if (priv->imsg_ring[mbox].imfq_base == NULL) {
  1850. tsi_err(&priv->pdev->dev,
  1851. "Failed to allocate free queue for IB MBOX%d", mbox);
  1852. rc = -ENOMEM;
  1853. goto out_buf;
  1854. }
  1855. /* Allocate memory for Inbound message descriptors */
  1856. priv->imsg_ring[mbox].imd_base =
  1857. dma_alloc_coherent(&priv->pdev->dev,
  1858. entries * sizeof(struct tsi721_imsg_desc),
  1859. &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
  1860. if (priv->imsg_ring[mbox].imd_base == NULL) {
  1861. tsi_err(&priv->pdev->dev,
  1862. "Failed to allocate descriptor memory for IB MBOX%d",
  1863. mbox);
  1864. rc = -ENOMEM;
  1865. goto out_dma;
  1866. }
  1867. /* Fill free buffer pointer list */
  1868. free_ptr = priv->imsg_ring[mbox].imfq_base;
  1869. for (i = 0; i < entries; i++)
  1870. free_ptr[i] = cpu_to_le64(
  1871. (u64)(priv->imsg_ring[mbox].buf_phys) +
  1872. i * 0x1000);
  1873. mb();
  1874. /*
  1875. * For mapping of inbound SRIO Messages into appropriate queues we need
  1876. * to set Inbound Device ID register in the messaging engine. We do it
  1877. * once when first inbound mailbox is requested.
  1878. */
  1879. if (!(priv->flags & TSI721_IMSGID_SET)) {
  1880. iowrite32((u32)priv->mport.host_deviceid,
  1881. priv->regs + TSI721_IB_DEVID);
  1882. priv->flags |= TSI721_IMSGID_SET;
  1883. }
  1884. /*
  1885. * Configure Inbound Messaging channel (ch = mbox + 4)
  1886. */
  1887. /* Setup Inbound Message free queue */
  1888. iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
  1889. priv->regs + TSI721_IBDMAC_FQBH(ch));
  1890. iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
  1891. TSI721_IBDMAC_FQBL_MASK),
  1892. priv->regs+TSI721_IBDMAC_FQBL(ch));
  1893. iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
  1894. priv->regs + TSI721_IBDMAC_FQSZ(ch));
  1895. /* Setup Inbound Message descriptor queue */
  1896. iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
  1897. priv->regs + TSI721_IBDMAC_DQBH(ch));
  1898. iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
  1899. (u32)TSI721_IBDMAC_DQBL_MASK),
  1900. priv->regs+TSI721_IBDMAC_DQBL(ch));
  1901. iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
  1902. priv->regs + TSI721_IBDMAC_DQSZ(ch));
  1903. /* Enable interrupts */
  1904. #ifdef CONFIG_PCI_MSI
  1905. if (priv->flags & TSI721_USING_MSIX) {
  1906. int idx = TSI721_VECT_IMB0_RCV + mbox;
  1907. /* Request interrupt service if we are in MSI-X mode */
  1908. rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
  1909. priv->msix[idx].irq_name, (void *)priv);
  1910. if (rc) {
  1911. tsi_debug(IMSG, &priv->pdev->dev,
  1912. "Unable to get MSI-X IRQ for IBOX%d-DONE",
  1913. mbox);
  1914. goto out_desc;
  1915. }
  1916. idx = TSI721_VECT_IMB0_INT + mbox;
  1917. rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
  1918. priv->msix[idx].irq_name, (void *)priv);
  1919. if (rc) {
  1920. tsi_debug(IMSG, &priv->pdev->dev,
  1921. "Unable to get MSI-X IRQ for IBOX%d-INT", mbox);
  1922. free_irq(
  1923. priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
  1924. (void *)priv);
  1925. goto out_desc;
  1926. }
  1927. }
  1928. #endif /* CONFIG_PCI_MSI */
  1929. tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
  1930. /* Initialize Inbound Message Engine */
  1931. iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
  1932. ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
  1933. udelay(10);
  1934. priv->imsg_ring[mbox].fq_wrptr = entries - 1;
  1935. iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
  1936. priv->imsg_init[mbox] = 1;
  1937. return 0;
  1938. #ifdef CONFIG_PCI_MSI
  1939. out_desc:
  1940. dma_free_coherent(&priv->pdev->dev,
  1941. priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
  1942. priv->imsg_ring[mbox].imd_base,
  1943. priv->imsg_ring[mbox].imd_phys);
  1944. priv->imsg_ring[mbox].imd_base = NULL;
  1945. #endif /* CONFIG_PCI_MSI */
  1946. out_dma:
  1947. dma_free_coherent(&priv->pdev->dev,
  1948. priv->imsg_ring[mbox].size * 8,
  1949. priv->imsg_ring[mbox].imfq_base,
  1950. priv->imsg_ring[mbox].imfq_phys);
  1951. priv->imsg_ring[mbox].imfq_base = NULL;
  1952. out_buf:
  1953. dma_free_coherent(&priv->pdev->dev,
  1954. priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
  1955. priv->imsg_ring[mbox].buf_base,
  1956. priv->imsg_ring[mbox].buf_phys);
  1957. priv->imsg_ring[mbox].buf_base = NULL;
  1958. out:
  1959. return rc;
  1960. }
  1961. /**
  1962. * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
  1963. * @mport: Master port implementing the Inbound Messaging Engine
  1964. * @mbox: Mailbox to close
  1965. */
  1966. static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
  1967. {
  1968. struct tsi721_device *priv = mport->priv;
  1969. u32 rx_slot;
  1970. int ch = mbox + 4;
  1971. if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
  1972. return;
  1973. priv->imsg_init[mbox] = 0;
  1974. /* Disable Inbound Messaging Engine */
  1975. /* Disable Interrupts */
  1976. tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
  1977. #ifdef CONFIG_PCI_MSI
  1978. if (priv->flags & TSI721_USING_MSIX) {
  1979. free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
  1980. (void *)priv);
  1981. free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
  1982. (void *)priv);
  1983. }
  1984. #endif /* CONFIG_PCI_MSI */
  1985. /* Clear Inbound Buffer Queue */
  1986. for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
  1987. priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
  1988. /* Free memory allocated for message buffers */
  1989. dma_free_coherent(&priv->pdev->dev,
  1990. priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
  1991. priv->imsg_ring[mbox].buf_base,
  1992. priv->imsg_ring[mbox].buf_phys);
  1993. priv->imsg_ring[mbox].buf_base = NULL;
  1994. /* Free memory allocated for free pointr list */
  1995. dma_free_coherent(&priv->pdev->dev,
  1996. priv->imsg_ring[mbox].size * 8,
  1997. priv->imsg_ring[mbox].imfq_base,
  1998. priv->imsg_ring[mbox].imfq_phys);
  1999. priv->imsg_ring[mbox].imfq_base = NULL;
  2000. /* Free memory allocated for RX descriptors */
  2001. dma_free_coherent(&priv->pdev->dev,
  2002. priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
  2003. priv->imsg_ring[mbox].imd_base,
  2004. priv->imsg_ring[mbox].imd_phys);
  2005. priv->imsg_ring[mbox].imd_base = NULL;
  2006. }
  2007. /**
  2008. * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
  2009. * @mport: Master port implementing the Inbound Messaging Engine
  2010. * @mbox: Inbound mailbox number
  2011. * @buf: Buffer to add to inbound queue
  2012. */
  2013. static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
  2014. {
  2015. struct tsi721_device *priv = mport->priv;
  2016. u32 rx_slot;
  2017. int rc = 0;
  2018. rx_slot = priv->imsg_ring[mbox].rx_slot;
  2019. if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
  2020. tsi_err(&priv->pdev->dev,
  2021. "Error adding inbound buffer %d, buffer exists",
  2022. rx_slot);
  2023. rc = -EINVAL;
  2024. goto out;
  2025. }
  2026. priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
  2027. if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
  2028. priv->imsg_ring[mbox].rx_slot = 0;
  2029. out:
  2030. return rc;
  2031. }
  2032. /**
  2033. * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
  2034. * @mport: Master port implementing the Inbound Messaging Engine
  2035. * @mbox: Inbound mailbox number
  2036. *
  2037. * Returns pointer to the message on success or NULL on failure.
  2038. */
  2039. static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
  2040. {
  2041. struct tsi721_device *priv = mport->priv;
  2042. struct tsi721_imsg_desc *desc;
  2043. u32 rx_slot;
  2044. void *rx_virt = NULL;
  2045. u64 rx_phys;
  2046. void *buf = NULL;
  2047. u64 *free_ptr;
  2048. int ch = mbox + 4;
  2049. int msg_size;
  2050. if (!priv->imsg_init[mbox])
  2051. return NULL;
  2052. desc = priv->imsg_ring[mbox].imd_base;
  2053. desc += priv->imsg_ring[mbox].desc_rdptr;
  2054. if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
  2055. goto out;
  2056. rx_slot = priv->imsg_ring[mbox].rx_slot;
  2057. while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
  2058. if (++rx_slot == priv->imsg_ring[mbox].size)
  2059. rx_slot = 0;
  2060. }
  2061. rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
  2062. le32_to_cpu(desc->bufptr_lo);
  2063. rx_virt = priv->imsg_ring[mbox].buf_base +
  2064. (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
  2065. buf = priv->imsg_ring[mbox].imq_base[rx_slot];
  2066. msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
  2067. if (msg_size == 0)
  2068. msg_size = RIO_MAX_MSG_SIZE;
  2069. memcpy(buf, rx_virt, msg_size);
  2070. priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
  2071. desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
  2072. if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
  2073. priv->imsg_ring[mbox].desc_rdptr = 0;
  2074. iowrite32(priv->imsg_ring[mbox].desc_rdptr,
  2075. priv->regs + TSI721_IBDMAC_DQRP(ch));
  2076. /* Return free buffer into the pointer list */
  2077. free_ptr = priv->imsg_ring[mbox].imfq_base;
  2078. free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
  2079. if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
  2080. priv->imsg_ring[mbox].fq_wrptr = 0;
  2081. iowrite32(priv->imsg_ring[mbox].fq_wrptr,
  2082. priv->regs + TSI721_IBDMAC_FQWP(ch));
  2083. out:
  2084. return buf;
  2085. }
  2086. /**
  2087. * tsi721_messages_init - Initialization of Messaging Engine
  2088. * @priv: pointer to tsi721 private data
  2089. *
  2090. * Configures Tsi721 messaging engine.
  2091. */
  2092. static int tsi721_messages_init(struct tsi721_device *priv)
  2093. {
  2094. int ch;
  2095. iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
  2096. iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
  2097. iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
  2098. /* Set SRIO Message Request/Response Timeout */
  2099. iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
  2100. /* Initialize Inbound Messaging Engine Registers */
  2101. for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
  2102. /* Clear interrupt bits */
  2103. iowrite32(TSI721_IBDMAC_INT_MASK,
  2104. priv->regs + TSI721_IBDMAC_INT(ch));
  2105. /* Clear Status */
  2106. iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
  2107. iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
  2108. priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
  2109. iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
  2110. priv->regs + TSI721_SMSG_ECC_NCOR(ch));
  2111. }
  2112. return 0;
  2113. }
  2114. /**
  2115. * tsi721_query_mport - Fetch inbound message from the Tsi721 MSG Queue
  2116. * @mport: Master port implementing the Inbound Messaging Engine
  2117. * @mbox: Inbound mailbox number
  2118. *
  2119. * Returns pointer to the message on success or NULL on failure.
  2120. */
  2121. static int tsi721_query_mport(struct rio_mport *mport,
  2122. struct rio_mport_attr *attr)
  2123. {
  2124. struct tsi721_device *priv = mport->priv;
  2125. u32 rval;
  2126. rval = ioread32(priv->regs + 0x100 + RIO_PORT_N_ERR_STS_CSR(0, 0));
  2127. if (rval & RIO_PORT_N_ERR_STS_PORT_OK) {
  2128. rval = ioread32(priv->regs + 0x100 + RIO_PORT_N_CTL2_CSR(0, 0));
  2129. attr->link_speed = (rval & RIO_PORT_N_CTL2_SEL_BAUD) >> 28;
  2130. rval = ioread32(priv->regs + 0x100 + RIO_PORT_N_CTL_CSR(0, 0));
  2131. attr->link_width = (rval & RIO_PORT_N_CTL_IPW) >> 27;
  2132. } else
  2133. attr->link_speed = RIO_LINK_DOWN;
  2134. #ifdef CONFIG_RAPIDIO_DMA_ENGINE
  2135. attr->flags = RIO_MPORT_DMA | RIO_MPORT_DMA_SG;
  2136. attr->dma_max_sge = 0;
  2137. attr->dma_max_size = TSI721_BDMA_MAX_BCOUNT;
  2138. attr->dma_align = 0;
  2139. #else
  2140. attr->flags = 0;
  2141. #endif
  2142. return 0;
  2143. }
  2144. /**
  2145. * tsi721_disable_ints - disables all device interrupts
  2146. * @priv: pointer to tsi721 private data
  2147. */
  2148. static void tsi721_disable_ints(struct tsi721_device *priv)
  2149. {
  2150. int ch;
  2151. /* Disable all device level interrupts */
  2152. iowrite32(0, priv->regs + TSI721_DEV_INTE);
  2153. /* Disable all Device Channel interrupts */
  2154. iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
  2155. /* Disable all Inbound Msg Channel interrupts */
  2156. for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
  2157. iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
  2158. /* Disable all Outbound Msg Channel interrupts */
  2159. for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
  2160. iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
  2161. /* Disable all general messaging interrupts */
  2162. iowrite32(0, priv->regs + TSI721_SMSG_INTE);
  2163. /* Disable all BDMA Channel interrupts */
  2164. for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
  2165. iowrite32(0,
  2166. priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
  2167. /* Disable all general BDMA interrupts */
  2168. iowrite32(0, priv->regs + TSI721_BDMA_INTE);
  2169. /* Disable all SRIO Channel interrupts */
  2170. for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
  2171. iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
  2172. /* Disable all general SR2PC interrupts */
  2173. iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
  2174. /* Disable all PC2SR interrupts */
  2175. iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
  2176. /* Disable all I2C interrupts */
  2177. iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
  2178. /* Disable SRIO MAC interrupts */
  2179. iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
  2180. iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
  2181. }
  2182. static struct rio_ops tsi721_rio_ops = {
  2183. .lcread = tsi721_lcread,
  2184. .lcwrite = tsi721_lcwrite,
  2185. .cread = tsi721_cread_dma,
  2186. .cwrite = tsi721_cwrite_dma,
  2187. .dsend = tsi721_dsend,
  2188. .open_inb_mbox = tsi721_open_inb_mbox,
  2189. .close_inb_mbox = tsi721_close_inb_mbox,
  2190. .open_outb_mbox = tsi721_open_outb_mbox,
  2191. .close_outb_mbox = tsi721_close_outb_mbox,
  2192. .add_outb_message = tsi721_add_outb_message,
  2193. .add_inb_buffer = tsi721_add_inb_buffer,
  2194. .get_inb_message = tsi721_get_inb_message,
  2195. .map_inb = tsi721_rio_map_inb_mem,
  2196. .unmap_inb = tsi721_rio_unmap_inb_mem,
  2197. .pwenable = tsi721_pw_enable,
  2198. .query_mport = tsi721_query_mport,
  2199. .map_outb = tsi721_map_outb_win,
  2200. .unmap_outb = tsi721_unmap_outb_win,
  2201. };
  2202. static void tsi721_mport_release(struct device *dev)
  2203. {
  2204. struct rio_mport *mport = to_rio_mport(dev);
  2205. tsi_debug(EXIT, dev, "%s id=%d", mport->name, mport->id);
  2206. }
  2207. /**
  2208. * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
  2209. * @priv: pointer to tsi721 private data
  2210. *
  2211. * Configures Tsi721 as RapidIO master port.
  2212. */
  2213. static int tsi721_setup_mport(struct tsi721_device *priv)
  2214. {
  2215. struct pci_dev *pdev = priv->pdev;
  2216. int err = 0;
  2217. struct rio_mport *mport = &priv->mport;
  2218. err = rio_mport_initialize(mport);
  2219. if (err)
  2220. return err;
  2221. mport->ops = &tsi721_rio_ops;
  2222. mport->index = 0;
  2223. mport->sys_size = 0; /* small system */
  2224. mport->priv = (void *)priv;
  2225. mport->phys_efptr = 0x100;
  2226. mport->phys_rmap = 1;
  2227. mport->dev.parent = &pdev->dev;
  2228. mport->dev.release = tsi721_mport_release;
  2229. INIT_LIST_HEAD(&mport->dbells);
  2230. rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
  2231. rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
  2232. rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
  2233. snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)",
  2234. dev_driver_string(&pdev->dev), dev_name(&pdev->dev));
  2235. /* Hook up interrupt handler */
  2236. #ifdef CONFIG_PCI_MSI
  2237. if (!tsi721_enable_msix(priv))
  2238. priv->flags |= TSI721_USING_MSIX;
  2239. else if (!pci_enable_msi(pdev))
  2240. priv->flags |= TSI721_USING_MSI;
  2241. else
  2242. tsi_debug(MPORT, &pdev->dev,
  2243. "MSI/MSI-X is not available. Using legacy INTx.");
  2244. #endif /* CONFIG_PCI_MSI */
  2245. err = tsi721_request_irq(priv);
  2246. if (err) {
  2247. tsi_err(&pdev->dev, "Unable to get PCI IRQ %02X (err=0x%x)",
  2248. pdev->irq, err);
  2249. return err;
  2250. }
  2251. #ifdef CONFIG_RAPIDIO_DMA_ENGINE
  2252. err = tsi721_register_dma(priv);
  2253. if (err)
  2254. goto err_exit;
  2255. #endif
  2256. /* Enable SRIO link */
  2257. iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
  2258. TSI721_DEVCTL_SRBOOT_CMPL,
  2259. priv->regs + TSI721_DEVCTL);
  2260. if (mport->host_deviceid >= 0)
  2261. iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
  2262. RIO_PORT_GEN_DISCOVERED,
  2263. priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
  2264. else
  2265. iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
  2266. err = rio_register_mport(mport);
  2267. if (err) {
  2268. tsi721_unregister_dma(priv);
  2269. goto err_exit;
  2270. }
  2271. return 0;
  2272. err_exit:
  2273. tsi721_free_irq(priv);
  2274. return err;
  2275. }
  2276. static int tsi721_probe(struct pci_dev *pdev,
  2277. const struct pci_device_id *id)
  2278. {
  2279. struct tsi721_device *priv;
  2280. int err;
  2281. priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
  2282. if (!priv) {
  2283. err = -ENOMEM;
  2284. goto err_exit;
  2285. }
  2286. err = pci_enable_device(pdev);
  2287. if (err) {
  2288. tsi_err(&pdev->dev, "Failed to enable PCI device");
  2289. goto err_clean;
  2290. }
  2291. priv->pdev = pdev;
  2292. #ifdef DEBUG
  2293. {
  2294. int i;
  2295. for (i = 0; i < PCI_STD_NUM_BARS; i++) {
  2296. tsi_debug(INIT, &pdev->dev, "res%d %pR",
  2297. i, &pdev->resource[i]);
  2298. }
  2299. }
  2300. #endif
  2301. /*
  2302. * Verify BAR configuration
  2303. */
  2304. /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
  2305. if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
  2306. pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
  2307. pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
  2308. tsi_err(&pdev->dev, "Missing or misconfigured CSR BAR0");
  2309. err = -ENODEV;
  2310. goto err_disable_pdev;
  2311. }
  2312. /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
  2313. if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
  2314. pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
  2315. pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
  2316. tsi_err(&pdev->dev, "Missing or misconfigured Doorbell BAR1");
  2317. err = -ENODEV;
  2318. goto err_disable_pdev;
  2319. }
  2320. /*
  2321. * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
  2322. * space.
  2323. * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
  2324. * It may be a good idea to keep them disabled using HW configuration
  2325. * to save PCI memory space.
  2326. */
  2327. priv->p2r_bar[0].size = priv->p2r_bar[1].size = 0;
  2328. if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64) {
  2329. if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_PREFETCH)
  2330. tsi_debug(INIT, &pdev->dev,
  2331. "Prefetchable OBW BAR2 will not be used");
  2332. else {
  2333. priv->p2r_bar[0].base = pci_resource_start(pdev, BAR_2);
  2334. priv->p2r_bar[0].size = pci_resource_len(pdev, BAR_2);
  2335. }
  2336. }
  2337. if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64) {
  2338. if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_PREFETCH)
  2339. tsi_debug(INIT, &pdev->dev,
  2340. "Prefetchable OBW BAR4 will not be used");
  2341. else {
  2342. priv->p2r_bar[1].base = pci_resource_start(pdev, BAR_4);
  2343. priv->p2r_bar[1].size = pci_resource_len(pdev, BAR_4);
  2344. }
  2345. }
  2346. err = pci_request_regions(pdev, DRV_NAME);
  2347. if (err) {
  2348. tsi_err(&pdev->dev, "Unable to obtain PCI resources");
  2349. goto err_disable_pdev;
  2350. }
  2351. pci_set_master(pdev);
  2352. priv->regs = pci_ioremap_bar(pdev, BAR_0);
  2353. if (!priv->regs) {
  2354. tsi_err(&pdev->dev, "Unable to map device registers space");
  2355. err = -ENOMEM;
  2356. goto err_free_res;
  2357. }
  2358. priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
  2359. if (!priv->odb_base) {
  2360. tsi_err(&pdev->dev, "Unable to map outbound doorbells space");
  2361. err = -ENOMEM;
  2362. goto err_unmap_bars;
  2363. }
  2364. /* Configure DMA attributes. */
  2365. if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
  2366. err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  2367. if (err) {
  2368. tsi_err(&pdev->dev, "Unable to set DMA mask");
  2369. goto err_unmap_bars;
  2370. }
  2371. if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
  2372. tsi_info(&pdev->dev, "Unable to set consistent DMA mask");
  2373. } else {
  2374. err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  2375. if (err)
  2376. tsi_info(&pdev->dev, "Unable to set consistent DMA mask");
  2377. }
  2378. BUG_ON(!pci_is_pcie(pdev));
  2379. /* Clear "no snoop" and "relaxed ordering" bits. */
  2380. pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
  2381. PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN, 0);
  2382. /* Override PCIe Maximum Read Request Size setting if requested */
  2383. if (pcie_mrrs >= 0) {
  2384. if (pcie_mrrs <= 5)
  2385. pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
  2386. PCI_EXP_DEVCTL_READRQ, pcie_mrrs << 12);
  2387. else
  2388. tsi_info(&pdev->dev,
  2389. "Invalid MRRS override value %d", pcie_mrrs);
  2390. }
  2391. /* Set PCIe completion timeout to 1-10ms */
  2392. pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2,
  2393. PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0x2);
  2394. /*
  2395. * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
  2396. */
  2397. pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
  2398. pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
  2399. TSI721_MSIXTBL_OFFSET);
  2400. pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
  2401. TSI721_MSIXPBA_OFFSET);
  2402. pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
  2403. /* End of FIXUP */
  2404. tsi721_disable_ints(priv);
  2405. tsi721_init_pc2sr_mapping(priv);
  2406. tsi721_init_sr2pc_mapping(priv);
  2407. if (tsi721_bdma_maint_init(priv)) {
  2408. tsi_err(&pdev->dev, "BDMA initialization failed");
  2409. err = -ENOMEM;
  2410. goto err_unmap_bars;
  2411. }
  2412. err = tsi721_doorbell_init(priv);
  2413. if (err)
  2414. goto err_free_bdma;
  2415. tsi721_port_write_init(priv);
  2416. err = tsi721_messages_init(priv);
  2417. if (err)
  2418. goto err_free_consistent;
  2419. err = tsi721_setup_mport(priv);
  2420. if (err)
  2421. goto err_free_consistent;
  2422. pci_set_drvdata(pdev, priv);
  2423. tsi721_interrupts_init(priv);
  2424. return 0;
  2425. err_free_consistent:
  2426. tsi721_port_write_free(priv);
  2427. tsi721_doorbell_free(priv);
  2428. err_free_bdma:
  2429. tsi721_bdma_maint_free(priv);
  2430. err_unmap_bars:
  2431. if (priv->regs)
  2432. iounmap(priv->regs);
  2433. if (priv->odb_base)
  2434. iounmap(priv->odb_base);
  2435. err_free_res:
  2436. pci_release_regions(pdev);
  2437. pci_clear_master(pdev);
  2438. err_disable_pdev:
  2439. pci_disable_device(pdev);
  2440. err_clean:
  2441. kfree(priv);
  2442. err_exit:
  2443. return err;
  2444. }
  2445. static void tsi721_remove(struct pci_dev *pdev)
  2446. {
  2447. struct tsi721_device *priv = pci_get_drvdata(pdev);
  2448. tsi_debug(EXIT, &pdev->dev, "enter");
  2449. tsi721_disable_ints(priv);
  2450. tsi721_free_irq(priv);
  2451. flush_scheduled_work();
  2452. rio_unregister_mport(&priv->mport);
  2453. tsi721_unregister_dma(priv);
  2454. tsi721_bdma_maint_free(priv);
  2455. tsi721_doorbell_free(priv);
  2456. tsi721_port_write_free(priv);
  2457. tsi721_close_sr2pc_mapping(priv);
  2458. if (priv->regs)
  2459. iounmap(priv->regs);
  2460. if (priv->odb_base)
  2461. iounmap(priv->odb_base);
  2462. #ifdef CONFIG_PCI_MSI
  2463. if (priv->flags & TSI721_USING_MSIX)
  2464. pci_disable_msix(priv->pdev);
  2465. else if (priv->flags & TSI721_USING_MSI)
  2466. pci_disable_msi(priv->pdev);
  2467. #endif
  2468. pci_release_regions(pdev);
  2469. pci_clear_master(pdev);
  2470. pci_disable_device(pdev);
  2471. pci_set_drvdata(pdev, NULL);
  2472. kfree(priv);
  2473. tsi_debug(EXIT, &pdev->dev, "exit");
  2474. }
  2475. static void tsi721_shutdown(struct pci_dev *pdev)
  2476. {
  2477. struct tsi721_device *priv = pci_get_drvdata(pdev);
  2478. tsi_debug(EXIT, &pdev->dev, "enter");
  2479. tsi721_disable_ints(priv);
  2480. tsi721_dma_stop_all(priv);
  2481. pci_clear_master(pdev);
  2482. pci_disable_device(pdev);
  2483. }
  2484. static const struct pci_device_id tsi721_pci_tbl[] = {
  2485. { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
  2486. { 0, } /* terminate list */
  2487. };
  2488. MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
  2489. static struct pci_driver tsi721_driver = {
  2490. .name = "tsi721",
  2491. .id_table = tsi721_pci_tbl,
  2492. .probe = tsi721_probe,
  2493. .remove = tsi721_remove,
  2494. .shutdown = tsi721_shutdown,
  2495. };
  2496. module_pci_driver(tsi721_driver);
  2497. MODULE_DESCRIPTION("IDT Tsi721 PCIExpress-to-SRIO bridge driver");
  2498. MODULE_AUTHOR("Integrated Device Technology, Inc.");
  2499. MODULE_LICENSE("GPL");