bcm-pdc-mailbox.c 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2016 Broadcom
  4. */
  5. /*
  6. * Broadcom PDC Mailbox Driver
  7. * The PDC provides a ring based programming interface to one or more hardware
  8. * offload engines. For example, the PDC driver works with both SPU-M and SPU2
  9. * cryptographic offload hardware. In some chips the PDC is referred to as MDE,
  10. * and in others the FA2/FA+ hardware is used with this PDC driver.
  11. *
  12. * The PDC driver registers with the Linux mailbox framework as a mailbox
  13. * controller, once for each PDC instance. Ring 0 for each PDC is registered as
  14. * a mailbox channel. The PDC driver uses interrupts to determine when data
  15. * transfers to and from an offload engine are complete. The PDC driver uses
  16. * threaded IRQs so that response messages are handled outside of interrupt
  17. * context.
  18. *
  19. * The PDC driver allows multiple messages to be pending in the descriptor
  20. * rings. The tx_msg_start descriptor index indicates where the last message
  21. * starts. The txin_numd value at this index indicates how many descriptor
  22. * indexes make up the message. Similar state is kept on the receive side. When
  23. * an rx interrupt indicates a response is ready, the PDC driver processes numd
  24. * descriptors from the tx and rx ring, thus processing one response at a time.
  25. */
  26. #include <linux/errno.h>
  27. #include <linux/module.h>
  28. #include <linux/init.h>
  29. #include <linux/slab.h>
  30. #include <linux/debugfs.h>
  31. #include <linux/interrupt.h>
  32. #include <linux/wait.h>
  33. #include <linux/platform_device.h>
  34. #include <linux/io.h>
  35. #include <linux/of.h>
  36. #include <linux/of_device.h>
  37. #include <linux/of_address.h>
  38. #include <linux/of_irq.h>
  39. #include <linux/mailbox_controller.h>
  40. #include <linux/mailbox/brcm-message.h>
  41. #include <linux/scatterlist.h>
  42. #include <linux/dma-direction.h>
  43. #include <linux/dma-mapping.h>
  44. #include <linux/dmapool.h>
  45. #define PDC_SUCCESS 0
  46. #define RING_ENTRY_SIZE sizeof(struct dma64dd)
  47. /* # entries in PDC dma ring */
  48. #define PDC_RING_ENTRIES 512
  49. /*
  50. * Minimum number of ring descriptor entries that must be free to tell mailbox
  51. * framework that it can submit another request
  52. */
  53. #define PDC_RING_SPACE_MIN 15
  54. #define PDC_RING_SIZE (PDC_RING_ENTRIES * RING_ENTRY_SIZE)
  55. /* Rings are 8k aligned */
  56. #define RING_ALIGN_ORDER 13
  57. #define RING_ALIGN BIT(RING_ALIGN_ORDER)
  58. #define RX_BUF_ALIGN_ORDER 5
  59. #define RX_BUF_ALIGN BIT(RX_BUF_ALIGN_ORDER)
  60. /* descriptor bumping macros */
  61. #define XXD(x, max_mask) ((x) & (max_mask))
  62. #define TXD(x, max_mask) XXD((x), (max_mask))
  63. #define RXD(x, max_mask) XXD((x), (max_mask))
  64. #define NEXTTXD(i, max_mask) TXD((i) + 1, (max_mask))
  65. #define PREVTXD(i, max_mask) TXD((i) - 1, (max_mask))
  66. #define NEXTRXD(i, max_mask) RXD((i) + 1, (max_mask))
  67. #define PREVRXD(i, max_mask) RXD((i) - 1, (max_mask))
  68. #define NTXDACTIVE(h, t, max_mask) TXD((t) - (h), (max_mask))
  69. #define NRXDACTIVE(h, t, max_mask) RXD((t) - (h), (max_mask))
  70. /* Length of BCM header at start of SPU msg, in bytes */
  71. #define BCM_HDR_LEN 8
  72. /*
  73. * PDC driver reserves ringset 0 on each SPU for its own use. The driver does
  74. * not currently support use of multiple ringsets on a single PDC engine.
  75. */
  76. #define PDC_RINGSET 0
  77. /*
  78. * Interrupt mask and status definitions. Enable interrupts for tx and rx on
  79. * ring 0
  80. */
  81. #define PDC_RCVINT_0 (16 + PDC_RINGSET)
  82. #define PDC_RCVINTEN_0 BIT(PDC_RCVINT_0)
  83. #define PDC_INTMASK (PDC_RCVINTEN_0)
  84. #define PDC_LAZY_FRAMECOUNT 1
  85. #define PDC_LAZY_TIMEOUT 10000
  86. #define PDC_LAZY_INT (PDC_LAZY_TIMEOUT | (PDC_LAZY_FRAMECOUNT << 24))
  87. #define PDC_INTMASK_OFFSET 0x24
  88. #define PDC_INTSTATUS_OFFSET 0x20
  89. #define PDC_RCVLAZY0_OFFSET (0x30 + 4 * PDC_RINGSET)
  90. #define FA_RCVLAZY0_OFFSET 0x100
  91. /*
  92. * For SPU2, configure MDE_CKSUM_CONTROL to write 17 bytes of metadata
  93. * before frame
  94. */
  95. #define PDC_SPU2_RESP_HDR_LEN 17
  96. #define PDC_CKSUM_CTRL BIT(27)
  97. #define PDC_CKSUM_CTRL_OFFSET 0x400
  98. #define PDC_SPUM_RESP_HDR_LEN 32
  99. /*
  100. * Sets the following bits for write to transmit control reg:
  101. * 11 - PtyChkDisable - parity check is disabled
  102. * 20:18 - BurstLen = 3 -> 2^7 = 128 byte data reads from memory
  103. */
  104. #define PDC_TX_CTL 0x000C0800
  105. /* Bit in tx control reg to enable tx channel */
  106. #define PDC_TX_ENABLE 0x1
  107. /*
  108. * Sets the following bits for write to receive control reg:
  109. * 7:1 - RcvOffset - size in bytes of status region at start of rx frame buf
  110. * 9 - SepRxHdrDescEn - place start of new frames only in descriptors
  111. * that have StartOfFrame set
  112. * 10 - OflowContinue - on rx FIFO overflow, clear rx fifo, discard all
  113. * remaining bytes in current frame, report error
  114. * in rx frame status for current frame
  115. * 11 - PtyChkDisable - parity check is disabled
  116. * 20:18 - BurstLen = 3 -> 2^7 = 128 byte data reads from memory
  117. */
  118. #define PDC_RX_CTL 0x000C0E00
  119. /* Bit in rx control reg to enable rx channel */
  120. #define PDC_RX_ENABLE 0x1
  121. #define CRYPTO_D64_RS0_CD_MASK ((PDC_RING_ENTRIES * RING_ENTRY_SIZE) - 1)
  122. /* descriptor flags */
  123. #define D64_CTRL1_EOT BIT(28) /* end of descriptor table */
  124. #define D64_CTRL1_IOC BIT(29) /* interrupt on complete */
  125. #define D64_CTRL1_EOF BIT(30) /* end of frame */
  126. #define D64_CTRL1_SOF BIT(31) /* start of frame */
  127. #define RX_STATUS_OVERFLOW 0x00800000
  128. #define RX_STATUS_LEN 0x0000FFFF
  129. #define PDC_TXREGS_OFFSET 0x200
  130. #define PDC_RXREGS_OFFSET 0x220
  131. /* Maximum size buffer the DMA engine can handle */
  132. #define PDC_DMA_BUF_MAX 16384
  133. enum pdc_hw {
  134. FA_HW, /* FA2/FA+ hardware (i.e. Northstar Plus) */
  135. PDC_HW /* PDC/MDE hardware (i.e. Northstar 2, Pegasus) */
  136. };
  137. struct pdc_dma_map {
  138. void *ctx; /* opaque context associated with frame */
  139. };
  140. /* dma descriptor */
  141. struct dma64dd {
  142. u32 ctrl1; /* misc control bits */
  143. u32 ctrl2; /* buffer count and address extension */
  144. u32 addrlow; /* memory address of the date buffer, bits 31:0 */
  145. u32 addrhigh; /* memory address of the date buffer, bits 63:32 */
  146. };
  147. /* dma registers per channel(xmt or rcv) */
  148. struct dma64_regs {
  149. u32 control; /* enable, et al */
  150. u32 ptr; /* last descriptor posted to chip */
  151. u32 addrlow; /* descriptor ring base address low 32-bits */
  152. u32 addrhigh; /* descriptor ring base address bits 63:32 */
  153. u32 status0; /* last rx descriptor written by hw */
  154. u32 status1; /* driver does not use */
  155. };
  156. /* cpp contortions to concatenate w/arg prescan */
  157. #ifndef PAD
  158. #define _PADLINE(line) pad ## line
  159. #define _XSTR(line) _PADLINE(line)
  160. #define PAD _XSTR(__LINE__)
  161. #endif /* PAD */
  162. /* dma registers. matches hw layout. */
  163. struct dma64 {
  164. struct dma64_regs dmaxmt; /* dma tx */
  165. u32 PAD[2];
  166. struct dma64_regs dmarcv; /* dma rx */
  167. u32 PAD[2];
  168. };
  169. /* PDC registers */
  170. struct pdc_regs {
  171. u32 devcontrol; /* 0x000 */
  172. u32 devstatus; /* 0x004 */
  173. u32 PAD;
  174. u32 biststatus; /* 0x00c */
  175. u32 PAD[4];
  176. u32 intstatus; /* 0x020 */
  177. u32 intmask; /* 0x024 */
  178. u32 gptimer; /* 0x028 */
  179. u32 PAD;
  180. u32 intrcvlazy_0; /* 0x030 (Only in PDC, not FA2) */
  181. u32 intrcvlazy_1; /* 0x034 (Only in PDC, not FA2) */
  182. u32 intrcvlazy_2; /* 0x038 (Only in PDC, not FA2) */
  183. u32 intrcvlazy_3; /* 0x03c (Only in PDC, not FA2) */
  184. u32 PAD[48];
  185. u32 fa_intrecvlazy; /* 0x100 (Only in FA2, not PDC) */
  186. u32 flowctlthresh; /* 0x104 */
  187. u32 wrrthresh; /* 0x108 */
  188. u32 gmac_idle_cnt_thresh; /* 0x10c */
  189. u32 PAD[4];
  190. u32 ifioaccessaddr; /* 0x120 */
  191. u32 ifioaccessbyte; /* 0x124 */
  192. u32 ifioaccessdata; /* 0x128 */
  193. u32 PAD[21];
  194. u32 phyaccess; /* 0x180 */
  195. u32 PAD;
  196. u32 phycontrol; /* 0x188 */
  197. u32 txqctl; /* 0x18c */
  198. u32 rxqctl; /* 0x190 */
  199. u32 gpioselect; /* 0x194 */
  200. u32 gpio_output_en; /* 0x198 */
  201. u32 PAD; /* 0x19c */
  202. u32 txq_rxq_mem_ctl; /* 0x1a0 */
  203. u32 memory_ecc_status; /* 0x1a4 */
  204. u32 serdes_ctl; /* 0x1a8 */
  205. u32 serdes_status0; /* 0x1ac */
  206. u32 serdes_status1; /* 0x1b0 */
  207. u32 PAD[11]; /* 0x1b4-1dc */
  208. u32 clk_ctl_st; /* 0x1e0 */
  209. u32 hw_war; /* 0x1e4 (Only in PDC, not FA2) */
  210. u32 pwrctl; /* 0x1e8 */
  211. u32 PAD[5];
  212. #define PDC_NUM_DMA_RINGS 4
  213. struct dma64 dmaregs[PDC_NUM_DMA_RINGS]; /* 0x0200 - 0x2fc */
  214. /* more registers follow, but we don't use them */
  215. };
  216. /* structure for allocating/freeing DMA rings */
  217. struct pdc_ring_alloc {
  218. dma_addr_t dmabase; /* DMA address of start of ring */
  219. void *vbase; /* base kernel virtual address of ring */
  220. u32 size; /* ring allocation size in bytes */
  221. };
  222. /*
  223. * context associated with a receive descriptor.
  224. * @rxp_ctx: opaque context associated with frame that starts at each
  225. * rx ring index.
  226. * @dst_sg: Scatterlist used to form reply frames beginning at a given ring
  227. * index. Retained in order to unmap each sg after reply is processed.
  228. * @rxin_numd: Number of rx descriptors associated with the message that starts
  229. * at a descriptor index. Not set for every index. For example,
  230. * if descriptor index i points to a scatterlist with 4 entries,
  231. * then the next three descriptor indexes don't have a value set.
  232. * @resp_hdr: Virtual address of buffer used to catch DMA rx status
  233. * @resp_hdr_daddr: physical address of DMA rx status buffer
  234. */
  235. struct pdc_rx_ctx {
  236. void *rxp_ctx;
  237. struct scatterlist *dst_sg;
  238. u32 rxin_numd;
  239. void *resp_hdr;
  240. dma_addr_t resp_hdr_daddr;
  241. };
  242. /* PDC state structure */
  243. struct pdc_state {
  244. /* Index of the PDC whose state is in this structure instance */
  245. u8 pdc_idx;
  246. /* Platform device for this PDC instance */
  247. struct platform_device *pdev;
  248. /*
  249. * Each PDC instance has a mailbox controller. PDC receives request
  250. * messages through mailboxes, and sends response messages through the
  251. * mailbox framework.
  252. */
  253. struct mbox_controller mbc;
  254. unsigned int pdc_irq;
  255. /* tasklet for deferred processing after DMA rx interrupt */
  256. struct tasklet_struct rx_tasklet;
  257. /* Number of bytes of receive status prior to each rx frame */
  258. u32 rx_status_len;
  259. /* Whether a BCM header is prepended to each frame */
  260. bool use_bcm_hdr;
  261. /* Sum of length of BCM header and rx status header */
  262. u32 pdc_resp_hdr_len;
  263. /* The base virtual address of DMA hw registers */
  264. void __iomem *pdc_reg_vbase;
  265. /* Pool for allocation of DMA rings */
  266. struct dma_pool *ring_pool;
  267. /* Pool for allocation of metadata buffers for response messages */
  268. struct dma_pool *rx_buf_pool;
  269. /*
  270. * The base virtual address of DMA tx/rx descriptor rings. Corresponding
  271. * DMA address and size of ring allocation.
  272. */
  273. struct pdc_ring_alloc tx_ring_alloc;
  274. struct pdc_ring_alloc rx_ring_alloc;
  275. struct pdc_regs *regs; /* start of PDC registers */
  276. struct dma64_regs *txregs_64; /* dma tx engine registers */
  277. struct dma64_regs *rxregs_64; /* dma rx engine registers */
  278. /*
  279. * Arrays of PDC_RING_ENTRIES descriptors
  280. * To use multiple ringsets, this needs to be extended
  281. */
  282. struct dma64dd *txd_64; /* tx descriptor ring */
  283. struct dma64dd *rxd_64; /* rx descriptor ring */
  284. /* descriptor ring sizes */
  285. u32 ntxd; /* # tx descriptors */
  286. u32 nrxd; /* # rx descriptors */
  287. u32 nrxpost; /* # rx buffers to keep posted */
  288. u32 ntxpost; /* max number of tx buffers that can be posted */
  289. /*
  290. * Index of next tx descriptor to reclaim. That is, the descriptor
  291. * index of the oldest tx buffer for which the host has yet to process
  292. * the corresponding response.
  293. */
  294. u32 txin;
  295. /*
  296. * Index of the first receive descriptor for the sequence of
  297. * message fragments currently under construction. Used to build up
  298. * the rxin_numd count for a message. Updated to rxout when the host
  299. * starts a new sequence of rx buffers for a new message.
  300. */
  301. u32 tx_msg_start;
  302. /* Index of next tx descriptor to post. */
  303. u32 txout;
  304. /*
  305. * Number of tx descriptors associated with the message that starts
  306. * at this tx descriptor index.
  307. */
  308. u32 txin_numd[PDC_RING_ENTRIES];
  309. /*
  310. * Index of next rx descriptor to reclaim. This is the index of
  311. * the next descriptor whose data has yet to be processed by the host.
  312. */
  313. u32 rxin;
  314. /*
  315. * Index of the first receive descriptor for the sequence of
  316. * message fragments currently under construction. Used to build up
  317. * the rxin_numd count for a message. Updated to rxout when the host
  318. * starts a new sequence of rx buffers for a new message.
  319. */
  320. u32 rx_msg_start;
  321. /*
  322. * Saved value of current hardware rx descriptor index.
  323. * The last rx buffer written by the hw is the index previous to
  324. * this one.
  325. */
  326. u32 last_rx_curr;
  327. /* Index of next rx descriptor to post. */
  328. u32 rxout;
  329. struct pdc_rx_ctx rx_ctx[PDC_RING_ENTRIES];
  330. /*
  331. * Scatterlists used to form request and reply frames beginning at a
  332. * given ring index. Retained in order to unmap each sg after reply
  333. * is processed
  334. */
  335. struct scatterlist *src_sg[PDC_RING_ENTRIES];
  336. /* counters */
  337. u32 pdc_requests; /* number of request messages submitted */
  338. u32 pdc_replies; /* number of reply messages received */
  339. u32 last_tx_not_done; /* too few tx descriptors to indicate done */
  340. u32 tx_ring_full; /* unable to accept msg because tx ring full */
  341. u32 rx_ring_full; /* unable to accept msg because rx ring full */
  342. u32 txnobuf; /* unable to create tx descriptor */
  343. u32 rxnobuf; /* unable to create rx descriptor */
  344. u32 rx_oflow; /* count of rx overflows */
  345. /* hardware type - FA2 or PDC/MDE */
  346. enum pdc_hw hw_type;
  347. };
  348. /* Global variables */
  349. struct pdc_globals {
  350. /* Actual number of SPUs in hardware, as reported by device tree */
  351. u32 num_spu;
  352. };
  353. static struct pdc_globals pdcg;
  354. /* top level debug FS directory for PDC driver */
  355. static struct dentry *debugfs_dir;
  356. static ssize_t pdc_debugfs_read(struct file *filp, char __user *ubuf,
  357. size_t count, loff_t *offp)
  358. {
  359. struct pdc_state *pdcs;
  360. char *buf;
  361. ssize_t ret, out_offset, out_count;
  362. out_count = 512;
  363. buf = kmalloc(out_count, GFP_KERNEL);
  364. if (!buf)
  365. return -ENOMEM;
  366. pdcs = filp->private_data;
  367. out_offset = 0;
  368. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  369. "SPU %u stats:\n", pdcs->pdc_idx);
  370. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  371. "PDC requests....................%u\n",
  372. pdcs->pdc_requests);
  373. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  374. "PDC responses...................%u\n",
  375. pdcs->pdc_replies);
  376. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  377. "Tx not done.....................%u\n",
  378. pdcs->last_tx_not_done);
  379. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  380. "Tx ring full....................%u\n",
  381. pdcs->tx_ring_full);
  382. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  383. "Rx ring full....................%u\n",
  384. pdcs->rx_ring_full);
  385. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  386. "Tx desc write fail. Ring full...%u\n",
  387. pdcs->txnobuf);
  388. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  389. "Rx desc write fail. Ring full...%u\n",
  390. pdcs->rxnobuf);
  391. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  392. "Receive overflow................%u\n",
  393. pdcs->rx_oflow);
  394. out_offset += scnprintf(buf + out_offset, out_count - out_offset,
  395. "Num frags in rx ring............%u\n",
  396. NRXDACTIVE(pdcs->rxin, pdcs->last_rx_curr,
  397. pdcs->nrxpost));
  398. if (out_offset > out_count)
  399. out_offset = out_count;
  400. ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
  401. kfree(buf);
  402. return ret;
  403. }
  404. static const struct file_operations pdc_debugfs_stats = {
  405. .owner = THIS_MODULE,
  406. .open = simple_open,
  407. .read = pdc_debugfs_read,
  408. };
  409. /**
  410. * pdc_setup_debugfs() - Create the debug FS directories. If the top-level
  411. * directory has not yet been created, create it now. Create a stats file in
  412. * this directory for a SPU.
  413. * @pdcs: PDC state structure
  414. */
  415. static void pdc_setup_debugfs(struct pdc_state *pdcs)
  416. {
  417. char spu_stats_name[16];
  418. if (!debugfs_initialized())
  419. return;
  420. snprintf(spu_stats_name, 16, "pdc%d_stats", pdcs->pdc_idx);
  421. if (!debugfs_dir)
  422. debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
  423. /* S_IRUSR == 0400 */
  424. debugfs_create_file(spu_stats_name, 0400, debugfs_dir, pdcs,
  425. &pdc_debugfs_stats);
  426. }
  427. static void pdc_free_debugfs(void)
  428. {
  429. debugfs_remove_recursive(debugfs_dir);
  430. debugfs_dir = NULL;
  431. }
  432. /**
  433. * pdc_build_rxd() - Build DMA descriptor to receive SPU result.
  434. * @pdcs: PDC state for SPU that will generate result
  435. * @dma_addr: DMA address of buffer that descriptor is being built for
  436. * @buf_len: Length of the receive buffer, in bytes
  437. * @flags: Flags to be stored in descriptor
  438. */
  439. static inline void
  440. pdc_build_rxd(struct pdc_state *pdcs, dma_addr_t dma_addr,
  441. u32 buf_len, u32 flags)
  442. {
  443. struct device *dev = &pdcs->pdev->dev;
  444. struct dma64dd *rxd = &pdcs->rxd_64[pdcs->rxout];
  445. dev_dbg(dev,
  446. "Writing rx descriptor for PDC %u at index %u with length %u. flags %#x\n",
  447. pdcs->pdc_idx, pdcs->rxout, buf_len, flags);
  448. rxd->addrlow = cpu_to_le32(lower_32_bits(dma_addr));
  449. rxd->addrhigh = cpu_to_le32(upper_32_bits(dma_addr));
  450. rxd->ctrl1 = cpu_to_le32(flags);
  451. rxd->ctrl2 = cpu_to_le32(buf_len);
  452. /* bump ring index and return */
  453. pdcs->rxout = NEXTRXD(pdcs->rxout, pdcs->nrxpost);
  454. }
  455. /**
  456. * pdc_build_txd() - Build a DMA descriptor to transmit a SPU request to
  457. * hardware.
  458. * @pdcs: PDC state for the SPU that will process this request
  459. * @dma_addr: DMA address of packet to be transmitted
  460. * @buf_len: Length of tx buffer, in bytes
  461. * @flags: Flags to be stored in descriptor
  462. */
  463. static inline void
  464. pdc_build_txd(struct pdc_state *pdcs, dma_addr_t dma_addr, u32 buf_len,
  465. u32 flags)
  466. {
  467. struct device *dev = &pdcs->pdev->dev;
  468. struct dma64dd *txd = &pdcs->txd_64[pdcs->txout];
  469. dev_dbg(dev,
  470. "Writing tx descriptor for PDC %u at index %u with length %u, flags %#x\n",
  471. pdcs->pdc_idx, pdcs->txout, buf_len, flags);
  472. txd->addrlow = cpu_to_le32(lower_32_bits(dma_addr));
  473. txd->addrhigh = cpu_to_le32(upper_32_bits(dma_addr));
  474. txd->ctrl1 = cpu_to_le32(flags);
  475. txd->ctrl2 = cpu_to_le32(buf_len);
  476. /* bump ring index and return */
  477. pdcs->txout = NEXTTXD(pdcs->txout, pdcs->ntxpost);
  478. }
  479. /**
  480. * pdc_receive_one() - Receive a response message from a given SPU.
  481. * @pdcs: PDC state for the SPU to receive from
  482. *
  483. * When the return code indicates success, the response message is available in
  484. * the receive buffers provided prior to submission of the request.
  485. *
  486. * Return: PDC_SUCCESS if one or more receive descriptors was processed
  487. * -EAGAIN indicates that no response message is available
  488. * -EIO an error occurred
  489. */
  490. static int
  491. pdc_receive_one(struct pdc_state *pdcs)
  492. {
  493. struct device *dev = &pdcs->pdev->dev;
  494. struct mbox_controller *mbc;
  495. struct mbox_chan *chan;
  496. struct brcm_message mssg;
  497. u32 len, rx_status;
  498. u32 num_frags;
  499. u8 *resp_hdr; /* virtual addr of start of resp message DMA header */
  500. u32 frags_rdy; /* number of fragments ready to read */
  501. u32 rx_idx; /* ring index of start of receive frame */
  502. dma_addr_t resp_hdr_daddr;
  503. struct pdc_rx_ctx *rx_ctx;
  504. mbc = &pdcs->mbc;
  505. chan = &mbc->chans[0];
  506. mssg.type = BRCM_MESSAGE_SPU;
  507. /*
  508. * return if a complete response message is not yet ready.
  509. * rxin_numd[rxin] is the number of fragments in the next msg
  510. * to read.
  511. */
  512. frags_rdy = NRXDACTIVE(pdcs->rxin, pdcs->last_rx_curr, pdcs->nrxpost);
  513. if ((frags_rdy == 0) ||
  514. (frags_rdy < pdcs->rx_ctx[pdcs->rxin].rxin_numd))
  515. /* No response ready */
  516. return -EAGAIN;
  517. num_frags = pdcs->txin_numd[pdcs->txin];
  518. WARN_ON(num_frags == 0);
  519. dma_unmap_sg(dev, pdcs->src_sg[pdcs->txin],
  520. sg_nents(pdcs->src_sg[pdcs->txin]), DMA_TO_DEVICE);
  521. pdcs->txin = (pdcs->txin + num_frags) & pdcs->ntxpost;
  522. dev_dbg(dev, "PDC %u reclaimed %d tx descriptors",
  523. pdcs->pdc_idx, num_frags);
  524. rx_idx = pdcs->rxin;
  525. rx_ctx = &pdcs->rx_ctx[rx_idx];
  526. num_frags = rx_ctx->rxin_numd;
  527. /* Return opaque context with result */
  528. mssg.ctx = rx_ctx->rxp_ctx;
  529. rx_ctx->rxp_ctx = NULL;
  530. resp_hdr = rx_ctx->resp_hdr;
  531. resp_hdr_daddr = rx_ctx->resp_hdr_daddr;
  532. dma_unmap_sg(dev, rx_ctx->dst_sg, sg_nents(rx_ctx->dst_sg),
  533. DMA_FROM_DEVICE);
  534. pdcs->rxin = (pdcs->rxin + num_frags) & pdcs->nrxpost;
  535. dev_dbg(dev, "PDC %u reclaimed %d rx descriptors",
  536. pdcs->pdc_idx, num_frags);
  537. dev_dbg(dev,
  538. "PDC %u txin %u, txout %u, rxin %u, rxout %u, last_rx_curr %u\n",
  539. pdcs->pdc_idx, pdcs->txin, pdcs->txout, pdcs->rxin,
  540. pdcs->rxout, pdcs->last_rx_curr);
  541. if (pdcs->pdc_resp_hdr_len == PDC_SPUM_RESP_HDR_LEN) {
  542. /*
  543. * For SPU-M, get length of response msg and rx overflow status.
  544. */
  545. rx_status = *((u32 *)resp_hdr);
  546. len = rx_status & RX_STATUS_LEN;
  547. dev_dbg(dev,
  548. "SPU response length %u bytes", len);
  549. if (unlikely(((rx_status & RX_STATUS_OVERFLOW) || (!len)))) {
  550. if (rx_status & RX_STATUS_OVERFLOW) {
  551. dev_err_ratelimited(dev,
  552. "crypto receive overflow");
  553. pdcs->rx_oflow++;
  554. } else {
  555. dev_info_ratelimited(dev, "crypto rx len = 0");
  556. }
  557. return -EIO;
  558. }
  559. }
  560. dma_pool_free(pdcs->rx_buf_pool, resp_hdr, resp_hdr_daddr);
  561. mbox_chan_received_data(chan, &mssg);
  562. pdcs->pdc_replies++;
  563. return PDC_SUCCESS;
  564. }
  565. /**
  566. * pdc_receive() - Process as many responses as are available in the rx ring.
  567. * @pdcs: PDC state
  568. *
  569. * Called within the hard IRQ.
  570. * Return:
  571. */
  572. static int
  573. pdc_receive(struct pdc_state *pdcs)
  574. {
  575. int rx_status;
  576. /* read last_rx_curr from register once */
  577. pdcs->last_rx_curr =
  578. (ioread32((const void __iomem *)&pdcs->rxregs_64->status0) &
  579. CRYPTO_D64_RS0_CD_MASK) / RING_ENTRY_SIZE;
  580. do {
  581. /* Could be many frames ready */
  582. rx_status = pdc_receive_one(pdcs);
  583. } while (rx_status == PDC_SUCCESS);
  584. return 0;
  585. }
  586. /**
  587. * pdc_tx_list_sg_add() - Add the buffers in a scatterlist to the transmit
  588. * descriptors for a given SPU. The scatterlist buffers contain the data for a
  589. * SPU request message.
  590. * @spu_idx: The index of the SPU to submit the request to, [0, max_spu)
  591. * @sg: Scatterlist whose buffers contain part of the SPU request
  592. *
  593. * If a scatterlist buffer is larger than PDC_DMA_BUF_MAX, multiple descriptors
  594. * are written for that buffer, each <= PDC_DMA_BUF_MAX byte in length.
  595. *
  596. * Return: PDC_SUCCESS if successful
  597. * < 0 otherwise
  598. */
  599. static int pdc_tx_list_sg_add(struct pdc_state *pdcs, struct scatterlist *sg)
  600. {
  601. u32 flags = 0;
  602. u32 eot;
  603. u32 tx_avail;
  604. /*
  605. * Num descriptors needed. Conservatively assume we need a descriptor
  606. * for every entry in sg.
  607. */
  608. u32 num_desc;
  609. u32 desc_w = 0; /* Number of tx descriptors written */
  610. u32 bufcnt; /* Number of bytes of buffer pointed to by descriptor */
  611. dma_addr_t databufptr; /* DMA address to put in descriptor */
  612. num_desc = (u32)sg_nents(sg);
  613. /* check whether enough tx descriptors are available */
  614. tx_avail = pdcs->ntxpost - NTXDACTIVE(pdcs->txin, pdcs->txout,
  615. pdcs->ntxpost);
  616. if (unlikely(num_desc > tx_avail)) {
  617. pdcs->txnobuf++;
  618. return -ENOSPC;
  619. }
  620. /* build tx descriptors */
  621. if (pdcs->tx_msg_start == pdcs->txout) {
  622. /* Start of frame */
  623. pdcs->txin_numd[pdcs->tx_msg_start] = 0;
  624. pdcs->src_sg[pdcs->txout] = sg;
  625. flags = D64_CTRL1_SOF;
  626. }
  627. while (sg) {
  628. if (unlikely(pdcs->txout == (pdcs->ntxd - 1)))
  629. eot = D64_CTRL1_EOT;
  630. else
  631. eot = 0;
  632. /*
  633. * If sg buffer larger than PDC limit, split across
  634. * multiple descriptors
  635. */
  636. bufcnt = sg_dma_len(sg);
  637. databufptr = sg_dma_address(sg);
  638. while (bufcnt > PDC_DMA_BUF_MAX) {
  639. pdc_build_txd(pdcs, databufptr, PDC_DMA_BUF_MAX,
  640. flags | eot);
  641. desc_w++;
  642. bufcnt -= PDC_DMA_BUF_MAX;
  643. databufptr += PDC_DMA_BUF_MAX;
  644. if (unlikely(pdcs->txout == (pdcs->ntxd - 1)))
  645. eot = D64_CTRL1_EOT;
  646. else
  647. eot = 0;
  648. }
  649. sg = sg_next(sg);
  650. if (!sg)
  651. /* Writing last descriptor for frame */
  652. flags |= (D64_CTRL1_EOF | D64_CTRL1_IOC);
  653. pdc_build_txd(pdcs, databufptr, bufcnt, flags | eot);
  654. desc_w++;
  655. /* Clear start of frame after first descriptor */
  656. flags &= ~D64_CTRL1_SOF;
  657. }
  658. pdcs->txin_numd[pdcs->tx_msg_start] += desc_w;
  659. return PDC_SUCCESS;
  660. }
  661. /**
  662. * pdc_tx_list_final() - Initiate DMA transfer of last frame written to tx
  663. * ring.
  664. * @pdcs: PDC state for SPU to process the request
  665. *
  666. * Sets the index of the last descriptor written in both the rx and tx ring.
  667. *
  668. * Return: PDC_SUCCESS
  669. */
  670. static int pdc_tx_list_final(struct pdc_state *pdcs)
  671. {
  672. /*
  673. * write barrier to ensure all register writes are complete
  674. * before chip starts to process new request
  675. */
  676. wmb();
  677. iowrite32(pdcs->rxout << 4, &pdcs->rxregs_64->ptr);
  678. iowrite32(pdcs->txout << 4, &pdcs->txregs_64->ptr);
  679. pdcs->pdc_requests++;
  680. return PDC_SUCCESS;
  681. }
  682. /**
  683. * pdc_rx_list_init() - Start a new receive descriptor list for a given PDC.
  684. * @pdcs: PDC state for SPU handling request
  685. * @dst_sg: scatterlist providing rx buffers for response to be returned to
  686. * mailbox client
  687. * @ctx: Opaque context for this request
  688. *
  689. * Posts a single receive descriptor to hold the metadata that precedes a
  690. * response. For example, with SPU-M, the metadata is a 32-byte DMA header and
  691. * an 8-byte BCM header. Moves the msg_start descriptor indexes for both tx and
  692. * rx to indicate the start of a new message.
  693. *
  694. * Return: PDC_SUCCESS if successful
  695. * < 0 if an error (e.g., rx ring is full)
  696. */
  697. static int pdc_rx_list_init(struct pdc_state *pdcs, struct scatterlist *dst_sg,
  698. void *ctx)
  699. {
  700. u32 flags = 0;
  701. u32 rx_avail;
  702. u32 rx_pkt_cnt = 1; /* Adding a single rx buffer */
  703. dma_addr_t daddr;
  704. void *vaddr;
  705. struct pdc_rx_ctx *rx_ctx;
  706. rx_avail = pdcs->nrxpost - NRXDACTIVE(pdcs->rxin, pdcs->rxout,
  707. pdcs->nrxpost);
  708. if (unlikely(rx_pkt_cnt > rx_avail)) {
  709. pdcs->rxnobuf++;
  710. return -ENOSPC;
  711. }
  712. /* allocate a buffer for the dma rx status */
  713. vaddr = dma_pool_zalloc(pdcs->rx_buf_pool, GFP_ATOMIC, &daddr);
  714. if (unlikely(!vaddr))
  715. return -ENOMEM;
  716. /*
  717. * Update msg_start indexes for both tx and rx to indicate the start
  718. * of a new sequence of descriptor indexes that contain the fragments
  719. * of the same message.
  720. */
  721. pdcs->rx_msg_start = pdcs->rxout;
  722. pdcs->tx_msg_start = pdcs->txout;
  723. /* This is always the first descriptor in the receive sequence */
  724. flags = D64_CTRL1_SOF;
  725. pdcs->rx_ctx[pdcs->rx_msg_start].rxin_numd = 1;
  726. if (unlikely(pdcs->rxout == (pdcs->nrxd - 1)))
  727. flags |= D64_CTRL1_EOT;
  728. rx_ctx = &pdcs->rx_ctx[pdcs->rxout];
  729. rx_ctx->rxp_ctx = ctx;
  730. rx_ctx->dst_sg = dst_sg;
  731. rx_ctx->resp_hdr = vaddr;
  732. rx_ctx->resp_hdr_daddr = daddr;
  733. pdc_build_rxd(pdcs, daddr, pdcs->pdc_resp_hdr_len, flags);
  734. return PDC_SUCCESS;
  735. }
  736. /**
  737. * pdc_rx_list_sg_add() - Add the buffers in a scatterlist to the receive
  738. * descriptors for a given SPU. The caller must have already DMA mapped the
  739. * scatterlist.
  740. * @spu_idx: Indicates which SPU the buffers are for
  741. * @sg: Scatterlist whose buffers are added to the receive ring
  742. *
  743. * If a receive buffer in the scatterlist is larger than PDC_DMA_BUF_MAX,
  744. * multiple receive descriptors are written, each with a buffer <=
  745. * PDC_DMA_BUF_MAX.
  746. *
  747. * Return: PDC_SUCCESS if successful
  748. * < 0 otherwise (e.g., receive ring is full)
  749. */
  750. static int pdc_rx_list_sg_add(struct pdc_state *pdcs, struct scatterlist *sg)
  751. {
  752. u32 flags = 0;
  753. u32 rx_avail;
  754. /*
  755. * Num descriptors needed. Conservatively assume we need a descriptor
  756. * for every entry from our starting point in the scatterlist.
  757. */
  758. u32 num_desc;
  759. u32 desc_w = 0; /* Number of tx descriptors written */
  760. u32 bufcnt; /* Number of bytes of buffer pointed to by descriptor */
  761. dma_addr_t databufptr; /* DMA address to put in descriptor */
  762. num_desc = (u32)sg_nents(sg);
  763. rx_avail = pdcs->nrxpost - NRXDACTIVE(pdcs->rxin, pdcs->rxout,
  764. pdcs->nrxpost);
  765. if (unlikely(num_desc > rx_avail)) {
  766. pdcs->rxnobuf++;
  767. return -ENOSPC;
  768. }
  769. while (sg) {
  770. if (unlikely(pdcs->rxout == (pdcs->nrxd - 1)))
  771. flags = D64_CTRL1_EOT;
  772. else
  773. flags = 0;
  774. /*
  775. * If sg buffer larger than PDC limit, split across
  776. * multiple descriptors
  777. */
  778. bufcnt = sg_dma_len(sg);
  779. databufptr = sg_dma_address(sg);
  780. while (bufcnt > PDC_DMA_BUF_MAX) {
  781. pdc_build_rxd(pdcs, databufptr, PDC_DMA_BUF_MAX, flags);
  782. desc_w++;
  783. bufcnt -= PDC_DMA_BUF_MAX;
  784. databufptr += PDC_DMA_BUF_MAX;
  785. if (unlikely(pdcs->rxout == (pdcs->nrxd - 1)))
  786. flags = D64_CTRL1_EOT;
  787. else
  788. flags = 0;
  789. }
  790. pdc_build_rxd(pdcs, databufptr, bufcnt, flags);
  791. desc_w++;
  792. sg = sg_next(sg);
  793. }
  794. pdcs->rx_ctx[pdcs->rx_msg_start].rxin_numd += desc_w;
  795. return PDC_SUCCESS;
  796. }
  797. /**
  798. * pdc_irq_handler() - Interrupt handler called in interrupt context.
  799. * @irq: Interrupt number that has fired
  800. * @data: device struct for DMA engine that generated the interrupt
  801. *
  802. * We have to clear the device interrupt status flags here. So cache the
  803. * status for later use in the thread function. Other than that, just return
  804. * WAKE_THREAD to invoke the thread function.
  805. *
  806. * Return: IRQ_WAKE_THREAD if interrupt is ours
  807. * IRQ_NONE otherwise
  808. */
  809. static irqreturn_t pdc_irq_handler(int irq, void *data)
  810. {
  811. struct device *dev = (struct device *)data;
  812. struct pdc_state *pdcs = dev_get_drvdata(dev);
  813. u32 intstatus = ioread32(pdcs->pdc_reg_vbase + PDC_INTSTATUS_OFFSET);
  814. if (unlikely(intstatus == 0))
  815. return IRQ_NONE;
  816. /* Disable interrupts until soft handler runs */
  817. iowrite32(0, pdcs->pdc_reg_vbase + PDC_INTMASK_OFFSET);
  818. /* Clear interrupt flags in device */
  819. iowrite32(intstatus, pdcs->pdc_reg_vbase + PDC_INTSTATUS_OFFSET);
  820. /* Wakeup IRQ thread */
  821. tasklet_schedule(&pdcs->rx_tasklet);
  822. return IRQ_HANDLED;
  823. }
  824. /**
  825. * pdc_tasklet_cb() - Tasklet callback that runs the deferred processing after
  826. * a DMA receive interrupt. Reenables the receive interrupt.
  827. * @data: PDC state structure
  828. */
  829. static void pdc_tasklet_cb(struct tasklet_struct *t)
  830. {
  831. struct pdc_state *pdcs = from_tasklet(pdcs, t, rx_tasklet);
  832. pdc_receive(pdcs);
  833. /* reenable interrupts */
  834. iowrite32(PDC_INTMASK, pdcs->pdc_reg_vbase + PDC_INTMASK_OFFSET);
  835. }
  836. /**
  837. * pdc_ring_init() - Allocate DMA rings and initialize constant fields of
  838. * descriptors in one ringset.
  839. * @pdcs: PDC instance state
  840. * @ringset: index of ringset being used
  841. *
  842. * Return: PDC_SUCCESS if ring initialized
  843. * < 0 otherwise
  844. */
  845. static int pdc_ring_init(struct pdc_state *pdcs, int ringset)
  846. {
  847. int i;
  848. int err = PDC_SUCCESS;
  849. struct dma64 *dma_reg;
  850. struct device *dev = &pdcs->pdev->dev;
  851. struct pdc_ring_alloc tx;
  852. struct pdc_ring_alloc rx;
  853. /* Allocate tx ring */
  854. tx.vbase = dma_pool_zalloc(pdcs->ring_pool, GFP_KERNEL, &tx.dmabase);
  855. if (unlikely(!tx.vbase)) {
  856. err = -ENOMEM;
  857. goto done;
  858. }
  859. /* Allocate rx ring */
  860. rx.vbase = dma_pool_zalloc(pdcs->ring_pool, GFP_KERNEL, &rx.dmabase);
  861. if (unlikely(!rx.vbase)) {
  862. err = -ENOMEM;
  863. goto fail_dealloc;
  864. }
  865. dev_dbg(dev, " - base DMA addr of tx ring %pad", &tx.dmabase);
  866. dev_dbg(dev, " - base virtual addr of tx ring %p", tx.vbase);
  867. dev_dbg(dev, " - base DMA addr of rx ring %pad", &rx.dmabase);
  868. dev_dbg(dev, " - base virtual addr of rx ring %p", rx.vbase);
  869. memcpy(&pdcs->tx_ring_alloc, &tx, sizeof(tx));
  870. memcpy(&pdcs->rx_ring_alloc, &rx, sizeof(rx));
  871. pdcs->rxin = 0;
  872. pdcs->rx_msg_start = 0;
  873. pdcs->last_rx_curr = 0;
  874. pdcs->rxout = 0;
  875. pdcs->txin = 0;
  876. pdcs->tx_msg_start = 0;
  877. pdcs->txout = 0;
  878. /* Set descriptor array base addresses */
  879. pdcs->txd_64 = (struct dma64dd *)pdcs->tx_ring_alloc.vbase;
  880. pdcs->rxd_64 = (struct dma64dd *)pdcs->rx_ring_alloc.vbase;
  881. /* Tell device the base DMA address of each ring */
  882. dma_reg = &pdcs->regs->dmaregs[ringset];
  883. /* But first disable DMA and set curptr to 0 for both TX & RX */
  884. iowrite32(PDC_TX_CTL, &dma_reg->dmaxmt.control);
  885. iowrite32((PDC_RX_CTL + (pdcs->rx_status_len << 1)),
  886. &dma_reg->dmarcv.control);
  887. iowrite32(0, &dma_reg->dmaxmt.ptr);
  888. iowrite32(0, &dma_reg->dmarcv.ptr);
  889. /* Set base DMA addresses */
  890. iowrite32(lower_32_bits(pdcs->tx_ring_alloc.dmabase),
  891. &dma_reg->dmaxmt.addrlow);
  892. iowrite32(upper_32_bits(pdcs->tx_ring_alloc.dmabase),
  893. &dma_reg->dmaxmt.addrhigh);
  894. iowrite32(lower_32_bits(pdcs->rx_ring_alloc.dmabase),
  895. &dma_reg->dmarcv.addrlow);
  896. iowrite32(upper_32_bits(pdcs->rx_ring_alloc.dmabase),
  897. &dma_reg->dmarcv.addrhigh);
  898. /* Re-enable DMA */
  899. iowrite32(PDC_TX_CTL | PDC_TX_ENABLE, &dma_reg->dmaxmt.control);
  900. iowrite32((PDC_RX_CTL | PDC_RX_ENABLE | (pdcs->rx_status_len << 1)),
  901. &dma_reg->dmarcv.control);
  902. /* Initialize descriptors */
  903. for (i = 0; i < PDC_RING_ENTRIES; i++) {
  904. /* Every tx descriptor can be used for start of frame. */
  905. if (i != pdcs->ntxpost) {
  906. iowrite32(D64_CTRL1_SOF | D64_CTRL1_EOF,
  907. &pdcs->txd_64[i].ctrl1);
  908. } else {
  909. /* Last descriptor in ringset. Set End of Table. */
  910. iowrite32(D64_CTRL1_SOF | D64_CTRL1_EOF |
  911. D64_CTRL1_EOT, &pdcs->txd_64[i].ctrl1);
  912. }
  913. /* Every rx descriptor can be used for start of frame */
  914. if (i != pdcs->nrxpost) {
  915. iowrite32(D64_CTRL1_SOF,
  916. &pdcs->rxd_64[i].ctrl1);
  917. } else {
  918. /* Last descriptor in ringset. Set End of Table. */
  919. iowrite32(D64_CTRL1_SOF | D64_CTRL1_EOT,
  920. &pdcs->rxd_64[i].ctrl1);
  921. }
  922. }
  923. return PDC_SUCCESS;
  924. fail_dealloc:
  925. dma_pool_free(pdcs->ring_pool, tx.vbase, tx.dmabase);
  926. done:
  927. return err;
  928. }
  929. static void pdc_ring_free(struct pdc_state *pdcs)
  930. {
  931. if (pdcs->tx_ring_alloc.vbase) {
  932. dma_pool_free(pdcs->ring_pool, pdcs->tx_ring_alloc.vbase,
  933. pdcs->tx_ring_alloc.dmabase);
  934. pdcs->tx_ring_alloc.vbase = NULL;
  935. }
  936. if (pdcs->rx_ring_alloc.vbase) {
  937. dma_pool_free(pdcs->ring_pool, pdcs->rx_ring_alloc.vbase,
  938. pdcs->rx_ring_alloc.dmabase);
  939. pdcs->rx_ring_alloc.vbase = NULL;
  940. }
  941. }
  942. /**
  943. * pdc_desc_count() - Count the number of DMA descriptors that will be required
  944. * for a given scatterlist. Account for the max length of a DMA buffer.
  945. * @sg: Scatterlist to be DMA'd
  946. * Return: Number of descriptors required
  947. */
  948. static u32 pdc_desc_count(struct scatterlist *sg)
  949. {
  950. u32 cnt = 0;
  951. while (sg) {
  952. cnt += ((sg->length / PDC_DMA_BUF_MAX) + 1);
  953. sg = sg_next(sg);
  954. }
  955. return cnt;
  956. }
  957. /**
  958. * pdc_rings_full() - Check whether the tx ring has room for tx_cnt descriptors
  959. * and the rx ring has room for rx_cnt descriptors.
  960. * @pdcs: PDC state
  961. * @tx_cnt: The number of descriptors required in the tx ring
  962. * @rx_cnt: The number of descriptors required i the rx ring
  963. *
  964. * Return: true if one of the rings does not have enough space
  965. * false if sufficient space is available in both rings
  966. */
  967. static bool pdc_rings_full(struct pdc_state *pdcs, int tx_cnt, int rx_cnt)
  968. {
  969. u32 rx_avail;
  970. u32 tx_avail;
  971. bool full = false;
  972. /* Check if the tx and rx rings are likely to have enough space */
  973. rx_avail = pdcs->nrxpost - NRXDACTIVE(pdcs->rxin, pdcs->rxout,
  974. pdcs->nrxpost);
  975. if (unlikely(rx_cnt > rx_avail)) {
  976. pdcs->rx_ring_full++;
  977. full = true;
  978. }
  979. if (likely(!full)) {
  980. tx_avail = pdcs->ntxpost - NTXDACTIVE(pdcs->txin, pdcs->txout,
  981. pdcs->ntxpost);
  982. if (unlikely(tx_cnt > tx_avail)) {
  983. pdcs->tx_ring_full++;
  984. full = true;
  985. }
  986. }
  987. return full;
  988. }
  989. /**
  990. * pdc_last_tx_done() - If both the tx and rx rings have at least
  991. * PDC_RING_SPACE_MIN descriptors available, then indicate that the mailbox
  992. * framework can submit another message.
  993. * @chan: mailbox channel to check
  994. * Return: true if PDC can accept another message on this channel
  995. */
  996. static bool pdc_last_tx_done(struct mbox_chan *chan)
  997. {
  998. struct pdc_state *pdcs = chan->con_priv;
  999. bool ret;
  1000. if (unlikely(pdc_rings_full(pdcs, PDC_RING_SPACE_MIN,
  1001. PDC_RING_SPACE_MIN))) {
  1002. pdcs->last_tx_not_done++;
  1003. ret = false;
  1004. } else {
  1005. ret = true;
  1006. }
  1007. return ret;
  1008. }
  1009. /**
  1010. * pdc_send_data() - mailbox send_data function
  1011. * @chan: The mailbox channel on which the data is sent. The channel
  1012. * corresponds to a DMA ringset.
  1013. * @data: The mailbox message to be sent. The message must be a
  1014. * brcm_message structure.
  1015. *
  1016. * This function is registered as the send_data function for the mailbox
  1017. * controller. From the destination scatterlist in the mailbox message, it
  1018. * creates a sequence of receive descriptors in the rx ring. From the source
  1019. * scatterlist, it creates a sequence of transmit descriptors in the tx ring.
  1020. * After creating the descriptors, it writes the rx ptr and tx ptr registers to
  1021. * initiate the DMA transfer.
  1022. *
  1023. * This function does the DMA map and unmap of the src and dst scatterlists in
  1024. * the mailbox message.
  1025. *
  1026. * Return: 0 if successful
  1027. * -ENOTSUPP if the mailbox message is a type this driver does not
  1028. * support
  1029. * < 0 if an error
  1030. */
  1031. static int pdc_send_data(struct mbox_chan *chan, void *data)
  1032. {
  1033. struct pdc_state *pdcs = chan->con_priv;
  1034. struct device *dev = &pdcs->pdev->dev;
  1035. struct brcm_message *mssg = data;
  1036. int err = PDC_SUCCESS;
  1037. int src_nent;
  1038. int dst_nent;
  1039. int nent;
  1040. u32 tx_desc_req;
  1041. u32 rx_desc_req;
  1042. if (unlikely(mssg->type != BRCM_MESSAGE_SPU))
  1043. return -ENOTSUPP;
  1044. src_nent = sg_nents(mssg->spu.src);
  1045. if (likely(src_nent)) {
  1046. nent = dma_map_sg(dev, mssg->spu.src, src_nent, DMA_TO_DEVICE);
  1047. if (unlikely(nent == 0))
  1048. return -EIO;
  1049. }
  1050. dst_nent = sg_nents(mssg->spu.dst);
  1051. if (likely(dst_nent)) {
  1052. nent = dma_map_sg(dev, mssg->spu.dst, dst_nent,
  1053. DMA_FROM_DEVICE);
  1054. if (unlikely(nent == 0)) {
  1055. dma_unmap_sg(dev, mssg->spu.src, src_nent,
  1056. DMA_TO_DEVICE);
  1057. return -EIO;
  1058. }
  1059. }
  1060. /*
  1061. * Check if the tx and rx rings have enough space. Do this prior to
  1062. * writing any tx or rx descriptors. Need to ensure that we do not write
  1063. * a partial set of descriptors, or write just rx descriptors but
  1064. * corresponding tx descriptors don't fit. Note that we want this check
  1065. * and the entire sequence of descriptor to happen without another
  1066. * thread getting in. The channel spin lock in the mailbox framework
  1067. * ensures this.
  1068. */
  1069. tx_desc_req = pdc_desc_count(mssg->spu.src);
  1070. rx_desc_req = pdc_desc_count(mssg->spu.dst);
  1071. if (unlikely(pdc_rings_full(pdcs, tx_desc_req, rx_desc_req + 1)))
  1072. return -ENOSPC;
  1073. /* Create rx descriptors to SPU catch response */
  1074. err = pdc_rx_list_init(pdcs, mssg->spu.dst, mssg->ctx);
  1075. err |= pdc_rx_list_sg_add(pdcs, mssg->spu.dst);
  1076. /* Create tx descriptors to submit SPU request */
  1077. err |= pdc_tx_list_sg_add(pdcs, mssg->spu.src);
  1078. err |= pdc_tx_list_final(pdcs); /* initiate transfer */
  1079. if (unlikely(err))
  1080. dev_err(&pdcs->pdev->dev,
  1081. "%s failed with error %d", __func__, err);
  1082. return err;
  1083. }
  1084. static int pdc_startup(struct mbox_chan *chan)
  1085. {
  1086. return pdc_ring_init(chan->con_priv, PDC_RINGSET);
  1087. }
  1088. static void pdc_shutdown(struct mbox_chan *chan)
  1089. {
  1090. struct pdc_state *pdcs = chan->con_priv;
  1091. if (!pdcs)
  1092. return;
  1093. dev_dbg(&pdcs->pdev->dev,
  1094. "Shutdown mailbox channel for PDC %u", pdcs->pdc_idx);
  1095. pdc_ring_free(pdcs);
  1096. }
  1097. /**
  1098. * pdc_hw_init() - Use the given initialization parameters to initialize the
  1099. * state for one of the PDCs.
  1100. * @pdcs: state of the PDC
  1101. */
  1102. static
  1103. void pdc_hw_init(struct pdc_state *pdcs)
  1104. {
  1105. struct platform_device *pdev;
  1106. struct device *dev;
  1107. struct dma64 *dma_reg;
  1108. int ringset = PDC_RINGSET;
  1109. pdev = pdcs->pdev;
  1110. dev = &pdev->dev;
  1111. dev_dbg(dev, "PDC %u initial values:", pdcs->pdc_idx);
  1112. dev_dbg(dev, "state structure: %p",
  1113. pdcs);
  1114. dev_dbg(dev, " - base virtual addr of hw regs %p",
  1115. pdcs->pdc_reg_vbase);
  1116. /* initialize data structures */
  1117. pdcs->regs = (struct pdc_regs *)pdcs->pdc_reg_vbase;
  1118. pdcs->txregs_64 = (struct dma64_regs *)
  1119. (((u8 *)pdcs->pdc_reg_vbase) +
  1120. PDC_TXREGS_OFFSET + (sizeof(struct dma64) * ringset));
  1121. pdcs->rxregs_64 = (struct dma64_regs *)
  1122. (((u8 *)pdcs->pdc_reg_vbase) +
  1123. PDC_RXREGS_OFFSET + (sizeof(struct dma64) * ringset));
  1124. pdcs->ntxd = PDC_RING_ENTRIES;
  1125. pdcs->nrxd = PDC_RING_ENTRIES;
  1126. pdcs->ntxpost = PDC_RING_ENTRIES - 1;
  1127. pdcs->nrxpost = PDC_RING_ENTRIES - 1;
  1128. iowrite32(0, &pdcs->regs->intmask);
  1129. dma_reg = &pdcs->regs->dmaregs[ringset];
  1130. /* Configure DMA but will enable later in pdc_ring_init() */
  1131. iowrite32(PDC_TX_CTL, &dma_reg->dmaxmt.control);
  1132. iowrite32(PDC_RX_CTL + (pdcs->rx_status_len << 1),
  1133. &dma_reg->dmarcv.control);
  1134. /* Reset current index pointers after making sure DMA is disabled */
  1135. iowrite32(0, &dma_reg->dmaxmt.ptr);
  1136. iowrite32(0, &dma_reg->dmarcv.ptr);
  1137. if (pdcs->pdc_resp_hdr_len == PDC_SPU2_RESP_HDR_LEN)
  1138. iowrite32(PDC_CKSUM_CTRL,
  1139. pdcs->pdc_reg_vbase + PDC_CKSUM_CTRL_OFFSET);
  1140. }
  1141. /**
  1142. * pdc_hw_disable() - Disable the tx and rx control in the hw.
  1143. * @pdcs: PDC state structure
  1144. *
  1145. */
  1146. static void pdc_hw_disable(struct pdc_state *pdcs)
  1147. {
  1148. struct dma64 *dma_reg;
  1149. dma_reg = &pdcs->regs->dmaregs[PDC_RINGSET];
  1150. iowrite32(PDC_TX_CTL, &dma_reg->dmaxmt.control);
  1151. iowrite32(PDC_RX_CTL + (pdcs->rx_status_len << 1),
  1152. &dma_reg->dmarcv.control);
  1153. }
  1154. /**
  1155. * pdc_rx_buf_pool_create() - Pool of receive buffers used to catch the metadata
  1156. * header returned with each response message.
  1157. * @pdcs: PDC state structure
  1158. *
  1159. * The metadata is not returned to the mailbox client. So the PDC driver
  1160. * manages these buffers.
  1161. *
  1162. * Return: PDC_SUCCESS
  1163. * -ENOMEM if pool creation fails
  1164. */
  1165. static int pdc_rx_buf_pool_create(struct pdc_state *pdcs)
  1166. {
  1167. struct platform_device *pdev;
  1168. struct device *dev;
  1169. pdev = pdcs->pdev;
  1170. dev = &pdev->dev;
  1171. pdcs->pdc_resp_hdr_len = pdcs->rx_status_len;
  1172. if (pdcs->use_bcm_hdr)
  1173. pdcs->pdc_resp_hdr_len += BCM_HDR_LEN;
  1174. pdcs->rx_buf_pool = dma_pool_create("pdc rx bufs", dev,
  1175. pdcs->pdc_resp_hdr_len,
  1176. RX_BUF_ALIGN, 0);
  1177. if (!pdcs->rx_buf_pool)
  1178. return -ENOMEM;
  1179. return PDC_SUCCESS;
  1180. }
  1181. /**
  1182. * pdc_interrupts_init() - Initialize the interrupt configuration for a PDC and
  1183. * specify a threaded IRQ handler for deferred handling of interrupts outside of
  1184. * interrupt context.
  1185. * @pdcs: PDC state
  1186. *
  1187. * Set the interrupt mask for transmit and receive done.
  1188. * Set the lazy interrupt frame count to generate an interrupt for just one pkt.
  1189. *
  1190. * Return: PDC_SUCCESS
  1191. * <0 if threaded irq request fails
  1192. */
  1193. static int pdc_interrupts_init(struct pdc_state *pdcs)
  1194. {
  1195. struct platform_device *pdev = pdcs->pdev;
  1196. struct device *dev = &pdev->dev;
  1197. struct device_node *dn = pdev->dev.of_node;
  1198. int err;
  1199. /* interrupt configuration */
  1200. iowrite32(PDC_INTMASK, pdcs->pdc_reg_vbase + PDC_INTMASK_OFFSET);
  1201. if (pdcs->hw_type == FA_HW)
  1202. iowrite32(PDC_LAZY_INT, pdcs->pdc_reg_vbase +
  1203. FA_RCVLAZY0_OFFSET);
  1204. else
  1205. iowrite32(PDC_LAZY_INT, pdcs->pdc_reg_vbase +
  1206. PDC_RCVLAZY0_OFFSET);
  1207. /* read irq from device tree */
  1208. pdcs->pdc_irq = irq_of_parse_and_map(dn, 0);
  1209. dev_dbg(dev, "pdc device %s irq %u for pdcs %p",
  1210. dev_name(dev), pdcs->pdc_irq, pdcs);
  1211. err = devm_request_irq(dev, pdcs->pdc_irq, pdc_irq_handler, 0,
  1212. dev_name(dev), dev);
  1213. if (err) {
  1214. dev_err(dev, "IRQ %u request failed with err %d\n",
  1215. pdcs->pdc_irq, err);
  1216. return err;
  1217. }
  1218. return PDC_SUCCESS;
  1219. }
  1220. static const struct mbox_chan_ops pdc_mbox_chan_ops = {
  1221. .send_data = pdc_send_data,
  1222. .last_tx_done = pdc_last_tx_done,
  1223. .startup = pdc_startup,
  1224. .shutdown = pdc_shutdown
  1225. };
  1226. /**
  1227. * pdc_mb_init() - Initialize the mailbox controller.
  1228. * @pdcs: PDC state
  1229. *
  1230. * Each PDC is a mailbox controller. Each ringset is a mailbox channel. Kernel
  1231. * driver only uses one ringset and thus one mb channel. PDC uses the transmit
  1232. * complete interrupt to determine when a mailbox message has successfully been
  1233. * transmitted.
  1234. *
  1235. * Return: 0 on success
  1236. * < 0 if there is an allocation or registration failure
  1237. */
  1238. static int pdc_mb_init(struct pdc_state *pdcs)
  1239. {
  1240. struct device *dev = &pdcs->pdev->dev;
  1241. struct mbox_controller *mbc;
  1242. int chan_index;
  1243. int err;
  1244. mbc = &pdcs->mbc;
  1245. mbc->dev = dev;
  1246. mbc->ops = &pdc_mbox_chan_ops;
  1247. mbc->num_chans = 1;
  1248. mbc->chans = devm_kcalloc(dev, mbc->num_chans, sizeof(*mbc->chans),
  1249. GFP_KERNEL);
  1250. if (!mbc->chans)
  1251. return -ENOMEM;
  1252. mbc->txdone_irq = false;
  1253. mbc->txdone_poll = true;
  1254. mbc->txpoll_period = 1;
  1255. for (chan_index = 0; chan_index < mbc->num_chans; chan_index++)
  1256. mbc->chans[chan_index].con_priv = pdcs;
  1257. /* Register mailbox controller */
  1258. err = devm_mbox_controller_register(dev, mbc);
  1259. if (err) {
  1260. dev_crit(dev,
  1261. "Failed to register PDC mailbox controller. Error %d.",
  1262. err);
  1263. return err;
  1264. }
  1265. return 0;
  1266. }
  1267. /* Device tree API */
  1268. static const int pdc_hw = PDC_HW;
  1269. static const int fa_hw = FA_HW;
  1270. static const struct of_device_id pdc_mbox_of_match[] = {
  1271. {.compatible = "brcm,iproc-pdc-mbox", .data = &pdc_hw},
  1272. {.compatible = "brcm,iproc-fa2-mbox", .data = &fa_hw},
  1273. { /* sentinel */ }
  1274. };
  1275. MODULE_DEVICE_TABLE(of, pdc_mbox_of_match);
  1276. /**
  1277. * pdc_dt_read() - Read application-specific data from device tree.
  1278. * @pdev: Platform device
  1279. * @pdcs: PDC state
  1280. *
  1281. * Reads the number of bytes of receive status that precede each received frame.
  1282. * Reads whether transmit and received frames should be preceded by an 8-byte
  1283. * BCM header.
  1284. *
  1285. * Return: 0 if successful
  1286. * -ENODEV if device not available
  1287. */
  1288. static int pdc_dt_read(struct platform_device *pdev, struct pdc_state *pdcs)
  1289. {
  1290. struct device *dev = &pdev->dev;
  1291. struct device_node *dn = pdev->dev.of_node;
  1292. const struct of_device_id *match;
  1293. const int *hw_type;
  1294. int err;
  1295. err = of_property_read_u32(dn, "brcm,rx-status-len",
  1296. &pdcs->rx_status_len);
  1297. if (err < 0)
  1298. dev_err(dev,
  1299. "%s failed to get DMA receive status length from device tree",
  1300. __func__);
  1301. pdcs->use_bcm_hdr = of_property_read_bool(dn, "brcm,use-bcm-hdr");
  1302. pdcs->hw_type = PDC_HW;
  1303. match = of_match_device(of_match_ptr(pdc_mbox_of_match), dev);
  1304. if (match != NULL) {
  1305. hw_type = match->data;
  1306. pdcs->hw_type = *hw_type;
  1307. }
  1308. return 0;
  1309. }
  1310. /**
  1311. * pdc_probe() - Probe function for PDC driver.
  1312. * @pdev: PDC platform device
  1313. *
  1314. * Reserve and map register regions defined in device tree.
  1315. * Allocate and initialize tx and rx DMA rings.
  1316. * Initialize a mailbox controller for each PDC.
  1317. *
  1318. * Return: 0 if successful
  1319. * < 0 if an error
  1320. */
  1321. static int pdc_probe(struct platform_device *pdev)
  1322. {
  1323. int err = 0;
  1324. struct device *dev = &pdev->dev;
  1325. struct resource *pdc_regs;
  1326. struct pdc_state *pdcs;
  1327. /* PDC state for one SPU */
  1328. pdcs = devm_kzalloc(dev, sizeof(*pdcs), GFP_KERNEL);
  1329. if (!pdcs) {
  1330. err = -ENOMEM;
  1331. goto cleanup;
  1332. }
  1333. pdcs->pdev = pdev;
  1334. platform_set_drvdata(pdev, pdcs);
  1335. pdcs->pdc_idx = pdcg.num_spu;
  1336. pdcg.num_spu++;
  1337. err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(39));
  1338. if (err) {
  1339. dev_warn(dev, "PDC device cannot perform DMA. Error %d.", err);
  1340. goto cleanup;
  1341. }
  1342. /* Create DMA pool for tx ring */
  1343. pdcs->ring_pool = dma_pool_create("pdc rings", dev, PDC_RING_SIZE,
  1344. RING_ALIGN, 0);
  1345. if (!pdcs->ring_pool) {
  1346. err = -ENOMEM;
  1347. goto cleanup;
  1348. }
  1349. err = pdc_dt_read(pdev, pdcs);
  1350. if (err)
  1351. goto cleanup_ring_pool;
  1352. pdc_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1353. if (!pdc_regs) {
  1354. err = -ENODEV;
  1355. goto cleanup_ring_pool;
  1356. }
  1357. dev_dbg(dev, "PDC register region res.start = %pa, res.end = %pa",
  1358. &pdc_regs->start, &pdc_regs->end);
  1359. pdcs->pdc_reg_vbase = devm_ioremap_resource(&pdev->dev, pdc_regs);
  1360. if (IS_ERR(pdcs->pdc_reg_vbase)) {
  1361. err = PTR_ERR(pdcs->pdc_reg_vbase);
  1362. dev_err(&pdev->dev, "Failed to map registers: %d\n", err);
  1363. goto cleanup_ring_pool;
  1364. }
  1365. /* create rx buffer pool after dt read to know how big buffers are */
  1366. err = pdc_rx_buf_pool_create(pdcs);
  1367. if (err)
  1368. goto cleanup_ring_pool;
  1369. pdc_hw_init(pdcs);
  1370. /* Init tasklet for deferred DMA rx processing */
  1371. tasklet_setup(&pdcs->rx_tasklet, pdc_tasklet_cb);
  1372. err = pdc_interrupts_init(pdcs);
  1373. if (err)
  1374. goto cleanup_buf_pool;
  1375. /* Initialize mailbox controller */
  1376. err = pdc_mb_init(pdcs);
  1377. if (err)
  1378. goto cleanup_buf_pool;
  1379. pdc_setup_debugfs(pdcs);
  1380. dev_dbg(dev, "pdc_probe() successful");
  1381. return PDC_SUCCESS;
  1382. cleanup_buf_pool:
  1383. tasklet_kill(&pdcs->rx_tasklet);
  1384. dma_pool_destroy(pdcs->rx_buf_pool);
  1385. cleanup_ring_pool:
  1386. dma_pool_destroy(pdcs->ring_pool);
  1387. cleanup:
  1388. return err;
  1389. }
  1390. static int pdc_remove(struct platform_device *pdev)
  1391. {
  1392. struct pdc_state *pdcs = platform_get_drvdata(pdev);
  1393. pdc_free_debugfs();
  1394. tasklet_kill(&pdcs->rx_tasklet);
  1395. pdc_hw_disable(pdcs);
  1396. dma_pool_destroy(pdcs->rx_buf_pool);
  1397. dma_pool_destroy(pdcs->ring_pool);
  1398. return 0;
  1399. }
  1400. static struct platform_driver pdc_mbox_driver = {
  1401. .probe = pdc_probe,
  1402. .remove = pdc_remove,
  1403. .driver = {
  1404. .name = "brcm-iproc-pdc-mbox",
  1405. .of_match_table = of_match_ptr(pdc_mbox_of_match),
  1406. },
  1407. };
  1408. module_platform_driver(pdc_mbox_driver);
  1409. MODULE_AUTHOR("Rob Rice <rob.rice@broadcom.com>");
  1410. MODULE_DESCRIPTION("Broadcom PDC mailbox driver");
  1411. MODULE_LICENSE("GPL v2");