bcm-flexrm-mailbox.c 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704
  1. /*
  2. * Copyright (C) 2017 Broadcom
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public License as
  6. * published by the Free Software Foundation version 2.
  7. *
  8. * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  9. * kind, whether express or implied; without even the implied warranty
  10. * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU General Public License for more details.
  12. */
  13. /*
  14. * Broadcom FlexRM Mailbox Driver
  15. *
  16. * Each Broadcom FlexSparx4 offload engine is implemented as an
  17. * extension to Broadcom FlexRM ring manager. The FlexRM ring
  18. * manager provides a set of rings which can be used to submit
  19. * work to a FlexSparx4 offload engine.
  20. *
  21. * This driver creates a mailbox controller using a set of FlexRM
  22. * rings where each mailbox channel represents a separate FlexRM ring.
  23. */
  24. #include <asm/barrier.h>
  25. #include <asm/byteorder.h>
  26. #include <linux/atomic.h>
  27. #include <linux/bitmap.h>
  28. #include <linux/debugfs.h>
  29. #include <linux/delay.h>
  30. #include <linux/device.h>
  31. #include <linux/dma-mapping.h>
  32. #include <linux/dmapool.h>
  33. #include <linux/err.h>
  34. #include <linux/interrupt.h>
  35. #include <linux/kernel.h>
  36. #include <linux/mailbox_controller.h>
  37. #include <linux/mailbox_client.h>
  38. #include <linux/mailbox/brcm-message.h>
  39. #include <linux/module.h>
  40. #include <linux/msi.h>
  41. #include <linux/of_address.h>
  42. #include <linux/of_irq.h>
  43. #include <linux/platform_device.h>
  44. #include <linux/spinlock.h>
  45. /* ====== FlexRM register defines ===== */
  46. /* FlexRM configuration */
  47. #define RING_REGS_SIZE 0x10000
  48. #define RING_DESC_SIZE 8
  49. #define RING_DESC_INDEX(offset) \
  50. ((offset) / RING_DESC_SIZE)
  51. #define RING_DESC_OFFSET(index) \
  52. ((index) * RING_DESC_SIZE)
  53. #define RING_MAX_REQ_COUNT 1024
  54. #define RING_BD_ALIGN_ORDER 12
  55. #define RING_BD_ALIGN_CHECK(addr) \
  56. (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
  57. #define RING_BD_TOGGLE_INVALID(offset) \
  58. (((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
  59. #define RING_BD_TOGGLE_VALID(offset) \
  60. (!RING_BD_TOGGLE_INVALID(offset))
  61. #define RING_BD_DESC_PER_REQ 32
  62. #define RING_BD_DESC_COUNT \
  63. (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
  64. #define RING_BD_SIZE \
  65. (RING_BD_DESC_COUNT * RING_DESC_SIZE)
  66. #define RING_CMPL_ALIGN_ORDER 13
  67. #define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT
  68. #define RING_CMPL_SIZE \
  69. (RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
  70. #define RING_VER_MAGIC 0x76303031
  71. /* Per-Ring register offsets */
  72. #define RING_VER 0x000
  73. #define RING_BD_START_ADDR 0x004
  74. #define RING_BD_READ_PTR 0x008
  75. #define RING_BD_WRITE_PTR 0x00c
  76. #define RING_BD_READ_PTR_DDR_LS 0x010
  77. #define RING_BD_READ_PTR_DDR_MS 0x014
  78. #define RING_CMPL_START_ADDR 0x018
  79. #define RING_CMPL_WRITE_PTR 0x01c
  80. #define RING_NUM_REQ_RECV_LS 0x020
  81. #define RING_NUM_REQ_RECV_MS 0x024
  82. #define RING_NUM_REQ_TRANS_LS 0x028
  83. #define RING_NUM_REQ_TRANS_MS 0x02c
  84. #define RING_NUM_REQ_OUTSTAND 0x030
  85. #define RING_CONTROL 0x034
  86. #define RING_FLUSH_DONE 0x038
  87. #define RING_MSI_ADDR_LS 0x03c
  88. #define RING_MSI_ADDR_MS 0x040
  89. #define RING_MSI_CONTROL 0x048
  90. #define RING_BD_READ_PTR_DDR_CONTROL 0x04c
  91. #define RING_MSI_DATA_VALUE 0x064
  92. /* Register RING_BD_START_ADDR fields */
  93. #define BD_LAST_UPDATE_HW_SHIFT 28
  94. #define BD_LAST_UPDATE_HW_MASK 0x1
  95. #define BD_START_ADDR_VALUE(pa) \
  96. ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
  97. #define BD_START_ADDR_DECODE(val) \
  98. ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
  99. /* Register RING_CMPL_START_ADDR fields */
  100. #define CMPL_START_ADDR_VALUE(pa) \
  101. ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
  102. /* Register RING_CONTROL fields */
  103. #define CONTROL_MASK_DISABLE_CONTROL 12
  104. #define CONTROL_FLUSH_SHIFT 5
  105. #define CONTROL_ACTIVE_SHIFT 4
  106. #define CONTROL_RATE_ADAPT_MASK 0xf
  107. #define CONTROL_RATE_DYNAMIC 0x0
  108. #define CONTROL_RATE_FAST 0x8
  109. #define CONTROL_RATE_MEDIUM 0x9
  110. #define CONTROL_RATE_SLOW 0xa
  111. #define CONTROL_RATE_IDLE 0xb
  112. /* Register RING_FLUSH_DONE fields */
  113. #define FLUSH_DONE_MASK 0x1
  114. /* Register RING_MSI_CONTROL fields */
  115. #define MSI_TIMER_VAL_SHIFT 16
  116. #define MSI_TIMER_VAL_MASK 0xffff
  117. #define MSI_ENABLE_SHIFT 15
  118. #define MSI_ENABLE_MASK 0x1
  119. #define MSI_COUNT_SHIFT 0
  120. #define MSI_COUNT_MASK 0x3ff
  121. /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
  122. #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16
  123. #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff
  124. #define BD_READ_PTR_DDR_ENABLE_SHIFT 15
  125. #define BD_READ_PTR_DDR_ENABLE_MASK 0x1
  126. /* ====== FlexRM ring descriptor defines ===== */
  127. /* Completion descriptor format */
  128. #define CMPL_OPAQUE_SHIFT 0
  129. #define CMPL_OPAQUE_MASK 0xffff
  130. #define CMPL_ENGINE_STATUS_SHIFT 16
  131. #define CMPL_ENGINE_STATUS_MASK 0xffff
  132. #define CMPL_DME_STATUS_SHIFT 32
  133. #define CMPL_DME_STATUS_MASK 0xffff
  134. #define CMPL_RM_STATUS_SHIFT 48
  135. #define CMPL_RM_STATUS_MASK 0xffff
  136. /* Completion DME status code */
  137. #define DME_STATUS_MEM_COR_ERR BIT(0)
  138. #define DME_STATUS_MEM_UCOR_ERR BIT(1)
  139. #define DME_STATUS_FIFO_UNDERFLOW BIT(2)
  140. #define DME_STATUS_FIFO_OVERFLOW BIT(3)
  141. #define DME_STATUS_RRESP_ERR BIT(4)
  142. #define DME_STATUS_BRESP_ERR BIT(5)
  143. #define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \
  144. DME_STATUS_MEM_UCOR_ERR | \
  145. DME_STATUS_FIFO_UNDERFLOW | \
  146. DME_STATUS_FIFO_OVERFLOW | \
  147. DME_STATUS_RRESP_ERR | \
  148. DME_STATUS_BRESP_ERR)
  149. /* Completion RM status code */
  150. #define RM_STATUS_CODE_SHIFT 0
  151. #define RM_STATUS_CODE_MASK 0x3ff
  152. #define RM_STATUS_CODE_GOOD 0x0
  153. #define RM_STATUS_CODE_AE_TIMEOUT 0x3ff
  154. /* General descriptor format */
  155. #define DESC_TYPE_SHIFT 60
  156. #define DESC_TYPE_MASK 0xf
  157. #define DESC_PAYLOAD_SHIFT 0
  158. #define DESC_PAYLOAD_MASK 0x0fffffffffffffff
  159. /* Null descriptor format */
  160. #define NULL_TYPE 0
  161. #define NULL_TOGGLE_SHIFT 58
  162. #define NULL_TOGGLE_MASK 0x1
  163. /* Header descriptor format */
  164. #define HEADER_TYPE 1
  165. #define HEADER_TOGGLE_SHIFT 58
  166. #define HEADER_TOGGLE_MASK 0x1
  167. #define HEADER_ENDPKT_SHIFT 57
  168. #define HEADER_ENDPKT_MASK 0x1
  169. #define HEADER_STARTPKT_SHIFT 56
  170. #define HEADER_STARTPKT_MASK 0x1
  171. #define HEADER_BDCOUNT_SHIFT 36
  172. #define HEADER_BDCOUNT_MASK 0x1f
  173. #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK
  174. #define HEADER_FLAGS_SHIFT 16
  175. #define HEADER_FLAGS_MASK 0xffff
  176. #define HEADER_OPAQUE_SHIFT 0
  177. #define HEADER_OPAQUE_MASK 0xffff
  178. /* Source (SRC) descriptor format */
  179. #define SRC_TYPE 2
  180. #define SRC_LENGTH_SHIFT 44
  181. #define SRC_LENGTH_MASK 0xffff
  182. #define SRC_ADDR_SHIFT 0
  183. #define SRC_ADDR_MASK 0x00000fffffffffff
  184. /* Destination (DST) descriptor format */
  185. #define DST_TYPE 3
  186. #define DST_LENGTH_SHIFT 44
  187. #define DST_LENGTH_MASK 0xffff
  188. #define DST_ADDR_SHIFT 0
  189. #define DST_ADDR_MASK 0x00000fffffffffff
  190. /* Immediate (IMM) descriptor format */
  191. #define IMM_TYPE 4
  192. #define IMM_DATA_SHIFT 0
  193. #define IMM_DATA_MASK 0x0fffffffffffffff
  194. /* Next pointer (NPTR) descriptor format */
  195. #define NPTR_TYPE 5
  196. #define NPTR_TOGGLE_SHIFT 58
  197. #define NPTR_TOGGLE_MASK 0x1
  198. #define NPTR_ADDR_SHIFT 0
  199. #define NPTR_ADDR_MASK 0x00000fffffffffff
  200. /* Mega source (MSRC) descriptor format */
  201. #define MSRC_TYPE 6
  202. #define MSRC_LENGTH_SHIFT 44
  203. #define MSRC_LENGTH_MASK 0xffff
  204. #define MSRC_ADDR_SHIFT 0
  205. #define MSRC_ADDR_MASK 0x00000fffffffffff
  206. /* Mega destination (MDST) descriptor format */
  207. #define MDST_TYPE 7
  208. #define MDST_LENGTH_SHIFT 44
  209. #define MDST_LENGTH_MASK 0xffff
  210. #define MDST_ADDR_SHIFT 0
  211. #define MDST_ADDR_MASK 0x00000fffffffffff
  212. /* Source with tlast (SRCT) descriptor format */
  213. #define SRCT_TYPE 8
  214. #define SRCT_LENGTH_SHIFT 44
  215. #define SRCT_LENGTH_MASK 0xffff
  216. #define SRCT_ADDR_SHIFT 0
  217. #define SRCT_ADDR_MASK 0x00000fffffffffff
  218. /* Destination with tlast (DSTT) descriptor format */
  219. #define DSTT_TYPE 9
  220. #define DSTT_LENGTH_SHIFT 44
  221. #define DSTT_LENGTH_MASK 0xffff
  222. #define DSTT_ADDR_SHIFT 0
  223. #define DSTT_ADDR_MASK 0x00000fffffffffff
  224. /* Immediate with tlast (IMMT) descriptor format */
  225. #define IMMT_TYPE 10
  226. #define IMMT_DATA_SHIFT 0
  227. #define IMMT_DATA_MASK 0x0fffffffffffffff
  228. /* Descriptor helper macros */
  229. #define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m))
  230. #define DESC_ENC(_d, _v, _s, _m) \
  231. do { \
  232. (_d) &= ~((u64)(_m) << (_s)); \
  233. (_d) |= (((u64)(_v) & (_m)) << (_s)); \
  234. } while (0)
  235. /* ====== FlexRM data structures ===== */
  236. struct flexrm_ring {
  237. /* Unprotected members */
  238. int num;
  239. struct flexrm_mbox *mbox;
  240. void __iomem *regs;
  241. bool irq_requested;
  242. unsigned int irq;
  243. cpumask_t irq_aff_hint;
  244. unsigned int msi_timer_val;
  245. unsigned int msi_count_threshold;
  246. struct brcm_message *requests[RING_MAX_REQ_COUNT];
  247. void *bd_base;
  248. dma_addr_t bd_dma_base;
  249. u32 bd_write_offset;
  250. void *cmpl_base;
  251. dma_addr_t cmpl_dma_base;
  252. /* Atomic stats */
  253. atomic_t msg_send_count;
  254. atomic_t msg_cmpl_count;
  255. /* Protected members */
  256. spinlock_t lock;
  257. DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
  258. u32 cmpl_read_offset;
  259. };
  260. struct flexrm_mbox {
  261. struct device *dev;
  262. void __iomem *regs;
  263. u32 num_rings;
  264. struct flexrm_ring *rings;
  265. struct dma_pool *bd_pool;
  266. struct dma_pool *cmpl_pool;
  267. struct dentry *root;
  268. struct mbox_controller controller;
  269. };
  270. /* ====== FlexRM ring descriptor helper routines ===== */
  271. static u64 flexrm_read_desc(void *desc_ptr)
  272. {
  273. return le64_to_cpu(*((u64 *)desc_ptr));
  274. }
  275. static void flexrm_write_desc(void *desc_ptr, u64 desc)
  276. {
  277. *((u64 *)desc_ptr) = cpu_to_le64(desc);
  278. }
  279. static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
  280. {
  281. return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
  282. }
  283. static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
  284. {
  285. u32 status;
  286. status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
  287. CMPL_DME_STATUS_MASK);
  288. if (status & DME_STATUS_ERROR_MASK)
  289. return -EIO;
  290. status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
  291. CMPL_RM_STATUS_MASK);
  292. status &= RM_STATUS_CODE_MASK;
  293. if (status == RM_STATUS_CODE_AE_TIMEOUT)
  294. return -ETIMEDOUT;
  295. return 0;
  296. }
  297. static bool flexrm_is_next_table_desc(void *desc_ptr)
  298. {
  299. u64 desc = flexrm_read_desc(desc_ptr);
  300. u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  301. return (type == NPTR_TYPE) ? true : false;
  302. }
  303. static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
  304. {
  305. u64 desc = 0;
  306. DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  307. DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
  308. DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
  309. return desc;
  310. }
  311. static u64 flexrm_null_desc(u32 toggle)
  312. {
  313. u64 desc = 0;
  314. DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  315. DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
  316. return desc;
  317. }
  318. static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
  319. {
  320. u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
  321. if (!(nhcnt % HEADER_BDCOUNT_MAX))
  322. hcnt += 1;
  323. return hcnt;
  324. }
  325. static void flexrm_flip_header_toggle(void *desc_ptr)
  326. {
  327. u64 desc = flexrm_read_desc(desc_ptr);
  328. if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
  329. desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
  330. else
  331. desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
  332. flexrm_write_desc(desc_ptr, desc);
  333. }
  334. static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
  335. u32 bdcount, u32 flags, u32 opaque)
  336. {
  337. u64 desc = 0;
  338. DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  339. DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
  340. DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
  341. DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
  342. DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
  343. DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
  344. DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
  345. return desc;
  346. }
  347. static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
  348. u64 desc, void **desc_ptr, u32 *toggle,
  349. void *start_desc, void *end_desc)
  350. {
  351. u64 d;
  352. u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
  353. /* Sanity check */
  354. if (nhcnt <= nhpos)
  355. return;
  356. /*
  357. * Each request or packet start with a HEADER descriptor followed
  358. * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
  359. * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
  360. * following a HEADER descriptor is represented by BDCOUNT field
  361. * of HEADER descriptor. The max value of BDCOUNT field is 31 which
  362. * means we can only have 31 non-HEADER descriptors following one
  363. * HEADER descriptor.
  364. *
  365. * In general use, number of non-HEADER descriptors can easily go
  366. * beyond 31. To tackle this situation, we have packet (or request)
  367. * extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor.
  368. *
  369. * To use packet extension, the first HEADER descriptor of request
  370. * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
  371. * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
  372. * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
  373. * TOGGLE bit of the first HEADER will be set to invalid state to
  374. * ensure that FlexRM does not start fetching descriptors till all
  375. * descriptors are enqueued. The user of this function will flip
  376. * the TOGGLE bit of first HEADER after all descriptors are
  377. * enqueued.
  378. */
  379. if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
  380. /* Prepare the header descriptor */
  381. nhavail = (nhcnt - nhpos);
  382. _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
  383. _startpkt = (nhpos == 0) ? 0x1 : 0x0;
  384. _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
  385. _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
  386. nhavail : HEADER_BDCOUNT_MAX;
  387. if (nhavail <= HEADER_BDCOUNT_MAX)
  388. _bdcount = nhavail;
  389. else
  390. _bdcount = HEADER_BDCOUNT_MAX;
  391. d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
  392. _bdcount, 0x0, reqid);
  393. /* Write header descriptor */
  394. flexrm_write_desc(*desc_ptr, d);
  395. /* Point to next descriptor */
  396. *desc_ptr += sizeof(desc);
  397. if (*desc_ptr == end_desc)
  398. *desc_ptr = start_desc;
  399. /* Skip next pointer descriptors */
  400. while (flexrm_is_next_table_desc(*desc_ptr)) {
  401. *toggle = (*toggle) ? 0 : 1;
  402. *desc_ptr += sizeof(desc);
  403. if (*desc_ptr == end_desc)
  404. *desc_ptr = start_desc;
  405. }
  406. }
  407. /* Write desired descriptor */
  408. flexrm_write_desc(*desc_ptr, desc);
  409. /* Point to next descriptor */
  410. *desc_ptr += sizeof(desc);
  411. if (*desc_ptr == end_desc)
  412. *desc_ptr = start_desc;
  413. /* Skip next pointer descriptors */
  414. while (flexrm_is_next_table_desc(*desc_ptr)) {
  415. *toggle = (*toggle) ? 0 : 1;
  416. *desc_ptr += sizeof(desc);
  417. if (*desc_ptr == end_desc)
  418. *desc_ptr = start_desc;
  419. }
  420. }
  421. static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
  422. {
  423. u64 desc = 0;
  424. DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  425. DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
  426. DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
  427. return desc;
  428. }
  429. static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
  430. {
  431. u64 desc = 0;
  432. DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  433. DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
  434. DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
  435. return desc;
  436. }
  437. static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
  438. {
  439. u64 desc = 0;
  440. DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  441. DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
  442. DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
  443. return desc;
  444. }
  445. static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
  446. {
  447. u64 desc = 0;
  448. DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  449. DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
  450. DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
  451. return desc;
  452. }
  453. static u64 flexrm_imm_desc(u64 data)
  454. {
  455. u64 desc = 0;
  456. DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  457. DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
  458. return desc;
  459. }
  460. static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
  461. {
  462. u64 desc = 0;
  463. DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  464. DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
  465. DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
  466. return desc;
  467. }
  468. static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
  469. {
  470. u64 desc = 0;
  471. DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  472. DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
  473. DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
  474. return desc;
  475. }
  476. static u64 flexrm_immt_desc(u64 data)
  477. {
  478. u64 desc = 0;
  479. DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
  480. DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
  481. return desc;
  482. }
  483. static bool flexrm_spu_sanity_check(struct brcm_message *msg)
  484. {
  485. struct scatterlist *sg;
  486. if (!msg->spu.src || !msg->spu.dst)
  487. return false;
  488. for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
  489. if (sg->length & 0xf) {
  490. if (sg->length > SRC_LENGTH_MASK)
  491. return false;
  492. } else {
  493. if (sg->length > (MSRC_LENGTH_MASK * 16))
  494. return false;
  495. }
  496. }
  497. for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
  498. if (sg->length & 0xf) {
  499. if (sg->length > DST_LENGTH_MASK)
  500. return false;
  501. } else {
  502. if (sg->length > (MDST_LENGTH_MASK * 16))
  503. return false;
  504. }
  505. }
  506. return true;
  507. }
  508. static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
  509. {
  510. u32 cnt = 0;
  511. unsigned int dst_target = 0;
  512. struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
  513. while (src_sg || dst_sg) {
  514. if (src_sg) {
  515. cnt++;
  516. dst_target = src_sg->length;
  517. src_sg = sg_next(src_sg);
  518. } else
  519. dst_target = UINT_MAX;
  520. while (dst_target && dst_sg) {
  521. cnt++;
  522. if (dst_sg->length < dst_target)
  523. dst_target -= dst_sg->length;
  524. else
  525. dst_target = 0;
  526. dst_sg = sg_next(dst_sg);
  527. }
  528. }
  529. return cnt;
  530. }
  531. static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
  532. {
  533. int rc;
  534. rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
  535. DMA_TO_DEVICE);
  536. if (rc < 0)
  537. return rc;
  538. rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
  539. DMA_FROM_DEVICE);
  540. if (rc < 0) {
  541. dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
  542. DMA_TO_DEVICE);
  543. return rc;
  544. }
  545. return 0;
  546. }
  547. static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
  548. {
  549. dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
  550. DMA_FROM_DEVICE);
  551. dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
  552. DMA_TO_DEVICE);
  553. }
  554. static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
  555. u32 reqid, void *desc_ptr, u32 toggle,
  556. void *start_desc, void *end_desc)
  557. {
  558. u64 d;
  559. u32 nhpos = 0;
  560. void *orig_desc_ptr = desc_ptr;
  561. unsigned int dst_target = 0;
  562. struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
  563. while (src_sg || dst_sg) {
  564. if (src_sg) {
  565. if (sg_dma_len(src_sg) & 0xf)
  566. d = flexrm_src_desc(sg_dma_address(src_sg),
  567. sg_dma_len(src_sg));
  568. else
  569. d = flexrm_msrc_desc(sg_dma_address(src_sg),
  570. sg_dma_len(src_sg)/16);
  571. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  572. d, &desc_ptr, &toggle,
  573. start_desc, end_desc);
  574. nhpos++;
  575. dst_target = sg_dma_len(src_sg);
  576. src_sg = sg_next(src_sg);
  577. } else
  578. dst_target = UINT_MAX;
  579. while (dst_target && dst_sg) {
  580. if (sg_dma_len(dst_sg) & 0xf)
  581. d = flexrm_dst_desc(sg_dma_address(dst_sg),
  582. sg_dma_len(dst_sg));
  583. else
  584. d = flexrm_mdst_desc(sg_dma_address(dst_sg),
  585. sg_dma_len(dst_sg)/16);
  586. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  587. d, &desc_ptr, &toggle,
  588. start_desc, end_desc);
  589. nhpos++;
  590. if (sg_dma_len(dst_sg) < dst_target)
  591. dst_target -= sg_dma_len(dst_sg);
  592. else
  593. dst_target = 0;
  594. dst_sg = sg_next(dst_sg);
  595. }
  596. }
  597. /* Null descriptor with invalid toggle bit */
  598. flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
  599. /* Ensure that descriptors have been written to memory */
  600. wmb();
  601. /* Flip toggle bit in header */
  602. flexrm_flip_header_toggle(orig_desc_ptr);
  603. return desc_ptr;
  604. }
  605. static bool flexrm_sba_sanity_check(struct brcm_message *msg)
  606. {
  607. u32 i;
  608. if (!msg->sba.cmds || !msg->sba.cmds_count)
  609. return false;
  610. for (i = 0; i < msg->sba.cmds_count; i++) {
  611. if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
  612. (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
  613. (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
  614. return false;
  615. if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
  616. (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
  617. return false;
  618. if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
  619. (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
  620. return false;
  621. if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
  622. (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
  623. return false;
  624. if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
  625. (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
  626. return false;
  627. }
  628. return true;
  629. }
  630. static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
  631. {
  632. u32 i, cnt;
  633. cnt = 0;
  634. for (i = 0; i < msg->sba.cmds_count; i++) {
  635. cnt++;
  636. if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
  637. (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
  638. cnt++;
  639. if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
  640. cnt++;
  641. if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
  642. cnt++;
  643. }
  644. return cnt;
  645. }
  646. static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
  647. u32 reqid, void *desc_ptr, u32 toggle,
  648. void *start_desc, void *end_desc)
  649. {
  650. u64 d;
  651. u32 i, nhpos = 0;
  652. struct brcm_sba_command *c;
  653. void *orig_desc_ptr = desc_ptr;
  654. /* Convert SBA commands into descriptors */
  655. for (i = 0; i < msg->sba.cmds_count; i++) {
  656. c = &msg->sba.cmds[i];
  657. if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
  658. (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
  659. /* Destination response descriptor */
  660. d = flexrm_dst_desc(c->resp, c->resp_len);
  661. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  662. d, &desc_ptr, &toggle,
  663. start_desc, end_desc);
  664. nhpos++;
  665. } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
  666. /* Destination response with tlast descriptor */
  667. d = flexrm_dstt_desc(c->resp, c->resp_len);
  668. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  669. d, &desc_ptr, &toggle,
  670. start_desc, end_desc);
  671. nhpos++;
  672. }
  673. if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
  674. /* Destination with tlast descriptor */
  675. d = flexrm_dstt_desc(c->data, c->data_len);
  676. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  677. d, &desc_ptr, &toggle,
  678. start_desc, end_desc);
  679. nhpos++;
  680. }
  681. if (c->flags & BRCM_SBA_CMD_TYPE_B) {
  682. /* Command as immediate descriptor */
  683. d = flexrm_imm_desc(c->cmd);
  684. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  685. d, &desc_ptr, &toggle,
  686. start_desc, end_desc);
  687. nhpos++;
  688. } else {
  689. /* Command as immediate descriptor with tlast */
  690. d = flexrm_immt_desc(c->cmd);
  691. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  692. d, &desc_ptr, &toggle,
  693. start_desc, end_desc);
  694. nhpos++;
  695. }
  696. if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
  697. (c->flags & BRCM_SBA_CMD_TYPE_C)) {
  698. /* Source with tlast descriptor */
  699. d = flexrm_srct_desc(c->data, c->data_len);
  700. flexrm_enqueue_desc(nhpos, nhcnt, reqid,
  701. d, &desc_ptr, &toggle,
  702. start_desc, end_desc);
  703. nhpos++;
  704. }
  705. }
  706. /* Null descriptor with invalid toggle bit */
  707. flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
  708. /* Ensure that descriptors have been written to memory */
  709. wmb();
  710. /* Flip toggle bit in header */
  711. flexrm_flip_header_toggle(orig_desc_ptr);
  712. return desc_ptr;
  713. }
  714. static bool flexrm_sanity_check(struct brcm_message *msg)
  715. {
  716. if (!msg)
  717. return false;
  718. switch (msg->type) {
  719. case BRCM_MESSAGE_SPU:
  720. return flexrm_spu_sanity_check(msg);
  721. case BRCM_MESSAGE_SBA:
  722. return flexrm_sba_sanity_check(msg);
  723. default:
  724. return false;
  725. };
  726. }
  727. static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
  728. {
  729. if (!msg)
  730. return 0;
  731. switch (msg->type) {
  732. case BRCM_MESSAGE_SPU:
  733. return flexrm_spu_estimate_nonheader_desc_count(msg);
  734. case BRCM_MESSAGE_SBA:
  735. return flexrm_sba_estimate_nonheader_desc_count(msg);
  736. default:
  737. return 0;
  738. };
  739. }
  740. static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
  741. {
  742. if (!dev || !msg)
  743. return -EINVAL;
  744. switch (msg->type) {
  745. case BRCM_MESSAGE_SPU:
  746. return flexrm_spu_dma_map(dev, msg);
  747. default:
  748. break;
  749. }
  750. return 0;
  751. }
  752. static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
  753. {
  754. if (!dev || !msg)
  755. return;
  756. switch (msg->type) {
  757. case BRCM_MESSAGE_SPU:
  758. flexrm_spu_dma_unmap(dev, msg);
  759. break;
  760. default:
  761. break;
  762. }
  763. }
  764. static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
  765. u32 reqid, void *desc_ptr, u32 toggle,
  766. void *start_desc, void *end_desc)
  767. {
  768. if (!msg || !desc_ptr || !start_desc || !end_desc)
  769. return ERR_PTR(-ENOTSUPP);
  770. if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
  771. return ERR_PTR(-ERANGE);
  772. switch (msg->type) {
  773. case BRCM_MESSAGE_SPU:
  774. return flexrm_spu_write_descs(msg, nhcnt, reqid,
  775. desc_ptr, toggle,
  776. start_desc, end_desc);
  777. case BRCM_MESSAGE_SBA:
  778. return flexrm_sba_write_descs(msg, nhcnt, reqid,
  779. desc_ptr, toggle,
  780. start_desc, end_desc);
  781. default:
  782. return ERR_PTR(-ENOTSUPP);
  783. };
  784. }
  785. /* ====== FlexRM driver helper routines ===== */
  786. static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
  787. struct seq_file *file)
  788. {
  789. int i;
  790. const char *state;
  791. struct flexrm_ring *ring;
  792. seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
  793. "Ring#", "State", "BD_Addr", "BD_Size",
  794. "Cmpl_Addr", "Cmpl_Size");
  795. for (i = 0; i < mbox->num_rings; i++) {
  796. ring = &mbox->rings[i];
  797. if (readl(ring->regs + RING_CONTROL) &
  798. BIT(CONTROL_ACTIVE_SHIFT))
  799. state = "active";
  800. else
  801. state = "inactive";
  802. seq_printf(file,
  803. "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
  804. ring->num, state,
  805. (unsigned long long)ring->bd_dma_base,
  806. (u32)RING_BD_SIZE,
  807. (unsigned long long)ring->cmpl_dma_base,
  808. (u32)RING_CMPL_SIZE);
  809. }
  810. }
  811. static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
  812. struct seq_file *file)
  813. {
  814. int i;
  815. u32 val, bd_read_offset;
  816. struct flexrm_ring *ring;
  817. seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
  818. "Ring#", "BD_Read", "BD_Write",
  819. "Cmpl_Read", "Submitted", "Completed");
  820. for (i = 0; i < mbox->num_rings; i++) {
  821. ring = &mbox->rings[i];
  822. bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
  823. val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
  824. bd_read_offset *= RING_DESC_SIZE;
  825. bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
  826. ring->bd_dma_base);
  827. seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
  828. ring->num,
  829. (u32)bd_read_offset,
  830. (u32)ring->bd_write_offset,
  831. (u32)ring->cmpl_read_offset,
  832. (u32)atomic_read(&ring->msg_send_count),
  833. (u32)atomic_read(&ring->msg_cmpl_count));
  834. }
  835. }
  836. static int flexrm_new_request(struct flexrm_ring *ring,
  837. struct brcm_message *batch_msg,
  838. struct brcm_message *msg)
  839. {
  840. void *next;
  841. unsigned long flags;
  842. u32 val, count, nhcnt;
  843. u32 read_offset, write_offset;
  844. bool exit_cleanup = false;
  845. int ret = 0, reqid;
  846. /* Do sanity check on message */
  847. if (!flexrm_sanity_check(msg))
  848. return -EIO;
  849. msg->error = 0;
  850. /* If no requests possible then save data pointer and goto done. */
  851. spin_lock_irqsave(&ring->lock, flags);
  852. reqid = bitmap_find_free_region(ring->requests_bmap,
  853. RING_MAX_REQ_COUNT, 0);
  854. spin_unlock_irqrestore(&ring->lock, flags);
  855. if (reqid < 0)
  856. return -ENOSPC;
  857. ring->requests[reqid] = msg;
  858. /* Do DMA mappings for the message */
  859. ret = flexrm_dma_map(ring->mbox->dev, msg);
  860. if (ret < 0) {
  861. ring->requests[reqid] = NULL;
  862. spin_lock_irqsave(&ring->lock, flags);
  863. bitmap_release_region(ring->requests_bmap, reqid, 0);
  864. spin_unlock_irqrestore(&ring->lock, flags);
  865. return ret;
  866. }
  867. /* Determine current HW BD read offset */
  868. read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
  869. val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
  870. read_offset *= RING_DESC_SIZE;
  871. read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
  872. /*
  873. * Number required descriptors = number of non-header descriptors +
  874. * number of header descriptors +
  875. * 1x null descriptor
  876. */
  877. nhcnt = flexrm_estimate_nonheader_desc_count(msg);
  878. count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
  879. /* Check for available descriptor space. */
  880. write_offset = ring->bd_write_offset;
  881. while (count) {
  882. if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
  883. count--;
  884. write_offset += RING_DESC_SIZE;
  885. if (write_offset == RING_BD_SIZE)
  886. write_offset = 0x0;
  887. if (write_offset == read_offset)
  888. break;
  889. }
  890. if (count) {
  891. ret = -ENOSPC;
  892. exit_cleanup = true;
  893. goto exit;
  894. }
  895. /* Write descriptors to ring */
  896. next = flexrm_write_descs(msg, nhcnt, reqid,
  897. ring->bd_base + ring->bd_write_offset,
  898. RING_BD_TOGGLE_VALID(ring->bd_write_offset),
  899. ring->bd_base, ring->bd_base + RING_BD_SIZE);
  900. if (IS_ERR(next)) {
  901. ret = PTR_ERR(next);
  902. exit_cleanup = true;
  903. goto exit;
  904. }
  905. /* Save ring BD write offset */
  906. ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
  907. /* Increment number of messages sent */
  908. atomic_inc_return(&ring->msg_send_count);
  909. exit:
  910. /* Update error status in message */
  911. msg->error = ret;
  912. /* Cleanup if we failed */
  913. if (exit_cleanup) {
  914. flexrm_dma_unmap(ring->mbox->dev, msg);
  915. ring->requests[reqid] = NULL;
  916. spin_lock_irqsave(&ring->lock, flags);
  917. bitmap_release_region(ring->requests_bmap, reqid, 0);
  918. spin_unlock_irqrestore(&ring->lock, flags);
  919. }
  920. return ret;
  921. }
  922. static int flexrm_process_completions(struct flexrm_ring *ring)
  923. {
  924. u64 desc;
  925. int err, count = 0;
  926. unsigned long flags;
  927. struct brcm_message *msg = NULL;
  928. u32 reqid, cmpl_read_offset, cmpl_write_offset;
  929. struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
  930. spin_lock_irqsave(&ring->lock, flags);
  931. /*
  932. * Get current completion read and write offset
  933. *
  934. * Note: We should read completion write pointer atleast once
  935. * after we get a MSI interrupt because HW maintains internal
  936. * MSI status which will allow next MSI interrupt only after
  937. * completion write pointer is read.
  938. */
  939. cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
  940. cmpl_write_offset *= RING_DESC_SIZE;
  941. cmpl_read_offset = ring->cmpl_read_offset;
  942. ring->cmpl_read_offset = cmpl_write_offset;
  943. spin_unlock_irqrestore(&ring->lock, flags);
  944. /* For each completed request notify mailbox clients */
  945. reqid = 0;
  946. while (cmpl_read_offset != cmpl_write_offset) {
  947. /* Dequeue next completion descriptor */
  948. desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
  949. /* Next read offset */
  950. cmpl_read_offset += RING_DESC_SIZE;
  951. if (cmpl_read_offset == RING_CMPL_SIZE)
  952. cmpl_read_offset = 0;
  953. /* Decode error from completion descriptor */
  954. err = flexrm_cmpl_desc_to_error(desc);
  955. if (err < 0) {
  956. dev_warn(ring->mbox->dev,
  957. "ring%d got completion desc=0x%lx with error %d\n",
  958. ring->num, (unsigned long)desc, err);
  959. }
  960. /* Determine request id from completion descriptor */
  961. reqid = flexrm_cmpl_desc_to_reqid(desc);
  962. /* Determine message pointer based on reqid */
  963. msg = ring->requests[reqid];
  964. if (!msg) {
  965. dev_warn(ring->mbox->dev,
  966. "ring%d null msg pointer for completion desc=0x%lx\n",
  967. ring->num, (unsigned long)desc);
  968. continue;
  969. }
  970. /* Release reqid for recycling */
  971. ring->requests[reqid] = NULL;
  972. spin_lock_irqsave(&ring->lock, flags);
  973. bitmap_release_region(ring->requests_bmap, reqid, 0);
  974. spin_unlock_irqrestore(&ring->lock, flags);
  975. /* Unmap DMA mappings */
  976. flexrm_dma_unmap(ring->mbox->dev, msg);
  977. /* Give-back message to mailbox client */
  978. msg->error = err;
  979. mbox_chan_received_data(chan, msg);
  980. /* Increment number of completions processed */
  981. atomic_inc_return(&ring->msg_cmpl_count);
  982. count++;
  983. }
  984. return count;
  985. }
  986. /* ====== FlexRM Debugfs callbacks ====== */
  987. static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
  988. {
  989. struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
  990. /* Write config in file */
  991. flexrm_write_config_in_seqfile(mbox, file);
  992. return 0;
  993. }
  994. static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
  995. {
  996. struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
  997. /* Write stats in file */
  998. flexrm_write_stats_in_seqfile(mbox, file);
  999. return 0;
  1000. }
  1001. /* ====== FlexRM interrupt handler ===== */
  1002. static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
  1003. {
  1004. /* We only have MSI for completions so just wakeup IRQ thread */
  1005. /* Ring related errors will be informed via completion descriptors */
  1006. return IRQ_WAKE_THREAD;
  1007. }
  1008. static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
  1009. {
  1010. flexrm_process_completions(dev_id);
  1011. return IRQ_HANDLED;
  1012. }
  1013. /* ====== FlexRM mailbox callbacks ===== */
  1014. static int flexrm_send_data(struct mbox_chan *chan, void *data)
  1015. {
  1016. int i, rc;
  1017. struct flexrm_ring *ring = chan->con_priv;
  1018. struct brcm_message *msg = data;
  1019. if (msg->type == BRCM_MESSAGE_BATCH) {
  1020. for (i = msg->batch.msgs_queued;
  1021. i < msg->batch.msgs_count; i++) {
  1022. rc = flexrm_new_request(ring, msg,
  1023. &msg->batch.msgs[i]);
  1024. if (rc) {
  1025. msg->error = rc;
  1026. return rc;
  1027. }
  1028. msg->batch.msgs_queued++;
  1029. }
  1030. return 0;
  1031. }
  1032. return flexrm_new_request(ring, NULL, data);
  1033. }
  1034. static bool flexrm_peek_data(struct mbox_chan *chan)
  1035. {
  1036. int cnt = flexrm_process_completions(chan->con_priv);
  1037. return (cnt > 0) ? true : false;
  1038. }
  1039. static int flexrm_startup(struct mbox_chan *chan)
  1040. {
  1041. u64 d;
  1042. u32 val, off;
  1043. int ret = 0;
  1044. dma_addr_t next_addr;
  1045. struct flexrm_ring *ring = chan->con_priv;
  1046. /* Allocate BD memory */
  1047. ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
  1048. GFP_KERNEL, &ring->bd_dma_base);
  1049. if (!ring->bd_base) {
  1050. dev_err(ring->mbox->dev,
  1051. "can't allocate BD memory for ring%d\n",
  1052. ring->num);
  1053. ret = -ENOMEM;
  1054. goto fail;
  1055. }
  1056. /* Configure next table pointer entries in BD memory */
  1057. for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
  1058. next_addr = off + RING_DESC_SIZE;
  1059. if (next_addr == RING_BD_SIZE)
  1060. next_addr = 0;
  1061. next_addr += ring->bd_dma_base;
  1062. if (RING_BD_ALIGN_CHECK(next_addr))
  1063. d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
  1064. next_addr);
  1065. else
  1066. d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
  1067. flexrm_write_desc(ring->bd_base + off, d);
  1068. }
  1069. /* Allocate completion memory */
  1070. ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
  1071. GFP_KERNEL, &ring->cmpl_dma_base);
  1072. if (!ring->cmpl_base) {
  1073. dev_err(ring->mbox->dev,
  1074. "can't allocate completion memory for ring%d\n",
  1075. ring->num);
  1076. ret = -ENOMEM;
  1077. goto fail_free_bd_memory;
  1078. }
  1079. /* Request IRQ */
  1080. if (ring->irq == UINT_MAX) {
  1081. dev_err(ring->mbox->dev,
  1082. "ring%d IRQ not available\n", ring->num);
  1083. ret = -ENODEV;
  1084. goto fail_free_cmpl_memory;
  1085. }
  1086. ret = request_threaded_irq(ring->irq,
  1087. flexrm_irq_event,
  1088. flexrm_irq_thread,
  1089. 0, dev_name(ring->mbox->dev), ring);
  1090. if (ret) {
  1091. dev_err(ring->mbox->dev,
  1092. "failed to request ring%d IRQ\n", ring->num);
  1093. goto fail_free_cmpl_memory;
  1094. }
  1095. ring->irq_requested = true;
  1096. /* Set IRQ affinity hint */
  1097. ring->irq_aff_hint = CPU_MASK_NONE;
  1098. val = ring->mbox->num_rings;
  1099. val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
  1100. cpumask_set_cpu((ring->num / val) % num_online_cpus(),
  1101. &ring->irq_aff_hint);
  1102. ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint);
  1103. if (ret) {
  1104. dev_err(ring->mbox->dev,
  1105. "failed to set IRQ affinity hint for ring%d\n",
  1106. ring->num);
  1107. goto fail_free_irq;
  1108. }
  1109. /* Disable/inactivate ring */
  1110. writel_relaxed(0x0, ring->regs + RING_CONTROL);
  1111. /* Program BD start address */
  1112. val = BD_START_ADDR_VALUE(ring->bd_dma_base);
  1113. writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
  1114. /* BD write pointer will be same as HW write pointer */
  1115. ring->bd_write_offset =
  1116. readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
  1117. ring->bd_write_offset *= RING_DESC_SIZE;
  1118. /* Program completion start address */
  1119. val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
  1120. writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
  1121. /* Completion read pointer will be same as HW write pointer */
  1122. ring->cmpl_read_offset =
  1123. readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
  1124. ring->cmpl_read_offset *= RING_DESC_SIZE;
  1125. /* Read ring Tx, Rx, and Outstanding counts to clear */
  1126. readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
  1127. readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
  1128. readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
  1129. readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
  1130. readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
  1131. /* Configure RING_MSI_CONTROL */
  1132. val = 0;
  1133. val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
  1134. val |= BIT(MSI_ENABLE_SHIFT);
  1135. val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
  1136. writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
  1137. /* Enable/activate ring */
  1138. val = BIT(CONTROL_ACTIVE_SHIFT);
  1139. writel_relaxed(val, ring->regs + RING_CONTROL);
  1140. /* Reset stats to zero */
  1141. atomic_set(&ring->msg_send_count, 0);
  1142. atomic_set(&ring->msg_cmpl_count, 0);
  1143. return 0;
  1144. fail_free_irq:
  1145. free_irq(ring->irq, ring);
  1146. ring->irq_requested = false;
  1147. fail_free_cmpl_memory:
  1148. dma_pool_free(ring->mbox->cmpl_pool,
  1149. ring->cmpl_base, ring->cmpl_dma_base);
  1150. ring->cmpl_base = NULL;
  1151. fail_free_bd_memory:
  1152. dma_pool_free(ring->mbox->bd_pool,
  1153. ring->bd_base, ring->bd_dma_base);
  1154. ring->bd_base = NULL;
  1155. fail:
  1156. return ret;
  1157. }
  1158. static void flexrm_shutdown(struct mbox_chan *chan)
  1159. {
  1160. u32 reqid;
  1161. unsigned int timeout;
  1162. struct brcm_message *msg;
  1163. struct flexrm_ring *ring = chan->con_priv;
  1164. /* Disable/inactivate ring */
  1165. writel_relaxed(0x0, ring->regs + RING_CONTROL);
  1166. /* Set ring flush state */
  1167. timeout = 1000; /* timeout of 1s */
  1168. writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
  1169. ring->regs + RING_CONTROL);
  1170. do {
  1171. if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
  1172. FLUSH_DONE_MASK)
  1173. break;
  1174. mdelay(1);
  1175. } while (--timeout);
  1176. if (!timeout)
  1177. dev_err(ring->mbox->dev,
  1178. "setting ring%d flush state timedout\n", ring->num);
  1179. /* Clear ring flush state */
  1180. timeout = 1000; /* timeout of 1s */
  1181. writel_relaxed(0x0, ring->regs + RING_CONTROL);
  1182. do {
  1183. if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
  1184. FLUSH_DONE_MASK))
  1185. break;
  1186. mdelay(1);
  1187. } while (--timeout);
  1188. if (!timeout)
  1189. dev_err(ring->mbox->dev,
  1190. "clearing ring%d flush state timedout\n", ring->num);
  1191. /* Abort all in-flight requests */
  1192. for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
  1193. msg = ring->requests[reqid];
  1194. if (!msg)
  1195. continue;
  1196. /* Release reqid for recycling */
  1197. ring->requests[reqid] = NULL;
  1198. /* Unmap DMA mappings */
  1199. flexrm_dma_unmap(ring->mbox->dev, msg);
  1200. /* Give-back message to mailbox client */
  1201. msg->error = -EIO;
  1202. mbox_chan_received_data(chan, msg);
  1203. }
  1204. /* Clear requests bitmap */
  1205. bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
  1206. /* Release IRQ */
  1207. if (ring->irq_requested) {
  1208. irq_set_affinity_hint(ring->irq, NULL);
  1209. free_irq(ring->irq, ring);
  1210. ring->irq_requested = false;
  1211. }
  1212. /* Free-up completion descriptor ring */
  1213. if (ring->cmpl_base) {
  1214. dma_pool_free(ring->mbox->cmpl_pool,
  1215. ring->cmpl_base, ring->cmpl_dma_base);
  1216. ring->cmpl_base = NULL;
  1217. }
  1218. /* Free-up BD descriptor ring */
  1219. if (ring->bd_base) {
  1220. dma_pool_free(ring->mbox->bd_pool,
  1221. ring->bd_base, ring->bd_dma_base);
  1222. ring->bd_base = NULL;
  1223. }
  1224. }
  1225. static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
  1226. .send_data = flexrm_send_data,
  1227. .startup = flexrm_startup,
  1228. .shutdown = flexrm_shutdown,
  1229. .peek_data = flexrm_peek_data,
  1230. };
  1231. static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
  1232. const struct of_phandle_args *pa)
  1233. {
  1234. struct mbox_chan *chan;
  1235. struct flexrm_ring *ring;
  1236. if (pa->args_count < 3)
  1237. return ERR_PTR(-EINVAL);
  1238. if (pa->args[0] >= cntlr->num_chans)
  1239. return ERR_PTR(-ENOENT);
  1240. if (pa->args[1] > MSI_COUNT_MASK)
  1241. return ERR_PTR(-EINVAL);
  1242. if (pa->args[2] > MSI_TIMER_VAL_MASK)
  1243. return ERR_PTR(-EINVAL);
  1244. chan = &cntlr->chans[pa->args[0]];
  1245. ring = chan->con_priv;
  1246. ring->msi_count_threshold = pa->args[1];
  1247. ring->msi_timer_val = pa->args[2];
  1248. return chan;
  1249. }
  1250. /* ====== FlexRM platform driver ===== */
  1251. static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
  1252. {
  1253. struct device *dev = msi_desc_to_dev(desc);
  1254. struct flexrm_mbox *mbox = dev_get_drvdata(dev);
  1255. struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index];
  1256. /* Configure per-Ring MSI registers */
  1257. writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
  1258. writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
  1259. writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
  1260. }
  1261. static int flexrm_mbox_probe(struct platform_device *pdev)
  1262. {
  1263. int index, ret = 0;
  1264. void __iomem *regs;
  1265. void __iomem *regs_end;
  1266. struct msi_desc *desc;
  1267. struct resource *iomem;
  1268. struct flexrm_ring *ring;
  1269. struct flexrm_mbox *mbox;
  1270. struct device *dev = &pdev->dev;
  1271. /* Allocate driver mailbox struct */
  1272. mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
  1273. if (!mbox) {
  1274. ret = -ENOMEM;
  1275. goto fail;
  1276. }
  1277. mbox->dev = dev;
  1278. platform_set_drvdata(pdev, mbox);
  1279. /* Get resource for registers */
  1280. iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1281. if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
  1282. ret = -ENODEV;
  1283. goto fail;
  1284. }
  1285. /* Map registers of all rings */
  1286. mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
  1287. if (IS_ERR(mbox->regs)) {
  1288. ret = PTR_ERR(mbox->regs);
  1289. dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret);
  1290. goto fail;
  1291. }
  1292. regs_end = mbox->regs + resource_size(iomem);
  1293. /* Scan and count available rings */
  1294. mbox->num_rings = 0;
  1295. for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
  1296. if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
  1297. mbox->num_rings++;
  1298. }
  1299. if (!mbox->num_rings) {
  1300. ret = -ENODEV;
  1301. goto fail;
  1302. }
  1303. /* Allocate driver ring structs */
  1304. ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
  1305. if (!ring) {
  1306. ret = -ENOMEM;
  1307. goto fail;
  1308. }
  1309. mbox->rings = ring;
  1310. /* Initialize members of driver ring structs */
  1311. regs = mbox->regs;
  1312. for (index = 0; index < mbox->num_rings; index++) {
  1313. ring = &mbox->rings[index];
  1314. ring->num = index;
  1315. ring->mbox = mbox;
  1316. while ((regs < regs_end) &&
  1317. (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
  1318. regs += RING_REGS_SIZE;
  1319. if (regs_end <= regs) {
  1320. ret = -ENODEV;
  1321. goto fail;
  1322. }
  1323. ring->regs = regs;
  1324. regs += RING_REGS_SIZE;
  1325. ring->irq = UINT_MAX;
  1326. ring->irq_requested = false;
  1327. ring->msi_timer_val = MSI_TIMER_VAL_MASK;
  1328. ring->msi_count_threshold = 0x1;
  1329. memset(ring->requests, 0, sizeof(ring->requests));
  1330. ring->bd_base = NULL;
  1331. ring->bd_dma_base = 0;
  1332. ring->cmpl_base = NULL;
  1333. ring->cmpl_dma_base = 0;
  1334. atomic_set(&ring->msg_send_count, 0);
  1335. atomic_set(&ring->msg_cmpl_count, 0);
  1336. spin_lock_init(&ring->lock);
  1337. bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
  1338. ring->cmpl_read_offset = 0;
  1339. }
  1340. /* FlexRM is capable of 40-bit physical addresses only */
  1341. ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
  1342. if (ret) {
  1343. ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
  1344. if (ret)
  1345. goto fail;
  1346. }
  1347. /* Create DMA pool for ring BD memory */
  1348. mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
  1349. 1 << RING_BD_ALIGN_ORDER, 0);
  1350. if (!mbox->bd_pool) {
  1351. ret = -ENOMEM;
  1352. goto fail;
  1353. }
  1354. /* Create DMA pool for ring completion memory */
  1355. mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
  1356. 1 << RING_CMPL_ALIGN_ORDER, 0);
  1357. if (!mbox->cmpl_pool) {
  1358. ret = -ENOMEM;
  1359. goto fail_destroy_bd_pool;
  1360. }
  1361. /* Allocate platform MSIs for each ring */
  1362. ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
  1363. flexrm_mbox_msi_write);
  1364. if (ret)
  1365. goto fail_destroy_cmpl_pool;
  1366. /* Save alloced IRQ numbers for each ring */
  1367. for_each_msi_entry(desc, dev) {
  1368. ring = &mbox->rings[desc->platform.msi_index];
  1369. ring->irq = desc->irq;
  1370. }
  1371. /* Check availability of debugfs */
  1372. if (!debugfs_initialized())
  1373. goto skip_debugfs;
  1374. /* Create debugfs root entry */
  1375. mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
  1376. /* Create debugfs config entry */
  1377. debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,
  1378. flexrm_debugfs_conf_show);
  1379. /* Create debugfs stats entry */
  1380. debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,
  1381. flexrm_debugfs_stats_show);
  1382. skip_debugfs:
  1383. /* Initialize mailbox controller */
  1384. mbox->controller.txdone_irq = false;
  1385. mbox->controller.txdone_poll = false;
  1386. mbox->controller.ops = &flexrm_mbox_chan_ops;
  1387. mbox->controller.dev = dev;
  1388. mbox->controller.num_chans = mbox->num_rings;
  1389. mbox->controller.of_xlate = flexrm_mbox_of_xlate;
  1390. mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
  1391. sizeof(*mbox->controller.chans), GFP_KERNEL);
  1392. if (!mbox->controller.chans) {
  1393. ret = -ENOMEM;
  1394. goto fail_free_debugfs_root;
  1395. }
  1396. for (index = 0; index < mbox->num_rings; index++)
  1397. mbox->controller.chans[index].con_priv = &mbox->rings[index];
  1398. /* Register mailbox controller */
  1399. ret = devm_mbox_controller_register(dev, &mbox->controller);
  1400. if (ret)
  1401. goto fail_free_debugfs_root;
  1402. dev_info(dev, "registered flexrm mailbox with %d channels\n",
  1403. mbox->controller.num_chans);
  1404. return 0;
  1405. fail_free_debugfs_root:
  1406. debugfs_remove_recursive(mbox->root);
  1407. platform_msi_domain_free_irqs(dev);
  1408. fail_destroy_cmpl_pool:
  1409. dma_pool_destroy(mbox->cmpl_pool);
  1410. fail_destroy_bd_pool:
  1411. dma_pool_destroy(mbox->bd_pool);
  1412. fail:
  1413. return ret;
  1414. }
  1415. static int flexrm_mbox_remove(struct platform_device *pdev)
  1416. {
  1417. struct device *dev = &pdev->dev;
  1418. struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
  1419. debugfs_remove_recursive(mbox->root);
  1420. platform_msi_domain_free_irqs(dev);
  1421. dma_pool_destroy(mbox->cmpl_pool);
  1422. dma_pool_destroy(mbox->bd_pool);
  1423. return 0;
  1424. }
  1425. static const struct of_device_id flexrm_mbox_of_match[] = {
  1426. { .compatible = "brcm,iproc-flexrm-mbox", },
  1427. {},
  1428. };
  1429. MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
  1430. static struct platform_driver flexrm_mbox_driver = {
  1431. .driver = {
  1432. .name = "brcm-flexrm-mbox",
  1433. .of_match_table = flexrm_mbox_of_match,
  1434. },
  1435. .probe = flexrm_mbox_probe,
  1436. .remove = flexrm_mbox_remove,
  1437. };
  1438. module_platform_driver(flexrm_mbox_driver);
  1439. MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
  1440. MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
  1441. MODULE_LICENSE("GPL v2");