k3-navss-ringacc.c 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
  4. *
  5. * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
  6. */
  7. #include <common.h>
  8. #include <cpu_func.h>
  9. #include <log.h>
  10. #include <asm/cache.h>
  11. #include <asm/io.h>
  12. #include <malloc.h>
  13. #include <asm/bitops.h>
  14. #include <dm.h>
  15. #include <dm/device_compat.h>
  16. #include <dm/devres.h>
  17. #include <dm/read.h>
  18. #include <dm/uclass.h>
  19. #include <linux/bitops.h>
  20. #include <linux/compat.h>
  21. #include <linux/dma-mapping.h>
  22. #include <linux/err.h>
  23. #include <linux/soc/ti/k3-navss-ringacc.h>
  24. #include <linux/soc/ti/ti_sci_protocol.h>
  25. #include <linux/soc/ti/cppi5.h>
  26. #define set_bit(bit, bitmap) __set_bit(bit, bitmap)
  27. #define clear_bit(bit, bitmap) __clear_bit(bit, bitmap)
  28. #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
  29. dma_free_coherent(cpu_addr)
  30. #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
  31. ({ \
  32. void *ring_mem_virt; \
  33. ring_mem_virt = dma_alloc_coherent((size), \
  34. (unsigned long *)(dma_handle)); \
  35. if (ring_mem_virt) \
  36. memset(ring_mem_virt, 0, (size)); \
  37. ring_mem_virt; \
  38. })
  39. static LIST_HEAD(k3_nav_ringacc_list);
  40. static void ringacc_writel(u32 v, void __iomem *reg)
  41. {
  42. pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
  43. writel(v, reg);
  44. }
  45. static u32 ringacc_readl(void __iomem *reg)
  46. {
  47. u32 v;
  48. v = readl(reg);
  49. pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
  50. return v;
  51. }
  52. #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
  53. #define K3_DMARING_RING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0)
  54. /**
  55. * struct k3_nav_ring_rt_regs - The RA Control/Status Registers region
  56. */
  57. struct k3_nav_ring_rt_regs {
  58. u32 resv_16[4];
  59. u32 db; /* RT Ring N Doorbell Register */
  60. u32 resv_4[1];
  61. u32 occ; /* RT Ring N Occupancy Register */
  62. u32 indx; /* RT Ring N Current Index Register */
  63. u32 hwocc; /* RT Ring N Hardware Occupancy Register */
  64. u32 hwindx; /* RT Ring N Current Index Register */
  65. };
  66. #define KNAV_RINGACC_RT_REGS_STEP 0x1000
  67. #define K3_DMARING_RING_RT_REGS_STEP 0x2000
  68. #define K3_DMARING_RING_RT_REGS_REVERSE_OFS 0x1000
  69. #define KNAV_RINGACC_RT_OCC_MASK GENMASK(20, 0)
  70. #define K3_DMARING_RING_RT_OCC_TDOWN_COMPLETE BIT(31)
  71. #define K3_DMARING_RING_RT_DB_ENTRY_MASK GENMASK(7, 0)
  72. #define K3_DMARING_RING_RT_DB_TDOWN_ACK BIT(31)
  73. /**
  74. * struct k3_nav_ring_fifo_regs - The Ring Accelerator Queues Registers region
  75. */
  76. struct k3_nav_ring_fifo_regs {
  77. u32 head_data[128]; /* Ring Head Entry Data Registers */
  78. u32 tail_data[128]; /* Ring Tail Entry Data Registers */
  79. u32 peek_head_data[128]; /* Ring Peek Head Entry Data Regs */
  80. u32 peek_tail_data[128]; /* Ring Peek Tail Entry Data Regs */
  81. };
  82. #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
  83. #define KNAV_RINGACC_FIFO_REGS_STEP 0x1000
  84. #define KNAV_RINGACC_MAX_DB_RING_CNT (127U)
  85. /**
  86. * struct k3_nav_ring_ops - Ring operations
  87. */
  88. struct k3_nav_ring_ops {
  89. int (*push_tail)(struct k3_nav_ring *ring, void *elm);
  90. int (*push_head)(struct k3_nav_ring *ring, void *elm);
  91. int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
  92. int (*pop_head)(struct k3_nav_ring *ring, void *elm);
  93. };
  94. /**
  95. * struct k3_nav_ring_state - Internal state tracking structure
  96. *
  97. * @free: Number of free entries
  98. * @occ: Occupancy
  99. * @windex: Write index
  100. * @rindex: Read index
  101. */
  102. struct k3_nav_ring_state {
  103. u32 free;
  104. u32 occ;
  105. u32 windex;
  106. u32 rindex;
  107. u32 tdown_complete:1;
  108. };
  109. /**
  110. * struct k3_nav_ring - RA Ring descriptor
  111. *
  112. * @cfg - Ring configuration registers
  113. * @rt - Ring control/status registers
  114. * @fifos - Ring queues registers
  115. * @ring_mem_dma - Ring buffer dma address
  116. * @ring_mem_virt - Ring buffer virt address
  117. * @ops - Ring operations
  118. * @size - Ring size in elements
  119. * @elm_size - Size of the ring element
  120. * @mode - Ring mode
  121. * @flags - flags
  122. * @ring_id - Ring Id
  123. * @parent - Pointer on struct @k3_nav_ringacc
  124. * @use_count - Use count for shared rings
  125. */
  126. struct k3_nav_ring {
  127. struct k3_nav_ring_cfg_regs __iomem *cfg;
  128. struct k3_nav_ring_rt_regs __iomem *rt;
  129. struct k3_nav_ring_fifo_regs __iomem *fifos;
  130. dma_addr_t ring_mem_dma;
  131. void *ring_mem_virt;
  132. struct k3_nav_ring_ops *ops;
  133. u32 size;
  134. enum k3_nav_ring_size elm_size;
  135. enum k3_nav_ring_mode mode;
  136. u32 flags;
  137. #define KNAV_RING_FLAG_BUSY BIT(1)
  138. #define K3_NAV_RING_FLAG_SHARED BIT(2)
  139. #define K3_NAV_RING_FLAG_REVERSE BIT(3)
  140. struct k3_nav_ring_state state;
  141. u32 ring_id;
  142. struct k3_nav_ringacc *parent;
  143. u32 use_count;
  144. };
  145. struct k3_nav_ringacc_ops {
  146. int (*init)(struct udevice *dev, struct k3_nav_ringacc *ringacc);
  147. };
  148. /**
  149. * struct k3_nav_ringacc - Rings accelerator descriptor
  150. *
  151. * @dev - pointer on RA device
  152. * @num_rings - number of ring in RA
  153. * @rm_gp_range - general purpose rings range from tisci
  154. * @dma_ring_reset_quirk - DMA reset w/a enable
  155. * @num_proxies - number of RA proxies
  156. * @rings - array of rings descriptors (struct @k3_nav_ring)
  157. * @list - list of RAs in the system
  158. * @tisci - pointer ti-sci handle
  159. * @tisci_ring_ops - ti-sci rings ops
  160. * @tisci_dev_id - ti-sci device id
  161. * @ops: SoC specific ringacc operation
  162. * @dual_ring: indicate k3_dmaring dual ring support
  163. */
  164. struct k3_nav_ringacc {
  165. struct udevice *dev;
  166. u32 num_rings; /* number of rings in Ringacc module */
  167. unsigned long *rings_inuse;
  168. struct ti_sci_resource *rm_gp_range;
  169. bool dma_ring_reset_quirk;
  170. u32 num_proxies;
  171. struct k3_nav_ring *rings;
  172. struct list_head list;
  173. const struct ti_sci_handle *tisci;
  174. const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
  175. u32 tisci_dev_id;
  176. const struct k3_nav_ringacc_ops *ops;
  177. bool dual_ring;
  178. };
  179. #include "k3-navss-ringacc-u-boot.c"
  180. static int k3_nav_ringacc_ring_read_occ(struct k3_nav_ring *ring)
  181. {
  182. return readl(&ring->rt->occ) & KNAV_RINGACC_RT_OCC_MASK;
  183. }
  184. static void k3_nav_ringacc_ring_update_occ(struct k3_nav_ring *ring)
  185. {
  186. u32 val;
  187. val = readl(&ring->rt->occ);
  188. ring->state.occ = val & KNAV_RINGACC_RT_OCC_MASK;
  189. ring->state.tdown_complete = !!(val & K3_DMARING_RING_RT_OCC_TDOWN_COMPLETE);
  190. }
  191. static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
  192. {
  193. return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
  194. }
  195. static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
  196. static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
  197. static int k3_dmaring_ring_fwd_pop_mem(struct k3_nav_ring *ring, void *elem);
  198. static int k3_dmaring_ring_reverse_pop_mem(struct k3_nav_ring *ring, void *elem);
  199. static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
  200. .push_tail = k3_nav_ringacc_ring_push_mem,
  201. .pop_head = k3_nav_ringacc_ring_pop_mem,
  202. };
  203. static struct k3_nav_ring_ops k3_dmaring_fwd_ring_ops = {
  204. .push_tail = k3_nav_ringacc_ring_push_mem,
  205. .pop_head = k3_dmaring_ring_fwd_pop_mem,
  206. };
  207. static struct k3_nav_ring_ops k3_dmaring_reverse_ring_ops = {
  208. .pop_head = k3_dmaring_ring_reverse_pop_mem,
  209. };
  210. struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
  211. {
  212. return ringacc->dev;
  213. }
  214. struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
  215. int id)
  216. {
  217. if (id == K3_NAV_RINGACC_RING_ID_ANY) {
  218. /* Request for any general purpose ring */
  219. struct ti_sci_resource_desc *gp_rings =
  220. &ringacc->rm_gp_range->desc[0];
  221. unsigned long size;
  222. size = gp_rings->start + gp_rings->num;
  223. id = find_next_zero_bit(ringacc->rings_inuse,
  224. size, gp_rings->start);
  225. if (id == size)
  226. goto error;
  227. } else if (id < 0) {
  228. goto error;
  229. }
  230. if (test_bit(id, ringacc->rings_inuse) &&
  231. !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
  232. goto error;
  233. else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
  234. goto out;
  235. if (!try_module_get(ringacc->dev->driver->owner))
  236. goto error;
  237. pr_debug("Giving ring#%d\n", id);
  238. set_bit(id, ringacc->rings_inuse);
  239. out:
  240. ringacc->rings[id].use_count++;
  241. return &ringacc->rings[id];
  242. error:
  243. return NULL;
  244. }
  245. static int k3_dmaring_ring_request_rings_pair(struct k3_nav_ringacc *ringacc,
  246. int fwd_id, int compl_id,
  247. struct k3_nav_ring **fwd_ring,
  248. struct k3_nav_ring **compl_ring)
  249. {
  250. /* k3_dmaring: fwd_id == compl_id, so we ignore compl_id */
  251. if (fwd_id < 0)
  252. return -EINVAL;
  253. if (test_bit(fwd_id, ringacc->rings_inuse))
  254. return -EBUSY;
  255. *fwd_ring = &ringacc->rings[fwd_id];
  256. *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
  257. set_bit(fwd_id, ringacc->rings_inuse);
  258. ringacc->rings[fwd_id].use_count++;
  259. dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
  260. return 0;
  261. }
  262. int k3_nav_ringacc_request_rings_pair(struct k3_nav_ringacc *ringacc,
  263. int fwd_id, int compl_id,
  264. struct k3_nav_ring **fwd_ring,
  265. struct k3_nav_ring **compl_ring)
  266. {
  267. int ret = 0;
  268. if (!fwd_ring || !compl_ring)
  269. return -EINVAL;
  270. if (ringacc->dual_ring)
  271. return k3_dmaring_ring_request_rings_pair(ringacc, fwd_id, compl_id,
  272. fwd_ring, compl_ring);
  273. *fwd_ring = k3_nav_ringacc_request_ring(ringacc, fwd_id);
  274. if (!(*fwd_ring))
  275. return -ENODEV;
  276. *compl_ring = k3_nav_ringacc_request_ring(ringacc, compl_id);
  277. if (!(*compl_ring)) {
  278. k3_nav_ringacc_ring_free(*fwd_ring);
  279. ret = -ENODEV;
  280. }
  281. return ret;
  282. }
  283. static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
  284. {
  285. struct k3_nav_ringacc *ringacc = ring->parent;
  286. int ret;
  287. if (IS_ENABLED(CONFIG_K3_DM_FW))
  288. return k3_ringacc_ring_reset_raw(ring);
  289. ret = ringacc->tisci_ring_ops->config(
  290. ringacc->tisci,
  291. TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
  292. ringacc->tisci_dev_id,
  293. ring->ring_id,
  294. 0,
  295. 0,
  296. ring->size,
  297. 0,
  298. 0,
  299. 0);
  300. if (ret)
  301. dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
  302. ret, ring->ring_id);
  303. }
  304. void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
  305. {
  306. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  307. return;
  308. memset(&ring->state, 0, sizeof(ring->state));
  309. k3_ringacc_ring_reset_sci(ring);
  310. }
  311. static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
  312. enum k3_nav_ring_mode mode)
  313. {
  314. struct k3_nav_ringacc *ringacc = ring->parent;
  315. int ret;
  316. if (IS_ENABLED(CONFIG_K3_DM_FW))
  317. return k3_ringacc_ring_reconfig_qmode_raw(ring, mode);
  318. ret = ringacc->tisci_ring_ops->config(
  319. ringacc->tisci,
  320. TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
  321. ringacc->tisci_dev_id,
  322. ring->ring_id,
  323. 0,
  324. 0,
  325. 0,
  326. mode,
  327. 0,
  328. 0);
  329. if (ret)
  330. dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
  331. ret, ring->ring_id);
  332. }
  333. void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
  334. {
  335. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  336. return;
  337. if (!ring->parent->dma_ring_reset_quirk) {
  338. k3_nav_ringacc_ring_reset(ring);
  339. return;
  340. }
  341. if (!occ)
  342. occ = ringacc_readl(&ring->rt->occ);
  343. if (occ) {
  344. u32 db_ring_cnt, db_ring_cnt_cur;
  345. pr_debug("%s %u occ: %u\n", __func__,
  346. ring->ring_id, occ);
  347. /* 2. Reset the ring */
  348. k3_ringacc_ring_reset_sci(ring);
  349. /*
  350. * 3. Setup the ring in ring/doorbell mode
  351. * (if not already in this mode)
  352. */
  353. if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
  354. k3_ringacc_ring_reconfig_qmode_sci(
  355. ring, K3_NAV_RINGACC_RING_MODE_RING);
  356. /*
  357. * 4. Ring the doorbell 2**22 – ringOcc times.
  358. * This will wrap the internal UDMAP ring state occupancy
  359. * counter (which is 21-bits wide) to 0.
  360. */
  361. db_ring_cnt = (1U << 22) - occ;
  362. while (db_ring_cnt != 0) {
  363. /*
  364. * Ring the doorbell with the maximum count each
  365. * iteration if possible to minimize the total
  366. * of writes
  367. */
  368. if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
  369. db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
  370. else
  371. db_ring_cnt_cur = db_ring_cnt;
  372. writel(db_ring_cnt_cur, &ring->rt->db);
  373. db_ring_cnt -= db_ring_cnt_cur;
  374. }
  375. /* 5. Restore the original ring mode (if not ring mode) */
  376. if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
  377. k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
  378. }
  379. /* 2. Reset the ring */
  380. k3_nav_ringacc_ring_reset(ring);
  381. }
  382. static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
  383. {
  384. struct k3_nav_ringacc *ringacc = ring->parent;
  385. int ret;
  386. if (IS_ENABLED(CONFIG_K3_DM_FW))
  387. return k3_ringacc_ring_free_raw(ring);
  388. ret = ringacc->tisci_ring_ops->config(
  389. ringacc->tisci,
  390. TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
  391. ringacc->tisci_dev_id,
  392. ring->ring_id,
  393. 0,
  394. 0,
  395. 0,
  396. 0,
  397. 0,
  398. 0);
  399. if (ret)
  400. dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
  401. ret, ring->ring_id);
  402. }
  403. int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
  404. {
  405. struct k3_nav_ringacc *ringacc;
  406. if (!ring)
  407. return -EINVAL;
  408. ringacc = ring->parent;
  409. /*
  410. * k3_dmaring: rings shared memory and configuration, only forward ring is
  411. * configured and reverse ring considered as slave.
  412. */
  413. if (ringacc->dual_ring && (ring->flags & K3_NAV_RING_FLAG_REVERSE))
  414. return 0;
  415. pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
  416. if (!test_bit(ring->ring_id, ringacc->rings_inuse))
  417. return -EINVAL;
  418. if (--ring->use_count)
  419. goto out;
  420. if (!(ring->flags & KNAV_RING_FLAG_BUSY))
  421. goto no_init;
  422. k3_ringacc_ring_free_sci(ring);
  423. dma_free_coherent(ringacc->dev,
  424. ring->size * (4 << ring->elm_size),
  425. ring->ring_mem_virt, ring->ring_mem_dma);
  426. ring->flags &= ~KNAV_RING_FLAG_BUSY;
  427. ring->ops = NULL;
  428. no_init:
  429. clear_bit(ring->ring_id, ringacc->rings_inuse);
  430. module_put(ringacc->dev->driver->owner);
  431. out:
  432. return 0;
  433. }
  434. u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
  435. {
  436. if (!ring)
  437. return -EINVAL;
  438. return ring->ring_id;
  439. }
  440. static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
  441. {
  442. struct k3_nav_ringacc *ringacc = ring->parent;
  443. u32 ring_idx;
  444. int ret;
  445. if (!ringacc->tisci)
  446. return -EINVAL;
  447. ring_idx = ring->ring_id;
  448. ret = ringacc->tisci_ring_ops->config(
  449. ringacc->tisci,
  450. TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
  451. ringacc->tisci_dev_id,
  452. ring_idx,
  453. lower_32_bits(ring->ring_mem_dma),
  454. upper_32_bits(ring->ring_mem_dma),
  455. ring->size,
  456. ring->mode,
  457. ring->elm_size,
  458. 0);
  459. if (ret) {
  460. dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
  461. ret, ring_idx);
  462. return ret;
  463. }
  464. /*
  465. * Above TI SCI call handles firewall configuration, cfg
  466. * register configuration still has to be done locally in
  467. * absence of RM services.
  468. */
  469. if (IS_ENABLED(CONFIG_K3_DM_FW))
  470. k3_nav_ringacc_ring_cfg_raw(ring);
  471. return 0;
  472. }
  473. static int k3_dmaring_ring_cfg(struct k3_nav_ring *ring, struct k3_nav_ring_cfg *cfg)
  474. {
  475. struct k3_nav_ringacc *ringacc;
  476. struct k3_nav_ring *reverse_ring;
  477. int ret = 0;
  478. if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
  479. cfg->mode != K3_NAV_RINGACC_RING_MODE_RING ||
  480. cfg->size & ~K3_DMARING_RING_CFG_RING_SIZE_ELCNT_MASK)
  481. return -EINVAL;
  482. ringacc = ring->parent;
  483. /*
  484. * k3_dmaring: rings shared memory and configuration, only forward ring is
  485. * configured and reverse ring considered as slave.
  486. */
  487. if (ringacc->dual_ring && (ring->flags & K3_NAV_RING_FLAG_REVERSE))
  488. return 0;
  489. if (!test_bit(ring->ring_id, ringacc->rings_inuse))
  490. return -EINVAL;
  491. ring->size = cfg->size;
  492. ring->elm_size = cfg->elm_size;
  493. ring->mode = cfg->mode;
  494. memset(&ring->state, 0, sizeof(ring->state));
  495. ring->ops = &k3_dmaring_fwd_ring_ops;
  496. ring->ring_mem_virt =
  497. dma_alloc_coherent(ring->size * (4 << ring->elm_size),
  498. (unsigned long *)&ring->ring_mem_dma);
  499. if (!ring->ring_mem_virt) {
  500. dev_err(ringacc->dev, "Failed to alloc ring mem\n");
  501. ret = -ENOMEM;
  502. goto err_free_ops;
  503. }
  504. ret = k3_nav_ringacc_ring_cfg_sci(ring);
  505. if (ret)
  506. goto err_free_mem;
  507. ring->flags |= KNAV_RING_FLAG_BUSY;
  508. /* k3_dmaring: configure reverse ring */
  509. reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
  510. reverse_ring->size = cfg->size;
  511. reverse_ring->elm_size = cfg->elm_size;
  512. reverse_ring->mode = cfg->mode;
  513. memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
  514. reverse_ring->ops = &k3_dmaring_reverse_ring_ops;
  515. reverse_ring->ring_mem_virt = ring->ring_mem_virt;
  516. reverse_ring->ring_mem_dma = ring->ring_mem_dma;
  517. reverse_ring->flags |= KNAV_RING_FLAG_BUSY;
  518. return 0;
  519. err_free_mem:
  520. dma_free_coherent(ringacc->dev,
  521. ring->size * (4 << ring->elm_size),
  522. ring->ring_mem_virt,
  523. ring->ring_mem_dma);
  524. err_free_ops:
  525. ring->ops = NULL;
  526. return ret;
  527. }
  528. int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
  529. struct k3_nav_ring_cfg *cfg)
  530. {
  531. struct k3_nav_ringacc *ringacc = ring->parent;
  532. int ret = 0;
  533. if (!ring || !cfg)
  534. return -EINVAL;
  535. if (ringacc->dual_ring)
  536. return k3_dmaring_ring_cfg(ring, cfg);
  537. if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
  538. cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
  539. cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
  540. !test_bit(ring->ring_id, ringacc->rings_inuse))
  541. return -EINVAL;
  542. if (ring->use_count != 1)
  543. return 0;
  544. ring->size = cfg->size;
  545. ring->elm_size = cfg->elm_size;
  546. ring->mode = cfg->mode;
  547. memset(&ring->state, 0, sizeof(ring->state));
  548. switch (ring->mode) {
  549. case K3_NAV_RINGACC_RING_MODE_RING:
  550. ring->ops = &k3_nav_mode_ring_ops;
  551. break;
  552. default:
  553. ring->ops = NULL;
  554. ret = -EINVAL;
  555. goto err_free_ops;
  556. };
  557. ring->ring_mem_virt =
  558. dma_zalloc_coherent(ringacc->dev,
  559. ring->size * (4 << ring->elm_size),
  560. &ring->ring_mem_dma, GFP_KERNEL);
  561. if (!ring->ring_mem_virt) {
  562. dev_err(ringacc->dev, "Failed to alloc ring mem\n");
  563. ret = -ENOMEM;
  564. goto err_free_ops;
  565. }
  566. ret = k3_nav_ringacc_ring_cfg_sci(ring);
  567. if (ret)
  568. goto err_free_mem;
  569. ring->flags |= KNAV_RING_FLAG_BUSY;
  570. ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
  571. K3_NAV_RING_FLAG_SHARED : 0;
  572. return 0;
  573. err_free_mem:
  574. dma_free_coherent(ringacc->dev,
  575. ring->size * (4 << ring->elm_size),
  576. ring->ring_mem_virt,
  577. ring->ring_mem_dma);
  578. err_free_ops:
  579. ring->ops = NULL;
  580. return ret;
  581. }
  582. u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
  583. {
  584. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  585. return -EINVAL;
  586. return ring->size;
  587. }
  588. u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
  589. {
  590. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  591. return -EINVAL;
  592. if (!ring->state.free)
  593. ring->state.free = ring->size - ringacc_readl(&ring->rt->occ);
  594. return ring->state.free;
  595. }
  596. u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
  597. {
  598. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  599. return -EINVAL;
  600. return ringacc_readl(&ring->rt->occ);
  601. }
  602. u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
  603. {
  604. return !k3_nav_ringacc_ring_get_free(ring);
  605. }
  606. enum k3_ringacc_access_mode {
  607. K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
  608. K3_RINGACC_ACCESS_MODE_POP_HEAD,
  609. K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
  610. K3_RINGACC_ACCESS_MODE_POP_TAIL,
  611. K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
  612. K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
  613. };
  614. static int k3_dmaring_ring_fwd_pop_mem(struct k3_nav_ring *ring, void *elem)
  615. {
  616. void *elem_ptr;
  617. u32 elem_idx;
  618. /*
  619. * k3_dmaring: forward ring is always tied DMA channel and HW does not
  620. * maintain any state data required for POP operation and its unknown
  621. * how much elements were consumed by HW. So, to actually
  622. * do POP, the read pointer has to be recalculated every time.
  623. */
  624. ring->state.occ = k3_nav_ringacc_ring_read_occ(ring);
  625. if (ring->state.windex >= ring->state.occ)
  626. elem_idx = ring->state.windex - ring->state.occ;
  627. else
  628. elem_idx = ring->size - (ring->state.occ - ring->state.windex);
  629. elem_ptr = k3_nav_ringacc_get_elm_addr(ring, elem_idx);
  630. invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
  631. ALIGN((unsigned long)ring->ring_mem_virt +
  632. ring->size * (4 << ring->elm_size),
  633. ARCH_DMA_MINALIGN));
  634. memcpy(elem, elem_ptr, (4 << ring->elm_size));
  635. ring->state.occ--;
  636. writel(-1, &ring->rt->db);
  637. dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
  638. __func__, ring->state.occ, ring->state.windex, elem_idx,
  639. elem_ptr);
  640. return 0;
  641. }
  642. static int k3_dmaring_ring_reverse_pop_mem(struct k3_nav_ring *ring, void *elem)
  643. {
  644. void *elem_ptr;
  645. elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);
  646. if (ring->state.occ) {
  647. invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
  648. ALIGN((unsigned long)ring->ring_mem_virt +
  649. ring->size * (4 << ring->elm_size),
  650. ARCH_DMA_MINALIGN));
  651. memcpy(elem, elem_ptr, (4 << ring->elm_size));
  652. ring->state.rindex = (ring->state.rindex + 1) % ring->size;
  653. ring->state.occ--;
  654. writel(-1 & K3_DMARING_RING_RT_DB_ENTRY_MASK, &ring->rt->db);
  655. }
  656. dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
  657. __func__, ring->state.occ, ring->state.rindex, elem_ptr);
  658. return 0;
  659. }
  660. static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
  661. {
  662. void *elem_ptr;
  663. elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.windex);
  664. memcpy(elem_ptr, elem, (4 << ring->elm_size));
  665. flush_dcache_range((unsigned long)ring->ring_mem_virt,
  666. ALIGN((unsigned long)ring->ring_mem_virt +
  667. ring->size * (4 << ring->elm_size),
  668. ARCH_DMA_MINALIGN));
  669. ring->state.windex = (ring->state.windex + 1) % ring->size;
  670. ring->state.free--;
  671. ringacc_writel(1, &ring->rt->db);
  672. pr_debug("ring_push_mem: free%d index%d\n",
  673. ring->state.free, ring->state.windex);
  674. return 0;
  675. }
  676. static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
  677. {
  678. void *elem_ptr;
  679. elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);
  680. invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
  681. ALIGN((unsigned long)ring->ring_mem_virt +
  682. ring->size * (4 << ring->elm_size),
  683. ARCH_DMA_MINALIGN));
  684. memcpy(elem, elem_ptr, (4 << ring->elm_size));
  685. ring->state.rindex = (ring->state.rindex + 1) % ring->size;
  686. ring->state.occ--;
  687. ringacc_writel(-1, &ring->rt->db);
  688. pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
  689. ring->state.occ, ring->state.rindex, elem_ptr);
  690. return 0;
  691. }
  692. int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
  693. {
  694. int ret = -EOPNOTSUPP;
  695. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  696. return -EINVAL;
  697. pr_debug("ring_push%d: free%d index%d\n",
  698. ring->ring_id, ring->state.free, ring->state.windex);
  699. if (k3_nav_ringacc_ring_is_full(ring))
  700. return -ENOMEM;
  701. if (ring->ops && ring->ops->push_tail)
  702. ret = ring->ops->push_tail(ring, elem);
  703. return ret;
  704. }
  705. int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
  706. {
  707. int ret = -EOPNOTSUPP;
  708. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  709. return -EINVAL;
  710. pr_debug("ring_push_head: free%d index%d\n",
  711. ring->state.free, ring->state.windex);
  712. if (k3_nav_ringacc_ring_is_full(ring))
  713. return -ENOMEM;
  714. if (ring->ops && ring->ops->push_head)
  715. ret = ring->ops->push_head(ring, elem);
  716. return ret;
  717. }
  718. int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
  719. {
  720. int ret = -EOPNOTSUPP;
  721. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  722. return -EINVAL;
  723. if (!ring->state.occ)
  724. k3_nav_ringacc_ring_update_occ(ring);
  725. pr_debug("ring_pop%d: occ%d index%d\n",
  726. ring->ring_id, ring->state.occ, ring->state.rindex);
  727. if (!ring->state.occ && !ring->state.tdown_complete)
  728. return -ENODATA;
  729. if (ring->ops && ring->ops->pop_head)
  730. ret = ring->ops->pop_head(ring, elem);
  731. return ret;
  732. }
  733. int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
  734. {
  735. int ret = -EOPNOTSUPP;
  736. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  737. return -EINVAL;
  738. if (!ring->state.occ)
  739. k3_nav_ringacc_ring_update_occ(ring);
  740. pr_debug("ring_pop_tail: occ%d index%d\n",
  741. ring->state.occ, ring->state.rindex);
  742. if (!ring->state.occ)
  743. return -ENODATA;
  744. if (ring->ops && ring->ops->pop_tail)
  745. ret = ring->ops->pop_tail(ring, elem);
  746. return ret;
  747. }
  748. static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
  749. {
  750. struct udevice *dev = ringacc->dev;
  751. struct udevice *devp = dev;
  752. struct udevice *tisci_dev = NULL;
  753. int ret;
  754. ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
  755. if (!ringacc->num_rings) {
  756. dev_err(dev, "ti,num-rings read failure %d\n", ret);
  757. return -EINVAL;
  758. }
  759. ringacc->dma_ring_reset_quirk =
  760. dev_read_bool(dev, "ti,dma-ring-reset-quirk");
  761. ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, devp,
  762. "ti,sci", &tisci_dev);
  763. if (ret) {
  764. pr_debug("TISCI RA RM get failed (%d)\n", ret);
  765. ringacc->tisci = NULL;
  766. return -ENODEV;
  767. }
  768. ringacc->tisci = (struct ti_sci_handle *)
  769. (ti_sci_get_handle_from_sysfw(tisci_dev));
  770. ret = dev_read_u32_default(devp, "ti,sci", 0);
  771. if (!ret) {
  772. dev_err(dev, "TISCI RA RM disabled\n");
  773. ringacc->tisci = NULL;
  774. return ret;
  775. }
  776. ret = dev_read_u32(devp, "ti,sci-dev-id", &ringacc->tisci_dev_id);
  777. if (ret) {
  778. dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
  779. ringacc->tisci = NULL;
  780. return ret;
  781. }
  782. ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
  783. ringacc->tisci, dev,
  784. ringacc->tisci_dev_id,
  785. "ti,sci-rm-range-gp-rings");
  786. if (IS_ERR(ringacc->rm_gp_range))
  787. ret = PTR_ERR(ringacc->rm_gp_range);
  788. return 0;
  789. }
  790. static int k3_nav_ringacc_init(struct udevice *dev, struct k3_nav_ringacc *ringacc)
  791. {
  792. void __iomem *base_cfg, *base_rt;
  793. int ret, i;
  794. ret = k3_nav_ringacc_probe_dt(ringacc);
  795. if (ret)
  796. return ret;
  797. base_cfg = dev_remap_addr_name(dev, "cfg");
  798. pr_debug("cfg %p\n", base_cfg);
  799. if (!base_cfg)
  800. return -EINVAL;
  801. base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
  802. pr_debug("rt %p\n", base_rt);
  803. if (IS_ERR(base_rt))
  804. return PTR_ERR(base_rt);
  805. ringacc->rings = devm_kzalloc(dev,
  806. sizeof(*ringacc->rings) *
  807. ringacc->num_rings,
  808. GFP_KERNEL);
  809. ringacc->rings_inuse = devm_kcalloc(dev,
  810. BITS_TO_LONGS(ringacc->num_rings),
  811. sizeof(unsigned long), GFP_KERNEL);
  812. if (!ringacc->rings || !ringacc->rings_inuse)
  813. return -ENOMEM;
  814. for (i = 0; i < ringacc->num_rings; i++) {
  815. ringacc->rings[i].cfg = base_cfg +
  816. KNAV_RINGACC_CFG_REGS_STEP * i;
  817. ringacc->rings[i].rt = base_rt +
  818. KNAV_RINGACC_RT_REGS_STEP * i;
  819. ringacc->rings[i].parent = ringacc;
  820. ringacc->rings[i].ring_id = i;
  821. }
  822. dev_set_drvdata(dev, ringacc);
  823. ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
  824. list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
  825. dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
  826. ringacc->num_rings,
  827. ringacc->rm_gp_range->desc[0].start,
  828. ringacc->rm_gp_range->desc[0].num,
  829. ringacc->tisci_dev_id);
  830. dev_info(dev, "dma-ring-reset-quirk: %s\n",
  831. ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
  832. return 0;
  833. }
  834. struct k3_nav_ringacc *k3_ringacc_dmarings_init(struct udevice *dev,
  835. struct k3_ringacc_init_data *data)
  836. {
  837. struct k3_nav_ringacc *ringacc;
  838. void __iomem *base_rt;
  839. int i;
  840. ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
  841. if (!ringacc)
  842. return ERR_PTR(-ENOMEM);
  843. ringacc->dual_ring = true;
  844. ringacc->dev = dev;
  845. ringacc->num_rings = data->num_rings;
  846. ringacc->tisci = data->tisci;
  847. ringacc->tisci_dev_id = data->tisci_dev_id;
  848. base_rt = (uint32_t *)devfdt_get_addr_name(dev, "ringrt");
  849. if (IS_ERR(base_rt))
  850. return base_rt;
  851. ringacc->rings = devm_kzalloc(dev,
  852. sizeof(*ringacc->rings) *
  853. ringacc->num_rings * 2,
  854. GFP_KERNEL);
  855. ringacc->rings_inuse = devm_kcalloc(dev,
  856. BITS_TO_LONGS(ringacc->num_rings),
  857. sizeof(unsigned long), GFP_KERNEL);
  858. if (!ringacc->rings || !ringacc->rings_inuse)
  859. return ERR_PTR(-ENOMEM);
  860. for (i = 0; i < ringacc->num_rings; i++) {
  861. struct k3_nav_ring *ring = &ringacc->rings[i];
  862. ring->rt = base_rt + K3_DMARING_RING_RT_REGS_STEP * i;
  863. ring->parent = ringacc;
  864. ring->ring_id = i;
  865. ring = &ringacc->rings[ringacc->num_rings + i];
  866. ring->rt = base_rt + K3_DMARING_RING_RT_REGS_STEP * i +
  867. K3_DMARING_RING_RT_REGS_REVERSE_OFS;
  868. ring->parent = ringacc;
  869. ring->ring_id = i;
  870. ring->flags = K3_NAV_RING_FLAG_REVERSE;
  871. }
  872. ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
  873. dev_info(dev, "k3_dmaring Ring probed rings:%u, sci-dev-id:%u\n",
  874. ringacc->num_rings,
  875. ringacc->tisci_dev_id);
  876. dev_info(dev, "dma-ring-reset-quirk: %s\n",
  877. ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
  878. return ringacc;
  879. }
  880. struct ringacc_match_data {
  881. struct k3_nav_ringacc_ops ops;
  882. };
  883. static struct ringacc_match_data k3_nav_ringacc_data = {
  884. .ops = {
  885. .init = k3_nav_ringacc_init,
  886. },
  887. };
  888. static const struct udevice_id knav_ringacc_ids[] = {
  889. { .compatible = "ti,am654-navss-ringacc", .data = (ulong)&k3_nav_ringacc_data, },
  890. {},
  891. };
  892. static int k3_nav_ringacc_probe(struct udevice *dev)
  893. {
  894. struct k3_nav_ringacc *ringacc;
  895. int ret;
  896. const struct ringacc_match_data *match_data;
  897. match_data = (struct ringacc_match_data *)dev_get_driver_data(dev);
  898. ringacc = dev_get_priv(dev);
  899. if (!ringacc)
  900. return -ENOMEM;
  901. ringacc->dev = dev;
  902. ringacc->ops = &match_data->ops;
  903. ret = ringacc->ops->init(dev, ringacc);
  904. if (ret)
  905. return ret;
  906. return 0;
  907. }
  908. U_BOOT_DRIVER(k3_navss_ringacc) = {
  909. .name = "k3-navss-ringacc",
  910. .id = UCLASS_MISC,
  911. .of_match = knav_ringacc_ids,
  912. .probe = k3_nav_ringacc_probe,
  913. .priv_auto = sizeof(struct k3_nav_ringacc),
  914. };