k3-navss-ringacc.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
  4. *
  5. * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
  6. */
  7. #include <common.h>
  8. #include <cpu_func.h>
  9. #include <log.h>
  10. #include <asm/cache.h>
  11. #include <asm/io.h>
  12. #include <malloc.h>
  13. #include <asm/bitops.h>
  14. #include <dm.h>
  15. #include <dm/device_compat.h>
  16. #include <dm/devres.h>
  17. #include <dm/read.h>
  18. #include <dm/uclass.h>
  19. #include <linux/bitops.h>
  20. #include <linux/compat.h>
  21. #include <linux/dma-mapping.h>
  22. #include <linux/err.h>
  23. #include <linux/soc/ti/k3-navss-ringacc.h>
  24. #include <linux/soc/ti/ti_sci_protocol.h>
  25. #define set_bit(bit, bitmap) __set_bit(bit, bitmap)
  26. #define clear_bit(bit, bitmap) __clear_bit(bit, bitmap)
  27. #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
  28. dma_free_coherent(cpu_addr)
  29. #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
  30. ({ \
  31. void *ring_mem_virt; \
  32. ring_mem_virt = dma_alloc_coherent((size), \
  33. (unsigned long *)(dma_handle)); \
  34. if (ring_mem_virt) \
  35. memset(ring_mem_virt, 0, (size)); \
  36. ring_mem_virt; \
  37. })
  38. static LIST_HEAD(k3_nav_ringacc_list);
  39. static void ringacc_writel(u32 v, void __iomem *reg)
  40. {
  41. pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
  42. writel(v, reg);
  43. }
  44. static u32 ringacc_readl(void __iomem *reg)
  45. {
  46. u32 v;
  47. v = readl(reg);
  48. pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
  49. return v;
  50. }
  51. #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
  52. /**
  53. * struct k3_nav_ring_rt_regs - The RA Control/Status Registers region
  54. */
  55. struct k3_nav_ring_rt_regs {
  56. u32 resv_16[4];
  57. u32 db; /* RT Ring N Doorbell Register */
  58. u32 resv_4[1];
  59. u32 occ; /* RT Ring N Occupancy Register */
  60. u32 indx; /* RT Ring N Current Index Register */
  61. u32 hwocc; /* RT Ring N Hardware Occupancy Register */
  62. u32 hwindx; /* RT Ring N Current Index Register */
  63. };
  64. #define KNAV_RINGACC_RT_REGS_STEP 0x1000
  65. /**
  66. * struct k3_nav_ring_fifo_regs - The Ring Accelerator Queues Registers region
  67. */
  68. struct k3_nav_ring_fifo_regs {
  69. u32 head_data[128]; /* Ring Head Entry Data Registers */
  70. u32 tail_data[128]; /* Ring Tail Entry Data Registers */
  71. u32 peek_head_data[128]; /* Ring Peek Head Entry Data Regs */
  72. u32 peek_tail_data[128]; /* Ring Peek Tail Entry Data Regs */
  73. };
  74. /**
  75. * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
  76. */
  77. struct k3_ringacc_proxy_gcfg_regs {
  78. u32 revision; /* Revision Register */
  79. u32 config; /* Config Register */
  80. };
  81. #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
  82. /**
  83. * struct k3_ringacc_proxy_target_regs - RA Proxy Datapath MMIO Region
  84. */
  85. struct k3_ringacc_proxy_target_regs {
  86. u32 control; /* Proxy Control Register */
  87. u32 status; /* Proxy Status Register */
  88. u8 resv_512[504];
  89. u32 data[128]; /* Proxy Data Register */
  90. };
  91. #define K3_RINGACC_PROXY_TARGET_STEP 0x1000
  92. #define K3_RINGACC_PROXY_NOT_USED (-1)
  93. enum k3_ringacc_proxy_access_mode {
  94. PROXY_ACCESS_MODE_HEAD = 0,
  95. PROXY_ACCESS_MODE_TAIL = 1,
  96. PROXY_ACCESS_MODE_PEEK_HEAD = 2,
  97. PROXY_ACCESS_MODE_PEEK_TAIL = 3,
  98. };
  99. #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
  100. #define KNAV_RINGACC_FIFO_REGS_STEP 0x1000
  101. #define KNAV_RINGACC_MAX_DB_RING_CNT (127U)
  102. /**
  103. * struct k3_nav_ring_ops - Ring operations
  104. */
  105. struct k3_nav_ring_ops {
  106. int (*push_tail)(struct k3_nav_ring *ring, void *elm);
  107. int (*push_head)(struct k3_nav_ring *ring, void *elm);
  108. int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
  109. int (*pop_head)(struct k3_nav_ring *ring, void *elm);
  110. };
  111. /**
  112. * struct k3_nav_ring_state - Internal state tracking structure
  113. *
  114. * @free: Number of free entries
  115. * @occ: Occupancy
  116. * @windex: Write index
  117. * @rindex: Read index
  118. */
  119. struct k3_nav_ring_state {
  120. u32 free;
  121. u32 occ;
  122. u32 windex;
  123. u32 rindex;
  124. u32 tdown_complete:1;
  125. };
  126. /**
  127. * struct k3_nav_ring - RA Ring descriptor
  128. *
  129. * @rt - Ring control/status registers
  130. * @fifos - Ring queues registers
  131. * @proxy - Ring Proxy Datapath registers
  132. * @ring_mem_dma - Ring buffer dma address
  133. * @ring_mem_virt - Ring buffer virt address
  134. * @ops - Ring operations
  135. * @size - Ring size in elements
  136. * @elm_size - Size of the ring element
  137. * @mode - Ring mode
  138. * @flags - flags
  139. * @ring_id - Ring Id
  140. * @parent - Pointer on struct @k3_nav_ringacc
  141. * @use_count - Use count for shared rings
  142. * @proxy_id - RA Ring Proxy Id (only if @K3_NAV_RINGACC_RING_USE_PROXY)
  143. */
  144. struct k3_nav_ring {
  145. struct k3_nav_ring_rt_regs __iomem *rt;
  146. struct k3_nav_ring_fifo_regs __iomem *fifos;
  147. struct k3_ringacc_proxy_target_regs __iomem *proxy;
  148. dma_addr_t ring_mem_dma;
  149. void *ring_mem_virt;
  150. struct k3_nav_ring_ops *ops;
  151. u32 size;
  152. enum k3_nav_ring_size elm_size;
  153. enum k3_nav_ring_mode mode;
  154. u32 flags;
  155. #define KNAV_RING_FLAG_BUSY BIT(1)
  156. #define K3_NAV_RING_FLAG_SHARED BIT(2)
  157. struct k3_nav_ring_state state;
  158. u32 ring_id;
  159. struct k3_nav_ringacc *parent;
  160. u32 use_count;
  161. int proxy_id;
  162. };
  163. struct k3_nav_ringacc_ops {
  164. int (*init)(struct udevice *dev, struct k3_nav_ringacc *ringacc);
  165. };
  166. /**
  167. * struct k3_nav_ringacc - Rings accelerator descriptor
  168. *
  169. * @dev - pointer on RA device
  170. * @proxy_gcfg - RA proxy global config registers
  171. * @proxy_target_base - RA proxy datapath region
  172. * @num_rings - number of ring in RA
  173. * @rm_gp_range - general purpose rings range from tisci
  174. * @dma_ring_reset_quirk - DMA reset w/a enable
  175. * @num_proxies - number of RA proxies
  176. * @rings - array of rings descriptors (struct @k3_nav_ring)
  177. * @list - list of RAs in the system
  178. * @tisci - pointer ti-sci handle
  179. * @tisci_ring_ops - ti-sci rings ops
  180. * @tisci_dev_id - ti-sci device id
  181. * @ops: SoC specific ringacc operation
  182. */
  183. struct k3_nav_ringacc {
  184. struct udevice *dev;
  185. struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
  186. void __iomem *proxy_target_base;
  187. u32 num_rings; /* number of rings in Ringacc module */
  188. unsigned long *rings_inuse;
  189. struct ti_sci_resource *rm_gp_range;
  190. bool dma_ring_reset_quirk;
  191. u32 num_proxies;
  192. unsigned long *proxy_inuse;
  193. struct k3_nav_ring *rings;
  194. struct list_head list;
  195. const struct ti_sci_handle *tisci;
  196. const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
  197. u32 tisci_dev_id;
  198. const struct k3_nav_ringacc_ops *ops;
  199. };
  200. static long k3_nav_ringacc_ring_get_fifo_pos(struct k3_nav_ring *ring)
  201. {
  202. return KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES -
  203. (4 << ring->elm_size);
  204. }
  205. static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
  206. {
  207. return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
  208. }
  209. static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
  210. static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
  211. static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
  212. .push_tail = k3_nav_ringacc_ring_push_mem,
  213. .pop_head = k3_nav_ringacc_ring_pop_mem,
  214. };
  215. static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem);
  216. static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem);
  217. static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
  218. void *elem);
  219. static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring,
  220. void *elem);
  221. static struct k3_nav_ring_ops k3_nav_mode_msg_ops = {
  222. .push_tail = k3_nav_ringacc_ring_push_io,
  223. .push_head = k3_nav_ringacc_ring_push_head_io,
  224. .pop_tail = k3_nav_ringacc_ring_pop_tail_io,
  225. .pop_head = k3_nav_ringacc_ring_pop_io,
  226. };
  227. static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring,
  228. void *elem);
  229. static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring,
  230. void *elem);
  231. static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem);
  232. static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem);
  233. static struct k3_nav_ring_ops k3_nav_mode_proxy_ops = {
  234. .push_tail = k3_ringacc_ring_push_tail_proxy,
  235. .push_head = k3_ringacc_ring_push_head_proxy,
  236. .pop_tail = k3_ringacc_ring_pop_tail_proxy,
  237. .pop_head = k3_ringacc_ring_pop_head_proxy,
  238. };
  239. struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
  240. {
  241. return ringacc->dev;
  242. }
  243. struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
  244. int id, u32 flags)
  245. {
  246. int proxy_id = K3_RINGACC_PROXY_NOT_USED;
  247. if (id == K3_NAV_RINGACC_RING_ID_ANY) {
  248. /* Request for any general purpose ring */
  249. struct ti_sci_resource_desc *gp_rings =
  250. &ringacc->rm_gp_range->desc[0];
  251. unsigned long size;
  252. size = gp_rings->start + gp_rings->num;
  253. id = find_next_zero_bit(ringacc->rings_inuse,
  254. size, gp_rings->start);
  255. if (id == size)
  256. goto error;
  257. } else if (id < 0) {
  258. goto error;
  259. }
  260. if (test_bit(id, ringacc->rings_inuse) &&
  261. !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
  262. goto error;
  263. else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
  264. goto out;
  265. if (flags & K3_NAV_RINGACC_RING_USE_PROXY) {
  266. proxy_id = find_next_zero_bit(ringacc->proxy_inuse,
  267. ringacc->num_proxies, 0);
  268. if (proxy_id == ringacc->num_proxies)
  269. goto error;
  270. }
  271. if (!try_module_get(ringacc->dev->driver->owner))
  272. goto error;
  273. if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
  274. set_bit(proxy_id, ringacc->proxy_inuse);
  275. ringacc->rings[id].proxy_id = proxy_id;
  276. pr_debug("Giving ring#%d proxy#%d\n",
  277. id, proxy_id);
  278. } else {
  279. pr_debug("Giving ring#%d\n", id);
  280. }
  281. set_bit(id, ringacc->rings_inuse);
  282. out:
  283. ringacc->rings[id].use_count++;
  284. return &ringacc->rings[id];
  285. error:
  286. return NULL;
  287. }
  288. int k3_nav_ringacc_request_rings_pair(struct k3_nav_ringacc *ringacc,
  289. int fwd_id, int compl_id,
  290. struct k3_nav_ring **fwd_ring,
  291. struct k3_nav_ring **compl_ring)
  292. {
  293. int ret = 0;
  294. if (!fwd_ring || !compl_ring)
  295. return -EINVAL;
  296. *fwd_ring = k3_nav_ringacc_request_ring(ringacc, fwd_id, 0);
  297. if (!(*fwd_ring))
  298. return -ENODEV;
  299. *compl_ring = k3_nav_ringacc_request_ring(ringacc, compl_id, 0);
  300. if (!(*compl_ring)) {
  301. k3_nav_ringacc_ring_free(*fwd_ring);
  302. ret = -ENODEV;
  303. }
  304. return ret;
  305. }
  306. static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
  307. {
  308. struct k3_nav_ringacc *ringacc = ring->parent;
  309. int ret;
  310. ret = ringacc->tisci_ring_ops->config(
  311. ringacc->tisci,
  312. TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
  313. ringacc->tisci_dev_id,
  314. ring->ring_id,
  315. 0,
  316. 0,
  317. ring->size,
  318. 0,
  319. 0,
  320. 0);
  321. if (ret)
  322. dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
  323. ret, ring->ring_id);
  324. }
  325. void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
  326. {
  327. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  328. return;
  329. memset(&ring->state, 0, sizeof(ring->state));
  330. k3_ringacc_ring_reset_sci(ring);
  331. }
  332. static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
  333. enum k3_nav_ring_mode mode)
  334. {
  335. struct k3_nav_ringacc *ringacc = ring->parent;
  336. int ret;
  337. ret = ringacc->tisci_ring_ops->config(
  338. ringacc->tisci,
  339. TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
  340. ringacc->tisci_dev_id,
  341. ring->ring_id,
  342. 0,
  343. 0,
  344. 0,
  345. mode,
  346. 0,
  347. 0);
  348. if (ret)
  349. dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
  350. ret, ring->ring_id);
  351. }
  352. void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
  353. {
  354. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  355. return;
  356. if (!ring->parent->dma_ring_reset_quirk) {
  357. k3_nav_ringacc_ring_reset(ring);
  358. return;
  359. }
  360. if (!occ)
  361. occ = ringacc_readl(&ring->rt->occ);
  362. if (occ) {
  363. u32 db_ring_cnt, db_ring_cnt_cur;
  364. pr_debug("%s %u occ: %u\n", __func__,
  365. ring->ring_id, occ);
  366. /* 2. Reset the ring */
  367. k3_ringacc_ring_reset_sci(ring);
  368. /*
  369. * 3. Setup the ring in ring/doorbell mode
  370. * (if not already in this mode)
  371. */
  372. if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
  373. k3_ringacc_ring_reconfig_qmode_sci(
  374. ring, K3_NAV_RINGACC_RING_MODE_RING);
  375. /*
  376. * 4. Ring the doorbell 2**22 – ringOcc times.
  377. * This will wrap the internal UDMAP ring state occupancy
  378. * counter (which is 21-bits wide) to 0.
  379. */
  380. db_ring_cnt = (1U << 22) - occ;
  381. while (db_ring_cnt != 0) {
  382. /*
  383. * Ring the doorbell with the maximum count each
  384. * iteration if possible to minimize the total
  385. * of writes
  386. */
  387. if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
  388. db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
  389. else
  390. db_ring_cnt_cur = db_ring_cnt;
  391. writel(db_ring_cnt_cur, &ring->rt->db);
  392. db_ring_cnt -= db_ring_cnt_cur;
  393. }
  394. /* 5. Restore the original ring mode (if not ring mode) */
  395. if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
  396. k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
  397. }
  398. /* 2. Reset the ring */
  399. k3_nav_ringacc_ring_reset(ring);
  400. }
  401. static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
  402. {
  403. struct k3_nav_ringacc *ringacc = ring->parent;
  404. int ret;
  405. ret = ringacc->tisci_ring_ops->config(
  406. ringacc->tisci,
  407. TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
  408. ringacc->tisci_dev_id,
  409. ring->ring_id,
  410. 0,
  411. 0,
  412. 0,
  413. 0,
  414. 0,
  415. 0);
  416. if (ret)
  417. dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
  418. ret, ring->ring_id);
  419. }
  420. int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
  421. {
  422. struct k3_nav_ringacc *ringacc;
  423. if (!ring)
  424. return -EINVAL;
  425. ringacc = ring->parent;
  426. pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
  427. if (!test_bit(ring->ring_id, ringacc->rings_inuse))
  428. return -EINVAL;
  429. if (--ring->use_count)
  430. goto out;
  431. if (!(ring->flags & KNAV_RING_FLAG_BUSY))
  432. goto no_init;
  433. k3_ringacc_ring_free_sci(ring);
  434. dma_free_coherent(ringacc->dev,
  435. ring->size * (4 << ring->elm_size),
  436. ring->ring_mem_virt, ring->ring_mem_dma);
  437. ring->flags &= ~KNAV_RING_FLAG_BUSY;
  438. ring->ops = NULL;
  439. if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
  440. clear_bit(ring->proxy_id, ringacc->proxy_inuse);
  441. ring->proxy = NULL;
  442. ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
  443. }
  444. no_init:
  445. clear_bit(ring->ring_id, ringacc->rings_inuse);
  446. module_put(ringacc->dev->driver->owner);
  447. out:
  448. return 0;
  449. }
  450. u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
  451. {
  452. if (!ring)
  453. return -EINVAL;
  454. return ring->ring_id;
  455. }
  456. static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
  457. {
  458. struct k3_nav_ringacc *ringacc = ring->parent;
  459. u32 ring_idx;
  460. int ret;
  461. if (!ringacc->tisci)
  462. return -EINVAL;
  463. ring_idx = ring->ring_id;
  464. ret = ringacc->tisci_ring_ops->config(
  465. ringacc->tisci,
  466. TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
  467. ringacc->tisci_dev_id,
  468. ring_idx,
  469. lower_32_bits(ring->ring_mem_dma),
  470. upper_32_bits(ring->ring_mem_dma),
  471. ring->size,
  472. ring->mode,
  473. ring->elm_size,
  474. 0);
  475. if (ret)
  476. dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
  477. ret, ring_idx);
  478. return ret;
  479. }
  480. int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
  481. struct k3_nav_ring_cfg *cfg)
  482. {
  483. struct k3_nav_ringacc *ringacc = ring->parent;
  484. int ret = 0;
  485. if (!ring || !cfg)
  486. return -EINVAL;
  487. if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
  488. cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
  489. cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
  490. !test_bit(ring->ring_id, ringacc->rings_inuse))
  491. return -EINVAL;
  492. if (ring->use_count != 1)
  493. return 0;
  494. ring->size = cfg->size;
  495. ring->elm_size = cfg->elm_size;
  496. ring->mode = cfg->mode;
  497. memset(&ring->state, 0, sizeof(ring->state));
  498. if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
  499. ring->proxy = ringacc->proxy_target_base +
  500. ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
  501. switch (ring->mode) {
  502. case K3_NAV_RINGACC_RING_MODE_RING:
  503. ring->ops = &k3_nav_mode_ring_ops;
  504. break;
  505. case K3_NAV_RINGACC_RING_MODE_QM:
  506. /*
  507. * In Queue mode elm_size can be 8 only and each operation
  508. * uses 2 element slots
  509. */
  510. if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
  511. cfg->size % 2)
  512. goto err_free_proxy;
  513. case K3_NAV_RINGACC_RING_MODE_MESSAGE:
  514. if (ring->proxy)
  515. ring->ops = &k3_nav_mode_proxy_ops;
  516. else
  517. ring->ops = &k3_nav_mode_msg_ops;
  518. break;
  519. default:
  520. ring->ops = NULL;
  521. ret = -EINVAL;
  522. goto err_free_proxy;
  523. };
  524. ring->ring_mem_virt =
  525. dma_zalloc_coherent(ringacc->dev,
  526. ring->size * (4 << ring->elm_size),
  527. &ring->ring_mem_dma, GFP_KERNEL);
  528. if (!ring->ring_mem_virt) {
  529. dev_err(ringacc->dev, "Failed to alloc ring mem\n");
  530. ret = -ENOMEM;
  531. goto err_free_ops;
  532. }
  533. ret = k3_nav_ringacc_ring_cfg_sci(ring);
  534. if (ret)
  535. goto err_free_mem;
  536. ring->flags |= KNAV_RING_FLAG_BUSY;
  537. ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
  538. K3_NAV_RING_FLAG_SHARED : 0;
  539. return 0;
  540. err_free_mem:
  541. dma_free_coherent(ringacc->dev,
  542. ring->size * (4 << ring->elm_size),
  543. ring->ring_mem_virt,
  544. ring->ring_mem_dma);
  545. err_free_ops:
  546. ring->ops = NULL;
  547. err_free_proxy:
  548. ring->proxy = NULL;
  549. return ret;
  550. }
  551. u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
  552. {
  553. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  554. return -EINVAL;
  555. return ring->size;
  556. }
  557. u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
  558. {
  559. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  560. return -EINVAL;
  561. if (!ring->state.free)
  562. ring->state.free = ring->size - ringacc_readl(&ring->rt->occ);
  563. return ring->state.free;
  564. }
  565. u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
  566. {
  567. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  568. return -EINVAL;
  569. return ringacc_readl(&ring->rt->occ);
  570. }
  571. u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
  572. {
  573. return !k3_nav_ringacc_ring_get_free(ring);
  574. }
  575. enum k3_ringacc_access_mode {
  576. K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
  577. K3_RINGACC_ACCESS_MODE_POP_HEAD,
  578. K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
  579. K3_RINGACC_ACCESS_MODE_POP_TAIL,
  580. K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
  581. K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
  582. };
  583. static int k3_ringacc_ring_cfg_proxy(struct k3_nav_ring *ring,
  584. enum k3_ringacc_proxy_access_mode mode)
  585. {
  586. u32 val;
  587. val = ring->ring_id;
  588. val |= mode << 16;
  589. val |= ring->elm_size << 24;
  590. ringacc_writel(val, &ring->proxy->control);
  591. return 0;
  592. }
  593. static int k3_nav_ringacc_ring_access_proxy(
  594. struct k3_nav_ring *ring, void *elem,
  595. enum k3_ringacc_access_mode access_mode)
  596. {
  597. void __iomem *ptr;
  598. ptr = (void __iomem *)&ring->proxy->data;
  599. switch (access_mode) {
  600. case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
  601. case K3_RINGACC_ACCESS_MODE_POP_HEAD:
  602. k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
  603. break;
  604. case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
  605. case K3_RINGACC_ACCESS_MODE_POP_TAIL:
  606. k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
  607. break;
  608. default:
  609. return -EINVAL;
  610. }
  611. ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
  612. switch (access_mode) {
  613. case K3_RINGACC_ACCESS_MODE_POP_HEAD:
  614. case K3_RINGACC_ACCESS_MODE_POP_TAIL:
  615. pr_debug("proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n",
  616. ptr, access_mode);
  617. memcpy_fromio(elem, ptr, (4 << ring->elm_size));
  618. ring->state.occ--;
  619. break;
  620. case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
  621. case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
  622. pr_debug("proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n",
  623. ptr, access_mode);
  624. memcpy_toio(ptr, elem, (4 << ring->elm_size));
  625. ring->state.free--;
  626. break;
  627. default:
  628. return -EINVAL;
  629. }
  630. pr_debug("proxy: free%d occ%d\n",
  631. ring->state.free, ring->state.occ);
  632. return 0;
  633. }
  634. static int k3_ringacc_ring_push_head_proxy(struct k3_nav_ring *ring, void *elem)
  635. {
  636. return k3_nav_ringacc_ring_access_proxy(
  637. ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
  638. }
  639. static int k3_ringacc_ring_push_tail_proxy(struct k3_nav_ring *ring, void *elem)
  640. {
  641. return k3_nav_ringacc_ring_access_proxy(
  642. ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
  643. }
  644. static int k3_ringacc_ring_pop_head_proxy(struct k3_nav_ring *ring, void *elem)
  645. {
  646. return k3_nav_ringacc_ring_access_proxy(
  647. ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
  648. }
  649. static int k3_ringacc_ring_pop_tail_proxy(struct k3_nav_ring *ring, void *elem)
  650. {
  651. return k3_nav_ringacc_ring_access_proxy(
  652. ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
  653. }
  654. static int k3_nav_ringacc_ring_access_io(
  655. struct k3_nav_ring *ring, void *elem,
  656. enum k3_ringacc_access_mode access_mode)
  657. {
  658. void __iomem *ptr;
  659. switch (access_mode) {
  660. case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
  661. case K3_RINGACC_ACCESS_MODE_POP_HEAD:
  662. ptr = (void __iomem *)&ring->fifos->head_data;
  663. break;
  664. case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
  665. case K3_RINGACC_ACCESS_MODE_POP_TAIL:
  666. ptr = (void __iomem *)&ring->fifos->tail_data;
  667. break;
  668. default:
  669. return -EINVAL;
  670. }
  671. ptr += k3_nav_ringacc_ring_get_fifo_pos(ring);
  672. switch (access_mode) {
  673. case K3_RINGACC_ACCESS_MODE_POP_HEAD:
  674. case K3_RINGACC_ACCESS_MODE_POP_TAIL:
  675. pr_debug("memcpy_fromio(x): --> ptr(%p), mode:%d\n",
  676. ptr, access_mode);
  677. memcpy_fromio(elem, ptr, (4 << ring->elm_size));
  678. ring->state.occ--;
  679. break;
  680. case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
  681. case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
  682. pr_debug("memcpy_toio(x): --> ptr(%p), mode:%d\n",
  683. ptr, access_mode);
  684. memcpy_toio(ptr, elem, (4 << ring->elm_size));
  685. ring->state.free--;
  686. break;
  687. default:
  688. return -EINVAL;
  689. }
  690. pr_debug("free%d index%d occ%d index%d\n",
  691. ring->state.free, ring->state.windex, ring->state.occ, ring->state.rindex);
  692. return 0;
  693. }
  694. static int k3_nav_ringacc_ring_push_head_io(struct k3_nav_ring *ring,
  695. void *elem)
  696. {
  697. return k3_nav_ringacc_ring_access_io(
  698. ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
  699. }
  700. static int k3_nav_ringacc_ring_push_io(struct k3_nav_ring *ring, void *elem)
  701. {
  702. return k3_nav_ringacc_ring_access_io(
  703. ring, elem, K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
  704. }
  705. static int k3_nav_ringacc_ring_pop_io(struct k3_nav_ring *ring, void *elem)
  706. {
  707. return k3_nav_ringacc_ring_access_io(
  708. ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
  709. }
  710. static int k3_nav_ringacc_ring_pop_tail_io(struct k3_nav_ring *ring, void *elem)
  711. {
  712. return k3_nav_ringacc_ring_access_io(
  713. ring, elem, K3_RINGACC_ACCESS_MODE_POP_HEAD);
  714. }
  715. static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
  716. {
  717. void *elem_ptr;
  718. elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.windex);
  719. memcpy(elem_ptr, elem, (4 << ring->elm_size));
  720. flush_dcache_range((unsigned long)ring->ring_mem_virt,
  721. ALIGN((unsigned long)ring->ring_mem_virt +
  722. ring->size * (4 << ring->elm_size),
  723. ARCH_DMA_MINALIGN));
  724. ring->state.windex = (ring->state.windex + 1) % ring->size;
  725. ring->state.free--;
  726. ringacc_writel(1, &ring->rt->db);
  727. pr_debug("ring_push_mem: free%d index%d\n",
  728. ring->state.free, ring->state.windex);
  729. return 0;
  730. }
  731. static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
  732. {
  733. void *elem_ptr;
  734. elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);
  735. invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
  736. ALIGN((unsigned long)ring->ring_mem_virt +
  737. ring->size * (4 << ring->elm_size),
  738. ARCH_DMA_MINALIGN));
  739. memcpy(elem, elem_ptr, (4 << ring->elm_size));
  740. ring->state.rindex = (ring->state.rindex + 1) % ring->size;
  741. ring->state.occ--;
  742. ringacc_writel(-1, &ring->rt->db);
  743. pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
  744. ring->state.occ, ring->state.rindex, elem_ptr);
  745. return 0;
  746. }
  747. int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
  748. {
  749. int ret = -EOPNOTSUPP;
  750. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  751. return -EINVAL;
  752. pr_debug("ring_push%d: free%d index%d\n",
  753. ring->ring_id, ring->state.free, ring->state.windex);
  754. if (k3_nav_ringacc_ring_is_full(ring))
  755. return -ENOMEM;
  756. if (ring->ops && ring->ops->push_tail)
  757. ret = ring->ops->push_tail(ring, elem);
  758. return ret;
  759. }
  760. int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
  761. {
  762. int ret = -EOPNOTSUPP;
  763. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  764. return -EINVAL;
  765. pr_debug("ring_push_head: free%d index%d\n",
  766. ring->state.free, ring->state.windex);
  767. if (k3_nav_ringacc_ring_is_full(ring))
  768. return -ENOMEM;
  769. if (ring->ops && ring->ops->push_head)
  770. ret = ring->ops->push_head(ring, elem);
  771. return ret;
  772. }
  773. int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
  774. {
  775. int ret = -EOPNOTSUPP;
  776. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  777. return -EINVAL;
  778. if (!ring->state.occ)
  779. ring->state.occ = k3_nav_ringacc_ring_get_occ(ring);
  780. pr_debug("ring_pop%d: occ%d index%d\n",
  781. ring->ring_id, ring->state.occ, ring->state.rindex);
  782. if (!ring->state.occ && !ring->state.tdown_complete)
  783. return -ENODATA;
  784. if (ring->ops && ring->ops->pop_head)
  785. ret = ring->ops->pop_head(ring, elem);
  786. return ret;
  787. }
  788. int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
  789. {
  790. int ret = -EOPNOTSUPP;
  791. if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
  792. return -EINVAL;
  793. if (!ring->state.occ)
  794. ring->state.occ = k3_nav_ringacc_ring_get_occ(ring);
  795. pr_debug("ring_pop_tail: occ%d index%d\n",
  796. ring->state.occ, ring->state.rindex);
  797. if (!ring->state.occ)
  798. return -ENODATA;
  799. if (ring->ops && ring->ops->pop_tail)
  800. ret = ring->ops->pop_tail(ring, elem);
  801. return ret;
  802. }
  803. static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
  804. {
  805. struct udevice *dev = ringacc->dev;
  806. struct udevice *tisci_dev = NULL;
  807. int ret;
  808. ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
  809. if (!ringacc->num_rings) {
  810. dev_err(dev, "ti,num-rings read failure %d\n", ret);
  811. return -EINVAL;
  812. }
  813. ringacc->dma_ring_reset_quirk =
  814. dev_read_bool(dev, "ti,dma-ring-reset-quirk");
  815. ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, dev,
  816. "ti,sci", &tisci_dev);
  817. if (ret) {
  818. pr_debug("TISCI RA RM get failed (%d)\n", ret);
  819. ringacc->tisci = NULL;
  820. return -ENODEV;
  821. }
  822. ringacc->tisci = (struct ti_sci_handle *)
  823. (ti_sci_get_handle_from_sysfw(tisci_dev));
  824. ret = dev_read_u32_default(dev, "ti,sci", 0);
  825. if (!ret) {
  826. dev_err(dev, "TISCI RA RM disabled\n");
  827. ringacc->tisci = NULL;
  828. return ret;
  829. }
  830. ret = dev_read_u32(dev, "ti,sci-dev-id", &ringacc->tisci_dev_id);
  831. if (ret) {
  832. dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
  833. ringacc->tisci = NULL;
  834. return ret;
  835. }
  836. ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
  837. ringacc->tisci, dev,
  838. ringacc->tisci_dev_id,
  839. "ti,sci-rm-range-gp-rings");
  840. if (IS_ERR(ringacc->rm_gp_range))
  841. ret = PTR_ERR(ringacc->rm_gp_range);
  842. return 0;
  843. }
  844. static int k3_nav_ringacc_init(struct udevice *dev, struct k3_nav_ringacc *ringacc)
  845. {
  846. void __iomem *base_fifo, *base_rt;
  847. int ret, i;
  848. ret = k3_nav_ringacc_probe_dt(ringacc);
  849. if (ret)
  850. return ret;
  851. base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
  852. pr_debug("rt %p\n", base_rt);
  853. if (IS_ERR(base_rt))
  854. return PTR_ERR(base_rt);
  855. base_fifo = (uint32_t *)devfdt_get_addr_name(dev, "fifos");
  856. pr_debug("fifos %p\n", base_fifo);
  857. if (IS_ERR(base_fifo))
  858. return PTR_ERR(base_fifo);
  859. ringacc->proxy_gcfg = (struct k3_ringacc_proxy_gcfg_regs __iomem *)
  860. devfdt_get_addr_name(dev, "proxy_gcfg");
  861. if (IS_ERR(ringacc->proxy_gcfg))
  862. return PTR_ERR(ringacc->proxy_gcfg);
  863. ringacc->proxy_target_base =
  864. (struct k3_ringacc_proxy_gcfg_regs __iomem *)
  865. devfdt_get_addr_name(dev, "proxy_target");
  866. if (IS_ERR(ringacc->proxy_target_base))
  867. return PTR_ERR(ringacc->proxy_target_base);
  868. ringacc->num_proxies = ringacc_readl(&ringacc->proxy_gcfg->config) &
  869. K3_RINGACC_PROXY_CFG_THREADS_MASK;
  870. ringacc->rings = devm_kzalloc(dev,
  871. sizeof(*ringacc->rings) *
  872. ringacc->num_rings,
  873. GFP_KERNEL);
  874. ringacc->rings_inuse = devm_kcalloc(dev,
  875. BITS_TO_LONGS(ringacc->num_rings),
  876. sizeof(unsigned long), GFP_KERNEL);
  877. ringacc->proxy_inuse = devm_kcalloc(dev,
  878. BITS_TO_LONGS(ringacc->num_proxies),
  879. sizeof(unsigned long), GFP_KERNEL);
  880. if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
  881. return -ENOMEM;
  882. for (i = 0; i < ringacc->num_rings; i++) {
  883. ringacc->rings[i].rt = base_rt +
  884. KNAV_RINGACC_RT_REGS_STEP * i;
  885. ringacc->rings[i].fifos = base_fifo +
  886. KNAV_RINGACC_FIFO_REGS_STEP * i;
  887. ringacc->rings[i].parent = ringacc;
  888. ringacc->rings[i].ring_id = i;
  889. ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
  890. }
  891. dev_set_drvdata(dev, ringacc);
  892. ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
  893. list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
  894. dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
  895. ringacc->num_rings,
  896. ringacc->rm_gp_range->desc[0].start,
  897. ringacc->rm_gp_range->desc[0].num,
  898. ringacc->tisci_dev_id);
  899. dev_info(dev, "dma-ring-reset-quirk: %s\n",
  900. ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
  901. dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
  902. ringacc_readl(&ringacc->proxy_gcfg->revision),
  903. ringacc->num_proxies);
  904. return 0;
  905. }
  906. struct ringacc_match_data {
  907. struct k3_nav_ringacc_ops ops;
  908. };
  909. static struct ringacc_match_data k3_nav_ringacc_data = {
  910. .ops = {
  911. .init = k3_nav_ringacc_init,
  912. },
  913. };
  914. static const struct udevice_id knav_ringacc_ids[] = {
  915. { .compatible = "ti,am654-navss-ringacc", .data = (ulong)&k3_nav_ringacc_data, },
  916. {},
  917. };
  918. static int k3_nav_ringacc_probe(struct udevice *dev)
  919. {
  920. struct k3_nav_ringacc *ringacc;
  921. int ret;
  922. const struct ringacc_match_data *match_data;
  923. match_data = (struct ringacc_match_data *)dev_get_driver_data(dev);
  924. ringacc = dev_get_priv(dev);
  925. if (!ringacc)
  926. return -ENOMEM;
  927. ringacc->dev = dev;
  928. ringacc->ops = &match_data->ops;
  929. ret = ringacc->ops->init(dev, ringacc);
  930. if (ret)
  931. return ret;
  932. return 0;
  933. }
  934. U_BOOT_DRIVER(k3_navss_ringacc) = {
  935. .name = "k3-navss-ringacc",
  936. .id = UCLASS_MISC,
  937. .of_match = knav_ringacc_ids,
  938. .probe = k3_nav_ringacc_probe,
  939. .priv_auto_alloc_size = sizeof(struct k3_nav_ringacc),
  940. };