vha_cnn.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. /*
  2. *****************************************************************************
  3. * Copyright (c) Imagination Technologies Ltd.
  4. *
  5. * The contents of this file are subject to the MIT license as set out below.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. *
  25. * Alternatively, the contents of this file may be used under the terms of the
  26. * GNU General Public License Version 2 ("GPL")in which case the provisions of
  27. * GPL are applicable instead of those above.
  28. *
  29. * If you wish to allow use of your version of this file only under the terms
  30. * of GPL, and not to allow others to use your version of this file under the
  31. * terms of the MIT license, indicate your decision by deleting the provisions
  32. * above and replace them with the notice and other provisions required by GPL
  33. * as set out in the file called "GPLHEADER" included in this distribution. If
  34. * you do not delete the provisions above, a recipient may use your version of
  35. * this file under the terms of either the MIT license or GPL.
  36. *
  37. * This License is also included in this distribution in the file called
  38. * "MIT_COPYING".
  39. *
  40. *****************************************************************************/
  41. #include <linux/slab.h>
  42. #include <linux/device.h>
  43. #include <linux/gfp.h>
  44. #include <linux/moduleparam.h>
  45. #include <linux/delay.h>
  46. #include <uapi/vha.h>
  47. #include "vha_common.h"
  48. #include "vha_plat.h"
  49. #include "vha_regs.h"
  50. static uint32_t cnn_pdump_poll_count = 10000000;
  51. module_param(cnn_pdump_poll_count, uint, 0444);
  52. MODULE_PARM_DESC(cnn_pdump_poll_count,
  53. "PDUMP: Number of times to poll for CNN status");
  54. static bool cnn_preloads_disable;
  55. module_param(cnn_preloads_disable, bool, 0444);
  56. MODULE_PARM_DESC(cnn_preloads_disable,
  57. "Disables CNN preloads");
  58. /*
  59. * submit a command stream to the CNN hardware
  60. * input buffers:
  61. * command
  62. * input
  63. * coeff
  64. * output buffers:
  65. * output
  66. * accum_load
  67. * data:
  68. * none
  69. */
  70. static int do_cmd_cnn_submit(struct vha_cmd *cmd)
  71. {
  72. int i;
  73. uint32_t val32;
  74. const struct vha_user_cnn_submit_cmd *user_cmd =
  75. (struct vha_user_cnn_submit_cmd *)&cmd->user_cmd;
  76. struct vha_session *session = cmd->session;
  77. struct vha_dev *vha = session->vha;
  78. struct vha_buffer *buf = NULL;
  79. struct vha_onchip_map *onchip_map = NULL;
  80. int ret = -EINVAL;
  81. uint64_t alt_addrs_used = 0;
  82. size_t user_cmd_size;
  83. if (vha->hw_bypass) {
  84. ret = -EAGAIN;
  85. dev_info(vha->dev, "%s skip\n", __func__);
  86. goto out_error;
  87. }
  88. img_pdump_printf("-- CNN_SETUP_BEGIN\n");
  89. /* Wait for the previous kick to be accepted */
  90. if (vha->low_latency != VHA_LL_DISABLED) {
  91. /* Sanity wait for the kick bit to be deasserted */
  92. IOPOLL64_PDUMP(0, 1000, 10, (uint64_t)VHA_CR_OS(CNN_CONTROL_START_EN),
  93. VHA_CR_OS(CNN_CONTROL));
  94. if (cmd->queued &&
  95. vha->low_latency == VHA_LL_SW_KICK)
  96. goto hw_kick;
  97. }
  98. if (vha->pendcmd[VHA_CNN_CMD].cmd != NULL &&
  99. vha->low_latency == VHA_LL_DISABLED) {
  100. dev_err(vha->dev, "%s: trying to submit cnn cmd when hw busy!\n",
  101. __func__);
  102. goto out_error;
  103. }
  104. user_cmd_size = sizeof(*user_cmd);
  105. if (user_cmd->subseg_num > 0)
  106. user_cmd_size += (user_cmd->subseg_num - 1) * sizeof(struct vha_subseg_info);
  107. if (cmd->size != user_cmd_size) {
  108. dev_err(vha->dev, "%s: command buffer wrong size: %zu/%zu",
  109. __func__, cmd->size, sizeof(*user_cmd));
  110. goto out_error;
  111. }
  112. if (!vha_dev_check_hw_capab(vha, user_cmd->expected_ip_capab)) {
  113. ret = -ENODEV;
  114. goto out_error;
  115. }
  116. /* at least CMD and (IN or OUT)*/
  117. if (user_cmd->msg.num_inbufs < 2 ||
  118. /* and maybe TMP and others */
  119. user_cmd->msg.num_bufs > VHA_CORE_MAX_ALT_ADDRS) {
  120. dev_err(vha->dev, "%s: wrong number of bufs: %u,%u\n",
  121. __func__,
  122. user_cmd->msg.num_inbufs, user_cmd->msg.num_bufs);
  123. goto out_error;
  124. }
  125. if (user_cmd->onchipram_map_id != 0) {
  126. onchip_map = idr_find(&session->onchip_maps, user_cmd->onchipram_map_id);
  127. if (!onchip_map) {
  128. dev_warn(vha->dev, "%s: idr_find failed\n", __func__);
  129. }
  130. }
  131. /*
  132. * write buffer address to each register,
  133. * and pdump LDB each of the the input buffers
  134. */
  135. img_pdump_printf("-- Load inputs\n");
  136. for (i = 0; i < user_cmd->msg.num_bufs; i++) {
  137. uint32_t offset;
  138. uint32_t size;
  139. uint32_t reg;
  140. /* buffer id == 0 means no buffer */
  141. if (user_cmd->msg.data[i] == 0)
  142. continue;
  143. buf = vha_find_bufid(session, user_cmd->msg.data[i]);
  144. if (buf == NULL) {
  145. dev_err(vha->dev, "%s: invalid buffer id:%d\n",
  146. __func__, user_cmd->msg.data[i]);
  147. goto out_error;
  148. }
  149. if (buf->id == user_cmd->cmdbuf) {
  150. /* cmdstream always starts at offset 0 */
  151. if (user_cmd->subseg_info[cmd->subseg_current].cmdbuf_size)
  152. size = cmd->stream_size = user_cmd->subseg_info[cmd->subseg_current].cmdbuf_size;
  153. else
  154. size = cmd->stream_size = buf->size;
  155. offset = user_cmd->subseg_info[cmd->subseg_current].cmdbuf_offset;
  156. if (size == 0) {
  157. dev_err(vha->dev,
  158. "%s: invalid cmdstream size\n",
  159. __func__);
  160. goto out_error;
  161. }
  162. reg = VHA_CR_OS(CNN_CMD_BASE_ADDRESS);
  163. img_pdump_printf("-- Setup command stream\n");
  164. } else {
  165. /*
  166. * offset can be specified for all
  167. * buffers except cmdstream buf
  168. */
  169. offset = user_cmd->bufoffsets[i-1];
  170. size = user_cmd->bufsizes[i-1];
  171. if (size + offset > buf->size) {
  172. dev_err(vha->dev,
  173. "%s: invalid size+offset: %x+%x > %zx\n",
  174. __func__, size, offset, buf->size);
  175. goto out_error;
  176. }
  177. reg = VHA_CR_OS(CNN_ALT_ADDRESS0)
  178. + user_cmd->regidx[i-1]
  179. * (VHA_CR_OS(CNN_ALT_ADDRESS1)
  180. - VHA_CR_OS(CNN_ALT_ADDRESS0));
  181. /* record what alt address is in use */
  182. alt_addrs_used |= 1 << user_cmd->regidx[i-1];
  183. #if defined(HW_AX3)
  184. /* Alternative addresses from 8 to 15 are
  185. * located in different place */
  186. if (user_cmd->regidx[i-1] >= 8) {
  187. reg = VHA_CR_OS(CNN_ALT_ADDRESS8)
  188. + (user_cmd->regidx[i-1] - 8)
  189. * (VHA_CR_OS(CNN_ALT_ADDRESS1)
  190. - VHA_CR_OS(CNN_ALT_ADDRESS0));
  191. }
  192. alt_addrs_used |= buf->req_type <<
  193. (VHA_CR_OS(CNN_ALT_ADDRESS_USED_ALT_ADDR0_BUF_TYPE_SHIFT) +
  194. user_cmd->regidx[i-1]);
  195. #elif defined(HW_AX2)
  196. if (user_cmd->regidx[i-1] > 8) {
  197. dev_err(vha->dev,
  198. "%s: extended alternative addresses not supported!\n",
  199. __func__);
  200. goto out_error;
  201. }
  202. #endif
  203. }
  204. /* pdump the input buffers (not filled by the hw),
  205. * try to cache buffers filled by SW,
  206. * to avoid unnecessary LDBs */
  207. if (i < user_cmd->msg.num_inbufs &&
  208. !(buf->status == VHA_BUF_FILLED_BY_HW))
  209. vha_pdump_ldb_buf(session, PDUMP_PRM,
  210. buf, offset, size,
  211. buf->status == VHA_BUF_FILLED_BY_SW);
  212. vha_dump_digest(session, buf, cmd);
  213. /*
  214. * write to all of the index registers.
  215. * in no-MMU mode, write phys address of a contig buffer.
  216. * in MMU mode, write virt address of buffer.
  217. * If onchip_map selected, use different virt address of buffer
  218. */
  219. if (onchip_map != NULL && onchip_map->bufid == buf->id)
  220. IOWRITE64_PDUMP(onchip_map->devvirt + offset, reg);
  221. else
  222. IOWRITE_PDUMP_BUFADDR(session, buf, offset, reg);
  223. if (vha_buf_needs_flush(session, buf->id))
  224. img_mem_sync_cpu_to_device(session->mem_ctx, buf->id);
  225. }
  226. if (!cnn_preloads_disable) {
  227. /* Inform the hw what alt addresses are in use,
  228. * so the command decoder can prefetch */
  229. img_pdump_printf("-- Setup CNN prefetch register\n");
  230. IOWRITE64_PDUMP(alt_addrs_used, VHA_CR_OS(CNN_ALT_ADDRESS_USED));
  231. }
  232. /* write the stream size only */
  233. ret = 0;
  234. if (vha->pendcmd[VHA_CNN_CMD].cmd) {
  235. vha->queuedcmd[VHA_CNN_CMD].cmd = cmd;
  236. cmd->queued = true;
  237. vha->stats.cnn_kicks_queued++;
  238. img_pdump_printf("-- CNN already kicked queueing!\n");
  239. dev_dbg(vha->dev, "%s: -> kicked:%p queueing:%p\n",
  240. __func__, vha->pendcmd[VHA_CNN_CMD].cmd, cmd);
  241. if (vha->low_latency == VHA_LL_SW_KICK)
  242. return ret;
  243. }
  244. hw_kick:
  245. /* Change mmu context */
  246. ret = vha_mmu_setup(cmd->session);
  247. if (ret) {
  248. dev_err(vha->dev,
  249. "%s: Error during MMU setup!\n", __func__);
  250. goto out_error;
  251. }
  252. /* Setup memory stuff */
  253. vha_dev_mh_setup(vha, session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id, NULL);
  254. /* Prepare debug buffer registers */
  255. vha_dbg_prepare_hwbufs(session, cmd, NULL);
  256. /* Setup cnn hw watchdog before kicking the hw */
  257. {
  258. uint64_t cycles, mode;
  259. ret = vha_dev_hwwdt_calculate(vha, cmd, &cycles, &mode);
  260. if (!ret)
  261. vha_dev_hwwdt_setup(session->vha, cycles, mode);
  262. else if (ret != -EIO) {
  263. dev_err(vha->dev,
  264. "%s: can't obtain HWWDT info!\n",
  265. __func__);
  266. goto out_error;
  267. }
  268. }
  269. if (CMD_EXEC_ON_HW(cmd)) {
  270. cmd->in_hw = true;
  271. if (!cmd->queued)
  272. vha->pendcmd[VHA_CNN_CMD].cmd = cmd;
  273. }
  274. #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
  275. /* Mark kick for dummy driver */
  276. cmd->dummy_kicked = true;
  277. #endif
  278. /* Consider this cmd as kicked. */
  279. vha->pri_q_counters[cmd->user_cmd.priority]--;
  280. cmd->subseg_current++;
  281. ret = 0;
  282. /* Setup kick info */
  283. val32 = vha_dev_kick_prepare(vha, cmd,
  284. session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id);
  285. img_pdump_printf("-- CNN_SETUP_END\n");
  286. /* Remember the time cnn is kicked */
  287. GETNSTIMEOFDAY(&cmd->hw_proc_start);
  288. vha->stats.hw_proc_start = cmd->hw_proc_start;
  289. /* Need to generate proper pdump */
  290. if (cmd->queued &&
  291. vha->low_latency == VHA_LL_SW_KICK) {
  292. /* Do not write to pdump
  293. * this needs to be done after irq POL*/
  294. IOWRITE64(vha->reg_base, VHA_CR_OS(CNN_CONTROL), val32);
  295. dev_dbg(vha->dev, "%s: CNN kick queued (%p)!\n",
  296. __func__, cmd);
  297. cmd->queued = false;
  298. } else {
  299. img_pdump_printf("-- CNN_KICK_BEGIN\n");
  300. img_pdump_printf("-- CNN kick!\n");
  301. IOWRITE64_PDUMP(val32, VHA_CR_OS(CNN_CONTROL));
  302. dev_dbg(vha->dev, "%s: CNN kick %s (%p)!\n",
  303. __func__, cmd->queued ? "queued" : "", cmd);
  304. img_pdump_printf("-- CNN_KICK_END\n");
  305. }
  306. vha->stats.cnn_kicks++;
  307. /* notify any observers of the submit event */
  308. if (vha_observers.submitted)
  309. vha_observers.submitted(vha->id, session->id, cmd->user_cmd.cmd_id,
  310. (cmd->subseg_current == VHA_CMD_SUBSEG_NUM(cmd)),
  311. cmd->user_cmd.priority);
  312. out_error:
  313. if (ret != 0) {
  314. /* Consider this cmd as kicked for errors too. */
  315. vha->pri_q_counters[cmd->user_cmd.priority]--;
  316. cmd->subseg_current++;
  317. }
  318. return ret;
  319. }
  320. /*
  321. * append a string to the pdump TXT file
  322. * buffers:
  323. * none
  324. * data:
  325. * string to be printed
  326. */
  327. static int do_cmd_cnn_pdump_msg(const struct vha_cmd *cmd)
  328. {
  329. const struct vha_user_cmd *user_cmd = &cmd->user_cmd;
  330. struct vha_session *session = cmd->session;
  331. struct vha_dev* vha = session->vha;
  332. int ret = 0;
  333. if (user_cmd->num_inbufs != 0 || user_cmd->num_bufs != 0) {
  334. dev_err(session->vha->dev, ">0 buffers in cmd is wrong\n");
  335. ret = -EINVAL;
  336. }
  337. /* remember the pdump message may not be null terminated */
  338. img_pdump_printf("%.*s\n", (int)cmd->size, (char *)user_cmd->data);
  339. return ret;
  340. }
  341. /*
  342. * Simple procedure that generates watchdog interrupt
  343. */
  344. void vha_cnn_start_calib(struct vha_dev *vha)
  345. {
  346. uint64_t clk;
  347. uint32_t start;
  348. /* Setup hw watchdog before kicking the hw */
  349. vha_dev_hwwdt_setup(vha, vha->calibration_cycles, 0);
  350. /* Disabling command decoder, so we can generate wdt interrupt,
  351. * without providing any buffer address */
  352. clk = IOREAD64(vha->reg_base, VHA_CR_CLK_CTRL0);
  353. VHA_CR_CLEARBITS(clk, CLK_CTRL0, CNN_CMD);
  354. IOWRITE64(vha->reg_base, VHA_CR_CLK_CTRL0, clk);
  355. /* To be sure the cmd clock has switched off*/
  356. udelay(100);
  357. /* Enable MMU bypass */
  358. IOWRITE64_PDUMP(VHA_CR_OS(MMU_CTRL_BYPASS_EN),
  359. VHA_CR_OS(MMU_CTRL));
  360. /* Set minimal command stream size */
  361. start = (2048/32-1) << VHA_CR_OS(CNN_CONTROL_CMD_SIZE_MIN1_SHIFT);
  362. start |= VHA_CR_OS(CNN_CONTROL_START_EN);
  363. /* write the START bit */
  364. IOWRITE64(vha->reg_base, VHA_CR_OS(CNN_CONTROL), start);
  365. /* Remember the time cnn is kicked */
  366. GETNSTIMEOFDAY(&vha->stats.hw_proc_start);
  367. }
  368. void vha_cnn_update_stats(struct vha_dev *vha)
  369. {
  370. vha->stats.cnn_last_proc_us =
  371. vha->stats.last_proc_us;
  372. vha->stats.cnn_total_proc_us +=
  373. vha->stats.last_proc_us;
  374. if (vha->stats.cnn_kicks) {
  375. uint64_t avg = vha->stats.cnn_total_proc_us;
  376. do_div(avg, vha->stats.cnn_kicks);
  377. vha->stats.cnn_avg_proc_us = avg;
  378. }
  379. #if defined(HW_AX2)
  380. vha->stats.cnn_last_cycles =
  381. IOREAD64(vha->reg_base, VHA_CR_CNN_WDT_TIMER);
  382. #elif defined(HW_AX3)
  383. vha->stats.cnn_last_cycles =
  384. IOREAD64(vha->reg_base, VHA_CR_OS(CNN_PERFORMANCE));
  385. #endif
  386. if (vha->stats.cnn_last_cycles && vha->freq_khz) {
  387. uint64_t est_proc_us = 1000UL * vha->stats.cnn_last_cycles;
  388. do_div(est_proc_us, vha->freq_khz);
  389. vha->stats.cnn_last_est_proc_us = est_proc_us;
  390. }
  391. vha->stats.cnn_total_cycles += vha->stats.cnn_last_cycles;
  392. if (vha->stats.cnn_kicks &&
  393. vha->stats.cnn_total_cycles && vha->freq_khz) {
  394. uint64_t avg = 1000UL * vha->stats.cnn_total_cycles;
  395. do_div(avg, vha->stats.cnn_kicks);
  396. do_div(avg, vha->freq_khz);
  397. vha->stats.cnn_avg_est_proc_us = avg;
  398. }
  399. }
  400. /*
  401. * a command has completed. sent notification to user
  402. */
  403. void vha_cnn_cmd_completed(struct vha_cmd *cmd, int status)
  404. {
  405. struct vha_session *session = cmd->session;
  406. struct vha_dev* vha = session->vha;
  407. struct vha_rsp *rsp = NULL;
  408. int i;
  409. struct vha_user_cnn_submit_rsp * cnn_submit_rsp = NULL;
  410. const struct vha_user_cmd *user_cmd = &cmd->user_cmd;
  411. switch (user_cmd->cmd_type) {
  412. case VHA_CMD_CNN_SUBMIT:
  413. {
  414. size_t mem_usage;
  415. /* allocate sufficient space for the response */
  416. size_t sz = sizeof(*rsp)
  417. + sizeof(struct vha_user_cnn_submit_rsp)
  418. - sizeof(struct vha_user_rsp);
  419. uint32_t status_mask;
  420. uint32_t ready_mask;
  421. uint32_t cmpl_val = VHA_CR_OS(VHA_EVENT_STATUS_VHA_CNN0_COMPLETE_EN);
  422. #if defined(HW_AX2)
  423. /* status change: wait for any status change:
  424. * WDT, MMU_PF, ERROR, COMPLETE
  425. */
  426. status_mask = 0xffffffff;
  427. ready_mask = 0xffffffff;
  428. #elif defined(HW_AX3)
  429. /* status mask: wait for a status change: either ERROR, COMPLETE:
  430. * note that, unlike the live driver, pdump will ignore the MMU_PF,
  431. * which will have to be detected by the WDT
  432. */
  433. status_mask = VHA_CR_OS(VHA_EVENT_STATUS_VHA_ERROR_CLRMSK)
  434. | VHA_CR_OS(VHA_EVENT_STATUS_VHA_CNN0_COMPLETE_CLRMSK);
  435. ready_mask = VHA_CR_OS(VHA_EVENT_STATUS_VHA_READY_CLRMSK);
  436. /* Ignore PARITY when waiting for status change */
  437. status_mask &= VHA_CR_OS(VHA_EVENT_STATUS_PARITY_CLRMSK);
  438. #ifdef VHA_SCF
  439. if (session->vha->hw_props.supported.parity &&
  440. !session->vha->parity_disable) {
  441. /* If complete bit is set then parity bit must be set as well ! */
  442. cmpl_val |= VHA_CR_OS(VHA_EVENT_STATUS_PARITY_EN);
  443. }
  444. #else
  445. /* Ignore PARITY, so that non-SCF pdump may work with SC CSIM */
  446. ready_mask &= VHA_CR_OS(VHA_EVENT_STATUS_PARITY_CLRMSK);
  447. #endif
  448. #endif
  449. rsp = kzalloc(sz, GFP_KERNEL);
  450. if (rsp == NULL) {
  451. session->oom = true;
  452. return;
  453. }
  454. cnn_submit_rsp = (struct vha_user_cnn_submit_rsp*)&rsp->user_rsp;
  455. rsp->size = sizeof(struct vha_user_cnn_submit_rsp);
  456. if (session->vha->hw_bypass) {
  457. session->vha->hw_bypass--;
  458. break;
  459. }
  460. img_pdump_printf("-- CNN_WAIT_BEGIN\n");
  461. /* pdump POL for status change
  462. * count=cnn_pdump_poll_count, delay=1000cycles
  463. */
  464. img_pdump_printf("-- Wait for any CNN status\n"
  465. "POL :REG:%#x 0 %#x 3 %u 1000\n",
  466. VHA_CR_OS(VHA_EVENT_STATUS),
  467. status_mask,
  468. cnn_pdump_poll_count);
  469. /* quick pdump POL for the status complete flag only:
  470. * count=1, delay=10cycles
  471. */
  472. img_pdump_printf("-- Check for CNN_COMPLETE flag only\n"
  473. "POL :REG:%#x %#x 0x%x 0 1 10\n",
  474. VHA_CR_OS(VHA_EVENT_STATUS),
  475. cmpl_val,
  476. ready_mask);
  477. #ifdef VHA_SCF
  478. if (session->vha->hw_props.supported.parity &&
  479. !session->vha->parity_disable) {
  480. /* Check CNN_STATUS parity */
  481. uint32_t cnn_status = VHA_CR_SETBITS_OS(CNN_STATUS,
  482. STREAM_COUNT, 1);
  483. cnn_status |= VHA_CR_SETBITS_OS(CNN_STATUS,
  484. PARITY, 1);
  485. img_pdump_printf("-- Check for CNN_STATUS parity\n"
  486. "POL :REG:%#x %#x 0xffffffff 0 1 10\n",
  487. VHA_CR_OS(CNN_STATUS), cnn_status);
  488. }
  489. #endif
  490. /* quick pdump POL for AXI errors:
  491. * count=1, delay=10cycles
  492. */
  493. img_pdump_printf("-- Post check of AXI status\n"
  494. "POL :REG:%#x 0 0xffffffff 0 1 10\n",
  495. VHA_CR_ACE_STATUS);
  496. /* We do clear interrupts in the irq handler,
  497. * but this is not recorded into pdump because
  498. * of the irq context, so do it here */
  499. img_pdump_printf("-- Clear CNN events\n"
  500. "WRW64 :REG:%#x %#x\n",
  501. VHA_CR_OS(VHA_EVENT_CLEAR),
  502. VHA_CR_OS(VHA_EVENT_CLEAR_VHA_CNN0_COMPLETE_EN) |
  503. VHA_CNN_ERR_EVNTS);
  504. /* Try to flush hw debug buffers first
  505. * - this does pdump SAB when proper checkpoint is set */
  506. vha_dbg_flush_hwbufs(session, 1, 0);
  507. /* pdump SAB for each of the output buffers */
  508. img_pdump_printf("-- Save outputs\n");
  509. for (i = user_cmd->num_inbufs; i < user_cmd->num_bufs; i++) {
  510. struct vha_buffer *buf;
  511. struct vha_user_cnn_submit_cmd *msg =
  512. container_of(user_cmd,
  513. struct vha_user_cnn_submit_cmd,
  514. msg);
  515. uint32_t offset;
  516. uint32_t size;
  517. buf = vha_find_bufid(session, user_cmd->data[i]);
  518. if (buf == NULL) {
  519. dev_err(session->vha->dev,
  520. "%s: invalid buffer id:%d\n",
  521. __func__, user_cmd->data[i]);
  522. continue;
  523. }
  524. if (buf->id == msg->cmdbuf) {
  525. offset = 0;
  526. size = buf->size;
  527. } else {
  528. offset = msg->bufoffsets[i-1];
  529. size = msg->bufsizes[i-1];
  530. }
  531. vha_pdump_sab_buf(session, PDUMP_RES,
  532. buf, offset, size);
  533. /* Update status, do not signal fence yet,
  534. * it's is done explicitly below, after cache invalidation */
  535. vha_set_buf_status(session, buf->id, VHA_BUF_FILLED_BY_HW,
  536. VHA_SYNC_NONE, false);
  537. if (vha_buf_needs_inval(session, buf->id) && !status)
  538. img_mem_sync_device_to_cpu(session->mem_ctx, buf->id);
  539. #ifdef KERNEL_DMA_FENCE_SUPPORT
  540. img_mem_signal_fence(session->mem_ctx, buf->id);
  541. #endif
  542. vha_dump_digest(session, buf, cmd);
  543. }
  544. if (session->vha->low_latency == VHA_LL_SW_KICK) {
  545. struct vha_cmd *cmd =
  546. session->vha->queuedcmd[VHA_CNN_CMD].cmd;
  547. if (cmd && cmd->queued) {
  548. /* Setup kick info */
  549. uint64_t val = vha_dev_kick_prepare(session->vha, cmd,
  550. session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id);
  551. img_pdump_printf("-- CNN kick (queued)!\n");
  552. img_pdump_printf("WRW64 :REG:%#x %#llx\n",
  553. VHA_CR_OS(CNN_CONTROL), val);
  554. }
  555. }
  556. img_pdump_printf("-- CNN_WAIT_END\n");
  557. img_mem_get_usage(session->mem_ctx, NULL, &mem_usage);
  558. /* send out an event when submit is complete */
  559. if (vha_observers.completed)
  560. vha_observers.completed(
  561. session->vha->id,
  562. session->id,
  563. user_cmd->cmd_id,
  564. status,
  565. session->vha->stats.cnn_last_cycles,
  566. mem_usage,
  567. user_cmd->priority);
  568. /* post some metrics about the hw to user space */
  569. #ifdef MEM_USAGE_LAST_METRICS_ARE_AVAILABLE
  570. cnn_submit_rsp->mem_usage = mem_usage;
  571. #else
  572. cnn_submit_rsp->mem_usage = ~0;
  573. #endif
  574. cnn_submit_rsp->last_proc_us = cmd->proc_us;
  575. cnn_submit_rsp->hw_cycles = cmd->hw_cycles;
  576. dev_dbg(session->vha->dev, "%s: %p, hw_cycles %llx\n", __func__,
  577. cmd, session->vha->stats.cnn_last_cycles);
  578. if (session->vha->stats.cnn_last_cycles > (uint32_t)~0)
  579. dev_warn(session->vha->dev,
  580. "%s: hw_cycles %llx exceeds 32bit limit\n",
  581. __func__,
  582. session->vha->stats.cnn_last_cycles);
  583. break;
  584. }
  585. case VHA_CMD_CNN_PDUMP_MSG:
  586. default:
  587. /* allocate space for standard response */
  588. rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
  589. if (rsp == NULL) {
  590. session->oom = true;
  591. return;
  592. }
  593. rsp->size = sizeof(rsp->user_rsp);
  594. break;
  595. }
  596. if (user_cmd->flags & VHA_CMDFLAG_NOTIFY) {
  597. rsp->user_rsp.cmd_id = cmd->user_cmd.cmd_id;
  598. rsp->user_rsp.err_no = session->vha->hw_bypass ? 0 : status;
  599. cmd->rsp = rsp;
  600. } else
  601. kfree(rsp);
  602. }
  603. /*
  604. * Perform a command, as requested by user.
  605. * note: this function is called with vha_dev.lock == locked
  606. */
  607. int vha_do_cnn_cmd(struct vha_cmd *cmd)
  608. {
  609. struct vha_session *session = cmd->session;
  610. const struct vha_user_cmd *user_cmd = &cmd->user_cmd;
  611. int status = -EINVAL;
  612. dev_dbg(session->vha->dev,
  613. "CNN command: id:%x type:%x nin:%x nbufs:%x\n",
  614. user_cmd->cmd_id, user_cmd->cmd_type,
  615. user_cmd->num_inbufs, user_cmd->num_bufs);
  616. #if 0
  617. print_hex_dump_debug("VHA CMD: ", DUMP_PREFIX_NONE, 4, 4,
  618. user_cmd, ALIGN(cmd->size, 4), false);
  619. #endif
  620. switch (user_cmd->cmd_type) {
  621. case VHA_CMD_CNN_SUBMIT:
  622. status = do_cmd_cnn_submit(cmd);
  623. #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
  624. if (cmd->dummy_kicked) {
  625. struct vha_dev *vha = cmd->session->vha;
  626. const struct vha_user_cnn_submit_cmd *cnn_user_cmd =
  627. (struct vha_user_cnn_submit_cmd *)&cmd->user_cmd;
  628. uint32_t estimated_cycles = cnn_user_cmd->estimated_cycles;
  629. if (estimated_cycles == 0)
  630. estimated_cycles = VHA_DUMMY_HW_PROCESSING_TIME_CYCLES;
  631. cmd->dummy_exec_time = (estimated_cycles / (vha->freq_khz / 1000));
  632. schedule_delayed_work(&vha->dummy_dwork,
  633. usecs_to_jiffies(cmd->dummy_exec_time));
  634. cmd->dummy_kicked = false;
  635. }
  636. #endif
  637. break;
  638. case VHA_CMD_CNN_PDUMP_MSG:
  639. status = do_cmd_cnn_pdump_msg(cmd);
  640. default:
  641. break;
  642. }
  643. /*
  644. * Immediately send notification to user if not using hw at all
  645. * or submitting failed.
  646. */
  647. if (!CMD_EXEC_ON_HW(cmd) || status) {
  648. vha_cnn_cmd_completed(cmd, status);
  649. vha_cmd_notify(cmd);
  650. return 1;
  651. }
  652. return 0;
  653. }
  654. void vha_cnn_dump_status(struct vha_dev *vha)
  655. {
  656. struct device *dev = vha->dev;
  657. dev_err(dev, " CNN_STATUS:%llx ",
  658. IOREAD64(vha->reg_base,
  659. VHA_CR_OS(CNN_STATUS)));
  660. #ifdef HW_AX2
  661. dev_err(dev, " CNN_WDT_COMPAREMATCH:%llx ",
  662. IOREAD64(vha->reg_base,
  663. VHA_CR_CNN_WDT_COMPAREMATCH));
  664. dev_err(dev, " CNN_WDT_TIMER:%llx ",
  665. IOREAD64(vha->reg_base,
  666. VHA_CR_CNN_WDT_TIMER));
  667. #endif
  668. dev_err(dev, " CNN_MEM_WDT_COMPAREMATCH:%llx ",
  669. IOREAD64(vha->reg_base,
  670. VHA_CR_CNN_MEM_WDT_COMPAREMATCH));
  671. dev_err(dev, " CNN_MEM_WDT_TIMER:%llx ",
  672. IOREAD64(vha->reg_base,
  673. VHA_CR_CNN_MEM_WDT_TIMER));
  674. dev_err(dev, " BIF_OUTSTANDING_READ:%llx\n",
  675. IOREAD64(vha->reg_base,
  676. VHA_CR_BIF_OUTSTANDING_READ));
  677. }