123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750 |
- /*
- *****************************************************************************
- * Copyright (c) Imagination Technologies Ltd.
- *
- * The contents of this file are subject to the MIT license as set out below.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- * Alternatively, the contents of this file may be used under the terms of the
- * GNU General Public License Version 2 ("GPL")in which case the provisions of
- * GPL are applicable instead of those above.
- *
- * If you wish to allow use of your version of this file only under the terms
- * of GPL, and not to allow others to use your version of this file under the
- * terms of the MIT license, indicate your decision by deleting the provisions
- * above and replace them with the notice and other provisions required by GPL
- * as set out in the file called "GPLHEADER" included in this distribution. If
- * you do not delete the provisions above, a recipient may use your version of
- * this file under the terms of either the MIT license or GPL.
- *
- * This License is also included in this distribution in the file called
- * "MIT_COPYING".
- *
- *****************************************************************************/
- #include <linux/slab.h>
- #include <linux/device.h>
- #include <linux/gfp.h>
- #include <linux/moduleparam.h>
- #include <linux/delay.h>
- #include <uapi/vha.h>
- #include "vha_common.h"
- #include "vha_plat.h"
- #include "vha_regs.h"
- static uint32_t cnn_pdump_poll_count = 10000000;
- module_param(cnn_pdump_poll_count, uint, 0444);
- MODULE_PARM_DESC(cnn_pdump_poll_count,
- "PDUMP: Number of times to poll for CNN status");
- static bool cnn_preloads_disable;
- module_param(cnn_preloads_disable, bool, 0444);
- MODULE_PARM_DESC(cnn_preloads_disable,
- "Disables CNN preloads");
- /*
- * submit a command stream to the CNN hardware
- * input buffers:
- * command
- * input
- * coeff
- * output buffers:
- * output
- * accum_load
- * data:
- * none
- */
- static int do_cmd_cnn_submit(struct vha_cmd *cmd)
- {
- int i;
- uint32_t val32;
- const struct vha_user_cnn_submit_cmd *user_cmd =
- (struct vha_user_cnn_submit_cmd *)&cmd->user_cmd;
- struct vha_session *session = cmd->session;
- struct vha_dev *vha = session->vha;
- struct vha_buffer *buf = NULL;
- struct vha_onchip_map *onchip_map = NULL;
- int ret = -EINVAL;
- uint64_t alt_addrs_used = 0;
- size_t user_cmd_size;
- if (vha->hw_bypass) {
- ret = -EAGAIN;
- dev_info(vha->dev, "%s skip\n", __func__);
- goto out_error;
- }
- img_pdump_printf("-- CNN_SETUP_BEGIN\n");
- /* Wait for the previous kick to be accepted */
- if (vha->low_latency != VHA_LL_DISABLED) {
- /* Sanity wait for the kick bit to be deasserted */
- IOPOLL64_PDUMP(0, 1000, 10, (uint64_t)VHA_CR_OS(CNN_CONTROL_START_EN),
- VHA_CR_OS(CNN_CONTROL));
- if (cmd->queued &&
- vha->low_latency == VHA_LL_SW_KICK)
- goto hw_kick;
- }
- if (vha->pendcmd[VHA_CNN_CMD].cmd != NULL &&
- vha->low_latency == VHA_LL_DISABLED) {
- dev_err(vha->dev, "%s: trying to submit cnn cmd when hw busy!\n",
- __func__);
- goto out_error;
- }
- user_cmd_size = sizeof(*user_cmd);
- if (user_cmd->subseg_num > 0)
- user_cmd_size += (user_cmd->subseg_num - 1) * sizeof(struct vha_subseg_info);
- if (cmd->size != user_cmd_size) {
- dev_err(vha->dev, "%s: command buffer wrong size: %zu/%zu",
- __func__, cmd->size, sizeof(*user_cmd));
- goto out_error;
- }
- if (!vha_dev_check_hw_capab(vha, user_cmd->expected_ip_capab)) {
- ret = -ENODEV;
- goto out_error;
- }
- /* at least CMD and (IN or OUT)*/
- if (user_cmd->msg.num_inbufs < 2 ||
- /* and maybe TMP and others */
- user_cmd->msg.num_bufs > VHA_CORE_MAX_ALT_ADDRS) {
- dev_err(vha->dev, "%s: wrong number of bufs: %u,%u\n",
- __func__,
- user_cmd->msg.num_inbufs, user_cmd->msg.num_bufs);
- goto out_error;
- }
- if (user_cmd->onchipram_map_id != 0) {
- onchip_map = idr_find(&session->onchip_maps, user_cmd->onchipram_map_id);
- if (!onchip_map) {
- dev_warn(vha->dev, "%s: idr_find failed\n", __func__);
- }
- }
- /*
- * write buffer address to each register,
- * and pdump LDB each of the the input buffers
- */
- img_pdump_printf("-- Load inputs\n");
- for (i = 0; i < user_cmd->msg.num_bufs; i++) {
- uint32_t offset;
- uint32_t size;
- uint32_t reg;
- /* buffer id == 0 means no buffer */
- if (user_cmd->msg.data[i] == 0)
- continue;
- buf = vha_find_bufid(session, user_cmd->msg.data[i]);
- if (buf == NULL) {
- dev_err(vha->dev, "%s: invalid buffer id:%d\n",
- __func__, user_cmd->msg.data[i]);
- goto out_error;
- }
- if (buf->id == user_cmd->cmdbuf) {
- /* cmdstream always starts at offset 0 */
- if (user_cmd->subseg_info[cmd->subseg_current].cmdbuf_size)
- size = cmd->stream_size = user_cmd->subseg_info[cmd->subseg_current].cmdbuf_size;
- else
- size = cmd->stream_size = buf->size;
- offset = user_cmd->subseg_info[cmd->subseg_current].cmdbuf_offset;
- if (size == 0) {
- dev_err(vha->dev,
- "%s: invalid cmdstream size\n",
- __func__);
- goto out_error;
- }
- reg = VHA_CR_OS(CNN_CMD_BASE_ADDRESS);
- img_pdump_printf("-- Setup command stream\n");
- } else {
- /*
- * offset can be specified for all
- * buffers except cmdstream buf
- */
- offset = user_cmd->bufoffsets[i-1];
- size = user_cmd->bufsizes[i-1];
- if (size + offset > buf->size) {
- dev_err(vha->dev,
- "%s: invalid size+offset: %x+%x > %zx\n",
- __func__, size, offset, buf->size);
- goto out_error;
- }
- reg = VHA_CR_OS(CNN_ALT_ADDRESS0)
- + user_cmd->regidx[i-1]
- * (VHA_CR_OS(CNN_ALT_ADDRESS1)
- - VHA_CR_OS(CNN_ALT_ADDRESS0));
- /* record what alt address is in use */
- alt_addrs_used |= 1 << user_cmd->regidx[i-1];
- #if defined(HW_AX3)
- /* Alternative addresses from 8 to 15 are
- * located in different place */
- if (user_cmd->regidx[i-1] >= 8) {
- reg = VHA_CR_OS(CNN_ALT_ADDRESS8)
- + (user_cmd->regidx[i-1] - 8)
- * (VHA_CR_OS(CNN_ALT_ADDRESS1)
- - VHA_CR_OS(CNN_ALT_ADDRESS0));
- }
- alt_addrs_used |= buf->req_type <<
- (VHA_CR_OS(CNN_ALT_ADDRESS_USED_ALT_ADDR0_BUF_TYPE_SHIFT) +
- user_cmd->regidx[i-1]);
- #elif defined(HW_AX2)
- if (user_cmd->regidx[i-1] > 8) {
- dev_err(vha->dev,
- "%s: extended alternative addresses not supported!\n",
- __func__);
- goto out_error;
- }
- #endif
- }
- /* pdump the input buffers (not filled by the hw),
- * try to cache buffers filled by SW,
- * to avoid unnecessary LDBs */
- if (i < user_cmd->msg.num_inbufs &&
- !(buf->status == VHA_BUF_FILLED_BY_HW))
- vha_pdump_ldb_buf(session, PDUMP_PRM,
- buf, offset, size,
- buf->status == VHA_BUF_FILLED_BY_SW);
- vha_dump_digest(session, buf, cmd);
- /*
- * write to all of the index registers.
- * in no-MMU mode, write phys address of a contig buffer.
- * in MMU mode, write virt address of buffer.
- * If onchip_map selected, use different virt address of buffer
- */
- if (onchip_map != NULL && onchip_map->bufid == buf->id)
- IOWRITE64_PDUMP(onchip_map->devvirt + offset, reg);
- else
- IOWRITE_PDUMP_BUFADDR(session, buf, offset, reg);
- if (vha_buf_needs_flush(session, buf->id))
- img_mem_sync_cpu_to_device(session->mem_ctx, buf->id);
- }
- if (!cnn_preloads_disable) {
- /* Inform the hw what alt addresses are in use,
- * so the command decoder can prefetch */
- img_pdump_printf("-- Setup CNN prefetch register\n");
- IOWRITE64_PDUMP(alt_addrs_used, VHA_CR_OS(CNN_ALT_ADDRESS_USED));
- }
- /* write the stream size only */
- ret = 0;
- if (vha->pendcmd[VHA_CNN_CMD].cmd) {
- vha->queuedcmd[VHA_CNN_CMD].cmd = cmd;
- cmd->queued = true;
- vha->stats.cnn_kicks_queued++;
- img_pdump_printf("-- CNN already kicked queueing!\n");
- dev_dbg(vha->dev, "%s: -> kicked:%p queueing:%p\n",
- __func__, vha->pendcmd[VHA_CNN_CMD].cmd, cmd);
- if (vha->low_latency == VHA_LL_SW_KICK)
- return ret;
- }
- hw_kick:
- /* Change mmu context */
- ret = vha_mmu_setup(cmd->session);
- if (ret) {
- dev_err(vha->dev,
- "%s: Error during MMU setup!\n", __func__);
- goto out_error;
- }
- /* Setup memory stuff */
- vha_dev_mh_setup(vha, session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id, NULL);
- /* Prepare debug buffer registers */
- vha_dbg_prepare_hwbufs(session, cmd, NULL);
- /* Setup cnn hw watchdog before kicking the hw */
- {
- uint64_t cycles, mode;
- ret = vha_dev_hwwdt_calculate(vha, cmd, &cycles, &mode);
- if (!ret)
- vha_dev_hwwdt_setup(session->vha, cycles, mode);
- else if (ret != -EIO) {
- dev_err(vha->dev,
- "%s: can't obtain HWWDT info!\n",
- __func__);
- goto out_error;
- }
- }
- if (CMD_EXEC_ON_HW(cmd)) {
- cmd->in_hw = true;
- if (!cmd->queued)
- vha->pendcmd[VHA_CNN_CMD].cmd = cmd;
- }
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- /* Mark kick for dummy driver */
- cmd->dummy_kicked = true;
- #endif
- /* Consider this cmd as kicked. */
- vha->pri_q_counters[cmd->user_cmd.priority]--;
- cmd->subseg_current++;
- ret = 0;
- /* Setup kick info */
- val32 = vha_dev_kick_prepare(vha, cmd,
- session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id);
- img_pdump_printf("-- CNN_SETUP_END\n");
- /* Remember the time cnn is kicked */
- GETNSTIMEOFDAY(&cmd->hw_proc_start);
- vha->stats.hw_proc_start = cmd->hw_proc_start;
- /* Need to generate proper pdump */
- if (cmd->queued &&
- vha->low_latency == VHA_LL_SW_KICK) {
- /* Do not write to pdump
- * this needs to be done after irq POL*/
- IOWRITE64(vha->reg_base, VHA_CR_OS(CNN_CONTROL), val32);
- dev_dbg(vha->dev, "%s: CNN kick queued (%p)!\n",
- __func__, cmd);
- cmd->queued = false;
- } else {
- img_pdump_printf("-- CNN_KICK_BEGIN\n");
- img_pdump_printf("-- CNN kick!\n");
- IOWRITE64_PDUMP(val32, VHA_CR_OS(CNN_CONTROL));
- dev_dbg(vha->dev, "%s: CNN kick %s (%p)!\n",
- __func__, cmd->queued ? "queued" : "", cmd);
- img_pdump_printf("-- CNN_KICK_END\n");
- }
- vha->stats.cnn_kicks++;
- /* notify any observers of the submit event */
- if (vha_observers.submitted)
- vha_observers.submitted(vha->id, session->id, cmd->user_cmd.cmd_id,
- (cmd->subseg_current == VHA_CMD_SUBSEG_NUM(cmd)),
- cmd->user_cmd.priority);
- out_error:
- if (ret != 0) {
- /* Consider this cmd as kicked for errors too. */
- vha->pri_q_counters[cmd->user_cmd.priority]--;
- cmd->subseg_current++;
- }
- return ret;
- }
- /*
- * append a string to the pdump TXT file
- * buffers:
- * none
- * data:
- * string to be printed
- */
- static int do_cmd_cnn_pdump_msg(const struct vha_cmd *cmd)
- {
- const struct vha_user_cmd *user_cmd = &cmd->user_cmd;
- struct vha_session *session = cmd->session;
- struct vha_dev* vha = session->vha;
- int ret = 0;
- if (user_cmd->num_inbufs != 0 || user_cmd->num_bufs != 0) {
- dev_err(session->vha->dev, ">0 buffers in cmd is wrong\n");
- ret = -EINVAL;
- }
- /* remember the pdump message may not be null terminated */
- img_pdump_printf("%.*s\n", (int)cmd->size, (char *)user_cmd->data);
- return ret;
- }
- /*
- * Simple procedure that generates watchdog interrupt
- */
- void vha_cnn_start_calib(struct vha_dev *vha)
- {
- uint64_t clk;
- uint32_t start;
- /* Setup hw watchdog before kicking the hw */
- vha_dev_hwwdt_setup(vha, vha->calibration_cycles, 0);
- /* Disabling command decoder, so we can generate wdt interrupt,
- * without providing any buffer address */
- clk = IOREAD64(vha->reg_base, VHA_CR_CLK_CTRL0);
- VHA_CR_CLEARBITS(clk, CLK_CTRL0, CNN_CMD);
- IOWRITE64(vha->reg_base, VHA_CR_CLK_CTRL0, clk);
- /* To be sure the cmd clock has switched off*/
- udelay(100);
- /* Enable MMU bypass */
- IOWRITE64_PDUMP(VHA_CR_OS(MMU_CTRL_BYPASS_EN),
- VHA_CR_OS(MMU_CTRL));
- /* Set minimal command stream size */
- start = (2048/32-1) << VHA_CR_OS(CNN_CONTROL_CMD_SIZE_MIN1_SHIFT);
- start |= VHA_CR_OS(CNN_CONTROL_START_EN);
- /* write the START bit */
- IOWRITE64(vha->reg_base, VHA_CR_OS(CNN_CONTROL), start);
- /* Remember the time cnn is kicked */
- GETNSTIMEOFDAY(&vha->stats.hw_proc_start);
- }
- void vha_cnn_update_stats(struct vha_dev *vha)
- {
- vha->stats.cnn_last_proc_us =
- vha->stats.last_proc_us;
- vha->stats.cnn_total_proc_us +=
- vha->stats.last_proc_us;
- if (vha->stats.cnn_kicks) {
- uint64_t avg = vha->stats.cnn_total_proc_us;
- do_div(avg, vha->stats.cnn_kicks);
- vha->stats.cnn_avg_proc_us = avg;
- }
- #if defined(HW_AX2)
- vha->stats.cnn_last_cycles =
- IOREAD64(vha->reg_base, VHA_CR_CNN_WDT_TIMER);
- #elif defined(HW_AX3)
- vha->stats.cnn_last_cycles =
- IOREAD64(vha->reg_base, VHA_CR_OS(CNN_PERFORMANCE));
- #endif
- if (vha->stats.cnn_last_cycles && vha->freq_khz) {
- uint64_t est_proc_us = 1000UL * vha->stats.cnn_last_cycles;
- do_div(est_proc_us, vha->freq_khz);
- vha->stats.cnn_last_est_proc_us = est_proc_us;
- }
- vha->stats.cnn_total_cycles += vha->stats.cnn_last_cycles;
- if (vha->stats.cnn_kicks &&
- vha->stats.cnn_total_cycles && vha->freq_khz) {
- uint64_t avg = 1000UL * vha->stats.cnn_total_cycles;
- do_div(avg, vha->stats.cnn_kicks);
- do_div(avg, vha->freq_khz);
- vha->stats.cnn_avg_est_proc_us = avg;
- }
- }
- /*
- * a command has completed. sent notification to user
- */
- void vha_cnn_cmd_completed(struct vha_cmd *cmd, int status)
- {
- struct vha_session *session = cmd->session;
- struct vha_dev* vha = session->vha;
- struct vha_rsp *rsp = NULL;
- int i;
- struct vha_user_cnn_submit_rsp * cnn_submit_rsp = NULL;
- const struct vha_user_cmd *user_cmd = &cmd->user_cmd;
- switch (user_cmd->cmd_type) {
- case VHA_CMD_CNN_SUBMIT:
- {
- size_t mem_usage;
- /* allocate sufficient space for the response */
- size_t sz = sizeof(*rsp)
- + sizeof(struct vha_user_cnn_submit_rsp)
- - sizeof(struct vha_user_rsp);
- uint32_t status_mask;
- uint32_t ready_mask;
- uint32_t cmpl_val = VHA_CR_OS(VHA_EVENT_STATUS_VHA_CNN0_COMPLETE_EN);
- #if defined(HW_AX2)
- /* status change: wait for any status change:
- * WDT, MMU_PF, ERROR, COMPLETE
- */
- status_mask = 0xffffffff;
- ready_mask = 0xffffffff;
- #elif defined(HW_AX3)
- /* status mask: wait for a status change: either ERROR, COMPLETE:
- * note that, unlike the live driver, pdump will ignore the MMU_PF,
- * which will have to be detected by the WDT
- */
- status_mask = VHA_CR_OS(VHA_EVENT_STATUS_VHA_ERROR_CLRMSK)
- | VHA_CR_OS(VHA_EVENT_STATUS_VHA_CNN0_COMPLETE_CLRMSK);
- ready_mask = VHA_CR_OS(VHA_EVENT_STATUS_VHA_READY_CLRMSK);
- /* Ignore PARITY when waiting for status change */
- status_mask &= VHA_CR_OS(VHA_EVENT_STATUS_PARITY_CLRMSK);
- #ifdef VHA_SCF
- if (session->vha->hw_props.supported.parity &&
- !session->vha->parity_disable) {
- /* If complete bit is set then parity bit must be set as well ! */
- cmpl_val |= VHA_CR_OS(VHA_EVENT_STATUS_PARITY_EN);
- }
- #else
- /* Ignore PARITY, so that non-SCF pdump may work with SC CSIM */
- ready_mask &= VHA_CR_OS(VHA_EVENT_STATUS_PARITY_CLRMSK);
- #endif
- #endif
- rsp = kzalloc(sz, GFP_KERNEL);
- if (rsp == NULL) {
- session->oom = true;
- return;
- }
- cnn_submit_rsp = (struct vha_user_cnn_submit_rsp*)&rsp->user_rsp;
- rsp->size = sizeof(struct vha_user_cnn_submit_rsp);
- if (session->vha->hw_bypass) {
- session->vha->hw_bypass--;
- break;
- }
- img_pdump_printf("-- CNN_WAIT_BEGIN\n");
- /* pdump POL for status change
- * count=cnn_pdump_poll_count, delay=1000cycles
- */
- img_pdump_printf("-- Wait for any CNN status\n"
- "POL :REG:%#x 0 %#x 3 %u 1000\n",
- VHA_CR_OS(VHA_EVENT_STATUS),
- status_mask,
- cnn_pdump_poll_count);
- /* quick pdump POL for the status complete flag only:
- * count=1, delay=10cycles
- */
- img_pdump_printf("-- Check for CNN_COMPLETE flag only\n"
- "POL :REG:%#x %#x 0x%x 0 1 10\n",
- VHA_CR_OS(VHA_EVENT_STATUS),
- cmpl_val,
- ready_mask);
- #ifdef VHA_SCF
- if (session->vha->hw_props.supported.parity &&
- !session->vha->parity_disable) {
- /* Check CNN_STATUS parity */
- uint32_t cnn_status = VHA_CR_SETBITS_OS(CNN_STATUS,
- STREAM_COUNT, 1);
- cnn_status |= VHA_CR_SETBITS_OS(CNN_STATUS,
- PARITY, 1);
- img_pdump_printf("-- Check for CNN_STATUS parity\n"
- "POL :REG:%#x %#x 0xffffffff 0 1 10\n",
- VHA_CR_OS(CNN_STATUS), cnn_status);
- }
- #endif
- /* quick pdump POL for AXI errors:
- * count=1, delay=10cycles
- */
- img_pdump_printf("-- Post check of AXI status\n"
- "POL :REG:%#x 0 0xffffffff 0 1 10\n",
- VHA_CR_ACE_STATUS);
- /* We do clear interrupts in the irq handler,
- * but this is not recorded into pdump because
- * of the irq context, so do it here */
- img_pdump_printf("-- Clear CNN events\n"
- "WRW64 :REG:%#x %#x\n",
- VHA_CR_OS(VHA_EVENT_CLEAR),
- VHA_CR_OS(VHA_EVENT_CLEAR_VHA_CNN0_COMPLETE_EN) |
- VHA_CNN_ERR_EVNTS);
- /* Try to flush hw debug buffers first
- * - this does pdump SAB when proper checkpoint is set */
- vha_dbg_flush_hwbufs(session, 1, 0);
- /* pdump SAB for each of the output buffers */
- img_pdump_printf("-- Save outputs\n");
- for (i = user_cmd->num_inbufs; i < user_cmd->num_bufs; i++) {
- struct vha_buffer *buf;
- struct vha_user_cnn_submit_cmd *msg =
- container_of(user_cmd,
- struct vha_user_cnn_submit_cmd,
- msg);
- uint32_t offset;
- uint32_t size;
- buf = vha_find_bufid(session, user_cmd->data[i]);
- if (buf == NULL) {
- dev_err(session->vha->dev,
- "%s: invalid buffer id:%d\n",
- __func__, user_cmd->data[i]);
- continue;
- }
- if (buf->id == msg->cmdbuf) {
- offset = 0;
- size = buf->size;
- } else {
- offset = msg->bufoffsets[i-1];
- size = msg->bufsizes[i-1];
- }
- vha_pdump_sab_buf(session, PDUMP_RES,
- buf, offset, size);
- /* Update status, do not signal fence yet,
- * it's is done explicitly below, after cache invalidation */
- vha_set_buf_status(session, buf->id, VHA_BUF_FILLED_BY_HW,
- VHA_SYNC_NONE, false);
- if (vha_buf_needs_inval(session, buf->id) && !status)
- img_mem_sync_device_to_cpu(session->mem_ctx, buf->id);
- #ifdef KERNEL_DMA_FENCE_SUPPORT
- img_mem_signal_fence(session->mem_ctx, buf->id);
- #endif
- vha_dump_digest(session, buf, cmd);
- }
- if (session->vha->low_latency == VHA_LL_SW_KICK) {
- struct vha_cmd *cmd =
- session->vha->queuedcmd[VHA_CNN_CMD].cmd;
- if (cmd && cmd->queued) {
- /* Setup kick info */
- uint64_t val = vha_dev_kick_prepare(session->vha, cmd,
- session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id);
- img_pdump_printf("-- CNN kick (queued)!\n");
- img_pdump_printf("WRW64 :REG:%#x %#llx\n",
- VHA_CR_OS(CNN_CONTROL), val);
- }
- }
- img_pdump_printf("-- CNN_WAIT_END\n");
- img_mem_get_usage(session->mem_ctx, NULL, &mem_usage);
- /* send out an event when submit is complete */
- if (vha_observers.completed)
- vha_observers.completed(
- session->vha->id,
- session->id,
- user_cmd->cmd_id,
- status,
- session->vha->stats.cnn_last_cycles,
- mem_usage,
- user_cmd->priority);
- /* post some metrics about the hw to user space */
- #ifdef MEM_USAGE_LAST_METRICS_ARE_AVAILABLE
- cnn_submit_rsp->mem_usage = mem_usage;
- #else
- cnn_submit_rsp->mem_usage = ~0;
- #endif
- cnn_submit_rsp->last_proc_us = cmd->proc_us;
- cnn_submit_rsp->hw_cycles = cmd->hw_cycles;
- dev_dbg(session->vha->dev, "%s: %p, hw_cycles %llx\n", __func__,
- cmd, session->vha->stats.cnn_last_cycles);
- if (session->vha->stats.cnn_last_cycles > (uint32_t)~0)
- dev_warn(session->vha->dev,
- "%s: hw_cycles %llx exceeds 32bit limit\n",
- __func__,
- session->vha->stats.cnn_last_cycles);
- break;
- }
- case VHA_CMD_CNN_PDUMP_MSG:
- default:
- /* allocate space for standard response */
- rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
- if (rsp == NULL) {
- session->oom = true;
- return;
- }
- rsp->size = sizeof(rsp->user_rsp);
- break;
- }
- if (user_cmd->flags & VHA_CMDFLAG_NOTIFY) {
- rsp->user_rsp.cmd_id = cmd->user_cmd.cmd_id;
- rsp->user_rsp.err_no = session->vha->hw_bypass ? 0 : status;
- cmd->rsp = rsp;
- } else
- kfree(rsp);
- }
- /*
- * Perform a command, as requested by user.
- * note: this function is called with vha_dev.lock == locked
- */
- int vha_do_cnn_cmd(struct vha_cmd *cmd)
- {
- struct vha_session *session = cmd->session;
- const struct vha_user_cmd *user_cmd = &cmd->user_cmd;
- int status = -EINVAL;
- dev_dbg(session->vha->dev,
- "CNN command: id:%x type:%x nin:%x nbufs:%x\n",
- user_cmd->cmd_id, user_cmd->cmd_type,
- user_cmd->num_inbufs, user_cmd->num_bufs);
- #if 0
- print_hex_dump_debug("VHA CMD: ", DUMP_PREFIX_NONE, 4, 4,
- user_cmd, ALIGN(cmd->size, 4), false);
- #endif
- switch (user_cmd->cmd_type) {
- case VHA_CMD_CNN_SUBMIT:
- status = do_cmd_cnn_submit(cmd);
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- if (cmd->dummy_kicked) {
- struct vha_dev *vha = cmd->session->vha;
- const struct vha_user_cnn_submit_cmd *cnn_user_cmd =
- (struct vha_user_cnn_submit_cmd *)&cmd->user_cmd;
- uint32_t estimated_cycles = cnn_user_cmd->estimated_cycles;
- if (estimated_cycles == 0)
- estimated_cycles = VHA_DUMMY_HW_PROCESSING_TIME_CYCLES;
- cmd->dummy_exec_time = (estimated_cycles / (vha->freq_khz / 1000));
- schedule_delayed_work(&vha->dummy_dwork,
- usecs_to_jiffies(cmd->dummy_exec_time));
- cmd->dummy_kicked = false;
- }
- #endif
- break;
- case VHA_CMD_CNN_PDUMP_MSG:
- status = do_cmd_cnn_pdump_msg(cmd);
- default:
- break;
- }
- /*
- * Immediately send notification to user if not using hw at all
- * or submitting failed.
- */
- if (!CMD_EXEC_ON_HW(cmd) || status) {
- vha_cnn_cmd_completed(cmd, status);
- vha_cmd_notify(cmd);
- return 1;
- }
- return 0;
- }
- void vha_cnn_dump_status(struct vha_dev *vha)
- {
- struct device *dev = vha->dev;
- dev_err(dev, " CNN_STATUS:%llx ",
- IOREAD64(vha->reg_base,
- VHA_CR_OS(CNN_STATUS)));
- #ifdef HW_AX2
- dev_err(dev, " CNN_WDT_COMPAREMATCH:%llx ",
- IOREAD64(vha->reg_base,
- VHA_CR_CNN_WDT_COMPAREMATCH));
- dev_err(dev, " CNN_WDT_TIMER:%llx ",
- IOREAD64(vha->reg_base,
- VHA_CR_CNN_WDT_TIMER));
- #endif
- dev_err(dev, " CNN_MEM_WDT_COMPAREMATCH:%llx ",
- IOREAD64(vha->reg_base,
- VHA_CR_CNN_MEM_WDT_COMPAREMATCH));
- dev_err(dev, " CNN_MEM_WDT_TIMER:%llx ",
- IOREAD64(vha->reg_base,
- VHA_CR_CNN_MEM_WDT_TIMER));
- dev_err(dev, " BIF_OUTSTANDING_READ:%llx\n",
- IOREAD64(vha->reg_base,
- VHA_CR_BIF_OUTSTANDING_READ));
- }
|