123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581 |
- /*
- *****************************************************************************
- * Copyright (c) Imagination Technologies Ltd.
- *
- * The contents of this file are subject to the MIT license as set out below.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- * Alternatively, the contents of this file may be used under the terms of the
- * GNU General Public License Version 2 ("GPL")in which case the provisions of
- * GPL are applicable instead of those above.
- *
- * If you wish to allow use of your version of this file only under the terms
- * of GPL, and not to allow others to use your version of this file under the
- * terms of the MIT license, indicate your decision by deleting the provisions
- * above and replace them with the notice and other provisions required by GPL
- * as set out in the file called "GPLHEADER" included in this distribution. If
- * you do not delete the provisions above, a recipient may use your version of
- * this file under the terms of either the MIT license or GPL.
- *
- * This License is also included in this distribution in the file called
- * "MIT_COPYING".
- *
- *****************************************************************************/
- #include <linux/delay.h>
- #include <linux/irq.h>
- #include <linux/moduleparam.h>
- #include <linux/pm_runtime.h>
- #include <linux/slab.h>
- #include <uapi/vha.h>
- #include "vha_common.h"
- #include "vha_plat.h"
- #include "vha_regs.h"
- #if defined(CFG_SYS_VAGUS)
- #include <hwdefs/nn_sys_cr_vagus.h>
- #endif
- #define ERR_EVENT_DESC(b) VHA_CR_OS(VHA_EVENT_STATUS_VHA_##b##_EN), __stringify(b)
- static void vha_dev_disable_events(struct vha_dev *vha)
- {
- img_pdump_printf("-- Clear CNN events\n");
- IOWRITE64_PDUMP(VHA_EVNTS_DEFAULT, VHA_CR_OS(VHA_EVENT_CLEAR));
- img_pdump_printf("-- Disable CNN events\n");
- IOWRITE64_PDUMP(0, VHA_CR_OS(VHA_EVENT_ENABLE));
- /* Clear the START bit !
- * Note: It is stated that writing 0 to this bit has no effect,
- * however in error cases, some hw blocks may start
- * to process previous requests after turning on the clocks
- * which was previously disabled */
- IOWRITE64_PDUMP(0, VHA_CR_OS(CNN_CONTROL));
- /* Disable core events */
- img_pdump_printf("-- Disable CORE events\n");
- IOWRITE64_PDUMP(0, VHA_CR_OS(VHA_EVENT_ENABLE));
- }
- __maybe_unused
- static void vha_dev_enable_clocks(struct vha_dev *vha)
- {
- uint64_t __maybe_unused sys_clks = 0;
- uint64_t __maybe_unused main_clks = 0;
- /* Always AUTO gating when needed */
- sys_clks = VHA_SYS_CLOCKS_DEFAULT(AUTO);
- main_clks = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
- /* Enable sys clocks ! */
- img_pdump_printf("-- Enable SYS clocks\n");
- IOWRITE64_PDUMP(sys_clks, VHA_CR_SYS_CLK_CTRL0);
- /* Enable main clocks ! */
- img_pdump_printf("-- Enable MAIN clocks\n");
- IOWRITE64_PDUMP(main_clks, VHA_CR_CLK_CTRL0);
- #if defined(CFG_SYS_VAGUS)
- img_pdump_printf("-- Enable NN_SYS clocks\n");
- IOWRITE64_PDUMP_REGIO(NN_SYS_CR_CLK_CTRL_MODE_AUTO,
- NN_SYS_CR_BASE, NN_SYS_CR_CLK_CTRL, "REG_NNSYS");
- #endif
- }
- static void vha_dev_ready(struct vha_dev *vha)
- {
- #ifndef CONFIG_VHA_DUMMY
- if (!vha->is_ready)
- return;
- #endif
- dev_dbg(vha->dev, "%s\n", __func__);
- vha_dev_wait(vha);
- /* Finally enable ALL events */
- img_pdump_printf("-- Enable ALL events\n");
- IOWRITE64_PDUMP(VHA_EVNTS_DEFAULT, VHA_CR_OS(VHA_EVENT_ENABLE));
- img_pdump_printf("-- Clear ALL events\n");
- IOWRITE64_PDUMP(VHA_EVNTS_DEFAULT, VHA_CR_OS(VHA_EVENT_CLEAR));
- #ifdef HW_AX2
- img_pdump_printf("-- Clear CNN status\n");
- IOWRITE64_PDUMP(0, VHA_CR_OS(CNN_STATUS));
- #endif
- img_pdump_printf("-- Clear MMU fault status\n");
- IOWRITE64_PDUMP(0, VHA_CR_OS(MMU_FAULT_STATUS1));
- img_pdump_printf("-- Clear SLC debug status\n");
- IOWRITE64_PDUMP(0, VHA_CR_SLC_STATUS_DEBUG);
- img_pdump_printf("-- Reset PERF counters\n");
- IOWRITE64_PDUMP(0, VHA_CR_PERF_RESET_FULL);
- }
- __maybe_unused
- static int vha_dev_reset(struct vha_dev *vha)
- {
- img_pdump_printf("-- Set RESET bits\n");
- #if defined(CFG_SYS_VAGUS)
- IOWRITE64_PDUMP_REGIO(NN_SYS_CR_RESET_CTRL_NN_SYS_EN,
- NN_SYS_CR_BASE, NN_SYS_CR_RESET_CTRL, "REG_NNSYS");
- #endif
- /* Perform reset procedure */
- IOWRITE64_PDUMP(VHA_RESET_DEFAULT, VHA_CR_RESET_CTRL);
- /* poll for reset deassertion
- * count=16, delay=256cycles
- */
- img_pdump_printf("-- Wait for RESET deassertion\n");
- #if defined(CFG_SYS_VAGUS)
- IOPOLL64_PDUMP_REGIO(0, 16, 256, NN_SYS_CR_RESET_CTRL_MASKFULL,
- NN_SYS_CR_BASE, NN_SYS_CR_RESET_CTRL, "REG_NNSYS");
- #endif
- IOPOLL64_PDUMP(0, 16, 256, VHA_CR_RESET_CTRL_MASKFULL,
- VHA_CR_RESET_CTRL);
- return 0;
- }
- __maybe_unused
- static int vha_dev_disable_clocks(struct vha_dev *vha)
- {
- /* If auto gating was turned on, wait for clocks idle state */
- img_pdump_printf("-- Wait for clocks IDLE state\n");
- IOPOLL64_PDUMP(0, 1000, 1000,
- VHA_CR_CLK_STATUS0_MASKFULL,
- VHA_CR_CLK_STATUS0);
- #if defined(CFG_SYS_VAGUS)
- IOPOLL64_PDUMP_REGIO(0, 100, 1000, NN_SYS_CR_CLK_STATUS_MASKFULL,
- NN_SYS_CR_BASE, NN_SYS_CR_CLK_STATUS, "REG_NNSYS");
- #endif
- /* Wait for MMU,CCM,RDI,XBAR IDLE state */
- img_pdump_printf("-- Wait for memory bus interface IDLE state\n");
- IOPOLL64_PDUMP(0xFFFF, 100, 1000, VHA_CR_SLC_IDLE_MASKFULL,
- VHA_CR_SLC_IDLE);
- /* Finally disable clocks */
- img_pdump_printf("-- Disable MAIN clocks\n");
- IOWRITE64_PDUMP(0, VHA_CR_CLK_CTRL0); /* main */
- img_pdump_printf("-- Disable SYS clocks\n");
- IOWRITE64_PDUMP(0, VHA_CR_SYS_CLK_CTRL0); /* sys */
- #if defined(CFG_SYS_VAGUS)
- img_pdump_printf("-- NN_SYS clocks\n");
- IOWRITE64_PDUMP_REGIO(0, NN_SYS_CR_BASE,
- NN_SYS_CR_CLK_CTRL, "REG_NNSYS"); /* nn_sys */
- #endif
- return 0;
- }
- /* start the device */
- int vha_dev_start(struct vha_dev *vha)
- {
- int ret = 0;
- /* Cancel APM request if new inference comes */
- cancel_delayed_work(&vha->apm_dworks[0].dwork);
- if (vha->state == VHA_STATE_ON)
- return 0; /* not an error */
- dev_dbg(vha->dev, "%s\n", __func__);
- /* Assuming OS0 is the privileged one */
- #if _OSID_ == 0 /* For HW_AX2 this is always true */
- pm_runtime_get_sync(vha->dev);
- /////////////// POWER ON //////////////////////////
- img_pdump_printf("-- POWER_ON_BEGIN\n");
-
- /* Prepare device ... */
- ret = vha_dev_prepare(vha);
- if (ret) {
- dev_err(vha->dev, "%s: Error preparing device!\n", __func__);
- return ret;
- }
- /* Reset device */
- ret = vha_dev_reset(vha);
- if (ret){
- dev_err(vha->dev, "%s: Error reseting device!\n", __func__);
- return ret;
- }
- /* Enable device clocks */
- vha_dev_enable_clocks(vha);
- img_pdump_printf("-- POWER_ON_END\n");
- /* Call device specific setup */
- vha_dev_setup(vha);
- /////////////////////////////////////////////////////
- #endif
- vha_dev_ready(vha);
- vha->state = VHA_STATE_ON;
- /* Remember the time hw is powered on */
- GETNSTIMEOFDAY(&vha->stats.hw_start);
- return ret;
- }
- /* stop the device */
- int vha_dev_stop(struct vha_dev *vha, bool reset)
- {
- int ret = 0;
- if (vha->state == VHA_STATE_OFF)
- return 0; /* not an error */
- /* Cancel APM request if we are about to power off the core */
- cancel_delayed_work(&vha->apm_dworks[0].dwork);
- dev_dbg(vha->dev, "%s\n", __func__);
- /* Disable events at first */
- vha_dev_disable_events(vha);
- vha->is_ready = false;
- /* Assuming OS0 is the privileged one */
- #if _OSID_ == 0 /* For HW_AX2 */
- /////////////// POWER_OFF //////////////////////////
- img_pdump_printf("-- POWER_OFF_BEGIN\n");
- /* Reset core in case of error or pending inference */
- if (reset) {
- /* ensure that clocks are set to AUTO before reset */
- vha_dev_enable_clocks(vha);
- ret = vha_dev_reset(vha);
- }
- if(ret)
- dev_warn(vha->dev,
- "%s: Problem with resetting device!\n",
- __func__);
- /* Disable device clocks */
- ret = vha_dev_disable_clocks(vha);
- if(ret)
- dev_warn(vha->dev,
- "%s: Problem with disabling clocks!\n",
- __func__);
- img_pdump_printf("-- POWER_OFF_END\n");
- /////////////////////////////////////////////////////
- if (reset) {
- pm_runtime_mark_last_busy(vha->dev);
- pm_runtime_put_sync_autosuspend(vha->dev);
- } else {
- pm_runtime_put_sync(vha->dev);
- }
- #endif
- vha->state = VHA_STATE_OFF;
- /* Update the up time of the core */
- if (!vha->do_calibration) {
- uint64_t tmp = 0;
- struct TIMESPEC now;
- GETNSTIMEOFDAY(&now);
- if (get_timespan_us(&vha->stats.hw_start, &now, &tmp)) {
- do_div(tmp, 1000UL);
- vha->stats.uptime_ms += tmp;
- if (vha->stats.uptime_ms)
- vha_update_utilization(vha);
- else
- dev_dbg(vha->dev,
- "%s Too short execution time to calculate utilization!\n",
- __func__);
- } else
- WARN_ON(1);
- }
- vha->active_mmu_ctx = VHA_INVALID_ID;
- spin_lock_irq(&vha->irq_lock);
- vha->irq_status = 0;
- vha->irq_count = 0;
- vha->stream_count = 0;
- spin_unlock_irq(&vha->irq_lock);
- return ret;
- }
- void vha_update_utilization(struct vha_dev *vha)
- {
- uint64_t tmp;
- tmp = vha->stats.cnn_total_proc_us;
- do_div(tmp, vha->stats.uptime_ms);
- vha->stats.cnn_utilization = tmp;
- }
- #ifdef VHA_EVENT_INJECT
- /*
- * Inject EVENT_STATUS bits, requested by respective debugfs nodes, to
- * the status register.
- */
- static inline void __inject_event_regs(struct vha_dev* vha, uint64_t* event_status)
- {
- if(!__EVENT_INJECT())
- return;
- if (*event_status & (1 << VHA_CR_VHA_EVENT_STATUS_TYPE_VHA_CNN0_COMPLETE_SHIFT))
- *event_status |= vha->injection.vha_cr_event;
- }
- #endif
- /* Top half */
- irqreturn_t vha_handle_irq(struct device *dev)
- {
- struct vha_dev *vha = vha_dev_get_drvdata(dev);
- int ret = IRQ_HANDLED;
- uint64_t event_status;
- if (!vha)
- return IRQ_NONE;
- event_status = IOREAD64(vha->reg_base, VHA_CR_OS(VHA_EVENT_STATUS));
- event_status &= IOREAD64(vha->reg_base, VHA_CR_OS(VHA_EVENT_ENABLE));
- /* On fpga platform it is possible to get
- * a spurious interrupt when the hw died
- * Do not proceed, just throw a warning */
- if (event_status == VHA_DEAD_HW || event_status == ~0) {
- WARN_ONCE(1, "Hardware is dead!");
- return IRQ_NONE;
- }
- #ifdef VHA_EVENT_INJECT
- __inject_event_regs(vha, &event_status);
- #endif
- #ifdef VHA_SCF
- if (vha->hw_props.supported.parity &&
- !vha->parity_disable) {
- bool par_bit = img_mem_calc_parity(event_status &
- ~VHA_CR_BITMASK(VHA_EVENT_STATUS_TYPE, PARITY));
- if (par_bit !=
- VHA_CR_GETBITS(VHA_EVENT_STATUS_TYPE, PARITY,
- event_status)) {
- dev_err(dev, "Event status register parity error!\n");
- /* Use the real event to indicate the error */
- event_status |= VHA_CR_OS(VHA_EVENT_STATUS_VHA_PARITY_ERROR_EN);
- }
- /* Clear the PARITY bit - it's not a valid event */
- VHA_CR_CLEARBITS(event_status, VHA_EVENT_STATUS_TYPE, PARITY);
- }
- #endif
- if (event_status & VHA_EVNTS_DEFAULT) {
- uint64_t cnn_status;
- uint8_t count;
- /* clear the interrupt:
- * best not to write pdump in interrupts */
- IOWRITE64(vha->reg_base, VHA_CR_OS(VHA_EVENT_CLEAR),
- event_status & VHA_EVNTS_DEFAULT);
- /* Read the stream count as single IRQ may be raised for multiple kicks */
- cnn_status = IOREAD64(vha->reg_base, VHA_CR_OS(CNN_STATUS));
- #ifdef VHA_SCF
- if (vha->hw_props.supported.parity &&
- !vha->parity_disable) {
- bool par_bit = img_mem_calc_parity(cnn_status &
- ~VHA_CR_BITMASK_OS(CNN_STATUS, PARITY));
- if (par_bit != VHA_CR_GETBITS_OS(CNN_STATUS, PARITY, cnn_status)) {
- dev_err(dev, "CNN status register parity error!\n");
- /* Use the real event to indicate the error */
- event_status |= VHA_CR_OS(VHA_EVENT_STATUS_VHA_PARITY_ERROR_EN);
- }
- }
- #endif
- if (vha->is_ready) {
- /* Post check for AXI bus errors */
- uint64_t ace_status = IOREAD64(vha->reg_base, VHA_CR_ACE_STATUS);
- if (ace_status) {
- dev_err(vha->dev, "AXI bus protocol error: %#llx\n",
- ace_status);
- /* Use AXI error event to indicate that */
- event_status |= VHA_CR_OS(VHA_EVENT_STATUS_VHA_AXI_ERROR_EN);
- }
- }
- /* Read the stream count as single IRQ may be raised for multiple kicks */
- count = VHA_CR_GETBITS_OS(CNN_STATUS, STREAM_COUNT, cnn_status);
- spin_lock(&vha->irq_lock);
- /* store the status to be processed later */
- if (vha->do_calibration ||
- vha_is_busy(vha)) {
- vha->irq_status |= event_status;
- if (vha->low_latency == VHA_LL_SELF_KICK)
- /* Two separate IRQs may be raised for multiple kicks */
- vha->irq_count += count - vha->stream_count;
- else
- /* Only single IRQ may be raised otherwise ... */
- vha->irq_count = count - vha->stream_count;
- vha->stream_count = count;
- /* Record hw processing end timestamps */
- vha->stats.hw_proc_end_prev = vha->stats.hw_proc_end;
- GETNSTIMEOFDAY(&vha->stats.hw_proc_end);
- } else {
- /* Command may have been aborted before this handler is executed */
- vha->irq_status = 0;
- vha->irq_count = 0;
- vha->stream_count = 0;
- }
- spin_unlock(&vha->irq_lock);
- ret = IRQ_WAKE_THREAD;
- } else
- return IRQ_NONE;
- dev_dbg(dev, "IRQ 0x%08llx\n", event_status);
- return ret;
- }
- static bool vha_rollback_cnn_cmds(struct vha_dev *vha)
- {
- bool processing = false;
- /* Not processed commands are still on the pending list
- * of each session, so just mark the hw pending lists as empty */
- if (vha->pendcmd[VHA_CNN_CMD].cmd) {
- struct vha_cmd *pendcmd = vha->pendcmd[VHA_CNN_CMD].cmd;
- pendcmd->in_hw = false;
- pendcmd->queued = false;
- pendcmd->rolled_back = true;
- processing = true;
- vha->stats.cnn_kicks_aborted += pendcmd->subseg_current;
- vha->stats.cnn_kicks_completed -= pendcmd->subsegs_completed;
- vha->pri_q_counters[pendcmd->user_cmd.priority] += pendcmd->subseg_current;
- pendcmd->subseg_current = 0;
- pendcmd->subsegs_completed = 0;
- vha->pendcmd[VHA_CNN_CMD].cmd = NULL;
- }
- /* low_latency ...*/
- if (vha->queuedcmd[VHA_CNN_CMD].cmd) {
- struct vha_cmd *queuedcmd = vha->queuedcmd[VHA_CNN_CMD].cmd;
- queuedcmd->in_hw = false;
- queuedcmd->queued = false;
- queuedcmd->rolled_back = true;
- vha->stats.cnn_kicks_aborted += queuedcmd->subseg_current;
- vha->stats.cnn_kicks_completed -= queuedcmd->subsegs_completed;
- vha->pri_q_counters[queuedcmd->user_cmd.priority] += queuedcmd->subseg_current;
- queuedcmd->subseg_current = 0;
- queuedcmd->subsegs_completed = 0;
- vha->queuedcmd[VHA_CNN_CMD].cmd = NULL;
- }
- dev_dbg(vha->dev, "%s: (%d)\n", __func__, processing);
- return processing;
- }
- bool vha_rollback_cmds(struct vha_dev *vha)
- {
- return vha_rollback_cnn_cmds(vha);
- }
- static bool vha_is_processing(struct vha_dev *vha)
- {
- return vha->pendcmd[VHA_CNN_CMD].cmd != NULL;
- }
- int vha_dev_suspend_work(struct vha_dev *vha)
- {
- bool processing = false;
- int ret;
- /* Check if anything is being processed right now. */
- processing = vha_is_processing(vha);
- /* Forcing hardware disable. */
- ret = vha_dev_stop(vha, processing);
- /* Rollback commands after hw is stopped. */
- vha_rollback_cmds(vha);
- return ret;
- }
- /*
- * handles the command already processed by the hw.
- */
- static bool vha_handle_cmd(struct vha_dev *vha, int status)
- {
- struct vha_cmd *cmd = NULL;
- cmd = vha->pendcmd[VHA_CNN_CMD].cmd;
- if (unlikely(!cmd)) {
- dev_dbg(vha->dev, "No command. Probably it has been aborted\n");
- return false;
- }
- {
- uint64_t proc_time = 0;
- struct TIMESPEC *from = &cmd->hw_proc_start;
- struct TIMESPEC *to = &vha->stats.hw_proc_end;
- if (TIMESPEC_COMPARE(&vha->stats.hw_proc_end_prev, &cmd->hw_proc_start) >= 0)
- from = &vha->stats.hw_proc_end_prev;
- if (get_timespan_us(from, to, &proc_time)) {
- vha->stats.last_proc_us = proc_time;
- } else {
- vha->stats.last_proc_us = 0;
- }
- /* Update cnn stats */
- vha_cnn_update_stats(vha);
- /* Update cmd stats. */
- cmd->proc_us += vha->stats.cnn_last_proc_us;
- cmd->hw_cycles += vha->stats.cnn_last_cycles;
- }
- /* Mark this subsegment as completed. */
- if (status == 0)
- vha->pendcmd[VHA_CNN_CMD].cmd->subsegs_completed++;
- /* If this isn't the last subsegment, just return to process the next one. */
- if ((cmd->subseg_current < VHA_CMD_SUBSEG_NUM(cmd)) && (status == 0)) {
- vha->pendcmd[VHA_CNN_CMD].cmd->in_hw = false;
- vha->pendcmd[VHA_CNN_CMD].cmd = NULL;
- return true;
- }
- vha_cnn_cmd_completed(cmd, status);
- if (status) {
- /* Rollback any queued command ... */
- vha_rollback_cnn_cmds(vha);
- /* Adjust for just rolled back pending cmd. */
- vha->pri_q_counters[cmd->user_cmd.priority] -= VHA_CMD_SUBSEG_NUM(cmd);
- /* Notify immediately current command */
- vha_cmd_notify(cmd);
- return false;
- }
- if (vha->queuedcmd[VHA_CNN_CMD].cmd)
- vha->pendcmd[VHA_CNN_CMD].cmd = vha->queuedcmd[VHA_CNN_CMD].cmd;
- else
- vha->pendcmd[VHA_CNN_CMD].cmd = NULL;
- vha->queuedcmd[VHA_CNN_CMD].cmd = NULL;
- dev_dbg(vha->dev,
- "%s: %p -> new pending %p\n",
- __func__, cmd, vha->pendcmd[VHA_CNN_CMD].cmd);
- vha_cmd_notify(cmd);
- return true;
- }
- static void vha_do_queued_cmd(struct vha_dev *vha)
- {
- struct vha_cmd *cmd, *pend;
- cmd = vha->queuedcmd[VHA_CNN_CMD].cmd;
- dev_dbg(vha->dev,
- "%s: queued %p pending %p\n",
- __func__, cmd, vha->pendcmd[VHA_CNN_CMD].cmd);
- if (!cmd || (cmd &&
- ((vha->low_latency == VHA_LL_DISABLED ||
- vha->low_latency == VHA_LL_SELF_KICK) ||
- !cmd->queued))) {
- dev_dbg(vha->dev, "%s: skipping!\n", __func__);
- return;
- }
- /* store actual pending command as it will be modified */
- pend = vha->pendcmd[VHA_CNN_CMD].cmd;
- /* at this point we should be able to process the cmd */
- vha_do_cnn_cmd(cmd);
- /* restore pending */
- vha->pendcmd[VHA_CNN_CMD].cmd = pend;
- }
- static int vha_report_failure(struct vha_dev *vha, uint64_t status,
- const struct vha_biterr bits[], int bits_size)
- {
- int error = 0;
- int i;
- int cmdid = -1;
- int sesid = -1;
- if (vha->pendcmd[VHA_CNN_CMD].cmd) {
- cmdid = vha->pendcmd[VHA_CNN_CMD].cmd->user_cmd.cmd_id;
- sesid = vha->pendcmd[VHA_CNN_CMD].cmd->session->id;
- }
- if (vha_observers.error)
- vha_observers.error(vha->id, sesid, cmdid, status);
- /* event status in human readable form */
- for (i = 0; i < bits_size; i++) {
- if (status & bits[i].b) {
- dev_err(vha->dev,
- " event status: %s\n",
- bits[i].s);
- /* convert from register bits into POSIX errno
- * if multiple errors, then arbitrary errno choice */
- error = bits[i].e;
- }
- }
- return error;
- }
- /* if vha event register reports CNN events, so handle them */
- static int vha_handle_cnn_event(struct vha_dev *vha, uint64_t event_status)
- {
- int err = 0;
- if (vha_check_calibration(vha))
- return 0;
- if (event_status & VHA_CNN_ERR_EVNTS) {
- static const struct vha_biterr err_bits[] = {
- {-ETIMEDOUT, ERR_EVENT_DESC(CNN0_MEM_WDT)},
- #ifdef HW_AX2
- {-ETIMEDOUT, ERR_EVENT_DESC(CNN0_WDT)},
- #endif
- {-EIO, ERR_EVENT_DESC(CNN0_ERROR)}
- };
- err = vha_report_failure(vha,
- event_status, err_bits, ARRAY_SIZE(err_bits));
- vha_cnn_dump_status(vha);
- }
- /* Poke the hw if there were already
- * command queued in the hw */
- if (!err)
- vha_do_queued_cmd(vha);
- /* Handle actual command */
- if (vha_handle_cmd(vha, err) == false)
- err = -ENOENT;
- return err;
- }
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- /* Simulating hw execution time by scheduling this delayed work. */
- void vha_dummy_worker(struct work_struct *work)
- {
- struct vha_dev *vha = container_of(work, struct vha_dev, dummy_dwork.work);
- mutex_lock(&vha->lock);
- if (vha->pendcmd[VHA_CNN_CMD].cmd) {
- /* Record hw processing end timestamps */
- vha->stats.hw_proc_end_prev = vha->stats.hw_proc_end;
- GETNSTIMEOFDAY(&vha->stats.hw_proc_end);
- /* Handle current pending command */
- vha_handle_cnn_event(vha, VHA_CNN_CMPLT_EVNT);
- vha->stats.cnn_kicks_completed++;
- /* Schedule following commands */
- vha_chk_cmd_queues(vha, true);
- }
- mutex_unlock(&vha->lock);
- }
- #endif
- /* Bottom half */
- irqreturn_t vha_handle_thread_irq(struct device *dev)
- {
- struct vha_dev *vha = vha_dev_get_drvdata(dev);
- irqreturn_t ret = IRQ_HANDLED;
- uint64_t status;
- uint8_t count, c = 0;
- int err = 0;
- if (!vha)
- return IRQ_NONE;
- mutex_lock(&vha->lock);
- #ifdef CONFIG_FAULT_INJECTION
- if (!vha->irq_bh_pid)
- vha->irq_bh_pid = task_pid_nr(current);
- if (vha->fault_inject & VHA_FI_IRQ_WORKER)
- current->make_it_fail = true;
- else
- current->make_it_fail = false;
- #endif
- spin_lock_irq(&vha->irq_lock);
- status = vha->irq_status;
- vha->irq_status = 0;
- count = vha->irq_count;
- vha->irq_count = 0;
- if (!count) {
- uint64_t proc_time = 0;
- if (get_timespan_us(&vha->stats.hw_proc_start, &vha->stats.hw_proc_end,
- &proc_time)) {
- vha->stats.last_proc_us = proc_time;
- } else {
- vha->stats.last_proc_us = 0;
- }
- }
- spin_unlock_irq(&vha->irq_lock);
- /* Command may have been aborted before this handler is executed */
- if (!status)
- goto exit;
- /* There can be two inferences already finished for self kick mode,
- * otherwise, only single inference at the time */
- if ((vha->low_latency == VHA_LL_SELF_KICK && count > 2) ||
- (vha->low_latency != VHA_LL_SELF_KICK && count > 1))
- WARN_ON(1);
- dev_dbg(dev, "%s: status:%llx count:%d\n",
- __func__, status, count);
- do {
- if (status & VHA_CORE_EVNTS) {
- static const struct vha_biterr err_bits[] = {
- {-EIO, ERR_EVENT_DESC(AXI_ERROR)},
- {-EFAULT, ERR_EVENT_DESC(MMU_PAGE_FAULT)},
- #ifdef HW_AX3
- #ifdef VHA_SCF
- {-EIO, ERR_EVENT_DESC(MMU_PARITY_ERROR)},
- {-EIO, ERR_EVENT_DESC(PARITY_ERROR)},
- {-EIO, ERR_EVENT_DESC(LOCKSTEP_ERROR)},
- #endif
- {-ETIMEDOUT, ERR_EVENT_DESC(HL_WDT)},
- {-EIO, ERR_EVENT_DESC(ERROR)}
- #endif
- };
- #ifdef HW_AX3
- if (status & VHA_EVENT_TYPE(HL_WDT)
- && vha->is_ready)
- if (vha_check_calibration(vha))
- break;
- if ((status & VHA_CORE_EVNTS)==
- VHA_EVENT_TYPE(READY)
- && !vha->is_ready) {
- vha->is_ready = true;
- vha_dev_ready(vha);
- if (vha->do_calibration) {
- vha_cnn_start_calib(vha);
- break;
- } else
- vha_chk_cmd_queues(vha, true);
- }
- #endif
- err = vha_report_failure(vha, status,
- err_bits, ARRAY_SIZE(err_bits));
- if (err) {
- dev_err(vha->dev, "NNA hw failure: %llx\n", status);
- dev_err(vha->dev, " CLK_STATUS0:%llx ",
- IOREAD64(vha->reg_base, VHA_CR_CLK_STATUS0));
- dev_err(vha->dev, " VHA_EVENT_STATUS:%llx ", status);
- }
- if (status & VHA_EVENT_TYPE(MMU_PAGE_FAULT))
- /* dump mmu status */
- vha_mmu_status(vha);
- }
- /* If no core level error process cnn events */
- if (!err && status & VHA_CNN_EVNTS)
- err = vha_handle_cnn_event(vha, status);
- #ifdef HW_AX3
- else if (status == VHA_EVENT_TYPE(ERROR)) {
- /* Resubmit command next time if no CNN error detected
- * and only ERROR bit is set.
- * That means other OS caused the error */
- vha_rollback_cnn_cmds(vha);
- }
- #endif
- else if (err && vha->is_ready) { /* Core level error */
- if (vha_handle_cmd(vha, err) == false)
- err = -ENOENT;
- }
- c++;
- } while (c < count && !err);
- if (err) {
- vha->stats.total_failures += count ? count : 1;
- vha_dev_stop(vha, true);
- /* Check queues ... */
- vha_chk_cmd_queues(vha, true);
- } else {
- /* Run in BH context! */
- vha_chk_cmd_queues(vha, false);
- }
- vha->stats.cnn_kicks_completed += count;
- exit:
- #ifdef CONFIG_FAULT_INJECTION
- if (vha->fault_inject & VHA_FI_IRQ_WORKER)
- current->make_it_fail = false;
- #endif
- mutex_unlock(&vha->lock);
- return ret;
- }
- bool vha_rm_session_cmds(struct vha_session *session)
- {
- struct vha_dev *vha = session->vha;
- bool pend_removed = false;
- bool queued_removed = false;
- bool reschedule = false;
- struct vha_cmd *cur_cmd, *tmp_cmd;
- uint8_t pri;
- /* Check if pend/queued commands will be removed. */
- if (vha->pendcmd[VHA_CNN_CMD].cmd &&
- vha->pendcmd[VHA_CNN_CMD].cmd->session == session) {
- dev_warn(vha->dev,
- "Removing a session while cnn cmd is still pending\n");
- pend_removed = true;
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- cancel_delayed_work(&vha->dummy_dwork);
- #endif
- }
- if (vha->queuedcmd[VHA_CNN_CMD].cmd &&
- vha->queuedcmd[VHA_CNN_CMD].cmd->session == session) {
- dev_warn(vha->dev,
- "Removing a session while cnn cmd is still queued\n");
- queued_removed = true;
- }
- /* Update session scheduling. */
- if (vha->queuedcmd[VHA_CNN_CMD].cmd &&
- (pend_removed && !queued_removed)) {
- uint8_t pri = vha->queuedcmd[VHA_CNN_CMD].cmd->user_cmd.priority;
- if (vha->queuedcmd[VHA_CNN_CMD].cmd->session !=
- list_entry(&vha->sched_sessions[pri], struct vha_session,
- sched_list[pri]))
- while(list_first_entry(&vha->sched_sessions[pri], struct vha_session,
- sched_list[pri]) != vha->queuedcmd[VHA_CNN_CMD].cmd->session)
- list_rotate_left(&vha->sched_sessions[pri]);
- }
- /* Remove pend/queued commands if needed. */
- if (pend_removed || queued_removed) {
- vha_rollback_cnn_cmds(vha);
- /* Need to reschedule too. */
- reschedule = true;
- }
- /* Remove session related commands. */
- for (pri = 0; pri < VHA_MAX_PRIORITIES; pri++) {
- list_for_each_entry_safe(cur_cmd, tmp_cmd, &session->cmds[pri], list[pri]) {
- /* rsp didn't make it to rsps list, free it now */
- kfree(cur_cmd->rsp);
- list_del(&cur_cmd->list[cur_cmd->user_cmd.priority]);
- vha->pri_q_counters[cur_cmd->user_cmd.priority] -=
- (VHA_CMD_SUBSEG_NUM(cur_cmd) - cur_cmd->subseg_current);
- if (vha_observers.canceled)
- vha_observers.canceled(vha->id, session->id, cur_cmd->user_cmd.cmd_id,
- cur_cmd->user_cmd.priority);
- kfree(cur_cmd);
- }
- }
- return reschedule;
- }
- bool vha_rm_session_cmds_masked(struct vha_session *session, uint32_t cmd_id,
- uint32_t cmd_id_mask)
- {
- struct vha_dev *vha = session->vha;
- bool reschedule = false;
- bool pend_removed = false;
- uint32_t pend_aborted_kicks_adj_val = 0;
- bool queued_removed = false;
- uint32_t queued_aborted_kicks_adj_val = 0;
- /* Check if pend/queued commands will be removed. */
- if (vha->pendcmd[VHA_CNN_CMD].cmd &&
- (vha->pendcmd[VHA_CNN_CMD].cmd->session == session) &&
- (vha->pendcmd[VHA_CNN_CMD].cmd->user_cmd.cmd_id & cmd_id_mask)
- == cmd_id) {
- pend_removed = true;
- vha->stats.cnn_kicks_cancelled += vha->pendcmd[VHA_CNN_CMD].cmd->subseg_current;
- pend_aborted_kicks_adj_val = vha->pendcmd[VHA_CNN_CMD].cmd->subseg_current;
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- cancel_delayed_work(&vha->dummy_dwork);
- #endif
- }
- if (vha->queuedcmd[VHA_CNN_CMD].cmd &&
- (vha->queuedcmd[VHA_CNN_CMD].cmd->session == session) &&
- (vha->queuedcmd[VHA_CNN_CMD].cmd->user_cmd.cmd_id & cmd_id_mask)
- == cmd_id) {
- queued_removed = true;
- vha->stats.cnn_kicks_cancelled += vha->queuedcmd[VHA_CNN_CMD].cmd->subseg_current;
- queued_aborted_kicks_adj_val = vha->pendcmd[VHA_CNN_CMD].cmd->subseg_current;
- }
- /* Update session scheduling. */
- if (vha->queuedcmd[VHA_CNN_CMD].cmd &&
- (pend_removed && !queued_removed)) {
- uint8_t pri = vha->queuedcmd[VHA_CNN_CMD].cmd->user_cmd.priority;
- if (vha->queuedcmd[VHA_CNN_CMD].cmd->session !=
- list_entry(&vha->sched_sessions[pri], struct vha_session,
- sched_list[pri]))
- while(list_first_entry(&vha->sched_sessions[pri], struct vha_session,
- sched_list[pri]) != vha->queuedcmd[VHA_CNN_CMD].cmd->session)
- list_rotate_left(&vha->sched_sessions[pri]);
- }
- /* Remove pend/queued commands if needed. */
- if (pend_removed || queued_removed) {
- vha_rollback_cnn_cmds(vha);
- /* Correct aborted stats. */
- if (queued_removed)
- vha->stats.cnn_kicks_aborted -= queued_aborted_kicks_adj_val;
- if (pend_removed)
- vha->stats.cnn_kicks_aborted -= pend_aborted_kicks_adj_val;
- reschedule = true;
- }
- return reschedule;
- }
- int vha_rm_cmds(struct vha_session *session, uint32_t cmd_id,
- uint32_t cmd_id_mask, bool respond)
- {
- struct vha_dev *vha = session->vha;
- struct vha_cmd *cur_cmd, *tmp_cmd;
- struct vha_rsp *cur_rsp, *tmp_rsp;
- bool reschedule = false;
- bool respond_aux = false;
- int ret = 0;
- uint8_t pri;
- mutex_lock(&vha->lock);
- /* Remove pend/queued session commands that match the cmd_id. */
- reschedule = vha_rm_session_cmds_masked(session, cmd_id, cmd_id_mask);
- /* Remove session related commands matching command id template. */
- for (pri = 0; pri < VHA_MAX_PRIORITIES; pri++) {
- list_for_each_entry_safe(cur_cmd, tmp_cmd, &session->cmds[pri], list[pri]) {
- if ((cur_cmd->user_cmd.cmd_id & cmd_id_mask) == cmd_id) {
- #ifdef KERNEL_DMA_FENCE_SUPPORT
- switch (cur_cmd->user_cmd.cmd_type)
- {
- case VHA_CMD_CNN_SUBMIT:
- {
- struct vha_user_cnn_submit_cmd *cnn_cmd =
- (struct vha_user_cnn_submit_cmd *)&cur_cmd->user_cmd;
- int j;
- for (j = 0; j < (cnn_cmd->msg.num_bufs - 1); j++) {
- struct vha_buffer *buf = vha_find_bufid(session, cnn_cmd->bufs[j]);
- if (buf == NULL) {
- dev_warn(vha->dev, "%s: could not find buf %x\n", __func__,
- cnn_cmd->bufs[j]);
- } else {
- vha_rm_buf_fence(session, buf);
- }
- }
- break;
- }
- default:
- dev_warn(vha->dev, "%s: invalid cmd type %x\n", __func__,
- cur_cmd->user_cmd.cmd_type);
- break;
- }
- #endif
- /* rsp didn't make it to rsps list; free it now. */
- kfree(cur_cmd->rsp);
- list_del(&cur_cmd->list[cur_cmd->user_cmd.priority]);
- vha->pri_q_counters[cur_cmd->user_cmd.priority] -=
- (VHA_CMD_SUBSEG_NUM(cur_cmd) - cur_cmd->subseg_current);
- if (vha_observers.canceled)
- vha_observers.canceled(vha->id, session->id, cur_cmd->user_cmd.cmd_id,
- cur_cmd->user_cmd.priority);
- kfree(cur_cmd);
- /* There were commands matching command id template in the list,
- * so respond to wake user space. */
- respond_aux = true;
- }
- }
- }
- /* Remove responses for session related commands
- * matching command id template. */
- list_for_each_entry_safe(cur_rsp, tmp_rsp, &session->rsps, list) {
- if ((cur_rsp->user_rsp.cmd_id & cmd_id_mask) == cmd_id) {
- list_del(&cur_rsp->list);
- kfree(cur_rsp);
- respond_aux = true;
- }
- }
- /* Reset hardware if required. */
- if (reschedule)
- ret = vha_dev_stop(vha, reschedule);
- /* Generate "cancel" response if any commands matching command id template
- * were removed. */
- if (respond_aux && respond) {
- /* Calculate space for the response. */
- size_t sz = sizeof(struct vha_rsp)
- + sizeof(struct vha_user_cnn_submit_rsp)
- - sizeof(struct vha_user_rsp);
- /* Allocate space for standard response. */
- struct vha_rsp *rsp = kzalloc(sz, GFP_KERNEL);
- if (rsp == NULL) {
- dev_crit(session->vha->dev,
- "Failed to allocate memory to notify cancel for cmds 0x%08x\n", cmd_id);
- session->oom = true;
- } else {
- rsp->size = sizeof(struct vha_user_cnn_submit_rsp);
- rsp->user_rsp.cmd_id = cmd_id;
- list_add_tail(&rsp->list, &session->rsps);
- }
- wake_up(&session->wq);
- }
- mutex_unlock(&vha->lock);
- /* Just return in case of oom. */
- if (session->oom)
- return -ENOMEM;
- /* Reschedule once all commands matching command id template are removed. */
- if (reschedule)
- vha_chk_cmd_queues(vha, true);
- return ret;
- }
- bool vha_is_busy(struct vha_dev *vha)
- {
- #ifndef CONFIG_VHA_DUMMY
- if (!vha->is_ready)
- return true;
- #endif
- if (vha->low_latency != VHA_LL_DISABLED) {
- return vha->pendcmd[VHA_CNN_CMD].cmd != NULL ||
- vha->queuedcmd[VHA_CNN_CMD].cmd != NULL;
- }
- return vha->pendcmd[VHA_CNN_CMD].cmd != NULL;
- }
- /* returns true if the cmd queue is full */
- bool vha_is_queue_full(struct vha_dev *vha, struct vha_cmd *cmd)
- {
- if (vha->low_latency != VHA_LL_DISABLED) {
- if (vha->low_latency == VHA_LL_SELF_KICK
- #ifdef HW_AX3
- /* if current command we are trying to queue belongs to a different session than pending one */
- && (vha->pendcmd[VHA_CNN_CMD].cmd != NULL && cmd != NULL &&
- vha->pendcmd[VHA_CNN_CMD].cmd->session != cmd->session)
- /* if session of the command we are trying to queue, shares the hw mmu ctx with the session of pending cmd */
- && (cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id ==
- vha->pendcmd[VHA_CNN_CMD].cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id)
- /* Sanity if hw mmu ctx is really shared at this point */
- && (vha->mmu_ctxs[cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id] > 1)
- ) {
- #else
- ) {
- dev_warn(vha->dev, "%s: LL=2 not supported!\n", __func__);
- #endif
- /* skip low latency mode */
- return vha->pendcmd[VHA_CNN_CMD].cmd != NULL;
- }
- return vha->pendcmd[VHA_CNN_CMD].cmd != NULL &&
- vha->queuedcmd[VHA_CNN_CMD].cmd != NULL;
- }
- return vha->pendcmd[VHA_CNN_CMD].cmd != NULL;
- }
- /* check all input buffers are filled and ready to go */
- bool vha_is_waiting_for_inputs(struct vha_session *session,
- struct vha_cmd *cmd)
- {
- if (!cmd->inbufs_ready) {
- const struct vha_user_cnn_submit_cmd *user_cmd =
- (struct vha_user_cnn_submit_cmd *)&cmd->user_cmd;
- int i;
- for (i = 0; i < cmd->user_cmd.num_inbufs - 1; i++) {
- struct vha_buffer *buf = vha_find_bufid(session, user_cmd->bufs[i]);
- if (buf && buf->status == VHA_BUF_UNFILLED) {
- dev_dbg(session->vha->dev,
- "%s: cmd %u waiting for input "
- "buf %d to be ready\n",
- __func__,
- cmd->user_cmd.cmd_id,
- buf->id);
- return true;
- }
- }
- }
- cmd->inbufs_ready = true;
- return false;
- }
- static bool vha_can_schedule(struct vha_dev *vha)
- {
- #ifndef CONFIG_VHA_DUMMY
- if (!vha->is_ready)
- return false;
- #endif
- if (vha->low_latency != VHA_LL_DISABLED) {
- return vha->pendcmd[VHA_CNN_CMD].cmd == NULL ||
- vha->queuedcmd[VHA_CNN_CMD].cmd == NULL;
- }
- return vha->pendcmd[VHA_CNN_CMD].cmd == NULL;
- }
- static void vha_scheduler_set_starting_session(struct vha_dev *vha,
- uint8_t priority, struct vha_session *session, bool set_next)
- {
- /* Rotate scheduling list to the current session
- * to make it a starting point for the next scheduling round. */
- if (session != list_entry(&vha->sched_sessions[priority],
- struct vha_session, sched_list[priority]))
- while(list_first_entry(&vha->sched_sessions[priority],
- struct vha_session, sched_list[priority]) != session)
- list_rotate_left(&vha->sched_sessions[priority]);
- /* Set a starting point session for the next scheduling round
- * to next to the current one if requested. */
- if (set_next)
- list_rotate_left(&vha->sched_sessions[priority]);
- }
- static uint8_t vha_scheduler_get_priority(struct vha_dev *vha)
- {
- uint8_t pri;
- /* Calculate current total window width. */
- for (pri = VHA_MAX_PRIORITIES - 1; (int8_t)pri >= 0; pri--)
- if (vha->pri_q_counters[pri] > 0)
- return pri;
- /* If there's no priority with WLs to schedule, just return 0. */
- return VHA_INVALID_PRI;
- }
- void vha_scheduler_loop(struct vha_dev *vha)
- {
- struct vha_cmd *cmd, *tmp;
- struct vha_session *session = NULL;
- enum do_cmd_status cmd_status = CMD_OK;
- bool scheduled = false;
- uint8_t current_pri = VHA_DEFAULT_PRI;
- if (vha_is_queue_full(vha, NULL)) {
- /* Postpone worker task if command queue is full. */
- dev_dbg(vha->dev, "%s Queue full. Postpone worker task!\n", __func__);
- return;
- }
- do {
- scheduled = false;
- current_pri = vha_scheduler_get_priority(vha);
- if (current_pri == VHA_INVALID_PRI)
- break;
- list_for_each_entry(session, &vha->sched_sessions[current_pri], sched_list[current_pri]) {
- list_for_each_entry_safe(cmd, tmp, &session->cmds[current_pri], list[current_pri]) {
- /* For hw commands... */
- if (CMD_EXEC_ON_HW(cmd)) {
- if (!VHA_IS_DUMMY(vha)) {
- /* Start device. */
- if(vha_dev_start(vha))
- return;
- }
- }
- /* Skip this workload as it's already scheduled. */
- if (cmd->queued || cmd->in_hw)
- continue;
- dev_dbg(vha->dev, "%s cur_prio=<%d>\n", __func__,current_pri);
- /* Attempt to schedule command for execution. */
- cmd_status = vha_do_cmd(cmd);
- /* Update scheduling loop based on command scheduling status. */
- if ((cmd_status == CMD_OK) || (cmd_status == CMD_HW_BUSY)) {
- bool set_next = false;
- if (cmd_status == CMD_OK) {
- scheduled = true;
- if (cmd->subseg_current == VHA_CMD_SUBSEG_NUM(cmd))
- set_next = true;
- }
- vha_scheduler_set_starting_session(vha, current_pri, session, set_next);
- goto exit_session_loop;
- }
- }
- }
- exit_session_loop:;
- /* Iterate until a workload was scheduled and no other can be scheduled. */
- } while (vha_can_schedule(vha) && scheduled);
- if (!VHA_IS_DUMMY(vha)) {
- /* Schedule APM if needed */
- if (!vha_is_busy(vha) &&
- !vha->no_clock_disable) {
- if (!vha->pm_delay) {
- if (vha_dev_stop(vha, false)) {
- dev_warn(vha->dev, "%s: Failed to soft stop device. trying with reset",
- __func__);
- if (vha_dev_stop(vha, true))
- dev_err(vha->dev, "%s: Failed to stop device with reset!", __func__);
- }
- }
- else {
- vha->apm_dworks[0].delay_ms = vha->pm_delay;
- vha_sched_apm(vha, &vha->apm_dworks[0]);
- }
- }
- }
- }
- void vha_dev_apm_stop(struct vha_dev *vha, struct vha_apm_work *apm_work)
- {
- if (!vha->do_calibration &&
- (vha->pendcmd[VHA_CNN_CMD].cmd == NULL &&
- vha->queuedcmd[VHA_CNN_CMD].cmd == NULL))
- if (vha_dev_stop(vha, false)) {
- dev_warn(vha->dev, "%s: Failed to soft stop device. trying with reset",
- __func__);
- if (vha_dev_stop(vha, true))
- dev_err(vha->dev, "%s: Failed to stop device with reset!", __func__);
- }
- }
- int vha_dev_get_props(struct vha_dev *vha, uint32_t onchipmem_size)
- {
- struct vha_hw_props *props = &vha->hw_props;
- uint64_t ip_config;
- uint32_t ocm_size_kb = 0;
- memset(props, 0, sizeof(*props));
- #ifdef CONFIG_VHA_DUMMY
- /* Note: dummy dev always reads zeroes from registers */
- props->product_id = 0x8070605040302010ULL;
- props->core_id = (long)HW_SERIES << (int)VHA_CR_CORE_ID_BRANCH_ID_SHIFT;
- props->core_id += 0x010203040505ULL; // provide a dummy core id
- props->dummy_dev = true;
- props->num_cnn_core_devs = 1;
- #else
- props->product_id = IOREAD64(vha->reg_base, VHA_CR_PRODUCT_ID);
- props->core_id = IOREAD64(vha->reg_base, VHA_CR_CORE_ID);
- #endif
- props->skip_bvnc_check = false;
- /*
- * New mmu version 3 and onwards operates on 40bit physical & virtual addresses
- */
- props->mmu_width = 40;
- /* HW from 1.1 onwards */
- ip_config = IOREAD64(vha->reg_base, VHA_CR_CORE_IP_CONFIG);
- #ifdef HW_AX3
- props->mmu_ver = VHA_CR_GETBITS(CORE_IP_CONFIG, MMU_VERSION, ip_config);
- #endif
- /* Mirage uses MMU version 3 hardware */
- if (!props->mmu_ver)
- props->mmu_ver = 3;
- if (VHA_CR_GETBITS(CORE_IP_CONFIG, CNN_SUPPORTED, ip_config))
- props->num_cnn_core_devs = 1;
- if (VHA_CR_GETBITS(CORE_IP_CONFIG, RTM_SUPPORTED, ip_config))
- props->supported.rtm = 1;
- #ifdef HW_AX3
- if (VHA_CR_GETBITS(CORE_IP_CONFIG, PARITY_REGISTERS, ip_config))
- props->supported.parity = 1;
- #if defined(CONFIG_VHA_DUMMY) && defined(VHA_SCF)
- /* Force parity for pdump generation */
- props->supported.parity = 1;
- #endif
- #endif
- if ((props->num_cnn_core_devs == 0)
- || VHA_CR_GETBITS(CORE_ID, BRANCH_ID, props->core_id) != HW_SERIES) {
- dev_err(vha->dev, "%s: Wrong core configuration detected. "
- "Expected BVNC %d.x.x.x, got %llu.x.x.x. "
- "Maybe kernel module was built with wrong params.\n",
- __func__, HW_SERIES,
- VHA_CR_GETBITS(CORE_ID, BRANCH_ID, props->core_id));
- return -ENODEV;
- }
- props->soc_axi = IOREAD64(vha->reg_base, VHA_CR_SOC_AXI);
- dev_info(vha->dev, "%s: Product id: %#llx\n",
- __func__, props->product_id);
- dev_info(vha->dev, "%s: Core id: %#llx\n",
- __func__, props->core_id);
- dev_info(vha->dev, "%s: MMU version:%d (%dbit)\n",
- __func__, props->mmu_ver, props->mmu_width);
- dev_dbg(vha->dev, "%s: supported: %#x\n",
- __func__, props->features);
- dev_dbg(vha->dev, "%s: soc_axi: %#llx\n",
- __func__, props->soc_axi);
- {
- uint64_t tmp = IOREAD64(vha->reg_base,
- VHA_CR_CORE_IP_INTEGRATOR_ID);
- dev_dbg(vha->dev, "%s: ip integrator id: %#llx\n",
- __func__, tmp);
- tmp = IOREAD64(vha->reg_base, VHA_CR_CORE_IP_CHANGELIST);
- dev_dbg(vha->dev, "%s: ip change list: %llu\n", __func__, tmp);
- }
- #if defined(CFG_SYS_VAGUS)
- ocm_size_kb = IOREAD64(vha->reg_base, NN_SYS_CR(CORE_IP_CONFIG)) &
- ~NN_SYS_CR_CORE_IP_CONFIG_NN_SYS_OCM_RAM_SIZE_4KB_CLRMSK;
- ocm_size_kb *= 4;
- #endif
- if (ocm_size_kb) {
- vha->hw_props.locm_size_bytes = ocm_size_kb * 1024;
- /* User may wanted to limit OCM ... */
- if (onchipmem_size) {
- if (onchipmem_size < vha->hw_props.locm_size_bytes) {
- dev_warn(vha->dev, "%s:Limiting onchip memory to %u bytes (available:%u)\n",
- __func__, onchipmem_size, vha->hw_props.locm_size_bytes);
- vha->hw_props.locm_size_bytes = onchipmem_size;
- } else if (onchipmem_size > vha->hw_props.locm_size_bytes) {
- dev_err(vha->dev, "%s: User defined onchip memory size exceeded (%u > %u))\n",
- __func__, onchipmem_size, vha->hw_props.locm_size_bytes);
- }
- }
- } else {
- vha->hw_props.locm_size_bytes = onchipmem_size;
- }
- dev_info(vha->dev, "%s: Total onchip memory: %u [kB]\n",
- __func__, vha->hw_props.locm_size_bytes / 1024);
- dev_info(vha->dev, "%s: Devices: DUMMY:%u CNN:%u\n", __func__,
- props->dummy_dev ? props->num_cnn_core_devs : 0,
- props->dummy_dev ? 0 : props->num_cnn_core_devs);
- return 0;
- }
- void vha_dev_ocm_configure(struct vha_dev *vha)
- {
- #if defined(CFG_SYS_VAGUS)
- dev_dbg(vha->dev, "%s: OCM address range: %#lx - %#lx\n",
- __func__, vha->ocm_paddr,
- vha->ocm_paddr + vha->hw_props.locm_size_bytes - 1);
- IOWRITE64(vha->reg_base, NN_SYS_CR(NOC_LOWER_ADDR1), vha->ocm_paddr);
- IOWRITE64(vha->reg_base, NN_SYS_CR(NOC_UPPER_ADDR1),
- vha->ocm_paddr + vha->hw_props.locm_size_bytes - 1);
- img_pdump_printf("-- Setup NN_SYS OCM phys address range\n"
- "WRW "_PMEM_":$0 :OCM:BLOCK_CACHE:0x0\n"
- "WRW64 :REG_NNSYS:%#x "_PMEM_":$0\n"
- "WRW "_PMEM_":$0 :OCM:BLOCK_CACHE:%#x\n"
- "WRW64 :REG_NNSYS:%#x "_PMEM_":$0\n",
- NN_SYS_CR_NOC_LOWER_ADDR1, vha->hw_props.locm_size_bytes-1,
- NN_SYS_CR_NOC_UPPER_ADDR1);
- #endif
- }
- /* prepare CRC and DEBUG data buffers */
- void vha_dbg_prepare_hwbufs(struct vha_session *session, struct vha_cmd *cmd,
- struct vha_crc_config_regs *regs)
- {
- struct vha_dev *vha = session->vha;
- (void)cmd;
- if (session->cnn_dbg.cnn_crc_buf[0]) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_crc_buf[0];
- uint64_t val64;
- /* enable CRC: address + mode */
- val64 = VHA_CR_SETBITS_OS(CNN_CRC_CONTROL, CNN_CRC_ENABLE,
- session->cnn_dbg.cnn_crc_mode);
- img_pdump_printf("-- CRC_CONTROL=%u buf 'CRC' size=%zx\n",
- session->cnn_dbg.cnn_crc_mode, buf->size);
- IOWRITE_PDUMP_BUFADDR(session, buf, 0, VHA_CR_OS(CNN_CRC_ADDRESS));
- IOWRITE64_PDUMP(val64, VHA_CR_OS(CNN_CRC_CONTROL));
- #ifdef HW_AX3
- img_pdump_printf("-- CRC_MASK=%#x\n", session->cnn_dbg.cnn_crc_mask);
- IOWRITE64_PDUMP(session->cnn_dbg.cnn_crc_mask, VHA_CR_OS(CNN_CRC_MASK_CTRL));
- #endif
- }
- if (session->cnn_dbg.cnn_dbg_buf[0] && session->cnn_dbg.cnn_dbg_pdump_enable) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_dbg_buf[0];
- uint64_t val64;
- /* enable DEBUG: address, perf mode, band mode */
- img_pdump_printf("-- DEBUG_CONTROL=%u,%u buf 'DBG' size=%zx\n",
- GET_CNN_DBG_MODE(PERF, session), GET_CNN_DBG_MODE(BAND, session),
- buf->size);
- IOWRITE_PDUMP_BUFADDR(session, buf, 0,
- VHA_CR_OS(CNN_DEBUG_ADDRESS));
- val64 = VHA_CR_ALIGN_SETBITS_OS(CNN_DEBUG_SIZE,
- CNN_DEBUG_SIZE,
- buf->size);
- IOWRITE64_PDUMP(val64, VHA_CR_OS(CNN_DEBUG_SIZE));
- /* Set the CONTROL register only if requested */
- if (CNN_DBG_MODE_ON(PERF, session) || CNN_DBG_MODE_ON(BAND, session)) {
- val64 = VHA_CR_SETBITS_OS(CNN_DEBUG_CONTROL, CNN_PERF_ENABLE,
- GET_CNN_DBG_MODE(PERF, session));
- val64 |= VHA_CR_SETBITS_OS(CNN_DEBUG_CONTROL, CNN_BAND_ENABLE,
- GET_CNN_DBG_MODE(BAND, session));
- IOWRITE64_PDUMP(val64, VHA_CR_OS(CNN_DEBUG_CONTROL));
- }
- }
- }
- /* flush CRC and DEBUG data buffers */
- void vha_dbg_flush_hwbufs(struct vha_session *session, char checkpoint, uint8_t mask)
- {
- struct vha_dev* vha = session->vha;
- (void)mask;
- if (session->cnn_dbg.cnn_dbg_flush != checkpoint)
- return;
- if (session->cnn_dbg.cnn_crc_buf[0]) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_crc_buf[0];
- /*
- * TOBEDONE: calculate CRC buffer size based
- * on num passes, num layers, etc
- */
- img_pdump_printf("-- Save signatures\n");
- img_pdump_printf("IF CHECK_CRCS\n");
- img_pdump_printf("COM Checking CRCs ...\n");
- vha_pdump_sab_buf(session, PDUMP_CRC,
- buf, 0, buf->size);
- img_pdump_printf("ELSE CHECK_CRCS\n");
- img_pdump_printf("COM Not checking CRCs!\n");
- img_pdump_printf("FI CHECK_CRCS\n");
- }
- if (session->cnn_dbg.cnn_dbg_buf[0] && session->cnn_dbg.cnn_dbg_pdump_enable) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_dbg_buf[0];
- /* read the size of the DEBUG buffer */
- uint64_t size = IOREAD64(vha->reg_base, VHA_CR_OS(CNN_DEBUG_STATUS));
- /*
- * SAB the DBG buffer, even though "it is not deterministic"
- */
- size = VHA_CR_GETBITS_OS(CNN_DEBUG_STATUS,
- CNN_DEBUG_OFFSET,
- size);
- img_pdump_printf("-- Save DEBUG info\n");
-
- vha_pdump_sab_buf(session, PDUMP_DBG, buf, 0, buf->size);
- }
- }
- /* stop capturing CRC and DEBUG data */
- void vha_dbg_stop_hwbufs(struct vha_session *session, uint8_t mask)
- {
- struct vha_dev *vha = session->vha;
- (void)mask;
- /* Flush hw debug buffers */
- vha_dbg_flush_hwbufs(session, 0, 0);
- if (session->cnn_dbg.cnn_crc_buf[0]) {
- IOWRITE64_PDUMP(0, VHA_CR_OS(CNN_CRC_CONTROL));
- }
- if (session->cnn_dbg.cnn_dbg_buf[0]) {
- /* read the size of the DEBUG buffer */
- uint64_t size = IOREAD64(vha->reg_base, VHA_CR_OS(CNN_DEBUG_STATUS));
- if (CNN_DBG_MODE_ON(PERF, session) || CNN_DBG_MODE_ON(BAND, session)) {
- IOWRITE64_PDUMP(0, VHA_CR_OS(CNN_DEBUG_CONTROL));
- /* just give a hint in the pdump:
- * dummy device returns 0 */
- img_pdump_printf(
- "-- POL64 :REG:%#x 0 0 0 1 1 -- DEBUG_STATUS=%llx\n",
- VHA_CR_OS(CNN_DEBUG_STATUS),
- size);
- }
- }
- }
- uint64_t vha_dbg_rtm_read(struct vha_dev *vha, uint64_t addr)
- {
- /* Turn on all clocks forcefully */
- IOWRITE64(vha->reg_base, VHA_CR_SYS_CLK_CTRL0, VHA_SYS_CLOCKS_DEFAULT(ON));
- IOWRITE64(vha->reg_base, VHA_CR_CLK_CTRL0, VHA_MAIN_CLOCKS_DEFAULT(ON));
- /* Set up address of the signal */
- IOWRITE64(vha->reg_base, VHA_CR_RTM_CTRL, addr | VHA_CR_RTM_CTRL_RTM_ENABLE_EN);
-
- /* but N_OF_RTM_STAGES is not accessible by SW*/
- /* so waiting 1 ms for now */
- msleep(1);
- /* Read the data */
- return IOREAD64(vha->reg_base, VHA_CR_RTM_DATA);
- }
- /* List of predefined registers to be shown in debugfs */
- const struct vha_reg vha_regs[] = {
- #define REG_DESC(reg) VHA_CR_##reg, VHA_CR_##reg##_MASKFULL
- #define REG_DESC_OS(reg) VHA_CR_OS(reg), VHA_CR_OS(reg##_MASKFULL)
- {"main_clocks_control ", REG_DESC(CLK_CTRL0)},
- {"main_clocks_status ", REG_DESC(CLK_STATUS0)},
- {"sys_clocks_control ", REG_DESC(SYS_CLK_CTRL0)},
- {"sys_clocks_status ", REG_DESC(SYS_CLK_STATUS0)},
- {"product_id ", REG_DESC(PRODUCT_ID)},
- {"core_id ", REG_DESC(CORE_ID)},
- {"soc_axi ", REG_DESC(SOC_AXI)},
- {"integrator_id ", REG_DESC(CORE_IP_INTEGRATOR_ID)},
- {"ip_changelist ", REG_DESC(CORE_IP_CHANGELIST)},
- {"core_ip_config ", REG_DESC(CORE_IP_CONFIG)},
- {"reset ", REG_DESC(RESET_CTRL)},
- {"event_enable ", REG_DESC_OS(VHA_EVENT_ENABLE)},
- {"event_status ", REG_DESC_OS(VHA_EVENT_STATUS)},
- {"cnn_control ", REG_DESC_OS(CNN_CONTROL)},
- {"cnn_status ", REG_DESC_OS(CNN_STATUS)},
- #ifdef HW_AX2
- {"cnn_wdt_cmpmatch ", REG_DESC(CNN_WDT_COMPAREMATCH)},
- {"cnn_wdt_control ", REG_DESC(CNN_WDT_CTRL)},
- {"cnn_wdt_timer ", REG_DESC(CNN_WDT_TIMER)},
- #endif
- {"cnn_mem_wdt_cmpmatch ", REG_DESC(CNN_MEM_WDT_COMPAREMATCH)},
- {"cnn_mem_wdt_control ", REG_DESC(CNN_MEM_WDT_CTRL)},
- {"cnn_mem_wdt_timer ", REG_DESC(CNN_MEM_WDT_TIMER)},
- {"mmu_control ", REG_DESC_OS(MMU_CTRL)},
- {"mmu_context ", REG_DESC_OS(MMU_CBASE_MAPPING_CONTEXT)},
- {"mmu_mapping ", REG_DESC_OS(MMU_CBASE_MAPPING)},
- {"mmu_status ", REG_DESC(MMU_STATUS)},
- {"mmu_fault_status1 ", REG_DESC_OS(MMU_FAULT_STATUS1)},
- {"mmu_fault_status2 ", REG_DESC_OS(MMU_FAULT_STATUS2)},
- {"slc_control ", REG_DESC(SLC_CTRL)},
- #if 0
- {"slc_bypass_control ", REG_DESC(SLC_BYPASS_CTRL)},
- #endif
- {"slc_status1 ", REG_DESC(SLC_STATUS1)},
- {"slc_status2 ", REG_DESC(SLC_STATUS2)},
- {"slc_status3 ", REG_DESC(SLC_STATUS3)},
- {"slc_idle ", REG_DESC(SLC_IDLE)},
- {"bif_outstanding_read ", REG_DESC(BIF_OUTSTANDING_READ)},
- #undef REG_DESC
- #undef REG_DESC_OS
- {NULL , 0},
- };
|