1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213 |
- /*
- *****************************************************************************
- * Copyright (c) Imagination Technologies Ltd.
- *
- * The contents of this file are subject to the MIT license as set out below.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- * Alternatively, the contents of this file may be used under the terms of the
- * GNU General Public License Version 2 ("GPL")in which case the provisions of
- * GPL are applicable instead of those above.
- *
- * If you wish to allow use of your version of this file only under the terms
- * of GPL, and not to allow others to use your version of this file under the
- * terms of the MIT license, indicate your decision by deleting the provisions
- * above and replace them with the notice and other provisions required by GPL
- * as set out in the file called "GPLHEADER" included in this distribution. If
- * you do not delete the provisions above, a recipient may use your version of
- * this file under the terms of either the MIT license or GPL.
- *
- * This License is also included in this distribution in the file called
- * "MIT_COPYING".
- *
- *****************************************************************************/
- #include <linux/moduleparam.h>
- #include <linux/delay.h>
- #include <linux/irq.h>
- #include <linux/preempt.h>
- #include <linux/pm_runtime.h>
- #include <linux/random.h>
- #include <linux/slab.h>
- #include <uapi/vha.h>
- #include <uapi/vha_errors.h>
- #include "vha_common.h"
- #include "vha_plat.h"
- #include "vha_regs.h"
- #include "vha_mt19937.h"
- static uint32_t shared_onchipmem_size;
- module_param(shared_onchipmem_size, uint, 0444);
- MODULE_PARM_DESC(shared_onchipmem_size,
- "Size of shared on-chip memory in bytes");
- /* WM debug statistics types */
- #define VHA_WM_DBG_MODE_PERF 0
- #define VHA_WM_DBG_MODE_BAND 1
- #define WM_DBG_MODE_ON(type) \
- (wm_dbg_ctrl[VHA_WM_DBG_MODE_##type])
- static uint32_t wm_dbg_ctrl[2] = { 0, 0 };
- module_param_array(wm_dbg_ctrl, uint, NULL, 0444);
- MODULE_PARM_DESC(wm_dbg_ctrl,
- "WM DEBUG CONTROL: switch for PERF and BAND: 0=disable 1=enable");
- static uint32_t slc_hash_mode;
- module_param(slc_hash_mode, uint, 0444);
- MODULE_PARM_DESC(slc_hash_mode,
- "SLC_CTRL_HASH_MODE: Address decoding for SLC. 0-none; 1-pvr_v3; 2-linear; 3-in_page. See TRM");
- #ifdef VHA_SCF
- static uint32_t sys_ram_correction_threshold = 0;
- module_param(sys_ram_correction_threshold, uint, 0444);
- MODULE_PARM_DESC(sys_ram_correction_threshold,
- "Threshold for system level ram correction");
- static uint32_t core_host_ram_correction_threshold = 0;
- module_param(core_host_ram_correction_threshold, uint, 0444);
- MODULE_PARM_DESC(core_host_ram_correction_threshold,
- "Threshold for host core level ram correction");
- static uint32_t core_wm_ram_correction_threshold = 0;
- module_param(core_wm_ram_correction_threshold, uint, 0444);
- MODULE_PARM_DESC(core_wm_ram_correction_threshold,
- "Threshold for wm core level ram correction");
- #endif
- #define CONF_WRITES_WAIT_TIMEOUT_MS 20
- #define SCHED_SEQ_CORES_MASK 0xff
- #define SCHED_SEQ_CORES_SHIFT 0
- #define SCHED_SEQ_WM_ID_MASK 0x7
- #define SCHED_SEQ_WM_ID_SHIFT 8
- #define SCHED_SEQ_GET_CORES(idx) \
- ((vha->scheduling_sequence[idx] >> SCHED_SEQ_CORES_SHIFT) & SCHED_SEQ_CORES_MASK)
- #define SCHED_SEQ_GET_WM(idx) \
- ((vha->scheduling_sequence[idx] >> SCHED_SEQ_WM_ID_SHIFT) & SCHED_SEQ_WM_ID_MASK)
- /*
- * scheduling_sequence can be used to force execution on specific WMs/cores.
- * It encodes the WM id (byte1) and the core mask (byte0), for example:
- * scheduling_sequence=0x001 -> WM0/core0
- * scheduling_sequence=0x204 -> WM2/core2
- * scheduling_sequence=0x520 -> WM5/core5
- * scheduling_sequence=0x310 -> WM3/core4
- */
- static int32_t scheduling_sequence_len;
- static uint32_t scheduling_sequence[VHA_MC_SCHED_SEQ_LEN_MAX] = { 0 };
- module_param_array(scheduling_sequence, uint, &scheduling_sequence_len, 0444);
- MODULE_PARM_DESC(scheduling_sequence, "multicore scheduling sequence");
- #define MAX_STALLING_DATA_ENTRIES 2
- static int32_t stalling_data_len;
- static uint32_t stalling[MAX_STALLING_DATA_ENTRIES] = { 0 };
- module_param_array(stalling, uint, &stalling_data_len, 0444);
- MODULE_PARM_DESC(stalling, "stalling data");
- static bool test_direct_events;
- module_param(test_direct_events, bool, 0444);
- MODULE_PARM_DESC(test_direct_events,
- "When set CORE&INTERCONNECT events are directly sent to host, to WM, otherwise");
- static int32_t pri_windows_list_len;
- static uint32_t pri_windows_list[VHA_MAX_PRIORITIES] = { 0 };
- module_param_array(pri_windows_list, uint, &pri_windows_list_len, 0444);
- MODULE_PARM_DESC(pri_windows_list,
- "priority window size list starting from lowest; all 0s mean no starvation avoidance");
- /* Priority scheduler local data. */
- struct vha_sched_local_data {
- void *rand_gen_handle;
- };
- /* Priority window array. */
- /* NOTE: Setting all to 0 implies strict priority scheduling (no starvation avoidance). */
- static uint32_t pri_windows[VHA_MAX_PRIORITIES] = {0};
- /* Parity related defines. */
- #ifdef VHA_SCF
- #define VHA_PARITY_READ_COUNT_MAX 4
- #endif
- struct vha_errcode {
- uint8_t e;
- const char* s;
- enum vha_reset_type reset_type;
- uint64_t rsp_err;
- };
- /* SYS event errors. */
- #define ERR_SYS_EVENT_DESC(b) VHA_SYS_EVENT_TYPE(b), __stringify(b)
- static const struct vha_biterr sys_err_bits[] = {
- {-EIO, ERR_SYS_EVENT_DESC(AXI_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_AXI_ERROR)},
- {-EFAULT, ERR_SYS_EVENT_DESC(MMU_PAGE_FAULT ), VHA_RESET_TYPE_MMU, VHA_RSP_ERROR(HW_SYS_MMU_PAGE_FAULT)},
- {-ETIMEDOUT, ERR_SYS_EVENT_DESC(SYS_MEM_WDT ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_SYS_MEM_WDT)},
- #ifdef VHA_SCF
- /*
- * Unfortunately, hw guys did not specify the way to identify the failed
- * WM. Waiting for them to fix this. */
- {-EIO, ERR_SYS_EVENT_DESC(AXI_MEMORY_PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_AXI_MEMORY_PARITY_ERROR)},
- {-EIO, ERR_SYS_EVENT_DESC(MMU_PARITY_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_MMU_PARITY_ERROR)}, /*VHA_RESET_TYPE_MMU},*/
- {-EIO, ERR_SYS_EVENT_DESC(RAM_CORRECTION ), VHA_RESET_TYPE_NONE, VHA_RSP_ERROR(HW_SYS_RAM_CORRECTION)},
- {-EIO, ERR_SYS_EVENT_DESC(RAM_DETECTION ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_RAM_DETECTION)},
- {-EIO, ERR_SYS_EVENT_DESC(LSYNC_INV_REQ ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_LSYNC_INV_REQ)},
- {-EIO, ERR_SYS_EVENT_DESC(LOGIC_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_LOGIC_ERROR)},
- {-EIO, VHA_REG_PARITY_ERROR_EN, __stringify(PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(SW_SYS_EVNT_PARITY_ERROR)},
- #endif
- {0}
- };
- /* WM event errors. */
- #define ERR_WM_EVENT_DESC(b) VHA_WM_EVENT_TYPE(b), __stringify(b)
- static const struct vha_biterr wm_err_bits[] = {
- {-ETIMEDOUT, ERR_WM_EVENT_DESC(WM_WL_WDT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_EVNT_WM_WL_WDT)},
- {-ETIMEDOUT, ERR_WM_EVENT_DESC(WM_WL_IDLE_WDT), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_EVNT_WM_WL_IDLE_WDT)},
- {-ETIMEDOUT, ERR_WM_EVENT_DESC(WM_SOCIF_WDT ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_EVNT_WM_SOCIF_WDT)},
- {-EFAULT, ERR_WM_EVENT_DESC(LOGIC_FAULT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_EVNT_LOGIC_FAULT)},
- #ifdef VHA_SCF
- {-EIO, VHA_REG_PARITY_ERROR_EN, __stringify(PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(SW_EVNT_WM_PARITY_ERROR)},
- #endif
- {0}
- };
- /* WM response FIFO status error codes. */
- #define ERR_WM_RSP_STATUS_DESC(v) VHA_WM_RESPONSE_ERROR_CODE(v), __stringify(v)
- static const struct vha_errcode wm_rsp_err_codes[] = {
- {ERR_WM_RSP_STATUS_DESC(CORE_IRQ_BEFORE_KICK ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_IRQ_BEFORE_KICK)},
- {ERR_WM_RSP_STATUS_DESC(INDIRECT_MASK_SET_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_INDIRECT_MASK_SET_ERROR)},
- {ERR_WM_RSP_STATUS_DESC(KICK_CORE_ACCESS_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_KICK_CORE_ACCESS_ERROR)},
- {ERR_WM_RSP_STATUS_DESC(CNN_CONTROL_START_HIGH ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CNN_CONTROL_START_HIGH)},
- {ERR_WM_RSP_STATUS_DESC(CNN_STATUS_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CNN_STATUS_ERROR)},
- {ERR_WM_RSP_STATUS_DESC(INT_CORE_ACCESS_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_INT_CORE_ACCESS_ERROR)},
- {ERR_WM_RSP_STATUS_DESC(CORE_EVENT_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_EVENT_ERROR)},
- {ERR_WM_RSP_STATUS_DESC(CORE_EVENT_NOT_CLEARED ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_EVENT_NOT_CLEARED)},
- {ERR_WM_RSP_STATUS_DESC(CORE_EVENT_IRQ_HIGH ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_EVENT_IRQ_HIGH)},
- {ERR_WM_RSP_STATUS_DESC(INTERCONNECT_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_INTERCONNECT_ERROR)},
- };
- /* CNN core status errors. */
- #define ERR_CORE_STATUS_DESC(b) VHA_CORE_STATUS(b), __stringify(b)
- static const struct vha_biterr core_err_bits[] = {
- {-EIO, ERR_CORE_STATUS_DESC(LOGIC_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_LOGIC_ERROR)},
- {-EIO, ERR_CORE_STATUS_DESC(RAM_CORRECTION ), VHA_RESET_TYPE_NONE, VHA_RSP_ERROR(HW_RAM_CORRECTION)},
- {-EIO, ERR_CORE_STATUS_DESC(RAM_DETECTION ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_RAM_DETECTION)},
- {-EIO, ERR_CORE_STATUS_DESC(CORE_SYNC_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_CORE_SYNC_ERROR)},
- {-ETIMEDOUT, ERR_CORE_STATUS_DESC(CORE_WDT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_WDT)},
- {-ETIMEDOUT, ERR_CORE_STATUS_DESC(CORE_MEM_WDT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_MEM_WDT)},
- {-EIO, ERR_CORE_STATUS_DESC(CNN_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_CNN_ERROR)},
- {0}
- };
- /* Interconnect status errors. */
- #define ERR_IC_STATUS_DESC(b) VHA_IC_STATUS(b), __stringify(b)
- static const struct vha_biterr ic_err_bits[] = {
- {-EIO, ERR_IC_STATUS_DESC(LOCKSTEP_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_LOCKSTEP_ERROR)},
- {-EIO, ERR_IC_STATUS_DESC(LOGIC_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_IC_LOGIC_ERROR)},
- {-EIO, ERR_IC_STATUS_DESC(SOCIF_READ_MISMATCH ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SOCIF_READ_MISMATCH)},
- {-EIO, ERR_IC_STATUS_DESC(SOCIF_READ_UNRESPONSIVE), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SOCIF_READ_UNRESPONSIVE)},
- #ifdef VHA_SCF
- {-EIO, VHA_REG_PARITY_ERROR_EN, __stringify(PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(SW_IC_PARITY_ERROR)},
- #endif
- {0}
- };
- bool vha_dev_dbg_params_check(struct vha_dev *vha)
- {
- if (vha->scheduling_sequence_len > 0) {
- uint32_t i;
- for (i = 0; i < vha->scheduling_sequence_len; i++) {
- uint8_t wm_id = SCHED_SEQ_GET_WM(i);
- uint8_t core_mask = SCHED_SEQ_GET_CORES(i);
- if ((wm_id >= vha->hw_props.num_cnn_core_devs) ||
- (~vha->full_core_mask & core_mask)) {
- dev_info(vha->dev,
- "%u/0x%02x -> %u/0x%02x (0x%02x)",
- wm_id, core_mask,
- vha->hw_props.num_cnn_core_devs, vha->full_core_mask,
- (~vha->full_core_mask & core_mask));
- dev_err(vha->dev,
- "'scheduling_sequence' contains cores that do not exist on this h/w.\n");
- return false;
- }
- }
- }
- return true;
- }
- bool vha_dev_dbg_params_init(struct vha_dev *vha)
- {
- vha->scheduling_sequence_len = scheduling_sequence_len;
- memcpy(vha->scheduling_sequence,
- scheduling_sequence, sizeof(scheduling_sequence));
- vha->scheduling_counter = 0;
- vha->stalling_sysbus_host_stall_ratio = stalling[0];
- vha->stalling_membus_sys_stall_ratio = stalling[1];
- return vha_dev_dbg_params_check(vha);
- }
- int vha_dev_scheduler_init(struct vha_dev *vha)
- {
- int ret;
- uint32_t seed, i;
- bool use_default_pri_windows = true;
- vha->hw_sched_status.num_cores_free = vha->hw_props.num_cnn_core_devs;
- vha->hw_sched_status.num_wms_free = vha->hw_props.num_cnn_core_devs;
- vha->hw_sched_status.free_core_mask =
- VHA_GET_CORE_MASK(vha->hw_props.num_cnn_core_devs);
- vha->hw_sched_status.free_wm_mask =
- VHA_GET_WM_MASK(vha->hw_props.num_cnn_core_devs);
- vha->full_core_mask = vha->hw_sched_status.free_core_mask;
- vha->wm_core_assignment = (uint64_t)(
- VHA_CR_CORE_ASSIGNMENT_CORE_7_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_6_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_5_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_4_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_3_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_2_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_1_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_0_WM_MAPPING_UNALLOCATED);
- vha->active_core_mask = 0;
- vha->apm_core_mask = 0;
- /* Allocate priority scheduler data. */
- vha->hw_sched_status.sched_data = kzalloc(sizeof(struct vha_sched_local_data), GFP_KERNEL);
- if (vha->hw_sched_status.sched_data == NULL) {
- dev_err(vha->dev, "%s: failed allocating scheduler data\n", __func__);
- return -ENOMEM;
- }
- /* Initialise random number generator for priority scheduling. */
- get_random_bytes(&seed, sizeof(seed));
- ret = vha_mt19937_init(seed, &vha->hw_sched_status.sched_data->rand_gen_handle);
- if (ret != 0) {
- dev_err(vha->dev, "%s: failed initialising random generator\n", __func__);
- kfree(vha->hw_sched_status.sched_data);
- return ret;
- }
- /* Attempt to set priority windows passed on to kernel module. */
- if (pri_windows_list_len == VHA_MAX_PRIORITIES) {
- uint32_t num_zeros = 0;
- for (i = 0; i < VHA_MAX_PRIORITIES; i++)
- if (pri_windows_list[i] == 0)
- num_zeros++;
- if (num_zeros < VHA_MAX_PRIORITIES)
- {
- dev_warn(vha->dev, "%s: some priority windows are set to 0; "
- "using default settings\n", __func__);
- } else {
- memcpy(pri_windows, pri_windows_list, sizeof(pri_windows));
- use_default_pri_windows = false;
- }
- } else if (pri_windows_list_len > 0) {
- dev_warn(vha->dev, "%s: too few priority windows provided (needed %u); "
- "using default settings\n", __func__, VHA_MAX_PRIORITIES);
- }
- /* Calculate priority windows. */
- if (use_default_pri_windows) {
- #define BASE_PRI_WINDOW_WIDTH 30
- for (i = 0; i < VHA_MAX_PRIORITIES; i++)
- pri_windows[i] = BASE_PRI_WINDOW_WIDTH + (i * 2 * BASE_PRI_WINDOW_WIDTH);
- #undef BASE_PRI_WINDOW_WIDTH
- }
- return 0;
- }
- int vha_dev_scheduler_deinit(struct vha_dev *vha)
- {
- int ret;
- if (vha->hw_sched_status.sched_data == NULL) {
- dev_warn(vha->dev, "%s: scheduler not initialised\n", __func__);
- return 0;
- }
- ret = vha_mt19937_deinit(vha->hw_sched_status.sched_data->rand_gen_handle);
- if (ret != 0) {
- dev_err(vha->dev, "%s: failed deinitialising random generator\n", __func__);
- }
- kfree(vha->hw_sched_status.sched_data);
- return ret;
- }
- void vha_dev_mh_setup(struct vha_dev *vha, int ctx_id, struct vha_mh_config_regs *regs)
- {
- uint64_t val64 = 0;
- regs->cnn_preload_control |= VHA_CR_SETBITS(OS0_CNN_PRELOAD_CONTROL,
- CBUF_N_REQS, VHA_CR_CNN_PRELOAD_CTRL_N_64);
- /* Setup preload for MMM */
- regs->cnn_preload_control |= VHA_CR_SETBITS(OS0_CNN_PRELOAD_CONTROL,
- MMM_RD_N_REQS, VHA_CR_CNN_PRELOAD_CTRL_N_256);
- regs->cnn_preload_control |= VHA_CR_SETBITS(OS0_CNN_PRELOAD_CONTROL,
- MMM_WR_N_REQS, VHA_CR_CNN_PRELOAD_CTRL_N_256);
- img_pdump_printf("-- MH setup:%d\n", ctx_id);
- IOWRITE64_CR_PDUMP(regs->cnn_preload_control, OS0_CNN_PRELOAD_CONTROL);
- regs->req_ctxt_override = VHA_SET_FIELD_SIMPLE_VAL(REQ_CTXT_OVERRIDE, OVERRIDE_OS0, EN);
- IOWRITE64_CR_PDUMP(regs->req_ctxt_override, REQ_CTXT_OVERRIDE);
- if (slc_hash_mode) {
- regs->slc_control = VHA_CR_SETBITS(SLC_CTRL, HASH_MODE,
- slc_hash_mode);
- IOWRITE64_CR_PDUMP(val64, SLC_CTRL);
- }
- }
- static int set_power_event(struct vha_dev *vha, uint64_t event)
- {
- int ret=0;
- uint64_t val64;
- /* Clear any pending power events */
- IOWRITE64_CR_PDUMP(0, POWER_EVENT);
- /* Confirm no power events are pending */
- ret = IOPOLL64_CR_PDUMP(0, 100, 1000,
- ((uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_COMPLETE) |
- (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_ABORT)),
- SYS_EVENT_STATUS);
- if(ret)
- return ret;
- /* Trigger power event */
- IOWRITE64_CR_PDUMP(event, POWER_EVENT);
- /* Wait for power complete */
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, POWER_COMPLETE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_COMPLETE),
- SYS_EVENT_STATUS);
- if(ret)
- return ret;
- /* Switch off power event */
- IOWRITE64_CR_PDUMP(0, POWER_EVENT);
- /* Clear power complete event status */
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, POWER_COMPLETE, EN);
- IOWRITE64_CR_PDUMP(val64, SYS_EVENT_CLEAR);
- /* Confirm power complete is cleared */
- ret = IOPOLL64_CR_PDUMP(0, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_COMPLETE),
- SYS_EVENT_STATUS);
- return ret;
- }
- #ifdef VHA_SCF
- static void ecc_correction_setup(struct vha_dev *vha)
- {
- uint64_t val64;
- val64 = VHA_CR_SETBITS(SYS_EVENT_THRESHOLD, RAM_CORRECTION,
- sys_ram_correction_threshold);
- IOWRITE64_CR_PDUMP(val64, SYS_EVENT_THRESHOLD);
- val64 = VHA_CR_SETBITS(CORE_EVENT_WM_THRESHOLD, RAM_CORRECTION,
- core_wm_ram_correction_threshold);
- IOWRITE64_CR_PDUMP(val64, CORE_EVENT_WM_THRESHOLD);
- val64 = VHA_CR_SETBITS(CORE_EVENT_HOST_THRESHOLD, RAM_CORRECTION,
- core_host_ram_correction_threshold);
- IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_THRESHOLD);
- }
- #endif
- static int vha_dev_prepare_cores(struct vha_dev *vha, uint8_t core_mask)
- {
- /* Enabling selected cores on the platform
- * Note: don't touch TLC, is an always ON domain */
- uint64_t val64 = VHA_CR_SETBITS(POWER_EVENT, DOMAIN,
- (core_mask << 1)) |
- VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, TYPE, POWER_UP) |
- VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, REQ, EN);
- img_pdump_printf("-- Trigger POWER UP domain event\n");
- return set_power_event(vha, val64);
- }
- static int vha_dev_flush_cores(struct vha_dev *vha, uint8_t core_mask)
- {
- uint64_t val64;
- img_pdump_printf("-- Deselect any cores\n");
- IOWRITE64_CR_PDUMP(0, CORE_CTRL_INDIRECT);
- /* Disabling selected cores on the platform
- * Note: don't touch TLC, is an always ON domain */
- val64 = VHA_CR_SETBITS(POWER_EVENT, DOMAIN, (core_mask << 1)) |
- VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, TYPE, POWER_DOWN) |
- VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, REQ, EN);
- img_pdump_printf("-- Trigger POWER DOWN domain event\n");
- return set_power_event(vha, val64);
- }
- void vha_dev_setup(struct vha_dev *vha)
- {
- uint64_t val64;
- img_pdump_printf("-- MMU set virtual address range0:%#llx-%#llx\n",
- IMG_MEM_VA_HEAP1_BASE, IMG_MEM_VA_HEAP1_SIZE);
- val64 = (uint64_t)vha->mmu_page_size <<
- VHA_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT;
- val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_ONE,
- BASE_ADDR, IMG_MEM_VA_HEAP1_BASE);
- val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_ONE,
- END_ADDR, (IMG_MEM_VA_HEAP1_BASE + IMG_MEM_VA_HEAP1_SIZE));
- IOWRITE64_PDUMP(val64, VHA_CR_MMU_PAGE_SIZE_RANGE_ONE);
- img_pdump_printf("-- MMU set virtual address range1:%#llx-%#llx\n",
- IMG_MEM_VA_HEAP2_BASE, IMG_MEM_VA_HEAP2_SIZE);
- val64 = (uint64_t)vha->mmu_page_size <<
- VHA_CR_MMU_PAGE_SIZE_RANGE_TWO_PAGE_SIZE_SHIFT ;
- val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_TWO,
- BASE_ADDR, IMG_MEM_VA_HEAP2_BASE);
- val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_TWO,
- END_ADDR, (IMG_MEM_VA_HEAP2_BASE + IMG_MEM_VA_HEAP2_SIZE));
- IOWRITE64_PDUMP(val64, VHA_CR_MMU_PAGE_SIZE_RANGE_TWO);
- #ifdef VHA_SCF
- ecc_correction_setup(vha);
- #endif
- }
- void vha_dev_wait(struct vha_dev *vha)
- {
- /* Nothing to do */
- }
- static void vha_dev_disable_events(struct vha_dev *vha, uint8_t core_mask, bool sys_release)
- {
- uint8_t id;
- if (sys_release) {
- img_pdump_printf("-- Disable SYS events\n");
- IOWRITE64_CR_PDUMP(0, SYS_EVENT_ENABLE);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- img_pdump_printf("-- Select WM%u\n", id);
- VHA_LOCK_WM();
- VHA_SELECT_WM(id);
- img_pdump_printf("-- Clear WM%u events\n", id);
- IOWRITE64_CR_PDUMP(VHA_WM_EVENTS_DEFAULT, WM_EVENT_CLEAR);
- img_pdump_printf("-- Disable WM%u events\n", id);
- IOWRITE64_CR_PDUMP(0, WM_EVENT_ENABLE);
- VHA_UNLOCK_WM();
- }
- }
- img_pdump_printf("-- Select cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
- if (test_direct_events) {
- img_pdump_printf("-- Disable CORE events to HOST\n");
- IOWRITE64_CR_PDUMP(0, CORE_EVENT_HOST_ENABLE);
- img_pdump_printf("-- Disable INTERCONNECT events to HOST\n");
- IOWRITE64_CR_PDUMP(0, INTERCONNECT_EVENT_HOST_ENABLE);
- } else {
- img_pdump_printf("-- Disable CORE events to WM\n");
- IOWRITE64_CR_PDUMP(0, CORE_EVENT_WM_ENABLE);
- img_pdump_printf("-- Disable INTERCONNECT events to WM\n");
- IOWRITE64_CR_PDUMP(0, INTERCONNECT_EVENT_WM_ENABLE);
- }
- }
- static void vha_dev_ready(struct vha_dev *vha, uint8_t core_mask, bool sys_setup)
- {
- uint8_t id;
- if (sys_setup) {
- img_pdump_printf("-- Enable SYS events\n");
- IOWRITE64_CR_PDUMP(VHA_SYS_EVENTS_DEFAULT, SYS_EVENT_ENABLE);
- img_pdump_printf("-- Clear SYS events\n");
- IOWRITE64_CR_PDUMP(VHA_SYS_EVENTS_DEFAULT, SYS_EVENT_CLEAR);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- img_pdump_printf("-- Select WM%u\n", id);
- VHA_LOCK_WM();
- VHA_SELECT_WM(id);
- img_pdump_printf("-- Enable WM%u events\n", id);
- IOWRITE64_CR_PDUMP(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
- img_pdump_printf("-- Clear WM%u events\n", id);
- IOWRITE64_CR_PDUMP(VHA_WM_EVENTS_DEFAULT, WM_EVENT_CLEAR);
- VHA_UNLOCK_WM();
- }
- }
- img_pdump_printf("-- Select cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
- if (test_direct_events) {
- img_pdump_printf("-- Enable CORE events to HOST\n");
- IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_HOST_ENABLE);
- img_pdump_printf("-- Clear CORE events on HOST\n");
- IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_HOST_CLEAR);
- img_pdump_printf("-- Enable INTERCONNECT events to HOST\n");
- IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_HOST_ENABLE);
- img_pdump_printf("-- Clear INTERCONNECT events on HOST\n");
- IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_HOST_CLEAR);
- } else {
- img_pdump_printf("-- Enable CORE events to WM\n");
- IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_ENABLE);
- img_pdump_printf("-- Clear CORE events on WM\n");
- IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_CLEAR);
- img_pdump_printf("-- Enable INTERCONNECT events to WM\n");
- IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_ENABLE);
- img_pdump_printf("-- Clear INTERCONNECT events on WM\n");
- IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_CLEAR);
- }
- }
- /* Global reset */
- static int vha_dev_reset(struct vha_dev *vha, uint8_t core_mask, bool sys_reset)
- {
- uint64_t val64 = 0;
- uint8_t mask = 0;
- uint8_t id;
- int ret = 0;
- WARN_ON(!mutex_is_locked(&vha->lock));
- dev_dbg(vha->dev, "%s core mask:%#x\n", __func__, core_mask);
- img_pdump_printf("-- Top level RESET sequence BEGIN\n");
- /* Perform reset procedure */
- if (sys_reset) {
- /* First reset all WMs with cores assigned. */
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- struct vha_hw_sched_info sched_info = {0};
- sched_info.core_mask = vha_wm_get_cores(vha, id);
- if (sched_info.core_mask) {
- sched_info.wm_id = id;
- vha_wm_reset(vha, &sched_info);
- core_mask &= ~sched_info.core_mask;
- }
- }
- }
- /* Core reset procedure. */
- img_pdump_printf("-- Resetting cores\n");
- /* Proceed core by core, unassigned cores only */
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (!(core_mask & VHA_CORE_ID_TO_MASK(id)))
- continue;
- /* Reset Assertion */
- /* 1. Select current core. */
- img_pdump_printf("-- Select core%u\n", id);
- mask = VHA_CORE_ID_TO_MASK(id);
- IOWRITE64_CR_PDUMP(mask, CORE_CTRL_INDIRECT);
- /* 3. Disable page fault interrupts for core while resetting. */
- img_pdump_printf("-- Disable page fault interrupts for core%u\n", id);
- val64 = IOREAD64_CR_REGIO(SYS_EVENT_ENABLE);
- val64 &= ~(VHA_CR_SETBITS(SYS_EVENT_ENABLE, MMU_PAGE_FAULT, mask));
- IOWRITE64_CR_PDUMP(val64, SYS_EVENT_ENABLE);
- /* 4. Force global clocks to ON on current core (others set to AUT0). */
- img_pdump_printf("-- Force global clocks ON for core%u (others set to AUTO)\n", id);
- val64 = VHA_SYS_CLOCK_MODE(INTERCONNECT, ON) |
- VHA_SYS_CLOCK_MODE_MULTI(CORE, ON, mask) |
- VHA_SYS_CLOCK_MODE_MULTI(CORE, AUTO, (uint8_t)~mask) |
- VHA_SYS_CLOCK_MODE_MULTI(NOC, AUTO, ~0) |
- VHA_SYS_CLOCK_MODE_MULTI(WM, AUTO, ~0) |
- VHA_SYS_CLOCK_MODE(AXI, AUTO) |
- VHA_SYS_CLOCK_MODE(SLC, AUTO) |
- VHA_SYS_CLOCK_MODE(LSYNC, AUTO) |
- VHA_SYS_CLOCK_MODE(SOCM, AUTO) |
- VHA_SYS_CLOCK_MODE(REGBANK, AUTO);
- IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
- /* 5. Set all core level clocks to AUTO. */
- img_pdump_printf("-- Set all core%u level clocks to AUTO\n", id);
- val64 = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
- IOWRITE64_CR_PDUMP(val64, CLK_CTRL0);
- /* 6. Move core into soft reset. */
- img_pdump_printf("-- Perform soft reset on core%u\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_SOFT_RESET, CORE_RESET, EN);
- IOWRITE64_CR_PDUMP(val64, CORE_SOFT_RESET);
- /* Dummy read to avoid race conditions in the hw. */
- val64 = IOREAD64_CR_PDUMP(CORE_SOFT_RESET);
- /* Clear reset. */
- IOWRITE64_CR_PDUMP(0, CORE_SOFT_RESET);
- /* 7. Wait until core memory bus reset has completed. */
- img_pdump_printf("-- Wait until core%u memory bus reset has completed\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, MEMBUS_RESET_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(CORE_EVENT_HOST_STATUS, MEMBUS_RESET_DONE),
- CORE_EVENT_HOST_STATUS);
- if(ret)
- return ret;
- /* 8. Clear core memory bus reset interrupt. */
- img_pdump_printf("-- Clear core%u memory bus reset interrupt\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_CLEAR, MEMBUS_RESET_DONE, EN);
- IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_CLEAR);
- /* 9. Clear the core indirect register. */
- img_pdump_printf("-- Deselect core%u\n", id);
- IOWRITE64_CR_PDUMP(0, CORE_CTRL_INDIRECT);
- /* 10. Ensure no resets are pending. */
- img_pdump_printf("-- Ensure no resets are pending\n");
- IOWRITE64_CR_PDUMP(0, SYS_RESET_CTRL);
- /* 11. Move current core into full reset state. */
- img_pdump_printf("-- Move core%u into full reset state\n", id);
- val64 = VHA_CR_SETBITS(SYS_RESET_CTRL, CORE, mask);
- IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
- /* 12. Dummy read to avoid race conditions in the hw. */
- val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
- /* Reset Deassertion */
- /* 1. Move current core out of reset state. */
- img_pdump_printf("-- Move core%u out of reset state\n", id);
- val64 &= ~(VHA_CR_SETBITS(SYS_RESET_CTRL, CORE, mask));
- IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
- /* Dummy read to avoid race conditions in the hw. */
- val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
- /* 2. Select current core again. */
- img_pdump_printf("-- Select core%u again\n", id);
- IOWRITE64_CR_PDUMP(mask, CORE_CTRL_INDIRECT);
- /* 5. Force core clocks to ON for everything. */
- img_pdump_printf("-- Force core clocks ON for everything\n");
- val64 = VHA_MAIN_CLOCKS_DEFAULT(ON);
- IOWRITE64_CR_PDUMP(val64, CLK_CTRL0);
- /* 6. Perform core level RAM initialisation. */
- img_pdump_printf("-- Perform core%u level RAM initialisation\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(FUSA_CONTROL, ECC_INIT_KICK, EN);
- IOWRITE64_CR_PDUMP(val64, FUSA_CONTROL);
- /* 7. Perform LOCM scrubbing. */
- img_pdump_printf("-- Perform core%u LOCM scrubbing\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(LOCM_SCRUB_CTRL, KICK, EN);
- IOWRITE64_CR_PDUMP(val64, LOCM_SCRUB_CTRL);
- /* 8. Wait until the RAM initialisation sequence has completed. */
- img_pdump_printf("-- Wait until the RAM initialisation sequence has completed\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, RAM_INIT_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(CORE_EVENT_HOST_STATUS, RAM_INIT_DONE),
- CORE_EVENT_HOST_STATUS);
- if(ret)
- return ret;
- /* 9. Clear core RAM reset interrupt. */
- img_pdump_printf("-- Clear core%u RAM reset interrupt\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_CLEAR, RAM_INIT_DONE, EN);
- IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_CLEAR);
- /* Confirm that 'RAM_INIT_DONE' field is cleared. */
- img_pdump_printf("-- Confirm that core%u RAM reset interrupt is cleared\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, RAM_INIT_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(0ULL, 10, 100, val64, CORE_EVENT_HOST_STATUS);
- if(ret)
- return ret;
- /* 10. Wait until the LOCM scrubbing sequence has completed. */
- img_pdump_printf("-- Wait until the LOCM scrubbing sequence has completed.\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, LOCM_SCRUB_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(CORE_EVENT_HOST_STATUS, LOCM_SCRUB_DONE),
- CORE_EVENT_HOST_STATUS);
- if(ret)
- return ret;
- /* 11. Deassert core LOCM scrubbing. */
- img_pdump_printf("-- Deassert core%u LOCM scrubbing\n", id);
- IOWRITE64_CR_PDUMP(0, LOCM_SCRUB_CTRL);
- /* 12. Clear core LOCM scrub interrupt. */
- img_pdump_printf("-- Clear core%u LOCM scrub interrupt\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_CLEAR, LOCM_SCRUB_DONE, EN);
- IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_CLEAR);
- /* Confirm that 'LOCM_SCRUB_DONE' field is cleared. */
- img_pdump_printf("-- Confirm that core%u LOCM scrub interrupt is cleared\n", id);
- val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, LOCM_SCRUB_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(0ULL, 10, 100, val64, CORE_EVENT_HOST_STATUS);
- if(ret)
- return ret;
- /* 13. Enable the interrupts from core to WM. */
- img_pdump_printf("-- Enable CORE events to WM\n");
- IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_ENABLE);
- /* 14. Clear all status from CORE_EVENT_WM (clears the RAM_INIT_DONE). */
- img_pdump_printf("-- Clear CORE events on WM\n");
- IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT |
- VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_WM_CLEAR, RAM_INIT_DONE, EN) |
- VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_WM_CLEAR, LOCM_SCRUB_DONE, EN) |
- VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_WM_CLEAR, MEMBUS_RESET_DONE, EN),
- CORE_EVENT_WM_CLEAR);
- /* 15. Enable the interrupts from interconnect to WM */
- img_pdump_printf("-- Enable INTERCONNECT events to WM\n");
- IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_ENABLE);
- /* 16. Disable all interrupts from the CORE to the HOST */
- img_pdump_printf("-- Disable CORE events on host\n");
- IOWRITE64_CR_PDUMP(0, CORE_EVENT_HOST_ENABLE);
- /* 17. Set all core level clocks back to AUTO. */
- img_pdump_printf("-- Set all core%u level clocks back to AUTO\n", id);
- val64 = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
- IOWRITE64_CR_PDUMP(val64, CLK_CTRL0);
- /* 18. Set core global clock back to AUTO. */
- img_pdump_printf("-- Set core%u global clock back to AUTO (others set to AUTO)\n", id);
- val64 = VHA_SYS_CLOCKS_DEFAULT(AUTO);
- IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
- /* Setup stalling if requested. */
- if (vha->stalling_membus_sys_stall_ratio != 0)
- IOWRITE64_CR_REGIO(vha->stalling_membus_sys_stall_ratio,
- NN_SYS2_MEMBUS_SYS_STALL_RATIO);
- }
- if (!sys_reset)
- return 0;
- dev_dbg(vha->dev, "%s handling system level reset\n", __func__);
- /* Move the rest of modules into reset state. */
- img_pdump_printf("-- Move other modules into reset state\n");
- val64 = VHA_SET_FIELD_SIMPLE_FULL(SYS_RESET_CTRL, WM) |
- VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, INTERCONNECT, EN) |
- VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, SLC, EN) |
- VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, MH, EN);
- IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
- /* Dummy read to avoid race conditions in the hw */
- val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
- /* Move the rest of modules out of reset state. */
- img_pdump_printf("-- Move other modules out of reset state\n");
- IOWRITE64_CR_PDUMP(0ULL, SYS_RESET_CTRL);
- /* Dummy read to avoid race conditions in the hw */
- val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
- /* Wait until core memory bus reset has completed. */
- img_pdump_printf("-- Wait until sys memory bus reset has completed\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, MEMBUS_RESET_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, MEMBUS_RESET_DONE),
- SYS_EVENT_STATUS);
- if(ret)
- return ret;
- /* Clear memory bus reset status. */
- img_pdump_printf("-- Clear memory bus reset status\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, MEMBUS_RESET_DONE, EN);
- IOWRITE64_CR_PDUMP(val64, SYS_EVENT_CLEAR);
- /* Force all system level clocks on. */
- img_pdump_printf("-- Force all system level clocks ON (except core)\n");
- val64 = IOREAD64_CR_REGIO(SYS_CLK_CTRL0);
- val64 &= VHA_SYS_CLOCKS_CORE_FULL_MASK;
- val64 |= VHA_SYS_CLOCKS_RESET(ON);
- IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
- /* Initiate system RAM initialisation. */
- img_pdump_printf("-- Initiate system RAM initialisation\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_RAM_INIT, KICK, EN);
- IOWRITE64_CR_PDUMP(val64, SYS_RAM_INIT);
- /* Initiate system SOCM scrubbing. */
- img_pdump_printf("-- Initiate system SOCM scrubbing\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SOCM_SCRUB_CTRL, KICK, EN);
- IOWRITE64_CR_PDUMP(val64, SOCM_SCRUB_CTRL);
- /* Wait until the RAM initialisation sequence has completed. */
- img_pdump_printf("-- Wait until the RAM initialisation sequence has completed\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, RAM_INIT_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, RAM_INIT_DONE),
- SYS_EVENT_STATUS);
- if(ret)
- return ret;
- /* Wait until the SOCM scrubbing sequence has completed. */
- img_pdump_printf("-- Wait until the SOCM scrubbing sequence has completed\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, SOCM_SCRUB_DONE, EN);
- ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
- (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, SOCM_SCRUB_DONE),
- SYS_EVENT_STATUS);
- if(ret)
- return ret;
- /* Deassert system SOCM scrubbing */
- img_pdump_printf("-- Deassert system SOCM scrubbing\n");
- IOWRITE64_CR_PDUMP(0, SOCM_SCRUB_CTRL);
- img_pdump_printf("-- Clear sys events\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, RAM_INIT_DONE, EN) |
- VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, SOCM_SCRUB_DONE, EN);
- IOWRITE64_CR_PDUMP(val64, SYS_EVENT_CLEAR);
- /* Set all clocks back to AUTO. */
- img_pdump_printf("-- Set all sys clocks back to AUTO\n");
- val64 = VHA_SYS_CLOCKS_DEFAULT(AUTO);
- IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
- /* Reset the system level register banks. */
- img_pdump_printf("-- Reset the system level register banks\n");
- val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, REGBANK, EN);
- IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
- /* Dummy read to avoid race conditions in the hw */
- val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
- /* Clear reset */
- IOWRITE64_CR_PDUMP(0, SYS_RESET_CTRL);
- /* Dummy read to avoid race conditions in the hw */
- val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
- img_pdump_printf("-- Top level RESET sequence END\n");
- vha->wm_core_assignment = (uint64_t)(
- VHA_CR_CORE_ASSIGNMENT_CORE_7_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_6_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_5_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_4_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_3_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_2_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_1_WM_MAPPING_UNALLOCATED |
- VHA_CR_CORE_ASSIGNMENT_CORE_0_WM_MAPPING_UNALLOCATED);
- /* Setup stalling if requested. */
- if (vha->stalling_sysbus_host_stall_ratio != 0)
- IOWRITE64_CR_REGIO(vha->stalling_sysbus_host_stall_ratio,
- NN_SYS2_SYSBUS_HOST_STALL_RATIO);
- return ret;
- }
- static void vha_dev_enable_clocks(struct vha_dev *vha, uint8_t core_mask)
- {
- uint64_t sys_clks = 0;
- uint64_t main_clks = 0;
- /* Always AUTO gating when needed */
- sys_clks = VHA_SYS_CLOCKS_DEFAULT(AUTO);
- main_clks = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
- /* Enable sys clocks */
- img_pdump_printf("-- Enable SYS clocks\n");
- IOWRITE64_CR_PDUMP(sys_clks, SYS_CLK_CTRL0);
- /* Dummy SYS clocks status read*/
- sys_clks = IOREAD64_CR_PDUMP(SYS_CLK_STATUS0);
- /* Enable main clocks on all cores */
- img_pdump_printf("-- Enable MAIN clocks on cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
- IOWRITE64_CR_PDUMP(main_clks, CLK_CTRL0);
- }
- static int vha_dev_disable_clocks(struct vha_dev *vha, uint8_t core_mask, bool sys_release)
- {
- uint64_t sys_clks = 0;
- uint8_t id;
- int ret = 0;
- if (sys_release) {
- /* Number of WMs equal to number of cores */
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- VHA_LOCK_WM();
- VHA_SELECT_WM(id);
- /* Check WM is idle, handle parity */
- img_pdump_printf("-- Wait for WM%d IDLE state\n", id);
- ret = IOPOLL64_CR_PDUMP_PARITY(VHA_CR_WM_STATUS_STATE_IDLE, 100, 1000,
- (uint64_t)VHA_CR_WM_STATUS_STATE_MASK,
- WM_STATUS);
- VHA_UNLOCK_WM();
- if(ret) {
- struct vha_hw_sched_info sched_info = {
- .wm_id = id,
- .core_mask = 0
- };
- dev_err(vha->dev, "Performing wm%d reset due to HW error detection.", id);
- vha_wm_reset(vha, &sched_info);
- dev_err(vha->dev, "%s Waiting for WM%d IDLE state failed!",
- __func__, id);
- return ret;
- }
- }
- }
- vha_wm_release_cores(vha, core_mask, true);
- img_pdump_printf("-- Address cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
- /* If auto gating was turned on, wait for clocks GATED state on all cores */
- img_pdump_printf("-- Wait for clocks IDLE state\n");
- ret = IOPOLL64_CR_PDUMP(0, 100, 1000,
- VHA_CR_CLK_STATUS0_MASKFULL,
- CLK_STATUS0);
- if(ret) {
- dev_err(vha->dev, "%s Waiting for clocks IDLE state failed!\n",
- __func__);
- return ret;
- }
- if (sys_release) {
- /* Wait for MMU,CCM,RDI,XBAR IDLE state */
- img_pdump_printf("-- Wait for memory bus interface IDLE state\n");
- ret = IOPOLL64_CR_PDUMP(VHA_CR_SLC_IDLE_MASKFULL, 1000, 1000,
- VHA_CR_SLC_IDLE_MASKFULL,
- SLC_IDLE);
- if(ret) {
- dev_err(vha->dev, "%s Waiting for memory bus interface IDLE state failed\n",
- __func__);
- return ret;
- }
- }
- /* Finally disable core clocks */
- img_pdump_printf("-- Disable MAIN clocks\n");
- IOWRITE64_CR_PDUMP(0, CLK_CTRL0); /* main */
- if (sys_release) {
- /* Finally disable sys clocks */
- img_pdump_printf("-- Disable SYS clocks (except REGBANK)\n");
- sys_clks = VHA_SYS_CLOCK_MODE(REGBANK, AUTO);
- IOWRITE64_CR_PDUMP(sys_clks, SYS_CLK_CTRL0); /* sys */
- }
- return ret;
- }
- void vha_update_utilization(struct vha_dev *vha)
- {
- uint8_t i;
- uint64_t tmp;
- uint64_t core_total_proc_us = 0ULL;
- for (i = 0; i < vha->hw_props.num_cnn_core_devs; i++) {
- /* Calculate core utilization. */
- tmp = vha->stats.core_stats[i].total_proc_us;
- do_div(tmp, vha->stats.uptime_ms);
- vha->stats.core_stats[i].utilization = tmp;
- /* Calculate WM utilization. */
- tmp = vha->stats.wm_stats[i].total_proc_us;
- do_div(tmp, vha->stats.uptime_ms);
- vha->stats.wm_stats[i].utilization = tmp;
- /* Calculate cumulative core processing time. */
- core_total_proc_us += vha->stats.core_stats[i].total_proc_us;
- }
- /* Calculate cluster utilization. */
- tmp = core_total_proc_us;
- do_div(tmp, (vha->stats.uptime_ms * vha->hw_props.num_cnn_core_devs));
- vha->stats.cnn_utilization = tmp;
- }
- #ifdef VHA_EVENT_INJECT
- /*
- * Inject EVENT_STATUS bits, requested by respective debugfs nodes, to
- * the registers defined by the currently handled WM.
- */
- static inline void __inject_event_regs(struct vha_dev* vha, struct vha_mc_irq_status* irq_status)
- {
- int id, wm_id;
- u32 mask, wm_mask;
- uint64_t vha_cr_sys_event = vha->injection.vha_cr_sys_event & VHA_CR_SYS_EVENT_INJECT_MASKFULL;
- uint64_t vha_cr_wm_event = vha->injection.vha_cr_wm_event & VHA_CR_WM_EVENT_INJECT_MASKFULL;
- uint64_t vha_cr_core_event = vha->injection.vha_cr_core_event & VHA_CR_CORE_EVENT_INJECT_MASKFULL;
- uint64_t vha_cr_interconnect_event = vha->injection.vha_cr_interconnect_event & VHA_CR_INTERCONNECT_EVENT_INJECT_MASKFULL;
- if(!__EVENT_INJECT())
- return;
- if(vha_cr_sys_event) {
- IOWRITE64_CR_REGIO(vha_cr_sys_event, SYS_EVENT_INJECT);
- }
- /* handle WM event injection */
- wm_mask = VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM, irq_status->event_source);
- if(!wm_mask)
- return;
- spin_lock_irqsave(&vha->irq_lock, vha->irq_flags);
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- if(~wm_mask & (1 << wm_id))
- continue; /* inject only to currently handled WM's */
- if(vha_cr_wm_event) {
- VHA_SELECT_WM(wm_id);
- IOWRITE64_CR_REGIO(vha_cr_wm_event, WM_EVENT_INJECT);
- }
- /* now handle WM's core and ic injections . IC sources are the same as core sources */
- if(!vha_cr_core_event && !vha_cr_interconnect_event)
- continue;
- /* get cores handled by specific WM, inject errors only to those cores */
- mask = vha_wm_get_cores(vha, wm_id);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if(~mask & (1 << id))
- continue; /* inject only to currently handled CORE's */
- if(vha_cr_core_event) {
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
- MASK, (1 << id)),
- CORE_CTRL_INDIRECT);
- IOWRITE64_CR_REGIO(vha_cr_core_event, CORE_EVENT_INJECT);
- }
- if(vha_cr_interconnect_event) {
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(IC_CORE_INDIRECT,
- MASK, (1 << id)),
- IC_CORE_INDIRECT);
- IOWRITE64_CR_REGIO(vha_cr_interconnect_event, INTERCONNECT_EVENT_INJECT);
- }
- }
- }
- /* read new injected event sources */
- irq_status->event_source |= IOREAD64_CR_REGIO(HOST_EVENT_SOURCE);
- spin_unlock_irqrestore(&vha->irq_lock, vha->irq_flags);
- }
- static inline void __inject_parity_err(struct vha_dev* vha, struct vha_mc_irq_status* irq_status) {
- int id, wm_id;
- u32 mask, wm_mask;
- if(!__EVENT_INJECT())
- return;
- if (VHA_REG_GET_PARITY_ERROR(vha->injection.vha_cr_sys_event)) {
- VHA_REG_SET_PARITY_ERROR(irq_status->sys_events);
- irq_status->event_source |= VHA_CR_HOST_EVENT_SOURCE_SYS_EN;
- }
- wm_mask = VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM, irq_status->event_source);
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- if(~wm_mask & (1 << wm_id))
- continue; /* inject only to currently handled WM's */
- if (VHA_REG_GET_PARITY_ERROR(vha->injection.vha_cr_wm_event)) {
- VHA_REG_SET_PARITY_ERROR(irq_status->wm_events[wm_id]);
- irq_status->event_source |= 1 << (wm_id + VHA_CR_HOST_EVENT_SOURCE_WM_SHIFT);
- }
- mask = vha_wm_get_cores(vha, wm_id);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if(~mask & (1 << id))
- continue; /* inject only to currently handled CORE's */
- if (VHA_REG_GET_PARITY_ERROR(vha->injection.vha_cr_interconnect_event)) {
- VHA_REG_SET_PARITY_ERROR(irq_status->ic_events[id]);
- irq_status->event_source |= 1 << (id + VHA_CR_HOST_EVENT_SOURCE_IC_SHIFT);
- }
- }
- }
- }
- #endif
- /* Top half */
- irqreturn_t vha_handle_irq(struct device *dev)
- {
- struct vha_dev *vha = vha_dev_get_drvdata(dev);
- irqreturn_t ret = IRQ_NONE;
- struct vha_mc_irq_status irq_status = {0};
- uint32_t multi_src_mask = 0;
- uint8_t id;
- struct TIMESPEC hw_proc_end[VHA_NUM_CORES] = {{0}};
- bool hw_proc_end_recorded[VHA_NUM_CORES] = {0};
- #define CHECK_FOR_DEAD_HW(r) \
- if (r == VHA_DEAD_HW || r == ~0) { \
- WARN_ONCE(1, "Hardware is dead!"); \
- if (!in_interrupt()) \
- mutex_unlock(&vha->lock); \
- return IRQ_NONE; \
- }
- /* ML: This thing is a complete mess in regdef file. The field is present
- * in most of these EVENT regs, but its definition varies a lot, so no idea
- * what it is actually meant to mean.
- */
- #define CHECK_FOR_LOGIC_ERROR(s, r) \
- if (r & VHA_##s##_EVENT_TYPE(LOGIC_ERROR)) { \
- WARN_ONCE(1, "Parity error detected!"); \
- if (!in_interrupt()) \
- mutex_unlock(&vha->lock); \
- return IRQ_NONE; \
- }
- if (!vha)
- return IRQ_NONE;
- /* Note: Top half can be called from the platform worker thread */
- if (!in_interrupt())
- mutex_lock(&vha->lock);
- irq_status.event_source = IOREAD64_CR_REGIO(HOST_EVENT_SOURCE);
- /* On fpga platform it is possible to get a spurious interrupt when the hw died.
- * Do not proceed, just throw a warning. */
- CHECK_FOR_DEAD_HW(irq_status.event_source);
- #ifdef VHA_EVENT_INJECT
- __inject_event_regs(vha, &irq_status);
- #endif
- if (VHA_CR_GETBITS(HOST_EVENT_SOURCE, SYS, irq_status.event_source)) {
- /* Read events. */
- irq_status.sys_events = IOREAD64_CR_REGIO(SYS_EVENT_STATUS);
- /* Just in case check for dead hw. */
- CHECK_FOR_DEAD_HW(irq_status.sys_events);
- #ifdef VHA_SCF
- if (vha->hw_props.supported.parity && !vha->parity_disable) {
- uint32_t i;
- for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
- /* Finish if bit parity is ok */
- if (!img_mem_calc_parity(irq_status.sys_events))
- break;
- /* Otherwise re-read the reg. */
- irq_status.sys_events = IOREAD64_CR_REGIO(SYS_EVENT_STATUS);
- }
- /* Raise an error if maximum re-read count is reached. */
- if (i == VHA_PARITY_READ_COUNT_MAX) {
- dev_err(dev, "SYS_EVENT_STATUS register parity error!\n");
- /* Use the real event to indicate the error */
- VHA_REG_SET_PARITY_ERROR(irq_status.sys_events);
- }
- }
- #endif
- /* Check for hw logic error. */
- /* ML: ??? */
- //CHECK_FOR_LOGIC_ERROR(SYS, irq_status.sys_events);
- /* wake thread even if only parity error is set. Erroneous event may occur that only
- * parity is set among other bits
- */
- if (irq_status.sys_events & (VHA_SYS_EVENTS_DEFAULT | VHA_REG_PARITY_ERROR_EN)) {
- /* Clear interrupts (best not to write pdump in ISR). */
- IOWRITE64_CR_REGIO(irq_status.sys_events & VHA_SYS_EVENTS_DEFAULT,
- SYS_EVENT_CLEAR);
- ret = IRQ_WAKE_THREAD;
- }
- }
- /* Read WM event source mask. */
- multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM,
- irq_status.event_source);
- if (multi_src_mask) {
- spin_lock_irqsave(&vha->irq_lock, vha->irq_flags);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (multi_src_mask & (1 << id)) {
- /* Select WM to read events from. */
- VHA_SELECT_WM(id);
- /* Read events. */
- irq_status.wm_events[id] = IOREAD64_CR_REGIO(WM_EVENT_STATUS);
- /* Just in case check for dead hw. */
- CHECK_FOR_DEAD_HW(irq_status.wm_events[id]);
- /* Record hw processing end timestamps */
- GETNSTIMEOFDAY(&hw_proc_end[id]);
- hw_proc_end_recorded[id] = true;
- #ifdef VHA_SCF
- if (vha->hw_props.supported.parity && !vha->parity_disable) {
- uint32_t i;
- for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
- /* Finish if parity is ok */
- if (!img_mem_calc_parity(irq_status.wm_events[id]))
- break;
- /* Otherwise re-read the reg. */
- irq_status.wm_events[id] = IOREAD64_CR_REGIO(WM_EVENT_STATUS);
- }
- /* Raise an error if maximum re-read count is reached. */
- if (i == VHA_PARITY_READ_COUNT_MAX) {
- dev_err(dev, "WM_EVENT_STATUS[%u] register parity error!\n", id);
- /* Use the real event to indicate the error */
- VHA_REG_SET_PARITY_ERROR(irq_status.wm_events[id]);
- }
- }
- #endif
- {
- /* Post check for AXI bus errors */
- uint64_t ace_status = IOREAD64(vha->reg_base, VHA_CR_ACE_STATUS);
- if (ace_status) {
- dev_err(vha->dev, "AXI bus protocol error: %#llx\n",
- ace_status);
- /* Use AXI error event to indicate that */
- irq_status.event_source |= VHA_CR_SETBITS(HOST_EVENT_SOURCE, SYS, 1);
- irq_status.sys_events |= VHA_CR_SETBITS(SYS_EVENT_TYPE, AXI_ERROR, 1);
- }
- }
- /* wake thread even if only parity error is set. Erroneous event may occur that only
- * parity is set among other bits
- */
- if (irq_status.wm_events[id] & (VHA_WM_EVENTS_DEFAULT | VHA_REG_PARITY_ERROR_EN)) {
- /* Events can't be cleared, disable to avoid interrupt storm */
- IOWRITE64_CR_REGIO(0, WM_EVENT_ENABLE);
- ret = IRQ_WAKE_THREAD;
- }
- }
- }
- spin_unlock_irqrestore(&vha->irq_lock, vha->irq_flags);
- }
- /* Read CORE event source mask. */
- multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, CORE,
- irq_status.event_source);
- /* Note: Direct (Host) core event is only used for frequency measurement,
- * Indirect (WM) core events are read in bottom handler */
- if (multi_src_mask) {
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (multi_src_mask & (1 << id)) {
- /* Select core to read events from. */
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
- MASK, (1 << id)),
- CORE_CTRL_INDIRECT);
- /* Read events. */
- /* In normal operation CORE events are routed to WM,
- * therefore there's no need to handle parity here
- */
- irq_status.core_events[id] = IOREAD64_CR_REGIO(CORE_EVENT_HOST_STATUS);
- /* Just in case check for dead hw. */
- CHECK_FOR_DEAD_HW(irq_status.core_events[id]);
- /* Check for hw logic error. */
- /* ML: ??? */
- //CHECK_FOR_LOGIC_ERROR(CORE, irq_status.core_events[id]);
- if (irq_status.core_events[id] & VHA_CORE_EVENTS_DEFAULT) {
- /* Clear interrupts (best not to write pdump in ISR). */
- IOWRITE64_CR_REGIO(irq_status.core_events[id] & VHA_CORE_EVENTS_DEFAULT,
- CORE_EVENT_HOST_CLEAR);
- /* Record hw processing end timestamps */
- /*
- * for regular workloads. This stat update is used
- * only for cluster clock measurement, so it is
- * executed only once after module is loaded. */
- GETNSTIMEOFDAY(&hw_proc_end[id]);
- hw_proc_end_recorded[id] = true;
- ret = IRQ_WAKE_THREAD;
- }
- }
- }
- }
- /* Read IC event source mask. */
- multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, IC,
- irq_status.event_source);
- /* Indirect (WM) interconnect events are read in bottom handler */
- if (multi_src_mask) {
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (ret || (multi_src_mask & (1 << id))) {
- /* Select IC to read events from. */
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(IC_CORE_INDIRECT,
- MASK, (1 << id)),
- IC_CORE_INDIRECT);
- /* Read events. */
- /* In normal operation IC events are routed to WM,
- * therefore there's no need to handle parity here
- */
- irq_status.ic_events[id] = IOREAD64_CR_REGIO(INTERCONNECT_EVENT_HOST_STATUS);
- #ifdef VHA_SCF
- if (vha->hw_props.supported.parity && !vha->parity_disable) {
- uint32_t i;
- for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
- /* Finish if parity is ok */
- if (!img_mem_calc_parity(irq_status.ic_events[id]))
- break;
- /* Otherwise re-read the reg. */
- irq_status.ic_events[id] = IOREAD64_CR_REGIO(INTERCONNECT_EVENT_HOST_STATUS);
- }
- /* Raise an error if maximum re-read count is reached. */
- if (i == VHA_PARITY_READ_COUNT_MAX) {
- dev_err(dev, "WM_EVENT_STATUS[%u] register parity error!\n", id);
- /* Use the real event to indicate the error */
- VHA_REG_SET_PARITY_ERROR(irq_status.ic_events[id]);
- }
- }
- #endif
- /* Just in case check for dead hw. */
- CHECK_FOR_DEAD_HW(irq_status.ic_events[id]);
- /* Check for hw logic error. */
- /* ML: ??? */
- //CHECK_FOR_LOGIC_ERROR(IC, irq_status.ic_events[id]);
- if (multi_src_mask && (irq_status.ic_events[id] & (VHA_IC_EVENTS_DEFAULT | VHA_REG_PARITY_ERROR_EN))) {
- /* Clear interrupts (best not to write pdump in ISR). */
- IOWRITE64_CR_REGIO(irq_status.ic_events[id] & VHA_IC_EVENTS_DEFAULT,
- INTERCONNECT_EVENT_HOST_CLEAR);
- ret = IRQ_WAKE_THREAD;
- }
- }
- }
- }
- #ifdef VHA_EVENT_INJECT
- __inject_parity_err(vha, &irq_status);
- #endif
- if (!in_interrupt())
- mutex_unlock(&vha->lock);
- if (ret == IRQ_WAKE_THREAD) {
- spin_lock(&vha->irq_lock);
- /* Store all the event info. */
- vha->irq_status.event_source |= irq_status.event_source;
- vha->irq_status.sys_events |= irq_status.sys_events;
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- vha->irq_status.wm_events[id] |= irq_status.wm_events[id];
- if (hw_proc_end_recorded[id]) {
- /* Record hw processing end timestamps */
- VHA_WM_STAT_SHIFT_PROC_END(vha, id);
- VHA_SET_WM_STAT(vha, hw_proc_end, id, hw_proc_end[id]);
- }
- }
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- vha->irq_status.core_events[id] |= irq_status.core_events[id];
- vha->irq_status.ic_events[id] |= irq_status.ic_events[id];
- }
- spin_unlock(&vha->irq_lock);
- }
- #undef CHECK_FOR_DEAD_HW
- #undef CHECK_FOR_LOGIC_ERROR
- if (ret) {
- dev_dbg(dev, "IRQ EVT:0x%08llx SYS:0x%08llx\n", irq_status.event_source, irq_status.sys_events);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
- if (irq_status.wm_events[id] || irq_status.core_events[id] || irq_status.ic_events[id])
- dev_dbg(dev, "WM%d:0x%08llx CORE%d:0x%08llx IC%d:0x%08llx\n",
- id, irq_status.wm_events[id], id, irq_status.core_events[id],
- id, irq_status.ic_events[id]);
- }
- return ret;
- }
- static void vha_do_queued_cmd(struct vha_dev *vha, uint8_t wm_id)
- {
- struct vha_cmd *cmd, *pend;
- cmd = vha->queuedcmd[wm_id].cmd;
- #if defined(DEBUG)
- {
- char queued_txt[24] = "none";
- char pending_txt[24] = "none";
- if (cmd)
- snprintf(queued_txt, 24, "0x%08x/%u",
- cmd->user_cmd.cmd_id, cmd->session->id);
- if (vha->pendcmd[wm_id].cmd)
- snprintf(pending_txt, 24, "0x%08x/%u",
- vha->pendcmd[wm_id].cmd->user_cmd.cmd_id,
- vha->pendcmd[wm_id].cmd->session->id);
- dev_dbg(vha->dev,
- "%s: WM%u pending %s, queued %s\n",
- __func__, wm_id, pending_txt, queued_txt);
- }
- #endif
- if (!cmd || (cmd &&
- ((vha->low_latency == VHA_LL_DISABLED ||
- vha->low_latency == VHA_LL_SELF_KICK) ||
- !cmd->queued))) {
- dev_dbg(vha->dev, "%s: skipping!\n", __func__);
- return;
- }
- /* store actual pending command as it will be modified */
- pend = vha->pendcmd[wm_id].cmd;
- /* at this point we should be able to process the cmd */
- vha_do_cnn_cmd(cmd);
- /* restore pending */
- vha->pendcmd[wm_id].cmd = pend;
- }
- /*
- * Roll back commands for a particular WM.
- */
- static bool vha_rollback_wm_cmds(struct vha_dev *vha, uint8_t wm_id,
- bool free_res)
- {
- bool processing = false;
- #if defined(DEBUG)
- char queued_txt[24] = "none";
- char pending_txt[24] = "none";
- #endif
- /* Not processed commands are still on the pending list
- * of each session, so just mark the hw pending lists as empty */
- if (vha->pendcmd[wm_id].cmd) {
- #if defined(DEBUG)
- snprintf(pending_txt, 24, "0x%08x/%u",
- vha->pendcmd[wm_id].cmd->user_cmd.cmd_id,
- vha->pendcmd[wm_id].cmd->session->id);
- #endif
- if (free_res) {
- /* Free command resources. */
- vha_wm_release_cores(vha,
- vha->pendcmd[wm_id].cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, vha->pendcmd[wm_id].cmd, false);
- vha->pri_q_counters[vha->pendcmd[wm_id].cmd->user_cmd.priority]++;
- }
- VHA_INC_WL_STAT(vha, kicks_aborted, vha->pendcmd[wm_id].cmd);
- vha->stats.cnn_kicks_aborted++;
- vha->pendcmd[wm_id].cmd->in_hw = false;
- vha->pendcmd[wm_id].cmd->queued = false;
- vha->pendcmd[wm_id].cmd->rolled_back = true;
- vha->pendcmd[wm_id].cmd = NULL;
- processing = true;
- }
- /* low_latency ...*/
- if (vha->queuedcmd[wm_id].cmd) {
- #if defined(DEBUG)
- snprintf(queued_txt, 24, "0x%08x/%u",
- vha->queuedcmd[wm_id].cmd->user_cmd.cmd_id,
- vha->queuedcmd[wm_id].cmd->session->id);
- #endif
- /* Free command resources. */
- vha_wm_release_cores(vha,
- vha->queuedcmd[wm_id].cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, vha->queuedcmd[wm_id].cmd, false);
- if (vha->low_latency == VHA_LL_SELF_KICK) {
- VHA_INC_WL_STAT(vha, kicks_aborted, vha->queuedcmd[wm_id].cmd);
- vha->stats.cnn_kicks_aborted++;
- vha->pri_q_counters[vha->queuedcmd[wm_id].cmd->user_cmd.priority]++;
- }
- vha->queuedcmd[wm_id].cmd->in_hw = false;
- vha->queuedcmd[wm_id].cmd->queued = false;
- vha->queuedcmd[wm_id].cmd->rolled_back = true;
- vha->queuedcmd[wm_id].cmd = NULL;
- }
- #if defined(DEBUG)
- dev_dbg(vha->dev, "%s: WM%u pending %s, queued %s\n",
- __func__, wm_id, pending_txt, queued_txt);
- #endif
- return processing;
- }
- bool vha_rollback_cmds(struct vha_dev *vha)
- {
- uint32_t wm_id;
- bool processing = false;
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- bool wm_processing = vha_rollback_wm_cmds(vha, wm_id, true);
- processing = processing || wm_processing;
- }
- return processing;
- }
- static void vha_stop_processing(struct vha_dev *vha)
- {
- uint32_t wm_id;
- VHA_LOCK_WM();
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
- if (vha->pendcmd[wm_id].cmd != NULL) {
- uint64_t wm_mask = VHA_CR_SETBITS(HOST_EVENT_SOURCE, WM, VHA_WM_ID_TO_MASK(wm_id));
- vha_wm_reset(vha, &vha->pendcmd[wm_id].cmd->hw_sched_info);
- VHA_SELECT_WM(wm_id);
- /* Remove WM related interrupt info if it happens to be set. */
- if (vha->irq_status.event_source & wm_mask)
- {
- /* Unset the WM related source bit. */
- vha->irq_status.event_source &= ~wm_mask;
- /* Clear all WM related events. */
- IOWRITE64_CR_REGIO(vha->irq_status.wm_events[wm_id] & VHA_WM_EVENTS_DEFAULT,
- WM_EVENT_CLEAR);
- vha->irq_status.wm_events[wm_id] = 0ULL;
- }
- }
- VHA_UNLOCK_WM();
- }
- int vha_dev_suspend_work(struct vha_dev *vha)
- {
- bool processing = false;
- int ret;
- /* Check if anything is being processed right now. */
- vha_stop_processing(vha);
- /* Rollback commands after hw is stopped. */
- processing = vha_rollback_cmds(vha);
- /* Forcing hardware disable. */
- ret = vha_dev_stop(vha, processing);
- return ret;
- }
- /*
- * Handles the command (of given cmd_idx) already processed by the hw.
- */
- static bool vha_handle_cmd(struct vha_dev *vha, uint8_t wm_id, uint64_t status,
- int err, uint64_t rsp_err_flags)
- {
- struct vha_cmd *cmd = NULL;
- if (wm_id >= vha->hw_props.num_cnn_core_devs)
- return false;
- cmd = vha->pendcmd[wm_id].cmd;
- if (unlikely(!cmd)) {
- dev_dbg(vha->dev, "No command. Probably it has been aborted\n");
- return false;
- }
- vha_cnn_cmd_completed(cmd, status, err, rsp_err_flags);
- if (status) {
- /* Rollback any queued command ... */
- vha_rollback_wm_cmds(vha, wm_id, false);
- /* Notify immediately current command */
- vha_cmd_notify(cmd);
- return false;
- }
- if (vha->queuedcmd[wm_id].cmd)
- vha->pendcmd[wm_id].cmd = vha->queuedcmd[wm_id].cmd;
- else
- vha->pendcmd[wm_id].cmd = NULL;
- vha->queuedcmd[wm_id].cmd = NULL;
- if (vha->pendcmd[wm_id].cmd)
- dev_dbg(vha->dev, "%s: WM%u 0x%08x/%u -> new pending 0x%08x/%u\n",
- __func__, wm_id, cmd->user_cmd.cmd_id, cmd->session->id,
- vha->pendcmd[wm_id].cmd->user_cmd.cmd_id,
- vha->pendcmd[wm_id].cmd->session->id);
- else
- dev_dbg(vha->dev, "%s: WM%u 0x%08x/%u -> no new pending\n",
- __func__, wm_id, cmd->user_cmd.cmd_id, cmd->session->id);
- vha_cmd_notify(cmd);
- return true;
- }
- void vha_dev_update_per_core_kicks(uint8_t core_mask, uint32_t *kicks_array)
- {
- while (core_mask != 0) {
- uint32_t curr_core_id = VHA_CORE_MASK_TO_ID(core_mask);
- core_mask &= ~(VHA_CORE_ID_TO_MASK(curr_core_id));
- kicks_array[curr_core_id]++;
- }
- }
- static int vha_report_wm_rsp_failure(struct vha_dev *vha, uint8_t wm_id,
- uint64_t wm_rsp_status, uint64_t *core_status, uint64_t *ic_status,
- enum vha_reset_type *reset_type, uint64_t *error_flags)
- {
- uint8_t err_code = VHA_WM_RESPONSE_GET_ERROR_CODE(wm_rsp_status);
- int cmdid = -1;
- int sesid = -1;
- uint32_t i = 0;
- int err = -EIO;
- if (vha->pendcmd[wm_id].cmd) {
- cmdid = vha->pendcmd[wm_id].cmd->user_cmd.cmd_id;
- sesid = vha->pendcmd[wm_id].cmd->session->id;
- }
- if (vha_observers.error)
- vha_observers.error(vha->id, sesid, cmdid, wm_rsp_status);
- if (VHA_REG_GET_PARITY_ERROR(wm_rsp_status)) {
- dev_err(vha->dev, " WM%u response error: PARITY\n", wm_id);
- *reset_type = VHA_RESET_TYPE_WM;
- *error_flags |= VHA_RSP_ERROR(SW_WM_PARITY_ERROR);
- } else if (VHA_REG_GET_WL_ID_MISMATCH_ERROR(wm_rsp_status)) {
- dev_err(vha->dev, " WM%u response error: WL_ID_MISMATCH\n", wm_id);
- *reset_type = VHA_RESET_TYPE_WM;
- *error_flags |= VHA_RSP_ERROR(SW_WL_ID_MISMATCH_ERROR);
- } else if (VHA_REG_GET_CONF_ERROR(wm_rsp_status)) {
- dev_err(vha->dev, " WM%u response error: CONFIRMATION_WRITES\n", wm_id);
- *reset_type = VHA_RESET_TYPE_WM;
- *error_flags |= VHA_RSP_ERROR(SW_CONF_ERROR);
- } else if (VHA_REG_GET_COMBINED_CRC_ERROR(wm_rsp_status)) {
- dev_err(vha->dev, " WM%u response error: COMBINED_CRC\n", wm_id);
- *reset_type = VHA_RESET_TYPE_WM;
- *error_flags |= VHA_RSP_ERROR(SW_CRC_MISMATCH_ERROR);
- } else {
- while (i < ARRAY_SIZE(wm_rsp_err_codes)) {
- if (wm_rsp_err_codes[i].e == err_code) {
- uint8_t core_id = VHA_WM_RESPONSE_GET_FAILED_CORE_IDX(wm_rsp_status);
- /* Store reset type. */
- *reset_type = wm_rsp_err_codes[i].reset_type;
- /* Error that caused the Workload Manager to halt*/
- dev_err(vha->dev, " WM%u error code:%d -> %s, failure on core%u\n",
- wm_id, err_code, wm_rsp_err_codes[i].s, core_id);
- *error_flags |= wm_rsp_err_codes[i].rsp_err;
- if (core_id < vha->hw_props.num_cnn_core_devs) {
- i = 0;
- while (core_err_bits[i].e != 0) {
- if (core_status[core_id] & core_err_bits[i].b) {
- dev_err(vha->dev, " %s\n", core_err_bits[i].s);
- err = core_err_bits[i].e;
- }
- i++;
- }
- i = 0;
- while (ic_err_bits[i].e != 0) {
- if (ic_status[core_id] & ic_err_bits[i].b) {
- dev_err(vha->dev, " %s\n", ic_err_bits[i].s);
- err = ic_err_bits[i].e;
- }
- i++;
- }
- } else
- dev_err(vha->dev, " invalid FAILED_CORE_ID, should be <%u\n",
- vha->hw_props.num_cnn_core_devs);
- goto exit;
- }
- i++;
- }
- dev_err(vha->dev, " invalid WM ERROR_CODE: %u\n", err_code);
- }
- exit:
- return err;
- }
- static void vha_handle_sys_failure(struct vha_dev *vha, uint64_t status, int err, uint64_t rsp_err_flags)
- {
- int cmdid = -1;
- int sesid = -1;
- uint32_t wm_id;
- struct vha_cmd *cmd = NULL;
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- cmd = vha->pendcmd[wm_id].cmd;
- if (cmd) {
- cmdid = cmd->user_cmd.cmd_id;
- sesid = cmd->session->id;
- }
- if (vha_observers.error)
- vha_observers.error(vha->id, sesid, cmdid, status);
- cmdid = -1;
- if (cmd) {
- /* Update stats. */
- vha->stats.total_failures++;
- vha->stats.cnn_kicks_completed++;
- VHA_INC_WL_STAT(vha, kicks_completed, cmd);
- vha_wm_reset(vha, &cmd->hw_sched_info);
- /* Free command resources. */
- vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, cmd, true);
- }
- /* Move command queue. */
- vha_do_queued_cmd(vha, wm_id);
- /* Handle actual command */
- vha_handle_cmd(vha, wm_id, status, err, rsp_err_flags);
- }
- }
- static void vha_handle_wm_failure(struct vha_dev *vha, uint8_t wm_id,
- uint64_t status, int err, uint64_t rsp_err_flags)
- {
- int cmdid = -1;
- int sesid = -1;
- struct vha_cmd *cmd = NULL;
- cmd = vha->pendcmd[wm_id].cmd;
- if (cmd) {
- cmdid = cmd->user_cmd.cmd_id;
- sesid = cmd->session->id;
- }
- if (vha_observers.error)
- vha_observers.error(vha->id, sesid, cmdid, status);
- if (cmd) {
- /* Update stats. */
- vha->stats.total_failures++;
- vha->stats.cnn_kicks_completed++;
- VHA_INC_WL_STAT(vha, kicks_completed, cmd);
- /* Free command resources. */
- vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, cmd, true);
- }
- /* Move command queue. */
- vha_do_queued_cmd(vha, wm_id);
- /* Handle actual command */
- vha_handle_cmd(vha, wm_id, status, err, rsp_err_flags);
- }
- static enum vha_reset_type vha_sys_get_reset_type(struct vha_dev *vha,
- uint64_t event_mask) {
- enum vha_reset_type sys_reset_type = VHA_RESET_TYPE_NONE;
- #ifdef VHA_SCF
- uint64_t sys_err_events = VHA_SYS_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
- #else
- uint64_t sys_err_events = VHA_SYS_ERR_EVENTS;
- #endif
- if (event_mask & sys_err_events) {
- uint32_t i = 0;
- while (sys_err_bits[i].e != 0) {
- if (event_mask & sys_err_bits[i].b) {
- /* Indicate the highest reset level of all errors. */
- if (sys_err_bits[i].reset_type > sys_reset_type)
- sys_reset_type = sys_err_bits[i].reset_type;
- }
- i++;
- }
- }
- return sys_reset_type;
- }
- static void vha_sys_get_wm_reset_types(struct vha_dev *vha, uint64_t event_mask,
- enum vha_reset_type *wm_reset_types) {
- uint8_t wm_id;
- uint8_t pf_errors;
- #ifdef VHA_SCF
- // uint8_t parity_errors;
- #endif
- /* Check MMU page fault errors. */
- pf_errors = (uint8_t)VHA_CR_GETBITS(SYS_EVENT_STATUS, MMU_PAGE_FAULT,
- event_mask);
- if (pf_errors) {
- wm_id = 0;
- while(wm_id < vha->hw_props.num_cnn_core_devs) {
- if (pf_errors & (1 << wm_id))
- wm_reset_types[wm_id] = VHA_RESET_TYPE_MMU;
- else
- wm_reset_types[wm_id] = VHA_RESET_TYPE_NONE;
- wm_id++;
- }
- }
- #ifdef VHA_SCF
- /* Check MMU parity errors. */
-
- // uint8_t parity_errors = (uint8_t)VHA_CR_GETBITS(SYS_EVENT_STATUS, MMU_PARITY_ERROR,
- // event_mask);
- // if (parity_errors) {
- // wm_id = 0;
- // while(wm_id < vha->hw_props.num_cnn_core_devs) {
- // if (parity_errors & (1 << wm_id))
- // wm_reset_types[wm_id] = VHA_RESET_TYPE_MMU;
- // else
- // wm_reset_types[wm_id] = VHA_RESET_TYPE_NONE;
- // wm_id++;
- // }
- // }
- #endif
- }
- static enum vha_reset_type vha_wm_get_reset_type(struct vha_dev *vha,
- uint64_t event_mask) {
- enum vha_reset_type wm_reset_type = VHA_RESET_TYPE_NONE;
- #ifdef VHA_SCF
- uint64_t wm_err_events = VHA_WM_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
- #else
- uint64_t wm_err_events = VHA_WM_ERR_EVENTS;
- #endif
- if (event_mask & wm_err_events) {
- uint32_t i = 0;
- while (wm_err_bits[i].e != 0) {
- if (event_mask & wm_err_bits[i].b) {
- /* Indicate the highest reset level of all errors. */
- if (wm_err_bits[i].reset_type > wm_reset_type)
- wm_reset_type = wm_err_bits[i].reset_type;
- }
- i++;
- }
- }
- return wm_reset_type;
- }
- static enum vha_reset_type vha_core_get_reset_type(struct vha_dev *vha,
- uint64_t event_mask) {
- enum vha_reset_type core_reset_type = VHA_RESET_TYPE_NONE;
- uint64_t core_err_events = VHA_CORE_ERR_EVENTS;
- if (event_mask & core_err_events) {
- uint32_t i = 0;
- while (core_err_bits[i].e != 0) {
- if (event_mask & core_err_bits[i].b) {
- /* Indicate the highest reset level of all errors. */
- if (core_err_bits[i].reset_type > core_reset_type)
- core_reset_type = core_err_bits[i].reset_type;
- }
- i++;
- }
- }
- return core_reset_type;
- }
- static enum vha_reset_type vha_ic_get_reset_type(struct vha_dev *vha,
- uint64_t event_mask) {
- enum vha_reset_type ic_reset_type = VHA_RESET_TYPE_NONE;
- #ifdef VHA_SCF
- uint64_t ic_err_events = VHA_IC_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
- #else
- uint64_t ic_err_events = VHA_IC_ERR_EVENTS;
- #endif
- if (event_mask & ic_err_events) {
- uint32_t i = 0;
- while (ic_err_bits[i].e != 0) {
- if (event_mask & ic_err_bits[i].b) {
- /* Indicate the highest reset level of all errors. */
- if (ic_err_bits[i].reset_type > ic_reset_type)
- ic_reset_type = ic_err_bits[i].reset_type;
- }
- i++;
- }
- }
- return ic_reset_type;
- }
- static int vha_report_sys_failures(struct vha_dev *vha, uint64_t event_mask, uint64_t *error_flags)
- {
- int error = 0;
- uint32_t i;
- bool print_header = true;
- uint8_t pf_status;
- /* Print event status in human readable form. */
- i = 0;
- while (sys_err_bits[i].e != 0) {
- if (event_mask & sys_err_bits[i].b) {
- if (print_header) {
- dev_err(vha->dev, " SYS event status:\n");
- print_header = false;
- }
- dev_err(vha->dev, " %s\n", sys_err_bits[i].s);
- /* Convert from register bits into POSIX errno.
- * If multiple errors, then arbitrary errno choice. */
- error = sys_err_bits[i].e;
- *error_flags |= sys_err_bits[i].rsp_err;
- }
- i++;
- }
- if (error) {
- dev_err(vha->dev, " SYS failure:\n");
- dev_err(vha->dev, " SYS_CLK_STATUS0: 0x%016llx\n",
- IOREAD64_CR_REGIO(SYS_CLK_STATUS0));
- dev_err(vha->dev, " SYS_EVENT_STATUS: 0x%016llx\n",
- event_mask);
- for (i = 0; i < vha->hw_props.num_cnn_core_devs; i++) {
- if (vha->active_core_mask & (1 << i)) {
- /* Select core to read clocks from. */
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
- MASK, (1 << i)),
- CORE_CTRL_INDIRECT);
- dev_err(vha->dev, " CORE%u CLK_STATUS0: 0x%016llx\n",
- i, IOREAD64_CR_REGIO(CLK_STATUS0));
- }
- }
- }
- if (error == -ETIMEDOUT) {
- dev_err(vha->dev, " SLC_STATUS1: 0x%016llx\n",
- IOREAD64_CR_REGIO(SLC_STATUS1));
- dev_err(vha->dev, " SLC_STATUS2: 0x%016llx\n",
- IOREAD64_CR_REGIO(SLC_STATUS2));
- dev_err(vha->dev, " SLC_IDLE: 0x%016llx\n",
- IOREAD64_CR_REGIO(SLC_IDLE));
- }
- /* Additionally report MMU PF failure if occurred. */
- pf_status = (uint8_t)VHA_CR_GETBITS(SYS_EVENT_STATUS, MMU_PAGE_FAULT,
- event_mask);
- if (pf_status) {
- /* dump mmu status */
- vha_mmu_status(vha, pf_status);
- }
- return error;
- }
- static int vha_report_wm_failures(struct vha_dev *vha, uint8_t wm_id, uint64_t event_mask, uint64_t *error_flags)
- {
- int error = 0;
- uint32_t i;
- bool print_header = true;
- /* Print event status in human readable form. */
- i = 0;
- print_header = true;
- while (wm_err_bits[i].e != 0) {
- if (event_mask & wm_err_bits[i].b) {
- if (print_header) {
- dev_err(vha->dev, " WM%u event status:\n", wm_id);
- print_header = false;
- }
- dev_err(vha->dev, " %s\n", wm_err_bits[i].s);
- /* Convert from register bits into POSIX errno.
- * If multiple errors, then arbitrary errno choice. */
- error = wm_err_bits[i].e;
- *error_flags |= wm_err_bits[i].rsp_err;
- }
- i++;
- }
- if (error == -ETIMEDOUT) {
- vha_wm_status(vha, wm_id, vha_wm_get_cores(vha, wm_id));
- }
- return error;
- }
- static int vha_report_core_failures(struct vha_dev *vha, uint8_t core_id, uint64_t event_mask, uint64_t *error_flags)
- {
- int error = 0;
- uint32_t i;
- bool print_header = true;
- /* Print event status in human readable form. */
- i = 0;
- print_header = true;
- while (core_err_bits[i].e != 0) {
- if (event_mask & core_err_bits[i].b) {
- if (print_header) {
- dev_err(vha->dev, " Core %u event status:\n", core_id);
- print_header = false;
- }
- dev_err(vha->dev, " %s\n", core_err_bits[i].s);
- /* Convert from register bits into POSIX errno.
- * If multiple errors, then arbitrary errno choice. */
- error = core_err_bits[i].e;
- *error_flags |= core_err_bits[i].rsp_err;
- }
- i++;
- }
- return error;
- }
- static int vha_report_ic_failures(struct vha_dev *vha, uint8_t core_id, uint64_t event_mask, uint64_t *error_flags)
- {
- int error = 0;
- uint32_t i;
- bool print_header = true;
- /* Print event status in human readable form. */
- i = 0;
- print_header = true;
- while (ic_err_bits[i].e != 0) {
- if (event_mask & ic_err_bits[i].b) {
- if (print_header) {
- dev_err(vha->dev, " IC %u event status:\n", core_id);
- print_header = false;
- }
- dev_err(vha->dev, " %s\n", ic_err_bits[i].s);
- /* Convert from register bits into POSIX errno.
- * If multiple errors, then arbitrary errno choice. */
- error = ic_err_bits[i].e;
- *error_flags |= ic_err_bits[i].rsp_err;
- }
- i++;
- }
- return error;
- }
- static uint8_t vha_events_process_errors(struct vha_dev *vha,
- struct vha_mc_irq_status *irq_status, bool *full_reset,
- bool *process_sys_events, uint64_t *error_flags) {
- int error = 0;
- int wm_error = 0;
- int core_error = 0;
- int ic_error = 0;
- uint8_t wm_process_mask = 0;
- uint8_t wm_source_mask = 0;
- uint8_t wm_id;
- enum vha_reset_type reset_type = VHA_RESET_TYPE_NONE;
- enum vha_reset_type wm_reset_type = VHA_RESET_TYPE_NONE;
- enum vha_reset_type wm_reset_types[VHA_NUM_CORES] = {0};
- uint64_t sys_err_status = 0;
- uint64_t wm_err_status_full_reset = 0;
- uint64_t wm_err_statuses[VHA_NUM_CORES] = {0};
- enum vha_reset_type core_reset_type = 0;
- enum vha_reset_type ic_reset_type = 0;
- uint8_t core_id;
- #define COMBINE_SYS_WM_STATUS(s, w) \
- (((w & ~((uint64_t)VHA_WM_ERR_EVENTS)) | s) | \
- ((w & ((uint64_t)VHA_WM_ERR_EVENTS)) << 32))
- #define INSERT_WM_ERROR(s, e) \
- ((s | ((uint64_t)e)) << 32)
- /* Assume no full reset. */
- *full_reset = false;
- /* Assume no SYS events. */
- *process_sys_events = false;
- /* Process SYS events. */
- if (VHA_CR_GETBITS(HOST_EVENT_SOURCE, SYS, irq_status->event_source)) {
- #ifdef VHA_SCF
- uint64_t sys_err_events = VHA_SYS_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
- #else
- uint64_t sys_err_events = VHA_SYS_ERR_EVENTS;
- #endif
- sys_err_status = irq_status->sys_events & sys_err_events;
- if (sys_err_status) {
- /* Determine reset types. */
- reset_type = vha_sys_get_reset_type(vha, irq_status->sys_events);
- if (reset_type < VHA_RESET_TYPE_FULL) {
- vha_sys_get_wm_reset_types(vha, irq_status->sys_events, wm_reset_types);
- wm_id = 0;
- while(wm_id < vha->hw_props.num_cnn_core_devs) {
- if (wm_reset_types[wm_id] > VHA_RESET_TYPE_NONE)
- wm_err_statuses[wm_id] = sys_err_status;
- wm_id++;
- }
- }
- /* Report SYS errors. */
- error = vha_report_sys_failures(vha, irq_status->sys_events, error_flags);
- }
- /* If no full reset is requested at this stage
- * and there are non-error SYS events raised,
- * signal them to be processed too. */
- if ((reset_type < VHA_RESET_TYPE_FULL) &&
- (irq_status->sys_events & ~sys_err_events))
- *process_sys_events = true;
- }
- /* Process WM events. */
- /* Read WM event source mask. */
- wm_source_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM,
- irq_status->event_source);
- if (wm_source_mask)
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
- if (wm_source_mask & (1 << wm_id)) {
- #ifdef VHA_SCF
- uint64_t wm_err_events = VHA_WM_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
- #else
- uint64_t wm_err_events = VHA_WM_ERR_EVENTS;
- #endif
- uint64_t wm_err_status = irq_status->wm_events[wm_id] & wm_err_events;
- if (wm_err_status) {
- /* If no full reset is requested... */
- if (reset_type < VHA_RESET_TYPE_FULL) {
- /* Determine reset type for this WM. */
- wm_reset_type = vha_wm_get_reset_type(
- vha, irq_status->wm_events[wm_id]);
- /* If full reset is requested for this WM, just skip
- * checking other ones. Otherwise update reset type
- * for this WM if needed. */
- if (wm_reset_type == VHA_RESET_TYPE_FULL) {
- reset_type = VHA_RESET_TYPE_FULL;
- wm_err_status_full_reset = wm_err_status;
- } else if (wm_reset_type > wm_reset_types[wm_id])
- wm_reset_types[wm_id] = wm_reset_type;
- }
- /* Compose accumulated error status. */
- wm_err_statuses[wm_id] =
- COMBINE_SYS_WM_STATUS(sys_err_status, wm_err_status);
- /* Report WM errors. */
- wm_error = vha_report_wm_failures(vha, wm_id,
- irq_status->wm_events[wm_id], error_flags);
- /* If no SYS error reported, get the first WM one. */
- if (error == 0)
- error = wm_error;
- }
- }
- /* Process core events */
- for (core_id = 0; core_id < vha->hw_props.num_cnn_core_devs; core_id++)
- if (irq_status->core_events[core_id] & VHA_CORE_ERR_EVENTS) {
- /* Determine reset type for this Core. */
- core_reset_type = vha_core_get_reset_type(vha, irq_status->core_events[core_id]);
- /* We do not reset the core itself, instead, we need to reset
- the WM that used it, so let's find it */
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
- if (vha->pendcmd[wm_id].cmd != NULL) {
- uint8_t core_mask = vha->pendcmd[wm_id].cmd->hw_sched_info.core_mask;
- if (core_mask & (1 << core_id)) {
- /* Override wm reset type */
- if (core_reset_type == VHA_RESET_TYPE_FULL)
- reset_type = VHA_RESET_TYPE_FULL;
- else if (core_reset_type > wm_reset_types[wm_id])
- wm_reset_types[wm_id] = core_reset_type;
- core_error = vha_report_core_failures(vha, core_id,
- irq_status->core_events[core_id], error_flags);
- /* If no SYS or WM error reported, get the first Core one. */
- if (error == 0)
- error = core_error;
- /* Add core error to this WM's status. */
- wm_err_statuses[wm_id] =
- INSERT_WM_ERROR(wm_err_statuses[wm_id], VHA_REG_WM_CORE_ERROR_EN);
- }
- }
- }
- /* Process IC events */
- for (core_id = 0; core_id < vha->hw_props.num_cnn_core_devs; core_id++) {
- #ifdef VHA_SCF
- uint64_t ic_err_events = VHA_IC_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
- #else
- uint64_t ic_err_events = VHA_IC_ERR_EVENTS;
- #endif
- if (irq_status->ic_events[core_id] & ic_err_events) {
- /* Determine reset type for this Core. */
- ic_reset_type = vha_ic_get_reset_type(vha, irq_status->ic_events[core_id]);
- /* We do not reset the core itself, instead, we need to reset
- the WM that used it, so let's find it */
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
- if (vha->pendcmd[wm_id].cmd != NULL) {
- uint8_t core_mask = vha->pendcmd[wm_id].cmd->hw_sched_info.core_mask;
- if (core_mask & (1 << core_id)) {
- /* Override wm reset type */
- if (ic_reset_type == VHA_RESET_TYPE_FULL)
- reset_type = VHA_RESET_TYPE_FULL;
- else if (ic_reset_type > wm_reset_types[wm_id])
- wm_reset_types[wm_id] = ic_reset_type;
- ic_error = vha_report_ic_failures(vha, core_id,
- irq_status->ic_events[core_id], error_flags);
- /* If no SYS or WM error reported, get the first Core one. */
- if (error == 0)
- error = ic_error;
- /* Add IC error to this WM's status. */
- wm_err_statuses[wm_id] =
- INSERT_WM_ERROR(wm_err_statuses[wm_id], VHA_REG_WM_IC_ERROR_EN);
- }
- }
- }
- }
- /* Perform selective resets. */
- if (reset_type < VHA_RESET_TYPE_FULL) {
- int ret;
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- struct vha_cmd *cmd = vha->pendcmd[wm_id].cmd;
- switch (wm_reset_types[wm_id]) {
- case VHA_RESET_TYPE_MMU:
- if (cmd) {
- /* Invalidate MMU. */
- ret = vha_mmu_flush_ctx(vha, cmd->session->mmu_ctxs[VHA_MMU_REQ_IO_CTXID].hw_id);
- if(ret) {
-
- dev_err(vha->dev, "Error during MMU flush, doing full reset\n");
- wm_err_status_full_reset = wm_err_statuses[wm_id];
- reset_type = VHA_RESET_TYPE_FULL;
- break;
- }
- }
- // fall through
- case VHA_RESET_TYPE_WM:
- dev_err(vha->dev, "Performing wm%d reset due to HW error detection.", wm_id);
- if (cmd)
- /* Reset WM and assigned cores. */
- ret = vha_wm_reset(vha, &cmd->hw_sched_info);
- else {
- /* Just reset WM. */
- struct vha_hw_sched_info sched_info = {
- .wm_id = wm_id,
- .core_mask = 0
- };
- ret = vha_wm_reset(vha, &sched_info);
- }
- if(ret) {
- dev_err(vha->dev, "Error during WM%d reset, doing full reset\n", wm_id);
- wm_err_status_full_reset = wm_err_statuses[wm_id];
- reset_type = VHA_RESET_TYPE_FULL;
- break;
- }
- VHA_LOCK_WM();
- VHA_SELECT_WM(wm_id);
- /* Clear all WM related events. */
- IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_CLEAR);
- /* Re-enable WM events here as this WM will not be handled further. */
- IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
- VHA_UNLOCK_WM();
- /* Handle pending command. */
- vha_handle_wm_failure(vha, wm_id, wm_err_statuses[wm_id], error, *error_flags);
- break;
- case VHA_RESET_TYPE_NONE:
- /* Mark WM source for normal processing if it was signalled. */
- if (cmd)
- wm_process_mask |= wm_source_mask & (1 << wm_id);
- break;
- default:
- break;
- }
- }
- }
- /* check once again, reset_type may have been updated due to failure during reset procedure */
- if(reset_type == VHA_RESET_TYPE_FULL){
- /* Handle all pending commands. */
- vha_handle_sys_failure(vha,
- COMBINE_SYS_WM_STATUS(sys_err_status, wm_err_status_full_reset), error, *error_flags);
- /* Full reset is requested anyway, so skip processing further SYS events. */
- *process_sys_events = false;
- /* Full reset will be executed outside. Just indicate here
- * that it's required.*/
- *full_reset = true;
- }
- return wm_process_mask;
- }
- /* if vha event register reports WM events, so handle them */
- static void vha_handle_wm_response(struct vha_dev *vha, uint8_t wm_id,
- uint64_t response_status, uint64_t *core_status_array,
- uint64_t *ic_status_array, bool *full_reset, uint64_t *error_flags)
- {
- enum vha_reset_type reset_type = VHA_RESET_TYPE_NONE;
- int err = *error_flags ? -EIO : 0 ;
- if (response_status &
- (VHA_WM_RESPONSE_STATUS(WL_FAILURE) |
- VHA_REG_PARITY_ERROR_EN |
- VHA_REG_WL_ID_MISMATCH_ERROR_EN |
- VHA_REG_CONF_ERROR_EN |
- VHA_REG_COMBINED_CRC_ERROR_EN)) {
- err = vha_report_wm_rsp_failure(vha, wm_id, response_status,
- core_status_array, ic_status_array, &reset_type, error_flags);
- }
- /* Move command queue. */
- switch (reset_type) {
- case VHA_RESET_TYPE_NONE:
- vha_do_queued_cmd(vha, wm_id);
- break;
- case VHA_RESET_TYPE_WM:
- if (!*full_reset && vha->pendcmd[wm_id].cmd) {
- dev_err(vha->dev, "Performing wm%d reset due to HW error detection.", wm_id);
- if (vha_wm_reset(vha, &vha->pendcmd[wm_id].cmd->hw_sched_info)) {
- dev_err(vha->dev, "%s: Error during WM%u reset, forcing full reset upon finish",
- __func__, wm_id);
- *full_reset = true;
- }
- }
- break;
- case VHA_RESET_TYPE_FULL:
- *full_reset = true;
- break;
- default:
- break;
- }
- /* Handle actual command */
- if (vha_handle_cmd(vha, wm_id, response_status, err, *error_flags) == false)
- reset_type = VHA_RESET_TYPE_NONE;
- }
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- /* Simulating hw execution time by scheduling this delayed work. */
- void vha_dummy_worker(struct work_struct *work)
- {
- struct vha_dummy_work *dummy_work =
- container_of(work, struct vha_dummy_work, dummy_dwork.work);
- struct vha_dev *vha = dummy_work->vha;
- struct vha_cmd *cmd;
- mutex_lock(&vha->lock);
- cmd = vha->pendcmd[dummy_work->wm_id].cmd;
- if (cmd) {
- uint64_t error_flags = 0;
- bool full_reset = false;
- /* Record hw processing end timestamps */
- VHA_WM_STAT_SHIFT_PROC_END(vha, cmd->hw_sched_info.wm_id);
- GETNSTIMEOFDAY(&vha->stats.wm_stats[cmd->hw_sched_info.wm_id].hw_proc_end);
- /* Update per core/WM stats. */
- VHA_INC_WL_STAT(vha, kicks_completed, cmd);
- vha->stats.cnn_kicks_completed++;
- /* Free command resources. */
- if (!vha->hw_sched_status.assignments[cmd->hw_sched_info.assignment_id].queued)
- vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, cmd, true);
- /* Handle current pending command */
- vha_handle_wm_response(vha, dummy_work->wm_id, 0, NULL, NULL, &full_reset, &error_flags);
- /* Schedule following commands */
- vha_chk_cmd_queues(vha, true);
- }
- mutex_unlock(&vha->lock);
- }
- #endif
- #ifdef VHA_SCF
- static void vha_handle_conf_status(struct vha_dev *vha, struct vha_cmd *cmd, bool *full_reset, uint64_t *status)
- {
- if (wait_for_completion_timeout(&cmd->conf_done, msecs_to_jiffies(CONF_WRITES_WAIT_TIMEOUT_MS))) {
- if (cmd->conf_top_error) {
- dev_err(vha->dev, "CONF_ERR_TOP\n");
- *full_reset = true;
- VHA_REG_SET_CONF_ERROR(*status);
- return;
- }
- if (cmd->conf_core_error) {
- dev_err(vha->dev, "CONF_ERR_BOTTOM\n");
- VHA_REG_SET_CONF_ERROR(*status);
- return;
- }
- } else {
- dev_err(vha->dev, "Confirmation writes procedure failed!\n");
- VHA_REG_SET_CONF_ERROR(*status);
- }
- }
- static void vha_check_crc(struct vha_dev *vha, struct vha_cmd *cmd, uint64_t *status)
- {
- struct vha_session *session = cmd->session;
- struct vha_hw_sched_info *sched_info = &cmd->hw_sched_info;
- uint32_t core_id = 0;
- uint32_t idx = 0;
- uint8_t num_cores = VHA_CORE_MASK_TO_NUM(sched_info->core_mask);
- uint32_t crcs[VHA_MAX_CORES];
- uint32_t *golden_crcs = NULL;
- struct vha_buffer *buf = session->cnn_dbg.cnn_combined_crc;
- bool crc_enabled = !!(cmd->user_cmd.flags & VHA_CHECK_CRC);
- if (!buf || !buf->kptr) {
- dev_err(vha->dev, "%s: Invalid crc buf\n", __func__);
- return;
- }
- img_mem_sync_device_to_cpu(session->mem_ctx, buf->id);
- for (core_id = 0; core_id < VHA_MAX_CORES; core_id++)
- if (sched_info->core_mask & (1 << core_id)) {
- memcpy(&crcs[idx], (uint8_t*)buf->kptr + core_id * VHA_COMBINED_CRC_CORE_OFFSET, sizeof(crcs[0]));
- idx++;
- }
- vha_update_crcs(vha, crcs, num_cores);
- if (crc_enabled) {
- struct vha_user_cnn_submit_multi_cmd *cnn_user_cmd =
- (struct vha_user_cnn_submit_multi_cmd *)&cmd->user_cmd;
- golden_crcs = cnn_user_cmd->crcs;
- for (idx = 0; idx < num_cores; idx++)
- if (crcs[idx] != golden_crcs[idx]) {
- VHA_REG_SET_COMBINED_CRC_ERROR(*status);
- dev_err(vha->dev, "%s: combined CRC mismatch !!!\n"
- "\tcrc %x\n"
- "\tgolden_crc %x\n", __func__, crcs[idx], golden_crcs[idx]);
- } else {
- dev_info(vha->dev, "%s: combined CRC ok, crc %x\n", __func__, crcs[idx]);
- }
- }
- }
- #endif
- /* Bottom half */
- irqreturn_t vha_handle_thread_irq(struct device *dev)
- {
- struct vha_dev *vha = vha_dev_get_drvdata(dev);
- irqreturn_t ret = IRQ_HANDLED;
- struct vha_mc_irq_status irq_status;
- uint64_t multi_src_mask = 0;
- uint8_t id;
- uint8_t wm_id;
- uint8_t wm_process_mask = 0;
- bool full_reset = false;
- bool process_sys_events = false;
- uint64_t error_flags = 0;
- if (!vha)
- return IRQ_NONE;
- mutex_lock(&vha->lock);
- #ifdef CONFIG_FAULT_INJECTION
- if (!vha->irq_bh_pid)
- vha->irq_bh_pid = task_pid_nr(current);
- if (vha->fault_inject & VHA_FI_IRQ_WORKER)
- current->make_it_fail = true;
- else
- current->make_it_fail = false;
- #endif
- spin_lock_irq(&vha->irq_lock);
- irq_status = vha->irq_status;
- memset(&vha->irq_status, 0, sizeof(vha->irq_status));
- if (irq_status.sys_events || vha->do_calibration) {
- uint64_t proc_time = 0;
- if (get_timespan_us(&vha->stats.wm_stats[VHA_CALIBRATION_WM_ID].hw_proc_start,
- &vha->stats.wm_stats[VHA_CALIBRATION_WM_ID].hw_proc_end,
- &proc_time)) {
- vha->stats.last_proc_us = proc_time;
- } else {
- vha->stats.last_proc_us = 0;
- }
- }
- spin_unlock_irq(&vha->irq_lock);
- /* Read CORE event source mask. */
- multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, CORE,
- irq_status.event_source);
- /* Check for clock calibration first. */
- if ((multi_src_mask == VHA_CALIBRATION_CORE_MASK) &&
- (irq_status.core_events[VHA_CALIBRATION_CORE_ID] &
- VHA_CORE_EVENT_TYPE(CORE_WDT))) {
- if (vha_check_calibration(vha)) {
- goto calibration_end;
- }
- }
- /* Read core/interconnect events if System or WM event occurred */
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- uint8_t mask = 0;
- if (irq_status.wm_events[wm_id])
- mask = vha_wm_get_cores(vha, wm_id);
- if (irq_status.sys_events)
- mask |= (1 << wm_id);
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if(mask & (1 << id)) {
- /* Select core to read events from. */
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
- MASK, (1 << id)),
- CORE_CTRL_INDIRECT);
- irq_status.core_events[id] |= IOREAD64_CR_REGIO(CORE_EVENT_WM_STATUS);
- if (irq_status.core_events[id] & VHA_CORE_ERR_EVENTS) {
- IOWRITE64_CR_REGIO(irq_status.core_events[id] & VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_CLEAR);
- irq_status.core_events[id] |= IOREAD64_CR_REGIO(CORE_EVENT_WM_STATUS);
- }
- /* Select IC to read events from. */
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(IC_CORE_INDIRECT,
- MASK, (1 << id)),
- IC_CORE_INDIRECT);
- irq_status.ic_events[id] |= IOREAD64_CR_REGIO(INTERCONNECT_EVENT_WM_STATUS);
- if (irq_status.ic_events[id] & VHA_IC_ERR_EVENTS) {
- IOWRITE64_CR_REGIO(irq_status.ic_events[id] & VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_CLEAR);
- irq_status.ic_events[id] |= IOREAD64_CR_REGIO(INTERCONNECT_EVENT_WM_STATUS);
- }
- }
- }
- }
- /* Process errors first. */
- wm_process_mask = vha_events_process_errors(vha, &irq_status,
- &full_reset, &process_sys_events, &error_flags);
- /* Process non-error system events. */
- if (process_sys_events) {
- /* Handle normal system events. */
- }
- /* Process non-failed WM events. */
- if (wm_process_mask) {
- uint64_t rsp_err_status = 0ULL;
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
- if (wm_process_mask & (1 << id)) {
- if (irq_status.wm_events[id] & VHA_WM_EVENTS) {
- uint16_t wm_cmd_id;
- uint64_t status;
- #ifdef VHA_SCF
- uint64_t wm_rsp_err_events =
- ((VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_MASKFULL |
- VHA_REG_PARITY_ERROR_EN | VHA_REG_WL_ID_MISMATCH_ERROR_EN) &
- ~((uint64_t)VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_SUCCESS_EN |
- (uint64_t)VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_PARITY_EN));
- #else
- uint64_t wm_rsp_err_events =
- ((VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_MASKFULL |
- VHA_REG_WL_ID_MISMATCH_ERROR_EN) &
- ~(VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_SUCCESS_EN |
- VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_PARITY_EN));
- #endif
- struct vha_cmd *cmd = vha->pendcmd[id].cmd;
- if (cmd != NULL) {
- /* Select WM to read response from. */
- VHA_LOCK_WM();
- VHA_SELECT_WM(id);
- /* Handle RESPONSE_FIFO. */
- /* Read RESPONSE_FIFO_WL_STATUS. */
- status = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_STATUS);
- #ifdef VHA_SCF
- if (vha->hw_props.supported.parity && !vha->parity_disable) {
- uint32_t i;
- for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
- /* Finish if parity is ok */
- if (!img_mem_calc_parity(status))
- break;
- /* Otherwise re-read the reg. */
- status = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_STATUS);
- }
- /* Raise an error if maximum re-read count is reached. */
- if (i == VHA_PARITY_READ_COUNT_MAX) {
- dev_err(dev, "WM_RESPONSE_FIFO_WL_STATUS register parity error!\n");
- /* Use the real event to indicate the error */
- VHA_REG_SET_PARITY_ERROR(status);
- dev_info(dev, "status: 0x%016llx!\n", status);
- }
- }
- #endif
- /* Read RESPONSE_FIFO_WL_ID. */
- wm_cmd_id = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_ID);
- /* Gather and process perf/stats data. */
- if (WM_DBG_MODE_ON(PERF))
- vha->stats.cnn_last_cycles = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_PERF);
- if (WM_DBG_MODE_ON(BAND)) {
- #define GET_MEM_STAT_TRANS(stat, reg) \
- vha->stats.last_mem_stats.stat##_transactions = \
- IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_BW_##reg)
- #define GET_MEM_STAT_WORDS(stat, reg) \
- vha->stats.last_mem_stats.stat##_words = \
- IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_BW_##reg##_WORD)
- GET_MEM_STAT_TRANS(locm_rd, LOCM_RD);
- GET_MEM_STAT_TRANS(locm_wr, LOCM_WR);
- GET_MEM_STAT_TRANS(locm_mwr, LOCM_MWR);
- GET_MEM_STAT_TRANS(socm_rd, SOCM_RD);
- GET_MEM_STAT_TRANS(socm_wr, SOCM_WR);
- GET_MEM_STAT_TRANS(socm_mwr, SOCM_MWR);
- GET_MEM_STAT_TRANS(ddr_rd, DDR_RD);
- GET_MEM_STAT_TRANS(ddr_wr, DDR_WR);
- GET_MEM_STAT_TRANS(ddr_mwr, DDR_MWR);
- GET_MEM_STAT_WORDS(locm_rd, LOCM_RD);
- GET_MEM_STAT_WORDS(locm_wr, LOCM_WR);
- GET_MEM_STAT_WORDS(socm_rd, SOCM_RD);
- GET_MEM_STAT_WORDS(socm_wr, SOCM_WR);
- GET_MEM_STAT_WORDS(ddr_rd, DDR_RD);
- GET_MEM_STAT_WORDS(ddr_wr, DDR_WR);
- #undef GET_MEM_STAT_TRANS
- #undef GET_MEM_STAT_WORDS
- }
- /* Pop response from RESPONSE_FIFO. */
- IOWRITE64_CR_REGIO(VHA_CR_WM_RESPONSE_FIFO_READ_FIFO_READ_EN,
- WM_RESPONSE_FIFO_READ);
- IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
- VHA_UNLOCK_WM();
- /* Check if id matches the command. */
- if (VHA_CR_GETBITS(WM_RESPONSE_FIFO_WL_ID, WL_ID, wm_cmd_id) !=
- cmd->wm_cmd_id) {
- dev_err(vha->dev, "%s: WM%u WL id mismatch for cmd 0x%08x/%u: "
- "0x%04x vs. 0x%04x\n", __func__, id,
- cmd->user_cmd.cmd_id, cmd->session->id, cmd->wm_cmd_id,
- (uint16_t)VHA_CR_GETBITS(
- WM_RESPONSE_FIFO_WL_ID, WL_ID,
- wm_cmd_id));
- /* Indicate WL id mismatch. */
- VHA_REG_SET_WL_ID_MISMATCH_ERROR(status);
- }
- /* Leave only potential errors. */
- status &= wm_rsp_err_events;
- /* Store the latest error status for potential full_reset. */
- if (status)
- rsp_err_status = status;
- #ifdef VHA_SCF
- if (vha->confirm_config_reg)
- vha_handle_conf_status(vha, cmd, &full_reset, &status);
- if (vha->cnn_combined_crc_enable)
- vha_check_crc(vha, cmd, &status);
- #endif
- /* Update per core/WM stats. */
- VHA_INC_WL_STAT(vha, kicks_completed, cmd);
- /* Free command resources. */
- if (!vha->hw_sched_status.assignments[cmd->hw_sched_info.assignment_id].queued)
- vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, cmd, true);
- /* Finally handle the response. */
- vha_handle_wm_response(vha, id, status, irq_status.core_events,
- irq_status.ic_events, &full_reset, &error_flags);
- if (status)
- vha->stats.total_failures++;
- vha->stats.cnn_kicks_completed++;
- } else {
- WARN_ON(1);
- }
- } else {
- /* Ignore or report??? */
- }
- }
- /* If any of processed WLs required full reset, all the WLs being
- * currently processed need to be failed and rolled back.
- * The reset itself will be executed at the end of the handler. */
- if (full_reset)
- vha_handle_sys_failure(vha, rsp_err_status, -EIO, error_flags);
- }
- /* Read core event source mask. */
- /* Debug purpose only ... */
- multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, CORE,
- irq_status.event_source);
- if (multi_src_mask)
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
- if (multi_src_mask & (1 << id)) {
- if (irq_status.core_events[id] & VHA_CORE_ERR_EVENTS)
- dev_err(vha->dev, "%s: Core%d error event has been detected: %llx\n",
- __func__, id, irq_status.core_events[id]);
- }
- /* Read IC event source mask. */
- /* Debug purpose only ... */
- multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, IC,
- irq_status.event_source);
- if (multi_src_mask)
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
- if (multi_src_mask & (1 << id)) {
- if (irq_status.ic_events[id] & VHA_IC_ERR_EVENTS)
- dev_err(vha->dev, "%s: Interconnect%d error event has been detected: %llx\n",
- __func__, id, irq_status.ic_events[id]);
- }
- calibration_end:
- if (full_reset) {
- dev_err(vha->dev, "Performing full system reset due to HW error detection.");
- /* Stop cores and execute the actual full reset finally. */
- ret = vha_dev_stop(vha, true);
- /* Check queues ... */
- vha_chk_cmd_queues(vha, true);
- } else {
- /* Run in BH context! */
- vha_chk_cmd_queues(vha, false);
- }
- #ifdef CONFIG_FAULT_INJECTION
- if (vha->fault_inject & VHA_FI_IRQ_WORKER)
- current->make_it_fail = false;
- #endif
- mutex_unlock(&vha->lock);
- return ret;
- }
- #ifdef CONFIG_VHA_DUMMY
- static int vha_dummy_dev_start(struct vha_dev *vha)
- {
- if (vha->state == VHA_STATE_ON)
- return 0; /* not an error */
- vha->state = VHA_STATE_ON;
- /* Remember the time hw is powered on */
- GETNSTIMEOFDAY(&vha->stats.hw_start);
- return 0;
- }
- static int vha_dummy_dev_stop(struct vha_dev *vha)
- {
- uint64_t tmp = 0;
- struct TIMESPEC now;
- if (vha->state == VHA_STATE_OFF)
- return -1;
- vha->state = VHA_STATE_OFF;
- /* Update the up time of the core */
- GETNSTIMEOFDAY(&now);
- if (get_timespan_us(&vha->stats.hw_start, &now, &tmp)) {
- do_div(tmp, 1000UL);
- vha->stats.uptime_ms += tmp;
- if (vha->stats.uptime_ms)
- vha_update_utilization(vha);
- else
- dev_dbg(vha->dev,
- "%s Too short execution time to calculate utilization!\n",
- __func__);
- } else
- WARN_ON(1);
- return 0;
- }
- #endif
- int vha_dev_start(struct vha_dev *vha)
- {
- int ret = 0;
- uint8_t core_mask;
- uint8_t active_core_mask = vha->full_core_mask;
- int id;
- #if defined(VHA_ENHANCED_APM) && !defined(CONFIG_VHA_DUMMY)
- active_core_mask &= ~vha->hw_sched_status.free_core_mask;
- #endif
- if (vha->do_calibration)
- active_core_mask |= VHA_CALIBRATION_CORE_MASK;
- /* If device disabled & no core active */
- if (vha->state == VHA_STATE_OFF && !vha->active_core_mask) {
- pm_runtime_get_sync(vha->dev);
- dev_dbg(vha->dev, "%s system power up\n", __func__);
- }
- /* Cancel any APM request for active cores that are busy at this point */
- {
- /* Find active cores that are busy and under APM */
- uint8_t apm_core_mask = active_core_mask & vha->apm_core_mask;
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (apm_core_mask & (1 << id))
- cancel_delayed_work(&vha->apm_dworks[id].dwork);
- }
- vha->apm_core_mask &= ~(apm_core_mask);
- }
- /* Find cores that have to be powered on */
- core_mask = (vha->active_core_mask ^ active_core_mask) &
- ~vha->active_core_mask;
- if (core_mask) {
- dev_dbg(vha->dev, "%s core mask:%#x (%#x -> %#x)\n",
- __func__, core_mask, vha->active_core_mask, active_core_mask);
- /////////////// POWER ON //////////////////////////
- img_pdump_printf("-- POWER_ON_BEGIN\n");
- /* Prepare device cores ... */
- ret = vha_dev_prepare_cores(vha, core_mask);
- if (ret) {
- dev_err(vha->dev, "%s: Error preparing device cores!\n", __func__);
- goto error;
- }
- /* Enable device cores clocks */
- vha_dev_enable_clocks(vha, core_mask);
- /* Reset device cores & system for the very first time */
- ret = vha_dev_reset(vha, core_mask,
- vha->active_core_mask ? false : true);
- if (ret){
- dev_err(vha->dev, "%s: Error reseting device cores!\n", __func__);
- goto error;
- }
- /* Enable device cores clocks */
- vha_dev_enable_clocks(vha, core_mask);
- img_pdump_printf("-- POWER_ON_END\n");
- /////////////////////////////////////////////////////
- vha_dev_ready(vha, core_mask,
- vha->active_core_mask ? false : true);
- /* Store actual status about active cores */
- vha->active_core_mask = active_core_mask;
- }
- if (vha->state == VHA_STATE_OFF) {
- /* Call device specific setup */
- vha_dev_setup(vha);
- /* Remember the time device is powered on */
- GETNSTIMEOFDAY(&vha->stats.hw_start);
- vha->state = VHA_STATE_ON;
- #ifdef VHA_SCF
- /* Start the SW watchdog */
- vha_start_swd(vha, 0);
- #endif
- }
- return 0;
- error:
- pm_runtime_put_sync(vha->dev);
- vha->state = VHA_STATE_OFF;
- vha->active_core_mask = 0;
- return ret;
- }
- static int vha_dev_stop_cores(struct vha_dev *vha, uint8_t core_mask, bool reset)
- {
- int ret = 0;
- if (core_mask) {
- /* Store actual status about active cores */
- vha->active_core_mask &= ~core_mask;
- /* Disable events at first */
- vha_dev_disable_events(vha, core_mask,
- vha->active_core_mask ? false : true);
- /////////////// POWER_OFF //////////////////////////
- img_pdump_printf("-- POWER_OFF_BEGIN\n");
- /* Reset core in case of error or pending inference */
- if (reset) {
- ret = vha_dev_reset(vha, core_mask,
- vha->active_core_mask ? false : true);
- if(ret)
- dev_warn(vha->dev,
- "%s: Problem with resetting device cores!\n",
- __func__);
- }
- /* Disable device clocks */
- ret = vha_dev_disable_clocks(vha, core_mask,
- vha->active_core_mask ? false : true);
- if(ret)
- dev_warn(vha->dev,
- "%s: Problem with disabling clocks for cores!\n",
- __func__);
- /* Execute any outstanding routines to flush the device cores */
- ret = vha_dev_flush_cores(vha, core_mask);
- if(ret)
- dev_warn(vha->dev,
- "%s: Problem with flushing device cores!\n",
- __func__);
- img_pdump_printf("-- POWER_OFF_END\n");
- /////////////////////////////////////////////////////
- }
- /* If device enabled & no core active */
- if (vha->state == VHA_STATE_ON && !vha->active_core_mask) {
- int id;
- /* Cancel APM requests if we are about to power off the device */
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
- cancel_delayed_work(&vha->apm_dworks[id].dwork);
- vha->apm_core_mask = 0;
- dev_dbg(vha->dev, "%s system power down\n", __func__);
- vha->state = VHA_STATE_OFF;
- /* Update the up time of the device */
- if (!vha->do_calibration) {
- uint64_t tmp = 0;
- struct TIMESPEC now;
- GETNSTIMEOFDAY(&now);
- if (get_timespan_us(&vha->stats.hw_start, &now, &tmp)) {
- do_div(tmp, 1000UL);
- vha->stats.uptime_ms += tmp;
- if (vha->stats.uptime_ms)
- vha_update_utilization(vha);
- else
- dev_dbg(vha->dev,
- "%s Too short execution time to calculate utilization!\n",
- __func__);
- } else
- WARN_ON(1);
- }
- vha->active_mmu_ctx = VHA_INVALID_ID;
- spin_lock_irq(&vha->irq_lock);
- memset(&vha->irq_status, 0, sizeof(vha->irq_status));
- spin_unlock_irq(&vha->irq_lock);
- if (reset) {
- pm_runtime_mark_last_busy(vha->dev);
- pm_runtime_put_sync_autosuspend(vha->dev);
- } else {
- pm_runtime_put_sync(vha->dev);
- }
- }
- return ret;
- }
- int vha_dev_stop(struct vha_dev *vha, bool reset)
- {
- int ret = 0;
- uint8_t active_core_mask = 0;
- uint8_t core_mask;
- #if defined(VHA_ENHANCED_APM) && !defined(CONFIG_VHA_DUMMY)
- active_core_mask = vha->full_core_mask &
- ~vha->hw_sched_status.free_core_mask;
- #endif
- if (vha->do_calibration)
- active_core_mask &= ~VHA_CALIBRATION_CORE_MASK;
- /* Find cores that have to be powered off */
- core_mask = (vha->active_core_mask ^ active_core_mask) &
- vha->active_core_mask;
- if (core_mask)
- dev_dbg(vha->dev, "%s core mask:%#x (%#x -> %#x)\n",
- __func__, core_mask, vha->active_core_mask, active_core_mask);
- ret = vha_dev_stop_cores(vha, core_mask, reset);
- return ret;
- }
- static bool vha_is_mmu_ctx_shared(struct vha_cmd *cmd)
- {
- struct vha_session *session = cmd->session;
- struct vha_dev *vha = session->vha;
- /* If the session of the command we are trying to execute shares
- * the hw mmu ctx with different session */
- if (vha->mmu_ctxs[session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id] > 1) {
- uint8_t id;
- /* Check currently processed commands */
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- /* Check if the mmu hw context is same as current command */
- if (vha->pendcmd[id].cmd != NULL &&
- vha->pendcmd[id].cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id ==
- session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id)
- return true;
- }
- }
- return false;
- }
- int vha_dev_schedule_cmd(struct vha_dev *vha, struct vha_cmd *cmd)
- {
- struct vha_hw_sched_status *status = &vha->hw_sched_status;
- struct vha_hw_sched_info *info;
- struct vha_user_cnn_submit_multi_cmd* user_cmd =
- (struct vha_user_cnn_submit_multi_cmd*)&cmd->user_cmd;
- uint8_t wm_id;
- uint8_t core_id = 0;
- uint8_t core_mask = 0;
- uint8_t assignment_id;
- uint8_t i;
- /* If no command provided, just check if anything can potentially
- * be scheduled. */
- if (cmd == NULL) {
- /* Calculate the number of cores in use. */
- uint8_t num_used_cores = 0;
- uint8_t assignment_id;
- for (assignment_id = 0; assignment_id < VHA_NUM_CORES; assignment_id++)
- if (status->assignments[assignment_id].core_mask) {
- uint8_t num_cores = VHA_CORE_MASK_TO_NUM(status->assignments[assignment_id].core_mask);
- num_used_cores += num_cores;
- if (vha->low_latency != VHA_LL_DISABLED)
- if (status->assignments[assignment_id].queued)
- num_used_cores += num_cores;
- }
- /* If all the cores are in use, nothing can be scheduled. */
- if (num_used_cores ==
- (vha->hw_props.num_cnn_core_devs * ((vha->low_latency != VHA_LL_DISABLED) ? 2 : 1)))
- return -1;
- return 0;
- }
- if (cmd->user_cmd.cmd_type != VHA_CMD_CNN_SUBMIT_MULTI)
- return 0;
- #define VHA_LL_BRANCH(l) \
- { \
- if (vha->low_latency == VHA_LL_DISABLED) \
- return -1; \
- else \
- goto l; \
- }
- /* Check for shared mmu hardware context, as we can't schedule command
- * on free cores, while other currently processing cores use the same
- * mmu hw context, because data would be overwritten */
- if (vha_is_mmu_ctx_shared(cmd)) {
- dev_dbg(vha->dev, "%s: Postpone command due to shared mmu context!\n",
- __func__);
- return -1;
- }
- info = &cmd->hw_sched_info;
- /* If external scheduling is requested... */
- if (vha->scheduling_sequence_len > 0) {
- uint8_t wm_mask;
- /* Queueing is not supported for external scheduling. */
- if (status->num_cores_free < user_cmd->num_cores)
- return -1;
- /* Read scheduling data for this workload from scheduling sequence. */
- wm_id = SCHED_SEQ_GET_WM(vha->scheduling_counter);
- wm_mask = VHA_WM_ID_TO_MASK(wm_id);
- core_mask = SCHED_SEQ_GET_CORES(vha->scheduling_counter);
- /* Sanity check the data. */
- if (((status->free_wm_mask & wm_mask) == 0) ||
- ((status->free_core_mask & core_mask) != core_mask))
- return -1;
- status->free_core_mask &= ~(core_mask);
- /* Increment scheduling counter. */
- vha->scheduling_counter =
- (vha->scheduling_counter + 1) %
- vha->scheduling_sequence_len;
- } else {
- /* Check if there are cores available. */
- if (status->num_cores_free < user_cmd->num_cores)
- VHA_LL_BRANCH(attempt_to_queue_multi);
- /* Check if there is a WM available. */
- if (status->num_wms_free == 0)
- VHA_LL_BRANCH(attempt_to_queue_multi);
- /* Find a free WM. */
- wm_id = ffs(status->free_wm_mask) - 1;
- /* Check if enough cores are available. */
- if (user_cmd->num_cores > status->num_cores_free)
- VHA_LL_BRANCH(attempt_to_queue_multi);
- /* Find the required number of free cores. */
- for (i = 0; i < user_cmd->num_cores; i++) {
- core_id = ffs(status->free_core_mask) - 1;
- core_mask |= VHA_CORE_ID_TO_MASK(core_id);
- status->free_core_mask &= ~(core_mask);
- }
- }
- /* Update resource status. */
- for (assignment_id = 0; assignment_id < VHA_NUM_CORES; assignment_id++)
- if (status->assignments[assignment_id].core_mask == 0)
- break;
- if (assignment_id == VHA_NUM_CORES) {
- dev_info(vha->dev, "%s: Scheduling data inconsistency detected!\n", __func__);
- return -1;
- }
- status->assignments[assignment_id].assignment_id = assignment_id;
- status->assignments[assignment_id].wm_id = wm_id;
- status->assignments[assignment_id].core_mask = core_mask;
- status->num_cores_free -= user_cmd->num_cores;
- status->num_wms_free--;
- status->free_wm_mask &= ~(VHA_WM_ID_TO_MASK(wm_id));
- /* Store command scheduling info. */
- *info = status->assignments[assignment_id];
- goto skip_label_attempt_to_queue_multi;
- attempt_to_queue_multi:
- /* Check if there is an assignment matching this scheduling request
- * available in the list of assignments. */
- for (assignment_id = 0; assignment_id < VHA_NUM_CORES; assignment_id++)
- /* If this assignment is not queued already and it has the same
- * number of cores as this scheduling request... */
- if (!status->assignments[assignment_id].queued &&
- status->assignments[assignment_id].core_mask &&
- (VHA_CORE_MASK_TO_NUM(status->assignments[assignment_id].core_mask) ==
- user_cmd->num_cores)) {
- wm_id = status->assignments[assignment_id].wm_id;
- if (vha->low_latency == VHA_LL_SELF_KICK
- /* If the current command we are trying to queue belongs
- * to a different session than the pending one. */
- && (vha->pendcmd[wm_id].cmd != NULL &&
- vha->pendcmd[wm_id].cmd->session != cmd->session)
- /* If the session of the command we are trying to queue shares
- * the hw mmu ctx with the session of pending cmd */
- && (cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id ==
- vha->pendcmd[wm_id].cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id)
- /* Sanity if hw mmu ctx is really shared at this point. */
- && (vha->mmu_ctxs[cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id] > 1)
- ) {
- /* Skip this assignment. */
- continue;
- }
- /* Make the assignment queued. */
- status->assignments[assignment_id].queued = true;
- /* Store command scheduling info. */
- *info = status->assignments[assignment_id];
- break;
- }
- /* Fail if no matching assignments found. */
- if (assignment_id == VHA_NUM_CORES)
- return -1;
- skip_label_attempt_to_queue_multi:
- /* For hw commands... */
- if (CMD_EXEC_ON_HW(cmd)) {
- if (!VHA_IS_DUMMY(vha)) {
- int tries = 3; /* magic number, just try harder to start the device */
- /* Start device. */
- while(tries--) {
- if (vha_dev_start(vha))
- dev_warn(vha->dev, "%s: Error starting device cores. Try once again.", __func__);
- else
- break;
- }
- }
- #ifdef CONFIG_VHA_DUMMY
- else
- vha_dummy_dev_start(vha);
- #endif
- }
- #undef VHA_LL_BRANCH
- dev_dbg(vha->dev, "%s: cmd 0x%08x/%u scheduled on WM%u/core(s) 0x%02x\n",
- __func__, cmd->user_cmd.cmd_id, cmd->session->id,
- info->wm_id, info->core_mask);
- return 0;
- }
- void vha_dev_free_cmd_res(struct vha_dev *vha, struct vha_cmd *cmd, bool update_stats)
- {
- struct vha_hw_sched_status *status = &vha->hw_sched_status;
- struct vha_hw_sched_info *info = &cmd->hw_sched_info;
- struct vha_user_cnn_submit_multi_cmd* user_cmd =
- (struct vha_user_cnn_submit_multi_cmd*)&cmd->user_cmd;
- if (update_stats) {
- uint64_t proc_time = 0;
- struct TIMESPEC *from = &cmd->hw_proc_start;
- struct TIMESPEC *to = &vha->stats.wm_stats[info->wm_id].hw_proc_end;
- if (TIMESPEC_COMPARE(&vha->stats.wm_stats[info->wm_id].hw_proc_end_prev,
- &cmd->hw_proc_start) >= 0)
- from = &vha->stats.wm_stats[info->wm_id].hw_proc_end_prev;
- if (get_timespan_us(from, to, &proc_time)) {
- vha->stats.last_proc_us = proc_time;
- } else {
- vha->stats.last_proc_us = 0;
- }
- /* Update WL stats. */
- VHA_UPDATE_WL_STAT(vha, total_proc_us, cmd, vha->stats.last_proc_us);
- /* Update common stats. */
- vha_cnn_update_stats(vha);
- }
- /* If assignment for this workload is queued... */
- if (status->assignments[info->assignment_id].queued) {
- /* Just mark it as not queued again. */
- status->assignments[info->assignment_id].queued = false;
- /* Clear scheduling info for this workload. */
- info->freed = true;
- /* Do not update the scheduling status. */
- return;
- }
- /* Update the scheduling status. */
- status->num_cores_free += user_cmd->num_cores;
- status->free_core_mask |= info->core_mask;
- status->num_wms_free++;
- status->free_wm_mask |= VHA_WM_ID_TO_MASK(info->wm_id);
- /* Clear the assignment and scheduling info for this workload. */
- memset(&status->assignments[info->assignment_id], 0,
- sizeof(struct vha_hw_sched_info));
- info->freed = true;
- }
- static void sched_apm_multi(struct vha_dev *vha)
- {
- struct vha_apm_work *apm_work = NULL;
- /* Find active cores that are free and not under APM */
- uint8_t apm_core_mask = vha->active_core_mask &
- vha->hw_sched_status.free_core_mask &
- ~vha->apm_core_mask;
- int id;
- /* Skip if nothing has changed */
- if (!apm_core_mask)
- return;
- dev_dbg(vha->dev, "%s core mask:%#x\n", __func__, apm_core_mask);
- /* Schedule for all cores, separately */
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (apm_core_mask & (1 << id)) {
- apm_work = &vha->apm_dworks[id];
- apm_work->core_mask = 1 << id;
- apm_work->delay_ms = vha->pm_delay;
- vha_sched_apm(vha, apm_work);
- }
- }
- /* Record actual status */
- vha->apm_core_mask |= apm_core_mask;
- }
- //#define MTSTK_MEASURE_MULTI_PRI
- #ifdef MTSTK_MEASURE_MULTI_PRI
- static uint8_t get_active_pri_count(struct vha_dev *vha)
- {
- uint8_t pris;
- uint8_t pri_count = 0;
- /* Calculate the number of priority levels with anything to schedule. */
- for (pris = 0; pris < VHA_MAX_PRIORITIES; pris++)
- if (vha->pri_q_counters[pris] > 0) {
- pri_count++;
- }
- return pri_count;
- }
- #endif
- static void vha_get_time_span(struct TIMESPEC *start, struct TIMESPEC *end,
- struct TIMESPEC *span)
- {
- if (start == NULL || end == NULL || span == NULL)
- return;
- /* Calculate the seconds span. */
- span->tv_sec = end->tv_sec - start->tv_sec;
- /* If there more than a second span, move one second to nanoseconds
- * to avoid potential negative nanosecond values. */
- if (span->tv_sec > 0) {
- span->tv_sec--;
- end->tv_nsec += 1000000000;
- }
- /* Calculate the nanoseconds span. */
- span->tv_nsec = end->tv_nsec - start->tv_nsec;
- /* If nanoseconds include a second, move it to seconds. */
- if (span->tv_nsec > 1000000000) {
- span->tv_sec++;
- span->tv_nsec -= 1000000000;
- }
- }
- static uint8_t vha_scheduler_get_priority(struct vha_dev *vha)
- {
- uint8_t ret_pri = 0;
- uint8_t pris;
- uint8_t pri_count = 0;
- uint32_t curr_window = 0;
- uint32_t curr_limit = 0;
- uint32_t rand_val;
- /* Calculate current total window width. */
- for (pris = 0; pris < VHA_MAX_PRIORITIES; pris++)
- if (vha->pri_q_counters[pris] > 0) {
- curr_window += pri_windows[pris];
- ret_pri = pris;
- pri_count++;
- }
- /* If there's no priority with WLs to schedule, just return 0. */
- if (pri_count == 0)
- return VHA_INVALID_PRI;
- /* If there's only one priority with WLs to schedule, just return it. */
- if (pri_count == 1)
- return ret_pri;
- /* If starvation avoidance is disabled, just return the highest priority
- * with WLs to schedule. */
- if (curr_window == 0)
- return ret_pri;
- /* If starvation avoidance is enabled, use 'lottery' based approach. */
- /* Generate random value within the current window. */
- vha_mt19937_gen_range(vha->hw_sched_status.sched_data->rand_gen_handle,
- 0, curr_window, &rand_val);
- /* Choose priority based on the value generated and available priorities. */
- for (pris = 0; pris < VHA_MAX_PRIORITIES; pris++)
- if (vha->pri_q_counters[pris] > 0) {
- curr_limit += pri_windows[pris];
- if (rand_val <= curr_limit) {
- ret_pri = pris;
- break;
- }
- }
- return ret_pri;
- }
- static void vha_scheduler_set_starting_session(struct vha_dev *vha,
- uint8_t priority, struct vha_session *session)
- {
- /* Set a starting point session for next scheduling round. */
- if (session != list_entry(&vha->sched_sessions[priority],
- struct vha_session, sched_list[priority]))
- while(list_first_entry(&vha->sched_sessions[priority],
- struct vha_session, sched_list[priority]) != session)
- list_rotate_left(&vha->sched_sessions[priority]);
- }
- void vha_scheduler_loop(struct vha_dev *vha)
- {
- struct vha_cmd *cmd, *tmp;
- struct vha_session *session = NULL;
- bool scheduled = false;
- enum do_cmd_status cmd_status = CMD_OK;
- uint8_t current_pri = VHA_DEFAULT_PRI;
- bool log_pri_sched_info = true;
- if (vha_dev_schedule_cmd(vha, NULL) != 0) {
- /* Postpone worker task if nothing can be scheduled. */
- dev_dbg(vha->dev, "%s Nothing can be scheduled at the moment. "
- "Postpone worker task!\n", __func__);
- return;
- }
- #ifdef MTSTK_MEASURE_MULTI_PRI
- log_pri_sched_info = (get_active_pri_count(vha) > 1) ? true : false;
- #endif
- /* Main scheduling loop. */
- do {
- scheduled = false;
- current_pri = vha_scheduler_get_priority(vha);
- if (current_pri == VHA_INVALID_PRI)
- break;
- list_for_each_entry(session, &vha->sched_sessions[current_pri], sched_list[current_pri]) {
- list_for_each_entry_safe(cmd, tmp, &session->cmds[current_pri], list[current_pri]) {
- #if defined(VHA_ENHANCED_APM)
- /* Schedule APM/power down cores in the middle if possible */
- if (!VHA_IS_DUMMY(vha)) {
- if (!vha->no_clock_disable) {
- if (!vha->pm_delay) {
- if (vha_dev_stop(vha, false)) {
- dev_warn(vha->dev, "%s: Failed to soft stop device. Trying harder with reset",
- __func__);
- if (vha_dev_stop(vha, true))
- dev_err(vha->dev, "%s: Failed to stop device with reset!", __func__);
- }
- } else
- sched_apm_multi(vha);
- }
- }
- #endif
- /* Skip this workload as it's already scheduled. */
- if (cmd->hw_sched_info.core_mask && !cmd->hw_sched_info.freed)
- continue;
- /* Attempt to schedule command for execution. */
- cmd_status = vha_do_cmd(cmd);
- if ((cmd_status == CMD_OK) || (cmd_status == CMD_HW_BUSY)) {
- if (cmd_status == CMD_OK) {
- scheduled = true;
- if (log_pri_sched_info && !cmd->rolled_back) {
- struct TIMESPEC sched_ts, sched_span = {0};
- GETNSTIMEOFDAY(&sched_ts);
- vha_get_time_span(&cmd->submit_ts, &sched_ts, &sched_span);
- #ifdef LOG_PRI_SCHEDULING_INFO
- dev_info(vha->dev, "@@@ scheduled 0x%08x/%u/%u, span: %llu\n",
- cmd->user_cmd.cmd_id, session->id, cmd->user_cmd.priority,
- (uint64_t)sched_span.tv_sec * 1000000000ULL +
- (uint64_t)sched_span.tv_nsec);
- #endif
- VHA_UPDATE_SCHED_STAT_MTSTK(vha, cmd, &sched_span);
- }
- session = list_next_entry(session, sched_list[current_pri]);
- }
- vha_scheduler_set_starting_session(vha, current_pri, session);
- goto exit_session_loop;
- }
- }
- }
- exit_session_loop:;
- /* Iterate until a workload was scheduled and no other can be scheduled. */
- } while (vha_dev_schedule_cmd(vha, NULL) == 0 && scheduled);
- /* Schedule APM/power down cores if possible at end */
- if (!VHA_IS_DUMMY(vha)) {
- bool skip = vha->no_clock_disable;
- #if !defined(VHA_ENHANCED_APM)
- skip |= vha_is_busy(vha);
- #endif
- if (!skip) {
- if (!vha->pm_delay) {
- if (vha_dev_stop(vha, false)) {
- dev_warn(vha->dev, "%s: Failed to soft stop device. Trying harder with reset",
- __func__);
- if (vha_dev_stop(vha, true))
- dev_err(vha->dev, "%s: Failed to stop device with reset!", __func__);
- }
- } else
- sched_apm_multi(vha);
- }
- }
- #ifdef CONFIG_VHA_DUMMY
- else if (!vha_is_busy(vha))
- vha_dummy_dev_stop(vha);
- #endif
- }
- bool vha_rm_session_cmds(struct vha_session *session)
- {
- struct vha_dev *vha = session->vha;
- bool reschedule = false;
- uint32_t wm_id;
- struct vha_hw_sched_info sched_info = {0};
- struct vha_cmd *cur_cmd, *tmp_cmd;
- uint8_t pri;
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- bool pend_removed = false;
- bool queued_removed = false;
- /* Check if pend/queued WLs will be removed. */
- if (vha->pendcmd[wm_id].cmd &&
- vha->pendcmd[wm_id].cmd->session == session) {
- dev_warn(vha->dev,
- "Removing a session while cnn cmd is still pending\n");
- pend_removed = true;
- sched_info = vha->pendcmd[wm_id].cmd->hw_sched_info;
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- cancel_delayed_work(&vha->dummy_dworks[wm_id].dummy_dwork);
- #endif
- }
- if (vha->queuedcmd[wm_id].cmd &&
- vha->queuedcmd[wm_id].cmd->session == session) {
- dev_warn(vha->dev,
- "Removing a session while cnn cmd is still queued\n");
- queued_removed = true;
- sched_info = vha->queuedcmd[wm_id].cmd->hw_sched_info;
- }
- /* Update session scheduling. */
- if (vha->queuedcmd[wm_id].cmd &&
- (pend_removed && !queued_removed)) {
- if (vha->queuedcmd[wm_id].cmd->session !=
- list_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
- struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]))
- while(list_first_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
- struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]) !=
- vha->queuedcmd[wm_id].cmd->session)
- list_rotate_left(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority]);
- }
- /* Remove pend/queued WLs if needed. */
- if (pend_removed || queued_removed) {
- uint64_t wm_mask = VHA_CR_SETBITS(HOST_EVENT_SOURCE, WM, VHA_WM_ID_TO_MASK(wm_id));
- /* Reset WM/cores. */
- vha_wm_reset(vha, &sched_info);
- VHA_LOCK_WM();
- VHA_SELECT_WM(wm_id);
- /* Remove WM related interrupt info if it happens to be set. */
- if (vha->irq_status.event_source & wm_mask)
- {
- /* Unset the WM related source bit. */
- vha->irq_status.event_source &= ~wm_mask;
- /* Clear all WM related events. */
- IOWRITE64_CR_REGIO(vha->irq_status.wm_events[wm_id] & VHA_WM_EVENTS_DEFAULT,
- WM_EVENT_CLEAR);
- vha->irq_status.wm_events[wm_id] = 0ULL;
- }
- /* Re-enable WM events here as this WM will not be handled further. */
- IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
- VHA_UNLOCK_WM();
- /* Rollback all WLs from this WM. */
- vha_rollback_wm_cmds(vha, wm_id, true);
- /* Need to reschedule too. */
- reschedule = true;
- }
- }
- /* Remove session related commands. */
- for (pri = 0; pri < VHA_MAX_PRIORITIES; pri++) {
- list_for_each_entry_safe(cur_cmd, tmp_cmd, &session->cmds[pri], list[pri]) {
- /* rsp didn't make it to rsps list, free it now */
- kfree(cur_cmd->rsp);
- list_del(&cur_cmd->list[cur_cmd->user_cmd.priority]);
- vha->pri_q_counters[cur_cmd->user_cmd.priority]--;
- kfree(cur_cmd);
- }
- }
- return reschedule;
- }
- bool vha_rm_session_cmds_masked(struct vha_session *session, uint32_t cmd_id,
- uint32_t cmd_id_mask)
- {
- struct vha_dev *vha = session->vha;
- bool reschedule = false;
- uint32_t wm_id;
- struct vha_hw_sched_info sched_info = {0};
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- bool pend_removed = false;
- bool queued_removed = false;
- /* Check if pend/queued WLs will be removed. */
- if (vha->pendcmd[wm_id].cmd &&
- (vha->pendcmd[wm_id].cmd->session == session) &&
- (vha->pendcmd[wm_id].cmd->user_cmd.cmd_id & cmd_id_mask)
- == cmd_id) {
- pend_removed = true;
- sched_info = vha->pendcmd[wm_id].cmd->hw_sched_info;
- #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
- cancel_delayed_work(&vha->dummy_dworks[wm_id].dummy_dwork);
- #endif
- VHA_INC_WL_STAT(vha, kicks_cancelled, vha->pendcmd[wm_id].cmd);
- vha->stats.cnn_kicks_cancelled++;
- }
- if (vha->queuedcmd[wm_id].cmd &&
- (vha->queuedcmd[wm_id].cmd->session == session) &&
- (vha->queuedcmd[wm_id].cmd->user_cmd.cmd_id & cmd_id_mask)
- == cmd_id) {
- sched_info = vha->queuedcmd[wm_id].cmd->hw_sched_info;
- queued_removed = true;
- if (vha->low_latency == VHA_LL_SELF_KICK) {
- VHA_INC_WL_STAT(vha, kicks_cancelled, vha->queuedcmd[wm_id].cmd);
- vha->stats.cnn_kicks_cancelled++;
- }
- }
- /* Update session scheduling. */
- if (vha->queuedcmd[wm_id].cmd &&
- (pend_removed && !queued_removed)) {
- if (vha->queuedcmd[wm_id].cmd->session !=
- list_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
- struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]))
- while(list_first_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
- struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]) !=
- vha->queuedcmd[wm_id].cmd->session)
- list_rotate_left(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority]);
- }
- /* Remove pend/queued WLs if needed. */
- if (pend_removed || queued_removed) {
- uint64_t wm_mask = VHA_CR_SETBITS(HOST_EVENT_SOURCE, WM, VHA_WM_ID_TO_MASK(wm_id));
- /* Reset WM/cores. */
- vha_wm_reset(vha, &sched_info);
- VHA_LOCK_WM();
- VHA_SELECT_WM(wm_id);
- /* Remove WM related interrupt info if it happens to be set. */
- if (vha->irq_status.event_source & wm_mask)
- {
- /* Unset the WM related source bit. */
- vha->irq_status.event_source &= ~wm_mask;
- /* Clear all WM related events. */
- IOWRITE64_CR_REGIO(vha->irq_status.wm_events[wm_id] & VHA_WM_EVENTS_DEFAULT,
- WM_EVENT_CLEAR);
- vha->irq_status.wm_events[wm_id] = 0ULL;
- }
- /* Re-enable WM events here as this WM will not be handled further. */
- IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
- VHA_UNLOCK_WM();
- /* Rollback all WLs from this WM. */
- vha_rollback_wm_cmds(vha, wm_id, true);
- /* Correct aborted stats. */
- if (queued_removed) {
- VHA_UPDATE_WM_STAT(vha, kicks_aborted, sched_info.wm_id, -1);
- VHA_UPDATE_CORE_GROUP_STAT(vha, kicks_aborted, sched_info.core_mask, -1);
- vha->stats.cnn_kicks_aborted--;
- }
- if (pend_removed) {
- VHA_UPDATE_WM_STAT(vha, kicks_aborted, sched_info.wm_id, -1);
- VHA_UPDATE_CORE_GROUP_STAT(vha, kicks_aborted, sched_info.core_mask, -1);
- vha->stats.cnn_kicks_aborted--;
- }
- reschedule = true;
- }
- }
- return reschedule;
- }
- int vha_rm_cmds(struct vha_session *session, uint32_t cmd_id,
- uint32_t cmd_id_mask, bool respond)
- {
- struct vha_dev *vha = session->vha;
- struct vha_cmd *cur_cmd, *tmp_cmd;
- struct vha_rsp *cur_rsp, *tmp_rsp;
- bool reschedule = false;
- bool respond_aux = false;
- int ret = 0;
- uint8_t pri;
- mutex_lock(&vha->lock);
- /* Remove pend/queued session commands that match the cmd_id. */
- reschedule = vha_rm_session_cmds_masked(session, cmd_id, cmd_id_mask);
- /* Remove session related commands matching command id template. */
- for (pri = 0; pri < VHA_MAX_PRIORITIES; pri++) {
- list_for_each_entry_safe(cur_cmd, tmp_cmd, &session->cmds[pri], list[pri]) {
- if ((cur_cmd->user_cmd.cmd_id & cmd_id_mask) == cmd_id) {
- #ifdef KERNEL_DMA_FENCE_SUPPORT
- switch (cur_cmd->user_cmd.cmd_type)
- {
- case VHA_CMD_CNN_SUBMIT_MULTI:
- {
- struct vha_user_cnn_submit_multi_cmd *cnn_cmd =
- (struct vha_user_cnn_submit_multi_cmd *)&cur_cmd->user_cmd;
- int j;
- for (j = 0; j < (cnn_cmd->msg.num_bufs - cnn_cmd->num_cores); j++) {
- struct vha_buffer *buf = vha_find_bufid(session, cnn_cmd->bufs[j]);
- if (buf == NULL) {
- dev_warn(vha->dev, "%s: could not find buf %x\n", __func__,
- cnn_cmd->bufs[j]);
- } else {
- vha_rm_buf_fence(session, buf);
- }
- }
- break;
- }
- default:
- dev_warn(vha->dev, "%s: invalid cmd type %x\n", __func__,
- cur_cmd->user_cmd.cmd_type);
- break;
- }
- #endif
- /* rsp didn't make it to rsps list; free it now. */
- kfree(cur_cmd->rsp);
- list_del(&cur_cmd->list[cur_cmd->user_cmd.priority]);
- vha->pri_q_counters[cur_cmd->user_cmd.priority]--;
- kfree(cur_cmd);
- /* There were commands matching command id template in the list,
- * so respond to wake user space. */
- respond_aux = true;
- }
- }
- }
- /* Remove responses for session related commands
- * matching command id template. */
- list_for_each_entry_safe(cur_rsp, tmp_rsp, &session->rsps, list) {
- if ((cur_rsp->user_rsp.cmd_id & cmd_id_mask) == cmd_id) {
- list_del(&cur_rsp->list);
- kfree(cur_rsp);
- respond_aux = true;
- }
- }
- /* Reset hardware if required. */
- if (reschedule)
- ret = vha_dev_stop(vha, reschedule);
- /* Generate "cancel" response if any commands matching command id template
- * were removed. */
- if (respond_aux && respond) {
- /* Calculate space for the response. */
- size_t sz = sizeof(struct vha_rsp)
- + sizeof(struct vha_user_cnn_submit_rsp)
- - sizeof(struct vha_user_rsp);
- /* Allocate space for standard response. */
- struct vha_rsp *rsp = kzalloc(sz, GFP_KERNEL);
- if (rsp == NULL) {
- dev_crit(session->vha->dev,
- "Failed to allocate memory to notify cancel for cmds 0x%08x\n", cmd_id);
- session->oom = true;
- } else {
- rsp->size = sizeof(struct vha_user_cnn_submit_rsp);
- rsp->user_rsp.cmd_id = cmd_id;
- list_add_tail(&rsp->list, &session->rsps);
- }
- wake_up(&session->wq);
- }
- mutex_unlock(&vha->lock);
- /* Just return in case of oom. */
- if (session->oom)
- return -ENOMEM;
- /* Reschedule once all commands matching command id template are removed. */
- if (reschedule)
- vha_chk_cmd_queues(vha, true);
- return ret;
- }
- bool vha_is_busy(struct vha_dev *vha)
- {
- return (vha->hw_sched_status.num_cores_free < vha->hw_props.num_cnn_core_devs);
- }
- /* check all input buffers are filled and ready to go */
- bool vha_is_waiting_for_inputs(struct vha_session *session,
- struct vha_cmd *cmd)
- {
- if (!cmd->inbufs_ready) {
- const struct vha_user_cnn_submit_multi_cmd *user_cmd =
- (struct vha_user_cnn_submit_multi_cmd *)&cmd->user_cmd;
- int i;
- for (i = 0; i < cmd->user_cmd.num_inbufs - user_cmd->num_cores; i++) {
- struct vha_buffer *buf = vha_find_bufid(session, user_cmd->bufs[i]);
- if (buf && buf->status == VHA_BUF_UNFILLED) {
- dev_dbg(session->vha->dev,
- "%s: cmd %u waiting for input "
- "buf %d to be ready\n",
- __func__,
- cmd->user_cmd.cmd_id,
- buf->id);
- return true;
- }
- }
- }
- cmd->inbufs_ready = true;
- return false;
- }
- void vha_dev_apm_stop(struct vha_dev *vha, struct vha_apm_work *apm_work)
- {
- /* Find active cores that are not busy and under APM for this apm request */
- uint8_t apm_core_mask = vha->active_core_mask &
- vha->hw_sched_status.free_core_mask &
- vha->apm_core_mask &
- apm_work->core_mask;
- vha->apm_core_mask &= ~apm_core_mask;
- if (vha->do_calibration)
- return;
- if (apm_core_mask) {
- dev_dbg(vha->dev, "%s core mask:%#x\n", __func__, apm_core_mask);
- if (vha_dev_stop_cores(vha, apm_core_mask, false)) {
- dev_warn(vha->dev, "%s: Failed to soft stop cores. Trying harder with reset",
- __func__);
- if (vha_dev_stop_cores(vha, apm_core_mask, true))
- dev_err(vha->dev, "%s: Failed to stop cores with reset!", __func__);
- }
- }
- }
- int vha_dev_get_props(struct vha_dev *vha, uint32_t onchipmem_size)
- {
- struct vha_hw_props *props = &vha->hw_props;
- uint64_t ip_config;
- uint32_t locm_size_kb = 0;
- uint32_t socm_size_kb = 0;
- uint8_t socm_num_sb, socm_num_ba, socm_num_bg;
- uint8_t ext_mem_bus_width;
- memset(props, 0, sizeof(*props));
- #ifdef CONFIG_VHA_DUMMY
- /* Note: dummy dev always reads zeroes from registers */
- props->product_id = 0x8070605040302010ULL;
- props->core_id = (long)HW_SERIES << (int)VHA_CR_CORE_ID_BRANCH_ID_SHIFT;
- props->core_id += 0x010203040506ULL; // provide a dummy core id
- props->dummy_dev = true;
- props->num_cnn_core_devs = VHA_NUM_CORES;
- #else
- props->product_id = IOREAD64_CR_REGIO(PRODUCT_ID);
- props->core_id = IOREAD64_CR_REGIO(CORE_ID);
- #endif
- props->skip_bvnc_check = false;
- /*
- * New mmu version 3 and onwards operates on 40bit physical & virtual addresses
- */
- props->mmu_width = 40;
- /* HW from 1.1 onwards */
- ip_config = IOREAD64_CR_REGIO(CORE_IP_CONFIG);
- #ifdef HW_AX3
- props->mmu_ver = VHA_CR_GETBITS(CORE_IP_CONFIG, MMU_VERSION, ip_config);
- #endif
- /* Mirage uses MMU version 3 hardware */
- if (!props->mmu_ver)
- props->mmu_ver = 3;
- ;
- /* Read num cores supported (number of WMs must be the same). */
- if (VHA_CR_GETBITS(CORE_IP_CONFIG, CNN_SUPPORTED, ip_config)) {
- uint64_t ip_config1 = IOREAD64_CR_REGIO(CORE_IP_CONFIG1);
- props->num_cnn_core_devs =
- 1 + VHA_CR_GETBITS(CORE_IP_CONFIG1, NUM_CORES_MIN1, ip_config1);
- }
- if (VHA_CR_GETBITS(CORE_IP_CONFIG, RTM_SUPPORTED, ip_config))
- props->supported.rtm = 1;
- #ifdef HW_AX3
- if (VHA_CR_GETBITS(CORE_IP_CONFIG, PARITY_REGISTERS, ip_config))
- props->supported.parity = 1;
- #if defined(CONFIG_VHA_DUMMY) && defined(VHA_SCF)
- /* Force parity for pdump generation */
- props->supported.parity = 1;
- #endif
- #endif
- if ((props->num_cnn_core_devs == 0)
- || VHA_CR_GETBITS(CORE_ID, BRANCH_ID, props->core_id) != HW_SERIES) {
- dev_err(vha->dev, "%s: Wrong core configuration detected. "
- "Expected BVNC %d.x.x.x, got %llu.x.x.x. "
- "Maybe kernel module was built with wrong params.\n",
- __func__, HW_SERIES,
- VHA_CR_GETBITS(CORE_ID, BRANCH_ID, props->core_id));
- return -ENODEV;
- }
- dev_info(vha->dev, "%s: Product id: %#llx\n",
- __func__, props->product_id);
- dev_info(vha->dev, "%s: Core id: %#llx\n",
- __func__, props->core_id);
- dev_info(vha->dev, "%s: MMU version:%d (%dbit)\n",
- __func__, props->mmu_ver, props->mmu_width);
- dev_dbg(vha->dev, "%s: supported: %#x\n",
- __func__, props->features);
- {
- uint64_t tmp = IOREAD64_CR_REGIO(CORE_IP_INTEGRATOR_ID);
- dev_dbg(vha->dev, "%s: ip integrator id: %#llx\n",
- __func__, tmp);
- tmp = IOREAD64_CR_REGIO(CORE_IP_CHANGELIST);
- dev_dbg(vha->dev, "%s: ip change list: %llu\n", __func__, tmp);
- }
- /* Read OCM info */
- {
- uint64_t ip_config1 = IOREAD64_CR_REGIO(CORE_IP_CONFIG1);
- /* Power of 2 Look-up table */
- uint8_t pow_2_lut[8] = { 1, 2, 4, 8, 16, 32, 64, 128 };
- /* LOCM per core size */
- locm_size_kb = VHA_CR_GETBITS(CORE_IP_CONFIG1, CORE_OCM_RAM_SIZE_4KB, ip_config1) * 4;
- /* SOCM total size */
- socm_size_kb = VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_RAM_SIZE_4KB, ip_config1) * 4;
- /* SOCM number of subbanks per bank array which is stored in hw reg as Log2 */
- socm_num_sb = pow_2_lut[VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_NUM_SUBBANKS_LOG2, ip_config1)];
- /* SOCM number of arrays per bank group */
- socm_num_ba = 1 + VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_NUM_BANK_ARRAYS_MIN1, ip_config1);
- /* SOCM number of bank groups */
- socm_num_bg = 1 + VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_NUM_BANK_GROUPS_MIN1, ip_config1);
- /* External memory interface width which is stored in hw reg as 8 * Log2 */
- ext_mem_bus_width = 8 * pow_2_lut[VHA_CR_GETBITS(CORE_IP_CONFIG1, EXT_MEM_BUS_WIDTH, ip_config1)];
- }
- if (locm_size_kb) {
- props->locm_size_bytes = locm_size_kb * 1024;
- /* User may wanted to limit local OCM ... */
- if (onchipmem_size) {
- if (onchipmem_size < props->locm_size_bytes) {
- dev_warn(vha->dev, "%s:Limiting local onchip memory to %u bytes (available:%u)\n",
- __func__, onchipmem_size, props->locm_size_bytes);
- props->locm_size_bytes = onchipmem_size;
- } else if (onchipmem_size > props->locm_size_bytes) {
- dev_warn(vha->dev, "%s: User defined local onchip memory size exceeded (%u > %u))\n",
- __func__, onchipmem_size, props->locm_size_bytes);
- }
- }
- } else {
- props->locm_size_bytes = onchipmem_size;
- }
- if (socm_size_kb) {
- props->socm_size_bytes = socm_size_kb * 1024;
- /* User may wanted to limit shared OCM ... */
- if (shared_onchipmem_size) {
- if (shared_onchipmem_size < props->socm_size_bytes) {
- dev_warn(vha->dev, "%s:Limiting shared onchip memory to %u bytes (available:%u)\n",
- __func__, shared_onchipmem_size, props->socm_size_bytes);
- props->socm_size_bytes = shared_onchipmem_size;
- } else if (shared_onchipmem_size > props->socm_size_bytes) {
- dev_warn(vha->dev, "%s: User defined shared onchip memory size exceeded (%u > %u))\n",
- __func__, shared_onchipmem_size, props->socm_size_bytes);
- }
- }
- {
- /* SOCM per core must be must be a multiple of socm_total_sb & ext_mem_bus_width */
- uint16_t socm_total_sb = socm_num_sb * socm_num_ba * socm_num_bg;
- if (socm_total_sb && ext_mem_bus_width) {
- /* The below division will round down */
- props->socm_core_size_bytes = props->socm_size_bytes /
- (props->num_cnn_core_devs * socm_total_sb * ext_mem_bus_width);
- /* Scale it back */
- props->socm_core_size_bytes *= socm_total_sb * ext_mem_bus_width;
- } else {
- /* Divide by number of cores as for dummy driver */
- props->socm_core_size_bytes = shared_onchipmem_size / props->num_cnn_core_devs;
- dev_warn(vha->dev, "%s: Shared onchip memory size per core can't be rounded"
- " based on SB:%d BA:%d BG:%d BW:%d!\n", __func__,
- socm_num_sb, socm_num_ba, socm_num_bg, ext_mem_bus_width);
- }
- }
- } else {
- props->socm_size_bytes = shared_onchipmem_size;
- /* Just divide by number of cores (dummy driver) */
- props->socm_core_size_bytes = shared_onchipmem_size / props->num_cnn_core_devs;
- }
- dev_info(vha->dev, "%s: Total onchip memory, Local: %u [kB], Shared total: %u [kB]"
- " per core: %u [kB]\n", __func__, props->locm_size_bytes / 1024,
- props->socm_size_bytes / 1024, props->socm_core_size_bytes / 1024);
- dev_info(vha->dev, "%s: Devices: DUMMY:%u CNN:%u\n", __func__,
- props->dummy_dev ? props->num_cnn_core_devs : 0,
- props->dummy_dev ? 0 : props->num_cnn_core_devs);
- return 0;
- }
- /* prepare CRC and DEBUG data buffers */
- void vha_dbg_prepare_hwbufs(struct vha_session *session, struct vha_cmd *cmd,
- struct vha_crc_config_regs *regs)
- {
- struct vha_dev *vha = session->vha;
- uint8_t mask = cmd->hw_sched_info.core_mask;
- if (session->cnn_dbg.cnn_crc_buf[0] || vha->cnn_combined_crc_enable) {
- uint8_t id;
- /* Note: all buffers have the same size */
- img_pdump_printf("-- Select cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
- /* enable CRC: address + mode */
- if (session->cnn_dbg.cnn_crc_buf[0])
- regs->crc_control |= VHA_CR_SETBITS(OS0_CNN_CRC_CONTROL, CNN_CRC_ENABLE,
- session->cnn_dbg.cnn_crc_mode);
- if (vha->cnn_combined_crc_enable)
- regs->crc_control |= VHA_CR_SETBITS(OS0_CNN_CRC_CONTROL, COMBINED_CNN_CRC_ENABLE, 1);
- img_pdump_printf("-- CRC_CONTROL=%llx buf 'CRC' size=%zx\n",
- regs->crc_control,
- session->cnn_dbg.cnn_crc_buf[0] ? session->cnn_dbg.cnn_crc_buf[0]->size : 0);
- IOWRITE64_CR_PDUMP(regs->crc_control, OS0_CNN_CRC_CONTROL);
- img_pdump_printf("-- CRC_MASK=%#x\n", session->cnn_dbg.cnn_crc_mask);
- IOWRITE64_CR_PDUMP(session->cnn_dbg.cnn_crc_mask, OS0_CNN_CRC_MASK_CTRL);
- regs->crc_mask_ctrl = session->cnn_dbg.cnn_crc_mask;
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (mask & (1 << id)) {
- /* Select core to be set */
- IOWRITE64_CR_PDUMP(VHA_CR_SETBITS(CORE_CTRL_INDIRECT, MASK, (1 << id)),
- CORE_CTRL_INDIRECT);
- if (session->cnn_dbg.cnn_crc_buf[0]) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_crc_buf[id];
- IOWRITE_PDUMP_BUFADDR(session, buf, 0, VHA_CR_OS0_CNN_CRC_ADDRESS);
- SET_BUFADDR(session, buf, 0, ®s->crc_address[id]);
- }
- if (vha->cnn_combined_crc_enable) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_combined_crc;
- IOWRITE_PDUMP_BUFADDR(session, buf, id * VHA_COMBINED_CRC_CORE_OFFSET,
- VHA_CR_OS0_COMBINED_CNN_CRC_ADDRESS);
- SET_BUFADDR(session, buf, id * VHA_COMBINED_CRC_CORE_OFFSET, ®s->crc_combined_address[id]);
- }
- }
- }
- }
- if (session->cnn_dbg.cnn_dbg_buf[0] && session->cnn_dbg.cnn_dbg_pdump_enable) {
- uint64_t val64;
- uint8_t id;
- /* Note: all buffers have the same size */
- img_pdump_printf("-- Select cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
- /* enable DEBUG: address, perf mode, band mode */
- img_pdump_printf("-- DEBUG_CONTROL=%u,%u buf 'DBG' size=%zx\n",
- GET_CNN_DBG_MODE(PERF, session), GET_CNN_DBG_MODE(BAND, session),
- session->cnn_dbg.cnn_dbg_buf[0]->size);
- val64 = VHA_CR_ALIGN_SETBITS(OS0_CNN_DEBUG_SIZE_LEGACY,
- CNN_DEBUG_SIZE, session->cnn_dbg.cnn_dbg_buf[0]->size);
- IOWRITE64_CR_PDUMP(val64, OS0_CNN_DEBUG_SIZE_LEGACY);
- /* Set the CONTROL register only if requested */
- if (CNN_DBG_MODE_ON(PERF, session) || CNN_DBG_MODE_ON(BAND, session)) {
- val64 = VHA_CR_SETBITS(OS0_CNN_DEBUG_CONTROL, CNN_PERF_ENABLE,
- GET_CNN_DBG_MODE(PERF, session));
- val64 |= VHA_CR_SETBITS(OS0_CNN_DEBUG_CONTROL, CNN_BAND_ENABLE,
- GET_CNN_DBG_MODE(BAND, session));
- img_pdump_printf("IF DUMP_DBG\n");
- IOWRITE64_CR_PDUMP(val64, OS0_CNN_DEBUG_CONTROL);
- img_pdump_printf("FI DUMP_DBG\n");
- }
- for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
- if (mask & (1 << id)) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_dbg_buf[id];
- /* Select core to be set */
- IOWRITE64_CR_PDUMP(VHA_CR_SETBITS(CORE_CTRL_INDIRECT, MASK, (1 << id)),
- CORE_CTRL_INDIRECT);
- IOWRITE_PDUMP_BUFADDR(session, buf, 0, VHA_CR_OS0_CNN_DEBUG_ADDRESS);
- }
- }
- }
- /* WM Performance & Bandwidth measurement */
- if (WM_DBG_MODE_ON(PERF) || WM_DBG_MODE_ON(BAND)) {
- uint64_t dbg_ctrl = 0;
- img_pdump_printf("IF CHECK_PERF_BW\n");
- if (WM_DBG_MODE_ON(PERF)) /* PERF */
- dbg_ctrl = VHA_SET_FIELD_SIMPLE_VAL(WM_DEBUG_CONTROL, PERF_ENABLE, EN);
- if (WM_DBG_MODE_ON(BAND)) { /* BW */
- uint64_t hw_brns = cmd->user_cmd.cmd_type == VHA_CMD_CNN_SUBMIT_MULTI ?
- ((struct vha_user_cnn_submit_multi_cmd*)&cmd->user_cmd)->hw_brns :
- ((struct vha_user_cnn_submit_cmd*)&cmd->user_cmd)->hw_brns;
- IOWRITE64_CR_PDUMP(VHA_CR_NOC_BWM_CONTROL_MASKFULL, NOC_BWM_CONTROL);
- dbg_ctrl |= VHA_SET_FIELD_SIMPLE_VAL(WM_DEBUG_CONTROL, BW_ENABLE, EN);
- if (VHA_IS_BRN(hw_brns, 71649)) {
- img_pdump_printf("-- BRN71649_START\n");
- IOWRITE64_CR_PDUMP(16, IDLE_HYSTERESIS_COUNT);
- IOWRITE64_CR_PDUMP(16, PWR_MAN_HYSTERESIS);
- img_pdump_printf("-- BRN71649_END\n");
- }
- }
- IOWRITE64_CR_PDUMP(dbg_ctrl, WM_DEBUG_CONTROL);
- img_pdump_printf("FI CHECK_PERF_BW\n");
- }
- }
- /* flush CRC and DEBUG data buffers */
- void vha_dbg_flush_hwbufs(struct vha_session *session, char checkpoint, uint8_t mask)
- {
- struct vha_dev *vha = session->vha;
- if (session->cnn_dbg.cnn_dbg_flush != checkpoint)
- return;
- if (session->cnn_dbg.cnn_crc_buf[0] || vha->cnn_combined_crc_enable) {
- int id;
- /* Note: all buffers have the same size */
- /*
- * TOBEDONE: calculate CRC buffer size based
- * on num passes, num layers, etc
- */
- img_pdump_printf("-- Save signatures\n");
- img_pdump_printf("IF SKIP_CHECK_CRCS\n");
- img_pdump_printf("COM Not checking CRCs!\n");
- img_pdump_printf("ELSE SKIP_CHECK_CRCS\n");
- img_pdump_printf("COM Checking CRCs ...\n");
- if (session->cnn_dbg.cnn_crc_buf[0]) {
- for (id = 0; id < session->vha->hw_props.num_cnn_core_devs; id++) {
- if (mask & (1 << id)) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_crc_buf[id];
- vha_pdump_sab_buf(session, PDUMP_CRC, buf, 0, buf->size);
- }
- }
- }
- if (vha->cnn_combined_crc_enable) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_combined_crc;
- vha_pdump_sab_buf(session, PDUMP_CRC_CMB, buf, 0, buf->size);
- }
- img_pdump_printf("FI SKIP_CHECK_CRCS\n");
- }
- if (session->cnn_dbg.cnn_dbg_buf[0] && session->cnn_dbg.cnn_dbg_pdump_enable) {
- int id;
- img_pdump_printf("-- Save DEBUG info\n");
- img_pdump_printf("IF DUMP_DBG\n");
- img_pdump_printf("COM Dumping debug data ...\n");
- for (id = 0; id < session->vha->hw_props.num_cnn_core_devs; id++) {
- if (mask & (1 << id)) {
- struct vha_buffer *buf = session->cnn_dbg.cnn_dbg_buf[id];
- vha_pdump_sab_buf(session, PDUMP_DBG, buf, 0, buf->size);
- }
- }
- img_pdump_printf("ELSE DUMP_DBG\n");
- img_pdump_printf("COM Not dumping debug data!\n");
- img_pdump_printf("FI DUMP_DBG\n");
- }
- }
- /* stop capturing CRC and DEBUG data */
- void vha_dbg_stop_hwbufs(struct vha_session *session, uint8_t mask)
- {
- struct vha_dev *vha = session->vha;
- /* Flush hw debug buffers */
- vha_dbg_flush_hwbufs(session, 0, mask);
- if (session->cnn_dbg.cnn_crc_buf[0]) {
- img_pdump_printf("-- Select cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
- IOWRITE64_CR_PDUMP(0, OS0_CNN_CRC_CONTROL);
- }
- if (session->cnn_dbg.cnn_dbg_buf[0]) {
- uint64_t size = 0;
- int id;
- for (id = 0; id < session->vha->hw_props.num_cnn_core_devs; id++) {
- uint64_t val;
- if (mask & (1 << id)) {
- val = IOREAD64_CR_REGIO(OS0_CNN_DEBUG_STATUS);
- if (val > size)
- size = val;
- }
- }
- if (CNN_DBG_MODE_ON(PERF, session) || CNN_DBG_MODE_ON(BAND, session)) {
- img_pdump_printf("IF DUMP_DBG\n");
- img_pdump_printf("-- Select cores\n");
- IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
- IOWRITE64_CR_PDUMP(0, OS0_CNN_DEBUG_CONTROL);
- /* just give a hint in the pdump:
- * dummy device returns 0 */
- img_pdump_printf(
- "-- POL64 :REG:%#x 0 0 0 1 1 -- DEBUG_STATUS=%llx\n",
- VHA_CR_OS0_CNN_DEBUG_STATUS,
- size);
- img_pdump_printf("FI DUMP_DBG\n");
- }
- }
- }
- uint64_t vha_dbg_rtm_read(struct vha_dev *vha, uint64_t addr)
- {
-
- return 0ULL;
- }
- const struct vha_reg vha_regs[] = {
- #define REG_DESC(reg) VHA_CR_##reg, VHA_CR_##reg##_MASKFULL
- {"product_id ", REG_DESC(PRODUCT_ID)},
- {"core_id ", REG_DESC(CORE_ID)},
- {"integrator_id ", REG_DESC(CORE_IP_INTEGRATOR_ID)},
- {"ip_changelist ", REG_DESC(CORE_IP_CHANGELIST)},
- {"core_ip_config ", REG_DESC(CORE_IP_CONFIG)},
- #undef REG_DESC
- {NULL , 0},
- };
- #ifdef VHA_SCF
- void wd_timer_callback(struct work_struct *work)
- {
- struct vha_dev *vha =
- container_of(work, struct vha_dev, swd_dwork.work);
- struct vha_cmd *cmd = NULL;
- unsigned int wm_id;
- mutex_lock(&vha->lock);
- for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
- cmd = vha->pendcmd[wm_id].cmd;
- if (cmd) {
- uint8_t core_mask = vha_wm_get_cores(vha, wm_id);
- uint8_t layer_count;
- uint8_t pass_count;
- bool lockup = false;
- uint64_t exec_time_us;
- uint64_t cmd_time_us;
- struct TIMESPEC now;
- GETNSTIMEOFDAY(&now);
- if (cmd->user_cmd.flags & VHA_EXEC_TIME_SET) {
- struct vha_user_cnn_submit_multi_cmd *cnn_user_cmd =
- (struct vha_user_cnn_submit_multi_cmd *)&cmd->user_cmd;
- cmd_time_us = cnn_user_cmd->exec_time;
- } else if (vha->swd_timeout_default)
- cmd_time_us = vha->swd_timeout_default;
- else //SW WDT disabled for this cmd
- continue;
- cmd_time_us *= vha->swd_timeout_m0;
- if (get_timespan_us(&cmd->hw_proc_start, &now, &exec_time_us)) {
- uint64_t expected_exec_time = do_div(cmd_time_us, 100) + vha->swd_timeout_m1;
- if (exec_time_us > expected_exec_time) {
- lockup = true;
- dev_err(vha->dev, "SW WDT lockup detected\n"
- " measured time: %llu\n"
- " cmd time: %llu\n"
- " cmd expected_exec_time: %llu\n",
- exec_time_us, cmd_time_us, expected_exec_time);
- }
- }
- while (core_mask != 0 && !lockup) {
- uint32_t core_id = VHA_CORE_MASK_TO_ID(core_mask);
- uint64_t cnn_status;
- uint64_t cnn_status2;
- core_mask &= ~(VHA_CORE_ID_TO_MASK(core_id));
- IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT, MASK, (1 << core_id)),
- CORE_CTRL_INDIRECT);
- cnn_status = IOREAD64_CR_REGIO(OS0_CNN_STATUS);
- cnn_status2 = IOREAD64_CR_REGIO(OS0_CNN_STATUS2);
- layer_count = VHA_CR_GETBITS_OS(CNN_STATUS, LAYER_COUNT, cnn_status);
- pass_count = VHA_CR_GETBITS_OS(CNN_STATUS2, PASS_COUNT, cnn_status2);
- if (cmd->layer_count[core_id] == layer_count &&
- cmd->pass_count[core_id] == pass_count) {
- lockup = true;
- dev_err(vha->dev, "SW WDT lockup detected\n"
- " layer_count: %d\n"
- " pass_count: %d\n", layer_count, pass_count);
- }
- cmd->layer_count[core_id] = layer_count;
- cmd->pass_count[core_id] = pass_count;
- }
- if (lockup) {
- if (vha_observers.error)
- vha_observers.error(vha->id, cmd->session->id, cmd->user_cmd.cmd_id, -EIO);
- /* Update stats. */
- vha->stats.total_failures++;
- vha->stats.cnn_kicks_completed++;
- VHA_INC_WL_STAT(vha, kicks_completed, cmd);
- vha_wm_reset(vha, &cmd->hw_sched_info);
- /* Free command resources. */
- vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
- vha_dev_free_cmd_res(vha, cmd, true);
- /* Move command queue. */
- vha_do_queued_cmd(vha, wm_id);
- /* Handle actual command */
- vha_handle_cmd(vha, wm_id, -EIO, -EIO, VHA_RSP_ERROR(SW_WDT_EXPIRED));
- }
- }
- }
- if (vha->state == VHA_STATE_ON)
- schedule_delayed_work(&vha->swd_dwork, msecs_to_jiffies(vha->swd_period));
- mutex_unlock(&vha->lock);
- }
- #endif
|