vha_dev.c 143 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213
  1. /*
  2. *****************************************************************************
  3. * Copyright (c) Imagination Technologies Ltd.
  4. *
  5. * The contents of this file are subject to the MIT license as set out below.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. *
  25. * Alternatively, the contents of this file may be used under the terms of the
  26. * GNU General Public License Version 2 ("GPL")in which case the provisions of
  27. * GPL are applicable instead of those above.
  28. *
  29. * If you wish to allow use of your version of this file only under the terms
  30. * of GPL, and not to allow others to use your version of this file under the
  31. * terms of the MIT license, indicate your decision by deleting the provisions
  32. * above and replace them with the notice and other provisions required by GPL
  33. * as set out in the file called "GPLHEADER" included in this distribution. If
  34. * you do not delete the provisions above, a recipient may use your version of
  35. * this file under the terms of either the MIT license or GPL.
  36. *
  37. * This License is also included in this distribution in the file called
  38. * "MIT_COPYING".
  39. *
  40. *****************************************************************************/
  41. #include <linux/moduleparam.h>
  42. #include <linux/delay.h>
  43. #include <linux/irq.h>
  44. #include <linux/preempt.h>
  45. #include <linux/pm_runtime.h>
  46. #include <linux/random.h>
  47. #include <linux/slab.h>
  48. #include <uapi/vha.h>
  49. #include <uapi/vha_errors.h>
  50. #include "vha_common.h"
  51. #include "vha_plat.h"
  52. #include "vha_regs.h"
  53. #include "vha_mt19937.h"
  54. static uint32_t shared_onchipmem_size;
  55. module_param(shared_onchipmem_size, uint, 0444);
  56. MODULE_PARM_DESC(shared_onchipmem_size,
  57. "Size of shared on-chip memory in bytes");
  58. /* WM debug statistics types */
  59. #define VHA_WM_DBG_MODE_PERF 0
  60. #define VHA_WM_DBG_MODE_BAND 1
  61. #define WM_DBG_MODE_ON(type) \
  62. (wm_dbg_ctrl[VHA_WM_DBG_MODE_##type])
  63. static uint32_t wm_dbg_ctrl[2] = { 0, 0 };
  64. module_param_array(wm_dbg_ctrl, uint, NULL, 0444);
  65. MODULE_PARM_DESC(wm_dbg_ctrl,
  66. "WM DEBUG CONTROL: switch for PERF and BAND: 0=disable 1=enable");
  67. static uint32_t slc_hash_mode;
  68. module_param(slc_hash_mode, uint, 0444);
  69. MODULE_PARM_DESC(slc_hash_mode,
  70. "SLC_CTRL_HASH_MODE: Address decoding for SLC. 0-none; 1-pvr_v3; 2-linear; 3-in_page. See TRM");
  71. #ifdef VHA_SCF
  72. static uint32_t sys_ram_correction_threshold = 0;
  73. module_param(sys_ram_correction_threshold, uint, 0444);
  74. MODULE_PARM_DESC(sys_ram_correction_threshold,
  75. "Threshold for system level ram correction");
  76. static uint32_t core_host_ram_correction_threshold = 0;
  77. module_param(core_host_ram_correction_threshold, uint, 0444);
  78. MODULE_PARM_DESC(core_host_ram_correction_threshold,
  79. "Threshold for host core level ram correction");
  80. static uint32_t core_wm_ram_correction_threshold = 0;
  81. module_param(core_wm_ram_correction_threshold, uint, 0444);
  82. MODULE_PARM_DESC(core_wm_ram_correction_threshold,
  83. "Threshold for wm core level ram correction");
  84. #endif
  85. #define CONF_WRITES_WAIT_TIMEOUT_MS 20
  86. #define SCHED_SEQ_CORES_MASK 0xff
  87. #define SCHED_SEQ_CORES_SHIFT 0
  88. #define SCHED_SEQ_WM_ID_MASK 0x7
  89. #define SCHED_SEQ_WM_ID_SHIFT 8
  90. #define SCHED_SEQ_GET_CORES(idx) \
  91. ((vha->scheduling_sequence[idx] >> SCHED_SEQ_CORES_SHIFT) & SCHED_SEQ_CORES_MASK)
  92. #define SCHED_SEQ_GET_WM(idx) \
  93. ((vha->scheduling_sequence[idx] >> SCHED_SEQ_WM_ID_SHIFT) & SCHED_SEQ_WM_ID_MASK)
  94. /*
  95. * scheduling_sequence can be used to force execution on specific WMs/cores.
  96. * It encodes the WM id (byte1) and the core mask (byte0), for example:
  97. * scheduling_sequence=0x001 -> WM0/core0
  98. * scheduling_sequence=0x204 -> WM2/core2
  99. * scheduling_sequence=0x520 -> WM5/core5
  100. * scheduling_sequence=0x310 -> WM3/core4
  101. */
  102. static int32_t scheduling_sequence_len;
  103. static uint32_t scheduling_sequence[VHA_MC_SCHED_SEQ_LEN_MAX] = { 0 };
  104. module_param_array(scheduling_sequence, uint, &scheduling_sequence_len, 0444);
  105. MODULE_PARM_DESC(scheduling_sequence, "multicore scheduling sequence");
  106. #define MAX_STALLING_DATA_ENTRIES 2
  107. static int32_t stalling_data_len;
  108. static uint32_t stalling[MAX_STALLING_DATA_ENTRIES] = { 0 };
  109. module_param_array(stalling, uint, &stalling_data_len, 0444);
  110. MODULE_PARM_DESC(stalling, "stalling data");
  111. static bool test_direct_events;
  112. module_param(test_direct_events, bool, 0444);
  113. MODULE_PARM_DESC(test_direct_events,
  114. "When set CORE&INTERCONNECT events are directly sent to host, to WM, otherwise");
  115. static int32_t pri_windows_list_len;
  116. static uint32_t pri_windows_list[VHA_MAX_PRIORITIES] = { 0 };
  117. module_param_array(pri_windows_list, uint, &pri_windows_list_len, 0444);
  118. MODULE_PARM_DESC(pri_windows_list,
  119. "priority window size list starting from lowest; all 0s mean no starvation avoidance");
  120. /* Priority scheduler local data. */
  121. struct vha_sched_local_data {
  122. void *rand_gen_handle;
  123. };
  124. /* Priority window array. */
  125. /* NOTE: Setting all to 0 implies strict priority scheduling (no starvation avoidance). */
  126. static uint32_t pri_windows[VHA_MAX_PRIORITIES] = {0};
  127. /* Parity related defines. */
  128. #ifdef VHA_SCF
  129. #define VHA_PARITY_READ_COUNT_MAX 4
  130. #endif
  131. struct vha_errcode {
  132. uint8_t e;
  133. const char* s;
  134. enum vha_reset_type reset_type;
  135. uint64_t rsp_err;
  136. };
  137. /* SYS event errors. */
  138. #define ERR_SYS_EVENT_DESC(b) VHA_SYS_EVENT_TYPE(b), __stringify(b)
  139. static const struct vha_biterr sys_err_bits[] = {
  140. {-EIO, ERR_SYS_EVENT_DESC(AXI_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_AXI_ERROR)},
  141. {-EFAULT, ERR_SYS_EVENT_DESC(MMU_PAGE_FAULT ), VHA_RESET_TYPE_MMU, VHA_RSP_ERROR(HW_SYS_MMU_PAGE_FAULT)},
  142. {-ETIMEDOUT, ERR_SYS_EVENT_DESC(SYS_MEM_WDT ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_SYS_MEM_WDT)},
  143. #ifdef VHA_SCF
  144. /*
  145. * Unfortunately, hw guys did not specify the way to identify the failed
  146. * WM. Waiting for them to fix this. */
  147. {-EIO, ERR_SYS_EVENT_DESC(AXI_MEMORY_PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_AXI_MEMORY_PARITY_ERROR)},
  148. {-EIO, ERR_SYS_EVENT_DESC(MMU_PARITY_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_MMU_PARITY_ERROR)}, /*VHA_RESET_TYPE_MMU},*/
  149. {-EIO, ERR_SYS_EVENT_DESC(RAM_CORRECTION ), VHA_RESET_TYPE_NONE, VHA_RSP_ERROR(HW_SYS_RAM_CORRECTION)},
  150. {-EIO, ERR_SYS_EVENT_DESC(RAM_DETECTION ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_RAM_DETECTION)},
  151. {-EIO, ERR_SYS_EVENT_DESC(LSYNC_INV_REQ ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_LSYNC_INV_REQ)},
  152. {-EIO, ERR_SYS_EVENT_DESC(LOGIC_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SYS_LOGIC_ERROR)},
  153. {-EIO, VHA_REG_PARITY_ERROR_EN, __stringify(PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(SW_SYS_EVNT_PARITY_ERROR)},
  154. #endif
  155. {0}
  156. };
  157. /* WM event errors. */
  158. #define ERR_WM_EVENT_DESC(b) VHA_WM_EVENT_TYPE(b), __stringify(b)
  159. static const struct vha_biterr wm_err_bits[] = {
  160. {-ETIMEDOUT, ERR_WM_EVENT_DESC(WM_WL_WDT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_EVNT_WM_WL_WDT)},
  161. {-ETIMEDOUT, ERR_WM_EVENT_DESC(WM_WL_IDLE_WDT), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_EVNT_WM_WL_IDLE_WDT)},
  162. {-ETIMEDOUT, ERR_WM_EVENT_DESC(WM_SOCIF_WDT ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_EVNT_WM_SOCIF_WDT)},
  163. {-EFAULT, ERR_WM_EVENT_DESC(LOGIC_FAULT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_EVNT_LOGIC_FAULT)},
  164. #ifdef VHA_SCF
  165. {-EIO, VHA_REG_PARITY_ERROR_EN, __stringify(PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(SW_EVNT_WM_PARITY_ERROR)},
  166. #endif
  167. {0}
  168. };
  169. /* WM response FIFO status error codes. */
  170. #define ERR_WM_RSP_STATUS_DESC(v) VHA_WM_RESPONSE_ERROR_CODE(v), __stringify(v)
  171. static const struct vha_errcode wm_rsp_err_codes[] = {
  172. {ERR_WM_RSP_STATUS_DESC(CORE_IRQ_BEFORE_KICK ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_IRQ_BEFORE_KICK)},
  173. {ERR_WM_RSP_STATUS_DESC(INDIRECT_MASK_SET_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_INDIRECT_MASK_SET_ERROR)},
  174. {ERR_WM_RSP_STATUS_DESC(KICK_CORE_ACCESS_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_KICK_CORE_ACCESS_ERROR)},
  175. {ERR_WM_RSP_STATUS_DESC(CNN_CONTROL_START_HIGH ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CNN_CONTROL_START_HIGH)},
  176. {ERR_WM_RSP_STATUS_DESC(CNN_STATUS_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CNN_STATUS_ERROR)},
  177. {ERR_WM_RSP_STATUS_DESC(INT_CORE_ACCESS_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_INT_CORE_ACCESS_ERROR)},
  178. {ERR_WM_RSP_STATUS_DESC(CORE_EVENT_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_EVENT_ERROR)},
  179. {ERR_WM_RSP_STATUS_DESC(CORE_EVENT_NOT_CLEARED ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_EVENT_NOT_CLEARED)},
  180. {ERR_WM_RSP_STATUS_DESC(CORE_EVENT_IRQ_HIGH ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_EVENT_IRQ_HIGH)},
  181. {ERR_WM_RSP_STATUS_DESC(INTERCONNECT_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_INTERCONNECT_ERROR)},
  182. };
  183. /* CNN core status errors. */
  184. #define ERR_CORE_STATUS_DESC(b) VHA_CORE_STATUS(b), __stringify(b)
  185. static const struct vha_biterr core_err_bits[] = {
  186. {-EIO, ERR_CORE_STATUS_DESC(LOGIC_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_LOGIC_ERROR)},
  187. {-EIO, ERR_CORE_STATUS_DESC(RAM_CORRECTION ), VHA_RESET_TYPE_NONE, VHA_RSP_ERROR(HW_RAM_CORRECTION)},
  188. {-EIO, ERR_CORE_STATUS_DESC(RAM_DETECTION ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_RAM_DETECTION)},
  189. {-EIO, ERR_CORE_STATUS_DESC(CORE_SYNC_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_CORE_SYNC_ERROR)},
  190. {-ETIMEDOUT, ERR_CORE_STATUS_DESC(CORE_WDT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_WDT)},
  191. {-ETIMEDOUT, ERR_CORE_STATUS_DESC(CORE_MEM_WDT ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_MEM_WDT)},
  192. {-EIO, ERR_CORE_STATUS_DESC(CNN_ERROR ), VHA_RESET_TYPE_WM, VHA_RSP_ERROR(HW_CORE_CNN_ERROR)},
  193. {0}
  194. };
  195. /* Interconnect status errors. */
  196. #define ERR_IC_STATUS_DESC(b) VHA_IC_STATUS(b), __stringify(b)
  197. static const struct vha_biterr ic_err_bits[] = {
  198. {-EIO, ERR_IC_STATUS_DESC(LOCKSTEP_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_LOCKSTEP_ERROR)},
  199. {-EIO, ERR_IC_STATUS_DESC(LOGIC_ERROR ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_IC_LOGIC_ERROR)},
  200. {-EIO, ERR_IC_STATUS_DESC(SOCIF_READ_MISMATCH ), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SOCIF_READ_MISMATCH)},
  201. {-EIO, ERR_IC_STATUS_DESC(SOCIF_READ_UNRESPONSIVE), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(HW_SOCIF_READ_UNRESPONSIVE)},
  202. #ifdef VHA_SCF
  203. {-EIO, VHA_REG_PARITY_ERROR_EN, __stringify(PARITY_ERROR), VHA_RESET_TYPE_FULL, VHA_RSP_ERROR(SW_IC_PARITY_ERROR)},
  204. #endif
  205. {0}
  206. };
  207. bool vha_dev_dbg_params_check(struct vha_dev *vha)
  208. {
  209. if (vha->scheduling_sequence_len > 0) {
  210. uint32_t i;
  211. for (i = 0; i < vha->scheduling_sequence_len; i++) {
  212. uint8_t wm_id = SCHED_SEQ_GET_WM(i);
  213. uint8_t core_mask = SCHED_SEQ_GET_CORES(i);
  214. if ((wm_id >= vha->hw_props.num_cnn_core_devs) ||
  215. (~vha->full_core_mask & core_mask)) {
  216. dev_info(vha->dev,
  217. "%u/0x%02x -> %u/0x%02x (0x%02x)",
  218. wm_id, core_mask,
  219. vha->hw_props.num_cnn_core_devs, vha->full_core_mask,
  220. (~vha->full_core_mask & core_mask));
  221. dev_err(vha->dev,
  222. "'scheduling_sequence' contains cores that do not exist on this h/w.\n");
  223. return false;
  224. }
  225. }
  226. }
  227. return true;
  228. }
  229. bool vha_dev_dbg_params_init(struct vha_dev *vha)
  230. {
  231. vha->scheduling_sequence_len = scheduling_sequence_len;
  232. memcpy(vha->scheduling_sequence,
  233. scheduling_sequence, sizeof(scheduling_sequence));
  234. vha->scheduling_counter = 0;
  235. vha->stalling_sysbus_host_stall_ratio = stalling[0];
  236. vha->stalling_membus_sys_stall_ratio = stalling[1];
  237. return vha_dev_dbg_params_check(vha);
  238. }
  239. int vha_dev_scheduler_init(struct vha_dev *vha)
  240. {
  241. int ret;
  242. uint32_t seed, i;
  243. bool use_default_pri_windows = true;
  244. vha->hw_sched_status.num_cores_free = vha->hw_props.num_cnn_core_devs;
  245. vha->hw_sched_status.num_wms_free = vha->hw_props.num_cnn_core_devs;
  246. vha->hw_sched_status.free_core_mask =
  247. VHA_GET_CORE_MASK(vha->hw_props.num_cnn_core_devs);
  248. vha->hw_sched_status.free_wm_mask =
  249. VHA_GET_WM_MASK(vha->hw_props.num_cnn_core_devs);
  250. vha->full_core_mask = vha->hw_sched_status.free_core_mask;
  251. vha->wm_core_assignment = (uint64_t)(
  252. VHA_CR_CORE_ASSIGNMENT_CORE_7_WM_MAPPING_UNALLOCATED |
  253. VHA_CR_CORE_ASSIGNMENT_CORE_6_WM_MAPPING_UNALLOCATED |
  254. VHA_CR_CORE_ASSIGNMENT_CORE_5_WM_MAPPING_UNALLOCATED |
  255. VHA_CR_CORE_ASSIGNMENT_CORE_4_WM_MAPPING_UNALLOCATED |
  256. VHA_CR_CORE_ASSIGNMENT_CORE_3_WM_MAPPING_UNALLOCATED |
  257. VHA_CR_CORE_ASSIGNMENT_CORE_2_WM_MAPPING_UNALLOCATED |
  258. VHA_CR_CORE_ASSIGNMENT_CORE_1_WM_MAPPING_UNALLOCATED |
  259. VHA_CR_CORE_ASSIGNMENT_CORE_0_WM_MAPPING_UNALLOCATED);
  260. vha->active_core_mask = 0;
  261. vha->apm_core_mask = 0;
  262. /* Allocate priority scheduler data. */
  263. vha->hw_sched_status.sched_data = kzalloc(sizeof(struct vha_sched_local_data), GFP_KERNEL);
  264. if (vha->hw_sched_status.sched_data == NULL) {
  265. dev_err(vha->dev, "%s: failed allocating scheduler data\n", __func__);
  266. return -ENOMEM;
  267. }
  268. /* Initialise random number generator for priority scheduling. */
  269. get_random_bytes(&seed, sizeof(seed));
  270. ret = vha_mt19937_init(seed, &vha->hw_sched_status.sched_data->rand_gen_handle);
  271. if (ret != 0) {
  272. dev_err(vha->dev, "%s: failed initialising random generator\n", __func__);
  273. kfree(vha->hw_sched_status.sched_data);
  274. return ret;
  275. }
  276. /* Attempt to set priority windows passed on to kernel module. */
  277. if (pri_windows_list_len == VHA_MAX_PRIORITIES) {
  278. uint32_t num_zeros = 0;
  279. for (i = 0; i < VHA_MAX_PRIORITIES; i++)
  280. if (pri_windows_list[i] == 0)
  281. num_zeros++;
  282. if (num_zeros < VHA_MAX_PRIORITIES)
  283. {
  284. dev_warn(vha->dev, "%s: some priority windows are set to 0; "
  285. "using default settings\n", __func__);
  286. } else {
  287. memcpy(pri_windows, pri_windows_list, sizeof(pri_windows));
  288. use_default_pri_windows = false;
  289. }
  290. } else if (pri_windows_list_len > 0) {
  291. dev_warn(vha->dev, "%s: too few priority windows provided (needed %u); "
  292. "using default settings\n", __func__, VHA_MAX_PRIORITIES);
  293. }
  294. /* Calculate priority windows. */
  295. if (use_default_pri_windows) {
  296. #define BASE_PRI_WINDOW_WIDTH 30
  297. for (i = 0; i < VHA_MAX_PRIORITIES; i++)
  298. pri_windows[i] = BASE_PRI_WINDOW_WIDTH + (i * 2 * BASE_PRI_WINDOW_WIDTH);
  299. #undef BASE_PRI_WINDOW_WIDTH
  300. }
  301. return 0;
  302. }
  303. int vha_dev_scheduler_deinit(struct vha_dev *vha)
  304. {
  305. int ret;
  306. if (vha->hw_sched_status.sched_data == NULL) {
  307. dev_warn(vha->dev, "%s: scheduler not initialised\n", __func__);
  308. return 0;
  309. }
  310. ret = vha_mt19937_deinit(vha->hw_sched_status.sched_data->rand_gen_handle);
  311. if (ret != 0) {
  312. dev_err(vha->dev, "%s: failed deinitialising random generator\n", __func__);
  313. }
  314. kfree(vha->hw_sched_status.sched_data);
  315. return ret;
  316. }
  317. void vha_dev_mh_setup(struct vha_dev *vha, int ctx_id, struct vha_mh_config_regs *regs)
  318. {
  319. uint64_t val64 = 0;
  320. regs->cnn_preload_control |= VHA_CR_SETBITS(OS0_CNN_PRELOAD_CONTROL,
  321. CBUF_N_REQS, VHA_CR_CNN_PRELOAD_CTRL_N_64);
  322. /* Setup preload for MMM */
  323. regs->cnn_preload_control |= VHA_CR_SETBITS(OS0_CNN_PRELOAD_CONTROL,
  324. MMM_RD_N_REQS, VHA_CR_CNN_PRELOAD_CTRL_N_256);
  325. regs->cnn_preload_control |= VHA_CR_SETBITS(OS0_CNN_PRELOAD_CONTROL,
  326. MMM_WR_N_REQS, VHA_CR_CNN_PRELOAD_CTRL_N_256);
  327. img_pdump_printf("-- MH setup:%d\n", ctx_id);
  328. IOWRITE64_CR_PDUMP(regs->cnn_preload_control, OS0_CNN_PRELOAD_CONTROL);
  329. regs->req_ctxt_override = VHA_SET_FIELD_SIMPLE_VAL(REQ_CTXT_OVERRIDE, OVERRIDE_OS0, EN);
  330. IOWRITE64_CR_PDUMP(regs->req_ctxt_override, REQ_CTXT_OVERRIDE);
  331. if (slc_hash_mode) {
  332. regs->slc_control = VHA_CR_SETBITS(SLC_CTRL, HASH_MODE,
  333. slc_hash_mode);
  334. IOWRITE64_CR_PDUMP(val64, SLC_CTRL);
  335. }
  336. }
  337. static int set_power_event(struct vha_dev *vha, uint64_t event)
  338. {
  339. int ret=0;
  340. uint64_t val64;
  341. /* Clear any pending power events */
  342. IOWRITE64_CR_PDUMP(0, POWER_EVENT);
  343. /* Confirm no power events are pending */
  344. ret = IOPOLL64_CR_PDUMP(0, 100, 1000,
  345. ((uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_COMPLETE) |
  346. (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_ABORT)),
  347. SYS_EVENT_STATUS);
  348. if(ret)
  349. return ret;
  350. /* Trigger power event */
  351. IOWRITE64_CR_PDUMP(event, POWER_EVENT);
  352. /* Wait for power complete */
  353. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, POWER_COMPLETE, EN);
  354. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  355. (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_COMPLETE),
  356. SYS_EVENT_STATUS);
  357. if(ret)
  358. return ret;
  359. /* Switch off power event */
  360. IOWRITE64_CR_PDUMP(0, POWER_EVENT);
  361. /* Clear power complete event status */
  362. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, POWER_COMPLETE, EN);
  363. IOWRITE64_CR_PDUMP(val64, SYS_EVENT_CLEAR);
  364. /* Confirm power complete is cleared */
  365. ret = IOPOLL64_CR_PDUMP(0, 100, 1000,
  366. (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, POWER_COMPLETE),
  367. SYS_EVENT_STATUS);
  368. return ret;
  369. }
  370. #ifdef VHA_SCF
  371. static void ecc_correction_setup(struct vha_dev *vha)
  372. {
  373. uint64_t val64;
  374. val64 = VHA_CR_SETBITS(SYS_EVENT_THRESHOLD, RAM_CORRECTION,
  375. sys_ram_correction_threshold);
  376. IOWRITE64_CR_PDUMP(val64, SYS_EVENT_THRESHOLD);
  377. val64 = VHA_CR_SETBITS(CORE_EVENT_WM_THRESHOLD, RAM_CORRECTION,
  378. core_wm_ram_correction_threshold);
  379. IOWRITE64_CR_PDUMP(val64, CORE_EVENT_WM_THRESHOLD);
  380. val64 = VHA_CR_SETBITS(CORE_EVENT_HOST_THRESHOLD, RAM_CORRECTION,
  381. core_host_ram_correction_threshold);
  382. IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_THRESHOLD);
  383. }
  384. #endif
  385. static int vha_dev_prepare_cores(struct vha_dev *vha, uint8_t core_mask)
  386. {
  387. /* Enabling selected cores on the platform
  388. * Note: don't touch TLC, is an always ON domain */
  389. uint64_t val64 = VHA_CR_SETBITS(POWER_EVENT, DOMAIN,
  390. (core_mask << 1)) |
  391. VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, TYPE, POWER_UP) |
  392. VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, REQ, EN);
  393. img_pdump_printf("-- Trigger POWER UP domain event\n");
  394. return set_power_event(vha, val64);
  395. }
  396. static int vha_dev_flush_cores(struct vha_dev *vha, uint8_t core_mask)
  397. {
  398. uint64_t val64;
  399. img_pdump_printf("-- Deselect any cores\n");
  400. IOWRITE64_CR_PDUMP(0, CORE_CTRL_INDIRECT);
  401. /* Disabling selected cores on the platform
  402. * Note: don't touch TLC, is an always ON domain */
  403. val64 = VHA_CR_SETBITS(POWER_EVENT, DOMAIN, (core_mask << 1)) |
  404. VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, TYPE, POWER_DOWN) |
  405. VHA_SET_FIELD_SIMPLE_VAL(POWER_EVENT, REQ, EN);
  406. img_pdump_printf("-- Trigger POWER DOWN domain event\n");
  407. return set_power_event(vha, val64);
  408. }
  409. void vha_dev_setup(struct vha_dev *vha)
  410. {
  411. uint64_t val64;
  412. img_pdump_printf("-- MMU set virtual address range0:%#llx-%#llx\n",
  413. IMG_MEM_VA_HEAP1_BASE, IMG_MEM_VA_HEAP1_SIZE);
  414. val64 = (uint64_t)vha->mmu_page_size <<
  415. VHA_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT;
  416. val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_ONE,
  417. BASE_ADDR, IMG_MEM_VA_HEAP1_BASE);
  418. val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_ONE,
  419. END_ADDR, (IMG_MEM_VA_HEAP1_BASE + IMG_MEM_VA_HEAP1_SIZE));
  420. IOWRITE64_PDUMP(val64, VHA_CR_MMU_PAGE_SIZE_RANGE_ONE);
  421. img_pdump_printf("-- MMU set virtual address range1:%#llx-%#llx\n",
  422. IMG_MEM_VA_HEAP2_BASE, IMG_MEM_VA_HEAP2_SIZE);
  423. val64 = (uint64_t)vha->mmu_page_size <<
  424. VHA_CR_MMU_PAGE_SIZE_RANGE_TWO_PAGE_SIZE_SHIFT ;
  425. val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_TWO,
  426. BASE_ADDR, IMG_MEM_VA_HEAP2_BASE);
  427. val64 |= VHA_CR_ALIGN_SETBITS(MMU_PAGE_SIZE_RANGE_TWO,
  428. END_ADDR, (IMG_MEM_VA_HEAP2_BASE + IMG_MEM_VA_HEAP2_SIZE));
  429. IOWRITE64_PDUMP(val64, VHA_CR_MMU_PAGE_SIZE_RANGE_TWO);
  430. #ifdef VHA_SCF
  431. ecc_correction_setup(vha);
  432. #endif
  433. }
  434. void vha_dev_wait(struct vha_dev *vha)
  435. {
  436. /* Nothing to do */
  437. }
  438. static void vha_dev_disable_events(struct vha_dev *vha, uint8_t core_mask, bool sys_release)
  439. {
  440. uint8_t id;
  441. if (sys_release) {
  442. img_pdump_printf("-- Disable SYS events\n");
  443. IOWRITE64_CR_PDUMP(0, SYS_EVENT_ENABLE);
  444. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  445. img_pdump_printf("-- Select WM%u\n", id);
  446. VHA_LOCK_WM();
  447. VHA_SELECT_WM(id);
  448. img_pdump_printf("-- Clear WM%u events\n", id);
  449. IOWRITE64_CR_PDUMP(VHA_WM_EVENTS_DEFAULT, WM_EVENT_CLEAR);
  450. img_pdump_printf("-- Disable WM%u events\n", id);
  451. IOWRITE64_CR_PDUMP(0, WM_EVENT_ENABLE);
  452. VHA_UNLOCK_WM();
  453. }
  454. }
  455. img_pdump_printf("-- Select cores\n");
  456. IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
  457. if (test_direct_events) {
  458. img_pdump_printf("-- Disable CORE events to HOST\n");
  459. IOWRITE64_CR_PDUMP(0, CORE_EVENT_HOST_ENABLE);
  460. img_pdump_printf("-- Disable INTERCONNECT events to HOST\n");
  461. IOWRITE64_CR_PDUMP(0, INTERCONNECT_EVENT_HOST_ENABLE);
  462. } else {
  463. img_pdump_printf("-- Disable CORE events to WM\n");
  464. IOWRITE64_CR_PDUMP(0, CORE_EVENT_WM_ENABLE);
  465. img_pdump_printf("-- Disable INTERCONNECT events to WM\n");
  466. IOWRITE64_CR_PDUMP(0, INTERCONNECT_EVENT_WM_ENABLE);
  467. }
  468. }
  469. static void vha_dev_ready(struct vha_dev *vha, uint8_t core_mask, bool sys_setup)
  470. {
  471. uint8_t id;
  472. if (sys_setup) {
  473. img_pdump_printf("-- Enable SYS events\n");
  474. IOWRITE64_CR_PDUMP(VHA_SYS_EVENTS_DEFAULT, SYS_EVENT_ENABLE);
  475. img_pdump_printf("-- Clear SYS events\n");
  476. IOWRITE64_CR_PDUMP(VHA_SYS_EVENTS_DEFAULT, SYS_EVENT_CLEAR);
  477. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  478. img_pdump_printf("-- Select WM%u\n", id);
  479. VHA_LOCK_WM();
  480. VHA_SELECT_WM(id);
  481. img_pdump_printf("-- Enable WM%u events\n", id);
  482. IOWRITE64_CR_PDUMP(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
  483. img_pdump_printf("-- Clear WM%u events\n", id);
  484. IOWRITE64_CR_PDUMP(VHA_WM_EVENTS_DEFAULT, WM_EVENT_CLEAR);
  485. VHA_UNLOCK_WM();
  486. }
  487. }
  488. img_pdump_printf("-- Select cores\n");
  489. IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
  490. if (test_direct_events) {
  491. img_pdump_printf("-- Enable CORE events to HOST\n");
  492. IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_HOST_ENABLE);
  493. img_pdump_printf("-- Clear CORE events on HOST\n");
  494. IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_HOST_CLEAR);
  495. img_pdump_printf("-- Enable INTERCONNECT events to HOST\n");
  496. IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_HOST_ENABLE);
  497. img_pdump_printf("-- Clear INTERCONNECT events on HOST\n");
  498. IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_HOST_CLEAR);
  499. } else {
  500. img_pdump_printf("-- Enable CORE events to WM\n");
  501. IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_ENABLE);
  502. img_pdump_printf("-- Clear CORE events on WM\n");
  503. IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_CLEAR);
  504. img_pdump_printf("-- Enable INTERCONNECT events to WM\n");
  505. IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_ENABLE);
  506. img_pdump_printf("-- Clear INTERCONNECT events on WM\n");
  507. IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_CLEAR);
  508. }
  509. }
  510. /* Global reset */
  511. static int vha_dev_reset(struct vha_dev *vha, uint8_t core_mask, bool sys_reset)
  512. {
  513. uint64_t val64 = 0;
  514. uint8_t mask = 0;
  515. uint8_t id;
  516. int ret = 0;
  517. WARN_ON(!mutex_is_locked(&vha->lock));
  518. dev_dbg(vha->dev, "%s core mask:%#x\n", __func__, core_mask);
  519. img_pdump_printf("-- Top level RESET sequence BEGIN\n");
  520. /* Perform reset procedure */
  521. if (sys_reset) {
  522. /* First reset all WMs with cores assigned. */
  523. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  524. struct vha_hw_sched_info sched_info = {0};
  525. sched_info.core_mask = vha_wm_get_cores(vha, id);
  526. if (sched_info.core_mask) {
  527. sched_info.wm_id = id;
  528. vha_wm_reset(vha, &sched_info);
  529. core_mask &= ~sched_info.core_mask;
  530. }
  531. }
  532. }
  533. /* Core reset procedure. */
  534. img_pdump_printf("-- Resetting cores\n");
  535. /* Proceed core by core, unassigned cores only */
  536. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  537. if (!(core_mask & VHA_CORE_ID_TO_MASK(id)))
  538. continue;
  539. /* Reset Assertion */
  540. /* 1. Select current core. */
  541. img_pdump_printf("-- Select core%u\n", id);
  542. mask = VHA_CORE_ID_TO_MASK(id);
  543. IOWRITE64_CR_PDUMP(mask, CORE_CTRL_INDIRECT);
  544. /* 3. Disable page fault interrupts for core while resetting. */
  545. img_pdump_printf("-- Disable page fault interrupts for core%u\n", id);
  546. val64 = IOREAD64_CR_REGIO(SYS_EVENT_ENABLE);
  547. val64 &= ~(VHA_CR_SETBITS(SYS_EVENT_ENABLE, MMU_PAGE_FAULT, mask));
  548. IOWRITE64_CR_PDUMP(val64, SYS_EVENT_ENABLE);
  549. /* 4. Force global clocks to ON on current core (others set to AUT0). */
  550. img_pdump_printf("-- Force global clocks ON for core%u (others set to AUTO)\n", id);
  551. val64 = VHA_SYS_CLOCK_MODE(INTERCONNECT, ON) |
  552. VHA_SYS_CLOCK_MODE_MULTI(CORE, ON, mask) |
  553. VHA_SYS_CLOCK_MODE_MULTI(CORE, AUTO, (uint8_t)~mask) |
  554. VHA_SYS_CLOCK_MODE_MULTI(NOC, AUTO, ~0) |
  555. VHA_SYS_CLOCK_MODE_MULTI(WM, AUTO, ~0) |
  556. VHA_SYS_CLOCK_MODE(AXI, AUTO) |
  557. VHA_SYS_CLOCK_MODE(SLC, AUTO) |
  558. VHA_SYS_CLOCK_MODE(LSYNC, AUTO) |
  559. VHA_SYS_CLOCK_MODE(SOCM, AUTO) |
  560. VHA_SYS_CLOCK_MODE(REGBANK, AUTO);
  561. IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
  562. /* 5. Set all core level clocks to AUTO. */
  563. img_pdump_printf("-- Set all core%u level clocks to AUTO\n", id);
  564. val64 = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
  565. IOWRITE64_CR_PDUMP(val64, CLK_CTRL0);
  566. /* 6. Move core into soft reset. */
  567. img_pdump_printf("-- Perform soft reset on core%u\n", id);
  568. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_SOFT_RESET, CORE_RESET, EN);
  569. IOWRITE64_CR_PDUMP(val64, CORE_SOFT_RESET);
  570. /* Dummy read to avoid race conditions in the hw. */
  571. val64 = IOREAD64_CR_PDUMP(CORE_SOFT_RESET);
  572. /* Clear reset. */
  573. IOWRITE64_CR_PDUMP(0, CORE_SOFT_RESET);
  574. /* 7. Wait until core memory bus reset has completed. */
  575. img_pdump_printf("-- Wait until core%u memory bus reset has completed\n", id);
  576. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, MEMBUS_RESET_DONE, EN);
  577. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  578. (uint64_t)VHA_CR_BITMASK(CORE_EVENT_HOST_STATUS, MEMBUS_RESET_DONE),
  579. CORE_EVENT_HOST_STATUS);
  580. if(ret)
  581. return ret;
  582. /* 8. Clear core memory bus reset interrupt. */
  583. img_pdump_printf("-- Clear core%u memory bus reset interrupt\n", id);
  584. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_CLEAR, MEMBUS_RESET_DONE, EN);
  585. IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_CLEAR);
  586. /* 9. Clear the core indirect register. */
  587. img_pdump_printf("-- Deselect core%u\n", id);
  588. IOWRITE64_CR_PDUMP(0, CORE_CTRL_INDIRECT);
  589. /* 10. Ensure no resets are pending. */
  590. img_pdump_printf("-- Ensure no resets are pending\n");
  591. IOWRITE64_CR_PDUMP(0, SYS_RESET_CTRL);
  592. /* 11. Move current core into full reset state. */
  593. img_pdump_printf("-- Move core%u into full reset state\n", id);
  594. val64 = VHA_CR_SETBITS(SYS_RESET_CTRL, CORE, mask);
  595. IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
  596. /* 12. Dummy read to avoid race conditions in the hw. */
  597. val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
  598. /* Reset Deassertion */
  599. /* 1. Move current core out of reset state. */
  600. img_pdump_printf("-- Move core%u out of reset state\n", id);
  601. val64 &= ~(VHA_CR_SETBITS(SYS_RESET_CTRL, CORE, mask));
  602. IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
  603. /* Dummy read to avoid race conditions in the hw. */
  604. val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
  605. /* 2. Select current core again. */
  606. img_pdump_printf("-- Select core%u again\n", id);
  607. IOWRITE64_CR_PDUMP(mask, CORE_CTRL_INDIRECT);
  608. /* 5. Force core clocks to ON for everything. */
  609. img_pdump_printf("-- Force core clocks ON for everything\n");
  610. val64 = VHA_MAIN_CLOCKS_DEFAULT(ON);
  611. IOWRITE64_CR_PDUMP(val64, CLK_CTRL0);
  612. /* 6. Perform core level RAM initialisation. */
  613. img_pdump_printf("-- Perform core%u level RAM initialisation\n", id);
  614. val64 = VHA_SET_FIELD_SIMPLE_VAL(FUSA_CONTROL, ECC_INIT_KICK, EN);
  615. IOWRITE64_CR_PDUMP(val64, FUSA_CONTROL);
  616. /* 7. Perform LOCM scrubbing. */
  617. img_pdump_printf("-- Perform core%u LOCM scrubbing\n", id);
  618. val64 = VHA_SET_FIELD_SIMPLE_VAL(LOCM_SCRUB_CTRL, KICK, EN);
  619. IOWRITE64_CR_PDUMP(val64, LOCM_SCRUB_CTRL);
  620. /* 8. Wait until the RAM initialisation sequence has completed. */
  621. img_pdump_printf("-- Wait until the RAM initialisation sequence has completed\n");
  622. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, RAM_INIT_DONE, EN);
  623. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  624. (uint64_t)VHA_CR_BITMASK(CORE_EVENT_HOST_STATUS, RAM_INIT_DONE),
  625. CORE_EVENT_HOST_STATUS);
  626. if(ret)
  627. return ret;
  628. /* 9. Clear core RAM reset interrupt. */
  629. img_pdump_printf("-- Clear core%u RAM reset interrupt\n", id);
  630. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_CLEAR, RAM_INIT_DONE, EN);
  631. IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_CLEAR);
  632. /* Confirm that 'RAM_INIT_DONE' field is cleared. */
  633. img_pdump_printf("-- Confirm that core%u RAM reset interrupt is cleared\n", id);
  634. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, RAM_INIT_DONE, EN);
  635. ret = IOPOLL64_CR_PDUMP(0ULL, 10, 100, val64, CORE_EVENT_HOST_STATUS);
  636. if(ret)
  637. return ret;
  638. /* 10. Wait until the LOCM scrubbing sequence has completed. */
  639. img_pdump_printf("-- Wait until the LOCM scrubbing sequence has completed.\n");
  640. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, LOCM_SCRUB_DONE, EN);
  641. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  642. (uint64_t)VHA_CR_BITMASK(CORE_EVENT_HOST_STATUS, LOCM_SCRUB_DONE),
  643. CORE_EVENT_HOST_STATUS);
  644. if(ret)
  645. return ret;
  646. /* 11. Deassert core LOCM scrubbing. */
  647. img_pdump_printf("-- Deassert core%u LOCM scrubbing\n", id);
  648. IOWRITE64_CR_PDUMP(0, LOCM_SCRUB_CTRL);
  649. /* 12. Clear core LOCM scrub interrupt. */
  650. img_pdump_printf("-- Clear core%u LOCM scrub interrupt\n", id);
  651. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_CLEAR, LOCM_SCRUB_DONE, EN);
  652. IOWRITE64_CR_PDUMP(val64, CORE_EVENT_HOST_CLEAR);
  653. /* Confirm that 'LOCM_SCRUB_DONE' field is cleared. */
  654. img_pdump_printf("-- Confirm that core%u LOCM scrub interrupt is cleared\n", id);
  655. val64 = VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_HOST_STATUS, LOCM_SCRUB_DONE, EN);
  656. ret = IOPOLL64_CR_PDUMP(0ULL, 10, 100, val64, CORE_EVENT_HOST_STATUS);
  657. if(ret)
  658. return ret;
  659. /* 13. Enable the interrupts from core to WM. */
  660. img_pdump_printf("-- Enable CORE events to WM\n");
  661. IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_ENABLE);
  662. /* 14. Clear all status from CORE_EVENT_WM (clears the RAM_INIT_DONE). */
  663. img_pdump_printf("-- Clear CORE events on WM\n");
  664. IOWRITE64_CR_PDUMP(VHA_CORE_EVENTS_DEFAULT |
  665. VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_WM_CLEAR, RAM_INIT_DONE, EN) |
  666. VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_WM_CLEAR, LOCM_SCRUB_DONE, EN) |
  667. VHA_SET_FIELD_SIMPLE_VAL(CORE_EVENT_WM_CLEAR, MEMBUS_RESET_DONE, EN),
  668. CORE_EVENT_WM_CLEAR);
  669. /* 15. Enable the interrupts from interconnect to WM */
  670. img_pdump_printf("-- Enable INTERCONNECT events to WM\n");
  671. IOWRITE64_CR_PDUMP(VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_ENABLE);
  672. /* 16. Disable all interrupts from the CORE to the HOST */
  673. img_pdump_printf("-- Disable CORE events on host\n");
  674. IOWRITE64_CR_PDUMP(0, CORE_EVENT_HOST_ENABLE);
  675. /* 17. Set all core level clocks back to AUTO. */
  676. img_pdump_printf("-- Set all core%u level clocks back to AUTO\n", id);
  677. val64 = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
  678. IOWRITE64_CR_PDUMP(val64, CLK_CTRL0);
  679. /* 18. Set core global clock back to AUTO. */
  680. img_pdump_printf("-- Set core%u global clock back to AUTO (others set to AUTO)\n", id);
  681. val64 = VHA_SYS_CLOCKS_DEFAULT(AUTO);
  682. IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
  683. /* Setup stalling if requested. */
  684. if (vha->stalling_membus_sys_stall_ratio != 0)
  685. IOWRITE64_CR_REGIO(vha->stalling_membus_sys_stall_ratio,
  686. NN_SYS2_MEMBUS_SYS_STALL_RATIO);
  687. }
  688. if (!sys_reset)
  689. return 0;
  690. dev_dbg(vha->dev, "%s handling system level reset\n", __func__);
  691. /* Move the rest of modules into reset state. */
  692. img_pdump_printf("-- Move other modules into reset state\n");
  693. val64 = VHA_SET_FIELD_SIMPLE_FULL(SYS_RESET_CTRL, WM) |
  694. VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, INTERCONNECT, EN) |
  695. VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, SLC, EN) |
  696. VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, MH, EN);
  697. IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
  698. /* Dummy read to avoid race conditions in the hw */
  699. val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
  700. /* Move the rest of modules out of reset state. */
  701. img_pdump_printf("-- Move other modules out of reset state\n");
  702. IOWRITE64_CR_PDUMP(0ULL, SYS_RESET_CTRL);
  703. /* Dummy read to avoid race conditions in the hw */
  704. val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
  705. /* Wait until core memory bus reset has completed. */
  706. img_pdump_printf("-- Wait until sys memory bus reset has completed\n");
  707. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, MEMBUS_RESET_DONE, EN);
  708. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  709. (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, MEMBUS_RESET_DONE),
  710. SYS_EVENT_STATUS);
  711. if(ret)
  712. return ret;
  713. /* Clear memory bus reset status. */
  714. img_pdump_printf("-- Clear memory bus reset status\n");
  715. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, MEMBUS_RESET_DONE, EN);
  716. IOWRITE64_CR_PDUMP(val64, SYS_EVENT_CLEAR);
  717. /* Force all system level clocks on. */
  718. img_pdump_printf("-- Force all system level clocks ON (except core)\n");
  719. val64 = IOREAD64_CR_REGIO(SYS_CLK_CTRL0);
  720. val64 &= VHA_SYS_CLOCKS_CORE_FULL_MASK;
  721. val64 |= VHA_SYS_CLOCKS_RESET(ON);
  722. IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
  723. /* Initiate system RAM initialisation. */
  724. img_pdump_printf("-- Initiate system RAM initialisation\n");
  725. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_RAM_INIT, KICK, EN);
  726. IOWRITE64_CR_PDUMP(val64, SYS_RAM_INIT);
  727. /* Initiate system SOCM scrubbing. */
  728. img_pdump_printf("-- Initiate system SOCM scrubbing\n");
  729. val64 = VHA_SET_FIELD_SIMPLE_VAL(SOCM_SCRUB_CTRL, KICK, EN);
  730. IOWRITE64_CR_PDUMP(val64, SOCM_SCRUB_CTRL);
  731. /* Wait until the RAM initialisation sequence has completed. */
  732. img_pdump_printf("-- Wait until the RAM initialisation sequence has completed\n");
  733. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, RAM_INIT_DONE, EN);
  734. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  735. (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, RAM_INIT_DONE),
  736. SYS_EVENT_STATUS);
  737. if(ret)
  738. return ret;
  739. /* Wait until the SOCM scrubbing sequence has completed. */
  740. img_pdump_printf("-- Wait until the SOCM scrubbing sequence has completed\n");
  741. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_STATUS, SOCM_SCRUB_DONE, EN);
  742. ret = IOPOLL64_CR_PDUMP(val64, 100, 1000,
  743. (uint64_t)VHA_CR_BITMASK(SYS_EVENT_STATUS, SOCM_SCRUB_DONE),
  744. SYS_EVENT_STATUS);
  745. if(ret)
  746. return ret;
  747. /* Deassert system SOCM scrubbing */
  748. img_pdump_printf("-- Deassert system SOCM scrubbing\n");
  749. IOWRITE64_CR_PDUMP(0, SOCM_SCRUB_CTRL);
  750. img_pdump_printf("-- Clear sys events\n");
  751. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, RAM_INIT_DONE, EN) |
  752. VHA_SET_FIELD_SIMPLE_VAL(SYS_EVENT_CLEAR, SOCM_SCRUB_DONE, EN);
  753. IOWRITE64_CR_PDUMP(val64, SYS_EVENT_CLEAR);
  754. /* Set all clocks back to AUTO. */
  755. img_pdump_printf("-- Set all sys clocks back to AUTO\n");
  756. val64 = VHA_SYS_CLOCKS_DEFAULT(AUTO);
  757. IOWRITE64_CR_PDUMP(val64, SYS_CLK_CTRL0);
  758. /* Reset the system level register banks. */
  759. img_pdump_printf("-- Reset the system level register banks\n");
  760. val64 = VHA_SET_FIELD_SIMPLE_VAL(SYS_RESET_CTRL, REGBANK, EN);
  761. IOWRITE64_CR_PDUMP(val64, SYS_RESET_CTRL);
  762. /* Dummy read to avoid race conditions in the hw */
  763. val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
  764. /* Clear reset */
  765. IOWRITE64_CR_PDUMP(0, SYS_RESET_CTRL);
  766. /* Dummy read to avoid race conditions in the hw */
  767. val64 = IOREAD64_CR_PDUMP(SYS_RESET_CTRL);
  768. img_pdump_printf("-- Top level RESET sequence END\n");
  769. vha->wm_core_assignment = (uint64_t)(
  770. VHA_CR_CORE_ASSIGNMENT_CORE_7_WM_MAPPING_UNALLOCATED |
  771. VHA_CR_CORE_ASSIGNMENT_CORE_6_WM_MAPPING_UNALLOCATED |
  772. VHA_CR_CORE_ASSIGNMENT_CORE_5_WM_MAPPING_UNALLOCATED |
  773. VHA_CR_CORE_ASSIGNMENT_CORE_4_WM_MAPPING_UNALLOCATED |
  774. VHA_CR_CORE_ASSIGNMENT_CORE_3_WM_MAPPING_UNALLOCATED |
  775. VHA_CR_CORE_ASSIGNMENT_CORE_2_WM_MAPPING_UNALLOCATED |
  776. VHA_CR_CORE_ASSIGNMENT_CORE_1_WM_MAPPING_UNALLOCATED |
  777. VHA_CR_CORE_ASSIGNMENT_CORE_0_WM_MAPPING_UNALLOCATED);
  778. /* Setup stalling if requested. */
  779. if (vha->stalling_sysbus_host_stall_ratio != 0)
  780. IOWRITE64_CR_REGIO(vha->stalling_sysbus_host_stall_ratio,
  781. NN_SYS2_SYSBUS_HOST_STALL_RATIO);
  782. return ret;
  783. }
  784. static void vha_dev_enable_clocks(struct vha_dev *vha, uint8_t core_mask)
  785. {
  786. uint64_t sys_clks = 0;
  787. uint64_t main_clks = 0;
  788. /* Always AUTO gating when needed */
  789. sys_clks = VHA_SYS_CLOCKS_DEFAULT(AUTO);
  790. main_clks = VHA_MAIN_CLOCKS_DEFAULT(AUTO);
  791. /* Enable sys clocks */
  792. img_pdump_printf("-- Enable SYS clocks\n");
  793. IOWRITE64_CR_PDUMP(sys_clks, SYS_CLK_CTRL0);
  794. /* Dummy SYS clocks status read*/
  795. sys_clks = IOREAD64_CR_PDUMP(SYS_CLK_STATUS0);
  796. /* Enable main clocks on all cores */
  797. img_pdump_printf("-- Enable MAIN clocks on cores\n");
  798. IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
  799. IOWRITE64_CR_PDUMP(main_clks, CLK_CTRL0);
  800. }
  801. static int vha_dev_disable_clocks(struct vha_dev *vha, uint8_t core_mask, bool sys_release)
  802. {
  803. uint64_t sys_clks = 0;
  804. uint8_t id;
  805. int ret = 0;
  806. if (sys_release) {
  807. /* Number of WMs equal to number of cores */
  808. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  809. VHA_LOCK_WM();
  810. VHA_SELECT_WM(id);
  811. /* Check WM is idle, handle parity */
  812. img_pdump_printf("-- Wait for WM%d IDLE state\n", id);
  813. ret = IOPOLL64_CR_PDUMP_PARITY(VHA_CR_WM_STATUS_STATE_IDLE, 100, 1000,
  814. (uint64_t)VHA_CR_WM_STATUS_STATE_MASK,
  815. WM_STATUS);
  816. VHA_UNLOCK_WM();
  817. if(ret) {
  818. struct vha_hw_sched_info sched_info = {
  819. .wm_id = id,
  820. .core_mask = 0
  821. };
  822. dev_err(vha->dev, "Performing wm%d reset due to HW error detection.", id);
  823. vha_wm_reset(vha, &sched_info);
  824. dev_err(vha->dev, "%s Waiting for WM%d IDLE state failed!",
  825. __func__, id);
  826. return ret;
  827. }
  828. }
  829. }
  830. vha_wm_release_cores(vha, core_mask, true);
  831. img_pdump_printf("-- Address cores\n");
  832. IOWRITE64_CR_PDUMP((uint64_t)core_mask, CORE_CTRL_INDIRECT);
  833. /* If auto gating was turned on, wait for clocks GATED state on all cores */
  834. img_pdump_printf("-- Wait for clocks IDLE state\n");
  835. ret = IOPOLL64_CR_PDUMP(0, 100, 1000,
  836. VHA_CR_CLK_STATUS0_MASKFULL,
  837. CLK_STATUS0);
  838. if(ret) {
  839. dev_err(vha->dev, "%s Waiting for clocks IDLE state failed!\n",
  840. __func__);
  841. return ret;
  842. }
  843. if (sys_release) {
  844. /* Wait for MMU,CCM,RDI,XBAR IDLE state */
  845. img_pdump_printf("-- Wait for memory bus interface IDLE state\n");
  846. ret = IOPOLL64_CR_PDUMP(VHA_CR_SLC_IDLE_MASKFULL, 1000, 1000,
  847. VHA_CR_SLC_IDLE_MASKFULL,
  848. SLC_IDLE);
  849. if(ret) {
  850. dev_err(vha->dev, "%s Waiting for memory bus interface IDLE state failed\n",
  851. __func__);
  852. return ret;
  853. }
  854. }
  855. /* Finally disable core clocks */
  856. img_pdump_printf("-- Disable MAIN clocks\n");
  857. IOWRITE64_CR_PDUMP(0, CLK_CTRL0); /* main */
  858. if (sys_release) {
  859. /* Finally disable sys clocks */
  860. img_pdump_printf("-- Disable SYS clocks (except REGBANK)\n");
  861. sys_clks = VHA_SYS_CLOCK_MODE(REGBANK, AUTO);
  862. IOWRITE64_CR_PDUMP(sys_clks, SYS_CLK_CTRL0); /* sys */
  863. }
  864. return ret;
  865. }
  866. void vha_update_utilization(struct vha_dev *vha)
  867. {
  868. uint8_t i;
  869. uint64_t tmp;
  870. uint64_t core_total_proc_us = 0ULL;
  871. for (i = 0; i < vha->hw_props.num_cnn_core_devs; i++) {
  872. /* Calculate core utilization. */
  873. tmp = vha->stats.core_stats[i].total_proc_us;
  874. do_div(tmp, vha->stats.uptime_ms);
  875. vha->stats.core_stats[i].utilization = tmp;
  876. /* Calculate WM utilization. */
  877. tmp = vha->stats.wm_stats[i].total_proc_us;
  878. do_div(tmp, vha->stats.uptime_ms);
  879. vha->stats.wm_stats[i].utilization = tmp;
  880. /* Calculate cumulative core processing time. */
  881. core_total_proc_us += vha->stats.core_stats[i].total_proc_us;
  882. }
  883. /* Calculate cluster utilization. */
  884. tmp = core_total_proc_us;
  885. do_div(tmp, (vha->stats.uptime_ms * vha->hw_props.num_cnn_core_devs));
  886. vha->stats.cnn_utilization = tmp;
  887. }
  888. #ifdef VHA_EVENT_INJECT
  889. /*
  890. * Inject EVENT_STATUS bits, requested by respective debugfs nodes, to
  891. * the registers defined by the currently handled WM.
  892. */
  893. static inline void __inject_event_regs(struct vha_dev* vha, struct vha_mc_irq_status* irq_status)
  894. {
  895. int id, wm_id;
  896. u32 mask, wm_mask;
  897. uint64_t vha_cr_sys_event = vha->injection.vha_cr_sys_event & VHA_CR_SYS_EVENT_INJECT_MASKFULL;
  898. uint64_t vha_cr_wm_event = vha->injection.vha_cr_wm_event & VHA_CR_WM_EVENT_INJECT_MASKFULL;
  899. uint64_t vha_cr_core_event = vha->injection.vha_cr_core_event & VHA_CR_CORE_EVENT_INJECT_MASKFULL;
  900. uint64_t vha_cr_interconnect_event = vha->injection.vha_cr_interconnect_event & VHA_CR_INTERCONNECT_EVENT_INJECT_MASKFULL;
  901. if(!__EVENT_INJECT())
  902. return;
  903. if(vha_cr_sys_event) {
  904. IOWRITE64_CR_REGIO(vha_cr_sys_event, SYS_EVENT_INJECT);
  905. }
  906. /* handle WM event injection */
  907. wm_mask = VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM, irq_status->event_source);
  908. if(!wm_mask)
  909. return;
  910. spin_lock_irqsave(&vha->irq_lock, vha->irq_flags);
  911. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  912. if(~wm_mask & (1 << wm_id))
  913. continue; /* inject only to currently handled WM's */
  914. if(vha_cr_wm_event) {
  915. VHA_SELECT_WM(wm_id);
  916. IOWRITE64_CR_REGIO(vha_cr_wm_event, WM_EVENT_INJECT);
  917. }
  918. /* now handle WM's core and ic injections . IC sources are the same as core sources */
  919. if(!vha_cr_core_event && !vha_cr_interconnect_event)
  920. continue;
  921. /* get cores handled by specific WM, inject errors only to those cores */
  922. mask = vha_wm_get_cores(vha, wm_id);
  923. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  924. if(~mask & (1 << id))
  925. continue; /* inject only to currently handled CORE's */
  926. if(vha_cr_core_event) {
  927. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
  928. MASK, (1 << id)),
  929. CORE_CTRL_INDIRECT);
  930. IOWRITE64_CR_REGIO(vha_cr_core_event, CORE_EVENT_INJECT);
  931. }
  932. if(vha_cr_interconnect_event) {
  933. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(IC_CORE_INDIRECT,
  934. MASK, (1 << id)),
  935. IC_CORE_INDIRECT);
  936. IOWRITE64_CR_REGIO(vha_cr_interconnect_event, INTERCONNECT_EVENT_INJECT);
  937. }
  938. }
  939. }
  940. /* read new injected event sources */
  941. irq_status->event_source |= IOREAD64_CR_REGIO(HOST_EVENT_SOURCE);
  942. spin_unlock_irqrestore(&vha->irq_lock, vha->irq_flags);
  943. }
  944. static inline void __inject_parity_err(struct vha_dev* vha, struct vha_mc_irq_status* irq_status) {
  945. int id, wm_id;
  946. u32 mask, wm_mask;
  947. if(!__EVENT_INJECT())
  948. return;
  949. if (VHA_REG_GET_PARITY_ERROR(vha->injection.vha_cr_sys_event)) {
  950. VHA_REG_SET_PARITY_ERROR(irq_status->sys_events);
  951. irq_status->event_source |= VHA_CR_HOST_EVENT_SOURCE_SYS_EN;
  952. }
  953. wm_mask = VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM, irq_status->event_source);
  954. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  955. if(~wm_mask & (1 << wm_id))
  956. continue; /* inject only to currently handled WM's */
  957. if (VHA_REG_GET_PARITY_ERROR(vha->injection.vha_cr_wm_event)) {
  958. VHA_REG_SET_PARITY_ERROR(irq_status->wm_events[wm_id]);
  959. irq_status->event_source |= 1 << (wm_id + VHA_CR_HOST_EVENT_SOURCE_WM_SHIFT);
  960. }
  961. mask = vha_wm_get_cores(vha, wm_id);
  962. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  963. if(~mask & (1 << id))
  964. continue; /* inject only to currently handled CORE's */
  965. if (VHA_REG_GET_PARITY_ERROR(vha->injection.vha_cr_interconnect_event)) {
  966. VHA_REG_SET_PARITY_ERROR(irq_status->ic_events[id]);
  967. irq_status->event_source |= 1 << (id + VHA_CR_HOST_EVENT_SOURCE_IC_SHIFT);
  968. }
  969. }
  970. }
  971. }
  972. #endif
  973. /* Top half */
  974. irqreturn_t vha_handle_irq(struct device *dev)
  975. {
  976. struct vha_dev *vha = vha_dev_get_drvdata(dev);
  977. irqreturn_t ret = IRQ_NONE;
  978. struct vha_mc_irq_status irq_status = {0};
  979. uint32_t multi_src_mask = 0;
  980. uint8_t id;
  981. struct TIMESPEC hw_proc_end[VHA_NUM_CORES] = {{0}};
  982. bool hw_proc_end_recorded[VHA_NUM_CORES] = {0};
  983. #define CHECK_FOR_DEAD_HW(r) \
  984. if (r == VHA_DEAD_HW || r == ~0) { \
  985. WARN_ONCE(1, "Hardware is dead!"); \
  986. if (!in_interrupt()) \
  987. mutex_unlock(&vha->lock); \
  988. return IRQ_NONE; \
  989. }
  990. /* ML: This thing is a complete mess in regdef file. The field is present
  991. * in most of these EVENT regs, but its definition varies a lot, so no idea
  992. * what it is actually meant to mean.
  993. */
  994. #define CHECK_FOR_LOGIC_ERROR(s, r) \
  995. if (r & VHA_##s##_EVENT_TYPE(LOGIC_ERROR)) { \
  996. WARN_ONCE(1, "Parity error detected!"); \
  997. if (!in_interrupt()) \
  998. mutex_unlock(&vha->lock); \
  999. return IRQ_NONE; \
  1000. }
  1001. if (!vha)
  1002. return IRQ_NONE;
  1003. /* Note: Top half can be called from the platform worker thread */
  1004. if (!in_interrupt())
  1005. mutex_lock(&vha->lock);
  1006. irq_status.event_source = IOREAD64_CR_REGIO(HOST_EVENT_SOURCE);
  1007. /* On fpga platform it is possible to get a spurious interrupt when the hw died.
  1008. * Do not proceed, just throw a warning. */
  1009. CHECK_FOR_DEAD_HW(irq_status.event_source);
  1010. #ifdef VHA_EVENT_INJECT
  1011. __inject_event_regs(vha, &irq_status);
  1012. #endif
  1013. if (VHA_CR_GETBITS(HOST_EVENT_SOURCE, SYS, irq_status.event_source)) {
  1014. /* Read events. */
  1015. irq_status.sys_events = IOREAD64_CR_REGIO(SYS_EVENT_STATUS);
  1016. /* Just in case check for dead hw. */
  1017. CHECK_FOR_DEAD_HW(irq_status.sys_events);
  1018. #ifdef VHA_SCF
  1019. if (vha->hw_props.supported.parity && !vha->parity_disable) {
  1020. uint32_t i;
  1021. for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
  1022. /* Finish if bit parity is ok */
  1023. if (!img_mem_calc_parity(irq_status.sys_events))
  1024. break;
  1025. /* Otherwise re-read the reg. */
  1026. irq_status.sys_events = IOREAD64_CR_REGIO(SYS_EVENT_STATUS);
  1027. }
  1028. /* Raise an error if maximum re-read count is reached. */
  1029. if (i == VHA_PARITY_READ_COUNT_MAX) {
  1030. dev_err(dev, "SYS_EVENT_STATUS register parity error!\n");
  1031. /* Use the real event to indicate the error */
  1032. VHA_REG_SET_PARITY_ERROR(irq_status.sys_events);
  1033. }
  1034. }
  1035. #endif
  1036. /* Check for hw logic error. */
  1037. /* ML: ??? */
  1038. //CHECK_FOR_LOGIC_ERROR(SYS, irq_status.sys_events);
  1039. /* wake thread even if only parity error is set. Erroneous event may occur that only
  1040. * parity is set among other bits
  1041. */
  1042. if (irq_status.sys_events & (VHA_SYS_EVENTS_DEFAULT | VHA_REG_PARITY_ERROR_EN)) {
  1043. /* Clear interrupts (best not to write pdump in ISR). */
  1044. IOWRITE64_CR_REGIO(irq_status.sys_events & VHA_SYS_EVENTS_DEFAULT,
  1045. SYS_EVENT_CLEAR);
  1046. ret = IRQ_WAKE_THREAD;
  1047. }
  1048. }
  1049. /* Read WM event source mask. */
  1050. multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM,
  1051. irq_status.event_source);
  1052. if (multi_src_mask) {
  1053. spin_lock_irqsave(&vha->irq_lock, vha->irq_flags);
  1054. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  1055. if (multi_src_mask & (1 << id)) {
  1056. /* Select WM to read events from. */
  1057. VHA_SELECT_WM(id);
  1058. /* Read events. */
  1059. irq_status.wm_events[id] = IOREAD64_CR_REGIO(WM_EVENT_STATUS);
  1060. /* Just in case check for dead hw. */
  1061. CHECK_FOR_DEAD_HW(irq_status.wm_events[id]);
  1062. /* Record hw processing end timestamps */
  1063. GETNSTIMEOFDAY(&hw_proc_end[id]);
  1064. hw_proc_end_recorded[id] = true;
  1065. #ifdef VHA_SCF
  1066. if (vha->hw_props.supported.parity && !vha->parity_disable) {
  1067. uint32_t i;
  1068. for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
  1069. /* Finish if parity is ok */
  1070. if (!img_mem_calc_parity(irq_status.wm_events[id]))
  1071. break;
  1072. /* Otherwise re-read the reg. */
  1073. irq_status.wm_events[id] = IOREAD64_CR_REGIO(WM_EVENT_STATUS);
  1074. }
  1075. /* Raise an error if maximum re-read count is reached. */
  1076. if (i == VHA_PARITY_READ_COUNT_MAX) {
  1077. dev_err(dev, "WM_EVENT_STATUS[%u] register parity error!\n", id);
  1078. /* Use the real event to indicate the error */
  1079. VHA_REG_SET_PARITY_ERROR(irq_status.wm_events[id]);
  1080. }
  1081. }
  1082. #endif
  1083. {
  1084. /* Post check for AXI bus errors */
  1085. uint64_t ace_status = IOREAD64(vha->reg_base, VHA_CR_ACE_STATUS);
  1086. if (ace_status) {
  1087. dev_err(vha->dev, "AXI bus protocol error: %#llx\n",
  1088. ace_status);
  1089. /* Use AXI error event to indicate that */
  1090. irq_status.event_source |= VHA_CR_SETBITS(HOST_EVENT_SOURCE, SYS, 1);
  1091. irq_status.sys_events |= VHA_CR_SETBITS(SYS_EVENT_TYPE, AXI_ERROR, 1);
  1092. }
  1093. }
  1094. /* wake thread even if only parity error is set. Erroneous event may occur that only
  1095. * parity is set among other bits
  1096. */
  1097. if (irq_status.wm_events[id] & (VHA_WM_EVENTS_DEFAULT | VHA_REG_PARITY_ERROR_EN)) {
  1098. /* Events can't be cleared, disable to avoid interrupt storm */
  1099. IOWRITE64_CR_REGIO(0, WM_EVENT_ENABLE);
  1100. ret = IRQ_WAKE_THREAD;
  1101. }
  1102. }
  1103. }
  1104. spin_unlock_irqrestore(&vha->irq_lock, vha->irq_flags);
  1105. }
  1106. /* Read CORE event source mask. */
  1107. multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, CORE,
  1108. irq_status.event_source);
  1109. /* Note: Direct (Host) core event is only used for frequency measurement,
  1110. * Indirect (WM) core events are read in bottom handler */
  1111. if (multi_src_mask) {
  1112. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  1113. if (multi_src_mask & (1 << id)) {
  1114. /* Select core to read events from. */
  1115. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
  1116. MASK, (1 << id)),
  1117. CORE_CTRL_INDIRECT);
  1118. /* Read events. */
  1119. /* In normal operation CORE events are routed to WM,
  1120. * therefore there's no need to handle parity here
  1121. */
  1122. irq_status.core_events[id] = IOREAD64_CR_REGIO(CORE_EVENT_HOST_STATUS);
  1123. /* Just in case check for dead hw. */
  1124. CHECK_FOR_DEAD_HW(irq_status.core_events[id]);
  1125. /* Check for hw logic error. */
  1126. /* ML: ??? */
  1127. //CHECK_FOR_LOGIC_ERROR(CORE, irq_status.core_events[id]);
  1128. if (irq_status.core_events[id] & VHA_CORE_EVENTS_DEFAULT) {
  1129. /* Clear interrupts (best not to write pdump in ISR). */
  1130. IOWRITE64_CR_REGIO(irq_status.core_events[id] & VHA_CORE_EVENTS_DEFAULT,
  1131. CORE_EVENT_HOST_CLEAR);
  1132. /* Record hw processing end timestamps */
  1133. /*
  1134. * for regular workloads. This stat update is used
  1135. * only for cluster clock measurement, so it is
  1136. * executed only once after module is loaded. */
  1137. GETNSTIMEOFDAY(&hw_proc_end[id]);
  1138. hw_proc_end_recorded[id] = true;
  1139. ret = IRQ_WAKE_THREAD;
  1140. }
  1141. }
  1142. }
  1143. }
  1144. /* Read IC event source mask. */
  1145. multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, IC,
  1146. irq_status.event_source);
  1147. /* Indirect (WM) interconnect events are read in bottom handler */
  1148. if (multi_src_mask) {
  1149. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  1150. if (ret || (multi_src_mask & (1 << id))) {
  1151. /* Select IC to read events from. */
  1152. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(IC_CORE_INDIRECT,
  1153. MASK, (1 << id)),
  1154. IC_CORE_INDIRECT);
  1155. /* Read events. */
  1156. /* In normal operation IC events are routed to WM,
  1157. * therefore there's no need to handle parity here
  1158. */
  1159. irq_status.ic_events[id] = IOREAD64_CR_REGIO(INTERCONNECT_EVENT_HOST_STATUS);
  1160. #ifdef VHA_SCF
  1161. if (vha->hw_props.supported.parity && !vha->parity_disable) {
  1162. uint32_t i;
  1163. for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
  1164. /* Finish if parity is ok */
  1165. if (!img_mem_calc_parity(irq_status.ic_events[id]))
  1166. break;
  1167. /* Otherwise re-read the reg. */
  1168. irq_status.ic_events[id] = IOREAD64_CR_REGIO(INTERCONNECT_EVENT_HOST_STATUS);
  1169. }
  1170. /* Raise an error if maximum re-read count is reached. */
  1171. if (i == VHA_PARITY_READ_COUNT_MAX) {
  1172. dev_err(dev, "WM_EVENT_STATUS[%u] register parity error!\n", id);
  1173. /* Use the real event to indicate the error */
  1174. VHA_REG_SET_PARITY_ERROR(irq_status.ic_events[id]);
  1175. }
  1176. }
  1177. #endif
  1178. /* Just in case check for dead hw. */
  1179. CHECK_FOR_DEAD_HW(irq_status.ic_events[id]);
  1180. /* Check for hw logic error. */
  1181. /* ML: ??? */
  1182. //CHECK_FOR_LOGIC_ERROR(IC, irq_status.ic_events[id]);
  1183. if (multi_src_mask && (irq_status.ic_events[id] & (VHA_IC_EVENTS_DEFAULT | VHA_REG_PARITY_ERROR_EN))) {
  1184. /* Clear interrupts (best not to write pdump in ISR). */
  1185. IOWRITE64_CR_REGIO(irq_status.ic_events[id] & VHA_IC_EVENTS_DEFAULT,
  1186. INTERCONNECT_EVENT_HOST_CLEAR);
  1187. ret = IRQ_WAKE_THREAD;
  1188. }
  1189. }
  1190. }
  1191. }
  1192. #ifdef VHA_EVENT_INJECT
  1193. __inject_parity_err(vha, &irq_status);
  1194. #endif
  1195. if (!in_interrupt())
  1196. mutex_unlock(&vha->lock);
  1197. if (ret == IRQ_WAKE_THREAD) {
  1198. spin_lock(&vha->irq_lock);
  1199. /* Store all the event info. */
  1200. vha->irq_status.event_source |= irq_status.event_source;
  1201. vha->irq_status.sys_events |= irq_status.sys_events;
  1202. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  1203. vha->irq_status.wm_events[id] |= irq_status.wm_events[id];
  1204. if (hw_proc_end_recorded[id]) {
  1205. /* Record hw processing end timestamps */
  1206. VHA_WM_STAT_SHIFT_PROC_END(vha, id);
  1207. VHA_SET_WM_STAT(vha, hw_proc_end, id, hw_proc_end[id]);
  1208. }
  1209. }
  1210. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  1211. vha->irq_status.core_events[id] |= irq_status.core_events[id];
  1212. vha->irq_status.ic_events[id] |= irq_status.ic_events[id];
  1213. }
  1214. spin_unlock(&vha->irq_lock);
  1215. }
  1216. #undef CHECK_FOR_DEAD_HW
  1217. #undef CHECK_FOR_LOGIC_ERROR
  1218. if (ret) {
  1219. dev_dbg(dev, "IRQ EVT:0x%08llx SYS:0x%08llx\n", irq_status.event_source, irq_status.sys_events);
  1220. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
  1221. if (irq_status.wm_events[id] || irq_status.core_events[id] || irq_status.ic_events[id])
  1222. dev_dbg(dev, "WM%d:0x%08llx CORE%d:0x%08llx IC%d:0x%08llx\n",
  1223. id, irq_status.wm_events[id], id, irq_status.core_events[id],
  1224. id, irq_status.ic_events[id]);
  1225. }
  1226. return ret;
  1227. }
  1228. static void vha_do_queued_cmd(struct vha_dev *vha, uint8_t wm_id)
  1229. {
  1230. struct vha_cmd *cmd, *pend;
  1231. cmd = vha->queuedcmd[wm_id].cmd;
  1232. #if defined(DEBUG)
  1233. {
  1234. char queued_txt[24] = "none";
  1235. char pending_txt[24] = "none";
  1236. if (cmd)
  1237. snprintf(queued_txt, 24, "0x%08x/%u",
  1238. cmd->user_cmd.cmd_id, cmd->session->id);
  1239. if (vha->pendcmd[wm_id].cmd)
  1240. snprintf(pending_txt, 24, "0x%08x/%u",
  1241. vha->pendcmd[wm_id].cmd->user_cmd.cmd_id,
  1242. vha->pendcmd[wm_id].cmd->session->id);
  1243. dev_dbg(vha->dev,
  1244. "%s: WM%u pending %s, queued %s\n",
  1245. __func__, wm_id, pending_txt, queued_txt);
  1246. }
  1247. #endif
  1248. if (!cmd || (cmd &&
  1249. ((vha->low_latency == VHA_LL_DISABLED ||
  1250. vha->low_latency == VHA_LL_SELF_KICK) ||
  1251. !cmd->queued))) {
  1252. dev_dbg(vha->dev, "%s: skipping!\n", __func__);
  1253. return;
  1254. }
  1255. /* store actual pending command as it will be modified */
  1256. pend = vha->pendcmd[wm_id].cmd;
  1257. /* at this point we should be able to process the cmd */
  1258. vha_do_cnn_cmd(cmd);
  1259. /* restore pending */
  1260. vha->pendcmd[wm_id].cmd = pend;
  1261. }
  1262. /*
  1263. * Roll back commands for a particular WM.
  1264. */
  1265. static bool vha_rollback_wm_cmds(struct vha_dev *vha, uint8_t wm_id,
  1266. bool free_res)
  1267. {
  1268. bool processing = false;
  1269. #if defined(DEBUG)
  1270. char queued_txt[24] = "none";
  1271. char pending_txt[24] = "none";
  1272. #endif
  1273. /* Not processed commands are still on the pending list
  1274. * of each session, so just mark the hw pending lists as empty */
  1275. if (vha->pendcmd[wm_id].cmd) {
  1276. #if defined(DEBUG)
  1277. snprintf(pending_txt, 24, "0x%08x/%u",
  1278. vha->pendcmd[wm_id].cmd->user_cmd.cmd_id,
  1279. vha->pendcmd[wm_id].cmd->session->id);
  1280. #endif
  1281. if (free_res) {
  1282. /* Free command resources. */
  1283. vha_wm_release_cores(vha,
  1284. vha->pendcmd[wm_id].cmd->hw_sched_info.core_mask, false);
  1285. vha_dev_free_cmd_res(vha, vha->pendcmd[wm_id].cmd, false);
  1286. vha->pri_q_counters[vha->pendcmd[wm_id].cmd->user_cmd.priority]++;
  1287. }
  1288. VHA_INC_WL_STAT(vha, kicks_aborted, vha->pendcmd[wm_id].cmd);
  1289. vha->stats.cnn_kicks_aborted++;
  1290. vha->pendcmd[wm_id].cmd->in_hw = false;
  1291. vha->pendcmd[wm_id].cmd->queued = false;
  1292. vha->pendcmd[wm_id].cmd->rolled_back = true;
  1293. vha->pendcmd[wm_id].cmd = NULL;
  1294. processing = true;
  1295. }
  1296. /* low_latency ...*/
  1297. if (vha->queuedcmd[wm_id].cmd) {
  1298. #if defined(DEBUG)
  1299. snprintf(queued_txt, 24, "0x%08x/%u",
  1300. vha->queuedcmd[wm_id].cmd->user_cmd.cmd_id,
  1301. vha->queuedcmd[wm_id].cmd->session->id);
  1302. #endif
  1303. /* Free command resources. */
  1304. vha_wm_release_cores(vha,
  1305. vha->queuedcmd[wm_id].cmd->hw_sched_info.core_mask, false);
  1306. vha_dev_free_cmd_res(vha, vha->queuedcmd[wm_id].cmd, false);
  1307. if (vha->low_latency == VHA_LL_SELF_KICK) {
  1308. VHA_INC_WL_STAT(vha, kicks_aborted, vha->queuedcmd[wm_id].cmd);
  1309. vha->stats.cnn_kicks_aborted++;
  1310. vha->pri_q_counters[vha->queuedcmd[wm_id].cmd->user_cmd.priority]++;
  1311. }
  1312. vha->queuedcmd[wm_id].cmd->in_hw = false;
  1313. vha->queuedcmd[wm_id].cmd->queued = false;
  1314. vha->queuedcmd[wm_id].cmd->rolled_back = true;
  1315. vha->queuedcmd[wm_id].cmd = NULL;
  1316. }
  1317. #if defined(DEBUG)
  1318. dev_dbg(vha->dev, "%s: WM%u pending %s, queued %s\n",
  1319. __func__, wm_id, pending_txt, queued_txt);
  1320. #endif
  1321. return processing;
  1322. }
  1323. bool vha_rollback_cmds(struct vha_dev *vha)
  1324. {
  1325. uint32_t wm_id;
  1326. bool processing = false;
  1327. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  1328. bool wm_processing = vha_rollback_wm_cmds(vha, wm_id, true);
  1329. processing = processing || wm_processing;
  1330. }
  1331. return processing;
  1332. }
  1333. static void vha_stop_processing(struct vha_dev *vha)
  1334. {
  1335. uint32_t wm_id;
  1336. VHA_LOCK_WM();
  1337. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
  1338. if (vha->pendcmd[wm_id].cmd != NULL) {
  1339. uint64_t wm_mask = VHA_CR_SETBITS(HOST_EVENT_SOURCE, WM, VHA_WM_ID_TO_MASK(wm_id));
  1340. vha_wm_reset(vha, &vha->pendcmd[wm_id].cmd->hw_sched_info);
  1341. VHA_SELECT_WM(wm_id);
  1342. /* Remove WM related interrupt info if it happens to be set. */
  1343. if (vha->irq_status.event_source & wm_mask)
  1344. {
  1345. /* Unset the WM related source bit. */
  1346. vha->irq_status.event_source &= ~wm_mask;
  1347. /* Clear all WM related events. */
  1348. IOWRITE64_CR_REGIO(vha->irq_status.wm_events[wm_id] & VHA_WM_EVENTS_DEFAULT,
  1349. WM_EVENT_CLEAR);
  1350. vha->irq_status.wm_events[wm_id] = 0ULL;
  1351. }
  1352. }
  1353. VHA_UNLOCK_WM();
  1354. }
  1355. int vha_dev_suspend_work(struct vha_dev *vha)
  1356. {
  1357. bool processing = false;
  1358. int ret;
  1359. /* Check if anything is being processed right now. */
  1360. vha_stop_processing(vha);
  1361. /* Rollback commands after hw is stopped. */
  1362. processing = vha_rollback_cmds(vha);
  1363. /* Forcing hardware disable. */
  1364. ret = vha_dev_stop(vha, processing);
  1365. return ret;
  1366. }
  1367. /*
  1368. * Handles the command (of given cmd_idx) already processed by the hw.
  1369. */
  1370. static bool vha_handle_cmd(struct vha_dev *vha, uint8_t wm_id, uint64_t status,
  1371. int err, uint64_t rsp_err_flags)
  1372. {
  1373. struct vha_cmd *cmd = NULL;
  1374. if (wm_id >= vha->hw_props.num_cnn_core_devs)
  1375. return false;
  1376. cmd = vha->pendcmd[wm_id].cmd;
  1377. if (unlikely(!cmd)) {
  1378. dev_dbg(vha->dev, "No command. Probably it has been aborted\n");
  1379. return false;
  1380. }
  1381. vha_cnn_cmd_completed(cmd, status, err, rsp_err_flags);
  1382. if (status) {
  1383. /* Rollback any queued command ... */
  1384. vha_rollback_wm_cmds(vha, wm_id, false);
  1385. /* Notify immediately current command */
  1386. vha_cmd_notify(cmd);
  1387. return false;
  1388. }
  1389. if (vha->queuedcmd[wm_id].cmd)
  1390. vha->pendcmd[wm_id].cmd = vha->queuedcmd[wm_id].cmd;
  1391. else
  1392. vha->pendcmd[wm_id].cmd = NULL;
  1393. vha->queuedcmd[wm_id].cmd = NULL;
  1394. if (vha->pendcmd[wm_id].cmd)
  1395. dev_dbg(vha->dev, "%s: WM%u 0x%08x/%u -> new pending 0x%08x/%u\n",
  1396. __func__, wm_id, cmd->user_cmd.cmd_id, cmd->session->id,
  1397. vha->pendcmd[wm_id].cmd->user_cmd.cmd_id,
  1398. vha->pendcmd[wm_id].cmd->session->id);
  1399. else
  1400. dev_dbg(vha->dev, "%s: WM%u 0x%08x/%u -> no new pending\n",
  1401. __func__, wm_id, cmd->user_cmd.cmd_id, cmd->session->id);
  1402. vha_cmd_notify(cmd);
  1403. return true;
  1404. }
  1405. void vha_dev_update_per_core_kicks(uint8_t core_mask, uint32_t *kicks_array)
  1406. {
  1407. while (core_mask != 0) {
  1408. uint32_t curr_core_id = VHA_CORE_MASK_TO_ID(core_mask);
  1409. core_mask &= ~(VHA_CORE_ID_TO_MASK(curr_core_id));
  1410. kicks_array[curr_core_id]++;
  1411. }
  1412. }
  1413. static int vha_report_wm_rsp_failure(struct vha_dev *vha, uint8_t wm_id,
  1414. uint64_t wm_rsp_status, uint64_t *core_status, uint64_t *ic_status,
  1415. enum vha_reset_type *reset_type, uint64_t *error_flags)
  1416. {
  1417. uint8_t err_code = VHA_WM_RESPONSE_GET_ERROR_CODE(wm_rsp_status);
  1418. int cmdid = -1;
  1419. int sesid = -1;
  1420. uint32_t i = 0;
  1421. int err = -EIO;
  1422. if (vha->pendcmd[wm_id].cmd) {
  1423. cmdid = vha->pendcmd[wm_id].cmd->user_cmd.cmd_id;
  1424. sesid = vha->pendcmd[wm_id].cmd->session->id;
  1425. }
  1426. if (vha_observers.error)
  1427. vha_observers.error(vha->id, sesid, cmdid, wm_rsp_status);
  1428. if (VHA_REG_GET_PARITY_ERROR(wm_rsp_status)) {
  1429. dev_err(vha->dev, " WM%u response error: PARITY\n", wm_id);
  1430. *reset_type = VHA_RESET_TYPE_WM;
  1431. *error_flags |= VHA_RSP_ERROR(SW_WM_PARITY_ERROR);
  1432. } else if (VHA_REG_GET_WL_ID_MISMATCH_ERROR(wm_rsp_status)) {
  1433. dev_err(vha->dev, " WM%u response error: WL_ID_MISMATCH\n", wm_id);
  1434. *reset_type = VHA_RESET_TYPE_WM;
  1435. *error_flags |= VHA_RSP_ERROR(SW_WL_ID_MISMATCH_ERROR);
  1436. } else if (VHA_REG_GET_CONF_ERROR(wm_rsp_status)) {
  1437. dev_err(vha->dev, " WM%u response error: CONFIRMATION_WRITES\n", wm_id);
  1438. *reset_type = VHA_RESET_TYPE_WM;
  1439. *error_flags |= VHA_RSP_ERROR(SW_CONF_ERROR);
  1440. } else if (VHA_REG_GET_COMBINED_CRC_ERROR(wm_rsp_status)) {
  1441. dev_err(vha->dev, " WM%u response error: COMBINED_CRC\n", wm_id);
  1442. *reset_type = VHA_RESET_TYPE_WM;
  1443. *error_flags |= VHA_RSP_ERROR(SW_CRC_MISMATCH_ERROR);
  1444. } else {
  1445. while (i < ARRAY_SIZE(wm_rsp_err_codes)) {
  1446. if (wm_rsp_err_codes[i].e == err_code) {
  1447. uint8_t core_id = VHA_WM_RESPONSE_GET_FAILED_CORE_IDX(wm_rsp_status);
  1448. /* Store reset type. */
  1449. *reset_type = wm_rsp_err_codes[i].reset_type;
  1450. /* Error that caused the Workload Manager to halt*/
  1451. dev_err(vha->dev, " WM%u error code:%d -> %s, failure on core%u\n",
  1452. wm_id, err_code, wm_rsp_err_codes[i].s, core_id);
  1453. *error_flags |= wm_rsp_err_codes[i].rsp_err;
  1454. if (core_id < vha->hw_props.num_cnn_core_devs) {
  1455. i = 0;
  1456. while (core_err_bits[i].e != 0) {
  1457. if (core_status[core_id] & core_err_bits[i].b) {
  1458. dev_err(vha->dev, " %s\n", core_err_bits[i].s);
  1459. err = core_err_bits[i].e;
  1460. }
  1461. i++;
  1462. }
  1463. i = 0;
  1464. while (ic_err_bits[i].e != 0) {
  1465. if (ic_status[core_id] & ic_err_bits[i].b) {
  1466. dev_err(vha->dev, " %s\n", ic_err_bits[i].s);
  1467. err = ic_err_bits[i].e;
  1468. }
  1469. i++;
  1470. }
  1471. } else
  1472. dev_err(vha->dev, " invalid FAILED_CORE_ID, should be <%u\n",
  1473. vha->hw_props.num_cnn_core_devs);
  1474. goto exit;
  1475. }
  1476. i++;
  1477. }
  1478. dev_err(vha->dev, " invalid WM ERROR_CODE: %u\n", err_code);
  1479. }
  1480. exit:
  1481. return err;
  1482. }
  1483. static void vha_handle_sys_failure(struct vha_dev *vha, uint64_t status, int err, uint64_t rsp_err_flags)
  1484. {
  1485. int cmdid = -1;
  1486. int sesid = -1;
  1487. uint32_t wm_id;
  1488. struct vha_cmd *cmd = NULL;
  1489. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  1490. cmd = vha->pendcmd[wm_id].cmd;
  1491. if (cmd) {
  1492. cmdid = cmd->user_cmd.cmd_id;
  1493. sesid = cmd->session->id;
  1494. }
  1495. if (vha_observers.error)
  1496. vha_observers.error(vha->id, sesid, cmdid, status);
  1497. cmdid = -1;
  1498. if (cmd) {
  1499. /* Update stats. */
  1500. vha->stats.total_failures++;
  1501. vha->stats.cnn_kicks_completed++;
  1502. VHA_INC_WL_STAT(vha, kicks_completed, cmd);
  1503. vha_wm_reset(vha, &cmd->hw_sched_info);
  1504. /* Free command resources. */
  1505. vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
  1506. vha_dev_free_cmd_res(vha, cmd, true);
  1507. }
  1508. /* Move command queue. */
  1509. vha_do_queued_cmd(vha, wm_id);
  1510. /* Handle actual command */
  1511. vha_handle_cmd(vha, wm_id, status, err, rsp_err_flags);
  1512. }
  1513. }
  1514. static void vha_handle_wm_failure(struct vha_dev *vha, uint8_t wm_id,
  1515. uint64_t status, int err, uint64_t rsp_err_flags)
  1516. {
  1517. int cmdid = -1;
  1518. int sesid = -1;
  1519. struct vha_cmd *cmd = NULL;
  1520. cmd = vha->pendcmd[wm_id].cmd;
  1521. if (cmd) {
  1522. cmdid = cmd->user_cmd.cmd_id;
  1523. sesid = cmd->session->id;
  1524. }
  1525. if (vha_observers.error)
  1526. vha_observers.error(vha->id, sesid, cmdid, status);
  1527. if (cmd) {
  1528. /* Update stats. */
  1529. vha->stats.total_failures++;
  1530. vha->stats.cnn_kicks_completed++;
  1531. VHA_INC_WL_STAT(vha, kicks_completed, cmd);
  1532. /* Free command resources. */
  1533. vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
  1534. vha_dev_free_cmd_res(vha, cmd, true);
  1535. }
  1536. /* Move command queue. */
  1537. vha_do_queued_cmd(vha, wm_id);
  1538. /* Handle actual command */
  1539. vha_handle_cmd(vha, wm_id, status, err, rsp_err_flags);
  1540. }
  1541. static enum vha_reset_type vha_sys_get_reset_type(struct vha_dev *vha,
  1542. uint64_t event_mask) {
  1543. enum vha_reset_type sys_reset_type = VHA_RESET_TYPE_NONE;
  1544. #ifdef VHA_SCF
  1545. uint64_t sys_err_events = VHA_SYS_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
  1546. #else
  1547. uint64_t sys_err_events = VHA_SYS_ERR_EVENTS;
  1548. #endif
  1549. if (event_mask & sys_err_events) {
  1550. uint32_t i = 0;
  1551. while (sys_err_bits[i].e != 0) {
  1552. if (event_mask & sys_err_bits[i].b) {
  1553. /* Indicate the highest reset level of all errors. */
  1554. if (sys_err_bits[i].reset_type > sys_reset_type)
  1555. sys_reset_type = sys_err_bits[i].reset_type;
  1556. }
  1557. i++;
  1558. }
  1559. }
  1560. return sys_reset_type;
  1561. }
  1562. static void vha_sys_get_wm_reset_types(struct vha_dev *vha, uint64_t event_mask,
  1563. enum vha_reset_type *wm_reset_types) {
  1564. uint8_t wm_id;
  1565. uint8_t pf_errors;
  1566. #ifdef VHA_SCF
  1567. // uint8_t parity_errors;
  1568. #endif
  1569. /* Check MMU page fault errors. */
  1570. pf_errors = (uint8_t)VHA_CR_GETBITS(SYS_EVENT_STATUS, MMU_PAGE_FAULT,
  1571. event_mask);
  1572. if (pf_errors) {
  1573. wm_id = 0;
  1574. while(wm_id < vha->hw_props.num_cnn_core_devs) {
  1575. if (pf_errors & (1 << wm_id))
  1576. wm_reset_types[wm_id] = VHA_RESET_TYPE_MMU;
  1577. else
  1578. wm_reset_types[wm_id] = VHA_RESET_TYPE_NONE;
  1579. wm_id++;
  1580. }
  1581. }
  1582. #ifdef VHA_SCF
  1583. /* Check MMU parity errors. */
  1584. // uint8_t parity_errors = (uint8_t)VHA_CR_GETBITS(SYS_EVENT_STATUS, MMU_PARITY_ERROR,
  1585. // event_mask);
  1586. // if (parity_errors) {
  1587. // wm_id = 0;
  1588. // while(wm_id < vha->hw_props.num_cnn_core_devs) {
  1589. // if (parity_errors & (1 << wm_id))
  1590. // wm_reset_types[wm_id] = VHA_RESET_TYPE_MMU;
  1591. // else
  1592. // wm_reset_types[wm_id] = VHA_RESET_TYPE_NONE;
  1593. // wm_id++;
  1594. // }
  1595. // }
  1596. #endif
  1597. }
  1598. static enum vha_reset_type vha_wm_get_reset_type(struct vha_dev *vha,
  1599. uint64_t event_mask) {
  1600. enum vha_reset_type wm_reset_type = VHA_RESET_TYPE_NONE;
  1601. #ifdef VHA_SCF
  1602. uint64_t wm_err_events = VHA_WM_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
  1603. #else
  1604. uint64_t wm_err_events = VHA_WM_ERR_EVENTS;
  1605. #endif
  1606. if (event_mask & wm_err_events) {
  1607. uint32_t i = 0;
  1608. while (wm_err_bits[i].e != 0) {
  1609. if (event_mask & wm_err_bits[i].b) {
  1610. /* Indicate the highest reset level of all errors. */
  1611. if (wm_err_bits[i].reset_type > wm_reset_type)
  1612. wm_reset_type = wm_err_bits[i].reset_type;
  1613. }
  1614. i++;
  1615. }
  1616. }
  1617. return wm_reset_type;
  1618. }
  1619. static enum vha_reset_type vha_core_get_reset_type(struct vha_dev *vha,
  1620. uint64_t event_mask) {
  1621. enum vha_reset_type core_reset_type = VHA_RESET_TYPE_NONE;
  1622. uint64_t core_err_events = VHA_CORE_ERR_EVENTS;
  1623. if (event_mask & core_err_events) {
  1624. uint32_t i = 0;
  1625. while (core_err_bits[i].e != 0) {
  1626. if (event_mask & core_err_bits[i].b) {
  1627. /* Indicate the highest reset level of all errors. */
  1628. if (core_err_bits[i].reset_type > core_reset_type)
  1629. core_reset_type = core_err_bits[i].reset_type;
  1630. }
  1631. i++;
  1632. }
  1633. }
  1634. return core_reset_type;
  1635. }
  1636. static enum vha_reset_type vha_ic_get_reset_type(struct vha_dev *vha,
  1637. uint64_t event_mask) {
  1638. enum vha_reset_type ic_reset_type = VHA_RESET_TYPE_NONE;
  1639. #ifdef VHA_SCF
  1640. uint64_t ic_err_events = VHA_IC_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
  1641. #else
  1642. uint64_t ic_err_events = VHA_IC_ERR_EVENTS;
  1643. #endif
  1644. if (event_mask & ic_err_events) {
  1645. uint32_t i = 0;
  1646. while (ic_err_bits[i].e != 0) {
  1647. if (event_mask & ic_err_bits[i].b) {
  1648. /* Indicate the highest reset level of all errors. */
  1649. if (ic_err_bits[i].reset_type > ic_reset_type)
  1650. ic_reset_type = ic_err_bits[i].reset_type;
  1651. }
  1652. i++;
  1653. }
  1654. }
  1655. return ic_reset_type;
  1656. }
  1657. static int vha_report_sys_failures(struct vha_dev *vha, uint64_t event_mask, uint64_t *error_flags)
  1658. {
  1659. int error = 0;
  1660. uint32_t i;
  1661. bool print_header = true;
  1662. uint8_t pf_status;
  1663. /* Print event status in human readable form. */
  1664. i = 0;
  1665. while (sys_err_bits[i].e != 0) {
  1666. if (event_mask & sys_err_bits[i].b) {
  1667. if (print_header) {
  1668. dev_err(vha->dev, " SYS event status:\n");
  1669. print_header = false;
  1670. }
  1671. dev_err(vha->dev, " %s\n", sys_err_bits[i].s);
  1672. /* Convert from register bits into POSIX errno.
  1673. * If multiple errors, then arbitrary errno choice. */
  1674. error = sys_err_bits[i].e;
  1675. *error_flags |= sys_err_bits[i].rsp_err;
  1676. }
  1677. i++;
  1678. }
  1679. if (error) {
  1680. dev_err(vha->dev, " SYS failure:\n");
  1681. dev_err(vha->dev, " SYS_CLK_STATUS0: 0x%016llx\n",
  1682. IOREAD64_CR_REGIO(SYS_CLK_STATUS0));
  1683. dev_err(vha->dev, " SYS_EVENT_STATUS: 0x%016llx\n",
  1684. event_mask);
  1685. for (i = 0; i < vha->hw_props.num_cnn_core_devs; i++) {
  1686. if (vha->active_core_mask & (1 << i)) {
  1687. /* Select core to read clocks from. */
  1688. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
  1689. MASK, (1 << i)),
  1690. CORE_CTRL_INDIRECT);
  1691. dev_err(vha->dev, " CORE%u CLK_STATUS0: 0x%016llx\n",
  1692. i, IOREAD64_CR_REGIO(CLK_STATUS0));
  1693. }
  1694. }
  1695. }
  1696. if (error == -ETIMEDOUT) {
  1697. dev_err(vha->dev, " SLC_STATUS1: 0x%016llx\n",
  1698. IOREAD64_CR_REGIO(SLC_STATUS1));
  1699. dev_err(vha->dev, " SLC_STATUS2: 0x%016llx\n",
  1700. IOREAD64_CR_REGIO(SLC_STATUS2));
  1701. dev_err(vha->dev, " SLC_IDLE: 0x%016llx\n",
  1702. IOREAD64_CR_REGIO(SLC_IDLE));
  1703. }
  1704. /* Additionally report MMU PF failure if occurred. */
  1705. pf_status = (uint8_t)VHA_CR_GETBITS(SYS_EVENT_STATUS, MMU_PAGE_FAULT,
  1706. event_mask);
  1707. if (pf_status) {
  1708. /* dump mmu status */
  1709. vha_mmu_status(vha, pf_status);
  1710. }
  1711. return error;
  1712. }
  1713. static int vha_report_wm_failures(struct vha_dev *vha, uint8_t wm_id, uint64_t event_mask, uint64_t *error_flags)
  1714. {
  1715. int error = 0;
  1716. uint32_t i;
  1717. bool print_header = true;
  1718. /* Print event status in human readable form. */
  1719. i = 0;
  1720. print_header = true;
  1721. while (wm_err_bits[i].e != 0) {
  1722. if (event_mask & wm_err_bits[i].b) {
  1723. if (print_header) {
  1724. dev_err(vha->dev, " WM%u event status:\n", wm_id);
  1725. print_header = false;
  1726. }
  1727. dev_err(vha->dev, " %s\n", wm_err_bits[i].s);
  1728. /* Convert from register bits into POSIX errno.
  1729. * If multiple errors, then arbitrary errno choice. */
  1730. error = wm_err_bits[i].e;
  1731. *error_flags |= wm_err_bits[i].rsp_err;
  1732. }
  1733. i++;
  1734. }
  1735. if (error == -ETIMEDOUT) {
  1736. vha_wm_status(vha, wm_id, vha_wm_get_cores(vha, wm_id));
  1737. }
  1738. return error;
  1739. }
  1740. static int vha_report_core_failures(struct vha_dev *vha, uint8_t core_id, uint64_t event_mask, uint64_t *error_flags)
  1741. {
  1742. int error = 0;
  1743. uint32_t i;
  1744. bool print_header = true;
  1745. /* Print event status in human readable form. */
  1746. i = 0;
  1747. print_header = true;
  1748. while (core_err_bits[i].e != 0) {
  1749. if (event_mask & core_err_bits[i].b) {
  1750. if (print_header) {
  1751. dev_err(vha->dev, " Core %u event status:\n", core_id);
  1752. print_header = false;
  1753. }
  1754. dev_err(vha->dev, " %s\n", core_err_bits[i].s);
  1755. /* Convert from register bits into POSIX errno.
  1756. * If multiple errors, then arbitrary errno choice. */
  1757. error = core_err_bits[i].e;
  1758. *error_flags |= core_err_bits[i].rsp_err;
  1759. }
  1760. i++;
  1761. }
  1762. return error;
  1763. }
  1764. static int vha_report_ic_failures(struct vha_dev *vha, uint8_t core_id, uint64_t event_mask, uint64_t *error_flags)
  1765. {
  1766. int error = 0;
  1767. uint32_t i;
  1768. bool print_header = true;
  1769. /* Print event status in human readable form. */
  1770. i = 0;
  1771. print_header = true;
  1772. while (ic_err_bits[i].e != 0) {
  1773. if (event_mask & ic_err_bits[i].b) {
  1774. if (print_header) {
  1775. dev_err(vha->dev, " IC %u event status:\n", core_id);
  1776. print_header = false;
  1777. }
  1778. dev_err(vha->dev, " %s\n", ic_err_bits[i].s);
  1779. /* Convert from register bits into POSIX errno.
  1780. * If multiple errors, then arbitrary errno choice. */
  1781. error = ic_err_bits[i].e;
  1782. *error_flags |= ic_err_bits[i].rsp_err;
  1783. }
  1784. i++;
  1785. }
  1786. return error;
  1787. }
  1788. static uint8_t vha_events_process_errors(struct vha_dev *vha,
  1789. struct vha_mc_irq_status *irq_status, bool *full_reset,
  1790. bool *process_sys_events, uint64_t *error_flags) {
  1791. int error = 0;
  1792. int wm_error = 0;
  1793. int core_error = 0;
  1794. int ic_error = 0;
  1795. uint8_t wm_process_mask = 0;
  1796. uint8_t wm_source_mask = 0;
  1797. uint8_t wm_id;
  1798. enum vha_reset_type reset_type = VHA_RESET_TYPE_NONE;
  1799. enum vha_reset_type wm_reset_type = VHA_RESET_TYPE_NONE;
  1800. enum vha_reset_type wm_reset_types[VHA_NUM_CORES] = {0};
  1801. uint64_t sys_err_status = 0;
  1802. uint64_t wm_err_status_full_reset = 0;
  1803. uint64_t wm_err_statuses[VHA_NUM_CORES] = {0};
  1804. enum vha_reset_type core_reset_type = 0;
  1805. enum vha_reset_type ic_reset_type = 0;
  1806. uint8_t core_id;
  1807. #define COMBINE_SYS_WM_STATUS(s, w) \
  1808. (((w & ~((uint64_t)VHA_WM_ERR_EVENTS)) | s) | \
  1809. ((w & ((uint64_t)VHA_WM_ERR_EVENTS)) << 32))
  1810. #define INSERT_WM_ERROR(s, e) \
  1811. ((s | ((uint64_t)e)) << 32)
  1812. /* Assume no full reset. */
  1813. *full_reset = false;
  1814. /* Assume no SYS events. */
  1815. *process_sys_events = false;
  1816. /* Process SYS events. */
  1817. if (VHA_CR_GETBITS(HOST_EVENT_SOURCE, SYS, irq_status->event_source)) {
  1818. #ifdef VHA_SCF
  1819. uint64_t sys_err_events = VHA_SYS_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
  1820. #else
  1821. uint64_t sys_err_events = VHA_SYS_ERR_EVENTS;
  1822. #endif
  1823. sys_err_status = irq_status->sys_events & sys_err_events;
  1824. if (sys_err_status) {
  1825. /* Determine reset types. */
  1826. reset_type = vha_sys_get_reset_type(vha, irq_status->sys_events);
  1827. if (reset_type < VHA_RESET_TYPE_FULL) {
  1828. vha_sys_get_wm_reset_types(vha, irq_status->sys_events, wm_reset_types);
  1829. wm_id = 0;
  1830. while(wm_id < vha->hw_props.num_cnn_core_devs) {
  1831. if (wm_reset_types[wm_id] > VHA_RESET_TYPE_NONE)
  1832. wm_err_statuses[wm_id] = sys_err_status;
  1833. wm_id++;
  1834. }
  1835. }
  1836. /* Report SYS errors. */
  1837. error = vha_report_sys_failures(vha, irq_status->sys_events, error_flags);
  1838. }
  1839. /* If no full reset is requested at this stage
  1840. * and there are non-error SYS events raised,
  1841. * signal them to be processed too. */
  1842. if ((reset_type < VHA_RESET_TYPE_FULL) &&
  1843. (irq_status->sys_events & ~sys_err_events))
  1844. *process_sys_events = true;
  1845. }
  1846. /* Process WM events. */
  1847. /* Read WM event source mask. */
  1848. wm_source_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, WM,
  1849. irq_status->event_source);
  1850. if (wm_source_mask)
  1851. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
  1852. if (wm_source_mask & (1 << wm_id)) {
  1853. #ifdef VHA_SCF
  1854. uint64_t wm_err_events = VHA_WM_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
  1855. #else
  1856. uint64_t wm_err_events = VHA_WM_ERR_EVENTS;
  1857. #endif
  1858. uint64_t wm_err_status = irq_status->wm_events[wm_id] & wm_err_events;
  1859. if (wm_err_status) {
  1860. /* If no full reset is requested... */
  1861. if (reset_type < VHA_RESET_TYPE_FULL) {
  1862. /* Determine reset type for this WM. */
  1863. wm_reset_type = vha_wm_get_reset_type(
  1864. vha, irq_status->wm_events[wm_id]);
  1865. /* If full reset is requested for this WM, just skip
  1866. * checking other ones. Otherwise update reset type
  1867. * for this WM if needed. */
  1868. if (wm_reset_type == VHA_RESET_TYPE_FULL) {
  1869. reset_type = VHA_RESET_TYPE_FULL;
  1870. wm_err_status_full_reset = wm_err_status;
  1871. } else if (wm_reset_type > wm_reset_types[wm_id])
  1872. wm_reset_types[wm_id] = wm_reset_type;
  1873. }
  1874. /* Compose accumulated error status. */
  1875. wm_err_statuses[wm_id] =
  1876. COMBINE_SYS_WM_STATUS(sys_err_status, wm_err_status);
  1877. /* Report WM errors. */
  1878. wm_error = vha_report_wm_failures(vha, wm_id,
  1879. irq_status->wm_events[wm_id], error_flags);
  1880. /* If no SYS error reported, get the first WM one. */
  1881. if (error == 0)
  1882. error = wm_error;
  1883. }
  1884. }
  1885. /* Process core events */
  1886. for (core_id = 0; core_id < vha->hw_props.num_cnn_core_devs; core_id++)
  1887. if (irq_status->core_events[core_id] & VHA_CORE_ERR_EVENTS) {
  1888. /* Determine reset type for this Core. */
  1889. core_reset_type = vha_core_get_reset_type(vha, irq_status->core_events[core_id]);
  1890. /* We do not reset the core itself, instead, we need to reset
  1891. the WM that used it, so let's find it */
  1892. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
  1893. if (vha->pendcmd[wm_id].cmd != NULL) {
  1894. uint8_t core_mask = vha->pendcmd[wm_id].cmd->hw_sched_info.core_mask;
  1895. if (core_mask & (1 << core_id)) {
  1896. /* Override wm reset type */
  1897. if (core_reset_type == VHA_RESET_TYPE_FULL)
  1898. reset_type = VHA_RESET_TYPE_FULL;
  1899. else if (core_reset_type > wm_reset_types[wm_id])
  1900. wm_reset_types[wm_id] = core_reset_type;
  1901. core_error = vha_report_core_failures(vha, core_id,
  1902. irq_status->core_events[core_id], error_flags);
  1903. /* If no SYS or WM error reported, get the first Core one. */
  1904. if (error == 0)
  1905. error = core_error;
  1906. /* Add core error to this WM's status. */
  1907. wm_err_statuses[wm_id] =
  1908. INSERT_WM_ERROR(wm_err_statuses[wm_id], VHA_REG_WM_CORE_ERROR_EN);
  1909. }
  1910. }
  1911. }
  1912. /* Process IC events */
  1913. for (core_id = 0; core_id < vha->hw_props.num_cnn_core_devs; core_id++) {
  1914. #ifdef VHA_SCF
  1915. uint64_t ic_err_events = VHA_IC_ERR_EVENTS | VHA_REG_PARITY_ERROR_EN;
  1916. #else
  1917. uint64_t ic_err_events = VHA_IC_ERR_EVENTS;
  1918. #endif
  1919. if (irq_status->ic_events[core_id] & ic_err_events) {
  1920. /* Determine reset type for this Core. */
  1921. ic_reset_type = vha_ic_get_reset_type(vha, irq_status->ic_events[core_id]);
  1922. /* We do not reset the core itself, instead, we need to reset
  1923. the WM that used it, so let's find it */
  1924. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++)
  1925. if (vha->pendcmd[wm_id].cmd != NULL) {
  1926. uint8_t core_mask = vha->pendcmd[wm_id].cmd->hw_sched_info.core_mask;
  1927. if (core_mask & (1 << core_id)) {
  1928. /* Override wm reset type */
  1929. if (ic_reset_type == VHA_RESET_TYPE_FULL)
  1930. reset_type = VHA_RESET_TYPE_FULL;
  1931. else if (ic_reset_type > wm_reset_types[wm_id])
  1932. wm_reset_types[wm_id] = ic_reset_type;
  1933. ic_error = vha_report_ic_failures(vha, core_id,
  1934. irq_status->ic_events[core_id], error_flags);
  1935. /* If no SYS or WM error reported, get the first Core one. */
  1936. if (error == 0)
  1937. error = ic_error;
  1938. /* Add IC error to this WM's status. */
  1939. wm_err_statuses[wm_id] =
  1940. INSERT_WM_ERROR(wm_err_statuses[wm_id], VHA_REG_WM_IC_ERROR_EN);
  1941. }
  1942. }
  1943. }
  1944. }
  1945. /* Perform selective resets. */
  1946. if (reset_type < VHA_RESET_TYPE_FULL) {
  1947. int ret;
  1948. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  1949. struct vha_cmd *cmd = vha->pendcmd[wm_id].cmd;
  1950. switch (wm_reset_types[wm_id]) {
  1951. case VHA_RESET_TYPE_MMU:
  1952. if (cmd) {
  1953. /* Invalidate MMU. */
  1954. ret = vha_mmu_flush_ctx(vha, cmd->session->mmu_ctxs[VHA_MMU_REQ_IO_CTXID].hw_id);
  1955. if(ret) {
  1956. dev_err(vha->dev, "Error during MMU flush, doing full reset\n");
  1957. wm_err_status_full_reset = wm_err_statuses[wm_id];
  1958. reset_type = VHA_RESET_TYPE_FULL;
  1959. break;
  1960. }
  1961. }
  1962. // fall through
  1963. case VHA_RESET_TYPE_WM:
  1964. dev_err(vha->dev, "Performing wm%d reset due to HW error detection.", wm_id);
  1965. if (cmd)
  1966. /* Reset WM and assigned cores. */
  1967. ret = vha_wm_reset(vha, &cmd->hw_sched_info);
  1968. else {
  1969. /* Just reset WM. */
  1970. struct vha_hw_sched_info sched_info = {
  1971. .wm_id = wm_id,
  1972. .core_mask = 0
  1973. };
  1974. ret = vha_wm_reset(vha, &sched_info);
  1975. }
  1976. if(ret) {
  1977. dev_err(vha->dev, "Error during WM%d reset, doing full reset\n", wm_id);
  1978. wm_err_status_full_reset = wm_err_statuses[wm_id];
  1979. reset_type = VHA_RESET_TYPE_FULL;
  1980. break;
  1981. }
  1982. VHA_LOCK_WM();
  1983. VHA_SELECT_WM(wm_id);
  1984. /* Clear all WM related events. */
  1985. IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_CLEAR);
  1986. /* Re-enable WM events here as this WM will not be handled further. */
  1987. IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
  1988. VHA_UNLOCK_WM();
  1989. /* Handle pending command. */
  1990. vha_handle_wm_failure(vha, wm_id, wm_err_statuses[wm_id], error, *error_flags);
  1991. break;
  1992. case VHA_RESET_TYPE_NONE:
  1993. /* Mark WM source for normal processing if it was signalled. */
  1994. if (cmd)
  1995. wm_process_mask |= wm_source_mask & (1 << wm_id);
  1996. break;
  1997. default:
  1998. break;
  1999. }
  2000. }
  2001. }
  2002. /* check once again, reset_type may have been updated due to failure during reset procedure */
  2003. if(reset_type == VHA_RESET_TYPE_FULL){
  2004. /* Handle all pending commands. */
  2005. vha_handle_sys_failure(vha,
  2006. COMBINE_SYS_WM_STATUS(sys_err_status, wm_err_status_full_reset), error, *error_flags);
  2007. /* Full reset is requested anyway, so skip processing further SYS events. */
  2008. *process_sys_events = false;
  2009. /* Full reset will be executed outside. Just indicate here
  2010. * that it's required.*/
  2011. *full_reset = true;
  2012. }
  2013. return wm_process_mask;
  2014. }
  2015. /* if vha event register reports WM events, so handle them */
  2016. static void vha_handle_wm_response(struct vha_dev *vha, uint8_t wm_id,
  2017. uint64_t response_status, uint64_t *core_status_array,
  2018. uint64_t *ic_status_array, bool *full_reset, uint64_t *error_flags)
  2019. {
  2020. enum vha_reset_type reset_type = VHA_RESET_TYPE_NONE;
  2021. int err = *error_flags ? -EIO : 0 ;
  2022. if (response_status &
  2023. (VHA_WM_RESPONSE_STATUS(WL_FAILURE) |
  2024. VHA_REG_PARITY_ERROR_EN |
  2025. VHA_REG_WL_ID_MISMATCH_ERROR_EN |
  2026. VHA_REG_CONF_ERROR_EN |
  2027. VHA_REG_COMBINED_CRC_ERROR_EN)) {
  2028. err = vha_report_wm_rsp_failure(vha, wm_id, response_status,
  2029. core_status_array, ic_status_array, &reset_type, error_flags);
  2030. }
  2031. /* Move command queue. */
  2032. switch (reset_type) {
  2033. case VHA_RESET_TYPE_NONE:
  2034. vha_do_queued_cmd(vha, wm_id);
  2035. break;
  2036. case VHA_RESET_TYPE_WM:
  2037. if (!*full_reset && vha->pendcmd[wm_id].cmd) {
  2038. dev_err(vha->dev, "Performing wm%d reset due to HW error detection.", wm_id);
  2039. if (vha_wm_reset(vha, &vha->pendcmd[wm_id].cmd->hw_sched_info)) {
  2040. dev_err(vha->dev, "%s: Error during WM%u reset, forcing full reset upon finish",
  2041. __func__, wm_id);
  2042. *full_reset = true;
  2043. }
  2044. }
  2045. break;
  2046. case VHA_RESET_TYPE_FULL:
  2047. *full_reset = true;
  2048. break;
  2049. default:
  2050. break;
  2051. }
  2052. /* Handle actual command */
  2053. if (vha_handle_cmd(vha, wm_id, response_status, err, *error_flags) == false)
  2054. reset_type = VHA_RESET_TYPE_NONE;
  2055. }
  2056. #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
  2057. /* Simulating hw execution time by scheduling this delayed work. */
  2058. void vha_dummy_worker(struct work_struct *work)
  2059. {
  2060. struct vha_dummy_work *dummy_work =
  2061. container_of(work, struct vha_dummy_work, dummy_dwork.work);
  2062. struct vha_dev *vha = dummy_work->vha;
  2063. struct vha_cmd *cmd;
  2064. mutex_lock(&vha->lock);
  2065. cmd = vha->pendcmd[dummy_work->wm_id].cmd;
  2066. if (cmd) {
  2067. uint64_t error_flags = 0;
  2068. bool full_reset = false;
  2069. /* Record hw processing end timestamps */
  2070. VHA_WM_STAT_SHIFT_PROC_END(vha, cmd->hw_sched_info.wm_id);
  2071. GETNSTIMEOFDAY(&vha->stats.wm_stats[cmd->hw_sched_info.wm_id].hw_proc_end);
  2072. /* Update per core/WM stats. */
  2073. VHA_INC_WL_STAT(vha, kicks_completed, cmd);
  2074. vha->stats.cnn_kicks_completed++;
  2075. /* Free command resources. */
  2076. if (!vha->hw_sched_status.assignments[cmd->hw_sched_info.assignment_id].queued)
  2077. vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
  2078. vha_dev_free_cmd_res(vha, cmd, true);
  2079. /* Handle current pending command */
  2080. vha_handle_wm_response(vha, dummy_work->wm_id, 0, NULL, NULL, &full_reset, &error_flags);
  2081. /* Schedule following commands */
  2082. vha_chk_cmd_queues(vha, true);
  2083. }
  2084. mutex_unlock(&vha->lock);
  2085. }
  2086. #endif
  2087. #ifdef VHA_SCF
  2088. static void vha_handle_conf_status(struct vha_dev *vha, struct vha_cmd *cmd, bool *full_reset, uint64_t *status)
  2089. {
  2090. if (wait_for_completion_timeout(&cmd->conf_done, msecs_to_jiffies(CONF_WRITES_WAIT_TIMEOUT_MS))) {
  2091. if (cmd->conf_top_error) {
  2092. dev_err(vha->dev, "CONF_ERR_TOP\n");
  2093. *full_reset = true;
  2094. VHA_REG_SET_CONF_ERROR(*status);
  2095. return;
  2096. }
  2097. if (cmd->conf_core_error) {
  2098. dev_err(vha->dev, "CONF_ERR_BOTTOM\n");
  2099. VHA_REG_SET_CONF_ERROR(*status);
  2100. return;
  2101. }
  2102. } else {
  2103. dev_err(vha->dev, "Confirmation writes procedure failed!\n");
  2104. VHA_REG_SET_CONF_ERROR(*status);
  2105. }
  2106. }
  2107. static void vha_check_crc(struct vha_dev *vha, struct vha_cmd *cmd, uint64_t *status)
  2108. {
  2109. struct vha_session *session = cmd->session;
  2110. struct vha_hw_sched_info *sched_info = &cmd->hw_sched_info;
  2111. uint32_t core_id = 0;
  2112. uint32_t idx = 0;
  2113. uint8_t num_cores = VHA_CORE_MASK_TO_NUM(sched_info->core_mask);
  2114. uint32_t crcs[VHA_MAX_CORES];
  2115. uint32_t *golden_crcs = NULL;
  2116. struct vha_buffer *buf = session->cnn_dbg.cnn_combined_crc;
  2117. bool crc_enabled = !!(cmd->user_cmd.flags & VHA_CHECK_CRC);
  2118. if (!buf || !buf->kptr) {
  2119. dev_err(vha->dev, "%s: Invalid crc buf\n", __func__);
  2120. return;
  2121. }
  2122. img_mem_sync_device_to_cpu(session->mem_ctx, buf->id);
  2123. for (core_id = 0; core_id < VHA_MAX_CORES; core_id++)
  2124. if (sched_info->core_mask & (1 << core_id)) {
  2125. memcpy(&crcs[idx], (uint8_t*)buf->kptr + core_id * VHA_COMBINED_CRC_CORE_OFFSET, sizeof(crcs[0]));
  2126. idx++;
  2127. }
  2128. vha_update_crcs(vha, crcs, num_cores);
  2129. if (crc_enabled) {
  2130. struct vha_user_cnn_submit_multi_cmd *cnn_user_cmd =
  2131. (struct vha_user_cnn_submit_multi_cmd *)&cmd->user_cmd;
  2132. golden_crcs = cnn_user_cmd->crcs;
  2133. for (idx = 0; idx < num_cores; idx++)
  2134. if (crcs[idx] != golden_crcs[idx]) {
  2135. VHA_REG_SET_COMBINED_CRC_ERROR(*status);
  2136. dev_err(vha->dev, "%s: combined CRC mismatch !!!\n"
  2137. "\tcrc %x\n"
  2138. "\tgolden_crc %x\n", __func__, crcs[idx], golden_crcs[idx]);
  2139. } else {
  2140. dev_info(vha->dev, "%s: combined CRC ok, crc %x\n", __func__, crcs[idx]);
  2141. }
  2142. }
  2143. }
  2144. #endif
  2145. /* Bottom half */
  2146. irqreturn_t vha_handle_thread_irq(struct device *dev)
  2147. {
  2148. struct vha_dev *vha = vha_dev_get_drvdata(dev);
  2149. irqreturn_t ret = IRQ_HANDLED;
  2150. struct vha_mc_irq_status irq_status;
  2151. uint64_t multi_src_mask = 0;
  2152. uint8_t id;
  2153. uint8_t wm_id;
  2154. uint8_t wm_process_mask = 0;
  2155. bool full_reset = false;
  2156. bool process_sys_events = false;
  2157. uint64_t error_flags = 0;
  2158. if (!vha)
  2159. return IRQ_NONE;
  2160. mutex_lock(&vha->lock);
  2161. #ifdef CONFIG_FAULT_INJECTION
  2162. if (!vha->irq_bh_pid)
  2163. vha->irq_bh_pid = task_pid_nr(current);
  2164. if (vha->fault_inject & VHA_FI_IRQ_WORKER)
  2165. current->make_it_fail = true;
  2166. else
  2167. current->make_it_fail = false;
  2168. #endif
  2169. spin_lock_irq(&vha->irq_lock);
  2170. irq_status = vha->irq_status;
  2171. memset(&vha->irq_status, 0, sizeof(vha->irq_status));
  2172. if (irq_status.sys_events || vha->do_calibration) {
  2173. uint64_t proc_time = 0;
  2174. if (get_timespan_us(&vha->stats.wm_stats[VHA_CALIBRATION_WM_ID].hw_proc_start,
  2175. &vha->stats.wm_stats[VHA_CALIBRATION_WM_ID].hw_proc_end,
  2176. &proc_time)) {
  2177. vha->stats.last_proc_us = proc_time;
  2178. } else {
  2179. vha->stats.last_proc_us = 0;
  2180. }
  2181. }
  2182. spin_unlock_irq(&vha->irq_lock);
  2183. /* Read CORE event source mask. */
  2184. multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, CORE,
  2185. irq_status.event_source);
  2186. /* Check for clock calibration first. */
  2187. if ((multi_src_mask == VHA_CALIBRATION_CORE_MASK) &&
  2188. (irq_status.core_events[VHA_CALIBRATION_CORE_ID] &
  2189. VHA_CORE_EVENT_TYPE(CORE_WDT))) {
  2190. if (vha_check_calibration(vha)) {
  2191. goto calibration_end;
  2192. }
  2193. }
  2194. /* Read core/interconnect events if System or WM event occurred */
  2195. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  2196. uint8_t mask = 0;
  2197. if (irq_status.wm_events[wm_id])
  2198. mask = vha_wm_get_cores(vha, wm_id);
  2199. if (irq_status.sys_events)
  2200. mask |= (1 << wm_id);
  2201. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  2202. if(mask & (1 << id)) {
  2203. /* Select core to read events from. */
  2204. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT,
  2205. MASK, (1 << id)),
  2206. CORE_CTRL_INDIRECT);
  2207. irq_status.core_events[id] |= IOREAD64_CR_REGIO(CORE_EVENT_WM_STATUS);
  2208. if (irq_status.core_events[id] & VHA_CORE_ERR_EVENTS) {
  2209. IOWRITE64_CR_REGIO(irq_status.core_events[id] & VHA_CORE_EVENTS_DEFAULT, CORE_EVENT_WM_CLEAR);
  2210. irq_status.core_events[id] |= IOREAD64_CR_REGIO(CORE_EVENT_WM_STATUS);
  2211. }
  2212. /* Select IC to read events from. */
  2213. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(IC_CORE_INDIRECT,
  2214. MASK, (1 << id)),
  2215. IC_CORE_INDIRECT);
  2216. irq_status.ic_events[id] |= IOREAD64_CR_REGIO(INTERCONNECT_EVENT_WM_STATUS);
  2217. if (irq_status.ic_events[id] & VHA_IC_ERR_EVENTS) {
  2218. IOWRITE64_CR_REGIO(irq_status.ic_events[id] & VHA_IC_EVENTS_DEFAULT, INTERCONNECT_EVENT_WM_CLEAR);
  2219. irq_status.ic_events[id] |= IOREAD64_CR_REGIO(INTERCONNECT_EVENT_WM_STATUS);
  2220. }
  2221. }
  2222. }
  2223. }
  2224. /* Process errors first. */
  2225. wm_process_mask = vha_events_process_errors(vha, &irq_status,
  2226. &full_reset, &process_sys_events, &error_flags);
  2227. /* Process non-error system events. */
  2228. if (process_sys_events) {
  2229. /* Handle normal system events. */
  2230. }
  2231. /* Process non-failed WM events. */
  2232. if (wm_process_mask) {
  2233. uint64_t rsp_err_status = 0ULL;
  2234. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
  2235. if (wm_process_mask & (1 << id)) {
  2236. if (irq_status.wm_events[id] & VHA_WM_EVENTS) {
  2237. uint16_t wm_cmd_id;
  2238. uint64_t status;
  2239. #ifdef VHA_SCF
  2240. uint64_t wm_rsp_err_events =
  2241. ((VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_MASKFULL |
  2242. VHA_REG_PARITY_ERROR_EN | VHA_REG_WL_ID_MISMATCH_ERROR_EN) &
  2243. ~((uint64_t)VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_SUCCESS_EN |
  2244. (uint64_t)VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_PARITY_EN));
  2245. #else
  2246. uint64_t wm_rsp_err_events =
  2247. ((VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_MASKFULL |
  2248. VHA_REG_WL_ID_MISMATCH_ERROR_EN) &
  2249. ~(VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_SUCCESS_EN |
  2250. VHA_CR_WM_RESPONSE_FIFO_WL_STATUS_PARITY_EN));
  2251. #endif
  2252. struct vha_cmd *cmd = vha->pendcmd[id].cmd;
  2253. if (cmd != NULL) {
  2254. /* Select WM to read response from. */
  2255. VHA_LOCK_WM();
  2256. VHA_SELECT_WM(id);
  2257. /* Handle RESPONSE_FIFO. */
  2258. /* Read RESPONSE_FIFO_WL_STATUS. */
  2259. status = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_STATUS);
  2260. #ifdef VHA_SCF
  2261. if (vha->hw_props.supported.parity && !vha->parity_disable) {
  2262. uint32_t i;
  2263. for (i = 0; i < VHA_PARITY_READ_COUNT_MAX; i++) {
  2264. /* Finish if parity is ok */
  2265. if (!img_mem_calc_parity(status))
  2266. break;
  2267. /* Otherwise re-read the reg. */
  2268. status = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_STATUS);
  2269. }
  2270. /* Raise an error if maximum re-read count is reached. */
  2271. if (i == VHA_PARITY_READ_COUNT_MAX) {
  2272. dev_err(dev, "WM_RESPONSE_FIFO_WL_STATUS register parity error!\n");
  2273. /* Use the real event to indicate the error */
  2274. VHA_REG_SET_PARITY_ERROR(status);
  2275. dev_info(dev, "status: 0x%016llx!\n", status);
  2276. }
  2277. }
  2278. #endif
  2279. /* Read RESPONSE_FIFO_WL_ID. */
  2280. wm_cmd_id = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_ID);
  2281. /* Gather and process perf/stats data. */
  2282. if (WM_DBG_MODE_ON(PERF))
  2283. vha->stats.cnn_last_cycles = IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_PERF);
  2284. if (WM_DBG_MODE_ON(BAND)) {
  2285. #define GET_MEM_STAT_TRANS(stat, reg) \
  2286. vha->stats.last_mem_stats.stat##_transactions = \
  2287. IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_BW_##reg)
  2288. #define GET_MEM_STAT_WORDS(stat, reg) \
  2289. vha->stats.last_mem_stats.stat##_words = \
  2290. IOREAD64_CR_REGIO(WM_RESPONSE_FIFO_WL_BW_##reg##_WORD)
  2291. GET_MEM_STAT_TRANS(locm_rd, LOCM_RD);
  2292. GET_MEM_STAT_TRANS(locm_wr, LOCM_WR);
  2293. GET_MEM_STAT_TRANS(locm_mwr, LOCM_MWR);
  2294. GET_MEM_STAT_TRANS(socm_rd, SOCM_RD);
  2295. GET_MEM_STAT_TRANS(socm_wr, SOCM_WR);
  2296. GET_MEM_STAT_TRANS(socm_mwr, SOCM_MWR);
  2297. GET_MEM_STAT_TRANS(ddr_rd, DDR_RD);
  2298. GET_MEM_STAT_TRANS(ddr_wr, DDR_WR);
  2299. GET_MEM_STAT_TRANS(ddr_mwr, DDR_MWR);
  2300. GET_MEM_STAT_WORDS(locm_rd, LOCM_RD);
  2301. GET_MEM_STAT_WORDS(locm_wr, LOCM_WR);
  2302. GET_MEM_STAT_WORDS(socm_rd, SOCM_RD);
  2303. GET_MEM_STAT_WORDS(socm_wr, SOCM_WR);
  2304. GET_MEM_STAT_WORDS(ddr_rd, DDR_RD);
  2305. GET_MEM_STAT_WORDS(ddr_wr, DDR_WR);
  2306. #undef GET_MEM_STAT_TRANS
  2307. #undef GET_MEM_STAT_WORDS
  2308. }
  2309. /* Pop response from RESPONSE_FIFO. */
  2310. IOWRITE64_CR_REGIO(VHA_CR_WM_RESPONSE_FIFO_READ_FIFO_READ_EN,
  2311. WM_RESPONSE_FIFO_READ);
  2312. IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
  2313. VHA_UNLOCK_WM();
  2314. /* Check if id matches the command. */
  2315. if (VHA_CR_GETBITS(WM_RESPONSE_FIFO_WL_ID, WL_ID, wm_cmd_id) !=
  2316. cmd->wm_cmd_id) {
  2317. dev_err(vha->dev, "%s: WM%u WL id mismatch for cmd 0x%08x/%u: "
  2318. "0x%04x vs. 0x%04x\n", __func__, id,
  2319. cmd->user_cmd.cmd_id, cmd->session->id, cmd->wm_cmd_id,
  2320. (uint16_t)VHA_CR_GETBITS(
  2321. WM_RESPONSE_FIFO_WL_ID, WL_ID,
  2322. wm_cmd_id));
  2323. /* Indicate WL id mismatch. */
  2324. VHA_REG_SET_WL_ID_MISMATCH_ERROR(status);
  2325. }
  2326. /* Leave only potential errors. */
  2327. status &= wm_rsp_err_events;
  2328. /* Store the latest error status for potential full_reset. */
  2329. if (status)
  2330. rsp_err_status = status;
  2331. #ifdef VHA_SCF
  2332. if (vha->confirm_config_reg)
  2333. vha_handle_conf_status(vha, cmd, &full_reset, &status);
  2334. if (vha->cnn_combined_crc_enable)
  2335. vha_check_crc(vha, cmd, &status);
  2336. #endif
  2337. /* Update per core/WM stats. */
  2338. VHA_INC_WL_STAT(vha, kicks_completed, cmd);
  2339. /* Free command resources. */
  2340. if (!vha->hw_sched_status.assignments[cmd->hw_sched_info.assignment_id].queued)
  2341. vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
  2342. vha_dev_free_cmd_res(vha, cmd, true);
  2343. /* Finally handle the response. */
  2344. vha_handle_wm_response(vha, id, status, irq_status.core_events,
  2345. irq_status.ic_events, &full_reset, &error_flags);
  2346. if (status)
  2347. vha->stats.total_failures++;
  2348. vha->stats.cnn_kicks_completed++;
  2349. } else {
  2350. WARN_ON(1);
  2351. }
  2352. } else {
  2353. /* Ignore or report??? */
  2354. }
  2355. }
  2356. /* If any of processed WLs required full reset, all the WLs being
  2357. * currently processed need to be failed and rolled back.
  2358. * The reset itself will be executed at the end of the handler. */
  2359. if (full_reset)
  2360. vha_handle_sys_failure(vha, rsp_err_status, -EIO, error_flags);
  2361. }
  2362. /* Read core event source mask. */
  2363. /* Debug purpose only ... */
  2364. multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, CORE,
  2365. irq_status.event_source);
  2366. if (multi_src_mask)
  2367. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
  2368. if (multi_src_mask & (1 << id)) {
  2369. if (irq_status.core_events[id] & VHA_CORE_ERR_EVENTS)
  2370. dev_err(vha->dev, "%s: Core%d error event has been detected: %llx\n",
  2371. __func__, id, irq_status.core_events[id]);
  2372. }
  2373. /* Read IC event source mask. */
  2374. /* Debug purpose only ... */
  2375. multi_src_mask = (uint32_t)VHA_CR_GETBITS(HOST_EVENT_SOURCE, IC,
  2376. irq_status.event_source);
  2377. if (multi_src_mask)
  2378. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
  2379. if (multi_src_mask & (1 << id)) {
  2380. if (irq_status.ic_events[id] & VHA_IC_ERR_EVENTS)
  2381. dev_err(vha->dev, "%s: Interconnect%d error event has been detected: %llx\n",
  2382. __func__, id, irq_status.ic_events[id]);
  2383. }
  2384. calibration_end:
  2385. if (full_reset) {
  2386. dev_err(vha->dev, "Performing full system reset due to HW error detection.");
  2387. /* Stop cores and execute the actual full reset finally. */
  2388. ret = vha_dev_stop(vha, true);
  2389. /* Check queues ... */
  2390. vha_chk_cmd_queues(vha, true);
  2391. } else {
  2392. /* Run in BH context! */
  2393. vha_chk_cmd_queues(vha, false);
  2394. }
  2395. #ifdef CONFIG_FAULT_INJECTION
  2396. if (vha->fault_inject & VHA_FI_IRQ_WORKER)
  2397. current->make_it_fail = false;
  2398. #endif
  2399. mutex_unlock(&vha->lock);
  2400. return ret;
  2401. }
  2402. #ifdef CONFIG_VHA_DUMMY
  2403. static int vha_dummy_dev_start(struct vha_dev *vha)
  2404. {
  2405. if (vha->state == VHA_STATE_ON)
  2406. return 0; /* not an error */
  2407. vha->state = VHA_STATE_ON;
  2408. /* Remember the time hw is powered on */
  2409. GETNSTIMEOFDAY(&vha->stats.hw_start);
  2410. return 0;
  2411. }
  2412. static int vha_dummy_dev_stop(struct vha_dev *vha)
  2413. {
  2414. uint64_t tmp = 0;
  2415. struct TIMESPEC now;
  2416. if (vha->state == VHA_STATE_OFF)
  2417. return -1;
  2418. vha->state = VHA_STATE_OFF;
  2419. /* Update the up time of the core */
  2420. GETNSTIMEOFDAY(&now);
  2421. if (get_timespan_us(&vha->stats.hw_start, &now, &tmp)) {
  2422. do_div(tmp, 1000UL);
  2423. vha->stats.uptime_ms += tmp;
  2424. if (vha->stats.uptime_ms)
  2425. vha_update_utilization(vha);
  2426. else
  2427. dev_dbg(vha->dev,
  2428. "%s Too short execution time to calculate utilization!\n",
  2429. __func__);
  2430. } else
  2431. WARN_ON(1);
  2432. return 0;
  2433. }
  2434. #endif
  2435. int vha_dev_start(struct vha_dev *vha)
  2436. {
  2437. int ret = 0;
  2438. uint8_t core_mask;
  2439. uint8_t active_core_mask = vha->full_core_mask;
  2440. int id;
  2441. #if defined(VHA_ENHANCED_APM) && !defined(CONFIG_VHA_DUMMY)
  2442. active_core_mask &= ~vha->hw_sched_status.free_core_mask;
  2443. #endif
  2444. if (vha->do_calibration)
  2445. active_core_mask |= VHA_CALIBRATION_CORE_MASK;
  2446. /* If device disabled & no core active */
  2447. if (vha->state == VHA_STATE_OFF && !vha->active_core_mask) {
  2448. pm_runtime_get_sync(vha->dev);
  2449. dev_dbg(vha->dev, "%s system power up\n", __func__);
  2450. }
  2451. /* Cancel any APM request for active cores that are busy at this point */
  2452. {
  2453. /* Find active cores that are busy and under APM */
  2454. uint8_t apm_core_mask = active_core_mask & vha->apm_core_mask;
  2455. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  2456. if (apm_core_mask & (1 << id))
  2457. cancel_delayed_work(&vha->apm_dworks[id].dwork);
  2458. }
  2459. vha->apm_core_mask &= ~(apm_core_mask);
  2460. }
  2461. /* Find cores that have to be powered on */
  2462. core_mask = (vha->active_core_mask ^ active_core_mask) &
  2463. ~vha->active_core_mask;
  2464. if (core_mask) {
  2465. dev_dbg(vha->dev, "%s core mask:%#x (%#x -> %#x)\n",
  2466. __func__, core_mask, vha->active_core_mask, active_core_mask);
  2467. /////////////// POWER ON //////////////////////////
  2468. img_pdump_printf("-- POWER_ON_BEGIN\n");
  2469. /* Prepare device cores ... */
  2470. ret = vha_dev_prepare_cores(vha, core_mask);
  2471. if (ret) {
  2472. dev_err(vha->dev, "%s: Error preparing device cores!\n", __func__);
  2473. goto error;
  2474. }
  2475. /* Enable device cores clocks */
  2476. vha_dev_enable_clocks(vha, core_mask);
  2477. /* Reset device cores & system for the very first time */
  2478. ret = vha_dev_reset(vha, core_mask,
  2479. vha->active_core_mask ? false : true);
  2480. if (ret){
  2481. dev_err(vha->dev, "%s: Error reseting device cores!\n", __func__);
  2482. goto error;
  2483. }
  2484. /* Enable device cores clocks */
  2485. vha_dev_enable_clocks(vha, core_mask);
  2486. img_pdump_printf("-- POWER_ON_END\n");
  2487. /////////////////////////////////////////////////////
  2488. vha_dev_ready(vha, core_mask,
  2489. vha->active_core_mask ? false : true);
  2490. /* Store actual status about active cores */
  2491. vha->active_core_mask = active_core_mask;
  2492. }
  2493. if (vha->state == VHA_STATE_OFF) {
  2494. /* Call device specific setup */
  2495. vha_dev_setup(vha);
  2496. /* Remember the time device is powered on */
  2497. GETNSTIMEOFDAY(&vha->stats.hw_start);
  2498. vha->state = VHA_STATE_ON;
  2499. #ifdef VHA_SCF
  2500. /* Start the SW watchdog */
  2501. vha_start_swd(vha, 0);
  2502. #endif
  2503. }
  2504. return 0;
  2505. error:
  2506. pm_runtime_put_sync(vha->dev);
  2507. vha->state = VHA_STATE_OFF;
  2508. vha->active_core_mask = 0;
  2509. return ret;
  2510. }
  2511. static int vha_dev_stop_cores(struct vha_dev *vha, uint8_t core_mask, bool reset)
  2512. {
  2513. int ret = 0;
  2514. if (core_mask) {
  2515. /* Store actual status about active cores */
  2516. vha->active_core_mask &= ~core_mask;
  2517. /* Disable events at first */
  2518. vha_dev_disable_events(vha, core_mask,
  2519. vha->active_core_mask ? false : true);
  2520. /////////////// POWER_OFF //////////////////////////
  2521. img_pdump_printf("-- POWER_OFF_BEGIN\n");
  2522. /* Reset core in case of error or pending inference */
  2523. if (reset) {
  2524. ret = vha_dev_reset(vha, core_mask,
  2525. vha->active_core_mask ? false : true);
  2526. if(ret)
  2527. dev_warn(vha->dev,
  2528. "%s: Problem with resetting device cores!\n",
  2529. __func__);
  2530. }
  2531. /* Disable device clocks */
  2532. ret = vha_dev_disable_clocks(vha, core_mask,
  2533. vha->active_core_mask ? false : true);
  2534. if(ret)
  2535. dev_warn(vha->dev,
  2536. "%s: Problem with disabling clocks for cores!\n",
  2537. __func__);
  2538. /* Execute any outstanding routines to flush the device cores */
  2539. ret = vha_dev_flush_cores(vha, core_mask);
  2540. if(ret)
  2541. dev_warn(vha->dev,
  2542. "%s: Problem with flushing device cores!\n",
  2543. __func__);
  2544. img_pdump_printf("-- POWER_OFF_END\n");
  2545. /////////////////////////////////////////////////////
  2546. }
  2547. /* If device enabled & no core active */
  2548. if (vha->state == VHA_STATE_ON && !vha->active_core_mask) {
  2549. int id;
  2550. /* Cancel APM requests if we are about to power off the device */
  2551. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++)
  2552. cancel_delayed_work(&vha->apm_dworks[id].dwork);
  2553. vha->apm_core_mask = 0;
  2554. dev_dbg(vha->dev, "%s system power down\n", __func__);
  2555. vha->state = VHA_STATE_OFF;
  2556. /* Update the up time of the device */
  2557. if (!vha->do_calibration) {
  2558. uint64_t tmp = 0;
  2559. struct TIMESPEC now;
  2560. GETNSTIMEOFDAY(&now);
  2561. if (get_timespan_us(&vha->stats.hw_start, &now, &tmp)) {
  2562. do_div(tmp, 1000UL);
  2563. vha->stats.uptime_ms += tmp;
  2564. if (vha->stats.uptime_ms)
  2565. vha_update_utilization(vha);
  2566. else
  2567. dev_dbg(vha->dev,
  2568. "%s Too short execution time to calculate utilization!\n",
  2569. __func__);
  2570. } else
  2571. WARN_ON(1);
  2572. }
  2573. vha->active_mmu_ctx = VHA_INVALID_ID;
  2574. spin_lock_irq(&vha->irq_lock);
  2575. memset(&vha->irq_status, 0, sizeof(vha->irq_status));
  2576. spin_unlock_irq(&vha->irq_lock);
  2577. if (reset) {
  2578. pm_runtime_mark_last_busy(vha->dev);
  2579. pm_runtime_put_sync_autosuspend(vha->dev);
  2580. } else {
  2581. pm_runtime_put_sync(vha->dev);
  2582. }
  2583. }
  2584. return ret;
  2585. }
  2586. int vha_dev_stop(struct vha_dev *vha, bool reset)
  2587. {
  2588. int ret = 0;
  2589. uint8_t active_core_mask = 0;
  2590. uint8_t core_mask;
  2591. #if defined(VHA_ENHANCED_APM) && !defined(CONFIG_VHA_DUMMY)
  2592. active_core_mask = vha->full_core_mask &
  2593. ~vha->hw_sched_status.free_core_mask;
  2594. #endif
  2595. if (vha->do_calibration)
  2596. active_core_mask &= ~VHA_CALIBRATION_CORE_MASK;
  2597. /* Find cores that have to be powered off */
  2598. core_mask = (vha->active_core_mask ^ active_core_mask) &
  2599. vha->active_core_mask;
  2600. if (core_mask)
  2601. dev_dbg(vha->dev, "%s core mask:%#x (%#x -> %#x)\n",
  2602. __func__, core_mask, vha->active_core_mask, active_core_mask);
  2603. ret = vha_dev_stop_cores(vha, core_mask, reset);
  2604. return ret;
  2605. }
  2606. static bool vha_is_mmu_ctx_shared(struct vha_cmd *cmd)
  2607. {
  2608. struct vha_session *session = cmd->session;
  2609. struct vha_dev *vha = session->vha;
  2610. /* If the session of the command we are trying to execute shares
  2611. * the hw mmu ctx with different session */
  2612. if (vha->mmu_ctxs[session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id] > 1) {
  2613. uint8_t id;
  2614. /* Check currently processed commands */
  2615. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  2616. /* Check if the mmu hw context is same as current command */
  2617. if (vha->pendcmd[id].cmd != NULL &&
  2618. vha->pendcmd[id].cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id ==
  2619. session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id)
  2620. return true;
  2621. }
  2622. }
  2623. return false;
  2624. }
  2625. int vha_dev_schedule_cmd(struct vha_dev *vha, struct vha_cmd *cmd)
  2626. {
  2627. struct vha_hw_sched_status *status = &vha->hw_sched_status;
  2628. struct vha_hw_sched_info *info;
  2629. struct vha_user_cnn_submit_multi_cmd* user_cmd =
  2630. (struct vha_user_cnn_submit_multi_cmd*)&cmd->user_cmd;
  2631. uint8_t wm_id;
  2632. uint8_t core_id = 0;
  2633. uint8_t core_mask = 0;
  2634. uint8_t assignment_id;
  2635. uint8_t i;
  2636. /* If no command provided, just check if anything can potentially
  2637. * be scheduled. */
  2638. if (cmd == NULL) {
  2639. /* Calculate the number of cores in use. */
  2640. uint8_t num_used_cores = 0;
  2641. uint8_t assignment_id;
  2642. for (assignment_id = 0; assignment_id < VHA_NUM_CORES; assignment_id++)
  2643. if (status->assignments[assignment_id].core_mask) {
  2644. uint8_t num_cores = VHA_CORE_MASK_TO_NUM(status->assignments[assignment_id].core_mask);
  2645. num_used_cores += num_cores;
  2646. if (vha->low_latency != VHA_LL_DISABLED)
  2647. if (status->assignments[assignment_id].queued)
  2648. num_used_cores += num_cores;
  2649. }
  2650. /* If all the cores are in use, nothing can be scheduled. */
  2651. if (num_used_cores ==
  2652. (vha->hw_props.num_cnn_core_devs * ((vha->low_latency != VHA_LL_DISABLED) ? 2 : 1)))
  2653. return -1;
  2654. return 0;
  2655. }
  2656. if (cmd->user_cmd.cmd_type != VHA_CMD_CNN_SUBMIT_MULTI)
  2657. return 0;
  2658. #define VHA_LL_BRANCH(l) \
  2659. { \
  2660. if (vha->low_latency == VHA_LL_DISABLED) \
  2661. return -1; \
  2662. else \
  2663. goto l; \
  2664. }
  2665. /* Check for shared mmu hardware context, as we can't schedule command
  2666. * on free cores, while other currently processing cores use the same
  2667. * mmu hw context, because data would be overwritten */
  2668. if (vha_is_mmu_ctx_shared(cmd)) {
  2669. dev_dbg(vha->dev, "%s: Postpone command due to shared mmu context!\n",
  2670. __func__);
  2671. return -1;
  2672. }
  2673. info = &cmd->hw_sched_info;
  2674. /* If external scheduling is requested... */
  2675. if (vha->scheduling_sequence_len > 0) {
  2676. uint8_t wm_mask;
  2677. /* Queueing is not supported for external scheduling. */
  2678. if (status->num_cores_free < user_cmd->num_cores)
  2679. return -1;
  2680. /* Read scheduling data for this workload from scheduling sequence. */
  2681. wm_id = SCHED_SEQ_GET_WM(vha->scheduling_counter);
  2682. wm_mask = VHA_WM_ID_TO_MASK(wm_id);
  2683. core_mask = SCHED_SEQ_GET_CORES(vha->scheduling_counter);
  2684. /* Sanity check the data. */
  2685. if (((status->free_wm_mask & wm_mask) == 0) ||
  2686. ((status->free_core_mask & core_mask) != core_mask))
  2687. return -1;
  2688. status->free_core_mask &= ~(core_mask);
  2689. /* Increment scheduling counter. */
  2690. vha->scheduling_counter =
  2691. (vha->scheduling_counter + 1) %
  2692. vha->scheduling_sequence_len;
  2693. } else {
  2694. /* Check if there are cores available. */
  2695. if (status->num_cores_free < user_cmd->num_cores)
  2696. VHA_LL_BRANCH(attempt_to_queue_multi);
  2697. /* Check if there is a WM available. */
  2698. if (status->num_wms_free == 0)
  2699. VHA_LL_BRANCH(attempt_to_queue_multi);
  2700. /* Find a free WM. */
  2701. wm_id = ffs(status->free_wm_mask) - 1;
  2702. /* Check if enough cores are available. */
  2703. if (user_cmd->num_cores > status->num_cores_free)
  2704. VHA_LL_BRANCH(attempt_to_queue_multi);
  2705. /* Find the required number of free cores. */
  2706. for (i = 0; i < user_cmd->num_cores; i++) {
  2707. core_id = ffs(status->free_core_mask) - 1;
  2708. core_mask |= VHA_CORE_ID_TO_MASK(core_id);
  2709. status->free_core_mask &= ~(core_mask);
  2710. }
  2711. }
  2712. /* Update resource status. */
  2713. for (assignment_id = 0; assignment_id < VHA_NUM_CORES; assignment_id++)
  2714. if (status->assignments[assignment_id].core_mask == 0)
  2715. break;
  2716. if (assignment_id == VHA_NUM_CORES) {
  2717. dev_info(vha->dev, "%s: Scheduling data inconsistency detected!\n", __func__);
  2718. return -1;
  2719. }
  2720. status->assignments[assignment_id].assignment_id = assignment_id;
  2721. status->assignments[assignment_id].wm_id = wm_id;
  2722. status->assignments[assignment_id].core_mask = core_mask;
  2723. status->num_cores_free -= user_cmd->num_cores;
  2724. status->num_wms_free--;
  2725. status->free_wm_mask &= ~(VHA_WM_ID_TO_MASK(wm_id));
  2726. /* Store command scheduling info. */
  2727. *info = status->assignments[assignment_id];
  2728. goto skip_label_attempt_to_queue_multi;
  2729. attempt_to_queue_multi:
  2730. /* Check if there is an assignment matching this scheduling request
  2731. * available in the list of assignments. */
  2732. for (assignment_id = 0; assignment_id < VHA_NUM_CORES; assignment_id++)
  2733. /* If this assignment is not queued already and it has the same
  2734. * number of cores as this scheduling request... */
  2735. if (!status->assignments[assignment_id].queued &&
  2736. status->assignments[assignment_id].core_mask &&
  2737. (VHA_CORE_MASK_TO_NUM(status->assignments[assignment_id].core_mask) ==
  2738. user_cmd->num_cores)) {
  2739. wm_id = status->assignments[assignment_id].wm_id;
  2740. if (vha->low_latency == VHA_LL_SELF_KICK
  2741. /* If the current command we are trying to queue belongs
  2742. * to a different session than the pending one. */
  2743. && (vha->pendcmd[wm_id].cmd != NULL &&
  2744. vha->pendcmd[wm_id].cmd->session != cmd->session)
  2745. /* If the session of the command we are trying to queue shares
  2746. * the hw mmu ctx with the session of pending cmd */
  2747. && (cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id ==
  2748. vha->pendcmd[wm_id].cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id)
  2749. /* Sanity if hw mmu ctx is really shared at this point. */
  2750. && (vha->mmu_ctxs[cmd->session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].hw_id] > 1)
  2751. ) {
  2752. /* Skip this assignment. */
  2753. continue;
  2754. }
  2755. /* Make the assignment queued. */
  2756. status->assignments[assignment_id].queued = true;
  2757. /* Store command scheduling info. */
  2758. *info = status->assignments[assignment_id];
  2759. break;
  2760. }
  2761. /* Fail if no matching assignments found. */
  2762. if (assignment_id == VHA_NUM_CORES)
  2763. return -1;
  2764. skip_label_attempt_to_queue_multi:
  2765. /* For hw commands... */
  2766. if (CMD_EXEC_ON_HW(cmd)) {
  2767. if (!VHA_IS_DUMMY(vha)) {
  2768. int tries = 3; /* magic number, just try harder to start the device */
  2769. /* Start device. */
  2770. while(tries--) {
  2771. if (vha_dev_start(vha))
  2772. dev_warn(vha->dev, "%s: Error starting device cores. Try once again.", __func__);
  2773. else
  2774. break;
  2775. }
  2776. }
  2777. #ifdef CONFIG_VHA_DUMMY
  2778. else
  2779. vha_dummy_dev_start(vha);
  2780. #endif
  2781. }
  2782. #undef VHA_LL_BRANCH
  2783. dev_dbg(vha->dev, "%s: cmd 0x%08x/%u scheduled on WM%u/core(s) 0x%02x\n",
  2784. __func__, cmd->user_cmd.cmd_id, cmd->session->id,
  2785. info->wm_id, info->core_mask);
  2786. return 0;
  2787. }
  2788. void vha_dev_free_cmd_res(struct vha_dev *vha, struct vha_cmd *cmd, bool update_stats)
  2789. {
  2790. struct vha_hw_sched_status *status = &vha->hw_sched_status;
  2791. struct vha_hw_sched_info *info = &cmd->hw_sched_info;
  2792. struct vha_user_cnn_submit_multi_cmd* user_cmd =
  2793. (struct vha_user_cnn_submit_multi_cmd*)&cmd->user_cmd;
  2794. if (update_stats) {
  2795. uint64_t proc_time = 0;
  2796. struct TIMESPEC *from = &cmd->hw_proc_start;
  2797. struct TIMESPEC *to = &vha->stats.wm_stats[info->wm_id].hw_proc_end;
  2798. if (TIMESPEC_COMPARE(&vha->stats.wm_stats[info->wm_id].hw_proc_end_prev,
  2799. &cmd->hw_proc_start) >= 0)
  2800. from = &vha->stats.wm_stats[info->wm_id].hw_proc_end_prev;
  2801. if (get_timespan_us(from, to, &proc_time)) {
  2802. vha->stats.last_proc_us = proc_time;
  2803. } else {
  2804. vha->stats.last_proc_us = 0;
  2805. }
  2806. /* Update WL stats. */
  2807. VHA_UPDATE_WL_STAT(vha, total_proc_us, cmd, vha->stats.last_proc_us);
  2808. /* Update common stats. */
  2809. vha_cnn_update_stats(vha);
  2810. }
  2811. /* If assignment for this workload is queued... */
  2812. if (status->assignments[info->assignment_id].queued) {
  2813. /* Just mark it as not queued again. */
  2814. status->assignments[info->assignment_id].queued = false;
  2815. /* Clear scheduling info for this workload. */
  2816. info->freed = true;
  2817. /* Do not update the scheduling status. */
  2818. return;
  2819. }
  2820. /* Update the scheduling status. */
  2821. status->num_cores_free += user_cmd->num_cores;
  2822. status->free_core_mask |= info->core_mask;
  2823. status->num_wms_free++;
  2824. status->free_wm_mask |= VHA_WM_ID_TO_MASK(info->wm_id);
  2825. /* Clear the assignment and scheduling info for this workload. */
  2826. memset(&status->assignments[info->assignment_id], 0,
  2827. sizeof(struct vha_hw_sched_info));
  2828. info->freed = true;
  2829. }
  2830. static void sched_apm_multi(struct vha_dev *vha)
  2831. {
  2832. struct vha_apm_work *apm_work = NULL;
  2833. /* Find active cores that are free and not under APM */
  2834. uint8_t apm_core_mask = vha->active_core_mask &
  2835. vha->hw_sched_status.free_core_mask &
  2836. ~vha->apm_core_mask;
  2837. int id;
  2838. /* Skip if nothing has changed */
  2839. if (!apm_core_mask)
  2840. return;
  2841. dev_dbg(vha->dev, "%s core mask:%#x\n", __func__, apm_core_mask);
  2842. /* Schedule for all cores, separately */
  2843. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  2844. if (apm_core_mask & (1 << id)) {
  2845. apm_work = &vha->apm_dworks[id];
  2846. apm_work->core_mask = 1 << id;
  2847. apm_work->delay_ms = vha->pm_delay;
  2848. vha_sched_apm(vha, apm_work);
  2849. }
  2850. }
  2851. /* Record actual status */
  2852. vha->apm_core_mask |= apm_core_mask;
  2853. }
  2854. //#define MTSTK_MEASURE_MULTI_PRI
  2855. #ifdef MTSTK_MEASURE_MULTI_PRI
  2856. static uint8_t get_active_pri_count(struct vha_dev *vha)
  2857. {
  2858. uint8_t pris;
  2859. uint8_t pri_count = 0;
  2860. /* Calculate the number of priority levels with anything to schedule. */
  2861. for (pris = 0; pris < VHA_MAX_PRIORITIES; pris++)
  2862. if (vha->pri_q_counters[pris] > 0) {
  2863. pri_count++;
  2864. }
  2865. return pri_count;
  2866. }
  2867. #endif
  2868. static void vha_get_time_span(struct TIMESPEC *start, struct TIMESPEC *end,
  2869. struct TIMESPEC *span)
  2870. {
  2871. if (start == NULL || end == NULL || span == NULL)
  2872. return;
  2873. /* Calculate the seconds span. */
  2874. span->tv_sec = end->tv_sec - start->tv_sec;
  2875. /* If there more than a second span, move one second to nanoseconds
  2876. * to avoid potential negative nanosecond values. */
  2877. if (span->tv_sec > 0) {
  2878. span->tv_sec--;
  2879. end->tv_nsec += 1000000000;
  2880. }
  2881. /* Calculate the nanoseconds span. */
  2882. span->tv_nsec = end->tv_nsec - start->tv_nsec;
  2883. /* If nanoseconds include a second, move it to seconds. */
  2884. if (span->tv_nsec > 1000000000) {
  2885. span->tv_sec++;
  2886. span->tv_nsec -= 1000000000;
  2887. }
  2888. }
  2889. static uint8_t vha_scheduler_get_priority(struct vha_dev *vha)
  2890. {
  2891. uint8_t ret_pri = 0;
  2892. uint8_t pris;
  2893. uint8_t pri_count = 0;
  2894. uint32_t curr_window = 0;
  2895. uint32_t curr_limit = 0;
  2896. uint32_t rand_val;
  2897. /* Calculate current total window width. */
  2898. for (pris = 0; pris < VHA_MAX_PRIORITIES; pris++)
  2899. if (vha->pri_q_counters[pris] > 0) {
  2900. curr_window += pri_windows[pris];
  2901. ret_pri = pris;
  2902. pri_count++;
  2903. }
  2904. /* If there's no priority with WLs to schedule, just return 0. */
  2905. if (pri_count == 0)
  2906. return VHA_INVALID_PRI;
  2907. /* If there's only one priority with WLs to schedule, just return it. */
  2908. if (pri_count == 1)
  2909. return ret_pri;
  2910. /* If starvation avoidance is disabled, just return the highest priority
  2911. * with WLs to schedule. */
  2912. if (curr_window == 0)
  2913. return ret_pri;
  2914. /* If starvation avoidance is enabled, use 'lottery' based approach. */
  2915. /* Generate random value within the current window. */
  2916. vha_mt19937_gen_range(vha->hw_sched_status.sched_data->rand_gen_handle,
  2917. 0, curr_window, &rand_val);
  2918. /* Choose priority based on the value generated and available priorities. */
  2919. for (pris = 0; pris < VHA_MAX_PRIORITIES; pris++)
  2920. if (vha->pri_q_counters[pris] > 0) {
  2921. curr_limit += pri_windows[pris];
  2922. if (rand_val <= curr_limit) {
  2923. ret_pri = pris;
  2924. break;
  2925. }
  2926. }
  2927. return ret_pri;
  2928. }
  2929. static void vha_scheduler_set_starting_session(struct vha_dev *vha,
  2930. uint8_t priority, struct vha_session *session)
  2931. {
  2932. /* Set a starting point session for next scheduling round. */
  2933. if (session != list_entry(&vha->sched_sessions[priority],
  2934. struct vha_session, sched_list[priority]))
  2935. while(list_first_entry(&vha->sched_sessions[priority],
  2936. struct vha_session, sched_list[priority]) != session)
  2937. list_rotate_left(&vha->sched_sessions[priority]);
  2938. }
  2939. void vha_scheduler_loop(struct vha_dev *vha)
  2940. {
  2941. struct vha_cmd *cmd, *tmp;
  2942. struct vha_session *session = NULL;
  2943. bool scheduled = false;
  2944. enum do_cmd_status cmd_status = CMD_OK;
  2945. uint8_t current_pri = VHA_DEFAULT_PRI;
  2946. bool log_pri_sched_info = true;
  2947. if (vha_dev_schedule_cmd(vha, NULL) != 0) {
  2948. /* Postpone worker task if nothing can be scheduled. */
  2949. dev_dbg(vha->dev, "%s Nothing can be scheduled at the moment. "
  2950. "Postpone worker task!\n", __func__);
  2951. return;
  2952. }
  2953. #ifdef MTSTK_MEASURE_MULTI_PRI
  2954. log_pri_sched_info = (get_active_pri_count(vha) > 1) ? true : false;
  2955. #endif
  2956. /* Main scheduling loop. */
  2957. do {
  2958. scheduled = false;
  2959. current_pri = vha_scheduler_get_priority(vha);
  2960. if (current_pri == VHA_INVALID_PRI)
  2961. break;
  2962. list_for_each_entry(session, &vha->sched_sessions[current_pri], sched_list[current_pri]) {
  2963. list_for_each_entry_safe(cmd, tmp, &session->cmds[current_pri], list[current_pri]) {
  2964. #if defined(VHA_ENHANCED_APM)
  2965. /* Schedule APM/power down cores in the middle if possible */
  2966. if (!VHA_IS_DUMMY(vha)) {
  2967. if (!vha->no_clock_disable) {
  2968. if (!vha->pm_delay) {
  2969. if (vha_dev_stop(vha, false)) {
  2970. dev_warn(vha->dev, "%s: Failed to soft stop device. Trying harder with reset",
  2971. __func__);
  2972. if (vha_dev_stop(vha, true))
  2973. dev_err(vha->dev, "%s: Failed to stop device with reset!", __func__);
  2974. }
  2975. } else
  2976. sched_apm_multi(vha);
  2977. }
  2978. }
  2979. #endif
  2980. /* Skip this workload as it's already scheduled. */
  2981. if (cmd->hw_sched_info.core_mask && !cmd->hw_sched_info.freed)
  2982. continue;
  2983. /* Attempt to schedule command for execution. */
  2984. cmd_status = vha_do_cmd(cmd);
  2985. if ((cmd_status == CMD_OK) || (cmd_status == CMD_HW_BUSY)) {
  2986. if (cmd_status == CMD_OK) {
  2987. scheduled = true;
  2988. if (log_pri_sched_info && !cmd->rolled_back) {
  2989. struct TIMESPEC sched_ts, sched_span = {0};
  2990. GETNSTIMEOFDAY(&sched_ts);
  2991. vha_get_time_span(&cmd->submit_ts, &sched_ts, &sched_span);
  2992. #ifdef LOG_PRI_SCHEDULING_INFO
  2993. dev_info(vha->dev, "@@@ scheduled 0x%08x/%u/%u, span: %llu\n",
  2994. cmd->user_cmd.cmd_id, session->id, cmd->user_cmd.priority,
  2995. (uint64_t)sched_span.tv_sec * 1000000000ULL +
  2996. (uint64_t)sched_span.tv_nsec);
  2997. #endif
  2998. VHA_UPDATE_SCHED_STAT_MTSTK(vha, cmd, &sched_span);
  2999. }
  3000. session = list_next_entry(session, sched_list[current_pri]);
  3001. }
  3002. vha_scheduler_set_starting_session(vha, current_pri, session);
  3003. goto exit_session_loop;
  3004. }
  3005. }
  3006. }
  3007. exit_session_loop:;
  3008. /* Iterate until a workload was scheduled and no other can be scheduled. */
  3009. } while (vha_dev_schedule_cmd(vha, NULL) == 0 && scheduled);
  3010. /* Schedule APM/power down cores if possible at end */
  3011. if (!VHA_IS_DUMMY(vha)) {
  3012. bool skip = vha->no_clock_disable;
  3013. #if !defined(VHA_ENHANCED_APM)
  3014. skip |= vha_is_busy(vha);
  3015. #endif
  3016. if (!skip) {
  3017. if (!vha->pm_delay) {
  3018. if (vha_dev_stop(vha, false)) {
  3019. dev_warn(vha->dev, "%s: Failed to soft stop device. Trying harder with reset",
  3020. __func__);
  3021. if (vha_dev_stop(vha, true))
  3022. dev_err(vha->dev, "%s: Failed to stop device with reset!", __func__);
  3023. }
  3024. } else
  3025. sched_apm_multi(vha);
  3026. }
  3027. }
  3028. #ifdef CONFIG_VHA_DUMMY
  3029. else if (!vha_is_busy(vha))
  3030. vha_dummy_dev_stop(vha);
  3031. #endif
  3032. }
  3033. bool vha_rm_session_cmds(struct vha_session *session)
  3034. {
  3035. struct vha_dev *vha = session->vha;
  3036. bool reschedule = false;
  3037. uint32_t wm_id;
  3038. struct vha_hw_sched_info sched_info = {0};
  3039. struct vha_cmd *cur_cmd, *tmp_cmd;
  3040. uint8_t pri;
  3041. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  3042. bool pend_removed = false;
  3043. bool queued_removed = false;
  3044. /* Check if pend/queued WLs will be removed. */
  3045. if (vha->pendcmd[wm_id].cmd &&
  3046. vha->pendcmd[wm_id].cmd->session == session) {
  3047. dev_warn(vha->dev,
  3048. "Removing a session while cnn cmd is still pending\n");
  3049. pend_removed = true;
  3050. sched_info = vha->pendcmd[wm_id].cmd->hw_sched_info;
  3051. #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
  3052. cancel_delayed_work(&vha->dummy_dworks[wm_id].dummy_dwork);
  3053. #endif
  3054. }
  3055. if (vha->queuedcmd[wm_id].cmd &&
  3056. vha->queuedcmd[wm_id].cmd->session == session) {
  3057. dev_warn(vha->dev,
  3058. "Removing a session while cnn cmd is still queued\n");
  3059. queued_removed = true;
  3060. sched_info = vha->queuedcmd[wm_id].cmd->hw_sched_info;
  3061. }
  3062. /* Update session scheduling. */
  3063. if (vha->queuedcmd[wm_id].cmd &&
  3064. (pend_removed && !queued_removed)) {
  3065. if (vha->queuedcmd[wm_id].cmd->session !=
  3066. list_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
  3067. struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]))
  3068. while(list_first_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
  3069. struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]) !=
  3070. vha->queuedcmd[wm_id].cmd->session)
  3071. list_rotate_left(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority]);
  3072. }
  3073. /* Remove pend/queued WLs if needed. */
  3074. if (pend_removed || queued_removed) {
  3075. uint64_t wm_mask = VHA_CR_SETBITS(HOST_EVENT_SOURCE, WM, VHA_WM_ID_TO_MASK(wm_id));
  3076. /* Reset WM/cores. */
  3077. vha_wm_reset(vha, &sched_info);
  3078. VHA_LOCK_WM();
  3079. VHA_SELECT_WM(wm_id);
  3080. /* Remove WM related interrupt info if it happens to be set. */
  3081. if (vha->irq_status.event_source & wm_mask)
  3082. {
  3083. /* Unset the WM related source bit. */
  3084. vha->irq_status.event_source &= ~wm_mask;
  3085. /* Clear all WM related events. */
  3086. IOWRITE64_CR_REGIO(vha->irq_status.wm_events[wm_id] & VHA_WM_EVENTS_DEFAULT,
  3087. WM_EVENT_CLEAR);
  3088. vha->irq_status.wm_events[wm_id] = 0ULL;
  3089. }
  3090. /* Re-enable WM events here as this WM will not be handled further. */
  3091. IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
  3092. VHA_UNLOCK_WM();
  3093. /* Rollback all WLs from this WM. */
  3094. vha_rollback_wm_cmds(vha, wm_id, true);
  3095. /* Need to reschedule too. */
  3096. reschedule = true;
  3097. }
  3098. }
  3099. /* Remove session related commands. */
  3100. for (pri = 0; pri < VHA_MAX_PRIORITIES; pri++) {
  3101. list_for_each_entry_safe(cur_cmd, tmp_cmd, &session->cmds[pri], list[pri]) {
  3102. /* rsp didn't make it to rsps list, free it now */
  3103. kfree(cur_cmd->rsp);
  3104. list_del(&cur_cmd->list[cur_cmd->user_cmd.priority]);
  3105. vha->pri_q_counters[cur_cmd->user_cmd.priority]--;
  3106. kfree(cur_cmd);
  3107. }
  3108. }
  3109. return reschedule;
  3110. }
  3111. bool vha_rm_session_cmds_masked(struct vha_session *session, uint32_t cmd_id,
  3112. uint32_t cmd_id_mask)
  3113. {
  3114. struct vha_dev *vha = session->vha;
  3115. bool reschedule = false;
  3116. uint32_t wm_id;
  3117. struct vha_hw_sched_info sched_info = {0};
  3118. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  3119. bool pend_removed = false;
  3120. bool queued_removed = false;
  3121. /* Check if pend/queued WLs will be removed. */
  3122. if (vha->pendcmd[wm_id].cmd &&
  3123. (vha->pendcmd[wm_id].cmd->session == session) &&
  3124. (vha->pendcmd[wm_id].cmd->user_cmd.cmd_id & cmd_id_mask)
  3125. == cmd_id) {
  3126. pend_removed = true;
  3127. sched_info = vha->pendcmd[wm_id].cmd->hw_sched_info;
  3128. #ifdef CONFIG_VHA_DUMMY_SIMULATE_HW_PROCESSING_TIME
  3129. cancel_delayed_work(&vha->dummy_dworks[wm_id].dummy_dwork);
  3130. #endif
  3131. VHA_INC_WL_STAT(vha, kicks_cancelled, vha->pendcmd[wm_id].cmd);
  3132. vha->stats.cnn_kicks_cancelled++;
  3133. }
  3134. if (vha->queuedcmd[wm_id].cmd &&
  3135. (vha->queuedcmd[wm_id].cmd->session == session) &&
  3136. (vha->queuedcmd[wm_id].cmd->user_cmd.cmd_id & cmd_id_mask)
  3137. == cmd_id) {
  3138. sched_info = vha->queuedcmd[wm_id].cmd->hw_sched_info;
  3139. queued_removed = true;
  3140. if (vha->low_latency == VHA_LL_SELF_KICK) {
  3141. VHA_INC_WL_STAT(vha, kicks_cancelled, vha->queuedcmd[wm_id].cmd);
  3142. vha->stats.cnn_kicks_cancelled++;
  3143. }
  3144. }
  3145. /* Update session scheduling. */
  3146. if (vha->queuedcmd[wm_id].cmd &&
  3147. (pend_removed && !queued_removed)) {
  3148. if (vha->queuedcmd[wm_id].cmd->session !=
  3149. list_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
  3150. struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]))
  3151. while(list_first_entry(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority],
  3152. struct vha_session, sched_list[vha->queuedcmd[wm_id].cmd->user_cmd.priority]) !=
  3153. vha->queuedcmd[wm_id].cmd->session)
  3154. list_rotate_left(&vha->sched_sessions[vha->queuedcmd[wm_id].cmd->user_cmd.priority]);
  3155. }
  3156. /* Remove pend/queued WLs if needed. */
  3157. if (pend_removed || queued_removed) {
  3158. uint64_t wm_mask = VHA_CR_SETBITS(HOST_EVENT_SOURCE, WM, VHA_WM_ID_TO_MASK(wm_id));
  3159. /* Reset WM/cores. */
  3160. vha_wm_reset(vha, &sched_info);
  3161. VHA_LOCK_WM();
  3162. VHA_SELECT_WM(wm_id);
  3163. /* Remove WM related interrupt info if it happens to be set. */
  3164. if (vha->irq_status.event_source & wm_mask)
  3165. {
  3166. /* Unset the WM related source bit. */
  3167. vha->irq_status.event_source &= ~wm_mask;
  3168. /* Clear all WM related events. */
  3169. IOWRITE64_CR_REGIO(vha->irq_status.wm_events[wm_id] & VHA_WM_EVENTS_DEFAULT,
  3170. WM_EVENT_CLEAR);
  3171. vha->irq_status.wm_events[wm_id] = 0ULL;
  3172. }
  3173. /* Re-enable WM events here as this WM will not be handled further. */
  3174. IOWRITE64_CR_REGIO(VHA_WM_EVENTS_DEFAULT, WM_EVENT_ENABLE);
  3175. VHA_UNLOCK_WM();
  3176. /* Rollback all WLs from this WM. */
  3177. vha_rollback_wm_cmds(vha, wm_id, true);
  3178. /* Correct aborted stats. */
  3179. if (queued_removed) {
  3180. VHA_UPDATE_WM_STAT(vha, kicks_aborted, sched_info.wm_id, -1);
  3181. VHA_UPDATE_CORE_GROUP_STAT(vha, kicks_aborted, sched_info.core_mask, -1);
  3182. vha->stats.cnn_kicks_aborted--;
  3183. }
  3184. if (pend_removed) {
  3185. VHA_UPDATE_WM_STAT(vha, kicks_aborted, sched_info.wm_id, -1);
  3186. VHA_UPDATE_CORE_GROUP_STAT(vha, kicks_aborted, sched_info.core_mask, -1);
  3187. vha->stats.cnn_kicks_aborted--;
  3188. }
  3189. reschedule = true;
  3190. }
  3191. }
  3192. return reschedule;
  3193. }
  3194. int vha_rm_cmds(struct vha_session *session, uint32_t cmd_id,
  3195. uint32_t cmd_id_mask, bool respond)
  3196. {
  3197. struct vha_dev *vha = session->vha;
  3198. struct vha_cmd *cur_cmd, *tmp_cmd;
  3199. struct vha_rsp *cur_rsp, *tmp_rsp;
  3200. bool reschedule = false;
  3201. bool respond_aux = false;
  3202. int ret = 0;
  3203. uint8_t pri;
  3204. mutex_lock(&vha->lock);
  3205. /* Remove pend/queued session commands that match the cmd_id. */
  3206. reschedule = vha_rm_session_cmds_masked(session, cmd_id, cmd_id_mask);
  3207. /* Remove session related commands matching command id template. */
  3208. for (pri = 0; pri < VHA_MAX_PRIORITIES; pri++) {
  3209. list_for_each_entry_safe(cur_cmd, tmp_cmd, &session->cmds[pri], list[pri]) {
  3210. if ((cur_cmd->user_cmd.cmd_id & cmd_id_mask) == cmd_id) {
  3211. #ifdef KERNEL_DMA_FENCE_SUPPORT
  3212. switch (cur_cmd->user_cmd.cmd_type)
  3213. {
  3214. case VHA_CMD_CNN_SUBMIT_MULTI:
  3215. {
  3216. struct vha_user_cnn_submit_multi_cmd *cnn_cmd =
  3217. (struct vha_user_cnn_submit_multi_cmd *)&cur_cmd->user_cmd;
  3218. int j;
  3219. for (j = 0; j < (cnn_cmd->msg.num_bufs - cnn_cmd->num_cores); j++) {
  3220. struct vha_buffer *buf = vha_find_bufid(session, cnn_cmd->bufs[j]);
  3221. if (buf == NULL) {
  3222. dev_warn(vha->dev, "%s: could not find buf %x\n", __func__,
  3223. cnn_cmd->bufs[j]);
  3224. } else {
  3225. vha_rm_buf_fence(session, buf);
  3226. }
  3227. }
  3228. break;
  3229. }
  3230. default:
  3231. dev_warn(vha->dev, "%s: invalid cmd type %x\n", __func__,
  3232. cur_cmd->user_cmd.cmd_type);
  3233. break;
  3234. }
  3235. #endif
  3236. /* rsp didn't make it to rsps list; free it now. */
  3237. kfree(cur_cmd->rsp);
  3238. list_del(&cur_cmd->list[cur_cmd->user_cmd.priority]);
  3239. vha->pri_q_counters[cur_cmd->user_cmd.priority]--;
  3240. kfree(cur_cmd);
  3241. /* There were commands matching command id template in the list,
  3242. * so respond to wake user space. */
  3243. respond_aux = true;
  3244. }
  3245. }
  3246. }
  3247. /* Remove responses for session related commands
  3248. * matching command id template. */
  3249. list_for_each_entry_safe(cur_rsp, tmp_rsp, &session->rsps, list) {
  3250. if ((cur_rsp->user_rsp.cmd_id & cmd_id_mask) == cmd_id) {
  3251. list_del(&cur_rsp->list);
  3252. kfree(cur_rsp);
  3253. respond_aux = true;
  3254. }
  3255. }
  3256. /* Reset hardware if required. */
  3257. if (reschedule)
  3258. ret = vha_dev_stop(vha, reschedule);
  3259. /* Generate "cancel" response if any commands matching command id template
  3260. * were removed. */
  3261. if (respond_aux && respond) {
  3262. /* Calculate space for the response. */
  3263. size_t sz = sizeof(struct vha_rsp)
  3264. + sizeof(struct vha_user_cnn_submit_rsp)
  3265. - sizeof(struct vha_user_rsp);
  3266. /* Allocate space for standard response. */
  3267. struct vha_rsp *rsp = kzalloc(sz, GFP_KERNEL);
  3268. if (rsp == NULL) {
  3269. dev_crit(session->vha->dev,
  3270. "Failed to allocate memory to notify cancel for cmds 0x%08x\n", cmd_id);
  3271. session->oom = true;
  3272. } else {
  3273. rsp->size = sizeof(struct vha_user_cnn_submit_rsp);
  3274. rsp->user_rsp.cmd_id = cmd_id;
  3275. list_add_tail(&rsp->list, &session->rsps);
  3276. }
  3277. wake_up(&session->wq);
  3278. }
  3279. mutex_unlock(&vha->lock);
  3280. /* Just return in case of oom. */
  3281. if (session->oom)
  3282. return -ENOMEM;
  3283. /* Reschedule once all commands matching command id template are removed. */
  3284. if (reschedule)
  3285. vha_chk_cmd_queues(vha, true);
  3286. return ret;
  3287. }
  3288. bool vha_is_busy(struct vha_dev *vha)
  3289. {
  3290. return (vha->hw_sched_status.num_cores_free < vha->hw_props.num_cnn_core_devs);
  3291. }
  3292. /* check all input buffers are filled and ready to go */
  3293. bool vha_is_waiting_for_inputs(struct vha_session *session,
  3294. struct vha_cmd *cmd)
  3295. {
  3296. if (!cmd->inbufs_ready) {
  3297. const struct vha_user_cnn_submit_multi_cmd *user_cmd =
  3298. (struct vha_user_cnn_submit_multi_cmd *)&cmd->user_cmd;
  3299. int i;
  3300. for (i = 0; i < cmd->user_cmd.num_inbufs - user_cmd->num_cores; i++) {
  3301. struct vha_buffer *buf = vha_find_bufid(session, user_cmd->bufs[i]);
  3302. if (buf && buf->status == VHA_BUF_UNFILLED) {
  3303. dev_dbg(session->vha->dev,
  3304. "%s: cmd %u waiting for input "
  3305. "buf %d to be ready\n",
  3306. __func__,
  3307. cmd->user_cmd.cmd_id,
  3308. buf->id);
  3309. return true;
  3310. }
  3311. }
  3312. }
  3313. cmd->inbufs_ready = true;
  3314. return false;
  3315. }
  3316. void vha_dev_apm_stop(struct vha_dev *vha, struct vha_apm_work *apm_work)
  3317. {
  3318. /* Find active cores that are not busy and under APM for this apm request */
  3319. uint8_t apm_core_mask = vha->active_core_mask &
  3320. vha->hw_sched_status.free_core_mask &
  3321. vha->apm_core_mask &
  3322. apm_work->core_mask;
  3323. vha->apm_core_mask &= ~apm_core_mask;
  3324. if (vha->do_calibration)
  3325. return;
  3326. if (apm_core_mask) {
  3327. dev_dbg(vha->dev, "%s core mask:%#x\n", __func__, apm_core_mask);
  3328. if (vha_dev_stop_cores(vha, apm_core_mask, false)) {
  3329. dev_warn(vha->dev, "%s: Failed to soft stop cores. Trying harder with reset",
  3330. __func__);
  3331. if (vha_dev_stop_cores(vha, apm_core_mask, true))
  3332. dev_err(vha->dev, "%s: Failed to stop cores with reset!", __func__);
  3333. }
  3334. }
  3335. }
  3336. int vha_dev_get_props(struct vha_dev *vha, uint32_t onchipmem_size)
  3337. {
  3338. struct vha_hw_props *props = &vha->hw_props;
  3339. uint64_t ip_config;
  3340. uint32_t locm_size_kb = 0;
  3341. uint32_t socm_size_kb = 0;
  3342. uint8_t socm_num_sb, socm_num_ba, socm_num_bg;
  3343. uint8_t ext_mem_bus_width;
  3344. memset(props, 0, sizeof(*props));
  3345. #ifdef CONFIG_VHA_DUMMY
  3346. /* Note: dummy dev always reads zeroes from registers */
  3347. props->product_id = 0x8070605040302010ULL;
  3348. props->core_id = (long)HW_SERIES << (int)VHA_CR_CORE_ID_BRANCH_ID_SHIFT;
  3349. props->core_id += 0x010203040506ULL; // provide a dummy core id
  3350. props->dummy_dev = true;
  3351. props->num_cnn_core_devs = VHA_NUM_CORES;
  3352. #else
  3353. props->product_id = IOREAD64_CR_REGIO(PRODUCT_ID);
  3354. props->core_id = IOREAD64_CR_REGIO(CORE_ID);
  3355. #endif
  3356. props->skip_bvnc_check = false;
  3357. /*
  3358. * New mmu version 3 and onwards operates on 40bit physical & virtual addresses
  3359. */
  3360. props->mmu_width = 40;
  3361. /* HW from 1.1 onwards */
  3362. ip_config = IOREAD64_CR_REGIO(CORE_IP_CONFIG);
  3363. #ifdef HW_AX3
  3364. props->mmu_ver = VHA_CR_GETBITS(CORE_IP_CONFIG, MMU_VERSION, ip_config);
  3365. #endif
  3366. /* Mirage uses MMU version 3 hardware */
  3367. if (!props->mmu_ver)
  3368. props->mmu_ver = 3;
  3369. ;
  3370. /* Read num cores supported (number of WMs must be the same). */
  3371. if (VHA_CR_GETBITS(CORE_IP_CONFIG, CNN_SUPPORTED, ip_config)) {
  3372. uint64_t ip_config1 = IOREAD64_CR_REGIO(CORE_IP_CONFIG1);
  3373. props->num_cnn_core_devs =
  3374. 1 + VHA_CR_GETBITS(CORE_IP_CONFIG1, NUM_CORES_MIN1, ip_config1);
  3375. }
  3376. if (VHA_CR_GETBITS(CORE_IP_CONFIG, RTM_SUPPORTED, ip_config))
  3377. props->supported.rtm = 1;
  3378. #ifdef HW_AX3
  3379. if (VHA_CR_GETBITS(CORE_IP_CONFIG, PARITY_REGISTERS, ip_config))
  3380. props->supported.parity = 1;
  3381. #if defined(CONFIG_VHA_DUMMY) && defined(VHA_SCF)
  3382. /* Force parity for pdump generation */
  3383. props->supported.parity = 1;
  3384. #endif
  3385. #endif
  3386. if ((props->num_cnn_core_devs == 0)
  3387. || VHA_CR_GETBITS(CORE_ID, BRANCH_ID, props->core_id) != HW_SERIES) {
  3388. dev_err(vha->dev, "%s: Wrong core configuration detected. "
  3389. "Expected BVNC %d.x.x.x, got %llu.x.x.x. "
  3390. "Maybe kernel module was built with wrong params.\n",
  3391. __func__, HW_SERIES,
  3392. VHA_CR_GETBITS(CORE_ID, BRANCH_ID, props->core_id));
  3393. return -ENODEV;
  3394. }
  3395. dev_info(vha->dev, "%s: Product id: %#llx\n",
  3396. __func__, props->product_id);
  3397. dev_info(vha->dev, "%s: Core id: %#llx\n",
  3398. __func__, props->core_id);
  3399. dev_info(vha->dev, "%s: MMU version:%d (%dbit)\n",
  3400. __func__, props->mmu_ver, props->mmu_width);
  3401. dev_dbg(vha->dev, "%s: supported: %#x\n",
  3402. __func__, props->features);
  3403. {
  3404. uint64_t tmp = IOREAD64_CR_REGIO(CORE_IP_INTEGRATOR_ID);
  3405. dev_dbg(vha->dev, "%s: ip integrator id: %#llx\n",
  3406. __func__, tmp);
  3407. tmp = IOREAD64_CR_REGIO(CORE_IP_CHANGELIST);
  3408. dev_dbg(vha->dev, "%s: ip change list: %llu\n", __func__, tmp);
  3409. }
  3410. /* Read OCM info */
  3411. {
  3412. uint64_t ip_config1 = IOREAD64_CR_REGIO(CORE_IP_CONFIG1);
  3413. /* Power of 2 Look-up table */
  3414. uint8_t pow_2_lut[8] = { 1, 2, 4, 8, 16, 32, 64, 128 };
  3415. /* LOCM per core size */
  3416. locm_size_kb = VHA_CR_GETBITS(CORE_IP_CONFIG1, CORE_OCM_RAM_SIZE_4KB, ip_config1) * 4;
  3417. /* SOCM total size */
  3418. socm_size_kb = VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_RAM_SIZE_4KB, ip_config1) * 4;
  3419. /* SOCM number of subbanks per bank array which is stored in hw reg as Log2 */
  3420. socm_num_sb = pow_2_lut[VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_NUM_SUBBANKS_LOG2, ip_config1)];
  3421. /* SOCM number of arrays per bank group */
  3422. socm_num_ba = 1 + VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_NUM_BANK_ARRAYS_MIN1, ip_config1);
  3423. /* SOCM number of bank groups */
  3424. socm_num_bg = 1 + VHA_CR_GETBITS(CORE_IP_CONFIG1, SYS_OCM_NUM_BANK_GROUPS_MIN1, ip_config1);
  3425. /* External memory interface width which is stored in hw reg as 8 * Log2 */
  3426. ext_mem_bus_width = 8 * pow_2_lut[VHA_CR_GETBITS(CORE_IP_CONFIG1, EXT_MEM_BUS_WIDTH, ip_config1)];
  3427. }
  3428. if (locm_size_kb) {
  3429. props->locm_size_bytes = locm_size_kb * 1024;
  3430. /* User may wanted to limit local OCM ... */
  3431. if (onchipmem_size) {
  3432. if (onchipmem_size < props->locm_size_bytes) {
  3433. dev_warn(vha->dev, "%s:Limiting local onchip memory to %u bytes (available:%u)\n",
  3434. __func__, onchipmem_size, props->locm_size_bytes);
  3435. props->locm_size_bytes = onchipmem_size;
  3436. } else if (onchipmem_size > props->locm_size_bytes) {
  3437. dev_warn(vha->dev, "%s: User defined local onchip memory size exceeded (%u > %u))\n",
  3438. __func__, onchipmem_size, props->locm_size_bytes);
  3439. }
  3440. }
  3441. } else {
  3442. props->locm_size_bytes = onchipmem_size;
  3443. }
  3444. if (socm_size_kb) {
  3445. props->socm_size_bytes = socm_size_kb * 1024;
  3446. /* User may wanted to limit shared OCM ... */
  3447. if (shared_onchipmem_size) {
  3448. if (shared_onchipmem_size < props->socm_size_bytes) {
  3449. dev_warn(vha->dev, "%s:Limiting shared onchip memory to %u bytes (available:%u)\n",
  3450. __func__, shared_onchipmem_size, props->socm_size_bytes);
  3451. props->socm_size_bytes = shared_onchipmem_size;
  3452. } else if (shared_onchipmem_size > props->socm_size_bytes) {
  3453. dev_warn(vha->dev, "%s: User defined shared onchip memory size exceeded (%u > %u))\n",
  3454. __func__, shared_onchipmem_size, props->socm_size_bytes);
  3455. }
  3456. }
  3457. {
  3458. /* SOCM per core must be must be a multiple of socm_total_sb & ext_mem_bus_width */
  3459. uint16_t socm_total_sb = socm_num_sb * socm_num_ba * socm_num_bg;
  3460. if (socm_total_sb && ext_mem_bus_width) {
  3461. /* The below division will round down */
  3462. props->socm_core_size_bytes = props->socm_size_bytes /
  3463. (props->num_cnn_core_devs * socm_total_sb * ext_mem_bus_width);
  3464. /* Scale it back */
  3465. props->socm_core_size_bytes *= socm_total_sb * ext_mem_bus_width;
  3466. } else {
  3467. /* Divide by number of cores as for dummy driver */
  3468. props->socm_core_size_bytes = shared_onchipmem_size / props->num_cnn_core_devs;
  3469. dev_warn(vha->dev, "%s: Shared onchip memory size per core can't be rounded"
  3470. " based on SB:%d BA:%d BG:%d BW:%d!\n", __func__,
  3471. socm_num_sb, socm_num_ba, socm_num_bg, ext_mem_bus_width);
  3472. }
  3473. }
  3474. } else {
  3475. props->socm_size_bytes = shared_onchipmem_size;
  3476. /* Just divide by number of cores (dummy driver) */
  3477. props->socm_core_size_bytes = shared_onchipmem_size / props->num_cnn_core_devs;
  3478. }
  3479. dev_info(vha->dev, "%s: Total onchip memory, Local: %u [kB], Shared total: %u [kB]"
  3480. " per core: %u [kB]\n", __func__, props->locm_size_bytes / 1024,
  3481. props->socm_size_bytes / 1024, props->socm_core_size_bytes / 1024);
  3482. dev_info(vha->dev, "%s: Devices: DUMMY:%u CNN:%u\n", __func__,
  3483. props->dummy_dev ? props->num_cnn_core_devs : 0,
  3484. props->dummy_dev ? 0 : props->num_cnn_core_devs);
  3485. return 0;
  3486. }
  3487. /* prepare CRC and DEBUG data buffers */
  3488. void vha_dbg_prepare_hwbufs(struct vha_session *session, struct vha_cmd *cmd,
  3489. struct vha_crc_config_regs *regs)
  3490. {
  3491. struct vha_dev *vha = session->vha;
  3492. uint8_t mask = cmd->hw_sched_info.core_mask;
  3493. if (session->cnn_dbg.cnn_crc_buf[0] || vha->cnn_combined_crc_enable) {
  3494. uint8_t id;
  3495. /* Note: all buffers have the same size */
  3496. img_pdump_printf("-- Select cores\n");
  3497. IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
  3498. /* enable CRC: address + mode */
  3499. if (session->cnn_dbg.cnn_crc_buf[0])
  3500. regs->crc_control |= VHA_CR_SETBITS(OS0_CNN_CRC_CONTROL, CNN_CRC_ENABLE,
  3501. session->cnn_dbg.cnn_crc_mode);
  3502. if (vha->cnn_combined_crc_enable)
  3503. regs->crc_control |= VHA_CR_SETBITS(OS0_CNN_CRC_CONTROL, COMBINED_CNN_CRC_ENABLE, 1);
  3504. img_pdump_printf("-- CRC_CONTROL=%llx buf 'CRC' size=%zx\n",
  3505. regs->crc_control,
  3506. session->cnn_dbg.cnn_crc_buf[0] ? session->cnn_dbg.cnn_crc_buf[0]->size : 0);
  3507. IOWRITE64_CR_PDUMP(regs->crc_control, OS0_CNN_CRC_CONTROL);
  3508. img_pdump_printf("-- CRC_MASK=%#x\n", session->cnn_dbg.cnn_crc_mask);
  3509. IOWRITE64_CR_PDUMP(session->cnn_dbg.cnn_crc_mask, OS0_CNN_CRC_MASK_CTRL);
  3510. regs->crc_mask_ctrl = session->cnn_dbg.cnn_crc_mask;
  3511. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  3512. if (mask & (1 << id)) {
  3513. /* Select core to be set */
  3514. IOWRITE64_CR_PDUMP(VHA_CR_SETBITS(CORE_CTRL_INDIRECT, MASK, (1 << id)),
  3515. CORE_CTRL_INDIRECT);
  3516. if (session->cnn_dbg.cnn_crc_buf[0]) {
  3517. struct vha_buffer *buf = session->cnn_dbg.cnn_crc_buf[id];
  3518. IOWRITE_PDUMP_BUFADDR(session, buf, 0, VHA_CR_OS0_CNN_CRC_ADDRESS);
  3519. SET_BUFADDR(session, buf, 0, &regs->crc_address[id]);
  3520. }
  3521. if (vha->cnn_combined_crc_enable) {
  3522. struct vha_buffer *buf = session->cnn_dbg.cnn_combined_crc;
  3523. IOWRITE_PDUMP_BUFADDR(session, buf, id * VHA_COMBINED_CRC_CORE_OFFSET,
  3524. VHA_CR_OS0_COMBINED_CNN_CRC_ADDRESS);
  3525. SET_BUFADDR(session, buf, id * VHA_COMBINED_CRC_CORE_OFFSET, &regs->crc_combined_address[id]);
  3526. }
  3527. }
  3528. }
  3529. }
  3530. if (session->cnn_dbg.cnn_dbg_buf[0] && session->cnn_dbg.cnn_dbg_pdump_enable) {
  3531. uint64_t val64;
  3532. uint8_t id;
  3533. /* Note: all buffers have the same size */
  3534. img_pdump_printf("-- Select cores\n");
  3535. IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
  3536. /* enable DEBUG: address, perf mode, band mode */
  3537. img_pdump_printf("-- DEBUG_CONTROL=%u,%u buf 'DBG' size=%zx\n",
  3538. GET_CNN_DBG_MODE(PERF, session), GET_CNN_DBG_MODE(BAND, session),
  3539. session->cnn_dbg.cnn_dbg_buf[0]->size);
  3540. val64 = VHA_CR_ALIGN_SETBITS(OS0_CNN_DEBUG_SIZE_LEGACY,
  3541. CNN_DEBUG_SIZE, session->cnn_dbg.cnn_dbg_buf[0]->size);
  3542. IOWRITE64_CR_PDUMP(val64, OS0_CNN_DEBUG_SIZE_LEGACY);
  3543. /* Set the CONTROL register only if requested */
  3544. if (CNN_DBG_MODE_ON(PERF, session) || CNN_DBG_MODE_ON(BAND, session)) {
  3545. val64 = VHA_CR_SETBITS(OS0_CNN_DEBUG_CONTROL, CNN_PERF_ENABLE,
  3546. GET_CNN_DBG_MODE(PERF, session));
  3547. val64 |= VHA_CR_SETBITS(OS0_CNN_DEBUG_CONTROL, CNN_BAND_ENABLE,
  3548. GET_CNN_DBG_MODE(BAND, session));
  3549. img_pdump_printf("IF DUMP_DBG\n");
  3550. IOWRITE64_CR_PDUMP(val64, OS0_CNN_DEBUG_CONTROL);
  3551. img_pdump_printf("FI DUMP_DBG\n");
  3552. }
  3553. for (id = 0; id < vha->hw_props.num_cnn_core_devs; id++) {
  3554. if (mask & (1 << id)) {
  3555. struct vha_buffer *buf = session->cnn_dbg.cnn_dbg_buf[id];
  3556. /* Select core to be set */
  3557. IOWRITE64_CR_PDUMP(VHA_CR_SETBITS(CORE_CTRL_INDIRECT, MASK, (1 << id)),
  3558. CORE_CTRL_INDIRECT);
  3559. IOWRITE_PDUMP_BUFADDR(session, buf, 0, VHA_CR_OS0_CNN_DEBUG_ADDRESS);
  3560. }
  3561. }
  3562. }
  3563. /* WM Performance & Bandwidth measurement */
  3564. if (WM_DBG_MODE_ON(PERF) || WM_DBG_MODE_ON(BAND)) {
  3565. uint64_t dbg_ctrl = 0;
  3566. img_pdump_printf("IF CHECK_PERF_BW\n");
  3567. if (WM_DBG_MODE_ON(PERF)) /* PERF */
  3568. dbg_ctrl = VHA_SET_FIELD_SIMPLE_VAL(WM_DEBUG_CONTROL, PERF_ENABLE, EN);
  3569. if (WM_DBG_MODE_ON(BAND)) { /* BW */
  3570. uint64_t hw_brns = cmd->user_cmd.cmd_type == VHA_CMD_CNN_SUBMIT_MULTI ?
  3571. ((struct vha_user_cnn_submit_multi_cmd*)&cmd->user_cmd)->hw_brns :
  3572. ((struct vha_user_cnn_submit_cmd*)&cmd->user_cmd)->hw_brns;
  3573. IOWRITE64_CR_PDUMP(VHA_CR_NOC_BWM_CONTROL_MASKFULL, NOC_BWM_CONTROL);
  3574. dbg_ctrl |= VHA_SET_FIELD_SIMPLE_VAL(WM_DEBUG_CONTROL, BW_ENABLE, EN);
  3575. if (VHA_IS_BRN(hw_brns, 71649)) {
  3576. img_pdump_printf("-- BRN71649_START\n");
  3577. IOWRITE64_CR_PDUMP(16, IDLE_HYSTERESIS_COUNT);
  3578. IOWRITE64_CR_PDUMP(16, PWR_MAN_HYSTERESIS);
  3579. img_pdump_printf("-- BRN71649_END\n");
  3580. }
  3581. }
  3582. IOWRITE64_CR_PDUMP(dbg_ctrl, WM_DEBUG_CONTROL);
  3583. img_pdump_printf("FI CHECK_PERF_BW\n");
  3584. }
  3585. }
  3586. /* flush CRC and DEBUG data buffers */
  3587. void vha_dbg_flush_hwbufs(struct vha_session *session, char checkpoint, uint8_t mask)
  3588. {
  3589. struct vha_dev *vha = session->vha;
  3590. if (session->cnn_dbg.cnn_dbg_flush != checkpoint)
  3591. return;
  3592. if (session->cnn_dbg.cnn_crc_buf[0] || vha->cnn_combined_crc_enable) {
  3593. int id;
  3594. /* Note: all buffers have the same size */
  3595. /*
  3596. * TOBEDONE: calculate CRC buffer size based
  3597. * on num passes, num layers, etc
  3598. */
  3599. img_pdump_printf("-- Save signatures\n");
  3600. img_pdump_printf("IF SKIP_CHECK_CRCS\n");
  3601. img_pdump_printf("COM Not checking CRCs!\n");
  3602. img_pdump_printf("ELSE SKIP_CHECK_CRCS\n");
  3603. img_pdump_printf("COM Checking CRCs ...\n");
  3604. if (session->cnn_dbg.cnn_crc_buf[0]) {
  3605. for (id = 0; id < session->vha->hw_props.num_cnn_core_devs; id++) {
  3606. if (mask & (1 << id)) {
  3607. struct vha_buffer *buf = session->cnn_dbg.cnn_crc_buf[id];
  3608. vha_pdump_sab_buf(session, PDUMP_CRC, buf, 0, buf->size);
  3609. }
  3610. }
  3611. }
  3612. if (vha->cnn_combined_crc_enable) {
  3613. struct vha_buffer *buf = session->cnn_dbg.cnn_combined_crc;
  3614. vha_pdump_sab_buf(session, PDUMP_CRC_CMB, buf, 0, buf->size);
  3615. }
  3616. img_pdump_printf("FI SKIP_CHECK_CRCS\n");
  3617. }
  3618. if (session->cnn_dbg.cnn_dbg_buf[0] && session->cnn_dbg.cnn_dbg_pdump_enable) {
  3619. int id;
  3620. img_pdump_printf("-- Save DEBUG info\n");
  3621. img_pdump_printf("IF DUMP_DBG\n");
  3622. img_pdump_printf("COM Dumping debug data ...\n");
  3623. for (id = 0; id < session->vha->hw_props.num_cnn_core_devs; id++) {
  3624. if (mask & (1 << id)) {
  3625. struct vha_buffer *buf = session->cnn_dbg.cnn_dbg_buf[id];
  3626. vha_pdump_sab_buf(session, PDUMP_DBG, buf, 0, buf->size);
  3627. }
  3628. }
  3629. img_pdump_printf("ELSE DUMP_DBG\n");
  3630. img_pdump_printf("COM Not dumping debug data!\n");
  3631. img_pdump_printf("FI DUMP_DBG\n");
  3632. }
  3633. }
  3634. /* stop capturing CRC and DEBUG data */
  3635. void vha_dbg_stop_hwbufs(struct vha_session *session, uint8_t mask)
  3636. {
  3637. struct vha_dev *vha = session->vha;
  3638. /* Flush hw debug buffers */
  3639. vha_dbg_flush_hwbufs(session, 0, mask);
  3640. if (session->cnn_dbg.cnn_crc_buf[0]) {
  3641. img_pdump_printf("-- Select cores\n");
  3642. IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
  3643. IOWRITE64_CR_PDUMP(0, OS0_CNN_CRC_CONTROL);
  3644. }
  3645. if (session->cnn_dbg.cnn_dbg_buf[0]) {
  3646. uint64_t size = 0;
  3647. int id;
  3648. for (id = 0; id < session->vha->hw_props.num_cnn_core_devs; id++) {
  3649. uint64_t val;
  3650. if (mask & (1 << id)) {
  3651. val = IOREAD64_CR_REGIO(OS0_CNN_DEBUG_STATUS);
  3652. if (val > size)
  3653. size = val;
  3654. }
  3655. }
  3656. if (CNN_DBG_MODE_ON(PERF, session) || CNN_DBG_MODE_ON(BAND, session)) {
  3657. img_pdump_printf("IF DUMP_DBG\n");
  3658. img_pdump_printf("-- Select cores\n");
  3659. IOWRITE64_CR_PDUMP((uint64_t)mask, CORE_CTRL_INDIRECT);
  3660. IOWRITE64_CR_PDUMP(0, OS0_CNN_DEBUG_CONTROL);
  3661. /* just give a hint in the pdump:
  3662. * dummy device returns 0 */
  3663. img_pdump_printf(
  3664. "-- POL64 :REG:%#x 0 0 0 1 1 -- DEBUG_STATUS=%llx\n",
  3665. VHA_CR_OS0_CNN_DEBUG_STATUS,
  3666. size);
  3667. img_pdump_printf("FI DUMP_DBG\n");
  3668. }
  3669. }
  3670. }
  3671. uint64_t vha_dbg_rtm_read(struct vha_dev *vha, uint64_t addr)
  3672. {
  3673. return 0ULL;
  3674. }
  3675. const struct vha_reg vha_regs[] = {
  3676. #define REG_DESC(reg) VHA_CR_##reg, VHA_CR_##reg##_MASKFULL
  3677. {"product_id ", REG_DESC(PRODUCT_ID)},
  3678. {"core_id ", REG_DESC(CORE_ID)},
  3679. {"integrator_id ", REG_DESC(CORE_IP_INTEGRATOR_ID)},
  3680. {"ip_changelist ", REG_DESC(CORE_IP_CHANGELIST)},
  3681. {"core_ip_config ", REG_DESC(CORE_IP_CONFIG)},
  3682. #undef REG_DESC
  3683. {NULL , 0},
  3684. };
  3685. #ifdef VHA_SCF
  3686. void wd_timer_callback(struct work_struct *work)
  3687. {
  3688. struct vha_dev *vha =
  3689. container_of(work, struct vha_dev, swd_dwork.work);
  3690. struct vha_cmd *cmd = NULL;
  3691. unsigned int wm_id;
  3692. mutex_lock(&vha->lock);
  3693. for (wm_id = 0; wm_id < vha->hw_props.num_cnn_core_devs; wm_id++) {
  3694. cmd = vha->pendcmd[wm_id].cmd;
  3695. if (cmd) {
  3696. uint8_t core_mask = vha_wm_get_cores(vha, wm_id);
  3697. uint8_t layer_count;
  3698. uint8_t pass_count;
  3699. bool lockup = false;
  3700. uint64_t exec_time_us;
  3701. uint64_t cmd_time_us;
  3702. struct TIMESPEC now;
  3703. GETNSTIMEOFDAY(&now);
  3704. if (cmd->user_cmd.flags & VHA_EXEC_TIME_SET) {
  3705. struct vha_user_cnn_submit_multi_cmd *cnn_user_cmd =
  3706. (struct vha_user_cnn_submit_multi_cmd *)&cmd->user_cmd;
  3707. cmd_time_us = cnn_user_cmd->exec_time;
  3708. } else if (vha->swd_timeout_default)
  3709. cmd_time_us = vha->swd_timeout_default;
  3710. else //SW WDT disabled for this cmd
  3711. continue;
  3712. cmd_time_us *= vha->swd_timeout_m0;
  3713. if (get_timespan_us(&cmd->hw_proc_start, &now, &exec_time_us)) {
  3714. uint64_t expected_exec_time = do_div(cmd_time_us, 100) + vha->swd_timeout_m1;
  3715. if (exec_time_us > expected_exec_time) {
  3716. lockup = true;
  3717. dev_err(vha->dev, "SW WDT lockup detected\n"
  3718. " measured time: %llu\n"
  3719. " cmd time: %llu\n"
  3720. " cmd expected_exec_time: %llu\n",
  3721. exec_time_us, cmd_time_us, expected_exec_time);
  3722. }
  3723. }
  3724. while (core_mask != 0 && !lockup) {
  3725. uint32_t core_id = VHA_CORE_MASK_TO_ID(core_mask);
  3726. uint64_t cnn_status;
  3727. uint64_t cnn_status2;
  3728. core_mask &= ~(VHA_CORE_ID_TO_MASK(core_id));
  3729. IOWRITE64_CR_REGIO(VHA_CR_SETBITS(CORE_CTRL_INDIRECT, MASK, (1 << core_id)),
  3730. CORE_CTRL_INDIRECT);
  3731. cnn_status = IOREAD64_CR_REGIO(OS0_CNN_STATUS);
  3732. cnn_status2 = IOREAD64_CR_REGIO(OS0_CNN_STATUS2);
  3733. layer_count = VHA_CR_GETBITS_OS(CNN_STATUS, LAYER_COUNT, cnn_status);
  3734. pass_count = VHA_CR_GETBITS_OS(CNN_STATUS2, PASS_COUNT, cnn_status2);
  3735. if (cmd->layer_count[core_id] == layer_count &&
  3736. cmd->pass_count[core_id] == pass_count) {
  3737. lockup = true;
  3738. dev_err(vha->dev, "SW WDT lockup detected\n"
  3739. " layer_count: %d\n"
  3740. " pass_count: %d\n", layer_count, pass_count);
  3741. }
  3742. cmd->layer_count[core_id] = layer_count;
  3743. cmd->pass_count[core_id] = pass_count;
  3744. }
  3745. if (lockup) {
  3746. if (vha_observers.error)
  3747. vha_observers.error(vha->id, cmd->session->id, cmd->user_cmd.cmd_id, -EIO);
  3748. /* Update stats. */
  3749. vha->stats.total_failures++;
  3750. vha->stats.cnn_kicks_completed++;
  3751. VHA_INC_WL_STAT(vha, kicks_completed, cmd);
  3752. vha_wm_reset(vha, &cmd->hw_sched_info);
  3753. /* Free command resources. */
  3754. vha_wm_release_cores(vha, cmd->hw_sched_info.core_mask, false);
  3755. vha_dev_free_cmd_res(vha, cmd, true);
  3756. /* Move command queue. */
  3757. vha_do_queued_cmd(vha, wm_id);
  3758. /* Handle actual command */
  3759. vha_handle_cmd(vha, wm_id, -EIO, -EIO, VHA_RSP_ERROR(SW_WDT_EXPIRED));
  3760. }
  3761. }
  3762. }
  3763. if (vha->state == VHA_STATE_ON)
  3764. schedule_delayed_work(&vha->swd_dwork, msecs_to_jiffies(vha->swd_period));
  3765. mutex_unlock(&vha->lock);
  3766. }
  3767. #endif