book3s_hv.c 150 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  4. * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
  5. *
  6. * Authors:
  7. * Paul Mackerras <paulus@au1.ibm.com>
  8. * Alexander Graf <agraf@suse.de>
  9. * Kevin Wolf <mail@kevin-wolf.de>
  10. *
  11. * Description: KVM functions specific to running on Book 3S
  12. * processors in hypervisor mode (specifically POWER7 and later).
  13. *
  14. * This file is derived from arch/powerpc/kvm/book3s.c,
  15. * by Alexander Graf <agraf@suse.de>.
  16. */
  17. #include <linux/kvm_host.h>
  18. #include <linux/kernel.h>
  19. #include <linux/err.h>
  20. #include <linux/slab.h>
  21. #include <linux/preempt.h>
  22. #include <linux/sched/signal.h>
  23. #include <linux/sched/stat.h>
  24. #include <linux/delay.h>
  25. #include <linux/export.h>
  26. #include <linux/fs.h>
  27. #include <linux/anon_inodes.h>
  28. #include <linux/cpu.h>
  29. #include <linux/cpumask.h>
  30. #include <linux/spinlock.h>
  31. #include <linux/page-flags.h>
  32. #include <linux/srcu.h>
  33. #include <linux/miscdevice.h>
  34. #include <linux/debugfs.h>
  35. #include <linux/gfp.h>
  36. #include <linux/vmalloc.h>
  37. #include <linux/highmem.h>
  38. #include <linux/hugetlb.h>
  39. #include <linux/kvm_irqfd.h>
  40. #include <linux/irqbypass.h>
  41. #include <linux/module.h>
  42. #include <linux/compiler.h>
  43. #include <linux/of.h>
  44. #include <asm/ftrace.h>
  45. #include <asm/reg.h>
  46. #include <asm/ppc-opcode.h>
  47. #include <asm/asm-prototypes.h>
  48. #include <asm/archrandom.h>
  49. #include <asm/debug.h>
  50. #include <asm/disassemble.h>
  51. #include <asm/cputable.h>
  52. #include <asm/cacheflush.h>
  53. #include <linux/uaccess.h>
  54. #include <asm/io.h>
  55. #include <asm/kvm_ppc.h>
  56. #include <asm/kvm_book3s.h>
  57. #include <asm/mmu_context.h>
  58. #include <asm/lppaca.h>
  59. #include <asm/pmc.h>
  60. #include <asm/processor.h>
  61. #include <asm/cputhreads.h>
  62. #include <asm/page.h>
  63. #include <asm/hvcall.h>
  64. #include <asm/switch_to.h>
  65. #include <asm/smp.h>
  66. #include <asm/dbell.h>
  67. #include <asm/hmi.h>
  68. #include <asm/pnv-pci.h>
  69. #include <asm/mmu.h>
  70. #include <asm/opal.h>
  71. #include <asm/xics.h>
  72. #include <asm/xive.h>
  73. #include <asm/hw_breakpoint.h>
  74. #include <asm/kvm_book3s_uvmem.h>
  75. #include <asm/ultravisor.h>
  76. #include <asm/dtl.h>
  77. #include "book3s.h"
  78. #define CREATE_TRACE_POINTS
  79. #include "trace_hv.h"
  80. /* #define EXIT_DEBUG */
  81. /* #define EXIT_DEBUG_SIMPLE */
  82. /* #define EXIT_DEBUG_INT */
  83. /* Used to indicate that a guest page fault needs to be handled */
  84. #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1)
  85. /* Used to indicate that a guest passthrough interrupt needs to be handled */
  86. #define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2)
  87. /* Used as a "null" value for timebase values */
  88. #define TB_NIL (~(u64)0)
  89. static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
  90. static int dynamic_mt_modes = 6;
  91. module_param(dynamic_mt_modes, int, 0644);
  92. MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
  93. static int target_smt_mode;
  94. module_param(target_smt_mode, int, 0644);
  95. MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
  96. static bool indep_threads_mode = true;
  97. module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR);
  98. MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)");
  99. static bool one_vm_per_core;
  100. module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
  101. MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires indep_threads_mode=N)");
  102. #ifdef CONFIG_KVM_XICS
  103. static const struct kernel_param_ops module_param_ops = {
  104. .set = param_set_int,
  105. .get = param_get_int,
  106. };
  107. module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
  108. MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
  109. module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
  110. MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
  111. #endif
  112. /* If set, guests are allowed to create and control nested guests */
  113. static bool nested = true;
  114. module_param(nested, bool, S_IRUGO | S_IWUSR);
  115. MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
  116. static inline bool nesting_enabled(struct kvm *kvm)
  117. {
  118. return kvm->arch.nested_enable && kvm_is_radix(kvm);
  119. }
  120. /* If set, the threads on each CPU core have to be in the same MMU mode */
  121. static bool no_mixing_hpt_and_radix;
  122. static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
  123. /*
  124. * RWMR values for POWER8. These control the rate at which PURR
  125. * and SPURR count and should be set according to the number of
  126. * online threads in the vcore being run.
  127. */
  128. #define RWMR_RPA_P8_1THREAD 0x164520C62609AECAUL
  129. #define RWMR_RPA_P8_2THREAD 0x7FFF2908450D8DA9UL
  130. #define RWMR_RPA_P8_3THREAD 0x164520C62609AECAUL
  131. #define RWMR_RPA_P8_4THREAD 0x199A421245058DA9UL
  132. #define RWMR_RPA_P8_5THREAD 0x164520C62609AECAUL
  133. #define RWMR_RPA_P8_6THREAD 0x164520C62609AECAUL
  134. #define RWMR_RPA_P8_7THREAD 0x164520C62609AECAUL
  135. #define RWMR_RPA_P8_8THREAD 0x164520C62609AECAUL
  136. static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
  137. RWMR_RPA_P8_1THREAD,
  138. RWMR_RPA_P8_1THREAD,
  139. RWMR_RPA_P8_2THREAD,
  140. RWMR_RPA_P8_3THREAD,
  141. RWMR_RPA_P8_4THREAD,
  142. RWMR_RPA_P8_5THREAD,
  143. RWMR_RPA_P8_6THREAD,
  144. RWMR_RPA_P8_7THREAD,
  145. RWMR_RPA_P8_8THREAD,
  146. };
  147. static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
  148. int *ip)
  149. {
  150. int i = *ip;
  151. struct kvm_vcpu *vcpu;
  152. while (++i < MAX_SMT_THREADS) {
  153. vcpu = READ_ONCE(vc->runnable_threads[i]);
  154. if (vcpu) {
  155. *ip = i;
  156. return vcpu;
  157. }
  158. }
  159. return NULL;
  160. }
  161. /* Used to traverse the list of runnable threads for a given vcore */
  162. #define for_each_runnable_thread(i, vcpu, vc) \
  163. for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
  164. static bool kvmppc_ipi_thread(int cpu)
  165. {
  166. unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
  167. /* If we're a nested hypervisor, fall back to ordinary IPIs for now */
  168. if (kvmhv_on_pseries())
  169. return false;
  170. /* On POWER9 we can use msgsnd to IPI any cpu */
  171. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  172. msg |= get_hard_smp_processor_id(cpu);
  173. smp_mb();
  174. __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
  175. return true;
  176. }
  177. /* On POWER8 for IPIs to threads in the same core, use msgsnd */
  178. if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
  179. preempt_disable();
  180. if (cpu_first_thread_sibling(cpu) ==
  181. cpu_first_thread_sibling(smp_processor_id())) {
  182. msg |= cpu_thread_in_core(cpu);
  183. smp_mb();
  184. __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
  185. preempt_enable();
  186. return true;
  187. }
  188. preempt_enable();
  189. }
  190. #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
  191. if (cpu >= 0 && cpu < nr_cpu_ids) {
  192. if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
  193. xics_wake_cpu(cpu);
  194. return true;
  195. }
  196. opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
  197. return true;
  198. }
  199. #endif
  200. return false;
  201. }
  202. static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
  203. {
  204. int cpu;
  205. struct rcuwait *waitp;
  206. waitp = kvm_arch_vcpu_get_wait(vcpu);
  207. if (rcuwait_wake_up(waitp))
  208. ++vcpu->stat.halt_wakeup;
  209. cpu = READ_ONCE(vcpu->arch.thread_cpu);
  210. if (cpu >= 0 && kvmppc_ipi_thread(cpu))
  211. return;
  212. /* CPU points to the first thread of the core */
  213. cpu = vcpu->cpu;
  214. if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
  215. smp_send_reschedule(cpu);
  216. }
  217. /*
  218. * We use the vcpu_load/put functions to measure stolen time.
  219. * Stolen time is counted as time when either the vcpu is able to
  220. * run as part of a virtual core, but the task running the vcore
  221. * is preempted or sleeping, or when the vcpu needs something done
  222. * in the kernel by the task running the vcpu, but that task is
  223. * preempted or sleeping. Those two things have to be counted
  224. * separately, since one of the vcpu tasks will take on the job
  225. * of running the core, and the other vcpu tasks in the vcore will
  226. * sleep waiting for it to do that, but that sleep shouldn't count
  227. * as stolen time.
  228. *
  229. * Hence we accumulate stolen time when the vcpu can run as part of
  230. * a vcore using vc->stolen_tb, and the stolen time when the vcpu
  231. * needs its task to do other things in the kernel (for example,
  232. * service a page fault) in busy_stolen. We don't accumulate
  233. * stolen time for a vcore when it is inactive, or for a vcpu
  234. * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of
  235. * a misnomer; it means that the vcpu task is not executing in
  236. * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
  237. * the kernel. We don't have any way of dividing up that time
  238. * between time that the vcpu is genuinely stopped, time that
  239. * the task is actively working on behalf of the vcpu, and time
  240. * that the task is preempted, so we don't count any of it as
  241. * stolen.
  242. *
  243. * Updates to busy_stolen are protected by arch.tbacct_lock;
  244. * updates to vc->stolen_tb are protected by the vcore->stoltb_lock
  245. * lock. The stolen times are measured in units of timebase ticks.
  246. * (Note that the != TB_NIL checks below are purely defensive;
  247. * they should never fail.)
  248. */
  249. static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc)
  250. {
  251. unsigned long flags;
  252. spin_lock_irqsave(&vc->stoltb_lock, flags);
  253. vc->preempt_tb = mftb();
  254. spin_unlock_irqrestore(&vc->stoltb_lock, flags);
  255. }
  256. static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc)
  257. {
  258. unsigned long flags;
  259. spin_lock_irqsave(&vc->stoltb_lock, flags);
  260. if (vc->preempt_tb != TB_NIL) {
  261. vc->stolen_tb += mftb() - vc->preempt_tb;
  262. vc->preempt_tb = TB_NIL;
  263. }
  264. spin_unlock_irqrestore(&vc->stoltb_lock, flags);
  265. }
  266. static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
  267. {
  268. struct kvmppc_vcore *vc = vcpu->arch.vcore;
  269. unsigned long flags;
  270. /*
  271. * We can test vc->runner without taking the vcore lock,
  272. * because only this task ever sets vc->runner to this
  273. * vcpu, and once it is set to this vcpu, only this task
  274. * ever sets it to NULL.
  275. */
  276. if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
  277. kvmppc_core_end_stolen(vc);
  278. spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
  279. if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
  280. vcpu->arch.busy_preempt != TB_NIL) {
  281. vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt;
  282. vcpu->arch.busy_preempt = TB_NIL;
  283. }
  284. spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
  285. }
  286. static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
  287. {
  288. struct kvmppc_vcore *vc = vcpu->arch.vcore;
  289. unsigned long flags;
  290. if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
  291. kvmppc_core_start_stolen(vc);
  292. spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
  293. if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
  294. vcpu->arch.busy_preempt = mftb();
  295. spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
  296. }
  297. static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
  298. {
  299. vcpu->arch.pvr = pvr;
  300. }
  301. /* Dummy value used in computing PCR value below */
  302. #define PCR_ARCH_31 (PCR_ARCH_300 << 1)
  303. static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
  304. {
  305. unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
  306. struct kvmppc_vcore *vc = vcpu->arch.vcore;
  307. /* We can (emulate) our own architecture version and anything older */
  308. if (cpu_has_feature(CPU_FTR_ARCH_31))
  309. host_pcr_bit = PCR_ARCH_31;
  310. else if (cpu_has_feature(CPU_FTR_ARCH_300))
  311. host_pcr_bit = PCR_ARCH_300;
  312. else if (cpu_has_feature(CPU_FTR_ARCH_207S))
  313. host_pcr_bit = PCR_ARCH_207;
  314. else if (cpu_has_feature(CPU_FTR_ARCH_206))
  315. host_pcr_bit = PCR_ARCH_206;
  316. else
  317. host_pcr_bit = PCR_ARCH_205;
  318. /* Determine lowest PCR bit needed to run guest in given PVR level */
  319. guest_pcr_bit = host_pcr_bit;
  320. if (arch_compat) {
  321. switch (arch_compat) {
  322. case PVR_ARCH_205:
  323. guest_pcr_bit = PCR_ARCH_205;
  324. break;
  325. case PVR_ARCH_206:
  326. case PVR_ARCH_206p:
  327. guest_pcr_bit = PCR_ARCH_206;
  328. break;
  329. case PVR_ARCH_207:
  330. guest_pcr_bit = PCR_ARCH_207;
  331. break;
  332. case PVR_ARCH_300:
  333. guest_pcr_bit = PCR_ARCH_300;
  334. break;
  335. case PVR_ARCH_31:
  336. guest_pcr_bit = PCR_ARCH_31;
  337. break;
  338. default:
  339. return -EINVAL;
  340. }
  341. }
  342. /* Check requested PCR bits don't exceed our capabilities */
  343. if (guest_pcr_bit > host_pcr_bit)
  344. return -EINVAL;
  345. spin_lock(&vc->lock);
  346. vc->arch_compat = arch_compat;
  347. /*
  348. * Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit
  349. * Also set all reserved PCR bits
  350. */
  351. vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK;
  352. spin_unlock(&vc->lock);
  353. return 0;
  354. }
  355. static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
  356. {
  357. int r;
  358. pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
  359. pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
  360. vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
  361. for (r = 0; r < 16; ++r)
  362. pr_err("r%2d = %.16lx r%d = %.16lx\n",
  363. r, kvmppc_get_gpr(vcpu, r),
  364. r+16, kvmppc_get_gpr(vcpu, r+16));
  365. pr_err("ctr = %.16lx lr = %.16lx\n",
  366. vcpu->arch.regs.ctr, vcpu->arch.regs.link);
  367. pr_err("srr0 = %.16llx srr1 = %.16llx\n",
  368. vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
  369. pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
  370. vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
  371. pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
  372. vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
  373. pr_err("cr = %.8lx xer = %.16lx dsisr = %.8x\n",
  374. vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
  375. pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
  376. pr_err("fault dar = %.16lx dsisr = %.8x\n",
  377. vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
  378. pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
  379. for (r = 0; r < vcpu->arch.slb_max; ++r)
  380. pr_err(" ESID = %.16llx VSID = %.16llx\n",
  381. vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
  382. pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
  383. vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
  384. vcpu->arch.last_inst);
  385. }
  386. static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
  387. {
  388. return kvm_get_vcpu_by_id(kvm, id);
  389. }
  390. static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
  391. {
  392. vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
  393. vpa->yield_count = cpu_to_be32(1);
  394. }
  395. static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
  396. unsigned long addr, unsigned long len)
  397. {
  398. /* check address is cacheline aligned */
  399. if (addr & (L1_CACHE_BYTES - 1))
  400. return -EINVAL;
  401. spin_lock(&vcpu->arch.vpa_update_lock);
  402. if (v->next_gpa != addr || v->len != len) {
  403. v->next_gpa = addr;
  404. v->len = addr ? len : 0;
  405. v->update_pending = 1;
  406. }
  407. spin_unlock(&vcpu->arch.vpa_update_lock);
  408. return 0;
  409. }
  410. /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
  411. struct reg_vpa {
  412. u32 dummy;
  413. union {
  414. __be16 hword;
  415. __be32 word;
  416. } length;
  417. };
  418. static int vpa_is_registered(struct kvmppc_vpa *vpap)
  419. {
  420. if (vpap->update_pending)
  421. return vpap->next_gpa != 0;
  422. return vpap->pinned_addr != NULL;
  423. }
  424. static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
  425. unsigned long flags,
  426. unsigned long vcpuid, unsigned long vpa)
  427. {
  428. struct kvm *kvm = vcpu->kvm;
  429. unsigned long len, nb;
  430. void *va;
  431. struct kvm_vcpu *tvcpu;
  432. int err;
  433. int subfunc;
  434. struct kvmppc_vpa *vpap;
  435. tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
  436. if (!tvcpu)
  437. return H_PARAMETER;
  438. subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
  439. if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
  440. subfunc == H_VPA_REG_SLB) {
  441. /* Registering new area - address must be cache-line aligned */
  442. if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
  443. return H_PARAMETER;
  444. /* convert logical addr to kernel addr and read length */
  445. va = kvmppc_pin_guest_page(kvm, vpa, &nb);
  446. if (va == NULL)
  447. return H_PARAMETER;
  448. if (subfunc == H_VPA_REG_VPA)
  449. len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
  450. else
  451. len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
  452. kvmppc_unpin_guest_page(kvm, va, vpa, false);
  453. /* Check length */
  454. if (len > nb || len < sizeof(struct reg_vpa))
  455. return H_PARAMETER;
  456. } else {
  457. vpa = 0;
  458. len = 0;
  459. }
  460. err = H_PARAMETER;
  461. vpap = NULL;
  462. spin_lock(&tvcpu->arch.vpa_update_lock);
  463. switch (subfunc) {
  464. case H_VPA_REG_VPA: /* register VPA */
  465. /*
  466. * The size of our lppaca is 1kB because of the way we align
  467. * it for the guest to avoid crossing a 4kB boundary. We only
  468. * use 640 bytes of the structure though, so we should accept
  469. * clients that set a size of 640.
  470. */
  471. BUILD_BUG_ON(sizeof(struct lppaca) != 640);
  472. if (len < sizeof(struct lppaca))
  473. break;
  474. vpap = &tvcpu->arch.vpa;
  475. err = 0;
  476. break;
  477. case H_VPA_REG_DTL: /* register DTL */
  478. if (len < sizeof(struct dtl_entry))
  479. break;
  480. len -= len % sizeof(struct dtl_entry);
  481. /* Check that they have previously registered a VPA */
  482. err = H_RESOURCE;
  483. if (!vpa_is_registered(&tvcpu->arch.vpa))
  484. break;
  485. vpap = &tvcpu->arch.dtl;
  486. err = 0;
  487. break;
  488. case H_VPA_REG_SLB: /* register SLB shadow buffer */
  489. /* Check that they have previously registered a VPA */
  490. err = H_RESOURCE;
  491. if (!vpa_is_registered(&tvcpu->arch.vpa))
  492. break;
  493. vpap = &tvcpu->arch.slb_shadow;
  494. err = 0;
  495. break;
  496. case H_VPA_DEREG_VPA: /* deregister VPA */
  497. /* Check they don't still have a DTL or SLB buf registered */
  498. err = H_RESOURCE;
  499. if (vpa_is_registered(&tvcpu->arch.dtl) ||
  500. vpa_is_registered(&tvcpu->arch.slb_shadow))
  501. break;
  502. vpap = &tvcpu->arch.vpa;
  503. err = 0;
  504. break;
  505. case H_VPA_DEREG_DTL: /* deregister DTL */
  506. vpap = &tvcpu->arch.dtl;
  507. err = 0;
  508. break;
  509. case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */
  510. vpap = &tvcpu->arch.slb_shadow;
  511. err = 0;
  512. break;
  513. }
  514. if (vpap) {
  515. vpap->next_gpa = vpa;
  516. vpap->len = len;
  517. vpap->update_pending = 1;
  518. }
  519. spin_unlock(&tvcpu->arch.vpa_update_lock);
  520. return err;
  521. }
  522. static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
  523. {
  524. struct kvm *kvm = vcpu->kvm;
  525. void *va;
  526. unsigned long nb;
  527. unsigned long gpa;
  528. /*
  529. * We need to pin the page pointed to by vpap->next_gpa,
  530. * but we can't call kvmppc_pin_guest_page under the lock
  531. * as it does get_user_pages() and down_read(). So we
  532. * have to drop the lock, pin the page, then get the lock
  533. * again and check that a new area didn't get registered
  534. * in the meantime.
  535. */
  536. for (;;) {
  537. gpa = vpap->next_gpa;
  538. spin_unlock(&vcpu->arch.vpa_update_lock);
  539. va = NULL;
  540. nb = 0;
  541. if (gpa)
  542. va = kvmppc_pin_guest_page(kvm, gpa, &nb);
  543. spin_lock(&vcpu->arch.vpa_update_lock);
  544. if (gpa == vpap->next_gpa)
  545. break;
  546. /* sigh... unpin that one and try again */
  547. if (va)
  548. kvmppc_unpin_guest_page(kvm, va, gpa, false);
  549. }
  550. vpap->update_pending = 0;
  551. if (va && nb < vpap->len) {
  552. /*
  553. * If it's now too short, it must be that userspace
  554. * has changed the mappings underlying guest memory,
  555. * so unregister the region.
  556. */
  557. kvmppc_unpin_guest_page(kvm, va, gpa, false);
  558. va = NULL;
  559. }
  560. if (vpap->pinned_addr)
  561. kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
  562. vpap->dirty);
  563. vpap->gpa = gpa;
  564. vpap->pinned_addr = va;
  565. vpap->dirty = false;
  566. if (va)
  567. vpap->pinned_end = va + vpap->len;
  568. }
  569. static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
  570. {
  571. if (!(vcpu->arch.vpa.update_pending ||
  572. vcpu->arch.slb_shadow.update_pending ||
  573. vcpu->arch.dtl.update_pending))
  574. return;
  575. spin_lock(&vcpu->arch.vpa_update_lock);
  576. if (vcpu->arch.vpa.update_pending) {
  577. kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
  578. if (vcpu->arch.vpa.pinned_addr)
  579. init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
  580. }
  581. if (vcpu->arch.dtl.update_pending) {
  582. kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
  583. vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
  584. vcpu->arch.dtl_index = 0;
  585. }
  586. if (vcpu->arch.slb_shadow.update_pending)
  587. kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
  588. spin_unlock(&vcpu->arch.vpa_update_lock);
  589. }
  590. /*
  591. * Return the accumulated stolen time for the vcore up until `now'.
  592. * The caller should hold the vcore lock.
  593. */
  594. static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
  595. {
  596. u64 p;
  597. unsigned long flags;
  598. spin_lock_irqsave(&vc->stoltb_lock, flags);
  599. p = vc->stolen_tb;
  600. if (vc->vcore_state != VCORE_INACTIVE &&
  601. vc->preempt_tb != TB_NIL)
  602. p += now - vc->preempt_tb;
  603. spin_unlock_irqrestore(&vc->stoltb_lock, flags);
  604. return p;
  605. }
  606. static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
  607. struct kvmppc_vcore *vc)
  608. {
  609. struct dtl_entry *dt;
  610. struct lppaca *vpa;
  611. unsigned long stolen;
  612. unsigned long core_stolen;
  613. u64 now;
  614. unsigned long flags;
  615. dt = vcpu->arch.dtl_ptr;
  616. vpa = vcpu->arch.vpa.pinned_addr;
  617. now = mftb();
  618. core_stolen = vcore_stolen_time(vc, now);
  619. stolen = core_stolen - vcpu->arch.stolen_logged;
  620. vcpu->arch.stolen_logged = core_stolen;
  621. spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
  622. stolen += vcpu->arch.busy_stolen;
  623. vcpu->arch.busy_stolen = 0;
  624. spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
  625. if (!dt || !vpa)
  626. return;
  627. memset(dt, 0, sizeof(struct dtl_entry));
  628. dt->dispatch_reason = 7;
  629. dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid);
  630. dt->timebase = cpu_to_be64(now + vc->tb_offset);
  631. dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
  632. dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
  633. dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
  634. ++dt;
  635. if (dt == vcpu->arch.dtl.pinned_end)
  636. dt = vcpu->arch.dtl.pinned_addr;
  637. vcpu->arch.dtl_ptr = dt;
  638. /* order writing *dt vs. writing vpa->dtl_idx */
  639. smp_wmb();
  640. vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
  641. vcpu->arch.dtl.dirty = true;
  642. }
  643. /* See if there is a doorbell interrupt pending for a vcpu */
  644. static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
  645. {
  646. int thr;
  647. struct kvmppc_vcore *vc;
  648. if (vcpu->arch.doorbell_request)
  649. return true;
  650. /*
  651. * Ensure that the read of vcore->dpdes comes after the read
  652. * of vcpu->doorbell_request. This barrier matches the
  653. * smp_wmb() in kvmppc_guest_entry_inject().
  654. */
  655. smp_rmb();
  656. vc = vcpu->arch.vcore;
  657. thr = vcpu->vcpu_id - vc->first_vcpuid;
  658. return !!(vc->dpdes & (1 << thr));
  659. }
  660. static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
  661. {
  662. if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
  663. return true;
  664. if ((!vcpu->arch.vcore->arch_compat) &&
  665. cpu_has_feature(CPU_FTR_ARCH_207S))
  666. return true;
  667. return false;
  668. }
  669. static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
  670. unsigned long resource, unsigned long value1,
  671. unsigned long value2)
  672. {
  673. switch (resource) {
  674. case H_SET_MODE_RESOURCE_SET_CIABR:
  675. if (!kvmppc_power8_compatible(vcpu))
  676. return H_P2;
  677. if (value2)
  678. return H_P4;
  679. if (mflags)
  680. return H_UNSUPPORTED_FLAG_START;
  681. /* Guests can't breakpoint the hypervisor */
  682. if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
  683. return H_P3;
  684. vcpu->arch.ciabr = value1;
  685. return H_SUCCESS;
  686. case H_SET_MODE_RESOURCE_SET_DAWR0:
  687. if (!kvmppc_power8_compatible(vcpu))
  688. return H_P2;
  689. if (!ppc_breakpoint_available())
  690. return H_P2;
  691. if (mflags)
  692. return H_UNSUPPORTED_FLAG_START;
  693. if (value2 & DABRX_HYP)
  694. return H_P4;
  695. vcpu->arch.dawr = value1;
  696. vcpu->arch.dawrx = value2;
  697. return H_SUCCESS;
  698. case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
  699. /* KVM does not support mflags=2 (AIL=2) */
  700. if (mflags != 0 && mflags != 3)
  701. return H_UNSUPPORTED_FLAG_START;
  702. return H_TOO_HARD;
  703. default:
  704. return H_TOO_HARD;
  705. }
  706. }
  707. /* Copy guest memory in place - must reside within a single memslot */
  708. static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
  709. unsigned long len)
  710. {
  711. struct kvm_memory_slot *to_memslot = NULL;
  712. struct kvm_memory_slot *from_memslot = NULL;
  713. unsigned long to_addr, from_addr;
  714. int r;
  715. /* Get HPA for from address */
  716. from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
  717. if (!from_memslot)
  718. return -EFAULT;
  719. if ((from + len) >= ((from_memslot->base_gfn + from_memslot->npages)
  720. << PAGE_SHIFT))
  721. return -EINVAL;
  722. from_addr = gfn_to_hva_memslot(from_memslot, from >> PAGE_SHIFT);
  723. if (kvm_is_error_hva(from_addr))
  724. return -EFAULT;
  725. from_addr |= (from & (PAGE_SIZE - 1));
  726. /* Get HPA for to address */
  727. to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
  728. if (!to_memslot)
  729. return -EFAULT;
  730. if ((to + len) >= ((to_memslot->base_gfn + to_memslot->npages)
  731. << PAGE_SHIFT))
  732. return -EINVAL;
  733. to_addr = gfn_to_hva_memslot(to_memslot, to >> PAGE_SHIFT);
  734. if (kvm_is_error_hva(to_addr))
  735. return -EFAULT;
  736. to_addr |= (to & (PAGE_SIZE - 1));
  737. /* Perform copy */
  738. r = raw_copy_in_user((void __user *)to_addr, (void __user *)from_addr,
  739. len);
  740. if (r)
  741. return -EFAULT;
  742. mark_page_dirty(kvm, to >> PAGE_SHIFT);
  743. return 0;
  744. }
  745. static long kvmppc_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags,
  746. unsigned long dest, unsigned long src)
  747. {
  748. u64 pg_sz = SZ_4K; /* 4K page size */
  749. u64 pg_mask = SZ_4K - 1;
  750. int ret;
  751. /* Check for invalid flags (H_PAGE_SET_LOANED covers all CMO flags) */
  752. if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE |
  753. H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED))
  754. return H_PARAMETER;
  755. /* dest (and src if copy_page flag set) must be page aligned */
  756. if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask)))
  757. return H_PARAMETER;
  758. /* zero and/or copy the page as determined by the flags */
  759. if (flags & H_COPY_PAGE) {
  760. ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
  761. if (ret < 0)
  762. return H_PARAMETER;
  763. } else if (flags & H_ZERO_PAGE) {
  764. ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
  765. if (ret < 0)
  766. return H_PARAMETER;
  767. }
  768. /* We can ignore the remaining flags */
  769. return H_SUCCESS;
  770. }
  771. static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
  772. {
  773. struct kvmppc_vcore *vcore = target->arch.vcore;
  774. /*
  775. * We expect to have been called by the real mode handler
  776. * (kvmppc_rm_h_confer()) which would have directly returned
  777. * H_SUCCESS if the source vcore wasn't idle (e.g. if it may
  778. * have useful work to do and should not confer) so we don't
  779. * recheck that here.
  780. */
  781. spin_lock(&vcore->lock);
  782. if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
  783. vcore->vcore_state != VCORE_INACTIVE &&
  784. vcore->runner)
  785. target = vcore->runner;
  786. spin_unlock(&vcore->lock);
  787. return kvm_vcpu_yield_to(target);
  788. }
  789. static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
  790. {
  791. int yield_count = 0;
  792. struct lppaca *lppaca;
  793. spin_lock(&vcpu->arch.vpa_update_lock);
  794. lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
  795. if (lppaca)
  796. yield_count = be32_to_cpu(lppaca->yield_count);
  797. spin_unlock(&vcpu->arch.vpa_update_lock);
  798. return yield_count;
  799. }
  800. int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
  801. {
  802. unsigned long req = kvmppc_get_gpr(vcpu, 3);
  803. unsigned long target, ret = H_SUCCESS;
  804. int yield_count;
  805. struct kvm_vcpu *tvcpu;
  806. int idx, rc;
  807. if (req <= MAX_HCALL_OPCODE &&
  808. !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
  809. return RESUME_HOST;
  810. switch (req) {
  811. case H_CEDE:
  812. break;
  813. case H_PROD:
  814. target = kvmppc_get_gpr(vcpu, 4);
  815. tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
  816. if (!tvcpu) {
  817. ret = H_PARAMETER;
  818. break;
  819. }
  820. tvcpu->arch.prodded = 1;
  821. smp_mb();
  822. if (tvcpu->arch.ceded)
  823. kvmppc_fast_vcpu_kick_hv(tvcpu);
  824. break;
  825. case H_CONFER:
  826. target = kvmppc_get_gpr(vcpu, 4);
  827. if (target == -1)
  828. break;
  829. tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
  830. if (!tvcpu) {
  831. ret = H_PARAMETER;
  832. break;
  833. }
  834. yield_count = kvmppc_get_gpr(vcpu, 5);
  835. if (kvmppc_get_yield_count(tvcpu) != yield_count)
  836. break;
  837. kvm_arch_vcpu_yield_to(tvcpu);
  838. break;
  839. case H_REGISTER_VPA:
  840. ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
  841. kvmppc_get_gpr(vcpu, 5),
  842. kvmppc_get_gpr(vcpu, 6));
  843. break;
  844. case H_RTAS:
  845. if (list_empty(&vcpu->kvm->arch.rtas_tokens))
  846. return RESUME_HOST;
  847. idx = srcu_read_lock(&vcpu->kvm->srcu);
  848. rc = kvmppc_rtas_hcall(vcpu);
  849. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  850. if (rc == -ENOENT)
  851. return RESUME_HOST;
  852. else if (rc == 0)
  853. break;
  854. /* Send the error out to userspace via KVM_RUN */
  855. return rc;
  856. case H_LOGICAL_CI_LOAD:
  857. ret = kvmppc_h_logical_ci_load(vcpu);
  858. if (ret == H_TOO_HARD)
  859. return RESUME_HOST;
  860. break;
  861. case H_LOGICAL_CI_STORE:
  862. ret = kvmppc_h_logical_ci_store(vcpu);
  863. if (ret == H_TOO_HARD)
  864. return RESUME_HOST;
  865. break;
  866. case H_SET_MODE:
  867. ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
  868. kvmppc_get_gpr(vcpu, 5),
  869. kvmppc_get_gpr(vcpu, 6),
  870. kvmppc_get_gpr(vcpu, 7));
  871. if (ret == H_TOO_HARD)
  872. return RESUME_HOST;
  873. break;
  874. case H_XIRR:
  875. case H_CPPR:
  876. case H_EOI:
  877. case H_IPI:
  878. case H_IPOLL:
  879. case H_XIRR_X:
  880. if (kvmppc_xics_enabled(vcpu)) {
  881. if (xics_on_xive()) {
  882. ret = H_NOT_AVAILABLE;
  883. return RESUME_GUEST;
  884. }
  885. ret = kvmppc_xics_hcall(vcpu, req);
  886. break;
  887. }
  888. return RESUME_HOST;
  889. case H_SET_DABR:
  890. ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
  891. break;
  892. case H_SET_XDABR:
  893. ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
  894. kvmppc_get_gpr(vcpu, 5));
  895. break;
  896. #ifdef CONFIG_SPAPR_TCE_IOMMU
  897. case H_GET_TCE:
  898. ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
  899. kvmppc_get_gpr(vcpu, 5));
  900. if (ret == H_TOO_HARD)
  901. return RESUME_HOST;
  902. break;
  903. case H_PUT_TCE:
  904. ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
  905. kvmppc_get_gpr(vcpu, 5),
  906. kvmppc_get_gpr(vcpu, 6));
  907. if (ret == H_TOO_HARD)
  908. return RESUME_HOST;
  909. break;
  910. case H_PUT_TCE_INDIRECT:
  911. ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
  912. kvmppc_get_gpr(vcpu, 5),
  913. kvmppc_get_gpr(vcpu, 6),
  914. kvmppc_get_gpr(vcpu, 7));
  915. if (ret == H_TOO_HARD)
  916. return RESUME_HOST;
  917. break;
  918. case H_STUFF_TCE:
  919. ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
  920. kvmppc_get_gpr(vcpu, 5),
  921. kvmppc_get_gpr(vcpu, 6),
  922. kvmppc_get_gpr(vcpu, 7));
  923. if (ret == H_TOO_HARD)
  924. return RESUME_HOST;
  925. break;
  926. #endif
  927. case H_RANDOM:
  928. if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4]))
  929. ret = H_HARDWARE;
  930. break;
  931. case H_SET_PARTITION_TABLE:
  932. ret = H_FUNCTION;
  933. if (nesting_enabled(vcpu->kvm))
  934. ret = kvmhv_set_partition_table(vcpu);
  935. break;
  936. case H_ENTER_NESTED:
  937. ret = H_FUNCTION;
  938. if (!nesting_enabled(vcpu->kvm))
  939. break;
  940. ret = kvmhv_enter_nested_guest(vcpu);
  941. if (ret == H_INTERRUPT) {
  942. kvmppc_set_gpr(vcpu, 3, 0);
  943. vcpu->arch.hcall_needed = 0;
  944. return -EINTR;
  945. } else if (ret == H_TOO_HARD) {
  946. kvmppc_set_gpr(vcpu, 3, 0);
  947. vcpu->arch.hcall_needed = 0;
  948. return RESUME_HOST;
  949. }
  950. break;
  951. case H_TLB_INVALIDATE:
  952. ret = H_FUNCTION;
  953. if (nesting_enabled(vcpu->kvm))
  954. ret = kvmhv_do_nested_tlbie(vcpu);
  955. break;
  956. case H_COPY_TOFROM_GUEST:
  957. ret = H_FUNCTION;
  958. if (nesting_enabled(vcpu->kvm))
  959. ret = kvmhv_copy_tofrom_guest_nested(vcpu);
  960. break;
  961. case H_PAGE_INIT:
  962. ret = kvmppc_h_page_init(vcpu, kvmppc_get_gpr(vcpu, 4),
  963. kvmppc_get_gpr(vcpu, 5),
  964. kvmppc_get_gpr(vcpu, 6));
  965. break;
  966. case H_SVM_PAGE_IN:
  967. ret = H_UNSUPPORTED;
  968. if (kvmppc_get_srr1(vcpu) & MSR_S)
  969. ret = kvmppc_h_svm_page_in(vcpu->kvm,
  970. kvmppc_get_gpr(vcpu, 4),
  971. kvmppc_get_gpr(vcpu, 5),
  972. kvmppc_get_gpr(vcpu, 6));
  973. break;
  974. case H_SVM_PAGE_OUT:
  975. ret = H_UNSUPPORTED;
  976. if (kvmppc_get_srr1(vcpu) & MSR_S)
  977. ret = kvmppc_h_svm_page_out(vcpu->kvm,
  978. kvmppc_get_gpr(vcpu, 4),
  979. kvmppc_get_gpr(vcpu, 5),
  980. kvmppc_get_gpr(vcpu, 6));
  981. break;
  982. case H_SVM_INIT_START:
  983. ret = H_UNSUPPORTED;
  984. if (kvmppc_get_srr1(vcpu) & MSR_S)
  985. ret = kvmppc_h_svm_init_start(vcpu->kvm);
  986. break;
  987. case H_SVM_INIT_DONE:
  988. ret = H_UNSUPPORTED;
  989. if (kvmppc_get_srr1(vcpu) & MSR_S)
  990. ret = kvmppc_h_svm_init_done(vcpu->kvm);
  991. break;
  992. case H_SVM_INIT_ABORT:
  993. /*
  994. * Even if that call is made by the Ultravisor, the SSR1 value
  995. * is the guest context one, with the secure bit clear as it has
  996. * not yet been secured. So we can't check it here.
  997. * Instead the kvm->arch.secure_guest flag is checked inside
  998. * kvmppc_h_svm_init_abort().
  999. */
  1000. ret = kvmppc_h_svm_init_abort(vcpu->kvm);
  1001. break;
  1002. default:
  1003. return RESUME_HOST;
  1004. }
  1005. kvmppc_set_gpr(vcpu, 3, ret);
  1006. vcpu->arch.hcall_needed = 0;
  1007. return RESUME_GUEST;
  1008. }
  1009. /*
  1010. * Handle H_CEDE in the nested virtualization case where we haven't
  1011. * called the real-mode hcall handlers in book3s_hv_rmhandlers.S.
  1012. * This has to be done early, not in kvmppc_pseries_do_hcall(), so
  1013. * that the cede logic in kvmppc_run_single_vcpu() works properly.
  1014. */
  1015. static void kvmppc_nested_cede(struct kvm_vcpu *vcpu)
  1016. {
  1017. vcpu->arch.shregs.msr |= MSR_EE;
  1018. vcpu->arch.ceded = 1;
  1019. smp_mb();
  1020. if (vcpu->arch.prodded) {
  1021. vcpu->arch.prodded = 0;
  1022. smp_mb();
  1023. vcpu->arch.ceded = 0;
  1024. }
  1025. }
  1026. static int kvmppc_hcall_impl_hv(unsigned long cmd)
  1027. {
  1028. switch (cmd) {
  1029. case H_CEDE:
  1030. case H_PROD:
  1031. case H_CONFER:
  1032. case H_REGISTER_VPA:
  1033. case H_SET_MODE:
  1034. case H_LOGICAL_CI_LOAD:
  1035. case H_LOGICAL_CI_STORE:
  1036. #ifdef CONFIG_KVM_XICS
  1037. case H_XIRR:
  1038. case H_CPPR:
  1039. case H_EOI:
  1040. case H_IPI:
  1041. case H_IPOLL:
  1042. case H_XIRR_X:
  1043. #endif
  1044. case H_PAGE_INIT:
  1045. return 1;
  1046. }
  1047. /* See if it's in the real-mode table */
  1048. return kvmppc_hcall_impl_hv_realmode(cmd);
  1049. }
  1050. static int kvmppc_emulate_debug_inst(struct kvm_vcpu *vcpu)
  1051. {
  1052. u32 last_inst;
  1053. if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
  1054. EMULATE_DONE) {
  1055. /*
  1056. * Fetch failed, so return to guest and
  1057. * try executing it again.
  1058. */
  1059. return RESUME_GUEST;
  1060. }
  1061. if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
  1062. vcpu->run->exit_reason = KVM_EXIT_DEBUG;
  1063. vcpu->run->debug.arch.address = kvmppc_get_pc(vcpu);
  1064. return RESUME_HOST;
  1065. } else {
  1066. kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
  1067. return RESUME_GUEST;
  1068. }
  1069. }
  1070. static void do_nothing(void *x)
  1071. {
  1072. }
  1073. static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
  1074. {
  1075. int thr, cpu, pcpu, nthreads;
  1076. struct kvm_vcpu *v;
  1077. unsigned long dpdes;
  1078. nthreads = vcpu->kvm->arch.emul_smt_mode;
  1079. dpdes = 0;
  1080. cpu = vcpu->vcpu_id & ~(nthreads - 1);
  1081. for (thr = 0; thr < nthreads; ++thr, ++cpu) {
  1082. v = kvmppc_find_vcpu(vcpu->kvm, cpu);
  1083. if (!v)
  1084. continue;
  1085. /*
  1086. * If the vcpu is currently running on a physical cpu thread,
  1087. * interrupt it in order to pull it out of the guest briefly,
  1088. * which will update its vcore->dpdes value.
  1089. */
  1090. pcpu = READ_ONCE(v->cpu);
  1091. if (pcpu >= 0)
  1092. smp_call_function_single(pcpu, do_nothing, NULL, 1);
  1093. if (kvmppc_doorbell_pending(v))
  1094. dpdes |= 1 << thr;
  1095. }
  1096. return dpdes;
  1097. }
  1098. /*
  1099. * On POWER9, emulate doorbell-related instructions in order to
  1100. * give the guest the illusion of running on a multi-threaded core.
  1101. * The instructions emulated are msgsndp, msgclrp, mfspr TIR,
  1102. * and mfspr DPDES.
  1103. */
  1104. static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
  1105. {
  1106. u32 inst, rb, thr;
  1107. unsigned long arg;
  1108. struct kvm *kvm = vcpu->kvm;
  1109. struct kvm_vcpu *tvcpu;
  1110. if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
  1111. return RESUME_GUEST;
  1112. if (get_op(inst) != 31)
  1113. return EMULATE_FAIL;
  1114. rb = get_rb(inst);
  1115. thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
  1116. switch (get_xop(inst)) {
  1117. case OP_31_XOP_MSGSNDP:
  1118. arg = kvmppc_get_gpr(vcpu, rb);
  1119. if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
  1120. break;
  1121. arg &= 0x3f;
  1122. if (arg >= kvm->arch.emul_smt_mode)
  1123. break;
  1124. tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
  1125. if (!tvcpu)
  1126. break;
  1127. if (!tvcpu->arch.doorbell_request) {
  1128. tvcpu->arch.doorbell_request = 1;
  1129. kvmppc_fast_vcpu_kick_hv(tvcpu);
  1130. }
  1131. break;
  1132. case OP_31_XOP_MSGCLRP:
  1133. arg = kvmppc_get_gpr(vcpu, rb);
  1134. if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER)
  1135. break;
  1136. vcpu->arch.vcore->dpdes = 0;
  1137. vcpu->arch.doorbell_request = 0;
  1138. break;
  1139. case OP_31_XOP_MFSPR:
  1140. switch (get_sprn(inst)) {
  1141. case SPRN_TIR:
  1142. arg = thr;
  1143. break;
  1144. case SPRN_DPDES:
  1145. arg = kvmppc_read_dpdes(vcpu);
  1146. break;
  1147. default:
  1148. return EMULATE_FAIL;
  1149. }
  1150. kvmppc_set_gpr(vcpu, get_rt(inst), arg);
  1151. break;
  1152. default:
  1153. return EMULATE_FAIL;
  1154. }
  1155. kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
  1156. return RESUME_GUEST;
  1157. }
  1158. static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
  1159. struct task_struct *tsk)
  1160. {
  1161. struct kvm_run *run = vcpu->run;
  1162. int r = RESUME_HOST;
  1163. vcpu->stat.sum_exits++;
  1164. /*
  1165. * This can happen if an interrupt occurs in the last stages
  1166. * of guest entry or the first stages of guest exit (i.e. after
  1167. * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
  1168. * and before setting it to KVM_GUEST_MODE_HOST_HV).
  1169. * That can happen due to a bug, or due to a machine check
  1170. * occurring at just the wrong time.
  1171. */
  1172. if (vcpu->arch.shregs.msr & MSR_HV) {
  1173. printk(KERN_EMERG "KVM trap in HV mode!\n");
  1174. printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
  1175. vcpu->arch.trap, kvmppc_get_pc(vcpu),
  1176. vcpu->arch.shregs.msr);
  1177. kvmppc_dump_regs(vcpu);
  1178. run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  1179. run->hw.hardware_exit_reason = vcpu->arch.trap;
  1180. return RESUME_HOST;
  1181. }
  1182. run->exit_reason = KVM_EXIT_UNKNOWN;
  1183. run->ready_for_interrupt_injection = 1;
  1184. switch (vcpu->arch.trap) {
  1185. /* We're good on these - the host merely wanted to get our attention */
  1186. case BOOK3S_INTERRUPT_HV_DECREMENTER:
  1187. vcpu->stat.dec_exits++;
  1188. r = RESUME_GUEST;
  1189. break;
  1190. case BOOK3S_INTERRUPT_EXTERNAL:
  1191. case BOOK3S_INTERRUPT_H_DOORBELL:
  1192. case BOOK3S_INTERRUPT_H_VIRT:
  1193. vcpu->stat.ext_intr_exits++;
  1194. r = RESUME_GUEST;
  1195. break;
  1196. /* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
  1197. case BOOK3S_INTERRUPT_HMI:
  1198. case BOOK3S_INTERRUPT_PERFMON:
  1199. case BOOK3S_INTERRUPT_SYSTEM_RESET:
  1200. r = RESUME_GUEST;
  1201. break;
  1202. case BOOK3S_INTERRUPT_MACHINE_CHECK:
  1203. /* Print the MCE event to host console. */
  1204. machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
  1205. /*
  1206. * If the guest can do FWNMI, exit to userspace so it can
  1207. * deliver a FWNMI to the guest.
  1208. * Otherwise we synthesize a machine check for the guest
  1209. * so that it knows that the machine check occurred.
  1210. */
  1211. if (!vcpu->kvm->arch.fwnmi_enabled) {
  1212. ulong flags = vcpu->arch.shregs.msr & 0x083c0000;
  1213. kvmppc_core_queue_machine_check(vcpu, flags);
  1214. r = RESUME_GUEST;
  1215. break;
  1216. }
  1217. /* Exit to guest with KVM_EXIT_NMI as exit reason */
  1218. run->exit_reason = KVM_EXIT_NMI;
  1219. run->hw.hardware_exit_reason = vcpu->arch.trap;
  1220. /* Clear out the old NMI status from run->flags */
  1221. run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
  1222. /* Now set the NMI status */
  1223. if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
  1224. run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
  1225. else
  1226. run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
  1227. r = RESUME_HOST;
  1228. break;
  1229. case BOOK3S_INTERRUPT_PROGRAM:
  1230. {
  1231. ulong flags;
  1232. /*
  1233. * Normally program interrupts are delivered directly
  1234. * to the guest by the hardware, but we can get here
  1235. * as a result of a hypervisor emulation interrupt
  1236. * (e40) getting turned into a 700 by BML RTAS.
  1237. */
  1238. flags = vcpu->arch.shregs.msr & 0x1f0000ull;
  1239. kvmppc_core_queue_program(vcpu, flags);
  1240. r = RESUME_GUEST;
  1241. break;
  1242. }
  1243. case BOOK3S_INTERRUPT_SYSCALL:
  1244. {
  1245. /* hcall - punt to userspace */
  1246. int i;
  1247. /* hypercall with MSR_PR has already been handled in rmode,
  1248. * and never reaches here.
  1249. */
  1250. run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
  1251. for (i = 0; i < 9; ++i)
  1252. run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
  1253. run->exit_reason = KVM_EXIT_PAPR_HCALL;
  1254. vcpu->arch.hcall_needed = 1;
  1255. r = RESUME_HOST;
  1256. break;
  1257. }
  1258. /*
  1259. * We get these next two if the guest accesses a page which it thinks
  1260. * it has mapped but which is not actually present, either because
  1261. * it is for an emulated I/O device or because the corresonding
  1262. * host page has been paged out. Any other HDSI/HISI interrupts
  1263. * have been handled already.
  1264. */
  1265. case BOOK3S_INTERRUPT_H_DATA_STORAGE:
  1266. r = RESUME_PAGE_FAULT;
  1267. break;
  1268. case BOOK3S_INTERRUPT_H_INST_STORAGE:
  1269. vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
  1270. vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
  1271. DSISR_SRR1_MATCH_64S;
  1272. if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
  1273. vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
  1274. r = RESUME_PAGE_FAULT;
  1275. break;
  1276. /*
  1277. * This occurs if the guest executes an illegal instruction.
  1278. * If the guest debug is disabled, generate a program interrupt
  1279. * to the guest. If guest debug is enabled, we need to check
  1280. * whether the instruction is a software breakpoint instruction.
  1281. * Accordingly return to Guest or Host.
  1282. */
  1283. case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
  1284. if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
  1285. vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
  1286. swab32(vcpu->arch.emul_inst) :
  1287. vcpu->arch.emul_inst;
  1288. if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
  1289. r = kvmppc_emulate_debug_inst(vcpu);
  1290. } else {
  1291. kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
  1292. r = RESUME_GUEST;
  1293. }
  1294. break;
  1295. /*
  1296. * This occurs if the guest (kernel or userspace), does something that
  1297. * is prohibited by HFSCR.
  1298. * On POWER9, this could be a doorbell instruction that we need
  1299. * to emulate.
  1300. * Otherwise, we just generate a program interrupt to the guest.
  1301. */
  1302. case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
  1303. r = EMULATE_FAIL;
  1304. if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
  1305. cpu_has_feature(CPU_FTR_ARCH_300))
  1306. r = kvmppc_emulate_doorbell_instr(vcpu);
  1307. if (r == EMULATE_FAIL) {
  1308. kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
  1309. r = RESUME_GUEST;
  1310. }
  1311. break;
  1312. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1313. case BOOK3S_INTERRUPT_HV_SOFTPATCH:
  1314. /*
  1315. * This occurs for various TM-related instructions that
  1316. * we need to emulate on POWER9 DD2.2. We have already
  1317. * handled the cases where the guest was in real-suspend
  1318. * mode and was transitioning to transactional state.
  1319. */
  1320. r = kvmhv_p9_tm_emulation(vcpu);
  1321. break;
  1322. #endif
  1323. case BOOK3S_INTERRUPT_HV_RM_HARD:
  1324. r = RESUME_PASSTHROUGH;
  1325. break;
  1326. default:
  1327. kvmppc_dump_regs(vcpu);
  1328. printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
  1329. vcpu->arch.trap, kvmppc_get_pc(vcpu),
  1330. vcpu->arch.shregs.msr);
  1331. run->hw.hardware_exit_reason = vcpu->arch.trap;
  1332. r = RESUME_HOST;
  1333. break;
  1334. }
  1335. return r;
  1336. }
  1337. static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
  1338. {
  1339. int r;
  1340. int srcu_idx;
  1341. vcpu->stat.sum_exits++;
  1342. /*
  1343. * This can happen if an interrupt occurs in the last stages
  1344. * of guest entry or the first stages of guest exit (i.e. after
  1345. * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
  1346. * and before setting it to KVM_GUEST_MODE_HOST_HV).
  1347. * That can happen due to a bug, or due to a machine check
  1348. * occurring at just the wrong time.
  1349. */
  1350. if (vcpu->arch.shregs.msr & MSR_HV) {
  1351. pr_emerg("KVM trap in HV mode while nested!\n");
  1352. pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
  1353. vcpu->arch.trap, kvmppc_get_pc(vcpu),
  1354. vcpu->arch.shregs.msr);
  1355. kvmppc_dump_regs(vcpu);
  1356. return RESUME_HOST;
  1357. }
  1358. switch (vcpu->arch.trap) {
  1359. /* We're good on these - the host merely wanted to get our attention */
  1360. case BOOK3S_INTERRUPT_HV_DECREMENTER:
  1361. vcpu->stat.dec_exits++;
  1362. r = RESUME_GUEST;
  1363. break;
  1364. case BOOK3S_INTERRUPT_EXTERNAL:
  1365. vcpu->stat.ext_intr_exits++;
  1366. r = RESUME_HOST;
  1367. break;
  1368. case BOOK3S_INTERRUPT_H_DOORBELL:
  1369. case BOOK3S_INTERRUPT_H_VIRT:
  1370. vcpu->stat.ext_intr_exits++;
  1371. r = RESUME_GUEST;
  1372. break;
  1373. /* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
  1374. case BOOK3S_INTERRUPT_HMI:
  1375. case BOOK3S_INTERRUPT_PERFMON:
  1376. case BOOK3S_INTERRUPT_SYSTEM_RESET:
  1377. r = RESUME_GUEST;
  1378. break;
  1379. case BOOK3S_INTERRUPT_MACHINE_CHECK:
  1380. /* Pass the machine check to the L1 guest */
  1381. r = RESUME_HOST;
  1382. /* Print the MCE event to host console. */
  1383. machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
  1384. break;
  1385. /*
  1386. * We get these next two if the guest accesses a page which it thinks
  1387. * it has mapped but which is not actually present, either because
  1388. * it is for an emulated I/O device or because the corresonding
  1389. * host page has been paged out.
  1390. */
  1391. case BOOK3S_INTERRUPT_H_DATA_STORAGE:
  1392. srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  1393. r = kvmhv_nested_page_fault(vcpu);
  1394. srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
  1395. break;
  1396. case BOOK3S_INTERRUPT_H_INST_STORAGE:
  1397. vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
  1398. vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
  1399. DSISR_SRR1_MATCH_64S;
  1400. if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
  1401. vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
  1402. srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  1403. r = kvmhv_nested_page_fault(vcpu);
  1404. srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
  1405. break;
  1406. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1407. case BOOK3S_INTERRUPT_HV_SOFTPATCH:
  1408. /*
  1409. * This occurs for various TM-related instructions that
  1410. * we need to emulate on POWER9 DD2.2. We have already
  1411. * handled the cases where the guest was in real-suspend
  1412. * mode and was transitioning to transactional state.
  1413. */
  1414. r = kvmhv_p9_tm_emulation(vcpu);
  1415. break;
  1416. #endif
  1417. case BOOK3S_INTERRUPT_HV_RM_HARD:
  1418. vcpu->arch.trap = 0;
  1419. r = RESUME_GUEST;
  1420. if (!xics_on_xive())
  1421. kvmppc_xics_rm_complete(vcpu, 0);
  1422. break;
  1423. default:
  1424. r = RESUME_HOST;
  1425. break;
  1426. }
  1427. return r;
  1428. }
  1429. static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
  1430. struct kvm_sregs *sregs)
  1431. {
  1432. int i;
  1433. memset(sregs, 0, sizeof(struct kvm_sregs));
  1434. sregs->pvr = vcpu->arch.pvr;
  1435. for (i = 0; i < vcpu->arch.slb_max; i++) {
  1436. sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
  1437. sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
  1438. }
  1439. return 0;
  1440. }
  1441. static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
  1442. struct kvm_sregs *sregs)
  1443. {
  1444. int i, j;
  1445. /* Only accept the same PVR as the host's, since we can't spoof it */
  1446. if (sregs->pvr != vcpu->arch.pvr)
  1447. return -EINVAL;
  1448. j = 0;
  1449. for (i = 0; i < vcpu->arch.slb_nr; i++) {
  1450. if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
  1451. vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
  1452. vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
  1453. ++j;
  1454. }
  1455. }
  1456. vcpu->arch.slb_max = j;
  1457. return 0;
  1458. }
  1459. static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
  1460. bool preserve_top32)
  1461. {
  1462. struct kvm *kvm = vcpu->kvm;
  1463. struct kvmppc_vcore *vc = vcpu->arch.vcore;
  1464. u64 mask;
  1465. spin_lock(&vc->lock);
  1466. /*
  1467. * If ILE (interrupt little-endian) has changed, update the
  1468. * MSR_LE bit in the intr_msr for each vcpu in this vcore.
  1469. */
  1470. if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
  1471. struct kvm_vcpu *vcpu;
  1472. int i;
  1473. kvm_for_each_vcpu(i, vcpu, kvm) {
  1474. if (vcpu->arch.vcore != vc)
  1475. continue;
  1476. if (new_lpcr & LPCR_ILE)
  1477. vcpu->arch.intr_msr |= MSR_LE;
  1478. else
  1479. vcpu->arch.intr_msr &= ~MSR_LE;
  1480. }
  1481. }
  1482. /*
  1483. * Userspace can only modify DPFD (default prefetch depth),
  1484. * ILE (interrupt little-endian) and TC (translation control).
  1485. * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.).
  1486. */
  1487. mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
  1488. if (cpu_has_feature(CPU_FTR_ARCH_207S))
  1489. mask |= LPCR_AIL;
  1490. /*
  1491. * On POWER9, allow userspace to enable large decrementer for the
  1492. * guest, whether or not the host has it enabled.
  1493. */
  1494. if (cpu_has_feature(CPU_FTR_ARCH_300))
  1495. mask |= LPCR_LD;
  1496. /* Broken 32-bit version of LPCR must not clear top bits */
  1497. if (preserve_top32)
  1498. mask &= 0xFFFFFFFF;
  1499. vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
  1500. spin_unlock(&vc->lock);
  1501. }
  1502. static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
  1503. union kvmppc_one_reg *val)
  1504. {
  1505. int r = 0;
  1506. long int i;
  1507. switch (id) {
  1508. case KVM_REG_PPC_DEBUG_INST:
  1509. *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
  1510. break;
  1511. case KVM_REG_PPC_HIOR:
  1512. *val = get_reg_val(id, 0);
  1513. break;
  1514. case KVM_REG_PPC_DABR:
  1515. *val = get_reg_val(id, vcpu->arch.dabr);
  1516. break;
  1517. case KVM_REG_PPC_DABRX:
  1518. *val = get_reg_val(id, vcpu->arch.dabrx);
  1519. break;
  1520. case KVM_REG_PPC_DSCR:
  1521. *val = get_reg_val(id, vcpu->arch.dscr);
  1522. break;
  1523. case KVM_REG_PPC_PURR:
  1524. *val = get_reg_val(id, vcpu->arch.purr);
  1525. break;
  1526. case KVM_REG_PPC_SPURR:
  1527. *val = get_reg_val(id, vcpu->arch.spurr);
  1528. break;
  1529. case KVM_REG_PPC_AMR:
  1530. *val = get_reg_val(id, vcpu->arch.amr);
  1531. break;
  1532. case KVM_REG_PPC_UAMOR:
  1533. *val = get_reg_val(id, vcpu->arch.uamor);
  1534. break;
  1535. case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
  1536. i = id - KVM_REG_PPC_MMCR0;
  1537. *val = get_reg_val(id, vcpu->arch.mmcr[i]);
  1538. break;
  1539. case KVM_REG_PPC_MMCR2:
  1540. *val = get_reg_val(id, vcpu->arch.mmcr[2]);
  1541. break;
  1542. case KVM_REG_PPC_MMCRA:
  1543. *val = get_reg_val(id, vcpu->arch.mmcra);
  1544. break;
  1545. case KVM_REG_PPC_MMCRS:
  1546. *val = get_reg_val(id, vcpu->arch.mmcrs);
  1547. break;
  1548. case KVM_REG_PPC_MMCR3:
  1549. *val = get_reg_val(id, vcpu->arch.mmcr[3]);
  1550. break;
  1551. case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
  1552. i = id - KVM_REG_PPC_PMC1;
  1553. *val = get_reg_val(id, vcpu->arch.pmc[i]);
  1554. break;
  1555. case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
  1556. i = id - KVM_REG_PPC_SPMC1;
  1557. *val = get_reg_val(id, vcpu->arch.spmc[i]);
  1558. break;
  1559. case KVM_REG_PPC_SIAR:
  1560. *val = get_reg_val(id, vcpu->arch.siar);
  1561. break;
  1562. case KVM_REG_PPC_SDAR:
  1563. *val = get_reg_val(id, vcpu->arch.sdar);
  1564. break;
  1565. case KVM_REG_PPC_SIER:
  1566. *val = get_reg_val(id, vcpu->arch.sier[0]);
  1567. break;
  1568. case KVM_REG_PPC_SIER2:
  1569. *val = get_reg_val(id, vcpu->arch.sier[1]);
  1570. break;
  1571. case KVM_REG_PPC_SIER3:
  1572. *val = get_reg_val(id, vcpu->arch.sier[2]);
  1573. break;
  1574. case KVM_REG_PPC_IAMR:
  1575. *val = get_reg_val(id, vcpu->arch.iamr);
  1576. break;
  1577. case KVM_REG_PPC_PSPB:
  1578. *val = get_reg_val(id, vcpu->arch.pspb);
  1579. break;
  1580. case KVM_REG_PPC_DPDES:
  1581. /*
  1582. * On POWER9, where we are emulating msgsndp etc.,
  1583. * we return 1 bit for each vcpu, which can come from
  1584. * either vcore->dpdes or doorbell_request.
  1585. * On POWER8, doorbell_request is 0.
  1586. */
  1587. *val = get_reg_val(id, vcpu->arch.vcore->dpdes |
  1588. vcpu->arch.doorbell_request);
  1589. break;
  1590. case KVM_REG_PPC_VTB:
  1591. *val = get_reg_val(id, vcpu->arch.vcore->vtb);
  1592. break;
  1593. case KVM_REG_PPC_DAWR:
  1594. *val = get_reg_val(id, vcpu->arch.dawr);
  1595. break;
  1596. case KVM_REG_PPC_DAWRX:
  1597. *val = get_reg_val(id, vcpu->arch.dawrx);
  1598. break;
  1599. case KVM_REG_PPC_CIABR:
  1600. *val = get_reg_val(id, vcpu->arch.ciabr);
  1601. break;
  1602. case KVM_REG_PPC_CSIGR:
  1603. *val = get_reg_val(id, vcpu->arch.csigr);
  1604. break;
  1605. case KVM_REG_PPC_TACR:
  1606. *val = get_reg_val(id, vcpu->arch.tacr);
  1607. break;
  1608. case KVM_REG_PPC_TCSCR:
  1609. *val = get_reg_val(id, vcpu->arch.tcscr);
  1610. break;
  1611. case KVM_REG_PPC_PID:
  1612. *val = get_reg_val(id, vcpu->arch.pid);
  1613. break;
  1614. case KVM_REG_PPC_ACOP:
  1615. *val = get_reg_val(id, vcpu->arch.acop);
  1616. break;
  1617. case KVM_REG_PPC_WORT:
  1618. *val = get_reg_val(id, vcpu->arch.wort);
  1619. break;
  1620. case KVM_REG_PPC_TIDR:
  1621. *val = get_reg_val(id, vcpu->arch.tid);
  1622. break;
  1623. case KVM_REG_PPC_PSSCR:
  1624. *val = get_reg_val(id, vcpu->arch.psscr);
  1625. break;
  1626. case KVM_REG_PPC_VPA_ADDR:
  1627. spin_lock(&vcpu->arch.vpa_update_lock);
  1628. *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
  1629. spin_unlock(&vcpu->arch.vpa_update_lock);
  1630. break;
  1631. case KVM_REG_PPC_VPA_SLB:
  1632. spin_lock(&vcpu->arch.vpa_update_lock);
  1633. val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
  1634. val->vpaval.length = vcpu->arch.slb_shadow.len;
  1635. spin_unlock(&vcpu->arch.vpa_update_lock);
  1636. break;
  1637. case KVM_REG_PPC_VPA_DTL:
  1638. spin_lock(&vcpu->arch.vpa_update_lock);
  1639. val->vpaval.addr = vcpu->arch.dtl.next_gpa;
  1640. val->vpaval.length = vcpu->arch.dtl.len;
  1641. spin_unlock(&vcpu->arch.vpa_update_lock);
  1642. break;
  1643. case KVM_REG_PPC_TB_OFFSET:
  1644. *val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
  1645. break;
  1646. case KVM_REG_PPC_LPCR:
  1647. case KVM_REG_PPC_LPCR_64:
  1648. *val = get_reg_val(id, vcpu->arch.vcore->lpcr);
  1649. break;
  1650. case KVM_REG_PPC_PPR:
  1651. *val = get_reg_val(id, vcpu->arch.ppr);
  1652. break;
  1653. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1654. case KVM_REG_PPC_TFHAR:
  1655. *val = get_reg_val(id, vcpu->arch.tfhar);
  1656. break;
  1657. case KVM_REG_PPC_TFIAR:
  1658. *val = get_reg_val(id, vcpu->arch.tfiar);
  1659. break;
  1660. case KVM_REG_PPC_TEXASR:
  1661. *val = get_reg_val(id, vcpu->arch.texasr);
  1662. break;
  1663. case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
  1664. i = id - KVM_REG_PPC_TM_GPR0;
  1665. *val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
  1666. break;
  1667. case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
  1668. {
  1669. int j;
  1670. i = id - KVM_REG_PPC_TM_VSR0;
  1671. if (i < 32)
  1672. for (j = 0; j < TS_FPRWIDTH; j++)
  1673. val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
  1674. else {
  1675. if (cpu_has_feature(CPU_FTR_ALTIVEC))
  1676. val->vval = vcpu->arch.vr_tm.vr[i-32];
  1677. else
  1678. r = -ENXIO;
  1679. }
  1680. break;
  1681. }
  1682. case KVM_REG_PPC_TM_CR:
  1683. *val = get_reg_val(id, vcpu->arch.cr_tm);
  1684. break;
  1685. case KVM_REG_PPC_TM_XER:
  1686. *val = get_reg_val(id, vcpu->arch.xer_tm);
  1687. break;
  1688. case KVM_REG_PPC_TM_LR:
  1689. *val = get_reg_val(id, vcpu->arch.lr_tm);
  1690. break;
  1691. case KVM_REG_PPC_TM_CTR:
  1692. *val = get_reg_val(id, vcpu->arch.ctr_tm);
  1693. break;
  1694. case KVM_REG_PPC_TM_FPSCR:
  1695. *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
  1696. break;
  1697. case KVM_REG_PPC_TM_AMR:
  1698. *val = get_reg_val(id, vcpu->arch.amr_tm);
  1699. break;
  1700. case KVM_REG_PPC_TM_PPR:
  1701. *val = get_reg_val(id, vcpu->arch.ppr_tm);
  1702. break;
  1703. case KVM_REG_PPC_TM_VRSAVE:
  1704. *val = get_reg_val(id, vcpu->arch.vrsave_tm);
  1705. break;
  1706. case KVM_REG_PPC_TM_VSCR:
  1707. if (cpu_has_feature(CPU_FTR_ALTIVEC))
  1708. *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
  1709. else
  1710. r = -ENXIO;
  1711. break;
  1712. case KVM_REG_PPC_TM_DSCR:
  1713. *val = get_reg_val(id, vcpu->arch.dscr_tm);
  1714. break;
  1715. case KVM_REG_PPC_TM_TAR:
  1716. *val = get_reg_val(id, vcpu->arch.tar_tm);
  1717. break;
  1718. #endif
  1719. case KVM_REG_PPC_ARCH_COMPAT:
  1720. *val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
  1721. break;
  1722. case KVM_REG_PPC_DEC_EXPIRY:
  1723. *val = get_reg_val(id, vcpu->arch.dec_expires +
  1724. vcpu->arch.vcore->tb_offset);
  1725. break;
  1726. case KVM_REG_PPC_ONLINE:
  1727. *val = get_reg_val(id, vcpu->arch.online);
  1728. break;
  1729. case KVM_REG_PPC_PTCR:
  1730. *val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
  1731. break;
  1732. default:
  1733. r = -EINVAL;
  1734. break;
  1735. }
  1736. return r;
  1737. }
  1738. static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
  1739. union kvmppc_one_reg *val)
  1740. {
  1741. int r = 0;
  1742. long int i;
  1743. unsigned long addr, len;
  1744. switch (id) {
  1745. case KVM_REG_PPC_HIOR:
  1746. /* Only allow this to be set to zero */
  1747. if (set_reg_val(id, *val))
  1748. r = -EINVAL;
  1749. break;
  1750. case KVM_REG_PPC_DABR:
  1751. vcpu->arch.dabr = set_reg_val(id, *val);
  1752. break;
  1753. case KVM_REG_PPC_DABRX:
  1754. vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
  1755. break;
  1756. case KVM_REG_PPC_DSCR:
  1757. vcpu->arch.dscr = set_reg_val(id, *val);
  1758. break;
  1759. case KVM_REG_PPC_PURR:
  1760. vcpu->arch.purr = set_reg_val(id, *val);
  1761. break;
  1762. case KVM_REG_PPC_SPURR:
  1763. vcpu->arch.spurr = set_reg_val(id, *val);
  1764. break;
  1765. case KVM_REG_PPC_AMR:
  1766. vcpu->arch.amr = set_reg_val(id, *val);
  1767. break;
  1768. case KVM_REG_PPC_UAMOR:
  1769. vcpu->arch.uamor = set_reg_val(id, *val);
  1770. break;
  1771. case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
  1772. i = id - KVM_REG_PPC_MMCR0;
  1773. vcpu->arch.mmcr[i] = set_reg_val(id, *val);
  1774. break;
  1775. case KVM_REG_PPC_MMCR2:
  1776. vcpu->arch.mmcr[2] = set_reg_val(id, *val);
  1777. break;
  1778. case KVM_REG_PPC_MMCRA:
  1779. vcpu->arch.mmcra = set_reg_val(id, *val);
  1780. break;
  1781. case KVM_REG_PPC_MMCRS:
  1782. vcpu->arch.mmcrs = set_reg_val(id, *val);
  1783. break;
  1784. case KVM_REG_PPC_MMCR3:
  1785. *val = get_reg_val(id, vcpu->arch.mmcr[3]);
  1786. break;
  1787. case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
  1788. i = id - KVM_REG_PPC_PMC1;
  1789. vcpu->arch.pmc[i] = set_reg_val(id, *val);
  1790. break;
  1791. case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
  1792. i = id - KVM_REG_PPC_SPMC1;
  1793. vcpu->arch.spmc[i] = set_reg_val(id, *val);
  1794. break;
  1795. case KVM_REG_PPC_SIAR:
  1796. vcpu->arch.siar = set_reg_val(id, *val);
  1797. break;
  1798. case KVM_REG_PPC_SDAR:
  1799. vcpu->arch.sdar = set_reg_val(id, *val);
  1800. break;
  1801. case KVM_REG_PPC_SIER:
  1802. vcpu->arch.sier[0] = set_reg_val(id, *val);
  1803. break;
  1804. case KVM_REG_PPC_SIER2:
  1805. vcpu->arch.sier[1] = set_reg_val(id, *val);
  1806. break;
  1807. case KVM_REG_PPC_SIER3:
  1808. vcpu->arch.sier[2] = set_reg_val(id, *val);
  1809. break;
  1810. case KVM_REG_PPC_IAMR:
  1811. vcpu->arch.iamr = set_reg_val(id, *val);
  1812. break;
  1813. case KVM_REG_PPC_PSPB:
  1814. vcpu->arch.pspb = set_reg_val(id, *val);
  1815. break;
  1816. case KVM_REG_PPC_DPDES:
  1817. vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
  1818. break;
  1819. case KVM_REG_PPC_VTB:
  1820. vcpu->arch.vcore->vtb = set_reg_val(id, *val);
  1821. break;
  1822. case KVM_REG_PPC_DAWR:
  1823. vcpu->arch.dawr = set_reg_val(id, *val);
  1824. break;
  1825. case KVM_REG_PPC_DAWRX:
  1826. vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
  1827. break;
  1828. case KVM_REG_PPC_CIABR:
  1829. vcpu->arch.ciabr = set_reg_val(id, *val);
  1830. /* Don't allow setting breakpoints in hypervisor code */
  1831. if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
  1832. vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */
  1833. break;
  1834. case KVM_REG_PPC_CSIGR:
  1835. vcpu->arch.csigr = set_reg_val(id, *val);
  1836. break;
  1837. case KVM_REG_PPC_TACR:
  1838. vcpu->arch.tacr = set_reg_val(id, *val);
  1839. break;
  1840. case KVM_REG_PPC_TCSCR:
  1841. vcpu->arch.tcscr = set_reg_val(id, *val);
  1842. break;
  1843. case KVM_REG_PPC_PID:
  1844. vcpu->arch.pid = set_reg_val(id, *val);
  1845. break;
  1846. case KVM_REG_PPC_ACOP:
  1847. vcpu->arch.acop = set_reg_val(id, *val);
  1848. break;
  1849. case KVM_REG_PPC_WORT:
  1850. vcpu->arch.wort = set_reg_val(id, *val);
  1851. break;
  1852. case KVM_REG_PPC_TIDR:
  1853. vcpu->arch.tid = set_reg_val(id, *val);
  1854. break;
  1855. case KVM_REG_PPC_PSSCR:
  1856. vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
  1857. break;
  1858. case KVM_REG_PPC_VPA_ADDR:
  1859. addr = set_reg_val(id, *val);
  1860. r = -EINVAL;
  1861. if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
  1862. vcpu->arch.dtl.next_gpa))
  1863. break;
  1864. r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
  1865. break;
  1866. case KVM_REG_PPC_VPA_SLB:
  1867. addr = val->vpaval.addr;
  1868. len = val->vpaval.length;
  1869. r = -EINVAL;
  1870. if (addr && !vcpu->arch.vpa.next_gpa)
  1871. break;
  1872. r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
  1873. break;
  1874. case KVM_REG_PPC_VPA_DTL:
  1875. addr = val->vpaval.addr;
  1876. len = val->vpaval.length;
  1877. r = -EINVAL;
  1878. if (addr && (len < sizeof(struct dtl_entry) ||
  1879. !vcpu->arch.vpa.next_gpa))
  1880. break;
  1881. len -= len % sizeof(struct dtl_entry);
  1882. r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
  1883. break;
  1884. case KVM_REG_PPC_TB_OFFSET:
  1885. /* round up to multiple of 2^24 */
  1886. vcpu->arch.vcore->tb_offset =
  1887. ALIGN(set_reg_val(id, *val), 1UL << 24);
  1888. break;
  1889. case KVM_REG_PPC_LPCR:
  1890. kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
  1891. break;
  1892. case KVM_REG_PPC_LPCR_64:
  1893. kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
  1894. break;
  1895. case KVM_REG_PPC_PPR:
  1896. vcpu->arch.ppr = set_reg_val(id, *val);
  1897. break;
  1898. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1899. case KVM_REG_PPC_TFHAR:
  1900. vcpu->arch.tfhar = set_reg_val(id, *val);
  1901. break;
  1902. case KVM_REG_PPC_TFIAR:
  1903. vcpu->arch.tfiar = set_reg_val(id, *val);
  1904. break;
  1905. case KVM_REG_PPC_TEXASR:
  1906. vcpu->arch.texasr = set_reg_val(id, *val);
  1907. break;
  1908. case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
  1909. i = id - KVM_REG_PPC_TM_GPR0;
  1910. vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
  1911. break;
  1912. case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
  1913. {
  1914. int j;
  1915. i = id - KVM_REG_PPC_TM_VSR0;
  1916. if (i < 32)
  1917. for (j = 0; j < TS_FPRWIDTH; j++)
  1918. vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
  1919. else
  1920. if (cpu_has_feature(CPU_FTR_ALTIVEC))
  1921. vcpu->arch.vr_tm.vr[i-32] = val->vval;
  1922. else
  1923. r = -ENXIO;
  1924. break;
  1925. }
  1926. case KVM_REG_PPC_TM_CR:
  1927. vcpu->arch.cr_tm = set_reg_val(id, *val);
  1928. break;
  1929. case KVM_REG_PPC_TM_XER:
  1930. vcpu->arch.xer_tm = set_reg_val(id, *val);
  1931. break;
  1932. case KVM_REG_PPC_TM_LR:
  1933. vcpu->arch.lr_tm = set_reg_val(id, *val);
  1934. break;
  1935. case KVM_REG_PPC_TM_CTR:
  1936. vcpu->arch.ctr_tm = set_reg_val(id, *val);
  1937. break;
  1938. case KVM_REG_PPC_TM_FPSCR:
  1939. vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
  1940. break;
  1941. case KVM_REG_PPC_TM_AMR:
  1942. vcpu->arch.amr_tm = set_reg_val(id, *val);
  1943. break;
  1944. case KVM_REG_PPC_TM_PPR:
  1945. vcpu->arch.ppr_tm = set_reg_val(id, *val);
  1946. break;
  1947. case KVM_REG_PPC_TM_VRSAVE:
  1948. vcpu->arch.vrsave_tm = set_reg_val(id, *val);
  1949. break;
  1950. case KVM_REG_PPC_TM_VSCR:
  1951. if (cpu_has_feature(CPU_FTR_ALTIVEC))
  1952. vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
  1953. else
  1954. r = - ENXIO;
  1955. break;
  1956. case KVM_REG_PPC_TM_DSCR:
  1957. vcpu->arch.dscr_tm = set_reg_val(id, *val);
  1958. break;
  1959. case KVM_REG_PPC_TM_TAR:
  1960. vcpu->arch.tar_tm = set_reg_val(id, *val);
  1961. break;
  1962. #endif
  1963. case KVM_REG_PPC_ARCH_COMPAT:
  1964. r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
  1965. break;
  1966. case KVM_REG_PPC_DEC_EXPIRY:
  1967. vcpu->arch.dec_expires = set_reg_val(id, *val) -
  1968. vcpu->arch.vcore->tb_offset;
  1969. break;
  1970. case KVM_REG_PPC_ONLINE:
  1971. i = set_reg_val(id, *val);
  1972. if (i && !vcpu->arch.online)
  1973. atomic_inc(&vcpu->arch.vcore->online_count);
  1974. else if (!i && vcpu->arch.online)
  1975. atomic_dec(&vcpu->arch.vcore->online_count);
  1976. vcpu->arch.online = i;
  1977. break;
  1978. case KVM_REG_PPC_PTCR:
  1979. vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
  1980. break;
  1981. default:
  1982. r = -EINVAL;
  1983. break;
  1984. }
  1985. return r;
  1986. }
  1987. /*
  1988. * On POWER9, threads are independent and can be in different partitions.
  1989. * Therefore we consider each thread to be a subcore.
  1990. * There is a restriction that all threads have to be in the same
  1991. * MMU mode (radix or HPT), unfortunately, but since we only support
  1992. * HPT guests on a HPT host so far, that isn't an impediment yet.
  1993. */
  1994. static int threads_per_vcore(struct kvm *kvm)
  1995. {
  1996. if (kvm->arch.threads_indep)
  1997. return 1;
  1998. return threads_per_subcore;
  1999. }
  2000. static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
  2001. {
  2002. struct kvmppc_vcore *vcore;
  2003. vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
  2004. if (vcore == NULL)
  2005. return NULL;
  2006. spin_lock_init(&vcore->lock);
  2007. spin_lock_init(&vcore->stoltb_lock);
  2008. rcuwait_init(&vcore->wait);
  2009. vcore->preempt_tb = TB_NIL;
  2010. vcore->lpcr = kvm->arch.lpcr;
  2011. vcore->first_vcpuid = id;
  2012. vcore->kvm = kvm;
  2013. INIT_LIST_HEAD(&vcore->preempt_list);
  2014. return vcore;
  2015. }
  2016. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  2017. static struct debugfs_timings_element {
  2018. const char *name;
  2019. size_t offset;
  2020. } timings[] = {
  2021. {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)},
  2022. {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)},
  2023. {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)},
  2024. {"guest", offsetof(struct kvm_vcpu, arch.guest_time)},
  2025. {"cede", offsetof(struct kvm_vcpu, arch.cede_time)},
  2026. };
  2027. #define N_TIMINGS (ARRAY_SIZE(timings))
  2028. struct debugfs_timings_state {
  2029. struct kvm_vcpu *vcpu;
  2030. unsigned int buflen;
  2031. char buf[N_TIMINGS * 100];
  2032. };
  2033. static int debugfs_timings_open(struct inode *inode, struct file *file)
  2034. {
  2035. struct kvm_vcpu *vcpu = inode->i_private;
  2036. struct debugfs_timings_state *p;
  2037. p = kzalloc(sizeof(*p), GFP_KERNEL);
  2038. if (!p)
  2039. return -ENOMEM;
  2040. kvm_get_kvm(vcpu->kvm);
  2041. p->vcpu = vcpu;
  2042. file->private_data = p;
  2043. return nonseekable_open(inode, file);
  2044. }
  2045. static int debugfs_timings_release(struct inode *inode, struct file *file)
  2046. {
  2047. struct debugfs_timings_state *p = file->private_data;
  2048. kvm_put_kvm(p->vcpu->kvm);
  2049. kfree(p);
  2050. return 0;
  2051. }
  2052. static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
  2053. size_t len, loff_t *ppos)
  2054. {
  2055. struct debugfs_timings_state *p = file->private_data;
  2056. struct kvm_vcpu *vcpu = p->vcpu;
  2057. char *s, *buf_end;
  2058. struct kvmhv_tb_accumulator tb;
  2059. u64 count;
  2060. loff_t pos;
  2061. ssize_t n;
  2062. int i, loops;
  2063. bool ok;
  2064. if (!p->buflen) {
  2065. s = p->buf;
  2066. buf_end = s + sizeof(p->buf);
  2067. for (i = 0; i < N_TIMINGS; ++i) {
  2068. struct kvmhv_tb_accumulator *acc;
  2069. acc = (struct kvmhv_tb_accumulator *)
  2070. ((unsigned long)vcpu + timings[i].offset);
  2071. ok = false;
  2072. for (loops = 0; loops < 1000; ++loops) {
  2073. count = acc->seqcount;
  2074. if (!(count & 1)) {
  2075. smp_rmb();
  2076. tb = *acc;
  2077. smp_rmb();
  2078. if (count == acc->seqcount) {
  2079. ok = true;
  2080. break;
  2081. }
  2082. }
  2083. udelay(1);
  2084. }
  2085. if (!ok)
  2086. snprintf(s, buf_end - s, "%s: stuck\n",
  2087. timings[i].name);
  2088. else
  2089. snprintf(s, buf_end - s,
  2090. "%s: %llu %llu %llu %llu\n",
  2091. timings[i].name, count / 2,
  2092. tb_to_ns(tb.tb_total),
  2093. tb_to_ns(tb.tb_min),
  2094. tb_to_ns(tb.tb_max));
  2095. s += strlen(s);
  2096. }
  2097. p->buflen = s - p->buf;
  2098. }
  2099. pos = *ppos;
  2100. if (pos >= p->buflen)
  2101. return 0;
  2102. if (len > p->buflen - pos)
  2103. len = p->buflen - pos;
  2104. n = copy_to_user(buf, p->buf + pos, len);
  2105. if (n) {
  2106. if (n == len)
  2107. return -EFAULT;
  2108. len -= n;
  2109. }
  2110. *ppos = pos + len;
  2111. return len;
  2112. }
  2113. static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
  2114. size_t len, loff_t *ppos)
  2115. {
  2116. return -EACCES;
  2117. }
  2118. static const struct file_operations debugfs_timings_ops = {
  2119. .owner = THIS_MODULE,
  2120. .open = debugfs_timings_open,
  2121. .release = debugfs_timings_release,
  2122. .read = debugfs_timings_read,
  2123. .write = debugfs_timings_write,
  2124. .llseek = generic_file_llseek,
  2125. };
  2126. /* Create a debugfs directory for the vcpu */
  2127. static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
  2128. {
  2129. char buf[16];
  2130. struct kvm *kvm = vcpu->kvm;
  2131. snprintf(buf, sizeof(buf), "vcpu%u", id);
  2132. vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir);
  2133. debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, vcpu,
  2134. &debugfs_timings_ops);
  2135. }
  2136. #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
  2137. static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id)
  2138. {
  2139. }
  2140. #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
  2141. static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu)
  2142. {
  2143. int err;
  2144. int core;
  2145. struct kvmppc_vcore *vcore;
  2146. struct kvm *kvm;
  2147. unsigned int id;
  2148. kvm = vcpu->kvm;
  2149. id = vcpu->vcpu_id;
  2150. vcpu->arch.shared = &vcpu->arch.shregs;
  2151. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2152. /*
  2153. * The shared struct is never shared on HV,
  2154. * so we can always use host endianness
  2155. */
  2156. #ifdef __BIG_ENDIAN__
  2157. vcpu->arch.shared_big_endian = true;
  2158. #else
  2159. vcpu->arch.shared_big_endian = false;
  2160. #endif
  2161. #endif
  2162. vcpu->arch.mmcr[0] = MMCR0_FC;
  2163. vcpu->arch.ctrl = CTRL_RUNLATCH;
  2164. /* default to host PVR, since we can't spoof it */
  2165. kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
  2166. spin_lock_init(&vcpu->arch.vpa_update_lock);
  2167. spin_lock_init(&vcpu->arch.tbacct_lock);
  2168. vcpu->arch.busy_preempt = TB_NIL;
  2169. vcpu->arch.intr_msr = MSR_SF | MSR_ME;
  2170. /*
  2171. * Set the default HFSCR for the guest from the host value.
  2172. * This value is only used on POWER9.
  2173. * On POWER9, we want to virtualize the doorbell facility, so we
  2174. * don't set the HFSCR_MSGP bit, and that causes those instructions
  2175. * to trap and then we emulate them.
  2176. */
  2177. vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
  2178. HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP | HFSCR_PREFIX;
  2179. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  2180. vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
  2181. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2182. if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
  2183. vcpu->arch.hfscr |= HFSCR_TM;
  2184. #endif
  2185. }
  2186. if (cpu_has_feature(CPU_FTR_TM_COMP))
  2187. vcpu->arch.hfscr |= HFSCR_TM;
  2188. kvmppc_mmu_book3s_hv_init(vcpu);
  2189. vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
  2190. init_waitqueue_head(&vcpu->arch.cpu_run);
  2191. mutex_lock(&kvm->lock);
  2192. vcore = NULL;
  2193. err = -EINVAL;
  2194. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  2195. if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
  2196. pr_devel("KVM: VCPU ID too high\n");
  2197. core = KVM_MAX_VCORES;
  2198. } else {
  2199. BUG_ON(kvm->arch.smt_mode != 1);
  2200. core = kvmppc_pack_vcpu_id(kvm, id);
  2201. }
  2202. } else {
  2203. core = id / kvm->arch.smt_mode;
  2204. }
  2205. if (core < KVM_MAX_VCORES) {
  2206. vcore = kvm->arch.vcores[core];
  2207. if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
  2208. pr_devel("KVM: collision on id %u", id);
  2209. vcore = NULL;
  2210. } else if (!vcore) {
  2211. /*
  2212. * Take mmu_setup_lock for mutual exclusion
  2213. * with kvmppc_update_lpcr().
  2214. */
  2215. err = -ENOMEM;
  2216. vcore = kvmppc_vcore_create(kvm,
  2217. id & ~(kvm->arch.smt_mode - 1));
  2218. mutex_lock(&kvm->arch.mmu_setup_lock);
  2219. kvm->arch.vcores[core] = vcore;
  2220. kvm->arch.online_vcores++;
  2221. mutex_unlock(&kvm->arch.mmu_setup_lock);
  2222. }
  2223. }
  2224. mutex_unlock(&kvm->lock);
  2225. if (!vcore)
  2226. return err;
  2227. spin_lock(&vcore->lock);
  2228. ++vcore->num_threads;
  2229. spin_unlock(&vcore->lock);
  2230. vcpu->arch.vcore = vcore;
  2231. vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
  2232. vcpu->arch.thread_cpu = -1;
  2233. vcpu->arch.prev_cpu = -1;
  2234. vcpu->arch.cpu_type = KVM_CPU_3S_64;
  2235. kvmppc_sanity_check(vcpu);
  2236. debugfs_vcpu_init(vcpu, id);
  2237. return 0;
  2238. }
  2239. static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
  2240. unsigned long flags)
  2241. {
  2242. int err;
  2243. int esmt = 0;
  2244. if (flags)
  2245. return -EINVAL;
  2246. if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
  2247. return -EINVAL;
  2248. if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
  2249. /*
  2250. * On POWER8 (or POWER7), the threading mode is "strict",
  2251. * so we pack smt_mode vcpus per vcore.
  2252. */
  2253. if (smt_mode > threads_per_subcore)
  2254. return -EINVAL;
  2255. } else {
  2256. /*
  2257. * On POWER9, the threading mode is "loose",
  2258. * so each vcpu gets its own vcore.
  2259. */
  2260. esmt = smt_mode;
  2261. smt_mode = 1;
  2262. }
  2263. mutex_lock(&kvm->lock);
  2264. err = -EBUSY;
  2265. if (!kvm->arch.online_vcores) {
  2266. kvm->arch.smt_mode = smt_mode;
  2267. kvm->arch.emul_smt_mode = esmt;
  2268. err = 0;
  2269. }
  2270. mutex_unlock(&kvm->lock);
  2271. return err;
  2272. }
  2273. static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
  2274. {
  2275. if (vpa->pinned_addr)
  2276. kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
  2277. vpa->dirty);
  2278. }
  2279. static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
  2280. {
  2281. spin_lock(&vcpu->arch.vpa_update_lock);
  2282. unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
  2283. unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
  2284. unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
  2285. spin_unlock(&vcpu->arch.vpa_update_lock);
  2286. }
  2287. static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
  2288. {
  2289. /* Indicate we want to get back into the guest */
  2290. return 1;
  2291. }
  2292. static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
  2293. {
  2294. unsigned long dec_nsec, now;
  2295. now = get_tb();
  2296. if (now > vcpu->arch.dec_expires) {
  2297. /* decrementer has already gone negative */
  2298. kvmppc_core_queue_dec(vcpu);
  2299. kvmppc_core_prepare_to_enter(vcpu);
  2300. return;
  2301. }
  2302. dec_nsec = tb_to_ns(vcpu->arch.dec_expires - now);
  2303. hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
  2304. vcpu->arch.timer_running = 1;
  2305. }
  2306. extern int __kvmppc_vcore_entry(void);
  2307. static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
  2308. struct kvm_vcpu *vcpu)
  2309. {
  2310. u64 now;
  2311. if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
  2312. return;
  2313. spin_lock_irq(&vcpu->arch.tbacct_lock);
  2314. now = mftb();
  2315. vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
  2316. vcpu->arch.stolen_logged;
  2317. vcpu->arch.busy_preempt = now;
  2318. vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
  2319. spin_unlock_irq(&vcpu->arch.tbacct_lock);
  2320. --vc->n_runnable;
  2321. WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
  2322. }
  2323. static int kvmppc_grab_hwthread(int cpu)
  2324. {
  2325. struct paca_struct *tpaca;
  2326. long timeout = 10000;
  2327. tpaca = paca_ptrs[cpu];
  2328. /* Ensure the thread won't go into the kernel if it wakes */
  2329. tpaca->kvm_hstate.kvm_vcpu = NULL;
  2330. tpaca->kvm_hstate.kvm_vcore = NULL;
  2331. tpaca->kvm_hstate.napping = 0;
  2332. smp_wmb();
  2333. tpaca->kvm_hstate.hwthread_req = 1;
  2334. /*
  2335. * If the thread is already executing in the kernel (e.g. handling
  2336. * a stray interrupt), wait for it to get back to nap mode.
  2337. * The smp_mb() is to ensure that our setting of hwthread_req
  2338. * is visible before we look at hwthread_state, so if this
  2339. * races with the code at system_reset_pSeries and the thread
  2340. * misses our setting of hwthread_req, we are sure to see its
  2341. * setting of hwthread_state, and vice versa.
  2342. */
  2343. smp_mb();
  2344. while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
  2345. if (--timeout <= 0) {
  2346. pr_err("KVM: couldn't grab cpu %d\n", cpu);
  2347. return -EBUSY;
  2348. }
  2349. udelay(1);
  2350. }
  2351. return 0;
  2352. }
  2353. static void kvmppc_release_hwthread(int cpu)
  2354. {
  2355. struct paca_struct *tpaca;
  2356. tpaca = paca_ptrs[cpu];
  2357. tpaca->kvm_hstate.hwthread_req = 0;
  2358. tpaca->kvm_hstate.kvm_vcpu = NULL;
  2359. tpaca->kvm_hstate.kvm_vcore = NULL;
  2360. tpaca->kvm_hstate.kvm_split_mode = NULL;
  2361. }
  2362. static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
  2363. {
  2364. struct kvm_nested_guest *nested = vcpu->arch.nested;
  2365. cpumask_t *cpu_in_guest;
  2366. int i;
  2367. cpu = cpu_first_tlb_thread_sibling(cpu);
  2368. if (nested) {
  2369. cpumask_set_cpu(cpu, &nested->need_tlb_flush);
  2370. cpu_in_guest = &nested->cpu_in_guest;
  2371. } else {
  2372. cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush);
  2373. cpu_in_guest = &kvm->arch.cpu_in_guest;
  2374. }
  2375. /*
  2376. * Make sure setting of bit in need_tlb_flush precedes
  2377. * testing of cpu_in_guest bits. The matching barrier on
  2378. * the other side is the first smp_mb() in kvmppc_run_core().
  2379. */
  2380. smp_mb();
  2381. for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu);
  2382. i += cpu_tlb_thread_sibling_step())
  2383. if (cpumask_test_cpu(i, cpu_in_guest))
  2384. smp_call_function_single(i, do_nothing, NULL, 1);
  2385. }
  2386. static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
  2387. {
  2388. struct kvm_nested_guest *nested = vcpu->arch.nested;
  2389. struct kvm *kvm = vcpu->kvm;
  2390. int prev_cpu;
  2391. if (!cpu_has_feature(CPU_FTR_HVMODE))
  2392. return;
  2393. if (nested)
  2394. prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
  2395. else
  2396. prev_cpu = vcpu->arch.prev_cpu;
  2397. /*
  2398. * With radix, the guest can do TLB invalidations itself,
  2399. * and it could choose to use the local form (tlbiel) if
  2400. * it is invalidating a translation that has only ever been
  2401. * used on one vcpu. However, that doesn't mean it has
  2402. * only ever been used on one physical cpu, since vcpus
  2403. * can move around between pcpus. To cope with this, when
  2404. * a vcpu moves from one pcpu to another, we need to tell
  2405. * any vcpus running on the same core as this vcpu previously
  2406. * ran to flush the TLB. The TLB is shared between threads,
  2407. * so we use a single bit in .need_tlb_flush for all 4 threads.
  2408. */
  2409. if (prev_cpu != pcpu) {
  2410. if (prev_cpu >= 0 &&
  2411. cpu_first_tlb_thread_sibling(prev_cpu) !=
  2412. cpu_first_tlb_thread_sibling(pcpu))
  2413. radix_flush_cpu(kvm, prev_cpu, vcpu);
  2414. if (nested)
  2415. nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
  2416. else
  2417. vcpu->arch.prev_cpu = pcpu;
  2418. }
  2419. }
  2420. static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
  2421. {
  2422. int cpu;
  2423. struct paca_struct *tpaca;
  2424. struct kvm *kvm = vc->kvm;
  2425. cpu = vc->pcpu;
  2426. if (vcpu) {
  2427. if (vcpu->arch.timer_running) {
  2428. hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
  2429. vcpu->arch.timer_running = 0;
  2430. }
  2431. cpu += vcpu->arch.ptid;
  2432. vcpu->cpu = vc->pcpu;
  2433. vcpu->arch.thread_cpu = cpu;
  2434. cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest);
  2435. }
  2436. tpaca = paca_ptrs[cpu];
  2437. tpaca->kvm_hstate.kvm_vcpu = vcpu;
  2438. tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
  2439. tpaca->kvm_hstate.fake_suspend = 0;
  2440. /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */
  2441. smp_wmb();
  2442. tpaca->kvm_hstate.kvm_vcore = vc;
  2443. if (cpu != smp_processor_id())
  2444. kvmppc_ipi_thread(cpu);
  2445. }
  2446. static void kvmppc_wait_for_nap(int n_threads)
  2447. {
  2448. int cpu = smp_processor_id();
  2449. int i, loops;
  2450. if (n_threads <= 1)
  2451. return;
  2452. for (loops = 0; loops < 1000000; ++loops) {
  2453. /*
  2454. * Check if all threads are finished.
  2455. * We set the vcore pointer when starting a thread
  2456. * and the thread clears it when finished, so we look
  2457. * for any threads that still have a non-NULL vcore ptr.
  2458. */
  2459. for (i = 1; i < n_threads; ++i)
  2460. if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
  2461. break;
  2462. if (i == n_threads) {
  2463. HMT_medium();
  2464. return;
  2465. }
  2466. HMT_low();
  2467. }
  2468. HMT_medium();
  2469. for (i = 1; i < n_threads; ++i)
  2470. if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
  2471. pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
  2472. }
  2473. /*
  2474. * Check that we are on thread 0 and that any other threads in
  2475. * this core are off-line. Then grab the threads so they can't
  2476. * enter the kernel.
  2477. */
  2478. static int on_primary_thread(void)
  2479. {
  2480. int cpu = smp_processor_id();
  2481. int thr;
  2482. /* Are we on a primary subcore? */
  2483. if (cpu_thread_in_subcore(cpu))
  2484. return 0;
  2485. thr = 0;
  2486. while (++thr < threads_per_subcore)
  2487. if (cpu_online(cpu + thr))
  2488. return 0;
  2489. /* Grab all hw threads so they can't go into the kernel */
  2490. for (thr = 1; thr < threads_per_subcore; ++thr) {
  2491. if (kvmppc_grab_hwthread(cpu + thr)) {
  2492. /* Couldn't grab one; let the others go */
  2493. do {
  2494. kvmppc_release_hwthread(cpu + thr);
  2495. } while (--thr > 0);
  2496. return 0;
  2497. }
  2498. }
  2499. return 1;
  2500. }
  2501. /*
  2502. * A list of virtual cores for each physical CPU.
  2503. * These are vcores that could run but their runner VCPU tasks are
  2504. * (or may be) preempted.
  2505. */
  2506. struct preempted_vcore_list {
  2507. struct list_head list;
  2508. spinlock_t lock;
  2509. };
  2510. static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
  2511. static void init_vcore_lists(void)
  2512. {
  2513. int cpu;
  2514. for_each_possible_cpu(cpu) {
  2515. struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
  2516. spin_lock_init(&lp->lock);
  2517. INIT_LIST_HEAD(&lp->list);
  2518. }
  2519. }
  2520. static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
  2521. {
  2522. struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
  2523. vc->vcore_state = VCORE_PREEMPT;
  2524. vc->pcpu = smp_processor_id();
  2525. if (vc->num_threads < threads_per_vcore(vc->kvm)) {
  2526. spin_lock(&lp->lock);
  2527. list_add_tail(&vc->preempt_list, &lp->list);
  2528. spin_unlock(&lp->lock);
  2529. }
  2530. /* Start accumulating stolen time */
  2531. kvmppc_core_start_stolen(vc);
  2532. }
  2533. static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
  2534. {
  2535. struct preempted_vcore_list *lp;
  2536. kvmppc_core_end_stolen(vc);
  2537. if (!list_empty(&vc->preempt_list)) {
  2538. lp = &per_cpu(preempted_vcores, vc->pcpu);
  2539. spin_lock(&lp->lock);
  2540. list_del_init(&vc->preempt_list);
  2541. spin_unlock(&lp->lock);
  2542. }
  2543. vc->vcore_state = VCORE_INACTIVE;
  2544. }
  2545. /*
  2546. * This stores information about the virtual cores currently
  2547. * assigned to a physical core.
  2548. */
  2549. struct core_info {
  2550. int n_subcores;
  2551. int max_subcore_threads;
  2552. int total_threads;
  2553. int subcore_threads[MAX_SUBCORES];
  2554. struct kvmppc_vcore *vc[MAX_SUBCORES];
  2555. };
  2556. /*
  2557. * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7
  2558. * respectively in 2-way micro-threading (split-core) mode on POWER8.
  2559. */
  2560. static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
  2561. static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
  2562. {
  2563. memset(cip, 0, sizeof(*cip));
  2564. cip->n_subcores = 1;
  2565. cip->max_subcore_threads = vc->num_threads;
  2566. cip->total_threads = vc->num_threads;
  2567. cip->subcore_threads[0] = vc->num_threads;
  2568. cip->vc[0] = vc;
  2569. }
  2570. static bool subcore_config_ok(int n_subcores, int n_threads)
  2571. {
  2572. /*
  2573. * POWER9 "SMT4" cores are permanently in what is effectively a 4-way
  2574. * split-core mode, with one thread per subcore.
  2575. */
  2576. if (cpu_has_feature(CPU_FTR_ARCH_300))
  2577. return n_subcores <= 4 && n_threads == 1;
  2578. /* On POWER8, can only dynamically split if unsplit to begin with */
  2579. if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
  2580. return false;
  2581. if (n_subcores > MAX_SUBCORES)
  2582. return false;
  2583. if (n_subcores > 1) {
  2584. if (!(dynamic_mt_modes & 2))
  2585. n_subcores = 4;
  2586. if (n_subcores > 2 && !(dynamic_mt_modes & 4))
  2587. return false;
  2588. }
  2589. return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
  2590. }
  2591. static void init_vcore_to_run(struct kvmppc_vcore *vc)
  2592. {
  2593. vc->entry_exit_map = 0;
  2594. vc->in_guest = 0;
  2595. vc->napping_threads = 0;
  2596. vc->conferring_threads = 0;
  2597. vc->tb_offset_applied = 0;
  2598. }
  2599. static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
  2600. {
  2601. int n_threads = vc->num_threads;
  2602. int sub;
  2603. if (!cpu_has_feature(CPU_FTR_ARCH_207S))
  2604. return false;
  2605. /* In one_vm_per_core mode, require all vcores to be from the same vm */
  2606. if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
  2607. return false;
  2608. /* Some POWER9 chips require all threads to be in the same MMU mode */
  2609. if (no_mixing_hpt_and_radix &&
  2610. kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
  2611. return false;
  2612. if (n_threads < cip->max_subcore_threads)
  2613. n_threads = cip->max_subcore_threads;
  2614. if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
  2615. return false;
  2616. cip->max_subcore_threads = n_threads;
  2617. sub = cip->n_subcores;
  2618. ++cip->n_subcores;
  2619. cip->total_threads += vc->num_threads;
  2620. cip->subcore_threads[sub] = vc->num_threads;
  2621. cip->vc[sub] = vc;
  2622. init_vcore_to_run(vc);
  2623. list_del_init(&vc->preempt_list);
  2624. return true;
  2625. }
  2626. /*
  2627. * Work out whether it is possible to piggyback the execution of
  2628. * vcore *pvc onto the execution of the other vcores described in *cip.
  2629. */
  2630. static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
  2631. int target_threads)
  2632. {
  2633. if (cip->total_threads + pvc->num_threads > target_threads)
  2634. return false;
  2635. return can_dynamic_split(pvc, cip);
  2636. }
  2637. static void prepare_threads(struct kvmppc_vcore *vc)
  2638. {
  2639. int i;
  2640. struct kvm_vcpu *vcpu;
  2641. for_each_runnable_thread(i, vcpu, vc) {
  2642. if (signal_pending(vcpu->arch.run_task))
  2643. vcpu->arch.ret = -EINTR;
  2644. else if (vcpu->arch.vpa.update_pending ||
  2645. vcpu->arch.slb_shadow.update_pending ||
  2646. vcpu->arch.dtl.update_pending)
  2647. vcpu->arch.ret = RESUME_GUEST;
  2648. else
  2649. continue;
  2650. kvmppc_remove_runnable(vc, vcpu);
  2651. wake_up(&vcpu->arch.cpu_run);
  2652. }
  2653. }
  2654. static void collect_piggybacks(struct core_info *cip, int target_threads)
  2655. {
  2656. struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
  2657. struct kvmppc_vcore *pvc, *vcnext;
  2658. spin_lock(&lp->lock);
  2659. list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
  2660. if (!spin_trylock(&pvc->lock))
  2661. continue;
  2662. prepare_threads(pvc);
  2663. if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
  2664. list_del_init(&pvc->preempt_list);
  2665. if (pvc->runner == NULL) {
  2666. pvc->vcore_state = VCORE_INACTIVE;
  2667. kvmppc_core_end_stolen(pvc);
  2668. }
  2669. spin_unlock(&pvc->lock);
  2670. continue;
  2671. }
  2672. if (!can_piggyback(pvc, cip, target_threads)) {
  2673. spin_unlock(&pvc->lock);
  2674. continue;
  2675. }
  2676. kvmppc_core_end_stolen(pvc);
  2677. pvc->vcore_state = VCORE_PIGGYBACK;
  2678. if (cip->total_threads >= target_threads)
  2679. break;
  2680. }
  2681. spin_unlock(&lp->lock);
  2682. }
  2683. static bool recheck_signals_and_mmu(struct core_info *cip)
  2684. {
  2685. int sub, i;
  2686. struct kvm_vcpu *vcpu;
  2687. struct kvmppc_vcore *vc;
  2688. for (sub = 0; sub < cip->n_subcores; ++sub) {
  2689. vc = cip->vc[sub];
  2690. if (!vc->kvm->arch.mmu_ready)
  2691. return true;
  2692. for_each_runnable_thread(i, vcpu, vc)
  2693. if (signal_pending(vcpu->arch.run_task))
  2694. return true;
  2695. }
  2696. return false;
  2697. }
  2698. static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
  2699. {
  2700. int still_running = 0, i;
  2701. u64 now;
  2702. long ret;
  2703. struct kvm_vcpu *vcpu;
  2704. spin_lock(&vc->lock);
  2705. now = get_tb();
  2706. for_each_runnable_thread(i, vcpu, vc) {
  2707. /*
  2708. * It's safe to unlock the vcore in the loop here, because
  2709. * for_each_runnable_thread() is safe against removal of
  2710. * the vcpu, and the vcore state is VCORE_EXITING here,
  2711. * so any vcpus becoming runnable will have their arch.trap
  2712. * set to zero and can't actually run in the guest.
  2713. */
  2714. spin_unlock(&vc->lock);
  2715. /* cancel pending dec exception if dec is positive */
  2716. if (now < vcpu->arch.dec_expires &&
  2717. kvmppc_core_pending_dec(vcpu))
  2718. kvmppc_core_dequeue_dec(vcpu);
  2719. trace_kvm_guest_exit(vcpu);
  2720. ret = RESUME_GUEST;
  2721. if (vcpu->arch.trap)
  2722. ret = kvmppc_handle_exit_hv(vcpu,
  2723. vcpu->arch.run_task);
  2724. vcpu->arch.ret = ret;
  2725. vcpu->arch.trap = 0;
  2726. spin_lock(&vc->lock);
  2727. if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
  2728. if (vcpu->arch.pending_exceptions)
  2729. kvmppc_core_prepare_to_enter(vcpu);
  2730. if (vcpu->arch.ceded)
  2731. kvmppc_set_timer(vcpu);
  2732. else
  2733. ++still_running;
  2734. } else {
  2735. kvmppc_remove_runnable(vc, vcpu);
  2736. wake_up(&vcpu->arch.cpu_run);
  2737. }
  2738. }
  2739. if (!is_master) {
  2740. if (still_running > 0) {
  2741. kvmppc_vcore_preempt(vc);
  2742. } else if (vc->runner) {
  2743. vc->vcore_state = VCORE_PREEMPT;
  2744. kvmppc_core_start_stolen(vc);
  2745. } else {
  2746. vc->vcore_state = VCORE_INACTIVE;
  2747. }
  2748. if (vc->n_runnable > 0 && vc->runner == NULL) {
  2749. /* make sure there's a candidate runner awake */
  2750. i = -1;
  2751. vcpu = next_runnable_thread(vc, &i);
  2752. wake_up(&vcpu->arch.cpu_run);
  2753. }
  2754. }
  2755. spin_unlock(&vc->lock);
  2756. }
  2757. /*
  2758. * Clear core from the list of active host cores as we are about to
  2759. * enter the guest. Only do this if it is the primary thread of the
  2760. * core (not if a subcore) that is entering the guest.
  2761. */
  2762. static inline int kvmppc_clear_host_core(unsigned int cpu)
  2763. {
  2764. int core;
  2765. if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
  2766. return 0;
  2767. /*
  2768. * Memory barrier can be omitted here as we will do a smp_wmb()
  2769. * later in kvmppc_start_thread and we need ensure that state is
  2770. * visible to other CPUs only after we enter guest.
  2771. */
  2772. core = cpu >> threads_shift;
  2773. kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
  2774. return 0;
  2775. }
  2776. /*
  2777. * Advertise this core as an active host core since we exited the guest
  2778. * Only need to do this if it is the primary thread of the core that is
  2779. * exiting.
  2780. */
  2781. static inline int kvmppc_set_host_core(unsigned int cpu)
  2782. {
  2783. int core;
  2784. if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
  2785. return 0;
  2786. /*
  2787. * Memory barrier can be omitted here because we do a spin_unlock
  2788. * immediately after this which provides the memory barrier.
  2789. */
  2790. core = cpu >> threads_shift;
  2791. kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
  2792. return 0;
  2793. }
  2794. static void set_irq_happened(int trap)
  2795. {
  2796. switch (trap) {
  2797. case BOOK3S_INTERRUPT_EXTERNAL:
  2798. local_paca->irq_happened |= PACA_IRQ_EE;
  2799. break;
  2800. case BOOK3S_INTERRUPT_H_DOORBELL:
  2801. local_paca->irq_happened |= PACA_IRQ_DBELL;
  2802. break;
  2803. case BOOK3S_INTERRUPT_HMI:
  2804. local_paca->irq_happened |= PACA_IRQ_HMI;
  2805. break;
  2806. case BOOK3S_INTERRUPT_SYSTEM_RESET:
  2807. replay_system_reset();
  2808. break;
  2809. }
  2810. }
  2811. /*
  2812. * Run a set of guest threads on a physical core.
  2813. * Called with vc->lock held.
  2814. */
  2815. static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
  2816. {
  2817. struct kvm_vcpu *vcpu;
  2818. int i;
  2819. int srcu_idx;
  2820. struct core_info core_info;
  2821. struct kvmppc_vcore *pvc;
  2822. struct kvm_split_mode split_info, *sip;
  2823. int split, subcore_size, active;
  2824. int sub;
  2825. bool thr0_done;
  2826. unsigned long cmd_bit, stat_bit;
  2827. int pcpu, thr;
  2828. int target_threads;
  2829. int controlled_threads;
  2830. int trap;
  2831. bool is_power8;
  2832. bool hpt_on_radix;
  2833. /*
  2834. * Remove from the list any threads that have a signal pending
  2835. * or need a VPA update done
  2836. */
  2837. prepare_threads(vc);
  2838. /* if the runner is no longer runnable, let the caller pick a new one */
  2839. if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
  2840. return;
  2841. /*
  2842. * Initialize *vc.
  2843. */
  2844. init_vcore_to_run(vc);
  2845. vc->preempt_tb = TB_NIL;
  2846. /*
  2847. * Number of threads that we will be controlling: the same as
  2848. * the number of threads per subcore, except on POWER9,
  2849. * where it's 1 because the threads are (mostly) independent.
  2850. */
  2851. controlled_threads = threads_per_vcore(vc->kvm);
  2852. /*
  2853. * Make sure we are running on primary threads, and that secondary
  2854. * threads are offline. Also check if the number of threads in this
  2855. * guest are greater than the current system threads per guest.
  2856. * On POWER9, we need to be not in independent-threads mode if
  2857. * this is a HPT guest on a radix host machine where the
  2858. * CPU threads may not be in different MMU modes.
  2859. */
  2860. hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
  2861. !kvm_is_radix(vc->kvm);
  2862. if (((controlled_threads > 1) &&
  2863. ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
  2864. (hpt_on_radix && vc->kvm->arch.threads_indep)) {
  2865. for_each_runnable_thread(i, vcpu, vc) {
  2866. vcpu->arch.ret = -EBUSY;
  2867. kvmppc_remove_runnable(vc, vcpu);
  2868. wake_up(&vcpu->arch.cpu_run);
  2869. }
  2870. goto out;
  2871. }
  2872. /*
  2873. * See if we could run any other vcores on the physical core
  2874. * along with this one.
  2875. */
  2876. init_core_info(&core_info, vc);
  2877. pcpu = smp_processor_id();
  2878. target_threads = controlled_threads;
  2879. if (target_smt_mode && target_smt_mode < target_threads)
  2880. target_threads = target_smt_mode;
  2881. if (vc->num_threads < target_threads)
  2882. collect_piggybacks(&core_info, target_threads);
  2883. /*
  2884. * On radix, arrange for TLB flushing if necessary.
  2885. * This has to be done before disabling interrupts since
  2886. * it uses smp_call_function().
  2887. */
  2888. pcpu = smp_processor_id();
  2889. if (kvm_is_radix(vc->kvm)) {
  2890. for (sub = 0; sub < core_info.n_subcores; ++sub)
  2891. for_each_runnable_thread(i, vcpu, core_info.vc[sub])
  2892. kvmppc_prepare_radix_vcpu(vcpu, pcpu);
  2893. }
  2894. /*
  2895. * Hard-disable interrupts, and check resched flag and signals.
  2896. * If we need to reschedule or deliver a signal, clean up
  2897. * and return without going into the guest(s).
  2898. * If the mmu_ready flag has been cleared, don't go into the
  2899. * guest because that means a HPT resize operation is in progress.
  2900. */
  2901. local_irq_disable();
  2902. hard_irq_disable();
  2903. if (lazy_irq_pending() || need_resched() ||
  2904. recheck_signals_and_mmu(&core_info)) {
  2905. local_irq_enable();
  2906. vc->vcore_state = VCORE_INACTIVE;
  2907. /* Unlock all except the primary vcore */
  2908. for (sub = 1; sub < core_info.n_subcores; ++sub) {
  2909. pvc = core_info.vc[sub];
  2910. /* Put back on to the preempted vcores list */
  2911. kvmppc_vcore_preempt(pvc);
  2912. spin_unlock(&pvc->lock);
  2913. }
  2914. for (i = 0; i < controlled_threads; ++i)
  2915. kvmppc_release_hwthread(pcpu + i);
  2916. return;
  2917. }
  2918. kvmppc_clear_host_core(pcpu);
  2919. /* Decide on micro-threading (split-core) mode */
  2920. subcore_size = threads_per_subcore;
  2921. cmd_bit = stat_bit = 0;
  2922. split = core_info.n_subcores;
  2923. sip = NULL;
  2924. is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
  2925. && !cpu_has_feature(CPU_FTR_ARCH_300);
  2926. if (split > 1 || hpt_on_radix) {
  2927. sip = &split_info;
  2928. memset(&split_info, 0, sizeof(split_info));
  2929. for (sub = 0; sub < core_info.n_subcores; ++sub)
  2930. split_info.vc[sub] = core_info.vc[sub];
  2931. if (is_power8) {
  2932. if (split == 2 && (dynamic_mt_modes & 2)) {
  2933. cmd_bit = HID0_POWER8_1TO2LPAR;
  2934. stat_bit = HID0_POWER8_2LPARMODE;
  2935. } else {
  2936. split = 4;
  2937. cmd_bit = HID0_POWER8_1TO4LPAR;
  2938. stat_bit = HID0_POWER8_4LPARMODE;
  2939. }
  2940. subcore_size = MAX_SMT_THREADS / split;
  2941. split_info.rpr = mfspr(SPRN_RPR);
  2942. split_info.pmmar = mfspr(SPRN_PMMAR);
  2943. split_info.ldbar = mfspr(SPRN_LDBAR);
  2944. split_info.subcore_size = subcore_size;
  2945. } else {
  2946. split_info.subcore_size = 1;
  2947. if (hpt_on_radix) {
  2948. /* Use the split_info for LPCR/LPIDR changes */
  2949. split_info.lpcr_req = vc->lpcr;
  2950. split_info.lpidr_req = vc->kvm->arch.lpid;
  2951. split_info.host_lpcr = vc->kvm->arch.host_lpcr;
  2952. split_info.do_set = 1;
  2953. }
  2954. }
  2955. /* order writes to split_info before kvm_split_mode pointer */
  2956. smp_wmb();
  2957. }
  2958. for (thr = 0; thr < controlled_threads; ++thr) {
  2959. struct paca_struct *paca = paca_ptrs[pcpu + thr];
  2960. paca->kvm_hstate.tid = thr;
  2961. paca->kvm_hstate.napping = 0;
  2962. paca->kvm_hstate.kvm_split_mode = sip;
  2963. }
  2964. /* Initiate micro-threading (split-core) on POWER8 if required */
  2965. if (cmd_bit) {
  2966. unsigned long hid0 = mfspr(SPRN_HID0);
  2967. hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
  2968. mb();
  2969. mtspr(SPRN_HID0, hid0);
  2970. isync();
  2971. for (;;) {
  2972. hid0 = mfspr(SPRN_HID0);
  2973. if (hid0 & stat_bit)
  2974. break;
  2975. cpu_relax();
  2976. }
  2977. }
  2978. /*
  2979. * On POWER8, set RWMR register.
  2980. * Since it only affects PURR and SPURR, it doesn't affect
  2981. * the host, so we don't save/restore the host value.
  2982. */
  2983. if (is_power8) {
  2984. unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
  2985. int n_online = atomic_read(&vc->online_count);
  2986. /*
  2987. * Use the 8-thread value if we're doing split-core
  2988. * or if the vcore's online count looks bogus.
  2989. */
  2990. if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
  2991. n_online >= 1 && n_online <= MAX_SMT_THREADS)
  2992. rwmr_val = p8_rwmr_values[n_online];
  2993. mtspr(SPRN_RWMR, rwmr_val);
  2994. }
  2995. /* Start all the threads */
  2996. active = 0;
  2997. for (sub = 0; sub < core_info.n_subcores; ++sub) {
  2998. thr = is_power8 ? subcore_thread_map[sub] : sub;
  2999. thr0_done = false;
  3000. active |= 1 << thr;
  3001. pvc = core_info.vc[sub];
  3002. pvc->pcpu = pcpu + thr;
  3003. for_each_runnable_thread(i, vcpu, pvc) {
  3004. kvmppc_start_thread(vcpu, pvc);
  3005. kvmppc_create_dtl_entry(vcpu, pvc);
  3006. trace_kvm_guest_enter(vcpu);
  3007. if (!vcpu->arch.ptid)
  3008. thr0_done = true;
  3009. active |= 1 << (thr + vcpu->arch.ptid);
  3010. }
  3011. /*
  3012. * We need to start the first thread of each subcore
  3013. * even if it doesn't have a vcpu.
  3014. */
  3015. if (!thr0_done)
  3016. kvmppc_start_thread(NULL, pvc);
  3017. }
  3018. /*
  3019. * Ensure that split_info.do_nap is set after setting
  3020. * the vcore pointer in the PACA of the secondaries.
  3021. */
  3022. smp_mb();
  3023. /*
  3024. * When doing micro-threading, poke the inactive threads as well.
  3025. * This gets them to the nap instruction after kvm_do_nap,
  3026. * which reduces the time taken to unsplit later.
  3027. * For POWER9 HPT guest on radix host, we need all the secondary
  3028. * threads woken up so they can do the LPCR/LPIDR change.
  3029. */
  3030. if (cmd_bit || hpt_on_radix) {
  3031. split_info.do_nap = 1; /* ask secondaries to nap when done */
  3032. for (thr = 1; thr < threads_per_subcore; ++thr)
  3033. if (!(active & (1 << thr)))
  3034. kvmppc_ipi_thread(pcpu + thr);
  3035. }
  3036. vc->vcore_state = VCORE_RUNNING;
  3037. preempt_disable();
  3038. trace_kvmppc_run_core(vc, 0);
  3039. for (sub = 0; sub < core_info.n_subcores; ++sub)
  3040. spin_unlock(&core_info.vc[sub]->lock);
  3041. guest_enter_irqoff();
  3042. srcu_idx = srcu_read_lock(&vc->kvm->srcu);
  3043. this_cpu_disable_ftrace();
  3044. /*
  3045. * Interrupts will be enabled once we get into the guest,
  3046. * so tell lockdep that we're about to enable interrupts.
  3047. */
  3048. trace_hardirqs_on();
  3049. trap = __kvmppc_vcore_entry();
  3050. trace_hardirqs_off();
  3051. this_cpu_enable_ftrace();
  3052. srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
  3053. set_irq_happened(trap);
  3054. spin_lock(&vc->lock);
  3055. /* prevent other vcpu threads from doing kvmppc_start_thread() now */
  3056. vc->vcore_state = VCORE_EXITING;
  3057. /* wait for secondary threads to finish writing their state to memory */
  3058. kvmppc_wait_for_nap(controlled_threads);
  3059. /* Return to whole-core mode if we split the core earlier */
  3060. if (cmd_bit) {
  3061. unsigned long hid0 = mfspr(SPRN_HID0);
  3062. unsigned long loops = 0;
  3063. hid0 &= ~HID0_POWER8_DYNLPARDIS;
  3064. stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
  3065. mb();
  3066. mtspr(SPRN_HID0, hid0);
  3067. isync();
  3068. for (;;) {
  3069. hid0 = mfspr(SPRN_HID0);
  3070. if (!(hid0 & stat_bit))
  3071. break;
  3072. cpu_relax();
  3073. ++loops;
  3074. }
  3075. } else if (hpt_on_radix) {
  3076. /* Wait for all threads to have seen final sync */
  3077. for (thr = 1; thr < controlled_threads; ++thr) {
  3078. struct paca_struct *paca = paca_ptrs[pcpu + thr];
  3079. while (paca->kvm_hstate.kvm_split_mode) {
  3080. HMT_low();
  3081. barrier();
  3082. }
  3083. HMT_medium();
  3084. }
  3085. }
  3086. split_info.do_nap = 0;
  3087. kvmppc_set_host_core(pcpu);
  3088. local_irq_enable();
  3089. guest_exit();
  3090. /* Let secondaries go back to the offline loop */
  3091. for (i = 0; i < controlled_threads; ++i) {
  3092. kvmppc_release_hwthread(pcpu + i);
  3093. if (sip && sip->napped[i])
  3094. kvmppc_ipi_thread(pcpu + i);
  3095. cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest);
  3096. }
  3097. spin_unlock(&vc->lock);
  3098. /* make sure updates to secondary vcpu structs are visible now */
  3099. smp_mb();
  3100. preempt_enable();
  3101. for (sub = 0; sub < core_info.n_subcores; ++sub) {
  3102. pvc = core_info.vc[sub];
  3103. post_guest_process(pvc, pvc == vc);
  3104. }
  3105. spin_lock(&vc->lock);
  3106. out:
  3107. vc->vcore_state = VCORE_INACTIVE;
  3108. trace_kvmppc_run_core(vc, 1);
  3109. }
  3110. /*
  3111. * Load up hypervisor-mode registers on P9.
  3112. */
  3113. static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit,
  3114. unsigned long lpcr)
  3115. {
  3116. struct kvmppc_vcore *vc = vcpu->arch.vcore;
  3117. s64 hdec;
  3118. u64 tb, purr, spurr;
  3119. int trap;
  3120. unsigned long host_hfscr = mfspr(SPRN_HFSCR);
  3121. unsigned long host_ciabr = mfspr(SPRN_CIABR);
  3122. unsigned long host_dawr = mfspr(SPRN_DAWR0);
  3123. unsigned long host_dawrx = mfspr(SPRN_DAWRX0);
  3124. unsigned long host_psscr = mfspr(SPRN_PSSCR);
  3125. unsigned long host_pidr = mfspr(SPRN_PID);
  3126. /*
  3127. * P8 and P9 suppress the HDEC exception when LPCR[HDICE] = 0,
  3128. * so set HDICE before writing HDEC.
  3129. */
  3130. mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr | LPCR_HDICE);
  3131. isync();
  3132. hdec = time_limit - mftb();
  3133. if (hdec < 0) {
  3134. mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr);
  3135. isync();
  3136. return BOOK3S_INTERRUPT_HV_DECREMENTER;
  3137. }
  3138. mtspr(SPRN_HDEC, hdec);
  3139. if (vc->tb_offset) {
  3140. u64 new_tb = mftb() + vc->tb_offset;
  3141. mtspr(SPRN_TBU40, new_tb);
  3142. tb = mftb();
  3143. if ((tb & 0xffffff) < (new_tb & 0xffffff))
  3144. mtspr(SPRN_TBU40, new_tb + 0x1000000);
  3145. vc->tb_offset_applied = vc->tb_offset;
  3146. }
  3147. if (vc->pcr)
  3148. mtspr(SPRN_PCR, vc->pcr | PCR_MASK);
  3149. mtspr(SPRN_DPDES, vc->dpdes);
  3150. mtspr(SPRN_VTB, vc->vtb);
  3151. local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
  3152. local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
  3153. mtspr(SPRN_PURR, vcpu->arch.purr);
  3154. mtspr(SPRN_SPURR, vcpu->arch.spurr);
  3155. if (dawr_enabled()) {
  3156. mtspr(SPRN_DAWR0, vcpu->arch.dawr);
  3157. mtspr(SPRN_DAWRX0, vcpu->arch.dawrx);
  3158. }
  3159. mtspr(SPRN_CIABR, vcpu->arch.ciabr);
  3160. mtspr(SPRN_IC, vcpu->arch.ic);
  3161. mtspr(SPRN_PID, vcpu->arch.pid);
  3162. mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
  3163. (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
  3164. mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
  3165. mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
  3166. mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
  3167. mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
  3168. mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
  3169. mtspr(SPRN_AMOR, ~0UL);
  3170. mtspr(SPRN_LPCR, lpcr);
  3171. isync();
  3172. kvmppc_xive_push_vcpu(vcpu);
  3173. mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
  3174. mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
  3175. trap = __kvmhv_vcpu_entry_p9(vcpu);
  3176. /* Advance host PURR/SPURR by the amount used by guest */
  3177. purr = mfspr(SPRN_PURR);
  3178. spurr = mfspr(SPRN_SPURR);
  3179. mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr +
  3180. purr - vcpu->arch.purr);
  3181. mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr +
  3182. spurr - vcpu->arch.spurr);
  3183. vcpu->arch.purr = purr;
  3184. vcpu->arch.spurr = spurr;
  3185. vcpu->arch.ic = mfspr(SPRN_IC);
  3186. vcpu->arch.pid = mfspr(SPRN_PID);
  3187. vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS;
  3188. vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
  3189. vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
  3190. vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
  3191. vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
  3192. /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */
  3193. mtspr(SPRN_PSSCR, host_psscr |
  3194. (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
  3195. mtspr(SPRN_HFSCR, host_hfscr);
  3196. mtspr(SPRN_CIABR, host_ciabr);
  3197. mtspr(SPRN_DAWR0, host_dawr);
  3198. mtspr(SPRN_DAWRX0, host_dawrx);
  3199. mtspr(SPRN_PID, host_pidr);
  3200. /*
  3201. * Since this is radix, do a eieio; tlbsync; ptesync sequence in
  3202. * case we interrupted the guest between a tlbie and a ptesync.
  3203. */
  3204. asm volatile("eieio; tlbsync; ptesync");
  3205. /*
  3206. * cp_abort is required if the processor supports local copy-paste
  3207. * to clear the copy buffer that was under control of the guest.
  3208. */
  3209. if (cpu_has_feature(CPU_FTR_ARCH_31))
  3210. asm volatile(PPC_CP_ABORT);
  3211. mtspr(SPRN_LPID, vcpu->kvm->arch.host_lpid); /* restore host LPID */
  3212. isync();
  3213. vc->dpdes = mfspr(SPRN_DPDES);
  3214. vc->vtb = mfspr(SPRN_VTB);
  3215. mtspr(SPRN_DPDES, 0);
  3216. if (vc->pcr)
  3217. mtspr(SPRN_PCR, PCR_MASK);
  3218. if (vc->tb_offset_applied) {
  3219. u64 new_tb = mftb() - vc->tb_offset_applied;
  3220. mtspr(SPRN_TBU40, new_tb);
  3221. tb = mftb();
  3222. if ((tb & 0xffffff) < (new_tb & 0xffffff))
  3223. mtspr(SPRN_TBU40, new_tb + 0x1000000);
  3224. vc->tb_offset_applied = 0;
  3225. }
  3226. mtspr(SPRN_HDEC, 0x7fffffff);
  3227. mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr);
  3228. return trap;
  3229. }
  3230. /*
  3231. * Virtual-mode guest entry for POWER9 and later when the host and
  3232. * guest are both using the radix MMU. The LPIDR has already been set.
  3233. */
  3234. static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
  3235. unsigned long lpcr)
  3236. {
  3237. struct kvmppc_vcore *vc = vcpu->arch.vcore;
  3238. unsigned long host_dscr = mfspr(SPRN_DSCR);
  3239. unsigned long host_tidr = mfspr(SPRN_TIDR);
  3240. unsigned long host_iamr = mfspr(SPRN_IAMR);
  3241. unsigned long host_amr = mfspr(SPRN_AMR);
  3242. unsigned long host_fscr = mfspr(SPRN_FSCR);
  3243. s64 dec;
  3244. u64 tb;
  3245. int trap, save_pmu;
  3246. dec = mfspr(SPRN_DEC);
  3247. tb = mftb();
  3248. if (dec < 0)
  3249. return BOOK3S_INTERRUPT_HV_DECREMENTER;
  3250. local_paca->kvm_hstate.dec_expires = dec + tb;
  3251. if (local_paca->kvm_hstate.dec_expires < time_limit)
  3252. time_limit = local_paca->kvm_hstate.dec_expires;
  3253. vcpu->arch.ceded = 0;
  3254. kvmhv_save_host_pmu(); /* saves it to PACA kvm_hstate */
  3255. kvmppc_subcore_enter_guest();
  3256. vc->entry_exit_map = 1;
  3257. vc->in_guest = 1;
  3258. if (vcpu->arch.vpa.pinned_addr) {
  3259. struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
  3260. u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
  3261. lp->yield_count = cpu_to_be32(yield_count);
  3262. vcpu->arch.vpa.dirty = 1;
  3263. }
  3264. if (cpu_has_feature(CPU_FTR_TM) ||
  3265. cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
  3266. kvmppc_restore_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
  3267. #ifdef CONFIG_PPC_PSERIES
  3268. if (kvmhv_on_pseries()) {
  3269. barrier();
  3270. if (vcpu->arch.vpa.pinned_addr) {
  3271. struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
  3272. get_lppaca()->pmcregs_in_use = lp->pmcregs_in_use;
  3273. } else {
  3274. get_lppaca()->pmcregs_in_use = 1;
  3275. }
  3276. barrier();
  3277. }
  3278. #endif
  3279. kvmhv_load_guest_pmu(vcpu);
  3280. msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
  3281. load_fp_state(&vcpu->arch.fp);
  3282. #ifdef CONFIG_ALTIVEC
  3283. load_vr_state(&vcpu->arch.vr);
  3284. #endif
  3285. mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
  3286. mtspr(SPRN_DSCR, vcpu->arch.dscr);
  3287. mtspr(SPRN_IAMR, vcpu->arch.iamr);
  3288. mtspr(SPRN_PSPB, vcpu->arch.pspb);
  3289. mtspr(SPRN_FSCR, vcpu->arch.fscr);
  3290. mtspr(SPRN_TAR, vcpu->arch.tar);
  3291. mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
  3292. mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
  3293. mtspr(SPRN_BESCR, vcpu->arch.bescr);
  3294. mtspr(SPRN_WORT, vcpu->arch.wort);
  3295. mtspr(SPRN_TIDR, vcpu->arch.tid);
  3296. mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
  3297. mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
  3298. mtspr(SPRN_AMR, vcpu->arch.amr);
  3299. mtspr(SPRN_UAMOR, vcpu->arch.uamor);
  3300. if (!(vcpu->arch.ctrl & 1))
  3301. mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1);
  3302. mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb());
  3303. if (kvmhv_on_pseries()) {
  3304. /*
  3305. * We need to save and restore the guest visible part of the
  3306. * psscr (i.e. using SPRN_PSSCR_PR) since the hypervisor
  3307. * doesn't do this for us. Note only required if pseries since
  3308. * this is done in kvmhv_load_hv_regs_and_go() below otherwise.
  3309. */
  3310. unsigned long host_psscr;
  3311. /* call our hypervisor to load up HV regs and go */
  3312. struct hv_guest_state hvregs;
  3313. host_psscr = mfspr(SPRN_PSSCR_PR);
  3314. mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
  3315. kvmhv_save_hv_regs(vcpu, &hvregs);
  3316. hvregs.lpcr = lpcr;
  3317. vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
  3318. hvregs.version = HV_GUEST_STATE_VERSION;
  3319. if (vcpu->arch.nested) {
  3320. hvregs.lpid = vcpu->arch.nested->shadow_lpid;
  3321. hvregs.vcpu_token = vcpu->arch.nested_vcpu_id;
  3322. } else {
  3323. hvregs.lpid = vcpu->kvm->arch.lpid;
  3324. hvregs.vcpu_token = vcpu->vcpu_id;
  3325. }
  3326. hvregs.hdec_expiry = time_limit;
  3327. trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs),
  3328. __pa(&vcpu->arch.regs));
  3329. kvmhv_restore_hv_return_state(vcpu, &hvregs);
  3330. vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
  3331. vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
  3332. vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
  3333. vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
  3334. mtspr(SPRN_PSSCR_PR, host_psscr);
  3335. /* H_CEDE has to be handled now, not later */
  3336. if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested &&
  3337. kvmppc_get_gpr(vcpu, 3) == H_CEDE) {
  3338. kvmppc_nested_cede(vcpu);
  3339. kvmppc_set_gpr(vcpu, 3, 0);
  3340. trap = 0;
  3341. }
  3342. } else {
  3343. trap = kvmhv_load_hv_regs_and_go(vcpu, time_limit, lpcr);
  3344. }
  3345. vcpu->arch.slb_max = 0;
  3346. dec = mfspr(SPRN_DEC);
  3347. if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */
  3348. dec = (s32) dec;
  3349. tb = mftb();
  3350. vcpu->arch.dec_expires = dec + tb;
  3351. vcpu->cpu = -1;
  3352. vcpu->arch.thread_cpu = -1;
  3353. /* Save guest CTRL register, set runlatch to 1 */
  3354. vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
  3355. if (!(vcpu->arch.ctrl & 1))
  3356. mtspr(SPRN_CTRLT, vcpu->arch.ctrl | 1);
  3357. vcpu->arch.iamr = mfspr(SPRN_IAMR);
  3358. vcpu->arch.pspb = mfspr(SPRN_PSPB);
  3359. vcpu->arch.fscr = mfspr(SPRN_FSCR);
  3360. vcpu->arch.tar = mfspr(SPRN_TAR);
  3361. vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
  3362. vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
  3363. vcpu->arch.bescr = mfspr(SPRN_BESCR);
  3364. vcpu->arch.wort = mfspr(SPRN_WORT);
  3365. vcpu->arch.tid = mfspr(SPRN_TIDR);
  3366. vcpu->arch.amr = mfspr(SPRN_AMR);
  3367. vcpu->arch.uamor = mfspr(SPRN_UAMOR);
  3368. vcpu->arch.dscr = mfspr(SPRN_DSCR);
  3369. mtspr(SPRN_PSPB, 0);
  3370. mtspr(SPRN_WORT, 0);
  3371. mtspr(SPRN_UAMOR, 0);
  3372. mtspr(SPRN_DSCR, host_dscr);
  3373. mtspr(SPRN_TIDR, host_tidr);
  3374. mtspr(SPRN_IAMR, host_iamr);
  3375. mtspr(SPRN_PSPB, 0);
  3376. if (host_amr != vcpu->arch.amr)
  3377. mtspr(SPRN_AMR, host_amr);
  3378. if (host_fscr != vcpu->arch.fscr)
  3379. mtspr(SPRN_FSCR, host_fscr);
  3380. msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
  3381. store_fp_state(&vcpu->arch.fp);
  3382. #ifdef CONFIG_ALTIVEC
  3383. store_vr_state(&vcpu->arch.vr);
  3384. #endif
  3385. vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
  3386. if (cpu_has_feature(CPU_FTR_TM) ||
  3387. cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
  3388. kvmppc_save_tm_hv(vcpu, vcpu->arch.shregs.msr, true);
  3389. save_pmu = 1;
  3390. if (vcpu->arch.vpa.pinned_addr) {
  3391. struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
  3392. u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
  3393. lp->yield_count = cpu_to_be32(yield_count);
  3394. vcpu->arch.vpa.dirty = 1;
  3395. save_pmu = lp->pmcregs_in_use;
  3396. }
  3397. /* Must save pmu if this guest is capable of running nested guests */
  3398. save_pmu |= nesting_enabled(vcpu->kvm);
  3399. kvmhv_save_guest_pmu(vcpu, save_pmu);
  3400. #ifdef CONFIG_PPC_PSERIES
  3401. if (kvmhv_on_pseries()) {
  3402. barrier();
  3403. get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse();
  3404. barrier();
  3405. }
  3406. #endif
  3407. vc->entry_exit_map = 0x101;
  3408. vc->in_guest = 0;
  3409. mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb());
  3410. mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
  3411. kvmhv_load_host_pmu();
  3412. kvmppc_subcore_exit_guest();
  3413. return trap;
  3414. }
  3415. /*
  3416. * Wait for some other vcpu thread to execute us, and
  3417. * wake us up when we need to handle something in the host.
  3418. */
  3419. static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
  3420. struct kvm_vcpu *vcpu, int wait_state)
  3421. {
  3422. DEFINE_WAIT(wait);
  3423. prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
  3424. if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
  3425. spin_unlock(&vc->lock);
  3426. schedule();
  3427. spin_lock(&vc->lock);
  3428. }
  3429. finish_wait(&vcpu->arch.cpu_run, &wait);
  3430. }
  3431. static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
  3432. {
  3433. if (!halt_poll_ns_grow)
  3434. return;
  3435. vc->halt_poll_ns *= halt_poll_ns_grow;
  3436. if (vc->halt_poll_ns < halt_poll_ns_grow_start)
  3437. vc->halt_poll_ns = halt_poll_ns_grow_start;
  3438. }
  3439. static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
  3440. {
  3441. if (halt_poll_ns_shrink == 0)
  3442. vc->halt_poll_ns = 0;
  3443. else
  3444. vc->halt_poll_ns /= halt_poll_ns_shrink;
  3445. }
  3446. #ifdef CONFIG_KVM_XICS
  3447. static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
  3448. {
  3449. if (!xics_on_xive())
  3450. return false;
  3451. return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
  3452. vcpu->arch.xive_saved_state.cppr;
  3453. }
  3454. #else
  3455. static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
  3456. {
  3457. return false;
  3458. }
  3459. #endif /* CONFIG_KVM_XICS */
  3460. static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
  3461. {
  3462. if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
  3463. kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
  3464. return true;
  3465. return false;
  3466. }
  3467. /*
  3468. * Check to see if any of the runnable vcpus on the vcore have pending
  3469. * exceptions or are no longer ceded
  3470. */
  3471. static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
  3472. {
  3473. struct kvm_vcpu *vcpu;
  3474. int i;
  3475. for_each_runnable_thread(i, vcpu, vc) {
  3476. if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
  3477. return 1;
  3478. }
  3479. return 0;
  3480. }
  3481. /*
  3482. * All the vcpus in this vcore are idle, so wait for a decrementer
  3483. * or external interrupt to one of the vcpus. vc->lock is held.
  3484. */
  3485. static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
  3486. {
  3487. ktime_t cur, start_poll, start_wait;
  3488. int do_sleep = 1;
  3489. u64 block_ns;
  3490. /* Poll for pending exceptions and ceded state */
  3491. cur = start_poll = ktime_get();
  3492. if (vc->halt_poll_ns) {
  3493. ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
  3494. ++vc->runner->stat.halt_attempted_poll;
  3495. vc->vcore_state = VCORE_POLLING;
  3496. spin_unlock(&vc->lock);
  3497. do {
  3498. if (kvmppc_vcore_check_block(vc)) {
  3499. do_sleep = 0;
  3500. break;
  3501. }
  3502. cur = ktime_get();
  3503. } while (single_task_running() && ktime_before(cur, stop));
  3504. spin_lock(&vc->lock);
  3505. vc->vcore_state = VCORE_INACTIVE;
  3506. if (!do_sleep) {
  3507. ++vc->runner->stat.halt_successful_poll;
  3508. goto out;
  3509. }
  3510. }
  3511. prepare_to_rcuwait(&vc->wait);
  3512. set_current_state(TASK_INTERRUPTIBLE);
  3513. if (kvmppc_vcore_check_block(vc)) {
  3514. finish_rcuwait(&vc->wait);
  3515. do_sleep = 0;
  3516. /* If we polled, count this as a successful poll */
  3517. if (vc->halt_poll_ns)
  3518. ++vc->runner->stat.halt_successful_poll;
  3519. goto out;
  3520. }
  3521. start_wait = ktime_get();
  3522. vc->vcore_state = VCORE_SLEEPING;
  3523. trace_kvmppc_vcore_blocked(vc, 0);
  3524. spin_unlock(&vc->lock);
  3525. schedule();
  3526. finish_rcuwait(&vc->wait);
  3527. spin_lock(&vc->lock);
  3528. vc->vcore_state = VCORE_INACTIVE;
  3529. trace_kvmppc_vcore_blocked(vc, 1);
  3530. ++vc->runner->stat.halt_successful_wait;
  3531. cur = ktime_get();
  3532. out:
  3533. block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
  3534. /* Attribute wait time */
  3535. if (do_sleep) {
  3536. vc->runner->stat.halt_wait_ns +=
  3537. ktime_to_ns(cur) - ktime_to_ns(start_wait);
  3538. /* Attribute failed poll time */
  3539. if (vc->halt_poll_ns)
  3540. vc->runner->stat.halt_poll_fail_ns +=
  3541. ktime_to_ns(start_wait) -
  3542. ktime_to_ns(start_poll);
  3543. } else {
  3544. /* Attribute successful poll time */
  3545. if (vc->halt_poll_ns)
  3546. vc->runner->stat.halt_poll_success_ns +=
  3547. ktime_to_ns(cur) -
  3548. ktime_to_ns(start_poll);
  3549. }
  3550. /* Adjust poll time */
  3551. if (halt_poll_ns) {
  3552. if (block_ns <= vc->halt_poll_ns)
  3553. ;
  3554. /* We slept and blocked for longer than the max halt time */
  3555. else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
  3556. shrink_halt_poll_ns(vc);
  3557. /* We slept and our poll time is too small */
  3558. else if (vc->halt_poll_ns < halt_poll_ns &&
  3559. block_ns < halt_poll_ns)
  3560. grow_halt_poll_ns(vc);
  3561. if (vc->halt_poll_ns > halt_poll_ns)
  3562. vc->halt_poll_ns = halt_poll_ns;
  3563. } else
  3564. vc->halt_poll_ns = 0;
  3565. trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
  3566. }
  3567. /*
  3568. * This never fails for a radix guest, as none of the operations it does
  3569. * for a radix guest can fail or have a way to report failure.
  3570. * kvmhv_run_single_vcpu() relies on this fact.
  3571. */
  3572. static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
  3573. {
  3574. int r = 0;
  3575. struct kvm *kvm = vcpu->kvm;
  3576. mutex_lock(&kvm->arch.mmu_setup_lock);
  3577. if (!kvm->arch.mmu_ready) {
  3578. if (!kvm_is_radix(kvm))
  3579. r = kvmppc_hv_setup_htab_rma(vcpu);
  3580. if (!r) {
  3581. if (cpu_has_feature(CPU_FTR_ARCH_300))
  3582. kvmppc_setup_partition_table(kvm);
  3583. kvm->arch.mmu_ready = 1;
  3584. }
  3585. }
  3586. mutex_unlock(&kvm->arch.mmu_setup_lock);
  3587. return r;
  3588. }
  3589. static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu)
  3590. {
  3591. struct kvm_run *run = vcpu->run;
  3592. int n_ceded, i, r;
  3593. struct kvmppc_vcore *vc;
  3594. struct kvm_vcpu *v;
  3595. trace_kvmppc_run_vcpu_enter(vcpu);
  3596. run->exit_reason = 0;
  3597. vcpu->arch.ret = RESUME_GUEST;
  3598. vcpu->arch.trap = 0;
  3599. kvmppc_update_vpas(vcpu);
  3600. /*
  3601. * Synchronize with other threads in this virtual core
  3602. */
  3603. vc = vcpu->arch.vcore;
  3604. spin_lock(&vc->lock);
  3605. vcpu->arch.ceded = 0;
  3606. vcpu->arch.run_task = current;
  3607. vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
  3608. vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
  3609. vcpu->arch.busy_preempt = TB_NIL;
  3610. WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
  3611. ++vc->n_runnable;
  3612. /*
  3613. * This happens the first time this is called for a vcpu.
  3614. * If the vcore is already running, we may be able to start
  3615. * this thread straight away and have it join in.
  3616. */
  3617. if (!signal_pending(current)) {
  3618. if ((vc->vcore_state == VCORE_PIGGYBACK ||
  3619. vc->vcore_state == VCORE_RUNNING) &&
  3620. !VCORE_IS_EXITING(vc)) {
  3621. kvmppc_create_dtl_entry(vcpu, vc);
  3622. kvmppc_start_thread(vcpu, vc);
  3623. trace_kvm_guest_enter(vcpu);
  3624. } else if (vc->vcore_state == VCORE_SLEEPING) {
  3625. rcuwait_wake_up(&vc->wait);
  3626. }
  3627. }
  3628. while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
  3629. !signal_pending(current)) {
  3630. /* See if the MMU is ready to go */
  3631. if (!vcpu->kvm->arch.mmu_ready) {
  3632. spin_unlock(&vc->lock);
  3633. r = kvmhv_setup_mmu(vcpu);
  3634. spin_lock(&vc->lock);
  3635. if (r) {
  3636. run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  3637. run->fail_entry.
  3638. hardware_entry_failure_reason = 0;
  3639. vcpu->arch.ret = r;
  3640. break;
  3641. }
  3642. }
  3643. if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
  3644. kvmppc_vcore_end_preempt(vc);
  3645. if (vc->vcore_state != VCORE_INACTIVE) {
  3646. kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
  3647. continue;
  3648. }
  3649. for_each_runnable_thread(i, v, vc) {
  3650. kvmppc_core_prepare_to_enter(v);
  3651. if (signal_pending(v->arch.run_task)) {
  3652. kvmppc_remove_runnable(vc, v);
  3653. v->stat.signal_exits++;
  3654. v->run->exit_reason = KVM_EXIT_INTR;
  3655. v->arch.ret = -EINTR;
  3656. wake_up(&v->arch.cpu_run);
  3657. }
  3658. }
  3659. if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
  3660. break;
  3661. n_ceded = 0;
  3662. for_each_runnable_thread(i, v, vc) {
  3663. if (!kvmppc_vcpu_woken(v))
  3664. n_ceded += v->arch.ceded;
  3665. else
  3666. v->arch.ceded = 0;
  3667. }
  3668. vc->runner = vcpu;
  3669. if (n_ceded == vc->n_runnable) {
  3670. kvmppc_vcore_blocked(vc);
  3671. } else if (need_resched()) {
  3672. kvmppc_vcore_preempt(vc);
  3673. /* Let something else run */
  3674. cond_resched_lock(&vc->lock);
  3675. if (vc->vcore_state == VCORE_PREEMPT)
  3676. kvmppc_vcore_end_preempt(vc);
  3677. } else {
  3678. kvmppc_run_core(vc);
  3679. }
  3680. vc->runner = NULL;
  3681. }
  3682. while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
  3683. (vc->vcore_state == VCORE_RUNNING ||
  3684. vc->vcore_state == VCORE_EXITING ||
  3685. vc->vcore_state == VCORE_PIGGYBACK))
  3686. kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
  3687. if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
  3688. kvmppc_vcore_end_preempt(vc);
  3689. if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
  3690. kvmppc_remove_runnable(vc, vcpu);
  3691. vcpu->stat.signal_exits++;
  3692. run->exit_reason = KVM_EXIT_INTR;
  3693. vcpu->arch.ret = -EINTR;
  3694. }
  3695. if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
  3696. /* Wake up some vcpu to run the core */
  3697. i = -1;
  3698. v = next_runnable_thread(vc, &i);
  3699. wake_up(&v->arch.cpu_run);
  3700. }
  3701. trace_kvmppc_run_vcpu_exit(vcpu);
  3702. spin_unlock(&vc->lock);
  3703. return vcpu->arch.ret;
  3704. }
  3705. int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
  3706. unsigned long lpcr)
  3707. {
  3708. struct kvm_run *run = vcpu->run;
  3709. int trap, r, pcpu;
  3710. int srcu_idx, lpid;
  3711. struct kvmppc_vcore *vc;
  3712. struct kvm *kvm = vcpu->kvm;
  3713. struct kvm_nested_guest *nested = vcpu->arch.nested;
  3714. trace_kvmppc_run_vcpu_enter(vcpu);
  3715. run->exit_reason = 0;
  3716. vcpu->arch.ret = RESUME_GUEST;
  3717. vcpu->arch.trap = 0;
  3718. vc = vcpu->arch.vcore;
  3719. vcpu->arch.ceded = 0;
  3720. vcpu->arch.run_task = current;
  3721. vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
  3722. vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
  3723. vcpu->arch.busy_preempt = TB_NIL;
  3724. vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
  3725. vc->runnable_threads[0] = vcpu;
  3726. vc->n_runnable = 1;
  3727. vc->runner = vcpu;
  3728. /* See if the MMU is ready to go */
  3729. if (!kvm->arch.mmu_ready)
  3730. kvmhv_setup_mmu(vcpu);
  3731. if (need_resched())
  3732. cond_resched();
  3733. kvmppc_update_vpas(vcpu);
  3734. init_vcore_to_run(vc);
  3735. vc->preempt_tb = TB_NIL;
  3736. preempt_disable();
  3737. pcpu = smp_processor_id();
  3738. vc->pcpu = pcpu;
  3739. kvmppc_prepare_radix_vcpu(vcpu, pcpu);
  3740. local_irq_disable();
  3741. hard_irq_disable();
  3742. if (signal_pending(current))
  3743. goto sigpend;
  3744. if (lazy_irq_pending() || need_resched() || !kvm->arch.mmu_ready)
  3745. goto out;
  3746. if (!nested) {
  3747. kvmppc_core_prepare_to_enter(vcpu);
  3748. if (vcpu->arch.doorbell_request) {
  3749. vc->dpdes = 1;
  3750. smp_wmb();
  3751. vcpu->arch.doorbell_request = 0;
  3752. }
  3753. if (test_bit(BOOK3S_IRQPRIO_EXTERNAL,
  3754. &vcpu->arch.pending_exceptions))
  3755. lpcr |= LPCR_MER;
  3756. } else if (vcpu->arch.pending_exceptions ||
  3757. vcpu->arch.doorbell_request ||
  3758. xive_interrupt_pending(vcpu)) {
  3759. vcpu->arch.ret = RESUME_HOST;
  3760. goto out;
  3761. }
  3762. kvmppc_clear_host_core(pcpu);
  3763. local_paca->kvm_hstate.tid = 0;
  3764. local_paca->kvm_hstate.napping = 0;
  3765. local_paca->kvm_hstate.kvm_split_mode = NULL;
  3766. kvmppc_start_thread(vcpu, vc);
  3767. kvmppc_create_dtl_entry(vcpu, vc);
  3768. trace_kvm_guest_enter(vcpu);
  3769. vc->vcore_state = VCORE_RUNNING;
  3770. trace_kvmppc_run_core(vc, 0);
  3771. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  3772. lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
  3773. mtspr(SPRN_LPID, lpid);
  3774. isync();
  3775. kvmppc_check_need_tlb_flush(kvm, pcpu, nested);
  3776. }
  3777. guest_enter_irqoff();
  3778. srcu_idx = srcu_read_lock(&kvm->srcu);
  3779. this_cpu_disable_ftrace();
  3780. /* Tell lockdep that we're about to enable interrupts */
  3781. trace_hardirqs_on();
  3782. trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr);
  3783. vcpu->arch.trap = trap;
  3784. trace_hardirqs_off();
  3785. this_cpu_enable_ftrace();
  3786. srcu_read_unlock(&kvm->srcu, srcu_idx);
  3787. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  3788. mtspr(SPRN_LPID, kvm->arch.host_lpid);
  3789. isync();
  3790. }
  3791. set_irq_happened(trap);
  3792. kvmppc_set_host_core(pcpu);
  3793. local_irq_enable();
  3794. guest_exit();
  3795. cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest);
  3796. preempt_enable();
  3797. /*
  3798. * cancel pending decrementer exception if DEC is now positive, or if
  3799. * entering a nested guest in which case the decrementer is now owned
  3800. * by L2 and the L1 decrementer is provided in hdec_expires
  3801. */
  3802. if (kvmppc_core_pending_dec(vcpu) &&
  3803. ((get_tb() < vcpu->arch.dec_expires) ||
  3804. (trap == BOOK3S_INTERRUPT_SYSCALL &&
  3805. kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED)))
  3806. kvmppc_core_dequeue_dec(vcpu);
  3807. trace_kvm_guest_exit(vcpu);
  3808. r = RESUME_GUEST;
  3809. if (trap) {
  3810. if (!nested)
  3811. r = kvmppc_handle_exit_hv(vcpu, current);
  3812. else
  3813. r = kvmppc_handle_nested_exit(vcpu);
  3814. }
  3815. vcpu->arch.ret = r;
  3816. if (is_kvmppc_resume_guest(r) && vcpu->arch.ceded &&
  3817. !kvmppc_vcpu_woken(vcpu)) {
  3818. kvmppc_set_timer(vcpu);
  3819. while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) {
  3820. if (signal_pending(current)) {
  3821. vcpu->stat.signal_exits++;
  3822. run->exit_reason = KVM_EXIT_INTR;
  3823. vcpu->arch.ret = -EINTR;
  3824. break;
  3825. }
  3826. spin_lock(&vc->lock);
  3827. kvmppc_vcore_blocked(vc);
  3828. spin_unlock(&vc->lock);
  3829. }
  3830. }
  3831. vcpu->arch.ceded = 0;
  3832. vc->vcore_state = VCORE_INACTIVE;
  3833. trace_kvmppc_run_core(vc, 1);
  3834. done:
  3835. kvmppc_remove_runnable(vc, vcpu);
  3836. trace_kvmppc_run_vcpu_exit(vcpu);
  3837. return vcpu->arch.ret;
  3838. sigpend:
  3839. vcpu->stat.signal_exits++;
  3840. run->exit_reason = KVM_EXIT_INTR;
  3841. vcpu->arch.ret = -EINTR;
  3842. out:
  3843. local_irq_enable();
  3844. preempt_enable();
  3845. goto done;
  3846. }
  3847. static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu)
  3848. {
  3849. struct kvm_run *run = vcpu->run;
  3850. int r;
  3851. int srcu_idx;
  3852. unsigned long ebb_regs[3] = {}; /* shut up GCC */
  3853. unsigned long user_tar = 0;
  3854. unsigned int user_vrsave;
  3855. struct kvm *kvm;
  3856. if (!vcpu->arch.sane) {
  3857. run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  3858. return -EINVAL;
  3859. }
  3860. /*
  3861. * Don't allow entry with a suspended transaction, because
  3862. * the guest entry/exit code will lose it.
  3863. * If the guest has TM enabled, save away their TM-related SPRs
  3864. * (they will get restored by the TM unavailable interrupt).
  3865. */
  3866. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  3867. if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
  3868. (current->thread.regs->msr & MSR_TM)) {
  3869. if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
  3870. run->exit_reason = KVM_EXIT_FAIL_ENTRY;
  3871. run->fail_entry.hardware_entry_failure_reason = 0;
  3872. return -EINVAL;
  3873. }
  3874. /* Enable TM so we can read the TM SPRs */
  3875. mtmsr(mfmsr() | MSR_TM);
  3876. current->thread.tm_tfhar = mfspr(SPRN_TFHAR);
  3877. current->thread.tm_tfiar = mfspr(SPRN_TFIAR);
  3878. current->thread.tm_texasr = mfspr(SPRN_TEXASR);
  3879. current->thread.regs->msr &= ~MSR_TM;
  3880. }
  3881. #endif
  3882. /*
  3883. * Force online to 1 for the sake of old userspace which doesn't
  3884. * set it.
  3885. */
  3886. if (!vcpu->arch.online) {
  3887. atomic_inc(&vcpu->arch.vcore->online_count);
  3888. vcpu->arch.online = 1;
  3889. }
  3890. kvmppc_core_prepare_to_enter(vcpu);
  3891. /* No need to go into the guest when all we'll do is come back out */
  3892. if (signal_pending(current)) {
  3893. run->exit_reason = KVM_EXIT_INTR;
  3894. return -EINTR;
  3895. }
  3896. kvm = vcpu->kvm;
  3897. atomic_inc(&kvm->arch.vcpus_running);
  3898. /* Order vcpus_running vs. mmu_ready, see kvmppc_alloc_reset_hpt */
  3899. smp_mb();
  3900. flush_all_to_thread(current);
  3901. /* Save userspace EBB and other register values */
  3902. if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
  3903. ebb_regs[0] = mfspr(SPRN_EBBHR);
  3904. ebb_regs[1] = mfspr(SPRN_EBBRR);
  3905. ebb_regs[2] = mfspr(SPRN_BESCR);
  3906. user_tar = mfspr(SPRN_TAR);
  3907. }
  3908. user_vrsave = mfspr(SPRN_VRSAVE);
  3909. vcpu->arch.waitp = &vcpu->arch.vcore->wait;
  3910. vcpu->arch.pgdir = kvm->mm->pgd;
  3911. vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
  3912. do {
  3913. /*
  3914. * The early POWER9 chips that can't mix radix and HPT threads
  3915. * on the same core also need the workaround for the problem
  3916. * where the TLB would prefetch entries in the guest exit path
  3917. * for radix guests using the guest PIDR value and LPID 0.
  3918. * The workaround is in the old path (kvmppc_run_vcpu())
  3919. * but not the new path (kvmhv_run_single_vcpu()).
  3920. */
  3921. if (kvm->arch.threads_indep && kvm_is_radix(kvm) &&
  3922. !no_mixing_hpt_and_radix)
  3923. r = kvmhv_run_single_vcpu(vcpu, ~(u64)0,
  3924. vcpu->arch.vcore->lpcr);
  3925. else
  3926. r = kvmppc_run_vcpu(vcpu);
  3927. if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
  3928. !(vcpu->arch.shregs.msr & MSR_PR)) {
  3929. trace_kvm_hcall_enter(vcpu);
  3930. r = kvmppc_pseries_do_hcall(vcpu);
  3931. trace_kvm_hcall_exit(vcpu, r);
  3932. kvmppc_core_prepare_to_enter(vcpu);
  3933. } else if (r == RESUME_PAGE_FAULT) {
  3934. srcu_idx = srcu_read_lock(&kvm->srcu);
  3935. r = kvmppc_book3s_hv_page_fault(vcpu,
  3936. vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
  3937. srcu_read_unlock(&kvm->srcu, srcu_idx);
  3938. } else if (r == RESUME_PASSTHROUGH) {
  3939. if (WARN_ON(xics_on_xive()))
  3940. r = H_SUCCESS;
  3941. else
  3942. r = kvmppc_xics_rm_complete(vcpu, 0);
  3943. }
  3944. } while (is_kvmppc_resume_guest(r));
  3945. /* Restore userspace EBB and other register values */
  3946. if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
  3947. mtspr(SPRN_EBBHR, ebb_regs[0]);
  3948. mtspr(SPRN_EBBRR, ebb_regs[1]);
  3949. mtspr(SPRN_BESCR, ebb_regs[2]);
  3950. mtspr(SPRN_TAR, user_tar);
  3951. mtspr(SPRN_FSCR, current->thread.fscr);
  3952. }
  3953. mtspr(SPRN_VRSAVE, user_vrsave);
  3954. vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
  3955. atomic_dec(&kvm->arch.vcpus_running);
  3956. return r;
  3957. }
  3958. static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
  3959. int shift, int sllp)
  3960. {
  3961. (*sps)->page_shift = shift;
  3962. (*sps)->slb_enc = sllp;
  3963. (*sps)->enc[0].page_shift = shift;
  3964. (*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
  3965. /*
  3966. * Add 16MB MPSS support (may get filtered out by userspace)
  3967. */
  3968. if (shift != 24) {
  3969. int penc = kvmppc_pgsize_lp_encoding(shift, 24);
  3970. if (penc != -1) {
  3971. (*sps)->enc[1].page_shift = 24;
  3972. (*sps)->enc[1].pte_enc = penc;
  3973. }
  3974. }
  3975. (*sps)++;
  3976. }
  3977. static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
  3978. struct kvm_ppc_smmu_info *info)
  3979. {
  3980. struct kvm_ppc_one_seg_page_size *sps;
  3981. /*
  3982. * POWER7, POWER8 and POWER9 all support 32 storage keys for data.
  3983. * POWER7 doesn't support keys for instruction accesses,
  3984. * POWER8 and POWER9 do.
  3985. */
  3986. info->data_keys = 32;
  3987. info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
  3988. /* POWER7, 8 and 9 all have 1T segments and 32-entry SLB */
  3989. info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
  3990. info->slb_size = 32;
  3991. /* We only support these sizes for now, and no muti-size segments */
  3992. sps = &info->sps[0];
  3993. kvmppc_add_seg_page_size(&sps, 12, 0);
  3994. kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
  3995. kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
  3996. /* If running as a nested hypervisor, we don't support HPT guests */
  3997. if (kvmhv_on_pseries())
  3998. info->flags |= KVM_PPC_NO_HASH;
  3999. return 0;
  4000. }
  4001. /*
  4002. * Get (and clear) the dirty memory log for a memory slot.
  4003. */
  4004. static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
  4005. struct kvm_dirty_log *log)
  4006. {
  4007. struct kvm_memslots *slots;
  4008. struct kvm_memory_slot *memslot;
  4009. int i, r;
  4010. unsigned long n;
  4011. unsigned long *buf, *p;
  4012. struct kvm_vcpu *vcpu;
  4013. mutex_lock(&kvm->slots_lock);
  4014. r = -EINVAL;
  4015. if (log->slot >= KVM_USER_MEM_SLOTS)
  4016. goto out;
  4017. slots = kvm_memslots(kvm);
  4018. memslot = id_to_memslot(slots, log->slot);
  4019. r = -ENOENT;
  4020. if (!memslot || !memslot->dirty_bitmap)
  4021. goto out;
  4022. /*
  4023. * Use second half of bitmap area because both HPT and radix
  4024. * accumulate bits in the first half.
  4025. */
  4026. n = kvm_dirty_bitmap_bytes(memslot);
  4027. buf = memslot->dirty_bitmap + n / sizeof(long);
  4028. memset(buf, 0, n);
  4029. if (kvm_is_radix(kvm))
  4030. r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
  4031. else
  4032. r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
  4033. if (r)
  4034. goto out;
  4035. /*
  4036. * We accumulate dirty bits in the first half of the
  4037. * memslot's dirty_bitmap area, for when pages are paged
  4038. * out or modified by the host directly. Pick up these
  4039. * bits and add them to the map.
  4040. */
  4041. p = memslot->dirty_bitmap;
  4042. for (i = 0; i < n / sizeof(long); ++i)
  4043. buf[i] |= xchg(&p[i], 0);
  4044. /* Harvest dirty bits from VPA and DTL updates */
  4045. /* Note: we never modify the SLB shadow buffer areas */
  4046. kvm_for_each_vcpu(i, vcpu, kvm) {
  4047. spin_lock(&vcpu->arch.vpa_update_lock);
  4048. kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
  4049. kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
  4050. spin_unlock(&vcpu->arch.vpa_update_lock);
  4051. }
  4052. r = -EFAULT;
  4053. if (copy_to_user(log->dirty_bitmap, buf, n))
  4054. goto out;
  4055. r = 0;
  4056. out:
  4057. mutex_unlock(&kvm->slots_lock);
  4058. return r;
  4059. }
  4060. static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *slot)
  4061. {
  4062. vfree(slot->arch.rmap);
  4063. slot->arch.rmap = NULL;
  4064. }
  4065. static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
  4066. struct kvm_memory_slot *slot,
  4067. const struct kvm_userspace_memory_region *mem,
  4068. enum kvm_mr_change change)
  4069. {
  4070. unsigned long npages = mem->memory_size >> PAGE_SHIFT;
  4071. if (change == KVM_MR_CREATE) {
  4072. unsigned long size = array_size(npages, sizeof(*slot->arch.rmap));
  4073. if ((size >> PAGE_SHIFT) > totalram_pages())
  4074. return -ENOMEM;
  4075. slot->arch.rmap = vzalloc(size);
  4076. if (!slot->arch.rmap)
  4077. return -ENOMEM;
  4078. }
  4079. return 0;
  4080. }
  4081. static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
  4082. const struct kvm_userspace_memory_region *mem,
  4083. const struct kvm_memory_slot *old,
  4084. const struct kvm_memory_slot *new,
  4085. enum kvm_mr_change change)
  4086. {
  4087. unsigned long npages = mem->memory_size >> PAGE_SHIFT;
  4088. /*
  4089. * If we are making a new memslot, it might make
  4090. * some address that was previously cached as emulated
  4091. * MMIO be no longer emulated MMIO, so invalidate
  4092. * all the caches of emulated MMIO translations.
  4093. */
  4094. if (npages)
  4095. atomic64_inc(&kvm->arch.mmio_update);
  4096. /*
  4097. * For change == KVM_MR_MOVE or KVM_MR_DELETE, higher levels
  4098. * have already called kvm_arch_flush_shadow_memslot() to
  4099. * flush shadow mappings. For KVM_MR_CREATE we have no
  4100. * previous mappings. So the only case to handle is
  4101. * KVM_MR_FLAGS_ONLY when the KVM_MEM_LOG_DIRTY_PAGES bit
  4102. * has been changed.
  4103. * For radix guests, we flush on setting KVM_MEM_LOG_DIRTY_PAGES
  4104. * to get rid of any THP PTEs in the partition-scoped page tables
  4105. * so we can track dirtiness at the page level; we flush when
  4106. * clearing KVM_MEM_LOG_DIRTY_PAGES so that we can go back to
  4107. * using THP PTEs.
  4108. */
  4109. if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
  4110. ((new->flags ^ old->flags) & KVM_MEM_LOG_DIRTY_PAGES))
  4111. kvmppc_radix_flush_memslot(kvm, old);
  4112. /*
  4113. * If UV hasn't yet called H_SVM_INIT_START, don't register memslots.
  4114. */
  4115. if (!kvm->arch.secure_guest)
  4116. return;
  4117. switch (change) {
  4118. case KVM_MR_CREATE:
  4119. /*
  4120. * @TODO kvmppc_uvmem_memslot_create() can fail and
  4121. * return error. Fix this.
  4122. */
  4123. kvmppc_uvmem_memslot_create(kvm, new);
  4124. break;
  4125. case KVM_MR_DELETE:
  4126. kvmppc_uvmem_memslot_delete(kvm, old);
  4127. break;
  4128. default:
  4129. /* TODO: Handle KVM_MR_MOVE */
  4130. break;
  4131. }
  4132. }
  4133. /*
  4134. * Update LPCR values in kvm->arch and in vcores.
  4135. * Caller must hold kvm->arch.mmu_setup_lock (for mutual exclusion
  4136. * of kvm->arch.lpcr update).
  4137. */
  4138. void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
  4139. {
  4140. long int i;
  4141. u32 cores_done = 0;
  4142. if ((kvm->arch.lpcr & mask) == lpcr)
  4143. return;
  4144. kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
  4145. for (i = 0; i < KVM_MAX_VCORES; ++i) {
  4146. struct kvmppc_vcore *vc = kvm->arch.vcores[i];
  4147. if (!vc)
  4148. continue;
  4149. spin_lock(&vc->lock);
  4150. vc->lpcr = (vc->lpcr & ~mask) | lpcr;
  4151. spin_unlock(&vc->lock);
  4152. if (++cores_done >= kvm->arch.online_vcores)
  4153. break;
  4154. }
  4155. }
  4156. void kvmppc_setup_partition_table(struct kvm *kvm)
  4157. {
  4158. unsigned long dw0, dw1;
  4159. if (!kvm_is_radix(kvm)) {
  4160. /* PS field - page size for VRMA */
  4161. dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
  4162. ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
  4163. /* HTABSIZE and HTABORG fields */
  4164. dw0 |= kvm->arch.sdr1;
  4165. /* Second dword as set by userspace */
  4166. dw1 = kvm->arch.process_table;
  4167. } else {
  4168. dw0 = PATB_HR | radix__get_tree_size() |
  4169. __pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
  4170. dw1 = PATB_GR | kvm->arch.process_table;
  4171. }
  4172. kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
  4173. }
  4174. /*
  4175. * Set up HPT (hashed page table) and RMA (real-mode area).
  4176. * Must be called with kvm->arch.mmu_setup_lock held.
  4177. */
  4178. static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
  4179. {
  4180. int err = 0;
  4181. struct kvm *kvm = vcpu->kvm;
  4182. unsigned long hva;
  4183. struct kvm_memory_slot *memslot;
  4184. struct vm_area_struct *vma;
  4185. unsigned long lpcr = 0, senc;
  4186. unsigned long psize, porder;
  4187. int srcu_idx;
  4188. /* Allocate hashed page table (if not done already) and reset it */
  4189. if (!kvm->arch.hpt.virt) {
  4190. int order = KVM_DEFAULT_HPT_ORDER;
  4191. struct kvm_hpt_info info;
  4192. err = kvmppc_allocate_hpt(&info, order);
  4193. /* If we get here, it means userspace didn't specify a
  4194. * size explicitly. So, try successively smaller
  4195. * sizes if the default failed. */
  4196. while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
  4197. err = kvmppc_allocate_hpt(&info, order);
  4198. if (err < 0) {
  4199. pr_err("KVM: Couldn't alloc HPT\n");
  4200. goto out;
  4201. }
  4202. kvmppc_set_hpt(kvm, &info);
  4203. }
  4204. /* Look up the memslot for guest physical address 0 */
  4205. srcu_idx = srcu_read_lock(&kvm->srcu);
  4206. memslot = gfn_to_memslot(kvm, 0);
  4207. /* We must have some memory at 0 by now */
  4208. err = -EINVAL;
  4209. if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
  4210. goto out_srcu;
  4211. /* Look up the VMA for the start of this memory slot */
  4212. hva = memslot->userspace_addr;
  4213. mmap_read_lock(kvm->mm);
  4214. vma = find_vma(kvm->mm, hva);
  4215. if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
  4216. goto up_out;
  4217. psize = vma_kernel_pagesize(vma);
  4218. mmap_read_unlock(kvm->mm);
  4219. /* We can handle 4k, 64k or 16M pages in the VRMA */
  4220. if (psize >= 0x1000000)
  4221. psize = 0x1000000;
  4222. else if (psize >= 0x10000)
  4223. psize = 0x10000;
  4224. else
  4225. psize = 0x1000;
  4226. porder = __ilog2(psize);
  4227. senc = slb_pgsize_encoding(psize);
  4228. kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
  4229. (VRMA_VSID << SLB_VSID_SHIFT_1T);
  4230. /* Create HPTEs in the hash page table for the VRMA */
  4231. kvmppc_map_vrma(vcpu, memslot, porder);
  4232. /* Update VRMASD field in the LPCR */
  4233. if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
  4234. /* the -4 is to account for senc values starting at 0x10 */
  4235. lpcr = senc << (LPCR_VRMASD_SH - 4);
  4236. kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
  4237. }
  4238. /* Order updates to kvm->arch.lpcr etc. vs. mmu_ready */
  4239. smp_wmb();
  4240. err = 0;
  4241. out_srcu:
  4242. srcu_read_unlock(&kvm->srcu, srcu_idx);
  4243. out:
  4244. return err;
  4245. up_out:
  4246. mmap_read_unlock(kvm->mm);
  4247. goto out_srcu;
  4248. }
  4249. /*
  4250. * Must be called with kvm->arch.mmu_setup_lock held and
  4251. * mmu_ready = 0 and no vcpus running.
  4252. */
  4253. int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
  4254. {
  4255. if (nesting_enabled(kvm))
  4256. kvmhv_release_all_nested(kvm);
  4257. kvmppc_rmap_reset(kvm);
  4258. kvm->arch.process_table = 0;
  4259. /* Mutual exclusion with kvm_unmap_hva_range etc. */
  4260. spin_lock(&kvm->mmu_lock);
  4261. kvm->arch.radix = 0;
  4262. spin_unlock(&kvm->mmu_lock);
  4263. kvmppc_free_radix(kvm);
  4264. kvmppc_update_lpcr(kvm, LPCR_VPM1,
  4265. LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
  4266. return 0;
  4267. }
  4268. /*
  4269. * Must be called with kvm->arch.mmu_setup_lock held and
  4270. * mmu_ready = 0 and no vcpus running.
  4271. */
  4272. int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
  4273. {
  4274. int err;
  4275. err = kvmppc_init_vm_radix(kvm);
  4276. if (err)
  4277. return err;
  4278. kvmppc_rmap_reset(kvm);
  4279. /* Mutual exclusion with kvm_unmap_hva_range etc. */
  4280. spin_lock(&kvm->mmu_lock);
  4281. kvm->arch.radix = 1;
  4282. spin_unlock(&kvm->mmu_lock);
  4283. kvmppc_free_hpt(&kvm->arch.hpt);
  4284. kvmppc_update_lpcr(kvm, LPCR_UPRT | LPCR_GTSE | LPCR_HR,
  4285. LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR);
  4286. return 0;
  4287. }
  4288. #ifdef CONFIG_KVM_XICS
  4289. /*
  4290. * Allocate a per-core structure for managing state about which cores are
  4291. * running in the host versus the guest and for exchanging data between
  4292. * real mode KVM and CPU running in the host.
  4293. * This is only done for the first VM.
  4294. * The allocated structure stays even if all VMs have stopped.
  4295. * It is only freed when the kvm-hv module is unloaded.
  4296. * It's OK for this routine to fail, we just don't support host
  4297. * core operations like redirecting H_IPI wakeups.
  4298. */
  4299. void kvmppc_alloc_host_rm_ops(void)
  4300. {
  4301. struct kvmppc_host_rm_ops *ops;
  4302. unsigned long l_ops;
  4303. int cpu, core;
  4304. int size;
  4305. /* Not the first time here ? */
  4306. if (kvmppc_host_rm_ops_hv != NULL)
  4307. return;
  4308. ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
  4309. if (!ops)
  4310. return;
  4311. size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
  4312. ops->rm_core = kzalloc(size, GFP_KERNEL);
  4313. if (!ops->rm_core) {
  4314. kfree(ops);
  4315. return;
  4316. }
  4317. cpus_read_lock();
  4318. for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
  4319. if (!cpu_online(cpu))
  4320. continue;
  4321. core = cpu >> threads_shift;
  4322. ops->rm_core[core].rm_state.in_host = 1;
  4323. }
  4324. ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
  4325. /*
  4326. * Make the contents of the kvmppc_host_rm_ops structure visible
  4327. * to other CPUs before we assign it to the global variable.
  4328. * Do an atomic assignment (no locks used here), but if someone
  4329. * beats us to it, just free our copy and return.
  4330. */
  4331. smp_wmb();
  4332. l_ops = (unsigned long) ops;
  4333. if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
  4334. cpus_read_unlock();
  4335. kfree(ops->rm_core);
  4336. kfree(ops);
  4337. return;
  4338. }
  4339. cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
  4340. "ppc/kvm_book3s:prepare",
  4341. kvmppc_set_host_core,
  4342. kvmppc_clear_host_core);
  4343. cpus_read_unlock();
  4344. }
  4345. void kvmppc_free_host_rm_ops(void)
  4346. {
  4347. if (kvmppc_host_rm_ops_hv) {
  4348. cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
  4349. kfree(kvmppc_host_rm_ops_hv->rm_core);
  4350. kfree(kvmppc_host_rm_ops_hv);
  4351. kvmppc_host_rm_ops_hv = NULL;
  4352. }
  4353. }
  4354. #endif
  4355. static int kvmppc_core_init_vm_hv(struct kvm *kvm)
  4356. {
  4357. unsigned long lpcr, lpid;
  4358. char buf[32];
  4359. int ret;
  4360. mutex_init(&kvm->arch.uvmem_lock);
  4361. INIT_LIST_HEAD(&kvm->arch.uvmem_pfns);
  4362. mutex_init(&kvm->arch.mmu_setup_lock);
  4363. /* Allocate the guest's logical partition ID */
  4364. lpid = kvmppc_alloc_lpid();
  4365. if ((long)lpid < 0)
  4366. return -ENOMEM;
  4367. kvm->arch.lpid = lpid;
  4368. kvmppc_alloc_host_rm_ops();
  4369. kvmhv_vm_nested_init(kvm);
  4370. /*
  4371. * Since we don't flush the TLB when tearing down a VM,
  4372. * and this lpid might have previously been used,
  4373. * make sure we flush on each core before running the new VM.
  4374. * On POWER9, the tlbie in mmu_partition_table_set_entry()
  4375. * does this flush for us.
  4376. */
  4377. if (!cpu_has_feature(CPU_FTR_ARCH_300))
  4378. cpumask_setall(&kvm->arch.need_tlb_flush);
  4379. /* Start out with the default set of hcalls enabled */
  4380. memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
  4381. sizeof(kvm->arch.enabled_hcalls));
  4382. if (!cpu_has_feature(CPU_FTR_ARCH_300))
  4383. kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
  4384. /* Init LPCR for virtual RMA mode */
  4385. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  4386. kvm->arch.host_lpid = mfspr(SPRN_LPID);
  4387. kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
  4388. lpcr &= LPCR_PECE | LPCR_LPES;
  4389. } else {
  4390. lpcr = 0;
  4391. }
  4392. lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
  4393. LPCR_VPM0 | LPCR_VPM1;
  4394. kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
  4395. (VRMA_VSID << SLB_VSID_SHIFT_1T);
  4396. /* On POWER8 turn on online bit to enable PURR/SPURR */
  4397. if (cpu_has_feature(CPU_FTR_ARCH_207S))
  4398. lpcr |= LPCR_ONL;
  4399. /*
  4400. * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed)
  4401. * Set HVICE bit to enable hypervisor virtualization interrupts.
  4402. * Set HEIC to prevent OS interrupts to go to hypervisor (should
  4403. * be unnecessary but better safe than sorry in case we re-enable
  4404. * EE in HV mode with this LPCR still set)
  4405. */
  4406. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  4407. lpcr &= ~LPCR_VPM0;
  4408. lpcr |= LPCR_HVICE | LPCR_HEIC;
  4409. /*
  4410. * If xive is enabled, we route 0x500 interrupts directly
  4411. * to the guest.
  4412. */
  4413. if (xics_on_xive())
  4414. lpcr |= LPCR_LPES;
  4415. }
  4416. /*
  4417. * If the host uses radix, the guest starts out as radix.
  4418. */
  4419. if (radix_enabled()) {
  4420. kvm->arch.radix = 1;
  4421. kvm->arch.mmu_ready = 1;
  4422. lpcr &= ~LPCR_VPM1;
  4423. lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
  4424. ret = kvmppc_init_vm_radix(kvm);
  4425. if (ret) {
  4426. kvmppc_free_lpid(kvm->arch.lpid);
  4427. return ret;
  4428. }
  4429. kvmppc_setup_partition_table(kvm);
  4430. }
  4431. kvm->arch.lpcr = lpcr;
  4432. /* Initialization for future HPT resizes */
  4433. kvm->arch.resize_hpt = NULL;
  4434. /*
  4435. * Work out how many sets the TLB has, for the use of
  4436. * the TLB invalidation loop in book3s_hv_rmhandlers.S.
  4437. */
  4438. if (radix_enabled())
  4439. kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX; /* 128 */
  4440. else if (cpu_has_feature(CPU_FTR_ARCH_300))
  4441. kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH; /* 256 */
  4442. else if (cpu_has_feature(CPU_FTR_ARCH_207S))
  4443. kvm->arch.tlb_sets = POWER8_TLB_SETS; /* 512 */
  4444. else
  4445. kvm->arch.tlb_sets = POWER7_TLB_SETS; /* 128 */
  4446. /*
  4447. * Track that we now have a HV mode VM active. This blocks secondary
  4448. * CPU threads from coming online.
  4449. * On POWER9, we only need to do this if the "indep_threads_mode"
  4450. * module parameter has been set to N.
  4451. */
  4452. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  4453. if (!indep_threads_mode && !cpu_has_feature(CPU_FTR_HVMODE)) {
  4454. pr_warn("KVM: Ignoring indep_threads_mode=N in nested hypervisor\n");
  4455. kvm->arch.threads_indep = true;
  4456. } else {
  4457. kvm->arch.threads_indep = indep_threads_mode;
  4458. }
  4459. }
  4460. if (!kvm->arch.threads_indep)
  4461. kvm_hv_vm_activated();
  4462. /*
  4463. * Initialize smt_mode depending on processor.
  4464. * POWER8 and earlier have to use "strict" threading, where
  4465. * all vCPUs in a vcore have to run on the same (sub)core,
  4466. * whereas on POWER9 the threads can each run a different
  4467. * guest.
  4468. */
  4469. if (!cpu_has_feature(CPU_FTR_ARCH_300))
  4470. kvm->arch.smt_mode = threads_per_subcore;
  4471. else
  4472. kvm->arch.smt_mode = 1;
  4473. kvm->arch.emul_smt_mode = 1;
  4474. /*
  4475. * Create a debugfs directory for the VM
  4476. */
  4477. snprintf(buf, sizeof(buf), "vm%d", current->pid);
  4478. kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir);
  4479. kvmppc_mmu_debugfs_init(kvm);
  4480. if (radix_enabled())
  4481. kvmhv_radix_debugfs_init(kvm);
  4482. return 0;
  4483. }
  4484. static void kvmppc_free_vcores(struct kvm *kvm)
  4485. {
  4486. long int i;
  4487. for (i = 0; i < KVM_MAX_VCORES; ++i)
  4488. kfree(kvm->arch.vcores[i]);
  4489. kvm->arch.online_vcores = 0;
  4490. }
  4491. static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
  4492. {
  4493. debugfs_remove_recursive(kvm->arch.debugfs_dir);
  4494. if (!kvm->arch.threads_indep)
  4495. kvm_hv_vm_deactivated();
  4496. kvmppc_free_vcores(kvm);
  4497. if (kvm_is_radix(kvm))
  4498. kvmppc_free_radix(kvm);
  4499. else
  4500. kvmppc_free_hpt(&kvm->arch.hpt);
  4501. /* Perform global invalidation and return lpid to the pool */
  4502. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  4503. if (nesting_enabled(kvm))
  4504. kvmhv_release_all_nested(kvm);
  4505. kvm->arch.process_table = 0;
  4506. if (kvm->arch.secure_guest)
  4507. uv_svm_terminate(kvm->arch.lpid);
  4508. kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
  4509. }
  4510. kvmppc_free_lpid(kvm->arch.lpid);
  4511. kvmppc_free_pimap(kvm);
  4512. }
  4513. /* We don't need to emulate any privileged instructions or dcbz */
  4514. static int kvmppc_core_emulate_op_hv(struct kvm_vcpu *vcpu,
  4515. unsigned int inst, int *advance)
  4516. {
  4517. return EMULATE_FAIL;
  4518. }
  4519. static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
  4520. ulong spr_val)
  4521. {
  4522. return EMULATE_FAIL;
  4523. }
  4524. static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
  4525. ulong *spr_val)
  4526. {
  4527. return EMULATE_FAIL;
  4528. }
  4529. static int kvmppc_core_check_processor_compat_hv(void)
  4530. {
  4531. if (cpu_has_feature(CPU_FTR_HVMODE) &&
  4532. cpu_has_feature(CPU_FTR_ARCH_206))
  4533. return 0;
  4534. /* POWER9 in radix mode is capable of being a nested hypervisor. */
  4535. if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
  4536. return 0;
  4537. return -EIO;
  4538. }
  4539. #ifdef CONFIG_KVM_XICS
  4540. void kvmppc_free_pimap(struct kvm *kvm)
  4541. {
  4542. kfree(kvm->arch.pimap);
  4543. }
  4544. static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
  4545. {
  4546. return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
  4547. }
  4548. static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
  4549. {
  4550. struct irq_desc *desc;
  4551. struct kvmppc_irq_map *irq_map;
  4552. struct kvmppc_passthru_irqmap *pimap;
  4553. struct irq_chip *chip;
  4554. int i, rc = 0;
  4555. if (!kvm_irq_bypass)
  4556. return 1;
  4557. desc = irq_to_desc(host_irq);
  4558. if (!desc)
  4559. return -EIO;
  4560. mutex_lock(&kvm->lock);
  4561. pimap = kvm->arch.pimap;
  4562. if (pimap == NULL) {
  4563. /* First call, allocate structure to hold IRQ map */
  4564. pimap = kvmppc_alloc_pimap();
  4565. if (pimap == NULL) {
  4566. mutex_unlock(&kvm->lock);
  4567. return -ENOMEM;
  4568. }
  4569. kvm->arch.pimap = pimap;
  4570. }
  4571. /*
  4572. * For now, we only support interrupts for which the EOI operation
  4573. * is an OPAL call followed by a write to XIRR, since that's
  4574. * what our real-mode EOI code does, or a XIVE interrupt
  4575. */
  4576. chip = irq_data_get_irq_chip(&desc->irq_data);
  4577. if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) {
  4578. pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
  4579. host_irq, guest_gsi);
  4580. mutex_unlock(&kvm->lock);
  4581. return -ENOENT;
  4582. }
  4583. /*
  4584. * See if we already have an entry for this guest IRQ number.
  4585. * If it's mapped to a hardware IRQ number, that's an error,
  4586. * otherwise re-use this entry.
  4587. */
  4588. for (i = 0; i < pimap->n_mapped; i++) {
  4589. if (guest_gsi == pimap->mapped[i].v_hwirq) {
  4590. if (pimap->mapped[i].r_hwirq) {
  4591. mutex_unlock(&kvm->lock);
  4592. return -EINVAL;
  4593. }
  4594. break;
  4595. }
  4596. }
  4597. if (i == KVMPPC_PIRQ_MAPPED) {
  4598. mutex_unlock(&kvm->lock);
  4599. return -EAGAIN; /* table is full */
  4600. }
  4601. irq_map = &pimap->mapped[i];
  4602. irq_map->v_hwirq = guest_gsi;
  4603. irq_map->desc = desc;
  4604. /*
  4605. * Order the above two stores before the next to serialize with
  4606. * the KVM real mode handler.
  4607. */
  4608. smp_wmb();
  4609. irq_map->r_hwirq = desc->irq_data.hwirq;
  4610. if (i == pimap->n_mapped)
  4611. pimap->n_mapped++;
  4612. if (xics_on_xive())
  4613. rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc);
  4614. else
  4615. kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq);
  4616. if (rc)
  4617. irq_map->r_hwirq = 0;
  4618. mutex_unlock(&kvm->lock);
  4619. return 0;
  4620. }
  4621. static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
  4622. {
  4623. struct irq_desc *desc;
  4624. struct kvmppc_passthru_irqmap *pimap;
  4625. int i, rc = 0;
  4626. if (!kvm_irq_bypass)
  4627. return 0;
  4628. desc = irq_to_desc(host_irq);
  4629. if (!desc)
  4630. return -EIO;
  4631. mutex_lock(&kvm->lock);
  4632. if (!kvm->arch.pimap)
  4633. goto unlock;
  4634. pimap = kvm->arch.pimap;
  4635. for (i = 0; i < pimap->n_mapped; i++) {
  4636. if (guest_gsi == pimap->mapped[i].v_hwirq)
  4637. break;
  4638. }
  4639. if (i == pimap->n_mapped) {
  4640. mutex_unlock(&kvm->lock);
  4641. return -ENODEV;
  4642. }
  4643. if (xics_on_xive())
  4644. rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc);
  4645. else
  4646. kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
  4647. /* invalidate the entry (what do do on error from the above ?) */
  4648. pimap->mapped[i].r_hwirq = 0;
  4649. /*
  4650. * We don't free this structure even when the count goes to
  4651. * zero. The structure is freed when we destroy the VM.
  4652. */
  4653. unlock:
  4654. mutex_unlock(&kvm->lock);
  4655. return rc;
  4656. }
  4657. static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
  4658. struct irq_bypass_producer *prod)
  4659. {
  4660. int ret = 0;
  4661. struct kvm_kernel_irqfd *irqfd =
  4662. container_of(cons, struct kvm_kernel_irqfd, consumer);
  4663. irqfd->producer = prod;
  4664. ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
  4665. if (ret)
  4666. pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
  4667. prod->irq, irqfd->gsi, ret);
  4668. return ret;
  4669. }
  4670. static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
  4671. struct irq_bypass_producer *prod)
  4672. {
  4673. int ret;
  4674. struct kvm_kernel_irqfd *irqfd =
  4675. container_of(cons, struct kvm_kernel_irqfd, consumer);
  4676. irqfd->producer = NULL;
  4677. /*
  4678. * When producer of consumer is unregistered, we change back to
  4679. * default external interrupt handling mode - KVM real mode
  4680. * will switch back to host.
  4681. */
  4682. ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
  4683. if (ret)
  4684. pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
  4685. prod->irq, irqfd->gsi, ret);
  4686. }
  4687. #endif
  4688. static long kvm_arch_vm_ioctl_hv(struct file *filp,
  4689. unsigned int ioctl, unsigned long arg)
  4690. {
  4691. struct kvm *kvm __maybe_unused = filp->private_data;
  4692. void __user *argp = (void __user *)arg;
  4693. long r;
  4694. switch (ioctl) {
  4695. case KVM_PPC_ALLOCATE_HTAB: {
  4696. u32 htab_order;
  4697. /* If we're a nested hypervisor, we currently only support radix */
  4698. if (kvmhv_on_pseries()) {
  4699. r = -EOPNOTSUPP;
  4700. break;
  4701. }
  4702. r = -EFAULT;
  4703. if (get_user(htab_order, (u32 __user *)argp))
  4704. break;
  4705. r = kvmppc_alloc_reset_hpt(kvm, htab_order);
  4706. if (r)
  4707. break;
  4708. r = 0;
  4709. break;
  4710. }
  4711. case KVM_PPC_GET_HTAB_FD: {
  4712. struct kvm_get_htab_fd ghf;
  4713. r = -EFAULT;
  4714. if (copy_from_user(&ghf, argp, sizeof(ghf)))
  4715. break;
  4716. r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
  4717. break;
  4718. }
  4719. case KVM_PPC_RESIZE_HPT_PREPARE: {
  4720. struct kvm_ppc_resize_hpt rhpt;
  4721. r = -EFAULT;
  4722. if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
  4723. break;
  4724. r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
  4725. break;
  4726. }
  4727. case KVM_PPC_RESIZE_HPT_COMMIT: {
  4728. struct kvm_ppc_resize_hpt rhpt;
  4729. r = -EFAULT;
  4730. if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
  4731. break;
  4732. r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
  4733. break;
  4734. }
  4735. default:
  4736. r = -ENOTTY;
  4737. }
  4738. return r;
  4739. }
  4740. /*
  4741. * List of hcall numbers to enable by default.
  4742. * For compatibility with old userspace, we enable by default
  4743. * all hcalls that were implemented before the hcall-enabling
  4744. * facility was added. Note this list should not include H_RTAS.
  4745. */
  4746. static unsigned int default_hcall_list[] = {
  4747. H_REMOVE,
  4748. H_ENTER,
  4749. H_READ,
  4750. H_PROTECT,
  4751. H_BULK_REMOVE,
  4752. H_GET_TCE,
  4753. H_PUT_TCE,
  4754. H_SET_DABR,
  4755. H_SET_XDABR,
  4756. H_CEDE,
  4757. H_PROD,
  4758. H_CONFER,
  4759. H_REGISTER_VPA,
  4760. #ifdef CONFIG_KVM_XICS
  4761. H_EOI,
  4762. H_CPPR,
  4763. H_IPI,
  4764. H_IPOLL,
  4765. H_XIRR,
  4766. H_XIRR_X,
  4767. #endif
  4768. 0
  4769. };
  4770. static void init_default_hcalls(void)
  4771. {
  4772. int i;
  4773. unsigned int hcall;
  4774. for (i = 0; default_hcall_list[i]; ++i) {
  4775. hcall = default_hcall_list[i];
  4776. WARN_ON(!kvmppc_hcall_impl_hv(hcall));
  4777. __set_bit(hcall / 4, default_enabled_hcalls);
  4778. }
  4779. }
  4780. static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
  4781. {
  4782. unsigned long lpcr;
  4783. int radix;
  4784. int err;
  4785. /* If not on a POWER9, reject it */
  4786. if (!cpu_has_feature(CPU_FTR_ARCH_300))
  4787. return -ENODEV;
  4788. /* If any unknown flags set, reject it */
  4789. if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
  4790. return -EINVAL;
  4791. /* GR (guest radix) bit in process_table field must match */
  4792. radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
  4793. if (!!(cfg->process_table & PATB_GR) != radix)
  4794. return -EINVAL;
  4795. /* Process table size field must be reasonable, i.e. <= 24 */
  4796. if ((cfg->process_table & PRTS_MASK) > 24)
  4797. return -EINVAL;
  4798. /* We can change a guest to/from radix now, if the host is radix */
  4799. if (radix && !radix_enabled())
  4800. return -EINVAL;
  4801. /* If we're a nested hypervisor, we currently only support radix */
  4802. if (kvmhv_on_pseries() && !radix)
  4803. return -EINVAL;
  4804. mutex_lock(&kvm->arch.mmu_setup_lock);
  4805. if (radix != kvm_is_radix(kvm)) {
  4806. if (kvm->arch.mmu_ready) {
  4807. kvm->arch.mmu_ready = 0;
  4808. /* order mmu_ready vs. vcpus_running */
  4809. smp_mb();
  4810. if (atomic_read(&kvm->arch.vcpus_running)) {
  4811. kvm->arch.mmu_ready = 1;
  4812. err = -EBUSY;
  4813. goto out_unlock;
  4814. }
  4815. }
  4816. if (radix)
  4817. err = kvmppc_switch_mmu_to_radix(kvm);
  4818. else
  4819. err = kvmppc_switch_mmu_to_hpt(kvm);
  4820. if (err)
  4821. goto out_unlock;
  4822. }
  4823. kvm->arch.process_table = cfg->process_table;
  4824. kvmppc_setup_partition_table(kvm);
  4825. lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
  4826. kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
  4827. err = 0;
  4828. out_unlock:
  4829. mutex_unlock(&kvm->arch.mmu_setup_lock);
  4830. return err;
  4831. }
  4832. static int kvmhv_enable_nested(struct kvm *kvm)
  4833. {
  4834. if (!nested)
  4835. return -EPERM;
  4836. if (!cpu_has_feature(CPU_FTR_ARCH_300) || no_mixing_hpt_and_radix)
  4837. return -ENODEV;
  4838. /* kvm == NULL means the caller is testing if the capability exists */
  4839. if (kvm)
  4840. kvm->arch.nested_enable = true;
  4841. return 0;
  4842. }
  4843. static int kvmhv_load_from_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
  4844. int size)
  4845. {
  4846. int rc = -EINVAL;
  4847. if (kvmhv_vcpu_is_radix(vcpu)) {
  4848. rc = kvmhv_copy_from_guest_radix(vcpu, *eaddr, ptr, size);
  4849. if (rc > 0)
  4850. rc = -EINVAL;
  4851. }
  4852. /* For now quadrants are the only way to access nested guest memory */
  4853. if (rc && vcpu->arch.nested)
  4854. rc = -EAGAIN;
  4855. return rc;
  4856. }
  4857. static int kvmhv_store_to_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
  4858. int size)
  4859. {
  4860. int rc = -EINVAL;
  4861. if (kvmhv_vcpu_is_radix(vcpu)) {
  4862. rc = kvmhv_copy_to_guest_radix(vcpu, *eaddr, ptr, size);
  4863. if (rc > 0)
  4864. rc = -EINVAL;
  4865. }
  4866. /* For now quadrants are the only way to access nested guest memory */
  4867. if (rc && vcpu->arch.nested)
  4868. rc = -EAGAIN;
  4869. return rc;
  4870. }
  4871. static void unpin_vpa_reset(struct kvm *kvm, struct kvmppc_vpa *vpa)
  4872. {
  4873. unpin_vpa(kvm, vpa);
  4874. vpa->gpa = 0;
  4875. vpa->pinned_addr = NULL;
  4876. vpa->dirty = false;
  4877. vpa->update_pending = 0;
  4878. }
  4879. /*
  4880. * Enable a guest to become a secure VM, or test whether
  4881. * that could be enabled.
  4882. * Called when the KVM_CAP_PPC_SECURE_GUEST capability is
  4883. * tested (kvm == NULL) or enabled (kvm != NULL).
  4884. */
  4885. static int kvmhv_enable_svm(struct kvm *kvm)
  4886. {
  4887. if (!kvmppc_uvmem_available())
  4888. return -EINVAL;
  4889. if (kvm)
  4890. kvm->arch.svm_enabled = 1;
  4891. return 0;
  4892. }
  4893. /*
  4894. * IOCTL handler to turn off secure mode of guest
  4895. *
  4896. * - Release all device pages
  4897. * - Issue ucall to terminate the guest on the UV side
  4898. * - Unpin the VPA pages.
  4899. * - Reinit the partition scoped page tables
  4900. */
  4901. static int kvmhv_svm_off(struct kvm *kvm)
  4902. {
  4903. struct kvm_vcpu *vcpu;
  4904. int mmu_was_ready;
  4905. int srcu_idx;
  4906. int ret = 0;
  4907. int i;
  4908. if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
  4909. return ret;
  4910. mutex_lock(&kvm->arch.mmu_setup_lock);
  4911. mmu_was_ready = kvm->arch.mmu_ready;
  4912. if (kvm->arch.mmu_ready) {
  4913. kvm->arch.mmu_ready = 0;
  4914. /* order mmu_ready vs. vcpus_running */
  4915. smp_mb();
  4916. if (atomic_read(&kvm->arch.vcpus_running)) {
  4917. kvm->arch.mmu_ready = 1;
  4918. ret = -EBUSY;
  4919. goto out;
  4920. }
  4921. }
  4922. srcu_idx = srcu_read_lock(&kvm->srcu);
  4923. for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
  4924. struct kvm_memory_slot *memslot;
  4925. struct kvm_memslots *slots = __kvm_memslots(kvm, i);
  4926. if (!slots)
  4927. continue;
  4928. kvm_for_each_memslot(memslot, slots) {
  4929. kvmppc_uvmem_drop_pages(memslot, kvm, true);
  4930. uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
  4931. }
  4932. }
  4933. srcu_read_unlock(&kvm->srcu, srcu_idx);
  4934. ret = uv_svm_terminate(kvm->arch.lpid);
  4935. if (ret != U_SUCCESS) {
  4936. ret = -EINVAL;
  4937. goto out;
  4938. }
  4939. /*
  4940. * When secure guest is reset, all the guest pages are sent
  4941. * to UV via UV_PAGE_IN before the non-boot vcpus get a
  4942. * chance to run and unpin their VPA pages. Unpinning of all
  4943. * VPA pages is done here explicitly so that VPA pages
  4944. * can be migrated to the secure side.
  4945. *
  4946. * This is required to for the secure SMP guest to reboot
  4947. * correctly.
  4948. */
  4949. kvm_for_each_vcpu(i, vcpu, kvm) {
  4950. spin_lock(&vcpu->arch.vpa_update_lock);
  4951. unpin_vpa_reset(kvm, &vcpu->arch.dtl);
  4952. unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
  4953. unpin_vpa_reset(kvm, &vcpu->arch.vpa);
  4954. spin_unlock(&vcpu->arch.vpa_update_lock);
  4955. }
  4956. kvmppc_setup_partition_table(kvm);
  4957. kvm->arch.secure_guest = 0;
  4958. kvm->arch.mmu_ready = mmu_was_ready;
  4959. out:
  4960. mutex_unlock(&kvm->arch.mmu_setup_lock);
  4961. return ret;
  4962. }
  4963. static struct kvmppc_ops kvm_ops_hv = {
  4964. .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
  4965. .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
  4966. .get_one_reg = kvmppc_get_one_reg_hv,
  4967. .set_one_reg = kvmppc_set_one_reg_hv,
  4968. .vcpu_load = kvmppc_core_vcpu_load_hv,
  4969. .vcpu_put = kvmppc_core_vcpu_put_hv,
  4970. .inject_interrupt = kvmppc_inject_interrupt_hv,
  4971. .set_msr = kvmppc_set_msr_hv,
  4972. .vcpu_run = kvmppc_vcpu_run_hv,
  4973. .vcpu_create = kvmppc_core_vcpu_create_hv,
  4974. .vcpu_free = kvmppc_core_vcpu_free_hv,
  4975. .check_requests = kvmppc_core_check_requests_hv,
  4976. .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv,
  4977. .flush_memslot = kvmppc_core_flush_memslot_hv,
  4978. .prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
  4979. .commit_memory_region = kvmppc_core_commit_memory_region_hv,
  4980. .unmap_hva_range = kvm_unmap_hva_range_hv,
  4981. .age_hva = kvm_age_hva_hv,
  4982. .test_age_hva = kvm_test_age_hva_hv,
  4983. .set_spte_hva = kvm_set_spte_hva_hv,
  4984. .free_memslot = kvmppc_core_free_memslot_hv,
  4985. .init_vm = kvmppc_core_init_vm_hv,
  4986. .destroy_vm = kvmppc_core_destroy_vm_hv,
  4987. .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
  4988. .emulate_op = kvmppc_core_emulate_op_hv,
  4989. .emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
  4990. .emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
  4991. .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
  4992. .arch_vm_ioctl = kvm_arch_vm_ioctl_hv,
  4993. .hcall_implemented = kvmppc_hcall_impl_hv,
  4994. #ifdef CONFIG_KVM_XICS
  4995. .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
  4996. .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
  4997. #endif
  4998. .configure_mmu = kvmhv_configure_mmu,
  4999. .get_rmmu_info = kvmhv_get_rmmu_info,
  5000. .set_smt_mode = kvmhv_set_smt_mode,
  5001. .enable_nested = kvmhv_enable_nested,
  5002. .load_from_eaddr = kvmhv_load_from_eaddr,
  5003. .store_to_eaddr = kvmhv_store_to_eaddr,
  5004. .enable_svm = kvmhv_enable_svm,
  5005. .svm_off = kvmhv_svm_off,
  5006. };
  5007. static int kvm_init_subcore_bitmap(void)
  5008. {
  5009. int i, j;
  5010. int nr_cores = cpu_nr_cores();
  5011. struct sibling_subcore_state *sibling_subcore_state;
  5012. for (i = 0; i < nr_cores; i++) {
  5013. int first_cpu = i * threads_per_core;
  5014. int node = cpu_to_node(first_cpu);
  5015. /* Ignore if it is already allocated. */
  5016. if (paca_ptrs[first_cpu]->sibling_subcore_state)
  5017. continue;
  5018. sibling_subcore_state =
  5019. kzalloc_node(sizeof(struct sibling_subcore_state),
  5020. GFP_KERNEL, node);
  5021. if (!sibling_subcore_state)
  5022. return -ENOMEM;
  5023. for (j = 0; j < threads_per_core; j++) {
  5024. int cpu = first_cpu + j;
  5025. paca_ptrs[cpu]->sibling_subcore_state =
  5026. sibling_subcore_state;
  5027. }
  5028. }
  5029. return 0;
  5030. }
  5031. static int kvmppc_radix_possible(void)
  5032. {
  5033. return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
  5034. }
  5035. static int kvmppc_book3s_init_hv(void)
  5036. {
  5037. int r;
  5038. if (!tlbie_capable) {
  5039. pr_err("KVM-HV: Host does not support TLBIE\n");
  5040. return -ENODEV;
  5041. }
  5042. /*
  5043. * FIXME!! Do we need to check on all cpus ?
  5044. */
  5045. r = kvmppc_core_check_processor_compat_hv();
  5046. if (r < 0)
  5047. return -ENODEV;
  5048. r = kvmhv_nested_init();
  5049. if (r)
  5050. return r;
  5051. r = kvm_init_subcore_bitmap();
  5052. if (r)
  5053. return r;
  5054. /*
  5055. * We need a way of accessing the XICS interrupt controller,
  5056. * either directly, via paca_ptrs[cpu]->kvm_hstate.xics_phys, or
  5057. * indirectly, via OPAL.
  5058. */
  5059. #ifdef CONFIG_SMP
  5060. if (!xics_on_xive() && !kvmhv_on_pseries() &&
  5061. !local_paca->kvm_hstate.xics_phys) {
  5062. struct device_node *np;
  5063. np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
  5064. if (!np) {
  5065. pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
  5066. return -ENODEV;
  5067. }
  5068. /* presence of intc confirmed - node can be dropped again */
  5069. of_node_put(np);
  5070. }
  5071. #endif
  5072. kvm_ops_hv.owner = THIS_MODULE;
  5073. kvmppc_hv_ops = &kvm_ops_hv;
  5074. init_default_hcalls();
  5075. init_vcore_lists();
  5076. r = kvmppc_mmu_hv_init();
  5077. if (r)
  5078. return r;
  5079. if (kvmppc_radix_possible()) {
  5080. r = kvmppc_radix_init();
  5081. if (r)
  5082. return r;
  5083. }
  5084. /*
  5085. * POWER9 chips before version 2.02 can't have some threads in
  5086. * HPT mode and some in radix mode on the same core.
  5087. */
  5088. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  5089. unsigned int pvr = mfspr(SPRN_PVR);
  5090. if ((pvr >> 16) == PVR_POWER9 &&
  5091. (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
  5092. ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
  5093. no_mixing_hpt_and_radix = true;
  5094. }
  5095. r = kvmppc_uvmem_init();
  5096. if (r < 0)
  5097. pr_err("KVM-HV: kvmppc_uvmem_init failed %d\n", r);
  5098. return r;
  5099. }
  5100. static void kvmppc_book3s_exit_hv(void)
  5101. {
  5102. kvmppc_uvmem_free();
  5103. kvmppc_free_host_rm_ops();
  5104. if (kvmppc_radix_possible())
  5105. kvmppc_radix_exit();
  5106. kvmppc_hv_ops = NULL;
  5107. kvmhv_nested_exit();
  5108. }
  5109. module_init(kvmppc_book3s_init_hv);
  5110. module_exit(kvmppc_book3s_exit_hv);
  5111. MODULE_LICENSE("GPL");
  5112. MODULE_ALIAS_MISCDEV(KVM_MINOR);
  5113. MODULE_ALIAS("devname:kvm");