exceptions-64s.S 90 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * This file contains the 64-bit "server" PowerPC variant
  4. * of the low level exception handling including exception
  5. * vectors, exception return, part of the slb and stab
  6. * handling and other fixed offset specific things.
  7. *
  8. * This file is meant to be #included from head_64.S due to
  9. * position dependent assembly.
  10. *
  11. * Most of this originates from head_64.S and thus has the same
  12. * copyright history.
  13. *
  14. */
  15. #include <asm/hw_irq.h>
  16. #include <asm/exception-64s.h>
  17. #include <asm/ptrace.h>
  18. #include <asm/cpuidle.h>
  19. #include <asm/head-64.h>
  20. #include <asm/feature-fixups.h>
  21. #include <asm/kup.h>
  22. /* PACA save area offsets (exgen, exmc, etc) */
  23. #define EX_R9 0
  24. #define EX_R10 8
  25. #define EX_R11 16
  26. #define EX_R12 24
  27. #define EX_R13 32
  28. #define EX_DAR 40
  29. #define EX_DSISR 48
  30. #define EX_CCR 52
  31. #define EX_CFAR 56
  32. #define EX_PPR 64
  33. #define EX_CTR 72
  34. .if EX_SIZE != 10
  35. .error "EX_SIZE is wrong"
  36. .endif
  37. /*
  38. * Following are fixed section helper macros.
  39. *
  40. * EXC_REAL_BEGIN/END - real, unrelocated exception vectors
  41. * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors
  42. * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these)
  43. * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use)
  44. * EXC_COMMON - After switching to virtual, relocated mode.
  45. */
  46. #define EXC_REAL_BEGIN(name, start, size) \
  47. FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
  48. #define EXC_REAL_END(name, start, size) \
  49. FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
  50. #define EXC_VIRT_BEGIN(name, start, size) \
  51. FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
  52. #define EXC_VIRT_END(name, start, size) \
  53. FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
  54. #define EXC_COMMON_BEGIN(name) \
  55. USE_TEXT_SECTION(); \
  56. .balign IFETCH_ALIGN_BYTES; \
  57. .global name; \
  58. _ASM_NOKPROBE_SYMBOL(name); \
  59. DEFINE_FIXED_SYMBOL(name); \
  60. name:
  61. #define TRAMP_REAL_BEGIN(name) \
  62. FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name)
  63. #define TRAMP_VIRT_BEGIN(name) \
  64. FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
  65. #define EXC_REAL_NONE(start, size) \
  66. FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \
  67. FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size)
  68. #define EXC_VIRT_NONE(start, size) \
  69. FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \
  70. FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size)
  71. /*
  72. * We're short on space and time in the exception prolog, so we can't
  73. * use the normal LOAD_REG_IMMEDIATE macro to load the address of label.
  74. * Instead we get the base of the kernel from paca->kernelbase and or in the low
  75. * part of label. This requires that the label be within 64KB of kernelbase, and
  76. * that kernelbase be 64K aligned.
  77. */
  78. #define LOAD_HANDLER(reg, label) \
  79. ld reg,PACAKBASE(r13); /* get high part of &label */ \
  80. ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label)
  81. #define __LOAD_HANDLER(reg, label) \
  82. ld reg,PACAKBASE(r13); \
  83. ori reg,reg,(ABS_ADDR(label))@l
  84. /*
  85. * Branches from unrelocated code (e.g., interrupts) to labels outside
  86. * head-y require >64K offsets.
  87. */
  88. #define __LOAD_FAR_HANDLER(reg, label) \
  89. ld reg,PACAKBASE(r13); \
  90. ori reg,reg,(ABS_ADDR(label))@l; \
  91. addis reg,reg,(ABS_ADDR(label))@h
  92. /*
  93. * Branch to label using its 0xC000 address. This results in instruction
  94. * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
  95. * on using mtmsr rather than rfid.
  96. *
  97. * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
  98. * load KBASE for a slight optimisation.
  99. */
  100. #define BRANCH_TO_C000(reg, label) \
  101. __LOAD_FAR_HANDLER(reg, label); \
  102. mtctr reg; \
  103. bctr
  104. /*
  105. * Interrupt code generation macros
  106. */
  107. #define IVEC .L_IVEC_\name\() /* Interrupt vector address */
  108. #define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */
  109. #define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */
  110. #define IAREA .L_IAREA_\name\() /* PACA save area */
  111. #define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */
  112. #define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */
  113. #define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */
  114. #define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */
  115. #define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */
  116. #define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */
  117. #define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */
  118. #define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */
  119. #define IKVM_SKIP .L_IKVM_SKIP_\name\() /* Generate KVM skip handler */
  120. #define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */
  121. #define __IKVM_REAL(name) .L_IKVM_REAL_ ## name
  122. #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */
  123. #define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */
  124. #define __ISTACK(name) .L_ISTACK_ ## name
  125. #define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */
  126. #define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */
  127. #define INT_DEFINE_BEGIN(n) \
  128. .macro int_define_ ## n name
  129. #define INT_DEFINE_END(n) \
  130. .endm ; \
  131. int_define_ ## n n ; \
  132. do_define_int n
  133. .macro do_define_int name
  134. .ifndef IVEC
  135. .error "IVEC not defined"
  136. .endif
  137. .ifndef IHSRR
  138. IHSRR=0
  139. .endif
  140. .ifndef IHSRR_IF_HVMODE
  141. IHSRR_IF_HVMODE=0
  142. .endif
  143. .ifndef IAREA
  144. IAREA=PACA_EXGEN
  145. .endif
  146. .ifndef IVIRT
  147. IVIRT=1
  148. .endif
  149. .ifndef IISIDE
  150. IISIDE=0
  151. .endif
  152. .ifndef IDAR
  153. IDAR=0
  154. .endif
  155. .ifndef IDSISR
  156. IDSISR=0
  157. .endif
  158. .ifndef ISET_RI
  159. ISET_RI=1
  160. .endif
  161. .ifndef IBRANCH_TO_COMMON
  162. IBRANCH_TO_COMMON=1
  163. .endif
  164. .ifndef IREALMODE_COMMON
  165. IREALMODE_COMMON=0
  166. .else
  167. .if ! IBRANCH_TO_COMMON
  168. .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0"
  169. .endif
  170. .endif
  171. .ifndef IMASK
  172. IMASK=0
  173. .endif
  174. .ifndef IKVM_SKIP
  175. IKVM_SKIP=0
  176. .endif
  177. .ifndef IKVM_REAL
  178. IKVM_REAL=0
  179. .endif
  180. .ifndef IKVM_VIRT
  181. IKVM_VIRT=0
  182. .endif
  183. .ifndef ISTACK
  184. ISTACK=1
  185. .endif
  186. .ifndef IRECONCILE
  187. IRECONCILE=1
  188. .endif
  189. .ifndef IKUAP
  190. IKUAP=1
  191. .endif
  192. .endm
  193. #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
  194. #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
  195. /*
  196. * All interrupts which set HSRR registers, as well as SRESET and MCE and
  197. * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken,
  198. * so they all generally need to test whether they were taken in guest context.
  199. *
  200. * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be
  201. * taken with MSR[HV]=0.
  202. *
  203. * Interrupts which set SRR registers (with the above exceptions) do not
  204. * elevate to MSR[HV]=1 mode, though most can be taken when running with
  205. * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do
  206. * not need to test whether a guest is running because they get delivered to
  207. * the guest directly, including nested HV KVM guests.
  208. *
  209. * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host
  210. * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the
  211. * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be
  212. * delivered to the real-mode entry point, therefore such interrupts only test
  213. * KVM in their real mode handlers, and only when PR KVM is possible.
  214. *
  215. * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always
  216. * delivered in real-mode when the MMU is in hash mode because the MMU
  217. * registers are not set appropriately to translate host addresses. In nested
  218. * radix mode these can be delivered in virt-mode as the host translations are
  219. * used implicitly (see: effective LPID, effective PID).
  220. */
  221. /*
  222. * If an interrupt is taken while a guest is running, it is immediately routed
  223. * to KVM to handle. If both HV and PR KVM arepossible, KVM interrupts go first
  224. * to kvmppc_interrupt_hv, which handles the PR guest case.
  225. */
  226. #define kvmppc_interrupt kvmppc_interrupt_hv
  227. #else
  228. #define kvmppc_interrupt kvmppc_interrupt_pr
  229. #endif
  230. .macro KVMTEST name
  231. lbz r10,HSTATE_IN_GUEST(r13)
  232. cmpwi r10,0
  233. bne \name\()_kvm
  234. .endm
  235. .macro GEN_KVM name
  236. .balign IFETCH_ALIGN_BYTES
  237. \name\()_kvm:
  238. .if IKVM_SKIP
  239. cmpwi r10,KVM_GUEST_MODE_SKIP
  240. beq 89f
  241. .else
  242. BEGIN_FTR_SECTION
  243. ld r10,IAREA+EX_CFAR(r13)
  244. std r10,HSTATE_CFAR(r13)
  245. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  246. .endif
  247. ld r10,IAREA+EX_CTR(r13)
  248. mtctr r10
  249. BEGIN_FTR_SECTION
  250. ld r10,IAREA+EX_PPR(r13)
  251. std r10,HSTATE_PPR(r13)
  252. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  253. ld r11,IAREA+EX_R11(r13)
  254. ld r12,IAREA+EX_R12(r13)
  255. std r12,HSTATE_SCRATCH0(r13)
  256. sldi r12,r9,32
  257. ld r9,IAREA+EX_R9(r13)
  258. ld r10,IAREA+EX_R10(r13)
  259. /* HSRR variants have the 0x2 bit added to their trap number */
  260. .if IHSRR_IF_HVMODE
  261. BEGIN_FTR_SECTION
  262. ori r12,r12,(IVEC + 0x2)
  263. FTR_SECTION_ELSE
  264. ori r12,r12,(IVEC)
  265. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  266. .elseif IHSRR
  267. ori r12,r12,(IVEC+ 0x2)
  268. .else
  269. ori r12,r12,(IVEC)
  270. .endif
  271. b kvmppc_interrupt
  272. .if IKVM_SKIP
  273. 89: mtocrf 0x80,r9
  274. ld r10,IAREA+EX_CTR(r13)
  275. mtctr r10
  276. ld r9,IAREA+EX_R9(r13)
  277. ld r10,IAREA+EX_R10(r13)
  278. ld r11,IAREA+EX_R11(r13)
  279. ld r12,IAREA+EX_R12(r13)
  280. .if IHSRR_IF_HVMODE
  281. BEGIN_FTR_SECTION
  282. b kvmppc_skip_Hinterrupt
  283. FTR_SECTION_ELSE
  284. b kvmppc_skip_interrupt
  285. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  286. .elseif IHSRR
  287. b kvmppc_skip_Hinterrupt
  288. .else
  289. b kvmppc_skip_interrupt
  290. .endif
  291. .endif
  292. .endm
  293. #else
  294. .macro KVMTEST name
  295. .endm
  296. .macro GEN_KVM name
  297. .endm
  298. #endif
  299. /*
  300. * This is the BOOK3S interrupt entry code macro.
  301. *
  302. * This can result in one of several things happening:
  303. * - Branch to the _common handler, relocated, in virtual mode.
  304. * These are normal interrupts (synchronous and asynchronous) handled by
  305. * the kernel.
  306. * - Branch to KVM, relocated but real mode interrupts remain in real mode.
  307. * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by
  308. * / intended for host or guest kernel, but KVM must always be involved
  309. * because the machine state is set for guest execution.
  310. * - Branch to the masked handler, unrelocated.
  311. * These occur when maskable asynchronous interrupts are taken with the
  312. * irq_soft_mask set.
  313. * - Branch to an "early" handler in real mode but relocated.
  314. * This is done if early=1. MCE and HMI use these to handle errors in real
  315. * mode.
  316. * - Fall through and continue executing in real, unrelocated mode.
  317. * This is done if early=2.
  318. */
  319. .macro GEN_BRANCH_TO_COMMON name, virt
  320. .if IREALMODE_COMMON
  321. LOAD_HANDLER(r10, \name\()_common)
  322. mtctr r10
  323. bctr
  324. .else
  325. .if \virt
  326. #ifndef CONFIG_RELOCATABLE
  327. b \name\()_common_virt
  328. #else
  329. LOAD_HANDLER(r10, \name\()_common_virt)
  330. mtctr r10
  331. bctr
  332. #endif
  333. .else
  334. LOAD_HANDLER(r10, \name\()_common_real)
  335. mtctr r10
  336. bctr
  337. .endif
  338. .endif
  339. .endm
  340. .macro GEN_INT_ENTRY name, virt, ool=0
  341. SET_SCRATCH0(r13) /* save r13 */
  342. GET_PACA(r13)
  343. std r9,IAREA+EX_R9(r13) /* save r9 */
  344. BEGIN_FTR_SECTION
  345. mfspr r9,SPRN_PPR
  346. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  347. HMT_MEDIUM
  348. std r10,IAREA+EX_R10(r13) /* save r10 - r12 */
  349. BEGIN_FTR_SECTION
  350. mfspr r10,SPRN_CFAR
  351. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  352. .if \ool
  353. .if !\virt
  354. b tramp_real_\name
  355. .pushsection .text
  356. TRAMP_REAL_BEGIN(tramp_real_\name)
  357. .else
  358. b tramp_virt_\name
  359. .pushsection .text
  360. TRAMP_VIRT_BEGIN(tramp_virt_\name)
  361. .endif
  362. .endif
  363. BEGIN_FTR_SECTION
  364. std r9,IAREA+EX_PPR(r13)
  365. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  366. BEGIN_FTR_SECTION
  367. std r10,IAREA+EX_CFAR(r13)
  368. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  369. INTERRUPT_TO_KERNEL
  370. mfctr r10
  371. std r10,IAREA+EX_CTR(r13)
  372. mfcr r9
  373. std r11,IAREA+EX_R11(r13)
  374. std r12,IAREA+EX_R12(r13)
  375. /*
  376. * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],
  377. * because a d-side MCE will clobber those registers so is
  378. * not recoverable if they are live.
  379. */
  380. GET_SCRATCH0(r10)
  381. std r10,IAREA+EX_R13(r13)
  382. .if IDAR && !IISIDE
  383. .if IHSRR
  384. mfspr r10,SPRN_HDAR
  385. .else
  386. mfspr r10,SPRN_DAR
  387. .endif
  388. std r10,IAREA+EX_DAR(r13)
  389. .endif
  390. .if IDSISR && !IISIDE
  391. .if IHSRR
  392. mfspr r10,SPRN_HDSISR
  393. .else
  394. mfspr r10,SPRN_DSISR
  395. .endif
  396. stw r10,IAREA+EX_DSISR(r13)
  397. .endif
  398. .if IHSRR_IF_HVMODE
  399. BEGIN_FTR_SECTION
  400. mfspr r11,SPRN_HSRR0 /* save HSRR0 */
  401. mfspr r12,SPRN_HSRR1 /* and HSRR1 */
  402. FTR_SECTION_ELSE
  403. mfspr r11,SPRN_SRR0 /* save SRR0 */
  404. mfspr r12,SPRN_SRR1 /* and SRR1 */
  405. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  406. .elseif IHSRR
  407. mfspr r11,SPRN_HSRR0 /* save HSRR0 */
  408. mfspr r12,SPRN_HSRR1 /* and HSRR1 */
  409. .else
  410. mfspr r11,SPRN_SRR0 /* save SRR0 */
  411. mfspr r12,SPRN_SRR1 /* and SRR1 */
  412. .endif
  413. .if IBRANCH_TO_COMMON
  414. GEN_BRANCH_TO_COMMON \name \virt
  415. .endif
  416. .if \ool
  417. .popsection
  418. .endif
  419. .endm
  420. /*
  421. * __GEN_COMMON_ENTRY is required to receive the branch from interrupt
  422. * entry, except in the case of the real-mode handlers which require
  423. * __GEN_REALMODE_COMMON_ENTRY.
  424. *
  425. * This switches to virtual mode and sets MSR[RI].
  426. */
  427. .macro __GEN_COMMON_ENTRY name
  428. DEFINE_FIXED_SYMBOL(\name\()_common_real)
  429. \name\()_common_real:
  430. .if IKVM_REAL
  431. KVMTEST \name
  432. .endif
  433. ld r10,PACAKMSR(r13) /* get MSR value for kernel */
  434. /* MSR[RI] is clear iff using SRR regs */
  435. .if IHSRR_IF_HVMODE
  436. BEGIN_FTR_SECTION
  437. xori r10,r10,MSR_RI
  438. END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
  439. .elseif ! IHSRR
  440. xori r10,r10,MSR_RI
  441. .endif
  442. mtmsrd r10
  443. .if IVIRT
  444. .if IKVM_VIRT
  445. b 1f /* skip the virt test coming from real */
  446. .endif
  447. .balign IFETCH_ALIGN_BYTES
  448. DEFINE_FIXED_SYMBOL(\name\()_common_virt)
  449. \name\()_common_virt:
  450. .if IKVM_VIRT
  451. KVMTEST \name
  452. 1:
  453. .endif
  454. .endif /* IVIRT */
  455. .endm
  456. /*
  457. * Don't switch to virt mode. Used for early MCE and HMI handlers that
  458. * want to run in real mode.
  459. */
  460. .macro __GEN_REALMODE_COMMON_ENTRY name
  461. DEFINE_FIXED_SYMBOL(\name\()_common_real)
  462. \name\()_common_real:
  463. .if IKVM_REAL
  464. KVMTEST \name
  465. .endif
  466. .endm
  467. .macro __GEN_COMMON_BODY name
  468. .if IMASK
  469. .if ! ISTACK
  470. .error "No support for masked interrupt to use custom stack"
  471. .endif
  472. /* If coming from user, skip soft-mask tests. */
  473. andi. r10,r12,MSR_PR
  474. bne 2f
  475. /* Kernel code running below __end_interrupts is implicitly
  476. * soft-masked */
  477. LOAD_HANDLER(r10, __end_interrupts)
  478. cmpld r11,r10
  479. li r10,IMASK
  480. blt- 1f
  481. /* Test the soft mask state against our interrupt's bit */
  482. lbz r10,PACAIRQSOFTMASK(r13)
  483. 1: andi. r10,r10,IMASK
  484. /* Associate vector numbers with bits in paca->irq_happened */
  485. .if IVEC == 0x500 || IVEC == 0xea0
  486. li r10,PACA_IRQ_EE
  487. .elseif IVEC == 0x900
  488. li r10,PACA_IRQ_DEC
  489. .elseif IVEC == 0xa00 || IVEC == 0xe80
  490. li r10,PACA_IRQ_DBELL
  491. .elseif IVEC == 0xe60
  492. li r10,PACA_IRQ_HMI
  493. .elseif IVEC == 0xf00
  494. li r10,PACA_IRQ_PMI
  495. .else
  496. .abort "Bad maskable vector"
  497. .endif
  498. .if IHSRR_IF_HVMODE
  499. BEGIN_FTR_SECTION
  500. bne masked_Hinterrupt
  501. FTR_SECTION_ELSE
  502. bne masked_interrupt
  503. ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  504. .elseif IHSRR
  505. bne masked_Hinterrupt
  506. .else
  507. bne masked_interrupt
  508. .endif
  509. .endif
  510. .if ISTACK
  511. andi. r10,r12,MSR_PR /* See if coming from user */
  512. 2: mr r10,r1 /* Save r1 */
  513. subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */
  514. beq- 100f
  515. ld r1,PACAKSAVE(r13) /* kernel stack to use */
  516. 100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */
  517. EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0
  518. .endif
  519. std r9,_CCR(r1) /* save CR in stackframe */
  520. std r11,_NIP(r1) /* save SRR0 in stackframe */
  521. std r12,_MSR(r1) /* save SRR1 in stackframe */
  522. std r10,0(r1) /* make stack chain pointer */
  523. std r0,GPR0(r1) /* save r0 in stackframe */
  524. std r10,GPR1(r1) /* save r1 in stackframe */
  525. .if ISET_RI
  526. li r10,MSR_RI
  527. mtmsrd r10,1 /* Set MSR_RI */
  528. .endif
  529. .if ISTACK
  530. .if IKUAP
  531. kuap_save_amr_and_lock r9, r10, cr1, cr0
  532. .endif
  533. beq 101f /* if from kernel mode */
  534. ACCOUNT_CPU_USER_ENTRY(r13, r9, r10)
  535. BEGIN_FTR_SECTION
  536. ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */
  537. std r9,_PPR(r1)
  538. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  539. 101:
  540. .else
  541. .if IKUAP
  542. kuap_save_amr_and_lock r9, r10, cr1
  543. .endif
  544. .endif
  545. /* Save original regs values from save area to stack frame. */
  546. ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */
  547. ld r10,IAREA+EX_R10(r13)
  548. std r9,GPR9(r1)
  549. std r10,GPR10(r1)
  550. ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */
  551. ld r10,IAREA+EX_R12(r13)
  552. ld r11,IAREA+EX_R13(r13)
  553. std r9,GPR11(r1)
  554. std r10,GPR12(r1)
  555. std r11,GPR13(r1)
  556. SAVE_NVGPRS(r1)
  557. .if IDAR
  558. .if IISIDE
  559. ld r10,_NIP(r1)
  560. .else
  561. ld r10,IAREA+EX_DAR(r13)
  562. .endif
  563. std r10,_DAR(r1)
  564. .endif
  565. .if IDSISR
  566. .if IISIDE
  567. ld r10,_MSR(r1)
  568. lis r11,DSISR_SRR1_MATCH_64S@h
  569. and r10,r10,r11
  570. .else
  571. lwz r10,IAREA+EX_DSISR(r13)
  572. .endif
  573. std r10,_DSISR(r1)
  574. .endif
  575. BEGIN_FTR_SECTION
  576. ld r10,IAREA+EX_CFAR(r13)
  577. std r10,ORIG_GPR3(r1)
  578. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  579. ld r10,IAREA+EX_CTR(r13)
  580. std r10,_CTR(r1)
  581. std r2,GPR2(r1) /* save r2 in stackframe */
  582. SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */
  583. SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */
  584. mflr r9 /* Get LR, later save to stack */
  585. ld r2,PACATOC(r13) /* get kernel TOC into r2 */
  586. std r9,_LINK(r1)
  587. lbz r10,PACAIRQSOFTMASK(r13)
  588. mfspr r11,SPRN_XER /* save XER in stackframe */
  589. std r10,SOFTE(r1)
  590. std r11,_XER(r1)
  591. li r9,IVEC
  592. std r9,_TRAP(r1) /* set trap number */
  593. li r10,0
  594. ld r11,exception_marker@toc(r2)
  595. std r10,RESULT(r1) /* clear regs->result */
  596. std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */
  597. .if ISTACK
  598. ACCOUNT_STOLEN_TIME
  599. .endif
  600. .if IRECONCILE
  601. RECONCILE_IRQ_STATE(r10, r11)
  602. .endif
  603. .endm
  604. /*
  605. * On entry r13 points to the paca, r9-r13 are saved in the paca,
  606. * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
  607. * SRR1, and relocation is on.
  608. *
  609. * If stack=0, then the stack is already set in r1, and r1 is saved in r10.
  610. * PPR save and CPU accounting is not done for the !stack case (XXX why not?)
  611. */
  612. .macro GEN_COMMON name
  613. __GEN_COMMON_ENTRY \name
  614. __GEN_COMMON_BODY \name
  615. .endm
  616. /*
  617. * Restore all registers including H/SRR0/1 saved in a stack frame of a
  618. * standard exception.
  619. */
  620. .macro EXCEPTION_RESTORE_REGS hsrr=0
  621. /* Move original SRR0 and SRR1 into the respective regs */
  622. ld r9,_MSR(r1)
  623. .if \hsrr
  624. mtspr SPRN_HSRR1,r9
  625. .else
  626. mtspr SPRN_SRR1,r9
  627. .endif
  628. ld r9,_NIP(r1)
  629. .if \hsrr
  630. mtspr SPRN_HSRR0,r9
  631. .else
  632. mtspr SPRN_SRR0,r9
  633. .endif
  634. ld r9,_CTR(r1)
  635. mtctr r9
  636. ld r9,_XER(r1)
  637. mtxer r9
  638. ld r9,_LINK(r1)
  639. mtlr r9
  640. ld r9,_CCR(r1)
  641. mtcr r9
  642. REST_8GPRS(2, r1)
  643. REST_4GPRS(10, r1)
  644. REST_GPR(0, r1)
  645. /* restore original r1. */
  646. ld r1,GPR1(r1)
  647. .endm
  648. #define RUNLATCH_ON \
  649. BEGIN_FTR_SECTION \
  650. ld r3, PACA_THREAD_INFO(r13); \
  651. ld r4,TI_LOCAL_FLAGS(r3); \
  652. andi. r0,r4,_TLF_RUNLATCH; \
  653. beql ppc64_runlatch_on_trampoline; \
  654. END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
  655. /*
  656. * When the idle code in power4_idle puts the CPU into NAP mode,
  657. * it has to do so in a loop, and relies on the external interrupt
  658. * and decrementer interrupt entry code to get it out of the loop.
  659. * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
  660. * to signal that it is in the loop and needs help to get out.
  661. */
  662. #ifdef CONFIG_PPC_970_NAP
  663. #define FINISH_NAP \
  664. BEGIN_FTR_SECTION \
  665. ld r11, PACA_THREAD_INFO(r13); \
  666. ld r9,TI_LOCAL_FLAGS(r11); \
  667. andi. r10,r9,_TLF_NAPPING; \
  668. bnel power4_fixup_nap; \
  669. END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
  670. #else
  671. #define FINISH_NAP
  672. #endif
  673. /*
  674. * There are a few constraints to be concerned with.
  675. * - Real mode exceptions code/data must be located at their physical location.
  676. * - Virtual mode exceptions must be mapped at their 0xc000... location.
  677. * - Fixed location code must not call directly beyond the __end_interrupts
  678. * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence
  679. * must be used.
  680. * - LOAD_HANDLER targets must be within first 64K of physical 0 /
  681. * virtual 0xc00...
  682. * - Conditional branch targets must be within +/-32K of caller.
  683. *
  684. * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and
  685. * therefore don't have to run in physically located code or rfid to
  686. * virtual mode kernel code. However on relocatable kernels they do have
  687. * to branch to KERNELBASE offset because the rest of the kernel (outside
  688. * the exception vectors) may be located elsewhere.
  689. *
  690. * Virtual exceptions correspond with physical, except their entry points
  691. * are offset by 0xc000000000000000 and also tend to get an added 0x4000
  692. * offset applied. Virtual exceptions are enabled with the Alternate
  693. * Interrupt Location (AIL) bit set in the LPCR. However this does not
  694. * guarantee they will be delivered virtually. Some conditions (see the ISA)
  695. * cause exceptions to be delivered in real mode.
  696. *
  697. * The scv instructions are a special case. They get a 0x3000 offset applied.
  698. * scv exceptions have unique reentrancy properties, see below.
  699. *
  700. * It's impossible to receive interrupts below 0x300 via AIL.
  701. *
  702. * KVM: None of the virtual exceptions are from the guest. Anything that
  703. * escalated to HV=1 from HV=0 is delivered via real mode handlers.
  704. *
  705. *
  706. * We layout physical memory as follows:
  707. * 0x0000 - 0x00ff : Secondary processor spin code
  708. * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
  709. * 0x1900 - 0x2fff : Real mode trampolines
  710. * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
  711. * 0x5900 - 0x6fff : Relon mode trampolines
  712. * 0x7000 - 0x7fff : FWNMI data area
  713. * 0x8000 - .... : Common interrupt handlers, remaining early
  714. * setup code, rest of kernel.
  715. *
  716. * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
  717. * is necessary. Until then it's more consistent to explicitly put VIRT_NONE
  718. * vectors there.
  719. */
  720. OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900)
  721. OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000)
  722. OPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900)
  723. OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000)
  724. #ifdef CONFIG_PPC_POWERNV
  725. .globl start_real_trampolines
  726. .globl end_real_trampolines
  727. .globl start_virt_trampolines
  728. .globl end_virt_trampolines
  729. #endif
  730. #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
  731. /*
  732. * Data area reserved for FWNMI option.
  733. * This address (0x7000) is fixed by the RPA.
  734. * pseries and powernv need to keep the whole page from
  735. * 0x7000 to 0x8000 free for use by the firmware
  736. */
  737. ZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000)
  738. OPEN_TEXT_SECTION(0x8000)
  739. #else
  740. OPEN_TEXT_SECTION(0x7000)
  741. #endif
  742. USE_FIXED_SECTION(real_vectors)
  743. /*
  744. * This is the start of the interrupt handlers for pSeries
  745. * This code runs with relocation off.
  746. * Code from here to __end_interrupts gets copied down to real
  747. * address 0x100 when we are running a relocatable kernel.
  748. * Therefore any relative branches in this section must only
  749. * branch to labels in this section.
  750. */
  751. .globl __start_interrupts
  752. __start_interrupts:
  753. /**
  754. * Interrupt 0x3000 - System Call Vectored Interrupt (syscall).
  755. * This is a synchronous interrupt invoked with the "scv" instruction. The
  756. * system call does not alter the HV bit, so it is directed to the OS.
  757. *
  758. * Handling:
  759. * scv instructions enter the kernel without changing EE, RI, ME, or HV.
  760. * In particular, this means we can take a maskable interrupt at any point
  761. * in the scv handler, which is unlike any other interrupt. This is solved
  762. * by treating the instruction addresses below __end_interrupts as being
  763. * soft-masked.
  764. *
  765. * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and
  766. * ensure scv is never executed with relocation off, which means AIL-0
  767. * should never happen.
  768. *
  769. * Before leaving the below __end_interrupts text, at least of the following
  770. * must be true:
  771. * - MSR[PR]=1 (i.e., return to userspace)
  772. * - MSR_EE|MSR_RI is set (no reentrant exceptions)
  773. * - Standard kernel environment is set up (stack, paca, etc)
  774. *
  775. * Call convention:
  776. *
  777. * syscall register convention is in Documentation/powerpc/syscall64-abi.rst
  778. */
  779. EXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000)
  780. /* SCV 0 */
  781. mr r9,r13
  782. GET_PACA(r13)
  783. mflr r11
  784. mfctr r12
  785. li r10,IRQS_ALL_DISABLED
  786. stb r10,PACAIRQSOFTMASK(r13)
  787. #ifdef CONFIG_RELOCATABLE
  788. b system_call_vectored_tramp
  789. #else
  790. b system_call_vectored_common
  791. #endif
  792. nop
  793. /* SCV 1 - 127 */
  794. .rept 127
  795. mr r9,r13
  796. GET_PACA(r13)
  797. mflr r11
  798. mfctr r12
  799. li r10,IRQS_ALL_DISABLED
  800. stb r10,PACAIRQSOFTMASK(r13)
  801. li r0,-1 /* cause failure */
  802. #ifdef CONFIG_RELOCATABLE
  803. b system_call_vectored_sigill_tramp
  804. #else
  805. b system_call_vectored_sigill
  806. #endif
  807. .endr
  808. EXC_VIRT_END(system_call_vectored, 0x3000, 0x1000)
  809. #ifdef CONFIG_RELOCATABLE
  810. TRAMP_VIRT_BEGIN(system_call_vectored_tramp)
  811. __LOAD_HANDLER(r10, system_call_vectored_common)
  812. mtctr r10
  813. bctr
  814. TRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp)
  815. __LOAD_HANDLER(r10, system_call_vectored_sigill)
  816. mtctr r10
  817. bctr
  818. #endif
  819. /* No virt vectors corresponding with 0x0..0x100 */
  820. EXC_VIRT_NONE(0x4000, 0x100)
  821. /**
  822. * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI).
  823. * This is a non-maskable, asynchronous interrupt always taken in real-mode.
  824. * It is caused by:
  825. * - Wake from power-saving state, on powernv.
  826. * - An NMI from another CPU, triggered by firmware or hypercall.
  827. * - As crash/debug signal injected from BMC, firmware or hypervisor.
  828. *
  829. * Handling:
  830. * Power-save wakeup is the only performance critical path, so this is
  831. * determined quickly as possible first. In this case volatile registers
  832. * can be discarded and SPRs like CFAR don't need to be read.
  833. *
  834. * If not a powersave wakeup, then it's run as a regular interrupt, however
  835. * it uses its own stack and PACA save area to preserve the regular kernel
  836. * environment for debugging.
  837. *
  838. * This interrupt is not maskable, so triggering it when MSR[RI] is clear,
  839. * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely
  840. * correct to switch to virtual mode to run the regular interrupt handler
  841. * because it might be interrupted when the MMU is in a bad state (e.g., SLB
  842. * is clear).
  843. *
  844. * FWNMI:
  845. * PAPR specifies a "fwnmi" facility which sends the sreset to a different
  846. * entry point with a different register set up. Some hypervisors will
  847. * send the sreset to 0x100 in the guest if it is not fwnmi capable.
  848. *
  849. * KVM:
  850. * Unlike most SRR interrupts, this may be taken by the host while executing
  851. * in a guest, so a KVM test is required. KVM will pull the CPU out of guest
  852. * mode and then raise the sreset.
  853. */
  854. INT_DEFINE_BEGIN(system_reset)
  855. IVEC=0x100
  856. IAREA=PACA_EXNMI
  857. IVIRT=0 /* no virt entry point */
  858. /*
  859. * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is
  860. * being used, so a nested NMI exception would corrupt it.
  861. */
  862. ISET_RI=0
  863. ISTACK=0
  864. IRECONCILE=0
  865. IKVM_REAL=1
  866. INT_DEFINE_END(system_reset)
  867. EXC_REAL_BEGIN(system_reset, 0x100, 0x100)
  868. #ifdef CONFIG_PPC_P7_NAP
  869. /*
  870. * If running native on arch 2.06 or later, check if we are waking up
  871. * from nap/sleep/winkle, and branch to idle handler. This tests SRR1
  872. * bits 46:47. A non-0 value indicates that we are coming from a power
  873. * saving state. The idle wakeup handler initially runs in real mode,
  874. * but we branch to the 0xc000... address so we can turn on relocation
  875. * with mtmsrd later, after SPRs are restored.
  876. *
  877. * Careful to minimise cost for the fast path (idle wakeup) while
  878. * also avoiding clobbering CFAR for the debug path (non-idle).
  879. *
  880. * For the idle wake case volatile registers can be clobbered, which
  881. * is why we use those initially. If it turns out to not be an idle
  882. * wake, carefully put everything back the way it was, so we can use
  883. * common exception macros to handle it.
  884. */
  885. BEGIN_FTR_SECTION
  886. SET_SCRATCH0(r13)
  887. GET_PACA(r13)
  888. std r3,PACA_EXNMI+0*8(r13)
  889. std r4,PACA_EXNMI+1*8(r13)
  890. std r5,PACA_EXNMI+2*8(r13)
  891. mfspr r3,SPRN_SRR1
  892. mfocrf r4,0x80
  893. rlwinm. r5,r3,47-31,30,31
  894. bne+ system_reset_idle_wake
  895. /* Not powersave wakeup. Restore regs for regular interrupt handler. */
  896. mtocrf 0x80,r4
  897. ld r3,PACA_EXNMI+0*8(r13)
  898. ld r4,PACA_EXNMI+1*8(r13)
  899. ld r5,PACA_EXNMI+2*8(r13)
  900. GET_SCRATCH0(r13)
  901. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  902. #endif
  903. GEN_INT_ENTRY system_reset, virt=0
  904. /*
  905. * In theory, we should not enable relocation here if it was disabled
  906. * in SRR1, because the MMU may not be configured to support it (e.g.,
  907. * SLB may have been cleared). In practice, there should only be a few
  908. * small windows where that's the case, and sreset is considered to
  909. * be dangerous anyway.
  910. */
  911. EXC_REAL_END(system_reset, 0x100, 0x100)
  912. EXC_VIRT_NONE(0x4100, 0x100)
  913. #ifdef CONFIG_PPC_P7_NAP
  914. TRAMP_REAL_BEGIN(system_reset_idle_wake)
  915. /* We are waking up from idle, so may clobber any volatile register */
  916. cmpwi cr1,r5,2
  917. bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */
  918. BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss))
  919. #endif
  920. #ifdef CONFIG_PPC_PSERIES
  921. /*
  922. * Vectors for the FWNMI option. Share common code.
  923. */
  924. TRAMP_REAL_BEGIN(system_reset_fwnmi)
  925. GEN_INT_ENTRY system_reset, virt=0
  926. #endif /* CONFIG_PPC_PSERIES */
  927. EXC_COMMON_BEGIN(system_reset_common)
  928. __GEN_COMMON_ENTRY system_reset
  929. /*
  930. * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able
  931. * to recover, but nested NMI will notice in_nmi and not recover
  932. * because of the use of the NMI stack. in_nmi reentrancy is tested in
  933. * system_reset_exception.
  934. */
  935. lhz r10,PACA_IN_NMI(r13)
  936. addi r10,r10,1
  937. sth r10,PACA_IN_NMI(r13)
  938. li r10,MSR_RI
  939. mtmsrd r10,1
  940. mr r10,r1
  941. ld r1,PACA_NMI_EMERG_SP(r13)
  942. subi r1,r1,INT_FRAME_SIZE
  943. __GEN_COMMON_BODY system_reset
  944. /*
  945. * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
  946. * the right thing. We do not want to reconcile because that goes
  947. * through irq tracing which we don't want in NMI.
  948. *
  949. * Save PACAIRQHAPPENED to RESULT (otherwise unused), and set HARD_DIS
  950. * as we are running with MSR[EE]=0.
  951. */
  952. li r10,IRQS_ALL_DISABLED
  953. stb r10,PACAIRQSOFTMASK(r13)
  954. lbz r10,PACAIRQHAPPENED(r13)
  955. std r10,RESULT(r1)
  956. ori r10,r10,PACA_IRQ_HARD_DIS
  957. stb r10,PACAIRQHAPPENED(r13)
  958. addi r3,r1,STACK_FRAME_OVERHEAD
  959. bl system_reset_exception
  960. /* Clear MSR_RI before setting SRR0 and SRR1. */
  961. li r9,0
  962. mtmsrd r9,1
  963. /*
  964. * MSR_RI is clear, now we can decrement paca->in_nmi.
  965. */
  966. lhz r10,PACA_IN_NMI(r13)
  967. subi r10,r10,1
  968. sth r10,PACA_IN_NMI(r13)
  969. /*
  970. * Restore soft mask settings.
  971. */
  972. ld r10,RESULT(r1)
  973. stb r10,PACAIRQHAPPENED(r13)
  974. ld r10,SOFTE(r1)
  975. stb r10,PACAIRQSOFTMASK(r13)
  976. kuap_restore_amr r9, r10
  977. EXCEPTION_RESTORE_REGS
  978. RFI_TO_USER_OR_KERNEL
  979. GEN_KVM system_reset
  980. /**
  981. * Interrupt 0x200 - Machine Check Interrupt (MCE).
  982. * This is a non-maskable interrupt always taken in real-mode. It can be
  983. * synchronous or asynchronous, caused by hardware or software, and it may be
  984. * taken in a power-saving state.
  985. *
  986. * Handling:
  987. * Similarly to system reset, this uses its own stack and PACA save area,
  988. * the difference is re-entrancy is allowed on the machine check stack.
  989. *
  990. * machine_check_early is run in real mode, and carefully decodes the
  991. * machine check and tries to handle it (e.g., flush the SLB if there was an
  992. * error detected there), determines if it was recoverable and logs the
  993. * event.
  994. *
  995. * This early code does not "reconcile" irq soft-mask state like SRESET or
  996. * regular interrupts do, so irqs_disabled() among other things may not work
  997. * properly (irq disable/enable already doesn't work because irq tracing can
  998. * not work in real mode).
  999. *
  1000. * Then, depending on the execution context when the interrupt is taken, there
  1001. * are 3 main actions:
  1002. * - Executing in kernel mode. The event is queued with irq_work, which means
  1003. * it is handled when it is next safe to do so (i.e., the kernel has enabled
  1004. * interrupts), which could be immediately when the interrupt returns. This
  1005. * avoids nasty issues like switching to virtual mode when the MMU is in a
  1006. * bad state, or when executing OPAL code. (SRESET is exposed to such issues,
  1007. * but it has different priorities). Check to see if the CPU was in power
  1008. * save, and return via the wake up code if it was.
  1009. *
  1010. * - Executing in user mode. machine_check_exception is run like a normal
  1011. * interrupt handler, which processes the data generated by the early handler.
  1012. *
  1013. * - Executing in guest mode. The interrupt is run with its KVM test, and
  1014. * branches to KVM to deal with. KVM may queue the event for the host
  1015. * to report later.
  1016. *
  1017. * This interrupt is not maskable, so if it triggers when MSR[RI] is clear,
  1018. * or SCRATCH0 is in use, it may cause a crash.
  1019. *
  1020. * KVM:
  1021. * See SRESET.
  1022. */
  1023. INT_DEFINE_BEGIN(machine_check_early)
  1024. IVEC=0x200
  1025. IAREA=PACA_EXMC
  1026. IVIRT=0 /* no virt entry point */
  1027. IREALMODE_COMMON=1
  1028. /*
  1029. * MSR_RI is not enabled, because PACA_EXMC is being used, so a
  1030. * nested machine check corrupts it. machine_check_common enables
  1031. * MSR_RI.
  1032. */
  1033. ISET_RI=0
  1034. ISTACK=0
  1035. IDAR=1
  1036. IDSISR=1
  1037. IRECONCILE=0
  1038. IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
  1039. INT_DEFINE_END(machine_check_early)
  1040. INT_DEFINE_BEGIN(machine_check)
  1041. IVEC=0x200
  1042. IAREA=PACA_EXMC
  1043. IVIRT=0 /* no virt entry point */
  1044. ISET_RI=0
  1045. IDAR=1
  1046. IDSISR=1
  1047. IKVM_SKIP=1
  1048. IKVM_REAL=1
  1049. INT_DEFINE_END(machine_check)
  1050. EXC_REAL_BEGIN(machine_check, 0x200, 0x100)
  1051. GEN_INT_ENTRY machine_check_early, virt=0
  1052. EXC_REAL_END(machine_check, 0x200, 0x100)
  1053. EXC_VIRT_NONE(0x4200, 0x100)
  1054. #ifdef CONFIG_PPC_PSERIES
  1055. TRAMP_REAL_BEGIN(machine_check_fwnmi)
  1056. /* See comment at machine_check exception, don't turn on RI */
  1057. GEN_INT_ENTRY machine_check_early, virt=0
  1058. #endif
  1059. #define MACHINE_CHECK_HANDLER_WINDUP \
  1060. /* Clear MSR_RI before setting SRR0 and SRR1. */\
  1061. li r9,0; \
  1062. mtmsrd r9,1; /* Clear MSR_RI */ \
  1063. /* Decrement paca->in_mce now RI is clear. */ \
  1064. lhz r12,PACA_IN_MCE(r13); \
  1065. subi r12,r12,1; \
  1066. sth r12,PACA_IN_MCE(r13); \
  1067. EXCEPTION_RESTORE_REGS
  1068. EXC_COMMON_BEGIN(machine_check_early_common)
  1069. __GEN_REALMODE_COMMON_ENTRY machine_check_early
  1070. /*
  1071. * Switch to mc_emergency stack and handle re-entrancy (we limit
  1072. * the nested MCE upto level 4 to avoid stack overflow).
  1073. * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
  1074. *
  1075. * We use paca->in_mce to check whether this is the first entry or
  1076. * nested machine check. We increment paca->in_mce to track nested
  1077. * machine checks.
  1078. *
  1079. * If this is the first entry then set stack pointer to
  1080. * paca->mc_emergency_sp, otherwise r1 is already pointing to
  1081. * stack frame on mc_emergency stack.
  1082. *
  1083. * NOTE: We are here with MSR_ME=0 (off), which means we risk a
  1084. * checkstop if we get another machine check exception before we do
  1085. * rfid with MSR_ME=1.
  1086. *
  1087. * This interrupt can wake directly from idle. If that is the case,
  1088. * the machine check is handled then the idle wakeup code is called
  1089. * to restore state.
  1090. */
  1091. lhz r10,PACA_IN_MCE(r13)
  1092. cmpwi r10,0 /* Are we in nested machine check */
  1093. cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */
  1094. addi r10,r10,1 /* increment paca->in_mce */
  1095. sth r10,PACA_IN_MCE(r13)
  1096. mr r10,r1 /* Save r1 */
  1097. bne 1f
  1098. /* First machine check entry */
  1099. ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */
  1100. 1: /* Limit nested MCE to level 4 to avoid stack overflow */
  1101. bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */
  1102. subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
  1103. __GEN_COMMON_BODY machine_check_early
  1104. BEGIN_FTR_SECTION
  1105. bl enable_machine_check
  1106. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
  1107. li r10,MSR_RI
  1108. mtmsrd r10,1
  1109. /*
  1110. * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see
  1111. * system_reset_common)
  1112. */
  1113. li r10,IRQS_ALL_DISABLED
  1114. stb r10,PACAIRQSOFTMASK(r13)
  1115. lbz r10,PACAIRQHAPPENED(r13)
  1116. std r10,RESULT(r1)
  1117. ori r10,r10,PACA_IRQ_HARD_DIS
  1118. stb r10,PACAIRQHAPPENED(r13)
  1119. addi r3,r1,STACK_FRAME_OVERHEAD
  1120. bl machine_check_early
  1121. std r3,RESULT(r1) /* Save result */
  1122. ld r12,_MSR(r1)
  1123. /*
  1124. * Restore soft mask settings.
  1125. */
  1126. ld r10,RESULT(r1)
  1127. stb r10,PACAIRQHAPPENED(r13)
  1128. ld r10,SOFTE(r1)
  1129. stb r10,PACAIRQSOFTMASK(r13)
  1130. #ifdef CONFIG_PPC_P7_NAP
  1131. /*
  1132. * Check if thread was in power saving mode. We come here when any
  1133. * of the following is true:
  1134. * a. thread wasn't in power saving mode
  1135. * b. thread was in power saving mode with no state loss,
  1136. * supervisor state loss or hypervisor state loss.
  1137. *
  1138. * Go back to nap/sleep/winkle mode again if (b) is true.
  1139. */
  1140. BEGIN_FTR_SECTION
  1141. rlwinm. r11,r12,47-31,30,31
  1142. bne machine_check_idle_common
  1143. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
  1144. #endif
  1145. #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
  1146. /*
  1147. * Check if we are coming from guest. If yes, then run the normal
  1148. * exception handler which will take the
  1149. * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event
  1150. * to guest.
  1151. */
  1152. lbz r11,HSTATE_IN_GUEST(r13)
  1153. cmpwi r11,0 /* Check if coming from guest */
  1154. bne mce_deliver /* continue if we are. */
  1155. #endif
  1156. /*
  1157. * Check if we are coming from userspace. If yes, then run the normal
  1158. * exception handler which will deliver the MC event to this kernel.
  1159. */
  1160. andi. r11,r12,MSR_PR /* See if coming from user. */
  1161. bne mce_deliver /* continue in V mode if we are. */
  1162. /*
  1163. * At this point we are coming from kernel context.
  1164. * Queue up the MCE event and return from the interrupt.
  1165. * But before that, check if this is an un-recoverable exception.
  1166. * If yes, then stay on emergency stack and panic.
  1167. */
  1168. andi. r11,r12,MSR_RI
  1169. beq unrecoverable_mce
  1170. /*
  1171. * Check if we have successfully handled/recovered from error, if not
  1172. * then stay on emergency stack and panic.
  1173. */
  1174. ld r3,RESULT(r1) /* Load result */
  1175. cmpdi r3,0 /* see if we handled MCE successfully */
  1176. beq unrecoverable_mce /* if !handled then panic */
  1177. /*
  1178. * Return from MC interrupt.
  1179. * Queue up the MCE event so that we can log it later, while
  1180. * returning from kernel or opal call.
  1181. */
  1182. bl machine_check_queue_event
  1183. MACHINE_CHECK_HANDLER_WINDUP
  1184. RFI_TO_KERNEL
  1185. mce_deliver:
  1186. /*
  1187. * This is a host user or guest MCE. Restore all registers, then
  1188. * run the "late" handler. For host user, this will run the
  1189. * machine_check_exception handler in virtual mode like a normal
  1190. * interrupt handler. For guest, this will trigger the KVM test
  1191. * and branch to the KVM interrupt similarly to other interrupts.
  1192. */
  1193. BEGIN_FTR_SECTION
  1194. ld r10,ORIG_GPR3(r1)
  1195. mtspr SPRN_CFAR,r10
  1196. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  1197. MACHINE_CHECK_HANDLER_WINDUP
  1198. GEN_INT_ENTRY machine_check, virt=0
  1199. EXC_COMMON_BEGIN(machine_check_common)
  1200. /*
  1201. * Machine check is different because we use a different
  1202. * save area: PACA_EXMC instead of PACA_EXGEN.
  1203. */
  1204. GEN_COMMON machine_check
  1205. FINISH_NAP
  1206. /* Enable MSR_RI when finished with PACA_EXMC */
  1207. li r10,MSR_RI
  1208. mtmsrd r10,1
  1209. addi r3,r1,STACK_FRAME_OVERHEAD
  1210. bl machine_check_exception
  1211. b interrupt_return
  1212. GEN_KVM machine_check
  1213. #ifdef CONFIG_PPC_P7_NAP
  1214. /*
  1215. * This is an idle wakeup. Low level machine check has already been
  1216. * done. Queue the event then call the idle code to do the wake up.
  1217. */
  1218. EXC_COMMON_BEGIN(machine_check_idle_common)
  1219. bl machine_check_queue_event
  1220. /*
  1221. * GPR-loss wakeups are relatively straightforward, because the
  1222. * idle sleep code has saved all non-volatile registers on its
  1223. * own stack, and r1 in PACAR1.
  1224. *
  1225. * For no-loss wakeups the r1 and lr registers used by the
  1226. * early machine check handler have to be restored first. r2 is
  1227. * the kernel TOC, so no need to restore it.
  1228. *
  1229. * Then decrement MCE nesting after finishing with the stack.
  1230. */
  1231. ld r3,_MSR(r1)
  1232. ld r4,_LINK(r1)
  1233. ld r1,GPR1(r1)
  1234. lhz r11,PACA_IN_MCE(r13)
  1235. subi r11,r11,1
  1236. sth r11,PACA_IN_MCE(r13)
  1237. mtlr r4
  1238. rlwinm r10,r3,47-31,30,31
  1239. cmpwi cr1,r10,2
  1240. bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */
  1241. b idle_return_gpr_loss
  1242. #endif
  1243. EXC_COMMON_BEGIN(unrecoverable_mce)
  1244. /*
  1245. * We are going down. But there are chances that we might get hit by
  1246. * another MCE during panic path and we may run into unstable state
  1247. * with no way out. Hence, turn ME bit off while going down, so that
  1248. * when another MCE is hit during panic path, system will checkstop
  1249. * and hypervisor will get restarted cleanly by SP.
  1250. */
  1251. BEGIN_FTR_SECTION
  1252. li r10,0 /* clear MSR_RI */
  1253. mtmsrd r10,1
  1254. bl disable_machine_check
  1255. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
  1256. ld r10,PACAKMSR(r13)
  1257. li r3,MSR_ME
  1258. andc r10,r10,r3
  1259. mtmsrd r10
  1260. lhz r12,PACA_IN_MCE(r13)
  1261. subi r12,r12,1
  1262. sth r12,PACA_IN_MCE(r13)
  1263. /* Invoke machine_check_exception to print MCE event and panic. */
  1264. addi r3,r1,STACK_FRAME_OVERHEAD
  1265. bl machine_check_exception
  1266. /*
  1267. * We will not reach here. Even if we did, there is no way out.
  1268. * Call unrecoverable_exception and die.
  1269. */
  1270. addi r3,r1,STACK_FRAME_OVERHEAD
  1271. bl unrecoverable_exception
  1272. b .
  1273. /**
  1274. * Interrupt 0x300 - Data Storage Interrupt (DSI).
  1275. * This is a synchronous interrupt generated due to a data access exception,
  1276. * e.g., a load orstore which does not have a valid page table entry with
  1277. * permissions. DAWR matches also fault here, as do RC updates, and minor misc
  1278. * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc.
  1279. *
  1280. * Handling:
  1281. * - Hash MMU
  1282. * Go to do_hash_page first to see if the HPT can be filled from an entry in
  1283. * the Linux page table. Hash faults can hit in kernel mode in a fairly
  1284. * arbitrary state (e.g., interrupts disabled, locks held) when accessing
  1285. * "non-bolted" regions, e.g., vmalloc space. However these should always be
  1286. * backed by Linux page tables.
  1287. *
  1288. * If none is found, do a Linux page fault. Linux page faults can happen in
  1289. * kernel mode due to user copy operations of course.
  1290. *
  1291. * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest
  1292. * MMU context, which may cause a DSI in the host, which must go to the
  1293. * KVM handler. MSR[IR] is not enabled, so the real-mode handler will
  1294. * always be used regardless of AIL setting.
  1295. *
  1296. * - Radix MMU
  1297. * The hardware loads from the Linux page table directly, so a fault goes
  1298. * immediately to Linux page fault.
  1299. *
  1300. * Conditions like DAWR match are handled on the way in to Linux page fault.
  1301. */
  1302. INT_DEFINE_BEGIN(data_access)
  1303. IVEC=0x300
  1304. IDAR=1
  1305. IDSISR=1
  1306. IKVM_SKIP=1
  1307. IKVM_REAL=1
  1308. INT_DEFINE_END(data_access)
  1309. EXC_REAL_BEGIN(data_access, 0x300, 0x80)
  1310. GEN_INT_ENTRY data_access, virt=0
  1311. EXC_REAL_END(data_access, 0x300, 0x80)
  1312. EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
  1313. GEN_INT_ENTRY data_access, virt=1
  1314. EXC_VIRT_END(data_access, 0x4300, 0x80)
  1315. EXC_COMMON_BEGIN(data_access_common)
  1316. GEN_COMMON data_access
  1317. ld r4,_DAR(r1)
  1318. ld r5,_DSISR(r1)
  1319. BEGIN_MMU_FTR_SECTION
  1320. ld r6,_MSR(r1)
  1321. li r3,0x300
  1322. b do_hash_page /* Try to handle as hpte fault */
  1323. MMU_FTR_SECTION_ELSE
  1324. b handle_page_fault
  1325. ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
  1326. GEN_KVM data_access
  1327. /**
  1328. * Interrupt 0x380 - Data Segment Interrupt (DSLB).
  1329. * This is a synchronous interrupt in response to an MMU fault missing SLB
  1330. * entry for HPT, or an address outside RPT translation range.
  1331. *
  1332. * Handling:
  1333. * - HPT:
  1334. * This refills the SLB, or reports an access fault similarly to a bad page
  1335. * fault. When coming from user-mode, the SLB handler may access any kernel
  1336. * data, though it may itself take a DSLB. When coming from kernel mode,
  1337. * recursive faults must be avoided so access is restricted to the kernel
  1338. * image text/data, kernel stack, and any data allocated below
  1339. * ppc64_bolted_size (first segment). The kernel handler must avoid stomping
  1340. * on user-handler data structures.
  1341. *
  1342. * KVM: Same as 0x300, DSLB must test for KVM guest.
  1343. *
  1344. * A dedicated save area EXSLB is used (XXX: but it actually need not be
  1345. * these days, we could use EXGEN).
  1346. */
  1347. INT_DEFINE_BEGIN(data_access_slb)
  1348. IVEC=0x380
  1349. IAREA=PACA_EXSLB
  1350. IRECONCILE=0
  1351. IDAR=1
  1352. IKVM_SKIP=1
  1353. IKVM_REAL=1
  1354. INT_DEFINE_END(data_access_slb)
  1355. EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
  1356. GEN_INT_ENTRY data_access_slb, virt=0
  1357. EXC_REAL_END(data_access_slb, 0x380, 0x80)
  1358. EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
  1359. GEN_INT_ENTRY data_access_slb, virt=1
  1360. EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
  1361. EXC_COMMON_BEGIN(data_access_slb_common)
  1362. GEN_COMMON data_access_slb
  1363. ld r4,_DAR(r1)
  1364. addi r3,r1,STACK_FRAME_OVERHEAD
  1365. BEGIN_MMU_FTR_SECTION
  1366. /* HPT case, do SLB fault */
  1367. bl do_slb_fault
  1368. cmpdi r3,0
  1369. bne- 1f
  1370. b fast_interrupt_return
  1371. 1: /* Error case */
  1372. MMU_FTR_SECTION_ELSE
  1373. /* Radix case, access is outside page table range */
  1374. li r3,-EFAULT
  1375. ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
  1376. std r3,RESULT(r1)
  1377. RECONCILE_IRQ_STATE(r10, r11)
  1378. ld r4,_DAR(r1)
  1379. ld r5,RESULT(r1)
  1380. addi r3,r1,STACK_FRAME_OVERHEAD
  1381. bl do_bad_slb_fault
  1382. b interrupt_return
  1383. GEN_KVM data_access_slb
  1384. /**
  1385. * Interrupt 0x400 - Instruction Storage Interrupt (ISI).
  1386. * This is a synchronous interrupt in response to an MMU fault due to an
  1387. * instruction fetch.
  1388. *
  1389. * Handling:
  1390. * Similar to DSI, though in response to fetch. The faulting address is found
  1391. * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR).
  1392. */
  1393. INT_DEFINE_BEGIN(instruction_access)
  1394. IVEC=0x400
  1395. IISIDE=1
  1396. IDAR=1
  1397. IDSISR=1
  1398. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1399. IKVM_REAL=1
  1400. #endif
  1401. INT_DEFINE_END(instruction_access)
  1402. EXC_REAL_BEGIN(instruction_access, 0x400, 0x80)
  1403. GEN_INT_ENTRY instruction_access, virt=0
  1404. EXC_REAL_END(instruction_access, 0x400, 0x80)
  1405. EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
  1406. GEN_INT_ENTRY instruction_access, virt=1
  1407. EXC_VIRT_END(instruction_access, 0x4400, 0x80)
  1408. EXC_COMMON_BEGIN(instruction_access_common)
  1409. GEN_COMMON instruction_access
  1410. ld r4,_DAR(r1)
  1411. ld r5,_DSISR(r1)
  1412. BEGIN_MMU_FTR_SECTION
  1413. ld r6,_MSR(r1)
  1414. li r3,0x400
  1415. b do_hash_page /* Try to handle as hpte fault */
  1416. MMU_FTR_SECTION_ELSE
  1417. b handle_page_fault
  1418. ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
  1419. GEN_KVM instruction_access
  1420. /**
  1421. * Interrupt 0x480 - Instruction Segment Interrupt (ISLB).
  1422. * This is a synchronous interrupt in response to an MMU fault due to an
  1423. * instruction fetch.
  1424. *
  1425. * Handling:
  1426. * Similar to DSLB, though in response to fetch. The faulting address is found
  1427. * in SRR0 (rather than DAR).
  1428. */
  1429. INT_DEFINE_BEGIN(instruction_access_slb)
  1430. IVEC=0x480
  1431. IAREA=PACA_EXSLB
  1432. IRECONCILE=0
  1433. IISIDE=1
  1434. IDAR=1
  1435. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1436. IKVM_REAL=1
  1437. #endif
  1438. INT_DEFINE_END(instruction_access_slb)
  1439. EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80)
  1440. GEN_INT_ENTRY instruction_access_slb, virt=0
  1441. EXC_REAL_END(instruction_access_slb, 0x480, 0x80)
  1442. EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
  1443. GEN_INT_ENTRY instruction_access_slb, virt=1
  1444. EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
  1445. EXC_COMMON_BEGIN(instruction_access_slb_common)
  1446. GEN_COMMON instruction_access_slb
  1447. ld r4,_DAR(r1)
  1448. addi r3,r1,STACK_FRAME_OVERHEAD
  1449. BEGIN_MMU_FTR_SECTION
  1450. /* HPT case, do SLB fault */
  1451. bl do_slb_fault
  1452. cmpdi r3,0
  1453. bne- 1f
  1454. b fast_interrupt_return
  1455. 1: /* Error case */
  1456. MMU_FTR_SECTION_ELSE
  1457. /* Radix case, access is outside page table range */
  1458. li r3,-EFAULT
  1459. ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
  1460. std r3,RESULT(r1)
  1461. RECONCILE_IRQ_STATE(r10, r11)
  1462. ld r4,_DAR(r1)
  1463. ld r5,RESULT(r1)
  1464. addi r3,r1,STACK_FRAME_OVERHEAD
  1465. bl do_bad_slb_fault
  1466. b interrupt_return
  1467. GEN_KVM instruction_access_slb
  1468. /**
  1469. * Interrupt 0x500 - External Interrupt.
  1470. * This is an asynchronous maskable interrupt in response to an "external
  1471. * exception" from the interrupt controller or hypervisor (e.g., device
  1472. * interrupt). It is maskable in hardware by clearing MSR[EE], and
  1473. * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()).
  1474. *
  1475. * When running in HV mode, Linux sets up the LPCR[LPES] bit such that
  1476. * interrupts are delivered with HSRR registers, guests use SRRs, which
  1477. * reqiures IHSRR_IF_HVMODE.
  1478. *
  1479. * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that
  1480. * external interrupts are delivered as Hypervisor Virtualization Interrupts
  1481. * rather than External Interrupts.
  1482. *
  1483. * Handling:
  1484. * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead,
  1485. * because registers at the time of the interrupt are not so important as it is
  1486. * asynchronous.
  1487. *
  1488. * If soft masked, the masked handler will note the pending interrupt for
  1489. * replay, and clear MSR[EE] in the interrupted context.
  1490. */
  1491. INT_DEFINE_BEGIN(hardware_interrupt)
  1492. IVEC=0x500
  1493. IHSRR_IF_HVMODE=1
  1494. IMASK=IRQS_DISABLED
  1495. IKVM_REAL=1
  1496. IKVM_VIRT=1
  1497. INT_DEFINE_END(hardware_interrupt)
  1498. EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
  1499. GEN_INT_ENTRY hardware_interrupt, virt=0
  1500. EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
  1501. EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
  1502. GEN_INT_ENTRY hardware_interrupt, virt=1
  1503. EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
  1504. EXC_COMMON_BEGIN(hardware_interrupt_common)
  1505. GEN_COMMON hardware_interrupt
  1506. FINISH_NAP
  1507. RUNLATCH_ON
  1508. addi r3,r1,STACK_FRAME_OVERHEAD
  1509. bl do_IRQ
  1510. b interrupt_return
  1511. GEN_KVM hardware_interrupt
  1512. /**
  1513. * Interrupt 0x600 - Alignment Interrupt
  1514. * This is a synchronous interrupt in response to data alignment fault.
  1515. */
  1516. INT_DEFINE_BEGIN(alignment)
  1517. IVEC=0x600
  1518. IDAR=1
  1519. IDSISR=1
  1520. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1521. IKVM_REAL=1
  1522. #endif
  1523. INT_DEFINE_END(alignment)
  1524. EXC_REAL_BEGIN(alignment, 0x600, 0x100)
  1525. GEN_INT_ENTRY alignment, virt=0
  1526. EXC_REAL_END(alignment, 0x600, 0x100)
  1527. EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
  1528. GEN_INT_ENTRY alignment, virt=1
  1529. EXC_VIRT_END(alignment, 0x4600, 0x100)
  1530. EXC_COMMON_BEGIN(alignment_common)
  1531. GEN_COMMON alignment
  1532. addi r3,r1,STACK_FRAME_OVERHEAD
  1533. bl alignment_exception
  1534. REST_NVGPRS(r1) /* instruction emulation may change GPRs */
  1535. b interrupt_return
  1536. GEN_KVM alignment
  1537. /**
  1538. * Interrupt 0x700 - Program Interrupt (program check).
  1539. * This is a synchronous interrupt in response to various instruction faults:
  1540. * traps, privilege errors, TM errors, floating point exceptions.
  1541. *
  1542. * Handling:
  1543. * This interrupt may use the "emergency stack" in some cases when being taken
  1544. * from kernel context, which complicates handling.
  1545. */
  1546. INT_DEFINE_BEGIN(program_check)
  1547. IVEC=0x700
  1548. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1549. IKVM_REAL=1
  1550. #endif
  1551. INT_DEFINE_END(program_check)
  1552. EXC_REAL_BEGIN(program_check, 0x700, 0x100)
  1553. GEN_INT_ENTRY program_check, virt=0
  1554. EXC_REAL_END(program_check, 0x700, 0x100)
  1555. EXC_VIRT_BEGIN(program_check, 0x4700, 0x100)
  1556. GEN_INT_ENTRY program_check, virt=1
  1557. EXC_VIRT_END(program_check, 0x4700, 0x100)
  1558. EXC_COMMON_BEGIN(program_check_common)
  1559. __GEN_COMMON_ENTRY program_check
  1560. /*
  1561. * It's possible to receive a TM Bad Thing type program check with
  1562. * userspace register values (in particular r1), but with SRR1 reporting
  1563. * that we came from the kernel. Normally that would confuse the bad
  1564. * stack logic, and we would report a bad kernel stack pointer. Instead
  1565. * we switch to the emergency stack if we're taking a TM Bad Thing from
  1566. * the kernel.
  1567. */
  1568. andi. r10,r12,MSR_PR
  1569. bne .Lnormal_stack /* If userspace, go normal path */
  1570. andis. r10,r12,(SRR1_PROGTM)@h
  1571. bne .Lemergency_stack /* If TM, emergency */
  1572. cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */
  1573. blt .Lnormal_stack /* normal path if not */
  1574. /* Use the emergency stack */
  1575. .Lemergency_stack:
  1576. andi. r10,r12,MSR_PR /* Set CR0 correctly for label */
  1577. /* 3 in EXCEPTION_PROLOG_COMMON */
  1578. mr r10,r1 /* Save r1 */
  1579. ld r1,PACAEMERGSP(r13) /* Use emergency stack */
  1580. subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
  1581. __ISTACK(program_check)=0
  1582. __GEN_COMMON_BODY program_check
  1583. b .Ldo_program_check
  1584. .Lnormal_stack:
  1585. __ISTACK(program_check)=1
  1586. __GEN_COMMON_BODY program_check
  1587. .Ldo_program_check:
  1588. addi r3,r1,STACK_FRAME_OVERHEAD
  1589. bl program_check_exception
  1590. REST_NVGPRS(r1) /* instruction emulation may change GPRs */
  1591. b interrupt_return
  1592. GEN_KVM program_check
  1593. /*
  1594. * Interrupt 0x800 - Floating-Point Unavailable Interrupt.
  1595. * This is a synchronous interrupt in response to executing an fp instruction
  1596. * with MSR[FP]=0.
  1597. *
  1598. * Handling:
  1599. * This will load FP registers and enable the FP bit if coming from userspace,
  1600. * otherwise report a bad kernel use of FP.
  1601. */
  1602. INT_DEFINE_BEGIN(fp_unavailable)
  1603. IVEC=0x800
  1604. IRECONCILE=0
  1605. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1606. IKVM_REAL=1
  1607. #endif
  1608. INT_DEFINE_END(fp_unavailable)
  1609. EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100)
  1610. GEN_INT_ENTRY fp_unavailable, virt=0
  1611. EXC_REAL_END(fp_unavailable, 0x800, 0x100)
  1612. EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100)
  1613. GEN_INT_ENTRY fp_unavailable, virt=1
  1614. EXC_VIRT_END(fp_unavailable, 0x4800, 0x100)
  1615. EXC_COMMON_BEGIN(fp_unavailable_common)
  1616. GEN_COMMON fp_unavailable
  1617. bne 1f /* if from user, just load it up */
  1618. RECONCILE_IRQ_STATE(r10, r11)
  1619. addi r3,r1,STACK_FRAME_OVERHEAD
  1620. bl kernel_fp_unavailable_exception
  1621. 0: trap
  1622. EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0
  1623. 1:
  1624. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1625. BEGIN_FTR_SECTION
  1626. /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
  1627. * transaction), go do TM stuff
  1628. */
  1629. rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
  1630. bne- 2f
  1631. END_FTR_SECTION_IFSET(CPU_FTR_TM)
  1632. #endif
  1633. bl load_up_fpu
  1634. b fast_interrupt_return
  1635. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1636. 2: /* User process was in a transaction */
  1637. RECONCILE_IRQ_STATE(r10, r11)
  1638. addi r3,r1,STACK_FRAME_OVERHEAD
  1639. bl fp_unavailable_tm
  1640. b interrupt_return
  1641. #endif
  1642. GEN_KVM fp_unavailable
  1643. /**
  1644. * Interrupt 0x900 - Decrementer Interrupt.
  1645. * This is an asynchronous interrupt in response to a decrementer exception
  1646. * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing
  1647. * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e.,
  1648. * local_irq_disable()).
  1649. *
  1650. * Handling:
  1651. * This calls into Linux timer handler. NVGPRs are not saved (see 0x500).
  1652. *
  1653. * If soft masked, the masked handler will note the pending interrupt for
  1654. * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled
  1655. * in the interrupted context.
  1656. * If PPC_WATCHDOG is configured, the soft masked handler will actually set
  1657. * things back up to run soft_nmi_interrupt as a regular interrupt handler
  1658. * on the emergency stack.
  1659. */
  1660. INT_DEFINE_BEGIN(decrementer)
  1661. IVEC=0x900
  1662. IMASK=IRQS_DISABLED
  1663. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1664. IKVM_REAL=1
  1665. #endif
  1666. INT_DEFINE_END(decrementer)
  1667. EXC_REAL_BEGIN(decrementer, 0x900, 0x80)
  1668. GEN_INT_ENTRY decrementer, virt=0
  1669. EXC_REAL_END(decrementer, 0x900, 0x80)
  1670. EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
  1671. GEN_INT_ENTRY decrementer, virt=1
  1672. EXC_VIRT_END(decrementer, 0x4900, 0x80)
  1673. EXC_COMMON_BEGIN(decrementer_common)
  1674. GEN_COMMON decrementer
  1675. FINISH_NAP
  1676. RUNLATCH_ON
  1677. addi r3,r1,STACK_FRAME_OVERHEAD
  1678. bl timer_interrupt
  1679. b interrupt_return
  1680. GEN_KVM decrementer
  1681. /**
  1682. * Interrupt 0x980 - Hypervisor Decrementer Interrupt.
  1683. * This is an asynchronous interrupt, similar to 0x900 but for the HDEC
  1684. * register.
  1685. *
  1686. * Handling:
  1687. * Linux does not use this outside KVM where it's used to keep a host timer
  1688. * while the guest is given control of DEC. It should normally be caught by
  1689. * the KVM test and routed there.
  1690. */
  1691. INT_DEFINE_BEGIN(hdecrementer)
  1692. IVEC=0x980
  1693. IHSRR=1
  1694. ISTACK=0
  1695. IRECONCILE=0
  1696. IKVM_REAL=1
  1697. IKVM_VIRT=1
  1698. INT_DEFINE_END(hdecrementer)
  1699. EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80)
  1700. GEN_INT_ENTRY hdecrementer, virt=0
  1701. EXC_REAL_END(hdecrementer, 0x980, 0x80)
  1702. EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80)
  1703. GEN_INT_ENTRY hdecrementer, virt=1
  1704. EXC_VIRT_END(hdecrementer, 0x4980, 0x80)
  1705. EXC_COMMON_BEGIN(hdecrementer_common)
  1706. __GEN_COMMON_ENTRY hdecrementer
  1707. /*
  1708. * Hypervisor decrementer interrupts not caught by the KVM test
  1709. * shouldn't occur but are sometimes left pending on exit from a KVM
  1710. * guest. We don't need to do anything to clear them, as they are
  1711. * edge-triggered.
  1712. *
  1713. * Be careful to avoid touching the kernel stack.
  1714. */
  1715. ld r10,PACA_EXGEN+EX_CTR(r13)
  1716. mtctr r10
  1717. mtcrf 0x80,r9
  1718. ld r9,PACA_EXGEN+EX_R9(r13)
  1719. ld r10,PACA_EXGEN+EX_R10(r13)
  1720. ld r11,PACA_EXGEN+EX_R11(r13)
  1721. ld r12,PACA_EXGEN+EX_R12(r13)
  1722. ld r13,PACA_EXGEN+EX_R13(r13)
  1723. HRFI_TO_KERNEL
  1724. GEN_KVM hdecrementer
  1725. /**
  1726. * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt.
  1727. * This is an asynchronous interrupt in response to a msgsndp doorbell.
  1728. * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
  1729. * IRQS_DISABLED mask (i.e., local_irq_disable()).
  1730. *
  1731. * Handling:
  1732. * Guests may use this for IPIs between threads in a core if the
  1733. * hypervisor supports it. NVGPRS are not saved (see 0x500).
  1734. *
  1735. * If soft masked, the masked handler will note the pending interrupt for
  1736. * replay, leaving MSR[EE] enabled in the interrupted context because the
  1737. * doorbells are edge triggered.
  1738. */
  1739. INT_DEFINE_BEGIN(doorbell_super)
  1740. IVEC=0xa00
  1741. IMASK=IRQS_DISABLED
  1742. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1743. IKVM_REAL=1
  1744. #endif
  1745. INT_DEFINE_END(doorbell_super)
  1746. EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100)
  1747. GEN_INT_ENTRY doorbell_super, virt=0
  1748. EXC_REAL_END(doorbell_super, 0xa00, 0x100)
  1749. EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100)
  1750. GEN_INT_ENTRY doorbell_super, virt=1
  1751. EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
  1752. EXC_COMMON_BEGIN(doorbell_super_common)
  1753. GEN_COMMON doorbell_super
  1754. FINISH_NAP
  1755. RUNLATCH_ON
  1756. addi r3,r1,STACK_FRAME_OVERHEAD
  1757. #ifdef CONFIG_PPC_DOORBELL
  1758. bl doorbell_exception
  1759. #else
  1760. bl unknown_exception
  1761. #endif
  1762. b interrupt_return
  1763. GEN_KVM doorbell_super
  1764. EXC_REAL_NONE(0xb00, 0x100)
  1765. EXC_VIRT_NONE(0x4b00, 0x100)
  1766. /**
  1767. * Interrupt 0xc00 - System Call Interrupt (syscall, hcall).
  1768. * This is a synchronous interrupt invoked with the "sc" instruction. The
  1769. * system call is invoked with "sc 0" and does not alter the HV bit, so it
  1770. * is directed to the currently running OS. The hypercall is invoked with
  1771. * "sc 1" and it sets HV=1, so it elevates to hypervisor.
  1772. *
  1773. * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to
  1774. * 0x4c00 virtual mode.
  1775. *
  1776. * Handling:
  1777. * If the KVM test fires then it was due to a hypercall and is accordingly
  1778. * routed to KVM. Otherwise this executes a normal Linux system call.
  1779. *
  1780. * Call convention:
  1781. *
  1782. * syscall and hypercalls register conventions are documented in
  1783. * Documentation/powerpc/syscall64-abi.rst and
  1784. * Documentation/powerpc/papr_hcalls.rst respectively.
  1785. *
  1786. * The intersection of volatile registers that don't contain possible
  1787. * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry
  1788. * without saving, though xer is not a good idea to use, as hardware may
  1789. * interpret some bits so it may be costly to change them.
  1790. */
  1791. INT_DEFINE_BEGIN(system_call)
  1792. IVEC=0xc00
  1793. IKVM_REAL=1
  1794. IKVM_VIRT=1
  1795. INT_DEFINE_END(system_call)
  1796. .macro SYSTEM_CALL virt
  1797. #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
  1798. /*
  1799. * There is a little bit of juggling to get syscall and hcall
  1800. * working well. Save r13 in ctr to avoid using SPRG scratch
  1801. * register.
  1802. *
  1803. * Userspace syscalls have already saved the PPR, hcalls must save
  1804. * it before setting HMT_MEDIUM.
  1805. */
  1806. mtctr r13
  1807. GET_PACA(r13)
  1808. std r10,PACA_EXGEN+EX_R10(r13)
  1809. INTERRUPT_TO_KERNEL
  1810. KVMTEST system_call /* uses r10, branch to system_call_kvm */
  1811. mfctr r9
  1812. #else
  1813. mr r9,r13
  1814. GET_PACA(r13)
  1815. INTERRUPT_TO_KERNEL
  1816. #endif
  1817. #ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
  1818. BEGIN_FTR_SECTION
  1819. cmpdi r0,0x1ebe
  1820. beq- 1f
  1821. END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
  1822. #endif
  1823. /* We reach here with PACA in r13, r13 in r9. */
  1824. mfspr r11,SPRN_SRR0
  1825. mfspr r12,SPRN_SRR1
  1826. HMT_MEDIUM
  1827. .if ! \virt
  1828. __LOAD_HANDLER(r10, system_call_common)
  1829. mtspr SPRN_SRR0,r10
  1830. ld r10,PACAKMSR(r13)
  1831. mtspr SPRN_SRR1,r10
  1832. RFI_TO_KERNEL
  1833. b . /* prevent speculative execution */
  1834. .else
  1835. li r10,MSR_RI
  1836. mtmsrd r10,1 /* Set RI (EE=0) */
  1837. #ifdef CONFIG_RELOCATABLE
  1838. __LOAD_HANDLER(r10, system_call_common)
  1839. mtctr r10
  1840. bctr
  1841. #else
  1842. b system_call_common
  1843. #endif
  1844. .endif
  1845. #ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
  1846. /* Fast LE/BE switch system call */
  1847. 1: mfspr r12,SPRN_SRR1
  1848. xori r12,r12,MSR_LE
  1849. mtspr SPRN_SRR1,r12
  1850. mr r13,r9
  1851. RFI_TO_USER /* return to userspace */
  1852. b . /* prevent speculative execution */
  1853. #endif
  1854. .endm
  1855. EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
  1856. SYSTEM_CALL 0
  1857. EXC_REAL_END(system_call, 0xc00, 0x100)
  1858. EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
  1859. SYSTEM_CALL 1
  1860. EXC_VIRT_END(system_call, 0x4c00, 0x100)
  1861. #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
  1862. TRAMP_REAL_BEGIN(system_call_kvm)
  1863. /*
  1864. * This is a hcall, so register convention is as above, with these
  1865. * differences:
  1866. * r13 = PACA
  1867. * ctr = orig r13
  1868. * orig r10 saved in PACA
  1869. */
  1870. /*
  1871. * Save the PPR (on systems that support it) before changing to
  1872. * HMT_MEDIUM. That allows the KVM code to save that value into the
  1873. * guest state (it is the guest's PPR value).
  1874. */
  1875. BEGIN_FTR_SECTION
  1876. mfspr r10,SPRN_PPR
  1877. std r10,HSTATE_PPR(r13)
  1878. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  1879. HMT_MEDIUM
  1880. mfctr r10
  1881. SET_SCRATCH0(r10)
  1882. mfcr r10
  1883. std r12,HSTATE_SCRATCH0(r13)
  1884. sldi r12,r10,32
  1885. ori r12,r12,0xc00
  1886. #ifdef CONFIG_RELOCATABLE
  1887. /*
  1888. * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives
  1889. * outside the head section.
  1890. */
  1891. __LOAD_FAR_HANDLER(r10, kvmppc_interrupt)
  1892. mtctr r10
  1893. ld r10,PACA_EXGEN+EX_R10(r13)
  1894. bctr
  1895. #else
  1896. ld r10,PACA_EXGEN+EX_R10(r13)
  1897. b kvmppc_interrupt
  1898. #endif
  1899. #endif
  1900. /**
  1901. * Interrupt 0xd00 - Trace Interrupt.
  1902. * This is a synchronous interrupt in response to instruction step or
  1903. * breakpoint faults.
  1904. */
  1905. INT_DEFINE_BEGIN(single_step)
  1906. IVEC=0xd00
  1907. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1908. IKVM_REAL=1
  1909. #endif
  1910. INT_DEFINE_END(single_step)
  1911. EXC_REAL_BEGIN(single_step, 0xd00, 0x100)
  1912. GEN_INT_ENTRY single_step, virt=0
  1913. EXC_REAL_END(single_step, 0xd00, 0x100)
  1914. EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100)
  1915. GEN_INT_ENTRY single_step, virt=1
  1916. EXC_VIRT_END(single_step, 0x4d00, 0x100)
  1917. EXC_COMMON_BEGIN(single_step_common)
  1918. GEN_COMMON single_step
  1919. addi r3,r1,STACK_FRAME_OVERHEAD
  1920. bl single_step_exception
  1921. b interrupt_return
  1922. GEN_KVM single_step
  1923. /**
  1924. * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI).
  1925. * This is a synchronous interrupt in response to an MMU fault caused by a
  1926. * guest data access.
  1927. *
  1928. * Handling:
  1929. * This should always get routed to KVM. In radix MMU mode, this is caused
  1930. * by a guest nested radix access that can't be performed due to the
  1931. * partition scope page table. In hash mode, this can be caused by guests
  1932. * running with translation disabled (virtual real mode) or with VPM enabled.
  1933. * KVM will update the page table structures or disallow the access.
  1934. */
  1935. INT_DEFINE_BEGIN(h_data_storage)
  1936. IVEC=0xe00
  1937. IHSRR=1
  1938. IDAR=1
  1939. IDSISR=1
  1940. IKVM_SKIP=1
  1941. IKVM_REAL=1
  1942. IKVM_VIRT=1
  1943. INT_DEFINE_END(h_data_storage)
  1944. EXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20)
  1945. GEN_INT_ENTRY h_data_storage, virt=0, ool=1
  1946. EXC_REAL_END(h_data_storage, 0xe00, 0x20)
  1947. EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20)
  1948. GEN_INT_ENTRY h_data_storage, virt=1, ool=1
  1949. EXC_VIRT_END(h_data_storage, 0x4e00, 0x20)
  1950. EXC_COMMON_BEGIN(h_data_storage_common)
  1951. GEN_COMMON h_data_storage
  1952. addi r3,r1,STACK_FRAME_OVERHEAD
  1953. BEGIN_MMU_FTR_SECTION
  1954. ld r4,_DAR(r1)
  1955. li r5,SIGSEGV
  1956. bl bad_page_fault
  1957. MMU_FTR_SECTION_ELSE
  1958. bl unknown_exception
  1959. ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
  1960. b interrupt_return
  1961. GEN_KVM h_data_storage
  1962. /**
  1963. * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI).
  1964. * This is a synchronous interrupt in response to an MMU fault caused by a
  1965. * guest instruction fetch, similar to HDSI.
  1966. */
  1967. INT_DEFINE_BEGIN(h_instr_storage)
  1968. IVEC=0xe20
  1969. IHSRR=1
  1970. IKVM_REAL=1
  1971. IKVM_VIRT=1
  1972. INT_DEFINE_END(h_instr_storage)
  1973. EXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20)
  1974. GEN_INT_ENTRY h_instr_storage, virt=0, ool=1
  1975. EXC_REAL_END(h_instr_storage, 0xe20, 0x20)
  1976. EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20)
  1977. GEN_INT_ENTRY h_instr_storage, virt=1, ool=1
  1978. EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20)
  1979. EXC_COMMON_BEGIN(h_instr_storage_common)
  1980. GEN_COMMON h_instr_storage
  1981. addi r3,r1,STACK_FRAME_OVERHEAD
  1982. bl unknown_exception
  1983. b interrupt_return
  1984. GEN_KVM h_instr_storage
  1985. /**
  1986. * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt.
  1987. */
  1988. INT_DEFINE_BEGIN(emulation_assist)
  1989. IVEC=0xe40
  1990. IHSRR=1
  1991. IKVM_REAL=1
  1992. IKVM_VIRT=1
  1993. INT_DEFINE_END(emulation_assist)
  1994. EXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20)
  1995. GEN_INT_ENTRY emulation_assist, virt=0, ool=1
  1996. EXC_REAL_END(emulation_assist, 0xe40, 0x20)
  1997. EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20)
  1998. GEN_INT_ENTRY emulation_assist, virt=1, ool=1
  1999. EXC_VIRT_END(emulation_assist, 0x4e40, 0x20)
  2000. EXC_COMMON_BEGIN(emulation_assist_common)
  2001. GEN_COMMON emulation_assist
  2002. addi r3,r1,STACK_FRAME_OVERHEAD
  2003. bl emulation_assist_interrupt
  2004. REST_NVGPRS(r1) /* instruction emulation may change GPRs */
  2005. b interrupt_return
  2006. GEN_KVM emulation_assist
  2007. /**
  2008. * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI).
  2009. * This is an asynchronous interrupt caused by a Hypervisor Maintenance
  2010. * Exception. It is always taken in real mode but uses HSRR registers
  2011. * unlike SRESET and MCE.
  2012. *
  2013. * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable
  2014. * with IRQS_DISABLED mask (i.e., local_irq_disable()).
  2015. *
  2016. * Handling:
  2017. * This is a special case, this is handled similarly to machine checks, with an
  2018. * initial real mode handler that is not soft-masked, which attempts to fix the
  2019. * problem. Then a regular handler which is soft-maskable and reports the
  2020. * problem.
  2021. *
  2022. * The emergency stack is used for the early real mode handler.
  2023. *
  2024. * XXX: unclear why MCE and HMI schemes could not be made common, e.g.,
  2025. * either use soft-masking for the MCE, or use irq_work for the HMI.
  2026. *
  2027. * KVM:
  2028. * Unlike MCE, this calls into KVM without calling the real mode handler
  2029. * first.
  2030. */
  2031. INT_DEFINE_BEGIN(hmi_exception_early)
  2032. IVEC=0xe60
  2033. IHSRR=1
  2034. IREALMODE_COMMON=1
  2035. ISTACK=0
  2036. IRECONCILE=0
  2037. IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
  2038. IKVM_REAL=1
  2039. INT_DEFINE_END(hmi_exception_early)
  2040. INT_DEFINE_BEGIN(hmi_exception)
  2041. IVEC=0xe60
  2042. IHSRR=1
  2043. IMASK=IRQS_DISABLED
  2044. IKVM_REAL=1
  2045. INT_DEFINE_END(hmi_exception)
  2046. EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20)
  2047. GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1
  2048. EXC_REAL_END(hmi_exception, 0xe60, 0x20)
  2049. EXC_VIRT_NONE(0x4e60, 0x20)
  2050. EXC_COMMON_BEGIN(hmi_exception_early_common)
  2051. __GEN_REALMODE_COMMON_ENTRY hmi_exception_early
  2052. mr r10,r1 /* Save r1 */
  2053. ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */
  2054. subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
  2055. __GEN_COMMON_BODY hmi_exception_early
  2056. addi r3,r1,STACK_FRAME_OVERHEAD
  2057. bl hmi_exception_realmode
  2058. cmpdi cr0,r3,0
  2059. bne 1f
  2060. EXCEPTION_RESTORE_REGS hsrr=1
  2061. HRFI_TO_USER_OR_KERNEL
  2062. 1:
  2063. /*
  2064. * Go to virtual mode and pull the HMI event information from
  2065. * firmware.
  2066. */
  2067. EXCEPTION_RESTORE_REGS hsrr=1
  2068. GEN_INT_ENTRY hmi_exception, virt=0
  2069. GEN_KVM hmi_exception_early
  2070. EXC_COMMON_BEGIN(hmi_exception_common)
  2071. GEN_COMMON hmi_exception
  2072. FINISH_NAP
  2073. RUNLATCH_ON
  2074. addi r3,r1,STACK_FRAME_OVERHEAD
  2075. bl handle_hmi_exception
  2076. b interrupt_return
  2077. GEN_KVM hmi_exception
  2078. /**
  2079. * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt.
  2080. * This is an asynchronous interrupt in response to a msgsnd doorbell.
  2081. * Similar to the 0xa00 doorbell but for host rather than guest.
  2082. */
  2083. INT_DEFINE_BEGIN(h_doorbell)
  2084. IVEC=0xe80
  2085. IHSRR=1
  2086. IMASK=IRQS_DISABLED
  2087. IKVM_REAL=1
  2088. IKVM_VIRT=1
  2089. INT_DEFINE_END(h_doorbell)
  2090. EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20)
  2091. GEN_INT_ENTRY h_doorbell, virt=0, ool=1
  2092. EXC_REAL_END(h_doorbell, 0xe80, 0x20)
  2093. EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20)
  2094. GEN_INT_ENTRY h_doorbell, virt=1, ool=1
  2095. EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
  2096. EXC_COMMON_BEGIN(h_doorbell_common)
  2097. GEN_COMMON h_doorbell
  2098. FINISH_NAP
  2099. RUNLATCH_ON
  2100. addi r3,r1,STACK_FRAME_OVERHEAD
  2101. #ifdef CONFIG_PPC_DOORBELL
  2102. bl doorbell_exception
  2103. #else
  2104. bl unknown_exception
  2105. #endif
  2106. b interrupt_return
  2107. GEN_KVM h_doorbell
  2108. /**
  2109. * Interrupt 0xea0 - Hypervisor Virtualization Interrupt.
  2110. * This is an asynchronous interrupt in response to an "external exception".
  2111. * Similar to 0x500 but for host only.
  2112. */
  2113. INT_DEFINE_BEGIN(h_virt_irq)
  2114. IVEC=0xea0
  2115. IHSRR=1
  2116. IMASK=IRQS_DISABLED
  2117. IKVM_REAL=1
  2118. IKVM_VIRT=1
  2119. INT_DEFINE_END(h_virt_irq)
  2120. EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20)
  2121. GEN_INT_ENTRY h_virt_irq, virt=0, ool=1
  2122. EXC_REAL_END(h_virt_irq, 0xea0, 0x20)
  2123. EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20)
  2124. GEN_INT_ENTRY h_virt_irq, virt=1, ool=1
  2125. EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
  2126. EXC_COMMON_BEGIN(h_virt_irq_common)
  2127. GEN_COMMON h_virt_irq
  2128. FINISH_NAP
  2129. RUNLATCH_ON
  2130. addi r3,r1,STACK_FRAME_OVERHEAD
  2131. bl do_IRQ
  2132. b interrupt_return
  2133. GEN_KVM h_virt_irq
  2134. EXC_REAL_NONE(0xec0, 0x20)
  2135. EXC_VIRT_NONE(0x4ec0, 0x20)
  2136. EXC_REAL_NONE(0xee0, 0x20)
  2137. EXC_VIRT_NONE(0x4ee0, 0x20)
  2138. /*
  2139. * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU).
  2140. * This is an asynchronous interrupt in response to a PMU exception.
  2141. * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
  2142. * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()).
  2143. *
  2144. * Handling:
  2145. * This calls into the perf subsystem.
  2146. *
  2147. * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it
  2148. * runs under local_irq_disable. However it may be soft-masked in
  2149. * powerpc-specific code.
  2150. *
  2151. * If soft masked, the masked handler will note the pending interrupt for
  2152. * replay, and clear MSR[EE] in the interrupted context.
  2153. */
  2154. INT_DEFINE_BEGIN(performance_monitor)
  2155. IVEC=0xf00
  2156. IMASK=IRQS_PMI_DISABLED
  2157. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2158. IKVM_REAL=1
  2159. #endif
  2160. INT_DEFINE_END(performance_monitor)
  2161. EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20)
  2162. GEN_INT_ENTRY performance_monitor, virt=0, ool=1
  2163. EXC_REAL_END(performance_monitor, 0xf00, 0x20)
  2164. EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20)
  2165. GEN_INT_ENTRY performance_monitor, virt=1, ool=1
  2166. EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
  2167. EXC_COMMON_BEGIN(performance_monitor_common)
  2168. GEN_COMMON performance_monitor
  2169. FINISH_NAP
  2170. RUNLATCH_ON
  2171. addi r3,r1,STACK_FRAME_OVERHEAD
  2172. bl performance_monitor_exception
  2173. b interrupt_return
  2174. GEN_KVM performance_monitor
  2175. /**
  2176. * Interrupt 0xf20 - Vector Unavailable Interrupt.
  2177. * This is a synchronous interrupt in response to
  2178. * executing a vector (or altivec) instruction with MSR[VEC]=0.
  2179. * Similar to FP unavailable.
  2180. */
  2181. INT_DEFINE_BEGIN(altivec_unavailable)
  2182. IVEC=0xf20
  2183. IRECONCILE=0
  2184. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2185. IKVM_REAL=1
  2186. #endif
  2187. INT_DEFINE_END(altivec_unavailable)
  2188. EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20)
  2189. GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1
  2190. EXC_REAL_END(altivec_unavailable, 0xf20, 0x20)
  2191. EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20)
  2192. GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1
  2193. EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20)
  2194. EXC_COMMON_BEGIN(altivec_unavailable_common)
  2195. GEN_COMMON altivec_unavailable
  2196. #ifdef CONFIG_ALTIVEC
  2197. BEGIN_FTR_SECTION
  2198. beq 1f
  2199. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2200. BEGIN_FTR_SECTION_NESTED(69)
  2201. /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
  2202. * transaction), go do TM stuff
  2203. */
  2204. rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
  2205. bne- 2f
  2206. END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
  2207. #endif
  2208. bl load_up_altivec
  2209. b fast_interrupt_return
  2210. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2211. 2: /* User process was in a transaction */
  2212. RECONCILE_IRQ_STATE(r10, r11)
  2213. addi r3,r1,STACK_FRAME_OVERHEAD
  2214. bl altivec_unavailable_tm
  2215. b interrupt_return
  2216. #endif
  2217. 1:
  2218. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  2219. #endif
  2220. RECONCILE_IRQ_STATE(r10, r11)
  2221. addi r3,r1,STACK_FRAME_OVERHEAD
  2222. bl altivec_unavailable_exception
  2223. b interrupt_return
  2224. GEN_KVM altivec_unavailable
  2225. /**
  2226. * Interrupt 0xf40 - VSX Unavailable Interrupt.
  2227. * This is a synchronous interrupt in response to
  2228. * executing a VSX instruction with MSR[VSX]=0.
  2229. * Similar to FP unavailable.
  2230. */
  2231. INT_DEFINE_BEGIN(vsx_unavailable)
  2232. IVEC=0xf40
  2233. IRECONCILE=0
  2234. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2235. IKVM_REAL=1
  2236. #endif
  2237. INT_DEFINE_END(vsx_unavailable)
  2238. EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20)
  2239. GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1
  2240. EXC_REAL_END(vsx_unavailable, 0xf40, 0x20)
  2241. EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20)
  2242. GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1
  2243. EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20)
  2244. EXC_COMMON_BEGIN(vsx_unavailable_common)
  2245. GEN_COMMON vsx_unavailable
  2246. #ifdef CONFIG_VSX
  2247. BEGIN_FTR_SECTION
  2248. beq 1f
  2249. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2250. BEGIN_FTR_SECTION_NESTED(69)
  2251. /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
  2252. * transaction), go do TM stuff
  2253. */
  2254. rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
  2255. bne- 2f
  2256. END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
  2257. #endif
  2258. b load_up_vsx
  2259. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2260. 2: /* User process was in a transaction */
  2261. RECONCILE_IRQ_STATE(r10, r11)
  2262. addi r3,r1,STACK_FRAME_OVERHEAD
  2263. bl vsx_unavailable_tm
  2264. b interrupt_return
  2265. #endif
  2266. 1:
  2267. END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  2268. #endif
  2269. RECONCILE_IRQ_STATE(r10, r11)
  2270. addi r3,r1,STACK_FRAME_OVERHEAD
  2271. bl vsx_unavailable_exception
  2272. b interrupt_return
  2273. GEN_KVM vsx_unavailable
  2274. /**
  2275. * Interrupt 0xf60 - Facility Unavailable Interrupt.
  2276. * This is a synchronous interrupt in response to
  2277. * executing an instruction without access to the facility that can be
  2278. * resolved by the OS (e.g., FSCR, MSR).
  2279. * Similar to FP unavailable.
  2280. */
  2281. INT_DEFINE_BEGIN(facility_unavailable)
  2282. IVEC=0xf60
  2283. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2284. IKVM_REAL=1
  2285. #endif
  2286. INT_DEFINE_END(facility_unavailable)
  2287. EXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20)
  2288. GEN_INT_ENTRY facility_unavailable, virt=0, ool=1
  2289. EXC_REAL_END(facility_unavailable, 0xf60, 0x20)
  2290. EXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20)
  2291. GEN_INT_ENTRY facility_unavailable, virt=1, ool=1
  2292. EXC_VIRT_END(facility_unavailable, 0x4f60, 0x20)
  2293. EXC_COMMON_BEGIN(facility_unavailable_common)
  2294. GEN_COMMON facility_unavailable
  2295. addi r3,r1,STACK_FRAME_OVERHEAD
  2296. bl facility_unavailable_exception
  2297. REST_NVGPRS(r1) /* instruction emulation may change GPRs */
  2298. b interrupt_return
  2299. GEN_KVM facility_unavailable
  2300. /**
  2301. * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt.
  2302. * This is a synchronous interrupt in response to
  2303. * executing an instruction without access to the facility that can only
  2304. * be resolved in HV mode (e.g., HFSCR).
  2305. * Similar to FP unavailable.
  2306. */
  2307. INT_DEFINE_BEGIN(h_facility_unavailable)
  2308. IVEC=0xf80
  2309. IHSRR=1
  2310. IKVM_REAL=1
  2311. IKVM_VIRT=1
  2312. INT_DEFINE_END(h_facility_unavailable)
  2313. EXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20)
  2314. GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1
  2315. EXC_REAL_END(h_facility_unavailable, 0xf80, 0x20)
  2316. EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20)
  2317. GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1
  2318. EXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20)
  2319. EXC_COMMON_BEGIN(h_facility_unavailable_common)
  2320. GEN_COMMON h_facility_unavailable
  2321. addi r3,r1,STACK_FRAME_OVERHEAD
  2322. bl facility_unavailable_exception
  2323. REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
  2324. b interrupt_return
  2325. GEN_KVM h_facility_unavailable
  2326. EXC_REAL_NONE(0xfa0, 0x20)
  2327. EXC_VIRT_NONE(0x4fa0, 0x20)
  2328. EXC_REAL_NONE(0xfc0, 0x20)
  2329. EXC_VIRT_NONE(0x4fc0, 0x20)
  2330. EXC_REAL_NONE(0xfe0, 0x20)
  2331. EXC_VIRT_NONE(0x4fe0, 0x20)
  2332. EXC_REAL_NONE(0x1000, 0x100)
  2333. EXC_VIRT_NONE(0x5000, 0x100)
  2334. EXC_REAL_NONE(0x1100, 0x100)
  2335. EXC_VIRT_NONE(0x5100, 0x100)
  2336. #ifdef CONFIG_CBE_RAS
  2337. INT_DEFINE_BEGIN(cbe_system_error)
  2338. IVEC=0x1200
  2339. IHSRR=1
  2340. IKVM_SKIP=1
  2341. IKVM_REAL=1
  2342. INT_DEFINE_END(cbe_system_error)
  2343. EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)
  2344. GEN_INT_ENTRY cbe_system_error, virt=0
  2345. EXC_REAL_END(cbe_system_error, 0x1200, 0x100)
  2346. EXC_VIRT_NONE(0x5200, 0x100)
  2347. EXC_COMMON_BEGIN(cbe_system_error_common)
  2348. GEN_COMMON cbe_system_error
  2349. addi r3,r1,STACK_FRAME_OVERHEAD
  2350. bl cbe_system_error_exception
  2351. b interrupt_return
  2352. GEN_KVM cbe_system_error
  2353. #else /* CONFIG_CBE_RAS */
  2354. EXC_REAL_NONE(0x1200, 0x100)
  2355. EXC_VIRT_NONE(0x5200, 0x100)
  2356. #endif
  2357. INT_DEFINE_BEGIN(instruction_breakpoint)
  2358. IVEC=0x1300
  2359. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2360. IKVM_SKIP=1
  2361. IKVM_REAL=1
  2362. #endif
  2363. INT_DEFINE_END(instruction_breakpoint)
  2364. EXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100)
  2365. GEN_INT_ENTRY instruction_breakpoint, virt=0
  2366. EXC_REAL_END(instruction_breakpoint, 0x1300, 0x100)
  2367. EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100)
  2368. GEN_INT_ENTRY instruction_breakpoint, virt=1
  2369. EXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100)
  2370. EXC_COMMON_BEGIN(instruction_breakpoint_common)
  2371. GEN_COMMON instruction_breakpoint
  2372. addi r3,r1,STACK_FRAME_OVERHEAD
  2373. bl instruction_breakpoint_exception
  2374. b interrupt_return
  2375. GEN_KVM instruction_breakpoint
  2376. EXC_REAL_NONE(0x1400, 0x100)
  2377. EXC_VIRT_NONE(0x5400, 0x100)
  2378. /**
  2379. * Interrupt 0x1500 - Soft Patch Interrupt
  2380. *
  2381. * Handling:
  2382. * This is an implementation specific interrupt which can be used for a
  2383. * range of exceptions.
  2384. *
  2385. * This interrupt handler is unique in that it runs the denormal assist
  2386. * code even for guests (and even in guest context) without going to KVM,
  2387. * for speed. POWER9 does not raise denorm exceptions, so this special case
  2388. * could be phased out in future to reduce special cases.
  2389. */
  2390. INT_DEFINE_BEGIN(denorm_exception)
  2391. IVEC=0x1500
  2392. IHSRR=1
  2393. IBRANCH_TO_COMMON=0
  2394. IKVM_REAL=1
  2395. INT_DEFINE_END(denorm_exception)
  2396. EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100)
  2397. GEN_INT_ENTRY denorm_exception, virt=0
  2398. #ifdef CONFIG_PPC_DENORMALISATION
  2399. andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */
  2400. bne+ denorm_assist
  2401. #endif
  2402. GEN_BRANCH_TO_COMMON denorm_exception, virt=0
  2403. EXC_REAL_END(denorm_exception, 0x1500, 0x100)
  2404. #ifdef CONFIG_PPC_DENORMALISATION
  2405. EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100)
  2406. GEN_INT_ENTRY denorm_exception, virt=1
  2407. andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */
  2408. bne+ denorm_assist
  2409. GEN_BRANCH_TO_COMMON denorm_exception, virt=1
  2410. EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
  2411. #else
  2412. EXC_VIRT_NONE(0x5500, 0x100)
  2413. #endif
  2414. #ifdef CONFIG_PPC_DENORMALISATION
  2415. TRAMP_REAL_BEGIN(denorm_assist)
  2416. BEGIN_FTR_SECTION
  2417. /*
  2418. * To denormalise we need to move a copy of the register to itself.
  2419. * For POWER6 do that here for all FP regs.
  2420. */
  2421. mfmsr r10
  2422. ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
  2423. xori r10,r10,(MSR_FE0|MSR_FE1)
  2424. mtmsrd r10
  2425. sync
  2426. .Lreg=0
  2427. .rept 32
  2428. fmr .Lreg,.Lreg
  2429. .Lreg=.Lreg+1
  2430. .endr
  2431. FTR_SECTION_ELSE
  2432. /*
  2433. * To denormalise we need to move a copy of the register to itself.
  2434. * For POWER7 do that here for the first 32 VSX registers only.
  2435. */
  2436. mfmsr r10
  2437. oris r10,r10,MSR_VSX@h
  2438. mtmsrd r10
  2439. sync
  2440. .Lreg=0
  2441. .rept 32
  2442. XVCPSGNDP(.Lreg,.Lreg,.Lreg)
  2443. .Lreg=.Lreg+1
  2444. .endr
  2445. ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
  2446. BEGIN_FTR_SECTION
  2447. b denorm_done
  2448. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
  2449. /*
  2450. * To denormalise we need to move a copy of the register to itself.
  2451. * For POWER8 we need to do that for all 64 VSX registers
  2452. */
  2453. .Lreg=32
  2454. .rept 32
  2455. XVCPSGNDP(.Lreg,.Lreg,.Lreg)
  2456. .Lreg=.Lreg+1
  2457. .endr
  2458. denorm_done:
  2459. mfspr r11,SPRN_HSRR0
  2460. subi r11,r11,4
  2461. mtspr SPRN_HSRR0,r11
  2462. mtcrf 0x80,r9
  2463. ld r9,PACA_EXGEN+EX_R9(r13)
  2464. BEGIN_FTR_SECTION
  2465. ld r10,PACA_EXGEN+EX_PPR(r13)
  2466. mtspr SPRN_PPR,r10
  2467. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  2468. BEGIN_FTR_SECTION
  2469. ld r10,PACA_EXGEN+EX_CFAR(r13)
  2470. mtspr SPRN_CFAR,r10
  2471. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  2472. ld r10,PACA_EXGEN+EX_R10(r13)
  2473. ld r11,PACA_EXGEN+EX_R11(r13)
  2474. ld r12,PACA_EXGEN+EX_R12(r13)
  2475. ld r13,PACA_EXGEN+EX_R13(r13)
  2476. HRFI_TO_UNKNOWN
  2477. b .
  2478. #endif
  2479. EXC_COMMON_BEGIN(denorm_exception_common)
  2480. GEN_COMMON denorm_exception
  2481. addi r3,r1,STACK_FRAME_OVERHEAD
  2482. bl unknown_exception
  2483. b interrupt_return
  2484. GEN_KVM denorm_exception
  2485. #ifdef CONFIG_CBE_RAS
  2486. INT_DEFINE_BEGIN(cbe_maintenance)
  2487. IVEC=0x1600
  2488. IHSRR=1
  2489. IKVM_SKIP=1
  2490. IKVM_REAL=1
  2491. INT_DEFINE_END(cbe_maintenance)
  2492. EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)
  2493. GEN_INT_ENTRY cbe_maintenance, virt=0
  2494. EXC_REAL_END(cbe_maintenance, 0x1600, 0x100)
  2495. EXC_VIRT_NONE(0x5600, 0x100)
  2496. EXC_COMMON_BEGIN(cbe_maintenance_common)
  2497. GEN_COMMON cbe_maintenance
  2498. addi r3,r1,STACK_FRAME_OVERHEAD
  2499. bl cbe_maintenance_exception
  2500. b interrupt_return
  2501. GEN_KVM cbe_maintenance
  2502. #else /* CONFIG_CBE_RAS */
  2503. EXC_REAL_NONE(0x1600, 0x100)
  2504. EXC_VIRT_NONE(0x5600, 0x100)
  2505. #endif
  2506. INT_DEFINE_BEGIN(altivec_assist)
  2507. IVEC=0x1700
  2508. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  2509. IKVM_REAL=1
  2510. #endif
  2511. INT_DEFINE_END(altivec_assist)
  2512. EXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100)
  2513. GEN_INT_ENTRY altivec_assist, virt=0
  2514. EXC_REAL_END(altivec_assist, 0x1700, 0x100)
  2515. EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100)
  2516. GEN_INT_ENTRY altivec_assist, virt=1
  2517. EXC_VIRT_END(altivec_assist, 0x5700, 0x100)
  2518. EXC_COMMON_BEGIN(altivec_assist_common)
  2519. GEN_COMMON altivec_assist
  2520. addi r3,r1,STACK_FRAME_OVERHEAD
  2521. #ifdef CONFIG_ALTIVEC
  2522. bl altivec_assist_exception
  2523. REST_NVGPRS(r1) /* instruction emulation may change GPRs */
  2524. #else
  2525. bl unknown_exception
  2526. #endif
  2527. b interrupt_return
  2528. GEN_KVM altivec_assist
  2529. #ifdef CONFIG_CBE_RAS
  2530. INT_DEFINE_BEGIN(cbe_thermal)
  2531. IVEC=0x1800
  2532. IHSRR=1
  2533. IKVM_SKIP=1
  2534. IKVM_REAL=1
  2535. INT_DEFINE_END(cbe_thermal)
  2536. EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
  2537. GEN_INT_ENTRY cbe_thermal, virt=0
  2538. EXC_REAL_END(cbe_thermal, 0x1800, 0x100)
  2539. EXC_VIRT_NONE(0x5800, 0x100)
  2540. EXC_COMMON_BEGIN(cbe_thermal_common)
  2541. GEN_COMMON cbe_thermal
  2542. addi r3,r1,STACK_FRAME_OVERHEAD
  2543. bl cbe_thermal_exception
  2544. b interrupt_return
  2545. GEN_KVM cbe_thermal
  2546. #else /* CONFIG_CBE_RAS */
  2547. EXC_REAL_NONE(0x1800, 0x100)
  2548. EXC_VIRT_NONE(0x5800, 0x100)
  2549. #endif
  2550. #ifdef CONFIG_PPC_WATCHDOG
  2551. INT_DEFINE_BEGIN(soft_nmi)
  2552. IVEC=0x900
  2553. ISTACK=0
  2554. IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */
  2555. INT_DEFINE_END(soft_nmi)
  2556. /*
  2557. * Branch to soft_nmi_interrupt using the emergency stack. The emergency
  2558. * stack is one that is usable by maskable interrupts so long as MSR_EE
  2559. * remains off. It is used for recovery when something has corrupted the
  2560. * normal kernel stack, for example. The "soft NMI" must not use the process
  2561. * stack because we want irq disabled sections to avoid touching the stack
  2562. * at all (other than PMU interrupts), so use the emergency stack for this,
  2563. * and run it entirely with interrupts hard disabled.
  2564. */
  2565. EXC_COMMON_BEGIN(soft_nmi_common)
  2566. mfspr r11,SPRN_SRR0
  2567. mr r10,r1
  2568. ld r1,PACAEMERGSP(r13)
  2569. subi r1,r1,INT_FRAME_SIZE
  2570. __GEN_COMMON_BODY soft_nmi
  2571. /*
  2572. * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see
  2573. * system_reset_common)
  2574. */
  2575. li r10,IRQS_ALL_DISABLED
  2576. stb r10,PACAIRQSOFTMASK(r13)
  2577. lbz r10,PACAIRQHAPPENED(r13)
  2578. std r10,RESULT(r1)
  2579. ori r10,r10,PACA_IRQ_HARD_DIS
  2580. stb r10,PACAIRQHAPPENED(r13)
  2581. addi r3,r1,STACK_FRAME_OVERHEAD
  2582. bl soft_nmi_interrupt
  2583. /* Clear MSR_RI before setting SRR0 and SRR1. */
  2584. li r9,0
  2585. mtmsrd r9,1
  2586. /*
  2587. * Restore soft mask settings.
  2588. */
  2589. ld r10,RESULT(r1)
  2590. stb r10,PACAIRQHAPPENED(r13)
  2591. ld r10,SOFTE(r1)
  2592. stb r10,PACAIRQSOFTMASK(r13)
  2593. kuap_restore_amr r9, r10
  2594. EXCEPTION_RESTORE_REGS hsrr=0
  2595. RFI_TO_KERNEL
  2596. #endif /* CONFIG_PPC_WATCHDOG */
  2597. /*
  2598. * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
  2599. * - If it was a decrementer interrupt, we bump the dec to max and and return.
  2600. * - If it was a doorbell we return immediately since doorbells are edge
  2601. * triggered and won't automatically refire.
  2602. * - If it was a HMI we return immediately since we handled it in realmode
  2603. * and it won't refire.
  2604. * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
  2605. * This is called with r10 containing the value to OR to the paca field.
  2606. */
  2607. .macro MASKED_INTERRUPT hsrr=0
  2608. .if \hsrr
  2609. masked_Hinterrupt:
  2610. .else
  2611. masked_interrupt:
  2612. .endif
  2613. lbz r11,PACAIRQHAPPENED(r13)
  2614. or r11,r11,r10
  2615. stb r11,PACAIRQHAPPENED(r13)
  2616. cmpwi r10,PACA_IRQ_DEC
  2617. bne 1f
  2618. lis r10,0x7fff
  2619. ori r10,r10,0xffff
  2620. mtspr SPRN_DEC,r10
  2621. #ifdef CONFIG_PPC_WATCHDOG
  2622. b soft_nmi_common
  2623. #else
  2624. b 2f
  2625. #endif
  2626. 1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK
  2627. beq 2f
  2628. xori r12,r12,MSR_EE /* clear MSR_EE */
  2629. .if \hsrr
  2630. mtspr SPRN_HSRR1,r12
  2631. .else
  2632. mtspr SPRN_SRR1,r12
  2633. .endif
  2634. ori r11,r11,PACA_IRQ_HARD_DIS
  2635. stb r11,PACAIRQHAPPENED(r13)
  2636. 2: /* done */
  2637. ld r10,PACA_EXGEN+EX_CTR(r13)
  2638. mtctr r10
  2639. mtcrf 0x80,r9
  2640. std r1,PACAR1(r13)
  2641. ld r9,PACA_EXGEN+EX_R9(r13)
  2642. ld r10,PACA_EXGEN+EX_R10(r13)
  2643. ld r11,PACA_EXGEN+EX_R11(r13)
  2644. ld r12,PACA_EXGEN+EX_R12(r13)
  2645. ld r13,PACA_EXGEN+EX_R13(r13)
  2646. /* May return to masked low address where r13 is not set up */
  2647. .if \hsrr
  2648. HRFI_TO_KERNEL
  2649. .else
  2650. RFI_TO_KERNEL
  2651. .endif
  2652. b .
  2653. .endm
  2654. TRAMP_REAL_BEGIN(stf_barrier_fallback)
  2655. std r9,PACA_EXRFI+EX_R9(r13)
  2656. std r10,PACA_EXRFI+EX_R10(r13)
  2657. sync
  2658. ld r9,PACA_EXRFI+EX_R9(r13)
  2659. ld r10,PACA_EXRFI+EX_R10(r13)
  2660. ori 31,31,0
  2661. .rept 14
  2662. b 1f
  2663. 1:
  2664. .endr
  2665. blr
  2666. /* Clobbers r10, r11, ctr */
  2667. .macro L1D_DISPLACEMENT_FLUSH
  2668. ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
  2669. ld r11,PACA_L1D_FLUSH_SIZE(r13)
  2670. srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
  2671. mtctr r11
  2672. DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
  2673. /* order ld/st prior to dcbt stop all streams with flushing */
  2674. sync
  2675. /*
  2676. * The load addresses are at staggered offsets within cachelines,
  2677. * which suits some pipelines better (on others it should not
  2678. * hurt).
  2679. */
  2680. 1:
  2681. ld r11,(0x80 + 8)*0(r10)
  2682. ld r11,(0x80 + 8)*1(r10)
  2683. ld r11,(0x80 + 8)*2(r10)
  2684. ld r11,(0x80 + 8)*3(r10)
  2685. ld r11,(0x80 + 8)*4(r10)
  2686. ld r11,(0x80 + 8)*5(r10)
  2687. ld r11,(0x80 + 8)*6(r10)
  2688. ld r11,(0x80 + 8)*7(r10)
  2689. addi r10,r10,0x80*8
  2690. bdnz 1b
  2691. .endm
  2692. TRAMP_REAL_BEGIN(entry_flush_fallback)
  2693. std r9,PACA_EXRFI+EX_R9(r13)
  2694. std r10,PACA_EXRFI+EX_R10(r13)
  2695. std r11,PACA_EXRFI+EX_R11(r13)
  2696. mfctr r9
  2697. L1D_DISPLACEMENT_FLUSH
  2698. mtctr r9
  2699. ld r9,PACA_EXRFI+EX_R9(r13)
  2700. ld r10,PACA_EXRFI+EX_R10(r13)
  2701. ld r11,PACA_EXRFI+EX_R11(r13)
  2702. blr
  2703. /*
  2704. * The SCV entry flush happens with interrupts enabled, so it must disable
  2705. * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10
  2706. * (containing LR) does not need to be preserved here because scv entry
  2707. * puts 0 in the pt_regs, CTR can be clobbered for the same reason.
  2708. */
  2709. TRAMP_REAL_BEGIN(scv_entry_flush_fallback)
  2710. li r10,0
  2711. mtmsrd r10,1
  2712. lbz r10,PACAIRQHAPPENED(r13)
  2713. ori r10,r10,PACA_IRQ_HARD_DIS
  2714. stb r10,PACAIRQHAPPENED(r13)
  2715. std r11,PACA_EXRFI+EX_R11(r13)
  2716. L1D_DISPLACEMENT_FLUSH
  2717. ld r11,PACA_EXRFI+EX_R11(r13)
  2718. li r10,MSR_RI
  2719. mtmsrd r10,1
  2720. blr
  2721. TRAMP_REAL_BEGIN(rfi_flush_fallback)
  2722. SET_SCRATCH0(r13);
  2723. GET_PACA(r13);
  2724. std r1,PACA_EXRFI+EX_R12(r13)
  2725. ld r1,PACAKSAVE(r13)
  2726. std r9,PACA_EXRFI+EX_R9(r13)
  2727. std r10,PACA_EXRFI+EX_R10(r13)
  2728. std r11,PACA_EXRFI+EX_R11(r13)
  2729. mfctr r9
  2730. L1D_DISPLACEMENT_FLUSH
  2731. mtctr r9
  2732. ld r9,PACA_EXRFI+EX_R9(r13)
  2733. ld r10,PACA_EXRFI+EX_R10(r13)
  2734. ld r11,PACA_EXRFI+EX_R11(r13)
  2735. ld r1,PACA_EXRFI+EX_R12(r13)
  2736. GET_SCRATCH0(r13);
  2737. rfid
  2738. TRAMP_REAL_BEGIN(hrfi_flush_fallback)
  2739. SET_SCRATCH0(r13);
  2740. GET_PACA(r13);
  2741. std r1,PACA_EXRFI+EX_R12(r13)
  2742. ld r1,PACAKSAVE(r13)
  2743. std r9,PACA_EXRFI+EX_R9(r13)
  2744. std r10,PACA_EXRFI+EX_R10(r13)
  2745. std r11,PACA_EXRFI+EX_R11(r13)
  2746. mfctr r9
  2747. L1D_DISPLACEMENT_FLUSH
  2748. mtctr r9
  2749. ld r9,PACA_EXRFI+EX_R9(r13)
  2750. ld r10,PACA_EXRFI+EX_R10(r13)
  2751. ld r11,PACA_EXRFI+EX_R11(r13)
  2752. ld r1,PACA_EXRFI+EX_R12(r13)
  2753. GET_SCRATCH0(r13);
  2754. hrfid
  2755. TRAMP_REAL_BEGIN(rfscv_flush_fallback)
  2756. /* system call volatile */
  2757. mr r7,r13
  2758. GET_PACA(r13);
  2759. mr r8,r1
  2760. ld r1,PACAKSAVE(r13)
  2761. mfctr r9
  2762. ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
  2763. ld r11,PACA_L1D_FLUSH_SIZE(r13)
  2764. srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
  2765. mtctr r11
  2766. DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
  2767. /* order ld/st prior to dcbt stop all streams with flushing */
  2768. sync
  2769. /*
  2770. * The load adresses are at staggered offsets within cachelines,
  2771. * which suits some pipelines better (on others it should not
  2772. * hurt).
  2773. */
  2774. 1:
  2775. ld r11,(0x80 + 8)*0(r10)
  2776. ld r11,(0x80 + 8)*1(r10)
  2777. ld r11,(0x80 + 8)*2(r10)
  2778. ld r11,(0x80 + 8)*3(r10)
  2779. ld r11,(0x80 + 8)*4(r10)
  2780. ld r11,(0x80 + 8)*5(r10)
  2781. ld r11,(0x80 + 8)*6(r10)
  2782. ld r11,(0x80 + 8)*7(r10)
  2783. addi r10,r10,0x80*8
  2784. bdnz 1b
  2785. mtctr r9
  2786. li r9,0
  2787. li r10,0
  2788. li r11,0
  2789. mr r1,r8
  2790. mr r13,r7
  2791. RFSCV
  2792. USE_TEXT_SECTION()
  2793. _GLOBAL(do_uaccess_flush)
  2794. UACCESS_FLUSH_FIXUP_SECTION
  2795. nop
  2796. nop
  2797. nop
  2798. blr
  2799. L1D_DISPLACEMENT_FLUSH
  2800. blr
  2801. _ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
  2802. EXPORT_SYMBOL(do_uaccess_flush)
  2803. MASKED_INTERRUPT
  2804. MASKED_INTERRUPT hsrr=1
  2805. #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
  2806. kvmppc_skip_interrupt:
  2807. /*
  2808. * Here all GPRs are unchanged from when the interrupt happened
  2809. * except for r13, which is saved in SPRG_SCRATCH0.
  2810. */
  2811. mfspr r13, SPRN_SRR0
  2812. addi r13, r13, 4
  2813. mtspr SPRN_SRR0, r13
  2814. GET_SCRATCH0(r13)
  2815. RFI_TO_KERNEL
  2816. b .
  2817. kvmppc_skip_Hinterrupt:
  2818. /*
  2819. * Here all GPRs are unchanged from when the interrupt happened
  2820. * except for r13, which is saved in SPRG_SCRATCH0.
  2821. */
  2822. mfspr r13, SPRN_HSRR0
  2823. addi r13, r13, 4
  2824. mtspr SPRN_HSRR0, r13
  2825. GET_SCRATCH0(r13)
  2826. HRFI_TO_KERNEL
  2827. b .
  2828. #endif
  2829. /*
  2830. * Relocation-on interrupts: A subset of the interrupts can be delivered
  2831. * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
  2832. * it. Addresses are the same as the original interrupt addresses, but
  2833. * offset by 0xc000000000004000.
  2834. * It's impossible to receive interrupts below 0x300 via this mechanism.
  2835. * KVM: None of these traps are from the guest ; anything that escalated
  2836. * to HV=1 from HV=0 is delivered via real mode handlers.
  2837. */
  2838. /*
  2839. * This uses the standard macro, since the original 0x300 vector
  2840. * only has extra guff for STAB-based processors -- which never
  2841. * come here.
  2842. */
  2843. EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
  2844. b __ppc64_runlatch_on
  2845. USE_FIXED_SECTION(virt_trampolines)
  2846. /*
  2847. * All code below __end_interrupts is treated as soft-masked. If
  2848. * any code runs here with MSR[EE]=1, it must then cope with pending
  2849. * soft interrupt being raised (i.e., by ensuring it is replayed).
  2850. *
  2851. * The __end_interrupts marker must be past the out-of-line (OOL)
  2852. * handlers, so that they are copied to real address 0x100 when running
  2853. * a relocatable kernel. This ensures they can be reached from the short
  2854. * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
  2855. * directly, without using LOAD_HANDLER().
  2856. */
  2857. .align 7
  2858. .globl __end_interrupts
  2859. __end_interrupts:
  2860. DEFINE_FIXED_SYMBOL(__end_interrupts)
  2861. #ifdef CONFIG_PPC_970_NAP
  2862. /*
  2863. * Called by exception entry code if _TLF_NAPPING was set, this clears
  2864. * the NAPPING flag, and redirects the exception exit to
  2865. * power4_fixup_nap_return.
  2866. */
  2867. .globl power4_fixup_nap
  2868. EXC_COMMON_BEGIN(power4_fixup_nap)
  2869. andc r9,r9,r10
  2870. std r9,TI_LOCAL_FLAGS(r11)
  2871. LOAD_REG_ADDR(r10, power4_idle_nap_return)
  2872. std r10,_NIP(r1)
  2873. blr
  2874. power4_idle_nap_return:
  2875. blr
  2876. #endif
  2877. CLOSE_FIXED_SECTION(real_vectors);
  2878. CLOSE_FIXED_SECTION(real_trampolines);
  2879. CLOSE_FIXED_SECTION(virt_vectors);
  2880. CLOSE_FIXED_SECTION(virt_trampolines);
  2881. USE_TEXT_SECTION()
  2882. /* MSR[RI] should be clear because this uses SRR[01] */
  2883. enable_machine_check:
  2884. mflr r0
  2885. bcl 20,31,$+4
  2886. 0: mflr r3
  2887. addi r3,r3,(1f - 0b)
  2888. mtspr SPRN_SRR0,r3
  2889. mfmsr r3
  2890. ori r3,r3,MSR_ME
  2891. mtspr SPRN_SRR1,r3
  2892. RFI_TO_KERNEL
  2893. 1: mtlr r0
  2894. blr
  2895. /* MSR[RI] should be clear because this uses SRR[01] */
  2896. disable_machine_check:
  2897. mflr r0
  2898. bcl 20,31,$+4
  2899. 0: mflr r3
  2900. addi r3,r3,(1f - 0b)
  2901. mtspr SPRN_SRR0,r3
  2902. mfmsr r3
  2903. li r4,MSR_ME
  2904. andc r3,r3,r4
  2905. mtspr SPRN_SRR1,r3
  2906. RFI_TO_KERNEL
  2907. 1: mtlr r0
  2908. blr
  2909. /*
  2910. * Hash table stuff
  2911. */
  2912. .balign IFETCH_ALIGN_BYTES
  2913. do_hash_page:
  2914. #ifdef CONFIG_PPC_BOOK3S_64
  2915. lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h
  2916. ori r0,r0,DSISR_BAD_FAULT_64S@l
  2917. and. r0,r5,r0 /* weird error? */
  2918. bne- handle_page_fault /* if not, try to insert a HPTE */
  2919. /*
  2920. * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
  2921. * don't call hash_page, just fail the fault. This is required to
  2922. * prevent re-entrancy problems in the hash code, namely perf
  2923. * interrupts hitting while something holds H_PAGE_BUSY, and taking a
  2924. * hash fault. See the comment in hash_preload().
  2925. */
  2926. ld r11, PACA_THREAD_INFO(r13)
  2927. lwz r0,TI_PREEMPT(r11)
  2928. andis. r0,r0,NMI_MASK@h
  2929. bne 77f
  2930. /*
  2931. * r3 contains the trap number
  2932. * r4 contains the faulting address
  2933. * r5 contains dsisr
  2934. * r6 msr
  2935. *
  2936. * at return r3 = 0 for success, 1 for page fault, negative for error
  2937. */
  2938. bl __hash_page /* build HPTE if possible */
  2939. cmpdi r3,0 /* see if __hash_page succeeded */
  2940. /* Success */
  2941. beq interrupt_return /* Return from exception on success */
  2942. /* Error */
  2943. blt- 13f
  2944. /* Reload DAR/DSISR into r4/r5 for the DABR check below */
  2945. ld r4,_DAR(r1)
  2946. ld r5,_DSISR(r1)
  2947. #endif /* CONFIG_PPC_BOOK3S_64 */
  2948. /* Here we have a page fault that hash_page can't handle. */
  2949. handle_page_fault:
  2950. 11: andis. r0,r5,DSISR_DABRMATCH@h
  2951. bne- handle_dabr_fault
  2952. addi r3,r1,STACK_FRAME_OVERHEAD
  2953. bl do_page_fault
  2954. cmpdi r3,0
  2955. beq+ interrupt_return
  2956. mr r5,r3
  2957. addi r3,r1,STACK_FRAME_OVERHEAD
  2958. ld r4,_DAR(r1)
  2959. bl bad_page_fault
  2960. b interrupt_return
  2961. /* We have a data breakpoint exception - handle it */
  2962. handle_dabr_fault:
  2963. ld r4,_DAR(r1)
  2964. ld r5,_DSISR(r1)
  2965. addi r3,r1,STACK_FRAME_OVERHEAD
  2966. bl do_break
  2967. /*
  2968. * do_break() may have changed the NV GPRS while handling a breakpoint.
  2969. * If so, we need to restore them with their updated values.
  2970. */
  2971. REST_NVGPRS(r1)
  2972. b interrupt_return
  2973. #ifdef CONFIG_PPC_BOOK3S_64
  2974. /* We have a page fault that hash_page could handle but HV refused
  2975. * the PTE insertion
  2976. */
  2977. 13: mr r5,r3
  2978. addi r3,r1,STACK_FRAME_OVERHEAD
  2979. ld r4,_DAR(r1)
  2980. bl low_hash_fault
  2981. b interrupt_return
  2982. #endif
  2983. /*
  2984. * We come here as a result of a DSI at a point where we don't want
  2985. * to call hash_page, such as when we are accessing memory (possibly
  2986. * user memory) inside a PMU interrupt that occurred while interrupts
  2987. * were soft-disabled. We want to invoke the exception handler for
  2988. * the access, or panic if there isn't a handler.
  2989. */
  2990. 77: addi r3,r1,STACK_FRAME_OVERHEAD
  2991. li r5,SIGSEGV
  2992. bl bad_page_fault
  2993. b interrupt_return