bookehv_interrupts.S 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. *
  4. * Copyright (C) 2010-2011 Freescale Semiconductor, Inc.
  5. *
  6. * Author: Varun Sethi <varun.sethi@freescale.com>
  7. * Author: Scott Wood <scotwood@freescale.com>
  8. * Author: Mihai Caraman <mihai.caraman@freescale.com>
  9. *
  10. * This file is derived from arch/powerpc/kvm/booke_interrupts.S
  11. */
  12. #include <asm/ppc_asm.h>
  13. #include <asm/kvm_asm.h>
  14. #include <asm/reg.h>
  15. #include <asm/page.h>
  16. #include <asm/asm-compat.h>
  17. #include <asm/asm-offsets.h>
  18. #include <asm/bitsperlong.h>
  19. #ifdef CONFIG_64BIT
  20. #include <asm/exception-64e.h>
  21. #include <asm/hw_irq.h>
  22. #include <asm/irqflags.h>
  23. #else
  24. #include "../kernel/head_booke.h" /* for THREAD_NORMSAVE() */
  25. #endif
  26. #define LONGBYTES (BITS_PER_LONG / 8)
  27. #define VCPU_GUEST_SPRG(n) (VCPU_GUEST_SPRGS + (n * LONGBYTES))
  28. /* The host stack layout: */
  29. #define HOST_R1 0 /* Implied by stwu. */
  30. #define HOST_CALLEE_LR PPC_LR_STKOFF
  31. #define HOST_RUN (HOST_CALLEE_LR + LONGBYTES)
  32. /*
  33. * r2 is special: it holds 'current', and it made nonvolatile in the
  34. * kernel with the -ffixed-r2 gcc option.
  35. */
  36. #define HOST_R2 (HOST_RUN + LONGBYTES)
  37. #define HOST_CR (HOST_R2 + LONGBYTES)
  38. #define HOST_NV_GPRS (HOST_CR + LONGBYTES)
  39. #define __HOST_NV_GPR(n) (HOST_NV_GPRS + ((n - 14) * LONGBYTES))
  40. #define HOST_NV_GPR(n) __HOST_NV_GPR(__REG_##n)
  41. #define HOST_MIN_STACK_SIZE (HOST_NV_GPR(R31) + LONGBYTES)
  42. #define HOST_STACK_SIZE ((HOST_MIN_STACK_SIZE + 15) & ~15) /* Align. */
  43. /* LR in caller stack frame. */
  44. #define HOST_STACK_LR (HOST_STACK_SIZE + PPC_LR_STKOFF)
  45. #define NEED_EMU 0x00000001 /* emulation -- save nv regs */
  46. #define NEED_DEAR 0x00000002 /* save faulting DEAR */
  47. #define NEED_ESR 0x00000004 /* save faulting ESR */
  48. /*
  49. * On entry:
  50. * r4 = vcpu, r5 = srr0, r6 = srr1
  51. * saved in vcpu: cr, ctr, r3-r13
  52. */
  53. .macro kvm_handler_common intno, srr0, flags
  54. /* Restore host stack pointer */
  55. PPC_STL r1, VCPU_GPR(R1)(r4)
  56. PPC_STL r2, VCPU_GPR(R2)(r4)
  57. PPC_LL r1, VCPU_HOST_STACK(r4)
  58. PPC_LL r2, HOST_R2(r1)
  59. START_BTB_FLUSH_SECTION
  60. BTB_FLUSH(r10)
  61. END_BTB_FLUSH_SECTION
  62. mfspr r10, SPRN_PID
  63. lwz r8, VCPU_HOST_PID(r4)
  64. PPC_LL r11, VCPU_SHARED(r4)
  65. PPC_STL r14, VCPU_GPR(R14)(r4) /* We need a non-volatile GPR. */
  66. li r14, \intno
  67. stw r10, VCPU_GUEST_PID(r4)
  68. mtspr SPRN_PID, r8
  69. #ifdef CONFIG_KVM_EXIT_TIMING
  70. /* save exit time */
  71. 1: mfspr r7, SPRN_TBRU
  72. mfspr r8, SPRN_TBRL
  73. mfspr r9, SPRN_TBRU
  74. cmpw r9, r7
  75. stw r8, VCPU_TIMING_EXIT_TBL(r4)
  76. bne- 1b
  77. stw r9, VCPU_TIMING_EXIT_TBU(r4)
  78. #endif
  79. oris r8, r6, MSR_CE@h
  80. PPC_STD(r6, VCPU_SHARED_MSR, r11)
  81. ori r8, r8, MSR_ME | MSR_RI
  82. PPC_STL r5, VCPU_PC(r4)
  83. /*
  84. * Make sure CE/ME/RI are set (if appropriate for exception type)
  85. * whether or not the guest had it set. Since mfmsr/mtmsr are
  86. * somewhat expensive, skip in the common case where the guest
  87. * had all these bits set (and thus they're still set if
  88. * appropriate for the exception type).
  89. */
  90. cmpw r6, r8
  91. beq 1f
  92. mfmsr r7
  93. .if \srr0 != SPRN_MCSRR0 && \srr0 != SPRN_CSRR0
  94. oris r7, r7, MSR_CE@h
  95. .endif
  96. .if \srr0 != SPRN_MCSRR0
  97. ori r7, r7, MSR_ME | MSR_RI
  98. .endif
  99. mtmsr r7
  100. 1:
  101. .if \flags & NEED_EMU
  102. PPC_STL r15, VCPU_GPR(R15)(r4)
  103. PPC_STL r16, VCPU_GPR(R16)(r4)
  104. PPC_STL r17, VCPU_GPR(R17)(r4)
  105. PPC_STL r18, VCPU_GPR(R18)(r4)
  106. PPC_STL r19, VCPU_GPR(R19)(r4)
  107. PPC_STL r20, VCPU_GPR(R20)(r4)
  108. PPC_STL r21, VCPU_GPR(R21)(r4)
  109. PPC_STL r22, VCPU_GPR(R22)(r4)
  110. PPC_STL r23, VCPU_GPR(R23)(r4)
  111. PPC_STL r24, VCPU_GPR(R24)(r4)
  112. PPC_STL r25, VCPU_GPR(R25)(r4)
  113. PPC_STL r26, VCPU_GPR(R26)(r4)
  114. PPC_STL r27, VCPU_GPR(R27)(r4)
  115. PPC_STL r28, VCPU_GPR(R28)(r4)
  116. PPC_STL r29, VCPU_GPR(R29)(r4)
  117. PPC_STL r30, VCPU_GPR(R30)(r4)
  118. PPC_STL r31, VCPU_GPR(R31)(r4)
  119. /*
  120. * We don't use external PID support. lwepx faults would need to be
  121. * handled by KVM and this implies aditional code in DO_KVM (for
  122. * DTB_MISS, DSI and LRAT) to check ESR[EPID] and EPLC[EGS] which
  123. * is too intrusive for the host. Get last instuction in
  124. * kvmppc_get_last_inst().
  125. */
  126. li r9, KVM_INST_FETCH_FAILED
  127. stw r9, VCPU_LAST_INST(r4)
  128. .endif
  129. .if \flags & NEED_ESR
  130. mfspr r8, SPRN_ESR
  131. PPC_STL r8, VCPU_FAULT_ESR(r4)
  132. .endif
  133. .if \flags & NEED_DEAR
  134. mfspr r9, SPRN_DEAR
  135. PPC_STL r9, VCPU_FAULT_DEAR(r4)
  136. .endif
  137. b kvmppc_resume_host
  138. .endm
  139. #ifdef CONFIG_64BIT
  140. /* Exception types */
  141. #define EX_GEN 1
  142. #define EX_GDBELL 2
  143. #define EX_DBG 3
  144. #define EX_MC 4
  145. #define EX_CRIT 5
  146. #define EX_TLB 6
  147. /*
  148. * For input register values, see arch/powerpc/include/asm/kvm_booke_hv_asm.h
  149. */
  150. .macro kvm_handler intno type scratch, paca_ex, ex_r10, ex_r11, srr0, srr1, flags
  151. _GLOBAL(kvmppc_handler_\intno\()_\srr1)
  152. mr r11, r4
  153. /*
  154. * Get vcpu from Paca: paca->__current.thread->kvm_vcpu
  155. */
  156. PPC_LL r4, PACACURRENT(r13)
  157. PPC_LL r4, (THREAD + THREAD_KVM_VCPU)(r4)
  158. PPC_STL r10, VCPU_CR(r4)
  159. PPC_STL r11, VCPU_GPR(R4)(r4)
  160. PPC_STL r5, VCPU_GPR(R5)(r4)
  161. PPC_STL r6, VCPU_GPR(R6)(r4)
  162. PPC_STL r8, VCPU_GPR(R8)(r4)
  163. PPC_STL r9, VCPU_GPR(R9)(r4)
  164. .if \type == EX_TLB
  165. PPC_LL r5, EX_TLB_R13(r12)
  166. PPC_LL r6, EX_TLB_R10(r12)
  167. PPC_LL r8, EX_TLB_R11(r12)
  168. mfspr r12, \scratch
  169. .else
  170. mfspr r5, \scratch
  171. PPC_LL r6, (\paca_ex + \ex_r10)(r13)
  172. PPC_LL r8, (\paca_ex + \ex_r11)(r13)
  173. .endif
  174. PPC_STL r5, VCPU_GPR(R13)(r4)
  175. PPC_STL r3, VCPU_GPR(R3)(r4)
  176. PPC_STL r7, VCPU_GPR(R7)(r4)
  177. PPC_STL r12, VCPU_GPR(R12)(r4)
  178. PPC_STL r6, VCPU_GPR(R10)(r4)
  179. PPC_STL r8, VCPU_GPR(R11)(r4)
  180. mfctr r5
  181. PPC_STL r5, VCPU_CTR(r4)
  182. mfspr r5, \srr0
  183. mfspr r6, \srr1
  184. kvm_handler_common \intno, \srr0, \flags
  185. .endm
  186. #define EX_PARAMS(type) \
  187. EX_##type, \
  188. SPRN_SPRG_##type##_SCRATCH, \
  189. PACA_EX##type, \
  190. EX_R10, \
  191. EX_R11
  192. #define EX_PARAMS_TLB \
  193. EX_TLB, \
  194. SPRN_SPRG_GEN_SCRATCH, \
  195. PACA_EXTLB, \
  196. EX_TLB_R10, \
  197. EX_TLB_R11
  198. kvm_handler BOOKE_INTERRUPT_CRITICAL, EX_PARAMS(CRIT), \
  199. SPRN_CSRR0, SPRN_CSRR1, 0
  200. kvm_handler BOOKE_INTERRUPT_MACHINE_CHECK, EX_PARAMS(MC), \
  201. SPRN_MCSRR0, SPRN_MCSRR1, 0
  202. kvm_handler BOOKE_INTERRUPT_DATA_STORAGE, EX_PARAMS(GEN), \
  203. SPRN_SRR0, SPRN_SRR1,(NEED_EMU | NEED_DEAR | NEED_ESR)
  204. kvm_handler BOOKE_INTERRUPT_INST_STORAGE, EX_PARAMS(GEN), \
  205. SPRN_SRR0, SPRN_SRR1, NEED_ESR
  206. kvm_handler BOOKE_INTERRUPT_EXTERNAL, EX_PARAMS(GEN), \
  207. SPRN_SRR0, SPRN_SRR1, 0
  208. kvm_handler BOOKE_INTERRUPT_ALIGNMENT, EX_PARAMS(GEN), \
  209. SPRN_SRR0, SPRN_SRR1,(NEED_DEAR | NEED_ESR)
  210. kvm_handler BOOKE_INTERRUPT_PROGRAM, EX_PARAMS(GEN), \
  211. SPRN_SRR0, SPRN_SRR1, (NEED_ESR | NEED_EMU)
  212. kvm_handler BOOKE_INTERRUPT_FP_UNAVAIL, EX_PARAMS(GEN), \
  213. SPRN_SRR0, SPRN_SRR1, 0
  214. kvm_handler BOOKE_INTERRUPT_AP_UNAVAIL, EX_PARAMS(GEN), \
  215. SPRN_SRR0, SPRN_SRR1, 0
  216. kvm_handler BOOKE_INTERRUPT_DECREMENTER, EX_PARAMS(GEN), \
  217. SPRN_SRR0, SPRN_SRR1, 0
  218. kvm_handler BOOKE_INTERRUPT_FIT, EX_PARAMS(GEN), \
  219. SPRN_SRR0, SPRN_SRR1, 0
  220. kvm_handler BOOKE_INTERRUPT_WATCHDOG, EX_PARAMS(CRIT),\
  221. SPRN_CSRR0, SPRN_CSRR1, 0
  222. /*
  223. * Only bolted TLB miss exception handlers are supported for now
  224. */
  225. kvm_handler BOOKE_INTERRUPT_DTLB_MISS, EX_PARAMS_TLB, \
  226. SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR)
  227. kvm_handler BOOKE_INTERRUPT_ITLB_MISS, EX_PARAMS_TLB, \
  228. SPRN_SRR0, SPRN_SRR1, 0
  229. kvm_handler BOOKE_INTERRUPT_ALTIVEC_UNAVAIL, EX_PARAMS(GEN), \
  230. SPRN_SRR0, SPRN_SRR1, 0
  231. kvm_handler BOOKE_INTERRUPT_ALTIVEC_ASSIST, EX_PARAMS(GEN), \
  232. SPRN_SRR0, SPRN_SRR1, 0
  233. kvm_handler BOOKE_INTERRUPT_PERFORMANCE_MONITOR, EX_PARAMS(GEN), \
  234. SPRN_SRR0, SPRN_SRR1, 0
  235. kvm_handler BOOKE_INTERRUPT_DOORBELL, EX_PARAMS(GEN), \
  236. SPRN_SRR0, SPRN_SRR1, 0
  237. kvm_handler BOOKE_INTERRUPT_DOORBELL_CRITICAL, EX_PARAMS(CRIT), \
  238. SPRN_CSRR0, SPRN_CSRR1, 0
  239. kvm_handler BOOKE_INTERRUPT_HV_PRIV, EX_PARAMS(GEN), \
  240. SPRN_SRR0, SPRN_SRR1, NEED_EMU
  241. kvm_handler BOOKE_INTERRUPT_HV_SYSCALL, EX_PARAMS(GEN), \
  242. SPRN_SRR0, SPRN_SRR1, 0
  243. kvm_handler BOOKE_INTERRUPT_GUEST_DBELL, EX_PARAMS(GDBELL), \
  244. SPRN_GSRR0, SPRN_GSRR1, 0
  245. kvm_handler BOOKE_INTERRUPT_GUEST_DBELL_CRIT, EX_PARAMS(CRIT), \
  246. SPRN_CSRR0, SPRN_CSRR1, 0
  247. kvm_handler BOOKE_INTERRUPT_DEBUG, EX_PARAMS(DBG), \
  248. SPRN_DSRR0, SPRN_DSRR1, 0
  249. kvm_handler BOOKE_INTERRUPT_DEBUG, EX_PARAMS(CRIT), \
  250. SPRN_CSRR0, SPRN_CSRR1, 0
  251. kvm_handler BOOKE_INTERRUPT_LRAT_ERROR, EX_PARAMS(GEN), \
  252. SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR)
  253. #else
  254. /*
  255. * For input register values, see arch/powerpc/include/asm/kvm_booke_hv_asm.h
  256. */
  257. .macro kvm_handler intno srr0, srr1, flags
  258. _GLOBAL(kvmppc_handler_\intno\()_\srr1)
  259. PPC_LL r11, THREAD_KVM_VCPU(r10)
  260. PPC_STL r3, VCPU_GPR(R3)(r11)
  261. mfspr r3, SPRN_SPRG_RSCRATCH0
  262. PPC_STL r4, VCPU_GPR(R4)(r11)
  263. PPC_LL r4, THREAD_NORMSAVE(0)(r10)
  264. PPC_STL r5, VCPU_GPR(R5)(r11)
  265. PPC_STL r13, VCPU_CR(r11)
  266. mfspr r5, \srr0
  267. PPC_STL r3, VCPU_GPR(R10)(r11)
  268. PPC_LL r3, THREAD_NORMSAVE(2)(r10)
  269. PPC_STL r6, VCPU_GPR(R6)(r11)
  270. PPC_STL r4, VCPU_GPR(R11)(r11)
  271. mfspr r6, \srr1
  272. PPC_STL r7, VCPU_GPR(R7)(r11)
  273. PPC_STL r8, VCPU_GPR(R8)(r11)
  274. PPC_STL r9, VCPU_GPR(R9)(r11)
  275. PPC_STL r3, VCPU_GPR(R13)(r11)
  276. mfctr r7
  277. PPC_STL r12, VCPU_GPR(R12)(r11)
  278. PPC_STL r7, VCPU_CTR(r11)
  279. mr r4, r11
  280. kvm_handler_common \intno, \srr0, \flags
  281. .endm
  282. .macro kvm_lvl_handler intno scratch srr0, srr1, flags
  283. _GLOBAL(kvmppc_handler_\intno\()_\srr1)
  284. mfspr r10, SPRN_SPRG_THREAD
  285. PPC_LL r11, THREAD_KVM_VCPU(r10)
  286. PPC_STL r3, VCPU_GPR(R3)(r11)
  287. mfspr r3, \scratch
  288. PPC_STL r4, VCPU_GPR(R4)(r11)
  289. PPC_LL r4, GPR9(r8)
  290. PPC_STL r5, VCPU_GPR(R5)(r11)
  291. PPC_STL r9, VCPU_CR(r11)
  292. mfspr r5, \srr0
  293. PPC_STL r3, VCPU_GPR(R8)(r11)
  294. PPC_LL r3, GPR10(r8)
  295. PPC_STL r6, VCPU_GPR(R6)(r11)
  296. PPC_STL r4, VCPU_GPR(R9)(r11)
  297. mfspr r6, \srr1
  298. PPC_LL r4, GPR11(r8)
  299. PPC_STL r7, VCPU_GPR(R7)(r11)
  300. PPC_STL r3, VCPU_GPR(R10)(r11)
  301. mfctr r7
  302. PPC_STL r12, VCPU_GPR(R12)(r11)
  303. PPC_STL r13, VCPU_GPR(R13)(r11)
  304. PPC_STL r4, VCPU_GPR(R11)(r11)
  305. PPC_STL r7, VCPU_CTR(r11)
  306. mr r4, r11
  307. kvm_handler_common \intno, \srr0, \flags
  308. .endm
  309. kvm_lvl_handler BOOKE_INTERRUPT_CRITICAL, \
  310. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  311. kvm_lvl_handler BOOKE_INTERRUPT_MACHINE_CHECK, \
  312. SPRN_SPRG_RSCRATCH_MC, SPRN_MCSRR0, SPRN_MCSRR1, 0
  313. kvm_handler BOOKE_INTERRUPT_DATA_STORAGE, \
  314. SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR)
  315. kvm_handler BOOKE_INTERRUPT_INST_STORAGE, SPRN_SRR0, SPRN_SRR1, NEED_ESR
  316. kvm_handler BOOKE_INTERRUPT_EXTERNAL, SPRN_SRR0, SPRN_SRR1, 0
  317. kvm_handler BOOKE_INTERRUPT_ALIGNMENT, \
  318. SPRN_SRR0, SPRN_SRR1, (NEED_DEAR | NEED_ESR)
  319. kvm_handler BOOKE_INTERRUPT_PROGRAM, SPRN_SRR0, SPRN_SRR1, (NEED_ESR | NEED_EMU)
  320. kvm_handler BOOKE_INTERRUPT_FP_UNAVAIL, SPRN_SRR0, SPRN_SRR1, 0
  321. kvm_handler BOOKE_INTERRUPT_SYSCALL, SPRN_SRR0, SPRN_SRR1, 0
  322. kvm_handler BOOKE_INTERRUPT_AP_UNAVAIL, SPRN_SRR0, SPRN_SRR1, 0
  323. kvm_handler BOOKE_INTERRUPT_DECREMENTER, SPRN_SRR0, SPRN_SRR1, 0
  324. kvm_handler BOOKE_INTERRUPT_FIT, SPRN_SRR0, SPRN_SRR1, 0
  325. kvm_lvl_handler BOOKE_INTERRUPT_WATCHDOG, \
  326. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  327. kvm_handler BOOKE_INTERRUPT_DTLB_MISS, \
  328. SPRN_SRR0, SPRN_SRR1, (NEED_EMU | NEED_DEAR | NEED_ESR)
  329. kvm_handler BOOKE_INTERRUPT_ITLB_MISS, SPRN_SRR0, SPRN_SRR1, 0
  330. kvm_handler BOOKE_INTERRUPT_PERFORMANCE_MONITOR, SPRN_SRR0, SPRN_SRR1, 0
  331. kvm_handler BOOKE_INTERRUPT_DOORBELL, SPRN_SRR0, SPRN_SRR1, 0
  332. kvm_lvl_handler BOOKE_INTERRUPT_DOORBELL_CRITICAL, \
  333. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  334. kvm_handler BOOKE_INTERRUPT_HV_PRIV, SPRN_SRR0, SPRN_SRR1, NEED_EMU
  335. kvm_handler BOOKE_INTERRUPT_HV_SYSCALL, SPRN_SRR0, SPRN_SRR1, 0
  336. kvm_handler BOOKE_INTERRUPT_GUEST_DBELL, SPRN_GSRR0, SPRN_GSRR1, 0
  337. kvm_lvl_handler BOOKE_INTERRUPT_GUEST_DBELL_CRIT, \
  338. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  339. kvm_lvl_handler BOOKE_INTERRUPT_DEBUG, \
  340. SPRN_SPRG_RSCRATCH_CRIT, SPRN_CSRR0, SPRN_CSRR1, 0
  341. kvm_lvl_handler BOOKE_INTERRUPT_DEBUG, \
  342. SPRN_SPRG_RSCRATCH_DBG, SPRN_DSRR0, SPRN_DSRR1, 0
  343. #endif
  344. /* Registers:
  345. * SPRG_SCRATCH0: guest r10
  346. * r4: vcpu pointer
  347. * r11: vcpu->arch.shared
  348. * r14: KVM exit number
  349. */
  350. _GLOBAL(kvmppc_resume_host)
  351. /* Save remaining volatile guest register state to vcpu. */
  352. mfspr r3, SPRN_VRSAVE
  353. PPC_STL r0, VCPU_GPR(R0)(r4)
  354. mflr r5
  355. mfspr r6, SPRN_SPRG4
  356. PPC_STL r5, VCPU_LR(r4)
  357. mfspr r7, SPRN_SPRG5
  358. stw r3, VCPU_VRSAVE(r4)
  359. #ifdef CONFIG_64BIT
  360. PPC_LL r3, PACA_SPRG_VDSO(r13)
  361. #endif
  362. mfspr r5, SPRN_SPRG9
  363. PPC_STD(r6, VCPU_SHARED_SPRG4, r11)
  364. mfspr r8, SPRN_SPRG6
  365. PPC_STD(r7, VCPU_SHARED_SPRG5, r11)
  366. mfspr r9, SPRN_SPRG7
  367. #ifdef CONFIG_64BIT
  368. mtspr SPRN_SPRG_VDSO_WRITE, r3
  369. #endif
  370. PPC_STD(r5, VCPU_SPRG9, r4)
  371. PPC_STD(r8, VCPU_SHARED_SPRG6, r11)
  372. mfxer r3
  373. PPC_STD(r9, VCPU_SHARED_SPRG7, r11)
  374. /* save guest MAS registers and restore host mas4 & mas6 */
  375. mfspr r5, SPRN_MAS0
  376. PPC_STL r3, VCPU_XER(r4)
  377. mfspr r6, SPRN_MAS1
  378. stw r5, VCPU_SHARED_MAS0(r11)
  379. mfspr r7, SPRN_MAS2
  380. stw r6, VCPU_SHARED_MAS1(r11)
  381. PPC_STD(r7, VCPU_SHARED_MAS2, r11)
  382. mfspr r5, SPRN_MAS3
  383. mfspr r6, SPRN_MAS4
  384. stw r5, VCPU_SHARED_MAS7_3+4(r11)
  385. mfspr r7, SPRN_MAS6
  386. stw r6, VCPU_SHARED_MAS4(r11)
  387. mfspr r5, SPRN_MAS7
  388. lwz r6, VCPU_HOST_MAS4(r4)
  389. stw r7, VCPU_SHARED_MAS6(r11)
  390. lwz r8, VCPU_HOST_MAS6(r4)
  391. mtspr SPRN_MAS4, r6
  392. stw r5, VCPU_SHARED_MAS7_3+0(r11)
  393. mtspr SPRN_MAS6, r8
  394. /* Enable MAS register updates via exception */
  395. mfspr r3, SPRN_EPCR
  396. rlwinm r3, r3, 0, ~SPRN_EPCR_DMIUH
  397. mtspr SPRN_EPCR, r3
  398. isync
  399. #ifdef CONFIG_64BIT
  400. /*
  401. * We enter with interrupts disabled in hardware, but
  402. * we need to call RECONCILE_IRQ_STATE to ensure
  403. * that the software state is kept in sync.
  404. */
  405. RECONCILE_IRQ_STATE(r3,r5)
  406. #endif
  407. /* Switch to kernel stack and jump to handler. */
  408. mr r3, r4
  409. mr r5, r14 /* intno */
  410. mr r14, r4 /* Save vcpu pointer. */
  411. mr r4, r5
  412. bl kvmppc_handle_exit
  413. /* Restore vcpu pointer and the nonvolatiles we used. */
  414. mr r4, r14
  415. PPC_LL r14, VCPU_GPR(R14)(r4)
  416. andi. r5, r3, RESUME_FLAG_NV
  417. beq skip_nv_load
  418. PPC_LL r15, VCPU_GPR(R15)(r4)
  419. PPC_LL r16, VCPU_GPR(R16)(r4)
  420. PPC_LL r17, VCPU_GPR(R17)(r4)
  421. PPC_LL r18, VCPU_GPR(R18)(r4)
  422. PPC_LL r19, VCPU_GPR(R19)(r4)
  423. PPC_LL r20, VCPU_GPR(R20)(r4)
  424. PPC_LL r21, VCPU_GPR(R21)(r4)
  425. PPC_LL r22, VCPU_GPR(R22)(r4)
  426. PPC_LL r23, VCPU_GPR(R23)(r4)
  427. PPC_LL r24, VCPU_GPR(R24)(r4)
  428. PPC_LL r25, VCPU_GPR(R25)(r4)
  429. PPC_LL r26, VCPU_GPR(R26)(r4)
  430. PPC_LL r27, VCPU_GPR(R27)(r4)
  431. PPC_LL r28, VCPU_GPR(R28)(r4)
  432. PPC_LL r29, VCPU_GPR(R29)(r4)
  433. PPC_LL r30, VCPU_GPR(R30)(r4)
  434. PPC_LL r31, VCPU_GPR(R31)(r4)
  435. skip_nv_load:
  436. /* Should we return to the guest? */
  437. andi. r5, r3, RESUME_FLAG_HOST
  438. beq lightweight_exit
  439. srawi r3, r3, 2 /* Shift -ERR back down. */
  440. heavyweight_exit:
  441. /* Not returning to guest. */
  442. PPC_LL r5, HOST_STACK_LR(r1)
  443. lwz r6, HOST_CR(r1)
  444. /*
  445. * We already saved guest volatile register state; now save the
  446. * non-volatiles.
  447. */
  448. PPC_STL r15, VCPU_GPR(R15)(r4)
  449. PPC_STL r16, VCPU_GPR(R16)(r4)
  450. PPC_STL r17, VCPU_GPR(R17)(r4)
  451. PPC_STL r18, VCPU_GPR(R18)(r4)
  452. PPC_STL r19, VCPU_GPR(R19)(r4)
  453. PPC_STL r20, VCPU_GPR(R20)(r4)
  454. PPC_STL r21, VCPU_GPR(R21)(r4)
  455. PPC_STL r22, VCPU_GPR(R22)(r4)
  456. PPC_STL r23, VCPU_GPR(R23)(r4)
  457. PPC_STL r24, VCPU_GPR(R24)(r4)
  458. PPC_STL r25, VCPU_GPR(R25)(r4)
  459. PPC_STL r26, VCPU_GPR(R26)(r4)
  460. PPC_STL r27, VCPU_GPR(R27)(r4)
  461. PPC_STL r28, VCPU_GPR(R28)(r4)
  462. PPC_STL r29, VCPU_GPR(R29)(r4)
  463. PPC_STL r30, VCPU_GPR(R30)(r4)
  464. PPC_STL r31, VCPU_GPR(R31)(r4)
  465. /* Load host non-volatile register state from host stack. */
  466. PPC_LL r14, HOST_NV_GPR(R14)(r1)
  467. PPC_LL r15, HOST_NV_GPR(R15)(r1)
  468. PPC_LL r16, HOST_NV_GPR(R16)(r1)
  469. PPC_LL r17, HOST_NV_GPR(R17)(r1)
  470. PPC_LL r18, HOST_NV_GPR(R18)(r1)
  471. PPC_LL r19, HOST_NV_GPR(R19)(r1)
  472. PPC_LL r20, HOST_NV_GPR(R20)(r1)
  473. PPC_LL r21, HOST_NV_GPR(R21)(r1)
  474. PPC_LL r22, HOST_NV_GPR(R22)(r1)
  475. PPC_LL r23, HOST_NV_GPR(R23)(r1)
  476. PPC_LL r24, HOST_NV_GPR(R24)(r1)
  477. PPC_LL r25, HOST_NV_GPR(R25)(r1)
  478. PPC_LL r26, HOST_NV_GPR(R26)(r1)
  479. PPC_LL r27, HOST_NV_GPR(R27)(r1)
  480. PPC_LL r28, HOST_NV_GPR(R28)(r1)
  481. PPC_LL r29, HOST_NV_GPR(R29)(r1)
  482. PPC_LL r30, HOST_NV_GPR(R30)(r1)
  483. PPC_LL r31, HOST_NV_GPR(R31)(r1)
  484. /* Return to kvm_vcpu_run(). */
  485. mtlr r5
  486. mtcr r6
  487. addi r1, r1, HOST_STACK_SIZE
  488. /* r3 still contains the return code from kvmppc_handle_exit(). */
  489. blr
  490. /* Registers:
  491. * r3: vcpu pointer
  492. */
  493. _GLOBAL(__kvmppc_vcpu_run)
  494. stwu r1, -HOST_STACK_SIZE(r1)
  495. PPC_STL r1, VCPU_HOST_STACK(r3) /* Save stack pointer to vcpu. */
  496. /* Save host state to stack. */
  497. mr r4, r3
  498. mflr r3
  499. mfcr r5
  500. PPC_STL r3, HOST_STACK_LR(r1)
  501. stw r5, HOST_CR(r1)
  502. /* Save host non-volatile register state to stack. */
  503. PPC_STL r14, HOST_NV_GPR(R14)(r1)
  504. PPC_STL r15, HOST_NV_GPR(R15)(r1)
  505. PPC_STL r16, HOST_NV_GPR(R16)(r1)
  506. PPC_STL r17, HOST_NV_GPR(R17)(r1)
  507. PPC_STL r18, HOST_NV_GPR(R18)(r1)
  508. PPC_STL r19, HOST_NV_GPR(R19)(r1)
  509. PPC_STL r20, HOST_NV_GPR(R20)(r1)
  510. PPC_STL r21, HOST_NV_GPR(R21)(r1)
  511. PPC_STL r22, HOST_NV_GPR(R22)(r1)
  512. PPC_STL r23, HOST_NV_GPR(R23)(r1)
  513. PPC_STL r24, HOST_NV_GPR(R24)(r1)
  514. PPC_STL r25, HOST_NV_GPR(R25)(r1)
  515. PPC_STL r26, HOST_NV_GPR(R26)(r1)
  516. PPC_STL r27, HOST_NV_GPR(R27)(r1)
  517. PPC_STL r28, HOST_NV_GPR(R28)(r1)
  518. PPC_STL r29, HOST_NV_GPR(R29)(r1)
  519. PPC_STL r30, HOST_NV_GPR(R30)(r1)
  520. PPC_STL r31, HOST_NV_GPR(R31)(r1)
  521. /* Load guest non-volatiles. */
  522. PPC_LL r14, VCPU_GPR(R14)(r4)
  523. PPC_LL r15, VCPU_GPR(R15)(r4)
  524. PPC_LL r16, VCPU_GPR(R16)(r4)
  525. PPC_LL r17, VCPU_GPR(R17)(r4)
  526. PPC_LL r18, VCPU_GPR(R18)(r4)
  527. PPC_LL r19, VCPU_GPR(R19)(r4)
  528. PPC_LL r20, VCPU_GPR(R20)(r4)
  529. PPC_LL r21, VCPU_GPR(R21)(r4)
  530. PPC_LL r22, VCPU_GPR(R22)(r4)
  531. PPC_LL r23, VCPU_GPR(R23)(r4)
  532. PPC_LL r24, VCPU_GPR(R24)(r4)
  533. PPC_LL r25, VCPU_GPR(R25)(r4)
  534. PPC_LL r26, VCPU_GPR(R26)(r4)
  535. PPC_LL r27, VCPU_GPR(R27)(r4)
  536. PPC_LL r28, VCPU_GPR(R28)(r4)
  537. PPC_LL r29, VCPU_GPR(R29)(r4)
  538. PPC_LL r30, VCPU_GPR(R30)(r4)
  539. PPC_LL r31, VCPU_GPR(R31)(r4)
  540. lightweight_exit:
  541. PPC_STL r2, HOST_R2(r1)
  542. mfspr r3, SPRN_PID
  543. stw r3, VCPU_HOST_PID(r4)
  544. lwz r3, VCPU_GUEST_PID(r4)
  545. mtspr SPRN_PID, r3
  546. PPC_LL r11, VCPU_SHARED(r4)
  547. /* Disable MAS register updates via exception */
  548. mfspr r3, SPRN_EPCR
  549. oris r3, r3, SPRN_EPCR_DMIUH@h
  550. mtspr SPRN_EPCR, r3
  551. isync
  552. /* Save host mas4 and mas6 and load guest MAS registers */
  553. mfspr r3, SPRN_MAS4
  554. stw r3, VCPU_HOST_MAS4(r4)
  555. mfspr r3, SPRN_MAS6
  556. stw r3, VCPU_HOST_MAS6(r4)
  557. lwz r3, VCPU_SHARED_MAS0(r11)
  558. lwz r5, VCPU_SHARED_MAS1(r11)
  559. PPC_LD(r6, VCPU_SHARED_MAS2, r11)
  560. lwz r7, VCPU_SHARED_MAS7_3+4(r11)
  561. lwz r8, VCPU_SHARED_MAS4(r11)
  562. mtspr SPRN_MAS0, r3
  563. mtspr SPRN_MAS1, r5
  564. mtspr SPRN_MAS2, r6
  565. mtspr SPRN_MAS3, r7
  566. mtspr SPRN_MAS4, r8
  567. lwz r3, VCPU_SHARED_MAS6(r11)
  568. lwz r5, VCPU_SHARED_MAS7_3+0(r11)
  569. mtspr SPRN_MAS6, r3
  570. mtspr SPRN_MAS7, r5
  571. /*
  572. * Host interrupt handlers may have clobbered these guest-readable
  573. * SPRGs, so we need to reload them here with the guest's values.
  574. */
  575. lwz r3, VCPU_VRSAVE(r4)
  576. PPC_LD(r5, VCPU_SHARED_SPRG4, r11)
  577. mtspr SPRN_VRSAVE, r3
  578. PPC_LD(r6, VCPU_SHARED_SPRG5, r11)
  579. mtspr SPRN_SPRG4W, r5
  580. PPC_LD(r7, VCPU_SHARED_SPRG6, r11)
  581. mtspr SPRN_SPRG5W, r6
  582. PPC_LD(r8, VCPU_SHARED_SPRG7, r11)
  583. mtspr SPRN_SPRG6W, r7
  584. PPC_LD(r5, VCPU_SPRG9, r4)
  585. mtspr SPRN_SPRG7W, r8
  586. mtspr SPRN_SPRG9, r5
  587. /* Load some guest volatiles. */
  588. PPC_LL r3, VCPU_LR(r4)
  589. PPC_LL r5, VCPU_XER(r4)
  590. PPC_LL r6, VCPU_CTR(r4)
  591. PPC_LL r7, VCPU_CR(r4)
  592. PPC_LL r8, VCPU_PC(r4)
  593. PPC_LD(r9, VCPU_SHARED_MSR, r11)
  594. PPC_LL r0, VCPU_GPR(R0)(r4)
  595. PPC_LL r1, VCPU_GPR(R1)(r4)
  596. PPC_LL r2, VCPU_GPR(R2)(r4)
  597. PPC_LL r10, VCPU_GPR(R10)(r4)
  598. PPC_LL r11, VCPU_GPR(R11)(r4)
  599. PPC_LL r12, VCPU_GPR(R12)(r4)
  600. PPC_LL r13, VCPU_GPR(R13)(r4)
  601. mtlr r3
  602. mtxer r5
  603. mtctr r6
  604. mtsrr0 r8
  605. mtsrr1 r9
  606. #ifdef CONFIG_KVM_EXIT_TIMING
  607. /* save enter time */
  608. 1:
  609. mfspr r6, SPRN_TBRU
  610. mfspr r9, SPRN_TBRL
  611. mfspr r8, SPRN_TBRU
  612. cmpw r8, r6
  613. stw r9, VCPU_TIMING_LAST_ENTER_TBL(r4)
  614. bne 1b
  615. stw r8, VCPU_TIMING_LAST_ENTER_TBU(r4)
  616. #endif
  617. /*
  618. * Don't execute any instruction which can change CR after
  619. * below instruction.
  620. */
  621. mtcr r7
  622. /* Finish loading guest volatiles and jump to guest. */
  623. PPC_LL r5, VCPU_GPR(R5)(r4)
  624. PPC_LL r6, VCPU_GPR(R6)(r4)
  625. PPC_LL r7, VCPU_GPR(R7)(r4)
  626. PPC_LL r8, VCPU_GPR(R8)(r4)
  627. PPC_LL r9, VCPU_GPR(R9)(r4)
  628. PPC_LL r3, VCPU_GPR(R3)(r4)
  629. PPC_LL r4, VCPU_GPR(R4)(r4)
  630. rfi