booke.c 55 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright IBM Corp. 2007
  5. * Copyright 2010-2011 Freescale Semiconductor, Inc.
  6. *
  7. * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  8. * Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
  9. * Scott Wood <scottwood@freescale.com>
  10. * Varun Sethi <varun.sethi@freescale.com>
  11. */
  12. #include <linux/errno.h>
  13. #include <linux/err.h>
  14. #include <linux/kvm_host.h>
  15. #include <linux/gfp.h>
  16. #include <linux/module.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/fs.h>
  19. #include <asm/cputable.h>
  20. #include <linux/uaccess.h>
  21. #include <asm/kvm_ppc.h>
  22. #include <asm/cacheflush.h>
  23. #include <asm/dbell.h>
  24. #include <asm/hw_irq.h>
  25. #include <asm/irq.h>
  26. #include <asm/time.h>
  27. #include "timing.h"
  28. #include "booke.h"
  29. #define CREATE_TRACE_POINTS
  30. #include "trace_booke.h"
  31. unsigned long kvmppc_booke_handlers;
  32. struct kvm_stats_debugfs_item debugfs_entries[] = {
  33. VCPU_STAT("mmio", mmio_exits),
  34. VCPU_STAT("sig", signal_exits),
  35. VCPU_STAT("itlb_r", itlb_real_miss_exits),
  36. VCPU_STAT("itlb_v", itlb_virt_miss_exits),
  37. VCPU_STAT("dtlb_r", dtlb_real_miss_exits),
  38. VCPU_STAT("dtlb_v", dtlb_virt_miss_exits),
  39. VCPU_STAT("sysc", syscall_exits),
  40. VCPU_STAT("isi", isi_exits),
  41. VCPU_STAT("dsi", dsi_exits),
  42. VCPU_STAT("inst_emu", emulated_inst_exits),
  43. VCPU_STAT("dec", dec_exits),
  44. VCPU_STAT("ext_intr", ext_intr_exits),
  45. VCPU_STAT("halt_successful_poll", halt_successful_poll),
  46. VCPU_STAT("halt_attempted_poll", halt_attempted_poll),
  47. VCPU_STAT("halt_poll_invalid", halt_poll_invalid),
  48. VCPU_STAT("halt_wakeup", halt_wakeup),
  49. VCPU_STAT("doorbell", dbell_exits),
  50. VCPU_STAT("guest doorbell", gdbell_exits),
  51. VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns),
  52. VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),
  53. VM_STAT("remote_tlb_flush", remote_tlb_flush),
  54. { NULL }
  55. };
  56. /* TODO: use vcpu_printf() */
  57. void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
  58. {
  59. int i;
  60. printk("pc: %08lx msr: %08llx\n", vcpu->arch.regs.nip,
  61. vcpu->arch.shared->msr);
  62. printk("lr: %08lx ctr: %08lx\n", vcpu->arch.regs.link,
  63. vcpu->arch.regs.ctr);
  64. printk("srr0: %08llx srr1: %08llx\n", vcpu->arch.shared->srr0,
  65. vcpu->arch.shared->srr1);
  66. printk("exceptions: %08lx\n", vcpu->arch.pending_exceptions);
  67. for (i = 0; i < 32; i += 4) {
  68. printk("gpr%02d: %08lx %08lx %08lx %08lx\n", i,
  69. kvmppc_get_gpr(vcpu, i),
  70. kvmppc_get_gpr(vcpu, i+1),
  71. kvmppc_get_gpr(vcpu, i+2),
  72. kvmppc_get_gpr(vcpu, i+3));
  73. }
  74. }
  75. #ifdef CONFIG_SPE
  76. void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu)
  77. {
  78. preempt_disable();
  79. enable_kernel_spe();
  80. kvmppc_save_guest_spe(vcpu);
  81. disable_kernel_spe();
  82. vcpu->arch.shadow_msr &= ~MSR_SPE;
  83. preempt_enable();
  84. }
  85. static void kvmppc_vcpu_enable_spe(struct kvm_vcpu *vcpu)
  86. {
  87. preempt_disable();
  88. enable_kernel_spe();
  89. kvmppc_load_guest_spe(vcpu);
  90. disable_kernel_spe();
  91. vcpu->arch.shadow_msr |= MSR_SPE;
  92. preempt_enable();
  93. }
  94. static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
  95. {
  96. if (vcpu->arch.shared->msr & MSR_SPE) {
  97. if (!(vcpu->arch.shadow_msr & MSR_SPE))
  98. kvmppc_vcpu_enable_spe(vcpu);
  99. } else if (vcpu->arch.shadow_msr & MSR_SPE) {
  100. kvmppc_vcpu_disable_spe(vcpu);
  101. }
  102. }
  103. #else
  104. static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
  105. {
  106. }
  107. #endif
  108. /*
  109. * Load up guest vcpu FP state if it's needed.
  110. * It also set the MSR_FP in thread so that host know
  111. * we're holding FPU, and then host can help to save
  112. * guest vcpu FP state if other threads require to use FPU.
  113. * This simulates an FP unavailable fault.
  114. *
  115. * It requires to be called with preemption disabled.
  116. */
  117. static inline void kvmppc_load_guest_fp(struct kvm_vcpu *vcpu)
  118. {
  119. #ifdef CONFIG_PPC_FPU
  120. if (!(current->thread.regs->msr & MSR_FP)) {
  121. enable_kernel_fp();
  122. load_fp_state(&vcpu->arch.fp);
  123. disable_kernel_fp();
  124. current->thread.fp_save_area = &vcpu->arch.fp;
  125. current->thread.regs->msr |= MSR_FP;
  126. }
  127. #endif
  128. }
  129. /*
  130. * Save guest vcpu FP state into thread.
  131. * It requires to be called with preemption disabled.
  132. */
  133. static inline void kvmppc_save_guest_fp(struct kvm_vcpu *vcpu)
  134. {
  135. #ifdef CONFIG_PPC_FPU
  136. if (current->thread.regs->msr & MSR_FP)
  137. giveup_fpu(current);
  138. current->thread.fp_save_area = NULL;
  139. #endif
  140. }
  141. static void kvmppc_vcpu_sync_fpu(struct kvm_vcpu *vcpu)
  142. {
  143. #if defined(CONFIG_PPC_FPU) && !defined(CONFIG_KVM_BOOKE_HV)
  144. /* We always treat the FP bit as enabled from the host
  145. perspective, so only need to adjust the shadow MSR */
  146. vcpu->arch.shadow_msr &= ~MSR_FP;
  147. vcpu->arch.shadow_msr |= vcpu->arch.shared->msr & MSR_FP;
  148. #endif
  149. }
  150. /*
  151. * Simulate AltiVec unavailable fault to load guest state
  152. * from thread to AltiVec unit.
  153. * It requires to be called with preemption disabled.
  154. */
  155. static inline void kvmppc_load_guest_altivec(struct kvm_vcpu *vcpu)
  156. {
  157. #ifdef CONFIG_ALTIVEC
  158. if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
  159. if (!(current->thread.regs->msr & MSR_VEC)) {
  160. enable_kernel_altivec();
  161. load_vr_state(&vcpu->arch.vr);
  162. disable_kernel_altivec();
  163. current->thread.vr_save_area = &vcpu->arch.vr;
  164. current->thread.regs->msr |= MSR_VEC;
  165. }
  166. }
  167. #endif
  168. }
  169. /*
  170. * Save guest vcpu AltiVec state into thread.
  171. * It requires to be called with preemption disabled.
  172. */
  173. static inline void kvmppc_save_guest_altivec(struct kvm_vcpu *vcpu)
  174. {
  175. #ifdef CONFIG_ALTIVEC
  176. if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
  177. if (current->thread.regs->msr & MSR_VEC)
  178. giveup_altivec(current);
  179. current->thread.vr_save_area = NULL;
  180. }
  181. #endif
  182. }
  183. static void kvmppc_vcpu_sync_debug(struct kvm_vcpu *vcpu)
  184. {
  185. /* Synchronize guest's desire to get debug interrupts into shadow MSR */
  186. #ifndef CONFIG_KVM_BOOKE_HV
  187. vcpu->arch.shadow_msr &= ~MSR_DE;
  188. vcpu->arch.shadow_msr |= vcpu->arch.shared->msr & MSR_DE;
  189. #endif
  190. /* Force enable debug interrupts when user space wants to debug */
  191. if (vcpu->guest_debug) {
  192. #ifdef CONFIG_KVM_BOOKE_HV
  193. /*
  194. * Since there is no shadow MSR, sync MSR_DE into the guest
  195. * visible MSR.
  196. */
  197. vcpu->arch.shared->msr |= MSR_DE;
  198. #else
  199. vcpu->arch.shadow_msr |= MSR_DE;
  200. vcpu->arch.shared->msr &= ~MSR_DE;
  201. #endif
  202. }
  203. }
  204. /*
  205. * Helper function for "full" MSR writes. No need to call this if only
  206. * EE/CE/ME/DE/RI are changing.
  207. */
  208. void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
  209. {
  210. u32 old_msr = vcpu->arch.shared->msr;
  211. #ifdef CONFIG_KVM_BOOKE_HV
  212. new_msr |= MSR_GS;
  213. #endif
  214. vcpu->arch.shared->msr = new_msr;
  215. kvmppc_mmu_msr_notify(vcpu, old_msr);
  216. kvmppc_vcpu_sync_spe(vcpu);
  217. kvmppc_vcpu_sync_fpu(vcpu);
  218. kvmppc_vcpu_sync_debug(vcpu);
  219. }
  220. static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
  221. unsigned int priority)
  222. {
  223. trace_kvm_booke_queue_irqprio(vcpu, priority);
  224. set_bit(priority, &vcpu->arch.pending_exceptions);
  225. }
  226. void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu,
  227. ulong dear_flags, ulong esr_flags)
  228. {
  229. vcpu->arch.queued_dear = dear_flags;
  230. vcpu->arch.queued_esr = esr_flags;
  231. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS);
  232. }
  233. void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu,
  234. ulong dear_flags, ulong esr_flags)
  235. {
  236. vcpu->arch.queued_dear = dear_flags;
  237. vcpu->arch.queued_esr = esr_flags;
  238. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DATA_STORAGE);
  239. }
  240. void kvmppc_core_queue_itlb_miss(struct kvm_vcpu *vcpu)
  241. {
  242. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
  243. }
  244. void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu, ulong esr_flags)
  245. {
  246. vcpu->arch.queued_esr = esr_flags;
  247. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_INST_STORAGE);
  248. }
  249. static void kvmppc_core_queue_alignment(struct kvm_vcpu *vcpu, ulong dear_flags,
  250. ulong esr_flags)
  251. {
  252. vcpu->arch.queued_dear = dear_flags;
  253. vcpu->arch.queued_esr = esr_flags;
  254. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALIGNMENT);
  255. }
  256. void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong esr_flags)
  257. {
  258. vcpu->arch.queued_esr = esr_flags;
  259. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_PROGRAM);
  260. }
  261. void kvmppc_core_queue_fpunavail(struct kvm_vcpu *vcpu)
  262. {
  263. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
  264. }
  265. #ifdef CONFIG_ALTIVEC
  266. void kvmppc_core_queue_vec_unavail(struct kvm_vcpu *vcpu)
  267. {
  268. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_UNAVAIL);
  269. }
  270. #endif
  271. void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
  272. {
  273. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DECREMENTER);
  274. }
  275. int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
  276. {
  277. return test_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
  278. }
  279. void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
  280. {
  281. clear_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
  282. }
  283. void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
  284. struct kvm_interrupt *irq)
  285. {
  286. unsigned int prio = BOOKE_IRQPRIO_EXTERNAL;
  287. if (irq->irq == KVM_INTERRUPT_SET_LEVEL)
  288. prio = BOOKE_IRQPRIO_EXTERNAL_LEVEL;
  289. kvmppc_booke_queue_irqprio(vcpu, prio);
  290. }
  291. void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu)
  292. {
  293. clear_bit(BOOKE_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
  294. clear_bit(BOOKE_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions);
  295. }
  296. static void kvmppc_core_queue_watchdog(struct kvm_vcpu *vcpu)
  297. {
  298. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_WATCHDOG);
  299. }
  300. static void kvmppc_core_dequeue_watchdog(struct kvm_vcpu *vcpu)
  301. {
  302. clear_bit(BOOKE_IRQPRIO_WATCHDOG, &vcpu->arch.pending_exceptions);
  303. }
  304. void kvmppc_core_queue_debug(struct kvm_vcpu *vcpu)
  305. {
  306. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DEBUG);
  307. }
  308. void kvmppc_core_dequeue_debug(struct kvm_vcpu *vcpu)
  309. {
  310. clear_bit(BOOKE_IRQPRIO_DEBUG, &vcpu->arch.pending_exceptions);
  311. }
  312. static void set_guest_srr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  313. {
  314. kvmppc_set_srr0(vcpu, srr0);
  315. kvmppc_set_srr1(vcpu, srr1);
  316. }
  317. static void set_guest_csrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  318. {
  319. vcpu->arch.csrr0 = srr0;
  320. vcpu->arch.csrr1 = srr1;
  321. }
  322. static void set_guest_dsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  323. {
  324. if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) {
  325. vcpu->arch.dsrr0 = srr0;
  326. vcpu->arch.dsrr1 = srr1;
  327. } else {
  328. set_guest_csrr(vcpu, srr0, srr1);
  329. }
  330. }
  331. static void set_guest_mcsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  332. {
  333. vcpu->arch.mcsrr0 = srr0;
  334. vcpu->arch.mcsrr1 = srr1;
  335. }
  336. /* Deliver the interrupt of the corresponding priority, if possible. */
  337. static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
  338. unsigned int priority)
  339. {
  340. int allowed = 0;
  341. ulong msr_mask = 0;
  342. bool update_esr = false, update_dear = false, update_epr = false;
  343. ulong crit_raw = vcpu->arch.shared->critical;
  344. ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
  345. bool crit;
  346. bool keep_irq = false;
  347. enum int_class int_class;
  348. ulong new_msr = vcpu->arch.shared->msr;
  349. /* Truncate crit indicators in 32 bit mode */
  350. if (!(vcpu->arch.shared->msr & MSR_SF)) {
  351. crit_raw &= 0xffffffff;
  352. crit_r1 &= 0xffffffff;
  353. }
  354. /* Critical section when crit == r1 */
  355. crit = (crit_raw == crit_r1);
  356. /* ... and we're in supervisor mode */
  357. crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
  358. if (priority == BOOKE_IRQPRIO_EXTERNAL_LEVEL) {
  359. priority = BOOKE_IRQPRIO_EXTERNAL;
  360. keep_irq = true;
  361. }
  362. if ((priority == BOOKE_IRQPRIO_EXTERNAL) && vcpu->arch.epr_flags)
  363. update_epr = true;
  364. switch (priority) {
  365. case BOOKE_IRQPRIO_DTLB_MISS:
  366. case BOOKE_IRQPRIO_DATA_STORAGE:
  367. case BOOKE_IRQPRIO_ALIGNMENT:
  368. update_dear = true;
  369. fallthrough;
  370. case BOOKE_IRQPRIO_INST_STORAGE:
  371. case BOOKE_IRQPRIO_PROGRAM:
  372. update_esr = true;
  373. fallthrough;
  374. case BOOKE_IRQPRIO_ITLB_MISS:
  375. case BOOKE_IRQPRIO_SYSCALL:
  376. case BOOKE_IRQPRIO_FP_UNAVAIL:
  377. #ifdef CONFIG_SPE_POSSIBLE
  378. case BOOKE_IRQPRIO_SPE_UNAVAIL:
  379. case BOOKE_IRQPRIO_SPE_FP_DATA:
  380. case BOOKE_IRQPRIO_SPE_FP_ROUND:
  381. #endif
  382. #ifdef CONFIG_ALTIVEC
  383. case BOOKE_IRQPRIO_ALTIVEC_UNAVAIL:
  384. case BOOKE_IRQPRIO_ALTIVEC_ASSIST:
  385. #endif
  386. case BOOKE_IRQPRIO_AP_UNAVAIL:
  387. allowed = 1;
  388. msr_mask = MSR_CE | MSR_ME | MSR_DE;
  389. int_class = INT_CLASS_NONCRIT;
  390. break;
  391. case BOOKE_IRQPRIO_WATCHDOG:
  392. case BOOKE_IRQPRIO_CRITICAL:
  393. case BOOKE_IRQPRIO_DBELL_CRIT:
  394. allowed = vcpu->arch.shared->msr & MSR_CE;
  395. allowed = allowed && !crit;
  396. msr_mask = MSR_ME;
  397. int_class = INT_CLASS_CRIT;
  398. break;
  399. case BOOKE_IRQPRIO_MACHINE_CHECK:
  400. allowed = vcpu->arch.shared->msr & MSR_ME;
  401. allowed = allowed && !crit;
  402. int_class = INT_CLASS_MC;
  403. break;
  404. case BOOKE_IRQPRIO_DECREMENTER:
  405. case BOOKE_IRQPRIO_FIT:
  406. keep_irq = true;
  407. fallthrough;
  408. case BOOKE_IRQPRIO_EXTERNAL:
  409. case BOOKE_IRQPRIO_DBELL:
  410. allowed = vcpu->arch.shared->msr & MSR_EE;
  411. allowed = allowed && !crit;
  412. msr_mask = MSR_CE | MSR_ME | MSR_DE;
  413. int_class = INT_CLASS_NONCRIT;
  414. break;
  415. case BOOKE_IRQPRIO_DEBUG:
  416. allowed = vcpu->arch.shared->msr & MSR_DE;
  417. allowed = allowed && !crit;
  418. msr_mask = MSR_ME;
  419. if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
  420. int_class = INT_CLASS_DBG;
  421. else
  422. int_class = INT_CLASS_CRIT;
  423. break;
  424. }
  425. if (allowed) {
  426. switch (int_class) {
  427. case INT_CLASS_NONCRIT:
  428. set_guest_srr(vcpu, vcpu->arch.regs.nip,
  429. vcpu->arch.shared->msr);
  430. break;
  431. case INT_CLASS_CRIT:
  432. set_guest_csrr(vcpu, vcpu->arch.regs.nip,
  433. vcpu->arch.shared->msr);
  434. break;
  435. case INT_CLASS_DBG:
  436. set_guest_dsrr(vcpu, vcpu->arch.regs.nip,
  437. vcpu->arch.shared->msr);
  438. break;
  439. case INT_CLASS_MC:
  440. set_guest_mcsrr(vcpu, vcpu->arch.regs.nip,
  441. vcpu->arch.shared->msr);
  442. break;
  443. }
  444. vcpu->arch.regs.nip = vcpu->arch.ivpr |
  445. vcpu->arch.ivor[priority];
  446. if (update_esr == true)
  447. kvmppc_set_esr(vcpu, vcpu->arch.queued_esr);
  448. if (update_dear == true)
  449. kvmppc_set_dar(vcpu, vcpu->arch.queued_dear);
  450. if (update_epr == true) {
  451. if (vcpu->arch.epr_flags & KVMPPC_EPR_USER)
  452. kvm_make_request(KVM_REQ_EPR_EXIT, vcpu);
  453. else if (vcpu->arch.epr_flags & KVMPPC_EPR_KERNEL) {
  454. BUG_ON(vcpu->arch.irq_type != KVMPPC_IRQ_MPIC);
  455. kvmppc_mpic_set_epr(vcpu);
  456. }
  457. }
  458. new_msr &= msr_mask;
  459. #if defined(CONFIG_64BIT)
  460. if (vcpu->arch.epcr & SPRN_EPCR_ICM)
  461. new_msr |= MSR_CM;
  462. #endif
  463. kvmppc_set_msr(vcpu, new_msr);
  464. if (!keep_irq)
  465. clear_bit(priority, &vcpu->arch.pending_exceptions);
  466. }
  467. #ifdef CONFIG_KVM_BOOKE_HV
  468. /*
  469. * If an interrupt is pending but masked, raise a guest doorbell
  470. * so that we are notified when the guest enables the relevant
  471. * MSR bit.
  472. */
  473. if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_EE)
  474. kvmppc_set_pending_interrupt(vcpu, INT_CLASS_NONCRIT);
  475. if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_CE)
  476. kvmppc_set_pending_interrupt(vcpu, INT_CLASS_CRIT);
  477. if (vcpu->arch.pending_exceptions & BOOKE_IRQPRIO_MACHINE_CHECK)
  478. kvmppc_set_pending_interrupt(vcpu, INT_CLASS_MC);
  479. #endif
  480. return allowed;
  481. }
  482. /*
  483. * Return the number of jiffies until the next timeout. If the timeout is
  484. * longer than the NEXT_TIMER_MAX_DELTA, then return NEXT_TIMER_MAX_DELTA
  485. * because the larger value can break the timer APIs.
  486. */
  487. static unsigned long watchdog_next_timeout(struct kvm_vcpu *vcpu)
  488. {
  489. u64 tb, wdt_tb, wdt_ticks = 0;
  490. u64 nr_jiffies = 0;
  491. u32 period = TCR_GET_WP(vcpu->arch.tcr);
  492. wdt_tb = 1ULL << (63 - period);
  493. tb = get_tb();
  494. /*
  495. * The watchdog timeout will hapeen when TB bit corresponding
  496. * to watchdog will toggle from 0 to 1.
  497. */
  498. if (tb & wdt_tb)
  499. wdt_ticks = wdt_tb;
  500. wdt_ticks += wdt_tb - (tb & (wdt_tb - 1));
  501. /* Convert timebase ticks to jiffies */
  502. nr_jiffies = wdt_ticks;
  503. if (do_div(nr_jiffies, tb_ticks_per_jiffy))
  504. nr_jiffies++;
  505. return min_t(unsigned long long, nr_jiffies, NEXT_TIMER_MAX_DELTA);
  506. }
  507. static void arm_next_watchdog(struct kvm_vcpu *vcpu)
  508. {
  509. unsigned long nr_jiffies;
  510. unsigned long flags;
  511. /*
  512. * If TSR_ENW and TSR_WIS are not set then no need to exit to
  513. * userspace, so clear the KVM_REQ_WATCHDOG request.
  514. */
  515. if ((vcpu->arch.tsr & (TSR_ENW | TSR_WIS)) != (TSR_ENW | TSR_WIS))
  516. kvm_clear_request(KVM_REQ_WATCHDOG, vcpu);
  517. spin_lock_irqsave(&vcpu->arch.wdt_lock, flags);
  518. nr_jiffies = watchdog_next_timeout(vcpu);
  519. /*
  520. * If the number of jiffies of watchdog timer >= NEXT_TIMER_MAX_DELTA
  521. * then do not run the watchdog timer as this can break timer APIs.
  522. */
  523. if (nr_jiffies < NEXT_TIMER_MAX_DELTA)
  524. mod_timer(&vcpu->arch.wdt_timer, jiffies + nr_jiffies);
  525. else
  526. del_timer(&vcpu->arch.wdt_timer);
  527. spin_unlock_irqrestore(&vcpu->arch.wdt_lock, flags);
  528. }
  529. void kvmppc_watchdog_func(struct timer_list *t)
  530. {
  531. struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.wdt_timer);
  532. u32 tsr, new_tsr;
  533. int final;
  534. do {
  535. new_tsr = tsr = vcpu->arch.tsr;
  536. final = 0;
  537. /* Time out event */
  538. if (tsr & TSR_ENW) {
  539. if (tsr & TSR_WIS)
  540. final = 1;
  541. else
  542. new_tsr = tsr | TSR_WIS;
  543. } else {
  544. new_tsr = tsr | TSR_ENW;
  545. }
  546. } while (cmpxchg(&vcpu->arch.tsr, tsr, new_tsr) != tsr);
  547. if (new_tsr & TSR_WIS) {
  548. smp_wmb();
  549. kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
  550. kvm_vcpu_kick(vcpu);
  551. }
  552. /*
  553. * If this is final watchdog expiry and some action is required
  554. * then exit to userspace.
  555. */
  556. if (final && (vcpu->arch.tcr & TCR_WRC_MASK) &&
  557. vcpu->arch.watchdog_enabled) {
  558. smp_wmb();
  559. kvm_make_request(KVM_REQ_WATCHDOG, vcpu);
  560. kvm_vcpu_kick(vcpu);
  561. }
  562. /*
  563. * Stop running the watchdog timer after final expiration to
  564. * prevent the host from being flooded with timers if the
  565. * guest sets a short period.
  566. * Timers will resume when TSR/TCR is updated next time.
  567. */
  568. if (!final)
  569. arm_next_watchdog(vcpu);
  570. }
  571. static void update_timer_ints(struct kvm_vcpu *vcpu)
  572. {
  573. if ((vcpu->arch.tcr & TCR_DIE) && (vcpu->arch.tsr & TSR_DIS))
  574. kvmppc_core_queue_dec(vcpu);
  575. else
  576. kvmppc_core_dequeue_dec(vcpu);
  577. if ((vcpu->arch.tcr & TCR_WIE) && (vcpu->arch.tsr & TSR_WIS))
  578. kvmppc_core_queue_watchdog(vcpu);
  579. else
  580. kvmppc_core_dequeue_watchdog(vcpu);
  581. }
  582. static void kvmppc_core_check_exceptions(struct kvm_vcpu *vcpu)
  583. {
  584. unsigned long *pending = &vcpu->arch.pending_exceptions;
  585. unsigned int priority;
  586. priority = __ffs(*pending);
  587. while (priority < BOOKE_IRQPRIO_MAX) {
  588. if (kvmppc_booke_irqprio_deliver(vcpu, priority))
  589. break;
  590. priority = find_next_bit(pending,
  591. BITS_PER_BYTE * sizeof(*pending),
  592. priority + 1);
  593. }
  594. /* Tell the guest about our interrupt status */
  595. vcpu->arch.shared->int_pending = !!*pending;
  596. }
  597. /* Check pending exceptions and deliver one, if possible. */
  598. int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu)
  599. {
  600. int r = 0;
  601. WARN_ON_ONCE(!irqs_disabled());
  602. kvmppc_core_check_exceptions(vcpu);
  603. if (kvm_request_pending(vcpu)) {
  604. /* Exception delivery raised request; start over */
  605. return 1;
  606. }
  607. if (vcpu->arch.shared->msr & MSR_WE) {
  608. local_irq_enable();
  609. kvm_vcpu_block(vcpu);
  610. kvm_clear_request(KVM_REQ_UNHALT, vcpu);
  611. hard_irq_disable();
  612. kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
  613. r = 1;
  614. };
  615. return r;
  616. }
  617. int kvmppc_core_check_requests(struct kvm_vcpu *vcpu)
  618. {
  619. int r = 1; /* Indicate we want to get back into the guest */
  620. if (kvm_check_request(KVM_REQ_PENDING_TIMER, vcpu))
  621. update_timer_ints(vcpu);
  622. #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
  623. if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
  624. kvmppc_core_flush_tlb(vcpu);
  625. #endif
  626. if (kvm_check_request(KVM_REQ_WATCHDOG, vcpu)) {
  627. vcpu->run->exit_reason = KVM_EXIT_WATCHDOG;
  628. r = 0;
  629. }
  630. if (kvm_check_request(KVM_REQ_EPR_EXIT, vcpu)) {
  631. vcpu->run->epr.epr = 0;
  632. vcpu->arch.epr_needed = true;
  633. vcpu->run->exit_reason = KVM_EXIT_EPR;
  634. r = 0;
  635. }
  636. return r;
  637. }
  638. int kvmppc_vcpu_run(struct kvm_vcpu *vcpu)
  639. {
  640. int ret, s;
  641. struct debug_reg debug;
  642. if (!vcpu->arch.sane) {
  643. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  644. return -EINVAL;
  645. }
  646. s = kvmppc_prepare_to_enter(vcpu);
  647. if (s <= 0) {
  648. ret = s;
  649. goto out;
  650. }
  651. /* interrupts now hard-disabled */
  652. #ifdef CONFIG_PPC_FPU
  653. /* Save userspace FPU state in stack */
  654. enable_kernel_fp();
  655. /*
  656. * Since we can't trap on MSR_FP in GS-mode, we consider the guest
  657. * as always using the FPU.
  658. */
  659. kvmppc_load_guest_fp(vcpu);
  660. #endif
  661. #ifdef CONFIG_ALTIVEC
  662. /* Save userspace AltiVec state in stack */
  663. if (cpu_has_feature(CPU_FTR_ALTIVEC))
  664. enable_kernel_altivec();
  665. /*
  666. * Since we can't trap on MSR_VEC in GS-mode, we consider the guest
  667. * as always using the AltiVec.
  668. */
  669. kvmppc_load_guest_altivec(vcpu);
  670. #endif
  671. /* Switch to guest debug context */
  672. debug = vcpu->arch.dbg_reg;
  673. switch_booke_debug_regs(&debug);
  674. debug = current->thread.debug;
  675. current->thread.debug = vcpu->arch.dbg_reg;
  676. vcpu->arch.pgdir = vcpu->kvm->mm->pgd;
  677. kvmppc_fix_ee_before_entry();
  678. ret = __kvmppc_vcpu_run(vcpu);
  679. /* No need for guest_exit. It's done in handle_exit.
  680. We also get here with interrupts enabled. */
  681. /* Switch back to user space debug context */
  682. switch_booke_debug_regs(&debug);
  683. current->thread.debug = debug;
  684. #ifdef CONFIG_PPC_FPU
  685. kvmppc_save_guest_fp(vcpu);
  686. #endif
  687. #ifdef CONFIG_ALTIVEC
  688. kvmppc_save_guest_altivec(vcpu);
  689. #endif
  690. out:
  691. vcpu->mode = OUTSIDE_GUEST_MODE;
  692. return ret;
  693. }
  694. static int emulation_exit(struct kvm_vcpu *vcpu)
  695. {
  696. enum emulation_result er;
  697. er = kvmppc_emulate_instruction(vcpu);
  698. switch (er) {
  699. case EMULATE_DONE:
  700. /* don't overwrite subtypes, just account kvm_stats */
  701. kvmppc_account_exit_stat(vcpu, EMULATED_INST_EXITS);
  702. /* Future optimization: only reload non-volatiles if
  703. * they were actually modified by emulation. */
  704. return RESUME_GUEST_NV;
  705. case EMULATE_AGAIN:
  706. return RESUME_GUEST;
  707. case EMULATE_FAIL:
  708. printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
  709. __func__, vcpu->arch.regs.nip, vcpu->arch.last_inst);
  710. /* For debugging, encode the failing instruction and
  711. * report it to userspace. */
  712. vcpu->run->hw.hardware_exit_reason = ~0ULL << 32;
  713. vcpu->run->hw.hardware_exit_reason |= vcpu->arch.last_inst;
  714. kvmppc_core_queue_program(vcpu, ESR_PIL);
  715. return RESUME_HOST;
  716. case EMULATE_EXIT_USER:
  717. return RESUME_HOST;
  718. default:
  719. BUG();
  720. }
  721. }
  722. static int kvmppc_handle_debug(struct kvm_vcpu *vcpu)
  723. {
  724. struct kvm_run *run = vcpu->run;
  725. struct debug_reg *dbg_reg = &(vcpu->arch.dbg_reg);
  726. u32 dbsr = vcpu->arch.dbsr;
  727. if (vcpu->guest_debug == 0) {
  728. /*
  729. * Debug resources belong to Guest.
  730. * Imprecise debug event is not injected
  731. */
  732. if (dbsr & DBSR_IDE) {
  733. dbsr &= ~DBSR_IDE;
  734. if (!dbsr)
  735. return RESUME_GUEST;
  736. }
  737. if (dbsr && (vcpu->arch.shared->msr & MSR_DE) &&
  738. (vcpu->arch.dbg_reg.dbcr0 & DBCR0_IDM))
  739. kvmppc_core_queue_debug(vcpu);
  740. /* Inject a program interrupt if trap debug is not allowed */
  741. if ((dbsr & DBSR_TIE) && !(vcpu->arch.shared->msr & MSR_DE))
  742. kvmppc_core_queue_program(vcpu, ESR_PTR);
  743. return RESUME_GUEST;
  744. }
  745. /*
  746. * Debug resource owned by userspace.
  747. * Clear guest dbsr (vcpu->arch.dbsr)
  748. */
  749. vcpu->arch.dbsr = 0;
  750. run->debug.arch.status = 0;
  751. run->debug.arch.address = vcpu->arch.regs.nip;
  752. if (dbsr & (DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4)) {
  753. run->debug.arch.status |= KVMPPC_DEBUG_BREAKPOINT;
  754. } else {
  755. if (dbsr & (DBSR_DAC1W | DBSR_DAC2W))
  756. run->debug.arch.status |= KVMPPC_DEBUG_WATCH_WRITE;
  757. else if (dbsr & (DBSR_DAC1R | DBSR_DAC2R))
  758. run->debug.arch.status |= KVMPPC_DEBUG_WATCH_READ;
  759. if (dbsr & (DBSR_DAC1R | DBSR_DAC1W))
  760. run->debug.arch.address = dbg_reg->dac1;
  761. else if (dbsr & (DBSR_DAC2R | DBSR_DAC2W))
  762. run->debug.arch.address = dbg_reg->dac2;
  763. }
  764. return RESUME_HOST;
  765. }
  766. static void kvmppc_fill_pt_regs(struct pt_regs *regs)
  767. {
  768. ulong r1, ip, msr, lr;
  769. asm("mr %0, 1" : "=r"(r1));
  770. asm("mflr %0" : "=r"(lr));
  771. asm("mfmsr %0" : "=r"(msr));
  772. asm("bl 1f; 1: mflr %0" : "=r"(ip));
  773. memset(regs, 0, sizeof(*regs));
  774. regs->gpr[1] = r1;
  775. regs->nip = ip;
  776. regs->msr = msr;
  777. regs->link = lr;
  778. }
  779. /*
  780. * For interrupts needed to be handled by host interrupt handlers,
  781. * corresponding host handler are called from here in similar way
  782. * (but not exact) as they are called from low level handler
  783. * (such as from arch/powerpc/kernel/head_fsl_booke.S).
  784. */
  785. static void kvmppc_restart_interrupt(struct kvm_vcpu *vcpu,
  786. unsigned int exit_nr)
  787. {
  788. struct pt_regs regs;
  789. switch (exit_nr) {
  790. case BOOKE_INTERRUPT_EXTERNAL:
  791. kvmppc_fill_pt_regs(&regs);
  792. do_IRQ(&regs);
  793. break;
  794. case BOOKE_INTERRUPT_DECREMENTER:
  795. kvmppc_fill_pt_regs(&regs);
  796. timer_interrupt(&regs);
  797. break;
  798. #if defined(CONFIG_PPC_DOORBELL)
  799. case BOOKE_INTERRUPT_DOORBELL:
  800. kvmppc_fill_pt_regs(&regs);
  801. doorbell_exception(&regs);
  802. break;
  803. #endif
  804. case BOOKE_INTERRUPT_MACHINE_CHECK:
  805. /* FIXME */
  806. break;
  807. case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
  808. kvmppc_fill_pt_regs(&regs);
  809. performance_monitor_exception(&regs);
  810. break;
  811. case BOOKE_INTERRUPT_WATCHDOG:
  812. kvmppc_fill_pt_regs(&regs);
  813. #ifdef CONFIG_BOOKE_WDT
  814. WatchdogException(&regs);
  815. #else
  816. unknown_exception(&regs);
  817. #endif
  818. break;
  819. case BOOKE_INTERRUPT_CRITICAL:
  820. kvmppc_fill_pt_regs(&regs);
  821. unknown_exception(&regs);
  822. break;
  823. case BOOKE_INTERRUPT_DEBUG:
  824. /* Save DBSR before preemption is enabled */
  825. vcpu->arch.dbsr = mfspr(SPRN_DBSR);
  826. kvmppc_clear_dbsr();
  827. break;
  828. }
  829. }
  830. static int kvmppc_resume_inst_load(struct kvm_vcpu *vcpu,
  831. enum emulation_result emulated, u32 last_inst)
  832. {
  833. switch (emulated) {
  834. case EMULATE_AGAIN:
  835. return RESUME_GUEST;
  836. case EMULATE_FAIL:
  837. pr_debug("%s: load instruction from guest address %lx failed\n",
  838. __func__, vcpu->arch.regs.nip);
  839. /* For debugging, encode the failing instruction and
  840. * report it to userspace. */
  841. vcpu->run->hw.hardware_exit_reason = ~0ULL << 32;
  842. vcpu->run->hw.hardware_exit_reason |= last_inst;
  843. kvmppc_core_queue_program(vcpu, ESR_PIL);
  844. return RESUME_HOST;
  845. default:
  846. BUG();
  847. }
  848. }
  849. /**
  850. * kvmppc_handle_exit
  851. *
  852. * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
  853. */
  854. int kvmppc_handle_exit(struct kvm_vcpu *vcpu, unsigned int exit_nr)
  855. {
  856. struct kvm_run *run = vcpu->run;
  857. int r = RESUME_HOST;
  858. int s;
  859. int idx;
  860. u32 last_inst = KVM_INST_FETCH_FAILED;
  861. enum emulation_result emulated = EMULATE_DONE;
  862. /* update before a new last_exit_type is rewritten */
  863. kvmppc_update_timing_stats(vcpu);
  864. /* restart interrupts if they were meant for the host */
  865. kvmppc_restart_interrupt(vcpu, exit_nr);
  866. /*
  867. * get last instruction before being preempted
  868. * TODO: for e6500 check also BOOKE_INTERRUPT_LRAT_ERROR & ESR_DATA
  869. */
  870. switch (exit_nr) {
  871. case BOOKE_INTERRUPT_DATA_STORAGE:
  872. case BOOKE_INTERRUPT_DTLB_MISS:
  873. case BOOKE_INTERRUPT_HV_PRIV:
  874. emulated = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
  875. break;
  876. case BOOKE_INTERRUPT_PROGRAM:
  877. /* SW breakpoints arrive as illegal instructions on HV */
  878. if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
  879. emulated = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
  880. break;
  881. default:
  882. break;
  883. }
  884. trace_kvm_exit(exit_nr, vcpu);
  885. guest_exit_irqoff();
  886. local_irq_enable();
  887. run->exit_reason = KVM_EXIT_UNKNOWN;
  888. run->ready_for_interrupt_injection = 1;
  889. if (emulated != EMULATE_DONE) {
  890. r = kvmppc_resume_inst_load(vcpu, emulated, last_inst);
  891. goto out;
  892. }
  893. switch (exit_nr) {
  894. case BOOKE_INTERRUPT_MACHINE_CHECK:
  895. printk("MACHINE CHECK: %lx\n", mfspr(SPRN_MCSR));
  896. kvmppc_dump_vcpu(vcpu);
  897. /* For debugging, send invalid exit reason to user space */
  898. run->hw.hardware_exit_reason = ~1ULL << 32;
  899. run->hw.hardware_exit_reason |= mfspr(SPRN_MCSR);
  900. r = RESUME_HOST;
  901. break;
  902. case BOOKE_INTERRUPT_EXTERNAL:
  903. kvmppc_account_exit(vcpu, EXT_INTR_EXITS);
  904. r = RESUME_GUEST;
  905. break;
  906. case BOOKE_INTERRUPT_DECREMENTER:
  907. kvmppc_account_exit(vcpu, DEC_EXITS);
  908. r = RESUME_GUEST;
  909. break;
  910. case BOOKE_INTERRUPT_WATCHDOG:
  911. r = RESUME_GUEST;
  912. break;
  913. case BOOKE_INTERRUPT_DOORBELL:
  914. kvmppc_account_exit(vcpu, DBELL_EXITS);
  915. r = RESUME_GUEST;
  916. break;
  917. case BOOKE_INTERRUPT_GUEST_DBELL_CRIT:
  918. kvmppc_account_exit(vcpu, GDBELL_EXITS);
  919. /*
  920. * We are here because there is a pending guest interrupt
  921. * which could not be delivered as MSR_CE or MSR_ME was not
  922. * set. Once we break from here we will retry delivery.
  923. */
  924. r = RESUME_GUEST;
  925. break;
  926. case BOOKE_INTERRUPT_GUEST_DBELL:
  927. kvmppc_account_exit(vcpu, GDBELL_EXITS);
  928. /*
  929. * We are here because there is a pending guest interrupt
  930. * which could not be delivered as MSR_EE was not set. Once
  931. * we break from here we will retry delivery.
  932. */
  933. r = RESUME_GUEST;
  934. break;
  935. case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
  936. r = RESUME_GUEST;
  937. break;
  938. case BOOKE_INTERRUPT_HV_PRIV:
  939. r = emulation_exit(vcpu);
  940. break;
  941. case BOOKE_INTERRUPT_PROGRAM:
  942. if ((vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) &&
  943. (last_inst == KVMPPC_INST_SW_BREAKPOINT)) {
  944. /*
  945. * We are here because of an SW breakpoint instr,
  946. * so lets return to host to handle.
  947. */
  948. r = kvmppc_handle_debug(vcpu);
  949. run->exit_reason = KVM_EXIT_DEBUG;
  950. kvmppc_account_exit(vcpu, DEBUG_EXITS);
  951. break;
  952. }
  953. if (vcpu->arch.shared->msr & (MSR_PR | MSR_GS)) {
  954. /*
  955. * Program traps generated by user-level software must
  956. * be handled by the guest kernel.
  957. *
  958. * In GS mode, hypervisor privileged instructions trap
  959. * on BOOKE_INTERRUPT_HV_PRIV, not here, so these are
  960. * actual program interrupts, handled by the guest.
  961. */
  962. kvmppc_core_queue_program(vcpu, vcpu->arch.fault_esr);
  963. r = RESUME_GUEST;
  964. kvmppc_account_exit(vcpu, USR_PR_INST);
  965. break;
  966. }
  967. r = emulation_exit(vcpu);
  968. break;
  969. case BOOKE_INTERRUPT_FP_UNAVAIL:
  970. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
  971. kvmppc_account_exit(vcpu, FP_UNAVAIL);
  972. r = RESUME_GUEST;
  973. break;
  974. #ifdef CONFIG_SPE
  975. case BOOKE_INTERRUPT_SPE_UNAVAIL: {
  976. if (vcpu->arch.shared->msr & MSR_SPE)
  977. kvmppc_vcpu_enable_spe(vcpu);
  978. else
  979. kvmppc_booke_queue_irqprio(vcpu,
  980. BOOKE_IRQPRIO_SPE_UNAVAIL);
  981. r = RESUME_GUEST;
  982. break;
  983. }
  984. case BOOKE_INTERRUPT_SPE_FP_DATA:
  985. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
  986. r = RESUME_GUEST;
  987. break;
  988. case BOOKE_INTERRUPT_SPE_FP_ROUND:
  989. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
  990. r = RESUME_GUEST;
  991. break;
  992. #elif defined(CONFIG_SPE_POSSIBLE)
  993. case BOOKE_INTERRUPT_SPE_UNAVAIL:
  994. /*
  995. * Guest wants SPE, but host kernel doesn't support it. Send
  996. * an "unimplemented operation" program check to the guest.
  997. */
  998. kvmppc_core_queue_program(vcpu, ESR_PUO | ESR_SPV);
  999. r = RESUME_GUEST;
  1000. break;
  1001. /*
  1002. * These really should never happen without CONFIG_SPE,
  1003. * as we should never enable the real MSR[SPE] in the guest.
  1004. */
  1005. case BOOKE_INTERRUPT_SPE_FP_DATA:
  1006. case BOOKE_INTERRUPT_SPE_FP_ROUND:
  1007. printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
  1008. __func__, exit_nr, vcpu->arch.regs.nip);
  1009. run->hw.hardware_exit_reason = exit_nr;
  1010. r = RESUME_HOST;
  1011. break;
  1012. #endif /* CONFIG_SPE_POSSIBLE */
  1013. /*
  1014. * On cores with Vector category, KVM is loaded only if CONFIG_ALTIVEC,
  1015. * see kvmppc_core_check_processor_compat().
  1016. */
  1017. #ifdef CONFIG_ALTIVEC
  1018. case BOOKE_INTERRUPT_ALTIVEC_UNAVAIL:
  1019. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_UNAVAIL);
  1020. r = RESUME_GUEST;
  1021. break;
  1022. case BOOKE_INTERRUPT_ALTIVEC_ASSIST:
  1023. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_ASSIST);
  1024. r = RESUME_GUEST;
  1025. break;
  1026. #endif
  1027. case BOOKE_INTERRUPT_DATA_STORAGE:
  1028. kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear,
  1029. vcpu->arch.fault_esr);
  1030. kvmppc_account_exit(vcpu, DSI_EXITS);
  1031. r = RESUME_GUEST;
  1032. break;
  1033. case BOOKE_INTERRUPT_INST_STORAGE:
  1034. kvmppc_core_queue_inst_storage(vcpu, vcpu->arch.fault_esr);
  1035. kvmppc_account_exit(vcpu, ISI_EXITS);
  1036. r = RESUME_GUEST;
  1037. break;
  1038. case BOOKE_INTERRUPT_ALIGNMENT:
  1039. kvmppc_core_queue_alignment(vcpu, vcpu->arch.fault_dear,
  1040. vcpu->arch.fault_esr);
  1041. r = RESUME_GUEST;
  1042. break;
  1043. #ifdef CONFIG_KVM_BOOKE_HV
  1044. case BOOKE_INTERRUPT_HV_SYSCALL:
  1045. if (!(vcpu->arch.shared->msr & MSR_PR)) {
  1046. kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
  1047. } else {
  1048. /*
  1049. * hcall from guest userspace -- send privileged
  1050. * instruction program check.
  1051. */
  1052. kvmppc_core_queue_program(vcpu, ESR_PPR);
  1053. }
  1054. r = RESUME_GUEST;
  1055. break;
  1056. #else
  1057. case BOOKE_INTERRUPT_SYSCALL:
  1058. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  1059. (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
  1060. /* KVM PV hypercalls */
  1061. kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
  1062. r = RESUME_GUEST;
  1063. } else {
  1064. /* Guest syscalls */
  1065. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SYSCALL);
  1066. }
  1067. kvmppc_account_exit(vcpu, SYSCALL_EXITS);
  1068. r = RESUME_GUEST;
  1069. break;
  1070. #endif
  1071. case BOOKE_INTERRUPT_DTLB_MISS: {
  1072. unsigned long eaddr = vcpu->arch.fault_dear;
  1073. int gtlb_index;
  1074. gpa_t gpaddr;
  1075. gfn_t gfn;
  1076. #ifdef CONFIG_KVM_E500V2
  1077. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  1078. (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
  1079. kvmppc_map_magic(vcpu);
  1080. kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
  1081. r = RESUME_GUEST;
  1082. break;
  1083. }
  1084. #endif
  1085. /* Check the guest TLB. */
  1086. gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
  1087. if (gtlb_index < 0) {
  1088. /* The guest didn't have a mapping for it. */
  1089. kvmppc_core_queue_dtlb_miss(vcpu,
  1090. vcpu->arch.fault_dear,
  1091. vcpu->arch.fault_esr);
  1092. kvmppc_mmu_dtlb_miss(vcpu);
  1093. kvmppc_account_exit(vcpu, DTLB_REAL_MISS_EXITS);
  1094. r = RESUME_GUEST;
  1095. break;
  1096. }
  1097. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1098. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  1099. gfn = gpaddr >> PAGE_SHIFT;
  1100. if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  1101. /* The guest TLB had a mapping, but the shadow TLB
  1102. * didn't, and it is RAM. This could be because:
  1103. * a) the entry is mapping the host kernel, or
  1104. * b) the guest used a large mapping which we're faking
  1105. * Either way, we need to satisfy the fault without
  1106. * invoking the guest. */
  1107. kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
  1108. kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
  1109. r = RESUME_GUEST;
  1110. } else {
  1111. /* Guest has mapped and accessed a page which is not
  1112. * actually RAM. */
  1113. vcpu->arch.paddr_accessed = gpaddr;
  1114. vcpu->arch.vaddr_accessed = eaddr;
  1115. r = kvmppc_emulate_mmio(vcpu);
  1116. kvmppc_account_exit(vcpu, MMIO_EXITS);
  1117. }
  1118. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1119. break;
  1120. }
  1121. case BOOKE_INTERRUPT_ITLB_MISS: {
  1122. unsigned long eaddr = vcpu->arch.regs.nip;
  1123. gpa_t gpaddr;
  1124. gfn_t gfn;
  1125. int gtlb_index;
  1126. r = RESUME_GUEST;
  1127. /* Check the guest TLB. */
  1128. gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
  1129. if (gtlb_index < 0) {
  1130. /* The guest didn't have a mapping for it. */
  1131. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
  1132. kvmppc_mmu_itlb_miss(vcpu);
  1133. kvmppc_account_exit(vcpu, ITLB_REAL_MISS_EXITS);
  1134. break;
  1135. }
  1136. kvmppc_account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
  1137. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1138. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  1139. gfn = gpaddr >> PAGE_SHIFT;
  1140. if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  1141. /* The guest TLB had a mapping, but the shadow TLB
  1142. * didn't. This could be because:
  1143. * a) the entry is mapping the host kernel, or
  1144. * b) the guest used a large mapping which we're faking
  1145. * Either way, we need to satisfy the fault without
  1146. * invoking the guest. */
  1147. kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
  1148. } else {
  1149. /* Guest mapped and leaped at non-RAM! */
  1150. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_MACHINE_CHECK);
  1151. }
  1152. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1153. break;
  1154. }
  1155. case BOOKE_INTERRUPT_DEBUG: {
  1156. r = kvmppc_handle_debug(vcpu);
  1157. if (r == RESUME_HOST)
  1158. run->exit_reason = KVM_EXIT_DEBUG;
  1159. kvmppc_account_exit(vcpu, DEBUG_EXITS);
  1160. break;
  1161. }
  1162. default:
  1163. printk(KERN_EMERG "exit_nr %d\n", exit_nr);
  1164. BUG();
  1165. }
  1166. out:
  1167. /*
  1168. * To avoid clobbering exit_reason, only check for signals if we
  1169. * aren't already exiting to userspace for some other reason.
  1170. */
  1171. if (!(r & RESUME_HOST)) {
  1172. s = kvmppc_prepare_to_enter(vcpu);
  1173. if (s <= 0)
  1174. r = (s << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
  1175. else {
  1176. /* interrupts now hard-disabled */
  1177. kvmppc_fix_ee_before_entry();
  1178. kvmppc_load_guest_fp(vcpu);
  1179. kvmppc_load_guest_altivec(vcpu);
  1180. }
  1181. }
  1182. return r;
  1183. }
  1184. static void kvmppc_set_tsr(struct kvm_vcpu *vcpu, u32 new_tsr)
  1185. {
  1186. u32 old_tsr = vcpu->arch.tsr;
  1187. vcpu->arch.tsr = new_tsr;
  1188. if ((old_tsr ^ vcpu->arch.tsr) & (TSR_ENW | TSR_WIS))
  1189. arm_next_watchdog(vcpu);
  1190. update_timer_ints(vcpu);
  1191. }
  1192. int kvmppc_subarch_vcpu_init(struct kvm_vcpu *vcpu)
  1193. {
  1194. /* setup watchdog timer once */
  1195. spin_lock_init(&vcpu->arch.wdt_lock);
  1196. timer_setup(&vcpu->arch.wdt_timer, kvmppc_watchdog_func, 0);
  1197. /*
  1198. * Clear DBSR.MRR to avoid guest debug interrupt as
  1199. * this is of host interest
  1200. */
  1201. mtspr(SPRN_DBSR, DBSR_MRR);
  1202. return 0;
  1203. }
  1204. void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu)
  1205. {
  1206. del_timer_sync(&vcpu->arch.wdt_timer);
  1207. }
  1208. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  1209. {
  1210. int i;
  1211. vcpu_load(vcpu);
  1212. regs->pc = vcpu->arch.regs.nip;
  1213. regs->cr = kvmppc_get_cr(vcpu);
  1214. regs->ctr = vcpu->arch.regs.ctr;
  1215. regs->lr = vcpu->arch.regs.link;
  1216. regs->xer = kvmppc_get_xer(vcpu);
  1217. regs->msr = vcpu->arch.shared->msr;
  1218. regs->srr0 = kvmppc_get_srr0(vcpu);
  1219. regs->srr1 = kvmppc_get_srr1(vcpu);
  1220. regs->pid = vcpu->arch.pid;
  1221. regs->sprg0 = kvmppc_get_sprg0(vcpu);
  1222. regs->sprg1 = kvmppc_get_sprg1(vcpu);
  1223. regs->sprg2 = kvmppc_get_sprg2(vcpu);
  1224. regs->sprg3 = kvmppc_get_sprg3(vcpu);
  1225. regs->sprg4 = kvmppc_get_sprg4(vcpu);
  1226. regs->sprg5 = kvmppc_get_sprg5(vcpu);
  1227. regs->sprg6 = kvmppc_get_sprg6(vcpu);
  1228. regs->sprg7 = kvmppc_get_sprg7(vcpu);
  1229. for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
  1230. regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
  1231. vcpu_put(vcpu);
  1232. return 0;
  1233. }
  1234. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  1235. {
  1236. int i;
  1237. vcpu_load(vcpu);
  1238. vcpu->arch.regs.nip = regs->pc;
  1239. kvmppc_set_cr(vcpu, regs->cr);
  1240. vcpu->arch.regs.ctr = regs->ctr;
  1241. vcpu->arch.regs.link = regs->lr;
  1242. kvmppc_set_xer(vcpu, regs->xer);
  1243. kvmppc_set_msr(vcpu, regs->msr);
  1244. kvmppc_set_srr0(vcpu, regs->srr0);
  1245. kvmppc_set_srr1(vcpu, regs->srr1);
  1246. kvmppc_set_pid(vcpu, regs->pid);
  1247. kvmppc_set_sprg0(vcpu, regs->sprg0);
  1248. kvmppc_set_sprg1(vcpu, regs->sprg1);
  1249. kvmppc_set_sprg2(vcpu, regs->sprg2);
  1250. kvmppc_set_sprg3(vcpu, regs->sprg3);
  1251. kvmppc_set_sprg4(vcpu, regs->sprg4);
  1252. kvmppc_set_sprg5(vcpu, regs->sprg5);
  1253. kvmppc_set_sprg6(vcpu, regs->sprg6);
  1254. kvmppc_set_sprg7(vcpu, regs->sprg7);
  1255. for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
  1256. kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
  1257. vcpu_put(vcpu);
  1258. return 0;
  1259. }
  1260. static void get_sregs_base(struct kvm_vcpu *vcpu,
  1261. struct kvm_sregs *sregs)
  1262. {
  1263. u64 tb = get_tb();
  1264. sregs->u.e.features |= KVM_SREGS_E_BASE;
  1265. sregs->u.e.csrr0 = vcpu->arch.csrr0;
  1266. sregs->u.e.csrr1 = vcpu->arch.csrr1;
  1267. sregs->u.e.mcsr = vcpu->arch.mcsr;
  1268. sregs->u.e.esr = kvmppc_get_esr(vcpu);
  1269. sregs->u.e.dear = kvmppc_get_dar(vcpu);
  1270. sregs->u.e.tsr = vcpu->arch.tsr;
  1271. sregs->u.e.tcr = vcpu->arch.tcr;
  1272. sregs->u.e.dec = kvmppc_get_dec(vcpu, tb);
  1273. sregs->u.e.tb = tb;
  1274. sregs->u.e.vrsave = vcpu->arch.vrsave;
  1275. }
  1276. static int set_sregs_base(struct kvm_vcpu *vcpu,
  1277. struct kvm_sregs *sregs)
  1278. {
  1279. if (!(sregs->u.e.features & KVM_SREGS_E_BASE))
  1280. return 0;
  1281. vcpu->arch.csrr0 = sregs->u.e.csrr0;
  1282. vcpu->arch.csrr1 = sregs->u.e.csrr1;
  1283. vcpu->arch.mcsr = sregs->u.e.mcsr;
  1284. kvmppc_set_esr(vcpu, sregs->u.e.esr);
  1285. kvmppc_set_dar(vcpu, sregs->u.e.dear);
  1286. vcpu->arch.vrsave = sregs->u.e.vrsave;
  1287. kvmppc_set_tcr(vcpu, sregs->u.e.tcr);
  1288. if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_DEC) {
  1289. vcpu->arch.dec = sregs->u.e.dec;
  1290. kvmppc_emulate_dec(vcpu);
  1291. }
  1292. if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_TSR)
  1293. kvmppc_set_tsr(vcpu, sregs->u.e.tsr);
  1294. return 0;
  1295. }
  1296. static void get_sregs_arch206(struct kvm_vcpu *vcpu,
  1297. struct kvm_sregs *sregs)
  1298. {
  1299. sregs->u.e.features |= KVM_SREGS_E_ARCH206;
  1300. sregs->u.e.pir = vcpu->vcpu_id;
  1301. sregs->u.e.mcsrr0 = vcpu->arch.mcsrr0;
  1302. sregs->u.e.mcsrr1 = vcpu->arch.mcsrr1;
  1303. sregs->u.e.decar = vcpu->arch.decar;
  1304. sregs->u.e.ivpr = vcpu->arch.ivpr;
  1305. }
  1306. static int set_sregs_arch206(struct kvm_vcpu *vcpu,
  1307. struct kvm_sregs *sregs)
  1308. {
  1309. if (!(sregs->u.e.features & KVM_SREGS_E_ARCH206))
  1310. return 0;
  1311. if (sregs->u.e.pir != vcpu->vcpu_id)
  1312. return -EINVAL;
  1313. vcpu->arch.mcsrr0 = sregs->u.e.mcsrr0;
  1314. vcpu->arch.mcsrr1 = sregs->u.e.mcsrr1;
  1315. vcpu->arch.decar = sregs->u.e.decar;
  1316. vcpu->arch.ivpr = sregs->u.e.ivpr;
  1317. return 0;
  1318. }
  1319. int kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  1320. {
  1321. sregs->u.e.features |= KVM_SREGS_E_IVOR;
  1322. sregs->u.e.ivor_low[0] = vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL];
  1323. sregs->u.e.ivor_low[1] = vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK];
  1324. sregs->u.e.ivor_low[2] = vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE];
  1325. sregs->u.e.ivor_low[3] = vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE];
  1326. sregs->u.e.ivor_low[4] = vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL];
  1327. sregs->u.e.ivor_low[5] = vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT];
  1328. sregs->u.e.ivor_low[6] = vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM];
  1329. sregs->u.e.ivor_low[7] = vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL];
  1330. sregs->u.e.ivor_low[8] = vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL];
  1331. sregs->u.e.ivor_low[9] = vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL];
  1332. sregs->u.e.ivor_low[10] = vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER];
  1333. sregs->u.e.ivor_low[11] = vcpu->arch.ivor[BOOKE_IRQPRIO_FIT];
  1334. sregs->u.e.ivor_low[12] = vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG];
  1335. sregs->u.e.ivor_low[13] = vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS];
  1336. sregs->u.e.ivor_low[14] = vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS];
  1337. sregs->u.e.ivor_low[15] = vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG];
  1338. return 0;
  1339. }
  1340. int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  1341. {
  1342. if (!(sregs->u.e.features & KVM_SREGS_E_IVOR))
  1343. return 0;
  1344. vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL] = sregs->u.e.ivor_low[0];
  1345. vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK] = sregs->u.e.ivor_low[1];
  1346. vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE] = sregs->u.e.ivor_low[2];
  1347. vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE] = sregs->u.e.ivor_low[3];
  1348. vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL] = sregs->u.e.ivor_low[4];
  1349. vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT] = sregs->u.e.ivor_low[5];
  1350. vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM] = sregs->u.e.ivor_low[6];
  1351. vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL] = sregs->u.e.ivor_low[7];
  1352. vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL] = sregs->u.e.ivor_low[8];
  1353. vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL] = sregs->u.e.ivor_low[9];
  1354. vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER] = sregs->u.e.ivor_low[10];
  1355. vcpu->arch.ivor[BOOKE_IRQPRIO_FIT] = sregs->u.e.ivor_low[11];
  1356. vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG] = sregs->u.e.ivor_low[12];
  1357. vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS] = sregs->u.e.ivor_low[13];
  1358. vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS] = sregs->u.e.ivor_low[14];
  1359. vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG] = sregs->u.e.ivor_low[15];
  1360. return 0;
  1361. }
  1362. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  1363. struct kvm_sregs *sregs)
  1364. {
  1365. int ret;
  1366. vcpu_load(vcpu);
  1367. sregs->pvr = vcpu->arch.pvr;
  1368. get_sregs_base(vcpu, sregs);
  1369. get_sregs_arch206(vcpu, sregs);
  1370. ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
  1371. vcpu_put(vcpu);
  1372. return ret;
  1373. }
  1374. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  1375. struct kvm_sregs *sregs)
  1376. {
  1377. int ret = -EINVAL;
  1378. vcpu_load(vcpu);
  1379. if (vcpu->arch.pvr != sregs->pvr)
  1380. goto out;
  1381. ret = set_sregs_base(vcpu, sregs);
  1382. if (ret < 0)
  1383. goto out;
  1384. ret = set_sregs_arch206(vcpu, sregs);
  1385. if (ret < 0)
  1386. goto out;
  1387. ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
  1388. out:
  1389. vcpu_put(vcpu);
  1390. return ret;
  1391. }
  1392. int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id,
  1393. union kvmppc_one_reg *val)
  1394. {
  1395. int r = 0;
  1396. switch (id) {
  1397. case KVM_REG_PPC_IAC1:
  1398. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac1);
  1399. break;
  1400. case KVM_REG_PPC_IAC2:
  1401. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac2);
  1402. break;
  1403. #if CONFIG_PPC_ADV_DEBUG_IACS > 2
  1404. case KVM_REG_PPC_IAC3:
  1405. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac3);
  1406. break;
  1407. case KVM_REG_PPC_IAC4:
  1408. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac4);
  1409. break;
  1410. #endif
  1411. case KVM_REG_PPC_DAC1:
  1412. *val = get_reg_val(id, vcpu->arch.dbg_reg.dac1);
  1413. break;
  1414. case KVM_REG_PPC_DAC2:
  1415. *val = get_reg_val(id, vcpu->arch.dbg_reg.dac2);
  1416. break;
  1417. case KVM_REG_PPC_EPR: {
  1418. u32 epr = kvmppc_get_epr(vcpu);
  1419. *val = get_reg_val(id, epr);
  1420. break;
  1421. }
  1422. #if defined(CONFIG_64BIT)
  1423. case KVM_REG_PPC_EPCR:
  1424. *val = get_reg_val(id, vcpu->arch.epcr);
  1425. break;
  1426. #endif
  1427. case KVM_REG_PPC_TCR:
  1428. *val = get_reg_val(id, vcpu->arch.tcr);
  1429. break;
  1430. case KVM_REG_PPC_TSR:
  1431. *val = get_reg_val(id, vcpu->arch.tsr);
  1432. break;
  1433. case KVM_REG_PPC_DEBUG_INST:
  1434. *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
  1435. break;
  1436. case KVM_REG_PPC_VRSAVE:
  1437. *val = get_reg_val(id, vcpu->arch.vrsave);
  1438. break;
  1439. default:
  1440. r = vcpu->kvm->arch.kvm_ops->get_one_reg(vcpu, id, val);
  1441. break;
  1442. }
  1443. return r;
  1444. }
  1445. int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id,
  1446. union kvmppc_one_reg *val)
  1447. {
  1448. int r = 0;
  1449. switch (id) {
  1450. case KVM_REG_PPC_IAC1:
  1451. vcpu->arch.dbg_reg.iac1 = set_reg_val(id, *val);
  1452. break;
  1453. case KVM_REG_PPC_IAC2:
  1454. vcpu->arch.dbg_reg.iac2 = set_reg_val(id, *val);
  1455. break;
  1456. #if CONFIG_PPC_ADV_DEBUG_IACS > 2
  1457. case KVM_REG_PPC_IAC3:
  1458. vcpu->arch.dbg_reg.iac3 = set_reg_val(id, *val);
  1459. break;
  1460. case KVM_REG_PPC_IAC4:
  1461. vcpu->arch.dbg_reg.iac4 = set_reg_val(id, *val);
  1462. break;
  1463. #endif
  1464. case KVM_REG_PPC_DAC1:
  1465. vcpu->arch.dbg_reg.dac1 = set_reg_val(id, *val);
  1466. break;
  1467. case KVM_REG_PPC_DAC2:
  1468. vcpu->arch.dbg_reg.dac2 = set_reg_val(id, *val);
  1469. break;
  1470. case KVM_REG_PPC_EPR: {
  1471. u32 new_epr = set_reg_val(id, *val);
  1472. kvmppc_set_epr(vcpu, new_epr);
  1473. break;
  1474. }
  1475. #if defined(CONFIG_64BIT)
  1476. case KVM_REG_PPC_EPCR: {
  1477. u32 new_epcr = set_reg_val(id, *val);
  1478. kvmppc_set_epcr(vcpu, new_epcr);
  1479. break;
  1480. }
  1481. #endif
  1482. case KVM_REG_PPC_OR_TSR: {
  1483. u32 tsr_bits = set_reg_val(id, *val);
  1484. kvmppc_set_tsr_bits(vcpu, tsr_bits);
  1485. break;
  1486. }
  1487. case KVM_REG_PPC_CLEAR_TSR: {
  1488. u32 tsr_bits = set_reg_val(id, *val);
  1489. kvmppc_clr_tsr_bits(vcpu, tsr_bits);
  1490. break;
  1491. }
  1492. case KVM_REG_PPC_TSR: {
  1493. u32 tsr = set_reg_val(id, *val);
  1494. kvmppc_set_tsr(vcpu, tsr);
  1495. break;
  1496. }
  1497. case KVM_REG_PPC_TCR: {
  1498. u32 tcr = set_reg_val(id, *val);
  1499. kvmppc_set_tcr(vcpu, tcr);
  1500. break;
  1501. }
  1502. case KVM_REG_PPC_VRSAVE:
  1503. vcpu->arch.vrsave = set_reg_val(id, *val);
  1504. break;
  1505. default:
  1506. r = vcpu->kvm->arch.kvm_ops->set_one_reg(vcpu, id, val);
  1507. break;
  1508. }
  1509. return r;
  1510. }
  1511. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  1512. {
  1513. return -EOPNOTSUPP;
  1514. }
  1515. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  1516. {
  1517. return -EOPNOTSUPP;
  1518. }
  1519. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  1520. struct kvm_translation *tr)
  1521. {
  1522. int r;
  1523. vcpu_load(vcpu);
  1524. r = kvmppc_core_vcpu_translate(vcpu, tr);
  1525. vcpu_put(vcpu);
  1526. return r;
  1527. }
  1528. void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
  1529. {
  1530. }
  1531. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
  1532. {
  1533. return -EOPNOTSUPP;
  1534. }
  1535. void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
  1536. {
  1537. }
  1538. int kvmppc_core_prepare_memory_region(struct kvm *kvm,
  1539. struct kvm_memory_slot *memslot,
  1540. const struct kvm_userspace_memory_region *mem,
  1541. enum kvm_mr_change change)
  1542. {
  1543. return 0;
  1544. }
  1545. void kvmppc_core_commit_memory_region(struct kvm *kvm,
  1546. const struct kvm_userspace_memory_region *mem,
  1547. const struct kvm_memory_slot *old,
  1548. const struct kvm_memory_slot *new,
  1549. enum kvm_mr_change change)
  1550. {
  1551. }
  1552. void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot)
  1553. {
  1554. }
  1555. void kvmppc_set_epcr(struct kvm_vcpu *vcpu, u32 new_epcr)
  1556. {
  1557. #if defined(CONFIG_64BIT)
  1558. vcpu->arch.epcr = new_epcr;
  1559. #ifdef CONFIG_KVM_BOOKE_HV
  1560. vcpu->arch.shadow_epcr &= ~SPRN_EPCR_GICM;
  1561. if (vcpu->arch.epcr & SPRN_EPCR_ICM)
  1562. vcpu->arch.shadow_epcr |= SPRN_EPCR_GICM;
  1563. #endif
  1564. #endif
  1565. }
  1566. void kvmppc_set_tcr(struct kvm_vcpu *vcpu, u32 new_tcr)
  1567. {
  1568. vcpu->arch.tcr = new_tcr;
  1569. arm_next_watchdog(vcpu);
  1570. update_timer_ints(vcpu);
  1571. }
  1572. void kvmppc_set_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
  1573. {
  1574. set_bits(tsr_bits, &vcpu->arch.tsr);
  1575. smp_wmb();
  1576. kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
  1577. kvm_vcpu_kick(vcpu);
  1578. }
  1579. void kvmppc_clr_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
  1580. {
  1581. clear_bits(tsr_bits, &vcpu->arch.tsr);
  1582. /*
  1583. * We may have stopped the watchdog due to
  1584. * being stuck on final expiration.
  1585. */
  1586. if (tsr_bits & (TSR_ENW | TSR_WIS))
  1587. arm_next_watchdog(vcpu);
  1588. update_timer_ints(vcpu);
  1589. }
  1590. void kvmppc_decrementer_func(struct kvm_vcpu *vcpu)
  1591. {
  1592. if (vcpu->arch.tcr & TCR_ARE) {
  1593. vcpu->arch.dec = vcpu->arch.decar;
  1594. kvmppc_emulate_dec(vcpu);
  1595. }
  1596. kvmppc_set_tsr_bits(vcpu, TSR_DIS);
  1597. }
  1598. static int kvmppc_booke_add_breakpoint(struct debug_reg *dbg_reg,
  1599. uint64_t addr, int index)
  1600. {
  1601. switch (index) {
  1602. case 0:
  1603. dbg_reg->dbcr0 |= DBCR0_IAC1;
  1604. dbg_reg->iac1 = addr;
  1605. break;
  1606. case 1:
  1607. dbg_reg->dbcr0 |= DBCR0_IAC2;
  1608. dbg_reg->iac2 = addr;
  1609. break;
  1610. #if CONFIG_PPC_ADV_DEBUG_IACS > 2
  1611. case 2:
  1612. dbg_reg->dbcr0 |= DBCR0_IAC3;
  1613. dbg_reg->iac3 = addr;
  1614. break;
  1615. case 3:
  1616. dbg_reg->dbcr0 |= DBCR0_IAC4;
  1617. dbg_reg->iac4 = addr;
  1618. break;
  1619. #endif
  1620. default:
  1621. return -EINVAL;
  1622. }
  1623. dbg_reg->dbcr0 |= DBCR0_IDM;
  1624. return 0;
  1625. }
  1626. static int kvmppc_booke_add_watchpoint(struct debug_reg *dbg_reg, uint64_t addr,
  1627. int type, int index)
  1628. {
  1629. switch (index) {
  1630. case 0:
  1631. if (type & KVMPPC_DEBUG_WATCH_READ)
  1632. dbg_reg->dbcr0 |= DBCR0_DAC1R;
  1633. if (type & KVMPPC_DEBUG_WATCH_WRITE)
  1634. dbg_reg->dbcr0 |= DBCR0_DAC1W;
  1635. dbg_reg->dac1 = addr;
  1636. break;
  1637. case 1:
  1638. if (type & KVMPPC_DEBUG_WATCH_READ)
  1639. dbg_reg->dbcr0 |= DBCR0_DAC2R;
  1640. if (type & KVMPPC_DEBUG_WATCH_WRITE)
  1641. dbg_reg->dbcr0 |= DBCR0_DAC2W;
  1642. dbg_reg->dac2 = addr;
  1643. break;
  1644. default:
  1645. return -EINVAL;
  1646. }
  1647. dbg_reg->dbcr0 |= DBCR0_IDM;
  1648. return 0;
  1649. }
  1650. void kvm_guest_protect_msr(struct kvm_vcpu *vcpu, ulong prot_bitmap, bool set)
  1651. {
  1652. /* XXX: Add similar MSR protection for BookE-PR */
  1653. #ifdef CONFIG_KVM_BOOKE_HV
  1654. BUG_ON(prot_bitmap & ~(MSRP_UCLEP | MSRP_DEP | MSRP_PMMP));
  1655. if (set) {
  1656. if (prot_bitmap & MSR_UCLE)
  1657. vcpu->arch.shadow_msrp |= MSRP_UCLEP;
  1658. if (prot_bitmap & MSR_DE)
  1659. vcpu->arch.shadow_msrp |= MSRP_DEP;
  1660. if (prot_bitmap & MSR_PMM)
  1661. vcpu->arch.shadow_msrp |= MSRP_PMMP;
  1662. } else {
  1663. if (prot_bitmap & MSR_UCLE)
  1664. vcpu->arch.shadow_msrp &= ~MSRP_UCLEP;
  1665. if (prot_bitmap & MSR_DE)
  1666. vcpu->arch.shadow_msrp &= ~MSRP_DEP;
  1667. if (prot_bitmap & MSR_PMM)
  1668. vcpu->arch.shadow_msrp &= ~MSRP_PMMP;
  1669. }
  1670. #endif
  1671. }
  1672. int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, enum xlate_instdata xlid,
  1673. enum xlate_readwrite xlrw, struct kvmppc_pte *pte)
  1674. {
  1675. int gtlb_index;
  1676. gpa_t gpaddr;
  1677. #ifdef CONFIG_KVM_E500V2
  1678. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  1679. (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
  1680. pte->eaddr = eaddr;
  1681. pte->raddr = (vcpu->arch.magic_page_pa & PAGE_MASK) |
  1682. (eaddr & ~PAGE_MASK);
  1683. pte->vpage = eaddr >> PAGE_SHIFT;
  1684. pte->may_read = true;
  1685. pte->may_write = true;
  1686. pte->may_execute = true;
  1687. return 0;
  1688. }
  1689. #endif
  1690. /* Check the guest TLB. */
  1691. switch (xlid) {
  1692. case XLATE_INST:
  1693. gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
  1694. break;
  1695. case XLATE_DATA:
  1696. gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
  1697. break;
  1698. default:
  1699. BUG();
  1700. }
  1701. /* Do we have a TLB entry at all? */
  1702. if (gtlb_index < 0)
  1703. return -ENOENT;
  1704. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  1705. pte->eaddr = eaddr;
  1706. pte->raddr = (gpaddr & PAGE_MASK) | (eaddr & ~PAGE_MASK);
  1707. pte->vpage = eaddr >> PAGE_SHIFT;
  1708. /* XXX read permissions from the guest TLB */
  1709. pte->may_read = true;
  1710. pte->may_write = true;
  1711. pte->may_execute = true;
  1712. return 0;
  1713. }
  1714. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  1715. struct kvm_guest_debug *dbg)
  1716. {
  1717. struct debug_reg *dbg_reg;
  1718. int n, b = 0, w = 0;
  1719. int ret = 0;
  1720. vcpu_load(vcpu);
  1721. if (!(dbg->control & KVM_GUESTDBG_ENABLE)) {
  1722. vcpu->arch.dbg_reg.dbcr0 = 0;
  1723. vcpu->guest_debug = 0;
  1724. kvm_guest_protect_msr(vcpu, MSR_DE, false);
  1725. goto out;
  1726. }
  1727. kvm_guest_protect_msr(vcpu, MSR_DE, true);
  1728. vcpu->guest_debug = dbg->control;
  1729. vcpu->arch.dbg_reg.dbcr0 = 0;
  1730. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  1731. vcpu->arch.dbg_reg.dbcr0 |= DBCR0_IDM | DBCR0_IC;
  1732. /* Code below handles only HW breakpoints */
  1733. dbg_reg = &(vcpu->arch.dbg_reg);
  1734. #ifdef CONFIG_KVM_BOOKE_HV
  1735. /*
  1736. * On BookE-HV (e500mc) the guest is always executed with MSR.GS=1
  1737. * DBCR1 and DBCR2 are set to trigger debug events when MSR.PR is 0
  1738. */
  1739. dbg_reg->dbcr1 = 0;
  1740. dbg_reg->dbcr2 = 0;
  1741. #else
  1742. /*
  1743. * On BookE-PR (e500v2) the guest is always executed with MSR.PR=1
  1744. * We set DBCR1 and DBCR2 to only trigger debug events when MSR.PR
  1745. * is set.
  1746. */
  1747. dbg_reg->dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US | DBCR1_IAC3US |
  1748. DBCR1_IAC4US;
  1749. dbg_reg->dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
  1750. #endif
  1751. if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
  1752. goto out;
  1753. ret = -EINVAL;
  1754. for (n = 0; n < (KVMPPC_BOOKE_IAC_NUM + KVMPPC_BOOKE_DAC_NUM); n++) {
  1755. uint64_t addr = dbg->arch.bp[n].addr;
  1756. uint32_t type = dbg->arch.bp[n].type;
  1757. if (type == KVMPPC_DEBUG_NONE)
  1758. continue;
  1759. if (type & ~(KVMPPC_DEBUG_WATCH_READ |
  1760. KVMPPC_DEBUG_WATCH_WRITE |
  1761. KVMPPC_DEBUG_BREAKPOINT))
  1762. goto out;
  1763. if (type & KVMPPC_DEBUG_BREAKPOINT) {
  1764. /* Setting H/W breakpoint */
  1765. if (kvmppc_booke_add_breakpoint(dbg_reg, addr, b++))
  1766. goto out;
  1767. } else {
  1768. /* Setting H/W watchpoint */
  1769. if (kvmppc_booke_add_watchpoint(dbg_reg, addr,
  1770. type, w++))
  1771. goto out;
  1772. }
  1773. }
  1774. ret = 0;
  1775. out:
  1776. vcpu_put(vcpu);
  1777. return ret;
  1778. }
  1779. void kvmppc_booke_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1780. {
  1781. vcpu->cpu = smp_processor_id();
  1782. current->thread.kvm_vcpu = vcpu;
  1783. }
  1784. void kvmppc_booke_vcpu_put(struct kvm_vcpu *vcpu)
  1785. {
  1786. current->thread.kvm_vcpu = NULL;
  1787. vcpu->cpu = -1;
  1788. /* Clear pending debug event in DBSR */
  1789. kvmppc_clear_dbsr();
  1790. }
  1791. int kvmppc_core_init_vm(struct kvm *kvm)
  1792. {
  1793. return kvm->arch.kvm_ops->init_vm(kvm);
  1794. }
  1795. int kvmppc_core_vcpu_create(struct kvm_vcpu *vcpu)
  1796. {
  1797. int i;
  1798. int r;
  1799. r = vcpu->kvm->arch.kvm_ops->vcpu_create(vcpu);
  1800. if (r)
  1801. return r;
  1802. /* Initial guest state: 16MB mapping 0 -> 0, PC = 0, MSR = 0, R1 = 16MB */
  1803. vcpu->arch.regs.nip = 0;
  1804. vcpu->arch.shared->pir = vcpu->vcpu_id;
  1805. kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
  1806. kvmppc_set_msr(vcpu, 0);
  1807. #ifndef CONFIG_KVM_BOOKE_HV
  1808. vcpu->arch.shadow_msr = MSR_USER | MSR_IS | MSR_DS;
  1809. vcpu->arch.shadow_pid = 1;
  1810. vcpu->arch.shared->msr = 0;
  1811. #endif
  1812. /* Eye-catching numbers so we know if the guest takes an interrupt
  1813. * before it's programmed its own IVPR/IVORs. */
  1814. vcpu->arch.ivpr = 0x55550000;
  1815. for (i = 0; i < BOOKE_IRQPRIO_MAX; i++)
  1816. vcpu->arch.ivor[i] = 0x7700 | i * 4;
  1817. kvmppc_init_timing_stats(vcpu);
  1818. r = kvmppc_core_vcpu_setup(vcpu);
  1819. if (r)
  1820. vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
  1821. kvmppc_sanity_check(vcpu);
  1822. return r;
  1823. }
  1824. void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
  1825. {
  1826. vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
  1827. }
  1828. void kvmppc_core_destroy_vm(struct kvm *kvm)
  1829. {
  1830. kvm->arch.kvm_ops->destroy_vm(kvm);
  1831. }
  1832. void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1833. {
  1834. vcpu->kvm->arch.kvm_ops->vcpu_load(vcpu, cpu);
  1835. }
  1836. void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
  1837. {
  1838. vcpu->kvm->arch.kvm_ops->vcpu_put(vcpu);
  1839. }
  1840. int __init kvmppc_booke_init(void)
  1841. {
  1842. #ifndef CONFIG_KVM_BOOKE_HV
  1843. unsigned long ivor[16];
  1844. unsigned long *handler = kvmppc_booke_handler_addr;
  1845. unsigned long max_ivor = 0;
  1846. unsigned long handler_len;
  1847. int i;
  1848. /* We install our own exception handlers by hijacking IVPR. IVPR must
  1849. * be 16-bit aligned, so we need a 64KB allocation. */
  1850. kvmppc_booke_handlers = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
  1851. VCPU_SIZE_ORDER);
  1852. if (!kvmppc_booke_handlers)
  1853. return -ENOMEM;
  1854. /* XXX make sure our handlers are smaller than Linux's */
  1855. /* Copy our interrupt handlers to match host IVORs. That way we don't
  1856. * have to swap the IVORs on every guest/host transition. */
  1857. ivor[0] = mfspr(SPRN_IVOR0);
  1858. ivor[1] = mfspr(SPRN_IVOR1);
  1859. ivor[2] = mfspr(SPRN_IVOR2);
  1860. ivor[3] = mfspr(SPRN_IVOR3);
  1861. ivor[4] = mfspr(SPRN_IVOR4);
  1862. ivor[5] = mfspr(SPRN_IVOR5);
  1863. ivor[6] = mfspr(SPRN_IVOR6);
  1864. ivor[7] = mfspr(SPRN_IVOR7);
  1865. ivor[8] = mfspr(SPRN_IVOR8);
  1866. ivor[9] = mfspr(SPRN_IVOR9);
  1867. ivor[10] = mfspr(SPRN_IVOR10);
  1868. ivor[11] = mfspr(SPRN_IVOR11);
  1869. ivor[12] = mfspr(SPRN_IVOR12);
  1870. ivor[13] = mfspr(SPRN_IVOR13);
  1871. ivor[14] = mfspr(SPRN_IVOR14);
  1872. ivor[15] = mfspr(SPRN_IVOR15);
  1873. for (i = 0; i < 16; i++) {
  1874. if (ivor[i] > max_ivor)
  1875. max_ivor = i;
  1876. handler_len = handler[i + 1] - handler[i];
  1877. memcpy((void *)kvmppc_booke_handlers + ivor[i],
  1878. (void *)handler[i], handler_len);
  1879. }
  1880. handler_len = handler[max_ivor + 1] - handler[max_ivor];
  1881. flush_icache_range(kvmppc_booke_handlers, kvmppc_booke_handlers +
  1882. ivor[max_ivor] + handler_len);
  1883. #endif /* !BOOKE_HV */
  1884. return 0;
  1885. }
  1886. void __exit kvmppc_booke_exit(void)
  1887. {
  1888. free_pages(kvmppc_booke_handlers, VCPU_SIZE_ORDER);
  1889. kvm_exit();
  1890. }