vmx.c 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "kvm.h"
  18. #include "vmx.h"
  19. #include "kvm_vmx.h"
  20. #include <linux/module.h>
  21. #include <linux/kernel.h>
  22. #include <linux/mm.h>
  23. #include <linux/highmem.h>
  24. #include <linux/profile.h>
  25. #include <asm/io.h>
  26. #include <asm/desc.h>
  27. #include "segment_descriptor.h"
  28. MODULE_AUTHOR("Qumranet");
  29. MODULE_LICENSE("GPL");
  30. static DEFINE_PER_CPU(struct vmcs *, vmxarea);
  31. static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
  32. #ifdef CONFIG_X86_64
  33. #define HOST_IS_64 1
  34. #else
  35. #define HOST_IS_64 0
  36. #endif
  37. static struct vmcs_descriptor {
  38. int size;
  39. int order;
  40. u32 revision_id;
  41. } vmcs_descriptor;
  42. #define VMX_SEGMENT_FIELD(seg) \
  43. [VCPU_SREG_##seg] = { \
  44. .selector = GUEST_##seg##_SELECTOR, \
  45. .base = GUEST_##seg##_BASE, \
  46. .limit = GUEST_##seg##_LIMIT, \
  47. .ar_bytes = GUEST_##seg##_AR_BYTES, \
  48. }
  49. static struct kvm_vmx_segment_field {
  50. unsigned selector;
  51. unsigned base;
  52. unsigned limit;
  53. unsigned ar_bytes;
  54. } kvm_vmx_segment_fields[] = {
  55. VMX_SEGMENT_FIELD(CS),
  56. VMX_SEGMENT_FIELD(DS),
  57. VMX_SEGMENT_FIELD(ES),
  58. VMX_SEGMENT_FIELD(FS),
  59. VMX_SEGMENT_FIELD(GS),
  60. VMX_SEGMENT_FIELD(SS),
  61. VMX_SEGMENT_FIELD(TR),
  62. VMX_SEGMENT_FIELD(LDTR),
  63. };
  64. static const u32 vmx_msr_index[] = {
  65. #ifdef CONFIG_X86_64
  66. MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE,
  67. #endif
  68. MSR_EFER, MSR_K6_STAR,
  69. };
  70. #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
  71. static inline int is_page_fault(u32 intr_info)
  72. {
  73. return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
  74. INTR_INFO_VALID_MASK)) ==
  75. (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
  76. }
  77. static inline int is_external_interrupt(u32 intr_info)
  78. {
  79. return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
  80. == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
  81. }
  82. static struct vmx_msr_entry *find_msr_entry(struct kvm_vcpu *vcpu, u32 msr)
  83. {
  84. int i;
  85. for (i = 0; i < vcpu->nmsrs; ++i)
  86. if (vcpu->guest_msrs[i].index == msr)
  87. return &vcpu->guest_msrs[i];
  88. return NULL;
  89. }
  90. static void vmcs_clear(struct vmcs *vmcs)
  91. {
  92. u64 phys_addr = __pa(vmcs);
  93. u8 error;
  94. asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0"
  95. : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
  96. : "cc", "memory");
  97. if (error)
  98. printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
  99. vmcs, phys_addr);
  100. }
  101. static void __vcpu_clear(void *arg)
  102. {
  103. struct kvm_vcpu *vcpu = arg;
  104. int cpu = raw_smp_processor_id();
  105. if (vcpu->cpu == cpu)
  106. vmcs_clear(vcpu->vmcs);
  107. if (per_cpu(current_vmcs, cpu) == vcpu->vmcs)
  108. per_cpu(current_vmcs, cpu) = NULL;
  109. }
  110. static void vcpu_clear(struct kvm_vcpu *vcpu)
  111. {
  112. if (vcpu->cpu != raw_smp_processor_id() && vcpu->cpu != -1)
  113. smp_call_function_single(vcpu->cpu, __vcpu_clear, vcpu, 0, 1);
  114. else
  115. __vcpu_clear(vcpu);
  116. vcpu->launched = 0;
  117. }
  118. static unsigned long vmcs_readl(unsigned long field)
  119. {
  120. unsigned long value;
  121. asm volatile (ASM_VMX_VMREAD_RDX_RAX
  122. : "=a"(value) : "d"(field) : "cc");
  123. return value;
  124. }
  125. static u16 vmcs_read16(unsigned long field)
  126. {
  127. return vmcs_readl(field);
  128. }
  129. static u32 vmcs_read32(unsigned long field)
  130. {
  131. return vmcs_readl(field);
  132. }
  133. static u64 vmcs_read64(unsigned long field)
  134. {
  135. #ifdef CONFIG_X86_64
  136. return vmcs_readl(field);
  137. #else
  138. return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
  139. #endif
  140. }
  141. static noinline void vmwrite_error(unsigned long field, unsigned long value)
  142. {
  143. printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
  144. field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
  145. dump_stack();
  146. }
  147. static void vmcs_writel(unsigned long field, unsigned long value)
  148. {
  149. u8 error;
  150. asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0"
  151. : "=q"(error) : "a"(value), "d"(field) : "cc" );
  152. if (unlikely(error))
  153. vmwrite_error(field, value);
  154. }
  155. static void vmcs_write16(unsigned long field, u16 value)
  156. {
  157. vmcs_writel(field, value);
  158. }
  159. static void vmcs_write32(unsigned long field, u32 value)
  160. {
  161. vmcs_writel(field, value);
  162. }
  163. static void vmcs_write64(unsigned long field, u64 value)
  164. {
  165. #ifdef CONFIG_X86_64
  166. vmcs_writel(field, value);
  167. #else
  168. vmcs_writel(field, value);
  169. asm volatile ("");
  170. vmcs_writel(field+1, value >> 32);
  171. #endif
  172. }
  173. /*
  174. * Switches to specified vcpu, until a matching vcpu_put(), but assumes
  175. * vcpu mutex is already taken.
  176. */
  177. static void vmx_vcpu_load(struct kvm_vcpu *vcpu)
  178. {
  179. u64 phys_addr = __pa(vcpu->vmcs);
  180. int cpu;
  181. cpu = get_cpu();
  182. if (vcpu->cpu != cpu)
  183. vcpu_clear(vcpu);
  184. if (per_cpu(current_vmcs, cpu) != vcpu->vmcs) {
  185. u8 error;
  186. per_cpu(current_vmcs, cpu) = vcpu->vmcs;
  187. asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0"
  188. : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
  189. : "cc");
  190. if (error)
  191. printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
  192. vcpu->vmcs, phys_addr);
  193. }
  194. if (vcpu->cpu != cpu) {
  195. struct descriptor_table dt;
  196. unsigned long sysenter_esp;
  197. vcpu->cpu = cpu;
  198. /*
  199. * Linux uses per-cpu TSS and GDT, so set these when switching
  200. * processors.
  201. */
  202. vmcs_writel(HOST_TR_BASE, read_tr_base()); /* 22.2.4 */
  203. get_gdt(&dt);
  204. vmcs_writel(HOST_GDTR_BASE, dt.base); /* 22.2.4 */
  205. rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
  206. vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
  207. }
  208. }
  209. static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
  210. {
  211. put_cpu();
  212. }
  213. static void vmx_vcpu_decache(struct kvm_vcpu *vcpu)
  214. {
  215. vcpu_clear(vcpu);
  216. }
  217. static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
  218. {
  219. return vmcs_readl(GUEST_RFLAGS);
  220. }
  221. static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  222. {
  223. vmcs_writel(GUEST_RFLAGS, rflags);
  224. }
  225. static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
  226. {
  227. unsigned long rip;
  228. u32 interruptibility;
  229. rip = vmcs_readl(GUEST_RIP);
  230. rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
  231. vmcs_writel(GUEST_RIP, rip);
  232. /*
  233. * We emulated an instruction, so temporary interrupt blocking
  234. * should be removed, if set.
  235. */
  236. interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
  237. if (interruptibility & 3)
  238. vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
  239. interruptibility & ~3);
  240. vcpu->interrupt_window_open = 1;
  241. }
  242. static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code)
  243. {
  244. printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n",
  245. vmcs_readl(GUEST_RIP));
  246. vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
  247. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
  248. GP_VECTOR |
  249. INTR_TYPE_EXCEPTION |
  250. INTR_INFO_DELIEVER_CODE_MASK |
  251. INTR_INFO_VALID_MASK);
  252. }
  253. /*
  254. * reads and returns guest's timestamp counter "register"
  255. * guest_tsc = host_tsc + tsc_offset -- 21.3
  256. */
  257. static u64 guest_read_tsc(void)
  258. {
  259. u64 host_tsc, tsc_offset;
  260. rdtscll(host_tsc);
  261. tsc_offset = vmcs_read64(TSC_OFFSET);
  262. return host_tsc + tsc_offset;
  263. }
  264. /*
  265. * writes 'guest_tsc' into guest's timestamp counter "register"
  266. * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
  267. */
  268. static void guest_write_tsc(u64 guest_tsc)
  269. {
  270. u64 host_tsc;
  271. rdtscll(host_tsc);
  272. vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
  273. }
  274. static void reload_tss(void)
  275. {
  276. #ifndef CONFIG_X86_64
  277. /*
  278. * VT restores TR but not its size. Useless.
  279. */
  280. struct descriptor_table gdt;
  281. struct segment_descriptor *descs;
  282. get_gdt(&gdt);
  283. descs = (void *)gdt.base;
  284. descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
  285. load_TR_desc();
  286. #endif
  287. }
  288. /*
  289. * Reads an msr value (of 'msr_index') into 'pdata'.
  290. * Returns 0 on success, non-0 otherwise.
  291. * Assumes vcpu_load() was already called.
  292. */
  293. static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  294. {
  295. u64 data;
  296. struct vmx_msr_entry *msr;
  297. if (!pdata) {
  298. printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
  299. return -EINVAL;
  300. }
  301. switch (msr_index) {
  302. #ifdef CONFIG_X86_64
  303. case MSR_FS_BASE:
  304. data = vmcs_readl(GUEST_FS_BASE);
  305. break;
  306. case MSR_GS_BASE:
  307. data = vmcs_readl(GUEST_GS_BASE);
  308. break;
  309. case MSR_EFER:
  310. return kvm_get_msr_common(vcpu, msr_index, pdata);
  311. #endif
  312. case MSR_IA32_TIME_STAMP_COUNTER:
  313. data = guest_read_tsc();
  314. break;
  315. case MSR_IA32_SYSENTER_CS:
  316. data = vmcs_read32(GUEST_SYSENTER_CS);
  317. break;
  318. case MSR_IA32_SYSENTER_EIP:
  319. data = vmcs_readl(GUEST_SYSENTER_EIP);
  320. break;
  321. case MSR_IA32_SYSENTER_ESP:
  322. data = vmcs_readl(GUEST_SYSENTER_ESP);
  323. break;
  324. default:
  325. msr = find_msr_entry(vcpu, msr_index);
  326. if (msr) {
  327. data = msr->data;
  328. break;
  329. }
  330. return kvm_get_msr_common(vcpu, msr_index, pdata);
  331. }
  332. *pdata = data;
  333. return 0;
  334. }
  335. /*
  336. * Writes msr value into into the appropriate "register".
  337. * Returns 0 on success, non-0 otherwise.
  338. * Assumes vcpu_load() was already called.
  339. */
  340. static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  341. {
  342. struct vmx_msr_entry *msr;
  343. switch (msr_index) {
  344. #ifdef CONFIG_X86_64
  345. case MSR_EFER:
  346. return kvm_set_msr_common(vcpu, msr_index, data);
  347. case MSR_FS_BASE:
  348. vmcs_writel(GUEST_FS_BASE, data);
  349. break;
  350. case MSR_GS_BASE:
  351. vmcs_writel(GUEST_GS_BASE, data);
  352. break;
  353. #endif
  354. case MSR_IA32_SYSENTER_CS:
  355. vmcs_write32(GUEST_SYSENTER_CS, data);
  356. break;
  357. case MSR_IA32_SYSENTER_EIP:
  358. vmcs_writel(GUEST_SYSENTER_EIP, data);
  359. break;
  360. case MSR_IA32_SYSENTER_ESP:
  361. vmcs_writel(GUEST_SYSENTER_ESP, data);
  362. break;
  363. case MSR_IA32_TIME_STAMP_COUNTER:
  364. guest_write_tsc(data);
  365. break;
  366. default:
  367. msr = find_msr_entry(vcpu, msr_index);
  368. if (msr) {
  369. msr->data = data;
  370. break;
  371. }
  372. return kvm_set_msr_common(vcpu, msr_index, data);
  373. msr->data = data;
  374. break;
  375. }
  376. return 0;
  377. }
  378. /*
  379. * Sync the rsp and rip registers into the vcpu structure. This allows
  380. * registers to be accessed by indexing vcpu->regs.
  381. */
  382. static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu)
  383. {
  384. vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
  385. vcpu->rip = vmcs_readl(GUEST_RIP);
  386. }
  387. /*
  388. * Syncs rsp and rip back into the vmcs. Should be called after possible
  389. * modification.
  390. */
  391. static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu)
  392. {
  393. vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]);
  394. vmcs_writel(GUEST_RIP, vcpu->rip);
  395. }
  396. static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
  397. {
  398. unsigned long dr7 = 0x400;
  399. u32 exception_bitmap;
  400. int old_singlestep;
  401. exception_bitmap = vmcs_read32(EXCEPTION_BITMAP);
  402. old_singlestep = vcpu->guest_debug.singlestep;
  403. vcpu->guest_debug.enabled = dbg->enabled;
  404. if (vcpu->guest_debug.enabled) {
  405. int i;
  406. dr7 |= 0x200; /* exact */
  407. for (i = 0; i < 4; ++i) {
  408. if (!dbg->breakpoints[i].enabled)
  409. continue;
  410. vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address;
  411. dr7 |= 2 << (i*2); /* global enable */
  412. dr7 |= 0 << (i*4+16); /* execution breakpoint */
  413. }
  414. exception_bitmap |= (1u << 1); /* Trap debug exceptions */
  415. vcpu->guest_debug.singlestep = dbg->singlestep;
  416. } else {
  417. exception_bitmap &= ~(1u << 1); /* Ignore debug exceptions */
  418. vcpu->guest_debug.singlestep = 0;
  419. }
  420. if (old_singlestep && !vcpu->guest_debug.singlestep) {
  421. unsigned long flags;
  422. flags = vmcs_readl(GUEST_RFLAGS);
  423. flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
  424. vmcs_writel(GUEST_RFLAGS, flags);
  425. }
  426. vmcs_write32(EXCEPTION_BITMAP, exception_bitmap);
  427. vmcs_writel(GUEST_DR7, dr7);
  428. return 0;
  429. }
  430. static __init int cpu_has_kvm_support(void)
  431. {
  432. unsigned long ecx = cpuid_ecx(1);
  433. return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */
  434. }
  435. static __init int vmx_disabled_by_bios(void)
  436. {
  437. u64 msr;
  438. rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
  439. return (msr & 5) == 1; /* locked but not enabled */
  440. }
  441. static void hardware_enable(void *garbage)
  442. {
  443. int cpu = raw_smp_processor_id();
  444. u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
  445. u64 old;
  446. rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
  447. if ((old & 5) != 5)
  448. /* enable and lock */
  449. wrmsrl(MSR_IA32_FEATURE_CONTROL, old | 5);
  450. write_cr4(read_cr4() | CR4_VMXE); /* FIXME: not cpu hotplug safe */
  451. asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr)
  452. : "memory", "cc");
  453. }
  454. static void hardware_disable(void *garbage)
  455. {
  456. asm volatile (ASM_VMX_VMXOFF : : : "cc");
  457. }
  458. static __init void setup_vmcs_descriptor(void)
  459. {
  460. u32 vmx_msr_low, vmx_msr_high;
  461. rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
  462. vmcs_descriptor.size = vmx_msr_high & 0x1fff;
  463. vmcs_descriptor.order = get_order(vmcs_descriptor.size);
  464. vmcs_descriptor.revision_id = vmx_msr_low;
  465. }
  466. static struct vmcs *alloc_vmcs_cpu(int cpu)
  467. {
  468. int node = cpu_to_node(cpu);
  469. struct page *pages;
  470. struct vmcs *vmcs;
  471. pages = alloc_pages_node(node, GFP_KERNEL, vmcs_descriptor.order);
  472. if (!pages)
  473. return NULL;
  474. vmcs = page_address(pages);
  475. memset(vmcs, 0, vmcs_descriptor.size);
  476. vmcs->revision_id = vmcs_descriptor.revision_id; /* vmcs revision id */
  477. return vmcs;
  478. }
  479. static struct vmcs *alloc_vmcs(void)
  480. {
  481. return alloc_vmcs_cpu(raw_smp_processor_id());
  482. }
  483. static void free_vmcs(struct vmcs *vmcs)
  484. {
  485. free_pages((unsigned long)vmcs, vmcs_descriptor.order);
  486. }
  487. static __exit void free_kvm_area(void)
  488. {
  489. int cpu;
  490. for_each_online_cpu(cpu)
  491. free_vmcs(per_cpu(vmxarea, cpu));
  492. }
  493. extern struct vmcs *alloc_vmcs_cpu(int cpu);
  494. static __init int alloc_kvm_area(void)
  495. {
  496. int cpu;
  497. for_each_online_cpu(cpu) {
  498. struct vmcs *vmcs;
  499. vmcs = alloc_vmcs_cpu(cpu);
  500. if (!vmcs) {
  501. free_kvm_area();
  502. return -ENOMEM;
  503. }
  504. per_cpu(vmxarea, cpu) = vmcs;
  505. }
  506. return 0;
  507. }
  508. static __init int hardware_setup(void)
  509. {
  510. setup_vmcs_descriptor();
  511. return alloc_kvm_area();
  512. }
  513. static __exit void hardware_unsetup(void)
  514. {
  515. free_kvm_area();
  516. }
  517. static void update_exception_bitmap(struct kvm_vcpu *vcpu)
  518. {
  519. if (vcpu->rmode.active)
  520. vmcs_write32(EXCEPTION_BITMAP, ~0);
  521. else
  522. vmcs_write32(EXCEPTION_BITMAP, 1 << PF_VECTOR);
  523. }
  524. static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
  525. {
  526. struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  527. if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
  528. vmcs_write16(sf->selector, save->selector);
  529. vmcs_writel(sf->base, save->base);
  530. vmcs_write32(sf->limit, save->limit);
  531. vmcs_write32(sf->ar_bytes, save->ar);
  532. } else {
  533. u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
  534. << AR_DPL_SHIFT;
  535. vmcs_write32(sf->ar_bytes, 0x93 | dpl);
  536. }
  537. }
  538. static void enter_pmode(struct kvm_vcpu *vcpu)
  539. {
  540. unsigned long flags;
  541. vcpu->rmode.active = 0;
  542. vmcs_writel(GUEST_TR_BASE, vcpu->rmode.tr.base);
  543. vmcs_write32(GUEST_TR_LIMIT, vcpu->rmode.tr.limit);
  544. vmcs_write32(GUEST_TR_AR_BYTES, vcpu->rmode.tr.ar);
  545. flags = vmcs_readl(GUEST_RFLAGS);
  546. flags &= ~(IOPL_MASK | X86_EFLAGS_VM);
  547. flags |= (vcpu->rmode.save_iopl << IOPL_SHIFT);
  548. vmcs_writel(GUEST_RFLAGS, flags);
  549. vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~CR4_VME_MASK) |
  550. (vmcs_readl(CR4_READ_SHADOW) & CR4_VME_MASK));
  551. update_exception_bitmap(vcpu);
  552. fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->rmode.es);
  553. fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->rmode.ds);
  554. fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->rmode.gs);
  555. fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->rmode.fs);
  556. vmcs_write16(GUEST_SS_SELECTOR, 0);
  557. vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
  558. vmcs_write16(GUEST_CS_SELECTOR,
  559. vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
  560. vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
  561. }
  562. static int rmode_tss_base(struct kvm* kvm)
  563. {
  564. gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3;
  565. return base_gfn << PAGE_SHIFT;
  566. }
  567. static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
  568. {
  569. struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  570. save->selector = vmcs_read16(sf->selector);
  571. save->base = vmcs_readl(sf->base);
  572. save->limit = vmcs_read32(sf->limit);
  573. save->ar = vmcs_read32(sf->ar_bytes);
  574. vmcs_write16(sf->selector, vmcs_readl(sf->base) >> 4);
  575. vmcs_write32(sf->limit, 0xffff);
  576. vmcs_write32(sf->ar_bytes, 0xf3);
  577. }
  578. static void enter_rmode(struct kvm_vcpu *vcpu)
  579. {
  580. unsigned long flags;
  581. vcpu->rmode.active = 1;
  582. vcpu->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
  583. vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
  584. vcpu->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
  585. vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
  586. vcpu->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
  587. vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
  588. flags = vmcs_readl(GUEST_RFLAGS);
  589. vcpu->rmode.save_iopl = (flags & IOPL_MASK) >> IOPL_SHIFT;
  590. flags |= IOPL_MASK | X86_EFLAGS_VM;
  591. vmcs_writel(GUEST_RFLAGS, flags);
  592. vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | CR4_VME_MASK);
  593. update_exception_bitmap(vcpu);
  594. vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
  595. vmcs_write32(GUEST_SS_LIMIT, 0xffff);
  596. vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
  597. vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
  598. vmcs_write32(GUEST_CS_LIMIT, 0xffff);
  599. vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
  600. fix_rmode_seg(VCPU_SREG_ES, &vcpu->rmode.es);
  601. fix_rmode_seg(VCPU_SREG_DS, &vcpu->rmode.ds);
  602. fix_rmode_seg(VCPU_SREG_GS, &vcpu->rmode.gs);
  603. fix_rmode_seg(VCPU_SREG_FS, &vcpu->rmode.fs);
  604. }
  605. #ifdef CONFIG_X86_64
  606. static void enter_lmode(struct kvm_vcpu *vcpu)
  607. {
  608. u32 guest_tr_ar;
  609. guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
  610. if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
  611. printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
  612. __FUNCTION__);
  613. vmcs_write32(GUEST_TR_AR_BYTES,
  614. (guest_tr_ar & ~AR_TYPE_MASK)
  615. | AR_TYPE_BUSY_64_TSS);
  616. }
  617. vcpu->shadow_efer |= EFER_LMA;
  618. find_msr_entry(vcpu, MSR_EFER)->data |= EFER_LMA | EFER_LME;
  619. vmcs_write32(VM_ENTRY_CONTROLS,
  620. vmcs_read32(VM_ENTRY_CONTROLS)
  621. | VM_ENTRY_CONTROLS_IA32E_MASK);
  622. }
  623. static void exit_lmode(struct kvm_vcpu *vcpu)
  624. {
  625. vcpu->shadow_efer &= ~EFER_LMA;
  626. vmcs_write32(VM_ENTRY_CONTROLS,
  627. vmcs_read32(VM_ENTRY_CONTROLS)
  628. & ~VM_ENTRY_CONTROLS_IA32E_MASK);
  629. }
  630. #endif
  631. static void vmx_decache_cr0_cr4_guest_bits(struct kvm_vcpu *vcpu)
  632. {
  633. vcpu->cr0 &= KVM_GUEST_CR0_MASK;
  634. vcpu->cr0 |= vmcs_readl(GUEST_CR0) & ~KVM_GUEST_CR0_MASK;
  635. vcpu->cr4 &= KVM_GUEST_CR4_MASK;
  636. vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK;
  637. }
  638. static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  639. {
  640. if (vcpu->rmode.active && (cr0 & CR0_PE_MASK))
  641. enter_pmode(vcpu);
  642. if (!vcpu->rmode.active && !(cr0 & CR0_PE_MASK))
  643. enter_rmode(vcpu);
  644. #ifdef CONFIG_X86_64
  645. if (vcpu->shadow_efer & EFER_LME) {
  646. if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK))
  647. enter_lmode(vcpu);
  648. if (is_paging(vcpu) && !(cr0 & CR0_PG_MASK))
  649. exit_lmode(vcpu);
  650. }
  651. #endif
  652. vmcs_writel(CR0_READ_SHADOW, cr0);
  653. vmcs_writel(GUEST_CR0,
  654. (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON);
  655. vcpu->cr0 = cr0;
  656. }
  657. /*
  658. * Used when restoring the VM to avoid corrupting segment registers
  659. */
  660. static void vmx_set_cr0_no_modeswitch(struct kvm_vcpu *vcpu, unsigned long cr0)
  661. {
  662. if (!vcpu->rmode.active && !(cr0 & CR0_PE_MASK))
  663. enter_rmode(vcpu);
  664. vcpu->rmode.active = ((cr0 & CR0_PE_MASK) == 0);
  665. update_exception_bitmap(vcpu);
  666. vmcs_writel(CR0_READ_SHADOW, cr0);
  667. vmcs_writel(GUEST_CR0,
  668. (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON);
  669. vcpu->cr0 = cr0;
  670. }
  671. static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  672. {
  673. vmcs_writel(GUEST_CR3, cr3);
  674. }
  675. static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  676. {
  677. vmcs_writel(CR4_READ_SHADOW, cr4);
  678. vmcs_writel(GUEST_CR4, cr4 | (vcpu->rmode.active ?
  679. KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON));
  680. vcpu->cr4 = cr4;
  681. }
  682. #ifdef CONFIG_X86_64
  683. static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
  684. {
  685. struct vmx_msr_entry *msr = find_msr_entry(vcpu, MSR_EFER);
  686. vcpu->shadow_efer = efer;
  687. if (efer & EFER_LMA) {
  688. vmcs_write32(VM_ENTRY_CONTROLS,
  689. vmcs_read32(VM_ENTRY_CONTROLS) |
  690. VM_ENTRY_CONTROLS_IA32E_MASK);
  691. msr->data = efer;
  692. } else {
  693. vmcs_write32(VM_ENTRY_CONTROLS,
  694. vmcs_read32(VM_ENTRY_CONTROLS) &
  695. ~VM_ENTRY_CONTROLS_IA32E_MASK);
  696. msr->data = efer & ~EFER_LME;
  697. }
  698. }
  699. #endif
  700. static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
  701. {
  702. struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  703. return vmcs_readl(sf->base);
  704. }
  705. static void vmx_get_segment(struct kvm_vcpu *vcpu,
  706. struct kvm_segment *var, int seg)
  707. {
  708. struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  709. u32 ar;
  710. var->base = vmcs_readl(sf->base);
  711. var->limit = vmcs_read32(sf->limit);
  712. var->selector = vmcs_read16(sf->selector);
  713. ar = vmcs_read32(sf->ar_bytes);
  714. if (ar & AR_UNUSABLE_MASK)
  715. ar = 0;
  716. var->type = ar & 15;
  717. var->s = (ar >> 4) & 1;
  718. var->dpl = (ar >> 5) & 3;
  719. var->present = (ar >> 7) & 1;
  720. var->avl = (ar >> 12) & 1;
  721. var->l = (ar >> 13) & 1;
  722. var->db = (ar >> 14) & 1;
  723. var->g = (ar >> 15) & 1;
  724. var->unusable = (ar >> 16) & 1;
  725. }
  726. static void vmx_set_segment(struct kvm_vcpu *vcpu,
  727. struct kvm_segment *var, int seg)
  728. {
  729. struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  730. u32 ar;
  731. vmcs_writel(sf->base, var->base);
  732. vmcs_write32(sf->limit, var->limit);
  733. vmcs_write16(sf->selector, var->selector);
  734. if (var->unusable)
  735. ar = 1 << 16;
  736. else {
  737. ar = var->type & 15;
  738. ar |= (var->s & 1) << 4;
  739. ar |= (var->dpl & 3) << 5;
  740. ar |= (var->present & 1) << 7;
  741. ar |= (var->avl & 1) << 12;
  742. ar |= (var->l & 1) << 13;
  743. ar |= (var->db & 1) << 14;
  744. ar |= (var->g & 1) << 15;
  745. }
  746. if (ar == 0) /* a 0 value means unusable */
  747. ar = AR_UNUSABLE_MASK;
  748. vmcs_write32(sf->ar_bytes, ar);
  749. }
  750. static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  751. {
  752. u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
  753. *db = (ar >> 14) & 1;
  754. *l = (ar >> 13) & 1;
  755. }
  756. static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  757. {
  758. dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
  759. dt->base = vmcs_readl(GUEST_IDTR_BASE);
  760. }
  761. static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  762. {
  763. vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
  764. vmcs_writel(GUEST_IDTR_BASE, dt->base);
  765. }
  766. static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  767. {
  768. dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
  769. dt->base = vmcs_readl(GUEST_GDTR_BASE);
  770. }
  771. static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  772. {
  773. vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
  774. vmcs_writel(GUEST_GDTR_BASE, dt->base);
  775. }
  776. static int init_rmode_tss(struct kvm* kvm)
  777. {
  778. struct page *p1, *p2, *p3;
  779. gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
  780. char *page;
  781. p1 = _gfn_to_page(kvm, fn++);
  782. p2 = _gfn_to_page(kvm, fn++);
  783. p3 = _gfn_to_page(kvm, fn);
  784. if (!p1 || !p2 || !p3) {
  785. kvm_printf(kvm,"%s: gfn_to_page failed\n", __FUNCTION__);
  786. return 0;
  787. }
  788. page = kmap_atomic(p1, KM_USER0);
  789. memset(page, 0, PAGE_SIZE);
  790. *(u16*)(page + 0x66) = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
  791. kunmap_atomic(page, KM_USER0);
  792. page = kmap_atomic(p2, KM_USER0);
  793. memset(page, 0, PAGE_SIZE);
  794. kunmap_atomic(page, KM_USER0);
  795. page = kmap_atomic(p3, KM_USER0);
  796. memset(page, 0, PAGE_SIZE);
  797. *(page + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1) = ~0;
  798. kunmap_atomic(page, KM_USER0);
  799. return 1;
  800. }
  801. static void vmcs_write32_fixedbits(u32 msr, u32 vmcs_field, u32 val)
  802. {
  803. u32 msr_high, msr_low;
  804. rdmsr(msr, msr_low, msr_high);
  805. val &= msr_high;
  806. val |= msr_low;
  807. vmcs_write32(vmcs_field, val);
  808. }
  809. static void seg_setup(int seg)
  810. {
  811. struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
  812. vmcs_write16(sf->selector, 0);
  813. vmcs_writel(sf->base, 0);
  814. vmcs_write32(sf->limit, 0xffff);
  815. vmcs_write32(sf->ar_bytes, 0x93);
  816. }
  817. /*
  818. * Sets up the vmcs for emulated real mode.
  819. */
  820. static int vmx_vcpu_setup(struct kvm_vcpu *vcpu)
  821. {
  822. u32 host_sysenter_cs;
  823. u32 junk;
  824. unsigned long a;
  825. struct descriptor_table dt;
  826. int i;
  827. int ret = 0;
  828. int nr_good_msrs;
  829. extern asmlinkage void kvm_vmx_return(void);
  830. if (!init_rmode_tss(vcpu->kvm)) {
  831. ret = -ENOMEM;
  832. goto out;
  833. }
  834. memset(vcpu->regs, 0, sizeof(vcpu->regs));
  835. vcpu->regs[VCPU_REGS_RDX] = get_rdx_init_val();
  836. vcpu->cr8 = 0;
  837. vcpu->apic_base = 0xfee00000 |
  838. /*for vcpu 0*/ MSR_IA32_APICBASE_BSP |
  839. MSR_IA32_APICBASE_ENABLE;
  840. fx_init(vcpu);
  841. /*
  842. * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
  843. * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh.
  844. */
  845. vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
  846. vmcs_writel(GUEST_CS_BASE, 0x000f0000);
  847. vmcs_write32(GUEST_CS_LIMIT, 0xffff);
  848. vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
  849. seg_setup(VCPU_SREG_DS);
  850. seg_setup(VCPU_SREG_ES);
  851. seg_setup(VCPU_SREG_FS);
  852. seg_setup(VCPU_SREG_GS);
  853. seg_setup(VCPU_SREG_SS);
  854. vmcs_write16(GUEST_TR_SELECTOR, 0);
  855. vmcs_writel(GUEST_TR_BASE, 0);
  856. vmcs_write32(GUEST_TR_LIMIT, 0xffff);
  857. vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
  858. vmcs_write16(GUEST_LDTR_SELECTOR, 0);
  859. vmcs_writel(GUEST_LDTR_BASE, 0);
  860. vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
  861. vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
  862. vmcs_write32(GUEST_SYSENTER_CS, 0);
  863. vmcs_writel(GUEST_SYSENTER_ESP, 0);
  864. vmcs_writel(GUEST_SYSENTER_EIP, 0);
  865. vmcs_writel(GUEST_RFLAGS, 0x02);
  866. vmcs_writel(GUEST_RIP, 0xfff0);
  867. vmcs_writel(GUEST_RSP, 0);
  868. //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0
  869. vmcs_writel(GUEST_DR7, 0x400);
  870. vmcs_writel(GUEST_GDTR_BASE, 0);
  871. vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
  872. vmcs_writel(GUEST_IDTR_BASE, 0);
  873. vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
  874. vmcs_write32(GUEST_ACTIVITY_STATE, 0);
  875. vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
  876. vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
  877. /* I/O */
  878. vmcs_write64(IO_BITMAP_A, 0);
  879. vmcs_write64(IO_BITMAP_B, 0);
  880. guest_write_tsc(0);
  881. vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
  882. /* Special registers */
  883. vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
  884. /* Control */
  885. vmcs_write32_fixedbits(MSR_IA32_VMX_PINBASED_CTLS,
  886. PIN_BASED_VM_EXEC_CONTROL,
  887. PIN_BASED_EXT_INTR_MASK /* 20.6.1 */
  888. | PIN_BASED_NMI_EXITING /* 20.6.1 */
  889. );
  890. vmcs_write32_fixedbits(MSR_IA32_VMX_PROCBASED_CTLS,
  891. CPU_BASED_VM_EXEC_CONTROL,
  892. CPU_BASED_HLT_EXITING /* 20.6.2 */
  893. | CPU_BASED_CR8_LOAD_EXITING /* 20.6.2 */
  894. | CPU_BASED_CR8_STORE_EXITING /* 20.6.2 */
  895. | CPU_BASED_UNCOND_IO_EXITING /* 20.6.2 */
  896. | CPU_BASED_MOV_DR_EXITING
  897. | CPU_BASED_USE_TSC_OFFSETING /* 21.3 */
  898. );
  899. vmcs_write32(EXCEPTION_BITMAP, 1 << PF_VECTOR);
  900. vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
  901. vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
  902. vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
  903. vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */
  904. vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
  905. vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
  906. vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
  907. vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
  908. vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
  909. vmcs_write16(HOST_FS_SELECTOR, read_fs()); /* 22.2.4 */
  910. vmcs_write16(HOST_GS_SELECTOR, read_gs()); /* 22.2.4 */
  911. vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
  912. #ifdef CONFIG_X86_64
  913. rdmsrl(MSR_FS_BASE, a);
  914. vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
  915. rdmsrl(MSR_GS_BASE, a);
  916. vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
  917. #else
  918. vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
  919. vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
  920. #endif
  921. vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
  922. get_idt(&dt);
  923. vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */
  924. vmcs_writel(HOST_RIP, (unsigned long)kvm_vmx_return); /* 22.2.5 */
  925. rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
  926. vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
  927. rdmsrl(MSR_IA32_SYSENTER_ESP, a);
  928. vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */
  929. rdmsrl(MSR_IA32_SYSENTER_EIP, a);
  930. vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */
  931. for (i = 0; i < NR_VMX_MSR; ++i) {
  932. u32 index = vmx_msr_index[i];
  933. u32 data_low, data_high;
  934. u64 data;
  935. int j = vcpu->nmsrs;
  936. if (rdmsr_safe(index, &data_low, &data_high) < 0)
  937. continue;
  938. if (wrmsr_safe(index, data_low, data_high) < 0)
  939. continue;
  940. data = data_low | ((u64)data_high << 32);
  941. vcpu->host_msrs[j].index = index;
  942. vcpu->host_msrs[j].reserved = 0;
  943. vcpu->host_msrs[j].data = data;
  944. vcpu->guest_msrs[j] = vcpu->host_msrs[j];
  945. ++vcpu->nmsrs;
  946. }
  947. printk(KERN_DEBUG "kvm: msrs: %d\n", vcpu->nmsrs);
  948. nr_good_msrs = vcpu->nmsrs - NR_BAD_MSRS;
  949. vmcs_writel(VM_ENTRY_MSR_LOAD_ADDR,
  950. virt_to_phys(vcpu->guest_msrs + NR_BAD_MSRS));
  951. vmcs_writel(VM_EXIT_MSR_STORE_ADDR,
  952. virt_to_phys(vcpu->guest_msrs + NR_BAD_MSRS));
  953. vmcs_writel(VM_EXIT_MSR_LOAD_ADDR,
  954. virt_to_phys(vcpu->host_msrs + NR_BAD_MSRS));
  955. vmcs_write32_fixedbits(MSR_IA32_VMX_EXIT_CTLS, VM_EXIT_CONTROLS,
  956. (HOST_IS_64 << 9)); /* 22.2,1, 20.7.1 */
  957. vmcs_write32(VM_EXIT_MSR_STORE_COUNT, nr_good_msrs); /* 22.2.2 */
  958. vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, nr_good_msrs); /* 22.2.2 */
  959. vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, nr_good_msrs); /* 22.2.2 */
  960. /* 22.2.1, 20.8.1 */
  961. vmcs_write32_fixedbits(MSR_IA32_VMX_ENTRY_CTLS,
  962. VM_ENTRY_CONTROLS, 0);
  963. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
  964. #ifdef CONFIG_X86_64
  965. vmcs_writel(VIRTUAL_APIC_PAGE_ADDR, 0);
  966. vmcs_writel(TPR_THRESHOLD, 0);
  967. #endif
  968. vmcs_writel(CR0_GUEST_HOST_MASK, KVM_GUEST_CR0_MASK);
  969. vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK);
  970. vcpu->cr0 = 0x60000010;
  971. vmx_set_cr0(vcpu, vcpu->cr0); // enter rmode
  972. vmx_set_cr4(vcpu, 0);
  973. #ifdef CONFIG_X86_64
  974. vmx_set_efer(vcpu, 0);
  975. #endif
  976. return 0;
  977. out:
  978. return ret;
  979. }
  980. static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq)
  981. {
  982. u16 ent[2];
  983. u16 cs;
  984. u16 ip;
  985. unsigned long flags;
  986. unsigned long ss_base = vmcs_readl(GUEST_SS_BASE);
  987. u16 sp = vmcs_readl(GUEST_RSP);
  988. u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT);
  989. if (sp > ss_limit || sp - 6 > sp) {
  990. vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n",
  991. __FUNCTION__,
  992. vmcs_readl(GUEST_RSP),
  993. vmcs_readl(GUEST_SS_BASE),
  994. vmcs_read32(GUEST_SS_LIMIT));
  995. return;
  996. }
  997. if (kvm_read_guest(vcpu, irq * sizeof(ent), sizeof(ent), &ent) !=
  998. sizeof(ent)) {
  999. vcpu_printf(vcpu, "%s: read guest err\n", __FUNCTION__);
  1000. return;
  1001. }
  1002. flags = vmcs_readl(GUEST_RFLAGS);
  1003. cs = vmcs_readl(GUEST_CS_BASE) >> 4;
  1004. ip = vmcs_readl(GUEST_RIP);
  1005. if (kvm_write_guest(vcpu, ss_base + sp - 2, 2, &flags) != 2 ||
  1006. kvm_write_guest(vcpu, ss_base + sp - 4, 2, &cs) != 2 ||
  1007. kvm_write_guest(vcpu, ss_base + sp - 6, 2, &ip) != 2) {
  1008. vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__);
  1009. return;
  1010. }
  1011. vmcs_writel(GUEST_RFLAGS, flags &
  1012. ~( X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF));
  1013. vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ;
  1014. vmcs_writel(GUEST_CS_BASE, ent[1] << 4);
  1015. vmcs_writel(GUEST_RIP, ent[0]);
  1016. vmcs_writel(GUEST_RSP, (vmcs_readl(GUEST_RSP) & ~0xffff) | (sp - 6));
  1017. }
  1018. static void kvm_do_inject_irq(struct kvm_vcpu *vcpu)
  1019. {
  1020. int word_index = __ffs(vcpu->irq_summary);
  1021. int bit_index = __ffs(vcpu->irq_pending[word_index]);
  1022. int irq = word_index * BITS_PER_LONG + bit_index;
  1023. clear_bit(bit_index, &vcpu->irq_pending[word_index]);
  1024. if (!vcpu->irq_pending[word_index])
  1025. clear_bit(word_index, &vcpu->irq_summary);
  1026. if (vcpu->rmode.active) {
  1027. inject_rmode_irq(vcpu, irq);
  1028. return;
  1029. }
  1030. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
  1031. irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
  1032. }
  1033. static void do_interrupt_requests(struct kvm_vcpu *vcpu,
  1034. struct kvm_run *kvm_run)
  1035. {
  1036. u32 cpu_based_vm_exec_control;
  1037. vcpu->interrupt_window_open =
  1038. ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
  1039. (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0);
  1040. if (vcpu->interrupt_window_open &&
  1041. vcpu->irq_summary &&
  1042. !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK))
  1043. /*
  1044. * If interrupts enabled, and not blocked by sti or mov ss. Good.
  1045. */
  1046. kvm_do_inject_irq(vcpu);
  1047. cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
  1048. if (!vcpu->interrupt_window_open &&
  1049. (vcpu->irq_summary || kvm_run->request_interrupt_window))
  1050. /*
  1051. * Interrupts blocked. Wait for unblock.
  1052. */
  1053. cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
  1054. else
  1055. cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
  1056. vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
  1057. }
  1058. static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu)
  1059. {
  1060. struct kvm_guest_debug *dbg = &vcpu->guest_debug;
  1061. set_debugreg(dbg->bp[0], 0);
  1062. set_debugreg(dbg->bp[1], 1);
  1063. set_debugreg(dbg->bp[2], 2);
  1064. set_debugreg(dbg->bp[3], 3);
  1065. if (dbg->singlestep) {
  1066. unsigned long flags;
  1067. flags = vmcs_readl(GUEST_RFLAGS);
  1068. flags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
  1069. vmcs_writel(GUEST_RFLAGS, flags);
  1070. }
  1071. }
  1072. static int handle_rmode_exception(struct kvm_vcpu *vcpu,
  1073. int vec, u32 err_code)
  1074. {
  1075. if (!vcpu->rmode.active)
  1076. return 0;
  1077. if (vec == GP_VECTOR && err_code == 0)
  1078. if (emulate_instruction(vcpu, NULL, 0, 0) == EMULATE_DONE)
  1079. return 1;
  1080. return 0;
  1081. }
  1082. static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1083. {
  1084. u32 intr_info, error_code;
  1085. unsigned long cr2, rip;
  1086. u32 vect_info;
  1087. enum emulation_result er;
  1088. int r;
  1089. vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
  1090. intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
  1091. if ((vect_info & VECTORING_INFO_VALID_MASK) &&
  1092. !is_page_fault(intr_info)) {
  1093. printk(KERN_ERR "%s: unexpected, vectoring info 0x%x "
  1094. "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info);
  1095. }
  1096. if (is_external_interrupt(vect_info)) {
  1097. int irq = vect_info & VECTORING_INFO_VECTOR_MASK;
  1098. set_bit(irq, vcpu->irq_pending);
  1099. set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary);
  1100. }
  1101. if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) { /* nmi */
  1102. asm ("int $2");
  1103. return 1;
  1104. }
  1105. error_code = 0;
  1106. rip = vmcs_readl(GUEST_RIP);
  1107. if (intr_info & INTR_INFO_DELIEVER_CODE_MASK)
  1108. error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
  1109. if (is_page_fault(intr_info)) {
  1110. cr2 = vmcs_readl(EXIT_QUALIFICATION);
  1111. spin_lock(&vcpu->kvm->lock);
  1112. r = kvm_mmu_page_fault(vcpu, cr2, error_code);
  1113. if (r < 0) {
  1114. spin_unlock(&vcpu->kvm->lock);
  1115. return r;
  1116. }
  1117. if (!r) {
  1118. spin_unlock(&vcpu->kvm->lock);
  1119. return 1;
  1120. }
  1121. er = emulate_instruction(vcpu, kvm_run, cr2, error_code);
  1122. spin_unlock(&vcpu->kvm->lock);
  1123. switch (er) {
  1124. case EMULATE_DONE:
  1125. return 1;
  1126. case EMULATE_DO_MMIO:
  1127. ++kvm_stat.mmio_exits;
  1128. kvm_run->exit_reason = KVM_EXIT_MMIO;
  1129. return 0;
  1130. case EMULATE_FAIL:
  1131. vcpu_printf(vcpu, "%s: emulate fail\n", __FUNCTION__);
  1132. break;
  1133. default:
  1134. BUG();
  1135. }
  1136. }
  1137. if (vcpu->rmode.active &&
  1138. handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
  1139. error_code))
  1140. return 1;
  1141. if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == (INTR_TYPE_EXCEPTION | 1)) {
  1142. kvm_run->exit_reason = KVM_EXIT_DEBUG;
  1143. return 0;
  1144. }
  1145. kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
  1146. kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK;
  1147. kvm_run->ex.error_code = error_code;
  1148. return 0;
  1149. }
  1150. static int handle_external_interrupt(struct kvm_vcpu *vcpu,
  1151. struct kvm_run *kvm_run)
  1152. {
  1153. ++kvm_stat.irq_exits;
  1154. return 1;
  1155. }
  1156. static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1157. {
  1158. kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
  1159. return 0;
  1160. }
  1161. static int get_io_count(struct kvm_vcpu *vcpu, u64 *count)
  1162. {
  1163. u64 inst;
  1164. gva_t rip;
  1165. int countr_size;
  1166. int i, n;
  1167. if ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_VM)) {
  1168. countr_size = 2;
  1169. } else {
  1170. u32 cs_ar = vmcs_read32(GUEST_CS_AR_BYTES);
  1171. countr_size = (cs_ar & AR_L_MASK) ? 8:
  1172. (cs_ar & AR_DB_MASK) ? 4: 2;
  1173. }
  1174. rip = vmcs_readl(GUEST_RIP);
  1175. if (countr_size != 8)
  1176. rip += vmcs_readl(GUEST_CS_BASE);
  1177. n = kvm_read_guest(vcpu, rip, sizeof(inst), &inst);
  1178. for (i = 0; i < n; i++) {
  1179. switch (((u8*)&inst)[i]) {
  1180. case 0xf0:
  1181. case 0xf2:
  1182. case 0xf3:
  1183. case 0x2e:
  1184. case 0x36:
  1185. case 0x3e:
  1186. case 0x26:
  1187. case 0x64:
  1188. case 0x65:
  1189. case 0x66:
  1190. break;
  1191. case 0x67:
  1192. countr_size = (countr_size == 2) ? 4: (countr_size >> 1);
  1193. default:
  1194. goto done;
  1195. }
  1196. }
  1197. return 0;
  1198. done:
  1199. countr_size *= 8;
  1200. *count = vcpu->regs[VCPU_REGS_RCX] & (~0ULL >> (64 - countr_size));
  1201. return 1;
  1202. }
  1203. static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1204. {
  1205. u64 exit_qualification;
  1206. ++kvm_stat.io_exits;
  1207. exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
  1208. kvm_run->exit_reason = KVM_EXIT_IO;
  1209. if (exit_qualification & 8)
  1210. kvm_run->io.direction = KVM_EXIT_IO_IN;
  1211. else
  1212. kvm_run->io.direction = KVM_EXIT_IO_OUT;
  1213. kvm_run->io.size = (exit_qualification & 7) + 1;
  1214. kvm_run->io.string = (exit_qualification & 16) != 0;
  1215. kvm_run->io.string_down
  1216. = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0;
  1217. kvm_run->io.rep = (exit_qualification & 32) != 0;
  1218. kvm_run->io.port = exit_qualification >> 16;
  1219. if (kvm_run->io.string) {
  1220. if (!get_io_count(vcpu, &kvm_run->io.count))
  1221. return 1;
  1222. kvm_run->io.address = vmcs_readl(GUEST_LINEAR_ADDRESS);
  1223. } else
  1224. kvm_run->io.value = vcpu->regs[VCPU_REGS_RAX]; /* rax */
  1225. return 0;
  1226. }
  1227. static void
  1228. vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
  1229. {
  1230. /*
  1231. * Patch in the VMCALL instruction:
  1232. */
  1233. hypercall[0] = 0x0f;
  1234. hypercall[1] = 0x01;
  1235. hypercall[2] = 0xc1;
  1236. hypercall[3] = 0xc3;
  1237. }
  1238. static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1239. {
  1240. u64 exit_qualification;
  1241. int cr;
  1242. int reg;
  1243. exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
  1244. cr = exit_qualification & 15;
  1245. reg = (exit_qualification >> 8) & 15;
  1246. switch ((exit_qualification >> 4) & 3) {
  1247. case 0: /* mov to cr */
  1248. switch (cr) {
  1249. case 0:
  1250. vcpu_load_rsp_rip(vcpu);
  1251. set_cr0(vcpu, vcpu->regs[reg]);
  1252. skip_emulated_instruction(vcpu);
  1253. return 1;
  1254. case 3:
  1255. vcpu_load_rsp_rip(vcpu);
  1256. set_cr3(vcpu, vcpu->regs[reg]);
  1257. skip_emulated_instruction(vcpu);
  1258. return 1;
  1259. case 4:
  1260. vcpu_load_rsp_rip(vcpu);
  1261. set_cr4(vcpu, vcpu->regs[reg]);
  1262. skip_emulated_instruction(vcpu);
  1263. return 1;
  1264. case 8:
  1265. vcpu_load_rsp_rip(vcpu);
  1266. set_cr8(vcpu, vcpu->regs[reg]);
  1267. skip_emulated_instruction(vcpu);
  1268. return 1;
  1269. };
  1270. break;
  1271. case 1: /*mov from cr*/
  1272. switch (cr) {
  1273. case 3:
  1274. vcpu_load_rsp_rip(vcpu);
  1275. vcpu->regs[reg] = vcpu->cr3;
  1276. vcpu_put_rsp_rip(vcpu);
  1277. skip_emulated_instruction(vcpu);
  1278. return 1;
  1279. case 8:
  1280. printk(KERN_DEBUG "handle_cr: read CR8 "
  1281. "cpu erratum AA15\n");
  1282. vcpu_load_rsp_rip(vcpu);
  1283. vcpu->regs[reg] = vcpu->cr8;
  1284. vcpu_put_rsp_rip(vcpu);
  1285. skip_emulated_instruction(vcpu);
  1286. return 1;
  1287. }
  1288. break;
  1289. case 3: /* lmsw */
  1290. lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f);
  1291. skip_emulated_instruction(vcpu);
  1292. return 1;
  1293. default:
  1294. break;
  1295. }
  1296. kvm_run->exit_reason = 0;
  1297. printk(KERN_ERR "kvm: unhandled control register: op %d cr %d\n",
  1298. (int)(exit_qualification >> 4) & 3, cr);
  1299. return 0;
  1300. }
  1301. static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1302. {
  1303. u64 exit_qualification;
  1304. unsigned long val;
  1305. int dr, reg;
  1306. /*
  1307. * FIXME: this code assumes the host is debugging the guest.
  1308. * need to deal with guest debugging itself too.
  1309. */
  1310. exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
  1311. dr = exit_qualification & 7;
  1312. reg = (exit_qualification >> 8) & 15;
  1313. vcpu_load_rsp_rip(vcpu);
  1314. if (exit_qualification & 16) {
  1315. /* mov from dr */
  1316. switch (dr) {
  1317. case 6:
  1318. val = 0xffff0ff0;
  1319. break;
  1320. case 7:
  1321. val = 0x400;
  1322. break;
  1323. default:
  1324. val = 0;
  1325. }
  1326. vcpu->regs[reg] = val;
  1327. } else {
  1328. /* mov to dr */
  1329. }
  1330. vcpu_put_rsp_rip(vcpu);
  1331. skip_emulated_instruction(vcpu);
  1332. return 1;
  1333. }
  1334. static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1335. {
  1336. kvm_run->exit_reason = KVM_EXIT_CPUID;
  1337. return 0;
  1338. }
  1339. static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1340. {
  1341. u32 ecx = vcpu->regs[VCPU_REGS_RCX];
  1342. u64 data;
  1343. if (vmx_get_msr(vcpu, ecx, &data)) {
  1344. vmx_inject_gp(vcpu, 0);
  1345. return 1;
  1346. }
  1347. /* FIXME: handling of bits 32:63 of rax, rdx */
  1348. vcpu->regs[VCPU_REGS_RAX] = data & -1u;
  1349. vcpu->regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
  1350. skip_emulated_instruction(vcpu);
  1351. return 1;
  1352. }
  1353. static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1354. {
  1355. u32 ecx = vcpu->regs[VCPU_REGS_RCX];
  1356. u64 data = (vcpu->regs[VCPU_REGS_RAX] & -1u)
  1357. | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32);
  1358. if (vmx_set_msr(vcpu, ecx, data) != 0) {
  1359. vmx_inject_gp(vcpu, 0);
  1360. return 1;
  1361. }
  1362. skip_emulated_instruction(vcpu);
  1363. return 1;
  1364. }
  1365. static void post_kvm_run_save(struct kvm_vcpu *vcpu,
  1366. struct kvm_run *kvm_run)
  1367. {
  1368. kvm_run->if_flag = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) != 0;
  1369. kvm_run->cr8 = vcpu->cr8;
  1370. kvm_run->apic_base = vcpu->apic_base;
  1371. kvm_run->ready_for_interrupt_injection = (vcpu->interrupt_window_open &&
  1372. vcpu->irq_summary == 0);
  1373. }
  1374. static int handle_interrupt_window(struct kvm_vcpu *vcpu,
  1375. struct kvm_run *kvm_run)
  1376. {
  1377. /*
  1378. * If the user space waits to inject interrupts, exit as soon as
  1379. * possible
  1380. */
  1381. if (kvm_run->request_interrupt_window &&
  1382. !vcpu->irq_summary) {
  1383. kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
  1384. ++kvm_stat.irq_window_exits;
  1385. return 0;
  1386. }
  1387. return 1;
  1388. }
  1389. static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1390. {
  1391. skip_emulated_instruction(vcpu);
  1392. if (vcpu->irq_summary)
  1393. return 1;
  1394. kvm_run->exit_reason = KVM_EXIT_HLT;
  1395. ++kvm_stat.halt_exits;
  1396. return 0;
  1397. }
  1398. static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1399. {
  1400. vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP)+3);
  1401. return kvm_hypercall(vcpu, kvm_run);
  1402. }
  1403. /*
  1404. * The exit handlers return 1 if the exit was handled fully and guest execution
  1405. * may resume. Otherwise they set the kvm_run parameter to indicate what needs
  1406. * to be done to userspace and return 0.
  1407. */
  1408. static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
  1409. struct kvm_run *kvm_run) = {
  1410. [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
  1411. [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
  1412. [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
  1413. [EXIT_REASON_IO_INSTRUCTION] = handle_io,
  1414. [EXIT_REASON_CR_ACCESS] = handle_cr,
  1415. [EXIT_REASON_DR_ACCESS] = handle_dr,
  1416. [EXIT_REASON_CPUID] = handle_cpuid,
  1417. [EXIT_REASON_MSR_READ] = handle_rdmsr,
  1418. [EXIT_REASON_MSR_WRITE] = handle_wrmsr,
  1419. [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window,
  1420. [EXIT_REASON_HLT] = handle_halt,
  1421. [EXIT_REASON_VMCALL] = handle_vmcall,
  1422. };
  1423. static const int kvm_vmx_max_exit_handlers =
  1424. sizeof(kvm_vmx_exit_handlers) / sizeof(*kvm_vmx_exit_handlers);
  1425. /*
  1426. * The guest has exited. See if we can fix it or if we need userspace
  1427. * assistance.
  1428. */
  1429. static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
  1430. {
  1431. u32 vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
  1432. u32 exit_reason = vmcs_read32(VM_EXIT_REASON);
  1433. if ( (vectoring_info & VECTORING_INFO_VALID_MASK) &&
  1434. exit_reason != EXIT_REASON_EXCEPTION_NMI )
  1435. printk(KERN_WARNING "%s: unexpected, valid vectoring info and "
  1436. "exit reason is 0x%x\n", __FUNCTION__, exit_reason);
  1437. kvm_run->instruction_length = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
  1438. if (exit_reason < kvm_vmx_max_exit_handlers
  1439. && kvm_vmx_exit_handlers[exit_reason])
  1440. return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run);
  1441. else {
  1442. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  1443. kvm_run->hw.hardware_exit_reason = exit_reason;
  1444. }
  1445. return 0;
  1446. }
  1447. /*
  1448. * Check if userspace requested an interrupt window, and that the
  1449. * interrupt window is open.
  1450. *
  1451. * No need to exit to userspace if we already have an interrupt queued.
  1452. */
  1453. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
  1454. struct kvm_run *kvm_run)
  1455. {
  1456. return (!vcpu->irq_summary &&
  1457. kvm_run->request_interrupt_window &&
  1458. vcpu->interrupt_window_open &&
  1459. (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF));
  1460. }
  1461. static int vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1462. {
  1463. u8 fail;
  1464. u16 fs_sel, gs_sel, ldt_sel;
  1465. int fs_gs_ldt_reload_needed;
  1466. int r;
  1467. again:
  1468. /*
  1469. * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
  1470. * allow segment selectors with cpl > 0 or ti == 1.
  1471. */
  1472. fs_sel = read_fs();
  1473. gs_sel = read_gs();
  1474. ldt_sel = read_ldt();
  1475. fs_gs_ldt_reload_needed = (fs_sel & 7) | (gs_sel & 7) | ldt_sel;
  1476. if (!fs_gs_ldt_reload_needed) {
  1477. vmcs_write16(HOST_FS_SELECTOR, fs_sel);
  1478. vmcs_write16(HOST_GS_SELECTOR, gs_sel);
  1479. } else {
  1480. vmcs_write16(HOST_FS_SELECTOR, 0);
  1481. vmcs_write16(HOST_GS_SELECTOR, 0);
  1482. }
  1483. #ifdef CONFIG_X86_64
  1484. vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
  1485. vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
  1486. #else
  1487. vmcs_writel(HOST_FS_BASE, segment_base(fs_sel));
  1488. vmcs_writel(HOST_GS_BASE, segment_base(gs_sel));
  1489. #endif
  1490. if (!vcpu->mmio_read_completed)
  1491. do_interrupt_requests(vcpu, kvm_run);
  1492. if (vcpu->guest_debug.enabled)
  1493. kvm_guest_debug_pre(vcpu);
  1494. fx_save(vcpu->host_fx_image);
  1495. fx_restore(vcpu->guest_fx_image);
  1496. save_msrs(vcpu->host_msrs, vcpu->nmsrs);
  1497. load_msrs(vcpu->guest_msrs, NR_BAD_MSRS);
  1498. asm (
  1499. /* Store host registers */
  1500. "pushf \n\t"
  1501. #ifdef CONFIG_X86_64
  1502. "push %%rax; push %%rbx; push %%rdx;"
  1503. "push %%rsi; push %%rdi; push %%rbp;"
  1504. "push %%r8; push %%r9; push %%r10; push %%r11;"
  1505. "push %%r12; push %%r13; push %%r14; push %%r15;"
  1506. "push %%rcx \n\t"
  1507. ASM_VMX_VMWRITE_RSP_RDX "\n\t"
  1508. #else
  1509. "pusha; push %%ecx \n\t"
  1510. ASM_VMX_VMWRITE_RSP_RDX "\n\t"
  1511. #endif
  1512. /* Check if vmlaunch of vmresume is needed */
  1513. "cmp $0, %1 \n\t"
  1514. /* Load guest registers. Don't clobber flags. */
  1515. #ifdef CONFIG_X86_64
  1516. "mov %c[cr2](%3), %%rax \n\t"
  1517. "mov %%rax, %%cr2 \n\t"
  1518. "mov %c[rax](%3), %%rax \n\t"
  1519. "mov %c[rbx](%3), %%rbx \n\t"
  1520. "mov %c[rdx](%3), %%rdx \n\t"
  1521. "mov %c[rsi](%3), %%rsi \n\t"
  1522. "mov %c[rdi](%3), %%rdi \n\t"
  1523. "mov %c[rbp](%3), %%rbp \n\t"
  1524. "mov %c[r8](%3), %%r8 \n\t"
  1525. "mov %c[r9](%3), %%r9 \n\t"
  1526. "mov %c[r10](%3), %%r10 \n\t"
  1527. "mov %c[r11](%3), %%r11 \n\t"
  1528. "mov %c[r12](%3), %%r12 \n\t"
  1529. "mov %c[r13](%3), %%r13 \n\t"
  1530. "mov %c[r14](%3), %%r14 \n\t"
  1531. "mov %c[r15](%3), %%r15 \n\t"
  1532. "mov %c[rcx](%3), %%rcx \n\t" /* kills %3 (rcx) */
  1533. #else
  1534. "mov %c[cr2](%3), %%eax \n\t"
  1535. "mov %%eax, %%cr2 \n\t"
  1536. "mov %c[rax](%3), %%eax \n\t"
  1537. "mov %c[rbx](%3), %%ebx \n\t"
  1538. "mov %c[rdx](%3), %%edx \n\t"
  1539. "mov %c[rsi](%3), %%esi \n\t"
  1540. "mov %c[rdi](%3), %%edi \n\t"
  1541. "mov %c[rbp](%3), %%ebp \n\t"
  1542. "mov %c[rcx](%3), %%ecx \n\t" /* kills %3 (ecx) */
  1543. #endif
  1544. /* Enter guest mode */
  1545. "jne launched \n\t"
  1546. ASM_VMX_VMLAUNCH "\n\t"
  1547. "jmp kvm_vmx_return \n\t"
  1548. "launched: " ASM_VMX_VMRESUME "\n\t"
  1549. ".globl kvm_vmx_return \n\t"
  1550. "kvm_vmx_return: "
  1551. /* Save guest registers, load host registers, keep flags */
  1552. #ifdef CONFIG_X86_64
  1553. "xchg %3, (%%rsp) \n\t"
  1554. "mov %%rax, %c[rax](%3) \n\t"
  1555. "mov %%rbx, %c[rbx](%3) \n\t"
  1556. "pushq (%%rsp); popq %c[rcx](%3) \n\t"
  1557. "mov %%rdx, %c[rdx](%3) \n\t"
  1558. "mov %%rsi, %c[rsi](%3) \n\t"
  1559. "mov %%rdi, %c[rdi](%3) \n\t"
  1560. "mov %%rbp, %c[rbp](%3) \n\t"
  1561. "mov %%r8, %c[r8](%3) \n\t"
  1562. "mov %%r9, %c[r9](%3) \n\t"
  1563. "mov %%r10, %c[r10](%3) \n\t"
  1564. "mov %%r11, %c[r11](%3) \n\t"
  1565. "mov %%r12, %c[r12](%3) \n\t"
  1566. "mov %%r13, %c[r13](%3) \n\t"
  1567. "mov %%r14, %c[r14](%3) \n\t"
  1568. "mov %%r15, %c[r15](%3) \n\t"
  1569. "mov %%cr2, %%rax \n\t"
  1570. "mov %%rax, %c[cr2](%3) \n\t"
  1571. "mov (%%rsp), %3 \n\t"
  1572. "pop %%rcx; pop %%r15; pop %%r14; pop %%r13; pop %%r12;"
  1573. "pop %%r11; pop %%r10; pop %%r9; pop %%r8;"
  1574. "pop %%rbp; pop %%rdi; pop %%rsi;"
  1575. "pop %%rdx; pop %%rbx; pop %%rax \n\t"
  1576. #else
  1577. "xchg %3, (%%esp) \n\t"
  1578. "mov %%eax, %c[rax](%3) \n\t"
  1579. "mov %%ebx, %c[rbx](%3) \n\t"
  1580. "pushl (%%esp); popl %c[rcx](%3) \n\t"
  1581. "mov %%edx, %c[rdx](%3) \n\t"
  1582. "mov %%esi, %c[rsi](%3) \n\t"
  1583. "mov %%edi, %c[rdi](%3) \n\t"
  1584. "mov %%ebp, %c[rbp](%3) \n\t"
  1585. "mov %%cr2, %%eax \n\t"
  1586. "mov %%eax, %c[cr2](%3) \n\t"
  1587. "mov (%%esp), %3 \n\t"
  1588. "pop %%ecx; popa \n\t"
  1589. #endif
  1590. "setbe %0 \n\t"
  1591. "popf \n\t"
  1592. : "=q" (fail)
  1593. : "r"(vcpu->launched), "d"((unsigned long)HOST_RSP),
  1594. "c"(vcpu),
  1595. [rax]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RAX])),
  1596. [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])),
  1597. [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])),
  1598. [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])),
  1599. [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])),
  1600. [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])),
  1601. [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])),
  1602. #ifdef CONFIG_X86_64
  1603. [r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])),
  1604. [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])),
  1605. [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])),
  1606. [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])),
  1607. [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])),
  1608. [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])),
  1609. [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])),
  1610. [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])),
  1611. #endif
  1612. [cr2]"i"(offsetof(struct kvm_vcpu, cr2))
  1613. : "cc", "memory" );
  1614. /*
  1615. * Reload segment selectors ASAP. (it's needed for a functional
  1616. * kernel: x86 relies on having __KERNEL_PDA in %fs and x86_64
  1617. * relies on having 0 in %gs for the CPU PDA to work.)
  1618. */
  1619. if (fs_gs_ldt_reload_needed) {
  1620. load_ldt(ldt_sel);
  1621. load_fs(fs_sel);
  1622. /*
  1623. * If we have to reload gs, we must take care to
  1624. * preserve our gs base.
  1625. */
  1626. local_irq_disable();
  1627. load_gs(gs_sel);
  1628. #ifdef CONFIG_X86_64
  1629. wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
  1630. #endif
  1631. local_irq_enable();
  1632. reload_tss();
  1633. }
  1634. ++kvm_stat.exits;
  1635. save_msrs(vcpu->guest_msrs, NR_BAD_MSRS);
  1636. load_msrs(vcpu->host_msrs, NR_BAD_MSRS);
  1637. fx_save(vcpu->guest_fx_image);
  1638. fx_restore(vcpu->host_fx_image);
  1639. vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0;
  1640. asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
  1641. kvm_run->exit_type = 0;
  1642. if (fail) {
  1643. kvm_run->exit_type = KVM_EXIT_TYPE_FAIL_ENTRY;
  1644. kvm_run->exit_reason = vmcs_read32(VM_INSTRUCTION_ERROR);
  1645. r = 0;
  1646. } else {
  1647. /*
  1648. * Profile KVM exit RIPs:
  1649. */
  1650. if (unlikely(prof_on == KVM_PROFILING))
  1651. profile_hit(KVM_PROFILING, (void *)vmcs_readl(GUEST_RIP));
  1652. vcpu->launched = 1;
  1653. kvm_run->exit_type = KVM_EXIT_TYPE_VM_EXIT;
  1654. r = kvm_handle_exit(kvm_run, vcpu);
  1655. if (r > 0) {
  1656. /* Give scheduler a change to reschedule. */
  1657. if (signal_pending(current)) {
  1658. ++kvm_stat.signal_exits;
  1659. post_kvm_run_save(vcpu, kvm_run);
  1660. return -EINTR;
  1661. }
  1662. if (dm_request_for_irq_injection(vcpu, kvm_run)) {
  1663. ++kvm_stat.request_irq_exits;
  1664. post_kvm_run_save(vcpu, kvm_run);
  1665. return -EINTR;
  1666. }
  1667. kvm_resched(vcpu);
  1668. goto again;
  1669. }
  1670. }
  1671. post_kvm_run_save(vcpu, kvm_run);
  1672. return r;
  1673. }
  1674. static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
  1675. {
  1676. vmcs_writel(GUEST_CR3, vmcs_readl(GUEST_CR3));
  1677. }
  1678. static void vmx_inject_page_fault(struct kvm_vcpu *vcpu,
  1679. unsigned long addr,
  1680. u32 err_code)
  1681. {
  1682. u32 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
  1683. ++kvm_stat.pf_guest;
  1684. if (is_page_fault(vect_info)) {
  1685. printk(KERN_DEBUG "inject_page_fault: "
  1686. "double fault 0x%lx @ 0x%lx\n",
  1687. addr, vmcs_readl(GUEST_RIP));
  1688. vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0);
  1689. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
  1690. DF_VECTOR |
  1691. INTR_TYPE_EXCEPTION |
  1692. INTR_INFO_DELIEVER_CODE_MASK |
  1693. INTR_INFO_VALID_MASK);
  1694. return;
  1695. }
  1696. vcpu->cr2 = addr;
  1697. vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code);
  1698. vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
  1699. PF_VECTOR |
  1700. INTR_TYPE_EXCEPTION |
  1701. INTR_INFO_DELIEVER_CODE_MASK |
  1702. INTR_INFO_VALID_MASK);
  1703. }
  1704. static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
  1705. {
  1706. if (vcpu->vmcs) {
  1707. on_each_cpu(__vcpu_clear, vcpu, 0, 1);
  1708. free_vmcs(vcpu->vmcs);
  1709. vcpu->vmcs = NULL;
  1710. }
  1711. }
  1712. static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
  1713. {
  1714. vmx_free_vmcs(vcpu);
  1715. }
  1716. static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
  1717. {
  1718. struct vmcs *vmcs;
  1719. vcpu->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
  1720. if (!vcpu->guest_msrs)
  1721. return -ENOMEM;
  1722. vcpu->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
  1723. if (!vcpu->host_msrs)
  1724. goto out_free_guest_msrs;
  1725. vmcs = alloc_vmcs();
  1726. if (!vmcs)
  1727. goto out_free_msrs;
  1728. vmcs_clear(vmcs);
  1729. vcpu->vmcs = vmcs;
  1730. vcpu->launched = 0;
  1731. return 0;
  1732. out_free_msrs:
  1733. kfree(vcpu->host_msrs);
  1734. vcpu->host_msrs = NULL;
  1735. out_free_guest_msrs:
  1736. kfree(vcpu->guest_msrs);
  1737. vcpu->guest_msrs = NULL;
  1738. return -ENOMEM;
  1739. }
  1740. static struct kvm_arch_ops vmx_arch_ops = {
  1741. .cpu_has_kvm_support = cpu_has_kvm_support,
  1742. .disabled_by_bios = vmx_disabled_by_bios,
  1743. .hardware_setup = hardware_setup,
  1744. .hardware_unsetup = hardware_unsetup,
  1745. .hardware_enable = hardware_enable,
  1746. .hardware_disable = hardware_disable,
  1747. .vcpu_create = vmx_create_vcpu,
  1748. .vcpu_free = vmx_free_vcpu,
  1749. .vcpu_load = vmx_vcpu_load,
  1750. .vcpu_put = vmx_vcpu_put,
  1751. .vcpu_decache = vmx_vcpu_decache,
  1752. .set_guest_debug = set_guest_debug,
  1753. .get_msr = vmx_get_msr,
  1754. .set_msr = vmx_set_msr,
  1755. .get_segment_base = vmx_get_segment_base,
  1756. .get_segment = vmx_get_segment,
  1757. .set_segment = vmx_set_segment,
  1758. .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
  1759. .decache_cr0_cr4_guest_bits = vmx_decache_cr0_cr4_guest_bits,
  1760. .set_cr0 = vmx_set_cr0,
  1761. .set_cr0_no_modeswitch = vmx_set_cr0_no_modeswitch,
  1762. .set_cr3 = vmx_set_cr3,
  1763. .set_cr4 = vmx_set_cr4,
  1764. #ifdef CONFIG_X86_64
  1765. .set_efer = vmx_set_efer,
  1766. #endif
  1767. .get_idt = vmx_get_idt,
  1768. .set_idt = vmx_set_idt,
  1769. .get_gdt = vmx_get_gdt,
  1770. .set_gdt = vmx_set_gdt,
  1771. .cache_regs = vcpu_load_rsp_rip,
  1772. .decache_regs = vcpu_put_rsp_rip,
  1773. .get_rflags = vmx_get_rflags,
  1774. .set_rflags = vmx_set_rflags,
  1775. .tlb_flush = vmx_flush_tlb,
  1776. .inject_page_fault = vmx_inject_page_fault,
  1777. .inject_gp = vmx_inject_gp,
  1778. .run = vmx_vcpu_run,
  1779. .skip_emulated_instruction = skip_emulated_instruction,
  1780. .vcpu_setup = vmx_vcpu_setup,
  1781. .patch_hypercall = vmx_patch_hypercall,
  1782. };
  1783. static int __init vmx_init(void)
  1784. {
  1785. return kvm_init_arch(&vmx_arch_ops, THIS_MODULE);
  1786. }
  1787. static void __exit vmx_exit(void)
  1788. {
  1789. kvm_exit_arch();
  1790. }
  1791. module_init(vmx_init)
  1792. module_exit(vmx_exit)