svm.c 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * AMD SVM support
  5. *
  6. * Copyright (C) 2006 Qumranet, Inc.
  7. *
  8. * Authors:
  9. * Yaniv Kamay <yaniv@qumranet.com>
  10. * Avi Kivity <avi@qumranet.com>
  11. *
  12. * This work is licensed under the terms of the GNU GPL, version 2. See
  13. * the COPYING file in the top-level directory.
  14. *
  15. */
  16. #include <linux/module.h>
  17. #include <linux/kernel.h>
  18. #include <linux/vmalloc.h>
  19. #include <linux/highmem.h>
  20. #include <linux/profile.h>
  21. #include <asm/desc.h>
  22. #include "kvm_svm.h"
  23. #include "x86_emulate.h"
  24. MODULE_AUTHOR("Qumranet");
  25. MODULE_LICENSE("GPL");
  26. #define IOPM_ALLOC_ORDER 2
  27. #define MSRPM_ALLOC_ORDER 1
  28. #define DB_VECTOR 1
  29. #define UD_VECTOR 6
  30. #define GP_VECTOR 13
  31. #define DR7_GD_MASK (1 << 13)
  32. #define DR6_BD_MASK (1 << 13)
  33. #define CR4_DE_MASK (1UL << 3)
  34. #define SEG_TYPE_LDT 2
  35. #define SEG_TYPE_BUSY_TSS16 3
  36. #define KVM_EFER_LMA (1 << 10)
  37. #define KVM_EFER_LME (1 << 8)
  38. unsigned long iopm_base;
  39. unsigned long msrpm_base;
  40. struct kvm_ldttss_desc {
  41. u16 limit0;
  42. u16 base0;
  43. unsigned base1 : 8, type : 5, dpl : 2, p : 1;
  44. unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8;
  45. u32 base3;
  46. u32 zero1;
  47. } __attribute__((packed));
  48. struct svm_cpu_data {
  49. int cpu;
  50. uint64_t asid_generation;
  51. uint32_t max_asid;
  52. uint32_t next_asid;
  53. struct kvm_ldttss_desc *tss_desc;
  54. struct page *save_area;
  55. };
  56. static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
  57. struct svm_init_data {
  58. int cpu;
  59. int r;
  60. };
  61. static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
  62. #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
  63. #define MSRS_RANGE_SIZE 2048
  64. #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
  65. #define MAX_INST_SIZE 15
  66. static unsigned get_addr_size(struct kvm_vcpu *vcpu)
  67. {
  68. struct vmcb_save_area *sa = &vcpu->svm->vmcb->save;
  69. u16 cs_attrib;
  70. if (!(sa->cr0 & CR0_PE_MASK) || (sa->rflags & X86_EFLAGS_VM))
  71. return 2;
  72. cs_attrib = sa->cs.attrib;
  73. return (cs_attrib & SVM_SELECTOR_L_MASK) ? 8 :
  74. (cs_attrib & SVM_SELECTOR_DB_MASK) ? 4 : 2;
  75. }
  76. static inline u8 pop_irq(struct kvm_vcpu *vcpu)
  77. {
  78. int word_index = __ffs(vcpu->irq_summary);
  79. int bit_index = __ffs(vcpu->irq_pending[word_index]);
  80. int irq = word_index * BITS_PER_LONG + bit_index;
  81. clear_bit(bit_index, &vcpu->irq_pending[word_index]);
  82. if (!vcpu->irq_pending[word_index])
  83. clear_bit(word_index, &vcpu->irq_summary);
  84. return irq;
  85. }
  86. static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq)
  87. {
  88. set_bit(irq, vcpu->irq_pending);
  89. set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary);
  90. }
  91. static inline void clgi(void)
  92. {
  93. asm volatile (SVM_CLGI);
  94. }
  95. static inline void stgi(void)
  96. {
  97. asm volatile (SVM_STGI);
  98. }
  99. static inline void invlpga(unsigned long addr, u32 asid)
  100. {
  101. asm volatile (SVM_INVLPGA :: "a"(addr), "c"(asid));
  102. }
  103. static inline unsigned long kvm_read_cr2(void)
  104. {
  105. unsigned long cr2;
  106. asm volatile ("mov %%cr2, %0" : "=r" (cr2));
  107. return cr2;
  108. }
  109. static inline void kvm_write_cr2(unsigned long val)
  110. {
  111. asm volatile ("mov %0, %%cr2" :: "r" (val));
  112. }
  113. static inline unsigned long read_dr6(void)
  114. {
  115. unsigned long dr6;
  116. asm volatile ("mov %%dr6, %0" : "=r" (dr6));
  117. return dr6;
  118. }
  119. static inline void write_dr6(unsigned long val)
  120. {
  121. asm volatile ("mov %0, %%dr6" :: "r" (val));
  122. }
  123. static inline unsigned long read_dr7(void)
  124. {
  125. unsigned long dr7;
  126. asm volatile ("mov %%dr7, %0" : "=r" (dr7));
  127. return dr7;
  128. }
  129. static inline void write_dr7(unsigned long val)
  130. {
  131. asm volatile ("mov %0, %%dr7" :: "r" (val));
  132. }
  133. static inline void force_new_asid(struct kvm_vcpu *vcpu)
  134. {
  135. vcpu->svm->asid_generation--;
  136. }
  137. static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
  138. {
  139. force_new_asid(vcpu);
  140. }
  141. static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
  142. {
  143. if (!(efer & KVM_EFER_LMA))
  144. efer &= ~KVM_EFER_LME;
  145. vcpu->svm->vmcb->save.efer = efer | MSR_EFER_SVME_MASK;
  146. vcpu->shadow_efer = efer;
  147. }
  148. static void svm_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code)
  149. {
  150. vcpu->svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  151. SVM_EVTINJ_VALID_ERR |
  152. SVM_EVTINJ_TYPE_EXEPT |
  153. GP_VECTOR;
  154. vcpu->svm->vmcb->control.event_inj_err = error_code;
  155. }
  156. static void inject_ud(struct kvm_vcpu *vcpu)
  157. {
  158. vcpu->svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  159. SVM_EVTINJ_TYPE_EXEPT |
  160. UD_VECTOR;
  161. }
  162. static void inject_db(struct kvm_vcpu *vcpu)
  163. {
  164. vcpu->svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  165. SVM_EVTINJ_TYPE_EXEPT |
  166. DB_VECTOR;
  167. }
  168. static int is_page_fault(uint32_t info)
  169. {
  170. info &= SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
  171. return info == (PF_VECTOR | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT);
  172. }
  173. static int is_external_interrupt(u32 info)
  174. {
  175. info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
  176. return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
  177. }
  178. static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
  179. {
  180. if (!vcpu->svm->next_rip) {
  181. printk(KERN_DEBUG "%s: NOP\n", __FUNCTION__);
  182. return;
  183. }
  184. if (vcpu->svm->next_rip - vcpu->svm->vmcb->save.rip > 15) {
  185. printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n",
  186. __FUNCTION__,
  187. vcpu->svm->vmcb->save.rip,
  188. vcpu->svm->next_rip);
  189. }
  190. vcpu->rip = vcpu->svm->vmcb->save.rip = vcpu->svm->next_rip;
  191. vcpu->svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
  192. vcpu->interrupt_window_open = 1;
  193. }
  194. static int has_svm(void)
  195. {
  196. uint32_t eax, ebx, ecx, edx;
  197. if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
  198. printk(KERN_INFO "has_svm: not amd\n");
  199. return 0;
  200. }
  201. cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
  202. if (eax < SVM_CPUID_FUNC) {
  203. printk(KERN_INFO "has_svm: can't execute cpuid_8000000a\n");
  204. return 0;
  205. }
  206. cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
  207. if (!(ecx & (1 << SVM_CPUID_FEATURE_SHIFT))) {
  208. printk(KERN_DEBUG "has_svm: svm not available\n");
  209. return 0;
  210. }
  211. return 1;
  212. }
  213. static void svm_hardware_disable(void *garbage)
  214. {
  215. struct svm_cpu_data *svm_data
  216. = per_cpu(svm_data, raw_smp_processor_id());
  217. if (svm_data) {
  218. uint64_t efer;
  219. wrmsrl(MSR_VM_HSAVE_PA, 0);
  220. rdmsrl(MSR_EFER, efer);
  221. wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK);
  222. per_cpu(svm_data, raw_smp_processor_id()) = NULL;
  223. __free_page(svm_data->save_area);
  224. kfree(svm_data);
  225. }
  226. }
  227. static void svm_hardware_enable(void *garbage)
  228. {
  229. struct svm_cpu_data *svm_data;
  230. uint64_t efer;
  231. #ifdef CONFIG_X86_64
  232. struct desc_ptr gdt_descr;
  233. #else
  234. struct Xgt_desc_struct gdt_descr;
  235. #endif
  236. struct desc_struct *gdt;
  237. int me = raw_smp_processor_id();
  238. if (!has_svm()) {
  239. printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
  240. return;
  241. }
  242. svm_data = per_cpu(svm_data, me);
  243. if (!svm_data) {
  244. printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
  245. me);
  246. return;
  247. }
  248. svm_data->asid_generation = 1;
  249. svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
  250. svm_data->next_asid = svm_data->max_asid + 1;
  251. asm volatile ( "sgdt %0" : "=m"(gdt_descr) );
  252. gdt = (struct desc_struct *)gdt_descr.address;
  253. svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
  254. rdmsrl(MSR_EFER, efer);
  255. wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK);
  256. wrmsrl(MSR_VM_HSAVE_PA,
  257. page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
  258. }
  259. static int svm_cpu_init(int cpu)
  260. {
  261. struct svm_cpu_data *svm_data;
  262. int r;
  263. svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
  264. if (!svm_data)
  265. return -ENOMEM;
  266. svm_data->cpu = cpu;
  267. svm_data->save_area = alloc_page(GFP_KERNEL);
  268. r = -ENOMEM;
  269. if (!svm_data->save_area)
  270. goto err_1;
  271. per_cpu(svm_data, cpu) = svm_data;
  272. return 0;
  273. err_1:
  274. kfree(svm_data);
  275. return r;
  276. }
  277. static int set_msr_interception(u32 *msrpm, unsigned msr,
  278. int read, int write)
  279. {
  280. int i;
  281. for (i = 0; i < NUM_MSR_MAPS; i++) {
  282. if (msr >= msrpm_ranges[i] &&
  283. msr < msrpm_ranges[i] + MSRS_IN_RANGE) {
  284. u32 msr_offset = (i * MSRS_IN_RANGE + msr -
  285. msrpm_ranges[i]) * 2;
  286. u32 *base = msrpm + (msr_offset / 32);
  287. u32 msr_shift = msr_offset % 32;
  288. u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1);
  289. *base = (*base & ~(0x3 << msr_shift)) |
  290. (mask << msr_shift);
  291. return 1;
  292. }
  293. }
  294. printk(KERN_DEBUG "%s: not found 0x%x\n", __FUNCTION__, msr);
  295. return 0;
  296. }
  297. static __init int svm_hardware_setup(void)
  298. {
  299. int cpu;
  300. struct page *iopm_pages;
  301. struct page *msrpm_pages;
  302. void *msrpm_va;
  303. int r;
  304. kvm_emulator_want_group7_invlpg();
  305. iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
  306. if (!iopm_pages)
  307. return -ENOMEM;
  308. memset(page_address(iopm_pages), 0xff,
  309. PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
  310. iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
  311. msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
  312. r = -ENOMEM;
  313. if (!msrpm_pages)
  314. goto err_1;
  315. msrpm_va = page_address(msrpm_pages);
  316. memset(msrpm_va, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
  317. msrpm_base = page_to_pfn(msrpm_pages) << PAGE_SHIFT;
  318. #ifdef CONFIG_X86_64
  319. set_msr_interception(msrpm_va, MSR_GS_BASE, 1, 1);
  320. set_msr_interception(msrpm_va, MSR_FS_BASE, 1, 1);
  321. set_msr_interception(msrpm_va, MSR_KERNEL_GS_BASE, 1, 1);
  322. set_msr_interception(msrpm_va, MSR_LSTAR, 1, 1);
  323. set_msr_interception(msrpm_va, MSR_CSTAR, 1, 1);
  324. set_msr_interception(msrpm_va, MSR_SYSCALL_MASK, 1, 1);
  325. #endif
  326. set_msr_interception(msrpm_va, MSR_K6_STAR, 1, 1);
  327. set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_CS, 1, 1);
  328. set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_ESP, 1, 1);
  329. set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_EIP, 1, 1);
  330. for_each_online_cpu(cpu) {
  331. r = svm_cpu_init(cpu);
  332. if (r)
  333. goto err_2;
  334. }
  335. return 0;
  336. err_2:
  337. __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
  338. msrpm_base = 0;
  339. err_1:
  340. __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
  341. iopm_base = 0;
  342. return r;
  343. }
  344. static __exit void svm_hardware_unsetup(void)
  345. {
  346. __free_pages(pfn_to_page(msrpm_base >> PAGE_SHIFT), MSRPM_ALLOC_ORDER);
  347. __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
  348. iopm_base = msrpm_base = 0;
  349. }
  350. static void init_seg(struct vmcb_seg *seg)
  351. {
  352. seg->selector = 0;
  353. seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
  354. SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
  355. seg->limit = 0xffff;
  356. seg->base = 0;
  357. }
  358. static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
  359. {
  360. seg->selector = 0;
  361. seg->attrib = SVM_SELECTOR_P_MASK | type;
  362. seg->limit = 0xffff;
  363. seg->base = 0;
  364. }
  365. static int svm_vcpu_setup(struct kvm_vcpu *vcpu)
  366. {
  367. return 0;
  368. }
  369. static void init_vmcb(struct vmcb *vmcb)
  370. {
  371. struct vmcb_control_area *control = &vmcb->control;
  372. struct vmcb_save_area *save = &vmcb->save;
  373. u64 tsc;
  374. control->intercept_cr_read = INTERCEPT_CR0_MASK |
  375. INTERCEPT_CR3_MASK |
  376. INTERCEPT_CR4_MASK;
  377. control->intercept_cr_write = INTERCEPT_CR0_MASK |
  378. INTERCEPT_CR3_MASK |
  379. INTERCEPT_CR4_MASK;
  380. control->intercept_dr_read = INTERCEPT_DR0_MASK |
  381. INTERCEPT_DR1_MASK |
  382. INTERCEPT_DR2_MASK |
  383. INTERCEPT_DR3_MASK;
  384. control->intercept_dr_write = INTERCEPT_DR0_MASK |
  385. INTERCEPT_DR1_MASK |
  386. INTERCEPT_DR2_MASK |
  387. INTERCEPT_DR3_MASK |
  388. INTERCEPT_DR5_MASK |
  389. INTERCEPT_DR7_MASK;
  390. control->intercept_exceptions = 1 << PF_VECTOR;
  391. control->intercept = (1ULL << INTERCEPT_INTR) |
  392. (1ULL << INTERCEPT_NMI) |
  393. (1ULL << INTERCEPT_SMI) |
  394. /*
  395. * selective cr0 intercept bug?
  396. * 0: 0f 22 d8 mov %eax,%cr3
  397. * 3: 0f 20 c0 mov %cr0,%eax
  398. * 6: 0d 00 00 00 80 or $0x80000000,%eax
  399. * b: 0f 22 c0 mov %eax,%cr0
  400. * set cr3 ->interception
  401. * get cr0 ->interception
  402. * set cr0 -> no interception
  403. */
  404. /* (1ULL << INTERCEPT_SELECTIVE_CR0) | */
  405. (1ULL << INTERCEPT_CPUID) |
  406. (1ULL << INTERCEPT_HLT) |
  407. (1ULL << INTERCEPT_INVLPGA) |
  408. (1ULL << INTERCEPT_IOIO_PROT) |
  409. (1ULL << INTERCEPT_MSR_PROT) |
  410. (1ULL << INTERCEPT_TASK_SWITCH) |
  411. (1ULL << INTERCEPT_SHUTDOWN) |
  412. (1ULL << INTERCEPT_VMRUN) |
  413. (1ULL << INTERCEPT_VMMCALL) |
  414. (1ULL << INTERCEPT_VMLOAD) |
  415. (1ULL << INTERCEPT_VMSAVE) |
  416. (1ULL << INTERCEPT_STGI) |
  417. (1ULL << INTERCEPT_CLGI) |
  418. (1ULL << INTERCEPT_SKINIT);
  419. control->iopm_base_pa = iopm_base;
  420. control->msrpm_base_pa = msrpm_base;
  421. rdtscll(tsc);
  422. control->tsc_offset = -tsc;
  423. control->int_ctl = V_INTR_MASKING_MASK;
  424. init_seg(&save->es);
  425. init_seg(&save->ss);
  426. init_seg(&save->ds);
  427. init_seg(&save->fs);
  428. init_seg(&save->gs);
  429. save->cs.selector = 0xf000;
  430. /* Executable/Readable Code Segment */
  431. save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
  432. SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
  433. save->cs.limit = 0xffff;
  434. /*
  435. * cs.base should really be 0xffff0000, but vmx can't handle that, so
  436. * be consistent with it.
  437. *
  438. * Replace when we have real mode working for vmx.
  439. */
  440. save->cs.base = 0xf0000;
  441. save->gdtr.limit = 0xffff;
  442. save->idtr.limit = 0xffff;
  443. init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
  444. init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
  445. save->efer = MSR_EFER_SVME_MASK;
  446. save->dr6 = 0xffff0ff0;
  447. save->dr7 = 0x400;
  448. save->rflags = 2;
  449. save->rip = 0x0000fff0;
  450. /*
  451. * cr0 val on cpu init should be 0x60000010, we enable cpu
  452. * cache by default. the orderly way is to enable cache in bios.
  453. */
  454. save->cr0 = 0x00000010 | CR0_PG_MASK | CR0_WP_MASK;
  455. save->cr4 = CR4_PAE_MASK;
  456. /* rdx = ?? */
  457. }
  458. static int svm_create_vcpu(struct kvm_vcpu *vcpu)
  459. {
  460. struct page *page;
  461. int r;
  462. r = -ENOMEM;
  463. vcpu->svm = kzalloc(sizeof *vcpu->svm, GFP_KERNEL);
  464. if (!vcpu->svm)
  465. goto out1;
  466. page = alloc_page(GFP_KERNEL);
  467. if (!page)
  468. goto out2;
  469. vcpu->svm->vmcb = page_address(page);
  470. memset(vcpu->svm->vmcb, 0, PAGE_SIZE);
  471. vcpu->svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
  472. vcpu->svm->cr0 = 0x00000010;
  473. vcpu->svm->asid_generation = 0;
  474. memset(vcpu->svm->db_regs, 0, sizeof(vcpu->svm->db_regs));
  475. init_vmcb(vcpu->svm->vmcb);
  476. fx_init(vcpu);
  477. return 0;
  478. out2:
  479. kfree(vcpu->svm);
  480. out1:
  481. return r;
  482. }
  483. static void svm_free_vcpu(struct kvm_vcpu *vcpu)
  484. {
  485. if (!vcpu->svm)
  486. return;
  487. if (vcpu->svm->vmcb)
  488. __free_page(pfn_to_page(vcpu->svm->vmcb_pa >> PAGE_SHIFT));
  489. kfree(vcpu->svm);
  490. }
  491. static void svm_vcpu_load(struct kvm_vcpu *vcpu)
  492. {
  493. get_cpu();
  494. }
  495. static void svm_vcpu_put(struct kvm_vcpu *vcpu)
  496. {
  497. put_cpu();
  498. }
  499. static void svm_vcpu_decache(struct kvm_vcpu *vcpu)
  500. {
  501. }
  502. static void svm_cache_regs(struct kvm_vcpu *vcpu)
  503. {
  504. vcpu->regs[VCPU_REGS_RAX] = vcpu->svm->vmcb->save.rax;
  505. vcpu->regs[VCPU_REGS_RSP] = vcpu->svm->vmcb->save.rsp;
  506. vcpu->rip = vcpu->svm->vmcb->save.rip;
  507. }
  508. static void svm_decache_regs(struct kvm_vcpu *vcpu)
  509. {
  510. vcpu->svm->vmcb->save.rax = vcpu->regs[VCPU_REGS_RAX];
  511. vcpu->svm->vmcb->save.rsp = vcpu->regs[VCPU_REGS_RSP];
  512. vcpu->svm->vmcb->save.rip = vcpu->rip;
  513. }
  514. static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
  515. {
  516. return vcpu->svm->vmcb->save.rflags;
  517. }
  518. static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
  519. {
  520. vcpu->svm->vmcb->save.rflags = rflags;
  521. }
  522. static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
  523. {
  524. struct vmcb_save_area *save = &vcpu->svm->vmcb->save;
  525. switch (seg) {
  526. case VCPU_SREG_CS: return &save->cs;
  527. case VCPU_SREG_DS: return &save->ds;
  528. case VCPU_SREG_ES: return &save->es;
  529. case VCPU_SREG_FS: return &save->fs;
  530. case VCPU_SREG_GS: return &save->gs;
  531. case VCPU_SREG_SS: return &save->ss;
  532. case VCPU_SREG_TR: return &save->tr;
  533. case VCPU_SREG_LDTR: return &save->ldtr;
  534. }
  535. BUG();
  536. return NULL;
  537. }
  538. static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
  539. {
  540. struct vmcb_seg *s = svm_seg(vcpu, seg);
  541. return s->base;
  542. }
  543. static void svm_get_segment(struct kvm_vcpu *vcpu,
  544. struct kvm_segment *var, int seg)
  545. {
  546. struct vmcb_seg *s = svm_seg(vcpu, seg);
  547. var->base = s->base;
  548. var->limit = s->limit;
  549. var->selector = s->selector;
  550. var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
  551. var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
  552. var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
  553. var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
  554. var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
  555. var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
  556. var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
  557. var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
  558. var->unusable = !var->present;
  559. }
  560. static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
  561. {
  562. struct vmcb_seg *s = svm_seg(vcpu, VCPU_SREG_CS);
  563. *db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
  564. *l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
  565. }
  566. static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  567. {
  568. dt->limit = vcpu->svm->vmcb->save.idtr.limit;
  569. dt->base = vcpu->svm->vmcb->save.idtr.base;
  570. }
  571. static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  572. {
  573. vcpu->svm->vmcb->save.idtr.limit = dt->limit;
  574. vcpu->svm->vmcb->save.idtr.base = dt->base ;
  575. }
  576. static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  577. {
  578. dt->limit = vcpu->svm->vmcb->save.gdtr.limit;
  579. dt->base = vcpu->svm->vmcb->save.gdtr.base;
  580. }
  581. static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
  582. {
  583. vcpu->svm->vmcb->save.gdtr.limit = dt->limit;
  584. vcpu->svm->vmcb->save.gdtr.base = dt->base ;
  585. }
  586. static void svm_decache_cr0_cr4_guest_bits(struct kvm_vcpu *vcpu)
  587. {
  588. }
  589. static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  590. {
  591. #ifdef CONFIG_X86_64
  592. if (vcpu->shadow_efer & KVM_EFER_LME) {
  593. if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
  594. vcpu->shadow_efer |= KVM_EFER_LMA;
  595. vcpu->svm->vmcb->save.efer |= KVM_EFER_LMA | KVM_EFER_LME;
  596. }
  597. if (is_paging(vcpu) && !(cr0 & CR0_PG_MASK) ) {
  598. vcpu->shadow_efer &= ~KVM_EFER_LMA;
  599. vcpu->svm->vmcb->save.efer &= ~(KVM_EFER_LMA | KVM_EFER_LME);
  600. }
  601. }
  602. #endif
  603. vcpu->svm->cr0 = cr0;
  604. vcpu->svm->vmcb->save.cr0 = cr0 | CR0_PG_MASK | CR0_WP_MASK;
  605. vcpu->cr0 = cr0;
  606. }
  607. static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  608. {
  609. vcpu->cr4 = cr4;
  610. vcpu->svm->vmcb->save.cr4 = cr4 | CR4_PAE_MASK;
  611. }
  612. static void svm_set_segment(struct kvm_vcpu *vcpu,
  613. struct kvm_segment *var, int seg)
  614. {
  615. struct vmcb_seg *s = svm_seg(vcpu, seg);
  616. s->base = var->base;
  617. s->limit = var->limit;
  618. s->selector = var->selector;
  619. if (var->unusable)
  620. s->attrib = 0;
  621. else {
  622. s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
  623. s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
  624. s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
  625. s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
  626. s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
  627. s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
  628. s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
  629. s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
  630. }
  631. if (seg == VCPU_SREG_CS)
  632. vcpu->svm->vmcb->save.cpl
  633. = (vcpu->svm->vmcb->save.cs.attrib
  634. >> SVM_SELECTOR_DPL_SHIFT) & 3;
  635. }
  636. /* FIXME:
  637. vcpu->svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
  638. vcpu->svm->vmcb->control.int_ctl |= (sregs->cr8 & V_TPR_MASK);
  639. */
  640. static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
  641. {
  642. return -EOPNOTSUPP;
  643. }
  644. static void load_host_msrs(struct kvm_vcpu *vcpu)
  645. {
  646. int i;
  647. for ( i = 0; i < NR_HOST_SAVE_MSRS; i++)
  648. wrmsrl(host_save_msrs[i], vcpu->svm->host_msrs[i]);
  649. }
  650. static void save_host_msrs(struct kvm_vcpu *vcpu)
  651. {
  652. int i;
  653. for ( i = 0; i < NR_HOST_SAVE_MSRS; i++)
  654. rdmsrl(host_save_msrs[i], vcpu->svm->host_msrs[i]);
  655. }
  656. static void new_asid(struct kvm_vcpu *vcpu, struct svm_cpu_data *svm_data)
  657. {
  658. if (svm_data->next_asid > svm_data->max_asid) {
  659. ++svm_data->asid_generation;
  660. svm_data->next_asid = 1;
  661. vcpu->svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
  662. }
  663. vcpu->cpu = svm_data->cpu;
  664. vcpu->svm->asid_generation = svm_data->asid_generation;
  665. vcpu->svm->vmcb->control.asid = svm_data->next_asid++;
  666. }
  667. static void svm_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  668. {
  669. invlpga(address, vcpu->svm->vmcb->control.asid); // is needed?
  670. }
  671. static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr)
  672. {
  673. return vcpu->svm->db_regs[dr];
  674. }
  675. static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value,
  676. int *exception)
  677. {
  678. *exception = 0;
  679. if (vcpu->svm->vmcb->save.dr7 & DR7_GD_MASK) {
  680. vcpu->svm->vmcb->save.dr7 &= ~DR7_GD_MASK;
  681. vcpu->svm->vmcb->save.dr6 |= DR6_BD_MASK;
  682. *exception = DB_VECTOR;
  683. return;
  684. }
  685. switch (dr) {
  686. case 0 ... 3:
  687. vcpu->svm->db_regs[dr] = value;
  688. return;
  689. case 4 ... 5:
  690. if (vcpu->cr4 & CR4_DE_MASK) {
  691. *exception = UD_VECTOR;
  692. return;
  693. }
  694. case 7: {
  695. if (value & ~((1ULL << 32) - 1)) {
  696. *exception = GP_VECTOR;
  697. return;
  698. }
  699. vcpu->svm->vmcb->save.dr7 = value;
  700. return;
  701. }
  702. default:
  703. printk(KERN_DEBUG "%s: unexpected dr %u\n",
  704. __FUNCTION__, dr);
  705. *exception = UD_VECTOR;
  706. return;
  707. }
  708. }
  709. static int pf_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  710. {
  711. u32 exit_int_info = vcpu->svm->vmcb->control.exit_int_info;
  712. u64 fault_address;
  713. u32 error_code;
  714. enum emulation_result er;
  715. int r;
  716. if (is_external_interrupt(exit_int_info))
  717. push_irq(vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK);
  718. spin_lock(&vcpu->kvm->lock);
  719. fault_address = vcpu->svm->vmcb->control.exit_info_2;
  720. error_code = vcpu->svm->vmcb->control.exit_info_1;
  721. r = kvm_mmu_page_fault(vcpu, fault_address, error_code);
  722. if (r < 0) {
  723. spin_unlock(&vcpu->kvm->lock);
  724. return r;
  725. }
  726. if (!r) {
  727. spin_unlock(&vcpu->kvm->lock);
  728. return 1;
  729. }
  730. er = emulate_instruction(vcpu, kvm_run, fault_address, error_code);
  731. spin_unlock(&vcpu->kvm->lock);
  732. switch (er) {
  733. case EMULATE_DONE:
  734. return 1;
  735. case EMULATE_DO_MMIO:
  736. ++kvm_stat.mmio_exits;
  737. kvm_run->exit_reason = KVM_EXIT_MMIO;
  738. return 0;
  739. case EMULATE_FAIL:
  740. vcpu_printf(vcpu, "%s: emulate fail\n", __FUNCTION__);
  741. break;
  742. default:
  743. BUG();
  744. }
  745. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  746. return 0;
  747. }
  748. static int shutdown_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  749. {
  750. /*
  751. * VMCB is undefined after a SHUTDOWN intercept
  752. * so reinitialize it.
  753. */
  754. memset(vcpu->svm->vmcb, 0, PAGE_SIZE);
  755. init_vmcb(vcpu->svm->vmcb);
  756. kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
  757. return 0;
  758. }
  759. static int io_get_override(struct kvm_vcpu *vcpu,
  760. struct vmcb_seg **seg,
  761. int *addr_override)
  762. {
  763. u8 inst[MAX_INST_SIZE];
  764. unsigned ins_length;
  765. gva_t rip;
  766. int i;
  767. rip = vcpu->svm->vmcb->save.rip;
  768. ins_length = vcpu->svm->next_rip - rip;
  769. rip += vcpu->svm->vmcb->save.cs.base;
  770. if (ins_length > MAX_INST_SIZE)
  771. printk(KERN_DEBUG
  772. "%s: inst length err, cs base 0x%llx rip 0x%llx "
  773. "next rip 0x%llx ins_length %u\n",
  774. __FUNCTION__,
  775. vcpu->svm->vmcb->save.cs.base,
  776. vcpu->svm->vmcb->save.rip,
  777. vcpu->svm->vmcb->control.exit_info_2,
  778. ins_length);
  779. if (kvm_read_guest(vcpu, rip, ins_length, inst) != ins_length)
  780. /* #PF */
  781. return 0;
  782. *addr_override = 0;
  783. *seg = NULL;
  784. for (i = 0; i < ins_length; i++)
  785. switch (inst[i]) {
  786. case 0xf0:
  787. case 0xf2:
  788. case 0xf3:
  789. case 0x66:
  790. continue;
  791. case 0x67:
  792. *addr_override = 1;
  793. continue;
  794. case 0x2e:
  795. *seg = &vcpu->svm->vmcb->save.cs;
  796. continue;
  797. case 0x36:
  798. *seg = &vcpu->svm->vmcb->save.ss;
  799. continue;
  800. case 0x3e:
  801. *seg = &vcpu->svm->vmcb->save.ds;
  802. continue;
  803. case 0x26:
  804. *seg = &vcpu->svm->vmcb->save.es;
  805. continue;
  806. case 0x64:
  807. *seg = &vcpu->svm->vmcb->save.fs;
  808. continue;
  809. case 0x65:
  810. *seg = &vcpu->svm->vmcb->save.gs;
  811. continue;
  812. default:
  813. return 1;
  814. }
  815. printk(KERN_DEBUG "%s: unexpected\n", __FUNCTION__);
  816. return 0;
  817. }
  818. static unsigned long io_adress(struct kvm_vcpu *vcpu, int ins, u64 *address)
  819. {
  820. unsigned long addr_mask;
  821. unsigned long *reg;
  822. struct vmcb_seg *seg;
  823. int addr_override;
  824. struct vmcb_save_area *save_area = &vcpu->svm->vmcb->save;
  825. u16 cs_attrib = save_area->cs.attrib;
  826. unsigned addr_size = get_addr_size(vcpu);
  827. if (!io_get_override(vcpu, &seg, &addr_override))
  828. return 0;
  829. if (addr_override)
  830. addr_size = (addr_size == 2) ? 4: (addr_size >> 1);
  831. if (ins) {
  832. reg = &vcpu->regs[VCPU_REGS_RDI];
  833. seg = &vcpu->svm->vmcb->save.es;
  834. } else {
  835. reg = &vcpu->regs[VCPU_REGS_RSI];
  836. seg = (seg) ? seg : &vcpu->svm->vmcb->save.ds;
  837. }
  838. addr_mask = ~0ULL >> (64 - (addr_size * 8));
  839. if ((cs_attrib & SVM_SELECTOR_L_MASK) &&
  840. !(vcpu->svm->vmcb->save.rflags & X86_EFLAGS_VM)) {
  841. *address = (*reg & addr_mask);
  842. return addr_mask;
  843. }
  844. if (!(seg->attrib & SVM_SELECTOR_P_SHIFT)) {
  845. svm_inject_gp(vcpu, 0);
  846. return 0;
  847. }
  848. *address = (*reg & addr_mask) + seg->base;
  849. return addr_mask;
  850. }
  851. static int io_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  852. {
  853. u32 io_info = vcpu->svm->vmcb->control.exit_info_1; //address size bug?
  854. int _in = io_info & SVM_IOIO_TYPE_MASK;
  855. ++kvm_stat.io_exits;
  856. vcpu->svm->next_rip = vcpu->svm->vmcb->control.exit_info_2;
  857. kvm_run->exit_reason = KVM_EXIT_IO;
  858. kvm_run->io.port = io_info >> 16;
  859. kvm_run->io.direction = (_in) ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
  860. kvm_run->io.size = ((io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT);
  861. kvm_run->io.string = (io_info & SVM_IOIO_STR_MASK) != 0;
  862. kvm_run->io.rep = (io_info & SVM_IOIO_REP_MASK) != 0;
  863. if (kvm_run->io.string) {
  864. unsigned addr_mask;
  865. addr_mask = io_adress(vcpu, _in, &kvm_run->io.address);
  866. if (!addr_mask) {
  867. printk(KERN_DEBUG "%s: get io address failed\n",
  868. __FUNCTION__);
  869. return 1;
  870. }
  871. if (kvm_run->io.rep) {
  872. kvm_run->io.count
  873. = vcpu->regs[VCPU_REGS_RCX] & addr_mask;
  874. kvm_run->io.string_down = (vcpu->svm->vmcb->save.rflags
  875. & X86_EFLAGS_DF) != 0;
  876. }
  877. } else
  878. kvm_run->io.value = vcpu->svm->vmcb->save.rax;
  879. return 0;
  880. }
  881. static int nop_on_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  882. {
  883. return 1;
  884. }
  885. static int halt_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  886. {
  887. vcpu->svm->next_rip = vcpu->svm->vmcb->save.rip + 1;
  888. skip_emulated_instruction(vcpu);
  889. if (vcpu->irq_summary)
  890. return 1;
  891. kvm_run->exit_reason = KVM_EXIT_HLT;
  892. ++kvm_stat.halt_exits;
  893. return 0;
  894. }
  895. static int vmmcall_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  896. {
  897. vcpu->svm->vmcb->save.rip += 3;
  898. return kvm_hypercall(vcpu, kvm_run);
  899. }
  900. static int invalid_op_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  901. {
  902. inject_ud(vcpu);
  903. return 1;
  904. }
  905. static int task_switch_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  906. {
  907. printk(KERN_DEBUG "%s: task swiche is unsupported\n", __FUNCTION__);
  908. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  909. return 0;
  910. }
  911. static int cpuid_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  912. {
  913. vcpu->svm->next_rip = vcpu->svm->vmcb->save.rip + 2;
  914. kvm_run->exit_reason = KVM_EXIT_CPUID;
  915. return 0;
  916. }
  917. static int emulate_on_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  918. {
  919. if (emulate_instruction(vcpu, NULL, 0, 0) != EMULATE_DONE)
  920. printk(KERN_ERR "%s: failed\n", __FUNCTION__);
  921. return 1;
  922. }
  923. static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
  924. {
  925. switch (ecx) {
  926. case MSR_IA32_TIME_STAMP_COUNTER: {
  927. u64 tsc;
  928. rdtscll(tsc);
  929. *data = vcpu->svm->vmcb->control.tsc_offset + tsc;
  930. break;
  931. }
  932. case MSR_K6_STAR:
  933. *data = vcpu->svm->vmcb->save.star;
  934. break;
  935. #ifdef CONFIG_X86_64
  936. case MSR_LSTAR:
  937. *data = vcpu->svm->vmcb->save.lstar;
  938. break;
  939. case MSR_CSTAR:
  940. *data = vcpu->svm->vmcb->save.cstar;
  941. break;
  942. case MSR_KERNEL_GS_BASE:
  943. *data = vcpu->svm->vmcb->save.kernel_gs_base;
  944. break;
  945. case MSR_SYSCALL_MASK:
  946. *data = vcpu->svm->vmcb->save.sfmask;
  947. break;
  948. #endif
  949. case MSR_IA32_SYSENTER_CS:
  950. *data = vcpu->svm->vmcb->save.sysenter_cs;
  951. break;
  952. case MSR_IA32_SYSENTER_EIP:
  953. *data = vcpu->svm->vmcb->save.sysenter_eip;
  954. break;
  955. case MSR_IA32_SYSENTER_ESP:
  956. *data = vcpu->svm->vmcb->save.sysenter_esp;
  957. break;
  958. default:
  959. return kvm_get_msr_common(vcpu, ecx, data);
  960. }
  961. return 0;
  962. }
  963. static int rdmsr_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  964. {
  965. u32 ecx = vcpu->regs[VCPU_REGS_RCX];
  966. u64 data;
  967. if (svm_get_msr(vcpu, ecx, &data))
  968. svm_inject_gp(vcpu, 0);
  969. else {
  970. vcpu->svm->vmcb->save.rax = data & 0xffffffff;
  971. vcpu->regs[VCPU_REGS_RDX] = data >> 32;
  972. vcpu->svm->next_rip = vcpu->svm->vmcb->save.rip + 2;
  973. skip_emulated_instruction(vcpu);
  974. }
  975. return 1;
  976. }
  977. static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
  978. {
  979. switch (ecx) {
  980. case MSR_IA32_TIME_STAMP_COUNTER: {
  981. u64 tsc;
  982. rdtscll(tsc);
  983. vcpu->svm->vmcb->control.tsc_offset = data - tsc;
  984. break;
  985. }
  986. case MSR_K6_STAR:
  987. vcpu->svm->vmcb->save.star = data;
  988. break;
  989. #ifdef CONFIG_X86_64
  990. case MSR_LSTAR:
  991. vcpu->svm->vmcb->save.lstar = data;
  992. break;
  993. case MSR_CSTAR:
  994. vcpu->svm->vmcb->save.cstar = data;
  995. break;
  996. case MSR_KERNEL_GS_BASE:
  997. vcpu->svm->vmcb->save.kernel_gs_base = data;
  998. break;
  999. case MSR_SYSCALL_MASK:
  1000. vcpu->svm->vmcb->save.sfmask = data;
  1001. break;
  1002. #endif
  1003. case MSR_IA32_SYSENTER_CS:
  1004. vcpu->svm->vmcb->save.sysenter_cs = data;
  1005. break;
  1006. case MSR_IA32_SYSENTER_EIP:
  1007. vcpu->svm->vmcb->save.sysenter_eip = data;
  1008. break;
  1009. case MSR_IA32_SYSENTER_ESP:
  1010. vcpu->svm->vmcb->save.sysenter_esp = data;
  1011. break;
  1012. default:
  1013. return kvm_set_msr_common(vcpu, ecx, data);
  1014. }
  1015. return 0;
  1016. }
  1017. static int wrmsr_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1018. {
  1019. u32 ecx = vcpu->regs[VCPU_REGS_RCX];
  1020. u64 data = (vcpu->svm->vmcb->save.rax & -1u)
  1021. | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32);
  1022. vcpu->svm->next_rip = vcpu->svm->vmcb->save.rip + 2;
  1023. if (svm_set_msr(vcpu, ecx, data))
  1024. svm_inject_gp(vcpu, 0);
  1025. else
  1026. skip_emulated_instruction(vcpu);
  1027. return 1;
  1028. }
  1029. static int msr_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1030. {
  1031. if (vcpu->svm->vmcb->control.exit_info_1)
  1032. return wrmsr_interception(vcpu, kvm_run);
  1033. else
  1034. return rdmsr_interception(vcpu, kvm_run);
  1035. }
  1036. static int interrupt_window_interception(struct kvm_vcpu *vcpu,
  1037. struct kvm_run *kvm_run)
  1038. {
  1039. /*
  1040. * If the user space waits to inject interrupts, exit as soon as
  1041. * possible
  1042. */
  1043. if (kvm_run->request_interrupt_window &&
  1044. !vcpu->irq_summary) {
  1045. ++kvm_stat.irq_window_exits;
  1046. kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
  1047. return 0;
  1048. }
  1049. return 1;
  1050. }
  1051. static int (*svm_exit_handlers[])(struct kvm_vcpu *vcpu,
  1052. struct kvm_run *kvm_run) = {
  1053. [SVM_EXIT_READ_CR0] = emulate_on_interception,
  1054. [SVM_EXIT_READ_CR3] = emulate_on_interception,
  1055. [SVM_EXIT_READ_CR4] = emulate_on_interception,
  1056. /* for now: */
  1057. [SVM_EXIT_WRITE_CR0] = emulate_on_interception,
  1058. [SVM_EXIT_WRITE_CR3] = emulate_on_interception,
  1059. [SVM_EXIT_WRITE_CR4] = emulate_on_interception,
  1060. [SVM_EXIT_READ_DR0] = emulate_on_interception,
  1061. [SVM_EXIT_READ_DR1] = emulate_on_interception,
  1062. [SVM_EXIT_READ_DR2] = emulate_on_interception,
  1063. [SVM_EXIT_READ_DR3] = emulate_on_interception,
  1064. [SVM_EXIT_WRITE_DR0] = emulate_on_interception,
  1065. [SVM_EXIT_WRITE_DR1] = emulate_on_interception,
  1066. [SVM_EXIT_WRITE_DR2] = emulate_on_interception,
  1067. [SVM_EXIT_WRITE_DR3] = emulate_on_interception,
  1068. [SVM_EXIT_WRITE_DR5] = emulate_on_interception,
  1069. [SVM_EXIT_WRITE_DR7] = emulate_on_interception,
  1070. [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
  1071. [SVM_EXIT_INTR] = nop_on_interception,
  1072. [SVM_EXIT_NMI] = nop_on_interception,
  1073. [SVM_EXIT_SMI] = nop_on_interception,
  1074. [SVM_EXIT_INIT] = nop_on_interception,
  1075. [SVM_EXIT_VINTR] = interrupt_window_interception,
  1076. /* [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, */
  1077. [SVM_EXIT_CPUID] = cpuid_interception,
  1078. [SVM_EXIT_HLT] = halt_interception,
  1079. [SVM_EXIT_INVLPG] = emulate_on_interception,
  1080. [SVM_EXIT_INVLPGA] = invalid_op_interception,
  1081. [SVM_EXIT_IOIO] = io_interception,
  1082. [SVM_EXIT_MSR] = msr_interception,
  1083. [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
  1084. [SVM_EXIT_SHUTDOWN] = shutdown_interception,
  1085. [SVM_EXIT_VMRUN] = invalid_op_interception,
  1086. [SVM_EXIT_VMMCALL] = vmmcall_interception,
  1087. [SVM_EXIT_VMLOAD] = invalid_op_interception,
  1088. [SVM_EXIT_VMSAVE] = invalid_op_interception,
  1089. [SVM_EXIT_STGI] = invalid_op_interception,
  1090. [SVM_EXIT_CLGI] = invalid_op_interception,
  1091. [SVM_EXIT_SKINIT] = invalid_op_interception,
  1092. };
  1093. static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1094. {
  1095. u32 exit_code = vcpu->svm->vmcb->control.exit_code;
  1096. kvm_run->exit_type = KVM_EXIT_TYPE_VM_EXIT;
  1097. if (is_external_interrupt(vcpu->svm->vmcb->control.exit_int_info) &&
  1098. exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR)
  1099. printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
  1100. "exit_code 0x%x\n",
  1101. __FUNCTION__, vcpu->svm->vmcb->control.exit_int_info,
  1102. exit_code);
  1103. if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
  1104. || svm_exit_handlers[exit_code] == 0) {
  1105. kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
  1106. printk(KERN_ERR "%s: 0x%x @ 0x%llx cr0 0x%lx rflags 0x%llx\n",
  1107. __FUNCTION__,
  1108. exit_code,
  1109. vcpu->svm->vmcb->save.rip,
  1110. vcpu->cr0,
  1111. vcpu->svm->vmcb->save.rflags);
  1112. return 0;
  1113. }
  1114. return svm_exit_handlers[exit_code](vcpu, kvm_run);
  1115. }
  1116. static void reload_tss(struct kvm_vcpu *vcpu)
  1117. {
  1118. int cpu = raw_smp_processor_id();
  1119. struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
  1120. svm_data->tss_desc->type = 9; //available 32/64-bit TSS
  1121. load_TR_desc();
  1122. }
  1123. static void pre_svm_run(struct kvm_vcpu *vcpu)
  1124. {
  1125. int cpu = raw_smp_processor_id();
  1126. struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu);
  1127. vcpu->svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
  1128. if (vcpu->cpu != cpu ||
  1129. vcpu->svm->asid_generation != svm_data->asid_generation)
  1130. new_asid(vcpu, svm_data);
  1131. }
  1132. static inline void kvm_do_inject_irq(struct kvm_vcpu *vcpu)
  1133. {
  1134. struct vmcb_control_area *control;
  1135. control = &vcpu->svm->vmcb->control;
  1136. control->int_vector = pop_irq(vcpu);
  1137. control->int_ctl &= ~V_INTR_PRIO_MASK;
  1138. control->int_ctl |= V_IRQ_MASK |
  1139. ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
  1140. }
  1141. static void kvm_reput_irq(struct kvm_vcpu *vcpu)
  1142. {
  1143. struct vmcb_control_area *control = &vcpu->svm->vmcb->control;
  1144. if (control->int_ctl & V_IRQ_MASK) {
  1145. control->int_ctl &= ~V_IRQ_MASK;
  1146. push_irq(vcpu, control->int_vector);
  1147. }
  1148. vcpu->interrupt_window_open =
  1149. !(control->int_state & SVM_INTERRUPT_SHADOW_MASK);
  1150. }
  1151. static void do_interrupt_requests(struct kvm_vcpu *vcpu,
  1152. struct kvm_run *kvm_run)
  1153. {
  1154. struct vmcb_control_area *control = &vcpu->svm->vmcb->control;
  1155. vcpu->interrupt_window_open =
  1156. (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) &&
  1157. (vcpu->svm->vmcb->save.rflags & X86_EFLAGS_IF));
  1158. if (vcpu->interrupt_window_open && vcpu->irq_summary)
  1159. /*
  1160. * If interrupts enabled, and not blocked by sti or mov ss. Good.
  1161. */
  1162. kvm_do_inject_irq(vcpu);
  1163. /*
  1164. * Interrupts blocked. Wait for unblock.
  1165. */
  1166. if (!vcpu->interrupt_window_open &&
  1167. (vcpu->irq_summary || kvm_run->request_interrupt_window)) {
  1168. control->intercept |= 1ULL << INTERCEPT_VINTR;
  1169. } else
  1170. control->intercept &= ~(1ULL << INTERCEPT_VINTR);
  1171. }
  1172. static void post_kvm_run_save(struct kvm_vcpu *vcpu,
  1173. struct kvm_run *kvm_run)
  1174. {
  1175. kvm_run->ready_for_interrupt_injection = (vcpu->interrupt_window_open &&
  1176. vcpu->irq_summary == 0);
  1177. kvm_run->if_flag = (vcpu->svm->vmcb->save.rflags & X86_EFLAGS_IF) != 0;
  1178. kvm_run->cr8 = vcpu->cr8;
  1179. kvm_run->apic_base = vcpu->apic_base;
  1180. }
  1181. /*
  1182. * Check if userspace requested an interrupt window, and that the
  1183. * interrupt window is open.
  1184. *
  1185. * No need to exit to userspace if we already have an interrupt queued.
  1186. */
  1187. static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
  1188. struct kvm_run *kvm_run)
  1189. {
  1190. return (!vcpu->irq_summary &&
  1191. kvm_run->request_interrupt_window &&
  1192. vcpu->interrupt_window_open &&
  1193. (vcpu->svm->vmcb->save.rflags & X86_EFLAGS_IF));
  1194. }
  1195. static void save_db_regs(unsigned long *db_regs)
  1196. {
  1197. asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0]));
  1198. asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1]));
  1199. asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2]));
  1200. asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3]));
  1201. }
  1202. static void load_db_regs(unsigned long *db_regs)
  1203. {
  1204. asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0]));
  1205. asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1]));
  1206. asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2]));
  1207. asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3]));
  1208. }
  1209. static int svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1210. {
  1211. u16 fs_selector;
  1212. u16 gs_selector;
  1213. u16 ldt_selector;
  1214. int r;
  1215. again:
  1216. if (!vcpu->mmio_read_completed)
  1217. do_interrupt_requests(vcpu, kvm_run);
  1218. clgi();
  1219. pre_svm_run(vcpu);
  1220. save_host_msrs(vcpu);
  1221. fs_selector = read_fs();
  1222. gs_selector = read_gs();
  1223. ldt_selector = read_ldt();
  1224. vcpu->svm->host_cr2 = kvm_read_cr2();
  1225. vcpu->svm->host_dr6 = read_dr6();
  1226. vcpu->svm->host_dr7 = read_dr7();
  1227. vcpu->svm->vmcb->save.cr2 = vcpu->cr2;
  1228. if (vcpu->svm->vmcb->save.dr7 & 0xff) {
  1229. write_dr7(0);
  1230. save_db_regs(vcpu->svm->host_db_regs);
  1231. load_db_regs(vcpu->svm->db_regs);
  1232. }
  1233. fx_save(vcpu->host_fx_image);
  1234. fx_restore(vcpu->guest_fx_image);
  1235. asm volatile (
  1236. #ifdef CONFIG_X86_64
  1237. "push %%rbx; push %%rcx; push %%rdx;"
  1238. "push %%rsi; push %%rdi; push %%rbp;"
  1239. "push %%r8; push %%r9; push %%r10; push %%r11;"
  1240. "push %%r12; push %%r13; push %%r14; push %%r15;"
  1241. #else
  1242. "push %%ebx; push %%ecx; push %%edx;"
  1243. "push %%esi; push %%edi; push %%ebp;"
  1244. #endif
  1245. #ifdef CONFIG_X86_64
  1246. "mov %c[rbx](%[vcpu]), %%rbx \n\t"
  1247. "mov %c[rcx](%[vcpu]), %%rcx \n\t"
  1248. "mov %c[rdx](%[vcpu]), %%rdx \n\t"
  1249. "mov %c[rsi](%[vcpu]), %%rsi \n\t"
  1250. "mov %c[rdi](%[vcpu]), %%rdi \n\t"
  1251. "mov %c[rbp](%[vcpu]), %%rbp \n\t"
  1252. "mov %c[r8](%[vcpu]), %%r8 \n\t"
  1253. "mov %c[r9](%[vcpu]), %%r9 \n\t"
  1254. "mov %c[r10](%[vcpu]), %%r10 \n\t"
  1255. "mov %c[r11](%[vcpu]), %%r11 \n\t"
  1256. "mov %c[r12](%[vcpu]), %%r12 \n\t"
  1257. "mov %c[r13](%[vcpu]), %%r13 \n\t"
  1258. "mov %c[r14](%[vcpu]), %%r14 \n\t"
  1259. "mov %c[r15](%[vcpu]), %%r15 \n\t"
  1260. #else
  1261. "mov %c[rbx](%[vcpu]), %%ebx \n\t"
  1262. "mov %c[rcx](%[vcpu]), %%ecx \n\t"
  1263. "mov %c[rdx](%[vcpu]), %%edx \n\t"
  1264. "mov %c[rsi](%[vcpu]), %%esi \n\t"
  1265. "mov %c[rdi](%[vcpu]), %%edi \n\t"
  1266. "mov %c[rbp](%[vcpu]), %%ebp \n\t"
  1267. #endif
  1268. #ifdef CONFIG_X86_64
  1269. /* Enter guest mode */
  1270. "push %%rax \n\t"
  1271. "mov %c[svm](%[vcpu]), %%rax \n\t"
  1272. "mov %c[vmcb](%%rax), %%rax \n\t"
  1273. SVM_VMLOAD "\n\t"
  1274. SVM_VMRUN "\n\t"
  1275. SVM_VMSAVE "\n\t"
  1276. "pop %%rax \n\t"
  1277. #else
  1278. /* Enter guest mode */
  1279. "push %%eax \n\t"
  1280. "mov %c[svm](%[vcpu]), %%eax \n\t"
  1281. "mov %c[vmcb](%%eax), %%eax \n\t"
  1282. SVM_VMLOAD "\n\t"
  1283. SVM_VMRUN "\n\t"
  1284. SVM_VMSAVE "\n\t"
  1285. "pop %%eax \n\t"
  1286. #endif
  1287. /* Save guest registers, load host registers */
  1288. #ifdef CONFIG_X86_64
  1289. "mov %%rbx, %c[rbx](%[vcpu]) \n\t"
  1290. "mov %%rcx, %c[rcx](%[vcpu]) \n\t"
  1291. "mov %%rdx, %c[rdx](%[vcpu]) \n\t"
  1292. "mov %%rsi, %c[rsi](%[vcpu]) \n\t"
  1293. "mov %%rdi, %c[rdi](%[vcpu]) \n\t"
  1294. "mov %%rbp, %c[rbp](%[vcpu]) \n\t"
  1295. "mov %%r8, %c[r8](%[vcpu]) \n\t"
  1296. "mov %%r9, %c[r9](%[vcpu]) \n\t"
  1297. "mov %%r10, %c[r10](%[vcpu]) \n\t"
  1298. "mov %%r11, %c[r11](%[vcpu]) \n\t"
  1299. "mov %%r12, %c[r12](%[vcpu]) \n\t"
  1300. "mov %%r13, %c[r13](%[vcpu]) \n\t"
  1301. "mov %%r14, %c[r14](%[vcpu]) \n\t"
  1302. "mov %%r15, %c[r15](%[vcpu]) \n\t"
  1303. "pop %%r15; pop %%r14; pop %%r13; pop %%r12;"
  1304. "pop %%r11; pop %%r10; pop %%r9; pop %%r8;"
  1305. "pop %%rbp; pop %%rdi; pop %%rsi;"
  1306. "pop %%rdx; pop %%rcx; pop %%rbx; \n\t"
  1307. #else
  1308. "mov %%ebx, %c[rbx](%[vcpu]) \n\t"
  1309. "mov %%ecx, %c[rcx](%[vcpu]) \n\t"
  1310. "mov %%edx, %c[rdx](%[vcpu]) \n\t"
  1311. "mov %%esi, %c[rsi](%[vcpu]) \n\t"
  1312. "mov %%edi, %c[rdi](%[vcpu]) \n\t"
  1313. "mov %%ebp, %c[rbp](%[vcpu]) \n\t"
  1314. "pop %%ebp; pop %%edi; pop %%esi;"
  1315. "pop %%edx; pop %%ecx; pop %%ebx; \n\t"
  1316. #endif
  1317. :
  1318. : [vcpu]"a"(vcpu),
  1319. [svm]"i"(offsetof(struct kvm_vcpu, svm)),
  1320. [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
  1321. [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])),
  1322. [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])),
  1323. [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])),
  1324. [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])),
  1325. [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])),
  1326. [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP]))
  1327. #ifdef CONFIG_X86_64
  1328. ,[r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])),
  1329. [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])),
  1330. [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])),
  1331. [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])),
  1332. [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])),
  1333. [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])),
  1334. [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])),
  1335. [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15]))
  1336. #endif
  1337. : "cc", "memory" );
  1338. fx_save(vcpu->guest_fx_image);
  1339. fx_restore(vcpu->host_fx_image);
  1340. if ((vcpu->svm->vmcb->save.dr7 & 0xff))
  1341. load_db_regs(vcpu->svm->host_db_regs);
  1342. vcpu->cr2 = vcpu->svm->vmcb->save.cr2;
  1343. write_dr6(vcpu->svm->host_dr6);
  1344. write_dr7(vcpu->svm->host_dr7);
  1345. kvm_write_cr2(vcpu->svm->host_cr2);
  1346. load_fs(fs_selector);
  1347. load_gs(gs_selector);
  1348. load_ldt(ldt_selector);
  1349. load_host_msrs(vcpu);
  1350. reload_tss(vcpu);
  1351. /*
  1352. * Profile KVM exit RIPs:
  1353. */
  1354. if (unlikely(prof_on == KVM_PROFILING))
  1355. profile_hit(KVM_PROFILING,
  1356. (void *)(unsigned long)vcpu->svm->vmcb->save.rip);
  1357. stgi();
  1358. kvm_reput_irq(vcpu);
  1359. vcpu->svm->next_rip = 0;
  1360. if (vcpu->svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
  1361. kvm_run->exit_type = KVM_EXIT_TYPE_FAIL_ENTRY;
  1362. kvm_run->exit_reason = vcpu->svm->vmcb->control.exit_code;
  1363. post_kvm_run_save(vcpu, kvm_run);
  1364. return 0;
  1365. }
  1366. r = handle_exit(vcpu, kvm_run);
  1367. if (r > 0) {
  1368. if (signal_pending(current)) {
  1369. ++kvm_stat.signal_exits;
  1370. post_kvm_run_save(vcpu, kvm_run);
  1371. return -EINTR;
  1372. }
  1373. if (dm_request_for_irq_injection(vcpu, kvm_run)) {
  1374. ++kvm_stat.request_irq_exits;
  1375. post_kvm_run_save(vcpu, kvm_run);
  1376. return -EINTR;
  1377. }
  1378. kvm_resched(vcpu);
  1379. goto again;
  1380. }
  1381. post_kvm_run_save(vcpu, kvm_run);
  1382. return r;
  1383. }
  1384. static void svm_flush_tlb(struct kvm_vcpu *vcpu)
  1385. {
  1386. force_new_asid(vcpu);
  1387. }
  1388. static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
  1389. {
  1390. vcpu->svm->vmcb->save.cr3 = root;
  1391. force_new_asid(vcpu);
  1392. }
  1393. static void svm_inject_page_fault(struct kvm_vcpu *vcpu,
  1394. unsigned long addr,
  1395. uint32_t err_code)
  1396. {
  1397. uint32_t exit_int_info = vcpu->svm->vmcb->control.exit_int_info;
  1398. ++kvm_stat.pf_guest;
  1399. if (is_page_fault(exit_int_info)) {
  1400. vcpu->svm->vmcb->control.event_inj_err = 0;
  1401. vcpu->svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  1402. SVM_EVTINJ_VALID_ERR |
  1403. SVM_EVTINJ_TYPE_EXEPT |
  1404. DF_VECTOR;
  1405. return;
  1406. }
  1407. vcpu->cr2 = addr;
  1408. vcpu->svm->vmcb->save.cr2 = addr;
  1409. vcpu->svm->vmcb->control.event_inj = SVM_EVTINJ_VALID |
  1410. SVM_EVTINJ_VALID_ERR |
  1411. SVM_EVTINJ_TYPE_EXEPT |
  1412. PF_VECTOR;
  1413. vcpu->svm->vmcb->control.event_inj_err = err_code;
  1414. }
  1415. static int is_disabled(void)
  1416. {
  1417. return 0;
  1418. }
  1419. static void
  1420. svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
  1421. {
  1422. /*
  1423. * Patch in the VMMCALL instruction:
  1424. */
  1425. hypercall[0] = 0x0f;
  1426. hypercall[1] = 0x01;
  1427. hypercall[2] = 0xd9;
  1428. hypercall[3] = 0xc3;
  1429. }
  1430. static struct kvm_arch_ops svm_arch_ops = {
  1431. .cpu_has_kvm_support = has_svm,
  1432. .disabled_by_bios = is_disabled,
  1433. .hardware_setup = svm_hardware_setup,
  1434. .hardware_unsetup = svm_hardware_unsetup,
  1435. .hardware_enable = svm_hardware_enable,
  1436. .hardware_disable = svm_hardware_disable,
  1437. .vcpu_create = svm_create_vcpu,
  1438. .vcpu_free = svm_free_vcpu,
  1439. .vcpu_load = svm_vcpu_load,
  1440. .vcpu_put = svm_vcpu_put,
  1441. .vcpu_decache = svm_vcpu_decache,
  1442. .set_guest_debug = svm_guest_debug,
  1443. .get_msr = svm_get_msr,
  1444. .set_msr = svm_set_msr,
  1445. .get_segment_base = svm_get_segment_base,
  1446. .get_segment = svm_get_segment,
  1447. .set_segment = svm_set_segment,
  1448. .get_cs_db_l_bits = svm_get_cs_db_l_bits,
  1449. .decache_cr0_cr4_guest_bits = svm_decache_cr0_cr4_guest_bits,
  1450. .set_cr0 = svm_set_cr0,
  1451. .set_cr0_no_modeswitch = svm_set_cr0,
  1452. .set_cr3 = svm_set_cr3,
  1453. .set_cr4 = svm_set_cr4,
  1454. .set_efer = svm_set_efer,
  1455. .get_idt = svm_get_idt,
  1456. .set_idt = svm_set_idt,
  1457. .get_gdt = svm_get_gdt,
  1458. .set_gdt = svm_set_gdt,
  1459. .get_dr = svm_get_dr,
  1460. .set_dr = svm_set_dr,
  1461. .cache_regs = svm_cache_regs,
  1462. .decache_regs = svm_decache_regs,
  1463. .get_rflags = svm_get_rflags,
  1464. .set_rflags = svm_set_rflags,
  1465. .invlpg = svm_invlpg,
  1466. .tlb_flush = svm_flush_tlb,
  1467. .inject_page_fault = svm_inject_page_fault,
  1468. .inject_gp = svm_inject_gp,
  1469. .run = svm_vcpu_run,
  1470. .skip_emulated_instruction = skip_emulated_instruction,
  1471. .vcpu_setup = svm_vcpu_setup,
  1472. .patch_hypercall = svm_patch_hypercall,
  1473. };
  1474. static int __init svm_init(void)
  1475. {
  1476. return kvm_init_arch(&svm_arch_ops, THIS_MODULE);
  1477. }
  1478. static void __exit svm_exit(void)
  1479. {
  1480. kvm_exit_arch();
  1481. }
  1482. module_init(svm_init)
  1483. module_exit(svm_exit)