kvm_main.c 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * Copyright (C) 2006 Qumranet, Inc.
  8. *
  9. * Authors:
  10. * Avi Kivity <avi@qumranet.com>
  11. * Yaniv Kamay <yaniv@qumranet.com>
  12. *
  13. * This work is licensed under the terms of the GNU GPL, version 2. See
  14. * the COPYING file in the top-level directory.
  15. *
  16. */
  17. #include "kvm.h"
  18. #include <linux/kvm.h>
  19. #include <linux/module.h>
  20. #include <linux/errno.h>
  21. #include <linux/magic.h>
  22. #include <asm/processor.h>
  23. #include <linux/percpu.h>
  24. #include <linux/gfp.h>
  25. #include <asm/msr.h>
  26. #include <linux/mm.h>
  27. #include <linux/miscdevice.h>
  28. #include <linux/vmalloc.h>
  29. #include <asm/uaccess.h>
  30. #include <linux/reboot.h>
  31. #include <asm/io.h>
  32. #include <linux/debugfs.h>
  33. #include <linux/highmem.h>
  34. #include <linux/file.h>
  35. #include <asm/desc.h>
  36. #include <linux/sysdev.h>
  37. #include <linux/cpu.h>
  38. #include <linux/file.h>
  39. #include <linux/fs.h>
  40. #include <linux/mount.h>
  41. #include "x86_emulate.h"
  42. #include "segment_descriptor.h"
  43. MODULE_AUTHOR("Qumranet");
  44. MODULE_LICENSE("GPL");
  45. static DEFINE_SPINLOCK(kvm_lock);
  46. static LIST_HEAD(vm_list);
  47. struct kvm_arch_ops *kvm_arch_ops;
  48. struct kvm_stat kvm_stat;
  49. EXPORT_SYMBOL_GPL(kvm_stat);
  50. static struct kvm_stats_debugfs_item {
  51. const char *name;
  52. u32 *data;
  53. struct dentry *dentry;
  54. } debugfs_entries[] = {
  55. { "pf_fixed", &kvm_stat.pf_fixed },
  56. { "pf_guest", &kvm_stat.pf_guest },
  57. { "tlb_flush", &kvm_stat.tlb_flush },
  58. { "invlpg", &kvm_stat.invlpg },
  59. { "exits", &kvm_stat.exits },
  60. { "io_exits", &kvm_stat.io_exits },
  61. { "mmio_exits", &kvm_stat.mmio_exits },
  62. { "signal_exits", &kvm_stat.signal_exits },
  63. { "irq_window", &kvm_stat.irq_window_exits },
  64. { "halt_exits", &kvm_stat.halt_exits },
  65. { "request_irq", &kvm_stat.request_irq_exits },
  66. { "irq_exits", &kvm_stat.irq_exits },
  67. { NULL, NULL }
  68. };
  69. static struct dentry *debugfs_dir;
  70. struct vfsmount *kvmfs_mnt;
  71. #define MAX_IO_MSRS 256
  72. #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  73. #define LMSW_GUEST_MASK 0x0eULL
  74. #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  75. #define CR8_RESEVED_BITS (~0x0fULL)
  76. #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  77. #ifdef CONFIG_X86_64
  78. // LDT or TSS descriptor in the GDT. 16 bytes.
  79. struct segment_descriptor_64 {
  80. struct segment_descriptor s;
  81. u32 base_higher;
  82. u32 pad_zero;
  83. };
  84. #endif
  85. static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  86. unsigned long arg);
  87. static struct inode *kvmfs_inode(struct file_operations *fops)
  88. {
  89. int error = -ENOMEM;
  90. struct inode *inode = new_inode(kvmfs_mnt->mnt_sb);
  91. if (!inode)
  92. goto eexit_1;
  93. inode->i_fop = fops;
  94. /*
  95. * Mark the inode dirty from the very beginning,
  96. * that way it will never be moved to the dirty
  97. * list because mark_inode_dirty() will think
  98. * that it already _is_ on the dirty list.
  99. */
  100. inode->i_state = I_DIRTY;
  101. inode->i_mode = S_IRUSR | S_IWUSR;
  102. inode->i_uid = current->fsuid;
  103. inode->i_gid = current->fsgid;
  104. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  105. return inode;
  106. eexit_1:
  107. return ERR_PTR(error);
  108. }
  109. static struct file *kvmfs_file(struct inode *inode, void *private_data)
  110. {
  111. struct file *file = get_empty_filp();
  112. if (!file)
  113. return ERR_PTR(-ENFILE);
  114. file->f_path.mnt = mntget(kvmfs_mnt);
  115. file->f_path.dentry = d_alloc_anon(inode);
  116. if (!file->f_path.dentry)
  117. return ERR_PTR(-ENOMEM);
  118. file->f_mapping = inode->i_mapping;
  119. file->f_pos = 0;
  120. file->f_flags = O_RDWR;
  121. file->f_op = inode->i_fop;
  122. file->f_mode = FMODE_READ | FMODE_WRITE;
  123. file->f_version = 0;
  124. file->private_data = private_data;
  125. return file;
  126. }
  127. unsigned long segment_base(u16 selector)
  128. {
  129. struct descriptor_table gdt;
  130. struct segment_descriptor *d;
  131. unsigned long table_base;
  132. typedef unsigned long ul;
  133. unsigned long v;
  134. if (selector == 0)
  135. return 0;
  136. asm ("sgdt %0" : "=m"(gdt));
  137. table_base = gdt.base;
  138. if (selector & 4) { /* from ldt */
  139. u16 ldt_selector;
  140. asm ("sldt %0" : "=g"(ldt_selector));
  141. table_base = segment_base(ldt_selector);
  142. }
  143. d = (struct segment_descriptor *)(table_base + (selector & ~7));
  144. v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
  145. #ifdef CONFIG_X86_64
  146. if (d->system == 0
  147. && (d->type == 2 || d->type == 9 || d->type == 11))
  148. v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
  149. #endif
  150. return v;
  151. }
  152. EXPORT_SYMBOL_GPL(segment_base);
  153. static inline int valid_vcpu(int n)
  154. {
  155. return likely(n >= 0 && n < KVM_MAX_VCPUS);
  156. }
  157. int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
  158. void *dest)
  159. {
  160. unsigned char *host_buf = dest;
  161. unsigned long req_size = size;
  162. while (size) {
  163. hpa_t paddr;
  164. unsigned now;
  165. unsigned offset;
  166. hva_t guest_buf;
  167. paddr = gva_to_hpa(vcpu, addr);
  168. if (is_error_hpa(paddr))
  169. break;
  170. guest_buf = (hva_t)kmap_atomic(
  171. pfn_to_page(paddr >> PAGE_SHIFT),
  172. KM_USER0);
  173. offset = addr & ~PAGE_MASK;
  174. guest_buf |= offset;
  175. now = min(size, PAGE_SIZE - offset);
  176. memcpy(host_buf, (void*)guest_buf, now);
  177. host_buf += now;
  178. addr += now;
  179. size -= now;
  180. kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
  181. }
  182. return req_size - size;
  183. }
  184. EXPORT_SYMBOL_GPL(kvm_read_guest);
  185. int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
  186. void *data)
  187. {
  188. unsigned char *host_buf = data;
  189. unsigned long req_size = size;
  190. while (size) {
  191. hpa_t paddr;
  192. unsigned now;
  193. unsigned offset;
  194. hva_t guest_buf;
  195. gfn_t gfn;
  196. paddr = gva_to_hpa(vcpu, addr);
  197. if (is_error_hpa(paddr))
  198. break;
  199. gfn = vcpu->mmu.gva_to_gpa(vcpu, addr) >> PAGE_SHIFT;
  200. mark_page_dirty(vcpu->kvm, gfn);
  201. guest_buf = (hva_t)kmap_atomic(
  202. pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
  203. offset = addr & ~PAGE_MASK;
  204. guest_buf |= offset;
  205. now = min(size, PAGE_SIZE - offset);
  206. memcpy((void*)guest_buf, host_buf, now);
  207. host_buf += now;
  208. addr += now;
  209. size -= now;
  210. kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
  211. }
  212. return req_size - size;
  213. }
  214. EXPORT_SYMBOL_GPL(kvm_write_guest);
  215. /*
  216. * Switches to specified vcpu, until a matching vcpu_put()
  217. */
  218. static void vcpu_load(struct kvm_vcpu *vcpu)
  219. {
  220. mutex_lock(&vcpu->mutex);
  221. kvm_arch_ops->vcpu_load(vcpu);
  222. }
  223. /*
  224. * Switches to specified vcpu, until a matching vcpu_put(). Will return NULL
  225. * if the slot is not populated.
  226. */
  227. static struct kvm_vcpu *vcpu_load_slot(struct kvm *kvm, int slot)
  228. {
  229. struct kvm_vcpu *vcpu = &kvm->vcpus[slot];
  230. mutex_lock(&vcpu->mutex);
  231. if (!vcpu->vmcs) {
  232. mutex_unlock(&vcpu->mutex);
  233. return NULL;
  234. }
  235. kvm_arch_ops->vcpu_load(vcpu);
  236. return vcpu;
  237. }
  238. static void vcpu_put(struct kvm_vcpu *vcpu)
  239. {
  240. kvm_arch_ops->vcpu_put(vcpu);
  241. mutex_unlock(&vcpu->mutex);
  242. }
  243. static struct kvm *kvm_create_vm(void)
  244. {
  245. struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
  246. int i;
  247. if (!kvm)
  248. return ERR_PTR(-ENOMEM);
  249. spin_lock_init(&kvm->lock);
  250. INIT_LIST_HEAD(&kvm->active_mmu_pages);
  251. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  252. struct kvm_vcpu *vcpu = &kvm->vcpus[i];
  253. mutex_init(&vcpu->mutex);
  254. vcpu->cpu = -1;
  255. vcpu->kvm = kvm;
  256. vcpu->mmu.root_hpa = INVALID_PAGE;
  257. INIT_LIST_HEAD(&vcpu->free_pages);
  258. spin_lock(&kvm_lock);
  259. list_add(&kvm->vm_list, &vm_list);
  260. spin_unlock(&kvm_lock);
  261. }
  262. return kvm;
  263. }
  264. static int kvm_dev_open(struct inode *inode, struct file *filp)
  265. {
  266. return 0;
  267. }
  268. /*
  269. * Free any memory in @free but not in @dont.
  270. */
  271. static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
  272. struct kvm_memory_slot *dont)
  273. {
  274. int i;
  275. if (!dont || free->phys_mem != dont->phys_mem)
  276. if (free->phys_mem) {
  277. for (i = 0; i < free->npages; ++i)
  278. if (free->phys_mem[i])
  279. __free_page(free->phys_mem[i]);
  280. vfree(free->phys_mem);
  281. }
  282. if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
  283. vfree(free->dirty_bitmap);
  284. free->phys_mem = NULL;
  285. free->npages = 0;
  286. free->dirty_bitmap = NULL;
  287. }
  288. static void kvm_free_physmem(struct kvm *kvm)
  289. {
  290. int i;
  291. for (i = 0; i < kvm->nmemslots; ++i)
  292. kvm_free_physmem_slot(&kvm->memslots[i], NULL);
  293. }
  294. static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
  295. {
  296. if (!vcpu->vmcs)
  297. return;
  298. vcpu_load(vcpu);
  299. kvm_mmu_destroy(vcpu);
  300. vcpu_put(vcpu);
  301. kvm_arch_ops->vcpu_free(vcpu);
  302. }
  303. static void kvm_free_vcpus(struct kvm *kvm)
  304. {
  305. unsigned int i;
  306. for (i = 0; i < KVM_MAX_VCPUS; ++i)
  307. kvm_free_vcpu(&kvm->vcpus[i]);
  308. }
  309. static int kvm_dev_release(struct inode *inode, struct file *filp)
  310. {
  311. return 0;
  312. }
  313. static void kvm_destroy_vm(struct kvm *kvm)
  314. {
  315. spin_lock(&kvm_lock);
  316. list_del(&kvm->vm_list);
  317. spin_unlock(&kvm_lock);
  318. kvm_free_vcpus(kvm);
  319. kvm_free_physmem(kvm);
  320. kfree(kvm);
  321. }
  322. static int kvm_vm_release(struct inode *inode, struct file *filp)
  323. {
  324. struct kvm *kvm = filp->private_data;
  325. kvm_destroy_vm(kvm);
  326. return 0;
  327. }
  328. static void inject_gp(struct kvm_vcpu *vcpu)
  329. {
  330. kvm_arch_ops->inject_gp(vcpu, 0);
  331. }
  332. /*
  333. * Load the pae pdptrs. Return true is they are all valid.
  334. */
  335. static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
  336. {
  337. gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
  338. unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
  339. int i;
  340. u64 pdpte;
  341. u64 *pdpt;
  342. int ret;
  343. struct kvm_memory_slot *memslot;
  344. spin_lock(&vcpu->kvm->lock);
  345. memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
  346. /* FIXME: !memslot - emulate? 0xff? */
  347. pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
  348. ret = 1;
  349. for (i = 0; i < 4; ++i) {
  350. pdpte = pdpt[offset + i];
  351. if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
  352. ret = 0;
  353. goto out;
  354. }
  355. }
  356. for (i = 0; i < 4; ++i)
  357. vcpu->pdptrs[i] = pdpt[offset + i];
  358. out:
  359. kunmap_atomic(pdpt, KM_USER0);
  360. spin_unlock(&vcpu->kvm->lock);
  361. return ret;
  362. }
  363. void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
  364. {
  365. if (cr0 & CR0_RESEVED_BITS) {
  366. printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
  367. cr0, vcpu->cr0);
  368. inject_gp(vcpu);
  369. return;
  370. }
  371. if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
  372. printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
  373. inject_gp(vcpu);
  374. return;
  375. }
  376. if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
  377. printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
  378. "and a clear PE flag\n");
  379. inject_gp(vcpu);
  380. return;
  381. }
  382. if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
  383. #ifdef CONFIG_X86_64
  384. if ((vcpu->shadow_efer & EFER_LME)) {
  385. int cs_db, cs_l;
  386. if (!is_pae(vcpu)) {
  387. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  388. "in long mode while PAE is disabled\n");
  389. inject_gp(vcpu);
  390. return;
  391. }
  392. kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  393. if (cs_l) {
  394. printk(KERN_DEBUG "set_cr0: #GP, start paging "
  395. "in long mode while CS.L == 1\n");
  396. inject_gp(vcpu);
  397. return;
  398. }
  399. } else
  400. #endif
  401. if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
  402. printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
  403. "reserved bits\n");
  404. inject_gp(vcpu);
  405. return;
  406. }
  407. }
  408. kvm_arch_ops->set_cr0(vcpu, cr0);
  409. vcpu->cr0 = cr0;
  410. spin_lock(&vcpu->kvm->lock);
  411. kvm_mmu_reset_context(vcpu);
  412. spin_unlock(&vcpu->kvm->lock);
  413. return;
  414. }
  415. EXPORT_SYMBOL_GPL(set_cr0);
  416. void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
  417. {
  418. kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
  419. set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
  420. }
  421. EXPORT_SYMBOL_GPL(lmsw);
  422. void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
  423. {
  424. if (cr4 & CR4_RESEVED_BITS) {
  425. printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
  426. inject_gp(vcpu);
  427. return;
  428. }
  429. if (is_long_mode(vcpu)) {
  430. if (!(cr4 & CR4_PAE_MASK)) {
  431. printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
  432. "in long mode\n");
  433. inject_gp(vcpu);
  434. return;
  435. }
  436. } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
  437. && !load_pdptrs(vcpu, vcpu->cr3)) {
  438. printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
  439. inject_gp(vcpu);
  440. }
  441. if (cr4 & CR4_VMXE_MASK) {
  442. printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
  443. inject_gp(vcpu);
  444. return;
  445. }
  446. kvm_arch_ops->set_cr4(vcpu, cr4);
  447. spin_lock(&vcpu->kvm->lock);
  448. kvm_mmu_reset_context(vcpu);
  449. spin_unlock(&vcpu->kvm->lock);
  450. }
  451. EXPORT_SYMBOL_GPL(set_cr4);
  452. void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
  453. {
  454. if (is_long_mode(vcpu)) {
  455. if (cr3 & CR3_L_MODE_RESEVED_BITS) {
  456. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  457. inject_gp(vcpu);
  458. return;
  459. }
  460. } else {
  461. if (cr3 & CR3_RESEVED_BITS) {
  462. printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
  463. inject_gp(vcpu);
  464. return;
  465. }
  466. if (is_paging(vcpu) && is_pae(vcpu) &&
  467. !load_pdptrs(vcpu, cr3)) {
  468. printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
  469. "reserved bits\n");
  470. inject_gp(vcpu);
  471. return;
  472. }
  473. }
  474. vcpu->cr3 = cr3;
  475. spin_lock(&vcpu->kvm->lock);
  476. /*
  477. * Does the new cr3 value map to physical memory? (Note, we
  478. * catch an invalid cr3 even in real-mode, because it would
  479. * cause trouble later on when we turn on paging anyway.)
  480. *
  481. * A real CPU would silently accept an invalid cr3 and would
  482. * attempt to use it - with largely undefined (and often hard
  483. * to debug) behavior on the guest side.
  484. */
  485. if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
  486. inject_gp(vcpu);
  487. else
  488. vcpu->mmu.new_cr3(vcpu);
  489. spin_unlock(&vcpu->kvm->lock);
  490. }
  491. EXPORT_SYMBOL_GPL(set_cr3);
  492. void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
  493. {
  494. if ( cr8 & CR8_RESEVED_BITS) {
  495. printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
  496. inject_gp(vcpu);
  497. return;
  498. }
  499. vcpu->cr8 = cr8;
  500. }
  501. EXPORT_SYMBOL_GPL(set_cr8);
  502. void fx_init(struct kvm_vcpu *vcpu)
  503. {
  504. struct __attribute__ ((__packed__)) fx_image_s {
  505. u16 control; //fcw
  506. u16 status; //fsw
  507. u16 tag; // ftw
  508. u16 opcode; //fop
  509. u64 ip; // fpu ip
  510. u64 operand;// fpu dp
  511. u32 mxcsr;
  512. u32 mxcsr_mask;
  513. } *fx_image;
  514. fx_save(vcpu->host_fx_image);
  515. fpu_init();
  516. fx_save(vcpu->guest_fx_image);
  517. fx_restore(vcpu->host_fx_image);
  518. fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
  519. fx_image->mxcsr = 0x1f80;
  520. memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
  521. 0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
  522. }
  523. EXPORT_SYMBOL_GPL(fx_init);
  524. static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
  525. {
  526. spin_lock(&vcpu->kvm->lock);
  527. kvm_mmu_slot_remove_write_access(vcpu, slot);
  528. spin_unlock(&vcpu->kvm->lock);
  529. }
  530. /*
  531. * Allocate some memory and give it an address in the guest physical address
  532. * space.
  533. *
  534. * Discontiguous memory is allowed, mostly for framebuffers.
  535. */
  536. static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
  537. struct kvm_memory_region *mem)
  538. {
  539. int r;
  540. gfn_t base_gfn;
  541. unsigned long npages;
  542. unsigned long i;
  543. struct kvm_memory_slot *memslot;
  544. struct kvm_memory_slot old, new;
  545. int memory_config_version;
  546. r = -EINVAL;
  547. /* General sanity checks */
  548. if (mem->memory_size & (PAGE_SIZE - 1))
  549. goto out;
  550. if (mem->guest_phys_addr & (PAGE_SIZE - 1))
  551. goto out;
  552. if (mem->slot >= KVM_MEMORY_SLOTS)
  553. goto out;
  554. if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
  555. goto out;
  556. memslot = &kvm->memslots[mem->slot];
  557. base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
  558. npages = mem->memory_size >> PAGE_SHIFT;
  559. if (!npages)
  560. mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
  561. raced:
  562. spin_lock(&kvm->lock);
  563. memory_config_version = kvm->memory_config_version;
  564. new = old = *memslot;
  565. new.base_gfn = base_gfn;
  566. new.npages = npages;
  567. new.flags = mem->flags;
  568. /* Disallow changing a memory slot's size. */
  569. r = -EINVAL;
  570. if (npages && old.npages && npages != old.npages)
  571. goto out_unlock;
  572. /* Check for overlaps */
  573. r = -EEXIST;
  574. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  575. struct kvm_memory_slot *s = &kvm->memslots[i];
  576. if (s == memslot)
  577. continue;
  578. if (!((base_gfn + npages <= s->base_gfn) ||
  579. (base_gfn >= s->base_gfn + s->npages)))
  580. goto out_unlock;
  581. }
  582. /*
  583. * Do memory allocations outside lock. memory_config_version will
  584. * detect any races.
  585. */
  586. spin_unlock(&kvm->lock);
  587. /* Deallocate if slot is being removed */
  588. if (!npages)
  589. new.phys_mem = NULL;
  590. /* Free page dirty bitmap if unneeded */
  591. if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
  592. new.dirty_bitmap = NULL;
  593. r = -ENOMEM;
  594. /* Allocate if a slot is being created */
  595. if (npages && !new.phys_mem) {
  596. new.phys_mem = vmalloc(npages * sizeof(struct page *));
  597. if (!new.phys_mem)
  598. goto out_free;
  599. memset(new.phys_mem, 0, npages * sizeof(struct page *));
  600. for (i = 0; i < npages; ++i) {
  601. new.phys_mem[i] = alloc_page(GFP_HIGHUSER
  602. | __GFP_ZERO);
  603. if (!new.phys_mem[i])
  604. goto out_free;
  605. set_page_private(new.phys_mem[i],0);
  606. }
  607. }
  608. /* Allocate page dirty bitmap if needed */
  609. if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
  610. unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
  611. new.dirty_bitmap = vmalloc(dirty_bytes);
  612. if (!new.dirty_bitmap)
  613. goto out_free;
  614. memset(new.dirty_bitmap, 0, dirty_bytes);
  615. }
  616. spin_lock(&kvm->lock);
  617. if (memory_config_version != kvm->memory_config_version) {
  618. spin_unlock(&kvm->lock);
  619. kvm_free_physmem_slot(&new, &old);
  620. goto raced;
  621. }
  622. r = -EAGAIN;
  623. if (kvm->busy)
  624. goto out_unlock;
  625. if (mem->slot >= kvm->nmemslots)
  626. kvm->nmemslots = mem->slot + 1;
  627. *memslot = new;
  628. ++kvm->memory_config_version;
  629. spin_unlock(&kvm->lock);
  630. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  631. struct kvm_vcpu *vcpu;
  632. vcpu = vcpu_load_slot(kvm, i);
  633. if (!vcpu)
  634. continue;
  635. if (new.flags & KVM_MEM_LOG_DIRTY_PAGES)
  636. do_remove_write_access(vcpu, mem->slot);
  637. kvm_mmu_reset_context(vcpu);
  638. vcpu_put(vcpu);
  639. }
  640. kvm_free_physmem_slot(&old, &new);
  641. return 0;
  642. out_unlock:
  643. spin_unlock(&kvm->lock);
  644. out_free:
  645. kvm_free_physmem_slot(&new, &old);
  646. out:
  647. return r;
  648. }
  649. /*
  650. * Get (and clear) the dirty memory log for a memory slot.
  651. */
  652. static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  653. struct kvm_dirty_log *log)
  654. {
  655. struct kvm_memory_slot *memslot;
  656. int r, i;
  657. int n;
  658. int cleared;
  659. unsigned long any = 0;
  660. spin_lock(&kvm->lock);
  661. /*
  662. * Prevent changes to guest memory configuration even while the lock
  663. * is not taken.
  664. */
  665. ++kvm->busy;
  666. spin_unlock(&kvm->lock);
  667. r = -EINVAL;
  668. if (log->slot >= KVM_MEMORY_SLOTS)
  669. goto out;
  670. memslot = &kvm->memslots[log->slot];
  671. r = -ENOENT;
  672. if (!memslot->dirty_bitmap)
  673. goto out;
  674. n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
  675. for (i = 0; !any && i < n/sizeof(long); ++i)
  676. any = memslot->dirty_bitmap[i];
  677. r = -EFAULT;
  678. if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
  679. goto out;
  680. if (any) {
  681. cleared = 0;
  682. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  683. struct kvm_vcpu *vcpu;
  684. vcpu = vcpu_load_slot(kvm, i);
  685. if (!vcpu)
  686. continue;
  687. if (!cleared) {
  688. do_remove_write_access(vcpu, log->slot);
  689. memset(memslot->dirty_bitmap, 0, n);
  690. cleared = 1;
  691. }
  692. kvm_arch_ops->tlb_flush(vcpu);
  693. vcpu_put(vcpu);
  694. }
  695. }
  696. r = 0;
  697. out:
  698. spin_lock(&kvm->lock);
  699. --kvm->busy;
  700. spin_unlock(&kvm->lock);
  701. return r;
  702. }
  703. struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
  704. {
  705. int i;
  706. for (i = 0; i < kvm->nmemslots; ++i) {
  707. struct kvm_memory_slot *memslot = &kvm->memslots[i];
  708. if (gfn >= memslot->base_gfn
  709. && gfn < memslot->base_gfn + memslot->npages)
  710. return memslot;
  711. }
  712. return NULL;
  713. }
  714. EXPORT_SYMBOL_GPL(gfn_to_memslot);
  715. void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
  716. {
  717. int i;
  718. struct kvm_memory_slot *memslot = NULL;
  719. unsigned long rel_gfn;
  720. for (i = 0; i < kvm->nmemslots; ++i) {
  721. memslot = &kvm->memslots[i];
  722. if (gfn >= memslot->base_gfn
  723. && gfn < memslot->base_gfn + memslot->npages) {
  724. if (!memslot || !memslot->dirty_bitmap)
  725. return;
  726. rel_gfn = gfn - memslot->base_gfn;
  727. /* avoid RMW */
  728. if (!test_bit(rel_gfn, memslot->dirty_bitmap))
  729. set_bit(rel_gfn, memslot->dirty_bitmap);
  730. return;
  731. }
  732. }
  733. }
  734. static int emulator_read_std(unsigned long addr,
  735. unsigned long *val,
  736. unsigned int bytes,
  737. struct x86_emulate_ctxt *ctxt)
  738. {
  739. struct kvm_vcpu *vcpu = ctxt->vcpu;
  740. void *data = val;
  741. while (bytes) {
  742. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  743. unsigned offset = addr & (PAGE_SIZE-1);
  744. unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
  745. unsigned long pfn;
  746. struct kvm_memory_slot *memslot;
  747. void *page;
  748. if (gpa == UNMAPPED_GVA)
  749. return X86EMUL_PROPAGATE_FAULT;
  750. pfn = gpa >> PAGE_SHIFT;
  751. memslot = gfn_to_memslot(vcpu->kvm, pfn);
  752. if (!memslot)
  753. return X86EMUL_UNHANDLEABLE;
  754. page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
  755. memcpy(data, page + offset, tocopy);
  756. kunmap_atomic(page, KM_USER0);
  757. bytes -= tocopy;
  758. data += tocopy;
  759. addr += tocopy;
  760. }
  761. return X86EMUL_CONTINUE;
  762. }
  763. static int emulator_write_std(unsigned long addr,
  764. unsigned long val,
  765. unsigned int bytes,
  766. struct x86_emulate_ctxt *ctxt)
  767. {
  768. printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
  769. addr, bytes);
  770. return X86EMUL_UNHANDLEABLE;
  771. }
  772. static int emulator_read_emulated(unsigned long addr,
  773. unsigned long *val,
  774. unsigned int bytes,
  775. struct x86_emulate_ctxt *ctxt)
  776. {
  777. struct kvm_vcpu *vcpu = ctxt->vcpu;
  778. if (vcpu->mmio_read_completed) {
  779. memcpy(val, vcpu->mmio_data, bytes);
  780. vcpu->mmio_read_completed = 0;
  781. return X86EMUL_CONTINUE;
  782. } else if (emulator_read_std(addr, val, bytes, ctxt)
  783. == X86EMUL_CONTINUE)
  784. return X86EMUL_CONTINUE;
  785. else {
  786. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  787. if (gpa == UNMAPPED_GVA)
  788. return X86EMUL_PROPAGATE_FAULT;
  789. vcpu->mmio_needed = 1;
  790. vcpu->mmio_phys_addr = gpa;
  791. vcpu->mmio_size = bytes;
  792. vcpu->mmio_is_write = 0;
  793. return X86EMUL_UNHANDLEABLE;
  794. }
  795. }
  796. static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
  797. unsigned long val, int bytes)
  798. {
  799. struct kvm_memory_slot *m;
  800. struct page *page;
  801. void *virt;
  802. if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
  803. return 0;
  804. m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
  805. if (!m)
  806. return 0;
  807. page = gfn_to_page(m, gpa >> PAGE_SHIFT);
  808. kvm_mmu_pre_write(vcpu, gpa, bytes);
  809. mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
  810. virt = kmap_atomic(page, KM_USER0);
  811. memcpy(virt + offset_in_page(gpa), &val, bytes);
  812. kunmap_atomic(virt, KM_USER0);
  813. kvm_mmu_post_write(vcpu, gpa, bytes);
  814. return 1;
  815. }
  816. static int emulator_write_emulated(unsigned long addr,
  817. unsigned long val,
  818. unsigned int bytes,
  819. struct x86_emulate_ctxt *ctxt)
  820. {
  821. struct kvm_vcpu *vcpu = ctxt->vcpu;
  822. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
  823. if (gpa == UNMAPPED_GVA)
  824. return X86EMUL_PROPAGATE_FAULT;
  825. if (emulator_write_phys(vcpu, gpa, val, bytes))
  826. return X86EMUL_CONTINUE;
  827. vcpu->mmio_needed = 1;
  828. vcpu->mmio_phys_addr = gpa;
  829. vcpu->mmio_size = bytes;
  830. vcpu->mmio_is_write = 1;
  831. memcpy(vcpu->mmio_data, &val, bytes);
  832. return X86EMUL_CONTINUE;
  833. }
  834. static int emulator_cmpxchg_emulated(unsigned long addr,
  835. unsigned long old,
  836. unsigned long new,
  837. unsigned int bytes,
  838. struct x86_emulate_ctxt *ctxt)
  839. {
  840. static int reported;
  841. if (!reported) {
  842. reported = 1;
  843. printk(KERN_WARNING "kvm: emulating exchange as write\n");
  844. }
  845. return emulator_write_emulated(addr, new, bytes, ctxt);
  846. }
  847. #ifdef CONFIG_X86_32
  848. static int emulator_cmpxchg8b_emulated(unsigned long addr,
  849. unsigned long old_lo,
  850. unsigned long old_hi,
  851. unsigned long new_lo,
  852. unsigned long new_hi,
  853. struct x86_emulate_ctxt *ctxt)
  854. {
  855. static int reported;
  856. int r;
  857. if (!reported) {
  858. reported = 1;
  859. printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
  860. }
  861. r = emulator_write_emulated(addr, new_lo, 4, ctxt);
  862. if (r != X86EMUL_CONTINUE)
  863. return r;
  864. return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
  865. }
  866. #endif
  867. static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
  868. {
  869. return kvm_arch_ops->get_segment_base(vcpu, seg);
  870. }
  871. int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
  872. {
  873. return X86EMUL_CONTINUE;
  874. }
  875. int emulate_clts(struct kvm_vcpu *vcpu)
  876. {
  877. unsigned long cr0;
  878. kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
  879. cr0 = vcpu->cr0 & ~CR0_TS_MASK;
  880. kvm_arch_ops->set_cr0(vcpu, cr0);
  881. return X86EMUL_CONTINUE;
  882. }
  883. int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
  884. {
  885. struct kvm_vcpu *vcpu = ctxt->vcpu;
  886. switch (dr) {
  887. case 0 ... 3:
  888. *dest = kvm_arch_ops->get_dr(vcpu, dr);
  889. return X86EMUL_CONTINUE;
  890. default:
  891. printk(KERN_DEBUG "%s: unexpected dr %u\n",
  892. __FUNCTION__, dr);
  893. return X86EMUL_UNHANDLEABLE;
  894. }
  895. }
  896. int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
  897. {
  898. unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
  899. int exception;
  900. kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
  901. if (exception) {
  902. /* FIXME: better handling */
  903. return X86EMUL_UNHANDLEABLE;
  904. }
  905. return X86EMUL_CONTINUE;
  906. }
  907. static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
  908. {
  909. static int reported;
  910. u8 opcodes[4];
  911. unsigned long rip = ctxt->vcpu->rip;
  912. unsigned long rip_linear;
  913. rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
  914. if (reported)
  915. return;
  916. emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
  917. printk(KERN_ERR "emulation failed but !mmio_needed?"
  918. " rip %lx %02x %02x %02x %02x\n",
  919. rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
  920. reported = 1;
  921. }
  922. struct x86_emulate_ops emulate_ops = {
  923. .read_std = emulator_read_std,
  924. .write_std = emulator_write_std,
  925. .read_emulated = emulator_read_emulated,
  926. .write_emulated = emulator_write_emulated,
  927. .cmpxchg_emulated = emulator_cmpxchg_emulated,
  928. #ifdef CONFIG_X86_32
  929. .cmpxchg8b_emulated = emulator_cmpxchg8b_emulated,
  930. #endif
  931. };
  932. int emulate_instruction(struct kvm_vcpu *vcpu,
  933. struct kvm_run *run,
  934. unsigned long cr2,
  935. u16 error_code)
  936. {
  937. struct x86_emulate_ctxt emulate_ctxt;
  938. int r;
  939. int cs_db, cs_l;
  940. kvm_arch_ops->cache_regs(vcpu);
  941. kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
  942. emulate_ctxt.vcpu = vcpu;
  943. emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
  944. emulate_ctxt.cr2 = cr2;
  945. emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
  946. ? X86EMUL_MODE_REAL : cs_l
  947. ? X86EMUL_MODE_PROT64 : cs_db
  948. ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
  949. if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
  950. emulate_ctxt.cs_base = 0;
  951. emulate_ctxt.ds_base = 0;
  952. emulate_ctxt.es_base = 0;
  953. emulate_ctxt.ss_base = 0;
  954. } else {
  955. emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
  956. emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
  957. emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
  958. emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
  959. }
  960. emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
  961. emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
  962. vcpu->mmio_is_write = 0;
  963. r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
  964. if ((r || vcpu->mmio_is_write) && run) {
  965. run->mmio.phys_addr = vcpu->mmio_phys_addr;
  966. memcpy(run->mmio.data, vcpu->mmio_data, 8);
  967. run->mmio.len = vcpu->mmio_size;
  968. run->mmio.is_write = vcpu->mmio_is_write;
  969. }
  970. if (r) {
  971. if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
  972. return EMULATE_DONE;
  973. if (!vcpu->mmio_needed) {
  974. report_emulation_failure(&emulate_ctxt);
  975. return EMULATE_FAIL;
  976. }
  977. return EMULATE_DO_MMIO;
  978. }
  979. kvm_arch_ops->decache_regs(vcpu);
  980. kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
  981. if (vcpu->mmio_is_write)
  982. return EMULATE_DO_MMIO;
  983. return EMULATE_DONE;
  984. }
  985. EXPORT_SYMBOL_GPL(emulate_instruction);
  986. int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
  987. {
  988. unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
  989. kvm_arch_ops->decache_regs(vcpu);
  990. ret = -KVM_EINVAL;
  991. #ifdef CONFIG_X86_64
  992. if (is_long_mode(vcpu)) {
  993. nr = vcpu->regs[VCPU_REGS_RAX];
  994. a0 = vcpu->regs[VCPU_REGS_RDI];
  995. a1 = vcpu->regs[VCPU_REGS_RSI];
  996. a2 = vcpu->regs[VCPU_REGS_RDX];
  997. a3 = vcpu->regs[VCPU_REGS_RCX];
  998. a4 = vcpu->regs[VCPU_REGS_R8];
  999. a5 = vcpu->regs[VCPU_REGS_R9];
  1000. } else
  1001. #endif
  1002. {
  1003. nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
  1004. a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
  1005. a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
  1006. a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
  1007. a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
  1008. a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
  1009. a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
  1010. }
  1011. switch (nr) {
  1012. default:
  1013. ;
  1014. }
  1015. vcpu->regs[VCPU_REGS_RAX] = ret;
  1016. kvm_arch_ops->cache_regs(vcpu);
  1017. return 1;
  1018. }
  1019. EXPORT_SYMBOL_GPL(kvm_hypercall);
  1020. static u64 mk_cr_64(u64 curr_cr, u32 new_val)
  1021. {
  1022. return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
  1023. }
  1024. void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  1025. {
  1026. struct descriptor_table dt = { limit, base };
  1027. kvm_arch_ops->set_gdt(vcpu, &dt);
  1028. }
  1029. void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
  1030. {
  1031. struct descriptor_table dt = { limit, base };
  1032. kvm_arch_ops->set_idt(vcpu, &dt);
  1033. }
  1034. void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
  1035. unsigned long *rflags)
  1036. {
  1037. lmsw(vcpu, msw);
  1038. *rflags = kvm_arch_ops->get_rflags(vcpu);
  1039. }
  1040. unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
  1041. {
  1042. kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
  1043. switch (cr) {
  1044. case 0:
  1045. return vcpu->cr0;
  1046. case 2:
  1047. return vcpu->cr2;
  1048. case 3:
  1049. return vcpu->cr3;
  1050. case 4:
  1051. return vcpu->cr4;
  1052. default:
  1053. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
  1054. return 0;
  1055. }
  1056. }
  1057. void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
  1058. unsigned long *rflags)
  1059. {
  1060. switch (cr) {
  1061. case 0:
  1062. set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
  1063. *rflags = kvm_arch_ops->get_rflags(vcpu);
  1064. break;
  1065. case 2:
  1066. vcpu->cr2 = val;
  1067. break;
  1068. case 3:
  1069. set_cr3(vcpu, val);
  1070. break;
  1071. case 4:
  1072. set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
  1073. break;
  1074. default:
  1075. vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
  1076. }
  1077. }
  1078. /*
  1079. * Register the para guest with the host:
  1080. */
  1081. static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
  1082. {
  1083. struct kvm_vcpu_para_state *para_state;
  1084. hpa_t para_state_hpa, hypercall_hpa;
  1085. struct page *para_state_page;
  1086. unsigned char *hypercall;
  1087. gpa_t hypercall_gpa;
  1088. printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
  1089. printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
  1090. /*
  1091. * Needs to be page aligned:
  1092. */
  1093. if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
  1094. goto err_gp;
  1095. para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
  1096. printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
  1097. if (is_error_hpa(para_state_hpa))
  1098. goto err_gp;
  1099. mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
  1100. para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
  1101. para_state = kmap_atomic(para_state_page, KM_USER0);
  1102. printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version);
  1103. printk(KERN_DEBUG ".... size: %d\n", para_state->size);
  1104. para_state->host_version = KVM_PARA_API_VERSION;
  1105. /*
  1106. * We cannot support guests that try to register themselves
  1107. * with a newer API version than the host supports:
  1108. */
  1109. if (para_state->guest_version > KVM_PARA_API_VERSION) {
  1110. para_state->ret = -KVM_EINVAL;
  1111. goto err_kunmap_skip;
  1112. }
  1113. hypercall_gpa = para_state->hypercall_gpa;
  1114. hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
  1115. printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
  1116. if (is_error_hpa(hypercall_hpa)) {
  1117. para_state->ret = -KVM_EINVAL;
  1118. goto err_kunmap_skip;
  1119. }
  1120. printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
  1121. vcpu->para_state_page = para_state_page;
  1122. vcpu->para_state_gpa = para_state_gpa;
  1123. vcpu->hypercall_gpa = hypercall_gpa;
  1124. mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
  1125. hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
  1126. KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
  1127. kvm_arch_ops->patch_hypercall(vcpu, hypercall);
  1128. kunmap_atomic(hypercall, KM_USER1);
  1129. para_state->ret = 0;
  1130. err_kunmap_skip:
  1131. kunmap_atomic(para_state, KM_USER0);
  1132. return 0;
  1133. err_gp:
  1134. return 1;
  1135. }
  1136. int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
  1137. {
  1138. u64 data;
  1139. switch (msr) {
  1140. case 0xc0010010: /* SYSCFG */
  1141. case 0xc0010015: /* HWCR */
  1142. case MSR_IA32_PLATFORM_ID:
  1143. case MSR_IA32_P5_MC_ADDR:
  1144. case MSR_IA32_P5_MC_TYPE:
  1145. case MSR_IA32_MC0_CTL:
  1146. case MSR_IA32_MCG_STATUS:
  1147. case MSR_IA32_MCG_CAP:
  1148. case MSR_IA32_MC0_MISC:
  1149. case MSR_IA32_MC0_MISC+4:
  1150. case MSR_IA32_MC0_MISC+8:
  1151. case MSR_IA32_MC0_MISC+12:
  1152. case MSR_IA32_MC0_MISC+16:
  1153. case MSR_IA32_UCODE_REV:
  1154. case MSR_IA32_PERF_STATUS:
  1155. /* MTRR registers */
  1156. case 0xfe:
  1157. case 0x200 ... 0x2ff:
  1158. data = 0;
  1159. break;
  1160. case 0xcd: /* fsb frequency */
  1161. data = 3;
  1162. break;
  1163. case MSR_IA32_APICBASE:
  1164. data = vcpu->apic_base;
  1165. break;
  1166. case MSR_IA32_MISC_ENABLE:
  1167. data = vcpu->ia32_misc_enable_msr;
  1168. break;
  1169. #ifdef CONFIG_X86_64
  1170. case MSR_EFER:
  1171. data = vcpu->shadow_efer;
  1172. break;
  1173. #endif
  1174. default:
  1175. printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
  1176. return 1;
  1177. }
  1178. *pdata = data;
  1179. return 0;
  1180. }
  1181. EXPORT_SYMBOL_GPL(kvm_get_msr_common);
  1182. /*
  1183. * Reads an msr value (of 'msr_index') into 'pdata'.
  1184. * Returns 0 on success, non-0 otherwise.
  1185. * Assumes vcpu_load() was already called.
  1186. */
  1187. static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
  1188. {
  1189. return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
  1190. }
  1191. #ifdef CONFIG_X86_64
  1192. static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
  1193. {
  1194. if (efer & EFER_RESERVED_BITS) {
  1195. printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
  1196. efer);
  1197. inject_gp(vcpu);
  1198. return;
  1199. }
  1200. if (is_paging(vcpu)
  1201. && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
  1202. printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
  1203. inject_gp(vcpu);
  1204. return;
  1205. }
  1206. kvm_arch_ops->set_efer(vcpu, efer);
  1207. efer &= ~EFER_LMA;
  1208. efer |= vcpu->shadow_efer & EFER_LMA;
  1209. vcpu->shadow_efer = efer;
  1210. }
  1211. #endif
  1212. int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  1213. {
  1214. switch (msr) {
  1215. #ifdef CONFIG_X86_64
  1216. case MSR_EFER:
  1217. set_efer(vcpu, data);
  1218. break;
  1219. #endif
  1220. case MSR_IA32_MC0_STATUS:
  1221. printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
  1222. __FUNCTION__, data);
  1223. break;
  1224. case MSR_IA32_UCODE_REV:
  1225. case MSR_IA32_UCODE_WRITE:
  1226. case 0x200 ... 0x2ff: /* MTRRs */
  1227. break;
  1228. case MSR_IA32_APICBASE:
  1229. vcpu->apic_base = data;
  1230. break;
  1231. case MSR_IA32_MISC_ENABLE:
  1232. vcpu->ia32_misc_enable_msr = data;
  1233. break;
  1234. /*
  1235. * This is the 'probe whether the host is KVM' logic:
  1236. */
  1237. case MSR_KVM_API_MAGIC:
  1238. return vcpu_register_para(vcpu, data);
  1239. default:
  1240. printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
  1241. return 1;
  1242. }
  1243. return 0;
  1244. }
  1245. EXPORT_SYMBOL_GPL(kvm_set_msr_common);
  1246. /*
  1247. * Writes msr value into into the appropriate "register".
  1248. * Returns 0 on success, non-0 otherwise.
  1249. * Assumes vcpu_load() was already called.
  1250. */
  1251. static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
  1252. {
  1253. return kvm_arch_ops->set_msr(vcpu, msr_index, data);
  1254. }
  1255. void kvm_resched(struct kvm_vcpu *vcpu)
  1256. {
  1257. vcpu_put(vcpu);
  1258. cond_resched();
  1259. vcpu_load(vcpu);
  1260. }
  1261. EXPORT_SYMBOL_GPL(kvm_resched);
  1262. void load_msrs(struct vmx_msr_entry *e, int n)
  1263. {
  1264. int i;
  1265. for (i = 0; i < n; ++i)
  1266. wrmsrl(e[i].index, e[i].data);
  1267. }
  1268. EXPORT_SYMBOL_GPL(load_msrs);
  1269. void save_msrs(struct vmx_msr_entry *e, int n)
  1270. {
  1271. int i;
  1272. for (i = 0; i < n; ++i)
  1273. rdmsrl(e[i].index, e[i].data);
  1274. }
  1275. EXPORT_SYMBOL_GPL(save_msrs);
  1276. static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
  1277. {
  1278. int r;
  1279. vcpu_load(vcpu);
  1280. /* re-sync apic's tpr */
  1281. vcpu->cr8 = kvm_run->cr8;
  1282. if (kvm_run->emulated) {
  1283. kvm_arch_ops->skip_emulated_instruction(vcpu);
  1284. kvm_run->emulated = 0;
  1285. }
  1286. if (kvm_run->mmio_completed) {
  1287. memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
  1288. vcpu->mmio_read_completed = 1;
  1289. }
  1290. vcpu->mmio_needed = 0;
  1291. r = kvm_arch_ops->run(vcpu, kvm_run);
  1292. vcpu_put(vcpu);
  1293. return r;
  1294. }
  1295. static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
  1296. struct kvm_regs *regs)
  1297. {
  1298. vcpu_load(vcpu);
  1299. kvm_arch_ops->cache_regs(vcpu);
  1300. regs->rax = vcpu->regs[VCPU_REGS_RAX];
  1301. regs->rbx = vcpu->regs[VCPU_REGS_RBX];
  1302. regs->rcx = vcpu->regs[VCPU_REGS_RCX];
  1303. regs->rdx = vcpu->regs[VCPU_REGS_RDX];
  1304. regs->rsi = vcpu->regs[VCPU_REGS_RSI];
  1305. regs->rdi = vcpu->regs[VCPU_REGS_RDI];
  1306. regs->rsp = vcpu->regs[VCPU_REGS_RSP];
  1307. regs->rbp = vcpu->regs[VCPU_REGS_RBP];
  1308. #ifdef CONFIG_X86_64
  1309. regs->r8 = vcpu->regs[VCPU_REGS_R8];
  1310. regs->r9 = vcpu->regs[VCPU_REGS_R9];
  1311. regs->r10 = vcpu->regs[VCPU_REGS_R10];
  1312. regs->r11 = vcpu->regs[VCPU_REGS_R11];
  1313. regs->r12 = vcpu->regs[VCPU_REGS_R12];
  1314. regs->r13 = vcpu->regs[VCPU_REGS_R13];
  1315. regs->r14 = vcpu->regs[VCPU_REGS_R14];
  1316. regs->r15 = vcpu->regs[VCPU_REGS_R15];
  1317. #endif
  1318. regs->rip = vcpu->rip;
  1319. regs->rflags = kvm_arch_ops->get_rflags(vcpu);
  1320. /*
  1321. * Don't leak debug flags in case they were set for guest debugging
  1322. */
  1323. if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
  1324. regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
  1325. vcpu_put(vcpu);
  1326. return 0;
  1327. }
  1328. static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
  1329. struct kvm_regs *regs)
  1330. {
  1331. vcpu_load(vcpu);
  1332. vcpu->regs[VCPU_REGS_RAX] = regs->rax;
  1333. vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
  1334. vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
  1335. vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
  1336. vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
  1337. vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
  1338. vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
  1339. vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
  1340. #ifdef CONFIG_X86_64
  1341. vcpu->regs[VCPU_REGS_R8] = regs->r8;
  1342. vcpu->regs[VCPU_REGS_R9] = regs->r9;
  1343. vcpu->regs[VCPU_REGS_R10] = regs->r10;
  1344. vcpu->regs[VCPU_REGS_R11] = regs->r11;
  1345. vcpu->regs[VCPU_REGS_R12] = regs->r12;
  1346. vcpu->regs[VCPU_REGS_R13] = regs->r13;
  1347. vcpu->regs[VCPU_REGS_R14] = regs->r14;
  1348. vcpu->regs[VCPU_REGS_R15] = regs->r15;
  1349. #endif
  1350. vcpu->rip = regs->rip;
  1351. kvm_arch_ops->set_rflags(vcpu, regs->rflags);
  1352. kvm_arch_ops->decache_regs(vcpu);
  1353. vcpu_put(vcpu);
  1354. return 0;
  1355. }
  1356. static void get_segment(struct kvm_vcpu *vcpu,
  1357. struct kvm_segment *var, int seg)
  1358. {
  1359. return kvm_arch_ops->get_segment(vcpu, var, seg);
  1360. }
  1361. static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  1362. struct kvm_sregs *sregs)
  1363. {
  1364. struct descriptor_table dt;
  1365. vcpu_load(vcpu);
  1366. get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  1367. get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  1368. get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  1369. get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  1370. get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  1371. get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  1372. get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  1373. get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  1374. kvm_arch_ops->get_idt(vcpu, &dt);
  1375. sregs->idt.limit = dt.limit;
  1376. sregs->idt.base = dt.base;
  1377. kvm_arch_ops->get_gdt(vcpu, &dt);
  1378. sregs->gdt.limit = dt.limit;
  1379. sregs->gdt.base = dt.base;
  1380. kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
  1381. sregs->cr0 = vcpu->cr0;
  1382. sregs->cr2 = vcpu->cr2;
  1383. sregs->cr3 = vcpu->cr3;
  1384. sregs->cr4 = vcpu->cr4;
  1385. sregs->cr8 = vcpu->cr8;
  1386. sregs->efer = vcpu->shadow_efer;
  1387. sregs->apic_base = vcpu->apic_base;
  1388. memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
  1389. sizeof sregs->interrupt_bitmap);
  1390. vcpu_put(vcpu);
  1391. return 0;
  1392. }
  1393. static void set_segment(struct kvm_vcpu *vcpu,
  1394. struct kvm_segment *var, int seg)
  1395. {
  1396. return kvm_arch_ops->set_segment(vcpu, var, seg);
  1397. }
  1398. static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  1399. struct kvm_sregs *sregs)
  1400. {
  1401. int mmu_reset_needed = 0;
  1402. int i;
  1403. struct descriptor_table dt;
  1404. vcpu_load(vcpu);
  1405. set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
  1406. set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
  1407. set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
  1408. set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
  1409. set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
  1410. set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
  1411. set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
  1412. set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
  1413. dt.limit = sregs->idt.limit;
  1414. dt.base = sregs->idt.base;
  1415. kvm_arch_ops->set_idt(vcpu, &dt);
  1416. dt.limit = sregs->gdt.limit;
  1417. dt.base = sregs->gdt.base;
  1418. kvm_arch_ops->set_gdt(vcpu, &dt);
  1419. vcpu->cr2 = sregs->cr2;
  1420. mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
  1421. vcpu->cr3 = sregs->cr3;
  1422. vcpu->cr8 = sregs->cr8;
  1423. mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
  1424. #ifdef CONFIG_X86_64
  1425. kvm_arch_ops->set_efer(vcpu, sregs->efer);
  1426. #endif
  1427. vcpu->apic_base = sregs->apic_base;
  1428. kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
  1429. mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
  1430. kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
  1431. mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
  1432. kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
  1433. if (!is_long_mode(vcpu) && is_pae(vcpu))
  1434. load_pdptrs(vcpu, vcpu->cr3);
  1435. if (mmu_reset_needed)
  1436. kvm_mmu_reset_context(vcpu);
  1437. memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
  1438. sizeof vcpu->irq_pending);
  1439. vcpu->irq_summary = 0;
  1440. for (i = 0; i < NR_IRQ_WORDS; ++i)
  1441. if (vcpu->irq_pending[i])
  1442. __set_bit(i, &vcpu->irq_summary);
  1443. vcpu_put(vcpu);
  1444. return 0;
  1445. }
  1446. /*
  1447. * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  1448. * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  1449. *
  1450. * This list is modified at module load time to reflect the
  1451. * capabilities of the host cpu.
  1452. */
  1453. static u32 msrs_to_save[] = {
  1454. MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  1455. MSR_K6_STAR,
  1456. #ifdef CONFIG_X86_64
  1457. MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
  1458. #endif
  1459. MSR_IA32_TIME_STAMP_COUNTER,
  1460. };
  1461. static unsigned num_msrs_to_save;
  1462. static u32 emulated_msrs[] = {
  1463. MSR_IA32_MISC_ENABLE,
  1464. };
  1465. static __init void kvm_init_msr_list(void)
  1466. {
  1467. u32 dummy[2];
  1468. unsigned i, j;
  1469. for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
  1470. if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
  1471. continue;
  1472. if (j < i)
  1473. msrs_to_save[j] = msrs_to_save[i];
  1474. j++;
  1475. }
  1476. num_msrs_to_save = j;
  1477. }
  1478. /*
  1479. * Adapt set_msr() to msr_io()'s calling convention
  1480. */
  1481. static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
  1482. {
  1483. return set_msr(vcpu, index, *data);
  1484. }
  1485. /*
  1486. * Read or write a bunch of msrs. All parameters are kernel addresses.
  1487. *
  1488. * @return number of msrs set successfully.
  1489. */
  1490. static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
  1491. struct kvm_msr_entry *entries,
  1492. int (*do_msr)(struct kvm_vcpu *vcpu,
  1493. unsigned index, u64 *data))
  1494. {
  1495. int i;
  1496. vcpu_load(vcpu);
  1497. for (i = 0; i < msrs->nmsrs; ++i)
  1498. if (do_msr(vcpu, entries[i].index, &entries[i].data))
  1499. break;
  1500. vcpu_put(vcpu);
  1501. return i;
  1502. }
  1503. /*
  1504. * Read or write a bunch of msrs. Parameters are user addresses.
  1505. *
  1506. * @return number of msrs set successfully.
  1507. */
  1508. static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
  1509. int (*do_msr)(struct kvm_vcpu *vcpu,
  1510. unsigned index, u64 *data),
  1511. int writeback)
  1512. {
  1513. struct kvm_msrs msrs;
  1514. struct kvm_msr_entry *entries;
  1515. int r, n;
  1516. unsigned size;
  1517. r = -EFAULT;
  1518. if (copy_from_user(&msrs, user_msrs, sizeof msrs))
  1519. goto out;
  1520. r = -E2BIG;
  1521. if (msrs.nmsrs >= MAX_IO_MSRS)
  1522. goto out;
  1523. r = -ENOMEM;
  1524. size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
  1525. entries = vmalloc(size);
  1526. if (!entries)
  1527. goto out;
  1528. r = -EFAULT;
  1529. if (copy_from_user(entries, user_msrs->entries, size))
  1530. goto out_free;
  1531. r = n = __msr_io(vcpu, &msrs, entries, do_msr);
  1532. if (r < 0)
  1533. goto out_free;
  1534. r = -EFAULT;
  1535. if (writeback && copy_to_user(user_msrs->entries, entries, size))
  1536. goto out_free;
  1537. r = n;
  1538. out_free:
  1539. vfree(entries);
  1540. out:
  1541. return r;
  1542. }
  1543. /*
  1544. * Translate a guest virtual address to a guest physical address.
  1545. */
  1546. static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  1547. struct kvm_translation *tr)
  1548. {
  1549. unsigned long vaddr = tr->linear_address;
  1550. gpa_t gpa;
  1551. vcpu_load(vcpu);
  1552. spin_lock(&vcpu->kvm->lock);
  1553. gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
  1554. tr->physical_address = gpa;
  1555. tr->valid = gpa != UNMAPPED_GVA;
  1556. tr->writeable = 1;
  1557. tr->usermode = 0;
  1558. spin_unlock(&vcpu->kvm->lock);
  1559. vcpu_put(vcpu);
  1560. return 0;
  1561. }
  1562. static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
  1563. struct kvm_interrupt *irq)
  1564. {
  1565. if (irq->irq < 0 || irq->irq >= 256)
  1566. return -EINVAL;
  1567. vcpu_load(vcpu);
  1568. set_bit(irq->irq, vcpu->irq_pending);
  1569. set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
  1570. vcpu_put(vcpu);
  1571. return 0;
  1572. }
  1573. static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
  1574. struct kvm_debug_guest *dbg)
  1575. {
  1576. int r;
  1577. vcpu_load(vcpu);
  1578. r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
  1579. vcpu_put(vcpu);
  1580. return r;
  1581. }
  1582. static int kvm_vcpu_release(struct inode *inode, struct file *filp)
  1583. {
  1584. struct kvm_vcpu *vcpu = filp->private_data;
  1585. fput(vcpu->kvm->filp);
  1586. return 0;
  1587. }
  1588. static struct file_operations kvm_vcpu_fops = {
  1589. .release = kvm_vcpu_release,
  1590. .unlocked_ioctl = kvm_vcpu_ioctl,
  1591. .compat_ioctl = kvm_vcpu_ioctl,
  1592. };
  1593. /*
  1594. * Allocates an inode for the vcpu.
  1595. */
  1596. static int create_vcpu_fd(struct kvm_vcpu *vcpu)
  1597. {
  1598. int fd, r;
  1599. struct inode *inode;
  1600. struct file *file;
  1601. atomic_inc(&vcpu->kvm->filp->f_count);
  1602. inode = kvmfs_inode(&kvm_vcpu_fops);
  1603. if (IS_ERR(inode)) {
  1604. r = PTR_ERR(inode);
  1605. goto out1;
  1606. }
  1607. file = kvmfs_file(inode, vcpu);
  1608. if (IS_ERR(file)) {
  1609. r = PTR_ERR(file);
  1610. goto out2;
  1611. }
  1612. r = get_unused_fd();
  1613. if (r < 0)
  1614. goto out3;
  1615. fd = r;
  1616. fd_install(fd, file);
  1617. return fd;
  1618. out3:
  1619. fput(file);
  1620. out2:
  1621. iput(inode);
  1622. out1:
  1623. fput(vcpu->kvm->filp);
  1624. return r;
  1625. }
  1626. /*
  1627. * Creates some virtual cpus. Good luck creating more than one.
  1628. */
  1629. static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
  1630. {
  1631. int r;
  1632. struct kvm_vcpu *vcpu;
  1633. r = -EINVAL;
  1634. if (!valid_vcpu(n))
  1635. goto out;
  1636. vcpu = &kvm->vcpus[n];
  1637. mutex_lock(&vcpu->mutex);
  1638. if (vcpu->vmcs) {
  1639. mutex_unlock(&vcpu->mutex);
  1640. return -EEXIST;
  1641. }
  1642. vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
  1643. FX_IMAGE_ALIGN);
  1644. vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
  1645. r = kvm_arch_ops->vcpu_create(vcpu);
  1646. if (r < 0)
  1647. goto out_free_vcpus;
  1648. r = kvm_mmu_create(vcpu);
  1649. if (r < 0)
  1650. goto out_free_vcpus;
  1651. kvm_arch_ops->vcpu_load(vcpu);
  1652. r = kvm_mmu_setup(vcpu);
  1653. if (r >= 0)
  1654. r = kvm_arch_ops->vcpu_setup(vcpu);
  1655. vcpu_put(vcpu);
  1656. if (r < 0)
  1657. goto out_free_vcpus;
  1658. r = create_vcpu_fd(vcpu);
  1659. if (r < 0)
  1660. goto out_free_vcpus;
  1661. return r;
  1662. out_free_vcpus:
  1663. kvm_free_vcpu(vcpu);
  1664. mutex_unlock(&vcpu->mutex);
  1665. out:
  1666. return r;
  1667. }
  1668. static long kvm_vcpu_ioctl(struct file *filp,
  1669. unsigned int ioctl, unsigned long arg)
  1670. {
  1671. struct kvm_vcpu *vcpu = filp->private_data;
  1672. void __user *argp = (void __user *)arg;
  1673. int r = -EINVAL;
  1674. switch (ioctl) {
  1675. case KVM_RUN: {
  1676. struct kvm_run kvm_run;
  1677. r = -EFAULT;
  1678. if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
  1679. goto out;
  1680. r = kvm_vcpu_ioctl_run(vcpu, &kvm_run);
  1681. if (r < 0 && r != -EINTR)
  1682. goto out;
  1683. if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
  1684. r = -EFAULT;
  1685. goto out;
  1686. }
  1687. break;
  1688. }
  1689. case KVM_GET_REGS: {
  1690. struct kvm_regs kvm_regs;
  1691. memset(&kvm_regs, 0, sizeof kvm_regs);
  1692. r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
  1693. if (r)
  1694. goto out;
  1695. r = -EFAULT;
  1696. if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
  1697. goto out;
  1698. r = 0;
  1699. break;
  1700. }
  1701. case KVM_SET_REGS: {
  1702. struct kvm_regs kvm_regs;
  1703. r = -EFAULT;
  1704. if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
  1705. goto out;
  1706. r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
  1707. if (r)
  1708. goto out;
  1709. r = 0;
  1710. break;
  1711. }
  1712. case KVM_GET_SREGS: {
  1713. struct kvm_sregs kvm_sregs;
  1714. memset(&kvm_sregs, 0, sizeof kvm_sregs);
  1715. r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
  1716. if (r)
  1717. goto out;
  1718. r = -EFAULT;
  1719. if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
  1720. goto out;
  1721. r = 0;
  1722. break;
  1723. }
  1724. case KVM_SET_SREGS: {
  1725. struct kvm_sregs kvm_sregs;
  1726. r = -EFAULT;
  1727. if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
  1728. goto out;
  1729. r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
  1730. if (r)
  1731. goto out;
  1732. r = 0;
  1733. break;
  1734. }
  1735. case KVM_TRANSLATE: {
  1736. struct kvm_translation tr;
  1737. r = -EFAULT;
  1738. if (copy_from_user(&tr, argp, sizeof tr))
  1739. goto out;
  1740. r = kvm_vcpu_ioctl_translate(vcpu, &tr);
  1741. if (r)
  1742. goto out;
  1743. r = -EFAULT;
  1744. if (copy_to_user(argp, &tr, sizeof tr))
  1745. goto out;
  1746. r = 0;
  1747. break;
  1748. }
  1749. case KVM_INTERRUPT: {
  1750. struct kvm_interrupt irq;
  1751. r = -EFAULT;
  1752. if (copy_from_user(&irq, argp, sizeof irq))
  1753. goto out;
  1754. r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
  1755. if (r)
  1756. goto out;
  1757. r = 0;
  1758. break;
  1759. }
  1760. case KVM_DEBUG_GUEST: {
  1761. struct kvm_debug_guest dbg;
  1762. r = -EFAULT;
  1763. if (copy_from_user(&dbg, argp, sizeof dbg))
  1764. goto out;
  1765. r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
  1766. if (r)
  1767. goto out;
  1768. r = 0;
  1769. break;
  1770. }
  1771. case KVM_GET_MSRS:
  1772. r = msr_io(vcpu, argp, get_msr, 1);
  1773. break;
  1774. case KVM_SET_MSRS:
  1775. r = msr_io(vcpu, argp, do_set_msr, 0);
  1776. break;
  1777. default:
  1778. ;
  1779. }
  1780. out:
  1781. return r;
  1782. }
  1783. static long kvm_vm_ioctl(struct file *filp,
  1784. unsigned int ioctl, unsigned long arg)
  1785. {
  1786. struct kvm *kvm = filp->private_data;
  1787. void __user *argp = (void __user *)arg;
  1788. int r = -EINVAL;
  1789. switch (ioctl) {
  1790. case KVM_CREATE_VCPU:
  1791. r = kvm_vm_ioctl_create_vcpu(kvm, arg);
  1792. if (r < 0)
  1793. goto out;
  1794. break;
  1795. case KVM_SET_MEMORY_REGION: {
  1796. struct kvm_memory_region kvm_mem;
  1797. r = -EFAULT;
  1798. if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
  1799. goto out;
  1800. r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
  1801. if (r)
  1802. goto out;
  1803. break;
  1804. }
  1805. case KVM_GET_DIRTY_LOG: {
  1806. struct kvm_dirty_log log;
  1807. r = -EFAULT;
  1808. if (copy_from_user(&log, argp, sizeof log))
  1809. goto out;
  1810. r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
  1811. if (r)
  1812. goto out;
  1813. break;
  1814. }
  1815. default:
  1816. ;
  1817. }
  1818. out:
  1819. return r;
  1820. }
  1821. static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
  1822. unsigned long address,
  1823. int *type)
  1824. {
  1825. struct kvm *kvm = vma->vm_file->private_data;
  1826. unsigned long pgoff;
  1827. struct kvm_memory_slot *slot;
  1828. struct page *page;
  1829. *type = VM_FAULT_MINOR;
  1830. pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1831. slot = gfn_to_memslot(kvm, pgoff);
  1832. if (!slot)
  1833. return NOPAGE_SIGBUS;
  1834. page = gfn_to_page(slot, pgoff);
  1835. if (!page)
  1836. return NOPAGE_SIGBUS;
  1837. get_page(page);
  1838. return page;
  1839. }
  1840. static struct vm_operations_struct kvm_vm_vm_ops = {
  1841. .nopage = kvm_vm_nopage,
  1842. };
  1843. static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
  1844. {
  1845. vma->vm_ops = &kvm_vm_vm_ops;
  1846. return 0;
  1847. }
  1848. static struct file_operations kvm_vm_fops = {
  1849. .release = kvm_vm_release,
  1850. .unlocked_ioctl = kvm_vm_ioctl,
  1851. .compat_ioctl = kvm_vm_ioctl,
  1852. .mmap = kvm_vm_mmap,
  1853. };
  1854. static int kvm_dev_ioctl_create_vm(void)
  1855. {
  1856. int fd, r;
  1857. struct inode *inode;
  1858. struct file *file;
  1859. struct kvm *kvm;
  1860. inode = kvmfs_inode(&kvm_vm_fops);
  1861. if (IS_ERR(inode)) {
  1862. r = PTR_ERR(inode);
  1863. goto out1;
  1864. }
  1865. kvm = kvm_create_vm();
  1866. if (IS_ERR(kvm)) {
  1867. r = PTR_ERR(kvm);
  1868. goto out2;
  1869. }
  1870. file = kvmfs_file(inode, kvm);
  1871. if (IS_ERR(file)) {
  1872. r = PTR_ERR(file);
  1873. goto out3;
  1874. }
  1875. kvm->filp = file;
  1876. r = get_unused_fd();
  1877. if (r < 0)
  1878. goto out4;
  1879. fd = r;
  1880. fd_install(fd, file);
  1881. return fd;
  1882. out4:
  1883. fput(file);
  1884. out3:
  1885. kvm_destroy_vm(kvm);
  1886. out2:
  1887. iput(inode);
  1888. out1:
  1889. return r;
  1890. }
  1891. static long kvm_dev_ioctl(struct file *filp,
  1892. unsigned int ioctl, unsigned long arg)
  1893. {
  1894. void __user *argp = (void __user *)arg;
  1895. int r = -EINVAL;
  1896. switch (ioctl) {
  1897. case KVM_GET_API_VERSION:
  1898. r = KVM_API_VERSION;
  1899. break;
  1900. case KVM_CREATE_VM:
  1901. r = kvm_dev_ioctl_create_vm();
  1902. break;
  1903. case KVM_GET_MSR_INDEX_LIST: {
  1904. struct kvm_msr_list __user *user_msr_list = argp;
  1905. struct kvm_msr_list msr_list;
  1906. unsigned n;
  1907. r = -EFAULT;
  1908. if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
  1909. goto out;
  1910. n = msr_list.nmsrs;
  1911. msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
  1912. if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
  1913. goto out;
  1914. r = -E2BIG;
  1915. if (n < num_msrs_to_save)
  1916. goto out;
  1917. r = -EFAULT;
  1918. if (copy_to_user(user_msr_list->indices, &msrs_to_save,
  1919. num_msrs_to_save * sizeof(u32)))
  1920. goto out;
  1921. if (copy_to_user(user_msr_list->indices
  1922. + num_msrs_to_save * sizeof(u32),
  1923. &emulated_msrs,
  1924. ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
  1925. goto out;
  1926. r = 0;
  1927. break;
  1928. }
  1929. default:
  1930. ;
  1931. }
  1932. out:
  1933. return r;
  1934. }
  1935. static struct file_operations kvm_chardev_ops = {
  1936. .open = kvm_dev_open,
  1937. .release = kvm_dev_release,
  1938. .unlocked_ioctl = kvm_dev_ioctl,
  1939. .compat_ioctl = kvm_dev_ioctl,
  1940. };
  1941. static struct miscdevice kvm_dev = {
  1942. MISC_DYNAMIC_MINOR,
  1943. "kvm",
  1944. &kvm_chardev_ops,
  1945. };
  1946. static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
  1947. void *v)
  1948. {
  1949. if (val == SYS_RESTART) {
  1950. /*
  1951. * Some (well, at least mine) BIOSes hang on reboot if
  1952. * in vmx root mode.
  1953. */
  1954. printk(KERN_INFO "kvm: exiting hardware virtualization\n");
  1955. on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
  1956. }
  1957. return NOTIFY_OK;
  1958. }
  1959. static struct notifier_block kvm_reboot_notifier = {
  1960. .notifier_call = kvm_reboot,
  1961. .priority = 0,
  1962. };
  1963. /*
  1964. * Make sure that a cpu that is being hot-unplugged does not have any vcpus
  1965. * cached on it.
  1966. */
  1967. static void decache_vcpus_on_cpu(int cpu)
  1968. {
  1969. struct kvm *vm;
  1970. struct kvm_vcpu *vcpu;
  1971. int i;
  1972. spin_lock(&kvm_lock);
  1973. list_for_each_entry(vm, &vm_list, vm_list)
  1974. for (i = 0; i < KVM_MAX_VCPUS; ++i) {
  1975. vcpu = &vm->vcpus[i];
  1976. /*
  1977. * If the vcpu is locked, then it is running on some
  1978. * other cpu and therefore it is not cached on the
  1979. * cpu in question.
  1980. *
  1981. * If it's not locked, check the last cpu it executed
  1982. * on.
  1983. */
  1984. if (mutex_trylock(&vcpu->mutex)) {
  1985. if (vcpu->cpu == cpu) {
  1986. kvm_arch_ops->vcpu_decache(vcpu);
  1987. vcpu->cpu = -1;
  1988. }
  1989. mutex_unlock(&vcpu->mutex);
  1990. }
  1991. }
  1992. spin_unlock(&kvm_lock);
  1993. }
  1994. static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  1995. void *v)
  1996. {
  1997. int cpu = (long)v;
  1998. switch (val) {
  1999. case CPU_DOWN_PREPARE:
  2000. case CPU_UP_CANCELED:
  2001. printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
  2002. cpu);
  2003. decache_vcpus_on_cpu(cpu);
  2004. smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
  2005. NULL, 0, 1);
  2006. break;
  2007. case CPU_ONLINE:
  2008. printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
  2009. cpu);
  2010. smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
  2011. NULL, 0, 1);
  2012. break;
  2013. }
  2014. return NOTIFY_OK;
  2015. }
  2016. static struct notifier_block kvm_cpu_notifier = {
  2017. .notifier_call = kvm_cpu_hotplug,
  2018. .priority = 20, /* must be > scheduler priority */
  2019. };
  2020. static __init void kvm_init_debug(void)
  2021. {
  2022. struct kvm_stats_debugfs_item *p;
  2023. debugfs_dir = debugfs_create_dir("kvm", NULL);
  2024. for (p = debugfs_entries; p->name; ++p)
  2025. p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
  2026. p->data);
  2027. }
  2028. static void kvm_exit_debug(void)
  2029. {
  2030. struct kvm_stats_debugfs_item *p;
  2031. for (p = debugfs_entries; p->name; ++p)
  2032. debugfs_remove(p->dentry);
  2033. debugfs_remove(debugfs_dir);
  2034. }
  2035. static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  2036. {
  2037. decache_vcpus_on_cpu(raw_smp_processor_id());
  2038. on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
  2039. return 0;
  2040. }
  2041. static int kvm_resume(struct sys_device *dev)
  2042. {
  2043. on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
  2044. return 0;
  2045. }
  2046. static struct sysdev_class kvm_sysdev_class = {
  2047. set_kset_name("kvm"),
  2048. .suspend = kvm_suspend,
  2049. .resume = kvm_resume,
  2050. };
  2051. static struct sys_device kvm_sysdev = {
  2052. .id = 0,
  2053. .cls = &kvm_sysdev_class,
  2054. };
  2055. hpa_t bad_page_address;
  2056. static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
  2057. const char *dev_name, void *data, struct vfsmount *mnt)
  2058. {
  2059. return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_SUPER_MAGIC, mnt);
  2060. }
  2061. static struct file_system_type kvm_fs_type = {
  2062. .name = "kvmfs",
  2063. .get_sb = kvmfs_get_sb,
  2064. .kill_sb = kill_anon_super,
  2065. };
  2066. int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
  2067. {
  2068. int r;
  2069. if (kvm_arch_ops) {
  2070. printk(KERN_ERR "kvm: already loaded the other module\n");
  2071. return -EEXIST;
  2072. }
  2073. if (!ops->cpu_has_kvm_support()) {
  2074. printk(KERN_ERR "kvm: no hardware support\n");
  2075. return -EOPNOTSUPP;
  2076. }
  2077. if (ops->disabled_by_bios()) {
  2078. printk(KERN_ERR "kvm: disabled by bios\n");
  2079. return -EOPNOTSUPP;
  2080. }
  2081. kvm_arch_ops = ops;
  2082. r = kvm_arch_ops->hardware_setup();
  2083. if (r < 0)
  2084. goto out;
  2085. on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
  2086. r = register_cpu_notifier(&kvm_cpu_notifier);
  2087. if (r)
  2088. goto out_free_1;
  2089. register_reboot_notifier(&kvm_reboot_notifier);
  2090. r = sysdev_class_register(&kvm_sysdev_class);
  2091. if (r)
  2092. goto out_free_2;
  2093. r = sysdev_register(&kvm_sysdev);
  2094. if (r)
  2095. goto out_free_3;
  2096. kvm_chardev_ops.owner = module;
  2097. r = misc_register(&kvm_dev);
  2098. if (r) {
  2099. printk (KERN_ERR "kvm: misc device register failed\n");
  2100. goto out_free;
  2101. }
  2102. return r;
  2103. out_free:
  2104. sysdev_unregister(&kvm_sysdev);
  2105. out_free_3:
  2106. sysdev_class_unregister(&kvm_sysdev_class);
  2107. out_free_2:
  2108. unregister_reboot_notifier(&kvm_reboot_notifier);
  2109. unregister_cpu_notifier(&kvm_cpu_notifier);
  2110. out_free_1:
  2111. on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
  2112. kvm_arch_ops->hardware_unsetup();
  2113. out:
  2114. kvm_arch_ops = NULL;
  2115. return r;
  2116. }
  2117. void kvm_exit_arch(void)
  2118. {
  2119. misc_deregister(&kvm_dev);
  2120. sysdev_unregister(&kvm_sysdev);
  2121. sysdev_class_unregister(&kvm_sysdev_class);
  2122. unregister_reboot_notifier(&kvm_reboot_notifier);
  2123. unregister_cpu_notifier(&kvm_cpu_notifier);
  2124. on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
  2125. kvm_arch_ops->hardware_unsetup();
  2126. kvm_arch_ops = NULL;
  2127. }
  2128. static __init int kvm_init(void)
  2129. {
  2130. static struct page *bad_page;
  2131. int r;
  2132. r = register_filesystem(&kvm_fs_type);
  2133. if (r)
  2134. goto out3;
  2135. kvmfs_mnt = kern_mount(&kvm_fs_type);
  2136. r = PTR_ERR(kvmfs_mnt);
  2137. if (IS_ERR(kvmfs_mnt))
  2138. goto out2;
  2139. kvm_init_debug();
  2140. kvm_init_msr_list();
  2141. if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
  2142. r = -ENOMEM;
  2143. goto out;
  2144. }
  2145. bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
  2146. memset(__va(bad_page_address), 0, PAGE_SIZE);
  2147. return 0;
  2148. out:
  2149. kvm_exit_debug();
  2150. mntput(kvmfs_mnt);
  2151. out2:
  2152. unregister_filesystem(&kvm_fs_type);
  2153. out3:
  2154. return r;
  2155. }
  2156. static __exit void kvm_exit(void)
  2157. {
  2158. kvm_exit_debug();
  2159. __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
  2160. mntput(kvmfs_mnt);
  2161. unregister_filesystem(&kvm_fs_type);
  2162. }
  2163. module_init(kvm_init)
  2164. module_exit(kvm_exit)
  2165. EXPORT_SYMBOL_GPL(kvm_init_arch);
  2166. EXPORT_SYMBOL_GPL(kvm_exit_arch);