mmu.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477
  1. /*
  2. * Kernel-based Virtual Machine driver for Linux
  3. *
  4. * This module enables machines with Intel VT-x extensions to run virtual
  5. * machines without emulation or binary translation.
  6. *
  7. * MMU support
  8. *
  9. * Copyright (C) 2006 Qumranet, Inc.
  10. *
  11. * Authors:
  12. * Yaniv Kamay <yaniv@qumranet.com>
  13. * Avi Kivity <avi@qumranet.com>
  14. *
  15. * This work is licensed under the terms of the GNU GPL, version 2. See
  16. * the COPYING file in the top-level directory.
  17. *
  18. */
  19. #include <linux/types.h>
  20. #include <linux/string.h>
  21. #include <asm/page.h>
  22. #include <linux/mm.h>
  23. #include <linux/highmem.h>
  24. #include <linux/module.h>
  25. #include "vmx.h"
  26. #include "kvm.h"
  27. #undef MMU_DEBUG
  28. #undef AUDIT
  29. #ifdef AUDIT
  30. static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
  31. #else
  32. static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
  33. #endif
  34. #ifdef MMU_DEBUG
  35. #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  36. #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  37. #else
  38. #define pgprintk(x...) do { } while (0)
  39. #define rmap_printk(x...) do { } while (0)
  40. #endif
  41. #if defined(MMU_DEBUG) || defined(AUDIT)
  42. static int dbg = 1;
  43. #endif
  44. #define ASSERT(x) \
  45. if (!(x)) { \
  46. printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
  47. __FILE__, __LINE__, #x); \
  48. }
  49. #define PT64_PT_BITS 9
  50. #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
  51. #define PT32_PT_BITS 10
  52. #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
  53. #define PT_WRITABLE_SHIFT 1
  54. #define PT_PRESENT_MASK (1ULL << 0)
  55. #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
  56. #define PT_USER_MASK (1ULL << 2)
  57. #define PT_PWT_MASK (1ULL << 3)
  58. #define PT_PCD_MASK (1ULL << 4)
  59. #define PT_ACCESSED_MASK (1ULL << 5)
  60. #define PT_DIRTY_MASK (1ULL << 6)
  61. #define PT_PAGE_SIZE_MASK (1ULL << 7)
  62. #define PT_PAT_MASK (1ULL << 7)
  63. #define PT_GLOBAL_MASK (1ULL << 8)
  64. #define PT64_NX_MASK (1ULL << 63)
  65. #define PT_PAT_SHIFT 7
  66. #define PT_DIR_PAT_SHIFT 12
  67. #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
  68. #define PT32_DIR_PSE36_SIZE 4
  69. #define PT32_DIR_PSE36_SHIFT 13
  70. #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
  71. #define PT32_PTE_COPY_MASK \
  72. (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
  73. #define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
  74. #define PT_FIRST_AVAIL_BITS_SHIFT 9
  75. #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  76. #define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  77. #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  78. #define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
  79. #define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
  80. #define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
  81. #define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
  82. #define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
  83. #define VALID_PAGE(x) ((x) != INVALID_PAGE)
  84. #define PT64_LEVEL_BITS 9
  85. #define PT64_LEVEL_SHIFT(level) \
  86. ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
  87. #define PT64_LEVEL_MASK(level) \
  88. (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
  89. #define PT64_INDEX(address, level)\
  90. (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  91. #define PT32_LEVEL_BITS 10
  92. #define PT32_LEVEL_SHIFT(level) \
  93. ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
  94. #define PT32_LEVEL_MASK(level) \
  95. (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
  96. #define PT32_INDEX(address, level)\
  97. (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
  98. #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
  99. #define PT64_DIR_BASE_ADDR_MASK \
  100. (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
  101. #define PT32_BASE_ADDR_MASK PAGE_MASK
  102. #define PT32_DIR_BASE_ADDR_MASK \
  103. (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
  104. #define PFERR_PRESENT_MASK (1U << 0)
  105. #define PFERR_WRITE_MASK (1U << 1)
  106. #define PFERR_USER_MASK (1U << 2)
  107. #define PFERR_FETCH_MASK (1U << 4)
  108. #define PT64_ROOT_LEVEL 4
  109. #define PT32_ROOT_LEVEL 2
  110. #define PT32E_ROOT_LEVEL 3
  111. #define PT_DIRECTORY_LEVEL 2
  112. #define PT_PAGE_TABLE_LEVEL 1
  113. #define RMAP_EXT 4
  114. struct kvm_rmap_desc {
  115. u64 *shadow_ptes[RMAP_EXT];
  116. struct kvm_rmap_desc *more;
  117. };
  118. static int is_write_protection(struct kvm_vcpu *vcpu)
  119. {
  120. return vcpu->cr0 & CR0_WP_MASK;
  121. }
  122. static int is_cpuid_PSE36(void)
  123. {
  124. return 1;
  125. }
  126. static int is_nx(struct kvm_vcpu *vcpu)
  127. {
  128. return vcpu->shadow_efer & EFER_NX;
  129. }
  130. static int is_present_pte(unsigned long pte)
  131. {
  132. return pte & PT_PRESENT_MASK;
  133. }
  134. static int is_writeble_pte(unsigned long pte)
  135. {
  136. return pte & PT_WRITABLE_MASK;
  137. }
  138. static int is_io_pte(unsigned long pte)
  139. {
  140. return pte & PT_SHADOW_IO_MARK;
  141. }
  142. static int is_rmap_pte(u64 pte)
  143. {
  144. return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
  145. == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
  146. }
  147. static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
  148. size_t objsize, int min)
  149. {
  150. void *obj;
  151. if (cache->nobjs >= min)
  152. return 0;
  153. while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
  154. obj = kzalloc(objsize, GFP_NOWAIT);
  155. if (!obj)
  156. return -ENOMEM;
  157. cache->objects[cache->nobjs++] = obj;
  158. }
  159. return 0;
  160. }
  161. static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
  162. {
  163. while (mc->nobjs)
  164. kfree(mc->objects[--mc->nobjs]);
  165. }
  166. static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
  167. {
  168. int r;
  169. r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
  170. sizeof(struct kvm_pte_chain), 4);
  171. if (r)
  172. goto out;
  173. r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
  174. sizeof(struct kvm_rmap_desc), 1);
  175. out:
  176. return r;
  177. }
  178. static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
  179. {
  180. mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
  181. mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
  182. }
  183. static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
  184. size_t size)
  185. {
  186. void *p;
  187. BUG_ON(!mc->nobjs);
  188. p = mc->objects[--mc->nobjs];
  189. memset(p, 0, size);
  190. return p;
  191. }
  192. static void mmu_memory_cache_free(struct kvm_mmu_memory_cache *mc, void *obj)
  193. {
  194. if (mc->nobjs < KVM_NR_MEM_OBJS)
  195. mc->objects[mc->nobjs++] = obj;
  196. else
  197. kfree(obj);
  198. }
  199. static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
  200. {
  201. return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
  202. sizeof(struct kvm_pte_chain));
  203. }
  204. static void mmu_free_pte_chain(struct kvm_vcpu *vcpu,
  205. struct kvm_pte_chain *pc)
  206. {
  207. mmu_memory_cache_free(&vcpu->mmu_pte_chain_cache, pc);
  208. }
  209. static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
  210. {
  211. return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
  212. sizeof(struct kvm_rmap_desc));
  213. }
  214. static void mmu_free_rmap_desc(struct kvm_vcpu *vcpu,
  215. struct kvm_rmap_desc *rd)
  216. {
  217. mmu_memory_cache_free(&vcpu->mmu_rmap_desc_cache, rd);
  218. }
  219. /*
  220. * Reverse mapping data structures:
  221. *
  222. * If page->private bit zero is zero, then page->private points to the
  223. * shadow page table entry that points to page_address(page).
  224. *
  225. * If page->private bit zero is one, (then page->private & ~1) points
  226. * to a struct kvm_rmap_desc containing more mappings.
  227. */
  228. static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
  229. {
  230. struct page *page;
  231. struct kvm_rmap_desc *desc;
  232. int i;
  233. if (!is_rmap_pte(*spte))
  234. return;
  235. page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
  236. if (!page_private(page)) {
  237. rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
  238. set_page_private(page,(unsigned long)spte);
  239. } else if (!(page_private(page) & 1)) {
  240. rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
  241. desc = mmu_alloc_rmap_desc(vcpu);
  242. desc->shadow_ptes[0] = (u64 *)page_private(page);
  243. desc->shadow_ptes[1] = spte;
  244. set_page_private(page,(unsigned long)desc | 1);
  245. } else {
  246. rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
  247. desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
  248. while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
  249. desc = desc->more;
  250. if (desc->shadow_ptes[RMAP_EXT-1]) {
  251. desc->more = mmu_alloc_rmap_desc(vcpu);
  252. desc = desc->more;
  253. }
  254. for (i = 0; desc->shadow_ptes[i]; ++i)
  255. ;
  256. desc->shadow_ptes[i] = spte;
  257. }
  258. }
  259. static void rmap_desc_remove_entry(struct kvm_vcpu *vcpu,
  260. struct page *page,
  261. struct kvm_rmap_desc *desc,
  262. int i,
  263. struct kvm_rmap_desc *prev_desc)
  264. {
  265. int j;
  266. for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
  267. ;
  268. desc->shadow_ptes[i] = desc->shadow_ptes[j];
  269. desc->shadow_ptes[j] = NULL;
  270. if (j != 0)
  271. return;
  272. if (!prev_desc && !desc->more)
  273. set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
  274. else
  275. if (prev_desc)
  276. prev_desc->more = desc->more;
  277. else
  278. set_page_private(page,(unsigned long)desc->more | 1);
  279. mmu_free_rmap_desc(vcpu, desc);
  280. }
  281. static void rmap_remove(struct kvm_vcpu *vcpu, u64 *spte)
  282. {
  283. struct page *page;
  284. struct kvm_rmap_desc *desc;
  285. struct kvm_rmap_desc *prev_desc;
  286. int i;
  287. if (!is_rmap_pte(*spte))
  288. return;
  289. page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
  290. if (!page_private(page)) {
  291. printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
  292. BUG();
  293. } else if (!(page_private(page) & 1)) {
  294. rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
  295. if ((u64 *)page_private(page) != spte) {
  296. printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
  297. spte, *spte);
  298. BUG();
  299. }
  300. set_page_private(page,0);
  301. } else {
  302. rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
  303. desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
  304. prev_desc = NULL;
  305. while (desc) {
  306. for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
  307. if (desc->shadow_ptes[i] == spte) {
  308. rmap_desc_remove_entry(vcpu, page,
  309. desc, i,
  310. prev_desc);
  311. return;
  312. }
  313. prev_desc = desc;
  314. desc = desc->more;
  315. }
  316. BUG();
  317. }
  318. }
  319. static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
  320. {
  321. struct kvm *kvm = vcpu->kvm;
  322. struct page *page;
  323. struct kvm_memory_slot *slot;
  324. struct kvm_rmap_desc *desc;
  325. u64 *spte;
  326. slot = gfn_to_memslot(kvm, gfn);
  327. BUG_ON(!slot);
  328. page = gfn_to_page(slot, gfn);
  329. while (page_private(page)) {
  330. if (!(page_private(page) & 1))
  331. spte = (u64 *)page_private(page);
  332. else {
  333. desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
  334. spte = desc->shadow_ptes[0];
  335. }
  336. BUG_ON(!spte);
  337. BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
  338. != page_to_pfn(page));
  339. BUG_ON(!(*spte & PT_PRESENT_MASK));
  340. BUG_ON(!(*spte & PT_WRITABLE_MASK));
  341. rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
  342. rmap_remove(vcpu, spte);
  343. kvm_arch_ops->tlb_flush(vcpu);
  344. *spte &= ~(u64)PT_WRITABLE_MASK;
  345. }
  346. }
  347. static int is_empty_shadow_page(hpa_t page_hpa)
  348. {
  349. u64 *pos;
  350. u64 *end;
  351. for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u64);
  352. pos != end; pos++)
  353. if (*pos != 0) {
  354. printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
  355. pos, *pos);
  356. return 0;
  357. }
  358. return 1;
  359. }
  360. static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
  361. {
  362. struct kvm_mmu_page *page_head = page_header(page_hpa);
  363. ASSERT(is_empty_shadow_page(page_hpa));
  364. list_del(&page_head->link);
  365. page_head->page_hpa = page_hpa;
  366. list_add(&page_head->link, &vcpu->free_pages);
  367. ++vcpu->kvm->n_free_mmu_pages;
  368. }
  369. static unsigned kvm_page_table_hashfn(gfn_t gfn)
  370. {
  371. return gfn;
  372. }
  373. static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
  374. u64 *parent_pte)
  375. {
  376. struct kvm_mmu_page *page;
  377. if (list_empty(&vcpu->free_pages))
  378. return NULL;
  379. page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
  380. list_del(&page->link);
  381. list_add(&page->link, &vcpu->kvm->active_mmu_pages);
  382. ASSERT(is_empty_shadow_page(page->page_hpa));
  383. page->slot_bitmap = 0;
  384. page->global = 1;
  385. page->multimapped = 0;
  386. page->parent_pte = parent_pte;
  387. --vcpu->kvm->n_free_mmu_pages;
  388. return page;
  389. }
  390. static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
  391. struct kvm_mmu_page *page, u64 *parent_pte)
  392. {
  393. struct kvm_pte_chain *pte_chain;
  394. struct hlist_node *node;
  395. int i;
  396. if (!parent_pte)
  397. return;
  398. if (!page->multimapped) {
  399. u64 *old = page->parent_pte;
  400. if (!old) {
  401. page->parent_pte = parent_pte;
  402. return;
  403. }
  404. page->multimapped = 1;
  405. pte_chain = mmu_alloc_pte_chain(vcpu);
  406. INIT_HLIST_HEAD(&page->parent_ptes);
  407. hlist_add_head(&pte_chain->link, &page->parent_ptes);
  408. pte_chain->parent_ptes[0] = old;
  409. }
  410. hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
  411. if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
  412. continue;
  413. for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
  414. if (!pte_chain->parent_ptes[i]) {
  415. pte_chain->parent_ptes[i] = parent_pte;
  416. return;
  417. }
  418. }
  419. pte_chain = mmu_alloc_pte_chain(vcpu);
  420. BUG_ON(!pte_chain);
  421. hlist_add_head(&pte_chain->link, &page->parent_ptes);
  422. pte_chain->parent_ptes[0] = parent_pte;
  423. }
  424. static void mmu_page_remove_parent_pte(struct kvm_vcpu *vcpu,
  425. struct kvm_mmu_page *page,
  426. u64 *parent_pte)
  427. {
  428. struct kvm_pte_chain *pte_chain;
  429. struct hlist_node *node;
  430. int i;
  431. if (!page->multimapped) {
  432. BUG_ON(page->parent_pte != parent_pte);
  433. page->parent_pte = NULL;
  434. return;
  435. }
  436. hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
  437. for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
  438. if (!pte_chain->parent_ptes[i])
  439. break;
  440. if (pte_chain->parent_ptes[i] != parent_pte)
  441. continue;
  442. while (i + 1 < NR_PTE_CHAIN_ENTRIES
  443. && pte_chain->parent_ptes[i + 1]) {
  444. pte_chain->parent_ptes[i]
  445. = pte_chain->parent_ptes[i + 1];
  446. ++i;
  447. }
  448. pte_chain->parent_ptes[i] = NULL;
  449. if (i == 0) {
  450. hlist_del(&pte_chain->link);
  451. mmu_free_pte_chain(vcpu, pte_chain);
  452. if (hlist_empty(&page->parent_ptes)) {
  453. page->multimapped = 0;
  454. page->parent_pte = NULL;
  455. }
  456. }
  457. return;
  458. }
  459. BUG();
  460. }
  461. static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
  462. gfn_t gfn)
  463. {
  464. unsigned index;
  465. struct hlist_head *bucket;
  466. struct kvm_mmu_page *page;
  467. struct hlist_node *node;
  468. pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
  469. index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
  470. bucket = &vcpu->kvm->mmu_page_hash[index];
  471. hlist_for_each_entry(page, node, bucket, hash_link)
  472. if (page->gfn == gfn && !page->role.metaphysical) {
  473. pgprintk("%s: found role %x\n",
  474. __FUNCTION__, page->role.word);
  475. return page;
  476. }
  477. return NULL;
  478. }
  479. static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
  480. gfn_t gfn,
  481. gva_t gaddr,
  482. unsigned level,
  483. int metaphysical,
  484. u64 *parent_pte)
  485. {
  486. union kvm_mmu_page_role role;
  487. unsigned index;
  488. unsigned quadrant;
  489. struct hlist_head *bucket;
  490. struct kvm_mmu_page *page;
  491. struct hlist_node *node;
  492. role.word = 0;
  493. role.glevels = vcpu->mmu.root_level;
  494. role.level = level;
  495. role.metaphysical = metaphysical;
  496. if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
  497. quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
  498. quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
  499. role.quadrant = quadrant;
  500. }
  501. pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
  502. gfn, role.word);
  503. index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
  504. bucket = &vcpu->kvm->mmu_page_hash[index];
  505. hlist_for_each_entry(page, node, bucket, hash_link)
  506. if (page->gfn == gfn && page->role.word == role.word) {
  507. mmu_page_add_parent_pte(vcpu, page, parent_pte);
  508. pgprintk("%s: found\n", __FUNCTION__);
  509. return page;
  510. }
  511. page = kvm_mmu_alloc_page(vcpu, parent_pte);
  512. if (!page)
  513. return page;
  514. pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
  515. page->gfn = gfn;
  516. page->role = role;
  517. hlist_add_head(&page->hash_link, bucket);
  518. if (!metaphysical)
  519. rmap_write_protect(vcpu, gfn);
  520. return page;
  521. }
  522. static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
  523. struct kvm_mmu_page *page)
  524. {
  525. unsigned i;
  526. u64 *pt;
  527. u64 ent;
  528. pt = __va(page->page_hpa);
  529. if (page->role.level == PT_PAGE_TABLE_LEVEL) {
  530. for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
  531. if (pt[i] & PT_PRESENT_MASK)
  532. rmap_remove(vcpu, &pt[i]);
  533. pt[i] = 0;
  534. }
  535. kvm_arch_ops->tlb_flush(vcpu);
  536. return;
  537. }
  538. for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
  539. ent = pt[i];
  540. pt[i] = 0;
  541. if (!(ent & PT_PRESENT_MASK))
  542. continue;
  543. ent &= PT64_BASE_ADDR_MASK;
  544. mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
  545. }
  546. }
  547. static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
  548. struct kvm_mmu_page *page,
  549. u64 *parent_pte)
  550. {
  551. mmu_page_remove_parent_pte(vcpu, page, parent_pte);
  552. }
  553. static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
  554. struct kvm_mmu_page *page)
  555. {
  556. u64 *parent_pte;
  557. while (page->multimapped || page->parent_pte) {
  558. if (!page->multimapped)
  559. parent_pte = page->parent_pte;
  560. else {
  561. struct kvm_pte_chain *chain;
  562. chain = container_of(page->parent_ptes.first,
  563. struct kvm_pte_chain, link);
  564. parent_pte = chain->parent_ptes[0];
  565. }
  566. BUG_ON(!parent_pte);
  567. kvm_mmu_put_page(vcpu, page, parent_pte);
  568. *parent_pte = 0;
  569. }
  570. kvm_mmu_page_unlink_children(vcpu, page);
  571. if (!page->root_count) {
  572. hlist_del(&page->hash_link);
  573. kvm_mmu_free_page(vcpu, page->page_hpa);
  574. } else {
  575. list_del(&page->link);
  576. list_add(&page->link, &vcpu->kvm->active_mmu_pages);
  577. }
  578. }
  579. static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
  580. {
  581. unsigned index;
  582. struct hlist_head *bucket;
  583. struct kvm_mmu_page *page;
  584. struct hlist_node *node, *n;
  585. int r;
  586. pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
  587. r = 0;
  588. index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
  589. bucket = &vcpu->kvm->mmu_page_hash[index];
  590. hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
  591. if (page->gfn == gfn && !page->role.metaphysical) {
  592. pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
  593. page->role.word);
  594. kvm_mmu_zap_page(vcpu, page);
  595. r = 1;
  596. }
  597. return r;
  598. }
  599. static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
  600. {
  601. int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
  602. struct kvm_mmu_page *page_head = page_header(__pa(pte));
  603. __set_bit(slot, &page_head->slot_bitmap);
  604. }
  605. hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
  606. {
  607. hpa_t hpa = gpa_to_hpa(vcpu, gpa);
  608. return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
  609. }
  610. hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
  611. {
  612. struct kvm_memory_slot *slot;
  613. struct page *page;
  614. ASSERT((gpa & HPA_ERR_MASK) == 0);
  615. slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
  616. if (!slot)
  617. return gpa | HPA_ERR_MASK;
  618. page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
  619. return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
  620. | (gpa & (PAGE_SIZE-1));
  621. }
  622. hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
  623. {
  624. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
  625. if (gpa == UNMAPPED_GVA)
  626. return UNMAPPED_GVA;
  627. return gpa_to_hpa(vcpu, gpa);
  628. }
  629. static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
  630. {
  631. }
  632. static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
  633. {
  634. int level = PT32E_ROOT_LEVEL;
  635. hpa_t table_addr = vcpu->mmu.root_hpa;
  636. for (; ; level--) {
  637. u32 index = PT64_INDEX(v, level);
  638. u64 *table;
  639. u64 pte;
  640. ASSERT(VALID_PAGE(table_addr));
  641. table = __va(table_addr);
  642. if (level == 1) {
  643. pte = table[index];
  644. if (is_present_pte(pte) && is_writeble_pte(pte))
  645. return 0;
  646. mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
  647. page_header_update_slot(vcpu->kvm, table, v);
  648. table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
  649. PT_USER_MASK;
  650. rmap_add(vcpu, &table[index]);
  651. return 0;
  652. }
  653. if (table[index] == 0) {
  654. struct kvm_mmu_page *new_table;
  655. gfn_t pseudo_gfn;
  656. pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
  657. >> PAGE_SHIFT;
  658. new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
  659. v, level - 1,
  660. 1, &table[index]);
  661. if (!new_table) {
  662. pgprintk("nonpaging_map: ENOMEM\n");
  663. return -ENOMEM;
  664. }
  665. table[index] = new_table->page_hpa | PT_PRESENT_MASK
  666. | PT_WRITABLE_MASK | PT_USER_MASK;
  667. }
  668. table_addr = table[index] & PT64_BASE_ADDR_MASK;
  669. }
  670. }
  671. static void mmu_free_roots(struct kvm_vcpu *vcpu)
  672. {
  673. int i;
  674. struct kvm_mmu_page *page;
  675. #ifdef CONFIG_X86_64
  676. if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  677. hpa_t root = vcpu->mmu.root_hpa;
  678. ASSERT(VALID_PAGE(root));
  679. page = page_header(root);
  680. --page->root_count;
  681. vcpu->mmu.root_hpa = INVALID_PAGE;
  682. return;
  683. }
  684. #endif
  685. for (i = 0; i < 4; ++i) {
  686. hpa_t root = vcpu->mmu.pae_root[i];
  687. ASSERT(VALID_PAGE(root));
  688. root &= PT64_BASE_ADDR_MASK;
  689. page = page_header(root);
  690. --page->root_count;
  691. vcpu->mmu.pae_root[i] = INVALID_PAGE;
  692. }
  693. vcpu->mmu.root_hpa = INVALID_PAGE;
  694. }
  695. static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
  696. {
  697. int i;
  698. gfn_t root_gfn;
  699. struct kvm_mmu_page *page;
  700. root_gfn = vcpu->cr3 >> PAGE_SHIFT;
  701. #ifdef CONFIG_X86_64
  702. if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
  703. hpa_t root = vcpu->mmu.root_hpa;
  704. ASSERT(!VALID_PAGE(root));
  705. page = kvm_mmu_get_page(vcpu, root_gfn, 0,
  706. PT64_ROOT_LEVEL, 0, NULL);
  707. root = page->page_hpa;
  708. ++page->root_count;
  709. vcpu->mmu.root_hpa = root;
  710. return;
  711. }
  712. #endif
  713. for (i = 0; i < 4; ++i) {
  714. hpa_t root = vcpu->mmu.pae_root[i];
  715. ASSERT(!VALID_PAGE(root));
  716. if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL)
  717. root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
  718. else if (vcpu->mmu.root_level == 0)
  719. root_gfn = 0;
  720. page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
  721. PT32_ROOT_LEVEL, !is_paging(vcpu),
  722. NULL);
  723. root = page->page_hpa;
  724. ++page->root_count;
  725. vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
  726. }
  727. vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
  728. }
  729. static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
  730. {
  731. return vaddr;
  732. }
  733. static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
  734. u32 error_code)
  735. {
  736. gpa_t addr = gva;
  737. hpa_t paddr;
  738. int r;
  739. r = mmu_topup_memory_caches(vcpu);
  740. if (r)
  741. return r;
  742. ASSERT(vcpu);
  743. ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
  744. paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
  745. if (is_error_hpa(paddr))
  746. return 1;
  747. return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
  748. }
  749. static void nonpaging_free(struct kvm_vcpu *vcpu)
  750. {
  751. mmu_free_roots(vcpu);
  752. }
  753. static int nonpaging_init_context(struct kvm_vcpu *vcpu)
  754. {
  755. struct kvm_mmu *context = &vcpu->mmu;
  756. context->new_cr3 = nonpaging_new_cr3;
  757. context->page_fault = nonpaging_page_fault;
  758. context->gva_to_gpa = nonpaging_gva_to_gpa;
  759. context->free = nonpaging_free;
  760. context->root_level = 0;
  761. context->shadow_root_level = PT32E_ROOT_LEVEL;
  762. mmu_alloc_roots(vcpu);
  763. ASSERT(VALID_PAGE(context->root_hpa));
  764. kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
  765. return 0;
  766. }
  767. static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
  768. {
  769. ++kvm_stat.tlb_flush;
  770. kvm_arch_ops->tlb_flush(vcpu);
  771. }
  772. static void paging_new_cr3(struct kvm_vcpu *vcpu)
  773. {
  774. pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
  775. mmu_free_roots(vcpu);
  776. if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES))
  777. kvm_mmu_free_some_pages(vcpu);
  778. mmu_alloc_roots(vcpu);
  779. kvm_mmu_flush_tlb(vcpu);
  780. kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
  781. }
  782. static void mark_pagetable_nonglobal(void *shadow_pte)
  783. {
  784. page_header(__pa(shadow_pte))->global = 0;
  785. }
  786. static inline void set_pte_common(struct kvm_vcpu *vcpu,
  787. u64 *shadow_pte,
  788. gpa_t gaddr,
  789. int dirty,
  790. u64 access_bits,
  791. gfn_t gfn)
  792. {
  793. hpa_t paddr;
  794. *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
  795. if (!dirty)
  796. access_bits &= ~PT_WRITABLE_MASK;
  797. paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
  798. *shadow_pte |= access_bits;
  799. if (!(*shadow_pte & PT_GLOBAL_MASK))
  800. mark_pagetable_nonglobal(shadow_pte);
  801. if (is_error_hpa(paddr)) {
  802. *shadow_pte |= gaddr;
  803. *shadow_pte |= PT_SHADOW_IO_MARK;
  804. *shadow_pte &= ~PT_PRESENT_MASK;
  805. return;
  806. }
  807. *shadow_pte |= paddr;
  808. if (access_bits & PT_WRITABLE_MASK) {
  809. struct kvm_mmu_page *shadow;
  810. shadow = kvm_mmu_lookup_page(vcpu, gfn);
  811. if (shadow) {
  812. pgprintk("%s: found shadow page for %lx, marking ro\n",
  813. __FUNCTION__, gfn);
  814. access_bits &= ~PT_WRITABLE_MASK;
  815. if (is_writeble_pte(*shadow_pte)) {
  816. *shadow_pte &= ~PT_WRITABLE_MASK;
  817. kvm_arch_ops->tlb_flush(vcpu);
  818. }
  819. }
  820. }
  821. if (access_bits & PT_WRITABLE_MASK)
  822. mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
  823. page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
  824. rmap_add(vcpu, shadow_pte);
  825. }
  826. static void inject_page_fault(struct kvm_vcpu *vcpu,
  827. u64 addr,
  828. u32 err_code)
  829. {
  830. kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
  831. }
  832. static inline int fix_read_pf(u64 *shadow_ent)
  833. {
  834. if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
  835. !(*shadow_ent & PT_USER_MASK)) {
  836. /*
  837. * If supervisor write protect is disabled, we shadow kernel
  838. * pages as user pages so we can trap the write access.
  839. */
  840. *shadow_ent |= PT_USER_MASK;
  841. *shadow_ent &= ~PT_WRITABLE_MASK;
  842. return 1;
  843. }
  844. return 0;
  845. }
  846. static void paging_free(struct kvm_vcpu *vcpu)
  847. {
  848. nonpaging_free(vcpu);
  849. }
  850. #define PTTYPE 64
  851. #include "paging_tmpl.h"
  852. #undef PTTYPE
  853. #define PTTYPE 32
  854. #include "paging_tmpl.h"
  855. #undef PTTYPE
  856. static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
  857. {
  858. struct kvm_mmu *context = &vcpu->mmu;
  859. ASSERT(is_pae(vcpu));
  860. context->new_cr3 = paging_new_cr3;
  861. context->page_fault = paging64_page_fault;
  862. context->gva_to_gpa = paging64_gva_to_gpa;
  863. context->free = paging_free;
  864. context->root_level = level;
  865. context->shadow_root_level = level;
  866. mmu_alloc_roots(vcpu);
  867. ASSERT(VALID_PAGE(context->root_hpa));
  868. kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
  869. (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
  870. return 0;
  871. }
  872. static int paging64_init_context(struct kvm_vcpu *vcpu)
  873. {
  874. return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
  875. }
  876. static int paging32_init_context(struct kvm_vcpu *vcpu)
  877. {
  878. struct kvm_mmu *context = &vcpu->mmu;
  879. context->new_cr3 = paging_new_cr3;
  880. context->page_fault = paging32_page_fault;
  881. context->gva_to_gpa = paging32_gva_to_gpa;
  882. context->free = paging_free;
  883. context->root_level = PT32_ROOT_LEVEL;
  884. context->shadow_root_level = PT32E_ROOT_LEVEL;
  885. mmu_alloc_roots(vcpu);
  886. ASSERT(VALID_PAGE(context->root_hpa));
  887. kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
  888. (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
  889. return 0;
  890. }
  891. static int paging32E_init_context(struct kvm_vcpu *vcpu)
  892. {
  893. return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
  894. }
  895. static int init_kvm_mmu(struct kvm_vcpu *vcpu)
  896. {
  897. ASSERT(vcpu);
  898. ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
  899. if (!is_paging(vcpu))
  900. return nonpaging_init_context(vcpu);
  901. else if (is_long_mode(vcpu))
  902. return paging64_init_context(vcpu);
  903. else if (is_pae(vcpu))
  904. return paging32E_init_context(vcpu);
  905. else
  906. return paging32_init_context(vcpu);
  907. }
  908. static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
  909. {
  910. ASSERT(vcpu);
  911. if (VALID_PAGE(vcpu->mmu.root_hpa)) {
  912. vcpu->mmu.free(vcpu);
  913. vcpu->mmu.root_hpa = INVALID_PAGE;
  914. }
  915. }
  916. int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
  917. {
  918. int r;
  919. destroy_kvm_mmu(vcpu);
  920. r = init_kvm_mmu(vcpu);
  921. if (r < 0)
  922. goto out;
  923. r = mmu_topup_memory_caches(vcpu);
  924. out:
  925. return r;
  926. }
  927. static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
  928. struct kvm_mmu_page *page,
  929. u64 *spte)
  930. {
  931. u64 pte;
  932. struct kvm_mmu_page *child;
  933. pte = *spte;
  934. if (is_present_pte(pte)) {
  935. if (page->role.level == PT_PAGE_TABLE_LEVEL)
  936. rmap_remove(vcpu, spte);
  937. else {
  938. child = page_header(pte & PT64_BASE_ADDR_MASK);
  939. mmu_page_remove_parent_pte(vcpu, child, spte);
  940. }
  941. }
  942. *spte = 0;
  943. }
  944. void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
  945. {
  946. gfn_t gfn = gpa >> PAGE_SHIFT;
  947. struct kvm_mmu_page *page;
  948. struct hlist_node *node, *n;
  949. struct hlist_head *bucket;
  950. unsigned index;
  951. u64 *spte;
  952. unsigned offset = offset_in_page(gpa);
  953. unsigned pte_size;
  954. unsigned page_offset;
  955. unsigned misaligned;
  956. int level;
  957. int flooded = 0;
  958. int npte;
  959. pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
  960. if (gfn == vcpu->last_pt_write_gfn) {
  961. ++vcpu->last_pt_write_count;
  962. if (vcpu->last_pt_write_count >= 3)
  963. flooded = 1;
  964. } else {
  965. vcpu->last_pt_write_gfn = gfn;
  966. vcpu->last_pt_write_count = 1;
  967. }
  968. index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
  969. bucket = &vcpu->kvm->mmu_page_hash[index];
  970. hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
  971. if (page->gfn != gfn || page->role.metaphysical)
  972. continue;
  973. pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
  974. misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
  975. if (misaligned || flooded) {
  976. /*
  977. * Misaligned accesses are too much trouble to fix
  978. * up; also, they usually indicate a page is not used
  979. * as a page table.
  980. *
  981. * If we're seeing too many writes to a page,
  982. * it may no longer be a page table, or we may be
  983. * forking, in which case it is better to unmap the
  984. * page.
  985. */
  986. pgprintk("misaligned: gpa %llx bytes %d role %x\n",
  987. gpa, bytes, page->role.word);
  988. kvm_mmu_zap_page(vcpu, page);
  989. continue;
  990. }
  991. page_offset = offset;
  992. level = page->role.level;
  993. npte = 1;
  994. if (page->role.glevels == PT32_ROOT_LEVEL) {
  995. page_offset <<= 1; /* 32->64 */
  996. /*
  997. * A 32-bit pde maps 4MB while the shadow pdes map
  998. * only 2MB. So we need to double the offset again
  999. * and zap two pdes instead of one.
  1000. */
  1001. if (level == PT32_ROOT_LEVEL) {
  1002. page_offset &= ~7; /* kill rounding error */
  1003. page_offset <<= 1;
  1004. npte = 2;
  1005. }
  1006. page_offset &= ~PAGE_MASK;
  1007. }
  1008. spte = __va(page->page_hpa);
  1009. spte += page_offset / sizeof(*spte);
  1010. while (npte--) {
  1011. mmu_pre_write_zap_pte(vcpu, page, spte);
  1012. ++spte;
  1013. }
  1014. }
  1015. }
  1016. void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
  1017. {
  1018. }
  1019. int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
  1020. {
  1021. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
  1022. return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
  1023. }
  1024. void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
  1025. {
  1026. while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
  1027. struct kvm_mmu_page *page;
  1028. page = container_of(vcpu->kvm->active_mmu_pages.prev,
  1029. struct kvm_mmu_page, link);
  1030. kvm_mmu_zap_page(vcpu, page);
  1031. }
  1032. }
  1033. EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages);
  1034. static void free_mmu_pages(struct kvm_vcpu *vcpu)
  1035. {
  1036. struct kvm_mmu_page *page;
  1037. while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
  1038. page = container_of(vcpu->kvm->active_mmu_pages.next,
  1039. struct kvm_mmu_page, link);
  1040. kvm_mmu_zap_page(vcpu, page);
  1041. }
  1042. while (!list_empty(&vcpu->free_pages)) {
  1043. page = list_entry(vcpu->free_pages.next,
  1044. struct kvm_mmu_page, link);
  1045. list_del(&page->link);
  1046. __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
  1047. page->page_hpa = INVALID_PAGE;
  1048. }
  1049. free_page((unsigned long)vcpu->mmu.pae_root);
  1050. }
  1051. static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
  1052. {
  1053. struct page *page;
  1054. int i;
  1055. ASSERT(vcpu);
  1056. for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
  1057. struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
  1058. INIT_LIST_HEAD(&page_header->link);
  1059. if ((page = alloc_page(GFP_KERNEL)) == NULL)
  1060. goto error_1;
  1061. set_page_private(page, (unsigned long)page_header);
  1062. page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
  1063. memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
  1064. list_add(&page_header->link, &vcpu->free_pages);
  1065. ++vcpu->kvm->n_free_mmu_pages;
  1066. }
  1067. /*
  1068. * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
  1069. * Therefore we need to allocate shadow page tables in the first
  1070. * 4GB of memory, which happens to fit the DMA32 zone.
  1071. */
  1072. page = alloc_page(GFP_KERNEL | __GFP_DMA32);
  1073. if (!page)
  1074. goto error_1;
  1075. vcpu->mmu.pae_root = page_address(page);
  1076. for (i = 0; i < 4; ++i)
  1077. vcpu->mmu.pae_root[i] = INVALID_PAGE;
  1078. return 0;
  1079. error_1:
  1080. free_mmu_pages(vcpu);
  1081. return -ENOMEM;
  1082. }
  1083. int kvm_mmu_create(struct kvm_vcpu *vcpu)
  1084. {
  1085. ASSERT(vcpu);
  1086. ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
  1087. ASSERT(list_empty(&vcpu->free_pages));
  1088. return alloc_mmu_pages(vcpu);
  1089. }
  1090. int kvm_mmu_setup(struct kvm_vcpu *vcpu)
  1091. {
  1092. ASSERT(vcpu);
  1093. ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
  1094. ASSERT(!list_empty(&vcpu->free_pages));
  1095. return init_kvm_mmu(vcpu);
  1096. }
  1097. void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
  1098. {
  1099. ASSERT(vcpu);
  1100. destroy_kvm_mmu(vcpu);
  1101. free_mmu_pages(vcpu);
  1102. mmu_free_memory_caches(vcpu);
  1103. }
  1104. void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
  1105. {
  1106. struct kvm *kvm = vcpu->kvm;
  1107. struct kvm_mmu_page *page;
  1108. list_for_each_entry(page, &kvm->active_mmu_pages, link) {
  1109. int i;
  1110. u64 *pt;
  1111. if (!test_bit(slot, &page->slot_bitmap))
  1112. continue;
  1113. pt = __va(page->page_hpa);
  1114. for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
  1115. /* avoid RMW */
  1116. if (pt[i] & PT_WRITABLE_MASK) {
  1117. rmap_remove(vcpu, &pt[i]);
  1118. pt[i] &= ~PT_WRITABLE_MASK;
  1119. }
  1120. }
  1121. }
  1122. #ifdef AUDIT
  1123. static const char *audit_msg;
  1124. static gva_t canonicalize(gva_t gva)
  1125. {
  1126. #ifdef CONFIG_X86_64
  1127. gva = (long long)(gva << 16) >> 16;
  1128. #endif
  1129. return gva;
  1130. }
  1131. static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
  1132. gva_t va, int level)
  1133. {
  1134. u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
  1135. int i;
  1136. gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
  1137. for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
  1138. u64 ent = pt[i];
  1139. if (!ent & PT_PRESENT_MASK)
  1140. continue;
  1141. va = canonicalize(va);
  1142. if (level > 1)
  1143. audit_mappings_page(vcpu, ent, va, level - 1);
  1144. else {
  1145. gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
  1146. hpa_t hpa = gpa_to_hpa(vcpu, gpa);
  1147. if ((ent & PT_PRESENT_MASK)
  1148. && (ent & PT64_BASE_ADDR_MASK) != hpa)
  1149. printk(KERN_ERR "audit error: (%s) levels %d"
  1150. " gva %lx gpa %llx hpa %llx ent %llx\n",
  1151. audit_msg, vcpu->mmu.root_level,
  1152. va, gpa, hpa, ent);
  1153. }
  1154. }
  1155. }
  1156. static void audit_mappings(struct kvm_vcpu *vcpu)
  1157. {
  1158. int i;
  1159. if (vcpu->mmu.root_level == 4)
  1160. audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
  1161. else
  1162. for (i = 0; i < 4; ++i)
  1163. if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
  1164. audit_mappings_page(vcpu,
  1165. vcpu->mmu.pae_root[i],
  1166. i << 30,
  1167. 2);
  1168. }
  1169. static int count_rmaps(struct kvm_vcpu *vcpu)
  1170. {
  1171. int nmaps = 0;
  1172. int i, j, k;
  1173. for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
  1174. struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
  1175. struct kvm_rmap_desc *d;
  1176. for (j = 0; j < m->npages; ++j) {
  1177. struct page *page = m->phys_mem[j];
  1178. if (!page->private)
  1179. continue;
  1180. if (!(page->private & 1)) {
  1181. ++nmaps;
  1182. continue;
  1183. }
  1184. d = (struct kvm_rmap_desc *)(page->private & ~1ul);
  1185. while (d) {
  1186. for (k = 0; k < RMAP_EXT; ++k)
  1187. if (d->shadow_ptes[k])
  1188. ++nmaps;
  1189. else
  1190. break;
  1191. d = d->more;
  1192. }
  1193. }
  1194. }
  1195. return nmaps;
  1196. }
  1197. static int count_writable_mappings(struct kvm_vcpu *vcpu)
  1198. {
  1199. int nmaps = 0;
  1200. struct kvm_mmu_page *page;
  1201. int i;
  1202. list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
  1203. u64 *pt = __va(page->page_hpa);
  1204. if (page->role.level != PT_PAGE_TABLE_LEVEL)
  1205. continue;
  1206. for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
  1207. u64 ent = pt[i];
  1208. if (!(ent & PT_PRESENT_MASK))
  1209. continue;
  1210. if (!(ent & PT_WRITABLE_MASK))
  1211. continue;
  1212. ++nmaps;
  1213. }
  1214. }
  1215. return nmaps;
  1216. }
  1217. static void audit_rmap(struct kvm_vcpu *vcpu)
  1218. {
  1219. int n_rmap = count_rmaps(vcpu);
  1220. int n_actual = count_writable_mappings(vcpu);
  1221. if (n_rmap != n_actual)
  1222. printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
  1223. __FUNCTION__, audit_msg, n_rmap, n_actual);
  1224. }
  1225. static void audit_write_protection(struct kvm_vcpu *vcpu)
  1226. {
  1227. struct kvm_mmu_page *page;
  1228. list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
  1229. hfn_t hfn;
  1230. struct page *pg;
  1231. if (page->role.metaphysical)
  1232. continue;
  1233. hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
  1234. >> PAGE_SHIFT;
  1235. pg = pfn_to_page(hfn);
  1236. if (pg->private)
  1237. printk(KERN_ERR "%s: (%s) shadow page has writable"
  1238. " mappings: gfn %lx role %x\n",
  1239. __FUNCTION__, audit_msg, page->gfn,
  1240. page->role.word);
  1241. }
  1242. }
  1243. static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
  1244. {
  1245. int olddbg = dbg;
  1246. dbg = 0;
  1247. audit_msg = msg;
  1248. audit_rmap(vcpu);
  1249. audit_write_protection(vcpu);
  1250. audit_mappings(vcpu);
  1251. dbg = olddbg;
  1252. }
  1253. #endif