book3s_64_mmu_radix.c 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright 2016 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  5. */
  6. #include <linux/types.h>
  7. #include <linux/string.h>
  8. #include <linux/kvm.h>
  9. #include <linux/kvm_host.h>
  10. #include <linux/anon_inodes.h>
  11. #include <linux/file.h>
  12. #include <linux/debugfs.h>
  13. #include <linux/pgtable.h>
  14. #include <asm/kvm_ppc.h>
  15. #include <asm/kvm_book3s.h>
  16. #include <asm/page.h>
  17. #include <asm/mmu.h>
  18. #include <asm/pgalloc.h>
  19. #include <asm/pte-walk.h>
  20. #include <asm/ultravisor.h>
  21. #include <asm/kvm_book3s_uvmem.h>
  22. /*
  23. * Supported radix tree geometry.
  24. * Like p9, we support either 5 or 9 bits at the first (lowest) level,
  25. * for a page size of 64k or 4k.
  26. */
  27. static int p9_supported_radix_bits[4] = { 5, 9, 9, 13 };
  28. unsigned long __kvmhv_copy_tofrom_guest_radix(int lpid, int pid,
  29. gva_t eaddr, void *to, void *from,
  30. unsigned long n)
  31. {
  32. int old_pid, old_lpid;
  33. unsigned long quadrant, ret = n;
  34. bool is_load = !!to;
  35. /* Can't access quadrants 1 or 2 in non-HV mode, call the HV to do it */
  36. if (kvmhv_on_pseries())
  37. return plpar_hcall_norets(H_COPY_TOFROM_GUEST, lpid, pid, eaddr,
  38. (to != NULL) ? __pa(to): 0,
  39. (from != NULL) ? __pa(from): 0, n);
  40. quadrant = 1;
  41. if (!pid)
  42. quadrant = 2;
  43. if (is_load)
  44. from = (void *) (eaddr | (quadrant << 62));
  45. else
  46. to = (void *) (eaddr | (quadrant << 62));
  47. preempt_disable();
  48. /* switch the lpid first to avoid running host with unallocated pid */
  49. old_lpid = mfspr(SPRN_LPID);
  50. if (old_lpid != lpid)
  51. mtspr(SPRN_LPID, lpid);
  52. if (quadrant == 1) {
  53. old_pid = mfspr(SPRN_PID);
  54. if (old_pid != pid)
  55. mtspr(SPRN_PID, pid);
  56. }
  57. isync();
  58. pagefault_disable();
  59. if (is_load)
  60. ret = __copy_from_user_inatomic(to, (const void __user *)from, n);
  61. else
  62. ret = __copy_to_user_inatomic((void __user *)to, from, n);
  63. pagefault_enable();
  64. /* switch the pid first to avoid running host with unallocated pid */
  65. if (quadrant == 1 && pid != old_pid)
  66. mtspr(SPRN_PID, old_pid);
  67. if (lpid != old_lpid)
  68. mtspr(SPRN_LPID, old_lpid);
  69. isync();
  70. preempt_enable();
  71. return ret;
  72. }
  73. EXPORT_SYMBOL_GPL(__kvmhv_copy_tofrom_guest_radix);
  74. static long kvmhv_copy_tofrom_guest_radix(struct kvm_vcpu *vcpu, gva_t eaddr,
  75. void *to, void *from, unsigned long n)
  76. {
  77. int lpid = vcpu->kvm->arch.lpid;
  78. int pid = vcpu->arch.pid;
  79. /* This would cause a data segment intr so don't allow the access */
  80. if (eaddr & (0x3FFUL << 52))
  81. return -EINVAL;
  82. /* Should we be using the nested lpid */
  83. if (vcpu->arch.nested)
  84. lpid = vcpu->arch.nested->shadow_lpid;
  85. /* If accessing quadrant 3 then pid is expected to be 0 */
  86. if (((eaddr >> 62) & 0x3) == 0x3)
  87. pid = 0;
  88. eaddr &= ~(0xFFFUL << 52);
  89. return __kvmhv_copy_tofrom_guest_radix(lpid, pid, eaddr, to, from, n);
  90. }
  91. long kvmhv_copy_from_guest_radix(struct kvm_vcpu *vcpu, gva_t eaddr, void *to,
  92. unsigned long n)
  93. {
  94. long ret;
  95. ret = kvmhv_copy_tofrom_guest_radix(vcpu, eaddr, to, NULL, n);
  96. if (ret > 0)
  97. memset(to + (n - ret), 0, ret);
  98. return ret;
  99. }
  100. EXPORT_SYMBOL_GPL(kvmhv_copy_from_guest_radix);
  101. long kvmhv_copy_to_guest_radix(struct kvm_vcpu *vcpu, gva_t eaddr, void *from,
  102. unsigned long n)
  103. {
  104. return kvmhv_copy_tofrom_guest_radix(vcpu, eaddr, NULL, from, n);
  105. }
  106. EXPORT_SYMBOL_GPL(kvmhv_copy_to_guest_radix);
  107. int kvmppc_mmu_walk_radix_tree(struct kvm_vcpu *vcpu, gva_t eaddr,
  108. struct kvmppc_pte *gpte, u64 root,
  109. u64 *pte_ret_p)
  110. {
  111. struct kvm *kvm = vcpu->kvm;
  112. int ret, level, ps;
  113. unsigned long rts, bits, offset, index;
  114. u64 pte, base, gpa;
  115. __be64 rpte;
  116. rts = ((root & RTS1_MASK) >> (RTS1_SHIFT - 3)) |
  117. ((root & RTS2_MASK) >> RTS2_SHIFT);
  118. bits = root & RPDS_MASK;
  119. base = root & RPDB_MASK;
  120. offset = rts + 31;
  121. /* Current implementations only support 52-bit space */
  122. if (offset != 52)
  123. return -EINVAL;
  124. /* Walk each level of the radix tree */
  125. for (level = 3; level >= 0; --level) {
  126. u64 addr;
  127. /* Check a valid size */
  128. if (level && bits != p9_supported_radix_bits[level])
  129. return -EINVAL;
  130. if (level == 0 && !(bits == 5 || bits == 9))
  131. return -EINVAL;
  132. offset -= bits;
  133. index = (eaddr >> offset) & ((1UL << bits) - 1);
  134. /* Check that low bits of page table base are zero */
  135. if (base & ((1UL << (bits + 3)) - 1))
  136. return -EINVAL;
  137. /* Read the entry from guest memory */
  138. addr = base + (index * sizeof(rpte));
  139. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  140. ret = kvm_read_guest(kvm, addr, &rpte, sizeof(rpte));
  141. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  142. if (ret) {
  143. if (pte_ret_p)
  144. *pte_ret_p = addr;
  145. return ret;
  146. }
  147. pte = __be64_to_cpu(rpte);
  148. if (!(pte & _PAGE_PRESENT))
  149. return -ENOENT;
  150. /* Check if a leaf entry */
  151. if (pte & _PAGE_PTE)
  152. break;
  153. /* Get ready to walk the next level */
  154. base = pte & RPDB_MASK;
  155. bits = pte & RPDS_MASK;
  156. }
  157. /* Need a leaf at lowest level; 512GB pages not supported */
  158. if (level < 0 || level == 3)
  159. return -EINVAL;
  160. /* We found a valid leaf PTE */
  161. /* Offset is now log base 2 of the page size */
  162. gpa = pte & 0x01fffffffffff000ul;
  163. if (gpa & ((1ul << offset) - 1))
  164. return -EINVAL;
  165. gpa |= eaddr & ((1ul << offset) - 1);
  166. for (ps = MMU_PAGE_4K; ps < MMU_PAGE_COUNT; ++ps)
  167. if (offset == mmu_psize_defs[ps].shift)
  168. break;
  169. gpte->page_size = ps;
  170. gpte->page_shift = offset;
  171. gpte->eaddr = eaddr;
  172. gpte->raddr = gpa;
  173. /* Work out permissions */
  174. gpte->may_read = !!(pte & _PAGE_READ);
  175. gpte->may_write = !!(pte & _PAGE_WRITE);
  176. gpte->may_execute = !!(pte & _PAGE_EXEC);
  177. gpte->rc = pte & (_PAGE_ACCESSED | _PAGE_DIRTY);
  178. if (pte_ret_p)
  179. *pte_ret_p = pte;
  180. return 0;
  181. }
  182. /*
  183. * Used to walk a partition or process table radix tree in guest memory
  184. * Note: We exploit the fact that a partition table and a process
  185. * table have the same layout, a partition-scoped page table and a
  186. * process-scoped page table have the same layout, and the 2nd
  187. * doubleword of a partition table entry has the same layout as
  188. * the PTCR register.
  189. */
  190. int kvmppc_mmu_radix_translate_table(struct kvm_vcpu *vcpu, gva_t eaddr,
  191. struct kvmppc_pte *gpte, u64 table,
  192. int table_index, u64 *pte_ret_p)
  193. {
  194. struct kvm *kvm = vcpu->kvm;
  195. int ret;
  196. unsigned long size, ptbl, root;
  197. struct prtb_entry entry;
  198. if ((table & PRTS_MASK) > 24)
  199. return -EINVAL;
  200. size = 1ul << ((table & PRTS_MASK) + 12);
  201. /* Is the table big enough to contain this entry? */
  202. if ((table_index * sizeof(entry)) >= size)
  203. return -EINVAL;
  204. /* Read the table to find the root of the radix tree */
  205. ptbl = (table & PRTB_MASK) + (table_index * sizeof(entry));
  206. vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
  207. ret = kvm_read_guest(kvm, ptbl, &entry, sizeof(entry));
  208. srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
  209. if (ret)
  210. return ret;
  211. /* Root is stored in the first double word */
  212. root = be64_to_cpu(entry.prtb0);
  213. return kvmppc_mmu_walk_radix_tree(vcpu, eaddr, gpte, root, pte_ret_p);
  214. }
  215. int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
  216. struct kvmppc_pte *gpte, bool data, bool iswrite)
  217. {
  218. u32 pid;
  219. u64 pte;
  220. int ret;
  221. /* Work out effective PID */
  222. switch (eaddr >> 62) {
  223. case 0:
  224. pid = vcpu->arch.pid;
  225. break;
  226. case 3:
  227. pid = 0;
  228. break;
  229. default:
  230. return -EINVAL;
  231. }
  232. ret = kvmppc_mmu_radix_translate_table(vcpu, eaddr, gpte,
  233. vcpu->kvm->arch.process_table, pid, &pte);
  234. if (ret)
  235. return ret;
  236. /* Check privilege (applies only to process scoped translations) */
  237. if (kvmppc_get_msr(vcpu) & MSR_PR) {
  238. if (pte & _PAGE_PRIVILEGED) {
  239. gpte->may_read = 0;
  240. gpte->may_write = 0;
  241. gpte->may_execute = 0;
  242. }
  243. } else {
  244. if (!(pte & _PAGE_PRIVILEGED)) {
  245. /* Check AMR/IAMR to see if strict mode is in force */
  246. if (vcpu->arch.amr & (1ul << 62))
  247. gpte->may_read = 0;
  248. if (vcpu->arch.amr & (1ul << 63))
  249. gpte->may_write = 0;
  250. if (vcpu->arch.iamr & (1ul << 62))
  251. gpte->may_execute = 0;
  252. }
  253. }
  254. return 0;
  255. }
  256. void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
  257. unsigned int pshift, unsigned int lpid)
  258. {
  259. unsigned long psize = PAGE_SIZE;
  260. int psi;
  261. long rc;
  262. unsigned long rb;
  263. if (pshift)
  264. psize = 1UL << pshift;
  265. else
  266. pshift = PAGE_SHIFT;
  267. addr &= ~(psize - 1);
  268. if (!kvmhv_on_pseries()) {
  269. radix__flush_tlb_lpid_page(lpid, addr, psize);
  270. return;
  271. }
  272. psi = shift_to_mmu_psize(pshift);
  273. rb = addr | (mmu_get_ap(psi) << PPC_BITLSHIFT(58));
  274. rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(0, 0, 1),
  275. lpid, rb);
  276. if (rc)
  277. pr_err("KVM: TLB page invalidation hcall failed, rc=%ld\n", rc);
  278. }
  279. static void kvmppc_radix_flush_pwc(struct kvm *kvm, unsigned int lpid)
  280. {
  281. long rc;
  282. if (!kvmhv_on_pseries()) {
  283. radix__flush_pwc_lpid(lpid);
  284. return;
  285. }
  286. rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(1, 0, 1),
  287. lpid, TLBIEL_INVAL_SET_LPID);
  288. if (rc)
  289. pr_err("KVM: TLB PWC invalidation hcall failed, rc=%ld\n", rc);
  290. }
  291. static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
  292. unsigned long clr, unsigned long set,
  293. unsigned long addr, unsigned int shift)
  294. {
  295. return __radix_pte_update(ptep, clr, set);
  296. }
  297. static void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
  298. pte_t *ptep, pte_t pte)
  299. {
  300. radix__set_pte_at(kvm->mm, addr, ptep, pte, 0);
  301. }
  302. static struct kmem_cache *kvm_pte_cache;
  303. static struct kmem_cache *kvm_pmd_cache;
  304. static pte_t *kvmppc_pte_alloc(void)
  305. {
  306. pte_t *pte;
  307. pte = kmem_cache_alloc(kvm_pte_cache, GFP_KERNEL);
  308. /* pmd_populate() will only reference _pa(pte). */
  309. kmemleak_ignore(pte);
  310. return pte;
  311. }
  312. static void kvmppc_pte_free(pte_t *ptep)
  313. {
  314. kmem_cache_free(kvm_pte_cache, ptep);
  315. }
  316. static pmd_t *kvmppc_pmd_alloc(void)
  317. {
  318. pmd_t *pmd;
  319. pmd = kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL);
  320. /* pud_populate() will only reference _pa(pmd). */
  321. kmemleak_ignore(pmd);
  322. return pmd;
  323. }
  324. static void kvmppc_pmd_free(pmd_t *pmdp)
  325. {
  326. kmem_cache_free(kvm_pmd_cache, pmdp);
  327. }
  328. /* Called with kvm->mmu_lock held */
  329. void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa,
  330. unsigned int shift,
  331. const struct kvm_memory_slot *memslot,
  332. unsigned int lpid)
  333. {
  334. unsigned long old;
  335. unsigned long gfn = gpa >> PAGE_SHIFT;
  336. unsigned long page_size = PAGE_SIZE;
  337. unsigned long hpa;
  338. old = kvmppc_radix_update_pte(kvm, pte, ~0UL, 0, gpa, shift);
  339. kvmppc_radix_tlbie_page(kvm, gpa, shift, lpid);
  340. /* The following only applies to L1 entries */
  341. if (lpid != kvm->arch.lpid)
  342. return;
  343. if (!memslot) {
  344. memslot = gfn_to_memslot(kvm, gfn);
  345. if (!memslot)
  346. return;
  347. }
  348. if (shift) { /* 1GB or 2MB page */
  349. page_size = 1ul << shift;
  350. if (shift == PMD_SHIFT)
  351. kvm->stat.num_2M_pages--;
  352. else if (shift == PUD_SHIFT)
  353. kvm->stat.num_1G_pages--;
  354. }
  355. gpa &= ~(page_size - 1);
  356. hpa = old & PTE_RPN_MASK;
  357. kvmhv_remove_nest_rmap_range(kvm, memslot, gpa, hpa, page_size);
  358. if ((old & _PAGE_DIRTY) && memslot->dirty_bitmap)
  359. kvmppc_update_dirty_map(memslot, gfn, page_size);
  360. }
  361. /*
  362. * kvmppc_free_p?d are used to free existing page tables, and recursively
  363. * descend and clear and free children.
  364. * Callers are responsible for flushing the PWC.
  365. *
  366. * When page tables are being unmapped/freed as part of page fault path
  367. * (full == false), valid ptes are generally not expected; however, there
  368. * is one situation where they arise, which is when dirty page logging is
  369. * turned off for a memslot while the VM is running. The new memslot
  370. * becomes visible to page faults before the memslot commit function
  371. * gets to flush the memslot, which can lead to a 2MB page mapping being
  372. * installed for a guest physical address where there are already 64kB
  373. * (or 4kB) mappings (of sub-pages of the same 2MB page).
  374. */
  375. static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full,
  376. unsigned int lpid)
  377. {
  378. if (full) {
  379. memset(pte, 0, sizeof(long) << RADIX_PTE_INDEX_SIZE);
  380. } else {
  381. pte_t *p = pte;
  382. unsigned long it;
  383. for (it = 0; it < PTRS_PER_PTE; ++it, ++p) {
  384. if (pte_val(*p) == 0)
  385. continue;
  386. kvmppc_unmap_pte(kvm, p,
  387. pte_pfn(*p) << PAGE_SHIFT,
  388. PAGE_SHIFT, NULL, lpid);
  389. }
  390. }
  391. kvmppc_pte_free(pte);
  392. }
  393. static void kvmppc_unmap_free_pmd(struct kvm *kvm, pmd_t *pmd, bool full,
  394. unsigned int lpid)
  395. {
  396. unsigned long im;
  397. pmd_t *p = pmd;
  398. for (im = 0; im < PTRS_PER_PMD; ++im, ++p) {
  399. if (!pmd_present(*p))
  400. continue;
  401. if (pmd_is_leaf(*p)) {
  402. if (full) {
  403. pmd_clear(p);
  404. } else {
  405. WARN_ON_ONCE(1);
  406. kvmppc_unmap_pte(kvm, (pte_t *)p,
  407. pte_pfn(*(pte_t *)p) << PAGE_SHIFT,
  408. PMD_SHIFT, NULL, lpid);
  409. }
  410. } else {
  411. pte_t *pte;
  412. pte = pte_offset_map(p, 0);
  413. kvmppc_unmap_free_pte(kvm, pte, full, lpid);
  414. pmd_clear(p);
  415. }
  416. }
  417. kvmppc_pmd_free(pmd);
  418. }
  419. static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,
  420. unsigned int lpid)
  421. {
  422. unsigned long iu;
  423. pud_t *p = pud;
  424. for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++p) {
  425. if (!pud_present(*p))
  426. continue;
  427. if (pud_is_leaf(*p)) {
  428. pud_clear(p);
  429. } else {
  430. pmd_t *pmd;
  431. pmd = pmd_offset(p, 0);
  432. kvmppc_unmap_free_pmd(kvm, pmd, true, lpid);
  433. pud_clear(p);
  434. }
  435. }
  436. pud_free(kvm->mm, pud);
  437. }
  438. void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, unsigned int lpid)
  439. {
  440. unsigned long ig;
  441. for (ig = 0; ig < PTRS_PER_PGD; ++ig, ++pgd) {
  442. p4d_t *p4d = p4d_offset(pgd, 0);
  443. pud_t *pud;
  444. if (!p4d_present(*p4d))
  445. continue;
  446. pud = pud_offset(p4d, 0);
  447. kvmppc_unmap_free_pud(kvm, pud, lpid);
  448. p4d_clear(p4d);
  449. }
  450. }
  451. void kvmppc_free_radix(struct kvm *kvm)
  452. {
  453. if (kvm->arch.pgtable) {
  454. kvmppc_free_pgtable_radix(kvm, kvm->arch.pgtable,
  455. kvm->arch.lpid);
  456. pgd_free(kvm->mm, kvm->arch.pgtable);
  457. kvm->arch.pgtable = NULL;
  458. }
  459. }
  460. static void kvmppc_unmap_free_pmd_entry_table(struct kvm *kvm, pmd_t *pmd,
  461. unsigned long gpa, unsigned int lpid)
  462. {
  463. pte_t *pte = pte_offset_kernel(pmd, 0);
  464. /*
  465. * Clearing the pmd entry then flushing the PWC ensures that the pte
  466. * page no longer be cached by the MMU, so can be freed without
  467. * flushing the PWC again.
  468. */
  469. pmd_clear(pmd);
  470. kvmppc_radix_flush_pwc(kvm, lpid);
  471. kvmppc_unmap_free_pte(kvm, pte, false, lpid);
  472. }
  473. static void kvmppc_unmap_free_pud_entry_table(struct kvm *kvm, pud_t *pud,
  474. unsigned long gpa, unsigned int lpid)
  475. {
  476. pmd_t *pmd = pmd_offset(pud, 0);
  477. /*
  478. * Clearing the pud entry then flushing the PWC ensures that the pmd
  479. * page and any children pte pages will no longer be cached by the MMU,
  480. * so can be freed without flushing the PWC again.
  481. */
  482. pud_clear(pud);
  483. kvmppc_radix_flush_pwc(kvm, lpid);
  484. kvmppc_unmap_free_pmd(kvm, pmd, false, lpid);
  485. }
  486. /*
  487. * There are a number of bits which may differ between different faults to
  488. * the same partition scope entry. RC bits, in the course of cleaning and
  489. * aging. And the write bit can change, either the access could have been
  490. * upgraded, or a read fault could happen concurrently with a write fault
  491. * that sets those bits first.
  492. */
  493. #define PTE_BITS_MUST_MATCH (~(_PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED))
  494. int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
  495. unsigned long gpa, unsigned int level,
  496. unsigned long mmu_seq, unsigned int lpid,
  497. unsigned long *rmapp, struct rmap_nested **n_rmap)
  498. {
  499. pgd_t *pgd;
  500. p4d_t *p4d;
  501. pud_t *pud, *new_pud = NULL;
  502. pmd_t *pmd, *new_pmd = NULL;
  503. pte_t *ptep, *new_ptep = NULL;
  504. int ret;
  505. /* Traverse the guest's 2nd-level tree, allocate new levels needed */
  506. pgd = pgtable + pgd_index(gpa);
  507. p4d = p4d_offset(pgd, gpa);
  508. pud = NULL;
  509. if (p4d_present(*p4d))
  510. pud = pud_offset(p4d, gpa);
  511. else
  512. new_pud = pud_alloc_one(kvm->mm, gpa);
  513. pmd = NULL;
  514. if (pud && pud_present(*pud) && !pud_is_leaf(*pud))
  515. pmd = pmd_offset(pud, gpa);
  516. else if (level <= 1)
  517. new_pmd = kvmppc_pmd_alloc();
  518. if (level == 0 && !(pmd && pmd_present(*pmd) && !pmd_is_leaf(*pmd)))
  519. new_ptep = kvmppc_pte_alloc();
  520. /* Check if we might have been invalidated; let the guest retry if so */
  521. spin_lock(&kvm->mmu_lock);
  522. ret = -EAGAIN;
  523. if (mmu_notifier_retry(kvm, mmu_seq))
  524. goto out_unlock;
  525. /* Now traverse again under the lock and change the tree */
  526. ret = -ENOMEM;
  527. if (p4d_none(*p4d)) {
  528. if (!new_pud)
  529. goto out_unlock;
  530. p4d_populate(kvm->mm, p4d, new_pud);
  531. new_pud = NULL;
  532. }
  533. pud = pud_offset(p4d, gpa);
  534. if (pud_is_leaf(*pud)) {
  535. unsigned long hgpa = gpa & PUD_MASK;
  536. /* Check if we raced and someone else has set the same thing */
  537. if (level == 2) {
  538. if (pud_raw(*pud) == pte_raw(pte)) {
  539. ret = 0;
  540. goto out_unlock;
  541. }
  542. /* Valid 1GB page here already, add our extra bits */
  543. WARN_ON_ONCE((pud_val(*pud) ^ pte_val(pte)) &
  544. PTE_BITS_MUST_MATCH);
  545. kvmppc_radix_update_pte(kvm, (pte_t *)pud,
  546. 0, pte_val(pte), hgpa, PUD_SHIFT);
  547. ret = 0;
  548. goto out_unlock;
  549. }
  550. /*
  551. * If we raced with another CPU which has just put
  552. * a 1GB pte in after we saw a pmd page, try again.
  553. */
  554. if (!new_pmd) {
  555. ret = -EAGAIN;
  556. goto out_unlock;
  557. }
  558. /* Valid 1GB page here already, remove it */
  559. kvmppc_unmap_pte(kvm, (pte_t *)pud, hgpa, PUD_SHIFT, NULL,
  560. lpid);
  561. }
  562. if (level == 2) {
  563. if (!pud_none(*pud)) {
  564. /*
  565. * There's a page table page here, but we wanted to
  566. * install a large page, so remove and free the page
  567. * table page.
  568. */
  569. kvmppc_unmap_free_pud_entry_table(kvm, pud, gpa, lpid);
  570. }
  571. kvmppc_radix_set_pte_at(kvm, gpa, (pte_t *)pud, pte);
  572. if (rmapp && n_rmap)
  573. kvmhv_insert_nest_rmap(kvm, rmapp, n_rmap);
  574. ret = 0;
  575. goto out_unlock;
  576. }
  577. if (pud_none(*pud)) {
  578. if (!new_pmd)
  579. goto out_unlock;
  580. pud_populate(kvm->mm, pud, new_pmd);
  581. new_pmd = NULL;
  582. }
  583. pmd = pmd_offset(pud, gpa);
  584. if (pmd_is_leaf(*pmd)) {
  585. unsigned long lgpa = gpa & PMD_MASK;
  586. /* Check if we raced and someone else has set the same thing */
  587. if (level == 1) {
  588. if (pmd_raw(*pmd) == pte_raw(pte)) {
  589. ret = 0;
  590. goto out_unlock;
  591. }
  592. /* Valid 2MB page here already, add our extra bits */
  593. WARN_ON_ONCE((pmd_val(*pmd) ^ pte_val(pte)) &
  594. PTE_BITS_MUST_MATCH);
  595. kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
  596. 0, pte_val(pte), lgpa, PMD_SHIFT);
  597. ret = 0;
  598. goto out_unlock;
  599. }
  600. /*
  601. * If we raced with another CPU which has just put
  602. * a 2MB pte in after we saw a pte page, try again.
  603. */
  604. if (!new_ptep) {
  605. ret = -EAGAIN;
  606. goto out_unlock;
  607. }
  608. /* Valid 2MB page here already, remove it */
  609. kvmppc_unmap_pte(kvm, pmdp_ptep(pmd), lgpa, PMD_SHIFT, NULL,
  610. lpid);
  611. }
  612. if (level == 1) {
  613. if (!pmd_none(*pmd)) {
  614. /*
  615. * There's a page table page here, but we wanted to
  616. * install a large page, so remove and free the page
  617. * table page.
  618. */
  619. kvmppc_unmap_free_pmd_entry_table(kvm, pmd, gpa, lpid);
  620. }
  621. kvmppc_radix_set_pte_at(kvm, gpa, pmdp_ptep(pmd), pte);
  622. if (rmapp && n_rmap)
  623. kvmhv_insert_nest_rmap(kvm, rmapp, n_rmap);
  624. ret = 0;
  625. goto out_unlock;
  626. }
  627. if (pmd_none(*pmd)) {
  628. if (!new_ptep)
  629. goto out_unlock;
  630. pmd_populate(kvm->mm, pmd, new_ptep);
  631. new_ptep = NULL;
  632. }
  633. ptep = pte_offset_kernel(pmd, gpa);
  634. if (pte_present(*ptep)) {
  635. /* Check if someone else set the same thing */
  636. if (pte_raw(*ptep) == pte_raw(pte)) {
  637. ret = 0;
  638. goto out_unlock;
  639. }
  640. /* Valid page here already, add our extra bits */
  641. WARN_ON_ONCE((pte_val(*ptep) ^ pte_val(pte)) &
  642. PTE_BITS_MUST_MATCH);
  643. kvmppc_radix_update_pte(kvm, ptep, 0, pte_val(pte), gpa, 0);
  644. ret = 0;
  645. goto out_unlock;
  646. }
  647. kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
  648. if (rmapp && n_rmap)
  649. kvmhv_insert_nest_rmap(kvm, rmapp, n_rmap);
  650. ret = 0;
  651. out_unlock:
  652. spin_unlock(&kvm->mmu_lock);
  653. if (new_pud)
  654. pud_free(kvm->mm, new_pud);
  655. if (new_pmd)
  656. kvmppc_pmd_free(new_pmd);
  657. if (new_ptep)
  658. kvmppc_pte_free(new_ptep);
  659. return ret;
  660. }
  661. bool kvmppc_hv_handle_set_rc(struct kvm *kvm, bool nested, bool writing,
  662. unsigned long gpa, unsigned int lpid)
  663. {
  664. unsigned long pgflags;
  665. unsigned int shift;
  666. pte_t *ptep;
  667. /*
  668. * Need to set an R or C bit in the 2nd-level tables;
  669. * since we are just helping out the hardware here,
  670. * it is sufficient to do what the hardware does.
  671. */
  672. pgflags = _PAGE_ACCESSED;
  673. if (writing)
  674. pgflags |= _PAGE_DIRTY;
  675. if (nested)
  676. ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
  677. else
  678. ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
  679. if (ptep && pte_present(*ptep) && (!writing || pte_write(*ptep))) {
  680. kvmppc_radix_update_pte(kvm, ptep, 0, pgflags, gpa, shift);
  681. return true;
  682. }
  683. return false;
  684. }
  685. int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu,
  686. unsigned long gpa,
  687. struct kvm_memory_slot *memslot,
  688. bool writing, bool kvm_ro,
  689. pte_t *inserted_pte, unsigned int *levelp)
  690. {
  691. struct kvm *kvm = vcpu->kvm;
  692. struct page *page = NULL;
  693. unsigned long mmu_seq;
  694. unsigned long hva, gfn = gpa >> PAGE_SHIFT;
  695. bool upgrade_write = false;
  696. bool *upgrade_p = &upgrade_write;
  697. pte_t pte, *ptep;
  698. unsigned int shift, level;
  699. int ret;
  700. bool large_enable;
  701. /* used to check for invalidations in progress */
  702. mmu_seq = kvm->mmu_notifier_seq;
  703. smp_rmb();
  704. /*
  705. * Do a fast check first, since __gfn_to_pfn_memslot doesn't
  706. * do it with !atomic && !async, which is how we call it.
  707. * We always ask for write permission since the common case
  708. * is that the page is writable.
  709. */
  710. hva = gfn_to_hva_memslot(memslot, gfn);
  711. if (!kvm_ro && get_user_page_fast_only(hva, FOLL_WRITE, &page)) {
  712. upgrade_write = true;
  713. } else {
  714. unsigned long pfn;
  715. /* Call KVM generic code to do the slow-path check */
  716. pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL,
  717. writing, upgrade_p);
  718. if (is_error_noslot_pfn(pfn))
  719. return -EFAULT;
  720. page = NULL;
  721. if (pfn_valid(pfn)) {
  722. page = pfn_to_page(pfn);
  723. if (PageReserved(page))
  724. page = NULL;
  725. }
  726. }
  727. /*
  728. * Read the PTE from the process' radix tree and use that
  729. * so we get the shift and attribute bits.
  730. */
  731. spin_lock(&kvm->mmu_lock);
  732. ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);
  733. pte = __pte(0);
  734. if (ptep)
  735. pte = READ_ONCE(*ptep);
  736. spin_unlock(&kvm->mmu_lock);
  737. /*
  738. * If the PTE disappeared temporarily due to a THP
  739. * collapse, just return and let the guest try again.
  740. */
  741. if (!pte_present(pte)) {
  742. if (page)
  743. put_page(page);
  744. return RESUME_GUEST;
  745. }
  746. /* If we're logging dirty pages, always map single pages */
  747. large_enable = !(memslot->flags & KVM_MEM_LOG_DIRTY_PAGES);
  748. /* Get pte level from shift/size */
  749. if (large_enable && shift == PUD_SHIFT &&
  750. (gpa & (PUD_SIZE - PAGE_SIZE)) ==
  751. (hva & (PUD_SIZE - PAGE_SIZE))) {
  752. level = 2;
  753. } else if (large_enable && shift == PMD_SHIFT &&
  754. (gpa & (PMD_SIZE - PAGE_SIZE)) ==
  755. (hva & (PMD_SIZE - PAGE_SIZE))) {
  756. level = 1;
  757. } else {
  758. level = 0;
  759. if (shift > PAGE_SHIFT) {
  760. /*
  761. * If the pte maps more than one page, bring over
  762. * bits from the virtual address to get the real
  763. * address of the specific single page we want.
  764. */
  765. unsigned long rpnmask = (1ul << shift) - PAGE_SIZE;
  766. pte = __pte(pte_val(pte) | (hva & rpnmask));
  767. }
  768. }
  769. pte = __pte(pte_val(pte) | _PAGE_EXEC | _PAGE_ACCESSED);
  770. if (writing || upgrade_write) {
  771. if (pte_val(pte) & _PAGE_WRITE)
  772. pte = __pte(pte_val(pte) | _PAGE_DIRTY);
  773. } else {
  774. pte = __pte(pte_val(pte) & ~(_PAGE_WRITE | _PAGE_DIRTY));
  775. }
  776. /* Allocate space in the tree and write the PTE */
  777. ret = kvmppc_create_pte(kvm, kvm->arch.pgtable, pte, gpa, level,
  778. mmu_seq, kvm->arch.lpid, NULL, NULL);
  779. if (inserted_pte)
  780. *inserted_pte = pte;
  781. if (levelp)
  782. *levelp = level;
  783. if (page) {
  784. if (!ret && (pte_val(pte) & _PAGE_WRITE))
  785. set_page_dirty_lock(page);
  786. put_page(page);
  787. }
  788. /* Increment number of large pages if we (successfully) inserted one */
  789. if (!ret) {
  790. if (level == 1)
  791. kvm->stat.num_2M_pages++;
  792. else if (level == 2)
  793. kvm->stat.num_1G_pages++;
  794. }
  795. return ret;
  796. }
  797. int kvmppc_book3s_radix_page_fault(struct kvm_vcpu *vcpu,
  798. unsigned long ea, unsigned long dsisr)
  799. {
  800. struct kvm *kvm = vcpu->kvm;
  801. unsigned long gpa, gfn;
  802. struct kvm_memory_slot *memslot;
  803. long ret;
  804. bool writing = !!(dsisr & DSISR_ISSTORE);
  805. bool kvm_ro = false;
  806. /* Check for unusual errors */
  807. if (dsisr & DSISR_UNSUPP_MMU) {
  808. pr_err("KVM: Got unsupported MMU fault\n");
  809. return -EFAULT;
  810. }
  811. if (dsisr & DSISR_BADACCESS) {
  812. /* Reflect to the guest as DSI */
  813. pr_err("KVM: Got radix HV page fault with DSISR=%lx\n", dsisr);
  814. kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
  815. return RESUME_GUEST;
  816. }
  817. /* Translate the logical address */
  818. gpa = vcpu->arch.fault_gpa & ~0xfffUL;
  819. gpa &= ~0xF000000000000000ul;
  820. gfn = gpa >> PAGE_SHIFT;
  821. if (!(dsisr & DSISR_PRTABLE_FAULT))
  822. gpa |= ea & 0xfff;
  823. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
  824. return kvmppc_send_page_to_uv(kvm, gfn);
  825. /* Get the corresponding memslot */
  826. memslot = gfn_to_memslot(kvm, gfn);
  827. /* No memslot means it's an emulated MMIO region */
  828. if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) {
  829. if (dsisr & (DSISR_PRTABLE_FAULT | DSISR_BADACCESS |
  830. DSISR_SET_RC)) {
  831. /*
  832. * Bad address in guest page table tree, or other
  833. * unusual error - reflect it to the guest as DSI.
  834. */
  835. kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
  836. return RESUME_GUEST;
  837. }
  838. return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, writing);
  839. }
  840. if (memslot->flags & KVM_MEM_READONLY) {
  841. if (writing) {
  842. /* give the guest a DSI */
  843. kvmppc_core_queue_data_storage(vcpu, ea, DSISR_ISSTORE |
  844. DSISR_PROTFAULT);
  845. return RESUME_GUEST;
  846. }
  847. kvm_ro = true;
  848. }
  849. /* Failed to set the reference/change bits */
  850. if (dsisr & DSISR_SET_RC) {
  851. spin_lock(&kvm->mmu_lock);
  852. if (kvmppc_hv_handle_set_rc(kvm, false, writing,
  853. gpa, kvm->arch.lpid))
  854. dsisr &= ~DSISR_SET_RC;
  855. spin_unlock(&kvm->mmu_lock);
  856. if (!(dsisr & (DSISR_BAD_FAULT_64S | DSISR_NOHPTE |
  857. DSISR_PROTFAULT | DSISR_SET_RC)))
  858. return RESUME_GUEST;
  859. }
  860. /* Try to insert a pte */
  861. ret = kvmppc_book3s_instantiate_page(vcpu, gpa, memslot, writing,
  862. kvm_ro, NULL, NULL);
  863. if (ret == 0 || ret == -EAGAIN)
  864. ret = RESUME_GUEST;
  865. return ret;
  866. }
  867. /* Called with kvm->mmu_lock held */
  868. int kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
  869. unsigned long gfn)
  870. {
  871. pte_t *ptep;
  872. unsigned long gpa = gfn << PAGE_SHIFT;
  873. unsigned int shift;
  874. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) {
  875. uv_page_inval(kvm->arch.lpid, gpa, PAGE_SHIFT);
  876. return 0;
  877. }
  878. ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
  879. if (ptep && pte_present(*ptep))
  880. kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot,
  881. kvm->arch.lpid);
  882. return 0;
  883. }
  884. /* Called with kvm->mmu_lock held */
  885. int kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
  886. unsigned long gfn)
  887. {
  888. pte_t *ptep;
  889. unsigned long gpa = gfn << PAGE_SHIFT;
  890. unsigned int shift;
  891. int ref = 0;
  892. unsigned long old, *rmapp;
  893. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
  894. return ref;
  895. ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
  896. if (ptep && pte_present(*ptep) && pte_young(*ptep)) {
  897. old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_ACCESSED, 0,
  898. gpa, shift);
  899. /* XXX need to flush tlb here? */
  900. /* Also clear bit in ptes in shadow pgtable for nested guests */
  901. rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
  902. kvmhv_update_nest_rmap_rc_list(kvm, rmapp, _PAGE_ACCESSED, 0,
  903. old & PTE_RPN_MASK,
  904. 1UL << shift);
  905. ref = 1;
  906. }
  907. return ref;
  908. }
  909. /* Called with kvm->mmu_lock held */
  910. int kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
  911. unsigned long gfn)
  912. {
  913. pte_t *ptep;
  914. unsigned long gpa = gfn << PAGE_SHIFT;
  915. unsigned int shift;
  916. int ref = 0;
  917. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
  918. return ref;
  919. ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
  920. if (ptep && pte_present(*ptep) && pte_young(*ptep))
  921. ref = 1;
  922. return ref;
  923. }
  924. /* Returns the number of PAGE_SIZE pages that are dirty */
  925. static int kvm_radix_test_clear_dirty(struct kvm *kvm,
  926. struct kvm_memory_slot *memslot, int pagenum)
  927. {
  928. unsigned long gfn = memslot->base_gfn + pagenum;
  929. unsigned long gpa = gfn << PAGE_SHIFT;
  930. pte_t *ptep, pte;
  931. unsigned int shift;
  932. int ret = 0;
  933. unsigned long old, *rmapp;
  934. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
  935. return ret;
  936. /*
  937. * For performance reasons we don't hold kvm->mmu_lock while walking the
  938. * partition scoped table.
  939. */
  940. ptep = find_kvm_secondary_pte_unlocked(kvm, gpa, &shift);
  941. if (!ptep)
  942. return 0;
  943. pte = READ_ONCE(*ptep);
  944. if (pte_present(pte) && pte_dirty(pte)) {
  945. spin_lock(&kvm->mmu_lock);
  946. /*
  947. * Recheck the pte again
  948. */
  949. if (pte_val(pte) != pte_val(*ptep)) {
  950. /*
  951. * We have KVM_MEM_LOG_DIRTY_PAGES enabled. Hence we can
  952. * only find PAGE_SIZE pte entries here. We can continue
  953. * to use the pte addr returned by above page table
  954. * walk.
  955. */
  956. if (!pte_present(*ptep) || !pte_dirty(*ptep)) {
  957. spin_unlock(&kvm->mmu_lock);
  958. return 0;
  959. }
  960. }
  961. ret = 1;
  962. VM_BUG_ON(shift);
  963. old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_DIRTY, 0,
  964. gpa, shift);
  965. kvmppc_radix_tlbie_page(kvm, gpa, shift, kvm->arch.lpid);
  966. /* Also clear bit in ptes in shadow pgtable for nested guests */
  967. rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
  968. kvmhv_update_nest_rmap_rc_list(kvm, rmapp, _PAGE_DIRTY, 0,
  969. old & PTE_RPN_MASK,
  970. 1UL << shift);
  971. spin_unlock(&kvm->mmu_lock);
  972. }
  973. return ret;
  974. }
  975. long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
  976. struct kvm_memory_slot *memslot, unsigned long *map)
  977. {
  978. unsigned long i, j;
  979. int npages;
  980. for (i = 0; i < memslot->npages; i = j) {
  981. npages = kvm_radix_test_clear_dirty(kvm, memslot, i);
  982. /*
  983. * Note that if npages > 0 then i must be a multiple of npages,
  984. * since huge pages are only used to back the guest at guest
  985. * real addresses that are a multiple of their size.
  986. * Since we have at most one PTE covering any given guest
  987. * real address, if npages > 1 we can skip to i + npages.
  988. */
  989. j = i + 1;
  990. if (npages) {
  991. set_dirty_bits(map, i, npages);
  992. j = i + npages;
  993. }
  994. }
  995. return 0;
  996. }
  997. void kvmppc_radix_flush_memslot(struct kvm *kvm,
  998. const struct kvm_memory_slot *memslot)
  999. {
  1000. unsigned long n;
  1001. pte_t *ptep;
  1002. unsigned long gpa;
  1003. unsigned int shift;
  1004. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)
  1005. kvmppc_uvmem_drop_pages(memslot, kvm, true);
  1006. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
  1007. return;
  1008. gpa = memslot->base_gfn << PAGE_SHIFT;
  1009. spin_lock(&kvm->mmu_lock);
  1010. for (n = memslot->npages; n; --n) {
  1011. ptep = find_kvm_secondary_pte(kvm, gpa, &shift);
  1012. if (ptep && pte_present(*ptep))
  1013. kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot,
  1014. kvm->arch.lpid);
  1015. gpa += PAGE_SIZE;
  1016. }
  1017. /*
  1018. * Increase the mmu notifier sequence number to prevent any page
  1019. * fault that read the memslot earlier from writing a PTE.
  1020. */
  1021. kvm->mmu_notifier_seq++;
  1022. spin_unlock(&kvm->mmu_lock);
  1023. }
  1024. static void add_rmmu_ap_encoding(struct kvm_ppc_rmmu_info *info,
  1025. int psize, int *indexp)
  1026. {
  1027. if (!mmu_psize_defs[psize].shift)
  1028. return;
  1029. info->ap_encodings[*indexp] = mmu_psize_defs[psize].shift |
  1030. (mmu_psize_defs[psize].ap << 29);
  1031. ++(*indexp);
  1032. }
  1033. int kvmhv_get_rmmu_info(struct kvm *kvm, struct kvm_ppc_rmmu_info *info)
  1034. {
  1035. int i;
  1036. if (!radix_enabled())
  1037. return -EINVAL;
  1038. memset(info, 0, sizeof(*info));
  1039. /* 4k page size */
  1040. info->geometries[0].page_shift = 12;
  1041. info->geometries[0].level_bits[0] = 9;
  1042. for (i = 1; i < 4; ++i)
  1043. info->geometries[0].level_bits[i] = p9_supported_radix_bits[i];
  1044. /* 64k page size */
  1045. info->geometries[1].page_shift = 16;
  1046. for (i = 0; i < 4; ++i)
  1047. info->geometries[1].level_bits[i] = p9_supported_radix_bits[i];
  1048. i = 0;
  1049. add_rmmu_ap_encoding(info, MMU_PAGE_4K, &i);
  1050. add_rmmu_ap_encoding(info, MMU_PAGE_64K, &i);
  1051. add_rmmu_ap_encoding(info, MMU_PAGE_2M, &i);
  1052. add_rmmu_ap_encoding(info, MMU_PAGE_1G, &i);
  1053. return 0;
  1054. }
  1055. int kvmppc_init_vm_radix(struct kvm *kvm)
  1056. {
  1057. kvm->arch.pgtable = pgd_alloc(kvm->mm);
  1058. if (!kvm->arch.pgtable)
  1059. return -ENOMEM;
  1060. return 0;
  1061. }
  1062. static void pte_ctor(void *addr)
  1063. {
  1064. memset(addr, 0, RADIX_PTE_TABLE_SIZE);
  1065. }
  1066. static void pmd_ctor(void *addr)
  1067. {
  1068. memset(addr, 0, RADIX_PMD_TABLE_SIZE);
  1069. }
  1070. struct debugfs_radix_state {
  1071. struct kvm *kvm;
  1072. struct mutex mutex;
  1073. unsigned long gpa;
  1074. int lpid;
  1075. int chars_left;
  1076. int buf_index;
  1077. char buf[128];
  1078. u8 hdr;
  1079. };
  1080. static int debugfs_radix_open(struct inode *inode, struct file *file)
  1081. {
  1082. struct kvm *kvm = inode->i_private;
  1083. struct debugfs_radix_state *p;
  1084. p = kzalloc(sizeof(*p), GFP_KERNEL);
  1085. if (!p)
  1086. return -ENOMEM;
  1087. kvm_get_kvm(kvm);
  1088. p->kvm = kvm;
  1089. mutex_init(&p->mutex);
  1090. file->private_data = p;
  1091. return nonseekable_open(inode, file);
  1092. }
  1093. static int debugfs_radix_release(struct inode *inode, struct file *file)
  1094. {
  1095. struct debugfs_radix_state *p = file->private_data;
  1096. kvm_put_kvm(p->kvm);
  1097. kfree(p);
  1098. return 0;
  1099. }
  1100. static ssize_t debugfs_radix_read(struct file *file, char __user *buf,
  1101. size_t len, loff_t *ppos)
  1102. {
  1103. struct debugfs_radix_state *p = file->private_data;
  1104. ssize_t ret, r;
  1105. unsigned long n;
  1106. struct kvm *kvm;
  1107. unsigned long gpa;
  1108. pgd_t *pgt;
  1109. struct kvm_nested_guest *nested;
  1110. pgd_t *pgdp;
  1111. p4d_t p4d, *p4dp;
  1112. pud_t pud, *pudp;
  1113. pmd_t pmd, *pmdp;
  1114. pte_t *ptep;
  1115. int shift;
  1116. unsigned long pte;
  1117. kvm = p->kvm;
  1118. if (!kvm_is_radix(kvm))
  1119. return 0;
  1120. ret = mutex_lock_interruptible(&p->mutex);
  1121. if (ret)
  1122. return ret;
  1123. if (p->chars_left) {
  1124. n = p->chars_left;
  1125. if (n > len)
  1126. n = len;
  1127. r = copy_to_user(buf, p->buf + p->buf_index, n);
  1128. n -= r;
  1129. p->chars_left -= n;
  1130. p->buf_index += n;
  1131. buf += n;
  1132. len -= n;
  1133. ret = n;
  1134. if (r) {
  1135. if (!n)
  1136. ret = -EFAULT;
  1137. goto out;
  1138. }
  1139. }
  1140. gpa = p->gpa;
  1141. nested = NULL;
  1142. pgt = NULL;
  1143. while (len != 0 && p->lpid >= 0) {
  1144. if (gpa >= RADIX_PGTABLE_RANGE) {
  1145. gpa = 0;
  1146. pgt = NULL;
  1147. if (nested) {
  1148. kvmhv_put_nested(nested);
  1149. nested = NULL;
  1150. }
  1151. p->lpid = kvmhv_nested_next_lpid(kvm, p->lpid);
  1152. p->hdr = 0;
  1153. if (p->lpid < 0)
  1154. break;
  1155. }
  1156. if (!pgt) {
  1157. if (p->lpid == 0) {
  1158. pgt = kvm->arch.pgtable;
  1159. } else {
  1160. nested = kvmhv_get_nested(kvm, p->lpid, false);
  1161. if (!nested) {
  1162. gpa = RADIX_PGTABLE_RANGE;
  1163. continue;
  1164. }
  1165. pgt = nested->shadow_pgtable;
  1166. }
  1167. }
  1168. n = 0;
  1169. if (!p->hdr) {
  1170. if (p->lpid > 0)
  1171. n = scnprintf(p->buf, sizeof(p->buf),
  1172. "\nNested LPID %d: ", p->lpid);
  1173. n += scnprintf(p->buf + n, sizeof(p->buf) - n,
  1174. "pgdir: %lx\n", (unsigned long)pgt);
  1175. p->hdr = 1;
  1176. goto copy;
  1177. }
  1178. pgdp = pgt + pgd_index(gpa);
  1179. p4dp = p4d_offset(pgdp, gpa);
  1180. p4d = READ_ONCE(*p4dp);
  1181. if (!(p4d_val(p4d) & _PAGE_PRESENT)) {
  1182. gpa = (gpa & P4D_MASK) + P4D_SIZE;
  1183. continue;
  1184. }
  1185. pudp = pud_offset(&p4d, gpa);
  1186. pud = READ_ONCE(*pudp);
  1187. if (!(pud_val(pud) & _PAGE_PRESENT)) {
  1188. gpa = (gpa & PUD_MASK) + PUD_SIZE;
  1189. continue;
  1190. }
  1191. if (pud_val(pud) & _PAGE_PTE) {
  1192. pte = pud_val(pud);
  1193. shift = PUD_SHIFT;
  1194. goto leaf;
  1195. }
  1196. pmdp = pmd_offset(&pud, gpa);
  1197. pmd = READ_ONCE(*pmdp);
  1198. if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
  1199. gpa = (gpa & PMD_MASK) + PMD_SIZE;
  1200. continue;
  1201. }
  1202. if (pmd_val(pmd) & _PAGE_PTE) {
  1203. pte = pmd_val(pmd);
  1204. shift = PMD_SHIFT;
  1205. goto leaf;
  1206. }
  1207. ptep = pte_offset_kernel(&pmd, gpa);
  1208. pte = pte_val(READ_ONCE(*ptep));
  1209. if (!(pte & _PAGE_PRESENT)) {
  1210. gpa += PAGE_SIZE;
  1211. continue;
  1212. }
  1213. shift = PAGE_SHIFT;
  1214. leaf:
  1215. n = scnprintf(p->buf, sizeof(p->buf),
  1216. " %lx: %lx %d\n", gpa, pte, shift);
  1217. gpa += 1ul << shift;
  1218. copy:
  1219. p->chars_left = n;
  1220. if (n > len)
  1221. n = len;
  1222. r = copy_to_user(buf, p->buf, n);
  1223. n -= r;
  1224. p->chars_left -= n;
  1225. p->buf_index = n;
  1226. buf += n;
  1227. len -= n;
  1228. ret += n;
  1229. if (r) {
  1230. if (!ret)
  1231. ret = -EFAULT;
  1232. break;
  1233. }
  1234. }
  1235. p->gpa = gpa;
  1236. if (nested)
  1237. kvmhv_put_nested(nested);
  1238. out:
  1239. mutex_unlock(&p->mutex);
  1240. return ret;
  1241. }
  1242. static ssize_t debugfs_radix_write(struct file *file, const char __user *buf,
  1243. size_t len, loff_t *ppos)
  1244. {
  1245. return -EACCES;
  1246. }
  1247. static const struct file_operations debugfs_radix_fops = {
  1248. .owner = THIS_MODULE,
  1249. .open = debugfs_radix_open,
  1250. .release = debugfs_radix_release,
  1251. .read = debugfs_radix_read,
  1252. .write = debugfs_radix_write,
  1253. .llseek = generic_file_llseek,
  1254. };
  1255. void kvmhv_radix_debugfs_init(struct kvm *kvm)
  1256. {
  1257. debugfs_create_file("radix", 0400, kvm->arch.debugfs_dir, kvm,
  1258. &debugfs_radix_fops);
  1259. }
  1260. int kvmppc_radix_init(void)
  1261. {
  1262. unsigned long size = sizeof(void *) << RADIX_PTE_INDEX_SIZE;
  1263. kvm_pte_cache = kmem_cache_create("kvm-pte", size, size, 0, pte_ctor);
  1264. if (!kvm_pte_cache)
  1265. return -ENOMEM;
  1266. size = sizeof(void *) << RADIX_PMD_INDEX_SIZE;
  1267. kvm_pmd_cache = kmem_cache_create("kvm-pmd", size, size, 0, pmd_ctor);
  1268. if (!kvm_pmd_cache) {
  1269. kmem_cache_destroy(kvm_pte_cache);
  1270. return -ENOMEM;
  1271. }
  1272. return 0;
  1273. }
  1274. void kvmppc_radix_exit(void)
  1275. {
  1276. kmem_cache_destroy(kvm_pte_cache);
  1277. kmem_cache_destroy(kvm_pmd_cache);
  1278. }