e500_mmu.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved.
  4. *
  5. * Author: Yu Liu, yu.liu@freescale.com
  6. * Scott Wood, scottwood@freescale.com
  7. * Ashish Kalra, ashish.kalra@freescale.com
  8. * Varun Sethi, varun.sethi@freescale.com
  9. * Alexander Graf, agraf@suse.de
  10. *
  11. * Description:
  12. * This file is based on arch/powerpc/kvm/44x_tlb.c,
  13. * by Hollis Blanchard <hollisb@us.ibm.com>.
  14. */
  15. #include <linux/kernel.h>
  16. #include <linux/types.h>
  17. #include <linux/slab.h>
  18. #include <linux/string.h>
  19. #include <linux/kvm.h>
  20. #include <linux/kvm_host.h>
  21. #include <linux/highmem.h>
  22. #include <linux/log2.h>
  23. #include <linux/uaccess.h>
  24. #include <linux/sched.h>
  25. #include <linux/rwsem.h>
  26. #include <linux/vmalloc.h>
  27. #include <linux/hugetlb.h>
  28. #include <asm/kvm_ppc.h>
  29. #include "e500.h"
  30. #include "trace_booke.h"
  31. #include "timing.h"
  32. #include "e500_mmu_host.h"
  33. static inline unsigned int gtlb0_get_next_victim(
  34. struct kvmppc_vcpu_e500 *vcpu_e500)
  35. {
  36. unsigned int victim;
  37. victim = vcpu_e500->gtlb_nv[0]++;
  38. if (unlikely(vcpu_e500->gtlb_nv[0] >= vcpu_e500->gtlb_params[0].ways))
  39. vcpu_e500->gtlb_nv[0] = 0;
  40. return victim;
  41. }
  42. static int tlb0_set_base(gva_t addr, int sets, int ways)
  43. {
  44. int set_base;
  45. set_base = (addr >> PAGE_SHIFT) & (sets - 1);
  46. set_base *= ways;
  47. return set_base;
  48. }
  49. static int gtlb0_set_base(struct kvmppc_vcpu_e500 *vcpu_e500, gva_t addr)
  50. {
  51. return tlb0_set_base(addr, vcpu_e500->gtlb_params[0].sets,
  52. vcpu_e500->gtlb_params[0].ways);
  53. }
  54. static unsigned int get_tlb_esel(struct kvm_vcpu *vcpu, int tlbsel)
  55. {
  56. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  57. int esel = get_tlb_esel_bit(vcpu);
  58. if (tlbsel == 0) {
  59. esel &= vcpu_e500->gtlb_params[0].ways - 1;
  60. esel += gtlb0_set_base(vcpu_e500, vcpu->arch.shared->mas2);
  61. } else {
  62. esel &= vcpu_e500->gtlb_params[tlbsel].entries - 1;
  63. }
  64. return esel;
  65. }
  66. /* Search the guest TLB for a matching entry. */
  67. static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500,
  68. gva_t eaddr, int tlbsel, unsigned int pid, int as)
  69. {
  70. int size = vcpu_e500->gtlb_params[tlbsel].entries;
  71. unsigned int set_base, offset;
  72. int i;
  73. if (tlbsel == 0) {
  74. set_base = gtlb0_set_base(vcpu_e500, eaddr);
  75. size = vcpu_e500->gtlb_params[0].ways;
  76. } else {
  77. if (eaddr < vcpu_e500->tlb1_min_eaddr ||
  78. eaddr > vcpu_e500->tlb1_max_eaddr)
  79. return -1;
  80. set_base = 0;
  81. }
  82. offset = vcpu_e500->gtlb_offset[tlbsel];
  83. for (i = 0; i < size; i++) {
  84. struct kvm_book3e_206_tlb_entry *tlbe =
  85. &vcpu_e500->gtlb_arch[offset + set_base + i];
  86. unsigned int tid;
  87. if (eaddr < get_tlb_eaddr(tlbe))
  88. continue;
  89. if (eaddr > get_tlb_end(tlbe))
  90. continue;
  91. tid = get_tlb_tid(tlbe);
  92. if (tid && (tid != pid))
  93. continue;
  94. if (!get_tlb_v(tlbe))
  95. continue;
  96. if (get_tlb_ts(tlbe) != as && as != -1)
  97. continue;
  98. return set_base + i;
  99. }
  100. return -1;
  101. }
  102. static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
  103. gva_t eaddr, int as)
  104. {
  105. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  106. unsigned int victim, tsized;
  107. int tlbsel;
  108. /* since we only have two TLBs, only lower bit is used. */
  109. tlbsel = (vcpu->arch.shared->mas4 >> 28) & 0x1;
  110. victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0;
  111. tsized = (vcpu->arch.shared->mas4 >> 7) & 0x1f;
  112. vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim)
  113. | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  114. vcpu->arch.shared->mas1 = MAS1_VALID | (as ? MAS1_TS : 0)
  115. | MAS1_TID(get_tlbmiss_tid(vcpu))
  116. | MAS1_TSIZE(tsized);
  117. vcpu->arch.shared->mas2 = (eaddr & MAS2_EPN)
  118. | (vcpu->arch.shared->mas4 & MAS2_ATTRIB_MASK);
  119. vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3;
  120. vcpu->arch.shared->mas6 = (vcpu->arch.shared->mas6 & MAS6_SPID1)
  121. | (get_cur_pid(vcpu) << 16)
  122. | (as ? MAS6_SAS : 0);
  123. }
  124. static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500)
  125. {
  126. int size = vcpu_e500->gtlb_params[1].entries;
  127. unsigned int offset;
  128. gva_t eaddr;
  129. int i;
  130. vcpu_e500->tlb1_min_eaddr = ~0UL;
  131. vcpu_e500->tlb1_max_eaddr = 0;
  132. offset = vcpu_e500->gtlb_offset[1];
  133. for (i = 0; i < size; i++) {
  134. struct kvm_book3e_206_tlb_entry *tlbe =
  135. &vcpu_e500->gtlb_arch[offset + i];
  136. if (!get_tlb_v(tlbe))
  137. continue;
  138. eaddr = get_tlb_eaddr(tlbe);
  139. vcpu_e500->tlb1_min_eaddr =
  140. min(vcpu_e500->tlb1_min_eaddr, eaddr);
  141. eaddr = get_tlb_end(tlbe);
  142. vcpu_e500->tlb1_max_eaddr =
  143. max(vcpu_e500->tlb1_max_eaddr, eaddr);
  144. }
  145. }
  146. static int kvmppc_need_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500,
  147. struct kvm_book3e_206_tlb_entry *gtlbe)
  148. {
  149. unsigned long start, end, size;
  150. size = get_tlb_bytes(gtlbe);
  151. start = get_tlb_eaddr(gtlbe) & ~(size - 1);
  152. end = start + size - 1;
  153. return vcpu_e500->tlb1_min_eaddr == start ||
  154. vcpu_e500->tlb1_max_eaddr == end;
  155. }
  156. /* This function is supposed to be called for a adding a new valid tlb entry */
  157. static void kvmppc_set_tlb1map_range(struct kvm_vcpu *vcpu,
  158. struct kvm_book3e_206_tlb_entry *gtlbe)
  159. {
  160. unsigned long start, end, size;
  161. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  162. if (!get_tlb_v(gtlbe))
  163. return;
  164. size = get_tlb_bytes(gtlbe);
  165. start = get_tlb_eaddr(gtlbe) & ~(size - 1);
  166. end = start + size - 1;
  167. vcpu_e500->tlb1_min_eaddr = min(vcpu_e500->tlb1_min_eaddr, start);
  168. vcpu_e500->tlb1_max_eaddr = max(vcpu_e500->tlb1_max_eaddr, end);
  169. }
  170. static inline int kvmppc_e500_gtlbe_invalidate(
  171. struct kvmppc_vcpu_e500 *vcpu_e500,
  172. int tlbsel, int esel)
  173. {
  174. struct kvm_book3e_206_tlb_entry *gtlbe =
  175. get_entry(vcpu_e500, tlbsel, esel);
  176. if (unlikely(get_tlb_iprot(gtlbe)))
  177. return -1;
  178. if (tlbsel == 1 && kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe))
  179. kvmppc_recalc_tlb1map_range(vcpu_e500);
  180. gtlbe->mas1 = 0;
  181. return 0;
  182. }
  183. int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value)
  184. {
  185. int esel;
  186. if (value & MMUCSR0_TLB0FI)
  187. for (esel = 0; esel < vcpu_e500->gtlb_params[0].entries; esel++)
  188. kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel);
  189. if (value & MMUCSR0_TLB1FI)
  190. for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++)
  191. kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel);
  192. /* Invalidate all host shadow mappings */
  193. kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
  194. return EMULATE_DONE;
  195. }
  196. int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, gva_t ea)
  197. {
  198. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  199. unsigned int ia;
  200. int esel, tlbsel;
  201. ia = (ea >> 2) & 0x1;
  202. /* since we only have two TLBs, only lower bit is used. */
  203. tlbsel = (ea >> 3) & 0x1;
  204. if (ia) {
  205. /* invalidate all entries */
  206. for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries;
  207. esel++)
  208. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  209. } else {
  210. ea &= 0xfffff000;
  211. esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel,
  212. get_cur_pid(vcpu), -1);
  213. if (esel >= 0)
  214. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  215. }
  216. /* Invalidate all host shadow mappings */
  217. kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
  218. return EMULATE_DONE;
  219. }
  220. static void tlbilx_all(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel,
  221. int pid, int type)
  222. {
  223. struct kvm_book3e_206_tlb_entry *tlbe;
  224. int tid, esel;
  225. /* invalidate all entries */
  226. for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; esel++) {
  227. tlbe = get_entry(vcpu_e500, tlbsel, esel);
  228. tid = get_tlb_tid(tlbe);
  229. if (type == 0 || tid == pid) {
  230. inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
  231. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  232. }
  233. }
  234. }
  235. static void tlbilx_one(struct kvmppc_vcpu_e500 *vcpu_e500, int pid,
  236. gva_t ea)
  237. {
  238. int tlbsel, esel;
  239. for (tlbsel = 0; tlbsel < 2; tlbsel++) {
  240. esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, -1);
  241. if (esel >= 0) {
  242. inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
  243. kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
  244. break;
  245. }
  246. }
  247. }
  248. int kvmppc_e500_emul_tlbilx(struct kvm_vcpu *vcpu, int type, gva_t ea)
  249. {
  250. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  251. int pid = get_cur_spid(vcpu);
  252. if (type == 0 || type == 1) {
  253. tlbilx_all(vcpu_e500, 0, pid, type);
  254. tlbilx_all(vcpu_e500, 1, pid, type);
  255. } else if (type == 3) {
  256. tlbilx_one(vcpu_e500, pid, ea);
  257. }
  258. return EMULATE_DONE;
  259. }
  260. int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu)
  261. {
  262. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  263. int tlbsel, esel;
  264. struct kvm_book3e_206_tlb_entry *gtlbe;
  265. tlbsel = get_tlb_tlbsel(vcpu);
  266. esel = get_tlb_esel(vcpu, tlbsel);
  267. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  268. vcpu->arch.shared->mas0 &= ~MAS0_NV(~0);
  269. vcpu->arch.shared->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  270. vcpu->arch.shared->mas1 = gtlbe->mas1;
  271. vcpu->arch.shared->mas2 = gtlbe->mas2;
  272. vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;
  273. return EMULATE_DONE;
  274. }
  275. int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea)
  276. {
  277. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  278. int as = !!get_cur_sas(vcpu);
  279. unsigned int pid = get_cur_spid(vcpu);
  280. int esel, tlbsel;
  281. struct kvm_book3e_206_tlb_entry *gtlbe = NULL;
  282. for (tlbsel = 0; tlbsel < 2; tlbsel++) {
  283. esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as);
  284. if (esel >= 0) {
  285. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  286. break;
  287. }
  288. }
  289. if (gtlbe) {
  290. esel &= vcpu_e500->gtlb_params[tlbsel].ways - 1;
  291. vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel)
  292. | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  293. vcpu->arch.shared->mas1 = gtlbe->mas1;
  294. vcpu->arch.shared->mas2 = gtlbe->mas2;
  295. vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;
  296. } else {
  297. int victim;
  298. /* since we only have two TLBs, only lower bit is used. */
  299. tlbsel = vcpu->arch.shared->mas4 >> 28 & 0x1;
  300. victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0;
  301. vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel)
  302. | MAS0_ESEL(victim)
  303. | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
  304. vcpu->arch.shared->mas1 =
  305. (vcpu->arch.shared->mas6 & MAS6_SPID0)
  306. | ((vcpu->arch.shared->mas6 & MAS6_SAS) ? MAS1_TS : 0)
  307. | (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0));
  308. vcpu->arch.shared->mas2 &= MAS2_EPN;
  309. vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 &
  310. MAS2_ATTRIB_MASK;
  311. vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 |
  312. MAS3_U2 | MAS3_U3;
  313. }
  314. kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS);
  315. return EMULATE_DONE;
  316. }
  317. int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
  318. {
  319. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  320. struct kvm_book3e_206_tlb_entry *gtlbe;
  321. int tlbsel, esel;
  322. int recal = 0;
  323. int idx;
  324. tlbsel = get_tlb_tlbsel(vcpu);
  325. esel = get_tlb_esel(vcpu, tlbsel);
  326. gtlbe = get_entry(vcpu_e500, tlbsel, esel);
  327. if (get_tlb_v(gtlbe)) {
  328. inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
  329. if ((tlbsel == 1) &&
  330. kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe))
  331. recal = 1;
  332. }
  333. gtlbe->mas1 = vcpu->arch.shared->mas1;
  334. gtlbe->mas2 = vcpu->arch.shared->mas2;
  335. if (!(vcpu->arch.shared->msr & MSR_CM))
  336. gtlbe->mas2 &= 0xffffffffUL;
  337. gtlbe->mas7_3 = vcpu->arch.shared->mas7_3;
  338. trace_kvm_booke206_gtlb_write(vcpu->arch.shared->mas0, gtlbe->mas1,
  339. gtlbe->mas2, gtlbe->mas7_3);
  340. if (tlbsel == 1) {
  341. /*
  342. * If a valid tlb1 entry is overwritten then recalculate the
  343. * min/max TLB1 map address range otherwise no need to look
  344. * in tlb1 array.
  345. */
  346. if (recal)
  347. kvmppc_recalc_tlb1map_range(vcpu_e500);
  348. else
  349. kvmppc_set_tlb1map_range(vcpu, gtlbe);
  350. }
  351. idx = srcu_read_lock(&vcpu->kvm->srcu);
  352. /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
  353. if (tlbe_is_host_safe(vcpu, gtlbe)) {
  354. u64 eaddr = get_tlb_eaddr(gtlbe);
  355. u64 raddr = get_tlb_raddr(gtlbe);
  356. if (tlbsel == 0) {
  357. gtlbe->mas1 &= ~MAS1_TSIZE(~0);
  358. gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K);
  359. }
  360. /* Premap the faulting page */
  361. kvmppc_mmu_map(vcpu, eaddr, raddr, index_of(tlbsel, esel));
  362. }
  363. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  364. kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS);
  365. return EMULATE_DONE;
  366. }
  367. static int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu,
  368. gva_t eaddr, unsigned int pid, int as)
  369. {
  370. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  371. int esel, tlbsel;
  372. for (tlbsel = 0; tlbsel < 2; tlbsel++) {
  373. esel = kvmppc_e500_tlb_index(vcpu_e500, eaddr, tlbsel, pid, as);
  374. if (esel >= 0)
  375. return index_of(tlbsel, esel);
  376. }
  377. return -1;
  378. }
  379. /* 'linear_address' is actually an encoding of AS|PID|EADDR . */
  380. int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu,
  381. struct kvm_translation *tr)
  382. {
  383. int index;
  384. gva_t eaddr;
  385. u8 pid;
  386. u8 as;
  387. eaddr = tr->linear_address;
  388. pid = (tr->linear_address >> 32) & 0xff;
  389. as = (tr->linear_address >> 40) & 0x1;
  390. index = kvmppc_e500_tlb_search(vcpu, eaddr, pid, as);
  391. if (index < 0) {
  392. tr->valid = 0;
  393. return 0;
  394. }
  395. tr->physical_address = kvmppc_mmu_xlate(vcpu, index, eaddr);
  396. /* XXX what does "writeable" and "usermode" even mean? */
  397. tr->valid = 1;
  398. return 0;
  399. }
  400. int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
  401. {
  402. unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
  403. return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);
  404. }
  405. int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
  406. {
  407. unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);
  408. return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);
  409. }
  410. void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu)
  411. {
  412. unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
  413. kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.regs.nip, as);
  414. }
  415. void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu)
  416. {
  417. unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);
  418. kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as);
  419. }
  420. gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index,
  421. gva_t eaddr)
  422. {
  423. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  424. struct kvm_book3e_206_tlb_entry *gtlbe;
  425. u64 pgmask;
  426. gtlbe = get_entry(vcpu_e500, tlbsel_of(index), esel_of(index));
  427. pgmask = get_tlb_bytes(gtlbe) - 1;
  428. return get_tlb_raddr(gtlbe) | (eaddr & pgmask);
  429. }
  430. /*****************************************/
  431. static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500)
  432. {
  433. int i;
  434. kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
  435. kfree(vcpu_e500->g2h_tlb1_map);
  436. kfree(vcpu_e500->gtlb_priv[0]);
  437. kfree(vcpu_e500->gtlb_priv[1]);
  438. if (vcpu_e500->shared_tlb_pages) {
  439. vfree((void *)(round_down((uintptr_t)vcpu_e500->gtlb_arch,
  440. PAGE_SIZE)));
  441. for (i = 0; i < vcpu_e500->num_shared_tlb_pages; i++) {
  442. set_page_dirty_lock(vcpu_e500->shared_tlb_pages[i]);
  443. put_page(vcpu_e500->shared_tlb_pages[i]);
  444. }
  445. vcpu_e500->num_shared_tlb_pages = 0;
  446. kfree(vcpu_e500->shared_tlb_pages);
  447. vcpu_e500->shared_tlb_pages = NULL;
  448. } else {
  449. kfree(vcpu_e500->gtlb_arch);
  450. }
  451. vcpu_e500->gtlb_arch = NULL;
  452. }
  453. void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  454. {
  455. sregs->u.e.mas0 = vcpu->arch.shared->mas0;
  456. sregs->u.e.mas1 = vcpu->arch.shared->mas1;
  457. sregs->u.e.mas2 = vcpu->arch.shared->mas2;
  458. sregs->u.e.mas7_3 = vcpu->arch.shared->mas7_3;
  459. sregs->u.e.mas4 = vcpu->arch.shared->mas4;
  460. sregs->u.e.mas6 = vcpu->arch.shared->mas6;
  461. sregs->u.e.mmucfg = vcpu->arch.mmucfg;
  462. sregs->u.e.tlbcfg[0] = vcpu->arch.tlbcfg[0];
  463. sregs->u.e.tlbcfg[1] = vcpu->arch.tlbcfg[1];
  464. sregs->u.e.tlbcfg[2] = 0;
  465. sregs->u.e.tlbcfg[3] = 0;
  466. }
  467. int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  468. {
  469. if (sregs->u.e.features & KVM_SREGS_E_ARCH206_MMU) {
  470. vcpu->arch.shared->mas0 = sregs->u.e.mas0;
  471. vcpu->arch.shared->mas1 = sregs->u.e.mas1;
  472. vcpu->arch.shared->mas2 = sregs->u.e.mas2;
  473. vcpu->arch.shared->mas7_3 = sregs->u.e.mas7_3;
  474. vcpu->arch.shared->mas4 = sregs->u.e.mas4;
  475. vcpu->arch.shared->mas6 = sregs->u.e.mas6;
  476. }
  477. return 0;
  478. }
  479. int kvmppc_get_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id,
  480. union kvmppc_one_reg *val)
  481. {
  482. int r = 0;
  483. long int i;
  484. switch (id) {
  485. case KVM_REG_PPC_MAS0:
  486. *val = get_reg_val(id, vcpu->arch.shared->mas0);
  487. break;
  488. case KVM_REG_PPC_MAS1:
  489. *val = get_reg_val(id, vcpu->arch.shared->mas1);
  490. break;
  491. case KVM_REG_PPC_MAS2:
  492. *val = get_reg_val(id, vcpu->arch.shared->mas2);
  493. break;
  494. case KVM_REG_PPC_MAS7_3:
  495. *val = get_reg_val(id, vcpu->arch.shared->mas7_3);
  496. break;
  497. case KVM_REG_PPC_MAS4:
  498. *val = get_reg_val(id, vcpu->arch.shared->mas4);
  499. break;
  500. case KVM_REG_PPC_MAS6:
  501. *val = get_reg_val(id, vcpu->arch.shared->mas6);
  502. break;
  503. case KVM_REG_PPC_MMUCFG:
  504. *val = get_reg_val(id, vcpu->arch.mmucfg);
  505. break;
  506. case KVM_REG_PPC_EPTCFG:
  507. *val = get_reg_val(id, vcpu->arch.eptcfg);
  508. break;
  509. case KVM_REG_PPC_TLB0CFG:
  510. case KVM_REG_PPC_TLB1CFG:
  511. case KVM_REG_PPC_TLB2CFG:
  512. case KVM_REG_PPC_TLB3CFG:
  513. i = id - KVM_REG_PPC_TLB0CFG;
  514. *val = get_reg_val(id, vcpu->arch.tlbcfg[i]);
  515. break;
  516. case KVM_REG_PPC_TLB0PS:
  517. case KVM_REG_PPC_TLB1PS:
  518. case KVM_REG_PPC_TLB2PS:
  519. case KVM_REG_PPC_TLB3PS:
  520. i = id - KVM_REG_PPC_TLB0PS;
  521. *val = get_reg_val(id, vcpu->arch.tlbps[i]);
  522. break;
  523. default:
  524. r = -EINVAL;
  525. break;
  526. }
  527. return r;
  528. }
  529. int kvmppc_set_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id,
  530. union kvmppc_one_reg *val)
  531. {
  532. int r = 0;
  533. long int i;
  534. switch (id) {
  535. case KVM_REG_PPC_MAS0:
  536. vcpu->arch.shared->mas0 = set_reg_val(id, *val);
  537. break;
  538. case KVM_REG_PPC_MAS1:
  539. vcpu->arch.shared->mas1 = set_reg_val(id, *val);
  540. break;
  541. case KVM_REG_PPC_MAS2:
  542. vcpu->arch.shared->mas2 = set_reg_val(id, *val);
  543. break;
  544. case KVM_REG_PPC_MAS7_3:
  545. vcpu->arch.shared->mas7_3 = set_reg_val(id, *val);
  546. break;
  547. case KVM_REG_PPC_MAS4:
  548. vcpu->arch.shared->mas4 = set_reg_val(id, *val);
  549. break;
  550. case KVM_REG_PPC_MAS6:
  551. vcpu->arch.shared->mas6 = set_reg_val(id, *val);
  552. break;
  553. /* Only allow MMU registers to be set to the config supported by KVM */
  554. case KVM_REG_PPC_MMUCFG: {
  555. u32 reg = set_reg_val(id, *val);
  556. if (reg != vcpu->arch.mmucfg)
  557. r = -EINVAL;
  558. break;
  559. }
  560. case KVM_REG_PPC_EPTCFG: {
  561. u32 reg = set_reg_val(id, *val);
  562. if (reg != vcpu->arch.eptcfg)
  563. r = -EINVAL;
  564. break;
  565. }
  566. case KVM_REG_PPC_TLB0CFG:
  567. case KVM_REG_PPC_TLB1CFG:
  568. case KVM_REG_PPC_TLB2CFG:
  569. case KVM_REG_PPC_TLB3CFG: {
  570. /* MMU geometry (N_ENTRY/ASSOC) can be set only using SW_TLB */
  571. u32 reg = set_reg_val(id, *val);
  572. i = id - KVM_REG_PPC_TLB0CFG;
  573. if (reg != vcpu->arch.tlbcfg[i])
  574. r = -EINVAL;
  575. break;
  576. }
  577. case KVM_REG_PPC_TLB0PS:
  578. case KVM_REG_PPC_TLB1PS:
  579. case KVM_REG_PPC_TLB2PS:
  580. case KVM_REG_PPC_TLB3PS: {
  581. u32 reg = set_reg_val(id, *val);
  582. i = id - KVM_REG_PPC_TLB0PS;
  583. if (reg != vcpu->arch.tlbps[i])
  584. r = -EINVAL;
  585. break;
  586. }
  587. default:
  588. r = -EINVAL;
  589. break;
  590. }
  591. return r;
  592. }
  593. static int vcpu_mmu_geometry_update(struct kvm_vcpu *vcpu,
  594. struct kvm_book3e_206_tlb_params *params)
  595. {
  596. vcpu->arch.tlbcfg[0] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  597. if (params->tlb_sizes[0] <= 2048)
  598. vcpu->arch.tlbcfg[0] |= params->tlb_sizes[0];
  599. vcpu->arch.tlbcfg[0] |= params->tlb_ways[0] << TLBnCFG_ASSOC_SHIFT;
  600. vcpu->arch.tlbcfg[1] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  601. vcpu->arch.tlbcfg[1] |= params->tlb_sizes[1];
  602. vcpu->arch.tlbcfg[1] |= params->tlb_ways[1] << TLBnCFG_ASSOC_SHIFT;
  603. return 0;
  604. }
  605. int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,
  606. struct kvm_config_tlb *cfg)
  607. {
  608. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  609. struct kvm_book3e_206_tlb_params params;
  610. char *virt;
  611. struct page **pages;
  612. struct tlbe_priv *privs[2] = {};
  613. u64 *g2h_bitmap;
  614. size_t array_len;
  615. u32 sets;
  616. int num_pages, ret, i;
  617. if (cfg->mmu_type != KVM_MMU_FSL_BOOKE_NOHV)
  618. return -EINVAL;
  619. if (copy_from_user(&params, (void __user *)(uintptr_t)cfg->params,
  620. sizeof(params)))
  621. return -EFAULT;
  622. if (params.tlb_sizes[1] > 64)
  623. return -EINVAL;
  624. if (params.tlb_ways[1] != params.tlb_sizes[1])
  625. return -EINVAL;
  626. if (params.tlb_sizes[2] != 0 || params.tlb_sizes[3] != 0)
  627. return -EINVAL;
  628. if (params.tlb_ways[2] != 0 || params.tlb_ways[3] != 0)
  629. return -EINVAL;
  630. if (!is_power_of_2(params.tlb_ways[0]))
  631. return -EINVAL;
  632. sets = params.tlb_sizes[0] >> ilog2(params.tlb_ways[0]);
  633. if (!is_power_of_2(sets))
  634. return -EINVAL;
  635. array_len = params.tlb_sizes[0] + params.tlb_sizes[1];
  636. array_len *= sizeof(struct kvm_book3e_206_tlb_entry);
  637. if (cfg->array_len < array_len)
  638. return -EINVAL;
  639. num_pages = DIV_ROUND_UP(cfg->array + array_len - 1, PAGE_SIZE) -
  640. cfg->array / PAGE_SIZE;
  641. pages = kmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL);
  642. if (!pages)
  643. return -ENOMEM;
  644. ret = get_user_pages_fast(cfg->array, num_pages, FOLL_WRITE, pages);
  645. if (ret < 0)
  646. goto free_pages;
  647. if (ret != num_pages) {
  648. num_pages = ret;
  649. ret = -EFAULT;
  650. goto put_pages;
  651. }
  652. virt = vmap(pages, num_pages, VM_MAP, PAGE_KERNEL);
  653. if (!virt) {
  654. ret = -ENOMEM;
  655. goto put_pages;
  656. }
  657. privs[0] = kcalloc(params.tlb_sizes[0], sizeof(*privs[0]), GFP_KERNEL);
  658. if (!privs[0]) {
  659. ret = -ENOMEM;
  660. goto put_pages;
  661. }
  662. privs[1] = kcalloc(params.tlb_sizes[1], sizeof(*privs[1]), GFP_KERNEL);
  663. if (!privs[1]) {
  664. ret = -ENOMEM;
  665. goto free_privs_first;
  666. }
  667. g2h_bitmap = kcalloc(params.tlb_sizes[1],
  668. sizeof(*g2h_bitmap),
  669. GFP_KERNEL);
  670. if (!g2h_bitmap) {
  671. ret = -ENOMEM;
  672. goto free_privs_second;
  673. }
  674. free_gtlb(vcpu_e500);
  675. vcpu_e500->gtlb_priv[0] = privs[0];
  676. vcpu_e500->gtlb_priv[1] = privs[1];
  677. vcpu_e500->g2h_tlb1_map = g2h_bitmap;
  678. vcpu_e500->gtlb_arch = (struct kvm_book3e_206_tlb_entry *)
  679. (virt + (cfg->array & (PAGE_SIZE - 1)));
  680. vcpu_e500->gtlb_params[0].entries = params.tlb_sizes[0];
  681. vcpu_e500->gtlb_params[1].entries = params.tlb_sizes[1];
  682. vcpu_e500->gtlb_offset[0] = 0;
  683. vcpu_e500->gtlb_offset[1] = params.tlb_sizes[0];
  684. /* Update vcpu's MMU geometry based on SW_TLB input */
  685. vcpu_mmu_geometry_update(vcpu, &params);
  686. vcpu_e500->shared_tlb_pages = pages;
  687. vcpu_e500->num_shared_tlb_pages = num_pages;
  688. vcpu_e500->gtlb_params[0].ways = params.tlb_ways[0];
  689. vcpu_e500->gtlb_params[0].sets = sets;
  690. vcpu_e500->gtlb_params[1].ways = params.tlb_sizes[1];
  691. vcpu_e500->gtlb_params[1].sets = 1;
  692. kvmppc_recalc_tlb1map_range(vcpu_e500);
  693. return 0;
  694. free_privs_second:
  695. kfree(privs[1]);
  696. free_privs_first:
  697. kfree(privs[0]);
  698. put_pages:
  699. for (i = 0; i < num_pages; i++)
  700. put_page(pages[i]);
  701. free_pages:
  702. kfree(pages);
  703. return ret;
  704. }
  705. int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu,
  706. struct kvm_dirty_tlb *dirty)
  707. {
  708. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  709. kvmppc_recalc_tlb1map_range(vcpu_e500);
  710. kvmppc_core_flush_tlb(vcpu);
  711. return 0;
  712. }
  713. /* Vcpu's MMU default configuration */
  714. static int vcpu_mmu_init(struct kvm_vcpu *vcpu,
  715. struct kvmppc_e500_tlb_params *params)
  716. {
  717. /* Initialize RASIZE, PIDSIZE, NTLBS and MAVN fields with host values*/
  718. vcpu->arch.mmucfg = mfspr(SPRN_MMUCFG) & ~MMUCFG_LPIDSIZE;
  719. /* Initialize TLBnCFG fields with host values and SW_TLB geometry*/
  720. vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) &
  721. ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  722. vcpu->arch.tlbcfg[0] |= params[0].entries;
  723. vcpu->arch.tlbcfg[0] |= params[0].ways << TLBnCFG_ASSOC_SHIFT;
  724. vcpu->arch.tlbcfg[1] = mfspr(SPRN_TLB1CFG) &
  725. ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
  726. vcpu->arch.tlbcfg[1] |= params[1].entries;
  727. vcpu->arch.tlbcfg[1] |= params[1].ways << TLBnCFG_ASSOC_SHIFT;
  728. if (has_feature(vcpu, VCPU_FTR_MMU_V2)) {
  729. vcpu->arch.tlbps[0] = mfspr(SPRN_TLB0PS);
  730. vcpu->arch.tlbps[1] = mfspr(SPRN_TLB1PS);
  731. vcpu->arch.mmucfg &= ~MMUCFG_LRAT;
  732. /* Guest mmu emulation currently doesn't handle E.PT */
  733. vcpu->arch.eptcfg = 0;
  734. vcpu->arch.tlbcfg[0] &= ~TLBnCFG_PT;
  735. vcpu->arch.tlbcfg[1] &= ~TLBnCFG_IND;
  736. }
  737. return 0;
  738. }
  739. int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
  740. {
  741. struct kvm_vcpu *vcpu = &vcpu_e500->vcpu;
  742. if (e500_mmu_host_init(vcpu_e500))
  743. goto free_vcpu;
  744. vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE;
  745. vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE;
  746. vcpu_e500->gtlb_params[0].ways = KVM_E500_TLB0_WAY_NUM;
  747. vcpu_e500->gtlb_params[0].sets =
  748. KVM_E500_TLB0_SIZE / KVM_E500_TLB0_WAY_NUM;
  749. vcpu_e500->gtlb_params[1].ways = KVM_E500_TLB1_SIZE;
  750. vcpu_e500->gtlb_params[1].sets = 1;
  751. vcpu_e500->gtlb_arch = kmalloc_array(KVM_E500_TLB0_SIZE +
  752. KVM_E500_TLB1_SIZE,
  753. sizeof(*vcpu_e500->gtlb_arch),
  754. GFP_KERNEL);
  755. if (!vcpu_e500->gtlb_arch)
  756. return -ENOMEM;
  757. vcpu_e500->gtlb_offset[0] = 0;
  758. vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE;
  759. vcpu_e500->gtlb_priv[0] = kcalloc(vcpu_e500->gtlb_params[0].entries,
  760. sizeof(struct tlbe_ref),
  761. GFP_KERNEL);
  762. if (!vcpu_e500->gtlb_priv[0])
  763. goto free_vcpu;
  764. vcpu_e500->gtlb_priv[1] = kcalloc(vcpu_e500->gtlb_params[1].entries,
  765. sizeof(struct tlbe_ref),
  766. GFP_KERNEL);
  767. if (!vcpu_e500->gtlb_priv[1])
  768. goto free_vcpu;
  769. vcpu_e500->g2h_tlb1_map = kcalloc(vcpu_e500->gtlb_params[1].entries,
  770. sizeof(*vcpu_e500->g2h_tlb1_map),
  771. GFP_KERNEL);
  772. if (!vcpu_e500->g2h_tlb1_map)
  773. goto free_vcpu;
  774. vcpu_mmu_init(vcpu, vcpu_e500->gtlb_params);
  775. kvmppc_recalc_tlb1map_range(vcpu_e500);
  776. return 0;
  777. free_vcpu:
  778. free_gtlb(vcpu_e500);
  779. return -1;
  780. }
  781. void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
  782. {
  783. free_gtlb(vcpu_e500);
  784. e500_mmu_host_uninit(vcpu_e500);
  785. }