e500.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
  4. *
  5. * Author: Yu Liu, <yu.liu@freescale.com>
  6. *
  7. * Description:
  8. * This file is derived from arch/powerpc/kvm/44x.c,
  9. * by Hollis Blanchard <hollisb@us.ibm.com>.
  10. */
  11. #include <linux/kvm_host.h>
  12. #include <linux/slab.h>
  13. #include <linux/err.h>
  14. #include <linux/export.h>
  15. #include <linux/module.h>
  16. #include <linux/miscdevice.h>
  17. #include <asm/reg.h>
  18. #include <asm/cputable.h>
  19. #include <asm/kvm_ppc.h>
  20. #include "../mm/mmu_decl.h"
  21. #include "booke.h"
  22. #include "e500.h"
  23. struct id {
  24. unsigned long val;
  25. struct id **pentry;
  26. };
  27. #define NUM_TIDS 256
  28. /*
  29. * This table provide mappings from:
  30. * (guestAS,guestTID,guestPR) --> ID of physical cpu
  31. * guestAS [0..1]
  32. * guestTID [0..255]
  33. * guestPR [0..1]
  34. * ID [1..255]
  35. * Each vcpu keeps one vcpu_id_table.
  36. */
  37. struct vcpu_id_table {
  38. struct id id[2][NUM_TIDS][2];
  39. };
  40. /*
  41. * This table provide reversed mappings of vcpu_id_table:
  42. * ID --> address of vcpu_id_table item.
  43. * Each physical core has one pcpu_id_table.
  44. */
  45. struct pcpu_id_table {
  46. struct id *entry[NUM_TIDS];
  47. };
  48. static DEFINE_PER_CPU(struct pcpu_id_table, pcpu_sids);
  49. /* This variable keeps last used shadow ID on local core.
  50. * The valid range of shadow ID is [1..255] */
  51. static DEFINE_PER_CPU(unsigned long, pcpu_last_used_sid);
  52. /*
  53. * Allocate a free shadow id and setup a valid sid mapping in given entry.
  54. * A mapping is only valid when vcpu_id_table and pcpu_id_table are match.
  55. *
  56. * The caller must have preemption disabled, and keep it that way until
  57. * it has finished with the returned shadow id (either written into the
  58. * TLB or arch.shadow_pid, or discarded).
  59. */
  60. static inline int local_sid_setup_one(struct id *entry)
  61. {
  62. unsigned long sid;
  63. int ret = -1;
  64. sid = __this_cpu_inc_return(pcpu_last_used_sid);
  65. if (sid < NUM_TIDS) {
  66. __this_cpu_write(pcpu_sids.entry[sid], entry);
  67. entry->val = sid;
  68. entry->pentry = this_cpu_ptr(&pcpu_sids.entry[sid]);
  69. ret = sid;
  70. }
  71. /*
  72. * If sid == NUM_TIDS, we've run out of sids. We return -1, and
  73. * the caller will invalidate everything and start over.
  74. *
  75. * sid > NUM_TIDS indicates a race, which we disable preemption to
  76. * avoid.
  77. */
  78. WARN_ON(sid > NUM_TIDS);
  79. return ret;
  80. }
  81. /*
  82. * Check if given entry contain a valid shadow id mapping.
  83. * An ID mapping is considered valid only if
  84. * both vcpu and pcpu know this mapping.
  85. *
  86. * The caller must have preemption disabled, and keep it that way until
  87. * it has finished with the returned shadow id (either written into the
  88. * TLB or arch.shadow_pid, or discarded).
  89. */
  90. static inline int local_sid_lookup(struct id *entry)
  91. {
  92. if (entry && entry->val != 0 &&
  93. __this_cpu_read(pcpu_sids.entry[entry->val]) == entry &&
  94. entry->pentry == this_cpu_ptr(&pcpu_sids.entry[entry->val]))
  95. return entry->val;
  96. return -1;
  97. }
  98. /* Invalidate all id mappings on local core -- call with preempt disabled */
  99. static inline void local_sid_destroy_all(void)
  100. {
  101. __this_cpu_write(pcpu_last_used_sid, 0);
  102. memset(this_cpu_ptr(&pcpu_sids), 0, sizeof(pcpu_sids));
  103. }
  104. static void *kvmppc_e500_id_table_alloc(struct kvmppc_vcpu_e500 *vcpu_e500)
  105. {
  106. vcpu_e500->idt = kzalloc(sizeof(struct vcpu_id_table), GFP_KERNEL);
  107. return vcpu_e500->idt;
  108. }
  109. static void kvmppc_e500_id_table_free(struct kvmppc_vcpu_e500 *vcpu_e500)
  110. {
  111. kfree(vcpu_e500->idt);
  112. vcpu_e500->idt = NULL;
  113. }
  114. /* Map guest pid to shadow.
  115. * We use PID to keep shadow of current guest non-zero PID,
  116. * and use PID1 to keep shadow of guest zero PID.
  117. * So that guest tlbe with TID=0 can be accessed at any time */
  118. static void kvmppc_e500_recalc_shadow_pid(struct kvmppc_vcpu_e500 *vcpu_e500)
  119. {
  120. preempt_disable();
  121. vcpu_e500->vcpu.arch.shadow_pid = kvmppc_e500_get_sid(vcpu_e500,
  122. get_cur_as(&vcpu_e500->vcpu),
  123. get_cur_pid(&vcpu_e500->vcpu),
  124. get_cur_pr(&vcpu_e500->vcpu), 1);
  125. vcpu_e500->vcpu.arch.shadow_pid1 = kvmppc_e500_get_sid(vcpu_e500,
  126. get_cur_as(&vcpu_e500->vcpu), 0,
  127. get_cur_pr(&vcpu_e500->vcpu), 1);
  128. preempt_enable();
  129. }
  130. /* Invalidate all mappings on vcpu */
  131. static void kvmppc_e500_id_table_reset_all(struct kvmppc_vcpu_e500 *vcpu_e500)
  132. {
  133. memset(vcpu_e500->idt, 0, sizeof(struct vcpu_id_table));
  134. /* Update shadow pid when mappings are changed */
  135. kvmppc_e500_recalc_shadow_pid(vcpu_e500);
  136. }
  137. /* Invalidate one ID mapping on vcpu */
  138. static inline void kvmppc_e500_id_table_reset_one(
  139. struct kvmppc_vcpu_e500 *vcpu_e500,
  140. int as, int pid, int pr)
  141. {
  142. struct vcpu_id_table *idt = vcpu_e500->idt;
  143. BUG_ON(as >= 2);
  144. BUG_ON(pid >= NUM_TIDS);
  145. BUG_ON(pr >= 2);
  146. idt->id[as][pid][pr].val = 0;
  147. idt->id[as][pid][pr].pentry = NULL;
  148. /* Update shadow pid when mappings are changed */
  149. kvmppc_e500_recalc_shadow_pid(vcpu_e500);
  150. }
  151. /*
  152. * Map guest (vcpu,AS,ID,PR) to physical core shadow id.
  153. * This function first lookup if a valid mapping exists,
  154. * if not, then creates a new one.
  155. *
  156. * The caller must have preemption disabled, and keep it that way until
  157. * it has finished with the returned shadow id (either written into the
  158. * TLB or arch.shadow_pid, or discarded).
  159. */
  160. unsigned int kvmppc_e500_get_sid(struct kvmppc_vcpu_e500 *vcpu_e500,
  161. unsigned int as, unsigned int gid,
  162. unsigned int pr, int avoid_recursion)
  163. {
  164. struct vcpu_id_table *idt = vcpu_e500->idt;
  165. int sid;
  166. BUG_ON(as >= 2);
  167. BUG_ON(gid >= NUM_TIDS);
  168. BUG_ON(pr >= 2);
  169. sid = local_sid_lookup(&idt->id[as][gid][pr]);
  170. while (sid <= 0) {
  171. /* No mapping yet */
  172. sid = local_sid_setup_one(&idt->id[as][gid][pr]);
  173. if (sid <= 0) {
  174. _tlbil_all();
  175. local_sid_destroy_all();
  176. }
  177. /* Update shadow pid when mappings are changed */
  178. if (!avoid_recursion)
  179. kvmppc_e500_recalc_shadow_pid(vcpu_e500);
  180. }
  181. return sid;
  182. }
  183. unsigned int kvmppc_e500_get_tlb_stid(struct kvm_vcpu *vcpu,
  184. struct kvm_book3e_206_tlb_entry *gtlbe)
  185. {
  186. return kvmppc_e500_get_sid(to_e500(vcpu), get_tlb_ts(gtlbe),
  187. get_tlb_tid(gtlbe), get_cur_pr(vcpu), 0);
  188. }
  189. void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid)
  190. {
  191. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  192. if (vcpu->arch.pid != pid) {
  193. vcpu_e500->pid[0] = vcpu->arch.pid = pid;
  194. kvmppc_e500_recalc_shadow_pid(vcpu_e500);
  195. }
  196. }
  197. /* gtlbe must not be mapped by more than one host tlbe */
  198. void kvmppc_e500_tlbil_one(struct kvmppc_vcpu_e500 *vcpu_e500,
  199. struct kvm_book3e_206_tlb_entry *gtlbe)
  200. {
  201. struct vcpu_id_table *idt = vcpu_e500->idt;
  202. unsigned int pr, tid, ts;
  203. int pid;
  204. u32 val, eaddr;
  205. unsigned long flags;
  206. ts = get_tlb_ts(gtlbe);
  207. tid = get_tlb_tid(gtlbe);
  208. preempt_disable();
  209. /* One guest ID may be mapped to two shadow IDs */
  210. for (pr = 0; pr < 2; pr++) {
  211. /*
  212. * The shadow PID can have a valid mapping on at most one
  213. * host CPU. In the common case, it will be valid on this
  214. * CPU, in which case we do a local invalidation of the
  215. * specific address.
  216. *
  217. * If the shadow PID is not valid on the current host CPU,
  218. * we invalidate the entire shadow PID.
  219. */
  220. pid = local_sid_lookup(&idt->id[ts][tid][pr]);
  221. if (pid <= 0) {
  222. kvmppc_e500_id_table_reset_one(vcpu_e500, ts, tid, pr);
  223. continue;
  224. }
  225. /*
  226. * The guest is invalidating a 4K entry which is in a PID
  227. * that has a valid shadow mapping on this host CPU. We
  228. * search host TLB to invalidate it's shadow TLB entry,
  229. * similar to __tlbil_va except that we need to look in AS1.
  230. */
  231. val = (pid << MAS6_SPID_SHIFT) | MAS6_SAS;
  232. eaddr = get_tlb_eaddr(gtlbe);
  233. local_irq_save(flags);
  234. mtspr(SPRN_MAS6, val);
  235. asm volatile("tlbsx 0, %[eaddr]" : : [eaddr] "r" (eaddr));
  236. val = mfspr(SPRN_MAS1);
  237. if (val & MAS1_VALID) {
  238. mtspr(SPRN_MAS1, val & ~MAS1_VALID);
  239. asm volatile("tlbwe");
  240. }
  241. local_irq_restore(flags);
  242. }
  243. preempt_enable();
  244. }
  245. void kvmppc_e500_tlbil_all(struct kvmppc_vcpu_e500 *vcpu_e500)
  246. {
  247. kvmppc_e500_id_table_reset_all(vcpu_e500);
  248. }
  249. void kvmppc_mmu_msr_notify(struct kvm_vcpu *vcpu, u32 old_msr)
  250. {
  251. /* Recalc shadow pid since MSR changes */
  252. kvmppc_e500_recalc_shadow_pid(to_e500(vcpu));
  253. }
  254. static void kvmppc_core_vcpu_load_e500(struct kvm_vcpu *vcpu, int cpu)
  255. {
  256. kvmppc_booke_vcpu_load(vcpu, cpu);
  257. /* Shadow PID may be expired on local core */
  258. kvmppc_e500_recalc_shadow_pid(to_e500(vcpu));
  259. }
  260. static void kvmppc_core_vcpu_put_e500(struct kvm_vcpu *vcpu)
  261. {
  262. #ifdef CONFIG_SPE
  263. if (vcpu->arch.shadow_msr & MSR_SPE)
  264. kvmppc_vcpu_disable_spe(vcpu);
  265. #endif
  266. kvmppc_booke_vcpu_put(vcpu);
  267. }
  268. int kvmppc_core_check_processor_compat(void)
  269. {
  270. int r;
  271. if (strcmp(cur_cpu_spec->cpu_name, "e500v2") == 0)
  272. r = 0;
  273. else
  274. r = -ENOTSUPP;
  275. return r;
  276. }
  277. static void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *vcpu_e500)
  278. {
  279. struct kvm_book3e_206_tlb_entry *tlbe;
  280. /* Insert large initial mapping for guest. */
  281. tlbe = get_entry(vcpu_e500, 1, 0);
  282. tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_256M);
  283. tlbe->mas2 = 0;
  284. tlbe->mas7_3 = E500_TLB_SUPER_PERM_MASK;
  285. /* 4K map for serial output. Used by kernel wrapper. */
  286. tlbe = get_entry(vcpu_e500, 1, 1);
  287. tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_4K);
  288. tlbe->mas2 = (0xe0004500 & 0xFFFFF000) | MAS2_I | MAS2_G;
  289. tlbe->mas7_3 = (0xe0004500 & 0xFFFFF000) | E500_TLB_SUPER_PERM_MASK;
  290. }
  291. int kvmppc_core_vcpu_setup(struct kvm_vcpu *vcpu)
  292. {
  293. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  294. kvmppc_e500_tlb_setup(vcpu_e500);
  295. /* Registers init */
  296. vcpu->arch.pvr = mfspr(SPRN_PVR);
  297. vcpu_e500->svr = mfspr(SPRN_SVR);
  298. vcpu->arch.cpu_type = KVM_CPU_E500V2;
  299. return 0;
  300. }
  301. static int kvmppc_core_get_sregs_e500(struct kvm_vcpu *vcpu,
  302. struct kvm_sregs *sregs)
  303. {
  304. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  305. sregs->u.e.features |= KVM_SREGS_E_ARCH206_MMU | KVM_SREGS_E_SPE |
  306. KVM_SREGS_E_PM;
  307. sregs->u.e.impl_id = KVM_SREGS_E_IMPL_FSL;
  308. sregs->u.e.impl.fsl.features = 0;
  309. sregs->u.e.impl.fsl.svr = vcpu_e500->svr;
  310. sregs->u.e.impl.fsl.hid0 = vcpu_e500->hid0;
  311. sregs->u.e.impl.fsl.mcar = vcpu_e500->mcar;
  312. sregs->u.e.ivor_high[0] = vcpu->arch.ivor[BOOKE_IRQPRIO_SPE_UNAVAIL];
  313. sregs->u.e.ivor_high[1] = vcpu->arch.ivor[BOOKE_IRQPRIO_SPE_FP_DATA];
  314. sregs->u.e.ivor_high[2] = vcpu->arch.ivor[BOOKE_IRQPRIO_SPE_FP_ROUND];
  315. sregs->u.e.ivor_high[3] =
  316. vcpu->arch.ivor[BOOKE_IRQPRIO_PERFORMANCE_MONITOR];
  317. kvmppc_get_sregs_ivor(vcpu, sregs);
  318. kvmppc_get_sregs_e500_tlb(vcpu, sregs);
  319. return 0;
  320. }
  321. static int kvmppc_core_set_sregs_e500(struct kvm_vcpu *vcpu,
  322. struct kvm_sregs *sregs)
  323. {
  324. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  325. int ret;
  326. if (sregs->u.e.impl_id == KVM_SREGS_E_IMPL_FSL) {
  327. vcpu_e500->svr = sregs->u.e.impl.fsl.svr;
  328. vcpu_e500->hid0 = sregs->u.e.impl.fsl.hid0;
  329. vcpu_e500->mcar = sregs->u.e.impl.fsl.mcar;
  330. }
  331. ret = kvmppc_set_sregs_e500_tlb(vcpu, sregs);
  332. if (ret < 0)
  333. return ret;
  334. if (!(sregs->u.e.features & KVM_SREGS_E_IVOR))
  335. return 0;
  336. if (sregs->u.e.features & KVM_SREGS_E_SPE) {
  337. vcpu->arch.ivor[BOOKE_IRQPRIO_SPE_UNAVAIL] =
  338. sregs->u.e.ivor_high[0];
  339. vcpu->arch.ivor[BOOKE_IRQPRIO_SPE_FP_DATA] =
  340. sregs->u.e.ivor_high[1];
  341. vcpu->arch.ivor[BOOKE_IRQPRIO_SPE_FP_ROUND] =
  342. sregs->u.e.ivor_high[2];
  343. }
  344. if (sregs->u.e.features & KVM_SREGS_E_PM) {
  345. vcpu->arch.ivor[BOOKE_IRQPRIO_PERFORMANCE_MONITOR] =
  346. sregs->u.e.ivor_high[3];
  347. }
  348. return kvmppc_set_sregs_ivor(vcpu, sregs);
  349. }
  350. static int kvmppc_get_one_reg_e500(struct kvm_vcpu *vcpu, u64 id,
  351. union kvmppc_one_reg *val)
  352. {
  353. int r = kvmppc_get_one_reg_e500_tlb(vcpu, id, val);
  354. return r;
  355. }
  356. static int kvmppc_set_one_reg_e500(struct kvm_vcpu *vcpu, u64 id,
  357. union kvmppc_one_reg *val)
  358. {
  359. int r = kvmppc_get_one_reg_e500_tlb(vcpu, id, val);
  360. return r;
  361. }
  362. static int kvmppc_core_vcpu_create_e500(struct kvm_vcpu *vcpu)
  363. {
  364. struct kvmppc_vcpu_e500 *vcpu_e500;
  365. int err;
  366. BUILD_BUG_ON(offsetof(struct kvmppc_vcpu_e500, vcpu) != 0);
  367. vcpu_e500 = to_e500(vcpu);
  368. if (kvmppc_e500_id_table_alloc(vcpu_e500) == NULL)
  369. return -ENOMEM;
  370. err = kvmppc_e500_tlb_init(vcpu_e500);
  371. if (err)
  372. goto uninit_id;
  373. vcpu->arch.shared = (void*)__get_free_page(GFP_KERNEL|__GFP_ZERO);
  374. if (!vcpu->arch.shared) {
  375. err = -ENOMEM;
  376. goto uninit_tlb;
  377. }
  378. return 0;
  379. uninit_tlb:
  380. kvmppc_e500_tlb_uninit(vcpu_e500);
  381. uninit_id:
  382. kvmppc_e500_id_table_free(vcpu_e500);
  383. return err;
  384. }
  385. static void kvmppc_core_vcpu_free_e500(struct kvm_vcpu *vcpu)
  386. {
  387. struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
  388. free_page((unsigned long)vcpu->arch.shared);
  389. kvmppc_e500_tlb_uninit(vcpu_e500);
  390. kvmppc_e500_id_table_free(vcpu_e500);
  391. }
  392. static int kvmppc_core_init_vm_e500(struct kvm *kvm)
  393. {
  394. return 0;
  395. }
  396. static void kvmppc_core_destroy_vm_e500(struct kvm *kvm)
  397. {
  398. }
  399. static struct kvmppc_ops kvm_ops_e500 = {
  400. .get_sregs = kvmppc_core_get_sregs_e500,
  401. .set_sregs = kvmppc_core_set_sregs_e500,
  402. .get_one_reg = kvmppc_get_one_reg_e500,
  403. .set_one_reg = kvmppc_set_one_reg_e500,
  404. .vcpu_load = kvmppc_core_vcpu_load_e500,
  405. .vcpu_put = kvmppc_core_vcpu_put_e500,
  406. .vcpu_create = kvmppc_core_vcpu_create_e500,
  407. .vcpu_free = kvmppc_core_vcpu_free_e500,
  408. .init_vm = kvmppc_core_init_vm_e500,
  409. .destroy_vm = kvmppc_core_destroy_vm_e500,
  410. .emulate_op = kvmppc_core_emulate_op_e500,
  411. .emulate_mtspr = kvmppc_core_emulate_mtspr_e500,
  412. .emulate_mfspr = kvmppc_core_emulate_mfspr_e500,
  413. };
  414. static int __init kvmppc_e500_init(void)
  415. {
  416. int r, i;
  417. unsigned long ivor[3];
  418. /* Process remaining handlers above the generic first 16 */
  419. unsigned long *handler = &kvmppc_booke_handler_addr[16];
  420. unsigned long handler_len;
  421. unsigned long max_ivor = 0;
  422. r = kvmppc_core_check_processor_compat();
  423. if (r)
  424. goto err_out;
  425. r = kvmppc_booke_init();
  426. if (r)
  427. goto err_out;
  428. /* copy extra E500 exception handlers */
  429. ivor[0] = mfspr(SPRN_IVOR32);
  430. ivor[1] = mfspr(SPRN_IVOR33);
  431. ivor[2] = mfspr(SPRN_IVOR34);
  432. for (i = 0; i < 3; i++) {
  433. if (ivor[i] > ivor[max_ivor])
  434. max_ivor = i;
  435. handler_len = handler[i + 1] - handler[i];
  436. memcpy((void *)kvmppc_booke_handlers + ivor[i],
  437. (void *)handler[i], handler_len);
  438. }
  439. handler_len = handler[max_ivor + 1] - handler[max_ivor];
  440. flush_icache_range(kvmppc_booke_handlers, kvmppc_booke_handlers +
  441. ivor[max_ivor] + handler_len);
  442. r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_e500), 0, THIS_MODULE);
  443. if (r)
  444. goto err_out;
  445. kvm_ops_e500.owner = THIS_MODULE;
  446. kvmppc_pr_ops = &kvm_ops_e500;
  447. err_out:
  448. return r;
  449. }
  450. static void __exit kvmppc_e500_exit(void)
  451. {
  452. kvmppc_pr_ops = NULL;
  453. kvmppc_booke_exit();
  454. }
  455. module_init(kvmppc_e500_init);
  456. module_exit(kvmppc_e500_exit);
  457. MODULE_ALIAS_MISCDEV(KVM_MINOR);
  458. MODULE_ALIAS("devname:kvm");