book3s_hv_builtin.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  4. */
  5. #include <linux/cpu.h>
  6. #include <linux/kvm_host.h>
  7. #include <linux/preempt.h>
  8. #include <linux/export.h>
  9. #include <linux/sched.h>
  10. #include <linux/spinlock.h>
  11. #include <linux/init.h>
  12. #include <linux/memblock.h>
  13. #include <linux/sizes.h>
  14. #include <linux/cma.h>
  15. #include <linux/bitops.h>
  16. #include <asm/asm-prototypes.h>
  17. #include <asm/cputable.h>
  18. #include <asm/kvm_ppc.h>
  19. #include <asm/kvm_book3s.h>
  20. #include <asm/archrandom.h>
  21. #include <asm/xics.h>
  22. #include <asm/xive.h>
  23. #include <asm/dbell.h>
  24. #include <asm/cputhreads.h>
  25. #include <asm/io.h>
  26. #include <asm/opal.h>
  27. #include <asm/smp.h>
  28. #define KVM_CMA_CHUNK_ORDER 18
  29. #include "book3s_xics.h"
  30. #include "book3s_xive.h"
  31. /*
  32. * The XIVE module will populate these when it loads
  33. */
  34. unsigned long (*__xive_vm_h_xirr)(struct kvm_vcpu *vcpu);
  35. unsigned long (*__xive_vm_h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server);
  36. int (*__xive_vm_h_ipi)(struct kvm_vcpu *vcpu, unsigned long server,
  37. unsigned long mfrr);
  38. int (*__xive_vm_h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr);
  39. int (*__xive_vm_h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr);
  40. EXPORT_SYMBOL_GPL(__xive_vm_h_xirr);
  41. EXPORT_SYMBOL_GPL(__xive_vm_h_ipoll);
  42. EXPORT_SYMBOL_GPL(__xive_vm_h_ipi);
  43. EXPORT_SYMBOL_GPL(__xive_vm_h_cppr);
  44. EXPORT_SYMBOL_GPL(__xive_vm_h_eoi);
  45. /*
  46. * Hash page table alignment on newer cpus(CPU_FTR_ARCH_206)
  47. * should be power of 2.
  48. */
  49. #define HPT_ALIGN_PAGES ((1 << 18) >> PAGE_SHIFT) /* 256k */
  50. /*
  51. * By default we reserve 5% of memory for hash pagetable allocation.
  52. */
  53. static unsigned long kvm_cma_resv_ratio = 5;
  54. static struct cma *kvm_cma;
  55. static int __init early_parse_kvm_cma_resv(char *p)
  56. {
  57. pr_debug("%s(%s)\n", __func__, p);
  58. if (!p)
  59. return -EINVAL;
  60. return kstrtoul(p, 0, &kvm_cma_resv_ratio);
  61. }
  62. early_param("kvm_cma_resv_ratio", early_parse_kvm_cma_resv);
  63. struct page *kvm_alloc_hpt_cma(unsigned long nr_pages)
  64. {
  65. VM_BUG_ON(order_base_2(nr_pages) < KVM_CMA_CHUNK_ORDER - PAGE_SHIFT);
  66. return cma_alloc(kvm_cma, nr_pages, order_base_2(HPT_ALIGN_PAGES),
  67. false);
  68. }
  69. EXPORT_SYMBOL_GPL(kvm_alloc_hpt_cma);
  70. void kvm_free_hpt_cma(struct page *page, unsigned long nr_pages)
  71. {
  72. cma_release(kvm_cma, page, nr_pages);
  73. }
  74. EXPORT_SYMBOL_GPL(kvm_free_hpt_cma);
  75. /**
  76. * kvm_cma_reserve() - reserve area for kvm hash pagetable
  77. *
  78. * This function reserves memory from early allocator. It should be
  79. * called by arch specific code once the memblock allocator
  80. * has been activated and all other subsystems have already allocated/reserved
  81. * memory.
  82. */
  83. void __init kvm_cma_reserve(void)
  84. {
  85. unsigned long align_size;
  86. phys_addr_t selected_size;
  87. /*
  88. * We need CMA reservation only when we are in HV mode
  89. */
  90. if (!cpu_has_feature(CPU_FTR_HVMODE))
  91. return;
  92. selected_size = PAGE_ALIGN(memblock_phys_mem_size() * kvm_cma_resv_ratio / 100);
  93. if (selected_size) {
  94. pr_info("%s: reserving %ld MiB for global area\n", __func__,
  95. (unsigned long)selected_size / SZ_1M);
  96. align_size = HPT_ALIGN_PAGES << PAGE_SHIFT;
  97. cma_declare_contiguous(0, selected_size, 0, align_size,
  98. KVM_CMA_CHUNK_ORDER - PAGE_SHIFT, false, "kvm_cma",
  99. &kvm_cma);
  100. }
  101. }
  102. /*
  103. * Real-mode H_CONFER implementation.
  104. * We check if we are the only vcpu out of this virtual core
  105. * still running in the guest and not ceded. If so, we pop up
  106. * to the virtual-mode implementation; if not, just return to
  107. * the guest.
  108. */
  109. long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target,
  110. unsigned int yield_count)
  111. {
  112. struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
  113. int ptid = local_paca->kvm_hstate.ptid;
  114. int threads_running;
  115. int threads_ceded;
  116. int threads_conferring;
  117. u64 stop = get_tb() + 10 * tb_ticks_per_usec;
  118. int rv = H_SUCCESS; /* => don't yield */
  119. set_bit(ptid, &vc->conferring_threads);
  120. while ((get_tb() < stop) && !VCORE_IS_EXITING(vc)) {
  121. threads_running = VCORE_ENTRY_MAP(vc);
  122. threads_ceded = vc->napping_threads;
  123. threads_conferring = vc->conferring_threads;
  124. if ((threads_ceded | threads_conferring) == threads_running) {
  125. rv = H_TOO_HARD; /* => do yield */
  126. break;
  127. }
  128. }
  129. clear_bit(ptid, &vc->conferring_threads);
  130. return rv;
  131. }
  132. /*
  133. * When running HV mode KVM we need to block certain operations while KVM VMs
  134. * exist in the system. We use a counter of VMs to track this.
  135. *
  136. * One of the operations we need to block is onlining of secondaries, so we
  137. * protect hv_vm_count with get/put_online_cpus().
  138. */
  139. static atomic_t hv_vm_count;
  140. void kvm_hv_vm_activated(void)
  141. {
  142. get_online_cpus();
  143. atomic_inc(&hv_vm_count);
  144. put_online_cpus();
  145. }
  146. EXPORT_SYMBOL_GPL(kvm_hv_vm_activated);
  147. void kvm_hv_vm_deactivated(void)
  148. {
  149. get_online_cpus();
  150. atomic_dec(&hv_vm_count);
  151. put_online_cpus();
  152. }
  153. EXPORT_SYMBOL_GPL(kvm_hv_vm_deactivated);
  154. bool kvm_hv_mode_active(void)
  155. {
  156. return atomic_read(&hv_vm_count) != 0;
  157. }
  158. extern int hcall_real_table[], hcall_real_table_end[];
  159. int kvmppc_hcall_impl_hv_realmode(unsigned long cmd)
  160. {
  161. cmd /= 4;
  162. if (cmd < hcall_real_table_end - hcall_real_table &&
  163. hcall_real_table[cmd])
  164. return 1;
  165. return 0;
  166. }
  167. EXPORT_SYMBOL_GPL(kvmppc_hcall_impl_hv_realmode);
  168. int kvmppc_hwrng_present(void)
  169. {
  170. return powernv_hwrng_present();
  171. }
  172. EXPORT_SYMBOL_GPL(kvmppc_hwrng_present);
  173. long kvmppc_h_random(struct kvm_vcpu *vcpu)
  174. {
  175. int r;
  176. /* Only need to do the expensive mfmsr() on radix */
  177. if (kvm_is_radix(vcpu->kvm) && (mfmsr() & MSR_IR))
  178. r = powernv_get_random_long(&vcpu->arch.regs.gpr[4]);
  179. else
  180. r = powernv_get_random_real_mode(&vcpu->arch.regs.gpr[4]);
  181. if (r)
  182. return H_SUCCESS;
  183. return H_HARDWARE;
  184. }
  185. /*
  186. * Send an interrupt or message to another CPU.
  187. * The caller needs to include any barrier needed to order writes
  188. * to memory vs. the IPI/message.
  189. */
  190. void kvmhv_rm_send_ipi(int cpu)
  191. {
  192. void __iomem *xics_phys;
  193. unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
  194. /* For a nested hypervisor, use the XICS via hcall */
  195. if (kvmhv_on_pseries()) {
  196. unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
  197. plpar_hcall_raw(H_IPI, retbuf, get_hard_smp_processor_id(cpu),
  198. IPI_PRIORITY);
  199. return;
  200. }
  201. /* On POWER9 we can use msgsnd for any destination cpu. */
  202. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  203. msg |= get_hard_smp_processor_id(cpu);
  204. __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
  205. return;
  206. }
  207. /* On POWER8 for IPIs to threads in the same core, use msgsnd. */
  208. if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
  209. cpu_first_thread_sibling(cpu) ==
  210. cpu_first_thread_sibling(raw_smp_processor_id())) {
  211. msg |= cpu_thread_in_core(cpu);
  212. __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
  213. return;
  214. }
  215. /* We should never reach this */
  216. if (WARN_ON_ONCE(xics_on_xive()))
  217. return;
  218. /* Else poke the target with an IPI */
  219. xics_phys = paca_ptrs[cpu]->kvm_hstate.xics_phys;
  220. if (xics_phys)
  221. __raw_rm_writeb(IPI_PRIORITY, xics_phys + XICS_MFRR);
  222. else
  223. opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
  224. }
  225. /*
  226. * The following functions are called from the assembly code
  227. * in book3s_hv_rmhandlers.S.
  228. */
  229. static void kvmhv_interrupt_vcore(struct kvmppc_vcore *vc, int active)
  230. {
  231. int cpu = vc->pcpu;
  232. /* Order setting of exit map vs. msgsnd/IPI */
  233. smp_mb();
  234. for (; active; active >>= 1, ++cpu)
  235. if (active & 1)
  236. kvmhv_rm_send_ipi(cpu);
  237. }
  238. void kvmhv_commence_exit(int trap)
  239. {
  240. struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
  241. int ptid = local_paca->kvm_hstate.ptid;
  242. struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode;
  243. int me, ee, i, t;
  244. int cpu0;
  245. /* Set our bit in the threads-exiting-guest map in the 0xff00
  246. bits of vcore->entry_exit_map */
  247. me = 0x100 << ptid;
  248. do {
  249. ee = vc->entry_exit_map;
  250. } while (cmpxchg(&vc->entry_exit_map, ee, ee | me) != ee);
  251. /* Are we the first here? */
  252. if ((ee >> 8) != 0)
  253. return;
  254. /*
  255. * Trigger the other threads in this vcore to exit the guest.
  256. * If this is a hypervisor decrementer interrupt then they
  257. * will be already on their way out of the guest.
  258. */
  259. if (trap != BOOK3S_INTERRUPT_HV_DECREMENTER)
  260. kvmhv_interrupt_vcore(vc, ee & ~(1 << ptid));
  261. /*
  262. * If we are doing dynamic micro-threading, interrupt the other
  263. * subcores to pull them out of their guests too.
  264. */
  265. if (!sip)
  266. return;
  267. for (i = 0; i < MAX_SUBCORES; ++i) {
  268. vc = sip->vc[i];
  269. if (!vc)
  270. break;
  271. do {
  272. ee = vc->entry_exit_map;
  273. /* Already asked to exit? */
  274. if ((ee >> 8) != 0)
  275. break;
  276. } while (cmpxchg(&vc->entry_exit_map, ee,
  277. ee | VCORE_EXIT_REQ) != ee);
  278. if ((ee >> 8) == 0)
  279. kvmhv_interrupt_vcore(vc, ee);
  280. }
  281. /*
  282. * On POWER9 when running a HPT guest on a radix host (sip != NULL),
  283. * we have to interrupt inactive CPU threads to get them to
  284. * restore the host LPCR value.
  285. */
  286. if (sip->lpcr_req) {
  287. if (cmpxchg(&sip->do_restore, 0, 1) == 0) {
  288. vc = local_paca->kvm_hstate.kvm_vcore;
  289. cpu0 = vc->pcpu + ptid - local_paca->kvm_hstate.tid;
  290. for (t = 1; t < threads_per_core; ++t) {
  291. if (sip->napped[t])
  292. kvmhv_rm_send_ipi(cpu0 + t);
  293. }
  294. }
  295. }
  296. }
  297. struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv;
  298. EXPORT_SYMBOL_GPL(kvmppc_host_rm_ops_hv);
  299. #ifdef CONFIG_KVM_XICS
  300. static struct kvmppc_irq_map *get_irqmap(struct kvmppc_passthru_irqmap *pimap,
  301. u32 xisr)
  302. {
  303. int i;
  304. /*
  305. * We access the mapped array here without a lock. That
  306. * is safe because we never reduce the number of entries
  307. * in the array and we never change the v_hwirq field of
  308. * an entry once it is set.
  309. *
  310. * We have also carefully ordered the stores in the writer
  311. * and the loads here in the reader, so that if we find a matching
  312. * hwirq here, the associated GSI and irq_desc fields are valid.
  313. */
  314. for (i = 0; i < pimap->n_mapped; i++) {
  315. if (xisr == pimap->mapped[i].r_hwirq) {
  316. /*
  317. * Order subsequent reads in the caller to serialize
  318. * with the writer.
  319. */
  320. smp_rmb();
  321. return &pimap->mapped[i];
  322. }
  323. }
  324. return NULL;
  325. }
  326. /*
  327. * If we have an interrupt that's not an IPI, check if we have a
  328. * passthrough adapter and if so, check if this external interrupt
  329. * is for the adapter.
  330. * We will attempt to deliver the IRQ directly to the target VCPU's
  331. * ICP, the virtual ICP (based on affinity - the xive value in ICS).
  332. *
  333. * If the delivery fails or if this is not for a passthrough adapter,
  334. * return to the host to handle this interrupt. We earlier
  335. * saved a copy of the XIRR in the PACA, it will be picked up by
  336. * the host ICP driver.
  337. */
  338. static int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
  339. {
  340. struct kvmppc_passthru_irqmap *pimap;
  341. struct kvmppc_irq_map *irq_map;
  342. struct kvm_vcpu *vcpu;
  343. vcpu = local_paca->kvm_hstate.kvm_vcpu;
  344. if (!vcpu)
  345. return 1;
  346. pimap = kvmppc_get_passthru_irqmap(vcpu->kvm);
  347. if (!pimap)
  348. return 1;
  349. irq_map = get_irqmap(pimap, xisr);
  350. if (!irq_map)
  351. return 1;
  352. /* We're handling this interrupt, generic code doesn't need to */
  353. local_paca->kvm_hstate.saved_xirr = 0;
  354. return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap, again);
  355. }
  356. #else
  357. static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
  358. {
  359. return 1;
  360. }
  361. #endif
  362. /*
  363. * Determine what sort of external interrupt is pending (if any).
  364. * Returns:
  365. * 0 if no interrupt is pending
  366. * 1 if an interrupt is pending that needs to be handled by the host
  367. * 2 Passthrough that needs completion in the host
  368. * -1 if there was a guest wakeup IPI (which has now been cleared)
  369. * -2 if there is PCI passthrough external interrupt that was handled
  370. */
  371. static long kvmppc_read_one_intr(bool *again);
  372. long kvmppc_read_intr(void)
  373. {
  374. long ret = 0;
  375. long rc;
  376. bool again;
  377. if (xive_enabled())
  378. return 1;
  379. do {
  380. again = false;
  381. rc = kvmppc_read_one_intr(&again);
  382. if (rc && (ret == 0 || rc > ret))
  383. ret = rc;
  384. } while (again);
  385. return ret;
  386. }
  387. static long kvmppc_read_one_intr(bool *again)
  388. {
  389. void __iomem *xics_phys;
  390. u32 h_xirr;
  391. __be32 xirr;
  392. u32 xisr;
  393. u8 host_ipi;
  394. int64_t rc;
  395. if (xive_enabled())
  396. return 1;
  397. /* see if a host IPI is pending */
  398. host_ipi = local_paca->kvm_hstate.host_ipi;
  399. if (host_ipi)
  400. return 1;
  401. /* Now read the interrupt from the ICP */
  402. if (kvmhv_on_pseries()) {
  403. unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
  404. rc = plpar_hcall_raw(H_XIRR, retbuf, 0xFF);
  405. xirr = cpu_to_be32(retbuf[0]);
  406. } else {
  407. xics_phys = local_paca->kvm_hstate.xics_phys;
  408. rc = 0;
  409. if (!xics_phys)
  410. rc = opal_int_get_xirr(&xirr, false);
  411. else
  412. xirr = __raw_rm_readl(xics_phys + XICS_XIRR);
  413. }
  414. if (rc < 0)
  415. return 1;
  416. /*
  417. * Save XIRR for later. Since we get control in reverse endian
  418. * on LE systems, save it byte reversed and fetch it back in
  419. * host endian. Note that xirr is the value read from the
  420. * XIRR register, while h_xirr is the host endian version.
  421. */
  422. h_xirr = be32_to_cpu(xirr);
  423. local_paca->kvm_hstate.saved_xirr = h_xirr;
  424. xisr = h_xirr & 0xffffff;
  425. /*
  426. * Ensure that the store/load complete to guarantee all side
  427. * effects of loading from XIRR has completed
  428. */
  429. smp_mb();
  430. /* if nothing pending in the ICP */
  431. if (!xisr)
  432. return 0;
  433. /* We found something in the ICP...
  434. *
  435. * If it is an IPI, clear the MFRR and EOI it.
  436. */
  437. if (xisr == XICS_IPI) {
  438. rc = 0;
  439. if (kvmhv_on_pseries()) {
  440. unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
  441. plpar_hcall_raw(H_IPI, retbuf,
  442. hard_smp_processor_id(), 0xff);
  443. plpar_hcall_raw(H_EOI, retbuf, h_xirr);
  444. } else if (xics_phys) {
  445. __raw_rm_writeb(0xff, xics_phys + XICS_MFRR);
  446. __raw_rm_writel(xirr, xics_phys + XICS_XIRR);
  447. } else {
  448. opal_int_set_mfrr(hard_smp_processor_id(), 0xff);
  449. rc = opal_int_eoi(h_xirr);
  450. }
  451. /* If rc > 0, there is another interrupt pending */
  452. *again = rc > 0;
  453. /*
  454. * Need to ensure side effects of above stores
  455. * complete before proceeding.
  456. */
  457. smp_mb();
  458. /*
  459. * We need to re-check host IPI now in case it got set in the
  460. * meantime. If it's clear, we bounce the interrupt to the
  461. * guest
  462. */
  463. host_ipi = local_paca->kvm_hstate.host_ipi;
  464. if (unlikely(host_ipi != 0)) {
  465. /* We raced with the host,
  466. * we need to resend that IPI, bummer
  467. */
  468. if (kvmhv_on_pseries()) {
  469. unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
  470. plpar_hcall_raw(H_IPI, retbuf,
  471. hard_smp_processor_id(),
  472. IPI_PRIORITY);
  473. } else if (xics_phys)
  474. __raw_rm_writeb(IPI_PRIORITY,
  475. xics_phys + XICS_MFRR);
  476. else
  477. opal_int_set_mfrr(hard_smp_processor_id(),
  478. IPI_PRIORITY);
  479. /* Let side effects complete */
  480. smp_mb();
  481. return 1;
  482. }
  483. /* OK, it's an IPI for us */
  484. local_paca->kvm_hstate.saved_xirr = 0;
  485. return -1;
  486. }
  487. return kvmppc_check_passthru(xisr, xirr, again);
  488. }
  489. #ifdef CONFIG_KVM_XICS
  490. static inline bool is_rm(void)
  491. {
  492. return !(mfmsr() & MSR_DR);
  493. }
  494. unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu)
  495. {
  496. if (!kvmppc_xics_enabled(vcpu))
  497. return H_TOO_HARD;
  498. if (xics_on_xive()) {
  499. if (is_rm())
  500. return xive_rm_h_xirr(vcpu);
  501. if (unlikely(!__xive_vm_h_xirr))
  502. return H_NOT_AVAILABLE;
  503. return __xive_vm_h_xirr(vcpu);
  504. } else
  505. return xics_rm_h_xirr(vcpu);
  506. }
  507. unsigned long kvmppc_rm_h_xirr_x(struct kvm_vcpu *vcpu)
  508. {
  509. if (!kvmppc_xics_enabled(vcpu))
  510. return H_TOO_HARD;
  511. vcpu->arch.regs.gpr[5] = get_tb();
  512. if (xics_on_xive()) {
  513. if (is_rm())
  514. return xive_rm_h_xirr(vcpu);
  515. if (unlikely(!__xive_vm_h_xirr))
  516. return H_NOT_AVAILABLE;
  517. return __xive_vm_h_xirr(vcpu);
  518. } else
  519. return xics_rm_h_xirr(vcpu);
  520. }
  521. unsigned long kvmppc_rm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
  522. {
  523. if (!kvmppc_xics_enabled(vcpu))
  524. return H_TOO_HARD;
  525. if (xics_on_xive()) {
  526. if (is_rm())
  527. return xive_rm_h_ipoll(vcpu, server);
  528. if (unlikely(!__xive_vm_h_ipoll))
  529. return H_NOT_AVAILABLE;
  530. return __xive_vm_h_ipoll(vcpu, server);
  531. } else
  532. return H_TOO_HARD;
  533. }
  534. int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
  535. unsigned long mfrr)
  536. {
  537. if (!kvmppc_xics_enabled(vcpu))
  538. return H_TOO_HARD;
  539. if (xics_on_xive()) {
  540. if (is_rm())
  541. return xive_rm_h_ipi(vcpu, server, mfrr);
  542. if (unlikely(!__xive_vm_h_ipi))
  543. return H_NOT_AVAILABLE;
  544. return __xive_vm_h_ipi(vcpu, server, mfrr);
  545. } else
  546. return xics_rm_h_ipi(vcpu, server, mfrr);
  547. }
  548. int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
  549. {
  550. if (!kvmppc_xics_enabled(vcpu))
  551. return H_TOO_HARD;
  552. if (xics_on_xive()) {
  553. if (is_rm())
  554. return xive_rm_h_cppr(vcpu, cppr);
  555. if (unlikely(!__xive_vm_h_cppr))
  556. return H_NOT_AVAILABLE;
  557. return __xive_vm_h_cppr(vcpu, cppr);
  558. } else
  559. return xics_rm_h_cppr(vcpu, cppr);
  560. }
  561. int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
  562. {
  563. if (!kvmppc_xics_enabled(vcpu))
  564. return H_TOO_HARD;
  565. if (xics_on_xive()) {
  566. if (is_rm())
  567. return xive_rm_h_eoi(vcpu, xirr);
  568. if (unlikely(!__xive_vm_h_eoi))
  569. return H_NOT_AVAILABLE;
  570. return __xive_vm_h_eoi(vcpu, xirr);
  571. } else
  572. return xics_rm_h_eoi(vcpu, xirr);
  573. }
  574. #endif /* CONFIG_KVM_XICS */
  575. void kvmppc_bad_interrupt(struct pt_regs *regs)
  576. {
  577. /*
  578. * 100 could happen at any time, 200 can happen due to invalid real
  579. * address access for example (or any time due to a hardware problem).
  580. */
  581. if (TRAP(regs) == 0x100) {
  582. get_paca()->in_nmi++;
  583. system_reset_exception(regs);
  584. get_paca()->in_nmi--;
  585. } else if (TRAP(regs) == 0x200) {
  586. machine_check_exception(regs);
  587. } else {
  588. die("Bad interrupt in KVM entry/exit code", regs, SIGABRT);
  589. }
  590. panic("Bad KVM trap");
  591. }
  592. /*
  593. * Functions used to switch LPCR HR and UPRT bits on all threads
  594. * when entering and exiting HPT guests on a radix host.
  595. */
  596. #define PHASE_REALMODE 1 /* in real mode */
  597. #define PHASE_SET_LPCR 2 /* have set LPCR */
  598. #define PHASE_OUT_OF_GUEST 4 /* have finished executing in guest */
  599. #define PHASE_RESET_LPCR 8 /* have reset LPCR to host value */
  600. #define ALL(p) (((p) << 24) | ((p) << 16) | ((p) << 8) | (p))
  601. static void wait_for_sync(struct kvm_split_mode *sip, int phase)
  602. {
  603. int thr = local_paca->kvm_hstate.tid;
  604. sip->lpcr_sync.phase[thr] |= phase;
  605. phase = ALL(phase);
  606. while ((sip->lpcr_sync.allphases & phase) != phase) {
  607. HMT_low();
  608. barrier();
  609. }
  610. HMT_medium();
  611. }
  612. void kvmhv_p9_set_lpcr(struct kvm_split_mode *sip)
  613. {
  614. unsigned long rb, set;
  615. /* wait for every other thread to get to real mode */
  616. wait_for_sync(sip, PHASE_REALMODE);
  617. /* Set LPCR and LPIDR */
  618. mtspr(SPRN_LPCR, sip->lpcr_req);
  619. mtspr(SPRN_LPID, sip->lpidr_req);
  620. isync();
  621. /* Invalidate the TLB on thread 0 */
  622. if (local_paca->kvm_hstate.tid == 0) {
  623. sip->do_set = 0;
  624. asm volatile("ptesync" : : : "memory");
  625. for (set = 0; set < POWER9_TLB_SETS_RADIX; ++set) {
  626. rb = TLBIEL_INVAL_SET_LPID +
  627. (set << TLBIEL_INVAL_SET_SHIFT);
  628. asm volatile(PPC_TLBIEL(%0, %1, 0, 0, 0) : :
  629. "r" (rb), "r" (0));
  630. }
  631. asm volatile("ptesync" : : : "memory");
  632. }
  633. /* indicate that we have done so and wait for others */
  634. wait_for_sync(sip, PHASE_SET_LPCR);
  635. /* order read of sip->lpcr_sync.allphases vs. sip->do_set */
  636. smp_rmb();
  637. }
  638. /*
  639. * Called when a thread that has been in the guest needs
  640. * to reload the host LPCR value - but only on POWER9 when
  641. * running a HPT guest on a radix host.
  642. */
  643. void kvmhv_p9_restore_lpcr(struct kvm_split_mode *sip)
  644. {
  645. /* we're out of the guest... */
  646. wait_for_sync(sip, PHASE_OUT_OF_GUEST);
  647. mtspr(SPRN_LPID, 0);
  648. mtspr(SPRN_LPCR, sip->host_lpcr);
  649. isync();
  650. if (local_paca->kvm_hstate.tid == 0) {
  651. sip->do_restore = 0;
  652. smp_wmb(); /* order store of do_restore vs. phase */
  653. }
  654. wait_for_sync(sip, PHASE_RESET_LPCR);
  655. smp_mb();
  656. local_paca->kvm_hstate.kvm_split_mode = NULL;
  657. }
  658. static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
  659. {
  660. vcpu->arch.ceded = 0;
  661. if (vcpu->arch.timer_running) {
  662. hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
  663. vcpu->arch.timer_running = 0;
  664. }
  665. }
  666. void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
  667. {
  668. /*
  669. * Check for illegal transactional state bit combination
  670. * and if we find it, force the TS field to a safe state.
  671. */
  672. if ((msr & MSR_TS_MASK) == MSR_TS_MASK)
  673. msr &= ~MSR_TS_MASK;
  674. vcpu->arch.shregs.msr = msr;
  675. kvmppc_end_cede(vcpu);
  676. }
  677. EXPORT_SYMBOL_GPL(kvmppc_set_msr_hv);
  678. static void inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags)
  679. {
  680. unsigned long msr, pc, new_msr, new_pc;
  681. msr = kvmppc_get_msr(vcpu);
  682. pc = kvmppc_get_pc(vcpu);
  683. new_msr = vcpu->arch.intr_msr;
  684. new_pc = vec;
  685. /* If transactional, change to suspend mode on IRQ delivery */
  686. if (MSR_TM_TRANSACTIONAL(msr))
  687. new_msr |= MSR_TS_S;
  688. else
  689. new_msr |= msr & MSR_TS_MASK;
  690. /*
  691. * Perform MSR and PC adjustment for LPCR[AIL]=3 if it is set and
  692. * applicable. AIL=2 is not supported.
  693. *
  694. * AIL does not apply to SRESET, MCE, or HMI (which is never
  695. * delivered to the guest), and does not apply if IR=0 or DR=0.
  696. */
  697. if (vec != BOOK3S_INTERRUPT_SYSTEM_RESET &&
  698. vec != BOOK3S_INTERRUPT_MACHINE_CHECK &&
  699. (vcpu->arch.vcore->lpcr & LPCR_AIL) == LPCR_AIL_3 &&
  700. (msr & (MSR_IR|MSR_DR)) == (MSR_IR|MSR_DR) ) {
  701. new_msr |= MSR_IR | MSR_DR;
  702. new_pc += 0xC000000000004000ULL;
  703. }
  704. kvmppc_set_srr0(vcpu, pc);
  705. kvmppc_set_srr1(vcpu, (msr & SRR1_MSR_BITS) | srr1_flags);
  706. kvmppc_set_pc(vcpu, new_pc);
  707. vcpu->arch.shregs.msr = new_msr;
  708. }
  709. void kvmppc_inject_interrupt_hv(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags)
  710. {
  711. inject_interrupt(vcpu, vec, srr1_flags);
  712. kvmppc_end_cede(vcpu);
  713. }
  714. EXPORT_SYMBOL_GPL(kvmppc_inject_interrupt_hv);
  715. /*
  716. * Is there a PRIV_DOORBELL pending for the guest (on POWER9)?
  717. * Can we inject a Decrementer or a External interrupt?
  718. */
  719. void kvmppc_guest_entry_inject_int(struct kvm_vcpu *vcpu)
  720. {
  721. int ext;
  722. unsigned long lpcr;
  723. /* Insert EXTERNAL bit into LPCR at the MER bit position */
  724. ext = (vcpu->arch.pending_exceptions >> BOOK3S_IRQPRIO_EXTERNAL) & 1;
  725. lpcr = mfspr(SPRN_LPCR);
  726. lpcr |= ext << LPCR_MER_SH;
  727. mtspr(SPRN_LPCR, lpcr);
  728. isync();
  729. if (vcpu->arch.shregs.msr & MSR_EE) {
  730. if (ext) {
  731. inject_interrupt(vcpu, BOOK3S_INTERRUPT_EXTERNAL, 0);
  732. } else {
  733. long int dec = mfspr(SPRN_DEC);
  734. if (!(lpcr & LPCR_LD))
  735. dec = (int) dec;
  736. if (dec < 0)
  737. inject_interrupt(vcpu,
  738. BOOK3S_INTERRUPT_DECREMENTER, 0);
  739. }
  740. }
  741. if (vcpu->arch.doorbell_request) {
  742. mtspr(SPRN_DPDES, 1);
  743. vcpu->arch.vcore->dpdes = 1;
  744. smp_wmb();
  745. vcpu->arch.doorbell_request = 0;
  746. }
  747. }
  748. static void flush_guest_tlb(struct kvm *kvm)
  749. {
  750. unsigned long rb, set;
  751. rb = PPC_BIT(52); /* IS = 2 */
  752. if (kvm_is_radix(kvm)) {
  753. /* R=1 PRS=1 RIC=2 */
  754. asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
  755. : : "r" (rb), "i" (1), "i" (1), "i" (2),
  756. "r" (0) : "memory");
  757. for (set = 1; set < kvm->arch.tlb_sets; ++set) {
  758. rb += PPC_BIT(51); /* increment set number */
  759. /* R=1 PRS=1 RIC=0 */
  760. asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
  761. : : "r" (rb), "i" (1), "i" (1), "i" (0),
  762. "r" (0) : "memory");
  763. }
  764. asm volatile("ptesync": : :"memory");
  765. // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
  766. asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
  767. } else {
  768. for (set = 0; set < kvm->arch.tlb_sets; ++set) {
  769. /* R=0 PRS=0 RIC=0 */
  770. asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
  771. : : "r" (rb), "i" (0), "i" (0), "i" (0),
  772. "r" (0) : "memory");
  773. rb += PPC_BIT(51); /* increment set number */
  774. }
  775. asm volatile("ptesync": : :"memory");
  776. // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
  777. if (cpu_has_feature(CPU_FTR_ARCH_300))
  778. asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
  779. }
  780. }
  781. void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu,
  782. struct kvm_nested_guest *nested)
  783. {
  784. cpumask_t *need_tlb_flush;
  785. /*
  786. * On POWER9, individual threads can come in here, but the
  787. * TLB is shared between the 4 threads in a core, hence
  788. * invalidating on one thread invalidates for all.
  789. * Thus we make all 4 threads use the same bit.
  790. */
  791. if (cpu_has_feature(CPU_FTR_ARCH_300))
  792. pcpu = cpu_first_tlb_thread_sibling(pcpu);
  793. if (nested)
  794. need_tlb_flush = &nested->need_tlb_flush;
  795. else
  796. need_tlb_flush = &kvm->arch.need_tlb_flush;
  797. if (cpumask_test_cpu(pcpu, need_tlb_flush)) {
  798. flush_guest_tlb(kvm);
  799. /* Clear the bit after the TLB flush */
  800. cpumask_clear_cpu(pcpu, need_tlb_flush);
  801. }
  802. }
  803. EXPORT_SYMBOL_GPL(kvmppc_check_need_tlb_flush);