book3s_hv_rm_xics.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2012 Michael Ellerman, IBM Corporation.
  4. * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/kvm_host.h>
  8. #include <linux/err.h>
  9. #include <linux/kernel_stat.h>
  10. #include <linux/pgtable.h>
  11. #include <asm/kvm_book3s.h>
  12. #include <asm/kvm_ppc.h>
  13. #include <asm/hvcall.h>
  14. #include <asm/xics.h>
  15. #include <asm/synch.h>
  16. #include <asm/cputhreads.h>
  17. #include <asm/ppc-opcode.h>
  18. #include <asm/pnv-pci.h>
  19. #include <asm/opal.h>
  20. #include <asm/smp.h>
  21. #include "book3s_xics.h"
  22. #define DEBUG_PASSUP
  23. int h_ipi_redirect = 1;
  24. EXPORT_SYMBOL(h_ipi_redirect);
  25. int kvm_irq_bypass = 1;
  26. EXPORT_SYMBOL(kvm_irq_bypass);
  27. static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
  28. u32 new_irq, bool check_resend);
  29. static int xics_opal_set_server(unsigned int hw_irq, int server_cpu);
  30. /* -- ICS routines -- */
  31. static void ics_rm_check_resend(struct kvmppc_xics *xics,
  32. struct kvmppc_ics *ics, struct kvmppc_icp *icp)
  33. {
  34. int i;
  35. for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
  36. struct ics_irq_state *state = &ics->irq_state[i];
  37. if (state->resend)
  38. icp_rm_deliver_irq(xics, icp, state->number, true);
  39. }
  40. }
  41. /* -- ICP routines -- */
  42. #ifdef CONFIG_SMP
  43. static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu)
  44. {
  45. int hcpu;
  46. hcpu = hcore << threads_shift;
  47. kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu;
  48. smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION);
  49. kvmppc_set_host_ipi(hcpu);
  50. smp_mb();
  51. kvmhv_rm_send_ipi(hcpu);
  52. }
  53. #else
  54. static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { }
  55. #endif
  56. /*
  57. * We start the search from our current CPU Id in the core map
  58. * and go in a circle until we get back to our ID looking for a
  59. * core that is running in host context and that hasn't already
  60. * been targeted for another rm_host_ops.
  61. *
  62. * In the future, could consider using a fairer algorithm (one
  63. * that distributes the IPIs better)
  64. *
  65. * Returns -1, if no CPU could be found in the host
  66. * Else, returns a CPU Id which has been reserved for use
  67. */
  68. static inline int grab_next_hostcore(int start,
  69. struct kvmppc_host_rm_core *rm_core, int max, int action)
  70. {
  71. bool success;
  72. int core;
  73. union kvmppc_rm_state old, new;
  74. for (core = start + 1; core < max; core++) {
  75. old = new = READ_ONCE(rm_core[core].rm_state);
  76. if (!old.in_host || old.rm_action)
  77. continue;
  78. /* Try to grab this host core if not taken already. */
  79. new.rm_action = action;
  80. success = cmpxchg64(&rm_core[core].rm_state.raw,
  81. old.raw, new.raw) == old.raw;
  82. if (success) {
  83. /*
  84. * Make sure that the store to the rm_action is made
  85. * visible before we return to caller (and the
  86. * subsequent store to rm_data) to synchronize with
  87. * the IPI handler.
  88. */
  89. smp_wmb();
  90. return core;
  91. }
  92. }
  93. return -1;
  94. }
  95. static inline int find_available_hostcore(int action)
  96. {
  97. int core;
  98. int my_core = smp_processor_id() >> threads_shift;
  99. struct kvmppc_host_rm_core *rm_core = kvmppc_host_rm_ops_hv->rm_core;
  100. core = grab_next_hostcore(my_core, rm_core, cpu_nr_cores(), action);
  101. if (core == -1)
  102. core = grab_next_hostcore(core, rm_core, my_core, action);
  103. return core;
  104. }
  105. static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu,
  106. struct kvm_vcpu *this_vcpu)
  107. {
  108. struct kvmppc_icp *this_icp = this_vcpu->arch.icp;
  109. int cpu;
  110. int hcore;
  111. /* Mark the target VCPU as having an interrupt pending */
  112. vcpu->stat.queue_intr++;
  113. set_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
  114. /* Kick self ? Just set MER and return */
  115. if (vcpu == this_vcpu) {
  116. mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER);
  117. return;
  118. }
  119. if (xive_enabled() && kvmhv_on_pseries()) {
  120. /* No XICS access or hypercalls available, too hard */
  121. this_icp->rm_action |= XICS_RM_KICK_VCPU;
  122. this_icp->rm_kick_target = vcpu;
  123. return;
  124. }
  125. /*
  126. * Check if the core is loaded,
  127. * if not, find an available host core to post to wake the VCPU,
  128. * if we can't find one, set up state to eventually return too hard.
  129. */
  130. cpu = vcpu->arch.thread_cpu;
  131. if (cpu < 0 || cpu >= nr_cpu_ids) {
  132. hcore = -1;
  133. if (kvmppc_host_rm_ops_hv && h_ipi_redirect)
  134. hcore = find_available_hostcore(XICS_RM_KICK_VCPU);
  135. if (hcore != -1) {
  136. icp_send_hcore_msg(hcore, vcpu);
  137. } else {
  138. this_icp->rm_action |= XICS_RM_KICK_VCPU;
  139. this_icp->rm_kick_target = vcpu;
  140. }
  141. return;
  142. }
  143. smp_mb();
  144. kvmhv_rm_send_ipi(cpu);
  145. }
  146. static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu)
  147. {
  148. /* Note: Only called on self ! */
  149. clear_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
  150. mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER);
  151. }
  152. static inline bool icp_rm_try_update(struct kvmppc_icp *icp,
  153. union kvmppc_icp_state old,
  154. union kvmppc_icp_state new)
  155. {
  156. struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu;
  157. bool success;
  158. /* Calculate new output value */
  159. new.out_ee = (new.xisr && (new.pending_pri < new.cppr));
  160. /* Attempt atomic update */
  161. success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw;
  162. if (!success)
  163. goto bail;
  164. /*
  165. * Check for output state update
  166. *
  167. * Note that this is racy since another processor could be updating
  168. * the state already. This is why we never clear the interrupt output
  169. * here, we only ever set it. The clear only happens prior to doing
  170. * an update and only by the processor itself. Currently we do it
  171. * in Accept (H_XIRR) and Up_Cppr (H_XPPR).
  172. *
  173. * We also do not try to figure out whether the EE state has changed,
  174. * we unconditionally set it if the new state calls for it. The reason
  175. * for that is that we opportunistically remove the pending interrupt
  176. * flag when raising CPPR, so we need to set it back here if an
  177. * interrupt is still pending.
  178. */
  179. if (new.out_ee)
  180. icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu);
  181. /* Expose the state change for debug purposes */
  182. this_vcpu->arch.icp->rm_dbgstate = new;
  183. this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu;
  184. bail:
  185. return success;
  186. }
  187. static inline int check_too_hard(struct kvmppc_xics *xics,
  188. struct kvmppc_icp *icp)
  189. {
  190. return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS;
  191. }
  192. static void icp_rm_check_resend(struct kvmppc_xics *xics,
  193. struct kvmppc_icp *icp)
  194. {
  195. u32 icsid;
  196. /* Order this load with the test for need_resend in the caller */
  197. smp_rmb();
  198. for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) {
  199. struct kvmppc_ics *ics = xics->ics[icsid];
  200. if (!test_and_clear_bit(icsid, icp->resend_map))
  201. continue;
  202. if (!ics)
  203. continue;
  204. ics_rm_check_resend(xics, ics, icp);
  205. }
  206. }
  207. static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority,
  208. u32 *reject)
  209. {
  210. union kvmppc_icp_state old_state, new_state;
  211. bool success;
  212. do {
  213. old_state = new_state = READ_ONCE(icp->state);
  214. *reject = 0;
  215. /* See if we can deliver */
  216. success = new_state.cppr > priority &&
  217. new_state.mfrr > priority &&
  218. new_state.pending_pri > priority;
  219. /*
  220. * If we can, check for a rejection and perform the
  221. * delivery
  222. */
  223. if (success) {
  224. *reject = new_state.xisr;
  225. new_state.xisr = irq;
  226. new_state.pending_pri = priority;
  227. } else {
  228. /*
  229. * If we failed to deliver we set need_resend
  230. * so a subsequent CPPR state change causes us
  231. * to try a new delivery.
  232. */
  233. new_state.need_resend = true;
  234. }
  235. } while (!icp_rm_try_update(icp, old_state, new_state));
  236. return success;
  237. }
  238. static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
  239. u32 new_irq, bool check_resend)
  240. {
  241. struct ics_irq_state *state;
  242. struct kvmppc_ics *ics;
  243. u32 reject;
  244. u16 src;
  245. /*
  246. * This is used both for initial delivery of an interrupt and
  247. * for subsequent rejection.
  248. *
  249. * Rejection can be racy vs. resends. We have evaluated the
  250. * rejection in an atomic ICP transaction which is now complete,
  251. * so potentially the ICP can already accept the interrupt again.
  252. *
  253. * So we need to retry the delivery. Essentially the reject path
  254. * boils down to a failed delivery. Always.
  255. *
  256. * Now the interrupt could also have moved to a different target,
  257. * thus we may need to re-do the ICP lookup as well
  258. */
  259. again:
  260. /* Get the ICS state and lock it */
  261. ics = kvmppc_xics_find_ics(xics, new_irq, &src);
  262. if (!ics) {
  263. /* Unsafe increment, but this does not need to be accurate */
  264. xics->err_noics++;
  265. return;
  266. }
  267. state = &ics->irq_state[src];
  268. /* Get a lock on the ICS */
  269. arch_spin_lock(&ics->lock);
  270. /* Get our server */
  271. if (!icp || state->server != icp->server_num) {
  272. icp = kvmppc_xics_find_server(xics->kvm, state->server);
  273. if (!icp) {
  274. /* Unsafe increment again*/
  275. xics->err_noicp++;
  276. goto out;
  277. }
  278. }
  279. if (check_resend)
  280. if (!state->resend)
  281. goto out;
  282. /* Clear the resend bit of that interrupt */
  283. state->resend = 0;
  284. /*
  285. * If masked, bail out
  286. *
  287. * Note: PAPR doesn't mention anything about masked pending
  288. * when doing a resend, only when doing a delivery.
  289. *
  290. * However that would have the effect of losing a masked
  291. * interrupt that was rejected and isn't consistent with
  292. * the whole masked_pending business which is about not
  293. * losing interrupts that occur while masked.
  294. *
  295. * I don't differentiate normal deliveries and resends, this
  296. * implementation will differ from PAPR and not lose such
  297. * interrupts.
  298. */
  299. if (state->priority == MASKED) {
  300. state->masked_pending = 1;
  301. goto out;
  302. }
  303. /*
  304. * Try the delivery, this will set the need_resend flag
  305. * in the ICP as part of the atomic transaction if the
  306. * delivery is not possible.
  307. *
  308. * Note that if successful, the new delivery might have itself
  309. * rejected an interrupt that was "delivered" before we took the
  310. * ics spin lock.
  311. *
  312. * In this case we do the whole sequence all over again for the
  313. * new guy. We cannot assume that the rejected interrupt is less
  314. * favored than the new one, and thus doesn't need to be delivered,
  315. * because by the time we exit icp_rm_try_to_deliver() the target
  316. * processor may well have already consumed & completed it, and thus
  317. * the rejected interrupt might actually be already acceptable.
  318. */
  319. if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) {
  320. /*
  321. * Delivery was successful, did we reject somebody else ?
  322. */
  323. if (reject && reject != XICS_IPI) {
  324. arch_spin_unlock(&ics->lock);
  325. icp->n_reject++;
  326. new_irq = reject;
  327. check_resend = 0;
  328. goto again;
  329. }
  330. } else {
  331. /*
  332. * We failed to deliver the interrupt we need to set the
  333. * resend map bit and mark the ICS state as needing a resend
  334. */
  335. state->resend = 1;
  336. /*
  337. * Make sure when checking resend, we don't miss the resend
  338. * if resend_map bit is seen and cleared.
  339. */
  340. smp_wmb();
  341. set_bit(ics->icsid, icp->resend_map);
  342. /*
  343. * If the need_resend flag got cleared in the ICP some time
  344. * between icp_rm_try_to_deliver() atomic update and now, then
  345. * we know it might have missed the resend_map bit. So we
  346. * retry
  347. */
  348. smp_mb();
  349. if (!icp->state.need_resend) {
  350. state->resend = 0;
  351. arch_spin_unlock(&ics->lock);
  352. check_resend = 0;
  353. goto again;
  354. }
  355. }
  356. out:
  357. arch_spin_unlock(&ics->lock);
  358. }
  359. static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
  360. u8 new_cppr)
  361. {
  362. union kvmppc_icp_state old_state, new_state;
  363. bool resend;
  364. /*
  365. * This handles several related states in one operation:
  366. *
  367. * ICP State: Down_CPPR
  368. *
  369. * Load CPPR with new value and if the XISR is 0
  370. * then check for resends:
  371. *
  372. * ICP State: Resend
  373. *
  374. * If MFRR is more favored than CPPR, check for IPIs
  375. * and notify ICS of a potential resend. This is done
  376. * asynchronously (when used in real mode, we will have
  377. * to exit here).
  378. *
  379. * We do not handle the complete Check_IPI as documented
  380. * here. In the PAPR, this state will be used for both
  381. * Set_MFRR and Down_CPPR. However, we know that we aren't
  382. * changing the MFRR state here so we don't need to handle
  383. * the case of an MFRR causing a reject of a pending irq,
  384. * this will have been handled when the MFRR was set in the
  385. * first place.
  386. *
  387. * Thus we don't have to handle rejects, only resends.
  388. *
  389. * When implementing real mode for HV KVM, resend will lead to
  390. * a H_TOO_HARD return and the whole transaction will be handled
  391. * in virtual mode.
  392. */
  393. do {
  394. old_state = new_state = READ_ONCE(icp->state);
  395. /* Down_CPPR */
  396. new_state.cppr = new_cppr;
  397. /*
  398. * Cut down Resend / Check_IPI / IPI
  399. *
  400. * The logic is that we cannot have a pending interrupt
  401. * trumped by an IPI at this point (see above), so we
  402. * know that either the pending interrupt is already an
  403. * IPI (in which case we don't care to override it) or
  404. * it's either more favored than us or non existent
  405. */
  406. if (new_state.mfrr < new_cppr &&
  407. new_state.mfrr <= new_state.pending_pri) {
  408. new_state.pending_pri = new_state.mfrr;
  409. new_state.xisr = XICS_IPI;
  410. }
  411. /* Latch/clear resend bit */
  412. resend = new_state.need_resend;
  413. new_state.need_resend = 0;
  414. } while (!icp_rm_try_update(icp, old_state, new_state));
  415. /*
  416. * Now handle resend checks. Those are asynchronous to the ICP
  417. * state update in HW (ie bus transactions) so we can handle them
  418. * separately here as well.
  419. */
  420. if (resend) {
  421. icp->n_check_resend++;
  422. icp_rm_check_resend(xics, icp);
  423. }
  424. }
  425. unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu)
  426. {
  427. union kvmppc_icp_state old_state, new_state;
  428. struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
  429. struct kvmppc_icp *icp = vcpu->arch.icp;
  430. u32 xirr;
  431. if (!xics || !xics->real_mode)
  432. return H_TOO_HARD;
  433. /* First clear the interrupt */
  434. icp_rm_clr_vcpu_irq(icp->vcpu);
  435. /*
  436. * ICP State: Accept_Interrupt
  437. *
  438. * Return the pending interrupt (if any) along with the
  439. * current CPPR, then clear the XISR & set CPPR to the
  440. * pending priority
  441. */
  442. do {
  443. old_state = new_state = READ_ONCE(icp->state);
  444. xirr = old_state.xisr | (((u32)old_state.cppr) << 24);
  445. if (!old_state.xisr)
  446. break;
  447. new_state.cppr = new_state.pending_pri;
  448. new_state.pending_pri = 0xff;
  449. new_state.xisr = 0;
  450. } while (!icp_rm_try_update(icp, old_state, new_state));
  451. /* Return the result in GPR4 */
  452. vcpu->arch.regs.gpr[4] = xirr;
  453. return check_too_hard(xics, icp);
  454. }
  455. int xics_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
  456. unsigned long mfrr)
  457. {
  458. union kvmppc_icp_state old_state, new_state;
  459. struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
  460. struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp;
  461. u32 reject;
  462. bool resend;
  463. bool local;
  464. if (!xics || !xics->real_mode)
  465. return H_TOO_HARD;
  466. local = this_icp->server_num == server;
  467. if (local)
  468. icp = this_icp;
  469. else
  470. icp = kvmppc_xics_find_server(vcpu->kvm, server);
  471. if (!icp)
  472. return H_PARAMETER;
  473. /*
  474. * ICP state: Set_MFRR
  475. *
  476. * If the CPPR is more favored than the new MFRR, then
  477. * nothing needs to be done as there can be no XISR to
  478. * reject.
  479. *
  480. * ICP state: Check_IPI
  481. *
  482. * If the CPPR is less favored, then we might be replacing
  483. * an interrupt, and thus need to possibly reject it.
  484. *
  485. * ICP State: IPI
  486. *
  487. * Besides rejecting any pending interrupts, we also
  488. * update XISR and pending_pri to mark IPI as pending.
  489. *
  490. * PAPR does not describe this state, but if the MFRR is being
  491. * made less favored than its earlier value, there might be
  492. * a previously-rejected interrupt needing to be resent.
  493. * Ideally, we would want to resend only if
  494. * prio(pending_interrupt) < mfrr &&
  495. * prio(pending_interrupt) < cppr
  496. * where pending interrupt is the one that was rejected. But
  497. * we don't have that state, so we simply trigger a resend
  498. * whenever the MFRR is made less favored.
  499. */
  500. do {
  501. old_state = new_state = READ_ONCE(icp->state);
  502. /* Set_MFRR */
  503. new_state.mfrr = mfrr;
  504. /* Check_IPI */
  505. reject = 0;
  506. resend = false;
  507. if (mfrr < new_state.cppr) {
  508. /* Reject a pending interrupt if not an IPI */
  509. if (mfrr <= new_state.pending_pri) {
  510. reject = new_state.xisr;
  511. new_state.pending_pri = mfrr;
  512. new_state.xisr = XICS_IPI;
  513. }
  514. }
  515. if (mfrr > old_state.mfrr) {
  516. resend = new_state.need_resend;
  517. new_state.need_resend = 0;
  518. }
  519. } while (!icp_rm_try_update(icp, old_state, new_state));
  520. /* Handle reject in real mode */
  521. if (reject && reject != XICS_IPI) {
  522. this_icp->n_reject++;
  523. icp_rm_deliver_irq(xics, icp, reject, false);
  524. }
  525. /* Handle resends in real mode */
  526. if (resend) {
  527. this_icp->n_check_resend++;
  528. icp_rm_check_resend(xics, icp);
  529. }
  530. return check_too_hard(xics, this_icp);
  531. }
  532. int xics_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
  533. {
  534. union kvmppc_icp_state old_state, new_state;
  535. struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
  536. struct kvmppc_icp *icp = vcpu->arch.icp;
  537. u32 reject;
  538. if (!xics || !xics->real_mode)
  539. return H_TOO_HARD;
  540. /*
  541. * ICP State: Set_CPPR
  542. *
  543. * We can safely compare the new value with the current
  544. * value outside of the transaction as the CPPR is only
  545. * ever changed by the processor on itself
  546. */
  547. if (cppr > icp->state.cppr) {
  548. icp_rm_down_cppr(xics, icp, cppr);
  549. goto bail;
  550. } else if (cppr == icp->state.cppr)
  551. return H_SUCCESS;
  552. /*
  553. * ICP State: Up_CPPR
  554. *
  555. * The processor is raising its priority, this can result
  556. * in a rejection of a pending interrupt:
  557. *
  558. * ICP State: Reject_Current
  559. *
  560. * We can remove EE from the current processor, the update
  561. * transaction will set it again if needed
  562. */
  563. icp_rm_clr_vcpu_irq(icp->vcpu);
  564. do {
  565. old_state = new_state = READ_ONCE(icp->state);
  566. reject = 0;
  567. new_state.cppr = cppr;
  568. if (cppr <= new_state.pending_pri) {
  569. reject = new_state.xisr;
  570. new_state.xisr = 0;
  571. new_state.pending_pri = 0xff;
  572. }
  573. } while (!icp_rm_try_update(icp, old_state, new_state));
  574. /*
  575. * Check for rejects. They are handled by doing a new delivery
  576. * attempt (see comments in icp_rm_deliver_irq).
  577. */
  578. if (reject && reject != XICS_IPI) {
  579. icp->n_reject++;
  580. icp_rm_deliver_irq(xics, icp, reject, false);
  581. }
  582. bail:
  583. return check_too_hard(xics, icp);
  584. }
  585. static int ics_rm_eoi(struct kvm_vcpu *vcpu, u32 irq)
  586. {
  587. struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
  588. struct kvmppc_icp *icp = vcpu->arch.icp;
  589. struct kvmppc_ics *ics;
  590. struct ics_irq_state *state;
  591. u16 src;
  592. u32 pq_old, pq_new;
  593. /*
  594. * ICS EOI handling: For LSI, if P bit is still set, we need to
  595. * resend it.
  596. *
  597. * For MSI, we move Q bit into P (and clear Q). If it is set,
  598. * resend it.
  599. */
  600. ics = kvmppc_xics_find_ics(xics, irq, &src);
  601. if (!ics)
  602. goto bail;
  603. state = &ics->irq_state[src];
  604. if (state->lsi)
  605. pq_new = state->pq_state;
  606. else
  607. do {
  608. pq_old = state->pq_state;
  609. pq_new = pq_old >> 1;
  610. } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);
  611. if (pq_new & PQ_PRESENTED)
  612. icp_rm_deliver_irq(xics, NULL, irq, false);
  613. if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) {
  614. icp->rm_action |= XICS_RM_NOTIFY_EOI;
  615. icp->rm_eoied_irq = irq;
  616. }
  617. if (state->host_irq) {
  618. ++vcpu->stat.pthru_all;
  619. if (state->intr_cpu != -1) {
  620. int pcpu = raw_smp_processor_id();
  621. pcpu = cpu_first_thread_sibling(pcpu);
  622. ++vcpu->stat.pthru_host;
  623. if (state->intr_cpu != pcpu) {
  624. ++vcpu->stat.pthru_bad_aff;
  625. xics_opal_set_server(state->host_irq, pcpu);
  626. }
  627. state->intr_cpu = -1;
  628. }
  629. }
  630. bail:
  631. return check_too_hard(xics, icp);
  632. }
  633. int xics_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
  634. {
  635. struct kvmppc_xics *xics = vcpu->kvm->arch.xics;
  636. struct kvmppc_icp *icp = vcpu->arch.icp;
  637. u32 irq = xirr & 0x00ffffff;
  638. if (!xics || !xics->real_mode)
  639. return H_TOO_HARD;
  640. /*
  641. * ICP State: EOI
  642. *
  643. * Note: If EOI is incorrectly used by SW to lower the CPPR
  644. * value (ie more favored), we do not check for rejection of
  645. * a pending interrupt, this is a SW error and PAPR specifies
  646. * that we don't have to deal with it.
  647. *
  648. * The sending of an EOI to the ICS is handled after the
  649. * CPPR update
  650. *
  651. * ICP State: Down_CPPR which we handle
  652. * in a separate function as it's shared with H_CPPR.
  653. */
  654. icp_rm_down_cppr(xics, icp, xirr >> 24);
  655. /* IPIs have no EOI */
  656. if (irq == XICS_IPI)
  657. return check_too_hard(xics, icp);
  658. return ics_rm_eoi(vcpu, irq);
  659. }
  660. static unsigned long eoi_rc;
  661. static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again)
  662. {
  663. void __iomem *xics_phys;
  664. int64_t rc;
  665. if (kvmhv_on_pseries()) {
  666. unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
  667. iosync();
  668. plpar_hcall_raw(H_EOI, retbuf, hwirq);
  669. return;
  670. }
  671. rc = pnv_opal_pci_msi_eoi(c, hwirq);
  672. if (rc)
  673. eoi_rc = rc;
  674. iosync();
  675. /* EOI it */
  676. xics_phys = local_paca->kvm_hstate.xics_phys;
  677. if (xics_phys) {
  678. __raw_rm_writel(xirr, xics_phys + XICS_XIRR);
  679. } else {
  680. rc = opal_int_eoi(be32_to_cpu(xirr));
  681. *again = rc > 0;
  682. }
  683. }
  684. static int xics_opal_set_server(unsigned int hw_irq, int server_cpu)
  685. {
  686. unsigned int mangle_cpu = get_hard_smp_processor_id(server_cpu) << 2;
  687. return opal_set_xive(hw_irq, mangle_cpu, DEFAULT_PRIORITY);
  688. }
  689. /*
  690. * Increment a per-CPU 32-bit unsigned integer variable.
  691. * Safe to call in real-mode. Handles vmalloc'ed addresses
  692. *
  693. * ToDo: Make this work for any integral type
  694. */
  695. static inline void this_cpu_inc_rm(unsigned int __percpu *addr)
  696. {
  697. unsigned long l;
  698. unsigned int *raddr;
  699. int cpu = smp_processor_id();
  700. raddr = per_cpu_ptr(addr, cpu);
  701. l = (unsigned long)raddr;
  702. if (get_region_id(l) == VMALLOC_REGION_ID) {
  703. l = vmalloc_to_phys(raddr);
  704. raddr = (unsigned int *)l;
  705. }
  706. ++*raddr;
  707. }
  708. /*
  709. * We don't try to update the flags in the irq_desc 'istate' field in
  710. * here as would happen in the normal IRQ handling path for several reasons:
  711. * - state flags represent internal IRQ state and are not expected to be
  712. * updated outside the IRQ subsystem
  713. * - more importantly, these are useful for edge triggered interrupts,
  714. * IRQ probing, etc., but we are only handling MSI/MSIx interrupts here
  715. * and these states shouldn't apply to us.
  716. *
  717. * However, we do update irq_stats - we somewhat duplicate the code in
  718. * kstat_incr_irqs_this_cpu() for this since this function is defined
  719. * in irq/internal.h which we don't want to include here.
  720. * The only difference is that desc->kstat_irqs is an allocated per CPU
  721. * variable and could have been vmalloc'ed, so we can't directly
  722. * call __this_cpu_inc() on it. The kstat structure is a static
  723. * per CPU variable and it should be accessible by real-mode KVM.
  724. *
  725. */
  726. static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc)
  727. {
  728. this_cpu_inc_rm(desc->kstat_irqs);
  729. __this_cpu_inc(kstat.irqs_sum);
  730. }
  731. long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu,
  732. __be32 xirr,
  733. struct kvmppc_irq_map *irq_map,
  734. struct kvmppc_passthru_irqmap *pimap,
  735. bool *again)
  736. {
  737. struct kvmppc_xics *xics;
  738. struct kvmppc_icp *icp;
  739. struct kvmppc_ics *ics;
  740. struct ics_irq_state *state;
  741. u32 irq;
  742. u16 src;
  743. u32 pq_old, pq_new;
  744. irq = irq_map->v_hwirq;
  745. xics = vcpu->kvm->arch.xics;
  746. icp = vcpu->arch.icp;
  747. kvmppc_rm_handle_irq_desc(irq_map->desc);
  748. ics = kvmppc_xics_find_ics(xics, irq, &src);
  749. if (!ics)
  750. return 2;
  751. state = &ics->irq_state[src];
  752. /* only MSIs register bypass producers, so it must be MSI here */
  753. do {
  754. pq_old = state->pq_state;
  755. pq_new = ((pq_old << 1) & 3) | PQ_PRESENTED;
  756. } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old);
  757. /* Test P=1, Q=0, this is the only case where we present */
  758. if (pq_new == PQ_PRESENTED)
  759. icp_rm_deliver_irq(xics, icp, irq, false);
  760. /* EOI the interrupt */
  761. icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr,
  762. again);
  763. if (check_too_hard(xics, icp) == H_TOO_HARD)
  764. return 2;
  765. else
  766. return -2;
  767. }
  768. /* --- Non-real mode XICS-related built-in routines --- */
  769. /**
  770. * Host Operations poked by RM KVM
  771. */
  772. static void rm_host_ipi_action(int action, void *data)
  773. {
  774. switch (action) {
  775. case XICS_RM_KICK_VCPU:
  776. kvmppc_host_rm_ops_hv->vcpu_kick(data);
  777. break;
  778. default:
  779. WARN(1, "Unexpected rm_action=%d data=%p\n", action, data);
  780. break;
  781. }
  782. }
  783. void kvmppc_xics_ipi_action(void)
  784. {
  785. int core;
  786. unsigned int cpu = smp_processor_id();
  787. struct kvmppc_host_rm_core *rm_corep;
  788. core = cpu >> threads_shift;
  789. rm_corep = &kvmppc_host_rm_ops_hv->rm_core[core];
  790. if (rm_corep->rm_data) {
  791. rm_host_ipi_action(rm_corep->rm_state.rm_action,
  792. rm_corep->rm_data);
  793. /* Order these stores against the real mode KVM */
  794. rm_corep->rm_data = NULL;
  795. smp_wmb();
  796. rm_corep->rm_state.rm_action = 0;
  797. }
  798. }