book3s_xive.c 59 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
  4. */
  5. #define pr_fmt(fmt) "xive-kvm: " fmt
  6. #include <linux/kernel.h>
  7. #include <linux/kvm_host.h>
  8. #include <linux/err.h>
  9. #include <linux/gfp.h>
  10. #include <linux/spinlock.h>
  11. #include <linux/delay.h>
  12. #include <linux/percpu.h>
  13. #include <linux/cpumask.h>
  14. #include <linux/uaccess.h>
  15. #include <asm/kvm_book3s.h>
  16. #include <asm/kvm_ppc.h>
  17. #include <asm/hvcall.h>
  18. #include <asm/xics.h>
  19. #include <asm/xive.h>
  20. #include <asm/xive-regs.h>
  21. #include <asm/debug.h>
  22. #include <asm/debugfs.h>
  23. #include <asm/time.h>
  24. #include <asm/opal.h>
  25. #include <linux/debugfs.h>
  26. #include <linux/seq_file.h>
  27. #include "book3s_xive.h"
  28. /*
  29. * Virtual mode variants of the hcalls for use on radix/radix
  30. * with AIL. They require the VCPU's VP to be "pushed"
  31. *
  32. * We still instantiate them here because we use some of the
  33. * generated utility functions as well in this file.
  34. */
  35. #define XIVE_RUNTIME_CHECKS
  36. #define X_PFX xive_vm_
  37. #define X_STATIC static
  38. #define X_STAT_PFX stat_vm_
  39. #define __x_tima xive_tima
  40. #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio))
  41. #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio))
  42. #define __x_writeb __raw_writeb
  43. #define __x_readw __raw_readw
  44. #define __x_readq __raw_readq
  45. #define __x_writeq __raw_writeq
  46. #include "book3s_xive_template.c"
  47. /*
  48. * We leave a gap of a couple of interrupts in the queue to
  49. * account for the IPI and additional safety guard.
  50. */
  51. #define XIVE_Q_GAP 2
  52. /*
  53. * Push a vcpu's context to the XIVE on guest entry.
  54. * This assumes we are in virtual mode (MMU on)
  55. */
  56. void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
  57. {
  58. void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
  59. u64 pq;
  60. /*
  61. * Nothing to do if the platform doesn't have a XIVE
  62. * or this vCPU doesn't have its own XIVE context
  63. * (e.g. because it's not using an in-kernel interrupt controller).
  64. */
  65. if (!tima || !vcpu->arch.xive_cam_word)
  66. return;
  67. eieio();
  68. __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
  69. __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
  70. vcpu->arch.xive_pushed = 1;
  71. eieio();
  72. /*
  73. * We clear the irq_pending flag. There is a small chance of a
  74. * race vs. the escalation interrupt happening on another
  75. * processor setting it again, but the only consequence is to
  76. * cause a spurious wakeup on the next H_CEDE, which is not an
  77. * issue.
  78. */
  79. vcpu->arch.irq_pending = 0;
  80. /*
  81. * In single escalation mode, if the escalation interrupt is
  82. * on, we mask it.
  83. */
  84. if (vcpu->arch.xive_esc_on) {
  85. pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
  86. XIVE_ESB_SET_PQ_01));
  87. mb();
  88. /*
  89. * We have a possible subtle race here: The escalation
  90. * interrupt might have fired and be on its way to the
  91. * host queue while we mask it, and if we unmask it
  92. * early enough (re-cede right away), there is a
  93. * theorical possibility that it fires again, thus
  94. * landing in the target queue more than once which is
  95. * a big no-no.
  96. *
  97. * Fortunately, solving this is rather easy. If the
  98. * above load setting PQ to 01 returns a previous
  99. * value where P is set, then we know the escalation
  100. * interrupt is somewhere on its way to the host. In
  101. * that case we simply don't clear the xive_esc_on
  102. * flag below. It will be eventually cleared by the
  103. * handler for the escalation interrupt.
  104. *
  105. * Then, when doing a cede, we check that flag again
  106. * before re-enabling the escalation interrupt, and if
  107. * set, we abort the cede.
  108. */
  109. if (!(pq & XIVE_ESB_VAL_P))
  110. /* Now P is 0, we can clear the flag */
  111. vcpu->arch.xive_esc_on = 0;
  112. }
  113. }
  114. EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
  115. /*
  116. * This is a simple trigger for a generic XIVE IRQ. This must
  117. * only be called for interrupts that support a trigger page
  118. */
  119. static bool xive_irq_trigger(struct xive_irq_data *xd)
  120. {
  121. /* This should be only for MSIs */
  122. if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
  123. return false;
  124. /* Those interrupts should always have a trigger page */
  125. if (WARN_ON(!xd->trig_mmio))
  126. return false;
  127. out_be64(xd->trig_mmio, 0);
  128. return true;
  129. }
  130. static irqreturn_t xive_esc_irq(int irq, void *data)
  131. {
  132. struct kvm_vcpu *vcpu = data;
  133. vcpu->arch.irq_pending = 1;
  134. smp_mb();
  135. if (vcpu->arch.ceded)
  136. kvmppc_fast_vcpu_kick(vcpu);
  137. /* Since we have the no-EOI flag, the interrupt is effectively
  138. * disabled now. Clearing xive_esc_on means we won't bother
  139. * doing so on the next entry.
  140. *
  141. * This also allows the entry code to know that if a PQ combination
  142. * of 10 is observed while xive_esc_on is true, it means the queue
  143. * contains an unprocessed escalation interrupt. We don't make use of
  144. * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
  145. */
  146. vcpu->arch.xive_esc_on = false;
  147. /* This orders xive_esc_on = false vs. subsequent stale_p = true */
  148. smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
  149. return IRQ_HANDLED;
  150. }
  151. int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
  152. bool single_escalation)
  153. {
  154. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  155. struct xive_q *q = &xc->queues[prio];
  156. char *name = NULL;
  157. int rc;
  158. /* Already there ? */
  159. if (xc->esc_virq[prio])
  160. return 0;
  161. /* Hook up the escalation interrupt */
  162. xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
  163. if (!xc->esc_virq[prio]) {
  164. pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
  165. prio, xc->server_num);
  166. return -EIO;
  167. }
  168. if (single_escalation)
  169. name = kasprintf(GFP_KERNEL, "kvm-%d-%d",
  170. vcpu->kvm->arch.lpid, xc->server_num);
  171. else
  172. name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
  173. vcpu->kvm->arch.lpid, xc->server_num, prio);
  174. if (!name) {
  175. pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
  176. prio, xc->server_num);
  177. rc = -ENOMEM;
  178. goto error;
  179. }
  180. pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
  181. rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
  182. IRQF_NO_THREAD, name, vcpu);
  183. if (rc) {
  184. pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
  185. prio, xc->server_num);
  186. goto error;
  187. }
  188. xc->esc_virq_names[prio] = name;
  189. /* In single escalation mode, we grab the ESB MMIO of the
  190. * interrupt and mask it. Also populate the VCPU v/raddr
  191. * of the ESB page for use by asm entry/exit code. Finally
  192. * set the XIVE_IRQ_NO_EOI flag which will prevent the
  193. * core code from performing an EOI on the escalation
  194. * interrupt, thus leaving it effectively masked after
  195. * it fires once.
  196. */
  197. if (single_escalation) {
  198. struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
  199. struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
  200. xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
  201. vcpu->arch.xive_esc_raddr = xd->eoi_page;
  202. vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
  203. xd->flags |= XIVE_IRQ_NO_EOI;
  204. }
  205. return 0;
  206. error:
  207. irq_dispose_mapping(xc->esc_virq[prio]);
  208. xc->esc_virq[prio] = 0;
  209. kfree(name);
  210. return rc;
  211. }
  212. static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
  213. {
  214. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  215. struct kvmppc_xive *xive = xc->xive;
  216. struct xive_q *q = &xc->queues[prio];
  217. void *qpage;
  218. int rc;
  219. if (WARN_ON(q->qpage))
  220. return 0;
  221. /* Allocate the queue and retrieve infos on current node for now */
  222. qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
  223. if (!qpage) {
  224. pr_err("Failed to allocate queue %d for VCPU %d\n",
  225. prio, xc->server_num);
  226. return -ENOMEM;
  227. }
  228. memset(qpage, 0, 1 << xive->q_order);
  229. /*
  230. * Reconfigure the queue. This will set q->qpage only once the
  231. * queue is fully configured. This is a requirement for prio 0
  232. * as we will stop doing EOIs for every IPI as soon as we observe
  233. * qpage being non-NULL, and instead will only EOI when we receive
  234. * corresponding queue 0 entries
  235. */
  236. rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
  237. xive->q_order, true);
  238. if (rc)
  239. pr_err("Failed to configure queue %d for VCPU %d\n",
  240. prio, xc->server_num);
  241. return rc;
  242. }
  243. /* Called with xive->lock held */
  244. static int xive_check_provisioning(struct kvm *kvm, u8 prio)
  245. {
  246. struct kvmppc_xive *xive = kvm->arch.xive;
  247. struct kvm_vcpu *vcpu;
  248. int i, rc;
  249. lockdep_assert_held(&xive->lock);
  250. /* Already provisioned ? */
  251. if (xive->qmap & (1 << prio))
  252. return 0;
  253. pr_devel("Provisioning prio... %d\n", prio);
  254. /* Provision each VCPU and enable escalations if needed */
  255. kvm_for_each_vcpu(i, vcpu, kvm) {
  256. if (!vcpu->arch.xive_vcpu)
  257. continue;
  258. rc = xive_provision_queue(vcpu, prio);
  259. if (rc == 0 && !xive->single_escalation)
  260. kvmppc_xive_attach_escalation(vcpu, prio,
  261. xive->single_escalation);
  262. if (rc)
  263. return rc;
  264. }
  265. /* Order previous stores and mark it as provisioned */
  266. mb();
  267. xive->qmap |= (1 << prio);
  268. return 0;
  269. }
  270. static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
  271. {
  272. struct kvm_vcpu *vcpu;
  273. struct kvmppc_xive_vcpu *xc;
  274. struct xive_q *q;
  275. /* Locate target server */
  276. vcpu = kvmppc_xive_find_server(kvm, server);
  277. if (!vcpu) {
  278. pr_warn("%s: Can't find server %d\n", __func__, server);
  279. return;
  280. }
  281. xc = vcpu->arch.xive_vcpu;
  282. if (WARN_ON(!xc))
  283. return;
  284. q = &xc->queues[prio];
  285. atomic_inc(&q->pending_count);
  286. }
  287. static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
  288. {
  289. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  290. struct xive_q *q;
  291. u32 max;
  292. if (WARN_ON(!xc))
  293. return -ENXIO;
  294. if (!xc->valid)
  295. return -ENXIO;
  296. q = &xc->queues[prio];
  297. if (WARN_ON(!q->qpage))
  298. return -ENXIO;
  299. /* Calculate max number of interrupts in that queue. */
  300. max = (q->msk + 1) - XIVE_Q_GAP;
  301. return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
  302. }
  303. int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
  304. {
  305. struct kvm_vcpu *vcpu;
  306. int i, rc;
  307. /* Locate target server */
  308. vcpu = kvmppc_xive_find_server(kvm, *server);
  309. if (!vcpu) {
  310. pr_devel("Can't find server %d\n", *server);
  311. return -EINVAL;
  312. }
  313. pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
  314. /* Try pick it */
  315. rc = xive_try_pick_queue(vcpu, prio);
  316. if (rc == 0)
  317. return rc;
  318. pr_devel(" .. failed, looking up candidate...\n");
  319. /* Failed, pick another VCPU */
  320. kvm_for_each_vcpu(i, vcpu, kvm) {
  321. if (!vcpu->arch.xive_vcpu)
  322. continue;
  323. rc = xive_try_pick_queue(vcpu, prio);
  324. if (rc == 0) {
  325. *server = vcpu->arch.xive_vcpu->server_num;
  326. pr_devel(" found on 0x%x/%d\n", *server, prio);
  327. return rc;
  328. }
  329. }
  330. pr_devel(" no available target !\n");
  331. /* No available target ! */
  332. return -EBUSY;
  333. }
  334. static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
  335. struct kvmppc_xive_src_block *sb,
  336. struct kvmppc_xive_irq_state *state)
  337. {
  338. struct xive_irq_data *xd;
  339. u32 hw_num;
  340. u8 old_prio;
  341. u64 val;
  342. /*
  343. * Take the lock, set masked, try again if racing
  344. * with H_EOI
  345. */
  346. for (;;) {
  347. arch_spin_lock(&sb->lock);
  348. old_prio = state->guest_priority;
  349. state->guest_priority = MASKED;
  350. mb();
  351. if (!state->in_eoi)
  352. break;
  353. state->guest_priority = old_prio;
  354. arch_spin_unlock(&sb->lock);
  355. }
  356. /* No change ? Bail */
  357. if (old_prio == MASKED)
  358. return old_prio;
  359. /* Get the right irq */
  360. kvmppc_xive_select_irq(state, &hw_num, &xd);
  361. /*
  362. * If the interrupt is marked as needing masking via
  363. * firmware, we do it here. Firmware masking however
  364. * is "lossy", it won't return the old p and q bits
  365. * and won't set the interrupt to a state where it will
  366. * record queued ones. If this is an issue we should do
  367. * lazy masking instead.
  368. *
  369. * For now, we work around this in unmask by forcing
  370. * an interrupt whenever we unmask a non-LSI via FW
  371. * (if ever).
  372. */
  373. if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) {
  374. xive_native_configure_irq(hw_num,
  375. kvmppc_xive_vp(xive, state->act_server),
  376. MASKED, state->number);
  377. /* set old_p so we can track if an H_EOI was done */
  378. state->old_p = true;
  379. state->old_q = false;
  380. } else {
  381. /* Set PQ to 10, return old P and old Q and remember them */
  382. val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
  383. state->old_p = !!(val & 2);
  384. state->old_q = !!(val & 1);
  385. /*
  386. * Synchronize hardware to sensure the queues are updated
  387. * when masking
  388. */
  389. xive_native_sync_source(hw_num);
  390. }
  391. return old_prio;
  392. }
  393. static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
  394. struct kvmppc_xive_irq_state *state)
  395. {
  396. /*
  397. * Take the lock try again if racing with H_EOI
  398. */
  399. for (;;) {
  400. arch_spin_lock(&sb->lock);
  401. if (!state->in_eoi)
  402. break;
  403. arch_spin_unlock(&sb->lock);
  404. }
  405. }
  406. static void xive_finish_unmask(struct kvmppc_xive *xive,
  407. struct kvmppc_xive_src_block *sb,
  408. struct kvmppc_xive_irq_state *state,
  409. u8 prio)
  410. {
  411. struct xive_irq_data *xd;
  412. u32 hw_num;
  413. /* If we aren't changing a thing, move on */
  414. if (state->guest_priority != MASKED)
  415. goto bail;
  416. /* Get the right irq */
  417. kvmppc_xive_select_irq(state, &hw_num, &xd);
  418. /*
  419. * See comment in xive_lock_and_mask() concerning masking
  420. * via firmware.
  421. */
  422. if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) {
  423. xive_native_configure_irq(hw_num,
  424. kvmppc_xive_vp(xive, state->act_server),
  425. state->act_priority, state->number);
  426. /* If an EOI is needed, do it here */
  427. if (!state->old_p)
  428. xive_vm_source_eoi(hw_num, xd);
  429. /* If this is not an LSI, force a trigger */
  430. if (!(xd->flags & OPAL_XIVE_IRQ_LSI))
  431. xive_irq_trigger(xd);
  432. goto bail;
  433. }
  434. /* Old Q set, set PQ to 11 */
  435. if (state->old_q)
  436. xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
  437. /*
  438. * If not old P, then perform an "effective" EOI,
  439. * on the source. This will handle the cases where
  440. * FW EOI is needed.
  441. */
  442. if (!state->old_p)
  443. xive_vm_source_eoi(hw_num, xd);
  444. /* Synchronize ordering and mark unmasked */
  445. mb();
  446. bail:
  447. state->guest_priority = prio;
  448. }
  449. /*
  450. * Target an interrupt to a given server/prio, this will fallback
  451. * to another server if necessary and perform the HW targetting
  452. * updates as needed
  453. *
  454. * NOTE: Must be called with the state lock held
  455. */
  456. static int xive_target_interrupt(struct kvm *kvm,
  457. struct kvmppc_xive_irq_state *state,
  458. u32 server, u8 prio)
  459. {
  460. struct kvmppc_xive *xive = kvm->arch.xive;
  461. u32 hw_num;
  462. int rc;
  463. /*
  464. * This will return a tentative server and actual
  465. * priority. The count for that new target will have
  466. * already been incremented.
  467. */
  468. rc = kvmppc_xive_select_target(kvm, &server, prio);
  469. /*
  470. * We failed to find a target ? Not much we can do
  471. * at least until we support the GIQ.
  472. */
  473. if (rc)
  474. return rc;
  475. /*
  476. * Increment the old queue pending count if there
  477. * was one so that the old queue count gets adjusted later
  478. * when observed to be empty.
  479. */
  480. if (state->act_priority != MASKED)
  481. xive_inc_q_pending(kvm,
  482. state->act_server,
  483. state->act_priority);
  484. /*
  485. * Update state and HW
  486. */
  487. state->act_priority = prio;
  488. state->act_server = server;
  489. /* Get the right irq */
  490. kvmppc_xive_select_irq(state, &hw_num, NULL);
  491. return xive_native_configure_irq(hw_num,
  492. kvmppc_xive_vp(xive, server),
  493. prio, state->number);
  494. }
  495. /*
  496. * Targetting rules: In order to avoid losing track of
  497. * pending interrupts accross mask and unmask, which would
  498. * allow queue overflows, we implement the following rules:
  499. *
  500. * - Unless it was never enabled (or we run out of capacity)
  501. * an interrupt is always targetted at a valid server/queue
  502. * pair even when "masked" by the guest. This pair tends to
  503. * be the last one used but it can be changed under some
  504. * circumstances. That allows us to separate targetting
  505. * from masking, we only handle accounting during (re)targetting,
  506. * this also allows us to let an interrupt drain into its target
  507. * queue after masking, avoiding complex schemes to remove
  508. * interrupts out of remote processor queues.
  509. *
  510. * - When masking, we set PQ to 10 and save the previous value
  511. * of P and Q.
  512. *
  513. * - When unmasking, if saved Q was set, we set PQ to 11
  514. * otherwise we leave PQ to the HW state which will be either
  515. * 10 if nothing happened or 11 if the interrupt fired while
  516. * masked. Effectively we are OR'ing the previous Q into the
  517. * HW Q.
  518. *
  519. * Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
  520. * which will unmask the interrupt and shoot a new one if Q was
  521. * set.
  522. *
  523. * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
  524. * effectively meaning an H_EOI from the guest is still expected
  525. * for that interrupt).
  526. *
  527. * - If H_EOI occurs while masked, we clear the saved P.
  528. *
  529. * - When changing target, we account on the new target and
  530. * increment a separate "pending" counter on the old one.
  531. * This pending counter will be used to decrement the old
  532. * target's count when its queue has been observed empty.
  533. */
  534. int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
  535. u32 priority)
  536. {
  537. struct kvmppc_xive *xive = kvm->arch.xive;
  538. struct kvmppc_xive_src_block *sb;
  539. struct kvmppc_xive_irq_state *state;
  540. u8 new_act_prio;
  541. int rc = 0;
  542. u16 idx;
  543. if (!xive)
  544. return -ENODEV;
  545. pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
  546. irq, server, priority);
  547. /* First, check provisioning of queues */
  548. if (priority != MASKED) {
  549. mutex_lock(&xive->lock);
  550. rc = xive_check_provisioning(xive->kvm,
  551. xive_prio_from_guest(priority));
  552. mutex_unlock(&xive->lock);
  553. }
  554. if (rc) {
  555. pr_devel(" provisioning failure %d !\n", rc);
  556. return rc;
  557. }
  558. sb = kvmppc_xive_find_source(xive, irq, &idx);
  559. if (!sb)
  560. return -EINVAL;
  561. state = &sb->irq_state[idx];
  562. /*
  563. * We first handle masking/unmasking since the locking
  564. * might need to be retried due to EOIs, we'll handle
  565. * targetting changes later. These functions will return
  566. * with the SB lock held.
  567. *
  568. * xive_lock_and_mask() will also set state->guest_priority
  569. * but won't otherwise change other fields of the state.
  570. *
  571. * xive_lock_for_unmask will not actually unmask, this will
  572. * be done later by xive_finish_unmask() once the targetting
  573. * has been done, so we don't try to unmask an interrupt
  574. * that hasn't yet been targetted.
  575. */
  576. if (priority == MASKED)
  577. xive_lock_and_mask(xive, sb, state);
  578. else
  579. xive_lock_for_unmask(sb, state);
  580. /*
  581. * Then we handle targetting.
  582. *
  583. * First calculate a new "actual priority"
  584. */
  585. new_act_prio = state->act_priority;
  586. if (priority != MASKED)
  587. new_act_prio = xive_prio_from_guest(priority);
  588. pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
  589. new_act_prio, state->act_server, state->act_priority);
  590. /*
  591. * Then check if we actually need to change anything,
  592. *
  593. * The condition for re-targetting the interrupt is that
  594. * we have a valid new priority (new_act_prio is not 0xff)
  595. * and either the server or the priority changed.
  596. *
  597. * Note: If act_priority was ff and the new priority is
  598. * also ff, we don't do anything and leave the interrupt
  599. * untargetted. An attempt of doing an int_on on an
  600. * untargetted interrupt will fail. If that is a problem
  601. * we could initialize interrupts with valid default
  602. */
  603. if (new_act_prio != MASKED &&
  604. (state->act_server != server ||
  605. state->act_priority != new_act_prio))
  606. rc = xive_target_interrupt(kvm, state, server, new_act_prio);
  607. /*
  608. * Perform the final unmasking of the interrupt source
  609. * if necessary
  610. */
  611. if (priority != MASKED)
  612. xive_finish_unmask(xive, sb, state, priority);
  613. /*
  614. * Finally Update saved_priority to match. Only int_on/off
  615. * set this field to a different value.
  616. */
  617. state->saved_priority = priority;
  618. arch_spin_unlock(&sb->lock);
  619. return rc;
  620. }
  621. int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
  622. u32 *priority)
  623. {
  624. struct kvmppc_xive *xive = kvm->arch.xive;
  625. struct kvmppc_xive_src_block *sb;
  626. struct kvmppc_xive_irq_state *state;
  627. u16 idx;
  628. if (!xive)
  629. return -ENODEV;
  630. sb = kvmppc_xive_find_source(xive, irq, &idx);
  631. if (!sb)
  632. return -EINVAL;
  633. state = &sb->irq_state[idx];
  634. arch_spin_lock(&sb->lock);
  635. *server = state->act_server;
  636. *priority = state->guest_priority;
  637. arch_spin_unlock(&sb->lock);
  638. return 0;
  639. }
  640. int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
  641. {
  642. struct kvmppc_xive *xive = kvm->arch.xive;
  643. struct kvmppc_xive_src_block *sb;
  644. struct kvmppc_xive_irq_state *state;
  645. u16 idx;
  646. if (!xive)
  647. return -ENODEV;
  648. sb = kvmppc_xive_find_source(xive, irq, &idx);
  649. if (!sb)
  650. return -EINVAL;
  651. state = &sb->irq_state[idx];
  652. pr_devel("int_on(irq=0x%x)\n", irq);
  653. /*
  654. * Check if interrupt was not targetted
  655. */
  656. if (state->act_priority == MASKED) {
  657. pr_devel("int_on on untargetted interrupt\n");
  658. return -EINVAL;
  659. }
  660. /* If saved_priority is 0xff, do nothing */
  661. if (state->saved_priority == MASKED)
  662. return 0;
  663. /*
  664. * Lock and unmask it.
  665. */
  666. xive_lock_for_unmask(sb, state);
  667. xive_finish_unmask(xive, sb, state, state->saved_priority);
  668. arch_spin_unlock(&sb->lock);
  669. return 0;
  670. }
  671. int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
  672. {
  673. struct kvmppc_xive *xive = kvm->arch.xive;
  674. struct kvmppc_xive_src_block *sb;
  675. struct kvmppc_xive_irq_state *state;
  676. u16 idx;
  677. if (!xive)
  678. return -ENODEV;
  679. sb = kvmppc_xive_find_source(xive, irq, &idx);
  680. if (!sb)
  681. return -EINVAL;
  682. state = &sb->irq_state[idx];
  683. pr_devel("int_off(irq=0x%x)\n", irq);
  684. /*
  685. * Lock and mask
  686. */
  687. state->saved_priority = xive_lock_and_mask(xive, sb, state);
  688. arch_spin_unlock(&sb->lock);
  689. return 0;
  690. }
  691. static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
  692. {
  693. struct kvmppc_xive_src_block *sb;
  694. struct kvmppc_xive_irq_state *state;
  695. u16 idx;
  696. sb = kvmppc_xive_find_source(xive, irq, &idx);
  697. if (!sb)
  698. return false;
  699. state = &sb->irq_state[idx];
  700. if (!state->valid)
  701. return false;
  702. /*
  703. * Trigger the IPI. This assumes we never restore a pass-through
  704. * interrupt which should be safe enough
  705. */
  706. xive_irq_trigger(&state->ipi_data);
  707. return true;
  708. }
  709. u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
  710. {
  711. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  712. if (!xc)
  713. return 0;
  714. /* Return the per-cpu state for state saving/migration */
  715. return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
  716. (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
  717. (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
  718. }
  719. int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
  720. {
  721. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  722. struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
  723. u8 cppr, mfrr;
  724. u32 xisr;
  725. if (!xc || !xive)
  726. return -ENOENT;
  727. /* Grab individual state fields. We don't use pending_pri */
  728. cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
  729. xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
  730. KVM_REG_PPC_ICP_XISR_MASK;
  731. mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
  732. pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
  733. xc->server_num, cppr, mfrr, xisr);
  734. /*
  735. * We can't update the state of a "pushed" VCPU, but that
  736. * shouldn't happen because the vcpu->mutex makes running a
  737. * vcpu mutually exclusive with doing one_reg get/set on it.
  738. */
  739. if (WARN_ON(vcpu->arch.xive_pushed))
  740. return -EIO;
  741. /* Update VCPU HW saved state */
  742. vcpu->arch.xive_saved_state.cppr = cppr;
  743. xc->hw_cppr = xc->cppr = cppr;
  744. /*
  745. * Update MFRR state. If it's not 0xff, we mark the VCPU as
  746. * having a pending MFRR change, which will re-evaluate the
  747. * target. The VCPU will thus potentially get a spurious
  748. * interrupt but that's not a big deal.
  749. */
  750. xc->mfrr = mfrr;
  751. if (mfrr < cppr)
  752. xive_irq_trigger(&xc->vp_ipi_data);
  753. /*
  754. * Now saved XIRR is "interesting". It means there's something in
  755. * the legacy "1 element" queue... for an IPI we simply ignore it,
  756. * as the MFRR restore will handle that. For anything else we need
  757. * to force a resend of the source.
  758. * However the source may not have been setup yet. If that's the
  759. * case, we keep that info and increment a counter in the xive to
  760. * tell subsequent xive_set_source() to go look.
  761. */
  762. if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
  763. xc->delayed_irq = xisr;
  764. xive->delayed_irqs++;
  765. pr_devel(" xisr restore delayed\n");
  766. }
  767. return 0;
  768. }
  769. int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
  770. struct irq_desc *host_desc)
  771. {
  772. struct kvmppc_xive *xive = kvm->arch.xive;
  773. struct kvmppc_xive_src_block *sb;
  774. struct kvmppc_xive_irq_state *state;
  775. struct irq_data *host_data = irq_desc_get_irq_data(host_desc);
  776. unsigned int host_irq = irq_desc_get_irq(host_desc);
  777. unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
  778. u16 idx;
  779. u8 prio;
  780. int rc;
  781. if (!xive)
  782. return -ENODEV;
  783. pr_devel("set_mapped girq 0x%lx host HW irq 0x%x...\n",guest_irq, hw_irq);
  784. sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
  785. if (!sb)
  786. return -EINVAL;
  787. state = &sb->irq_state[idx];
  788. /*
  789. * Mark the passed-through interrupt as going to a VCPU,
  790. * this will prevent further EOIs and similar operations
  791. * from the XIVE code. It will also mask the interrupt
  792. * to either PQ=10 or 11 state, the latter if the interrupt
  793. * is pending. This will allow us to unmask or retrigger it
  794. * after routing it to the guest with a simple EOI.
  795. *
  796. * The "state" argument is a "token", all it needs is to be
  797. * non-NULL to switch to passed-through or NULL for the
  798. * other way around. We may not yet have an actual VCPU
  799. * target here and we don't really care.
  800. */
  801. rc = irq_set_vcpu_affinity(host_irq, state);
  802. if (rc) {
  803. pr_err("Failed to set VCPU affinity for irq %d\n", host_irq);
  804. return rc;
  805. }
  806. /*
  807. * Mask and read state of IPI. We need to know if its P bit
  808. * is set as that means it's potentially already using a
  809. * queue entry in the target
  810. */
  811. prio = xive_lock_and_mask(xive, sb, state);
  812. pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
  813. state->old_p, state->old_q);
  814. /* Turn the IPI hard off */
  815. xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
  816. /*
  817. * Reset ESB guest mapping. Needed when ESB pages are exposed
  818. * to the guest in XIVE native mode
  819. */
  820. if (xive->ops && xive->ops->reset_mapped)
  821. xive->ops->reset_mapped(kvm, guest_irq);
  822. /* Grab info about irq */
  823. state->pt_number = hw_irq;
  824. state->pt_data = irq_data_get_irq_handler_data(host_data);
  825. /*
  826. * Configure the IRQ to match the existing configuration of
  827. * the IPI if it was already targetted. Otherwise this will
  828. * mask the interrupt in a lossy way (act_priority is 0xff)
  829. * which is fine for a never started interrupt.
  830. */
  831. xive_native_configure_irq(hw_irq,
  832. kvmppc_xive_vp(xive, state->act_server),
  833. state->act_priority, state->number);
  834. /*
  835. * We do an EOI to enable the interrupt (and retrigger if needed)
  836. * if the guest has the interrupt unmasked and the P bit was *not*
  837. * set in the IPI. If it was set, we know a slot may still be in
  838. * use in the target queue thus we have to wait for a guest
  839. * originated EOI
  840. */
  841. if (prio != MASKED && !state->old_p)
  842. xive_vm_source_eoi(hw_irq, state->pt_data);
  843. /* Clear old_p/old_q as they are no longer relevant */
  844. state->old_p = state->old_q = false;
  845. /* Restore guest prio (unlocks EOI) */
  846. mb();
  847. state->guest_priority = prio;
  848. arch_spin_unlock(&sb->lock);
  849. return 0;
  850. }
  851. EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
  852. int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
  853. struct irq_desc *host_desc)
  854. {
  855. struct kvmppc_xive *xive = kvm->arch.xive;
  856. struct kvmppc_xive_src_block *sb;
  857. struct kvmppc_xive_irq_state *state;
  858. unsigned int host_irq = irq_desc_get_irq(host_desc);
  859. u16 idx;
  860. u8 prio;
  861. int rc;
  862. if (!xive)
  863. return -ENODEV;
  864. pr_devel("clr_mapped girq 0x%lx...\n", guest_irq);
  865. sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
  866. if (!sb)
  867. return -EINVAL;
  868. state = &sb->irq_state[idx];
  869. /*
  870. * Mask and read state of IRQ. We need to know if its P bit
  871. * is set as that means it's potentially already using a
  872. * queue entry in the target
  873. */
  874. prio = xive_lock_and_mask(xive, sb, state);
  875. pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
  876. state->old_p, state->old_q);
  877. /*
  878. * If old_p is set, the interrupt is pending, we switch it to
  879. * PQ=11. This will force a resend in the host so the interrupt
  880. * isn't lost to whatver host driver may pick it up
  881. */
  882. if (state->old_p)
  883. xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
  884. /* Release the passed-through interrupt to the host */
  885. rc = irq_set_vcpu_affinity(host_irq, NULL);
  886. if (rc) {
  887. pr_err("Failed to clr VCPU affinity for irq %d\n", host_irq);
  888. return rc;
  889. }
  890. /* Forget about the IRQ */
  891. state->pt_number = 0;
  892. state->pt_data = NULL;
  893. /*
  894. * Reset ESB guest mapping. Needed when ESB pages are exposed
  895. * to the guest in XIVE native mode
  896. */
  897. if (xive->ops && xive->ops->reset_mapped) {
  898. xive->ops->reset_mapped(kvm, guest_irq);
  899. }
  900. /* Reconfigure the IPI */
  901. xive_native_configure_irq(state->ipi_number,
  902. kvmppc_xive_vp(xive, state->act_server),
  903. state->act_priority, state->number);
  904. /*
  905. * If old_p is set (we have a queue entry potentially
  906. * occupied) or the interrupt is masked, we set the IPI
  907. * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
  908. */
  909. if (prio == MASKED || state->old_p)
  910. xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
  911. else
  912. xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
  913. /* Restore guest prio (unlocks EOI) */
  914. mb();
  915. state->guest_priority = prio;
  916. arch_spin_unlock(&sb->lock);
  917. return 0;
  918. }
  919. EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
  920. void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
  921. {
  922. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  923. struct kvm *kvm = vcpu->kvm;
  924. struct kvmppc_xive *xive = kvm->arch.xive;
  925. int i, j;
  926. for (i = 0; i <= xive->max_sbid; i++) {
  927. struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
  928. if (!sb)
  929. continue;
  930. for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
  931. struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
  932. if (!state->valid)
  933. continue;
  934. if (state->act_priority == MASKED)
  935. continue;
  936. if (state->act_server != xc->server_num)
  937. continue;
  938. /* Clean it up */
  939. arch_spin_lock(&sb->lock);
  940. state->act_priority = MASKED;
  941. xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
  942. xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
  943. if (state->pt_number) {
  944. xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
  945. xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
  946. }
  947. arch_spin_unlock(&sb->lock);
  948. }
  949. }
  950. /* Disable vcpu's escalation interrupt */
  951. if (vcpu->arch.xive_esc_on) {
  952. __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
  953. XIVE_ESB_SET_PQ_01));
  954. vcpu->arch.xive_esc_on = false;
  955. }
  956. /*
  957. * Clear pointers to escalation interrupt ESB.
  958. * This is safe because the vcpu->mutex is held, preventing
  959. * any other CPU from concurrently executing a KVM_RUN ioctl.
  960. */
  961. vcpu->arch.xive_esc_vaddr = 0;
  962. vcpu->arch.xive_esc_raddr = 0;
  963. }
  964. /*
  965. * In single escalation mode, the escalation interrupt is marked so
  966. * that EOI doesn't re-enable it, but just sets the stale_p flag to
  967. * indicate that the P bit has already been dealt with. However, the
  968. * assembly code that enters the guest sets PQ to 00 without clearing
  969. * stale_p (because it has no easy way to address it). Hence we have
  970. * to adjust stale_p before shutting down the interrupt.
  971. */
  972. void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
  973. struct kvmppc_xive_vcpu *xc, int irq)
  974. {
  975. struct irq_data *d = irq_get_irq_data(irq);
  976. struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
  977. /*
  978. * This slightly odd sequence gives the right result
  979. * (i.e. stale_p set if xive_esc_on is false) even if
  980. * we race with xive_esc_irq() and xive_irq_eoi().
  981. */
  982. xd->stale_p = false;
  983. smp_mb(); /* paired with smb_wmb in xive_esc_irq */
  984. if (!vcpu->arch.xive_esc_on)
  985. xd->stale_p = true;
  986. }
  987. void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
  988. {
  989. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  990. struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
  991. int i;
  992. if (!kvmppc_xics_enabled(vcpu))
  993. return;
  994. if (!xc)
  995. return;
  996. pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
  997. /* Ensure no interrupt is still routed to that VP */
  998. xc->valid = false;
  999. kvmppc_xive_disable_vcpu_interrupts(vcpu);
  1000. /* Mask the VP IPI */
  1001. xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
  1002. /* Free escalations */
  1003. for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
  1004. if (xc->esc_virq[i]) {
  1005. if (xc->xive->single_escalation)
  1006. xive_cleanup_single_escalation(vcpu, xc,
  1007. xc->esc_virq[i]);
  1008. free_irq(xc->esc_virq[i], vcpu);
  1009. irq_dispose_mapping(xc->esc_virq[i]);
  1010. kfree(xc->esc_virq_names[i]);
  1011. }
  1012. }
  1013. /* Disable the VP */
  1014. xive_native_disable_vp(xc->vp_id);
  1015. /* Clear the cam word so guest entry won't try to push context */
  1016. vcpu->arch.xive_cam_word = 0;
  1017. /* Free the queues */
  1018. for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
  1019. struct xive_q *q = &xc->queues[i];
  1020. xive_native_disable_queue(xc->vp_id, q, i);
  1021. if (q->qpage) {
  1022. free_pages((unsigned long)q->qpage,
  1023. xive->q_page_order);
  1024. q->qpage = NULL;
  1025. }
  1026. }
  1027. /* Free the IPI */
  1028. if (xc->vp_ipi) {
  1029. xive_cleanup_irq_data(&xc->vp_ipi_data);
  1030. xive_native_free_irq(xc->vp_ipi);
  1031. }
  1032. /* Free the VP */
  1033. kfree(xc);
  1034. /* Cleanup the vcpu */
  1035. vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
  1036. vcpu->arch.xive_vcpu = NULL;
  1037. }
  1038. static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
  1039. {
  1040. /* We have a block of xive->nr_servers VPs. We just need to check
  1041. * packed vCPU ids are below that.
  1042. */
  1043. return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
  1044. }
  1045. int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
  1046. {
  1047. u32 vp_id;
  1048. if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
  1049. pr_devel("Out of bounds !\n");
  1050. return -EINVAL;
  1051. }
  1052. if (xive->vp_base == XIVE_INVALID_VP) {
  1053. xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
  1054. pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
  1055. if (xive->vp_base == XIVE_INVALID_VP)
  1056. return -ENOSPC;
  1057. }
  1058. vp_id = kvmppc_xive_vp(xive, cpu);
  1059. if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
  1060. pr_devel("Duplicate !\n");
  1061. return -EEXIST;
  1062. }
  1063. *vp = vp_id;
  1064. return 0;
  1065. }
  1066. int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
  1067. struct kvm_vcpu *vcpu, u32 cpu)
  1068. {
  1069. struct kvmppc_xive *xive = dev->private;
  1070. struct kvmppc_xive_vcpu *xc;
  1071. int i, r = -EBUSY;
  1072. u32 vp_id;
  1073. pr_devel("connect_vcpu(cpu=%d)\n", cpu);
  1074. if (dev->ops != &kvm_xive_ops) {
  1075. pr_devel("Wrong ops !\n");
  1076. return -EPERM;
  1077. }
  1078. if (xive->kvm != vcpu->kvm)
  1079. return -EPERM;
  1080. if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
  1081. return -EBUSY;
  1082. /* We need to synchronize with queue provisioning */
  1083. mutex_lock(&xive->lock);
  1084. r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
  1085. if (r)
  1086. goto bail;
  1087. xc = kzalloc(sizeof(*xc), GFP_KERNEL);
  1088. if (!xc) {
  1089. r = -ENOMEM;
  1090. goto bail;
  1091. }
  1092. vcpu->arch.xive_vcpu = xc;
  1093. xc->xive = xive;
  1094. xc->vcpu = vcpu;
  1095. xc->server_num = cpu;
  1096. xc->vp_id = vp_id;
  1097. xc->mfrr = 0xff;
  1098. xc->valid = true;
  1099. r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
  1100. if (r)
  1101. goto bail;
  1102. /* Configure VCPU fields for use by assembly push/pull */
  1103. vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
  1104. vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
  1105. /* Allocate IPI */
  1106. xc->vp_ipi = xive_native_alloc_irq();
  1107. if (!xc->vp_ipi) {
  1108. pr_err("Failed to allocate xive irq for VCPU IPI\n");
  1109. r = -EIO;
  1110. goto bail;
  1111. }
  1112. pr_devel(" IPI=0x%x\n", xc->vp_ipi);
  1113. r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
  1114. if (r)
  1115. goto bail;
  1116. /*
  1117. * Enable the VP first as the single escalation mode will
  1118. * affect escalation interrupts numbering
  1119. */
  1120. r = xive_native_enable_vp(xc->vp_id, xive->single_escalation);
  1121. if (r) {
  1122. pr_err("Failed to enable VP in OPAL, err %d\n", r);
  1123. goto bail;
  1124. }
  1125. /*
  1126. * Initialize queues. Initially we set them all for no queueing
  1127. * and we enable escalation for queue 0 only which we'll use for
  1128. * our mfrr change notifications. If the VCPU is hot-plugged, we
  1129. * do handle provisioning however based on the existing "map"
  1130. * of enabled queues.
  1131. */
  1132. for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
  1133. struct xive_q *q = &xc->queues[i];
  1134. /* Single escalation, no queue 7 */
  1135. if (i == 7 && xive->single_escalation)
  1136. break;
  1137. /* Is queue already enabled ? Provision it */
  1138. if (xive->qmap & (1 << i)) {
  1139. r = xive_provision_queue(vcpu, i);
  1140. if (r == 0 && !xive->single_escalation)
  1141. kvmppc_xive_attach_escalation(
  1142. vcpu, i, xive->single_escalation);
  1143. if (r)
  1144. goto bail;
  1145. } else {
  1146. r = xive_native_configure_queue(xc->vp_id,
  1147. q, i, NULL, 0, true);
  1148. if (r) {
  1149. pr_err("Failed to configure queue %d for VCPU %d\n",
  1150. i, cpu);
  1151. goto bail;
  1152. }
  1153. }
  1154. }
  1155. /* If not done above, attach priority 0 escalation */
  1156. r = kvmppc_xive_attach_escalation(vcpu, 0, xive->single_escalation);
  1157. if (r)
  1158. goto bail;
  1159. /* Route the IPI */
  1160. r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
  1161. if (!r)
  1162. xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
  1163. bail:
  1164. mutex_unlock(&xive->lock);
  1165. if (r) {
  1166. kvmppc_xive_cleanup_vcpu(vcpu);
  1167. return r;
  1168. }
  1169. vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
  1170. return 0;
  1171. }
  1172. /*
  1173. * Scanning of queues before/after migration save
  1174. */
  1175. static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
  1176. {
  1177. struct kvmppc_xive_src_block *sb;
  1178. struct kvmppc_xive_irq_state *state;
  1179. u16 idx;
  1180. sb = kvmppc_xive_find_source(xive, irq, &idx);
  1181. if (!sb)
  1182. return;
  1183. state = &sb->irq_state[idx];
  1184. /* Some sanity checking */
  1185. if (!state->valid) {
  1186. pr_err("invalid irq 0x%x in cpu queue!\n", irq);
  1187. return;
  1188. }
  1189. /*
  1190. * If the interrupt is in a queue it should have P set.
  1191. * We warn so that gets reported. A backtrace isn't useful
  1192. * so no need to use a WARN_ON.
  1193. */
  1194. if (!state->saved_p)
  1195. pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
  1196. /* Set flag */
  1197. state->in_queue = true;
  1198. }
  1199. static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
  1200. struct kvmppc_xive_src_block *sb,
  1201. u32 irq)
  1202. {
  1203. struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
  1204. if (!state->valid)
  1205. return;
  1206. /* Mask and save state, this will also sync HW queues */
  1207. state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
  1208. /* Transfer P and Q */
  1209. state->saved_p = state->old_p;
  1210. state->saved_q = state->old_q;
  1211. /* Unlock */
  1212. arch_spin_unlock(&sb->lock);
  1213. }
  1214. static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
  1215. struct kvmppc_xive_src_block *sb,
  1216. u32 irq)
  1217. {
  1218. struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
  1219. if (!state->valid)
  1220. return;
  1221. /*
  1222. * Lock / exclude EOI (not technically necessary if the
  1223. * guest isn't running concurrently. If this becomes a
  1224. * performance issue we can probably remove the lock.
  1225. */
  1226. xive_lock_for_unmask(sb, state);
  1227. /* Restore mask/prio if it wasn't masked */
  1228. if (state->saved_scan_prio != MASKED)
  1229. xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
  1230. /* Unlock */
  1231. arch_spin_unlock(&sb->lock);
  1232. }
  1233. static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
  1234. {
  1235. u32 idx = q->idx;
  1236. u32 toggle = q->toggle;
  1237. u32 irq;
  1238. do {
  1239. irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
  1240. if (irq > XICS_IPI)
  1241. xive_pre_save_set_queued(xive, irq);
  1242. } while(irq);
  1243. }
  1244. static void xive_pre_save_scan(struct kvmppc_xive *xive)
  1245. {
  1246. struct kvm_vcpu *vcpu = NULL;
  1247. int i, j;
  1248. /*
  1249. * See comment in xive_get_source() about how this
  1250. * work. Collect a stable state for all interrupts
  1251. */
  1252. for (i = 0; i <= xive->max_sbid; i++) {
  1253. struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
  1254. if (!sb)
  1255. continue;
  1256. for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
  1257. xive_pre_save_mask_irq(xive, sb, j);
  1258. }
  1259. /* Then scan the queues and update the "in_queue" flag */
  1260. kvm_for_each_vcpu(i, vcpu, xive->kvm) {
  1261. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  1262. if (!xc)
  1263. continue;
  1264. for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
  1265. if (xc->queues[j].qpage)
  1266. xive_pre_save_queue(xive, &xc->queues[j]);
  1267. }
  1268. }
  1269. /* Finally restore interrupt states */
  1270. for (i = 0; i <= xive->max_sbid; i++) {
  1271. struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
  1272. if (!sb)
  1273. continue;
  1274. for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
  1275. xive_pre_save_unmask_irq(xive, sb, j);
  1276. }
  1277. }
  1278. static void xive_post_save_scan(struct kvmppc_xive *xive)
  1279. {
  1280. u32 i, j;
  1281. /* Clear all the in_queue flags */
  1282. for (i = 0; i <= xive->max_sbid; i++) {
  1283. struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
  1284. if (!sb)
  1285. continue;
  1286. for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
  1287. sb->irq_state[j].in_queue = false;
  1288. }
  1289. /* Next get_source() will do a new scan */
  1290. xive->saved_src_count = 0;
  1291. }
  1292. /*
  1293. * This returns the source configuration and state to user space.
  1294. */
  1295. static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
  1296. {
  1297. struct kvmppc_xive_src_block *sb;
  1298. struct kvmppc_xive_irq_state *state;
  1299. u64 __user *ubufp = (u64 __user *) addr;
  1300. u64 val, prio;
  1301. u16 idx;
  1302. sb = kvmppc_xive_find_source(xive, irq, &idx);
  1303. if (!sb)
  1304. return -ENOENT;
  1305. state = &sb->irq_state[idx];
  1306. if (!state->valid)
  1307. return -ENOENT;
  1308. pr_devel("get_source(%ld)...\n", irq);
  1309. /*
  1310. * So to properly save the state into something that looks like a
  1311. * XICS migration stream we cannot treat interrupts individually.
  1312. *
  1313. * We need, instead, mask them all (& save their previous PQ state)
  1314. * to get a stable state in the HW, then sync them to ensure that
  1315. * any interrupt that had already fired hits its queue, and finally
  1316. * scan all the queues to collect which interrupts are still present
  1317. * in the queues, so we can set the "pending" flag on them and
  1318. * they can be resent on restore.
  1319. *
  1320. * So we do it all when the "first" interrupt gets saved, all the
  1321. * state is collected at that point, the rest of xive_get_source()
  1322. * will merely collect and convert that state to the expected
  1323. * userspace bit mask.
  1324. */
  1325. if (xive->saved_src_count == 0)
  1326. xive_pre_save_scan(xive);
  1327. xive->saved_src_count++;
  1328. /* Convert saved state into something compatible with xics */
  1329. val = state->act_server;
  1330. prio = state->saved_scan_prio;
  1331. if (prio == MASKED) {
  1332. val |= KVM_XICS_MASKED;
  1333. prio = state->saved_priority;
  1334. }
  1335. val |= prio << KVM_XICS_PRIORITY_SHIFT;
  1336. if (state->lsi) {
  1337. val |= KVM_XICS_LEVEL_SENSITIVE;
  1338. if (state->saved_p)
  1339. val |= KVM_XICS_PENDING;
  1340. } else {
  1341. if (state->saved_p)
  1342. val |= KVM_XICS_PRESENTED;
  1343. if (state->saved_q)
  1344. val |= KVM_XICS_QUEUED;
  1345. /*
  1346. * We mark it pending (which will attempt a re-delivery)
  1347. * if we are in a queue *or* we were masked and had
  1348. * Q set which is equivalent to the XICS "masked pending"
  1349. * state
  1350. */
  1351. if (state->in_queue || (prio == MASKED && state->saved_q))
  1352. val |= KVM_XICS_PENDING;
  1353. }
  1354. /*
  1355. * If that was the last interrupt saved, reset the
  1356. * in_queue flags
  1357. */
  1358. if (xive->saved_src_count == xive->src_count)
  1359. xive_post_save_scan(xive);
  1360. /* Copy the result to userspace */
  1361. if (put_user(val, ubufp))
  1362. return -EFAULT;
  1363. return 0;
  1364. }
  1365. struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
  1366. struct kvmppc_xive *xive, int irq)
  1367. {
  1368. struct kvmppc_xive_src_block *sb;
  1369. int i, bid;
  1370. bid = irq >> KVMPPC_XICS_ICS_SHIFT;
  1371. mutex_lock(&xive->lock);
  1372. /* block already exists - somebody else got here first */
  1373. if (xive->src_blocks[bid])
  1374. goto out;
  1375. /* Create the ICS */
  1376. sb = kzalloc(sizeof(*sb), GFP_KERNEL);
  1377. if (!sb)
  1378. goto out;
  1379. sb->id = bid;
  1380. for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
  1381. sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
  1382. sb->irq_state[i].eisn = 0;
  1383. sb->irq_state[i].guest_priority = MASKED;
  1384. sb->irq_state[i].saved_priority = MASKED;
  1385. sb->irq_state[i].act_priority = MASKED;
  1386. }
  1387. smp_wmb();
  1388. xive->src_blocks[bid] = sb;
  1389. if (bid > xive->max_sbid)
  1390. xive->max_sbid = bid;
  1391. out:
  1392. mutex_unlock(&xive->lock);
  1393. return xive->src_blocks[bid];
  1394. }
  1395. static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
  1396. {
  1397. struct kvm *kvm = xive->kvm;
  1398. struct kvm_vcpu *vcpu = NULL;
  1399. int i;
  1400. kvm_for_each_vcpu(i, vcpu, kvm) {
  1401. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  1402. if (!xc)
  1403. continue;
  1404. if (xc->delayed_irq == irq) {
  1405. xc->delayed_irq = 0;
  1406. xive->delayed_irqs--;
  1407. return true;
  1408. }
  1409. }
  1410. return false;
  1411. }
  1412. static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
  1413. {
  1414. struct kvmppc_xive_src_block *sb;
  1415. struct kvmppc_xive_irq_state *state;
  1416. u64 __user *ubufp = (u64 __user *) addr;
  1417. u16 idx;
  1418. u64 val;
  1419. u8 act_prio, guest_prio;
  1420. u32 server;
  1421. int rc = 0;
  1422. if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
  1423. return -ENOENT;
  1424. pr_devel("set_source(irq=0x%lx)\n", irq);
  1425. /* Find the source */
  1426. sb = kvmppc_xive_find_source(xive, irq, &idx);
  1427. if (!sb) {
  1428. pr_devel("No source, creating source block...\n");
  1429. sb = kvmppc_xive_create_src_block(xive, irq);
  1430. if (!sb) {
  1431. pr_devel("Failed to create block...\n");
  1432. return -ENOMEM;
  1433. }
  1434. }
  1435. state = &sb->irq_state[idx];
  1436. /* Read user passed data */
  1437. if (get_user(val, ubufp)) {
  1438. pr_devel("fault getting user info !\n");
  1439. return -EFAULT;
  1440. }
  1441. server = val & KVM_XICS_DESTINATION_MASK;
  1442. guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
  1443. pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n",
  1444. val, server, guest_prio);
  1445. /*
  1446. * If the source doesn't already have an IPI, allocate
  1447. * one and get the corresponding data
  1448. */
  1449. if (!state->ipi_number) {
  1450. state->ipi_number = xive_native_alloc_irq();
  1451. if (state->ipi_number == 0) {
  1452. pr_devel("Failed to allocate IPI !\n");
  1453. return -ENOMEM;
  1454. }
  1455. xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
  1456. pr_devel(" src_ipi=0x%x\n", state->ipi_number);
  1457. }
  1458. /*
  1459. * We use lock_and_mask() to set us in the right masked
  1460. * state. We will override that state from the saved state
  1461. * further down, but this will handle the cases of interrupts
  1462. * that need FW masking. We set the initial guest_priority to
  1463. * 0 before calling it to ensure it actually performs the masking.
  1464. */
  1465. state->guest_priority = 0;
  1466. xive_lock_and_mask(xive, sb, state);
  1467. /*
  1468. * Now, we select a target if we have one. If we don't we
  1469. * leave the interrupt untargetted. It means that an interrupt
  1470. * can become "untargetted" accross migration if it was masked
  1471. * by set_xive() but there is little we can do about it.
  1472. */
  1473. /* First convert prio and mark interrupt as untargetted */
  1474. act_prio = xive_prio_from_guest(guest_prio);
  1475. state->act_priority = MASKED;
  1476. /*
  1477. * We need to drop the lock due to the mutex below. Hopefully
  1478. * nothing is touching that interrupt yet since it hasn't been
  1479. * advertized to a running guest yet
  1480. */
  1481. arch_spin_unlock(&sb->lock);
  1482. /* If we have a priority target the interrupt */
  1483. if (act_prio != MASKED) {
  1484. /* First, check provisioning of queues */
  1485. mutex_lock(&xive->lock);
  1486. rc = xive_check_provisioning(xive->kvm, act_prio);
  1487. mutex_unlock(&xive->lock);
  1488. /* Target interrupt */
  1489. if (rc == 0)
  1490. rc = xive_target_interrupt(xive->kvm, state,
  1491. server, act_prio);
  1492. /*
  1493. * If provisioning or targetting failed, leave it
  1494. * alone and masked. It will remain disabled until
  1495. * the guest re-targets it.
  1496. */
  1497. }
  1498. /*
  1499. * Find out if this was a delayed irq stashed in an ICP,
  1500. * in which case, treat it as pending
  1501. */
  1502. if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
  1503. val |= KVM_XICS_PENDING;
  1504. pr_devel(" Found delayed ! forcing PENDING !\n");
  1505. }
  1506. /* Cleanup the SW state */
  1507. state->old_p = false;
  1508. state->old_q = false;
  1509. state->lsi = false;
  1510. state->asserted = false;
  1511. /* Restore LSI state */
  1512. if (val & KVM_XICS_LEVEL_SENSITIVE) {
  1513. state->lsi = true;
  1514. if (val & KVM_XICS_PENDING)
  1515. state->asserted = true;
  1516. pr_devel(" LSI ! Asserted=%d\n", state->asserted);
  1517. }
  1518. /*
  1519. * Restore P and Q. If the interrupt was pending, we
  1520. * force Q and !P, which will trigger a resend.
  1521. *
  1522. * That means that a guest that had both an interrupt
  1523. * pending (queued) and Q set will restore with only
  1524. * one instance of that interrupt instead of 2, but that
  1525. * is perfectly fine as coalescing interrupts that haven't
  1526. * been presented yet is always allowed.
  1527. */
  1528. if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
  1529. state->old_p = true;
  1530. if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
  1531. state->old_q = true;
  1532. pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q);
  1533. /*
  1534. * If the interrupt was unmasked, update guest priority and
  1535. * perform the appropriate state transition and do a
  1536. * re-trigger if necessary.
  1537. */
  1538. if (val & KVM_XICS_MASKED) {
  1539. pr_devel(" masked, saving prio\n");
  1540. state->guest_priority = MASKED;
  1541. state->saved_priority = guest_prio;
  1542. } else {
  1543. pr_devel(" unmasked, restoring to prio %d\n", guest_prio);
  1544. xive_finish_unmask(xive, sb, state, guest_prio);
  1545. state->saved_priority = guest_prio;
  1546. }
  1547. /* Increment the number of valid sources and mark this one valid */
  1548. if (!state->valid)
  1549. xive->src_count++;
  1550. state->valid = true;
  1551. return 0;
  1552. }
  1553. int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
  1554. bool line_status)
  1555. {
  1556. struct kvmppc_xive *xive = kvm->arch.xive;
  1557. struct kvmppc_xive_src_block *sb;
  1558. struct kvmppc_xive_irq_state *state;
  1559. u16 idx;
  1560. if (!xive)
  1561. return -ENODEV;
  1562. sb = kvmppc_xive_find_source(xive, irq, &idx);
  1563. if (!sb)
  1564. return -EINVAL;
  1565. /* Perform locklessly .... (we need to do some RCUisms here...) */
  1566. state = &sb->irq_state[idx];
  1567. if (!state->valid)
  1568. return -EINVAL;
  1569. /* We don't allow a trigger on a passed-through interrupt */
  1570. if (state->pt_number)
  1571. return -EINVAL;
  1572. if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
  1573. state->asserted = 1;
  1574. else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
  1575. state->asserted = 0;
  1576. return 0;
  1577. }
  1578. /* Trigger the IPI */
  1579. xive_irq_trigger(&state->ipi_data);
  1580. return 0;
  1581. }
  1582. int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
  1583. {
  1584. u32 __user *ubufp = (u32 __user *) addr;
  1585. u32 nr_servers;
  1586. int rc = 0;
  1587. if (get_user(nr_servers, ubufp))
  1588. return -EFAULT;
  1589. pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
  1590. if (!nr_servers || nr_servers > KVM_MAX_VCPU_ID)
  1591. return -EINVAL;
  1592. mutex_lock(&xive->lock);
  1593. if (xive->vp_base != XIVE_INVALID_VP)
  1594. /* The VP block is allocated once and freed when the device
  1595. * is released. Better not allow to change its size since its
  1596. * used by connect_vcpu to validate vCPU ids are valid (eg,
  1597. * setting it back to a higher value could allow connect_vcpu
  1598. * to come up with a VP id that goes beyond the VP block, which
  1599. * is likely to cause a crash in OPAL).
  1600. */
  1601. rc = -EBUSY;
  1602. else if (nr_servers > KVM_MAX_VCPUS)
  1603. /* We don't need more servers. Higher vCPU ids get packed
  1604. * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
  1605. */
  1606. xive->nr_servers = KVM_MAX_VCPUS;
  1607. else
  1608. xive->nr_servers = nr_servers;
  1609. mutex_unlock(&xive->lock);
  1610. return rc;
  1611. }
  1612. static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
  1613. {
  1614. struct kvmppc_xive *xive = dev->private;
  1615. /* We honor the existing XICS ioctl */
  1616. switch (attr->group) {
  1617. case KVM_DEV_XICS_GRP_SOURCES:
  1618. return xive_set_source(xive, attr->attr, attr->addr);
  1619. case KVM_DEV_XICS_GRP_CTRL:
  1620. switch (attr->attr) {
  1621. case KVM_DEV_XICS_NR_SERVERS:
  1622. return kvmppc_xive_set_nr_servers(xive, attr->addr);
  1623. }
  1624. }
  1625. return -ENXIO;
  1626. }
  1627. static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
  1628. {
  1629. struct kvmppc_xive *xive = dev->private;
  1630. /* We honor the existing XICS ioctl */
  1631. switch (attr->group) {
  1632. case KVM_DEV_XICS_GRP_SOURCES:
  1633. return xive_get_source(xive, attr->attr, attr->addr);
  1634. }
  1635. return -ENXIO;
  1636. }
  1637. static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
  1638. {
  1639. /* We honor the same limits as XICS, at least for now */
  1640. switch (attr->group) {
  1641. case KVM_DEV_XICS_GRP_SOURCES:
  1642. if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
  1643. attr->attr < KVMPPC_XICS_NR_IRQS)
  1644. return 0;
  1645. break;
  1646. case KVM_DEV_XICS_GRP_CTRL:
  1647. switch (attr->attr) {
  1648. case KVM_DEV_XICS_NR_SERVERS:
  1649. return 0;
  1650. }
  1651. }
  1652. return -ENXIO;
  1653. }
  1654. static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
  1655. {
  1656. xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
  1657. xive_native_configure_irq(hw_num, 0, MASKED, 0);
  1658. }
  1659. void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
  1660. {
  1661. int i;
  1662. for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
  1663. struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
  1664. if (!state->valid)
  1665. continue;
  1666. kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
  1667. xive_cleanup_irq_data(&state->ipi_data);
  1668. xive_native_free_irq(state->ipi_number);
  1669. /* Pass-through, cleanup too but keep IRQ hw data */
  1670. if (state->pt_number)
  1671. kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
  1672. state->valid = false;
  1673. }
  1674. }
  1675. /*
  1676. * Called when device fd is closed. kvm->lock is held.
  1677. */
  1678. static void kvmppc_xive_release(struct kvm_device *dev)
  1679. {
  1680. struct kvmppc_xive *xive = dev->private;
  1681. struct kvm *kvm = xive->kvm;
  1682. struct kvm_vcpu *vcpu;
  1683. int i;
  1684. pr_devel("Releasing xive device\n");
  1685. /*
  1686. * Since this is the device release function, we know that
  1687. * userspace does not have any open fd referring to the
  1688. * device. Therefore there can not be any of the device
  1689. * attribute set/get functions being executed concurrently,
  1690. * and similarly, the connect_vcpu and set/clr_mapped
  1691. * functions also cannot be being executed.
  1692. */
  1693. debugfs_remove(xive->dentry);
  1694. /*
  1695. * We should clean up the vCPU interrupt presenters first.
  1696. */
  1697. kvm_for_each_vcpu(i, vcpu, kvm) {
  1698. /*
  1699. * Take vcpu->mutex to ensure that no one_reg get/set ioctl
  1700. * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
  1701. * Holding the vcpu->mutex also means that the vcpu cannot
  1702. * be executing the KVM_RUN ioctl, and therefore it cannot
  1703. * be executing the XIVE push or pull code or accessing
  1704. * the XIVE MMIO regions.
  1705. */
  1706. mutex_lock(&vcpu->mutex);
  1707. kvmppc_xive_cleanup_vcpu(vcpu);
  1708. mutex_unlock(&vcpu->mutex);
  1709. }
  1710. /*
  1711. * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
  1712. * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
  1713. * against xive code getting called during vcpu execution or
  1714. * set/get one_reg operations.
  1715. */
  1716. kvm->arch.xive = NULL;
  1717. /* Mask and free interrupts */
  1718. for (i = 0; i <= xive->max_sbid; i++) {
  1719. if (xive->src_blocks[i])
  1720. kvmppc_xive_free_sources(xive->src_blocks[i]);
  1721. kfree(xive->src_blocks[i]);
  1722. xive->src_blocks[i] = NULL;
  1723. }
  1724. if (xive->vp_base != XIVE_INVALID_VP)
  1725. xive_native_free_vp_block(xive->vp_base);
  1726. /*
  1727. * A reference of the kvmppc_xive pointer is now kept under
  1728. * the xive_devices struct of the machine for reuse. It is
  1729. * freed when the VM is destroyed for now until we fix all the
  1730. * execution paths.
  1731. */
  1732. kfree(dev);
  1733. }
  1734. /*
  1735. * When the guest chooses the interrupt mode (XICS legacy or XIVE
  1736. * native), the VM will switch of KVM device. The previous device will
  1737. * be "released" before the new one is created.
  1738. *
  1739. * Until we are sure all execution paths are well protected, provide a
  1740. * fail safe (transitional) method for device destruction, in which
  1741. * the XIVE device pointer is recycled and not directly freed.
  1742. */
  1743. struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
  1744. {
  1745. struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
  1746. &kvm->arch.xive_devices.native :
  1747. &kvm->arch.xive_devices.xics_on_xive;
  1748. struct kvmppc_xive *xive = *kvm_xive_device;
  1749. if (!xive) {
  1750. xive = kzalloc(sizeof(*xive), GFP_KERNEL);
  1751. *kvm_xive_device = xive;
  1752. } else {
  1753. memset(xive, 0, sizeof(*xive));
  1754. }
  1755. return xive;
  1756. }
  1757. /*
  1758. * Create a XICS device with XIVE backend. kvm->lock is held.
  1759. */
  1760. static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
  1761. {
  1762. struct kvmppc_xive *xive;
  1763. struct kvm *kvm = dev->kvm;
  1764. pr_devel("Creating xive for partition\n");
  1765. /* Already there ? */
  1766. if (kvm->arch.xive)
  1767. return -EEXIST;
  1768. xive = kvmppc_xive_get_device(kvm, type);
  1769. if (!xive)
  1770. return -ENOMEM;
  1771. dev->private = xive;
  1772. xive->dev = dev;
  1773. xive->kvm = kvm;
  1774. mutex_init(&xive->lock);
  1775. /* We use the default queue size set by the host */
  1776. xive->q_order = xive_native_default_eq_shift();
  1777. if (xive->q_order < PAGE_SHIFT)
  1778. xive->q_page_order = 0;
  1779. else
  1780. xive->q_page_order = xive->q_order - PAGE_SHIFT;
  1781. /* VP allocation is delayed to the first call to connect_vcpu */
  1782. xive->vp_base = XIVE_INVALID_VP;
  1783. /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
  1784. * on a POWER9 system.
  1785. */
  1786. xive->nr_servers = KVM_MAX_VCPUS;
  1787. xive->single_escalation = xive_native_has_single_escalation();
  1788. kvm->arch.xive = xive;
  1789. return 0;
  1790. }
  1791. int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
  1792. {
  1793. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  1794. unsigned int i;
  1795. for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
  1796. struct xive_q *q = &xc->queues[i];
  1797. u32 i0, i1, idx;
  1798. if (!q->qpage && !xc->esc_virq[i])
  1799. continue;
  1800. seq_printf(m, " [q%d]: ", i);
  1801. if (q->qpage) {
  1802. idx = q->idx;
  1803. i0 = be32_to_cpup(q->qpage + idx);
  1804. idx = (idx + 1) & q->msk;
  1805. i1 = be32_to_cpup(q->qpage + idx);
  1806. seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
  1807. i0, i1);
  1808. }
  1809. if (xc->esc_virq[i]) {
  1810. struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
  1811. struct xive_irq_data *xd =
  1812. irq_data_get_irq_handler_data(d);
  1813. u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
  1814. seq_printf(m, "E:%c%c I(%d:%llx:%llx)",
  1815. (pq & XIVE_ESB_VAL_P) ? 'P' : 'p',
  1816. (pq & XIVE_ESB_VAL_Q) ? 'Q' : 'q',
  1817. xc->esc_virq[i], pq, xd->eoi_page);
  1818. seq_puts(m, "\n");
  1819. }
  1820. }
  1821. return 0;
  1822. }
  1823. static int xive_debug_show(struct seq_file *m, void *private)
  1824. {
  1825. struct kvmppc_xive *xive = m->private;
  1826. struct kvm *kvm = xive->kvm;
  1827. struct kvm_vcpu *vcpu;
  1828. u64 t_rm_h_xirr = 0;
  1829. u64 t_rm_h_ipoll = 0;
  1830. u64 t_rm_h_cppr = 0;
  1831. u64 t_rm_h_eoi = 0;
  1832. u64 t_rm_h_ipi = 0;
  1833. u64 t_vm_h_xirr = 0;
  1834. u64 t_vm_h_ipoll = 0;
  1835. u64 t_vm_h_cppr = 0;
  1836. u64 t_vm_h_eoi = 0;
  1837. u64 t_vm_h_ipi = 0;
  1838. unsigned int i;
  1839. if (!kvm)
  1840. return 0;
  1841. seq_printf(m, "=========\nVCPU state\n=========\n");
  1842. kvm_for_each_vcpu(i, vcpu, kvm) {
  1843. struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
  1844. if (!xc)
  1845. continue;
  1846. seq_printf(m, "cpu server %#x VP:%#x CPPR:%#x HWCPPR:%#x"
  1847. " MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
  1848. xc->server_num, xc->vp_id, xc->cppr, xc->hw_cppr,
  1849. xc->mfrr, xc->pending,
  1850. xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
  1851. kvmppc_xive_debug_show_queues(m, vcpu);
  1852. t_rm_h_xirr += xc->stat_rm_h_xirr;
  1853. t_rm_h_ipoll += xc->stat_rm_h_ipoll;
  1854. t_rm_h_cppr += xc->stat_rm_h_cppr;
  1855. t_rm_h_eoi += xc->stat_rm_h_eoi;
  1856. t_rm_h_ipi += xc->stat_rm_h_ipi;
  1857. t_vm_h_xirr += xc->stat_vm_h_xirr;
  1858. t_vm_h_ipoll += xc->stat_vm_h_ipoll;
  1859. t_vm_h_cppr += xc->stat_vm_h_cppr;
  1860. t_vm_h_eoi += xc->stat_vm_h_eoi;
  1861. t_vm_h_ipi += xc->stat_vm_h_ipi;
  1862. }
  1863. seq_printf(m, "Hcalls totals\n");
  1864. seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
  1865. seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
  1866. seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
  1867. seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
  1868. seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
  1869. return 0;
  1870. }
  1871. DEFINE_SHOW_ATTRIBUTE(xive_debug);
  1872. static void xive_debugfs_init(struct kvmppc_xive *xive)
  1873. {
  1874. char *name;
  1875. name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive);
  1876. if (!name) {
  1877. pr_err("%s: no memory for name\n", __func__);
  1878. return;
  1879. }
  1880. xive->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root,
  1881. xive, &xive_debug_fops);
  1882. pr_debug("%s: created %s\n", __func__, name);
  1883. kfree(name);
  1884. }
  1885. static void kvmppc_xive_init(struct kvm_device *dev)
  1886. {
  1887. struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private;
  1888. /* Register some debug interfaces */
  1889. xive_debugfs_init(xive);
  1890. }
  1891. struct kvm_device_ops kvm_xive_ops = {
  1892. .name = "kvm-xive",
  1893. .create = kvmppc_xive_create,
  1894. .init = kvmppc_xive_init,
  1895. .release = kvmppc_xive_release,
  1896. .set_attr = xive_set_attr,
  1897. .get_attr = xive_get_attr,
  1898. .has_attr = xive_has_attr,
  1899. };
  1900. void kvmppc_xive_init_module(void)
  1901. {
  1902. __xive_vm_h_xirr = xive_vm_h_xirr;
  1903. __xive_vm_h_ipoll = xive_vm_h_ipoll;
  1904. __xive_vm_h_ipi = xive_vm_h_ipi;
  1905. __xive_vm_h_cppr = xive_vm_h_cppr;
  1906. __xive_vm_h_eoi = xive_vm_h_eoi;
  1907. }
  1908. void kvmppc_xive_exit_module(void)
  1909. {
  1910. __xive_vm_h_xirr = NULL;
  1911. __xive_vm_h_ipoll = NULL;
  1912. __xive_vm_h_ipi = NULL;
  1913. __xive_vm_h_cppr = NULL;
  1914. __xive_vm_h_eoi = NULL;
  1915. }