eventfd.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kvm eventfd support - use eventfd objects to signal various KVM events
  4. *
  5. * Copyright 2009 Novell. All Rights Reserved.
  6. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  7. *
  8. * Author:
  9. * Gregory Haskins <ghaskins@novell.com>
  10. */
  11. #include <linux/kvm_host.h>
  12. #include <linux/kvm.h>
  13. #include <linux/kvm_irqfd.h>
  14. #include <linux/workqueue.h>
  15. #include <linux/syscalls.h>
  16. #include <linux/wait.h>
  17. #include <linux/poll.h>
  18. #include <linux/file.h>
  19. #include <linux/list.h>
  20. #include <linux/eventfd.h>
  21. #include <linux/kernel.h>
  22. #include <linux/srcu.h>
  23. #include <linux/slab.h>
  24. #include <linux/seqlock.h>
  25. #include <linux/irqbypass.h>
  26. #include <trace/events/kvm.h>
  27. #include <kvm/iodev.h>
  28. #ifdef CONFIG_HAVE_KVM_IRQFD
  29. static struct workqueue_struct *irqfd_cleanup_wq;
  30. bool __attribute__((weak))
  31. kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
  32. {
  33. return true;
  34. }
  35. static void
  36. irqfd_inject(struct work_struct *work)
  37. {
  38. struct kvm_kernel_irqfd *irqfd =
  39. container_of(work, struct kvm_kernel_irqfd, inject);
  40. struct kvm *kvm = irqfd->kvm;
  41. if (!irqfd->resampler) {
  42. kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,
  43. false);
  44. kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,
  45. false);
  46. } else
  47. kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  48. irqfd->gsi, 1, false);
  49. }
  50. /*
  51. * Since resampler irqfds share an IRQ source ID, we de-assert once
  52. * then notify all of the resampler irqfds using this GSI. We can't
  53. * do multiple de-asserts or we risk racing with incoming re-asserts.
  54. */
  55. static void
  56. irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian)
  57. {
  58. struct kvm_kernel_irqfd_resampler *resampler;
  59. struct kvm *kvm;
  60. struct kvm_kernel_irqfd *irqfd;
  61. int idx;
  62. resampler = container_of(kian,
  63. struct kvm_kernel_irqfd_resampler, notifier);
  64. kvm = resampler->kvm;
  65. kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  66. resampler->notifier.gsi, 0, false);
  67. idx = srcu_read_lock(&kvm->irq_srcu);
  68. list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)
  69. eventfd_signal(irqfd->resamplefd, 1);
  70. srcu_read_unlock(&kvm->irq_srcu, idx);
  71. }
  72. static void
  73. irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd)
  74. {
  75. struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;
  76. struct kvm *kvm = resampler->kvm;
  77. mutex_lock(&kvm->irqfds.resampler_lock);
  78. list_del_rcu(&irqfd->resampler_link);
  79. synchronize_srcu(&kvm->irq_srcu);
  80. if (list_empty(&resampler->list)) {
  81. list_del(&resampler->link);
  82. kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);
  83. kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
  84. resampler->notifier.gsi, 0, false);
  85. kfree(resampler);
  86. }
  87. mutex_unlock(&kvm->irqfds.resampler_lock);
  88. }
  89. /*
  90. * Race-free decouple logic (ordering is critical)
  91. */
  92. static void
  93. irqfd_shutdown(struct work_struct *work)
  94. {
  95. struct kvm_kernel_irqfd *irqfd =
  96. container_of(work, struct kvm_kernel_irqfd, shutdown);
  97. struct kvm *kvm = irqfd->kvm;
  98. u64 cnt;
  99. /* Make sure irqfd has been initialized in assign path. */
  100. synchronize_srcu(&kvm->irq_srcu);
  101. /*
  102. * Synchronize with the wait-queue and unhook ourselves to prevent
  103. * further events.
  104. */
  105. eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);
  106. /*
  107. * We know no new events will be scheduled at this point, so block
  108. * until all previously outstanding events have completed
  109. */
  110. flush_work(&irqfd->inject);
  111. if (irqfd->resampler) {
  112. irqfd_resampler_shutdown(irqfd);
  113. eventfd_ctx_put(irqfd->resamplefd);
  114. }
  115. /*
  116. * It is now safe to release the object's resources
  117. */
  118. #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
  119. irq_bypass_unregister_consumer(&irqfd->consumer);
  120. #endif
  121. eventfd_ctx_put(irqfd->eventfd);
  122. kfree(irqfd);
  123. }
  124. /* assumes kvm->irqfds.lock is held */
  125. static bool
  126. irqfd_is_active(struct kvm_kernel_irqfd *irqfd)
  127. {
  128. return list_empty(&irqfd->list) ? false : true;
  129. }
  130. /*
  131. * Mark the irqfd as inactive and schedule it for removal
  132. *
  133. * assumes kvm->irqfds.lock is held
  134. */
  135. static void
  136. irqfd_deactivate(struct kvm_kernel_irqfd *irqfd)
  137. {
  138. BUG_ON(!irqfd_is_active(irqfd));
  139. list_del_init(&irqfd->list);
  140. queue_work(irqfd_cleanup_wq, &irqfd->shutdown);
  141. }
  142. int __attribute__((weak)) kvm_arch_set_irq_inatomic(
  143. struct kvm_kernel_irq_routing_entry *irq,
  144. struct kvm *kvm, int irq_source_id,
  145. int level,
  146. bool line_status)
  147. {
  148. return -EWOULDBLOCK;
  149. }
  150. /*
  151. * Called with wqh->lock held and interrupts disabled
  152. */
  153. static int
  154. irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  155. {
  156. struct kvm_kernel_irqfd *irqfd =
  157. container_of(wait, struct kvm_kernel_irqfd, wait);
  158. __poll_t flags = key_to_poll(key);
  159. struct kvm_kernel_irq_routing_entry irq;
  160. struct kvm *kvm = irqfd->kvm;
  161. unsigned seq;
  162. int idx;
  163. if (flags & EPOLLIN) {
  164. idx = srcu_read_lock(&kvm->irq_srcu);
  165. do {
  166. seq = read_seqcount_begin(&irqfd->irq_entry_sc);
  167. irq = irqfd->irq_entry;
  168. } while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));
  169. /* An event has been signaled, inject an interrupt */
  170. if (kvm_arch_set_irq_inatomic(&irq, kvm,
  171. KVM_USERSPACE_IRQ_SOURCE_ID, 1,
  172. false) == -EWOULDBLOCK)
  173. schedule_work(&irqfd->inject);
  174. srcu_read_unlock(&kvm->irq_srcu, idx);
  175. }
  176. if (flags & EPOLLHUP) {
  177. /* The eventfd is closing, detach from KVM */
  178. unsigned long iflags;
  179. spin_lock_irqsave(&kvm->irqfds.lock, iflags);
  180. /*
  181. * We must check if someone deactivated the irqfd before
  182. * we could acquire the irqfds.lock since the item is
  183. * deactivated from the KVM side before it is unhooked from
  184. * the wait-queue. If it is already deactivated, we can
  185. * simply return knowing the other side will cleanup for us.
  186. * We cannot race against the irqfd going away since the
  187. * other side is required to acquire wqh->lock, which we hold
  188. */
  189. if (irqfd_is_active(irqfd))
  190. irqfd_deactivate(irqfd);
  191. spin_unlock_irqrestore(&kvm->irqfds.lock, iflags);
  192. }
  193. return 0;
  194. }
  195. static void
  196. irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh,
  197. poll_table *pt)
  198. {
  199. struct kvm_kernel_irqfd *irqfd =
  200. container_of(pt, struct kvm_kernel_irqfd, pt);
  201. add_wait_queue(wqh, &irqfd->wait);
  202. }
  203. /* Must be called under irqfds.lock */
  204. static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)
  205. {
  206. struct kvm_kernel_irq_routing_entry *e;
  207. struct kvm_kernel_irq_routing_entry entries[KVM_NR_IRQCHIPS];
  208. int n_entries;
  209. n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);
  210. write_seqcount_begin(&irqfd->irq_entry_sc);
  211. e = entries;
  212. if (n_entries == 1)
  213. irqfd->irq_entry = *e;
  214. else
  215. irqfd->irq_entry.type = 0;
  216. write_seqcount_end(&irqfd->irq_entry_sc);
  217. }
  218. #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
  219. void __attribute__((weak)) kvm_arch_irq_bypass_stop(
  220. struct irq_bypass_consumer *cons)
  221. {
  222. }
  223. void __attribute__((weak)) kvm_arch_irq_bypass_start(
  224. struct irq_bypass_consumer *cons)
  225. {
  226. }
  227. int __attribute__((weak)) kvm_arch_update_irqfd_routing(
  228. struct kvm *kvm, unsigned int host_irq,
  229. uint32_t guest_irq, bool set)
  230. {
  231. return 0;
  232. }
  233. #endif
  234. static int
  235. kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
  236. {
  237. struct kvm_kernel_irqfd *irqfd, *tmp;
  238. struct fd f;
  239. struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL;
  240. int ret;
  241. __poll_t events;
  242. int idx;
  243. if (!kvm_arch_intc_initialized(kvm))
  244. return -EAGAIN;
  245. if (!kvm_arch_irqfd_allowed(kvm, args))
  246. return -EINVAL;
  247. irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL_ACCOUNT);
  248. if (!irqfd)
  249. return -ENOMEM;
  250. irqfd->kvm = kvm;
  251. irqfd->gsi = args->gsi;
  252. INIT_LIST_HEAD(&irqfd->list);
  253. INIT_WORK(&irqfd->inject, irqfd_inject);
  254. INIT_WORK(&irqfd->shutdown, irqfd_shutdown);
  255. seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);
  256. f = fdget(args->fd);
  257. if (!f.file) {
  258. ret = -EBADF;
  259. goto out;
  260. }
  261. eventfd = eventfd_ctx_fileget(f.file);
  262. if (IS_ERR(eventfd)) {
  263. ret = PTR_ERR(eventfd);
  264. goto fail;
  265. }
  266. irqfd->eventfd = eventfd;
  267. if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) {
  268. struct kvm_kernel_irqfd_resampler *resampler;
  269. resamplefd = eventfd_ctx_fdget(args->resamplefd);
  270. if (IS_ERR(resamplefd)) {
  271. ret = PTR_ERR(resamplefd);
  272. goto fail;
  273. }
  274. irqfd->resamplefd = resamplefd;
  275. INIT_LIST_HEAD(&irqfd->resampler_link);
  276. mutex_lock(&kvm->irqfds.resampler_lock);
  277. list_for_each_entry(resampler,
  278. &kvm->irqfds.resampler_list, link) {
  279. if (resampler->notifier.gsi == irqfd->gsi) {
  280. irqfd->resampler = resampler;
  281. break;
  282. }
  283. }
  284. if (!irqfd->resampler) {
  285. resampler = kzalloc(sizeof(*resampler),
  286. GFP_KERNEL_ACCOUNT);
  287. if (!resampler) {
  288. ret = -ENOMEM;
  289. mutex_unlock(&kvm->irqfds.resampler_lock);
  290. goto fail;
  291. }
  292. resampler->kvm = kvm;
  293. INIT_LIST_HEAD(&resampler->list);
  294. resampler->notifier.gsi = irqfd->gsi;
  295. resampler->notifier.irq_acked = irqfd_resampler_ack;
  296. INIT_LIST_HEAD(&resampler->link);
  297. list_add(&resampler->link, &kvm->irqfds.resampler_list);
  298. kvm_register_irq_ack_notifier(kvm,
  299. &resampler->notifier);
  300. irqfd->resampler = resampler;
  301. }
  302. list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);
  303. synchronize_srcu(&kvm->irq_srcu);
  304. mutex_unlock(&kvm->irqfds.resampler_lock);
  305. }
  306. /*
  307. * Install our own custom wake-up handling so we are notified via
  308. * a callback whenever someone signals the underlying eventfd
  309. */
  310. init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);
  311. init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);
  312. spin_lock_irq(&kvm->irqfds.lock);
  313. ret = 0;
  314. list_for_each_entry(tmp, &kvm->irqfds.items, list) {
  315. if (irqfd->eventfd != tmp->eventfd)
  316. continue;
  317. /* This fd is used for another irq already. */
  318. ret = -EBUSY;
  319. spin_unlock_irq(&kvm->irqfds.lock);
  320. goto fail;
  321. }
  322. idx = srcu_read_lock(&kvm->irq_srcu);
  323. irqfd_update(kvm, irqfd);
  324. list_add_tail(&irqfd->list, &kvm->irqfds.items);
  325. spin_unlock_irq(&kvm->irqfds.lock);
  326. /*
  327. * Check if there was an event already pending on the eventfd
  328. * before we registered, and trigger it as if we didn't miss it.
  329. */
  330. events = vfs_poll(f.file, &irqfd->pt);
  331. if (events & EPOLLIN)
  332. schedule_work(&irqfd->inject);
  333. #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
  334. if (kvm_arch_has_irq_bypass()) {
  335. irqfd->consumer.token = (void *)irqfd->eventfd;
  336. irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;
  337. irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;
  338. irqfd->consumer.stop = kvm_arch_irq_bypass_stop;
  339. irqfd->consumer.start = kvm_arch_irq_bypass_start;
  340. ret = irq_bypass_register_consumer(&irqfd->consumer);
  341. if (ret)
  342. pr_info("irq bypass consumer (token %p) registration fails: %d\n",
  343. irqfd->consumer.token, ret);
  344. }
  345. #endif
  346. srcu_read_unlock(&kvm->irq_srcu, idx);
  347. /*
  348. * do not drop the file until the irqfd is fully initialized, otherwise
  349. * we might race against the EPOLLHUP
  350. */
  351. fdput(f);
  352. return 0;
  353. fail:
  354. if (irqfd->resampler)
  355. irqfd_resampler_shutdown(irqfd);
  356. if (resamplefd && !IS_ERR(resamplefd))
  357. eventfd_ctx_put(resamplefd);
  358. if (eventfd && !IS_ERR(eventfd))
  359. eventfd_ctx_put(eventfd);
  360. fdput(f);
  361. out:
  362. kfree(irqfd);
  363. return ret;
  364. }
  365. bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
  366. {
  367. struct kvm_irq_ack_notifier *kian;
  368. int gsi, idx;
  369. idx = srcu_read_lock(&kvm->irq_srcu);
  370. gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
  371. if (gsi != -1)
  372. hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
  373. link, srcu_read_lock_held(&kvm->irq_srcu))
  374. if (kian->gsi == gsi) {
  375. srcu_read_unlock(&kvm->irq_srcu, idx);
  376. return true;
  377. }
  378. srcu_read_unlock(&kvm->irq_srcu, idx);
  379. return false;
  380. }
  381. EXPORT_SYMBOL_GPL(kvm_irq_has_notifier);
  382. void kvm_notify_acked_gsi(struct kvm *kvm, int gsi)
  383. {
  384. struct kvm_irq_ack_notifier *kian;
  385. hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
  386. link, srcu_read_lock_held(&kvm->irq_srcu))
  387. if (kian->gsi == gsi)
  388. kian->irq_acked(kian);
  389. }
  390. void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
  391. {
  392. int gsi, idx;
  393. trace_kvm_ack_irq(irqchip, pin);
  394. idx = srcu_read_lock(&kvm->irq_srcu);
  395. gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
  396. if (gsi != -1)
  397. kvm_notify_acked_gsi(kvm, gsi);
  398. srcu_read_unlock(&kvm->irq_srcu, idx);
  399. }
  400. void kvm_register_irq_ack_notifier(struct kvm *kvm,
  401. struct kvm_irq_ack_notifier *kian)
  402. {
  403. mutex_lock(&kvm->irq_lock);
  404. hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);
  405. mutex_unlock(&kvm->irq_lock);
  406. kvm_arch_post_irq_ack_notifier_list_update(kvm);
  407. }
  408. void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
  409. struct kvm_irq_ack_notifier *kian)
  410. {
  411. mutex_lock(&kvm->irq_lock);
  412. hlist_del_init_rcu(&kian->link);
  413. mutex_unlock(&kvm->irq_lock);
  414. synchronize_srcu(&kvm->irq_srcu);
  415. kvm_arch_post_irq_ack_notifier_list_update(kvm);
  416. }
  417. #endif
  418. void
  419. kvm_eventfd_init(struct kvm *kvm)
  420. {
  421. #ifdef CONFIG_HAVE_KVM_IRQFD
  422. spin_lock_init(&kvm->irqfds.lock);
  423. INIT_LIST_HEAD(&kvm->irqfds.items);
  424. INIT_LIST_HEAD(&kvm->irqfds.resampler_list);
  425. mutex_init(&kvm->irqfds.resampler_lock);
  426. #endif
  427. INIT_LIST_HEAD(&kvm->ioeventfds);
  428. }
  429. #ifdef CONFIG_HAVE_KVM_IRQFD
  430. /*
  431. * shutdown any irqfd's that match fd+gsi
  432. */
  433. static int
  434. kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)
  435. {
  436. struct kvm_kernel_irqfd *irqfd, *tmp;
  437. struct eventfd_ctx *eventfd;
  438. eventfd = eventfd_ctx_fdget(args->fd);
  439. if (IS_ERR(eventfd))
  440. return PTR_ERR(eventfd);
  441. spin_lock_irq(&kvm->irqfds.lock);
  442. list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {
  443. if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {
  444. /*
  445. * This clearing of irq_entry.type is needed for when
  446. * another thread calls kvm_irq_routing_update before
  447. * we flush workqueue below (we synchronize with
  448. * kvm_irq_routing_update using irqfds.lock).
  449. */
  450. write_seqcount_begin(&irqfd->irq_entry_sc);
  451. irqfd->irq_entry.type = 0;
  452. write_seqcount_end(&irqfd->irq_entry_sc);
  453. irqfd_deactivate(irqfd);
  454. }
  455. }
  456. spin_unlock_irq(&kvm->irqfds.lock);
  457. eventfd_ctx_put(eventfd);
  458. /*
  459. * Block until we know all outstanding shutdown jobs have completed
  460. * so that we guarantee there will not be any more interrupts on this
  461. * gsi once this deassign function returns.
  462. */
  463. flush_workqueue(irqfd_cleanup_wq);
  464. return 0;
  465. }
  466. int
  467. kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
  468. {
  469. if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE))
  470. return -EINVAL;
  471. if (args->flags & KVM_IRQFD_FLAG_DEASSIGN)
  472. return kvm_irqfd_deassign(kvm, args);
  473. return kvm_irqfd_assign(kvm, args);
  474. }
  475. /*
  476. * This function is called as the kvm VM fd is being released. Shutdown all
  477. * irqfds that still remain open
  478. */
  479. void
  480. kvm_irqfd_release(struct kvm *kvm)
  481. {
  482. struct kvm_kernel_irqfd *irqfd, *tmp;
  483. spin_lock_irq(&kvm->irqfds.lock);
  484. list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)
  485. irqfd_deactivate(irqfd);
  486. spin_unlock_irq(&kvm->irqfds.lock);
  487. /*
  488. * Block until we know all outstanding shutdown jobs have completed
  489. * since we do not take a kvm* reference.
  490. */
  491. flush_workqueue(irqfd_cleanup_wq);
  492. }
  493. /*
  494. * Take note of a change in irq routing.
  495. * Caller must invoke synchronize_srcu(&kvm->irq_srcu) afterwards.
  496. */
  497. void kvm_irq_routing_update(struct kvm *kvm)
  498. {
  499. struct kvm_kernel_irqfd *irqfd;
  500. spin_lock_irq(&kvm->irqfds.lock);
  501. list_for_each_entry(irqfd, &kvm->irqfds.items, list) {
  502. irqfd_update(kvm, irqfd);
  503. #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
  504. if (irqfd->producer) {
  505. int ret = kvm_arch_update_irqfd_routing(
  506. irqfd->kvm, irqfd->producer->irq,
  507. irqfd->gsi, 1);
  508. WARN_ON(ret);
  509. }
  510. #endif
  511. }
  512. spin_unlock_irq(&kvm->irqfds.lock);
  513. }
  514. /*
  515. * create a host-wide workqueue for issuing deferred shutdown requests
  516. * aggregated from all vm* instances. We need our own isolated
  517. * queue to ease flushing work items when a VM exits.
  518. */
  519. int kvm_irqfd_init(void)
  520. {
  521. irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", 0, 0);
  522. if (!irqfd_cleanup_wq)
  523. return -ENOMEM;
  524. return 0;
  525. }
  526. void kvm_irqfd_exit(void)
  527. {
  528. destroy_workqueue(irqfd_cleanup_wq);
  529. }
  530. #endif
  531. /*
  532. * --------------------------------------------------------------------
  533. * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal.
  534. *
  535. * userspace can register a PIO/MMIO address with an eventfd for receiving
  536. * notification when the memory has been touched.
  537. * --------------------------------------------------------------------
  538. */
  539. struct _ioeventfd {
  540. struct list_head list;
  541. u64 addr;
  542. int length;
  543. struct eventfd_ctx *eventfd;
  544. u64 datamatch;
  545. struct kvm_io_device dev;
  546. u8 bus_idx;
  547. bool wildcard;
  548. };
  549. static inline struct _ioeventfd *
  550. to_ioeventfd(struct kvm_io_device *dev)
  551. {
  552. return container_of(dev, struct _ioeventfd, dev);
  553. }
  554. static void
  555. ioeventfd_release(struct _ioeventfd *p)
  556. {
  557. eventfd_ctx_put(p->eventfd);
  558. list_del(&p->list);
  559. kfree(p);
  560. }
  561. static bool
  562. ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val)
  563. {
  564. u64 _val;
  565. if (addr != p->addr)
  566. /* address must be precise for a hit */
  567. return false;
  568. if (!p->length)
  569. /* length = 0 means only look at the address, so always a hit */
  570. return true;
  571. if (len != p->length)
  572. /* address-range must be precise for a hit */
  573. return false;
  574. if (p->wildcard)
  575. /* all else equal, wildcard is always a hit */
  576. return true;
  577. /* otherwise, we have to actually compare the data */
  578. BUG_ON(!IS_ALIGNED((unsigned long)val, len));
  579. switch (len) {
  580. case 1:
  581. _val = *(u8 *)val;
  582. break;
  583. case 2:
  584. _val = *(u16 *)val;
  585. break;
  586. case 4:
  587. _val = *(u32 *)val;
  588. break;
  589. case 8:
  590. _val = *(u64 *)val;
  591. break;
  592. default:
  593. return false;
  594. }
  595. return _val == p->datamatch;
  596. }
  597. /* MMIO/PIO writes trigger an event if the addr/val match */
  598. static int
  599. ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr,
  600. int len, const void *val)
  601. {
  602. struct _ioeventfd *p = to_ioeventfd(this);
  603. if (!ioeventfd_in_range(p, addr, len, val))
  604. return -EOPNOTSUPP;
  605. eventfd_signal(p->eventfd, 1);
  606. return 0;
  607. }
  608. /*
  609. * This function is called as KVM is completely shutting down. We do not
  610. * need to worry about locking just nuke anything we have as quickly as possible
  611. */
  612. static void
  613. ioeventfd_destructor(struct kvm_io_device *this)
  614. {
  615. struct _ioeventfd *p = to_ioeventfd(this);
  616. ioeventfd_release(p);
  617. }
  618. static const struct kvm_io_device_ops ioeventfd_ops = {
  619. .write = ioeventfd_write,
  620. .destructor = ioeventfd_destructor,
  621. };
  622. /* assumes kvm->slots_lock held */
  623. static bool
  624. ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p)
  625. {
  626. struct _ioeventfd *_p;
  627. list_for_each_entry(_p, &kvm->ioeventfds, list)
  628. if (_p->bus_idx == p->bus_idx &&
  629. _p->addr == p->addr &&
  630. (!_p->length || !p->length ||
  631. (_p->length == p->length &&
  632. (_p->wildcard || p->wildcard ||
  633. _p->datamatch == p->datamatch))))
  634. return true;
  635. return false;
  636. }
  637. static enum kvm_bus ioeventfd_bus_from_flags(__u32 flags)
  638. {
  639. if (flags & KVM_IOEVENTFD_FLAG_PIO)
  640. return KVM_PIO_BUS;
  641. if (flags & KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY)
  642. return KVM_VIRTIO_CCW_NOTIFY_BUS;
  643. return KVM_MMIO_BUS;
  644. }
  645. static int kvm_assign_ioeventfd_idx(struct kvm *kvm,
  646. enum kvm_bus bus_idx,
  647. struct kvm_ioeventfd *args)
  648. {
  649. struct eventfd_ctx *eventfd;
  650. struct _ioeventfd *p;
  651. int ret;
  652. eventfd = eventfd_ctx_fdget(args->fd);
  653. if (IS_ERR(eventfd))
  654. return PTR_ERR(eventfd);
  655. p = kzalloc(sizeof(*p), GFP_KERNEL_ACCOUNT);
  656. if (!p) {
  657. ret = -ENOMEM;
  658. goto fail;
  659. }
  660. INIT_LIST_HEAD(&p->list);
  661. p->addr = args->addr;
  662. p->bus_idx = bus_idx;
  663. p->length = args->len;
  664. p->eventfd = eventfd;
  665. /* The datamatch feature is optional, otherwise this is a wildcard */
  666. if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH)
  667. p->datamatch = args->datamatch;
  668. else
  669. p->wildcard = true;
  670. mutex_lock(&kvm->slots_lock);
  671. /* Verify that there isn't a match already */
  672. if (ioeventfd_check_collision(kvm, p)) {
  673. ret = -EEXIST;
  674. goto unlock_fail;
  675. }
  676. kvm_iodevice_init(&p->dev, &ioeventfd_ops);
  677. ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,
  678. &p->dev);
  679. if (ret < 0)
  680. goto unlock_fail;
  681. kvm_get_bus(kvm, bus_idx)->ioeventfd_count++;
  682. list_add_tail(&p->list, &kvm->ioeventfds);
  683. mutex_unlock(&kvm->slots_lock);
  684. return 0;
  685. unlock_fail:
  686. mutex_unlock(&kvm->slots_lock);
  687. fail:
  688. kfree(p);
  689. eventfd_ctx_put(eventfd);
  690. return ret;
  691. }
  692. static int
  693. kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx,
  694. struct kvm_ioeventfd *args)
  695. {
  696. struct _ioeventfd *p, *tmp;
  697. struct eventfd_ctx *eventfd;
  698. struct kvm_io_bus *bus;
  699. int ret = -ENOENT;
  700. bool wildcard;
  701. eventfd = eventfd_ctx_fdget(args->fd);
  702. if (IS_ERR(eventfd))
  703. return PTR_ERR(eventfd);
  704. wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH);
  705. mutex_lock(&kvm->slots_lock);
  706. list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) {
  707. if (p->bus_idx != bus_idx ||
  708. p->eventfd != eventfd ||
  709. p->addr != args->addr ||
  710. p->length != args->len ||
  711. p->wildcard != wildcard)
  712. continue;
  713. if (!p->wildcard && p->datamatch != args->datamatch)
  714. continue;
  715. kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
  716. bus = kvm_get_bus(kvm, bus_idx);
  717. if (bus)
  718. bus->ioeventfd_count--;
  719. ioeventfd_release(p);
  720. ret = 0;
  721. break;
  722. }
  723. mutex_unlock(&kvm->slots_lock);
  724. eventfd_ctx_put(eventfd);
  725. return ret;
  726. }
  727. static int kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  728. {
  729. enum kvm_bus bus_idx = ioeventfd_bus_from_flags(args->flags);
  730. int ret = kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
  731. if (!args->len && bus_idx == KVM_MMIO_BUS)
  732. kvm_deassign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
  733. return ret;
  734. }
  735. static int
  736. kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  737. {
  738. enum kvm_bus bus_idx;
  739. int ret;
  740. bus_idx = ioeventfd_bus_from_flags(args->flags);
  741. /* must be natural-word sized, or 0 to ignore length */
  742. switch (args->len) {
  743. case 0:
  744. case 1:
  745. case 2:
  746. case 4:
  747. case 8:
  748. break;
  749. default:
  750. return -EINVAL;
  751. }
  752. /* check for range overflow */
  753. if (args->addr + args->len < args->addr)
  754. return -EINVAL;
  755. /* check for extra flags that we don't understand */
  756. if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)
  757. return -EINVAL;
  758. /* ioeventfd with no length can't be combined with DATAMATCH */
  759. if (!args->len && (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH))
  760. return -EINVAL;
  761. ret = kvm_assign_ioeventfd_idx(kvm, bus_idx, args);
  762. if (ret)
  763. goto fail;
  764. /* When length is ignored, MMIO is also put on a separate bus, for
  765. * faster lookups.
  766. */
  767. if (!args->len && bus_idx == KVM_MMIO_BUS) {
  768. ret = kvm_assign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
  769. if (ret < 0)
  770. goto fast_fail;
  771. }
  772. return 0;
  773. fast_fail:
  774. kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
  775. fail:
  776. return ret;
  777. }
  778. int
  779. kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
  780. {
  781. if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN)
  782. return kvm_deassign_ioeventfd(kvm, args);
  783. return kvm_assign_ioeventfd(kvm, args);
  784. }