irq_comm.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * irq_comm.c: Common API for in kernel interrupt controller
  4. * Copyright (c) 2007, Intel Corporation.
  5. *
  6. * Authors:
  7. * Yaozu (Eddie) Dong <Eddie.dong@intel.com>
  8. *
  9. * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  10. */
  11. #include <linux/kvm_host.h>
  12. #include <linux/slab.h>
  13. #include <linux/export.h>
  14. #include <linux/rculist.h>
  15. #include <trace/events/kvm.h>
  16. #include <asm/msidef.h>
  17. #include "irq.h"
  18. #include "ioapic.h"
  19. #include "lapic.h"
  20. #include "hyperv.h"
  21. #include "x86.h"
  22. static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e,
  23. struct kvm *kvm, int irq_source_id, int level,
  24. bool line_status)
  25. {
  26. struct kvm_pic *pic = kvm->arch.vpic;
  27. return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level);
  28. }
  29. static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e,
  30. struct kvm *kvm, int irq_source_id, int level,
  31. bool line_status)
  32. {
  33. struct kvm_ioapic *ioapic = kvm->arch.vioapic;
  34. return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level,
  35. line_status);
  36. }
  37. int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
  38. struct kvm_lapic_irq *irq, struct dest_map *dest_map)
  39. {
  40. int i, r = -1;
  41. struct kvm_vcpu *vcpu, *lowest = NULL;
  42. unsigned long dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)];
  43. unsigned int dest_vcpus = 0;
  44. if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map))
  45. return r;
  46. if (irq->dest_mode == APIC_DEST_PHYSICAL &&
  47. irq->dest_id == 0xff && kvm_lowest_prio_delivery(irq)) {
  48. printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n");
  49. irq->delivery_mode = APIC_DM_FIXED;
  50. }
  51. memset(dest_vcpu_bitmap, 0, sizeof(dest_vcpu_bitmap));
  52. kvm_for_each_vcpu(i, vcpu, kvm) {
  53. if (!kvm_apic_present(vcpu))
  54. continue;
  55. if (!kvm_apic_match_dest(vcpu, src, irq->shorthand,
  56. irq->dest_id, irq->dest_mode))
  57. continue;
  58. if (!kvm_lowest_prio_delivery(irq)) {
  59. if (r < 0)
  60. r = 0;
  61. r += kvm_apic_set_irq(vcpu, irq, dest_map);
  62. } else if (kvm_apic_sw_enabled(vcpu->arch.apic)) {
  63. if (!kvm_vector_hashing_enabled()) {
  64. if (!lowest)
  65. lowest = vcpu;
  66. else if (kvm_apic_compare_prio(vcpu, lowest) < 0)
  67. lowest = vcpu;
  68. } else {
  69. __set_bit(i, dest_vcpu_bitmap);
  70. dest_vcpus++;
  71. }
  72. }
  73. }
  74. if (dest_vcpus != 0) {
  75. int idx = kvm_vector_to_index(irq->vector, dest_vcpus,
  76. dest_vcpu_bitmap, KVM_MAX_VCPUS);
  77. lowest = kvm_get_vcpu(kvm, idx);
  78. }
  79. if (lowest)
  80. r = kvm_apic_set_irq(lowest, irq, dest_map);
  81. return r;
  82. }
  83. void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
  84. struct kvm_lapic_irq *irq)
  85. {
  86. trace_kvm_msi_set_irq(e->msi.address_lo | (kvm->arch.x2apic_format ?
  87. (u64)e->msi.address_hi << 32 : 0),
  88. e->msi.data);
  89. irq->dest_id = (e->msi.address_lo &
  90. MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT;
  91. if (kvm->arch.x2apic_format)
  92. irq->dest_id |= MSI_ADDR_EXT_DEST_ID(e->msi.address_hi);
  93. irq->vector = (e->msi.data &
  94. MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT;
  95. irq->dest_mode = kvm_lapic_irq_dest_mode(
  96. !!((1 << MSI_ADDR_DEST_MODE_SHIFT) & e->msi.address_lo));
  97. irq->trig_mode = (1 << MSI_DATA_TRIGGER_SHIFT) & e->msi.data;
  98. irq->delivery_mode = e->msi.data & 0x700;
  99. irq->msi_redir_hint = ((e->msi.address_lo
  100. & MSI_ADDR_REDIRECTION_LOWPRI) > 0);
  101. irq->level = 1;
  102. irq->shorthand = APIC_DEST_NOSHORT;
  103. }
  104. EXPORT_SYMBOL_GPL(kvm_set_msi_irq);
  105. static inline bool kvm_msi_route_invalid(struct kvm *kvm,
  106. struct kvm_kernel_irq_routing_entry *e)
  107. {
  108. return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff);
  109. }
  110. int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
  111. struct kvm *kvm, int irq_source_id, int level, bool line_status)
  112. {
  113. struct kvm_lapic_irq irq;
  114. if (kvm_msi_route_invalid(kvm, e))
  115. return -EINVAL;
  116. if (!level)
  117. return -1;
  118. kvm_set_msi_irq(kvm, e, &irq);
  119. return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL);
  120. }
  121. static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e,
  122. struct kvm *kvm, int irq_source_id, int level,
  123. bool line_status)
  124. {
  125. if (!level)
  126. return -1;
  127. return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint);
  128. }
  129. int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
  130. struct kvm *kvm, int irq_source_id, int level,
  131. bool line_status)
  132. {
  133. struct kvm_lapic_irq irq;
  134. int r;
  135. switch (e->type) {
  136. case KVM_IRQ_ROUTING_HV_SINT:
  137. return kvm_hv_set_sint(e, kvm, irq_source_id, level,
  138. line_status);
  139. case KVM_IRQ_ROUTING_MSI:
  140. if (kvm_msi_route_invalid(kvm, e))
  141. return -EINVAL;
  142. kvm_set_msi_irq(kvm, e, &irq);
  143. if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL))
  144. return r;
  145. break;
  146. default:
  147. break;
  148. }
  149. return -EWOULDBLOCK;
  150. }
  151. int kvm_request_irq_source_id(struct kvm *kvm)
  152. {
  153. unsigned long *bitmap = &kvm->arch.irq_sources_bitmap;
  154. int irq_source_id;
  155. mutex_lock(&kvm->irq_lock);
  156. irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG);
  157. if (irq_source_id >= BITS_PER_LONG) {
  158. printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n");
  159. irq_source_id = -EFAULT;
  160. goto unlock;
  161. }
  162. ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
  163. ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
  164. set_bit(irq_source_id, bitmap);
  165. unlock:
  166. mutex_unlock(&kvm->irq_lock);
  167. return irq_source_id;
  168. }
  169. void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id)
  170. {
  171. ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
  172. ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
  173. mutex_lock(&kvm->irq_lock);
  174. if (irq_source_id < 0 ||
  175. irq_source_id >= BITS_PER_LONG) {
  176. printk(KERN_ERR "kvm: IRQ source ID out of range!\n");
  177. goto unlock;
  178. }
  179. clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap);
  180. if (!irqchip_kernel(kvm))
  181. goto unlock;
  182. kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id);
  183. kvm_pic_clear_all(kvm->arch.vpic, irq_source_id);
  184. unlock:
  185. mutex_unlock(&kvm->irq_lock);
  186. }
  187. void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
  188. struct kvm_irq_mask_notifier *kimn)
  189. {
  190. mutex_lock(&kvm->irq_lock);
  191. kimn->irq = irq;
  192. hlist_add_head_rcu(&kimn->link, &kvm->arch.mask_notifier_list);
  193. mutex_unlock(&kvm->irq_lock);
  194. }
  195. void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
  196. struct kvm_irq_mask_notifier *kimn)
  197. {
  198. mutex_lock(&kvm->irq_lock);
  199. hlist_del_rcu(&kimn->link);
  200. mutex_unlock(&kvm->irq_lock);
  201. synchronize_srcu(&kvm->irq_srcu);
  202. }
  203. void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
  204. bool mask)
  205. {
  206. struct kvm_irq_mask_notifier *kimn;
  207. int idx, gsi;
  208. idx = srcu_read_lock(&kvm->irq_srcu);
  209. gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
  210. if (gsi != -1)
  211. hlist_for_each_entry_rcu(kimn, &kvm->arch.mask_notifier_list, link)
  212. if (kimn->irq == gsi)
  213. kimn->func(kimn, mask);
  214. srcu_read_unlock(&kvm->irq_srcu, idx);
  215. }
  216. bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
  217. {
  218. return irqchip_in_kernel(kvm);
  219. }
  220. int kvm_set_routing_entry(struct kvm *kvm,
  221. struct kvm_kernel_irq_routing_entry *e,
  222. const struct kvm_irq_routing_entry *ue)
  223. {
  224. /* We can't check irqchip_in_kernel() here as some callers are
  225. * currently inititalizing the irqchip. Other callers should therefore
  226. * check kvm_arch_can_set_irq_routing() before calling this function.
  227. */
  228. switch (ue->type) {
  229. case KVM_IRQ_ROUTING_IRQCHIP:
  230. if (irqchip_split(kvm))
  231. return -EINVAL;
  232. e->irqchip.pin = ue->u.irqchip.pin;
  233. switch (ue->u.irqchip.irqchip) {
  234. case KVM_IRQCHIP_PIC_SLAVE:
  235. e->irqchip.pin += PIC_NUM_PINS / 2;
  236. fallthrough;
  237. case KVM_IRQCHIP_PIC_MASTER:
  238. if (ue->u.irqchip.pin >= PIC_NUM_PINS / 2)
  239. return -EINVAL;
  240. e->set = kvm_set_pic_irq;
  241. break;
  242. case KVM_IRQCHIP_IOAPIC:
  243. if (ue->u.irqchip.pin >= KVM_IOAPIC_NUM_PINS)
  244. return -EINVAL;
  245. e->set = kvm_set_ioapic_irq;
  246. break;
  247. default:
  248. return -EINVAL;
  249. }
  250. e->irqchip.irqchip = ue->u.irqchip.irqchip;
  251. break;
  252. case KVM_IRQ_ROUTING_MSI:
  253. e->set = kvm_set_msi;
  254. e->msi.address_lo = ue->u.msi.address_lo;
  255. e->msi.address_hi = ue->u.msi.address_hi;
  256. e->msi.data = ue->u.msi.data;
  257. if (kvm_msi_route_invalid(kvm, e))
  258. return -EINVAL;
  259. break;
  260. case KVM_IRQ_ROUTING_HV_SINT:
  261. e->set = kvm_hv_set_sint;
  262. e->hv_sint.vcpu = ue->u.hv_sint.vcpu;
  263. e->hv_sint.sint = ue->u.hv_sint.sint;
  264. break;
  265. default:
  266. return -EINVAL;
  267. }
  268. return 0;
  269. }
  270. bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
  271. struct kvm_vcpu **dest_vcpu)
  272. {
  273. int i, r = 0;
  274. struct kvm_vcpu *vcpu;
  275. if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu))
  276. return true;
  277. kvm_for_each_vcpu(i, vcpu, kvm) {
  278. if (!kvm_apic_present(vcpu))
  279. continue;
  280. if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand,
  281. irq->dest_id, irq->dest_mode))
  282. continue;
  283. if (++r == 2)
  284. return false;
  285. *dest_vcpu = vcpu;
  286. }
  287. return r == 1;
  288. }
  289. EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu);
  290. #define IOAPIC_ROUTING_ENTRY(irq) \
  291. { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \
  292. .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } }
  293. #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq)
  294. #define PIC_ROUTING_ENTRY(irq) \
  295. { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \
  296. .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } }
  297. #define ROUTING_ENTRY2(irq) \
  298. IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq)
  299. static const struct kvm_irq_routing_entry default_routing[] = {
  300. ROUTING_ENTRY2(0), ROUTING_ENTRY2(1),
  301. ROUTING_ENTRY2(2), ROUTING_ENTRY2(3),
  302. ROUTING_ENTRY2(4), ROUTING_ENTRY2(5),
  303. ROUTING_ENTRY2(6), ROUTING_ENTRY2(7),
  304. ROUTING_ENTRY2(8), ROUTING_ENTRY2(9),
  305. ROUTING_ENTRY2(10), ROUTING_ENTRY2(11),
  306. ROUTING_ENTRY2(12), ROUTING_ENTRY2(13),
  307. ROUTING_ENTRY2(14), ROUTING_ENTRY2(15),
  308. ROUTING_ENTRY1(16), ROUTING_ENTRY1(17),
  309. ROUTING_ENTRY1(18), ROUTING_ENTRY1(19),
  310. ROUTING_ENTRY1(20), ROUTING_ENTRY1(21),
  311. ROUTING_ENTRY1(22), ROUTING_ENTRY1(23),
  312. };
  313. int kvm_setup_default_irq_routing(struct kvm *kvm)
  314. {
  315. return kvm_set_irq_routing(kvm, default_routing,
  316. ARRAY_SIZE(default_routing), 0);
  317. }
  318. static const struct kvm_irq_routing_entry empty_routing[] = {};
  319. int kvm_setup_empty_irq_routing(struct kvm *kvm)
  320. {
  321. return kvm_set_irq_routing(kvm, empty_routing, 0, 0);
  322. }
  323. void kvm_arch_post_irq_routing_update(struct kvm *kvm)
  324. {
  325. if (!irqchip_split(kvm))
  326. return;
  327. kvm_make_scan_ioapic_request(kvm);
  328. }
  329. void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu,
  330. ulong *ioapic_handled_vectors)
  331. {
  332. struct kvm *kvm = vcpu->kvm;
  333. struct kvm_kernel_irq_routing_entry *entry;
  334. struct kvm_irq_routing_table *table;
  335. u32 i, nr_ioapic_pins;
  336. int idx;
  337. idx = srcu_read_lock(&kvm->irq_srcu);
  338. table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
  339. nr_ioapic_pins = min_t(u32, table->nr_rt_entries,
  340. kvm->arch.nr_reserved_ioapic_pins);
  341. for (i = 0; i < nr_ioapic_pins; ++i) {
  342. hlist_for_each_entry(entry, &table->map[i], link) {
  343. struct kvm_lapic_irq irq;
  344. if (entry->type != KVM_IRQ_ROUTING_MSI)
  345. continue;
  346. kvm_set_msi_irq(vcpu->kvm, entry, &irq);
  347. if (irq.trig_mode &&
  348. kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
  349. irq.dest_id, irq.dest_mode))
  350. __set_bit(irq.vector, ioapic_handled_vectors);
  351. }
  352. }
  353. srcu_read_unlock(&kvm->irq_srcu, idx);
  354. }
  355. void kvm_arch_irq_routing_update(struct kvm *kvm)
  356. {
  357. kvm_hv_irq_routing_update(kvm);
  358. }