irq-gic-v4.c 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
  4. * Author: Marc Zyngier <marc.zyngier@arm.com>
  5. */
  6. #include <linux/interrupt.h>
  7. #include <linux/irq.h>
  8. #include <linux/irqdomain.h>
  9. #include <linux/msi.h>
  10. #include <linux/sched.h>
  11. #include <linux/irqchip/arm-gic-v4.h>
  12. /*
  13. * WARNING: The blurb below assumes that you understand the
  14. * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
  15. * translated into GICv4 commands. So it effectively targets at most
  16. * two individuals. You know who you are.
  17. *
  18. * The core GICv4 code is designed to *avoid* exposing too much of the
  19. * core GIC code (that would in turn leak into the hypervisor code),
  20. * and instead provide a hypervisor agnostic interface to the HW (of
  21. * course, the astute reader will quickly realize that hypervisor
  22. * agnostic actually means KVM-specific - what were you thinking?).
  23. *
  24. * In order to achieve a modicum of isolation, we try to hide most of
  25. * the GICv4 "stuff" behind normal irqchip operations:
  26. *
  27. * - Any guest-visible VLPI is backed by a Linux interrupt (and a
  28. * physical LPI which gets unmapped when the guest maps the
  29. * VLPI). This allows the same DevID/EventID pair to be either
  30. * mapped to the LPI (host) or the VLPI (guest). Note that this is
  31. * exclusive, and you cannot have both.
  32. *
  33. * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
  34. *
  35. * - Guest INT/CLEAR commands are implemented through
  36. * irq_set_irqchip_state().
  37. *
  38. * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
  39. * issuing an INV after changing a priority) gets shoved into the
  40. * irq_set_vcpu_affinity() method. While this is quite horrible
  41. * (let's face it, this is the irqchip version of an ioctl), it
  42. * confines the crap to a single location. And map/unmap really is
  43. * about setting the affinity of a VLPI to a vcpu, so only INV is
  44. * majorly out of place. So there.
  45. *
  46. * A number of commands are simply not provided by this interface, as
  47. * they do not make direct sense. For example, MAPD is purely local to
  48. * the virtual ITS (because it references a virtual device, and the
  49. * physical ITS is still very much in charge of the physical
  50. * device). Same goes for things like MAPC (the physical ITS deals
  51. * with the actual vPE affinity, and not the braindead concept of
  52. * collection). SYNC is not provided either, as each and every command
  53. * is followed by a VSYNC. This could be relaxed in the future, should
  54. * this be seen as a bottleneck (yes, this means *never*).
  55. *
  56. * But handling VLPIs is only one side of the job of the GICv4
  57. * code. The other (darker) side is to take care of the doorbell
  58. * interrupts which are delivered when a VLPI targeting a non-running
  59. * vcpu is being made pending.
  60. *
  61. * The choice made here is that each vcpu (VPE in old northern GICv4
  62. * dialect) gets a single doorbell LPI, no matter how many interrupts
  63. * are targeting it. This has a nice property, which is that the
  64. * interrupt becomes a handle for the VPE, and that the hypervisor
  65. * code can manipulate it through the normal interrupt API:
  66. *
  67. * - VMs (or rather the VM abstraction that matters to the GIC)
  68. * contain an irq domain where each interrupt maps to a VPE. In
  69. * turn, this domain sits on top of the normal LPI allocator, and a
  70. * specially crafted irq_chip implementation.
  71. *
  72. * - mask/unmask do what is expected on the doorbell interrupt.
  73. *
  74. * - irq_set_affinity is used to move a VPE from one redistributor to
  75. * another.
  76. *
  77. * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
  78. * creating a new sub-API, namely scheduling/descheduling a VPE
  79. * (which involves programming GICR_V{PROP,PEND}BASER) and
  80. * performing INVALL operations.
  81. */
  82. static struct irq_domain *gic_domain;
  83. static const struct irq_domain_ops *vpe_domain_ops;
  84. static const struct irq_domain_ops *sgi_domain_ops;
  85. static bool has_v4_1(void)
  86. {
  87. return !!sgi_domain_ops;
  88. }
  89. static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
  90. {
  91. char *name;
  92. int sgi_base;
  93. if (!has_v4_1())
  94. return 0;
  95. name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
  96. if (!name)
  97. goto err;
  98. vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
  99. if (!vpe->fwnode)
  100. goto err;
  101. kfree(name);
  102. name = NULL;
  103. vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
  104. sgi_domain_ops, vpe);
  105. if (!vpe->sgi_domain)
  106. goto err;
  107. sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
  108. NUMA_NO_NODE, vpe,
  109. false, NULL);
  110. if (sgi_base <= 0)
  111. goto err;
  112. return 0;
  113. err:
  114. if (vpe->sgi_domain)
  115. irq_domain_remove(vpe->sgi_domain);
  116. if (vpe->fwnode)
  117. irq_domain_free_fwnode(vpe->fwnode);
  118. kfree(name);
  119. return -ENOMEM;
  120. }
  121. int its_alloc_vcpu_irqs(struct its_vm *vm)
  122. {
  123. int vpe_base_irq, i;
  124. vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
  125. task_pid_nr(current));
  126. if (!vm->fwnode)
  127. goto err;
  128. vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
  129. vm->fwnode, vpe_domain_ops,
  130. vm);
  131. if (!vm->domain)
  132. goto err;
  133. for (i = 0; i < vm->nr_vpes; i++) {
  134. vm->vpes[i]->its_vm = vm;
  135. vm->vpes[i]->idai = true;
  136. }
  137. vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
  138. NUMA_NO_NODE, vm,
  139. false, NULL);
  140. if (vpe_base_irq <= 0)
  141. goto err;
  142. for (i = 0; i < vm->nr_vpes; i++) {
  143. int ret;
  144. vm->vpes[i]->irq = vpe_base_irq + i;
  145. ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
  146. if (ret)
  147. goto err;
  148. }
  149. return 0;
  150. err:
  151. if (vm->domain)
  152. irq_domain_remove(vm->domain);
  153. if (vm->fwnode)
  154. irq_domain_free_fwnode(vm->fwnode);
  155. return -ENOMEM;
  156. }
  157. static void its_free_sgi_irqs(struct its_vm *vm)
  158. {
  159. int i;
  160. if (!has_v4_1())
  161. return;
  162. for (i = 0; i < vm->nr_vpes; i++) {
  163. unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
  164. if (WARN_ON(!irq))
  165. continue;
  166. irq_domain_free_irqs(irq, 16);
  167. irq_domain_remove(vm->vpes[i]->sgi_domain);
  168. irq_domain_free_fwnode(vm->vpes[i]->fwnode);
  169. }
  170. }
  171. void its_free_vcpu_irqs(struct its_vm *vm)
  172. {
  173. its_free_sgi_irqs(vm);
  174. irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
  175. irq_domain_remove(vm->domain);
  176. irq_domain_free_fwnode(vm->fwnode);
  177. }
  178. static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
  179. {
  180. return irq_set_vcpu_affinity(vpe->irq, info);
  181. }
  182. int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
  183. {
  184. struct irq_desc *desc = irq_to_desc(vpe->irq);
  185. struct its_cmd_info info = { };
  186. int ret;
  187. WARN_ON(preemptible());
  188. info.cmd_type = DESCHEDULE_VPE;
  189. if (has_v4_1()) {
  190. /* GICv4.1 can directly deal with doorbells */
  191. info.req_db = db;
  192. } else {
  193. /* Undo the nested disable_irq() calls... */
  194. while (db && irqd_irq_disabled(&desc->irq_data))
  195. enable_irq(vpe->irq);
  196. }
  197. ret = its_send_vpe_cmd(vpe, &info);
  198. if (!ret)
  199. vpe->resident = false;
  200. vpe->ready = false;
  201. return ret;
  202. }
  203. int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
  204. {
  205. struct its_cmd_info info = { };
  206. int ret;
  207. WARN_ON(preemptible());
  208. info.cmd_type = SCHEDULE_VPE;
  209. if (has_v4_1()) {
  210. info.g0en = g0en;
  211. info.g1en = g1en;
  212. } else {
  213. /* Disabled the doorbell, as we're about to enter the guest */
  214. disable_irq_nosync(vpe->irq);
  215. }
  216. ret = its_send_vpe_cmd(vpe, &info);
  217. if (!ret)
  218. vpe->resident = true;
  219. return ret;
  220. }
  221. int its_commit_vpe(struct its_vpe *vpe)
  222. {
  223. struct its_cmd_info info = {
  224. .cmd_type = COMMIT_VPE,
  225. };
  226. int ret;
  227. WARN_ON(preemptible());
  228. ret = its_send_vpe_cmd(vpe, &info);
  229. if (!ret)
  230. vpe->ready = true;
  231. return ret;
  232. }
  233. int its_invall_vpe(struct its_vpe *vpe)
  234. {
  235. struct its_cmd_info info = {
  236. .cmd_type = INVALL_VPE,
  237. };
  238. return its_send_vpe_cmd(vpe, &info);
  239. }
  240. int its_map_vlpi(int irq, struct its_vlpi_map *map)
  241. {
  242. struct its_cmd_info info = {
  243. .cmd_type = MAP_VLPI,
  244. {
  245. .map = map,
  246. },
  247. };
  248. int ret;
  249. /*
  250. * The host will never see that interrupt firing again, so it
  251. * is vital that we don't do any lazy masking.
  252. */
  253. irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
  254. ret = irq_set_vcpu_affinity(irq, &info);
  255. if (ret)
  256. irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
  257. return ret;
  258. }
  259. int its_get_vlpi(int irq, struct its_vlpi_map *map)
  260. {
  261. struct its_cmd_info info = {
  262. .cmd_type = GET_VLPI,
  263. {
  264. .map = map,
  265. },
  266. };
  267. return irq_set_vcpu_affinity(irq, &info);
  268. }
  269. int its_unmap_vlpi(int irq)
  270. {
  271. irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
  272. return irq_set_vcpu_affinity(irq, NULL);
  273. }
  274. int its_prop_update_vlpi(int irq, u8 config, bool inv)
  275. {
  276. struct its_cmd_info info = {
  277. .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
  278. {
  279. .config = config,
  280. },
  281. };
  282. return irq_set_vcpu_affinity(irq, &info);
  283. }
  284. int its_prop_update_vsgi(int irq, u8 priority, bool group)
  285. {
  286. struct its_cmd_info info = {
  287. .cmd_type = PROP_UPDATE_VSGI,
  288. {
  289. .priority = priority,
  290. .group = group,
  291. },
  292. };
  293. return irq_set_vcpu_affinity(irq, &info);
  294. }
  295. int its_init_v4(struct irq_domain *domain,
  296. const struct irq_domain_ops *vpe_ops,
  297. const struct irq_domain_ops *sgi_ops)
  298. {
  299. if (domain) {
  300. pr_info("ITS: Enabling GICv4 support\n");
  301. gic_domain = domain;
  302. vpe_domain_ops = vpe_ops;
  303. sgi_domain_ops = sgi_ops;
  304. return 0;
  305. }
  306. pr_err("ITS: No GICv4 VPE domain allocated\n");
  307. return -ENODEV;
  308. }