irq-gic-v2m.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * ARM GIC v2m MSI(-X) support
  4. * Support for Message Signaled Interrupts for systems that
  5. * implement ARM Generic Interrupt Controller: GICv2m.
  6. *
  7. * Copyright (C) 2014 Advanced Micro Devices, Inc.
  8. * Authors: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  9. * Harish Kasiviswanathan <harish.kasiviswanathan@amd.com>
  10. * Brandon Anderson <brandon.anderson@amd.com>
  11. */
  12. #define pr_fmt(fmt) "GICv2m: " fmt
  13. #include <linux/acpi.h>
  14. #include <linux/dma-iommu.h>
  15. #include <linux/irq.h>
  16. #include <linux/irqdomain.h>
  17. #include <linux/kernel.h>
  18. #include <linux/pci.h>
  19. #include <linux/msi.h>
  20. #include <linux/of_address.h>
  21. #include <linux/of_pci.h>
  22. #include <linux/slab.h>
  23. #include <linux/spinlock.h>
  24. #include <linux/irqchip/arm-gic.h>
  25. /*
  26. * MSI_TYPER:
  27. * [31:26] Reserved
  28. * [25:16] lowest SPI assigned to MSI
  29. * [15:10] Reserved
  30. * [9:0] Numer of SPIs assigned to MSI
  31. */
  32. #define V2M_MSI_TYPER 0x008
  33. #define V2M_MSI_TYPER_BASE_SHIFT 16
  34. #define V2M_MSI_TYPER_BASE_MASK 0x3FF
  35. #define V2M_MSI_TYPER_NUM_MASK 0x3FF
  36. #define V2M_MSI_SETSPI_NS 0x040
  37. #define V2M_MIN_SPI 32
  38. #define V2M_MAX_SPI 1019
  39. #define V2M_MSI_IIDR 0xFCC
  40. #define V2M_MSI_TYPER_BASE_SPI(x) \
  41. (((x) >> V2M_MSI_TYPER_BASE_SHIFT) & V2M_MSI_TYPER_BASE_MASK)
  42. #define V2M_MSI_TYPER_NUM_SPI(x) ((x) & V2M_MSI_TYPER_NUM_MASK)
  43. /* APM X-Gene with GICv2m MSI_IIDR register value */
  44. #define XGENE_GICV2M_MSI_IIDR 0x06000170
  45. /* Broadcom NS2 GICv2m MSI_IIDR register value */
  46. #define BCM_NS2_GICV2M_MSI_IIDR 0x0000013f
  47. /* List of flags for specific v2m implementation */
  48. #define GICV2M_NEEDS_SPI_OFFSET 0x00000001
  49. #define GICV2M_GRAVITON_ADDRESS_ONLY 0x00000002
  50. static LIST_HEAD(v2m_nodes);
  51. static DEFINE_SPINLOCK(v2m_lock);
  52. struct v2m_data {
  53. struct list_head entry;
  54. struct fwnode_handle *fwnode;
  55. struct resource res; /* GICv2m resource */
  56. void __iomem *base; /* GICv2m virt address */
  57. u32 spi_start; /* The SPI number that MSIs start */
  58. u32 nr_spis; /* The number of SPIs for MSIs */
  59. u32 spi_offset; /* offset to be subtracted from SPI number */
  60. unsigned long *bm; /* MSI vector bitmap */
  61. u32 flags; /* v2m flags for specific implementation */
  62. };
  63. static void gicv2m_mask_msi_irq(struct irq_data *d)
  64. {
  65. pci_msi_mask_irq(d);
  66. irq_chip_mask_parent(d);
  67. }
  68. static void gicv2m_unmask_msi_irq(struct irq_data *d)
  69. {
  70. pci_msi_unmask_irq(d);
  71. irq_chip_unmask_parent(d);
  72. }
  73. static struct irq_chip gicv2m_msi_irq_chip = {
  74. .name = "MSI",
  75. .irq_mask = gicv2m_mask_msi_irq,
  76. .irq_unmask = gicv2m_unmask_msi_irq,
  77. .irq_eoi = irq_chip_eoi_parent,
  78. .irq_write_msi_msg = pci_msi_domain_write_msg,
  79. };
  80. static struct msi_domain_info gicv2m_msi_domain_info = {
  81. .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
  82. MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI),
  83. .chip = &gicv2m_msi_irq_chip,
  84. };
  85. static phys_addr_t gicv2m_get_msi_addr(struct v2m_data *v2m, int hwirq)
  86. {
  87. if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)
  88. return v2m->res.start | ((hwirq - 32) << 3);
  89. else
  90. return v2m->res.start + V2M_MSI_SETSPI_NS;
  91. }
  92. static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
  93. {
  94. struct v2m_data *v2m = irq_data_get_irq_chip_data(data);
  95. phys_addr_t addr = gicv2m_get_msi_addr(v2m, data->hwirq);
  96. msg->address_hi = upper_32_bits(addr);
  97. msg->address_lo = lower_32_bits(addr);
  98. if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)
  99. msg->data = 0;
  100. else
  101. msg->data = data->hwirq;
  102. if (v2m->flags & GICV2M_NEEDS_SPI_OFFSET)
  103. msg->data -= v2m->spi_offset;
  104. iommu_dma_compose_msi_msg(irq_data_get_msi_desc(data), msg);
  105. }
  106. static struct irq_chip gicv2m_irq_chip = {
  107. .name = "GICv2m",
  108. .irq_mask = irq_chip_mask_parent,
  109. .irq_unmask = irq_chip_unmask_parent,
  110. .irq_eoi = irq_chip_eoi_parent,
  111. .irq_set_affinity = irq_chip_set_affinity_parent,
  112. .irq_compose_msi_msg = gicv2m_compose_msi_msg,
  113. };
  114. static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain,
  115. unsigned int virq,
  116. irq_hw_number_t hwirq)
  117. {
  118. struct irq_fwspec fwspec;
  119. struct irq_data *d;
  120. int err;
  121. if (is_of_node(domain->parent->fwnode)) {
  122. fwspec.fwnode = domain->parent->fwnode;
  123. fwspec.param_count = 3;
  124. fwspec.param[0] = 0;
  125. fwspec.param[1] = hwirq - 32;
  126. fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
  127. } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
  128. fwspec.fwnode = domain->parent->fwnode;
  129. fwspec.param_count = 2;
  130. fwspec.param[0] = hwirq;
  131. fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
  132. } else {
  133. return -EINVAL;
  134. }
  135. err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
  136. if (err)
  137. return err;
  138. /* Configure the interrupt line to be edge */
  139. d = irq_domain_get_irq_data(domain->parent, virq);
  140. d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING);
  141. return 0;
  142. }
  143. static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq,
  144. int nr_irqs)
  145. {
  146. spin_lock(&v2m_lock);
  147. bitmap_release_region(v2m->bm, hwirq - v2m->spi_start,
  148. get_count_order(nr_irqs));
  149. spin_unlock(&v2m_lock);
  150. }
  151. static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
  152. unsigned int nr_irqs, void *args)
  153. {
  154. msi_alloc_info_t *info = args;
  155. struct v2m_data *v2m = NULL, *tmp;
  156. int hwirq, offset, i, err = 0;
  157. spin_lock(&v2m_lock);
  158. list_for_each_entry(tmp, &v2m_nodes, entry) {
  159. offset = bitmap_find_free_region(tmp->bm, tmp->nr_spis,
  160. get_count_order(nr_irqs));
  161. if (offset >= 0) {
  162. v2m = tmp;
  163. break;
  164. }
  165. }
  166. spin_unlock(&v2m_lock);
  167. if (!v2m)
  168. return -ENOSPC;
  169. hwirq = v2m->spi_start + offset;
  170. err = iommu_dma_prepare_msi(info->desc,
  171. gicv2m_get_msi_addr(v2m, hwirq));
  172. if (err)
  173. return err;
  174. for (i = 0; i < nr_irqs; i++) {
  175. err = gicv2m_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
  176. if (err)
  177. goto fail;
  178. irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
  179. &gicv2m_irq_chip, v2m);
  180. }
  181. return 0;
  182. fail:
  183. irq_domain_free_irqs_parent(domain, virq, nr_irqs);
  184. gicv2m_unalloc_msi(v2m, hwirq, nr_irqs);
  185. return err;
  186. }
  187. static void gicv2m_irq_domain_free(struct irq_domain *domain,
  188. unsigned int virq, unsigned int nr_irqs)
  189. {
  190. struct irq_data *d = irq_domain_get_irq_data(domain, virq);
  191. struct v2m_data *v2m = irq_data_get_irq_chip_data(d);
  192. gicv2m_unalloc_msi(v2m, d->hwirq, nr_irqs);
  193. irq_domain_free_irqs_parent(domain, virq, nr_irqs);
  194. }
  195. static const struct irq_domain_ops gicv2m_domain_ops = {
  196. .alloc = gicv2m_irq_domain_alloc,
  197. .free = gicv2m_irq_domain_free,
  198. };
  199. static bool is_msi_spi_valid(u32 base, u32 num)
  200. {
  201. if (base < V2M_MIN_SPI) {
  202. pr_err("Invalid MSI base SPI (base:%u)\n", base);
  203. return false;
  204. }
  205. if ((num == 0) || (base + num > V2M_MAX_SPI)) {
  206. pr_err("Number of SPIs (%u) exceed maximum (%u)\n",
  207. num, V2M_MAX_SPI - V2M_MIN_SPI + 1);
  208. return false;
  209. }
  210. return true;
  211. }
  212. static struct irq_chip gicv2m_pmsi_irq_chip = {
  213. .name = "pMSI",
  214. };
  215. static struct msi_domain_ops gicv2m_pmsi_ops = {
  216. };
  217. static struct msi_domain_info gicv2m_pmsi_domain_info = {
  218. .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
  219. .ops = &gicv2m_pmsi_ops,
  220. .chip = &gicv2m_pmsi_irq_chip,
  221. };
  222. static void gicv2m_teardown(void)
  223. {
  224. struct v2m_data *v2m, *tmp;
  225. list_for_each_entry_safe(v2m, tmp, &v2m_nodes, entry) {
  226. list_del(&v2m->entry);
  227. kfree(v2m->bm);
  228. iounmap(v2m->base);
  229. of_node_put(to_of_node(v2m->fwnode));
  230. if (is_fwnode_irqchip(v2m->fwnode))
  231. irq_domain_free_fwnode(v2m->fwnode);
  232. kfree(v2m);
  233. }
  234. }
  235. static int gicv2m_allocate_domains(struct irq_domain *parent)
  236. {
  237. struct irq_domain *inner_domain, *pci_domain, *plat_domain;
  238. struct v2m_data *v2m;
  239. v2m = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry);
  240. if (!v2m)
  241. return 0;
  242. inner_domain = irq_domain_create_tree(v2m->fwnode,
  243. &gicv2m_domain_ops, v2m);
  244. if (!inner_domain) {
  245. pr_err("Failed to create GICv2m domain\n");
  246. return -ENOMEM;
  247. }
  248. irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
  249. inner_domain->parent = parent;
  250. pci_domain = pci_msi_create_irq_domain(v2m->fwnode,
  251. &gicv2m_msi_domain_info,
  252. inner_domain);
  253. plat_domain = platform_msi_create_irq_domain(v2m->fwnode,
  254. &gicv2m_pmsi_domain_info,
  255. inner_domain);
  256. if (!pci_domain || !plat_domain) {
  257. pr_err("Failed to create MSI domains\n");
  258. if (plat_domain)
  259. irq_domain_remove(plat_domain);
  260. if (pci_domain)
  261. irq_domain_remove(pci_domain);
  262. irq_domain_remove(inner_domain);
  263. return -ENOMEM;
  264. }
  265. return 0;
  266. }
  267. static int __init gicv2m_init_one(struct fwnode_handle *fwnode,
  268. u32 spi_start, u32 nr_spis,
  269. struct resource *res, u32 flags)
  270. {
  271. int ret;
  272. struct v2m_data *v2m;
  273. v2m = kzalloc(sizeof(struct v2m_data), GFP_KERNEL);
  274. if (!v2m) {
  275. pr_err("Failed to allocate struct v2m_data.\n");
  276. return -ENOMEM;
  277. }
  278. INIT_LIST_HEAD(&v2m->entry);
  279. v2m->fwnode = fwnode;
  280. v2m->flags = flags;
  281. memcpy(&v2m->res, res, sizeof(struct resource));
  282. v2m->base = ioremap(v2m->res.start, resource_size(&v2m->res));
  283. if (!v2m->base) {
  284. pr_err("Failed to map GICv2m resource\n");
  285. ret = -ENOMEM;
  286. goto err_free_v2m;
  287. }
  288. if (spi_start && nr_spis) {
  289. v2m->spi_start = spi_start;
  290. v2m->nr_spis = nr_spis;
  291. } else {
  292. u32 typer;
  293. /* Graviton should always have explicit spi_start/nr_spis */
  294. if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) {
  295. ret = -EINVAL;
  296. goto err_iounmap;
  297. }
  298. typer = readl_relaxed(v2m->base + V2M_MSI_TYPER);
  299. v2m->spi_start = V2M_MSI_TYPER_BASE_SPI(typer);
  300. v2m->nr_spis = V2M_MSI_TYPER_NUM_SPI(typer);
  301. }
  302. if (!is_msi_spi_valid(v2m->spi_start, v2m->nr_spis)) {
  303. ret = -EINVAL;
  304. goto err_iounmap;
  305. }
  306. /*
  307. * APM X-Gene GICv2m implementation has an erratum where
  308. * the MSI data needs to be the offset from the spi_start
  309. * in order to trigger the correct MSI interrupt. This is
  310. * different from the standard GICv2m implementation where
  311. * the MSI data is the absolute value within the range from
  312. * spi_start to (spi_start + num_spis).
  313. *
  314. * Broadom NS2 GICv2m implementation has an erratum where the MSI data
  315. * is 'spi_number - 32'
  316. *
  317. * Reading that register fails on the Graviton implementation
  318. */
  319. if (!(v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)) {
  320. switch (readl_relaxed(v2m->base + V2M_MSI_IIDR)) {
  321. case XGENE_GICV2M_MSI_IIDR:
  322. v2m->flags |= GICV2M_NEEDS_SPI_OFFSET;
  323. v2m->spi_offset = v2m->spi_start;
  324. break;
  325. case BCM_NS2_GICV2M_MSI_IIDR:
  326. v2m->flags |= GICV2M_NEEDS_SPI_OFFSET;
  327. v2m->spi_offset = 32;
  328. break;
  329. }
  330. }
  331. v2m->bm = kcalloc(BITS_TO_LONGS(v2m->nr_spis), sizeof(long),
  332. GFP_KERNEL);
  333. if (!v2m->bm) {
  334. ret = -ENOMEM;
  335. goto err_iounmap;
  336. }
  337. list_add_tail(&v2m->entry, &v2m_nodes);
  338. pr_info("range%pR, SPI[%d:%d]\n", res,
  339. v2m->spi_start, (v2m->spi_start + v2m->nr_spis - 1));
  340. return 0;
  341. err_iounmap:
  342. iounmap(v2m->base);
  343. err_free_v2m:
  344. kfree(v2m);
  345. return ret;
  346. }
  347. static struct of_device_id gicv2m_device_id[] = {
  348. { .compatible = "arm,gic-v2m-frame", },
  349. {},
  350. };
  351. static int __init gicv2m_of_init(struct fwnode_handle *parent_handle,
  352. struct irq_domain *parent)
  353. {
  354. int ret = 0;
  355. struct device_node *node = to_of_node(parent_handle);
  356. struct device_node *child;
  357. for (child = of_find_matching_node(node, gicv2m_device_id); child;
  358. child = of_find_matching_node(child, gicv2m_device_id)) {
  359. u32 spi_start = 0, nr_spis = 0;
  360. struct resource res;
  361. if (!of_find_property(child, "msi-controller", NULL))
  362. continue;
  363. ret = of_address_to_resource(child, 0, &res);
  364. if (ret) {
  365. pr_err("Failed to allocate v2m resource.\n");
  366. break;
  367. }
  368. if (!of_property_read_u32(child, "arm,msi-base-spi",
  369. &spi_start) &&
  370. !of_property_read_u32(child, "arm,msi-num-spis", &nr_spis))
  371. pr_info("DT overriding V2M MSI_TYPER (base:%u, num:%u)\n",
  372. spi_start, nr_spis);
  373. ret = gicv2m_init_one(&child->fwnode, spi_start, nr_spis,
  374. &res, 0);
  375. if (ret) {
  376. of_node_put(child);
  377. break;
  378. }
  379. }
  380. if (!ret)
  381. ret = gicv2m_allocate_domains(parent);
  382. if (ret)
  383. gicv2m_teardown();
  384. return ret;
  385. }
  386. #ifdef CONFIG_ACPI
  387. static int acpi_num_msi;
  388. static struct fwnode_handle *gicv2m_get_fwnode(struct device *dev)
  389. {
  390. struct v2m_data *data;
  391. if (WARN_ON(acpi_num_msi <= 0))
  392. return NULL;
  393. /* We only return the fwnode of the first MSI frame. */
  394. data = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry);
  395. if (!data)
  396. return NULL;
  397. return data->fwnode;
  398. }
  399. static bool acpi_check_amazon_graviton_quirks(void)
  400. {
  401. static struct acpi_table_madt *madt;
  402. acpi_status status;
  403. bool rc = false;
  404. #define ACPI_AMZN_OEM_ID "AMAZON"
  405. status = acpi_get_table(ACPI_SIG_MADT, 0,
  406. (struct acpi_table_header **)&madt);
  407. if (ACPI_FAILURE(status) || !madt)
  408. return rc;
  409. rc = !memcmp(madt->header.oem_id, ACPI_AMZN_OEM_ID, ACPI_OEM_ID_SIZE);
  410. acpi_put_table((struct acpi_table_header *)madt);
  411. return rc;
  412. }
  413. static int __init
  414. acpi_parse_madt_msi(union acpi_subtable_headers *header,
  415. const unsigned long end)
  416. {
  417. int ret;
  418. struct resource res;
  419. u32 spi_start = 0, nr_spis = 0;
  420. struct acpi_madt_generic_msi_frame *m;
  421. struct fwnode_handle *fwnode;
  422. u32 flags = 0;
  423. m = (struct acpi_madt_generic_msi_frame *)header;
  424. if (BAD_MADT_ENTRY(m, end))
  425. return -EINVAL;
  426. res.start = m->base_address;
  427. res.end = m->base_address + SZ_4K - 1;
  428. res.flags = IORESOURCE_MEM;
  429. if (acpi_check_amazon_graviton_quirks()) {
  430. pr_info("applying Amazon Graviton quirk\n");
  431. res.end = res.start + SZ_8K - 1;
  432. flags |= GICV2M_GRAVITON_ADDRESS_ONLY;
  433. gicv2m_msi_domain_info.flags &= ~MSI_FLAG_MULTI_PCI_MSI;
  434. }
  435. if (m->flags & ACPI_MADT_OVERRIDE_SPI_VALUES) {
  436. spi_start = m->spi_base;
  437. nr_spis = m->spi_count;
  438. pr_info("ACPI overriding V2M MSI_TYPER (base:%u, num:%u)\n",
  439. spi_start, nr_spis);
  440. }
  441. fwnode = irq_domain_alloc_fwnode(&res.start);
  442. if (!fwnode) {
  443. pr_err("Unable to allocate GICv2m domain token\n");
  444. return -EINVAL;
  445. }
  446. ret = gicv2m_init_one(fwnode, spi_start, nr_spis, &res, flags);
  447. if (ret)
  448. irq_domain_free_fwnode(fwnode);
  449. return ret;
  450. }
  451. static int __init gicv2m_acpi_init(struct irq_domain *parent)
  452. {
  453. int ret;
  454. if (acpi_num_msi > 0)
  455. return 0;
  456. acpi_num_msi = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_MSI_FRAME,
  457. acpi_parse_madt_msi, 0);
  458. if (acpi_num_msi <= 0)
  459. goto err_out;
  460. ret = gicv2m_allocate_domains(parent);
  461. if (ret)
  462. goto err_out;
  463. pci_msi_register_fwnode_provider(&gicv2m_get_fwnode);
  464. return 0;
  465. err_out:
  466. gicv2m_teardown();
  467. return -EINVAL;
  468. }
  469. #else /* CONFIG_ACPI */
  470. static int __init gicv2m_acpi_init(struct irq_domain *parent)
  471. {
  472. return -EINVAL;
  473. }
  474. #endif /* CONFIG_ACPI */
  475. int __init gicv2m_init(struct fwnode_handle *parent_handle,
  476. struct irq_domain *parent)
  477. {
  478. if (is_of_node(parent_handle))
  479. return gicv2m_of_init(parent_handle, parent);
  480. return gicv2m_acpi_init(parent);
  481. }