irq-sifive-plic.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2017 SiFive
  4. * Copyright (C) 2018 Christoph Hellwig
  5. */
  6. #define pr_fmt(fmt) "plic: " fmt
  7. #include <linux/cpu.h>
  8. #include <linux/interrupt.h>
  9. #include <linux/io.h>
  10. #include <linux/irq.h>
  11. #include <linux/irqchip.h>
  12. #include <linux/irqchip/chained_irq.h>
  13. #include <linux/irqdomain.h>
  14. #include <linux/module.h>
  15. #include <linux/of.h>
  16. #include <linux/of_address.h>
  17. #include <linux/of_irq.h>
  18. #include <linux/platform_device.h>
  19. #include <linux/spinlock.h>
  20. #include <asm/smp.h>
  21. /*
  22. * This driver implements a version of the RISC-V PLIC with the actual layout
  23. * specified in chapter 8 of the SiFive U5 Coreplex Series Manual:
  24. *
  25. * https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf
  26. *
  27. * The largest number supported by devices marked as 'sifive,plic-1.0.0', is
  28. * 1024, of which device 0 is defined as non-existent by the RISC-V Privileged
  29. * Spec.
  30. */
  31. #define MAX_DEVICES 1024
  32. #define MAX_CONTEXTS 15872
  33. /*
  34. * Each interrupt source has a priority register associated with it.
  35. * We always hardwire it to one in Linux.
  36. */
  37. #define PRIORITY_BASE 0
  38. #define PRIORITY_PER_ID 4
  39. /*
  40. * Each hart context has a vector of interrupt enable bits associated with it.
  41. * There's one bit for each interrupt source.
  42. */
  43. #define ENABLE_BASE 0x2000
  44. #define ENABLE_PER_HART 0x80
  45. /*
  46. * Each hart context has a set of control registers associated with it. Right
  47. * now there's only two: a source priority threshold over which the hart will
  48. * take an interrupt, and a register to claim interrupts.
  49. */
  50. #define CONTEXT_BASE 0x200000
  51. #define CONTEXT_PER_HART 0x1000
  52. #define CONTEXT_THRESHOLD 0x00
  53. #define CONTEXT_CLAIM 0x04
  54. #define PLIC_DISABLE_THRESHOLD 0x7
  55. #define PLIC_ENABLE_THRESHOLD 0
  56. struct plic_priv {
  57. struct cpumask lmask;
  58. struct irq_domain *irqdomain;
  59. void __iomem *regs;
  60. unsigned int nr_irqs;
  61. };
  62. struct plic_handler {
  63. bool present;
  64. void __iomem *hart_base;
  65. /*
  66. * Protect mask operations on the registers given that we can't
  67. * assume atomic memory operations work on them.
  68. */
  69. raw_spinlock_t enable_lock;
  70. void __iomem *enable_base;
  71. struct plic_priv *priv;
  72. };
  73. static int plic_parent_irq;
  74. static bool plic_cpuhp_setup_done;
  75. static DEFINE_PER_CPU(struct plic_handler, plic_handlers);
  76. static inline void plic_toggle(struct plic_handler *handler,
  77. int hwirq, int enable)
  78. {
  79. u32 __iomem *reg = handler->enable_base + (hwirq / 32) * sizeof(u32);
  80. u32 hwirq_mask = 1 << (hwirq % 32);
  81. raw_spin_lock(&handler->enable_lock);
  82. if (enable)
  83. writel(readl(reg) | hwirq_mask, reg);
  84. else
  85. writel(readl(reg) & ~hwirq_mask, reg);
  86. raw_spin_unlock(&handler->enable_lock);
  87. }
  88. static inline void plic_irq_toggle(const struct cpumask *mask,
  89. struct irq_data *d, int enable)
  90. {
  91. int cpu;
  92. struct plic_priv *priv = irq_data_get_irq_chip_data(d);
  93. writel(enable, priv->regs + PRIORITY_BASE + d->hwirq * PRIORITY_PER_ID);
  94. for_each_cpu(cpu, mask) {
  95. struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
  96. if (handler->present &&
  97. cpumask_test_cpu(cpu, &handler->priv->lmask))
  98. plic_toggle(handler, d->hwirq, enable);
  99. }
  100. }
  101. static void plic_irq_unmask(struct irq_data *d)
  102. {
  103. struct cpumask amask;
  104. unsigned int cpu;
  105. struct plic_priv *priv = irq_data_get_irq_chip_data(d);
  106. cpumask_and(&amask, &priv->lmask, cpu_online_mask);
  107. cpu = cpumask_any_and(irq_data_get_affinity_mask(d),
  108. &amask);
  109. if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
  110. return;
  111. plic_irq_toggle(cpumask_of(cpu), d, 1);
  112. }
  113. static void plic_irq_mask(struct irq_data *d)
  114. {
  115. struct plic_priv *priv = irq_data_get_irq_chip_data(d);
  116. plic_irq_toggle(&priv->lmask, d, 0);
  117. }
  118. #ifdef CONFIG_SMP
  119. static int plic_set_affinity(struct irq_data *d,
  120. const struct cpumask *mask_val, bool force)
  121. {
  122. unsigned int cpu;
  123. struct cpumask amask;
  124. struct plic_priv *priv = irq_data_get_irq_chip_data(d);
  125. cpumask_and(&amask, &priv->lmask, mask_val);
  126. if (force)
  127. cpu = cpumask_first(&amask);
  128. else
  129. cpu = cpumask_any_and(&amask, cpu_online_mask);
  130. if (cpu >= nr_cpu_ids)
  131. return -EINVAL;
  132. plic_irq_toggle(cpu_online_mask, d, 0);
  133. plic_irq_toggle(cpumask_of(cpu), d, !irqd_irq_masked(d));
  134. irq_data_update_effective_affinity(d, cpumask_of(cpu));
  135. return IRQ_SET_MASK_OK_DONE;
  136. }
  137. #endif
  138. static void plic_irq_eoi(struct irq_data *d)
  139. {
  140. struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
  141. if (irqd_irq_masked(d)) {
  142. plic_irq_unmask(d);
  143. writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
  144. plic_irq_mask(d);
  145. } else {
  146. writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
  147. }
  148. }
  149. static struct irq_chip plic_chip = {
  150. .name = "SiFive PLIC",
  151. .irq_mask = plic_irq_mask,
  152. .irq_unmask = plic_irq_unmask,
  153. .irq_eoi = plic_irq_eoi,
  154. #ifdef CONFIG_SMP
  155. .irq_set_affinity = plic_set_affinity,
  156. #endif
  157. };
  158. static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq,
  159. irq_hw_number_t hwirq)
  160. {
  161. struct plic_priv *priv = d->host_data;
  162. irq_domain_set_info(d, irq, hwirq, &plic_chip, d->host_data,
  163. handle_fasteoi_irq, NULL, NULL);
  164. irq_set_noprobe(irq);
  165. irq_set_affinity(irq, &priv->lmask);
  166. return 0;
  167. }
  168. static int plic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
  169. unsigned int nr_irqs, void *arg)
  170. {
  171. int i, ret;
  172. irq_hw_number_t hwirq;
  173. unsigned int type;
  174. struct irq_fwspec *fwspec = arg;
  175. ret = irq_domain_translate_onecell(domain, fwspec, &hwirq, &type);
  176. if (ret)
  177. return ret;
  178. for (i = 0; i < nr_irqs; i++) {
  179. ret = plic_irqdomain_map(domain, virq + i, hwirq + i);
  180. if (ret)
  181. return ret;
  182. }
  183. return 0;
  184. }
  185. static const struct irq_domain_ops plic_irqdomain_ops = {
  186. .translate = irq_domain_translate_onecell,
  187. .alloc = plic_irq_domain_alloc,
  188. .free = irq_domain_free_irqs_top,
  189. };
  190. /*
  191. * Handling an interrupt is a two-step process: first you claim the interrupt
  192. * by reading the claim register, then you complete the interrupt by writing
  193. * that source ID back to the same claim register. This automatically enables
  194. * and disables the interrupt, so there's nothing else to do.
  195. */
  196. static void plic_handle_irq(struct irq_desc *desc)
  197. {
  198. struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
  199. struct irq_chip *chip = irq_desc_get_chip(desc);
  200. void __iomem *claim = handler->hart_base + CONTEXT_CLAIM;
  201. irq_hw_number_t hwirq;
  202. WARN_ON_ONCE(!handler->present);
  203. chained_irq_enter(chip, desc);
  204. while ((hwirq = readl(claim))) {
  205. int irq = irq_find_mapping(handler->priv->irqdomain, hwirq);
  206. if (unlikely(irq <= 0))
  207. pr_warn_ratelimited("can't find mapping for hwirq %lu\n",
  208. hwirq);
  209. else
  210. generic_handle_irq(irq);
  211. }
  212. chained_irq_exit(chip, desc);
  213. }
  214. static void plic_set_threshold(struct plic_handler *handler, u32 threshold)
  215. {
  216. /* priority must be > threshold to trigger an interrupt */
  217. writel(threshold, handler->hart_base + CONTEXT_THRESHOLD);
  218. }
  219. static int plic_dying_cpu(unsigned int cpu)
  220. {
  221. struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
  222. if (plic_parent_irq)
  223. disable_percpu_irq(plic_parent_irq);
  224. handler->present = false;
  225. return 0;
  226. }
  227. static int plic_starting_cpu(unsigned int cpu)
  228. {
  229. struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
  230. irq_hw_number_t hwirq;
  231. handler->present = true;
  232. for (hwirq = 1; hwirq <= handler->priv->nr_irqs; hwirq++)
  233. plic_toggle(handler, hwirq, 0);
  234. if (plic_parent_irq)
  235. enable_percpu_irq(plic_parent_irq,
  236. irq_get_trigger_type(plic_parent_irq));
  237. else
  238. pr_warn("cpu%d: parent irq not available\n", cpu);
  239. plic_set_threshold(handler, PLIC_ENABLE_THRESHOLD);
  240. return 0;
  241. }
  242. static int __init plic_init(struct device_node *node,
  243. struct device_node *parent)
  244. {
  245. int error = 0, nr_contexts, nr_handlers = 0, i;
  246. struct plic_priv *priv;
  247. struct plic_handler *handler;
  248. priv = kzalloc(sizeof(*priv), GFP_KERNEL);
  249. if (!priv)
  250. return -ENOMEM;
  251. priv->regs = of_iomap(node, 0);
  252. if (WARN_ON(!priv->regs)) {
  253. error = -EIO;
  254. goto out_free_priv;
  255. }
  256. error = -EINVAL;
  257. of_property_read_u32(node, "riscv,ndev", &priv->nr_irqs);
  258. if (WARN_ON(!priv->nr_irqs))
  259. goto out_iounmap;
  260. nr_contexts = of_irq_count(node);
  261. if (WARN_ON(!nr_contexts))
  262. goto out_iounmap;
  263. error = -ENOMEM;
  264. priv->irqdomain = irq_domain_add_linear(node, priv->nr_irqs + 1,
  265. &plic_irqdomain_ops, priv);
  266. if (WARN_ON(!priv->irqdomain))
  267. goto out_iounmap;
  268. for (i = 0; i < nr_contexts; i++) {
  269. struct of_phandle_args parent;
  270. int cpu, hartid;
  271. if (of_irq_parse_one(node, i, &parent)) {
  272. pr_err("failed to parse parent for context %d.\n", i);
  273. continue;
  274. }
  275. /*
  276. * Skip contexts other than external interrupts for our
  277. * privilege level.
  278. */
  279. if (parent.args[0] != RV_IRQ_EXT)
  280. continue;
  281. hartid = riscv_of_parent_hartid(parent.np);
  282. if (hartid < 0) {
  283. pr_warn("failed to parse hart ID for context %d.\n", i);
  284. continue;
  285. }
  286. cpu = riscv_hartid_to_cpuid(hartid);
  287. if (cpu < 0) {
  288. pr_warn("Invalid cpuid for context %d\n", i);
  289. continue;
  290. }
  291. /* Find parent domain and register chained handler */
  292. if (!plic_parent_irq && irq_find_host(parent.np)) {
  293. plic_parent_irq = irq_of_parse_and_map(node, i);
  294. if (plic_parent_irq)
  295. irq_set_chained_handler(plic_parent_irq,
  296. plic_handle_irq);
  297. }
  298. /*
  299. * When running in M-mode we need to ignore the S-mode handler.
  300. * Here we assume it always comes later, but that might be a
  301. * little fragile.
  302. */
  303. handler = per_cpu_ptr(&plic_handlers, cpu);
  304. if (handler->present) {
  305. pr_warn("handler already present for context %d.\n", i);
  306. plic_set_threshold(handler, PLIC_DISABLE_THRESHOLD);
  307. goto done;
  308. }
  309. cpumask_set_cpu(cpu, &priv->lmask);
  310. if (cpu == smp_processor_id())
  311. handler->present = true;
  312. handler->hart_base =
  313. priv->regs + CONTEXT_BASE + i * CONTEXT_PER_HART;
  314. raw_spin_lock_init(&handler->enable_lock);
  315. handler->enable_base =
  316. priv->regs + ENABLE_BASE + i * ENABLE_PER_HART;
  317. handler->priv = priv;
  318. done:
  319. nr_handlers++;
  320. }
  321. /*
  322. * We can have multiple PLIC instances so setup cpuhp state only
  323. * when context handler for current/boot CPU is present.
  324. */
  325. handler = this_cpu_ptr(&plic_handlers);
  326. if (handler->present && !plic_cpuhp_setup_done) {
  327. cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING,
  328. "irqchip/sifive/plic:starting",
  329. plic_starting_cpu, plic_dying_cpu);
  330. plic_cpuhp_setup_done = true;
  331. }
  332. pr_info("%pOFP: mapped %d interrupts with %d handlers for"
  333. " %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts);
  334. return 0;
  335. out_iounmap:
  336. iounmap(priv->regs);
  337. out_free_priv:
  338. kfree(priv);
  339. return error;
  340. }
  341. IRQCHIP_DECLARE(sifive_plic, "sifive,plic-1.0.0", plic_init);
  342. IRQCHIP_DECLARE(riscv_plic0, "riscv,plic0", plic_init); /* for legacy systems */
  343. IRQCHIP_DECLARE(thead_c900_plic, "thead,c900-plic", plic_init); /* for firmware driver */