irq-armada-370-xp.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804
  1. /*
  2. * Marvell Armada 370 and Armada XP SoC IRQ handling
  3. *
  4. * Copyright (C) 2012 Marvell
  5. *
  6. * Lior Amsalem <alior@marvell.com>
  7. * Gregory CLEMENT <gregory.clement@free-electrons.com>
  8. * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
  9. * Ben Dooks <ben.dooks@codethink.co.uk>
  10. *
  11. * This file is licensed under the terms of the GNU General Public
  12. * License version 2. This program is licensed "as is" without any
  13. * warranty of any kind, whether express or implied.
  14. */
  15. #include <linux/kernel.h>
  16. #include <linux/module.h>
  17. #include <linux/init.h>
  18. #include <linux/irq.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/irqchip.h>
  21. #include <linux/irqchip/chained_irq.h>
  22. #include <linux/cpu.h>
  23. #include <linux/io.h>
  24. #include <linux/of_address.h>
  25. #include <linux/of_irq.h>
  26. #include <linux/of_pci.h>
  27. #include <linux/irqdomain.h>
  28. #include <linux/slab.h>
  29. #include <linux/syscore_ops.h>
  30. #include <linux/msi.h>
  31. #include <asm/mach/arch.h>
  32. #include <asm/exception.h>
  33. #include <asm/smp_plat.h>
  34. #include <asm/mach/irq.h>
  35. /*
  36. * Overall diagram of the Armada XP interrupt controller:
  37. *
  38. * To CPU 0 To CPU 1
  39. *
  40. * /\ /\
  41. * || ||
  42. * +---------------+ +---------------+
  43. * | | | |
  44. * | per-CPU | | per-CPU |
  45. * | mask/unmask | | mask/unmask |
  46. * | CPU0 | | CPU1 |
  47. * | | | |
  48. * +---------------+ +---------------+
  49. * /\ /\
  50. * || ||
  51. * \\_______________________//
  52. * ||
  53. * +-------------------+
  54. * | |
  55. * | Global interrupt |
  56. * | mask/unmask |
  57. * | |
  58. * +-------------------+
  59. * /\
  60. * ||
  61. * interrupt from
  62. * device
  63. *
  64. * The "global interrupt mask/unmask" is modified using the
  65. * ARMADA_370_XP_INT_SET_ENABLE_OFFS and
  66. * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative
  67. * to "main_int_base".
  68. *
  69. * The "per-CPU mask/unmask" is modified using the
  70. * ARMADA_370_XP_INT_SET_MASK_OFFS and
  71. * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to
  72. * "per_cpu_int_base". This base address points to a special address,
  73. * which automatically accesses the registers of the current CPU.
  74. *
  75. * The per-CPU mask/unmask can also be adjusted using the global
  76. * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use
  77. * to configure interrupt affinity.
  78. *
  79. * Due to this model, all interrupts need to be mask/unmasked at two
  80. * different levels: at the global level and at the per-CPU level.
  81. *
  82. * This driver takes the following approach to deal with this:
  83. *
  84. * - For global interrupts:
  85. *
  86. * At ->map() time, a global interrupt is unmasked at the per-CPU
  87. * mask/unmask level. It is therefore unmasked at this level for
  88. * the current CPU, running the ->map() code. This allows to have
  89. * the interrupt unmasked at this level in non-SMP
  90. * configurations. In SMP configurations, the ->set_affinity()
  91. * callback is called, which using the
  92. * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask
  93. * for the interrupt.
  94. *
  95. * The ->mask() and ->unmask() operations only mask/unmask the
  96. * interrupt at the "global" level.
  97. *
  98. * So, a global interrupt is enabled at the per-CPU level as soon
  99. * as it is mapped. At run time, the masking/unmasking takes place
  100. * at the global level.
  101. *
  102. * - For per-CPU interrupts
  103. *
  104. * At ->map() time, a per-CPU interrupt is unmasked at the global
  105. * mask/unmask level.
  106. *
  107. * The ->mask() and ->unmask() operations mask/unmask the interrupt
  108. * at the per-CPU level.
  109. *
  110. * So, a per-CPU interrupt is enabled at the global level as soon
  111. * as it is mapped. At run time, the masking/unmasking takes place
  112. * at the per-CPU level.
  113. */
  114. /* Registers relative to main_int_base */
  115. #define ARMADA_370_XP_INT_CONTROL (0x00)
  116. #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04)
  117. #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30)
  118. #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34)
  119. #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4)
  120. #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF
  121. #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid)
  122. /* Registers relative to per_cpu_int_base */
  123. #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08)
  124. #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c)
  125. #define ARMADA_375_PPI_CAUSE (0x10)
  126. #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44)
  127. #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48)
  128. #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C)
  129. #define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54)
  130. #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu)
  131. #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28)
  132. #define IPI_DOORBELL_START (0)
  133. #define IPI_DOORBELL_END (8)
  134. #define IPI_DOORBELL_MASK 0xFF
  135. #define PCI_MSI_DOORBELL_START (16)
  136. #define PCI_MSI_DOORBELL_NR (16)
  137. #define PCI_MSI_DOORBELL_END (32)
  138. #define PCI_MSI_DOORBELL_MASK 0xFFFF0000
  139. static void __iomem *per_cpu_int_base;
  140. static void __iomem *main_int_base;
  141. static struct irq_domain *armada_370_xp_mpic_domain;
  142. static u32 doorbell_mask_reg;
  143. static int parent_irq;
  144. #ifdef CONFIG_PCI_MSI
  145. static struct irq_domain *armada_370_xp_msi_domain;
  146. static struct irq_domain *armada_370_xp_msi_inner_domain;
  147. static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR);
  148. static DEFINE_MUTEX(msi_used_lock);
  149. static phys_addr_t msi_doorbell_addr;
  150. #endif
  151. static inline bool is_percpu_irq(irq_hw_number_t irq)
  152. {
  153. if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS)
  154. return true;
  155. return false;
  156. }
  157. /*
  158. * In SMP mode:
  159. * For shared global interrupts, mask/unmask global enable bit
  160. * For CPU interrupts, mask/unmask the calling CPU's bit
  161. */
  162. static void armada_370_xp_irq_mask(struct irq_data *d)
  163. {
  164. irq_hw_number_t hwirq = irqd_to_hwirq(d);
  165. if (!is_percpu_irq(hwirq))
  166. writel(hwirq, main_int_base +
  167. ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS);
  168. else
  169. writel(hwirq, per_cpu_int_base +
  170. ARMADA_370_XP_INT_SET_MASK_OFFS);
  171. }
  172. static void armada_370_xp_irq_unmask(struct irq_data *d)
  173. {
  174. irq_hw_number_t hwirq = irqd_to_hwirq(d);
  175. if (!is_percpu_irq(hwirq))
  176. writel(hwirq, main_int_base +
  177. ARMADA_370_XP_INT_SET_ENABLE_OFFS);
  178. else
  179. writel(hwirq, per_cpu_int_base +
  180. ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  181. }
  182. #ifdef CONFIG_PCI_MSI
  183. static struct irq_chip armada_370_xp_msi_irq_chip = {
  184. .name = "MPIC MSI",
  185. .irq_mask = pci_msi_mask_irq,
  186. .irq_unmask = pci_msi_unmask_irq,
  187. };
  188. static struct msi_domain_info armada_370_xp_msi_domain_info = {
  189. .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
  190. MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX),
  191. .chip = &armada_370_xp_msi_irq_chip,
  192. };
  193. static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
  194. {
  195. msg->address_lo = lower_32_bits(msi_doorbell_addr);
  196. msg->address_hi = upper_32_bits(msi_doorbell_addr);
  197. msg->data = 0xf00 | (data->hwirq + PCI_MSI_DOORBELL_START);
  198. }
  199. static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data,
  200. const struct cpumask *mask, bool force)
  201. {
  202. return -EINVAL;
  203. }
  204. static struct irq_chip armada_370_xp_msi_bottom_irq_chip = {
  205. .name = "MPIC MSI",
  206. .irq_compose_msi_msg = armada_370_xp_compose_msi_msg,
  207. .irq_set_affinity = armada_370_xp_msi_set_affinity,
  208. };
  209. static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq,
  210. unsigned int nr_irqs, void *args)
  211. {
  212. int hwirq, i;
  213. mutex_lock(&msi_used_lock);
  214. hwirq = bitmap_find_free_region(msi_used, PCI_MSI_DOORBELL_NR,
  215. order_base_2(nr_irqs));
  216. mutex_unlock(&msi_used_lock);
  217. if (hwirq < 0)
  218. return -ENOSPC;
  219. for (i = 0; i < nr_irqs; i++) {
  220. irq_domain_set_info(domain, virq + i, hwirq + i,
  221. &armada_370_xp_msi_bottom_irq_chip,
  222. domain->host_data, handle_simple_irq,
  223. NULL, NULL);
  224. }
  225. return 0;
  226. }
  227. static void armada_370_xp_msi_free(struct irq_domain *domain,
  228. unsigned int virq, unsigned int nr_irqs)
  229. {
  230. struct irq_data *d = irq_domain_get_irq_data(domain, virq);
  231. mutex_lock(&msi_used_lock);
  232. bitmap_release_region(msi_used, d->hwirq, order_base_2(nr_irqs));
  233. mutex_unlock(&msi_used_lock);
  234. }
  235. static const struct irq_domain_ops armada_370_xp_msi_domain_ops = {
  236. .alloc = armada_370_xp_msi_alloc,
  237. .free = armada_370_xp_msi_free,
  238. };
  239. static int armada_370_xp_msi_init(struct device_node *node,
  240. phys_addr_t main_int_phys_base)
  241. {
  242. u32 reg;
  243. msi_doorbell_addr = main_int_phys_base +
  244. ARMADA_370_XP_SW_TRIG_INT_OFFS;
  245. armada_370_xp_msi_inner_domain =
  246. irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR,
  247. &armada_370_xp_msi_domain_ops, NULL);
  248. if (!armada_370_xp_msi_inner_domain)
  249. return -ENOMEM;
  250. armada_370_xp_msi_domain =
  251. pci_msi_create_irq_domain(of_node_to_fwnode(node),
  252. &armada_370_xp_msi_domain_info,
  253. armada_370_xp_msi_inner_domain);
  254. if (!armada_370_xp_msi_domain) {
  255. irq_domain_remove(armada_370_xp_msi_inner_domain);
  256. return -ENOMEM;
  257. }
  258. reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS)
  259. | PCI_MSI_DOORBELL_MASK;
  260. writel(reg, per_cpu_int_base +
  261. ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  262. /* Unmask IPI interrupt */
  263. writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  264. return 0;
  265. }
  266. #else
  267. static inline int armada_370_xp_msi_init(struct device_node *node,
  268. phys_addr_t main_int_phys_base)
  269. {
  270. return 0;
  271. }
  272. #endif
  273. static void armada_xp_mpic_perf_init(void)
  274. {
  275. unsigned long cpuid = cpu_logical_map(smp_processor_id());
  276. /* Enable Performance Counter Overflow interrupts */
  277. writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid),
  278. per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS);
  279. }
  280. #ifdef CONFIG_SMP
  281. static struct irq_domain *ipi_domain;
  282. static void armada_370_xp_ipi_mask(struct irq_data *d)
  283. {
  284. u32 reg;
  285. reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  286. reg &= ~BIT(d->hwirq);
  287. writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  288. }
  289. static void armada_370_xp_ipi_unmask(struct irq_data *d)
  290. {
  291. u32 reg;
  292. reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  293. reg |= BIT(d->hwirq);
  294. writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  295. }
  296. static void armada_370_xp_ipi_send_mask(struct irq_data *d,
  297. const struct cpumask *mask)
  298. {
  299. unsigned long map = 0;
  300. int cpu;
  301. /* Convert our logical CPU mask into a physical one. */
  302. for_each_cpu(cpu, mask)
  303. map |= 1 << cpu_logical_map(cpu);
  304. /*
  305. * Ensure that stores to Normal memory are visible to the
  306. * other CPUs before issuing the IPI.
  307. */
  308. dsb();
  309. /* submit softirq */
  310. writel((map << 8) | d->hwirq, main_int_base +
  311. ARMADA_370_XP_SW_TRIG_INT_OFFS);
  312. }
  313. static void armada_370_xp_ipi_eoi(struct irq_data *d)
  314. {
  315. writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
  316. }
  317. static struct irq_chip ipi_irqchip = {
  318. .name = "IPI",
  319. .irq_mask = armada_370_xp_ipi_mask,
  320. .irq_unmask = armada_370_xp_ipi_unmask,
  321. .irq_eoi = armada_370_xp_ipi_eoi,
  322. .ipi_send_mask = armada_370_xp_ipi_send_mask,
  323. };
  324. static int armada_370_xp_ipi_alloc(struct irq_domain *d,
  325. unsigned int virq,
  326. unsigned int nr_irqs, void *args)
  327. {
  328. int i;
  329. for (i = 0; i < nr_irqs; i++) {
  330. irq_set_percpu_devid(virq + i);
  331. irq_domain_set_info(d, virq + i, i, &ipi_irqchip,
  332. d->host_data,
  333. handle_percpu_devid_fasteoi_ipi,
  334. NULL, NULL);
  335. }
  336. return 0;
  337. }
  338. static void armada_370_xp_ipi_free(struct irq_domain *d,
  339. unsigned int virq,
  340. unsigned int nr_irqs)
  341. {
  342. /* Not freeing IPIs */
  343. }
  344. static const struct irq_domain_ops ipi_domain_ops = {
  345. .alloc = armada_370_xp_ipi_alloc,
  346. .free = armada_370_xp_ipi_free,
  347. };
  348. static void ipi_resume(void)
  349. {
  350. int i;
  351. for (i = 0; i < IPI_DOORBELL_END; i++) {
  352. int irq;
  353. irq = irq_find_mapping(ipi_domain, i);
  354. if (irq <= 0)
  355. continue;
  356. if (irq_percpu_is_enabled(irq)) {
  357. struct irq_data *d;
  358. d = irq_domain_get_irq_data(ipi_domain, irq);
  359. armada_370_xp_ipi_unmask(d);
  360. }
  361. }
  362. }
  363. static __init void armada_xp_ipi_init(struct device_node *node)
  364. {
  365. int base_ipi;
  366. ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node),
  367. IPI_DOORBELL_END,
  368. &ipi_domain_ops, NULL);
  369. if (WARN_ON(!ipi_domain))
  370. return;
  371. irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI);
  372. base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, IPI_DOORBELL_END,
  373. NUMA_NO_NODE, NULL, false, NULL);
  374. if (WARN_ON(!base_ipi))
  375. return;
  376. set_smp_ipi_range(base_ipi, IPI_DOORBELL_END);
  377. }
  378. static DEFINE_RAW_SPINLOCK(irq_controller_lock);
  379. static int armada_xp_set_affinity(struct irq_data *d,
  380. const struct cpumask *mask_val, bool force)
  381. {
  382. irq_hw_number_t hwirq = irqd_to_hwirq(d);
  383. unsigned long reg, mask;
  384. int cpu;
  385. /* Select a single core from the affinity mask which is online */
  386. cpu = cpumask_any_and(mask_val, cpu_online_mask);
  387. mask = 1UL << cpu_logical_map(cpu);
  388. raw_spin_lock(&irq_controller_lock);
  389. reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
  390. reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask;
  391. writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
  392. raw_spin_unlock(&irq_controller_lock);
  393. irq_data_update_effective_affinity(d, cpumask_of(cpu));
  394. return IRQ_SET_MASK_OK;
  395. }
  396. static void armada_xp_mpic_smp_cpu_init(void)
  397. {
  398. u32 control;
  399. int nr_irqs, i;
  400. control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL);
  401. nr_irqs = (control >> 2) & 0x3ff;
  402. for (i = 0; i < nr_irqs; i++)
  403. writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS);
  404. /* Disable all IPIs */
  405. writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  406. /* Clear pending IPIs */
  407. writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
  408. /* Unmask IPI interrupt */
  409. writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  410. }
  411. static void armada_xp_mpic_reenable_percpu(void)
  412. {
  413. unsigned int irq;
  414. /* Re-enable per-CPU interrupts that were enabled before suspend */
  415. for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) {
  416. struct irq_data *data;
  417. int virq;
  418. virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
  419. if (virq == 0)
  420. continue;
  421. data = irq_get_irq_data(virq);
  422. if (!irq_percpu_is_enabled(virq))
  423. continue;
  424. armada_370_xp_irq_unmask(data);
  425. }
  426. ipi_resume();
  427. }
  428. static int armada_xp_mpic_starting_cpu(unsigned int cpu)
  429. {
  430. armada_xp_mpic_perf_init();
  431. armada_xp_mpic_smp_cpu_init();
  432. armada_xp_mpic_reenable_percpu();
  433. return 0;
  434. }
  435. static int mpic_cascaded_starting_cpu(unsigned int cpu)
  436. {
  437. armada_xp_mpic_perf_init();
  438. armada_xp_mpic_reenable_percpu();
  439. enable_percpu_irq(parent_irq, IRQ_TYPE_NONE);
  440. return 0;
  441. }
  442. #else
  443. static void armada_xp_mpic_smp_cpu_init(void) {}
  444. static void ipi_resume(void) {}
  445. #endif
  446. static struct irq_chip armada_370_xp_irq_chip = {
  447. .name = "MPIC",
  448. .irq_mask = armada_370_xp_irq_mask,
  449. .irq_mask_ack = armada_370_xp_irq_mask,
  450. .irq_unmask = armada_370_xp_irq_unmask,
  451. #ifdef CONFIG_SMP
  452. .irq_set_affinity = armada_xp_set_affinity,
  453. #endif
  454. .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
  455. };
  456. static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
  457. unsigned int virq, irq_hw_number_t hw)
  458. {
  459. armada_370_xp_irq_mask(irq_get_irq_data(virq));
  460. if (!is_percpu_irq(hw))
  461. writel(hw, per_cpu_int_base +
  462. ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  463. else
  464. writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS);
  465. irq_set_status_flags(virq, IRQ_LEVEL);
  466. if (is_percpu_irq(hw)) {
  467. irq_set_percpu_devid(virq);
  468. irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
  469. handle_percpu_devid_irq);
  470. } else {
  471. irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
  472. handle_level_irq);
  473. irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
  474. }
  475. irq_set_probe(virq);
  476. return 0;
  477. }
  478. static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = {
  479. .map = armada_370_xp_mpic_irq_map,
  480. .xlate = irq_domain_xlate_onecell,
  481. };
  482. #ifdef CONFIG_PCI_MSI
  483. static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained)
  484. {
  485. u32 msimask, msinr;
  486. msimask = readl_relaxed(per_cpu_int_base +
  487. ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
  488. & PCI_MSI_DOORBELL_MASK;
  489. writel(~msimask, per_cpu_int_base +
  490. ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
  491. for (msinr = PCI_MSI_DOORBELL_START;
  492. msinr < PCI_MSI_DOORBELL_END; msinr++) {
  493. int irq;
  494. if (!(msimask & BIT(msinr)))
  495. continue;
  496. if (is_chained) {
  497. irq = irq_find_mapping(armada_370_xp_msi_inner_domain,
  498. msinr - PCI_MSI_DOORBELL_START);
  499. generic_handle_irq(irq);
  500. } else {
  501. irq = msinr - PCI_MSI_DOORBELL_START;
  502. handle_domain_irq(armada_370_xp_msi_inner_domain,
  503. irq, regs);
  504. }
  505. }
  506. }
  507. #else
  508. static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {}
  509. #endif
  510. static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc)
  511. {
  512. struct irq_chip *chip = irq_desc_get_chip(desc);
  513. unsigned long irqmap, irqn, irqsrc, cpuid;
  514. unsigned int cascade_irq;
  515. chained_irq_enter(chip, desc);
  516. irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE);
  517. cpuid = cpu_logical_map(smp_processor_id());
  518. for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) {
  519. irqsrc = readl_relaxed(main_int_base +
  520. ARMADA_370_XP_INT_SOURCE_CTL(irqn));
  521. /* Check if the interrupt is not masked on current CPU.
  522. * Test IRQ (0-1) and FIQ (8-9) mask bits.
  523. */
  524. if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid)))
  525. continue;
  526. if (irqn == 1) {
  527. armada_370_xp_handle_msi_irq(NULL, true);
  528. continue;
  529. }
  530. cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn);
  531. generic_handle_irq(cascade_irq);
  532. }
  533. chained_irq_exit(chip, desc);
  534. }
  535. static void __exception_irq_entry
  536. armada_370_xp_handle_irq(struct pt_regs *regs)
  537. {
  538. u32 irqstat, irqnr;
  539. do {
  540. irqstat = readl_relaxed(per_cpu_int_base +
  541. ARMADA_370_XP_CPU_INTACK_OFFS);
  542. irqnr = irqstat & 0x3FF;
  543. if (irqnr > 1022)
  544. break;
  545. if (irqnr > 1) {
  546. handle_domain_irq(armada_370_xp_mpic_domain,
  547. irqnr, regs);
  548. continue;
  549. }
  550. /* MSI handling */
  551. if (irqnr == 1)
  552. armada_370_xp_handle_msi_irq(regs, false);
  553. #ifdef CONFIG_SMP
  554. /* IPI Handling */
  555. if (irqnr == 0) {
  556. unsigned long ipimask;
  557. int ipi;
  558. ipimask = readl_relaxed(per_cpu_int_base +
  559. ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
  560. & IPI_DOORBELL_MASK;
  561. for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END)
  562. handle_domain_irq(ipi_domain, ipi, regs);
  563. }
  564. #endif
  565. } while (1);
  566. }
  567. static int armada_370_xp_mpic_suspend(void)
  568. {
  569. doorbell_mask_reg = readl(per_cpu_int_base +
  570. ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  571. return 0;
  572. }
  573. static void armada_370_xp_mpic_resume(void)
  574. {
  575. int nirqs;
  576. irq_hw_number_t irq;
  577. /* Re-enable interrupts */
  578. nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff;
  579. for (irq = 0; irq < nirqs; irq++) {
  580. struct irq_data *data;
  581. int virq;
  582. virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
  583. if (virq == 0)
  584. continue;
  585. data = irq_get_irq_data(virq);
  586. if (!is_percpu_irq(irq)) {
  587. /* Non per-CPU interrupts */
  588. writel(irq, per_cpu_int_base +
  589. ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  590. if (!irqd_irq_disabled(data))
  591. armada_370_xp_irq_unmask(data);
  592. } else {
  593. /* Per-CPU interrupts */
  594. writel(irq, main_int_base +
  595. ARMADA_370_XP_INT_SET_ENABLE_OFFS);
  596. /*
  597. * Re-enable on the current CPU,
  598. * armada_xp_mpic_reenable_percpu() will take
  599. * care of secondary CPUs when they come up.
  600. */
  601. if (irq_percpu_is_enabled(virq))
  602. armada_370_xp_irq_unmask(data);
  603. }
  604. }
  605. /* Reconfigure doorbells for IPIs and MSIs */
  606. writel(doorbell_mask_reg,
  607. per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
  608. if (doorbell_mask_reg & IPI_DOORBELL_MASK)
  609. writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  610. if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK)
  611. writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
  612. ipi_resume();
  613. }
  614. static struct syscore_ops armada_370_xp_mpic_syscore_ops = {
  615. .suspend = armada_370_xp_mpic_suspend,
  616. .resume = armada_370_xp_mpic_resume,
  617. };
  618. static int __init armada_370_xp_mpic_of_init(struct device_node *node,
  619. struct device_node *parent)
  620. {
  621. struct resource main_int_res, per_cpu_int_res;
  622. int nr_irqs, i;
  623. u32 control;
  624. BUG_ON(of_address_to_resource(node, 0, &main_int_res));
  625. BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res));
  626. BUG_ON(!request_mem_region(main_int_res.start,
  627. resource_size(&main_int_res),
  628. node->full_name));
  629. BUG_ON(!request_mem_region(per_cpu_int_res.start,
  630. resource_size(&per_cpu_int_res),
  631. node->full_name));
  632. main_int_base = ioremap(main_int_res.start,
  633. resource_size(&main_int_res));
  634. BUG_ON(!main_int_base);
  635. per_cpu_int_base = ioremap(per_cpu_int_res.start,
  636. resource_size(&per_cpu_int_res));
  637. BUG_ON(!per_cpu_int_base);
  638. control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL);
  639. nr_irqs = (control >> 2) & 0x3ff;
  640. for (i = 0; i < nr_irqs; i++)
  641. writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS);
  642. armada_370_xp_mpic_domain =
  643. irq_domain_add_linear(node, nr_irqs,
  644. &armada_370_xp_mpic_irq_ops, NULL);
  645. BUG_ON(!armada_370_xp_mpic_domain);
  646. irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED);
  647. /* Setup for the boot CPU */
  648. armada_xp_mpic_perf_init();
  649. armada_xp_mpic_smp_cpu_init();
  650. armada_370_xp_msi_init(node, main_int_res.start);
  651. parent_irq = irq_of_parse_and_map(node, 0);
  652. if (parent_irq <= 0) {
  653. irq_set_default_host(armada_370_xp_mpic_domain);
  654. set_handle_irq(armada_370_xp_handle_irq);
  655. #ifdef CONFIG_SMP
  656. armada_xp_ipi_init(node);
  657. cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING,
  658. "irqchip/armada/ipi:starting",
  659. armada_xp_mpic_starting_cpu, NULL);
  660. #endif
  661. } else {
  662. #ifdef CONFIG_SMP
  663. cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING,
  664. "irqchip/armada/cascade:starting",
  665. mpic_cascaded_starting_cpu, NULL);
  666. #endif
  667. irq_set_chained_handler(parent_irq,
  668. armada_370_xp_mpic_handle_cascade_irq);
  669. }
  670. register_syscore_ops(&armada_370_xp_mpic_syscore_ops);
  671. return 0;
  672. }
  673. IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init);