msi-xlp.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. /*
  2. * Copyright (c) 2003-2012 Broadcom Corporation
  3. * All Rights Reserved
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the Broadcom
  9. * license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or without
  12. * modification, are permitted provided that the following conditions
  13. * are met:
  14. *
  15. * 1. Redistributions of source code must retain the above copyright
  16. * notice, this list of conditions and the following disclaimer.
  17. * 2. Redistributions in binary form must reproduce the above copyright
  18. * notice, this list of conditions and the following disclaimer in
  19. * the documentation and/or other materials provided with the
  20. * distribution.
  21. *
  22. * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  23. * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  24. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25. * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  26. * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  29. * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  30. * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  31. * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  32. * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33. */
  34. #include <linux/types.h>
  35. #include <linux/pci.h>
  36. #include <linux/kernel.h>
  37. #include <linux/init.h>
  38. #include <linux/msi.h>
  39. #include <linux/mm.h>
  40. #include <linux/irq.h>
  41. #include <linux/irqdesc.h>
  42. #include <linux/console.h>
  43. #include <asm/io.h>
  44. #include <asm/netlogic/interrupt.h>
  45. #include <asm/netlogic/haldefs.h>
  46. #include <asm/netlogic/common.h>
  47. #include <asm/netlogic/mips-extns.h>
  48. #include <asm/netlogic/xlp-hal/iomap.h>
  49. #include <asm/netlogic/xlp-hal/xlp.h>
  50. #include <asm/netlogic/xlp-hal/pic.h>
  51. #include <asm/netlogic/xlp-hal/pcibus.h>
  52. #include <asm/netlogic/xlp-hal/bridge.h>
  53. #define XLP_MSIVEC_PER_LINK 32
  54. #define XLP_MSIXVEC_TOTAL (cpu_is_xlp9xx() ? 128 : 32)
  55. #define XLP_MSIXVEC_PER_LINK (cpu_is_xlp9xx() ? 32 : 8)
  56. /* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */
  57. static inline int nlm_link_msiirq(int link, int msivec)
  58. {
  59. return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec;
  60. }
  61. /* get the link MSI vector from irq number */
  62. static inline int nlm_irq_msivec(int irq)
  63. {
  64. return (irq - NLM_MSI_VEC_BASE) % XLP_MSIVEC_PER_LINK;
  65. }
  66. /* get the link from the irq number */
  67. static inline int nlm_irq_msilink(int irq)
  68. {
  69. int total_msivec = XLP_MSIVEC_PER_LINK * PCIE_NLINKS;
  70. return ((irq - NLM_MSI_VEC_BASE) % total_msivec) /
  71. XLP_MSIVEC_PER_LINK;
  72. }
  73. /*
  74. * For XLP 8xx/4xx/3xx/2xx, only 32 MSI-X vectors are possible because
  75. * there are only 32 PIC interrupts for MSI. We split them statically
  76. * and use 8 MSI-X vectors per link - this keeps the allocation and
  77. * lookup simple.
  78. * On XLP 9xx, there are 32 vectors per link, and the interrupts are
  79. * not routed thru PIC, so we can use all 128 MSI-X vectors.
  80. */
  81. static inline int nlm_link_msixirq(int link, int bit)
  82. {
  83. return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit;
  84. }
  85. /* get the link MSI vector from irq number */
  86. static inline int nlm_irq_msixvec(int irq)
  87. {
  88. return (irq - NLM_MSIX_VEC_BASE) % XLP_MSIXVEC_TOTAL;
  89. }
  90. /* get the link from MSIX vec */
  91. static inline int nlm_irq_msixlink(int msixvec)
  92. {
  93. return msixvec / XLP_MSIXVEC_PER_LINK;
  94. }
  95. /*
  96. * Per link MSI and MSI-X information, set as IRQ handler data for
  97. * MSI and MSI-X interrupts.
  98. */
  99. struct xlp_msi_data {
  100. struct nlm_soc_info *node;
  101. uint64_t lnkbase;
  102. uint32_t msi_enabled_mask;
  103. uint32_t msi_alloc_mask;
  104. uint32_t msix_alloc_mask;
  105. spinlock_t msi_lock;
  106. };
  107. /*
  108. * MSI Chip definitions
  109. *
  110. * On XLP, there is a PIC interrupt associated with each PCIe link on the
  111. * chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa
  112. * per link and 128 overall.
  113. *
  114. * When a device connected to the link raises a MSI interrupt, we get a
  115. * link interrupt and we then have to look at PCIE_MSI_STATUS register at
  116. * the bridge to map it to the IRQ
  117. */
  118. static void xlp_msi_enable(struct irq_data *d)
  119. {
  120. struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
  121. unsigned long flags;
  122. int vec;
  123. vec = nlm_irq_msivec(d->irq);
  124. spin_lock_irqsave(&md->msi_lock, flags);
  125. md->msi_enabled_mask |= 1u << vec;
  126. if (cpu_is_xlp9xx())
  127. nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
  128. md->msi_enabled_mask);
  129. else
  130. nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
  131. spin_unlock_irqrestore(&md->msi_lock, flags);
  132. }
  133. static void xlp_msi_disable(struct irq_data *d)
  134. {
  135. struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
  136. unsigned long flags;
  137. int vec;
  138. vec = nlm_irq_msivec(d->irq);
  139. spin_lock_irqsave(&md->msi_lock, flags);
  140. md->msi_enabled_mask &= ~(1u << vec);
  141. if (cpu_is_xlp9xx())
  142. nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
  143. md->msi_enabled_mask);
  144. else
  145. nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
  146. spin_unlock_irqrestore(&md->msi_lock, flags);
  147. }
  148. static void xlp_msi_mask_ack(struct irq_data *d)
  149. {
  150. struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
  151. int link, vec;
  152. link = nlm_irq_msilink(d->irq);
  153. vec = nlm_irq_msivec(d->irq);
  154. xlp_msi_disable(d);
  155. /* Ack MSI on bridge */
  156. if (cpu_is_xlp9xx())
  157. nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_STATUS, 1u << vec);
  158. else
  159. nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec);
  160. }
  161. static struct irq_chip xlp_msi_chip = {
  162. .name = "XLP-MSI",
  163. .irq_enable = xlp_msi_enable,
  164. .irq_disable = xlp_msi_disable,
  165. .irq_mask_ack = xlp_msi_mask_ack,
  166. .irq_unmask = xlp_msi_enable,
  167. };
  168. /*
  169. * XLP8XX/4XX/3XX/2XX:
  170. * The MSI-X interrupt handling is different from MSI, there are 32 MSI-X
  171. * interrupts generated by the PIC and each of these correspond to a MSI-X
  172. * vector (0-31) that can be assigned.
  173. *
  174. * We divide the MSI-X vectors to 8 per link and do a per-link allocation
  175. *
  176. * XLP9XX:
  177. * 32 MSI-X vectors are available per link, and the interrupts are not routed
  178. * thru the PIC. PIC ack not needed.
  179. *
  180. * Enable and disable done using standard MSI functions.
  181. */
  182. static void xlp_msix_mask_ack(struct irq_data *d)
  183. {
  184. struct xlp_msi_data *md;
  185. int link, msixvec;
  186. uint32_t status_reg, bit;
  187. msixvec = nlm_irq_msixvec(d->irq);
  188. link = nlm_irq_msixlink(msixvec);
  189. pci_msi_mask_irq(d);
  190. md = irq_data_get_irq_chip_data(d);
  191. /* Ack MSI on bridge */
  192. if (cpu_is_xlp9xx()) {
  193. status_reg = PCIE_9XX_MSIX_STATUSX(link);
  194. bit = msixvec % XLP_MSIXVEC_PER_LINK;
  195. } else {
  196. status_reg = PCIE_MSIX_STATUS;
  197. bit = msixvec;
  198. }
  199. nlm_write_reg(md->lnkbase, status_reg, 1u << bit);
  200. if (!cpu_is_xlp9xx())
  201. nlm_pic_ack(md->node->picbase,
  202. PIC_IRT_PCIE_MSIX_INDEX(msixvec));
  203. }
  204. static struct irq_chip xlp_msix_chip = {
  205. .name = "XLP-MSIX",
  206. .irq_enable = pci_msi_unmask_irq,
  207. .irq_disable = pci_msi_mask_irq,
  208. .irq_mask_ack = xlp_msix_mask_ack,
  209. .irq_unmask = pci_msi_unmask_irq,
  210. };
  211. void arch_teardown_msi_irq(unsigned int irq)
  212. {
  213. }
  214. /*
  215. * Setup a PCIe link for MSI. By default, the links are in
  216. * legacy interrupt mode. We will switch them to MSI mode
  217. * at the first MSI request.
  218. */
  219. static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr)
  220. {
  221. u32 val;
  222. if (cpu_is_xlp9xx()) {
  223. val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
  224. if ((val & 0x200) == 0) {
  225. val |= 0x200; /* MSI Interrupt enable */
  226. nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
  227. }
  228. } else {
  229. val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
  230. if ((val & 0x200) == 0) {
  231. val |= 0x200;
  232. nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
  233. }
  234. }
  235. val = nlm_read_reg(lnkbase, 0x1); /* CMD */
  236. if ((val & 0x0400) == 0) {
  237. val |= 0x0400;
  238. nlm_write_reg(lnkbase, 0x1, val);
  239. }
  240. /* Update IRQ in the PCI irq reg */
  241. val = nlm_read_pci_reg(lnkbase, 0xf);
  242. val &= ~0x1fu;
  243. val |= (1 << 8) | lirq;
  244. nlm_write_pci_reg(lnkbase, 0xf, val);
  245. /* MSI addr */
  246. nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32);
  247. nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff);
  248. /* MSI cap for bridge */
  249. val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP);
  250. if ((val & (1 << 16)) == 0) {
  251. val |= 0xb << 16; /* mmc32, msi enable */
  252. nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val);
  253. }
  254. }
  255. /*
  256. * Allocate a MSI vector on a link
  257. */
  258. static int xlp_setup_msi(uint64_t lnkbase, int node, int link,
  259. struct msi_desc *desc)
  260. {
  261. struct xlp_msi_data *md;
  262. struct msi_msg msg;
  263. unsigned long flags;
  264. int msivec, irt, lirq, xirq, ret;
  265. uint64_t msiaddr;
  266. /* Get MSI data for the link */
  267. lirq = PIC_PCIE_LINK_MSI_IRQ(link);
  268. xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
  269. md = irq_get_chip_data(xirq);
  270. msiaddr = MSI_LINK_ADDR(node, link);
  271. spin_lock_irqsave(&md->msi_lock, flags);
  272. if (md->msi_alloc_mask == 0) {
  273. xlp_config_link_msi(lnkbase, lirq, msiaddr);
  274. /* switch the link IRQ to MSI range */
  275. if (cpu_is_xlp9xx())
  276. irt = PIC_9XX_IRT_PCIE_LINK_INDEX(link);
  277. else
  278. irt = PIC_IRT_PCIE_LINK_INDEX(link);
  279. nlm_setup_pic_irq(node, lirq, lirq, irt);
  280. nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq,
  281. node * nlm_threads_per_node(), 1 /*en */);
  282. }
  283. /* allocate a MSI vec, and tell the bridge about it */
  284. msivec = fls(md->msi_alloc_mask);
  285. if (msivec == XLP_MSIVEC_PER_LINK) {
  286. spin_unlock_irqrestore(&md->msi_lock, flags);
  287. return -ENOMEM;
  288. }
  289. md->msi_alloc_mask |= (1u << msivec);
  290. spin_unlock_irqrestore(&md->msi_lock, flags);
  291. msg.address_hi = msiaddr >> 32;
  292. msg.address_lo = msiaddr & 0xffffffff;
  293. msg.data = 0xc00 | msivec;
  294. xirq = xirq + msivec; /* msi mapped to global irq space */
  295. ret = irq_set_msi_desc(xirq, desc);
  296. if (ret < 0)
  297. return ret;
  298. pci_write_msi_msg(xirq, &msg);
  299. return 0;
  300. }
  301. /*
  302. * Switch a link to MSI-X mode
  303. */
  304. static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr)
  305. {
  306. u32 val;
  307. val = nlm_read_reg(lnkbase, 0x2C);
  308. if ((val & 0x80000000U) == 0) {
  309. val |= 0x80000000U;
  310. nlm_write_reg(lnkbase, 0x2C, val);
  311. }
  312. if (cpu_is_xlp9xx()) {
  313. val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
  314. if ((val & 0x200) == 0) {
  315. val |= 0x200; /* MSI Interrupt enable */
  316. nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
  317. }
  318. } else {
  319. val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
  320. if ((val & 0x200) == 0) {
  321. val |= 0x200; /* MSI Interrupt enable */
  322. nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
  323. }
  324. }
  325. val = nlm_read_reg(lnkbase, 0x1); /* CMD */
  326. if ((val & 0x0400) == 0) {
  327. val |= 0x0400;
  328. nlm_write_reg(lnkbase, 0x1, val);
  329. }
  330. /* Update IRQ in the PCI irq reg */
  331. val = nlm_read_pci_reg(lnkbase, 0xf);
  332. val &= ~0x1fu;
  333. val |= (1 << 8) | lirq;
  334. nlm_write_pci_reg(lnkbase, 0xf, val);
  335. if (cpu_is_xlp9xx()) {
  336. /* MSI-X addresses */
  337. nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_BASE,
  338. msixaddr >> 8);
  339. nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_LIMIT,
  340. (msixaddr + MSI_ADDR_SZ) >> 8);
  341. } else {
  342. /* MSI-X addresses */
  343. nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE,
  344. msixaddr >> 8);
  345. nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT,
  346. (msixaddr + MSI_ADDR_SZ) >> 8);
  347. }
  348. }
  349. /*
  350. * Allocate a MSI-X vector
  351. */
  352. static int xlp_setup_msix(uint64_t lnkbase, int node, int link,
  353. struct msi_desc *desc)
  354. {
  355. struct xlp_msi_data *md;
  356. struct msi_msg msg;
  357. unsigned long flags;
  358. int t, msixvec, lirq, xirq, ret;
  359. uint64_t msixaddr;
  360. /* Get MSI data for the link */
  361. lirq = PIC_PCIE_MSIX_IRQ(link);
  362. xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
  363. md = irq_get_chip_data(xirq);
  364. msixaddr = MSIX_LINK_ADDR(node, link);
  365. spin_lock_irqsave(&md->msi_lock, flags);
  366. /* switch the PCIe link to MSI-X mode at the first alloc */
  367. if (md->msix_alloc_mask == 0)
  368. xlp_config_link_msix(lnkbase, lirq, msixaddr);
  369. /* allocate a MSI-X vec, and tell the bridge about it */
  370. t = fls(md->msix_alloc_mask);
  371. if (t == XLP_MSIXVEC_PER_LINK) {
  372. spin_unlock_irqrestore(&md->msi_lock, flags);
  373. return -ENOMEM;
  374. }
  375. md->msix_alloc_mask |= (1u << t);
  376. spin_unlock_irqrestore(&md->msi_lock, flags);
  377. xirq += t;
  378. msixvec = nlm_irq_msixvec(xirq);
  379. msg.address_hi = msixaddr >> 32;
  380. msg.address_lo = msixaddr & 0xffffffff;
  381. msg.data = 0xc00 | msixvec;
  382. ret = irq_set_msi_desc(xirq, desc);
  383. if (ret < 0)
  384. return ret;
  385. pci_write_msi_msg(xirq, &msg);
  386. return 0;
  387. }
  388. int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
  389. {
  390. struct pci_dev *lnkdev;
  391. uint64_t lnkbase;
  392. int node, link, slot;
  393. lnkdev = xlp_get_pcie_link(dev);
  394. if (lnkdev == NULL) {
  395. dev_err(&dev->dev, "Could not find bridge\n");
  396. return 1;
  397. }
  398. slot = PCI_SLOT(lnkdev->devfn);
  399. link = PCI_FUNC(lnkdev->devfn);
  400. node = slot / 8;
  401. lnkbase = nlm_get_pcie_base(node, link);
  402. if (desc->msi_attrib.is_msix)
  403. return xlp_setup_msix(lnkbase, node, link, desc);
  404. else
  405. return xlp_setup_msi(lnkbase, node, link, desc);
  406. }
  407. void __init xlp_init_node_msi_irqs(int node, int link)
  408. {
  409. struct nlm_soc_info *nodep;
  410. struct xlp_msi_data *md;
  411. int irq, i, irt, msixvec, val;
  412. pr_info("[%d %d] Init node PCI IRT\n", node, link);
  413. nodep = nlm_get_node(node);
  414. /* Alloc an MSI block for the link */
  415. md = kzalloc(sizeof(*md), GFP_KERNEL);
  416. spin_lock_init(&md->msi_lock);
  417. md->msi_enabled_mask = 0;
  418. md->msi_alloc_mask = 0;
  419. md->msix_alloc_mask = 0;
  420. md->node = nodep;
  421. md->lnkbase = nlm_get_pcie_base(node, link);
  422. /* extended space for MSI interrupts */
  423. irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
  424. for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) {
  425. irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq);
  426. irq_set_chip_data(i, md);
  427. }
  428. for (i = 0; i < XLP_MSIXVEC_PER_LINK ; i++) {
  429. if (cpu_is_xlp9xx()) {
  430. val = ((node * nlm_threads_per_node()) << 7 |
  431. PIC_PCIE_MSIX_IRQ(link) << 1 | 0 << 0);
  432. nlm_write_pcie_reg(md->lnkbase, PCIE_9XX_MSIX_VECX(i +
  433. (link * XLP_MSIXVEC_PER_LINK)), val);
  434. } else {
  435. /* Initialize MSI-X irts to generate one interrupt
  436. * per link
  437. */
  438. msixvec = link * XLP_MSIXVEC_PER_LINK + i;
  439. irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec);
  440. nlm_pic_init_irt(nodep->picbase, irt,
  441. PIC_PCIE_MSIX_IRQ(link),
  442. node * nlm_threads_per_node(), 1);
  443. }
  444. /* Initialize MSI-X extended irq space for the link */
  445. irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i));
  446. irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq);
  447. irq_set_chip_data(irq, md);
  448. }
  449. }
  450. void nlm_dispatch_msi(int node, int lirq)
  451. {
  452. struct xlp_msi_data *md;
  453. int link, i, irqbase;
  454. u32 status;
  455. link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE;
  456. irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
  457. md = irq_get_chip_data(irqbase);
  458. if (cpu_is_xlp9xx())
  459. status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSI_STATUS) &
  460. md->msi_enabled_mask;
  461. else
  462. status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) &
  463. md->msi_enabled_mask;
  464. while (status) {
  465. i = __ffs(status);
  466. do_IRQ(irqbase + i);
  467. status &= status - 1;
  468. }
  469. /* Ack at eirr and PIC */
  470. ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link));
  471. if (cpu_is_xlp9xx())
  472. nlm_pic_ack(md->node->picbase,
  473. PIC_9XX_IRT_PCIE_LINK_INDEX(link));
  474. else
  475. nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link));
  476. }
  477. void nlm_dispatch_msix(int node, int lirq)
  478. {
  479. struct xlp_msi_data *md;
  480. int link, i, irqbase;
  481. u32 status;
  482. link = lirq - PIC_PCIE_MSIX_IRQ_BASE;
  483. irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
  484. md = irq_get_chip_data(irqbase);
  485. if (cpu_is_xlp9xx())
  486. status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSIX_STATUSX(link));
  487. else
  488. status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS);
  489. /* narrow it down to the MSI-x vectors for our link */
  490. if (!cpu_is_xlp9xx())
  491. status = (status >> (link * XLP_MSIXVEC_PER_LINK)) &
  492. ((1 << XLP_MSIXVEC_PER_LINK) - 1);
  493. while (status) {
  494. i = __ffs(status);
  495. do_IRQ(irqbase + i);
  496. status &= status - 1;
  497. }
  498. /* Ack at eirr and PIC */
  499. ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link));
  500. }