msi.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2014 Intel Corp.
  4. * Author: Jiang Liu <jiang.liu@linux.intel.com>
  5. *
  6. * This file is licensed under GPLv2.
  7. *
  8. * This file contains common code to support Message Signalled Interrupt for
  9. * PCI compatible and non PCI compatible devices.
  10. */
  11. #include <linux/types.h>
  12. #include <linux/device.h>
  13. #include <linux/irq.h>
  14. #include <linux/irqdomain.h>
  15. #include <linux/msi.h>
  16. #include <linux/slab.h>
  17. #include "internals.h"
  18. /**
  19. * alloc_msi_entry - Allocate an initialize msi_entry
  20. * @dev: Pointer to the device for which this is allocated
  21. * @nvec: The number of vectors used in this entry
  22. * @affinity: Optional pointer to an affinity mask array size of @nvec
  23. *
  24. * If @affinity is not NULL then an affinity array[@nvec] is allocated
  25. * and the affinity masks and flags from @affinity are copied.
  26. */
  27. struct msi_desc *alloc_msi_entry(struct device *dev, int nvec,
  28. const struct irq_affinity_desc *affinity)
  29. {
  30. struct msi_desc *desc;
  31. desc = kzalloc(sizeof(*desc), GFP_KERNEL);
  32. if (!desc)
  33. return NULL;
  34. INIT_LIST_HEAD(&desc->list);
  35. desc->dev = dev;
  36. desc->nvec_used = nvec;
  37. if (affinity) {
  38. desc->affinity = kmemdup(affinity,
  39. nvec * sizeof(*desc->affinity), GFP_KERNEL);
  40. if (!desc->affinity) {
  41. kfree(desc);
  42. return NULL;
  43. }
  44. }
  45. return desc;
  46. }
  47. void free_msi_entry(struct msi_desc *entry)
  48. {
  49. kfree(entry->affinity);
  50. kfree(entry);
  51. }
  52. void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
  53. {
  54. *msg = entry->msg;
  55. }
  56. void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
  57. {
  58. struct msi_desc *entry = irq_get_msi_desc(irq);
  59. __get_cached_msi_msg(entry, msg);
  60. }
  61. EXPORT_SYMBOL_GPL(get_cached_msi_msg);
  62. #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
  63. static inline void irq_chip_write_msi_msg(struct irq_data *data,
  64. struct msi_msg *msg)
  65. {
  66. data->chip->irq_write_msi_msg(data, msg);
  67. }
  68. static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg)
  69. {
  70. struct msi_domain_info *info = domain->host_data;
  71. /*
  72. * If the MSI provider has messed with the second message and
  73. * not advertized that it is level-capable, signal the breakage.
  74. */
  75. WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&
  76. (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) &&
  77. (msg[1].address_lo || msg[1].address_hi || msg[1].data));
  78. }
  79. /**
  80. * msi_domain_set_affinity - Generic affinity setter function for MSI domains
  81. * @irq_data: The irq data associated to the interrupt
  82. * @mask: The affinity mask to set
  83. * @force: Flag to enforce setting (disable online checks)
  84. *
  85. * Intended to be used by MSI interrupt controllers which are
  86. * implemented with hierarchical domains.
  87. */
  88. int msi_domain_set_affinity(struct irq_data *irq_data,
  89. const struct cpumask *mask, bool force)
  90. {
  91. struct irq_data *parent = irq_data->parent_data;
  92. struct msi_msg msg[2] = { [1] = { }, };
  93. int ret;
  94. ret = parent->chip->irq_set_affinity(parent, mask, force);
  95. if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) {
  96. BUG_ON(irq_chip_compose_msi_msg(irq_data, msg));
  97. msi_check_level(irq_data->domain, msg);
  98. irq_chip_write_msi_msg(irq_data, msg);
  99. }
  100. return ret;
  101. }
  102. static int msi_domain_activate(struct irq_domain *domain,
  103. struct irq_data *irq_data, bool early)
  104. {
  105. struct msi_msg msg[2] = { [1] = { }, };
  106. BUG_ON(irq_chip_compose_msi_msg(irq_data, msg));
  107. msi_check_level(irq_data->domain, msg);
  108. irq_chip_write_msi_msg(irq_data, msg);
  109. return 0;
  110. }
  111. static void msi_domain_deactivate(struct irq_domain *domain,
  112. struct irq_data *irq_data)
  113. {
  114. struct msi_msg msg[2];
  115. memset(msg, 0, sizeof(msg));
  116. irq_chip_write_msi_msg(irq_data, msg);
  117. }
  118. static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
  119. unsigned int nr_irqs, void *arg)
  120. {
  121. struct msi_domain_info *info = domain->host_data;
  122. struct msi_domain_ops *ops = info->ops;
  123. irq_hw_number_t hwirq = ops->get_hwirq(info, arg);
  124. int i, ret;
  125. if (irq_find_mapping(domain, hwirq) > 0)
  126. return -EEXIST;
  127. if (domain->parent) {
  128. ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
  129. if (ret < 0)
  130. return ret;
  131. }
  132. for (i = 0; i < nr_irqs; i++) {
  133. ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg);
  134. if (ret < 0) {
  135. if (ops->msi_free) {
  136. for (i--; i > 0; i--)
  137. ops->msi_free(domain, info, virq + i);
  138. }
  139. irq_domain_free_irqs_top(domain, virq, nr_irqs);
  140. return ret;
  141. }
  142. }
  143. return 0;
  144. }
  145. static void msi_domain_free(struct irq_domain *domain, unsigned int virq,
  146. unsigned int nr_irqs)
  147. {
  148. struct msi_domain_info *info = domain->host_data;
  149. int i;
  150. if (info->ops->msi_free) {
  151. for (i = 0; i < nr_irqs; i++)
  152. info->ops->msi_free(domain, info, virq + i);
  153. }
  154. irq_domain_free_irqs_top(domain, virq, nr_irqs);
  155. }
  156. static const struct irq_domain_ops msi_domain_ops = {
  157. .alloc = msi_domain_alloc,
  158. .free = msi_domain_free,
  159. .activate = msi_domain_activate,
  160. .deactivate = msi_domain_deactivate,
  161. };
  162. static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info,
  163. msi_alloc_info_t *arg)
  164. {
  165. return arg->hwirq;
  166. }
  167. static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev,
  168. int nvec, msi_alloc_info_t *arg)
  169. {
  170. memset(arg, 0, sizeof(*arg));
  171. return 0;
  172. }
  173. static void msi_domain_ops_set_desc(msi_alloc_info_t *arg,
  174. struct msi_desc *desc)
  175. {
  176. arg->desc = desc;
  177. }
  178. static int msi_domain_ops_init(struct irq_domain *domain,
  179. struct msi_domain_info *info,
  180. unsigned int virq, irq_hw_number_t hwirq,
  181. msi_alloc_info_t *arg)
  182. {
  183. irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip,
  184. info->chip_data);
  185. if (info->handler && info->handler_name) {
  186. __irq_set_handler(virq, info->handler, 0, info->handler_name);
  187. if (info->handler_data)
  188. irq_set_handler_data(virq, info->handler_data);
  189. }
  190. return 0;
  191. }
  192. static int msi_domain_ops_check(struct irq_domain *domain,
  193. struct msi_domain_info *info,
  194. struct device *dev)
  195. {
  196. return 0;
  197. }
  198. static struct msi_domain_ops msi_domain_ops_default = {
  199. .get_hwirq = msi_domain_ops_get_hwirq,
  200. .msi_init = msi_domain_ops_init,
  201. .msi_check = msi_domain_ops_check,
  202. .msi_prepare = msi_domain_ops_prepare,
  203. .set_desc = msi_domain_ops_set_desc,
  204. .domain_alloc_irqs = __msi_domain_alloc_irqs,
  205. .domain_free_irqs = __msi_domain_free_irqs,
  206. };
  207. static void msi_domain_update_dom_ops(struct msi_domain_info *info)
  208. {
  209. struct msi_domain_ops *ops = info->ops;
  210. if (ops == NULL) {
  211. info->ops = &msi_domain_ops_default;
  212. return;
  213. }
  214. if (ops->domain_alloc_irqs == NULL)
  215. ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs;
  216. if (ops->domain_free_irqs == NULL)
  217. ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs;
  218. if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))
  219. return;
  220. if (ops->get_hwirq == NULL)
  221. ops->get_hwirq = msi_domain_ops_default.get_hwirq;
  222. if (ops->msi_init == NULL)
  223. ops->msi_init = msi_domain_ops_default.msi_init;
  224. if (ops->msi_check == NULL)
  225. ops->msi_check = msi_domain_ops_default.msi_check;
  226. if (ops->msi_prepare == NULL)
  227. ops->msi_prepare = msi_domain_ops_default.msi_prepare;
  228. if (ops->set_desc == NULL)
  229. ops->set_desc = msi_domain_ops_default.set_desc;
  230. }
  231. static void msi_domain_update_chip_ops(struct msi_domain_info *info)
  232. {
  233. struct irq_chip *chip = info->chip;
  234. BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);
  235. if (!chip->irq_set_affinity)
  236. chip->irq_set_affinity = msi_domain_set_affinity;
  237. }
  238. /**
  239. * msi_create_irq_domain - Create a MSI interrupt domain
  240. * @fwnode: Optional fwnode of the interrupt controller
  241. * @info: MSI domain info
  242. * @parent: Parent irq domain
  243. */
  244. struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
  245. struct msi_domain_info *info,
  246. struct irq_domain *parent)
  247. {
  248. struct irq_domain *domain;
  249. msi_domain_update_dom_ops(info);
  250. if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
  251. msi_domain_update_chip_ops(info);
  252. domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0,
  253. fwnode, &msi_domain_ops, info);
  254. if (domain && !domain->name && info->chip)
  255. domain->name = info->chip->name;
  256. return domain;
  257. }
  258. int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,
  259. int nvec, msi_alloc_info_t *arg)
  260. {
  261. struct msi_domain_info *info = domain->host_data;
  262. struct msi_domain_ops *ops = info->ops;
  263. int ret;
  264. ret = ops->msi_check(domain, info, dev);
  265. if (ret == 0)
  266. ret = ops->msi_prepare(domain, dev, nvec, arg);
  267. return ret;
  268. }
  269. int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,
  270. int virq, int nvec, msi_alloc_info_t *arg)
  271. {
  272. struct msi_domain_info *info = domain->host_data;
  273. struct msi_domain_ops *ops = info->ops;
  274. struct msi_desc *desc;
  275. int ret = 0;
  276. for_each_msi_entry(desc, dev) {
  277. /* Don't even try the multi-MSI brain damage. */
  278. if (WARN_ON(!desc->irq || desc->nvec_used != 1)) {
  279. ret = -EINVAL;
  280. break;
  281. }
  282. if (!(desc->irq >= virq && desc->irq < (virq + nvec)))
  283. continue;
  284. ops->set_desc(arg, desc);
  285. /* Assumes the domain mutex is held! */
  286. ret = irq_domain_alloc_irqs_hierarchy(domain, desc->irq, 1,
  287. arg);
  288. if (ret)
  289. break;
  290. irq_set_msi_desc_off(desc->irq, 0, desc);
  291. }
  292. if (ret) {
  293. /* Mop up the damage */
  294. for_each_msi_entry(desc, dev) {
  295. if (!(desc->irq >= virq && desc->irq < (virq + nvec)))
  296. continue;
  297. irq_domain_free_irqs_common(domain, desc->irq, 1);
  298. }
  299. }
  300. return ret;
  301. }
  302. /*
  303. * Carefully check whether the device can use reservation mode. If
  304. * reservation mode is enabled then the early activation will assign a
  305. * dummy vector to the device. If the PCI/MSI device does not support
  306. * masking of the entry then this can result in spurious interrupts when
  307. * the device driver is not absolutely careful. But even then a malfunction
  308. * of the hardware could result in a spurious interrupt on the dummy vector
  309. * and render the device unusable. If the entry can be masked then the core
  310. * logic will prevent the spurious interrupt and reservation mode can be
  311. * used. For now reservation mode is restricted to PCI/MSI.
  312. */
  313. static bool msi_check_reservation_mode(struct irq_domain *domain,
  314. struct msi_domain_info *info,
  315. struct device *dev)
  316. {
  317. struct msi_desc *desc;
  318. switch(domain->bus_token) {
  319. case DOMAIN_BUS_PCI_MSI:
  320. case DOMAIN_BUS_VMD_MSI:
  321. break;
  322. default:
  323. return false;
  324. }
  325. if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))
  326. return false;
  327. if (IS_ENABLED(CONFIG_PCI_MSI) && pci_msi_ignore_mask)
  328. return false;
  329. /*
  330. * Checking the first MSI descriptor is sufficient. MSIX supports
  331. * masking and MSI does so when the maskbit is set.
  332. */
  333. desc = first_msi_entry(dev);
  334. return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit;
  335. }
  336. int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
  337. int nvec)
  338. {
  339. struct msi_domain_info *info = domain->host_data;
  340. struct msi_domain_ops *ops = info->ops;
  341. struct irq_data *irq_data;
  342. struct msi_desc *desc;
  343. msi_alloc_info_t arg;
  344. int i, ret, virq;
  345. bool can_reserve;
  346. ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg);
  347. if (ret)
  348. return ret;
  349. for_each_msi_entry(desc, dev) {
  350. ops->set_desc(&arg, desc);
  351. virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used,
  352. dev_to_node(dev), &arg, false,
  353. desc->affinity);
  354. if (virq < 0) {
  355. ret = -ENOSPC;
  356. if (ops->handle_error)
  357. ret = ops->handle_error(domain, desc, ret);
  358. if (ops->msi_finish)
  359. ops->msi_finish(&arg, ret);
  360. return ret;
  361. }
  362. for (i = 0; i < desc->nvec_used; i++) {
  363. irq_set_msi_desc_off(virq, i, desc);
  364. irq_debugfs_copy_devname(virq + i, dev);
  365. }
  366. }
  367. if (ops->msi_finish)
  368. ops->msi_finish(&arg, 0);
  369. can_reserve = msi_check_reservation_mode(domain, info, dev);
  370. /*
  371. * This flag is set by the PCI layer as we need to activate
  372. * the MSI entries before the PCI layer enables MSI in the
  373. * card. Otherwise the card latches a random msi message.
  374. */
  375. if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY))
  376. goto skip_activate;
  377. for_each_msi_vector(desc, i, dev) {
  378. if (desc->irq == i) {
  379. virq = desc->irq;
  380. dev_dbg(dev, "irq [%d-%d] for MSI\n",
  381. virq, virq + desc->nvec_used - 1);
  382. }
  383. irq_data = irq_domain_get_irq_data(domain, i);
  384. if (!can_reserve) {
  385. irqd_clr_can_reserve(irq_data);
  386. if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK)
  387. irqd_set_msi_nomask_quirk(irq_data);
  388. }
  389. ret = irq_domain_activate_irq(irq_data, can_reserve);
  390. if (ret)
  391. goto cleanup;
  392. }
  393. skip_activate:
  394. /*
  395. * If these interrupts use reservation mode, clear the activated bit
  396. * so request_irq() will assign the final vector.
  397. */
  398. if (can_reserve) {
  399. for_each_msi_vector(desc, i, dev) {
  400. irq_data = irq_domain_get_irq_data(domain, i);
  401. irqd_clr_activated(irq_data);
  402. }
  403. }
  404. return 0;
  405. cleanup:
  406. msi_domain_free_irqs(domain, dev);
  407. return ret;
  408. }
  409. /**
  410. * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
  411. * @domain: The domain to allocate from
  412. * @dev: Pointer to device struct of the device for which the interrupts
  413. * are allocated
  414. * @nvec: The number of interrupts to allocate
  415. *
  416. * Returns 0 on success or an error code.
  417. */
  418. int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
  419. int nvec)
  420. {
  421. struct msi_domain_info *info = domain->host_data;
  422. struct msi_domain_ops *ops = info->ops;
  423. return ops->domain_alloc_irqs(domain, dev, nvec);
  424. }
  425. void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
  426. {
  427. struct irq_data *irq_data;
  428. struct msi_desc *desc;
  429. int i;
  430. for_each_msi_vector(desc, i, dev) {
  431. irq_data = irq_domain_get_irq_data(domain, i);
  432. if (irqd_is_activated(irq_data))
  433. irq_domain_deactivate_irq(irq_data);
  434. }
  435. for_each_msi_entry(desc, dev) {
  436. /*
  437. * We might have failed to allocate an MSI early
  438. * enough that there is no IRQ associated to this
  439. * entry. If that's the case, don't do anything.
  440. */
  441. if (desc->irq) {
  442. irq_domain_free_irqs(desc->irq, desc->nvec_used);
  443. desc->irq = 0;
  444. }
  445. }
  446. }
  447. /**
  448. * __msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated tp @dev
  449. * @domain: The domain to managing the interrupts
  450. * @dev: Pointer to device struct of the device for which the interrupts
  451. * are free
  452. */
  453. void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
  454. {
  455. struct msi_domain_info *info = domain->host_data;
  456. struct msi_domain_ops *ops = info->ops;
  457. return ops->domain_free_irqs(domain, dev);
  458. }
  459. /**
  460. * msi_get_domain_info - Get the MSI interrupt domain info for @domain
  461. * @domain: The interrupt domain to retrieve data from
  462. *
  463. * Returns the pointer to the msi_domain_info stored in
  464. * @domain->host_data.
  465. */
  466. struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain)
  467. {
  468. return (struct msi_domain_info *)domain->host_data;
  469. }
  470. #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */