irq.c 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
  3. #include <linux/init.h>
  4. #include <linux/kernel.h>
  5. #include <linux/module.h>
  6. #include <linux/pci.h>
  7. #include <linux/io-64-nonatomic-lo-hi.h>
  8. #include <linux/dmaengine.h>
  9. #include <uapi/linux/idxd.h>
  10. #include "../dmaengine.h"
  11. #include "idxd.h"
  12. #include "registers.h"
  13. static void idxd_device_reinit(struct work_struct *work)
  14. {
  15. struct idxd_device *idxd = container_of(work, struct idxd_device, work);
  16. struct device *dev = &idxd->pdev->dev;
  17. int rc, i;
  18. idxd_device_reset(idxd);
  19. rc = idxd_device_config(idxd);
  20. if (rc < 0)
  21. goto out;
  22. rc = idxd_device_enable(idxd);
  23. if (rc < 0)
  24. goto out;
  25. for (i = 0; i < idxd->max_wqs; i++) {
  26. struct idxd_wq *wq = &idxd->wqs[i];
  27. if (wq->state == IDXD_WQ_ENABLED) {
  28. rc = idxd_wq_enable(wq);
  29. if (rc < 0) {
  30. dev_warn(dev, "Unable to re-enable wq %s\n",
  31. dev_name(&wq->conf_dev));
  32. }
  33. }
  34. }
  35. return;
  36. out:
  37. idxd_device_wqs_clear_state(idxd);
  38. }
  39. irqreturn_t idxd_irq_handler(int vec, void *data)
  40. {
  41. struct idxd_irq_entry *irq_entry = data;
  42. struct idxd_device *idxd = irq_entry->idxd;
  43. idxd_mask_msix_vector(idxd, irq_entry->id);
  44. return IRQ_WAKE_THREAD;
  45. }
  46. static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
  47. {
  48. struct device *dev = &idxd->pdev->dev;
  49. union gensts_reg gensts;
  50. u32 val = 0;
  51. int i;
  52. bool err = false;
  53. if (cause & IDXD_INTC_ERR) {
  54. spin_lock_bh(&idxd->dev_lock);
  55. for (i = 0; i < 4; i++)
  56. idxd->sw_err.bits[i] = ioread64(idxd->reg_base +
  57. IDXD_SWERR_OFFSET + i * sizeof(u64));
  58. iowrite64(idxd->sw_err.bits[0] & IDXD_SWERR_ACK,
  59. idxd->reg_base + IDXD_SWERR_OFFSET);
  60. if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) {
  61. int id = idxd->sw_err.wq_idx;
  62. struct idxd_wq *wq = &idxd->wqs[id];
  63. if (wq->type == IDXD_WQT_USER)
  64. wake_up_interruptible(&wq->err_queue);
  65. } else {
  66. int i;
  67. for (i = 0; i < idxd->max_wqs; i++) {
  68. struct idxd_wq *wq = &idxd->wqs[i];
  69. if (wq->type == IDXD_WQT_USER)
  70. wake_up_interruptible(&wq->err_queue);
  71. }
  72. }
  73. spin_unlock_bh(&idxd->dev_lock);
  74. val |= IDXD_INTC_ERR;
  75. for (i = 0; i < 4; i++)
  76. dev_warn(dev, "err[%d]: %#16.16llx\n",
  77. i, idxd->sw_err.bits[i]);
  78. err = true;
  79. }
  80. if (cause & IDXD_INTC_CMD) {
  81. val |= IDXD_INTC_CMD;
  82. complete(idxd->cmd_done);
  83. }
  84. if (cause & IDXD_INTC_OCCUPY) {
  85. /* Driver does not utilize occupancy interrupt */
  86. val |= IDXD_INTC_OCCUPY;
  87. }
  88. if (cause & IDXD_INTC_PERFMON_OVFL) {
  89. /*
  90. * Driver does not utilize perfmon counter overflow interrupt
  91. * yet.
  92. */
  93. val |= IDXD_INTC_PERFMON_OVFL;
  94. }
  95. val ^= cause;
  96. if (val)
  97. dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n",
  98. val);
  99. if (!err)
  100. return 0;
  101. gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
  102. if (gensts.state == IDXD_DEVICE_STATE_HALT) {
  103. idxd->state = IDXD_DEV_HALTED;
  104. if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) {
  105. /*
  106. * If we need a software reset, we will throw the work
  107. * on a system workqueue in order to allow interrupts
  108. * for the device command completions.
  109. */
  110. INIT_WORK(&idxd->work, idxd_device_reinit);
  111. queue_work(idxd->wq, &idxd->work);
  112. } else {
  113. spin_lock_bh(&idxd->dev_lock);
  114. idxd_device_wqs_clear_state(idxd);
  115. dev_err(&idxd->pdev->dev,
  116. "idxd halted, need %s.\n",
  117. gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
  118. "FLR" : "system reset");
  119. spin_unlock_bh(&idxd->dev_lock);
  120. return -ENXIO;
  121. }
  122. }
  123. return 0;
  124. }
  125. irqreturn_t idxd_misc_thread(int vec, void *data)
  126. {
  127. struct idxd_irq_entry *irq_entry = data;
  128. struct idxd_device *idxd = irq_entry->idxd;
  129. int rc;
  130. u32 cause;
  131. cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
  132. if (cause)
  133. iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
  134. while (cause) {
  135. rc = process_misc_interrupts(idxd, cause);
  136. if (rc < 0)
  137. break;
  138. cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
  139. if (cause)
  140. iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
  141. }
  142. idxd_unmask_msix_vector(idxd, irq_entry->id);
  143. return IRQ_HANDLED;
  144. }
  145. static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
  146. int *processed)
  147. {
  148. struct idxd_desc *desc, *t;
  149. struct llist_node *head;
  150. int queued = 0;
  151. *processed = 0;
  152. head = llist_del_all(&irq_entry->pending_llist);
  153. if (!head)
  154. return 0;
  155. llist_for_each_entry_safe(desc, t, head, llnode) {
  156. if (desc->completion->status) {
  157. idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
  158. idxd_free_desc(desc->wq, desc);
  159. (*processed)++;
  160. } else {
  161. list_add_tail(&desc->list, &irq_entry->work_list);
  162. queued++;
  163. }
  164. }
  165. return queued;
  166. }
  167. static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
  168. int *processed)
  169. {
  170. struct list_head *node, *next;
  171. int queued = 0;
  172. *processed = 0;
  173. if (list_empty(&irq_entry->work_list))
  174. return 0;
  175. list_for_each_safe(node, next, &irq_entry->work_list) {
  176. struct idxd_desc *desc =
  177. container_of(node, struct idxd_desc, list);
  178. if (desc->completion->status) {
  179. list_del(&desc->list);
  180. /* process and callback */
  181. idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
  182. idxd_free_desc(desc->wq, desc);
  183. (*processed)++;
  184. } else {
  185. queued++;
  186. }
  187. }
  188. return queued;
  189. }
  190. static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
  191. {
  192. int rc, processed, total = 0;
  193. /*
  194. * There are two lists we are processing. The pending_llist is where
  195. * submmiter adds all the submitted descriptor after sending it to
  196. * the workqueue. It's a lockless singly linked list. The work_list
  197. * is the common linux double linked list. We are in a scenario of
  198. * multiple producers and a single consumer. The producers are all
  199. * the kernel submitters of descriptors, and the consumer is the
  200. * kernel irq handler thread for the msix vector when using threaded
  201. * irq. To work with the restrictions of llist to remain lockless,
  202. * we are doing the following steps:
  203. * 1. Iterate through the work_list and process any completed
  204. * descriptor. Delete the completed entries during iteration.
  205. * 2. llist_del_all() from the pending list.
  206. * 3. Iterate through the llist that was deleted from the pending list
  207. * and process the completed entries.
  208. * 4. If the entry is still waiting on hardware, list_add_tail() to
  209. * the work_list.
  210. * 5. Repeat until no more descriptors.
  211. */
  212. do {
  213. rc = irq_process_work_list(irq_entry, &processed);
  214. total += processed;
  215. if (rc != 0)
  216. continue;
  217. rc = irq_process_pending_llist(irq_entry, &processed);
  218. total += processed;
  219. } while (rc != 0);
  220. return total;
  221. }
  222. irqreturn_t idxd_wq_thread(int irq, void *data)
  223. {
  224. struct idxd_irq_entry *irq_entry = data;
  225. int processed;
  226. processed = idxd_desc_process(irq_entry);
  227. idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id);
  228. if (processed == 0)
  229. return IRQ_NONE;
  230. return IRQ_HANDLED;
  231. }