irq.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Common interrupt code for 32 and 64 bit
  4. */
  5. #include <linux/cpu.h>
  6. #include <linux/interrupt.h>
  7. #include <linux/kernel_stat.h>
  8. #include <linux/of.h>
  9. #include <linux/seq_file.h>
  10. #include <linux/smp.h>
  11. #include <linux/ftrace.h>
  12. #include <linux/delay.h>
  13. #include <linux/export.h>
  14. #include <linux/irq.h>
  15. #include <asm/irq_stack.h>
  16. #include <asm/apic.h>
  17. #include <asm/io_apic.h>
  18. #include <asm/irq.h>
  19. #include <asm/mce.h>
  20. #include <asm/hw_irq.h>
  21. #include <asm/desc.h>
  22. #include <asm/traps.h>
  23. #define CREATE_TRACE_POINTS
  24. #include <asm/trace/irq_vectors.h>
  25. DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
  26. EXPORT_PER_CPU_SYMBOL(irq_stat);
  27. atomic_t irq_err_count;
  28. /*
  29. * 'what should we do if we get a hw irq event on an illegal vector'.
  30. * each architecture has to answer this themselves.
  31. */
  32. void ack_bad_irq(unsigned int irq)
  33. {
  34. if (printk_ratelimit())
  35. pr_err("unexpected IRQ trap at vector %02x\n", irq);
  36. /*
  37. * Currently unexpected vectors happen only on SMP and APIC.
  38. * We _must_ ack these because every local APIC has only N
  39. * irq slots per priority level, and a 'hanging, unacked' IRQ
  40. * holds up an irq slot - in excessive cases (when multiple
  41. * unexpected vectors occur) that might lock up the APIC
  42. * completely.
  43. * But only ack when the APIC is enabled -AK
  44. */
  45. ack_APIC_irq();
  46. }
  47. #define irq_stats(x) (&per_cpu(irq_stat, x))
  48. /*
  49. * /proc/interrupts printing for arch specific interrupts
  50. */
  51. int arch_show_interrupts(struct seq_file *p, int prec)
  52. {
  53. int j;
  54. seq_printf(p, "%*s: ", prec, "NMI");
  55. for_each_online_cpu(j)
  56. seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
  57. seq_puts(p, " Non-maskable interrupts\n");
  58. #ifdef CONFIG_X86_LOCAL_APIC
  59. seq_printf(p, "%*s: ", prec, "LOC");
  60. for_each_online_cpu(j)
  61. seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
  62. seq_puts(p, " Local timer interrupts\n");
  63. seq_printf(p, "%*s: ", prec, "SPU");
  64. for_each_online_cpu(j)
  65. seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
  66. seq_puts(p, " Spurious interrupts\n");
  67. seq_printf(p, "%*s: ", prec, "PMI");
  68. for_each_online_cpu(j)
  69. seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
  70. seq_puts(p, " Performance monitoring interrupts\n");
  71. seq_printf(p, "%*s: ", prec, "IWI");
  72. for_each_online_cpu(j)
  73. seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
  74. seq_puts(p, " IRQ work interrupts\n");
  75. seq_printf(p, "%*s: ", prec, "RTR");
  76. for_each_online_cpu(j)
  77. seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
  78. seq_puts(p, " APIC ICR read retries\n");
  79. if (x86_platform_ipi_callback) {
  80. seq_printf(p, "%*s: ", prec, "PLT");
  81. for_each_online_cpu(j)
  82. seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
  83. seq_puts(p, " Platform interrupts\n");
  84. }
  85. #endif
  86. #ifdef CONFIG_SMP
  87. seq_printf(p, "%*s: ", prec, "RES");
  88. for_each_online_cpu(j)
  89. seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
  90. seq_puts(p, " Rescheduling interrupts\n");
  91. seq_printf(p, "%*s: ", prec, "CAL");
  92. for_each_online_cpu(j)
  93. seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
  94. seq_puts(p, " Function call interrupts\n");
  95. seq_printf(p, "%*s: ", prec, "TLB");
  96. for_each_online_cpu(j)
  97. seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
  98. seq_puts(p, " TLB shootdowns\n");
  99. #endif
  100. #ifdef CONFIG_X86_THERMAL_VECTOR
  101. seq_printf(p, "%*s: ", prec, "TRM");
  102. for_each_online_cpu(j)
  103. seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
  104. seq_puts(p, " Thermal event interrupts\n");
  105. #endif
  106. #ifdef CONFIG_X86_MCE_THRESHOLD
  107. seq_printf(p, "%*s: ", prec, "THR");
  108. for_each_online_cpu(j)
  109. seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
  110. seq_puts(p, " Threshold APIC interrupts\n");
  111. #endif
  112. #ifdef CONFIG_X86_MCE_AMD
  113. seq_printf(p, "%*s: ", prec, "DFR");
  114. for_each_online_cpu(j)
  115. seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
  116. seq_puts(p, " Deferred Error APIC interrupts\n");
  117. #endif
  118. #ifdef CONFIG_X86_MCE
  119. seq_printf(p, "%*s: ", prec, "MCE");
  120. for_each_online_cpu(j)
  121. seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
  122. seq_puts(p, " Machine check exceptions\n");
  123. seq_printf(p, "%*s: ", prec, "MCP");
  124. for_each_online_cpu(j)
  125. seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
  126. seq_puts(p, " Machine check polls\n");
  127. #endif
  128. #ifdef CONFIG_X86_HV_CALLBACK_VECTOR
  129. if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
  130. seq_printf(p, "%*s: ", prec, "HYP");
  131. for_each_online_cpu(j)
  132. seq_printf(p, "%10u ",
  133. irq_stats(j)->irq_hv_callback_count);
  134. seq_puts(p, " Hypervisor callback interrupts\n");
  135. }
  136. #endif
  137. #if IS_ENABLED(CONFIG_HYPERV)
  138. if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
  139. seq_printf(p, "%*s: ", prec, "HRE");
  140. for_each_online_cpu(j)
  141. seq_printf(p, "%10u ",
  142. irq_stats(j)->irq_hv_reenlightenment_count);
  143. seq_puts(p, " Hyper-V reenlightenment interrupts\n");
  144. }
  145. if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) {
  146. seq_printf(p, "%*s: ", prec, "HVS");
  147. for_each_online_cpu(j)
  148. seq_printf(p, "%10u ",
  149. irq_stats(j)->hyperv_stimer0_count);
  150. seq_puts(p, " Hyper-V stimer0 interrupts\n");
  151. }
  152. #endif
  153. seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
  154. #if defined(CONFIG_X86_IO_APIC)
  155. seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
  156. #endif
  157. #ifdef CONFIG_HAVE_KVM
  158. seq_printf(p, "%*s: ", prec, "PIN");
  159. for_each_online_cpu(j)
  160. seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
  161. seq_puts(p, " Posted-interrupt notification event\n");
  162. seq_printf(p, "%*s: ", prec, "NPI");
  163. for_each_online_cpu(j)
  164. seq_printf(p, "%10u ",
  165. irq_stats(j)->kvm_posted_intr_nested_ipis);
  166. seq_puts(p, " Nested posted-interrupt event\n");
  167. seq_printf(p, "%*s: ", prec, "PIW");
  168. for_each_online_cpu(j)
  169. seq_printf(p, "%10u ",
  170. irq_stats(j)->kvm_posted_intr_wakeup_ipis);
  171. seq_puts(p, " Posted-interrupt wakeup event\n");
  172. #endif
  173. return 0;
  174. }
  175. /*
  176. * /proc/stat helpers
  177. */
  178. u64 arch_irq_stat_cpu(unsigned int cpu)
  179. {
  180. u64 sum = irq_stats(cpu)->__nmi_count;
  181. #ifdef CONFIG_X86_LOCAL_APIC
  182. sum += irq_stats(cpu)->apic_timer_irqs;
  183. sum += irq_stats(cpu)->irq_spurious_count;
  184. sum += irq_stats(cpu)->apic_perf_irqs;
  185. sum += irq_stats(cpu)->apic_irq_work_irqs;
  186. sum += irq_stats(cpu)->icr_read_retry_count;
  187. if (x86_platform_ipi_callback)
  188. sum += irq_stats(cpu)->x86_platform_ipis;
  189. #endif
  190. #ifdef CONFIG_SMP
  191. sum += irq_stats(cpu)->irq_resched_count;
  192. sum += irq_stats(cpu)->irq_call_count;
  193. #endif
  194. #ifdef CONFIG_X86_THERMAL_VECTOR
  195. sum += irq_stats(cpu)->irq_thermal_count;
  196. #endif
  197. #ifdef CONFIG_X86_MCE_THRESHOLD
  198. sum += irq_stats(cpu)->irq_threshold_count;
  199. #endif
  200. #ifdef CONFIG_X86_MCE
  201. sum += per_cpu(mce_exception_count, cpu);
  202. sum += per_cpu(mce_poll_count, cpu);
  203. #endif
  204. return sum;
  205. }
  206. u64 arch_irq_stat(void)
  207. {
  208. u64 sum = atomic_read(&irq_err_count);
  209. return sum;
  210. }
  211. static __always_inline void handle_irq(struct irq_desc *desc,
  212. struct pt_regs *regs)
  213. {
  214. if (IS_ENABLED(CONFIG_X86_64))
  215. run_irq_on_irqstack_cond(desc->handle_irq, desc, regs);
  216. else
  217. __handle_irq(desc, regs);
  218. }
  219. /*
  220. * common_interrupt() handles all normal device IRQ's (the special SMP
  221. * cross-CPU interrupts have their own entry points).
  222. */
  223. DEFINE_IDTENTRY_IRQ(common_interrupt)
  224. {
  225. struct pt_regs *old_regs = set_irq_regs(regs);
  226. struct irq_desc *desc;
  227. /* entry code tells RCU that we're not quiescent. Check it. */
  228. RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
  229. desc = __this_cpu_read(vector_irq[vector]);
  230. if (likely(!IS_ERR_OR_NULL(desc))) {
  231. handle_irq(desc, regs);
  232. } else {
  233. ack_APIC_irq();
  234. if (desc == VECTOR_UNUSED) {
  235. pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n",
  236. __func__, smp_processor_id(),
  237. vector);
  238. } else {
  239. __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
  240. }
  241. }
  242. set_irq_regs(old_regs);
  243. }
  244. #ifdef CONFIG_X86_LOCAL_APIC
  245. /* Function pointer for generic interrupt vector handling */
  246. void (*x86_platform_ipi_callback)(void) = NULL;
  247. /*
  248. * Handler for X86_PLATFORM_IPI_VECTOR.
  249. */
  250. DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
  251. {
  252. struct pt_regs *old_regs = set_irq_regs(regs);
  253. ack_APIC_irq();
  254. trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
  255. inc_irq_stat(x86_platform_ipis);
  256. if (x86_platform_ipi_callback)
  257. x86_platform_ipi_callback();
  258. trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
  259. set_irq_regs(old_regs);
  260. }
  261. #endif
  262. #ifdef CONFIG_HAVE_KVM
  263. static void dummy_handler(void) {}
  264. static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
  265. void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
  266. {
  267. if (handler)
  268. kvm_posted_intr_wakeup_handler = handler;
  269. else {
  270. kvm_posted_intr_wakeup_handler = dummy_handler;
  271. synchronize_rcu();
  272. }
  273. }
  274. EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
  275. /*
  276. * Handler for POSTED_INTERRUPT_VECTOR.
  277. */
  278. DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
  279. {
  280. ack_APIC_irq();
  281. inc_irq_stat(kvm_posted_intr_ipis);
  282. }
  283. /*
  284. * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
  285. */
  286. DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
  287. {
  288. ack_APIC_irq();
  289. inc_irq_stat(kvm_posted_intr_wakeup_ipis);
  290. kvm_posted_intr_wakeup_handler();
  291. }
  292. /*
  293. * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
  294. */
  295. DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi)
  296. {
  297. ack_APIC_irq();
  298. inc_irq_stat(kvm_posted_intr_nested_ipis);
  299. }
  300. #endif
  301. #ifdef CONFIG_HOTPLUG_CPU
  302. /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
  303. void fixup_irqs(void)
  304. {
  305. unsigned int irr, vector;
  306. struct irq_desc *desc;
  307. struct irq_data *data;
  308. struct irq_chip *chip;
  309. irq_migrate_all_off_this_cpu();
  310. /*
  311. * We can remove mdelay() and then send spuriuous interrupts to
  312. * new cpu targets for all the irqs that were handled previously by
  313. * this cpu. While it works, I have seen spurious interrupt messages
  314. * (nothing wrong but still...).
  315. *
  316. * So for now, retain mdelay(1) and check the IRR and then send those
  317. * interrupts to new targets as this cpu is already offlined...
  318. */
  319. mdelay(1);
  320. /*
  321. * We can walk the vector array of this cpu without holding
  322. * vector_lock because the cpu is already marked !online, so
  323. * nothing else will touch it.
  324. */
  325. for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
  326. if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
  327. continue;
  328. irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
  329. if (irr & (1 << (vector % 32))) {
  330. desc = __this_cpu_read(vector_irq[vector]);
  331. raw_spin_lock(&desc->lock);
  332. data = irq_desc_get_irq_data(desc);
  333. chip = irq_data_get_irq_chip(data);
  334. if (chip->irq_retrigger) {
  335. chip->irq_retrigger(data);
  336. __this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
  337. }
  338. raw_spin_unlock(&desc->lock);
  339. }
  340. if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
  341. __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
  342. }
  343. }
  344. #endif