smp.c 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Intel SMP support routines.
  4. *
  5. * (c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
  6. * (c) 1998-99, 2000, 2009 Ingo Molnar <mingo@redhat.com>
  7. * (c) 2002,2003 Andi Kleen, SuSE Labs.
  8. *
  9. * i386 and x86_64 integration by Glauber Costa <gcosta@redhat.com>
  10. */
  11. #include <linux/init.h>
  12. #include <linux/mm.h>
  13. #include <linux/delay.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/export.h>
  16. #include <linux/kernel_stat.h>
  17. #include <linux/mc146818rtc.h>
  18. #include <linux/cache.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/cpu.h>
  21. #include <linux/gfp.h>
  22. #include <asm/mtrr.h>
  23. #include <asm/tlbflush.h>
  24. #include <asm/mmu_context.h>
  25. #include <asm/proto.h>
  26. #include <asm/apic.h>
  27. #include <asm/idtentry.h>
  28. #include <asm/nmi.h>
  29. #include <asm/mce.h>
  30. #include <asm/trace/irq_vectors.h>
  31. #include <asm/kexec.h>
  32. #include <asm/virtext.h>
  33. /*
  34. * Some notes on x86 processor bugs affecting SMP operation:
  35. *
  36. * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
  37. * The Linux implications for SMP are handled as follows:
  38. *
  39. * Pentium III / [Xeon]
  40. * None of the E1AP-E3AP errata are visible to the user.
  41. *
  42. * E1AP. see PII A1AP
  43. * E2AP. see PII A2AP
  44. * E3AP. see PII A3AP
  45. *
  46. * Pentium II / [Xeon]
  47. * None of the A1AP-A3AP errata are visible to the user.
  48. *
  49. * A1AP. see PPro 1AP
  50. * A2AP. see PPro 2AP
  51. * A3AP. see PPro 7AP
  52. *
  53. * Pentium Pro
  54. * None of 1AP-9AP errata are visible to the normal user,
  55. * except occasional delivery of 'spurious interrupt' as trap #15.
  56. * This is very rare and a non-problem.
  57. *
  58. * 1AP. Linux maps APIC as non-cacheable
  59. * 2AP. worked around in hardware
  60. * 3AP. fixed in C0 and above steppings microcode update.
  61. * Linux does not use excessive STARTUP_IPIs.
  62. * 4AP. worked around in hardware
  63. * 5AP. symmetric IO mode (normal Linux operation) not affected.
  64. * 'noapic' mode has vector 0xf filled out properly.
  65. * 6AP. 'noapic' mode might be affected - fixed in later steppings
  66. * 7AP. We do not assume writes to the LVT deassering IRQs
  67. * 8AP. We do not enable low power mode (deep sleep) during MP bootup
  68. * 9AP. We do not use mixed mode
  69. *
  70. * Pentium
  71. * There is a marginal case where REP MOVS on 100MHz SMP
  72. * machines with B stepping processors can fail. XXX should provide
  73. * an L1cache=Writethrough or L1cache=off option.
  74. *
  75. * B stepping CPUs may hang. There are hardware work arounds
  76. * for this. We warn about it in case your board doesn't have the work
  77. * arounds. Basically that's so I can tell anyone with a B stepping
  78. * CPU and SMP problems "tough".
  79. *
  80. * Specific items [From Pentium Processor Specification Update]
  81. *
  82. * 1AP. Linux doesn't use remote read
  83. * 2AP. Linux doesn't trust APIC errors
  84. * 3AP. We work around this
  85. * 4AP. Linux never generated 3 interrupts of the same priority
  86. * to cause a lost local interrupt.
  87. * 5AP. Remote read is never used
  88. * 6AP. not affected - worked around in hardware
  89. * 7AP. not affected - worked around in hardware
  90. * 8AP. worked around in hardware - we get explicit CS errors if not
  91. * 9AP. only 'noapic' mode affected. Might generate spurious
  92. * interrupts, we log only the first one and count the
  93. * rest silently.
  94. * 10AP. not affected - worked around in hardware
  95. * 11AP. Linux reads the APIC between writes to avoid this, as per
  96. * the documentation. Make sure you preserve this as it affects
  97. * the C stepping chips too.
  98. * 12AP. not affected - worked around in hardware
  99. * 13AP. not affected - worked around in hardware
  100. * 14AP. we always deassert INIT during bootup
  101. * 15AP. not affected - worked around in hardware
  102. * 16AP. not affected - worked around in hardware
  103. * 17AP. not affected - worked around in hardware
  104. * 18AP. not affected - worked around in hardware
  105. * 19AP. not affected - worked around in BIOS
  106. *
  107. * If this sounds worrying believe me these bugs are either ___RARE___,
  108. * or are signal timing bugs worked around in hardware and there's
  109. * about nothing of note with C stepping upwards.
  110. */
  111. static atomic_t stopping_cpu = ATOMIC_INIT(-1);
  112. static bool smp_no_nmi_ipi = false;
  113. static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs)
  114. {
  115. /* We are registered on stopping cpu too, avoid spurious NMI */
  116. if (raw_smp_processor_id() == atomic_read(&stopping_cpu))
  117. return NMI_HANDLED;
  118. cpu_emergency_vmxoff();
  119. stop_this_cpu(NULL);
  120. return NMI_HANDLED;
  121. }
  122. /*
  123. * this function calls the 'stop' function on all other CPUs in the system.
  124. */
  125. DEFINE_IDTENTRY_SYSVEC(sysvec_reboot)
  126. {
  127. ack_APIC_irq();
  128. cpu_emergency_vmxoff();
  129. stop_this_cpu(NULL);
  130. }
  131. static int register_stop_handler(void)
  132. {
  133. return register_nmi_handler(NMI_LOCAL, smp_stop_nmi_callback,
  134. NMI_FLAG_FIRST, "smp_stop");
  135. }
  136. static void native_stop_other_cpus(int wait)
  137. {
  138. unsigned long flags;
  139. unsigned long timeout;
  140. if (reboot_force)
  141. return;
  142. /*
  143. * Use an own vector here because smp_call_function
  144. * does lots of things not suitable in a panic situation.
  145. */
  146. /*
  147. * We start by using the REBOOT_VECTOR irq.
  148. * The irq is treated as a sync point to allow critical
  149. * regions of code on other cpus to release their spin locks
  150. * and re-enable irqs. Jumping straight to an NMI might
  151. * accidentally cause deadlocks with further shutdown/panic
  152. * code. By syncing, we give the cpus up to one second to
  153. * finish their work before we force them off with the NMI.
  154. */
  155. if (num_online_cpus() > 1) {
  156. /* did someone beat us here? */
  157. if (atomic_cmpxchg(&stopping_cpu, -1, safe_smp_processor_id()) != -1)
  158. return;
  159. /* sync above data before sending IRQ */
  160. wmb();
  161. apic_send_IPI_allbutself(REBOOT_VECTOR);
  162. /*
  163. * Don't wait longer than a second for IPI completion. The
  164. * wait request is not checked here because that would
  165. * prevent an NMI shutdown attempt in case that not all
  166. * CPUs reach shutdown state.
  167. */
  168. timeout = USEC_PER_SEC;
  169. while (num_online_cpus() > 1 && timeout--)
  170. udelay(1);
  171. }
  172. /* if the REBOOT_VECTOR didn't work, try with the NMI */
  173. if (num_online_cpus() > 1) {
  174. /*
  175. * If NMI IPI is enabled, try to register the stop handler
  176. * and send the IPI. In any case try to wait for the other
  177. * CPUs to stop.
  178. */
  179. if (!smp_no_nmi_ipi && !register_stop_handler()) {
  180. /* Sync above data before sending IRQ */
  181. wmb();
  182. pr_emerg("Shutting down cpus with NMI\n");
  183. apic_send_IPI_allbutself(NMI_VECTOR);
  184. }
  185. /*
  186. * Don't wait longer than 10 ms if the caller didn't
  187. * reqeust it. If wait is true, the machine hangs here if
  188. * one or more CPUs do not reach shutdown state.
  189. */
  190. timeout = USEC_PER_MSEC * 10;
  191. while (num_online_cpus() > 1 && (wait || timeout--))
  192. udelay(1);
  193. }
  194. local_irq_save(flags);
  195. disable_local_APIC();
  196. mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
  197. local_irq_restore(flags);
  198. }
  199. /*
  200. * Reschedule call back. KVM uses this interrupt to force a cpu out of
  201. * guest mode.
  202. */
  203. DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
  204. {
  205. ack_APIC_irq();
  206. trace_reschedule_entry(RESCHEDULE_VECTOR);
  207. inc_irq_stat(irq_resched_count);
  208. scheduler_ipi();
  209. trace_reschedule_exit(RESCHEDULE_VECTOR);
  210. }
  211. DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
  212. {
  213. ack_APIC_irq();
  214. trace_call_function_entry(CALL_FUNCTION_VECTOR);
  215. inc_irq_stat(irq_call_count);
  216. generic_smp_call_function_interrupt();
  217. trace_call_function_exit(CALL_FUNCTION_VECTOR);
  218. }
  219. DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single)
  220. {
  221. ack_APIC_irq();
  222. trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR);
  223. inc_irq_stat(irq_call_count);
  224. generic_smp_call_function_single_interrupt();
  225. trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR);
  226. }
  227. static int __init nonmi_ipi_setup(char *str)
  228. {
  229. smp_no_nmi_ipi = true;
  230. return 1;
  231. }
  232. __setup("nonmi_ipi", nonmi_ipi_setup);
  233. struct smp_ops smp_ops = {
  234. .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
  235. .smp_prepare_cpus = native_smp_prepare_cpus,
  236. .smp_cpus_done = native_smp_cpus_done,
  237. .stop_other_cpus = native_stop_other_cpus,
  238. #if defined(CONFIG_KEXEC_CORE)
  239. .crash_stop_other_cpus = kdump_nmi_shootdown_cpus,
  240. #endif
  241. .smp_send_reschedule = native_smp_send_reschedule,
  242. .cpu_up = native_cpu_up,
  243. .cpu_die = native_cpu_die,
  244. .cpu_disable = native_cpu_disable,
  245. .play_dead = native_play_dead,
  246. .send_call_func_ipi = native_send_call_func_ipi,
  247. .send_call_func_single_ipi = native_send_call_func_single_ipi,
  248. };
  249. EXPORT_SYMBOL_GPL(smp_ops);