nmi.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 1991, 1992 Linus Torvalds
  4. * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
  5. * Copyright (C) 2011 Don Zickus Red Hat, Inc.
  6. *
  7. * Pentium III FXSR, SSE support
  8. * Gareth Hughes <gareth@valinux.com>, May 2000
  9. */
  10. /*
  11. * Handle hardware traps and faults.
  12. */
  13. #include <linux/spinlock.h>
  14. #include <linux/kprobes.h>
  15. #include <linux/kdebug.h>
  16. #include <linux/sched/debug.h>
  17. #include <linux/nmi.h>
  18. #include <linux/debugfs.h>
  19. #include <linux/delay.h>
  20. #include <linux/hardirq.h>
  21. #include <linux/ratelimit.h>
  22. #include <linux/slab.h>
  23. #include <linux/export.h>
  24. #include <linux/atomic.h>
  25. #include <linux/sched/clock.h>
  26. #include <asm/cpu_entry_area.h>
  27. #include <asm/traps.h>
  28. #include <asm/mach_traps.h>
  29. #include <asm/nmi.h>
  30. #include <asm/x86_init.h>
  31. #include <asm/reboot.h>
  32. #include <asm/cache.h>
  33. #include <asm/nospec-branch.h>
  34. #include <asm/sev-es.h>
  35. #define CREATE_TRACE_POINTS
  36. #include <trace/events/nmi.h>
  37. struct nmi_desc {
  38. raw_spinlock_t lock;
  39. struct list_head head;
  40. };
  41. static struct nmi_desc nmi_desc[NMI_MAX] =
  42. {
  43. {
  44. .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
  45. .head = LIST_HEAD_INIT(nmi_desc[0].head),
  46. },
  47. {
  48. .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
  49. .head = LIST_HEAD_INIT(nmi_desc[1].head),
  50. },
  51. {
  52. .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock),
  53. .head = LIST_HEAD_INIT(nmi_desc[2].head),
  54. },
  55. {
  56. .lock = __RAW_SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock),
  57. .head = LIST_HEAD_INIT(nmi_desc[3].head),
  58. },
  59. };
  60. struct nmi_stats {
  61. unsigned int normal;
  62. unsigned int unknown;
  63. unsigned int external;
  64. unsigned int swallow;
  65. };
  66. static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
  67. static int ignore_nmis __read_mostly;
  68. int unknown_nmi_panic;
  69. /*
  70. * Prevent NMI reason port (0x61) being accessed simultaneously, can
  71. * only be used in NMI handler.
  72. */
  73. static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
  74. static int __init setup_unknown_nmi_panic(char *str)
  75. {
  76. unknown_nmi_panic = 1;
  77. return 1;
  78. }
  79. __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
  80. #define nmi_to_desc(type) (&nmi_desc[type])
  81. static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
  82. static int __init nmi_warning_debugfs(void)
  83. {
  84. debugfs_create_u64("nmi_longest_ns", 0644,
  85. arch_debugfs_dir, &nmi_longest_ns);
  86. return 0;
  87. }
  88. fs_initcall(nmi_warning_debugfs);
  89. static void nmi_check_duration(struct nmiaction *action, u64 duration)
  90. {
  91. int remainder_ns, decimal_msecs;
  92. if (duration < nmi_longest_ns || duration < action->max_duration)
  93. return;
  94. action->max_duration = duration;
  95. remainder_ns = do_div(duration, (1000 * 1000));
  96. decimal_msecs = remainder_ns / 1000;
  97. printk_ratelimited(KERN_INFO
  98. "INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
  99. action->handler, duration, decimal_msecs);
  100. }
  101. static int nmi_handle(unsigned int type, struct pt_regs *regs)
  102. {
  103. struct nmi_desc *desc = nmi_to_desc(type);
  104. struct nmiaction *a;
  105. int handled=0;
  106. rcu_read_lock();
  107. /*
  108. * NMIs are edge-triggered, which means if you have enough
  109. * of them concurrently, you can lose some because only one
  110. * can be latched at any given time. Walk the whole list
  111. * to handle those situations.
  112. */
  113. list_for_each_entry_rcu(a, &desc->head, list) {
  114. int thishandled;
  115. u64 delta;
  116. delta = sched_clock();
  117. thishandled = a->handler(type, regs);
  118. handled += thishandled;
  119. delta = sched_clock() - delta;
  120. trace_nmi_handler(a->handler, (int)delta, thishandled);
  121. nmi_check_duration(a, delta);
  122. }
  123. rcu_read_unlock();
  124. /* return total number of NMI events handled */
  125. return handled;
  126. }
  127. NOKPROBE_SYMBOL(nmi_handle);
  128. int __register_nmi_handler(unsigned int type, struct nmiaction *action)
  129. {
  130. struct nmi_desc *desc = nmi_to_desc(type);
  131. unsigned long flags;
  132. if (!action->handler)
  133. return -EINVAL;
  134. raw_spin_lock_irqsave(&desc->lock, flags);
  135. /*
  136. * Indicate if there are multiple registrations on the
  137. * internal NMI handler call chains (SERR and IO_CHECK).
  138. */
  139. WARN_ON_ONCE(type == NMI_SERR && !list_empty(&desc->head));
  140. WARN_ON_ONCE(type == NMI_IO_CHECK && !list_empty(&desc->head));
  141. /*
  142. * some handlers need to be executed first otherwise a fake
  143. * event confuses some handlers (kdump uses this flag)
  144. */
  145. if (action->flags & NMI_FLAG_FIRST)
  146. list_add_rcu(&action->list, &desc->head);
  147. else
  148. list_add_tail_rcu(&action->list, &desc->head);
  149. raw_spin_unlock_irqrestore(&desc->lock, flags);
  150. return 0;
  151. }
  152. EXPORT_SYMBOL(__register_nmi_handler);
  153. void unregister_nmi_handler(unsigned int type, const char *name)
  154. {
  155. struct nmi_desc *desc = nmi_to_desc(type);
  156. struct nmiaction *n;
  157. unsigned long flags;
  158. raw_spin_lock_irqsave(&desc->lock, flags);
  159. list_for_each_entry_rcu(n, &desc->head, list) {
  160. /*
  161. * the name passed in to describe the nmi handler
  162. * is used as the lookup key
  163. */
  164. if (!strcmp(n->name, name)) {
  165. WARN(in_nmi(),
  166. "Trying to free NMI (%s) from NMI context!\n", n->name);
  167. list_del_rcu(&n->list);
  168. break;
  169. }
  170. }
  171. raw_spin_unlock_irqrestore(&desc->lock, flags);
  172. synchronize_rcu();
  173. }
  174. EXPORT_SYMBOL_GPL(unregister_nmi_handler);
  175. static void
  176. pci_serr_error(unsigned char reason, struct pt_regs *regs)
  177. {
  178. /* check to see if anyone registered against these types of errors */
  179. if (nmi_handle(NMI_SERR, regs))
  180. return;
  181. pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
  182. reason, smp_processor_id());
  183. if (panic_on_unrecovered_nmi)
  184. nmi_panic(regs, "NMI: Not continuing");
  185. pr_emerg("Dazed and confused, but trying to continue\n");
  186. /* Clear and disable the PCI SERR error line. */
  187. reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
  188. outb(reason, NMI_REASON_PORT);
  189. }
  190. NOKPROBE_SYMBOL(pci_serr_error);
  191. static void
  192. io_check_error(unsigned char reason, struct pt_regs *regs)
  193. {
  194. unsigned long i;
  195. /* check to see if anyone registered against these types of errors */
  196. if (nmi_handle(NMI_IO_CHECK, regs))
  197. return;
  198. pr_emerg(
  199. "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
  200. reason, smp_processor_id());
  201. show_regs(regs);
  202. if (panic_on_io_nmi) {
  203. nmi_panic(regs, "NMI IOCK error: Not continuing");
  204. /*
  205. * If we end up here, it means we have received an NMI while
  206. * processing panic(). Simply return without delaying and
  207. * re-enabling NMIs.
  208. */
  209. return;
  210. }
  211. /* Re-enable the IOCK line, wait for a few seconds */
  212. reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
  213. outb(reason, NMI_REASON_PORT);
  214. i = 20000;
  215. while (--i) {
  216. touch_nmi_watchdog();
  217. udelay(100);
  218. }
  219. reason &= ~NMI_REASON_CLEAR_IOCHK;
  220. outb(reason, NMI_REASON_PORT);
  221. }
  222. NOKPROBE_SYMBOL(io_check_error);
  223. static void
  224. unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
  225. {
  226. int handled;
  227. /*
  228. * Use 'false' as back-to-back NMIs are dealt with one level up.
  229. * Of course this makes having multiple 'unknown' handlers useless
  230. * as only the first one is ever run (unless it can actually determine
  231. * if it caused the NMI)
  232. */
  233. handled = nmi_handle(NMI_UNKNOWN, regs);
  234. if (handled) {
  235. __this_cpu_add(nmi_stats.unknown, handled);
  236. return;
  237. }
  238. __this_cpu_add(nmi_stats.unknown, 1);
  239. pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
  240. reason, smp_processor_id());
  241. pr_emerg("Do you have a strange power saving mode enabled?\n");
  242. if (unknown_nmi_panic || panic_on_unrecovered_nmi)
  243. nmi_panic(regs, "NMI: Not continuing");
  244. pr_emerg("Dazed and confused, but trying to continue\n");
  245. }
  246. NOKPROBE_SYMBOL(unknown_nmi_error);
  247. static DEFINE_PER_CPU(bool, swallow_nmi);
  248. static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
  249. static noinstr void default_do_nmi(struct pt_regs *regs)
  250. {
  251. unsigned char reason = 0;
  252. int handled;
  253. bool b2b = false;
  254. /*
  255. * CPU-specific NMI must be processed before non-CPU-specific
  256. * NMI, otherwise we may lose it, because the CPU-specific
  257. * NMI can not be detected/processed on other CPUs.
  258. */
  259. /*
  260. * Back-to-back NMIs are interesting because they can either
  261. * be two NMI or more than two NMIs (any thing over two is dropped
  262. * due to NMI being edge-triggered). If this is the second half
  263. * of the back-to-back NMI, assume we dropped things and process
  264. * more handlers. Otherwise reset the 'swallow' NMI behaviour
  265. */
  266. if (regs->ip == __this_cpu_read(last_nmi_rip))
  267. b2b = true;
  268. else
  269. __this_cpu_write(swallow_nmi, false);
  270. __this_cpu_write(last_nmi_rip, regs->ip);
  271. instrumentation_begin();
  272. handled = nmi_handle(NMI_LOCAL, regs);
  273. __this_cpu_add(nmi_stats.normal, handled);
  274. if (handled) {
  275. /*
  276. * There are cases when a NMI handler handles multiple
  277. * events in the current NMI. One of these events may
  278. * be queued for in the next NMI. Because the event is
  279. * already handled, the next NMI will result in an unknown
  280. * NMI. Instead lets flag this for a potential NMI to
  281. * swallow.
  282. */
  283. if (handled > 1)
  284. __this_cpu_write(swallow_nmi, true);
  285. goto out;
  286. }
  287. /*
  288. * Non-CPU-specific NMI: NMI sources can be processed on any CPU.
  289. *
  290. * Another CPU may be processing panic routines while holding
  291. * nmi_reason_lock. Check if the CPU issued the IPI for crash dumping,
  292. * and if so, call its callback directly. If there is no CPU preparing
  293. * crash dump, we simply loop here.
  294. */
  295. while (!raw_spin_trylock(&nmi_reason_lock)) {
  296. run_crash_ipi_callback(regs);
  297. cpu_relax();
  298. }
  299. reason = x86_platform.get_nmi_reason();
  300. if (reason & NMI_REASON_MASK) {
  301. if (reason & NMI_REASON_SERR)
  302. pci_serr_error(reason, regs);
  303. else if (reason & NMI_REASON_IOCHK)
  304. io_check_error(reason, regs);
  305. #ifdef CONFIG_X86_32
  306. /*
  307. * Reassert NMI in case it became active
  308. * meanwhile as it's edge-triggered:
  309. */
  310. reassert_nmi();
  311. #endif
  312. __this_cpu_add(nmi_stats.external, 1);
  313. raw_spin_unlock(&nmi_reason_lock);
  314. goto out;
  315. }
  316. raw_spin_unlock(&nmi_reason_lock);
  317. /*
  318. * Only one NMI can be latched at a time. To handle
  319. * this we may process multiple nmi handlers at once to
  320. * cover the case where an NMI is dropped. The downside
  321. * to this approach is we may process an NMI prematurely,
  322. * while its real NMI is sitting latched. This will cause
  323. * an unknown NMI on the next run of the NMI processing.
  324. *
  325. * We tried to flag that condition above, by setting the
  326. * swallow_nmi flag when we process more than one event.
  327. * This condition is also only present on the second half
  328. * of a back-to-back NMI, so we flag that condition too.
  329. *
  330. * If both are true, we assume we already processed this
  331. * NMI previously and we swallow it. Otherwise we reset
  332. * the logic.
  333. *
  334. * There are scenarios where we may accidentally swallow
  335. * a 'real' unknown NMI. For example, while processing
  336. * a perf NMI another perf NMI comes in along with a
  337. * 'real' unknown NMI. These two NMIs get combined into
  338. * one (as described above). When the next NMI gets
  339. * processed, it will be flagged by perf as handled, but
  340. * no one will know that there was a 'real' unknown NMI sent
  341. * also. As a result it gets swallowed. Or if the first
  342. * perf NMI returns two events handled then the second
  343. * NMI will get eaten by the logic below, again losing a
  344. * 'real' unknown NMI. But this is the best we can do
  345. * for now.
  346. */
  347. if (b2b && __this_cpu_read(swallow_nmi))
  348. __this_cpu_add(nmi_stats.swallow, 1);
  349. else
  350. unknown_nmi_error(reason, regs);
  351. out:
  352. instrumentation_end();
  353. }
  354. /*
  355. * NMIs can page fault or hit breakpoints which will cause it to lose
  356. * its NMI context with the CPU when the breakpoint or page fault does an IRET.
  357. *
  358. * As a result, NMIs can nest if NMIs get unmasked due an IRET during
  359. * NMI processing. On x86_64, the asm glue protects us from nested NMIs
  360. * if the outer NMI came from kernel mode, but we can still nest if the
  361. * outer NMI came from user mode.
  362. *
  363. * To handle these nested NMIs, we have three states:
  364. *
  365. * 1) not running
  366. * 2) executing
  367. * 3) latched
  368. *
  369. * When no NMI is in progress, it is in the "not running" state.
  370. * When an NMI comes in, it goes into the "executing" state.
  371. * Normally, if another NMI is triggered, it does not interrupt
  372. * the running NMI and the HW will simply latch it so that when
  373. * the first NMI finishes, it will restart the second NMI.
  374. * (Note, the latch is binary, thus multiple NMIs triggering,
  375. * when one is running, are ignored. Only one NMI is restarted.)
  376. *
  377. * If an NMI executes an iret, another NMI can preempt it. We do not
  378. * want to allow this new NMI to run, but we want to execute it when the
  379. * first one finishes. We set the state to "latched", and the exit of
  380. * the first NMI will perform a dec_return, if the result is zero
  381. * (NOT_RUNNING), then it will simply exit the NMI handler. If not, the
  382. * dec_return would have set the state to NMI_EXECUTING (what we want it
  383. * to be when we are running). In this case, we simply jump back to
  384. * rerun the NMI handler again, and restart the 'latched' NMI.
  385. *
  386. * No trap (breakpoint or page fault) should be hit before nmi_restart,
  387. * thus there is no race between the first check of state for NOT_RUNNING
  388. * and setting it to NMI_EXECUTING. The HW will prevent nested NMIs
  389. * at this point.
  390. *
  391. * In case the NMI takes a page fault, we need to save off the CR2
  392. * because the NMI could have preempted another page fault and corrupt
  393. * the CR2 that is about to be read. As nested NMIs must be restarted
  394. * and they can not take breakpoints or page faults, the update of the
  395. * CR2 must be done before converting the nmi state back to NOT_RUNNING.
  396. * Otherwise, there would be a race of another nested NMI coming in
  397. * after setting state to NOT_RUNNING but before updating the nmi_cr2.
  398. */
  399. enum nmi_states {
  400. NMI_NOT_RUNNING = 0,
  401. NMI_EXECUTING,
  402. NMI_LATCHED,
  403. };
  404. static DEFINE_PER_CPU(enum nmi_states, nmi_state);
  405. static DEFINE_PER_CPU(unsigned long, nmi_cr2);
  406. static DEFINE_PER_CPU(unsigned long, nmi_dr7);
  407. DEFINE_IDTENTRY_RAW(exc_nmi)
  408. {
  409. irqentry_state_t irq_state;
  410. /*
  411. * Re-enable NMIs right here when running as an SEV-ES guest. This might
  412. * cause nested NMIs, but those can be handled safely.
  413. */
  414. sev_es_nmi_complete();
  415. if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))
  416. return;
  417. if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) {
  418. this_cpu_write(nmi_state, NMI_LATCHED);
  419. return;
  420. }
  421. this_cpu_write(nmi_state, NMI_EXECUTING);
  422. this_cpu_write(nmi_cr2, read_cr2());
  423. nmi_restart:
  424. /*
  425. * Needs to happen before DR7 is accessed, because the hypervisor can
  426. * intercept DR7 reads/writes, turning those into #VC exceptions.
  427. */
  428. sev_es_ist_enter(regs);
  429. this_cpu_write(nmi_dr7, local_db_save());
  430. irq_state = irqentry_nmi_enter(regs);
  431. inc_irq_stat(__nmi_count);
  432. if (!ignore_nmis)
  433. default_do_nmi(regs);
  434. irqentry_nmi_exit(regs, irq_state);
  435. local_db_restore(this_cpu_read(nmi_dr7));
  436. sev_es_ist_exit();
  437. if (unlikely(this_cpu_read(nmi_cr2) != read_cr2()))
  438. write_cr2(this_cpu_read(nmi_cr2));
  439. if (this_cpu_dec_return(nmi_state))
  440. goto nmi_restart;
  441. if (user_mode(regs))
  442. mds_user_clear_cpu_buffers();
  443. }
  444. #if defined(CONFIG_X86_64) && IS_ENABLED(CONFIG_KVM_INTEL)
  445. DEFINE_IDTENTRY_RAW(exc_nmi_noist)
  446. {
  447. exc_nmi(regs);
  448. }
  449. #endif
  450. #if IS_MODULE(CONFIG_KVM_INTEL)
  451. EXPORT_SYMBOL_GPL(asm_exc_nmi_noist);
  452. #endif
  453. void stop_nmi(void)
  454. {
  455. ignore_nmis++;
  456. }
  457. void restart_nmi(void)
  458. {
  459. ignore_nmis--;
  460. }
  461. /* reset the back-to-back NMI logic */
  462. void local_touch_nmi(void)
  463. {
  464. __this_cpu_write(last_nmi_rip, 0);
  465. }
  466. EXPORT_SYMBOL_GPL(local_touch_nmi);