tree_stall.h 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * RCU CPU stall warnings for normal RCU grace periods
  4. *
  5. * Copyright IBM Corporation, 2019
  6. *
  7. * Author: Paul E. McKenney <paulmck@linux.ibm.com>
  8. */
  9. #include <linux/kvm_para.h>
  10. //////////////////////////////////////////////////////////////////////////////
  11. //
  12. // Controlling CPU stall warnings, including delay calculation.
  13. /* panic() on RCU Stall sysctl. */
  14. int sysctl_panic_on_rcu_stall __read_mostly;
  15. #ifdef CONFIG_PROVE_RCU
  16. #define RCU_STALL_DELAY_DELTA (5 * HZ)
  17. #else
  18. #define RCU_STALL_DELAY_DELTA 0
  19. #endif
  20. #define RCU_STALL_MIGHT_DIV 8
  21. #define RCU_STALL_MIGHT_MIN (2 * HZ)
  22. /* Limit-check stall timeouts specified at boottime and runtime. */
  23. int rcu_jiffies_till_stall_check(void)
  24. {
  25. int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);
  26. /*
  27. * Limit check must be consistent with the Kconfig limits
  28. * for CONFIG_RCU_CPU_STALL_TIMEOUT.
  29. */
  30. if (till_stall_check < 3) {
  31. WRITE_ONCE(rcu_cpu_stall_timeout, 3);
  32. till_stall_check = 3;
  33. } else if (till_stall_check > 300) {
  34. WRITE_ONCE(rcu_cpu_stall_timeout, 300);
  35. till_stall_check = 300;
  36. }
  37. return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
  38. }
  39. EXPORT_SYMBOL_GPL(rcu_jiffies_till_stall_check);
  40. /**
  41. * rcu_gp_might_be_stalled - Is it likely that the grace period is stalled?
  42. *
  43. * Returns @true if the current grace period is sufficiently old that
  44. * it is reasonable to assume that it might be stalled. This can be
  45. * useful when deciding whether to allocate memory to enable RCU-mediated
  46. * freeing on the one hand or just invoking synchronize_rcu() on the other.
  47. * The latter is preferable when the grace period is stalled.
  48. *
  49. * Note that sampling of the .gp_start and .gp_seq fields must be done
  50. * carefully to avoid false positives at the beginnings and ends of
  51. * grace periods.
  52. */
  53. bool rcu_gp_might_be_stalled(void)
  54. {
  55. unsigned long d = rcu_jiffies_till_stall_check() / RCU_STALL_MIGHT_DIV;
  56. unsigned long j = jiffies;
  57. if (d < RCU_STALL_MIGHT_MIN)
  58. d = RCU_STALL_MIGHT_MIN;
  59. smp_mb(); // jiffies before .gp_seq to avoid false positives.
  60. if (!rcu_gp_in_progress())
  61. return false;
  62. // Long delays at this point avoids false positive, but a delay
  63. // of ULONG_MAX/4 jiffies voids your no-false-positive warranty.
  64. smp_mb(); // .gp_seq before second .gp_start
  65. // And ditto here.
  66. return !time_before(j, READ_ONCE(rcu_state.gp_start) + d);
  67. }
  68. /* Don't do RCU CPU stall warnings during long sysrq printouts. */
  69. void rcu_sysrq_start(void)
  70. {
  71. if (!rcu_cpu_stall_suppress)
  72. rcu_cpu_stall_suppress = 2;
  73. }
  74. void rcu_sysrq_end(void)
  75. {
  76. if (rcu_cpu_stall_suppress == 2)
  77. rcu_cpu_stall_suppress = 0;
  78. }
  79. /* Don't print RCU CPU stall warnings during a kernel panic. */
  80. static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
  81. {
  82. rcu_cpu_stall_suppress = 1;
  83. return NOTIFY_DONE;
  84. }
  85. static struct notifier_block rcu_panic_block = {
  86. .notifier_call = rcu_panic,
  87. };
  88. static int __init check_cpu_stall_init(void)
  89. {
  90. atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
  91. return 0;
  92. }
  93. early_initcall(check_cpu_stall_init);
  94. /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
  95. static void panic_on_rcu_stall(void)
  96. {
  97. if (sysctl_panic_on_rcu_stall)
  98. panic("RCU Stall\n");
  99. }
  100. /**
  101. * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
  102. *
  103. * Set the stall-warning timeout way off into the future, thus preventing
  104. * any RCU CPU stall-warning messages from appearing in the current set of
  105. * RCU grace periods.
  106. *
  107. * The caller must disable hard irqs.
  108. */
  109. void rcu_cpu_stall_reset(void)
  110. {
  111. WRITE_ONCE(rcu_state.jiffies_stall, jiffies + ULONG_MAX / 2);
  112. }
  113. //////////////////////////////////////////////////////////////////////////////
  114. //
  115. // Interaction with RCU grace periods
  116. /* Start of new grace period, so record stall time (and forcing times). */
  117. static void record_gp_stall_check_time(void)
  118. {
  119. unsigned long j = jiffies;
  120. unsigned long j1;
  121. WRITE_ONCE(rcu_state.gp_start, j);
  122. j1 = rcu_jiffies_till_stall_check();
  123. smp_mb(); // ->gp_start before ->jiffies_stall and caller's ->gp_seq.
  124. WRITE_ONCE(rcu_state.jiffies_stall, j + j1);
  125. rcu_state.jiffies_resched = j + j1 / 2;
  126. rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);
  127. }
  128. /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */
  129. static void zero_cpu_stall_ticks(struct rcu_data *rdp)
  130. {
  131. rdp->ticks_this_gp = 0;
  132. rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
  133. WRITE_ONCE(rdp->last_fqs_resched, jiffies);
  134. }
  135. /*
  136. * If too much time has passed in the current grace period, and if
  137. * so configured, go kick the relevant kthreads.
  138. */
  139. static void rcu_stall_kick_kthreads(void)
  140. {
  141. unsigned long j;
  142. if (!READ_ONCE(rcu_kick_kthreads))
  143. return;
  144. j = READ_ONCE(rcu_state.jiffies_kick_kthreads);
  145. if (time_after(jiffies, j) && rcu_state.gp_kthread &&
  146. (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) {
  147. WARN_ONCE(1, "Kicking %s grace-period kthread\n",
  148. rcu_state.name);
  149. rcu_ftrace_dump(DUMP_ALL);
  150. wake_up_process(rcu_state.gp_kthread);
  151. WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ);
  152. }
  153. }
  154. /*
  155. * Handler for the irq_work request posted about halfway into the RCU CPU
  156. * stall timeout, and used to detect excessive irq disabling. Set state
  157. * appropriately, but just complain if there is unexpected state on entry.
  158. */
  159. static void rcu_iw_handler(struct irq_work *iwp)
  160. {
  161. struct rcu_data *rdp;
  162. struct rcu_node *rnp;
  163. rdp = container_of(iwp, struct rcu_data, rcu_iw);
  164. rnp = rdp->mynode;
  165. raw_spin_lock_rcu_node(rnp);
  166. if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
  167. rdp->rcu_iw_gp_seq = rnp->gp_seq;
  168. rdp->rcu_iw_pending = false;
  169. }
  170. raw_spin_unlock_rcu_node(rnp);
  171. }
  172. //////////////////////////////////////////////////////////////////////////////
  173. //
  174. // Printing RCU CPU stall warnings
  175. #ifdef CONFIG_PREEMPT_RCU
  176. /*
  177. * Dump detailed information for all tasks blocking the current RCU
  178. * grace period on the specified rcu_node structure.
  179. */
  180. static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
  181. {
  182. unsigned long flags;
  183. struct task_struct *t;
  184. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  185. if (!rcu_preempt_blocked_readers_cgp(rnp)) {
  186. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  187. return;
  188. }
  189. t = list_entry(rnp->gp_tasks->prev,
  190. struct task_struct, rcu_node_entry);
  191. list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
  192. /*
  193. * We could be printing a lot while holding a spinlock.
  194. * Avoid triggering hard lockup.
  195. */
  196. touch_nmi_watchdog();
  197. sched_show_task(t);
  198. }
  199. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  200. }
  201. // Communicate task state back to the RCU CPU stall warning request.
  202. struct rcu_stall_chk_rdr {
  203. int nesting;
  204. union rcu_special rs;
  205. bool on_blkd_list;
  206. };
  207. /*
  208. * Report out the state of a not-running task that is stalling the
  209. * current RCU grace period.
  210. */
  211. static bool check_slow_task(struct task_struct *t, void *arg)
  212. {
  213. struct rcu_stall_chk_rdr *rscrp = arg;
  214. if (task_curr(t))
  215. return false; // It is running, so decline to inspect it.
  216. rscrp->nesting = t->rcu_read_lock_nesting;
  217. rscrp->rs = t->rcu_read_unlock_special;
  218. rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry);
  219. return true;
  220. }
  221. /*
  222. * Scan the current list of tasks blocked within RCU read-side critical
  223. * sections, printing out the tid of each of the first few of them.
  224. */
  225. static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
  226. __releases(rnp->lock)
  227. {
  228. int i = 0;
  229. int ndetected = 0;
  230. struct rcu_stall_chk_rdr rscr;
  231. struct task_struct *t;
  232. struct task_struct *ts[8];
  233. lockdep_assert_irqs_disabled();
  234. if (!rcu_preempt_blocked_readers_cgp(rnp)) {
  235. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  236. return 0;
  237. }
  238. pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
  239. rnp->level, rnp->grplo, rnp->grphi);
  240. t = list_entry(rnp->gp_tasks->prev,
  241. struct task_struct, rcu_node_entry);
  242. list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
  243. get_task_struct(t);
  244. ts[i++] = t;
  245. if (i >= ARRAY_SIZE(ts))
  246. break;
  247. }
  248. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  249. while (i) {
  250. t = ts[--i];
  251. if (!try_invoke_on_locked_down_task(t, check_slow_task, &rscr))
  252. pr_cont(" P%d", t->pid);
  253. else
  254. pr_cont(" P%d/%d:%c%c%c%c",
  255. t->pid, rscr.nesting,
  256. ".b"[rscr.rs.b.blocked],
  257. ".q"[rscr.rs.b.need_qs],
  258. ".e"[rscr.rs.b.exp_hint],
  259. ".l"[rscr.on_blkd_list]);
  260. lockdep_assert_irqs_disabled();
  261. put_task_struct(t);
  262. ndetected++;
  263. }
  264. pr_cont("\n");
  265. return ndetected;
  266. }
  267. #else /* #ifdef CONFIG_PREEMPT_RCU */
  268. /*
  269. * Because preemptible RCU does not exist, we never have to check for
  270. * tasks blocked within RCU read-side critical sections.
  271. */
  272. static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
  273. {
  274. }
  275. /*
  276. * Because preemptible RCU does not exist, we never have to check for
  277. * tasks blocked within RCU read-side critical sections.
  278. */
  279. static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
  280. {
  281. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  282. return 0;
  283. }
  284. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  285. /*
  286. * Dump stacks of all tasks running on stalled CPUs. First try using
  287. * NMIs, but fall back to manual remote stack tracing on architectures
  288. * that don't support NMI-based stack dumps. The NMI-triggered stack
  289. * traces are more accurate because they are printed by the target CPU.
  290. */
  291. static void rcu_dump_cpu_stacks(void)
  292. {
  293. int cpu;
  294. unsigned long flags;
  295. struct rcu_node *rnp;
  296. rcu_for_each_leaf_node(rnp) {
  297. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  298. for_each_leaf_node_possible_cpu(rnp, cpu)
  299. if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))
  300. if (!trigger_single_cpu_backtrace(cpu))
  301. dump_cpu_task(cpu);
  302. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  303. }
  304. }
  305. #ifdef CONFIG_RCU_FAST_NO_HZ
  306. static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
  307. {
  308. struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
  309. sprintf(cp, "last_accelerate: %04lx/%04lx dyntick_enabled: %d",
  310. rdp->last_accelerate & 0xffff, jiffies & 0xffff,
  311. !!rdp->tick_nohz_enabled_snap);
  312. }
  313. #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
  314. static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
  315. {
  316. *cp = '\0';
  317. }
  318. #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
  319. static const char * const gp_state_names[] = {
  320. [RCU_GP_IDLE] = "RCU_GP_IDLE",
  321. [RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS",
  322. [RCU_GP_DONE_GPS] = "RCU_GP_DONE_GPS",
  323. [RCU_GP_ONOFF] = "RCU_GP_ONOFF",
  324. [RCU_GP_INIT] = "RCU_GP_INIT",
  325. [RCU_GP_WAIT_FQS] = "RCU_GP_WAIT_FQS",
  326. [RCU_GP_DOING_FQS] = "RCU_GP_DOING_FQS",
  327. [RCU_GP_CLEANUP] = "RCU_GP_CLEANUP",
  328. [RCU_GP_CLEANED] = "RCU_GP_CLEANED",
  329. };
  330. /*
  331. * Convert a ->gp_state value to a character string.
  332. */
  333. static const char *gp_state_getname(short gs)
  334. {
  335. if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names))
  336. return "???";
  337. return gp_state_names[gs];
  338. }
  339. /* Is the RCU grace-period kthread being starved of CPU time? */
  340. static bool rcu_is_gp_kthread_starving(unsigned long *jp)
  341. {
  342. unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity);
  343. if (jp)
  344. *jp = j;
  345. return j > 2 * HZ;
  346. }
  347. /*
  348. * Print out diagnostic information for the specified stalled CPU.
  349. *
  350. * If the specified CPU is aware of the current RCU grace period, then
  351. * print the number of scheduling clock interrupts the CPU has taken
  352. * during the time that it has been aware. Otherwise, print the number
  353. * of RCU grace periods that this CPU is ignorant of, for example, "1"
  354. * if the CPU was aware of the previous grace period.
  355. *
  356. * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
  357. */
  358. static void print_cpu_stall_info(int cpu)
  359. {
  360. unsigned long delta;
  361. bool falsepositive;
  362. char fast_no_hz[72];
  363. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  364. char *ticks_title;
  365. unsigned long ticks_value;
  366. /*
  367. * We could be printing a lot while holding a spinlock. Avoid
  368. * triggering hard lockup.
  369. */
  370. touch_nmi_watchdog();
  371. ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq);
  372. if (ticks_value) {
  373. ticks_title = "GPs behind";
  374. } else {
  375. ticks_title = "ticks this GP";
  376. ticks_value = rdp->ticks_this_gp;
  377. }
  378. print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
  379. delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);
  380. falsepositive = rcu_is_gp_kthread_starving(NULL) &&
  381. rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp));
  382. pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld %s%s\n",
  383. cpu,
  384. "O."[!!cpu_online(cpu)],
  385. "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
  386. "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
  387. !IS_ENABLED(CONFIG_IRQ_WORK) ? '?' :
  388. rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' :
  389. "!."[!delta],
  390. ticks_value, ticks_title,
  391. rcu_dynticks_snap(rdp) & 0xfff,
  392. rdp->dynticks_nesting, rdp->dynticks_nmi_nesting,
  393. rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
  394. data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
  395. fast_no_hz,
  396. falsepositive ? " (false positive?)" : "");
  397. }
  398. /* Complain about starvation of grace-period kthread. */
  399. static void rcu_check_gp_kthread_starvation(void)
  400. {
  401. struct task_struct *gpk = rcu_state.gp_kthread;
  402. unsigned long j;
  403. if (rcu_is_gp_kthread_starving(&j)) {
  404. pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
  405. rcu_state.name, j,
  406. (long)rcu_seq_current(&rcu_state.gp_seq),
  407. data_race(rcu_state.gp_flags),
  408. gp_state_getname(rcu_state.gp_state), rcu_state.gp_state,
  409. gpk ? gpk->state : ~0, gpk ? task_cpu(gpk) : -1);
  410. if (gpk) {
  411. pr_err("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name);
  412. pr_err("RCU grace-period kthread stack dump:\n");
  413. sched_show_task(gpk);
  414. wake_up_process(gpk);
  415. }
  416. }
  417. }
  418. static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
  419. {
  420. int cpu;
  421. unsigned long flags;
  422. unsigned long gpa;
  423. unsigned long j;
  424. int ndetected = 0;
  425. struct rcu_node *rnp;
  426. long totqlen = 0;
  427. lockdep_assert_irqs_disabled();
  428. /* Kick and suppress, if so configured. */
  429. rcu_stall_kick_kthreads();
  430. if (rcu_stall_is_suppressed())
  431. return;
  432. /*
  433. * OK, time to rat on our buddy...
  434. * See Documentation/RCU/stallwarn.rst for info on how to debug
  435. * RCU CPU stall warnings.
  436. */
  437. trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected"));
  438. pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name);
  439. rcu_for_each_leaf_node(rnp) {
  440. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  441. if (rnp->qsmask != 0) {
  442. for_each_leaf_node_possible_cpu(rnp, cpu)
  443. if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
  444. print_cpu_stall_info(cpu);
  445. ndetected++;
  446. }
  447. }
  448. ndetected += rcu_print_task_stall(rnp, flags); // Releases rnp->lock.
  449. lockdep_assert_irqs_disabled();
  450. }
  451. for_each_possible_cpu(cpu)
  452. totqlen += rcu_get_n_cbs_cpu(cpu);
  453. pr_cont("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n",
  454. smp_processor_id(), (long)(jiffies - gps),
  455. (long)rcu_seq_current(&rcu_state.gp_seq), totqlen);
  456. if (ndetected) {
  457. rcu_dump_cpu_stacks();
  458. /* Complain about tasks blocking the grace period. */
  459. rcu_for_each_leaf_node(rnp)
  460. rcu_print_detail_task_stall_rnp(rnp);
  461. } else {
  462. if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) {
  463. pr_err("INFO: Stall ended before state dump start\n");
  464. } else {
  465. j = jiffies;
  466. gpa = data_race(rcu_state.gp_activity);
  467. pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
  468. rcu_state.name, j - gpa, j, gpa,
  469. data_race(jiffies_till_next_fqs),
  470. rcu_get_root()->qsmask);
  471. }
  472. }
  473. /* Rewrite if needed in case of slow consoles. */
  474. if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
  475. WRITE_ONCE(rcu_state.jiffies_stall,
  476. jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
  477. rcu_check_gp_kthread_starvation();
  478. panic_on_rcu_stall();
  479. rcu_force_quiescent_state(); /* Kick them all. */
  480. }
  481. static void print_cpu_stall(unsigned long gps)
  482. {
  483. int cpu;
  484. unsigned long flags;
  485. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  486. struct rcu_node *rnp = rcu_get_root();
  487. long totqlen = 0;
  488. lockdep_assert_irqs_disabled();
  489. /* Kick and suppress, if so configured. */
  490. rcu_stall_kick_kthreads();
  491. if (rcu_stall_is_suppressed())
  492. return;
  493. /*
  494. * OK, time to rat on ourselves...
  495. * See Documentation/RCU/stallwarn.rst for info on how to debug
  496. * RCU CPU stall warnings.
  497. */
  498. trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected"));
  499. pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name);
  500. raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
  501. print_cpu_stall_info(smp_processor_id());
  502. raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
  503. for_each_possible_cpu(cpu)
  504. totqlen += rcu_get_n_cbs_cpu(cpu);
  505. pr_cont("\t(t=%lu jiffies g=%ld q=%lu)\n",
  506. jiffies - gps,
  507. (long)rcu_seq_current(&rcu_state.gp_seq), totqlen);
  508. rcu_check_gp_kthread_starvation();
  509. rcu_dump_cpu_stacks();
  510. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  511. /* Rewrite if needed in case of slow consoles. */
  512. if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
  513. WRITE_ONCE(rcu_state.jiffies_stall,
  514. jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
  515. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  516. panic_on_rcu_stall();
  517. /*
  518. * Attempt to revive the RCU machinery by forcing a context switch.
  519. *
  520. * A context switch would normally allow the RCU state machine to make
  521. * progress and it could be we're stuck in kernel space without context
  522. * switches for an entirely unreasonable amount of time.
  523. */
  524. set_tsk_need_resched(current);
  525. set_preempt_need_resched();
  526. }
  527. static void check_cpu_stall(struct rcu_data *rdp)
  528. {
  529. unsigned long gs1;
  530. unsigned long gs2;
  531. unsigned long gps;
  532. unsigned long j;
  533. unsigned long jn;
  534. unsigned long js;
  535. struct rcu_node *rnp;
  536. lockdep_assert_irqs_disabled();
  537. if ((rcu_stall_is_suppressed() && !READ_ONCE(rcu_kick_kthreads)) ||
  538. !rcu_gp_in_progress())
  539. return;
  540. rcu_stall_kick_kthreads();
  541. j = jiffies;
  542. /*
  543. * Lots of memory barriers to reject false positives.
  544. *
  545. * The idea is to pick up rcu_state.gp_seq, then
  546. * rcu_state.jiffies_stall, then rcu_state.gp_start, and finally
  547. * another copy of rcu_state.gp_seq. These values are updated in
  548. * the opposite order with memory barriers (or equivalent) during
  549. * grace-period initialization and cleanup. Now, a false positive
  550. * can occur if we get an new value of rcu_state.gp_start and a old
  551. * value of rcu_state.jiffies_stall. But given the memory barriers,
  552. * the only way that this can happen is if one grace period ends
  553. * and another starts between these two fetches. This is detected
  554. * by comparing the second fetch of rcu_state.gp_seq with the
  555. * previous fetch from rcu_state.gp_seq.
  556. *
  557. * Given this check, comparisons of jiffies, rcu_state.jiffies_stall,
  558. * and rcu_state.gp_start suffice to forestall false positives.
  559. */
  560. gs1 = READ_ONCE(rcu_state.gp_seq);
  561. smp_rmb(); /* Pick up ->gp_seq first... */
  562. js = READ_ONCE(rcu_state.jiffies_stall);
  563. smp_rmb(); /* ...then ->jiffies_stall before the rest... */
  564. gps = READ_ONCE(rcu_state.gp_start);
  565. smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */
  566. gs2 = READ_ONCE(rcu_state.gp_seq);
  567. if (gs1 != gs2 ||
  568. ULONG_CMP_LT(j, js) ||
  569. ULONG_CMP_GE(gps, js))
  570. return; /* No stall or GP completed since entering function. */
  571. rnp = rdp->mynode;
  572. jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
  573. if (rcu_gp_in_progress() &&
  574. (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
  575. cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
  576. /*
  577. * If a virtual machine is stopped by the host it can look to
  578. * the watchdog like an RCU stall. Check to see if the host
  579. * stopped the vm.
  580. */
  581. if (kvm_check_and_clear_guest_paused())
  582. return;
  583. /* We haven't checked in, so go dump stack. */
  584. print_cpu_stall(gps);
  585. if (READ_ONCE(rcu_cpu_stall_ftrace_dump))
  586. rcu_ftrace_dump(DUMP_ALL);
  587. } else if (rcu_gp_in_progress() &&
  588. ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
  589. cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
  590. /*
  591. * If a virtual machine is stopped by the host it can look to
  592. * the watchdog like an RCU stall. Check to see if the host
  593. * stopped the vm.
  594. */
  595. if (kvm_check_and_clear_guest_paused())
  596. return;
  597. /* They had a few time units to dump stack, so complain. */
  598. print_other_cpu_stall(gs2, gps);
  599. if (READ_ONCE(rcu_cpu_stall_ftrace_dump))
  600. rcu_ftrace_dump(DUMP_ALL);
  601. }
  602. }
  603. //////////////////////////////////////////////////////////////////////////////
  604. //
  605. // RCU forward-progress mechanisms, including of callback invocation.
  606. /*
  607. * Show the state of the grace-period kthreads.
  608. */
  609. void show_rcu_gp_kthreads(void)
  610. {
  611. unsigned long cbs = 0;
  612. int cpu;
  613. unsigned long j;
  614. unsigned long ja;
  615. unsigned long jr;
  616. unsigned long jw;
  617. struct rcu_data *rdp;
  618. struct rcu_node *rnp;
  619. struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
  620. j = jiffies;
  621. ja = j - data_race(rcu_state.gp_activity);
  622. jr = j - data_race(rcu_state.gp_req_activity);
  623. jw = j - data_race(rcu_state.gp_wake_time);
  624. pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
  625. rcu_state.name, gp_state_getname(rcu_state.gp_state),
  626. rcu_state.gp_state, t ? t->state : 0x1ffffL,
  627. ja, jr, jw, (long)data_race(rcu_state.gp_wake_seq),
  628. (long)data_race(rcu_state.gp_seq),
  629. (long)data_race(rcu_get_root()->gp_seq_needed),
  630. data_race(rcu_state.gp_flags));
  631. rcu_for_each_node_breadth_first(rnp) {
  632. if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
  633. READ_ONCE(rnp->gp_seq_needed)))
  634. continue;
  635. pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
  636. rnp->grplo, rnp->grphi, (long)data_race(rnp->gp_seq),
  637. (long)data_race(rnp->gp_seq_needed));
  638. if (!rcu_is_leaf_node(rnp))
  639. continue;
  640. for_each_leaf_node_possible_cpu(rnp, cpu) {
  641. rdp = per_cpu_ptr(&rcu_data, cpu);
  642. if (READ_ONCE(rdp->gpwrap) ||
  643. ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
  644. READ_ONCE(rdp->gp_seq_needed)))
  645. continue;
  646. pr_info("\tcpu %d ->gp_seq_needed %ld\n",
  647. cpu, (long)data_race(rdp->gp_seq_needed));
  648. }
  649. }
  650. for_each_possible_cpu(cpu) {
  651. rdp = per_cpu_ptr(&rcu_data, cpu);
  652. cbs += data_race(rdp->n_cbs_invoked);
  653. if (rcu_segcblist_is_offloaded(&rdp->cblist))
  654. show_rcu_nocb_state(rdp);
  655. }
  656. pr_info("RCU callbacks invoked since boot: %lu\n", cbs);
  657. show_rcu_tasks_gp_kthreads();
  658. }
  659. EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
  660. /*
  661. * This function checks for grace-period requests that fail to motivate
  662. * RCU to come out of its idle mode.
  663. */
  664. static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,
  665. const unsigned long gpssdelay)
  666. {
  667. unsigned long flags;
  668. unsigned long j;
  669. struct rcu_node *rnp_root = rcu_get_root();
  670. static atomic_t warned = ATOMIC_INIT(0);
  671. if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||
  672. ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
  673. READ_ONCE(rnp_root->gp_seq_needed)) ||
  674. !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.
  675. return;
  676. j = jiffies; /* Expensive access, and in common case don't get here. */
  677. if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
  678. time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
  679. atomic_read(&warned))
  680. return;
  681. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  682. j = jiffies;
  683. if (rcu_gp_in_progress() ||
  684. ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
  685. READ_ONCE(rnp_root->gp_seq_needed)) ||
  686. time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
  687. time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
  688. atomic_read(&warned)) {
  689. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  690. return;
  691. }
  692. /* Hold onto the leaf lock to make others see warned==1. */
  693. if (rnp_root != rnp)
  694. raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
  695. j = jiffies;
  696. if (rcu_gp_in_progress() ||
  697. ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
  698. READ_ONCE(rnp_root->gp_seq_needed)) ||
  699. time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
  700. time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
  701. atomic_xchg(&warned, 1)) {
  702. if (rnp_root != rnp)
  703. /* irqs remain disabled. */
  704. raw_spin_unlock_rcu_node(rnp_root);
  705. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  706. return;
  707. }
  708. WARN_ON(1);
  709. if (rnp_root != rnp)
  710. raw_spin_unlock_rcu_node(rnp_root);
  711. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  712. show_rcu_gp_kthreads();
  713. }
  714. /*
  715. * Do a forward-progress check for rcutorture. This is normally invoked
  716. * due to an OOM event. The argument "j" gives the time period during
  717. * which rcutorture would like progress to have been made.
  718. */
  719. void rcu_fwd_progress_check(unsigned long j)
  720. {
  721. unsigned long cbs;
  722. int cpu;
  723. unsigned long max_cbs = 0;
  724. int max_cpu = -1;
  725. struct rcu_data *rdp;
  726. if (rcu_gp_in_progress()) {
  727. pr_info("%s: GP age %lu jiffies\n",
  728. __func__, jiffies - rcu_state.gp_start);
  729. show_rcu_gp_kthreads();
  730. } else {
  731. pr_info("%s: Last GP end %lu jiffies ago\n",
  732. __func__, jiffies - rcu_state.gp_end);
  733. preempt_disable();
  734. rdp = this_cpu_ptr(&rcu_data);
  735. rcu_check_gp_start_stall(rdp->mynode, rdp, j);
  736. preempt_enable();
  737. }
  738. for_each_possible_cpu(cpu) {
  739. cbs = rcu_get_n_cbs_cpu(cpu);
  740. if (!cbs)
  741. continue;
  742. if (max_cpu < 0)
  743. pr_info("%s: callbacks", __func__);
  744. pr_cont(" %d: %lu", cpu, cbs);
  745. if (cbs <= max_cbs)
  746. continue;
  747. max_cbs = cbs;
  748. max_cpu = cpu;
  749. }
  750. if (max_cpu >= 0)
  751. pr_cont("\n");
  752. }
  753. EXPORT_SYMBOL_GPL(rcu_fwd_progress_check);
  754. /* Commandeer a sysrq key to dump RCU's tree. */
  755. static bool sysrq_rcu;
  756. module_param(sysrq_rcu, bool, 0444);
  757. /* Dump grace-period-request information due to commandeered sysrq. */
  758. static void sysrq_show_rcu(int key)
  759. {
  760. show_rcu_gp_kthreads();
  761. }
  762. static const struct sysrq_key_op sysrq_rcudump_op = {
  763. .handler = sysrq_show_rcu,
  764. .help_msg = "show-rcu(y)",
  765. .action_msg = "Show RCU tree",
  766. .enable_mask = SYSRQ_ENABLE_DUMP,
  767. };
  768. static int __init rcu_sysrq_init(void)
  769. {
  770. if (sysrq_rcu)
  771. return register_sysrq_key('y', &sysrq_rcudump_op);
  772. return 0;
  773. }
  774. early_initcall(rcu_sysrq_init);