tree_plugin.h 81 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611
  1. /* SPDX-License-Identifier: GPL-2.0+ */
  2. /*
  3. * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  4. * Internal non-public definitions that provide either classic
  5. * or preemptible semantics.
  6. *
  7. * Copyright Red Hat, 2009
  8. * Copyright IBM Corporation, 2009
  9. *
  10. * Author: Ingo Molnar <mingo@elte.hu>
  11. * Paul E. McKenney <paulmck@linux.ibm.com>
  12. */
  13. #include "../locking/rtmutex_common.h"
  14. #ifdef CONFIG_RCU_NOCB_CPU
  15. static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
  16. static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
  17. #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
  18. /*
  19. * Check the RCU kernel configuration parameters and print informative
  20. * messages about anything out of the ordinary.
  21. */
  22. static void __init rcu_bootup_announce_oddness(void)
  23. {
  24. if (IS_ENABLED(CONFIG_RCU_TRACE))
  25. pr_info("\tRCU event tracing is enabled.\n");
  26. if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
  27. (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
  28. pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d.\n",
  29. RCU_FANOUT);
  30. if (rcu_fanout_exact)
  31. pr_info("\tHierarchical RCU autobalancing is disabled.\n");
  32. if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
  33. pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
  34. if (IS_ENABLED(CONFIG_PROVE_RCU))
  35. pr_info("\tRCU lockdep checking is enabled.\n");
  36. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
  37. pr_info("\tRCU strict (and thus non-scalable) grace periods enabled.\n");
  38. if (RCU_NUM_LVLS >= 4)
  39. pr_info("\tFour(or more)-level hierarchy is enabled.\n");
  40. if (RCU_FANOUT_LEAF != 16)
  41. pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
  42. RCU_FANOUT_LEAF);
  43. if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
  44. pr_info("\tBoot-time adjustment of leaf fanout to %d.\n",
  45. rcu_fanout_leaf);
  46. if (nr_cpu_ids != NR_CPUS)
  47. pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%u.\n", NR_CPUS, nr_cpu_ids);
  48. #ifdef CONFIG_RCU_BOOST
  49. pr_info("\tRCU priority boosting: priority %d delay %d ms.\n",
  50. kthread_prio, CONFIG_RCU_BOOST_DELAY);
  51. #endif
  52. if (blimit != DEFAULT_RCU_BLIMIT)
  53. pr_info("\tBoot-time adjustment of callback invocation limit to %ld.\n", blimit);
  54. if (qhimark != DEFAULT_RCU_QHIMARK)
  55. pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
  56. if (qlowmark != DEFAULT_RCU_QLOMARK)
  57. pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
  58. if (qovld != DEFAULT_RCU_QOVLD)
  59. pr_info("\tBoot-time adjustment of callback overload level to %ld.\n", qovld);
  60. if (jiffies_till_first_fqs != ULONG_MAX)
  61. pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
  62. if (jiffies_till_next_fqs != ULONG_MAX)
  63. pr_info("\tBoot-time adjustment of subsequent FQS scan delay to %ld jiffies.\n", jiffies_till_next_fqs);
  64. if (jiffies_till_sched_qs != ULONG_MAX)
  65. pr_info("\tBoot-time adjustment of scheduler-enlistment delay to %ld jiffies.\n", jiffies_till_sched_qs);
  66. if (rcu_kick_kthreads)
  67. pr_info("\tKick kthreads if too-long grace period.\n");
  68. if (IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD))
  69. pr_info("\tRCU callback double-/use-after-free debug enabled.\n");
  70. if (gp_preinit_delay)
  71. pr_info("\tRCU debug GP pre-init slowdown %d jiffies.\n", gp_preinit_delay);
  72. if (gp_init_delay)
  73. pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_init_delay);
  74. if (gp_cleanup_delay)
  75. pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_cleanup_delay);
  76. if (!use_softirq)
  77. pr_info("\tRCU_SOFTIRQ processing moved to rcuc kthreads.\n");
  78. if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG))
  79. pr_info("\tRCU debug extended QS entry/exit.\n");
  80. rcupdate_announce_bootup_oddness();
  81. }
  82. #ifdef CONFIG_PREEMPT_RCU
  83. static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
  84. static void rcu_read_unlock_special(struct task_struct *t);
  85. /*
  86. * Tell them what RCU they are running.
  87. */
  88. static void __init rcu_bootup_announce(void)
  89. {
  90. pr_info("Preemptible hierarchical RCU implementation.\n");
  91. rcu_bootup_announce_oddness();
  92. }
  93. /* Flags for rcu_preempt_ctxt_queue() decision table. */
  94. #define RCU_GP_TASKS 0x8
  95. #define RCU_EXP_TASKS 0x4
  96. #define RCU_GP_BLKD 0x2
  97. #define RCU_EXP_BLKD 0x1
  98. /*
  99. * Queues a task preempted within an RCU-preempt read-side critical
  100. * section into the appropriate location within the ->blkd_tasks list,
  101. * depending on the states of any ongoing normal and expedited grace
  102. * periods. The ->gp_tasks pointer indicates which element the normal
  103. * grace period is waiting on (NULL if none), and the ->exp_tasks pointer
  104. * indicates which element the expedited grace period is waiting on (again,
  105. * NULL if none). If a grace period is waiting on a given element in the
  106. * ->blkd_tasks list, it also waits on all subsequent elements. Thus,
  107. * adding a task to the tail of the list blocks any grace period that is
  108. * already waiting on one of the elements. In contrast, adding a task
  109. * to the head of the list won't block any grace period that is already
  110. * waiting on one of the elements.
  111. *
  112. * This queuing is imprecise, and can sometimes make an ongoing grace
  113. * period wait for a task that is not strictly speaking blocking it.
  114. * Given the choice, we needlessly block a normal grace period rather than
  115. * blocking an expedited grace period.
  116. *
  117. * Note that an endless sequence of expedited grace periods still cannot
  118. * indefinitely postpone a normal grace period. Eventually, all of the
  119. * fixed number of preempted tasks blocking the normal grace period that are
  120. * not also blocking the expedited grace period will resume and complete
  121. * their RCU read-side critical sections. At that point, the ->gp_tasks
  122. * pointer will equal the ->exp_tasks pointer, at which point the end of
  123. * the corresponding expedited grace period will also be the end of the
  124. * normal grace period.
  125. */
  126. static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
  127. __releases(rnp->lock) /* But leaves rrupts disabled. */
  128. {
  129. int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
  130. (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
  131. (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
  132. (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
  133. struct task_struct *t = current;
  134. raw_lockdep_assert_held_rcu_node(rnp);
  135. WARN_ON_ONCE(rdp->mynode != rnp);
  136. WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
  137. /* RCU better not be waiting on newly onlined CPUs! */
  138. WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask &
  139. rdp->grpmask);
  140. /*
  141. * Decide where to queue the newly blocked task. In theory,
  142. * this could be an if-statement. In practice, when I tried
  143. * that, it was quite messy.
  144. */
  145. switch (blkd_state) {
  146. case 0:
  147. case RCU_EXP_TASKS:
  148. case RCU_EXP_TASKS + RCU_GP_BLKD:
  149. case RCU_GP_TASKS:
  150. case RCU_GP_TASKS + RCU_EXP_TASKS:
  151. /*
  152. * Blocking neither GP, or first task blocking the normal
  153. * GP but not blocking the already-waiting expedited GP.
  154. * Queue at the head of the list to avoid unnecessarily
  155. * blocking the already-waiting GPs.
  156. */
  157. list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
  158. break;
  159. case RCU_EXP_BLKD:
  160. case RCU_GP_BLKD:
  161. case RCU_GP_BLKD + RCU_EXP_BLKD:
  162. case RCU_GP_TASKS + RCU_EXP_BLKD:
  163. case RCU_GP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
  164. case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
  165. /*
  166. * First task arriving that blocks either GP, or first task
  167. * arriving that blocks the expedited GP (with the normal
  168. * GP already waiting), or a task arriving that blocks
  169. * both GPs with both GPs already waiting. Queue at the
  170. * tail of the list to avoid any GP waiting on any of the
  171. * already queued tasks that are not blocking it.
  172. */
  173. list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
  174. break;
  175. case RCU_EXP_TASKS + RCU_EXP_BLKD:
  176. case RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
  177. case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_EXP_BLKD:
  178. /*
  179. * Second or subsequent task blocking the expedited GP.
  180. * The task either does not block the normal GP, or is the
  181. * first task blocking the normal GP. Queue just after
  182. * the first task blocking the expedited GP.
  183. */
  184. list_add(&t->rcu_node_entry, rnp->exp_tasks);
  185. break;
  186. case RCU_GP_TASKS + RCU_GP_BLKD:
  187. case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
  188. /*
  189. * Second or subsequent task blocking the normal GP.
  190. * The task does not block the expedited GP. Queue just
  191. * after the first task blocking the normal GP.
  192. */
  193. list_add(&t->rcu_node_entry, rnp->gp_tasks);
  194. break;
  195. default:
  196. /* Yet another exercise in excessive paranoia. */
  197. WARN_ON_ONCE(1);
  198. break;
  199. }
  200. /*
  201. * We have now queued the task. If it was the first one to
  202. * block either grace period, update the ->gp_tasks and/or
  203. * ->exp_tasks pointers, respectively, to reference the newly
  204. * blocked tasks.
  205. */
  206. if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) {
  207. WRITE_ONCE(rnp->gp_tasks, &t->rcu_node_entry);
  208. WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq);
  209. }
  210. if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
  211. WRITE_ONCE(rnp->exp_tasks, &t->rcu_node_entry);
  212. WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) !=
  213. !(rnp->qsmask & rdp->grpmask));
  214. WARN_ON_ONCE(!(blkd_state & RCU_EXP_BLKD) !=
  215. !(rnp->expmask & rdp->grpmask));
  216. raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */
  217. /*
  218. * Report the quiescent state for the expedited GP. This expedited
  219. * GP should not be able to end until we report, so there should be
  220. * no need to check for a subsequent expedited GP. (Though we are
  221. * still in a quiescent state in any case.)
  222. */
  223. if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs)
  224. rcu_report_exp_rdp(rdp);
  225. else
  226. WARN_ON_ONCE(rdp->exp_deferred_qs);
  227. }
  228. /*
  229. * Record a preemptible-RCU quiescent state for the specified CPU.
  230. * Note that this does not necessarily mean that the task currently running
  231. * on the CPU is in a quiescent state: Instead, it means that the current
  232. * grace period need not wait on any RCU read-side critical section that
  233. * starts later on this CPU. It also means that if the current task is
  234. * in an RCU read-side critical section, it has already added itself to
  235. * some leaf rcu_node structure's ->blkd_tasks list. In addition to the
  236. * current task, there might be any number of other tasks blocked while
  237. * in an RCU read-side critical section.
  238. *
  239. * Callers to this function must disable preemption.
  240. */
  241. static void rcu_qs(void)
  242. {
  243. RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n");
  244. if (__this_cpu_read(rcu_data.cpu_no_qs.s)) {
  245. trace_rcu_grace_period(TPS("rcu_preempt"),
  246. __this_cpu_read(rcu_data.gp_seq),
  247. TPS("cpuqs"));
  248. __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
  249. barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */
  250. WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false);
  251. }
  252. }
  253. /*
  254. * We have entered the scheduler, and the current task might soon be
  255. * context-switched away from. If this task is in an RCU read-side
  256. * critical section, we will no longer be able to rely on the CPU to
  257. * record that fact, so we enqueue the task on the blkd_tasks list.
  258. * The task will dequeue itself when it exits the outermost enclosing
  259. * RCU read-side critical section. Therefore, the current grace period
  260. * cannot be permitted to complete until the blkd_tasks list entries
  261. * predating the current grace period drain, in other words, until
  262. * rnp->gp_tasks becomes NULL.
  263. *
  264. * Caller must disable interrupts.
  265. */
  266. void rcu_note_context_switch(bool preempt)
  267. {
  268. struct task_struct *t = current;
  269. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  270. struct rcu_node *rnp;
  271. trace_rcu_utilization(TPS("Start context switch"));
  272. lockdep_assert_irqs_disabled();
  273. WARN_ON_ONCE(!preempt && rcu_preempt_depth() > 0);
  274. if (rcu_preempt_depth() > 0 &&
  275. !t->rcu_read_unlock_special.b.blocked) {
  276. /* Possibly blocking in an RCU read-side critical section. */
  277. rnp = rdp->mynode;
  278. raw_spin_lock_rcu_node(rnp);
  279. t->rcu_read_unlock_special.b.blocked = true;
  280. t->rcu_blocked_node = rnp;
  281. /*
  282. * Verify the CPU's sanity, trace the preemption, and
  283. * then queue the task as required based on the states
  284. * of any ongoing and expedited grace periods.
  285. */
  286. WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
  287. WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
  288. trace_rcu_preempt_task(rcu_state.name,
  289. t->pid,
  290. (rnp->qsmask & rdp->grpmask)
  291. ? rnp->gp_seq
  292. : rcu_seq_snap(&rnp->gp_seq));
  293. rcu_preempt_ctxt_queue(rnp, rdp);
  294. } else {
  295. rcu_preempt_deferred_qs(t);
  296. }
  297. /*
  298. * Either we were not in an RCU read-side critical section to
  299. * begin with, or we have now recorded that critical section
  300. * globally. Either way, we can now note a quiescent state
  301. * for this CPU. Again, if we were in an RCU read-side critical
  302. * section, and if that critical section was blocking the current
  303. * grace period, then the fact that the task has been enqueued
  304. * means that we continue to block the current grace period.
  305. */
  306. rcu_qs();
  307. if (rdp->exp_deferred_qs)
  308. rcu_report_exp_rdp(rdp);
  309. rcu_tasks_qs(current, preempt);
  310. trace_rcu_utilization(TPS("End context switch"));
  311. }
  312. EXPORT_SYMBOL_GPL(rcu_note_context_switch);
  313. /*
  314. * Check for preempted RCU readers blocking the current grace period
  315. * for the specified rcu_node structure. If the caller needs a reliable
  316. * answer, it must hold the rcu_node's ->lock.
  317. */
  318. static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
  319. {
  320. return READ_ONCE(rnp->gp_tasks) != NULL;
  321. }
  322. /* limit value for ->rcu_read_lock_nesting. */
  323. #define RCU_NEST_PMAX (INT_MAX / 2)
  324. static void rcu_preempt_read_enter(void)
  325. {
  326. current->rcu_read_lock_nesting++;
  327. }
  328. static int rcu_preempt_read_exit(void)
  329. {
  330. return --current->rcu_read_lock_nesting;
  331. }
  332. static void rcu_preempt_depth_set(int val)
  333. {
  334. current->rcu_read_lock_nesting = val;
  335. }
  336. /*
  337. * Preemptible RCU implementation for rcu_read_lock().
  338. * Just increment ->rcu_read_lock_nesting, shared state will be updated
  339. * if we block.
  340. */
  341. void __rcu_read_lock(void)
  342. {
  343. rcu_preempt_read_enter();
  344. if (IS_ENABLED(CONFIG_PROVE_LOCKING))
  345. WARN_ON_ONCE(rcu_preempt_depth() > RCU_NEST_PMAX);
  346. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && rcu_state.gp_kthread)
  347. WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, true);
  348. barrier(); /* critical section after entry code. */
  349. }
  350. EXPORT_SYMBOL_GPL(__rcu_read_lock);
  351. /*
  352. * Preemptible RCU implementation for rcu_read_unlock().
  353. * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
  354. * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
  355. * invoke rcu_read_unlock_special() to clean up after a context switch
  356. * in an RCU read-side critical section and other special cases.
  357. */
  358. void __rcu_read_unlock(void)
  359. {
  360. struct task_struct *t = current;
  361. if (rcu_preempt_read_exit() == 0) {
  362. barrier(); /* critical section before exit code. */
  363. if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s)))
  364. rcu_read_unlock_special(t);
  365. }
  366. if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
  367. int rrln = rcu_preempt_depth();
  368. WARN_ON_ONCE(rrln < 0 || rrln > RCU_NEST_PMAX);
  369. }
  370. }
  371. EXPORT_SYMBOL_GPL(__rcu_read_unlock);
  372. /*
  373. * Advance a ->blkd_tasks-list pointer to the next entry, instead
  374. * returning NULL if at the end of the list.
  375. */
  376. static struct list_head *rcu_next_node_entry(struct task_struct *t,
  377. struct rcu_node *rnp)
  378. {
  379. struct list_head *np;
  380. np = t->rcu_node_entry.next;
  381. if (np == &rnp->blkd_tasks)
  382. np = NULL;
  383. return np;
  384. }
  385. /*
  386. * Return true if the specified rcu_node structure has tasks that were
  387. * preempted within an RCU read-side critical section.
  388. */
  389. static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
  390. {
  391. return !list_empty(&rnp->blkd_tasks);
  392. }
  393. /*
  394. * Report deferred quiescent states. The deferral time can
  395. * be quite short, for example, in the case of the call from
  396. * rcu_read_unlock_special().
  397. */
  398. static void
  399. rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)
  400. {
  401. bool empty_exp;
  402. bool empty_norm;
  403. bool empty_exp_now;
  404. struct list_head *np;
  405. bool drop_boost_mutex = false;
  406. struct rcu_data *rdp;
  407. struct rcu_node *rnp;
  408. union rcu_special special;
  409. /*
  410. * If RCU core is waiting for this CPU to exit its critical section,
  411. * report the fact that it has exited. Because irqs are disabled,
  412. * t->rcu_read_unlock_special cannot change.
  413. */
  414. special = t->rcu_read_unlock_special;
  415. rdp = this_cpu_ptr(&rcu_data);
  416. if (!special.s && !rdp->exp_deferred_qs) {
  417. local_irq_restore(flags);
  418. return;
  419. }
  420. t->rcu_read_unlock_special.s = 0;
  421. if (special.b.need_qs) {
  422. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
  423. rcu_report_qs_rdp(rdp);
  424. udelay(rcu_unlock_delay);
  425. } else {
  426. rcu_qs();
  427. }
  428. }
  429. /*
  430. * Respond to a request by an expedited grace period for a
  431. * quiescent state from this CPU. Note that requests from
  432. * tasks are handled when removing the task from the
  433. * blocked-tasks list below.
  434. */
  435. if (rdp->exp_deferred_qs)
  436. rcu_report_exp_rdp(rdp);
  437. /* Clean up if blocked during RCU read-side critical section. */
  438. if (special.b.blocked) {
  439. /*
  440. * Remove this task from the list it blocked on. The task
  441. * now remains queued on the rcu_node corresponding to the
  442. * CPU it first blocked on, so there is no longer any need
  443. * to loop. Retain a WARN_ON_ONCE() out of sheer paranoia.
  444. */
  445. rnp = t->rcu_blocked_node;
  446. raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
  447. WARN_ON_ONCE(rnp != t->rcu_blocked_node);
  448. WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
  449. empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
  450. WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq &&
  451. (!empty_norm || rnp->qsmask));
  452. empty_exp = sync_rcu_exp_done(rnp);
  453. smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
  454. np = rcu_next_node_entry(t, rnp);
  455. list_del_init(&t->rcu_node_entry);
  456. t->rcu_blocked_node = NULL;
  457. trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
  458. rnp->gp_seq, t->pid);
  459. if (&t->rcu_node_entry == rnp->gp_tasks)
  460. WRITE_ONCE(rnp->gp_tasks, np);
  461. if (&t->rcu_node_entry == rnp->exp_tasks)
  462. WRITE_ONCE(rnp->exp_tasks, np);
  463. if (IS_ENABLED(CONFIG_RCU_BOOST)) {
  464. /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
  465. drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
  466. if (&t->rcu_node_entry == rnp->boost_tasks)
  467. WRITE_ONCE(rnp->boost_tasks, np);
  468. }
  469. /*
  470. * If this was the last task on the current list, and if
  471. * we aren't waiting on any CPUs, report the quiescent state.
  472. * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
  473. * so we must take a snapshot of the expedited state.
  474. */
  475. empty_exp_now = sync_rcu_exp_done(rnp);
  476. if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
  477. trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
  478. rnp->gp_seq,
  479. 0, rnp->qsmask,
  480. rnp->level,
  481. rnp->grplo,
  482. rnp->grphi,
  483. !!rnp->gp_tasks);
  484. rcu_report_unblock_qs_rnp(rnp, flags);
  485. } else {
  486. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  487. }
  488. /*
  489. * If this was the last task on the expedited lists,
  490. * then we need to report up the rcu_node hierarchy.
  491. */
  492. if (!empty_exp && empty_exp_now)
  493. rcu_report_exp_rnp(rnp, true);
  494. /* Unboost if we were boosted. */
  495. if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
  496. rt_mutex_futex_unlock(&rnp->boost_mtx);
  497. } else {
  498. local_irq_restore(flags);
  499. }
  500. }
  501. /*
  502. * Is a deferred quiescent-state pending, and are we also not in
  503. * an RCU read-side critical section? It is the caller's responsibility
  504. * to ensure it is otherwise safe to report any deferred quiescent
  505. * states. The reason for this is that it is safe to report a
  506. * quiescent state during context switch even though preemption
  507. * is disabled. This function cannot be expected to understand these
  508. * nuances, so the caller must handle them.
  509. */
  510. static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
  511. {
  512. return (__this_cpu_read(rcu_data.exp_deferred_qs) ||
  513. READ_ONCE(t->rcu_read_unlock_special.s)) &&
  514. rcu_preempt_depth() == 0;
  515. }
  516. /*
  517. * Report a deferred quiescent state if needed and safe to do so.
  518. * As with rcu_preempt_need_deferred_qs(), "safe" involves only
  519. * not being in an RCU read-side critical section. The caller must
  520. * evaluate safety in terms of interrupt, softirq, and preemption
  521. * disabling.
  522. */
  523. static void rcu_preempt_deferred_qs(struct task_struct *t)
  524. {
  525. unsigned long flags;
  526. if (!rcu_preempt_need_deferred_qs(t))
  527. return;
  528. local_irq_save(flags);
  529. rcu_preempt_deferred_qs_irqrestore(t, flags);
  530. }
  531. /*
  532. * Minimal handler to give the scheduler a chance to re-evaluate.
  533. */
  534. static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp)
  535. {
  536. struct rcu_data *rdp;
  537. rdp = container_of(iwp, struct rcu_data, defer_qs_iw);
  538. rdp->defer_qs_iw_pending = false;
  539. }
  540. /*
  541. * Handle special cases during rcu_read_unlock(), such as needing to
  542. * notify RCU core processing or task having blocked during the RCU
  543. * read-side critical section.
  544. */
  545. static void rcu_read_unlock_special(struct task_struct *t)
  546. {
  547. unsigned long flags;
  548. bool preempt_bh_were_disabled =
  549. !!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK));
  550. bool irqs_were_disabled;
  551. /* NMI handlers cannot block and cannot safely manipulate state. */
  552. if (in_nmi())
  553. return;
  554. local_irq_save(flags);
  555. irqs_were_disabled = irqs_disabled_flags(flags);
  556. if (preempt_bh_were_disabled || irqs_were_disabled) {
  557. bool exp;
  558. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  559. struct rcu_node *rnp = rdp->mynode;
  560. exp = (t->rcu_blocked_node &&
  561. READ_ONCE(t->rcu_blocked_node->exp_tasks)) ||
  562. (rdp->grpmask & READ_ONCE(rnp->expmask));
  563. // Need to defer quiescent state until everything is enabled.
  564. if (use_softirq && (in_irq() || (exp && !irqs_were_disabled))) {
  565. // Using softirq, safe to awaken, and either the
  566. // wakeup is free or there is an expedited GP.
  567. raise_softirq_irqoff(RCU_SOFTIRQ);
  568. } else {
  569. // Enabling BH or preempt does reschedule, so...
  570. // Also if no expediting, slow is OK.
  571. // Plus nohz_full CPUs eventually get tick enabled.
  572. set_tsk_need_resched(current);
  573. set_preempt_need_resched();
  574. if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled &&
  575. !rdp->defer_qs_iw_pending && exp && cpu_online(rdp->cpu)) {
  576. // Get scheduler to re-evaluate and call hooks.
  577. // If !IRQ_WORK, FQS scan will eventually IPI.
  578. init_irq_work(&rdp->defer_qs_iw,
  579. rcu_preempt_deferred_qs_handler);
  580. rdp->defer_qs_iw_pending = true;
  581. irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);
  582. }
  583. }
  584. local_irq_restore(flags);
  585. return;
  586. }
  587. rcu_preempt_deferred_qs_irqrestore(t, flags);
  588. }
  589. /*
  590. * Check that the list of blocked tasks for the newly completed grace
  591. * period is in fact empty. It is a serious bug to complete a grace
  592. * period that still has RCU readers blocked! This function must be
  593. * invoked -before- updating this rnp's ->gp_seq.
  594. *
  595. * Also, if there are blocked tasks on the list, they automatically
  596. * block the newly created grace period, so set up ->gp_tasks accordingly.
  597. */
  598. static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
  599. {
  600. struct task_struct *t;
  601. RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n");
  602. raw_lockdep_assert_held_rcu_node(rnp);
  603. if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
  604. dump_blkd_tasks(rnp, 10);
  605. if (rcu_preempt_has_tasks(rnp) &&
  606. (rnp->qsmaskinit || rnp->wait_blkd_tasks)) {
  607. WRITE_ONCE(rnp->gp_tasks, rnp->blkd_tasks.next);
  608. t = container_of(rnp->gp_tasks, struct task_struct,
  609. rcu_node_entry);
  610. trace_rcu_unlock_preempted_task(TPS("rcu_preempt-GPS"),
  611. rnp->gp_seq, t->pid);
  612. }
  613. WARN_ON_ONCE(rnp->qsmask);
  614. }
  615. /*
  616. * Check for a quiescent state from the current CPU, including voluntary
  617. * context switches for Tasks RCU. When a task blocks, the task is
  618. * recorded in the corresponding CPU's rcu_node structure, which is checked
  619. * elsewhere, hence this function need only check for quiescent states
  620. * related to the current CPU, not to those related to tasks.
  621. */
  622. static void rcu_flavor_sched_clock_irq(int user)
  623. {
  624. struct task_struct *t = current;
  625. lockdep_assert_irqs_disabled();
  626. if (user || rcu_is_cpu_rrupt_from_idle()) {
  627. rcu_note_voluntary_context_switch(current);
  628. }
  629. if (rcu_preempt_depth() > 0 ||
  630. (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) {
  631. /* No QS, force context switch if deferred. */
  632. if (rcu_preempt_need_deferred_qs(t)) {
  633. set_tsk_need_resched(t);
  634. set_preempt_need_resched();
  635. }
  636. } else if (rcu_preempt_need_deferred_qs(t)) {
  637. rcu_preempt_deferred_qs(t); /* Report deferred QS. */
  638. return;
  639. } else if (!WARN_ON_ONCE(rcu_preempt_depth())) {
  640. rcu_qs(); /* Report immediate QS. */
  641. return;
  642. }
  643. /* If GP is oldish, ask for help from rcu_read_unlock_special(). */
  644. if (rcu_preempt_depth() > 0 &&
  645. __this_cpu_read(rcu_data.core_needs_qs) &&
  646. __this_cpu_read(rcu_data.cpu_no_qs.b.norm) &&
  647. !t->rcu_read_unlock_special.b.need_qs &&
  648. time_after(jiffies, rcu_state.gp_start + HZ))
  649. t->rcu_read_unlock_special.b.need_qs = true;
  650. }
  651. /*
  652. * Check for a task exiting while in a preemptible-RCU read-side
  653. * critical section, clean up if so. No need to issue warnings, as
  654. * debug_check_no_locks_held() already does this if lockdep is enabled.
  655. * Besides, if this function does anything other than just immediately
  656. * return, there was a bug of some sort. Spewing warnings from this
  657. * function is like as not to simply obscure important prior warnings.
  658. */
  659. void exit_rcu(void)
  660. {
  661. struct task_struct *t = current;
  662. if (unlikely(!list_empty(&current->rcu_node_entry))) {
  663. rcu_preempt_depth_set(1);
  664. barrier();
  665. WRITE_ONCE(t->rcu_read_unlock_special.b.blocked, true);
  666. } else if (unlikely(rcu_preempt_depth())) {
  667. rcu_preempt_depth_set(1);
  668. } else {
  669. return;
  670. }
  671. __rcu_read_unlock();
  672. rcu_preempt_deferred_qs(current);
  673. }
  674. /*
  675. * Dump the blocked-tasks state, but limit the list dump to the
  676. * specified number of elements.
  677. */
  678. static void
  679. dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
  680. {
  681. int cpu;
  682. int i;
  683. struct list_head *lhp;
  684. bool onl;
  685. struct rcu_data *rdp;
  686. struct rcu_node *rnp1;
  687. raw_lockdep_assert_held_rcu_node(rnp);
  688. pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
  689. __func__, rnp->grplo, rnp->grphi, rnp->level,
  690. (long)READ_ONCE(rnp->gp_seq), (long)rnp->completedqs);
  691. for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
  692. pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",
  693. __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);
  694. pr_info("%s: ->gp_tasks %p ->boost_tasks %p ->exp_tasks %p\n",
  695. __func__, READ_ONCE(rnp->gp_tasks), data_race(rnp->boost_tasks),
  696. READ_ONCE(rnp->exp_tasks));
  697. pr_info("%s: ->blkd_tasks", __func__);
  698. i = 0;
  699. list_for_each(lhp, &rnp->blkd_tasks) {
  700. pr_cont(" %p", lhp);
  701. if (++i >= ncheck)
  702. break;
  703. }
  704. pr_cont("\n");
  705. for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) {
  706. rdp = per_cpu_ptr(&rcu_data, cpu);
  707. onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
  708. pr_info("\t%d: %c online: %ld(%d) offline: %ld(%d)\n",
  709. cpu, ".o"[onl],
  710. (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
  711. (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
  712. }
  713. }
  714. #else /* #ifdef CONFIG_PREEMPT_RCU */
  715. /*
  716. * If strict grace periods are enabled, and if the calling
  717. * __rcu_read_unlock() marks the beginning of a quiescent state, immediately
  718. * report that quiescent state and, if requested, spin for a bit.
  719. */
  720. void rcu_read_unlock_strict(void)
  721. {
  722. struct rcu_data *rdp;
  723. if (!IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ||
  724. irqs_disabled() || preempt_count() || !rcu_state.gp_kthread)
  725. return;
  726. rdp = this_cpu_ptr(&rcu_data);
  727. rcu_report_qs_rdp(rdp);
  728. udelay(rcu_unlock_delay);
  729. }
  730. EXPORT_SYMBOL_GPL(rcu_read_unlock_strict);
  731. /*
  732. * Tell them what RCU they are running.
  733. */
  734. static void __init rcu_bootup_announce(void)
  735. {
  736. pr_info("Hierarchical RCU implementation.\n");
  737. rcu_bootup_announce_oddness();
  738. }
  739. /*
  740. * Note a quiescent state for PREEMPTION=n. Because we do not need to know
  741. * how many quiescent states passed, just if there was at least one since
  742. * the start of the grace period, this just sets a flag. The caller must
  743. * have disabled preemption.
  744. */
  745. static void rcu_qs(void)
  746. {
  747. RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!");
  748. if (!__this_cpu_read(rcu_data.cpu_no_qs.s))
  749. return;
  750. trace_rcu_grace_period(TPS("rcu_sched"),
  751. __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs"));
  752. __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
  753. if (!__this_cpu_read(rcu_data.cpu_no_qs.b.exp))
  754. return;
  755. __this_cpu_write(rcu_data.cpu_no_qs.b.exp, false);
  756. rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
  757. }
  758. /*
  759. * Register an urgently needed quiescent state. If there is an
  760. * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
  761. * dyntick-idle quiescent state visible to other CPUs, which will in
  762. * some cases serve for expedited as well as normal grace periods.
  763. * Either way, register a lightweight quiescent state.
  764. */
  765. void rcu_all_qs(void)
  766. {
  767. unsigned long flags;
  768. if (!raw_cpu_read(rcu_data.rcu_urgent_qs))
  769. return;
  770. preempt_disable();
  771. /* Load rcu_urgent_qs before other flags. */
  772. if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
  773. preempt_enable();
  774. return;
  775. }
  776. this_cpu_write(rcu_data.rcu_urgent_qs, false);
  777. if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) {
  778. local_irq_save(flags);
  779. rcu_momentary_dyntick_idle();
  780. local_irq_restore(flags);
  781. }
  782. rcu_qs();
  783. preempt_enable();
  784. }
  785. EXPORT_SYMBOL_GPL(rcu_all_qs);
  786. /*
  787. * Note a PREEMPTION=n context switch. The caller must have disabled interrupts.
  788. */
  789. void rcu_note_context_switch(bool preempt)
  790. {
  791. trace_rcu_utilization(TPS("Start context switch"));
  792. rcu_qs();
  793. /* Load rcu_urgent_qs before other flags. */
  794. if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs)))
  795. goto out;
  796. this_cpu_write(rcu_data.rcu_urgent_qs, false);
  797. if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs)))
  798. rcu_momentary_dyntick_idle();
  799. rcu_tasks_qs(current, preempt);
  800. out:
  801. trace_rcu_utilization(TPS("End context switch"));
  802. }
  803. EXPORT_SYMBOL_GPL(rcu_note_context_switch);
  804. /*
  805. * Because preemptible RCU does not exist, there are never any preempted
  806. * RCU readers.
  807. */
  808. static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
  809. {
  810. return 0;
  811. }
  812. /*
  813. * Because there is no preemptible RCU, there can be no readers blocked.
  814. */
  815. static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
  816. {
  817. return false;
  818. }
  819. /*
  820. * Because there is no preemptible RCU, there can be no deferred quiescent
  821. * states.
  822. */
  823. static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
  824. {
  825. return false;
  826. }
  827. static void rcu_preempt_deferred_qs(struct task_struct *t) { }
  828. /*
  829. * Because there is no preemptible RCU, there can be no readers blocked,
  830. * so there is no need to check for blocked tasks. So check only for
  831. * bogus qsmask values.
  832. */
  833. static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
  834. {
  835. WARN_ON_ONCE(rnp->qsmask);
  836. }
  837. /*
  838. * Check to see if this CPU is in a non-context-switch quiescent state,
  839. * namely user mode and idle loop.
  840. */
  841. static void rcu_flavor_sched_clock_irq(int user)
  842. {
  843. if (user || rcu_is_cpu_rrupt_from_idle()) {
  844. /*
  845. * Get here if this CPU took its interrupt from user
  846. * mode or from the idle loop, and if this is not a
  847. * nested interrupt. In this case, the CPU is in
  848. * a quiescent state, so note it.
  849. *
  850. * No memory barrier is required here because rcu_qs()
  851. * references only CPU-local variables that other CPUs
  852. * neither access nor modify, at least not while the
  853. * corresponding CPU is online.
  854. */
  855. rcu_qs();
  856. }
  857. }
  858. /*
  859. * Because preemptible RCU does not exist, tasks cannot possibly exit
  860. * while in preemptible RCU read-side critical sections.
  861. */
  862. void exit_rcu(void)
  863. {
  864. }
  865. /*
  866. * Dump the guaranteed-empty blocked-tasks state. Trust but verify.
  867. */
  868. static void
  869. dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
  870. {
  871. WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks));
  872. }
  873. #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
  874. /*
  875. * If boosting, set rcuc kthreads to realtime priority.
  876. */
  877. static void rcu_cpu_kthread_setup(unsigned int cpu)
  878. {
  879. #ifdef CONFIG_RCU_BOOST
  880. struct sched_param sp;
  881. sp.sched_priority = kthread_prio;
  882. sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
  883. #endif /* #ifdef CONFIG_RCU_BOOST */
  884. }
  885. #ifdef CONFIG_RCU_BOOST
  886. /*
  887. * Carry out RCU priority boosting on the task indicated by ->exp_tasks
  888. * or ->boost_tasks, advancing the pointer to the next task in the
  889. * ->blkd_tasks list.
  890. *
  891. * Note that irqs must be enabled: boosting the task can block.
  892. * Returns 1 if there are more tasks needing to be boosted.
  893. */
  894. static int rcu_boost(struct rcu_node *rnp)
  895. {
  896. unsigned long flags;
  897. struct task_struct *t;
  898. struct list_head *tb;
  899. if (READ_ONCE(rnp->exp_tasks) == NULL &&
  900. READ_ONCE(rnp->boost_tasks) == NULL)
  901. return 0; /* Nothing left to boost. */
  902. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  903. /*
  904. * Recheck under the lock: all tasks in need of boosting
  905. * might exit their RCU read-side critical sections on their own.
  906. */
  907. if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
  908. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  909. return 0;
  910. }
  911. /*
  912. * Preferentially boost tasks blocking expedited grace periods.
  913. * This cannot starve the normal grace periods because a second
  914. * expedited grace period must boost all blocked tasks, including
  915. * those blocking the pre-existing normal grace period.
  916. */
  917. if (rnp->exp_tasks != NULL)
  918. tb = rnp->exp_tasks;
  919. else
  920. tb = rnp->boost_tasks;
  921. /*
  922. * We boost task t by manufacturing an rt_mutex that appears to
  923. * be held by task t. We leave a pointer to that rt_mutex where
  924. * task t can find it, and task t will release the mutex when it
  925. * exits its outermost RCU read-side critical section. Then
  926. * simply acquiring this artificial rt_mutex will boost task
  927. * t's priority. (Thanks to tglx for suggesting this approach!)
  928. *
  929. * Note that task t must acquire rnp->lock to remove itself from
  930. * the ->blkd_tasks list, which it will do from exit() if from
  931. * nowhere else. We therefore are guaranteed that task t will
  932. * stay around at least until we drop rnp->lock. Note that
  933. * rnp->lock also resolves races between our priority boosting
  934. * and task t's exiting its outermost RCU read-side critical
  935. * section.
  936. */
  937. t = container_of(tb, struct task_struct, rcu_node_entry);
  938. rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
  939. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  940. /* Lock only for side effect: boosts task t's priority. */
  941. rt_mutex_lock(&rnp->boost_mtx);
  942. rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */
  943. return READ_ONCE(rnp->exp_tasks) != NULL ||
  944. READ_ONCE(rnp->boost_tasks) != NULL;
  945. }
  946. /*
  947. * Priority-boosting kthread, one per leaf rcu_node.
  948. */
  949. static int rcu_boost_kthread(void *arg)
  950. {
  951. struct rcu_node *rnp = (struct rcu_node *)arg;
  952. int spincnt = 0;
  953. int more2boost;
  954. trace_rcu_utilization(TPS("Start boost kthread@init"));
  955. for (;;) {
  956. WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_WAITING);
  957. trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
  958. rcu_wait(READ_ONCE(rnp->boost_tasks) ||
  959. READ_ONCE(rnp->exp_tasks));
  960. trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
  961. WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_RUNNING);
  962. more2boost = rcu_boost(rnp);
  963. if (more2boost)
  964. spincnt++;
  965. else
  966. spincnt = 0;
  967. if (spincnt > 10) {
  968. WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_YIELDING);
  969. trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
  970. schedule_timeout_idle(2);
  971. trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
  972. spincnt = 0;
  973. }
  974. }
  975. /* NOTREACHED */
  976. trace_rcu_utilization(TPS("End boost kthread@notreached"));
  977. return 0;
  978. }
  979. /*
  980. * Check to see if it is time to start boosting RCU readers that are
  981. * blocking the current grace period, and, if so, tell the per-rcu_node
  982. * kthread to start boosting them. If there is an expedited grace
  983. * period in progress, it is always time to boost.
  984. *
  985. * The caller must hold rnp->lock, which this function releases.
  986. * The ->boost_kthread_task is immortal, so we don't need to worry
  987. * about it going away.
  988. */
  989. static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
  990. __releases(rnp->lock)
  991. {
  992. raw_lockdep_assert_held_rcu_node(rnp);
  993. if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
  994. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  995. return;
  996. }
  997. if (rnp->exp_tasks != NULL ||
  998. (rnp->gp_tasks != NULL &&
  999. rnp->boost_tasks == NULL &&
  1000. rnp->qsmask == 0 &&
  1001. (!time_after(rnp->boost_time, jiffies) || rcu_state.cbovld))) {
  1002. if (rnp->exp_tasks == NULL)
  1003. WRITE_ONCE(rnp->boost_tasks, rnp->gp_tasks);
  1004. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1005. rcu_wake_cond(rnp->boost_kthread_task,
  1006. READ_ONCE(rnp->boost_kthread_status));
  1007. } else {
  1008. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1009. }
  1010. }
  1011. /*
  1012. * Is the current CPU running the RCU-callbacks kthread?
  1013. * Caller must have preemption disabled.
  1014. */
  1015. static bool rcu_is_callbacks_kthread(void)
  1016. {
  1017. return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current;
  1018. }
  1019. #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
  1020. /*
  1021. * Do priority-boost accounting for the start of a new grace period.
  1022. */
  1023. static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
  1024. {
  1025. rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
  1026. }
  1027. /*
  1028. * Create an RCU-boost kthread for the specified node if one does not
  1029. * already exist. We only create this kthread for preemptible RCU.
  1030. * Returns zero if all is well, a negated errno otherwise.
  1031. */
  1032. static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)
  1033. {
  1034. int rnp_index = rnp - rcu_get_root();
  1035. unsigned long flags;
  1036. struct sched_param sp;
  1037. struct task_struct *t;
  1038. if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
  1039. return;
  1040. if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
  1041. return;
  1042. rcu_state.boost = 1;
  1043. if (rnp->boost_kthread_task != NULL)
  1044. return;
  1045. t = kthread_create(rcu_boost_kthread, (void *)rnp,
  1046. "rcub/%d", rnp_index);
  1047. if (WARN_ON_ONCE(IS_ERR(t)))
  1048. return;
  1049. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  1050. rnp->boost_kthread_task = t;
  1051. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1052. sp.sched_priority = kthread_prio;
  1053. sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
  1054. wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
  1055. }
  1056. /*
  1057. * Set the per-rcu_node kthread's affinity to cover all CPUs that are
  1058. * served by the rcu_node in question. The CPU hotplug lock is still
  1059. * held, so the value of rnp->qsmaskinit will be stable.
  1060. *
  1061. * We don't include outgoingcpu in the affinity set, use -1 if there is
  1062. * no outgoing CPU. If there are no CPUs left in the affinity set,
  1063. * this function allows the kthread to execute on any CPU.
  1064. */
  1065. static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
  1066. {
  1067. struct task_struct *t = rnp->boost_kthread_task;
  1068. unsigned long mask = rcu_rnp_online_cpus(rnp);
  1069. cpumask_var_t cm;
  1070. int cpu;
  1071. if (!t)
  1072. return;
  1073. if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
  1074. return;
  1075. for_each_leaf_node_possible_cpu(rnp, cpu)
  1076. if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&
  1077. cpu != outgoingcpu)
  1078. cpumask_set_cpu(cpu, cm);
  1079. if (cpumask_weight(cm) == 0)
  1080. cpumask_setall(cm);
  1081. set_cpus_allowed_ptr(t, cm);
  1082. free_cpumask_var(cm);
  1083. }
  1084. /*
  1085. * Spawn boost kthreads -- called as soon as the scheduler is running.
  1086. */
  1087. static void __init rcu_spawn_boost_kthreads(void)
  1088. {
  1089. struct rcu_node *rnp;
  1090. rcu_for_each_leaf_node(rnp)
  1091. rcu_spawn_one_boost_kthread(rnp);
  1092. }
  1093. static void rcu_prepare_kthreads(int cpu)
  1094. {
  1095. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  1096. struct rcu_node *rnp = rdp->mynode;
  1097. /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
  1098. if (rcu_scheduler_fully_active)
  1099. rcu_spawn_one_boost_kthread(rnp);
  1100. }
  1101. #else /* #ifdef CONFIG_RCU_BOOST */
  1102. static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
  1103. __releases(rnp->lock)
  1104. {
  1105. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1106. }
  1107. static bool rcu_is_callbacks_kthread(void)
  1108. {
  1109. return false;
  1110. }
  1111. static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
  1112. {
  1113. }
  1114. static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
  1115. {
  1116. }
  1117. static void __init rcu_spawn_boost_kthreads(void)
  1118. {
  1119. }
  1120. static void rcu_prepare_kthreads(int cpu)
  1121. {
  1122. }
  1123. #endif /* #else #ifdef CONFIG_RCU_BOOST */
  1124. #if !defined(CONFIG_RCU_FAST_NO_HZ)
  1125. /*
  1126. * Check to see if any future non-offloaded RCU-related work will need
  1127. * to be done by the current CPU, even if none need be done immediately,
  1128. * returning 1 if so. This function is part of the RCU implementation;
  1129. * it is -not- an exported member of the RCU API.
  1130. *
  1131. * Because we not have RCU_FAST_NO_HZ, just check whether or not this
  1132. * CPU has RCU callbacks queued.
  1133. */
  1134. int rcu_needs_cpu(u64 basemono, u64 *nextevt)
  1135. {
  1136. *nextevt = KTIME_MAX;
  1137. return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) &&
  1138. !rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist);
  1139. }
  1140. /*
  1141. * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
  1142. * after it.
  1143. */
  1144. static void rcu_cleanup_after_idle(void)
  1145. {
  1146. }
  1147. /*
  1148. * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
  1149. * is nothing.
  1150. */
  1151. static void rcu_prepare_for_idle(void)
  1152. {
  1153. }
  1154. #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
  1155. /*
  1156. * This code is invoked when a CPU goes idle, at which point we want
  1157. * to have the CPU do everything required for RCU so that it can enter
  1158. * the energy-efficient dyntick-idle mode.
  1159. *
  1160. * The following preprocessor symbol controls this:
  1161. *
  1162. * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
  1163. * to sleep in dyntick-idle mode with RCU callbacks pending. This
  1164. * is sized to be roughly one RCU grace period. Those energy-efficiency
  1165. * benchmarkers who might otherwise be tempted to set this to a large
  1166. * number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
  1167. * system. And if you are -that- concerned about energy efficiency,
  1168. * just power the system down and be done with it!
  1169. *
  1170. * The value below works well in practice. If future workloads require
  1171. * adjustment, they can be converted into kernel config parameters, though
  1172. * making the state machine smarter might be a better option.
  1173. */
  1174. #define RCU_IDLE_GP_DELAY 4 /* Roughly one grace period. */
  1175. static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
  1176. module_param(rcu_idle_gp_delay, int, 0644);
  1177. /*
  1178. * Try to advance callbacks on the current CPU, but only if it has been
  1179. * awhile since the last time we did so. Afterwards, if there are any
  1180. * callbacks ready for immediate invocation, return true.
  1181. */
  1182. static bool __maybe_unused rcu_try_advance_all_cbs(void)
  1183. {
  1184. bool cbs_ready = false;
  1185. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  1186. struct rcu_node *rnp;
  1187. /* Exit early if we advanced recently. */
  1188. if (jiffies == rdp->last_advance_all)
  1189. return false;
  1190. rdp->last_advance_all = jiffies;
  1191. rnp = rdp->mynode;
  1192. /*
  1193. * Don't bother checking unless a grace period has
  1194. * completed since we last checked and there are
  1195. * callbacks not yet ready to invoke.
  1196. */
  1197. if ((rcu_seq_completed_gp(rdp->gp_seq,
  1198. rcu_seq_current(&rnp->gp_seq)) ||
  1199. unlikely(READ_ONCE(rdp->gpwrap))) &&
  1200. rcu_segcblist_pend_cbs(&rdp->cblist))
  1201. note_gp_changes(rdp);
  1202. if (rcu_segcblist_ready_cbs(&rdp->cblist))
  1203. cbs_ready = true;
  1204. return cbs_ready;
  1205. }
  1206. /*
  1207. * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
  1208. * to invoke. If the CPU has callbacks, try to advance them. Tell the
  1209. * caller about what to set the timeout.
  1210. *
  1211. * The caller must have disabled interrupts.
  1212. */
  1213. int rcu_needs_cpu(u64 basemono, u64 *nextevt)
  1214. {
  1215. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  1216. unsigned long dj;
  1217. lockdep_assert_irqs_disabled();
  1218. /* If no non-offloaded callbacks, RCU doesn't need the CPU. */
  1219. if (rcu_segcblist_empty(&rdp->cblist) ||
  1220. rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist)) {
  1221. *nextevt = KTIME_MAX;
  1222. return 0;
  1223. }
  1224. /* Attempt to advance callbacks. */
  1225. if (rcu_try_advance_all_cbs()) {
  1226. /* Some ready to invoke, so initiate later invocation. */
  1227. invoke_rcu_core();
  1228. return 1;
  1229. }
  1230. rdp->last_accelerate = jiffies;
  1231. /* Request timer and round. */
  1232. dj = round_up(rcu_idle_gp_delay + jiffies, rcu_idle_gp_delay) - jiffies;
  1233. *nextevt = basemono + dj * TICK_NSEC;
  1234. return 0;
  1235. }
  1236. /*
  1237. * Prepare a CPU for idle from an RCU perspective. The first major task is to
  1238. * sense whether nohz mode has been enabled or disabled via sysfs. The second
  1239. * major task is to accelerate (that is, assign grace-period numbers to) any
  1240. * recently arrived callbacks.
  1241. *
  1242. * The caller must have disabled interrupts.
  1243. */
  1244. static void rcu_prepare_for_idle(void)
  1245. {
  1246. bool needwake;
  1247. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  1248. struct rcu_node *rnp;
  1249. int tne;
  1250. lockdep_assert_irqs_disabled();
  1251. if (rcu_segcblist_is_offloaded(&rdp->cblist))
  1252. return;
  1253. /* Handle nohz enablement switches conservatively. */
  1254. tne = READ_ONCE(tick_nohz_active);
  1255. if (tne != rdp->tick_nohz_enabled_snap) {
  1256. if (!rcu_segcblist_empty(&rdp->cblist))
  1257. invoke_rcu_core(); /* force nohz to see update. */
  1258. rdp->tick_nohz_enabled_snap = tne;
  1259. return;
  1260. }
  1261. if (!tne)
  1262. return;
  1263. /*
  1264. * If we have not yet accelerated this jiffy, accelerate all
  1265. * callbacks on this CPU.
  1266. */
  1267. if (rdp->last_accelerate == jiffies)
  1268. return;
  1269. rdp->last_accelerate = jiffies;
  1270. if (rcu_segcblist_pend_cbs(&rdp->cblist)) {
  1271. rnp = rdp->mynode;
  1272. raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
  1273. needwake = rcu_accelerate_cbs(rnp, rdp);
  1274. raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
  1275. if (needwake)
  1276. rcu_gp_kthread_wake();
  1277. }
  1278. }
  1279. /*
  1280. * Clean up for exit from idle. Attempt to advance callbacks based on
  1281. * any grace periods that elapsed while the CPU was idle, and if any
  1282. * callbacks are now ready to invoke, initiate invocation.
  1283. */
  1284. static void rcu_cleanup_after_idle(void)
  1285. {
  1286. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  1287. lockdep_assert_irqs_disabled();
  1288. if (rcu_segcblist_is_offloaded(&rdp->cblist))
  1289. return;
  1290. if (rcu_try_advance_all_cbs())
  1291. invoke_rcu_core();
  1292. }
  1293. #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
  1294. #ifdef CONFIG_RCU_NOCB_CPU
  1295. /*
  1296. * Offload callback processing from the boot-time-specified set of CPUs
  1297. * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads
  1298. * created that pull the callbacks from the corresponding CPU, wait for
  1299. * a grace period to elapse, and invoke the callbacks. These kthreads
  1300. * are organized into GP kthreads, which manage incoming callbacks, wait for
  1301. * grace periods, and awaken CB kthreads, and the CB kthreads, which only
  1302. * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs
  1303. * do a wake_up() on their GP kthread when they insert a callback into any
  1304. * empty list, unless the rcu_nocb_poll boot parameter has been specified,
  1305. * in which case each kthread actively polls its CPU. (Which isn't so great
  1306. * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
  1307. *
  1308. * This is intended to be used in conjunction with Frederic Weisbecker's
  1309. * adaptive-idle work, which would seriously reduce OS jitter on CPUs
  1310. * running CPU-bound user-mode computations.
  1311. *
  1312. * Offloading of callbacks can also be used as an energy-efficiency
  1313. * measure because CPUs with no RCU callbacks queued are more aggressive
  1314. * about entering dyntick-idle mode.
  1315. */
  1316. /*
  1317. * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
  1318. * The string after the "rcu_nocbs=" is either "all" for all CPUs, or a
  1319. * comma-separated list of CPUs and/or CPU ranges. If an invalid list is
  1320. * given, a warning is emitted and all CPUs are offloaded.
  1321. */
  1322. static int __init rcu_nocb_setup(char *str)
  1323. {
  1324. alloc_bootmem_cpumask_var(&rcu_nocb_mask);
  1325. if (!strcasecmp(str, "all"))
  1326. cpumask_setall(rcu_nocb_mask);
  1327. else
  1328. if (cpulist_parse(str, rcu_nocb_mask)) {
  1329. pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");
  1330. cpumask_setall(rcu_nocb_mask);
  1331. }
  1332. return 1;
  1333. }
  1334. __setup("rcu_nocbs=", rcu_nocb_setup);
  1335. static int __init parse_rcu_nocb_poll(char *arg)
  1336. {
  1337. rcu_nocb_poll = true;
  1338. return 0;
  1339. }
  1340. early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
  1341. /*
  1342. * Don't bother bypassing ->cblist if the call_rcu() rate is low.
  1343. * After all, the main point of bypassing is to avoid lock contention
  1344. * on ->nocb_lock, which only can happen at high call_rcu() rates.
  1345. */
  1346. int nocb_nobypass_lim_per_jiffy = 16 * 1000 / HZ;
  1347. module_param(nocb_nobypass_lim_per_jiffy, int, 0);
  1348. /*
  1349. * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the
  1350. * lock isn't immediately available, increment ->nocb_lock_contended to
  1351. * flag the contention.
  1352. */
  1353. static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
  1354. __acquires(&rdp->nocb_bypass_lock)
  1355. {
  1356. lockdep_assert_irqs_disabled();
  1357. if (raw_spin_trylock(&rdp->nocb_bypass_lock))
  1358. return;
  1359. atomic_inc(&rdp->nocb_lock_contended);
  1360. WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
  1361. smp_mb__after_atomic(); /* atomic_inc() before lock. */
  1362. raw_spin_lock(&rdp->nocb_bypass_lock);
  1363. smp_mb__before_atomic(); /* atomic_dec() after lock. */
  1364. atomic_dec(&rdp->nocb_lock_contended);
  1365. }
  1366. /*
  1367. * Spinwait until the specified rcu_data structure's ->nocb_lock is
  1368. * not contended. Please note that this is extremely special-purpose,
  1369. * relying on the fact that at most two kthreads and one CPU contend for
  1370. * this lock, and also that the two kthreads are guaranteed to have frequent
  1371. * grace-period-duration time intervals between successive acquisitions
  1372. * of the lock. This allows us to use an extremely simple throttling
  1373. * mechanism, and further to apply it only to the CPU doing floods of
  1374. * call_rcu() invocations. Don't try this at home!
  1375. */
  1376. static void rcu_nocb_wait_contended(struct rcu_data *rdp)
  1377. {
  1378. WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
  1379. while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended)))
  1380. cpu_relax();
  1381. }
  1382. /*
  1383. * Conditionally acquire the specified rcu_data structure's
  1384. * ->nocb_bypass_lock.
  1385. */
  1386. static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)
  1387. {
  1388. lockdep_assert_irqs_disabled();
  1389. return raw_spin_trylock(&rdp->nocb_bypass_lock);
  1390. }
  1391. /*
  1392. * Release the specified rcu_data structure's ->nocb_bypass_lock.
  1393. */
  1394. static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
  1395. __releases(&rdp->nocb_bypass_lock)
  1396. {
  1397. lockdep_assert_irqs_disabled();
  1398. raw_spin_unlock(&rdp->nocb_bypass_lock);
  1399. }
  1400. /*
  1401. * Acquire the specified rcu_data structure's ->nocb_lock, but only
  1402. * if it corresponds to a no-CBs CPU.
  1403. */
  1404. static void rcu_nocb_lock(struct rcu_data *rdp)
  1405. {
  1406. lockdep_assert_irqs_disabled();
  1407. if (!rcu_segcblist_is_offloaded(&rdp->cblist))
  1408. return;
  1409. raw_spin_lock(&rdp->nocb_lock);
  1410. }
  1411. /*
  1412. * Release the specified rcu_data structure's ->nocb_lock, but only
  1413. * if it corresponds to a no-CBs CPU.
  1414. */
  1415. static void rcu_nocb_unlock(struct rcu_data *rdp)
  1416. {
  1417. if (rcu_segcblist_is_offloaded(&rdp->cblist)) {
  1418. lockdep_assert_irqs_disabled();
  1419. raw_spin_unlock(&rdp->nocb_lock);
  1420. }
  1421. }
  1422. /*
  1423. * Release the specified rcu_data structure's ->nocb_lock and restore
  1424. * interrupts, but only if it corresponds to a no-CBs CPU.
  1425. */
  1426. static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
  1427. unsigned long flags)
  1428. {
  1429. if (rcu_segcblist_is_offloaded(&rdp->cblist)) {
  1430. lockdep_assert_irqs_disabled();
  1431. raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
  1432. } else {
  1433. local_irq_restore(flags);
  1434. }
  1435. }
  1436. /* Lockdep check that ->cblist may be safely accessed. */
  1437. static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
  1438. {
  1439. lockdep_assert_irqs_disabled();
  1440. if (rcu_segcblist_is_offloaded(&rdp->cblist))
  1441. lockdep_assert_held(&rdp->nocb_lock);
  1442. }
  1443. /*
  1444. * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
  1445. * grace period.
  1446. */
  1447. static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
  1448. {
  1449. swake_up_all(sq);
  1450. }
  1451. static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
  1452. {
  1453. return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];
  1454. }
  1455. static void rcu_init_one_nocb(struct rcu_node *rnp)
  1456. {
  1457. init_swait_queue_head(&rnp->nocb_gp_wq[0]);
  1458. init_swait_queue_head(&rnp->nocb_gp_wq[1]);
  1459. }
  1460. /* Is the specified CPU a no-CBs CPU? */
  1461. bool rcu_is_nocb_cpu(int cpu)
  1462. {
  1463. if (cpumask_available(rcu_nocb_mask))
  1464. return cpumask_test_cpu(cpu, rcu_nocb_mask);
  1465. return false;
  1466. }
  1467. /*
  1468. * Kick the GP kthread for this NOCB group. Caller holds ->nocb_lock
  1469. * and this function releases it.
  1470. */
  1471. static void wake_nocb_gp(struct rcu_data *rdp, bool force,
  1472. unsigned long flags)
  1473. __releases(rdp->nocb_lock)
  1474. {
  1475. bool needwake = false;
  1476. struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
  1477. lockdep_assert_held(&rdp->nocb_lock);
  1478. if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {
  1479. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1480. TPS("AlreadyAwake"));
  1481. rcu_nocb_unlock_irqrestore(rdp, flags);
  1482. return;
  1483. }
  1484. if (READ_ONCE(rdp->nocb_defer_wakeup) > RCU_NOCB_WAKE_NOT) {
  1485. WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
  1486. del_timer(&rdp->nocb_timer);
  1487. }
  1488. rcu_nocb_unlock_irqrestore(rdp, flags);
  1489. raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
  1490. if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {
  1491. WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
  1492. needwake = true;
  1493. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));
  1494. }
  1495. raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
  1496. if (needwake)
  1497. wake_up_process(rdp_gp->nocb_gp_kthread);
  1498. }
  1499. /*
  1500. * Arrange to wake the GP kthread for this NOCB group at some future
  1501. * time when it is safe to do so.
  1502. */
  1503. static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,
  1504. const char *reason)
  1505. {
  1506. if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT)
  1507. mod_timer(&rdp->nocb_timer, jiffies + 1);
  1508. if (rdp->nocb_defer_wakeup < waketype)
  1509. WRITE_ONCE(rdp->nocb_defer_wakeup, waketype);
  1510. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);
  1511. }
  1512. /*
  1513. * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
  1514. * However, if there is a callback to be enqueued and if ->nocb_bypass
  1515. * proves to be initially empty, just return false because the no-CB GP
  1516. * kthread may need to be awakened in this case.
  1517. *
  1518. * Note that this function always returns true if rhp is NULL.
  1519. */
  1520. static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
  1521. unsigned long j)
  1522. {
  1523. struct rcu_cblist rcl;
  1524. WARN_ON_ONCE(!rcu_segcblist_is_offloaded(&rdp->cblist));
  1525. rcu_lockdep_assert_cblist_protected(rdp);
  1526. lockdep_assert_held(&rdp->nocb_bypass_lock);
  1527. if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {
  1528. raw_spin_unlock(&rdp->nocb_bypass_lock);
  1529. return false;
  1530. }
  1531. /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */
  1532. if (rhp)
  1533. rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
  1534. rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);
  1535. rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);
  1536. WRITE_ONCE(rdp->nocb_bypass_first, j);
  1537. rcu_nocb_bypass_unlock(rdp);
  1538. return true;
  1539. }
  1540. /*
  1541. * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
  1542. * However, if there is a callback to be enqueued and if ->nocb_bypass
  1543. * proves to be initially empty, just return false because the no-CB GP
  1544. * kthread may need to be awakened in this case.
  1545. *
  1546. * Note that this function always returns true if rhp is NULL.
  1547. */
  1548. static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
  1549. unsigned long j)
  1550. {
  1551. if (!rcu_segcblist_is_offloaded(&rdp->cblist))
  1552. return true;
  1553. rcu_lockdep_assert_cblist_protected(rdp);
  1554. rcu_nocb_bypass_lock(rdp);
  1555. return rcu_nocb_do_flush_bypass(rdp, rhp, j);
  1556. }
  1557. /*
  1558. * If the ->nocb_bypass_lock is immediately available, flush the
  1559. * ->nocb_bypass queue into ->cblist.
  1560. */
  1561. static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)
  1562. {
  1563. rcu_lockdep_assert_cblist_protected(rdp);
  1564. if (!rcu_segcblist_is_offloaded(&rdp->cblist) ||
  1565. !rcu_nocb_bypass_trylock(rdp))
  1566. return;
  1567. WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j));
  1568. }
  1569. /*
  1570. * See whether it is appropriate to use the ->nocb_bypass list in order
  1571. * to control contention on ->nocb_lock. A limited number of direct
  1572. * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass
  1573. * is non-empty, further callbacks must be placed into ->nocb_bypass,
  1574. * otherwise rcu_barrier() breaks. Use rcu_nocb_flush_bypass() to switch
  1575. * back to direct use of ->cblist. However, ->nocb_bypass should not be
  1576. * used if ->cblist is empty, because otherwise callbacks can be stranded
  1577. * on ->nocb_bypass because we cannot count on the current CPU ever again
  1578. * invoking call_rcu(). The general rule is that if ->nocb_bypass is
  1579. * non-empty, the corresponding no-CBs grace-period kthread must not be
  1580. * in an indefinite sleep state.
  1581. *
  1582. * Finally, it is not permitted to use the bypass during early boot,
  1583. * as doing so would confuse the auto-initialization code. Besides
  1584. * which, there is no point in worrying about lock contention while
  1585. * there is only one CPU in operation.
  1586. */
  1587. static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
  1588. bool *was_alldone, unsigned long flags)
  1589. {
  1590. unsigned long c;
  1591. unsigned long cur_gp_seq;
  1592. unsigned long j = jiffies;
  1593. long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
  1594. if (!rcu_segcblist_is_offloaded(&rdp->cblist)) {
  1595. *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
  1596. return false; /* Not offloaded, no bypassing. */
  1597. }
  1598. lockdep_assert_irqs_disabled();
  1599. // Don't use ->nocb_bypass during early boot.
  1600. if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
  1601. rcu_nocb_lock(rdp);
  1602. WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
  1603. *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
  1604. return false;
  1605. }
  1606. // If we have advanced to a new jiffy, reset counts to allow
  1607. // moving back from ->nocb_bypass to ->cblist.
  1608. if (j == rdp->nocb_nobypass_last) {
  1609. c = rdp->nocb_nobypass_count + 1;
  1610. } else {
  1611. WRITE_ONCE(rdp->nocb_nobypass_last, j);
  1612. c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;
  1613. if (ULONG_CMP_LT(rdp->nocb_nobypass_count,
  1614. nocb_nobypass_lim_per_jiffy))
  1615. c = 0;
  1616. else if (c > nocb_nobypass_lim_per_jiffy)
  1617. c = nocb_nobypass_lim_per_jiffy;
  1618. }
  1619. WRITE_ONCE(rdp->nocb_nobypass_count, c);
  1620. // If there hasn't yet been all that many ->cblist enqueues
  1621. // this jiffy, tell the caller to enqueue onto ->cblist. But flush
  1622. // ->nocb_bypass first.
  1623. if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) {
  1624. rcu_nocb_lock(rdp);
  1625. *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
  1626. if (*was_alldone)
  1627. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1628. TPS("FirstQ"));
  1629. WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j));
  1630. WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
  1631. return false; // Caller must enqueue the callback.
  1632. }
  1633. // If ->nocb_bypass has been used too long or is too full,
  1634. // flush ->nocb_bypass to ->cblist.
  1635. if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) ||
  1636. ncbs >= qhimark) {
  1637. rcu_nocb_lock(rdp);
  1638. if (!rcu_nocb_flush_bypass(rdp, rhp, j)) {
  1639. *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
  1640. if (*was_alldone)
  1641. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1642. TPS("FirstQ"));
  1643. WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
  1644. return false; // Caller must enqueue the callback.
  1645. }
  1646. if (j != rdp->nocb_gp_adv_time &&
  1647. rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
  1648. rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
  1649. rcu_advance_cbs_nowake(rdp->mynode, rdp);
  1650. rdp->nocb_gp_adv_time = j;
  1651. }
  1652. rcu_nocb_unlock_irqrestore(rdp, flags);
  1653. return true; // Callback already enqueued.
  1654. }
  1655. // We need to use the bypass.
  1656. rcu_nocb_wait_contended(rdp);
  1657. rcu_nocb_bypass_lock(rdp);
  1658. ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
  1659. rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
  1660. rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
  1661. if (!ncbs) {
  1662. WRITE_ONCE(rdp->nocb_bypass_first, j);
  1663. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));
  1664. }
  1665. rcu_nocb_bypass_unlock(rdp);
  1666. smp_mb(); /* Order enqueue before wake. */
  1667. if (ncbs) {
  1668. local_irq_restore(flags);
  1669. } else {
  1670. // No-CBs GP kthread might be indefinitely asleep, if so, wake.
  1671. rcu_nocb_lock(rdp); // Rare during call_rcu() flood.
  1672. if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {
  1673. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1674. TPS("FirstBQwake"));
  1675. __call_rcu_nocb_wake(rdp, true, flags);
  1676. } else {
  1677. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1678. TPS("FirstBQnoWake"));
  1679. rcu_nocb_unlock_irqrestore(rdp, flags);
  1680. }
  1681. }
  1682. return true; // Callback already enqueued.
  1683. }
  1684. /*
  1685. * Awaken the no-CBs grace-period kthead if needed, either due to it
  1686. * legitimately being asleep or due to overload conditions.
  1687. *
  1688. * If warranted, also wake up the kthread servicing this CPUs queues.
  1689. */
  1690. static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
  1691. unsigned long flags)
  1692. __releases(rdp->nocb_lock)
  1693. {
  1694. unsigned long cur_gp_seq;
  1695. unsigned long j;
  1696. long len;
  1697. struct task_struct *t;
  1698. // If we are being polled or there is no kthread, just leave.
  1699. t = READ_ONCE(rdp->nocb_gp_kthread);
  1700. if (rcu_nocb_poll || !t) {
  1701. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1702. TPS("WakeNotPoll"));
  1703. rcu_nocb_unlock_irqrestore(rdp, flags);
  1704. return;
  1705. }
  1706. // Need to actually to a wakeup.
  1707. len = rcu_segcblist_n_cbs(&rdp->cblist);
  1708. if (was_alldone) {
  1709. rdp->qlen_last_fqs_check = len;
  1710. if (!irqs_disabled_flags(flags)) {
  1711. /* ... if queue was empty ... */
  1712. wake_nocb_gp(rdp, false, flags);
  1713. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1714. TPS("WakeEmpty"));
  1715. } else {
  1716. wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,
  1717. TPS("WakeEmptyIsDeferred"));
  1718. rcu_nocb_unlock_irqrestore(rdp, flags);
  1719. }
  1720. } else if (len > rdp->qlen_last_fqs_check + qhimark) {
  1721. /* ... or if many callbacks queued. */
  1722. rdp->qlen_last_fqs_check = len;
  1723. j = jiffies;
  1724. if (j != rdp->nocb_gp_adv_time &&
  1725. rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
  1726. rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
  1727. rcu_advance_cbs_nowake(rdp->mynode, rdp);
  1728. rdp->nocb_gp_adv_time = j;
  1729. }
  1730. smp_mb(); /* Enqueue before timer_pending(). */
  1731. if ((rdp->nocb_cb_sleep ||
  1732. !rcu_segcblist_ready_cbs(&rdp->cblist)) &&
  1733. !timer_pending(&rdp->nocb_bypass_timer))
  1734. wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
  1735. TPS("WakeOvfIsDeferred"));
  1736. rcu_nocb_unlock_irqrestore(rdp, flags);
  1737. } else {
  1738. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
  1739. rcu_nocb_unlock_irqrestore(rdp, flags);
  1740. }
  1741. return;
  1742. }
  1743. /* Wake up the no-CBs GP kthread to flush ->nocb_bypass. */
  1744. static void do_nocb_bypass_wakeup_timer(struct timer_list *t)
  1745. {
  1746. unsigned long flags;
  1747. struct rcu_data *rdp = from_timer(rdp, t, nocb_bypass_timer);
  1748. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));
  1749. rcu_nocb_lock_irqsave(rdp, flags);
  1750. smp_mb__after_spinlock(); /* Timer expire before wakeup. */
  1751. __call_rcu_nocb_wake(rdp, true, flags);
  1752. }
  1753. /*
  1754. * No-CBs GP kthreads come here to wait for additional callbacks to show up
  1755. * or for grace periods to end.
  1756. */
  1757. static void nocb_gp_wait(struct rcu_data *my_rdp)
  1758. {
  1759. bool bypass = false;
  1760. long bypass_ncbs;
  1761. int __maybe_unused cpu = my_rdp->cpu;
  1762. unsigned long cur_gp_seq;
  1763. unsigned long flags;
  1764. bool gotcbs = false;
  1765. unsigned long j = jiffies;
  1766. bool needwait_gp = false; // This prevents actual uninitialized use.
  1767. bool needwake;
  1768. bool needwake_gp;
  1769. struct rcu_data *rdp;
  1770. struct rcu_node *rnp;
  1771. unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
  1772. bool wasempty = false;
  1773. /*
  1774. * Each pass through the following loop checks for CBs and for the
  1775. * nearest grace period (if any) to wait for next. The CB kthreads
  1776. * and the global grace-period kthread are awakened if needed.
  1777. */
  1778. WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp);
  1779. for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) {
  1780. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));
  1781. rcu_nocb_lock_irqsave(rdp, flags);
  1782. bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
  1783. if (bypass_ncbs &&
  1784. (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||
  1785. bypass_ncbs > 2 * qhimark)) {
  1786. // Bypass full or old, so flush it.
  1787. (void)rcu_nocb_try_flush_bypass(rdp, j);
  1788. bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
  1789. } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
  1790. rcu_nocb_unlock_irqrestore(rdp, flags);
  1791. continue; /* No callbacks here, try next. */
  1792. }
  1793. if (bypass_ncbs) {
  1794. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1795. TPS("Bypass"));
  1796. bypass = true;
  1797. }
  1798. rnp = rdp->mynode;
  1799. if (bypass) { // Avoid race with first bypass CB.
  1800. WRITE_ONCE(my_rdp->nocb_defer_wakeup,
  1801. RCU_NOCB_WAKE_NOT);
  1802. del_timer(&my_rdp->nocb_timer);
  1803. }
  1804. // Advance callbacks if helpful and low contention.
  1805. needwake_gp = false;
  1806. if (!rcu_segcblist_restempty(&rdp->cblist,
  1807. RCU_NEXT_READY_TAIL) ||
  1808. (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
  1809. rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
  1810. raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
  1811. needwake_gp = rcu_advance_cbs(rnp, rdp);
  1812. wasempty = rcu_segcblist_restempty(&rdp->cblist,
  1813. RCU_NEXT_READY_TAIL);
  1814. raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
  1815. }
  1816. // Need to wait on some grace period?
  1817. WARN_ON_ONCE(wasempty &&
  1818. !rcu_segcblist_restempty(&rdp->cblist,
  1819. RCU_NEXT_READY_TAIL));
  1820. if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
  1821. if (!needwait_gp ||
  1822. ULONG_CMP_LT(cur_gp_seq, wait_gp_seq))
  1823. wait_gp_seq = cur_gp_seq;
  1824. needwait_gp = true;
  1825. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
  1826. TPS("NeedWaitGP"));
  1827. }
  1828. if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
  1829. needwake = rdp->nocb_cb_sleep;
  1830. WRITE_ONCE(rdp->nocb_cb_sleep, false);
  1831. smp_mb(); /* CB invocation -after- GP end. */
  1832. } else {
  1833. needwake = false;
  1834. }
  1835. rcu_nocb_unlock_irqrestore(rdp, flags);
  1836. if (needwake) {
  1837. swake_up_one(&rdp->nocb_cb_wq);
  1838. gotcbs = true;
  1839. }
  1840. if (needwake_gp)
  1841. rcu_gp_kthread_wake();
  1842. }
  1843. my_rdp->nocb_gp_bypass = bypass;
  1844. my_rdp->nocb_gp_gp = needwait_gp;
  1845. my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0;
  1846. if (bypass && !rcu_nocb_poll) {
  1847. // At least one child with non-empty ->nocb_bypass, so set
  1848. // timer in order to avoid stranding its callbacks.
  1849. raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
  1850. mod_timer(&my_rdp->nocb_bypass_timer, j + 2);
  1851. raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
  1852. }
  1853. if (rcu_nocb_poll) {
  1854. /* Polling, so trace if first poll in the series. */
  1855. if (gotcbs)
  1856. trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));
  1857. schedule_timeout_idle(1);
  1858. } else if (!needwait_gp) {
  1859. /* Wait for callbacks to appear. */
  1860. trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));
  1861. swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq,
  1862. !READ_ONCE(my_rdp->nocb_gp_sleep));
  1863. trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));
  1864. } else {
  1865. rnp = my_rdp->mynode;
  1866. trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));
  1867. swait_event_interruptible_exclusive(
  1868. rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1],
  1869. rcu_seq_done(&rnp->gp_seq, wait_gp_seq) ||
  1870. !READ_ONCE(my_rdp->nocb_gp_sleep));
  1871. trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait"));
  1872. }
  1873. if (!rcu_nocb_poll) {
  1874. raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
  1875. if (bypass)
  1876. del_timer(&my_rdp->nocb_bypass_timer);
  1877. WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
  1878. raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
  1879. }
  1880. my_rdp->nocb_gp_seq = -1;
  1881. WARN_ON(signal_pending(current));
  1882. }
  1883. /*
  1884. * No-CBs grace-period-wait kthread. There is one of these per group
  1885. * of CPUs, but only once at least one CPU in that group has come online
  1886. * at least once since boot. This kthread checks for newly posted
  1887. * callbacks from any of the CPUs it is responsible for, waits for a
  1888. * grace period, then awakens all of the rcu_nocb_cb_kthread() instances
  1889. * that then have callback-invocation work to do.
  1890. */
  1891. static int rcu_nocb_gp_kthread(void *arg)
  1892. {
  1893. struct rcu_data *rdp = arg;
  1894. for (;;) {
  1895. WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);
  1896. nocb_gp_wait(rdp);
  1897. cond_resched_tasks_rcu_qs();
  1898. }
  1899. return 0;
  1900. }
  1901. /*
  1902. * Invoke any ready callbacks from the corresponding no-CBs CPU,
  1903. * then, if there are no more, wait for more to appear.
  1904. */
  1905. static void nocb_cb_wait(struct rcu_data *rdp)
  1906. {
  1907. unsigned long cur_gp_seq;
  1908. unsigned long flags;
  1909. bool needwake_gp = false;
  1910. struct rcu_node *rnp = rdp->mynode;
  1911. local_irq_save(flags);
  1912. rcu_momentary_dyntick_idle();
  1913. local_irq_restore(flags);
  1914. local_bh_disable();
  1915. rcu_do_batch(rdp);
  1916. local_bh_enable();
  1917. lockdep_assert_irqs_enabled();
  1918. rcu_nocb_lock_irqsave(rdp, flags);
  1919. if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
  1920. rcu_seq_done(&rnp->gp_seq, cur_gp_seq) &&
  1921. raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */
  1922. needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);
  1923. raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
  1924. }
  1925. if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
  1926. rcu_nocb_unlock_irqrestore(rdp, flags);
  1927. if (needwake_gp)
  1928. rcu_gp_kthread_wake();
  1929. return;
  1930. }
  1931. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));
  1932. WRITE_ONCE(rdp->nocb_cb_sleep, true);
  1933. rcu_nocb_unlock_irqrestore(rdp, flags);
  1934. if (needwake_gp)
  1935. rcu_gp_kthread_wake();
  1936. swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
  1937. !READ_ONCE(rdp->nocb_cb_sleep));
  1938. if (!smp_load_acquire(&rdp->nocb_cb_sleep)) { /* VVV */
  1939. /* ^^^ Ensure CB invocation follows _sleep test. */
  1940. return;
  1941. }
  1942. WARN_ON(signal_pending(current));
  1943. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
  1944. }
  1945. /*
  1946. * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke
  1947. * nocb_cb_wait() to do the dirty work.
  1948. */
  1949. static int rcu_nocb_cb_kthread(void *arg)
  1950. {
  1951. struct rcu_data *rdp = arg;
  1952. // Each pass through this loop does one callback batch, and,
  1953. // if there are no more ready callbacks, waits for them.
  1954. for (;;) {
  1955. nocb_cb_wait(rdp);
  1956. cond_resched_tasks_rcu_qs();
  1957. }
  1958. return 0;
  1959. }
  1960. /* Is a deferred wakeup of rcu_nocb_kthread() required? */
  1961. static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
  1962. {
  1963. return READ_ONCE(rdp->nocb_defer_wakeup);
  1964. }
  1965. /* Do a deferred wakeup of rcu_nocb_kthread(). */
  1966. static void do_nocb_deferred_wakeup_common(struct rcu_data *rdp)
  1967. {
  1968. unsigned long flags;
  1969. int ndw;
  1970. rcu_nocb_lock_irqsave(rdp, flags);
  1971. if (!rcu_nocb_need_deferred_wakeup(rdp)) {
  1972. rcu_nocb_unlock_irqrestore(rdp, flags);
  1973. return;
  1974. }
  1975. ndw = READ_ONCE(rdp->nocb_defer_wakeup);
  1976. wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);
  1977. trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));
  1978. }
  1979. /* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */
  1980. static void do_nocb_deferred_wakeup_timer(struct timer_list *t)
  1981. {
  1982. struct rcu_data *rdp = from_timer(rdp, t, nocb_timer);
  1983. do_nocb_deferred_wakeup_common(rdp);
  1984. }
  1985. /*
  1986. * Do a deferred wakeup of rcu_nocb_kthread() from fastpath.
  1987. * This means we do an inexact common-case check. Note that if
  1988. * we miss, ->nocb_timer will eventually clean things up.
  1989. */
  1990. static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
  1991. {
  1992. if (rcu_nocb_need_deferred_wakeup(rdp))
  1993. do_nocb_deferred_wakeup_common(rdp);
  1994. }
  1995. void rcu_nocb_flush_deferred_wakeup(void)
  1996. {
  1997. do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data));
  1998. }
  1999. void __init rcu_init_nohz(void)
  2000. {
  2001. int cpu;
  2002. bool need_rcu_nocb_mask = false;
  2003. struct rcu_data *rdp;
  2004. #if defined(CONFIG_NO_HZ_FULL)
  2005. if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
  2006. need_rcu_nocb_mask = true;
  2007. #endif /* #if defined(CONFIG_NO_HZ_FULL) */
  2008. if (!cpumask_available(rcu_nocb_mask) && need_rcu_nocb_mask) {
  2009. if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
  2010. pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
  2011. return;
  2012. }
  2013. }
  2014. if (!cpumask_available(rcu_nocb_mask))
  2015. return;
  2016. #if defined(CONFIG_NO_HZ_FULL)
  2017. if (tick_nohz_full_running)
  2018. cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
  2019. #endif /* #if defined(CONFIG_NO_HZ_FULL) */
  2020. if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
  2021. pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n");
  2022. cpumask_and(rcu_nocb_mask, cpu_possible_mask,
  2023. rcu_nocb_mask);
  2024. }
  2025. if (cpumask_empty(rcu_nocb_mask))
  2026. pr_info("\tOffload RCU callbacks from CPUs: (none).\n");
  2027. else
  2028. pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
  2029. cpumask_pr_args(rcu_nocb_mask));
  2030. if (rcu_nocb_poll)
  2031. pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
  2032. for_each_cpu(cpu, rcu_nocb_mask) {
  2033. rdp = per_cpu_ptr(&rcu_data, cpu);
  2034. if (rcu_segcblist_empty(&rdp->cblist))
  2035. rcu_segcblist_init(&rdp->cblist);
  2036. rcu_segcblist_offload(&rdp->cblist);
  2037. }
  2038. rcu_organize_nocb_kthreads();
  2039. }
  2040. /* Initialize per-rcu_data variables for no-CBs CPUs. */
  2041. static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
  2042. {
  2043. init_swait_queue_head(&rdp->nocb_cb_wq);
  2044. init_swait_queue_head(&rdp->nocb_gp_wq);
  2045. raw_spin_lock_init(&rdp->nocb_lock);
  2046. raw_spin_lock_init(&rdp->nocb_bypass_lock);
  2047. raw_spin_lock_init(&rdp->nocb_gp_lock);
  2048. timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
  2049. timer_setup(&rdp->nocb_bypass_timer, do_nocb_bypass_wakeup_timer, 0);
  2050. rcu_cblist_init(&rdp->nocb_bypass);
  2051. }
  2052. /*
  2053. * If the specified CPU is a no-CBs CPU that does not already have its
  2054. * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread
  2055. * for this CPU's group has not yet been created, spawn it as well.
  2056. */
  2057. static void rcu_spawn_one_nocb_kthread(int cpu)
  2058. {
  2059. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  2060. struct rcu_data *rdp_gp;
  2061. struct task_struct *t;
  2062. /*
  2063. * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
  2064. * then nothing to do.
  2065. */
  2066. if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread)
  2067. return;
  2068. /* If we didn't spawn the GP kthread first, reorganize! */
  2069. rdp_gp = rdp->nocb_gp_rdp;
  2070. if (!rdp_gp->nocb_gp_kthread) {
  2071. t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,
  2072. "rcuog/%d", rdp_gp->cpu);
  2073. if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__))
  2074. return;
  2075. WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
  2076. }
  2077. /* Spawn the kthread for this CPU. */
  2078. t = kthread_run(rcu_nocb_cb_kthread, rdp,
  2079. "rcuo%c/%d", rcu_state.abbr, cpu);
  2080. if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__))
  2081. return;
  2082. WRITE_ONCE(rdp->nocb_cb_kthread, t);
  2083. WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
  2084. }
  2085. /*
  2086. * If the specified CPU is a no-CBs CPU that does not already have its
  2087. * rcuo kthread, spawn it.
  2088. */
  2089. static void rcu_spawn_cpu_nocb_kthread(int cpu)
  2090. {
  2091. if (rcu_scheduler_fully_active)
  2092. rcu_spawn_one_nocb_kthread(cpu);
  2093. }
  2094. /*
  2095. * Once the scheduler is running, spawn rcuo kthreads for all online
  2096. * no-CBs CPUs. This assumes that the early_initcall()s happen before
  2097. * non-boot CPUs come online -- if this changes, we will need to add
  2098. * some mutual exclusion.
  2099. */
  2100. static void __init rcu_spawn_nocb_kthreads(void)
  2101. {
  2102. int cpu;
  2103. for_each_online_cpu(cpu)
  2104. rcu_spawn_cpu_nocb_kthread(cpu);
  2105. }
  2106. /* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
  2107. static int rcu_nocb_gp_stride = -1;
  2108. module_param(rcu_nocb_gp_stride, int, 0444);
  2109. /*
  2110. * Initialize GP-CB relationships for all no-CBs CPU.
  2111. */
  2112. static void __init rcu_organize_nocb_kthreads(void)
  2113. {
  2114. int cpu;
  2115. bool firsttime = true;
  2116. bool gotnocbs = false;
  2117. bool gotnocbscbs = true;
  2118. int ls = rcu_nocb_gp_stride;
  2119. int nl = 0; /* Next GP kthread. */
  2120. struct rcu_data *rdp;
  2121. struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */
  2122. struct rcu_data *rdp_prev = NULL;
  2123. if (!cpumask_available(rcu_nocb_mask))
  2124. return;
  2125. if (ls == -1) {
  2126. ls = nr_cpu_ids / int_sqrt(nr_cpu_ids);
  2127. rcu_nocb_gp_stride = ls;
  2128. }
  2129. /*
  2130. * Each pass through this loop sets up one rcu_data structure.
  2131. * Should the corresponding CPU come online in the future, then
  2132. * we will spawn the needed set of rcu_nocb_kthread() kthreads.
  2133. */
  2134. for_each_cpu(cpu, rcu_nocb_mask) {
  2135. rdp = per_cpu_ptr(&rcu_data, cpu);
  2136. if (rdp->cpu >= nl) {
  2137. /* New GP kthread, set up for CBs & next GP. */
  2138. gotnocbs = true;
  2139. nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
  2140. rdp->nocb_gp_rdp = rdp;
  2141. rdp_gp = rdp;
  2142. if (dump_tree) {
  2143. if (!firsttime)
  2144. pr_cont("%s\n", gotnocbscbs
  2145. ? "" : " (self only)");
  2146. gotnocbscbs = false;
  2147. firsttime = false;
  2148. pr_alert("%s: No-CB GP kthread CPU %d:",
  2149. __func__, cpu);
  2150. }
  2151. } else {
  2152. /* Another CB kthread, link to previous GP kthread. */
  2153. gotnocbscbs = true;
  2154. rdp->nocb_gp_rdp = rdp_gp;
  2155. rdp_prev->nocb_next_cb_rdp = rdp;
  2156. if (dump_tree)
  2157. pr_cont(" %d", cpu);
  2158. }
  2159. rdp_prev = rdp;
  2160. }
  2161. if (gotnocbs && dump_tree)
  2162. pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");
  2163. }
  2164. /*
  2165. * Bind the current task to the offloaded CPUs. If there are no offloaded
  2166. * CPUs, leave the task unbound. Splat if the bind attempt fails.
  2167. */
  2168. void rcu_bind_current_to_nocb(void)
  2169. {
  2170. if (cpumask_available(rcu_nocb_mask) && cpumask_weight(rcu_nocb_mask))
  2171. WARN_ON(sched_setaffinity(current->pid, rcu_nocb_mask));
  2172. }
  2173. EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb);
  2174. /*
  2175. * Dump out nocb grace-period kthread state for the specified rcu_data
  2176. * structure.
  2177. */
  2178. static void show_rcu_nocb_gp_state(struct rcu_data *rdp)
  2179. {
  2180. struct rcu_node *rnp = rdp->mynode;
  2181. pr_info("nocb GP %d %c%c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu\n",
  2182. rdp->cpu,
  2183. "kK"[!!rdp->nocb_gp_kthread],
  2184. "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],
  2185. "dD"[!!rdp->nocb_defer_wakeup],
  2186. "tT"[timer_pending(&rdp->nocb_timer)],
  2187. "bB"[timer_pending(&rdp->nocb_bypass_timer)],
  2188. "sS"[!!rdp->nocb_gp_sleep],
  2189. ".W"[swait_active(&rdp->nocb_gp_wq)],
  2190. ".W"[swait_active(&rnp->nocb_gp_wq[0])],
  2191. ".W"[swait_active(&rnp->nocb_gp_wq[1])],
  2192. ".B"[!!rdp->nocb_gp_bypass],
  2193. ".G"[!!rdp->nocb_gp_gp],
  2194. (long)rdp->nocb_gp_seq,
  2195. rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops));
  2196. }
  2197. /* Dump out nocb kthread state for the specified rcu_data structure. */
  2198. static void show_rcu_nocb_state(struct rcu_data *rdp)
  2199. {
  2200. struct rcu_segcblist *rsclp = &rdp->cblist;
  2201. bool waslocked;
  2202. bool wastimer;
  2203. bool wassleep;
  2204. if (rdp->nocb_gp_rdp == rdp)
  2205. show_rcu_nocb_gp_state(rdp);
  2206. pr_info(" CB %d->%d %c%c%c%c%c%c F%ld L%ld C%d %c%c%c%c%c q%ld\n",
  2207. rdp->cpu, rdp->nocb_gp_rdp->cpu,
  2208. "kK"[!!rdp->nocb_cb_kthread],
  2209. "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],
  2210. "cC"[!!atomic_read(&rdp->nocb_lock_contended)],
  2211. "lL"[raw_spin_is_locked(&rdp->nocb_lock)],
  2212. "sS"[!!rdp->nocb_cb_sleep],
  2213. ".W"[swait_active(&rdp->nocb_cb_wq)],
  2214. jiffies - rdp->nocb_bypass_first,
  2215. jiffies - rdp->nocb_nobypass_last,
  2216. rdp->nocb_nobypass_count,
  2217. ".D"[rcu_segcblist_ready_cbs(rsclp)],
  2218. ".W"[!rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)],
  2219. ".R"[!rcu_segcblist_restempty(rsclp, RCU_WAIT_TAIL)],
  2220. ".N"[!rcu_segcblist_restempty(rsclp, RCU_NEXT_READY_TAIL)],
  2221. ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],
  2222. rcu_segcblist_n_cbs(&rdp->cblist));
  2223. /* It is OK for GP kthreads to have GP state. */
  2224. if (rdp->nocb_gp_rdp == rdp)
  2225. return;
  2226. waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);
  2227. wastimer = timer_pending(&rdp->nocb_bypass_timer);
  2228. wassleep = swait_active(&rdp->nocb_gp_wq);
  2229. if (!rdp->nocb_gp_sleep && !waslocked && !wastimer && !wassleep)
  2230. return; /* Nothing untowards. */
  2231. pr_info(" nocb GP activity on CB-only CPU!!! %c%c%c%c %c\n",
  2232. "lL"[waslocked],
  2233. "dD"[!!rdp->nocb_defer_wakeup],
  2234. "tT"[wastimer],
  2235. "sS"[!!rdp->nocb_gp_sleep],
  2236. ".W"[wassleep]);
  2237. }
  2238. #else /* #ifdef CONFIG_RCU_NOCB_CPU */
  2239. /* No ->nocb_lock to acquire. */
  2240. static void rcu_nocb_lock(struct rcu_data *rdp)
  2241. {
  2242. }
  2243. /* No ->nocb_lock to release. */
  2244. static void rcu_nocb_unlock(struct rcu_data *rdp)
  2245. {
  2246. }
  2247. /* No ->nocb_lock to release. */
  2248. static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
  2249. unsigned long flags)
  2250. {
  2251. local_irq_restore(flags);
  2252. }
  2253. /* Lockdep check that ->cblist may be safely accessed. */
  2254. static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
  2255. {
  2256. lockdep_assert_irqs_disabled();
  2257. }
  2258. static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
  2259. {
  2260. }
  2261. static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
  2262. {
  2263. return NULL;
  2264. }
  2265. static void rcu_init_one_nocb(struct rcu_node *rnp)
  2266. {
  2267. }
  2268. static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
  2269. unsigned long j)
  2270. {
  2271. return true;
  2272. }
  2273. static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
  2274. bool *was_alldone, unsigned long flags)
  2275. {
  2276. return false;
  2277. }
  2278. static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
  2279. unsigned long flags)
  2280. {
  2281. WARN_ON_ONCE(1); /* Should be dead code! */
  2282. }
  2283. static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
  2284. {
  2285. }
  2286. static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
  2287. {
  2288. return false;
  2289. }
  2290. static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
  2291. {
  2292. }
  2293. static void rcu_spawn_cpu_nocb_kthread(int cpu)
  2294. {
  2295. }
  2296. static void __init rcu_spawn_nocb_kthreads(void)
  2297. {
  2298. }
  2299. static void show_rcu_nocb_state(struct rcu_data *rdp)
  2300. {
  2301. }
  2302. #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
  2303. /*
  2304. * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
  2305. * grace-period kthread will do force_quiescent_state() processing?
  2306. * The idea is to avoid waking up RCU core processing on such a
  2307. * CPU unless the grace period has extended for too long.
  2308. *
  2309. * This code relies on the fact that all NO_HZ_FULL CPUs are also
  2310. * CONFIG_RCU_NOCB_CPU CPUs.
  2311. */
  2312. static bool rcu_nohz_full_cpu(void)
  2313. {
  2314. #ifdef CONFIG_NO_HZ_FULL
  2315. if (tick_nohz_full_cpu(smp_processor_id()) &&
  2316. (!rcu_gp_in_progress() ||
  2317. time_before(jiffies, READ_ONCE(rcu_state.gp_start) + HZ)))
  2318. return true;
  2319. #endif /* #ifdef CONFIG_NO_HZ_FULL */
  2320. return false;
  2321. }
  2322. /*
  2323. * Bind the RCU grace-period kthreads to the housekeeping CPU.
  2324. */
  2325. static void rcu_bind_gp_kthread(void)
  2326. {
  2327. if (!tick_nohz_full_enabled())
  2328. return;
  2329. housekeeping_affine(current, HK_FLAG_RCU);
  2330. }
  2331. /* Record the current task on dyntick-idle entry. */
  2332. static __always_inline void rcu_dynticks_task_enter(void)
  2333. {
  2334. #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
  2335. WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
  2336. #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
  2337. }
  2338. /* Record no current task on dyntick-idle exit. */
  2339. static __always_inline void rcu_dynticks_task_exit(void)
  2340. {
  2341. #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
  2342. WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
  2343. #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
  2344. }
  2345. /* Turn on heavyweight RCU tasks trace readers on idle/user entry. */
  2346. static __always_inline void rcu_dynticks_task_trace_enter(void)
  2347. {
  2348. #ifdef CONFIG_TASKS_TRACE_RCU
  2349. if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
  2350. current->trc_reader_special.b.need_mb = true;
  2351. #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
  2352. }
  2353. /* Turn off heavyweight RCU tasks trace readers on idle/user exit. */
  2354. static __always_inline void rcu_dynticks_task_trace_exit(void)
  2355. {
  2356. #ifdef CONFIG_TASKS_TRACE_RCU
  2357. if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
  2358. current->trc_reader_special.b.need_mb = false;
  2359. #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
  2360. }