tree.c 144 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  4. *
  5. * Copyright IBM Corporation, 2008
  6. *
  7. * Authors: Dipankar Sarma <dipankar@in.ibm.com>
  8. * Manfred Spraul <manfred@colorfullife.com>
  9. * Paul E. McKenney <paulmck@linux.ibm.com>
  10. *
  11. * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
  12. * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
  13. *
  14. * For detailed explanation of Read-Copy Update mechanism see -
  15. * Documentation/RCU
  16. */
  17. #define pr_fmt(fmt) "rcu: " fmt
  18. #include <linux/types.h>
  19. #include <linux/kernel.h>
  20. #include <linux/init.h>
  21. #include <linux/spinlock.h>
  22. #include <linux/smp.h>
  23. #include <linux/rcupdate_wait.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/sched.h>
  26. #include <linux/sched/debug.h>
  27. #include <linux/nmi.h>
  28. #include <linux/atomic.h>
  29. #include <linux/bitops.h>
  30. #include <linux/export.h>
  31. #include <linux/completion.h>
  32. #include <linux/moduleparam.h>
  33. #include <linux/percpu.h>
  34. #include <linux/notifier.h>
  35. #include <linux/cpu.h>
  36. #include <linux/mutex.h>
  37. #include <linux/time.h>
  38. #include <linux/kernel_stat.h>
  39. #include <linux/wait.h>
  40. #include <linux/kthread.h>
  41. #include <uapi/linux/sched/types.h>
  42. #include <linux/prefetch.h>
  43. #include <linux/delay.h>
  44. #include <linux/random.h>
  45. #include <linux/trace_events.h>
  46. #include <linux/suspend.h>
  47. #include <linux/ftrace.h>
  48. #include <linux/tick.h>
  49. #include <linux/sysrq.h>
  50. #include <linux/kprobes.h>
  51. #include <linux/gfp.h>
  52. #include <linux/oom.h>
  53. #include <linux/smpboot.h>
  54. #include <linux/jiffies.h>
  55. #include <linux/slab.h>
  56. #include <linux/sched/isolation.h>
  57. #include <linux/sched/clock.h>
  58. #include <linux/vmalloc.h>
  59. #include <linux/mm.h>
  60. #include <linux/kasan.h>
  61. #include "../time/tick-internal.h"
  62. #include "tree.h"
  63. #include "rcu.h"
  64. #ifdef MODULE_PARAM_PREFIX
  65. #undef MODULE_PARAM_PREFIX
  66. #endif
  67. #define MODULE_PARAM_PREFIX "rcutree."
  68. /* Data structures. */
  69. /*
  70. * Steal a bit from the bottom of ->dynticks for idle entry/exit
  71. * control. Initially this is for TLB flushing.
  72. */
  73. #define RCU_DYNTICK_CTRL_MASK 0x1
  74. #define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
  75. static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
  76. .dynticks_nesting = 1,
  77. .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
  78. .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
  79. };
  80. static struct rcu_state rcu_state = {
  81. .level = { &rcu_state.node[0] },
  82. .gp_state = RCU_GP_IDLE,
  83. .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
  84. .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
  85. .name = RCU_NAME,
  86. .abbr = RCU_ABBR,
  87. .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
  88. .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
  89. .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
  90. };
  91. /* Dump rcu_node combining tree at boot to verify correct setup. */
  92. static bool dump_tree;
  93. module_param(dump_tree, bool, 0444);
  94. /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
  95. static bool use_softirq = true;
  96. module_param(use_softirq, bool, 0444);
  97. /* Control rcu_node-tree auto-balancing at boot time. */
  98. static bool rcu_fanout_exact;
  99. module_param(rcu_fanout_exact, bool, 0444);
  100. /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */
  101. static int rcu_fanout_leaf = RCU_FANOUT_LEAF;
  102. module_param(rcu_fanout_leaf, int, 0444);
  103. int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;
  104. /* Number of rcu_nodes at specified level. */
  105. int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
  106. int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
  107. /*
  108. * The rcu_scheduler_active variable is initialized to the value
  109. * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the
  110. * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE,
  111. * RCU can assume that there is but one task, allowing RCU to (for example)
  112. * optimize synchronize_rcu() to a simple barrier(). When this variable
  113. * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required
  114. * to detect real grace periods. This variable is also used to suppress
  115. * boot-time false positives from lockdep-RCU error checking. Finally, it
  116. * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU
  117. * is fully initialized, including all of its kthreads having been spawned.
  118. */
  119. int rcu_scheduler_active __read_mostly;
  120. EXPORT_SYMBOL_GPL(rcu_scheduler_active);
  121. /*
  122. * The rcu_scheduler_fully_active variable transitions from zero to one
  123. * during the early_initcall() processing, which is after the scheduler
  124. * is capable of creating new tasks. So RCU processing (for example,
  125. * creating tasks for RCU priority boosting) must be delayed until after
  126. * rcu_scheduler_fully_active transitions from zero to one. We also
  127. * currently delay invocation of any RCU callbacks until after this point.
  128. *
  129. * It might later prove better for people registering RCU callbacks during
  130. * early boot to take responsibility for these callbacks, but one step at
  131. * a time.
  132. */
  133. static int rcu_scheduler_fully_active __read_mostly;
  134. static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
  135. unsigned long gps, unsigned long flags);
  136. static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
  137. static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
  138. static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
  139. static void invoke_rcu_core(void);
  140. static void rcu_report_exp_rdp(struct rcu_data *rdp);
  141. static void sync_sched_exp_online_cleanup(int cpu);
  142. static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
  143. /* rcuc/rcub kthread realtime priority */
  144. static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
  145. module_param(kthread_prio, int, 0444);
  146. /* Delay in jiffies for grace-period initialization delays, debug only. */
  147. static int gp_preinit_delay;
  148. module_param(gp_preinit_delay, int, 0444);
  149. static int gp_init_delay;
  150. module_param(gp_init_delay, int, 0444);
  151. static int gp_cleanup_delay;
  152. module_param(gp_cleanup_delay, int, 0444);
  153. // Add delay to rcu_read_unlock() for strict grace periods.
  154. static int rcu_unlock_delay;
  155. #ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
  156. module_param(rcu_unlock_delay, int, 0444);
  157. #endif
  158. /*
  159. * This rcu parameter is runtime-read-only. It reflects
  160. * a minimum allowed number of objects which can be cached
  161. * per-CPU. Object size is equal to one page. This value
  162. * can be changed at boot time.
  163. */
  164. static int rcu_min_cached_objs = 5;
  165. module_param(rcu_min_cached_objs, int, 0444);
  166. /* Retrieve RCU kthreads priority for rcutorture */
  167. int rcu_get_gp_kthreads_prio(void)
  168. {
  169. return kthread_prio;
  170. }
  171. EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio);
  172. /*
  173. * Number of grace periods between delays, normalized by the duration of
  174. * the delay. The longer the delay, the more the grace periods between
  175. * each delay. The reason for this normalization is that it means that,
  176. * for non-zero delays, the overall slowdown of grace periods is constant
  177. * regardless of the duration of the delay. This arrangement balances
  178. * the need for long delays to increase some race probabilities with the
  179. * need for fast grace periods to increase other race probabilities.
  180. */
  181. #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */
  182. /*
  183. * Compute the mask of online CPUs for the specified rcu_node structure.
  184. * This will not be stable unless the rcu_node structure's ->lock is
  185. * held, but the bit corresponding to the current CPU will be stable
  186. * in most contexts.
  187. */
  188. static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
  189. {
  190. return READ_ONCE(rnp->qsmaskinitnext);
  191. }
  192. /*
  193. * Return true if an RCU grace period is in progress. The READ_ONCE()s
  194. * permit this function to be invoked without holding the root rcu_node
  195. * structure's ->lock, but of course results can be subject to change.
  196. */
  197. static int rcu_gp_in_progress(void)
  198. {
  199. return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
  200. }
  201. /*
  202. * Return the number of callbacks queued on the specified CPU.
  203. * Handles both the nocbs and normal cases.
  204. */
  205. static long rcu_get_n_cbs_cpu(int cpu)
  206. {
  207. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  208. if (rcu_segcblist_is_enabled(&rdp->cblist))
  209. return rcu_segcblist_n_cbs(&rdp->cblist);
  210. return 0;
  211. }
  212. void rcu_softirq_qs(void)
  213. {
  214. rcu_qs();
  215. rcu_preempt_deferred_qs(current);
  216. }
  217. /*
  218. * Record entry into an extended quiescent state. This is only to be
  219. * called when not already in an extended quiescent state, that is,
  220. * RCU is watching prior to the call to this function and is no longer
  221. * watching upon return.
  222. */
  223. static noinstr void rcu_dynticks_eqs_enter(void)
  224. {
  225. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  226. int seq;
  227. /*
  228. * CPUs seeing atomic_add_return() must see prior RCU read-side
  229. * critical sections, and we also must force ordering with the
  230. * next idle sojourn.
  231. */
  232. rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
  233. seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
  234. // RCU is no longer watching. Better be in extended quiescent state!
  235. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
  236. (seq & RCU_DYNTICK_CTRL_CTR));
  237. /* Better not have special action (TLB flush) pending! */
  238. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
  239. (seq & RCU_DYNTICK_CTRL_MASK));
  240. }
  241. /*
  242. * Record exit from an extended quiescent state. This is only to be
  243. * called from an extended quiescent state, that is, RCU is not watching
  244. * prior to the call to this function and is watching upon return.
  245. */
  246. static noinstr void rcu_dynticks_eqs_exit(void)
  247. {
  248. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  249. int seq;
  250. /*
  251. * CPUs seeing atomic_add_return() must see prior idle sojourns,
  252. * and we also must force ordering with the next RCU read-side
  253. * critical section.
  254. */
  255. seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
  256. // RCU is now watching. Better not be in an extended quiescent state!
  257. rcu_dynticks_task_trace_exit(); // After ->dynticks update!
  258. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
  259. !(seq & RCU_DYNTICK_CTRL_CTR));
  260. if (seq & RCU_DYNTICK_CTRL_MASK) {
  261. arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
  262. smp_mb__after_atomic(); /* _exit after clearing mask. */
  263. }
  264. }
  265. /*
  266. * Reset the current CPU's ->dynticks counter to indicate that the
  267. * newly onlined CPU is no longer in an extended quiescent state.
  268. * This will either leave the counter unchanged, or increment it
  269. * to the next non-quiescent value.
  270. *
  271. * The non-atomic test/increment sequence works because the upper bits
  272. * of the ->dynticks counter are manipulated only by the corresponding CPU,
  273. * or when the corresponding CPU is offline.
  274. */
  275. static void rcu_dynticks_eqs_online(void)
  276. {
  277. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  278. if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
  279. return;
  280. atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
  281. }
  282. /*
  283. * Is the current CPU in an extended quiescent state?
  284. *
  285. * No ordering, as we are sampling CPU-local information.
  286. */
  287. static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
  288. {
  289. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  290. return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
  291. }
  292. /*
  293. * Snapshot the ->dynticks counter with full ordering so as to allow
  294. * stable comparison of this counter with past and future snapshots.
  295. */
  296. static int rcu_dynticks_snap(struct rcu_data *rdp)
  297. {
  298. int snap = atomic_add_return(0, &rdp->dynticks);
  299. return snap & ~RCU_DYNTICK_CTRL_MASK;
  300. }
  301. /*
  302. * Return true if the snapshot returned from rcu_dynticks_snap()
  303. * indicates that RCU is in an extended quiescent state.
  304. */
  305. static bool rcu_dynticks_in_eqs(int snap)
  306. {
  307. return !(snap & RCU_DYNTICK_CTRL_CTR);
  308. }
  309. /*
  310. * Return true if the CPU corresponding to the specified rcu_data
  311. * structure has spent some time in an extended quiescent state since
  312. * rcu_dynticks_snap() returned the specified snapshot.
  313. */
  314. static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
  315. {
  316. return snap != rcu_dynticks_snap(rdp);
  317. }
  318. /*
  319. * Return true if the referenced integer is zero while the specified
  320. * CPU remains within a single extended quiescent state.
  321. */
  322. bool rcu_dynticks_zero_in_eqs(int cpu, int *vp)
  323. {
  324. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  325. int snap;
  326. // If not quiescent, force back to earlier extended quiescent state.
  327. snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK |
  328. RCU_DYNTICK_CTRL_CTR);
  329. smp_rmb(); // Order ->dynticks and *vp reads.
  330. if (READ_ONCE(*vp))
  331. return false; // Non-zero, so report failure;
  332. smp_rmb(); // Order *vp read and ->dynticks re-read.
  333. // If still in the same extended quiescent state, we are good!
  334. return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK);
  335. }
  336. /*
  337. * Set the special (bottom) bit of the specified CPU so that it
  338. * will take special action (such as flushing its TLB) on the
  339. * next exit from an extended quiescent state. Returns true if
  340. * the bit was successfully set, or false if the CPU was not in
  341. * an extended quiescent state.
  342. */
  343. bool rcu_eqs_special_set(int cpu)
  344. {
  345. int old;
  346. int new;
  347. int new_old;
  348. struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
  349. new_old = atomic_read(&rdp->dynticks);
  350. do {
  351. old = new_old;
  352. if (old & RCU_DYNTICK_CTRL_CTR)
  353. return false;
  354. new = old | RCU_DYNTICK_CTRL_MASK;
  355. new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
  356. } while (new_old != old);
  357. return true;
  358. }
  359. /*
  360. * Let the RCU core know that this CPU has gone through the scheduler,
  361. * which is a quiescent state. This is called when the need for a
  362. * quiescent state is urgent, so we burn an atomic operation and full
  363. * memory barriers to let the RCU core know about it, regardless of what
  364. * this CPU might (or might not) do in the near future.
  365. *
  366. * We inform the RCU core by emulating a zero-duration dyntick-idle period.
  367. *
  368. * The caller must have disabled interrupts and must not be idle.
  369. */
  370. notrace void rcu_momentary_dyntick_idle(void)
  371. {
  372. int special;
  373. raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
  374. special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
  375. &this_cpu_ptr(&rcu_data)->dynticks);
  376. /* It is illegal to call this from idle state. */
  377. WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
  378. rcu_preempt_deferred_qs(current);
  379. }
  380. EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
  381. /**
  382. * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle
  383. *
  384. * If the current CPU is idle and running at a first-level (not nested)
  385. * interrupt, or directly, from idle, return true.
  386. *
  387. * The caller must have at least disabled IRQs.
  388. */
  389. static int rcu_is_cpu_rrupt_from_idle(void)
  390. {
  391. long nesting;
  392. /*
  393. * Usually called from the tick; but also used from smp_function_call()
  394. * for expedited grace periods. This latter can result in running from
  395. * the idle task, instead of an actual IPI.
  396. */
  397. lockdep_assert_irqs_disabled();
  398. /* Check for counter underflows */
  399. RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
  400. "RCU dynticks_nesting counter underflow!");
  401. RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
  402. "RCU dynticks_nmi_nesting counter underflow/zero!");
  403. /* Are we at first interrupt nesting level? */
  404. nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting);
  405. if (nesting > 1)
  406. return false;
  407. /*
  408. * If we're not in an interrupt, we must be in the idle task!
  409. */
  410. WARN_ON_ONCE(!nesting && !is_idle_task(current));
  411. /* Does CPU appear to be idle from an RCU standpoint? */
  412. return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
  413. }
  414. #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
  415. // Maximum callbacks per rcu_do_batch ...
  416. #define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.
  417. static long blimit = DEFAULT_RCU_BLIMIT;
  418. #define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.
  419. static long qhimark = DEFAULT_RCU_QHIMARK;
  420. #define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.
  421. static long qlowmark = DEFAULT_RCU_QLOMARK;
  422. #define DEFAULT_RCU_QOVLD_MULT 2
  423. #define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
  424. static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.
  425. static long qovld_calc = -1; // No pre-initialization lock acquisitions!
  426. module_param(blimit, long, 0444);
  427. module_param(qhimark, long, 0444);
  428. module_param(qlowmark, long, 0444);
  429. module_param(qovld, long, 0444);
  430. static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;
  431. static ulong jiffies_till_next_fqs = ULONG_MAX;
  432. static bool rcu_kick_kthreads;
  433. static int rcu_divisor = 7;
  434. module_param(rcu_divisor, int, 0644);
  435. /* Force an exit from rcu_do_batch() after 3 milliseconds. */
  436. static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
  437. module_param(rcu_resched_ns, long, 0644);
  438. /*
  439. * How long the grace period must be before we start recruiting
  440. * quiescent-state help from rcu_note_context_switch().
  441. */
  442. static ulong jiffies_till_sched_qs = ULONG_MAX;
  443. module_param(jiffies_till_sched_qs, ulong, 0444);
  444. static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
  445. module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
  446. /*
  447. * Make sure that we give the grace-period kthread time to detect any
  448. * idle CPUs before taking active measures to force quiescent states.
  449. * However, don't go below 100 milliseconds, adjusted upwards for really
  450. * large systems.
  451. */
  452. static void adjust_jiffies_till_sched_qs(void)
  453. {
  454. unsigned long j;
  455. /* If jiffies_till_sched_qs was specified, respect the request. */
  456. if (jiffies_till_sched_qs != ULONG_MAX) {
  457. WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
  458. return;
  459. }
  460. /* Otherwise, set to third fqs scan, but bound below on large system. */
  461. j = READ_ONCE(jiffies_till_first_fqs) +
  462. 2 * READ_ONCE(jiffies_till_next_fqs);
  463. if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
  464. j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
  465. pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
  466. WRITE_ONCE(jiffies_to_sched_qs, j);
  467. }
  468. static int param_set_first_fqs_jiffies(const char *val, const struct kernel_param *kp)
  469. {
  470. ulong j;
  471. int ret = kstrtoul(val, 0, &j);
  472. if (!ret) {
  473. WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : j);
  474. adjust_jiffies_till_sched_qs();
  475. }
  476. return ret;
  477. }
  478. static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param *kp)
  479. {
  480. ulong j;
  481. int ret = kstrtoul(val, 0, &j);
  482. if (!ret) {
  483. WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
  484. adjust_jiffies_till_sched_qs();
  485. }
  486. return ret;
  487. }
  488. static struct kernel_param_ops first_fqs_jiffies_ops = {
  489. .set = param_set_first_fqs_jiffies,
  490. .get = param_get_ulong,
  491. };
  492. static struct kernel_param_ops next_fqs_jiffies_ops = {
  493. .set = param_set_next_fqs_jiffies,
  494. .get = param_get_ulong,
  495. };
  496. module_param_cb(jiffies_till_first_fqs, &first_fqs_jiffies_ops, &jiffies_till_first_fqs, 0644);
  497. module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
  498. module_param(rcu_kick_kthreads, bool, 0644);
  499. static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
  500. static int rcu_pending(int user);
  501. /*
  502. * Return the number of RCU GPs completed thus far for debug & stats.
  503. */
  504. unsigned long rcu_get_gp_seq(void)
  505. {
  506. return READ_ONCE(rcu_state.gp_seq);
  507. }
  508. EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
  509. /*
  510. * Return the number of RCU expedited batches completed thus far for
  511. * debug & stats. Odd numbers mean that a batch is in progress, even
  512. * numbers mean idle. The value returned will thus be roughly double
  513. * the cumulative batches since boot.
  514. */
  515. unsigned long rcu_exp_batches_completed(void)
  516. {
  517. return rcu_state.expedited_sequence;
  518. }
  519. EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
  520. /*
  521. * Return the root node of the rcu_state structure.
  522. */
  523. static struct rcu_node *rcu_get_root(void)
  524. {
  525. return &rcu_state.node[0];
  526. }
  527. /*
  528. * Send along grace-period-related data for rcutorture diagnostics.
  529. */
  530. void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
  531. unsigned long *gp_seq)
  532. {
  533. switch (test_type) {
  534. case RCU_FLAVOR:
  535. *flags = READ_ONCE(rcu_state.gp_flags);
  536. *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
  537. break;
  538. default:
  539. break;
  540. }
  541. }
  542. EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
  543. /*
  544. * Enter an RCU extended quiescent state, which can be either the
  545. * idle loop or adaptive-tickless usermode execution.
  546. *
  547. * We crowbar the ->dynticks_nmi_nesting field to zero to allow for
  548. * the possibility of usermode upcalls having messed up our count
  549. * of interrupt nesting level during the prior busy period.
  550. */
  551. static noinstr void rcu_eqs_enter(bool user)
  552. {
  553. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  554. WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
  555. WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
  556. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
  557. rdp->dynticks_nesting == 0);
  558. if (rdp->dynticks_nesting != 1) {
  559. // RCU will still be watching, so just do accounting and leave.
  560. rdp->dynticks_nesting--;
  561. return;
  562. }
  563. lockdep_assert_irqs_disabled();
  564. instrumentation_begin();
  565. trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
  566. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
  567. rdp = this_cpu_ptr(&rcu_data);
  568. rcu_prepare_for_idle();
  569. rcu_preempt_deferred_qs(current);
  570. // instrumentation for the noinstr rcu_dynticks_eqs_enter()
  571. instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
  572. instrumentation_end();
  573. WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
  574. // RCU is watching here ...
  575. rcu_dynticks_eqs_enter();
  576. // ... but is no longer watching here.
  577. rcu_dynticks_task_enter();
  578. }
  579. /**
  580. * rcu_idle_enter - inform RCU that current CPU is entering idle
  581. *
  582. * Enter idle mode, in other words, -leave- the mode in which RCU
  583. * read-side critical sections can occur. (Though RCU read-side
  584. * critical sections can occur in irq handlers in idle, a possibility
  585. * handled by irq_enter() and irq_exit().)
  586. *
  587. * If you add or remove a call to rcu_idle_enter(), be sure to test with
  588. * CONFIG_RCU_EQS_DEBUG=y.
  589. */
  590. void rcu_idle_enter(void)
  591. {
  592. lockdep_assert_irqs_disabled();
  593. rcu_eqs_enter(false);
  594. }
  595. EXPORT_SYMBOL_GPL(rcu_idle_enter);
  596. #ifdef CONFIG_NO_HZ_FULL
  597. /**
  598. * rcu_user_enter - inform RCU that we are resuming userspace.
  599. *
  600. * Enter RCU idle mode right before resuming userspace. No use of RCU
  601. * is permitted between this call and rcu_user_exit(). This way the
  602. * CPU doesn't need to maintain the tick for RCU maintenance purposes
  603. * when the CPU runs in userspace.
  604. *
  605. * If you add or remove a call to rcu_user_enter(), be sure to test with
  606. * CONFIG_RCU_EQS_DEBUG=y.
  607. */
  608. noinstr void rcu_user_enter(void)
  609. {
  610. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  611. lockdep_assert_irqs_disabled();
  612. instrumentation_begin();
  613. do_nocb_deferred_wakeup(rdp);
  614. instrumentation_end();
  615. rcu_eqs_enter(true);
  616. }
  617. #endif /* CONFIG_NO_HZ_FULL */
  618. /**
  619. * rcu_nmi_exit - inform RCU of exit from NMI context
  620. *
  621. * If we are returning from the outermost NMI handler that interrupted an
  622. * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
  623. * to let the RCU grace-period handling know that the CPU is back to
  624. * being RCU-idle.
  625. *
  626. * If you add or remove a call to rcu_nmi_exit(), be sure to test
  627. * with CONFIG_RCU_EQS_DEBUG=y.
  628. */
  629. noinstr void rcu_nmi_exit(void)
  630. {
  631. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  632. instrumentation_begin();
  633. /*
  634. * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
  635. * (We are exiting an NMI handler, so RCU better be paying attention
  636. * to us!)
  637. */
  638. WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
  639. WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
  640. /*
  641. * If the nesting level is not 1, the CPU wasn't RCU-idle, so
  642. * leave it in non-RCU-idle state.
  643. */
  644. if (rdp->dynticks_nmi_nesting != 1) {
  645. trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
  646. atomic_read(&rdp->dynticks));
  647. WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
  648. rdp->dynticks_nmi_nesting - 2);
  649. instrumentation_end();
  650. return;
  651. }
  652. /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
  653. trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
  654. WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
  655. if (!in_nmi())
  656. rcu_prepare_for_idle();
  657. // instrumentation for the noinstr rcu_dynticks_eqs_enter()
  658. instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
  659. instrumentation_end();
  660. // RCU is watching here ...
  661. rcu_dynticks_eqs_enter();
  662. // ... but is no longer watching here.
  663. if (!in_nmi())
  664. rcu_dynticks_task_enter();
  665. }
  666. /**
  667. * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
  668. *
  669. * Exit from an interrupt handler, which might possibly result in entering
  670. * idle mode, in other words, leaving the mode in which read-side critical
  671. * sections can occur. The caller must have disabled interrupts.
  672. *
  673. * This code assumes that the idle loop never does anything that might
  674. * result in unbalanced calls to irq_enter() and irq_exit(). If your
  675. * architecture's idle loop violates this assumption, RCU will give you what
  676. * you deserve, good and hard. But very infrequently and irreproducibly.
  677. *
  678. * Use things like work queues to work around this limitation.
  679. *
  680. * You have been warned.
  681. *
  682. * If you add or remove a call to rcu_irq_exit(), be sure to test with
  683. * CONFIG_RCU_EQS_DEBUG=y.
  684. */
  685. void noinstr rcu_irq_exit(void)
  686. {
  687. lockdep_assert_irqs_disabled();
  688. rcu_nmi_exit();
  689. }
  690. /**
  691. * rcu_irq_exit_preempt - Inform RCU that current CPU is exiting irq
  692. * towards in kernel preemption
  693. *
  694. * Same as rcu_irq_exit() but has a sanity check that scheduling is safe
  695. * from RCU point of view. Invoked from return from interrupt before kernel
  696. * preemption.
  697. */
  698. void rcu_irq_exit_preempt(void)
  699. {
  700. lockdep_assert_irqs_disabled();
  701. rcu_nmi_exit();
  702. RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
  703. "RCU dynticks_nesting counter underflow/zero!");
  704. RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
  705. DYNTICK_IRQ_NONIDLE,
  706. "Bad RCU dynticks_nmi_nesting counter\n");
  707. RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
  708. "RCU in extended quiescent state!");
  709. }
  710. #ifdef CONFIG_PROVE_RCU
  711. /**
  712. * rcu_irq_exit_check_preempt - Validate that scheduling is possible
  713. */
  714. void rcu_irq_exit_check_preempt(void)
  715. {
  716. lockdep_assert_irqs_disabled();
  717. RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
  718. "RCU dynticks_nesting counter underflow/zero!");
  719. RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
  720. DYNTICK_IRQ_NONIDLE,
  721. "Bad RCU dynticks_nmi_nesting counter\n");
  722. RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
  723. "RCU in extended quiescent state!");
  724. }
  725. #endif /* #ifdef CONFIG_PROVE_RCU */
  726. /*
  727. * Wrapper for rcu_irq_exit() where interrupts are enabled.
  728. *
  729. * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test
  730. * with CONFIG_RCU_EQS_DEBUG=y.
  731. */
  732. void rcu_irq_exit_irqson(void)
  733. {
  734. unsigned long flags;
  735. local_irq_save(flags);
  736. rcu_irq_exit();
  737. local_irq_restore(flags);
  738. }
  739. /*
  740. * Exit an RCU extended quiescent state, which can be either the
  741. * idle loop or adaptive-tickless usermode execution.
  742. *
  743. * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
  744. * allow for the possibility of usermode upcalls messing up our count of
  745. * interrupt nesting level during the busy period that is just now starting.
  746. */
  747. static void noinstr rcu_eqs_exit(bool user)
  748. {
  749. struct rcu_data *rdp;
  750. long oldval;
  751. lockdep_assert_irqs_disabled();
  752. rdp = this_cpu_ptr(&rcu_data);
  753. oldval = rdp->dynticks_nesting;
  754. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
  755. if (oldval) {
  756. // RCU was already watching, so just do accounting and leave.
  757. rdp->dynticks_nesting++;
  758. return;
  759. }
  760. rcu_dynticks_task_exit();
  761. // RCU is not watching here ...
  762. rcu_dynticks_eqs_exit();
  763. // ... but is watching here.
  764. instrumentation_begin();
  765. // instrumentation for the noinstr rcu_dynticks_eqs_exit()
  766. instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
  767. rcu_cleanup_after_idle();
  768. trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
  769. WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
  770. WRITE_ONCE(rdp->dynticks_nesting, 1);
  771. WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
  772. WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
  773. instrumentation_end();
  774. }
  775. /**
  776. * rcu_idle_exit - inform RCU that current CPU is leaving idle
  777. *
  778. * Exit idle mode, in other words, -enter- the mode in which RCU
  779. * read-side critical sections can occur.
  780. *
  781. * If you add or remove a call to rcu_idle_exit(), be sure to test with
  782. * CONFIG_RCU_EQS_DEBUG=y.
  783. */
  784. void rcu_idle_exit(void)
  785. {
  786. unsigned long flags;
  787. local_irq_save(flags);
  788. rcu_eqs_exit(false);
  789. local_irq_restore(flags);
  790. }
  791. EXPORT_SYMBOL_GPL(rcu_idle_exit);
  792. #ifdef CONFIG_NO_HZ_FULL
  793. /**
  794. * rcu_user_exit - inform RCU that we are exiting userspace.
  795. *
  796. * Exit RCU idle mode while entering the kernel because it can
  797. * run a RCU read side critical section anytime.
  798. *
  799. * If you add or remove a call to rcu_user_exit(), be sure to test with
  800. * CONFIG_RCU_EQS_DEBUG=y.
  801. */
  802. void noinstr rcu_user_exit(void)
  803. {
  804. rcu_eqs_exit(1);
  805. }
  806. /**
  807. * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it.
  808. *
  809. * The scheduler tick is not normally enabled when CPUs enter the kernel
  810. * from nohz_full userspace execution. After all, nohz_full userspace
  811. * execution is an RCU quiescent state and the time executing in the kernel
  812. * is quite short. Except of course when it isn't. And it is not hard to
  813. * cause a large system to spend tens of seconds or even minutes looping
  814. * in the kernel, which can cause a number of problems, include RCU CPU
  815. * stall warnings.
  816. *
  817. * Therefore, if a nohz_full CPU fails to report a quiescent state
  818. * in a timely manner, the RCU grace-period kthread sets that CPU's
  819. * ->rcu_urgent_qs flag with the expectation that the next interrupt or
  820. * exception will invoke this function, which will turn on the scheduler
  821. * tick, which will enable RCU to detect that CPU's quiescent states,
  822. * for example, due to cond_resched() calls in CONFIG_PREEMPT=n kernels.
  823. * The tick will be disabled once a quiescent state is reported for
  824. * this CPU.
  825. *
  826. * Of course, in carefully tuned systems, there might never be an
  827. * interrupt or exception. In that case, the RCU grace-period kthread
  828. * will eventually cause one to happen. However, in less carefully
  829. * controlled environments, this function allows RCU to get what it
  830. * needs without creating otherwise useless interruptions.
  831. */
  832. void __rcu_irq_enter_check_tick(void)
  833. {
  834. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  835. // If we're here from NMI there's nothing to do.
  836. if (in_nmi())
  837. return;
  838. RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
  839. "Illegal rcu_irq_enter_check_tick() from extended quiescent state");
  840. if (!tick_nohz_full_cpu(rdp->cpu) ||
  841. !READ_ONCE(rdp->rcu_urgent_qs) ||
  842. READ_ONCE(rdp->rcu_forced_tick)) {
  843. // RCU doesn't need nohz_full help from this CPU, or it is
  844. // already getting that help.
  845. return;
  846. }
  847. // We get here only when not in an extended quiescent state and
  848. // from interrupts (as opposed to NMIs). Therefore, (1) RCU is
  849. // already watching and (2) The fact that we are in an interrupt
  850. // handler and that the rcu_node lock is an irq-disabled lock
  851. // prevents self-deadlock. So we can safely recheck under the lock.
  852. // Note that the nohz_full state currently cannot change.
  853. raw_spin_lock_rcu_node(rdp->mynode);
  854. if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
  855. // A nohz_full CPU is in the kernel and RCU needs a
  856. // quiescent state. Turn on the tick!
  857. WRITE_ONCE(rdp->rcu_forced_tick, true);
  858. tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
  859. }
  860. raw_spin_unlock_rcu_node(rdp->mynode);
  861. }
  862. #endif /* CONFIG_NO_HZ_FULL */
  863. /**
  864. * rcu_nmi_enter - inform RCU of entry to NMI context
  865. *
  866. * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
  867. * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
  868. * that the CPU is active. This implementation permits nested NMIs, as
  869. * long as the nesting level does not overflow an int. (You will probably
  870. * run out of stack space first.)
  871. *
  872. * If you add or remove a call to rcu_nmi_enter(), be sure to test
  873. * with CONFIG_RCU_EQS_DEBUG=y.
  874. */
  875. noinstr void rcu_nmi_enter(void)
  876. {
  877. long incby = 2;
  878. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  879. /* Complain about underflow. */
  880. WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
  881. /*
  882. * If idle from RCU viewpoint, atomically increment ->dynticks
  883. * to mark non-idle and increment ->dynticks_nmi_nesting by one.
  884. * Otherwise, increment ->dynticks_nmi_nesting by two. This means
  885. * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
  886. * to be in the outermost NMI handler that interrupted an RCU-idle
  887. * period (observation due to Andy Lutomirski).
  888. */
  889. if (rcu_dynticks_curr_cpu_in_eqs()) {
  890. if (!in_nmi())
  891. rcu_dynticks_task_exit();
  892. // RCU is not watching here ...
  893. rcu_dynticks_eqs_exit();
  894. // ... but is watching here.
  895. if (!in_nmi()) {
  896. instrumentation_begin();
  897. rcu_cleanup_after_idle();
  898. instrumentation_end();
  899. }
  900. instrumentation_begin();
  901. // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
  902. instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
  903. // instrumentation for the noinstr rcu_dynticks_eqs_exit()
  904. instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
  905. incby = 1;
  906. } else if (!in_nmi()) {
  907. instrumentation_begin();
  908. rcu_irq_enter_check_tick();
  909. } else {
  910. instrumentation_begin();
  911. }
  912. trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
  913. rdp->dynticks_nmi_nesting,
  914. rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
  915. instrumentation_end();
  916. WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */
  917. rdp->dynticks_nmi_nesting + incby);
  918. barrier();
  919. }
  920. /**
  921. * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
  922. *
  923. * Enter an interrupt handler, which might possibly result in exiting
  924. * idle mode, in other words, entering the mode in which read-side critical
  925. * sections can occur. The caller must have disabled interrupts.
  926. *
  927. * Note that the Linux kernel is fully capable of entering an interrupt
  928. * handler that it never exits, for example when doing upcalls to user mode!
  929. * This code assumes that the idle loop never does upcalls to user mode.
  930. * If your architecture's idle loop does do upcalls to user mode (or does
  931. * anything else that results in unbalanced calls to the irq_enter() and
  932. * irq_exit() functions), RCU will give you what you deserve, good and hard.
  933. * But very infrequently and irreproducibly.
  934. *
  935. * Use things like work queues to work around this limitation.
  936. *
  937. * You have been warned.
  938. *
  939. * If you add or remove a call to rcu_irq_enter(), be sure to test with
  940. * CONFIG_RCU_EQS_DEBUG=y.
  941. */
  942. noinstr void rcu_irq_enter(void)
  943. {
  944. lockdep_assert_irqs_disabled();
  945. rcu_nmi_enter();
  946. }
  947. /*
  948. * Wrapper for rcu_irq_enter() where interrupts are enabled.
  949. *
  950. * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test
  951. * with CONFIG_RCU_EQS_DEBUG=y.
  952. */
  953. void rcu_irq_enter_irqson(void)
  954. {
  955. unsigned long flags;
  956. local_irq_save(flags);
  957. rcu_irq_enter();
  958. local_irq_restore(flags);
  959. }
  960. /*
  961. * If any sort of urgency was applied to the current CPU (for example,
  962. * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
  963. * to get to a quiescent state, disable it.
  964. */
  965. static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
  966. {
  967. raw_lockdep_assert_held_rcu_node(rdp->mynode);
  968. WRITE_ONCE(rdp->rcu_urgent_qs, false);
  969. WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
  970. if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
  971. tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
  972. WRITE_ONCE(rdp->rcu_forced_tick, false);
  973. }
  974. }
  975. /**
  976. * rcu_is_watching - see if RCU thinks that the current CPU is not idle
  977. *
  978. * Return true if RCU is watching the running CPU, which means that this
  979. * CPU can safely enter RCU read-side critical sections. In other words,
  980. * if the current CPU is not in its idle loop or is in an interrupt or
  981. * NMI handler, return true.
  982. *
  983. * Make notrace because it can be called by the internal functions of
  984. * ftrace, and making this notrace removes unnecessary recursion calls.
  985. */
  986. notrace bool rcu_is_watching(void)
  987. {
  988. bool ret;
  989. preempt_disable_notrace();
  990. ret = !rcu_dynticks_curr_cpu_in_eqs();
  991. preempt_enable_notrace();
  992. return ret;
  993. }
  994. EXPORT_SYMBOL_GPL(rcu_is_watching);
  995. /*
  996. * If a holdout task is actually running, request an urgent quiescent
  997. * state from its CPU. This is unsynchronized, so migrations can cause
  998. * the request to go to the wrong CPU. Which is OK, all that will happen
  999. * is that the CPU's next context switch will be a bit slower and next
  1000. * time around this task will generate another request.
  1001. */
  1002. void rcu_request_urgent_qs_task(struct task_struct *t)
  1003. {
  1004. int cpu;
  1005. barrier();
  1006. cpu = task_cpu(t);
  1007. if (!task_curr(t))
  1008. return; /* This task is not running on that CPU. */
  1009. smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
  1010. }
  1011. #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
  1012. /*
  1013. * Is the current CPU online as far as RCU is concerned?
  1014. *
  1015. * Disable preemption to avoid false positives that could otherwise
  1016. * happen due to the current CPU number being sampled, this task being
  1017. * preempted, its old CPU being taken offline, resuming on some other CPU,
  1018. * then determining that its old CPU is now offline.
  1019. *
  1020. * Disable checking if in an NMI handler because we cannot safely
  1021. * report errors from NMI handlers anyway. In addition, it is OK to use
  1022. * RCU on an offline processor during initial boot, hence the check for
  1023. * rcu_scheduler_fully_active.
  1024. */
  1025. bool rcu_lockdep_current_cpu_online(void)
  1026. {
  1027. struct rcu_data *rdp;
  1028. struct rcu_node *rnp;
  1029. bool ret = false;
  1030. if (in_nmi() || !rcu_scheduler_fully_active)
  1031. return true;
  1032. preempt_disable_notrace();
  1033. rdp = this_cpu_ptr(&rcu_data);
  1034. rnp = rdp->mynode;
  1035. if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
  1036. ret = true;
  1037. preempt_enable_notrace();
  1038. return ret;
  1039. }
  1040. EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
  1041. #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
  1042. /*
  1043. * We are reporting a quiescent state on behalf of some other CPU, so
  1044. * it is our responsibility to check for and handle potential overflow
  1045. * of the rcu_node ->gp_seq counter with respect to the rcu_data counters.
  1046. * After all, the CPU might be in deep idle state, and thus executing no
  1047. * code whatsoever.
  1048. */
  1049. static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp)
  1050. {
  1051. raw_lockdep_assert_held_rcu_node(rnp);
  1052. if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4,
  1053. rnp->gp_seq))
  1054. WRITE_ONCE(rdp->gpwrap, true);
  1055. if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
  1056. rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
  1057. }
  1058. /*
  1059. * Snapshot the specified CPU's dynticks counter so that we can later
  1060. * credit them with an implicit quiescent state. Return 1 if this CPU
  1061. * is in dynticks idle mode, which is an extended quiescent state.
  1062. */
  1063. static int dyntick_save_progress_counter(struct rcu_data *rdp)
  1064. {
  1065. rdp->dynticks_snap = rcu_dynticks_snap(rdp);
  1066. if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
  1067. trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
  1068. rcu_gpnum_ovf(rdp->mynode, rdp);
  1069. return 1;
  1070. }
  1071. return 0;
  1072. }
  1073. /*
  1074. * Return true if the specified CPU has passed through a quiescent
  1075. * state by virtue of being in or having passed through an dynticks
  1076. * idle state since the last call to dyntick_save_progress_counter()
  1077. * for this same CPU, or by virtue of having been offline.
  1078. */
  1079. static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
  1080. {
  1081. unsigned long jtsq;
  1082. bool *rnhqp;
  1083. bool *ruqp;
  1084. struct rcu_node *rnp = rdp->mynode;
  1085. /*
  1086. * If the CPU passed through or entered a dynticks idle phase with
  1087. * no active irq/NMI handlers, then we can safely pretend that the CPU
  1088. * already acknowledged the request to pass through a quiescent
  1089. * state. Either way, that CPU cannot possibly be in an RCU
  1090. * read-side critical section that started before the beginning
  1091. * of the current RCU grace period.
  1092. */
  1093. if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
  1094. trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
  1095. rcu_gpnum_ovf(rnp, rdp);
  1096. return 1;
  1097. }
  1098. /*
  1099. * Complain if a CPU that is considered to be offline from RCU's
  1100. * perspective has not yet reported a quiescent state. After all,
  1101. * the offline CPU should have reported a quiescent state during
  1102. * the CPU-offline process, or, failing that, by rcu_gp_init()
  1103. * if it ran concurrently with either the CPU going offline or the
  1104. * last task on a leaf rcu_node structure exiting its RCU read-side
  1105. * critical section while all CPUs corresponding to that structure
  1106. * are offline. This added warning detects bugs in any of these
  1107. * code paths.
  1108. *
  1109. * The rcu_node structure's ->lock is held here, which excludes
  1110. * the relevant portions the CPU-hotplug code, the grace-period
  1111. * initialization code, and the rcu_read_unlock() code paths.
  1112. *
  1113. * For more detail, please refer to the "Hotplug CPU" section
  1114. * of RCU's Requirements documentation.
  1115. */
  1116. if (WARN_ON_ONCE(!(rdp->grpmask & rcu_rnp_online_cpus(rnp)))) {
  1117. bool onl;
  1118. struct rcu_node *rnp1;
  1119. pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
  1120. __func__, rnp->grplo, rnp->grphi, rnp->level,
  1121. (long)rnp->gp_seq, (long)rnp->completedqs);
  1122. for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
  1123. pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx ->rcu_gp_init_mask %#lx\n",
  1124. __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext, rnp1->rcu_gp_init_mask);
  1125. onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
  1126. pr_info("%s %d: %c online: %ld(%d) offline: %ld(%d)\n",
  1127. __func__, rdp->cpu, ".o"[onl],
  1128. (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
  1129. (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
  1130. return 1; /* Break things loose after complaining. */
  1131. }
  1132. /*
  1133. * A CPU running for an extended time within the kernel can
  1134. * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
  1135. * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
  1136. * both .rcu_need_heavy_qs and .rcu_urgent_qs. Note that the
  1137. * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
  1138. * variable are safe because the assignments are repeated if this
  1139. * CPU failed to pass through a quiescent state. This code
  1140. * also checks .jiffies_resched in case jiffies_to_sched_qs
  1141. * is set way high.
  1142. */
  1143. jtsq = READ_ONCE(jiffies_to_sched_qs);
  1144. ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
  1145. rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
  1146. if (!READ_ONCE(*rnhqp) &&
  1147. (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||
  1148. time_after(jiffies, rcu_state.jiffies_resched) ||
  1149. rcu_state.cbovld)) {
  1150. WRITE_ONCE(*rnhqp, true);
  1151. /* Store rcu_need_heavy_qs before rcu_urgent_qs. */
  1152. smp_store_release(ruqp, true);
  1153. } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
  1154. WRITE_ONCE(*ruqp, true);
  1155. }
  1156. /*
  1157. * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
  1158. * The above code handles this, but only for straight cond_resched().
  1159. * And some in-kernel loops check need_resched() before calling
  1160. * cond_resched(), which defeats the above code for CPUs that are
  1161. * running in-kernel with scheduling-clock interrupts disabled.
  1162. * So hit them over the head with the resched_cpu() hammer!
  1163. */
  1164. if (tick_nohz_full_cpu(rdp->cpu) &&
  1165. (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) ||
  1166. rcu_state.cbovld)) {
  1167. WRITE_ONCE(*ruqp, true);
  1168. resched_cpu(rdp->cpu);
  1169. WRITE_ONCE(rdp->last_fqs_resched, jiffies);
  1170. }
  1171. /*
  1172. * If more than halfway to RCU CPU stall-warning time, invoke
  1173. * resched_cpu() more frequently to try to loosen things up a bit.
  1174. * Also check to see if the CPU is getting hammered with interrupts,
  1175. * but only once per grace period, just to keep the IPIs down to
  1176. * a dull roar.
  1177. */
  1178. if (time_after(jiffies, rcu_state.jiffies_resched)) {
  1179. if (time_after(jiffies,
  1180. READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
  1181. resched_cpu(rdp->cpu);
  1182. WRITE_ONCE(rdp->last_fqs_resched, jiffies);
  1183. }
  1184. if (IS_ENABLED(CONFIG_IRQ_WORK) &&
  1185. !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
  1186. (rnp->ffmask & rdp->grpmask)) {
  1187. init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
  1188. atomic_set(&rdp->rcu_iw.flags, IRQ_WORK_HARD_IRQ);
  1189. rdp->rcu_iw_pending = true;
  1190. rdp->rcu_iw_gp_seq = rnp->gp_seq;
  1191. irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
  1192. }
  1193. }
  1194. return 0;
  1195. }
  1196. /* Trace-event wrapper function for trace_rcu_future_grace_period. */
  1197. static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
  1198. unsigned long gp_seq_req, const char *s)
  1199. {
  1200. trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
  1201. gp_seq_req, rnp->level,
  1202. rnp->grplo, rnp->grphi, s);
  1203. }
  1204. /*
  1205. * rcu_start_this_gp - Request the start of a particular grace period
  1206. * @rnp_start: The leaf node of the CPU from which to start.
  1207. * @rdp: The rcu_data corresponding to the CPU from which to start.
  1208. * @gp_seq_req: The gp_seq of the grace period to start.
  1209. *
  1210. * Start the specified grace period, as needed to handle newly arrived
  1211. * callbacks. The required future grace periods are recorded in each
  1212. * rcu_node structure's ->gp_seq_needed field. Returns true if there
  1213. * is reason to awaken the grace-period kthread.
  1214. *
  1215. * The caller must hold the specified rcu_node structure's ->lock, which
  1216. * is why the caller is responsible for waking the grace-period kthread.
  1217. *
  1218. * Returns true if the GP thread needs to be awakened else false.
  1219. */
  1220. static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp,
  1221. unsigned long gp_seq_req)
  1222. {
  1223. bool ret = false;
  1224. struct rcu_node *rnp;
  1225. /*
  1226. * Use funnel locking to either acquire the root rcu_node
  1227. * structure's lock or bail out if the need for this grace period
  1228. * has already been recorded -- or if that grace period has in
  1229. * fact already started. If there is already a grace period in
  1230. * progress in a non-leaf node, no recording is needed because the
  1231. * end of the grace period will scan the leaf rcu_node structures.
  1232. * Note that rnp_start->lock must not be released.
  1233. */
  1234. raw_lockdep_assert_held_rcu_node(rnp_start);
  1235. trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf"));
  1236. for (rnp = rnp_start; 1; rnp = rnp->parent) {
  1237. if (rnp != rnp_start)
  1238. raw_spin_lock_rcu_node(rnp);
  1239. if (ULONG_CMP_GE(rnp->gp_seq_needed, gp_seq_req) ||
  1240. rcu_seq_started(&rnp->gp_seq, gp_seq_req) ||
  1241. (rnp != rnp_start &&
  1242. rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) {
  1243. trace_rcu_this_gp(rnp, rdp, gp_seq_req,
  1244. TPS("Prestarted"));
  1245. goto unlock_out;
  1246. }
  1247. WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
  1248. if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
  1249. /*
  1250. * We just marked the leaf or internal node, and a
  1251. * grace period is in progress, which means that
  1252. * rcu_gp_cleanup() will see the marking. Bail to
  1253. * reduce contention.
  1254. */
  1255. trace_rcu_this_gp(rnp_start, rdp, gp_seq_req,
  1256. TPS("Startedleaf"));
  1257. goto unlock_out;
  1258. }
  1259. if (rnp != rnp_start && rnp->parent != NULL)
  1260. raw_spin_unlock_rcu_node(rnp);
  1261. if (!rnp->parent)
  1262. break; /* At root, and perhaps also leaf. */
  1263. }
  1264. /* If GP already in progress, just leave, otherwise start one. */
  1265. if (rcu_gp_in_progress()) {
  1266. trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
  1267. goto unlock_out;
  1268. }
  1269. trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
  1270. WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);
  1271. WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
  1272. if (!READ_ONCE(rcu_state.gp_kthread)) {
  1273. trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
  1274. goto unlock_out;
  1275. }
  1276. trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
  1277. ret = true; /* Caller must wake GP kthread. */
  1278. unlock_out:
  1279. /* Push furthest requested GP to leaf node and rcu_data structure. */
  1280. if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
  1281. WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
  1282. WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
  1283. }
  1284. if (rnp != rnp_start)
  1285. raw_spin_unlock_rcu_node(rnp);
  1286. return ret;
  1287. }
  1288. /*
  1289. * Clean up any old requests for the just-ended grace period. Also return
  1290. * whether any additional grace periods have been requested.
  1291. */
  1292. static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
  1293. {
  1294. bool needmore;
  1295. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  1296. needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
  1297. if (!needmore)
  1298. rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */
  1299. trace_rcu_this_gp(rnp, rdp, rnp->gp_seq,
  1300. needmore ? TPS("CleanupMore") : TPS("Cleanup"));
  1301. return needmore;
  1302. }
  1303. /*
  1304. * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
  1305. * interrupt or softirq handler, in which case we just might immediately
  1306. * sleep upon return, resulting in a grace-period hang), and don't bother
  1307. * awakening when there is nothing for the grace-period kthread to do
  1308. * (as in several CPUs raced to awaken, we lost), and finally don't try
  1309. * to awaken a kthread that has not yet been created. If all those checks
  1310. * are passed, track some debug information and awaken.
  1311. *
  1312. * So why do the self-wakeup when in an interrupt or softirq handler
  1313. * in the grace-period kthread's context? Because the kthread might have
  1314. * been interrupted just as it was going to sleep, and just after the final
  1315. * pre-sleep check of the awaken condition. In this case, a wakeup really
  1316. * is required, and is therefore supplied.
  1317. */
  1318. static void rcu_gp_kthread_wake(void)
  1319. {
  1320. struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
  1321. if ((current == t && !in_irq() && !in_serving_softirq()) ||
  1322. !READ_ONCE(rcu_state.gp_flags) || !t)
  1323. return;
  1324. WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
  1325. WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
  1326. swake_up_one(&rcu_state.gp_wq);
  1327. }
  1328. /*
  1329. * If there is room, assign a ->gp_seq number to any callbacks on this
  1330. * CPU that have not already been assigned. Also accelerate any callbacks
  1331. * that were previously assigned a ->gp_seq number that has since proven
  1332. * to be too conservative, which can happen if callbacks get assigned a
  1333. * ->gp_seq number while RCU is idle, but with reference to a non-root
  1334. * rcu_node structure. This function is idempotent, so it does not hurt
  1335. * to call it repeatedly. Returns an flag saying that we should awaken
  1336. * the RCU grace-period kthread.
  1337. *
  1338. * The caller must hold rnp->lock with interrupts disabled.
  1339. */
  1340. static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
  1341. {
  1342. unsigned long gp_seq_req;
  1343. bool ret = false;
  1344. rcu_lockdep_assert_cblist_protected(rdp);
  1345. raw_lockdep_assert_held_rcu_node(rnp);
  1346. /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
  1347. if (!rcu_segcblist_pend_cbs(&rdp->cblist))
  1348. return false;
  1349. /*
  1350. * Callbacks are often registered with incomplete grace-period
  1351. * information. Something about the fact that getting exact
  1352. * information requires acquiring a global lock... RCU therefore
  1353. * makes a conservative estimate of the grace period number at which
  1354. * a given callback will become ready to invoke. The following
  1355. * code checks this estimate and improves it when possible, thus
  1356. * accelerating callback invocation to an earlier grace-period
  1357. * number.
  1358. */
  1359. gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
  1360. if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
  1361. ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
  1362. /* Trace depending on how much we were able to accelerate. */
  1363. if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
  1364. trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));
  1365. else
  1366. trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));
  1367. return ret;
  1368. }
  1369. /*
  1370. * Similar to rcu_accelerate_cbs(), but does not require that the leaf
  1371. * rcu_node structure's ->lock be held. It consults the cached value
  1372. * of ->gp_seq_needed in the rcu_data structure, and if that indicates
  1373. * that a new grace-period request be made, invokes rcu_accelerate_cbs()
  1374. * while holding the leaf rcu_node structure's ->lock.
  1375. */
  1376. static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
  1377. struct rcu_data *rdp)
  1378. {
  1379. unsigned long c;
  1380. bool needwake;
  1381. rcu_lockdep_assert_cblist_protected(rdp);
  1382. c = rcu_seq_snap(&rcu_state.gp_seq);
  1383. if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
  1384. /* Old request still live, so mark recent callbacks. */
  1385. (void)rcu_segcblist_accelerate(&rdp->cblist, c);
  1386. return;
  1387. }
  1388. raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
  1389. needwake = rcu_accelerate_cbs(rnp, rdp);
  1390. raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
  1391. if (needwake)
  1392. rcu_gp_kthread_wake();
  1393. }
  1394. /*
  1395. * Move any callbacks whose grace period has completed to the
  1396. * RCU_DONE_TAIL sublist, then compact the remaining sublists and
  1397. * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL
  1398. * sublist. This function is idempotent, so it does not hurt to
  1399. * invoke it repeatedly. As long as it is not invoked -too- often...
  1400. * Returns true if the RCU grace-period kthread needs to be awakened.
  1401. *
  1402. * The caller must hold rnp->lock with interrupts disabled.
  1403. */
  1404. static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
  1405. {
  1406. rcu_lockdep_assert_cblist_protected(rdp);
  1407. raw_lockdep_assert_held_rcu_node(rnp);
  1408. /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
  1409. if (!rcu_segcblist_pend_cbs(&rdp->cblist))
  1410. return false;
  1411. /*
  1412. * Find all callbacks whose ->gp_seq numbers indicate that they
  1413. * are ready to invoke, and put them into the RCU_DONE_TAIL sublist.
  1414. */
  1415. rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
  1416. /* Classify any remaining callbacks. */
  1417. return rcu_accelerate_cbs(rnp, rdp);
  1418. }
  1419. /*
  1420. * Move and classify callbacks, but only if doing so won't require
  1421. * that the RCU grace-period kthread be awakened.
  1422. */
  1423. static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
  1424. struct rcu_data *rdp)
  1425. {
  1426. rcu_lockdep_assert_cblist_protected(rdp);
  1427. if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp))
  1428. return;
  1429. // The grace period cannot end while we hold the rcu_node lock.
  1430. if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
  1431. WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
  1432. raw_spin_unlock_rcu_node(rnp);
  1433. }
  1434. /*
  1435. * In CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels, attempt to generate a
  1436. * quiescent state. This is intended to be invoked when the CPU notices
  1437. * a new grace period.
  1438. */
  1439. static void rcu_strict_gp_check_qs(void)
  1440. {
  1441. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
  1442. rcu_read_lock();
  1443. rcu_read_unlock();
  1444. }
  1445. }
  1446. /*
  1447. * Update CPU-local rcu_data state to record the beginnings and ends of
  1448. * grace periods. The caller must hold the ->lock of the leaf rcu_node
  1449. * structure corresponding to the current CPU, and must have irqs disabled.
  1450. * Returns true if the grace-period kthread needs to be awakened.
  1451. */
  1452. static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
  1453. {
  1454. bool ret = false;
  1455. bool need_qs;
  1456. const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  1457. rcu_segcblist_is_offloaded(&rdp->cblist);
  1458. raw_lockdep_assert_held_rcu_node(rnp);
  1459. if (rdp->gp_seq == rnp->gp_seq)
  1460. return false; /* Nothing to do. */
  1461. /* Handle the ends of any preceding grace periods first. */
  1462. if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||
  1463. unlikely(READ_ONCE(rdp->gpwrap))) {
  1464. if (!offloaded)
  1465. ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
  1466. rdp->core_needs_qs = false;
  1467. trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
  1468. } else {
  1469. if (!offloaded)
  1470. ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
  1471. if (rdp->core_needs_qs)
  1472. rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
  1473. }
  1474. /* Now handle the beginnings of any new-to-this-CPU grace periods. */
  1475. if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||
  1476. unlikely(READ_ONCE(rdp->gpwrap))) {
  1477. /*
  1478. * If the current grace period is waiting for this CPU,
  1479. * set up to detect a quiescent state, otherwise don't
  1480. * go looking for one.
  1481. */
  1482. trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
  1483. need_qs = !!(rnp->qsmask & rdp->grpmask);
  1484. rdp->cpu_no_qs.b.norm = need_qs;
  1485. rdp->core_needs_qs = need_qs;
  1486. zero_cpu_stall_ticks(rdp);
  1487. }
  1488. rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
  1489. if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)
  1490. WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
  1491. WRITE_ONCE(rdp->gpwrap, false);
  1492. rcu_gpnum_ovf(rnp, rdp);
  1493. return ret;
  1494. }
  1495. static void note_gp_changes(struct rcu_data *rdp)
  1496. {
  1497. unsigned long flags;
  1498. bool needwake;
  1499. struct rcu_node *rnp;
  1500. local_irq_save(flags);
  1501. rnp = rdp->mynode;
  1502. if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) &&
  1503. !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */
  1504. !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */
  1505. local_irq_restore(flags);
  1506. return;
  1507. }
  1508. needwake = __note_gp_changes(rnp, rdp);
  1509. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1510. rcu_strict_gp_check_qs();
  1511. if (needwake)
  1512. rcu_gp_kthread_wake();
  1513. }
  1514. static void rcu_gp_slow(int delay)
  1515. {
  1516. if (delay > 0 &&
  1517. !(rcu_seq_ctr(rcu_state.gp_seq) %
  1518. (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
  1519. schedule_timeout_idle(delay);
  1520. }
  1521. static unsigned long sleep_duration;
  1522. /* Allow rcutorture to stall the grace-period kthread. */
  1523. void rcu_gp_set_torture_wait(int duration)
  1524. {
  1525. if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST) && duration > 0)
  1526. WRITE_ONCE(sleep_duration, duration);
  1527. }
  1528. EXPORT_SYMBOL_GPL(rcu_gp_set_torture_wait);
  1529. /* Actually implement the aforementioned wait. */
  1530. static void rcu_gp_torture_wait(void)
  1531. {
  1532. unsigned long duration;
  1533. if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST))
  1534. return;
  1535. duration = xchg(&sleep_duration, 0UL);
  1536. if (duration > 0) {
  1537. pr_alert("%s: Waiting %lu jiffies\n", __func__, duration);
  1538. schedule_timeout_idle(duration);
  1539. pr_alert("%s: Wait complete\n", __func__);
  1540. }
  1541. }
  1542. /*
  1543. * Handler for on_each_cpu() to invoke the target CPU's RCU core
  1544. * processing.
  1545. */
  1546. static void rcu_strict_gp_boundary(void *unused)
  1547. {
  1548. invoke_rcu_core();
  1549. }
  1550. /*
  1551. * Initialize a new grace period. Return false if no grace period required.
  1552. */
  1553. static bool rcu_gp_init(void)
  1554. {
  1555. unsigned long flags;
  1556. unsigned long oldmask;
  1557. unsigned long mask;
  1558. struct rcu_data *rdp;
  1559. struct rcu_node *rnp = rcu_get_root();
  1560. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1561. raw_spin_lock_irq_rcu_node(rnp);
  1562. if (!READ_ONCE(rcu_state.gp_flags)) {
  1563. /* Spurious wakeup, tell caller to go back to sleep. */
  1564. raw_spin_unlock_irq_rcu_node(rnp);
  1565. return false;
  1566. }
  1567. WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
  1568. if (WARN_ON_ONCE(rcu_gp_in_progress())) {
  1569. /*
  1570. * Grace period already in progress, don't start another.
  1571. * Not supposed to be able to happen.
  1572. */
  1573. raw_spin_unlock_irq_rcu_node(rnp);
  1574. return false;
  1575. }
  1576. /* Advance to a new grace period and initialize state. */
  1577. record_gp_stall_check_time();
  1578. /* Record GP times before starting GP, hence rcu_seq_start(). */
  1579. rcu_seq_start(&rcu_state.gp_seq);
  1580. ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
  1581. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
  1582. raw_spin_unlock_irq_rcu_node(rnp);
  1583. /*
  1584. * Apply per-leaf buffered online and offline operations to
  1585. * the rcu_node tree. Note that this new grace period need not
  1586. * wait for subsequent online CPUs, and that RCU hooks in the CPU
  1587. * offlining path, when combined with checks in this function,
  1588. * will handle CPUs that are currently going offline or that will
  1589. * go offline later. Please also refer to "Hotplug CPU" section
  1590. * of RCU's Requirements documentation.
  1591. */
  1592. rcu_state.gp_state = RCU_GP_ONOFF;
  1593. rcu_for_each_leaf_node(rnp) {
  1594. raw_spin_lock(&rcu_state.ofl_lock);
  1595. raw_spin_lock_irq_rcu_node(rnp);
  1596. if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
  1597. !rnp->wait_blkd_tasks) {
  1598. /* Nothing to do on this leaf rcu_node structure. */
  1599. raw_spin_unlock_irq_rcu_node(rnp);
  1600. raw_spin_unlock(&rcu_state.ofl_lock);
  1601. continue;
  1602. }
  1603. /* Record old state, apply changes to ->qsmaskinit field. */
  1604. oldmask = rnp->qsmaskinit;
  1605. rnp->qsmaskinit = rnp->qsmaskinitnext;
  1606. /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
  1607. if (!oldmask != !rnp->qsmaskinit) {
  1608. if (!oldmask) { /* First online CPU for rcu_node. */
  1609. if (!rnp->wait_blkd_tasks) /* Ever offline? */
  1610. rcu_init_new_rnp(rnp);
  1611. } else if (rcu_preempt_has_tasks(rnp)) {
  1612. rnp->wait_blkd_tasks = true; /* blocked tasks */
  1613. } else { /* Last offline CPU and can propagate. */
  1614. rcu_cleanup_dead_rnp(rnp);
  1615. }
  1616. }
  1617. /*
  1618. * If all waited-on tasks from prior grace period are
  1619. * done, and if all this rcu_node structure's CPUs are
  1620. * still offline, propagate up the rcu_node tree and
  1621. * clear ->wait_blkd_tasks. Otherwise, if one of this
  1622. * rcu_node structure's CPUs has since come back online,
  1623. * simply clear ->wait_blkd_tasks.
  1624. */
  1625. if (rnp->wait_blkd_tasks &&
  1626. (!rcu_preempt_has_tasks(rnp) || rnp->qsmaskinit)) {
  1627. rnp->wait_blkd_tasks = false;
  1628. if (!rnp->qsmaskinit)
  1629. rcu_cleanup_dead_rnp(rnp);
  1630. }
  1631. raw_spin_unlock_irq_rcu_node(rnp);
  1632. raw_spin_unlock(&rcu_state.ofl_lock);
  1633. }
  1634. rcu_gp_slow(gp_preinit_delay); /* Races with CPU hotplug. */
  1635. /*
  1636. * Set the quiescent-state-needed bits in all the rcu_node
  1637. * structures for all currently online CPUs in breadth-first
  1638. * order, starting from the root rcu_node structure, relying on the
  1639. * layout of the tree within the rcu_state.node[] array. Note that
  1640. * other CPUs will access only the leaves of the hierarchy, thus
  1641. * seeing that no grace period is in progress, at least until the
  1642. * corresponding leaf node has been initialized.
  1643. *
  1644. * The grace period cannot complete until the initialization
  1645. * process finishes, because this kthread handles both.
  1646. */
  1647. rcu_state.gp_state = RCU_GP_INIT;
  1648. rcu_for_each_node_breadth_first(rnp) {
  1649. rcu_gp_slow(gp_init_delay);
  1650. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  1651. rdp = this_cpu_ptr(&rcu_data);
  1652. rcu_preempt_check_blocked_tasks(rnp);
  1653. rnp->qsmask = rnp->qsmaskinit;
  1654. WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
  1655. if (rnp == rdp->mynode)
  1656. (void)__note_gp_changes(rnp, rdp);
  1657. rcu_preempt_boost_start_gp(rnp);
  1658. trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
  1659. rnp->level, rnp->grplo,
  1660. rnp->grphi, rnp->qsmask);
  1661. /* Quiescent states for tasks on any now-offline CPUs. */
  1662. mask = rnp->qsmask & ~rnp->qsmaskinitnext;
  1663. rnp->rcu_gp_init_mask = mask;
  1664. if ((mask || rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
  1665. rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
  1666. else
  1667. raw_spin_unlock_irq_rcu_node(rnp);
  1668. cond_resched_tasks_rcu_qs();
  1669. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1670. }
  1671. // If strict, make all CPUs aware of new grace period.
  1672. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
  1673. on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
  1674. return true;
  1675. }
  1676. /*
  1677. * Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
  1678. * time.
  1679. */
  1680. static bool rcu_gp_fqs_check_wake(int *gfp)
  1681. {
  1682. struct rcu_node *rnp = rcu_get_root();
  1683. // If under overload conditions, force an immediate FQS scan.
  1684. if (*gfp & RCU_GP_FLAG_OVLD)
  1685. return true;
  1686. // Someone like call_rcu() requested a force-quiescent-state scan.
  1687. *gfp = READ_ONCE(rcu_state.gp_flags);
  1688. if (*gfp & RCU_GP_FLAG_FQS)
  1689. return true;
  1690. // The current grace period has completed.
  1691. if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
  1692. return true;
  1693. return false;
  1694. }
  1695. /*
  1696. * Do one round of quiescent-state forcing.
  1697. */
  1698. static void rcu_gp_fqs(bool first_time)
  1699. {
  1700. struct rcu_node *rnp = rcu_get_root();
  1701. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1702. WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
  1703. if (first_time) {
  1704. /* Collect dyntick-idle snapshots. */
  1705. force_qs_rnp(dyntick_save_progress_counter);
  1706. } else {
  1707. /* Handle dyntick-idle and offline CPUs. */
  1708. force_qs_rnp(rcu_implicit_dynticks_qs);
  1709. }
  1710. /* Clear flag to prevent immediate re-entry. */
  1711. if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
  1712. raw_spin_lock_irq_rcu_node(rnp);
  1713. WRITE_ONCE(rcu_state.gp_flags,
  1714. READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
  1715. raw_spin_unlock_irq_rcu_node(rnp);
  1716. }
  1717. }
  1718. /*
  1719. * Loop doing repeated quiescent-state forcing until the grace period ends.
  1720. */
  1721. static void rcu_gp_fqs_loop(void)
  1722. {
  1723. bool first_gp_fqs;
  1724. int gf = 0;
  1725. unsigned long j;
  1726. int ret;
  1727. struct rcu_node *rnp = rcu_get_root();
  1728. first_gp_fqs = true;
  1729. j = READ_ONCE(jiffies_till_first_fqs);
  1730. if (rcu_state.cbovld)
  1731. gf = RCU_GP_FLAG_OVLD;
  1732. ret = 0;
  1733. for (;;) {
  1734. if (!ret) {
  1735. rcu_state.jiffies_force_qs = jiffies + j;
  1736. WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
  1737. jiffies + (j ? 3 * j : 2));
  1738. }
  1739. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
  1740. TPS("fqswait"));
  1741. rcu_state.gp_state = RCU_GP_WAIT_FQS;
  1742. ret = swait_event_idle_timeout_exclusive(
  1743. rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
  1744. rcu_gp_torture_wait();
  1745. rcu_state.gp_state = RCU_GP_DOING_FQS;
  1746. /* Locking provides needed memory barriers. */
  1747. /* If grace period done, leave loop. */
  1748. if (!READ_ONCE(rnp->qsmask) &&
  1749. !rcu_preempt_blocked_readers_cgp(rnp))
  1750. break;
  1751. /* If time for quiescent-state forcing, do it. */
  1752. if (!time_after(rcu_state.jiffies_force_qs, jiffies) ||
  1753. (gf & (RCU_GP_FLAG_FQS | RCU_GP_FLAG_OVLD))) {
  1754. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
  1755. TPS("fqsstart"));
  1756. rcu_gp_fqs(first_gp_fqs);
  1757. gf = 0;
  1758. if (first_gp_fqs) {
  1759. first_gp_fqs = false;
  1760. gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
  1761. }
  1762. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
  1763. TPS("fqsend"));
  1764. cond_resched_tasks_rcu_qs();
  1765. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1766. ret = 0; /* Force full wait till next FQS. */
  1767. j = READ_ONCE(jiffies_till_next_fqs);
  1768. } else {
  1769. /* Deal with stray signal. */
  1770. cond_resched_tasks_rcu_qs();
  1771. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1772. WARN_ON(signal_pending(current));
  1773. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
  1774. TPS("fqswaitsig"));
  1775. ret = 1; /* Keep old FQS timing. */
  1776. j = jiffies;
  1777. if (time_after(jiffies, rcu_state.jiffies_force_qs))
  1778. j = 1;
  1779. else
  1780. j = rcu_state.jiffies_force_qs - j;
  1781. gf = 0;
  1782. }
  1783. }
  1784. }
  1785. /*
  1786. * Clean up after the old grace period.
  1787. */
  1788. static void rcu_gp_cleanup(void)
  1789. {
  1790. int cpu;
  1791. bool needgp = false;
  1792. unsigned long gp_duration;
  1793. unsigned long new_gp_seq;
  1794. bool offloaded;
  1795. struct rcu_data *rdp;
  1796. struct rcu_node *rnp = rcu_get_root();
  1797. struct swait_queue_head *sq;
  1798. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1799. raw_spin_lock_irq_rcu_node(rnp);
  1800. rcu_state.gp_end = jiffies;
  1801. gp_duration = rcu_state.gp_end - rcu_state.gp_start;
  1802. if (gp_duration > rcu_state.gp_max)
  1803. rcu_state.gp_max = gp_duration;
  1804. /*
  1805. * We know the grace period is complete, but to everyone else
  1806. * it appears to still be ongoing. But it is also the case
  1807. * that to everyone else it looks like there is nothing that
  1808. * they can do to advance the grace period. It is therefore
  1809. * safe for us to drop the lock in order to mark the grace
  1810. * period as completed in all of the rcu_node structures.
  1811. */
  1812. raw_spin_unlock_irq_rcu_node(rnp);
  1813. /*
  1814. * Propagate new ->gp_seq value to rcu_node structures so that
  1815. * other CPUs don't have to wait until the start of the next grace
  1816. * period to process their callbacks. This also avoids some nasty
  1817. * RCU grace-period initialization races by forcing the end of
  1818. * the current grace period to be completely recorded in all of
  1819. * the rcu_node structures before the beginning of the next grace
  1820. * period is recorded in any of the rcu_node structures.
  1821. */
  1822. new_gp_seq = rcu_state.gp_seq;
  1823. rcu_seq_end(&new_gp_seq);
  1824. rcu_for_each_node_breadth_first(rnp) {
  1825. raw_spin_lock_irq_rcu_node(rnp);
  1826. if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
  1827. dump_blkd_tasks(rnp, 10);
  1828. WARN_ON_ONCE(rnp->qsmask);
  1829. WRITE_ONCE(rnp->gp_seq, new_gp_seq);
  1830. rdp = this_cpu_ptr(&rcu_data);
  1831. if (rnp == rdp->mynode)
  1832. needgp = __note_gp_changes(rnp, rdp) || needgp;
  1833. /* smp_mb() provided by prior unlock-lock pair. */
  1834. needgp = rcu_future_gp_cleanup(rnp) || needgp;
  1835. // Reset overload indication for CPUs no longer overloaded
  1836. if (rcu_is_leaf_node(rnp))
  1837. for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
  1838. rdp = per_cpu_ptr(&rcu_data, cpu);
  1839. check_cb_ovld_locked(rdp, rnp);
  1840. }
  1841. sq = rcu_nocb_gp_get(rnp);
  1842. raw_spin_unlock_irq_rcu_node(rnp);
  1843. rcu_nocb_gp_cleanup(sq);
  1844. cond_resched_tasks_rcu_qs();
  1845. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1846. rcu_gp_slow(gp_cleanup_delay);
  1847. }
  1848. rnp = rcu_get_root();
  1849. raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
  1850. /* Declare grace period done, trace first to use old GP number. */
  1851. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
  1852. rcu_seq_end(&rcu_state.gp_seq);
  1853. ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
  1854. rcu_state.gp_state = RCU_GP_IDLE;
  1855. /* Check for GP requests since above loop. */
  1856. rdp = this_cpu_ptr(&rcu_data);
  1857. if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
  1858. trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
  1859. TPS("CleanupMore"));
  1860. needgp = true;
  1861. }
  1862. /* Advance CBs to reduce false positives below. */
  1863. offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  1864. rcu_segcblist_is_offloaded(&rdp->cblist);
  1865. if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
  1866. WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
  1867. WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
  1868. trace_rcu_grace_period(rcu_state.name,
  1869. rcu_state.gp_seq,
  1870. TPS("newreq"));
  1871. } else {
  1872. WRITE_ONCE(rcu_state.gp_flags,
  1873. rcu_state.gp_flags & RCU_GP_FLAG_INIT);
  1874. }
  1875. raw_spin_unlock_irq_rcu_node(rnp);
  1876. // If strict, make all CPUs aware of the end of the old grace period.
  1877. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
  1878. on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
  1879. }
  1880. /*
  1881. * Body of kthread that handles grace periods.
  1882. */
  1883. static int __noreturn rcu_gp_kthread(void *unused)
  1884. {
  1885. rcu_bind_gp_kthread();
  1886. for (;;) {
  1887. /* Handle grace-period start. */
  1888. for (;;) {
  1889. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
  1890. TPS("reqwait"));
  1891. rcu_state.gp_state = RCU_GP_WAIT_GPS;
  1892. swait_event_idle_exclusive(rcu_state.gp_wq,
  1893. READ_ONCE(rcu_state.gp_flags) &
  1894. RCU_GP_FLAG_INIT);
  1895. rcu_gp_torture_wait();
  1896. rcu_state.gp_state = RCU_GP_DONE_GPS;
  1897. /* Locking provides needed memory barrier. */
  1898. if (rcu_gp_init())
  1899. break;
  1900. cond_resched_tasks_rcu_qs();
  1901. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  1902. WARN_ON(signal_pending(current));
  1903. trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
  1904. TPS("reqwaitsig"));
  1905. }
  1906. /* Handle quiescent-state forcing. */
  1907. rcu_gp_fqs_loop();
  1908. /* Handle grace-period end. */
  1909. rcu_state.gp_state = RCU_GP_CLEANUP;
  1910. rcu_gp_cleanup();
  1911. rcu_state.gp_state = RCU_GP_CLEANED;
  1912. }
  1913. }
  1914. /*
  1915. * Report a full set of quiescent states to the rcu_state data structure.
  1916. * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
  1917. * another grace period is required. Whether we wake the grace-period
  1918. * kthread or it awakens itself for the next round of quiescent-state
  1919. * forcing, that kthread will clean up after the just-completed grace
  1920. * period. Note that the caller must hold rnp->lock, which is released
  1921. * before return.
  1922. */
  1923. static void rcu_report_qs_rsp(unsigned long flags)
  1924. __releases(rcu_get_root()->lock)
  1925. {
  1926. raw_lockdep_assert_held_rcu_node(rcu_get_root());
  1927. WARN_ON_ONCE(!rcu_gp_in_progress());
  1928. WRITE_ONCE(rcu_state.gp_flags,
  1929. READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
  1930. raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
  1931. rcu_gp_kthread_wake();
  1932. }
  1933. /*
  1934. * Similar to rcu_report_qs_rdp(), for which it is a helper function.
  1935. * Allows quiescent states for a group of CPUs to be reported at one go
  1936. * to the specified rcu_node structure, though all the CPUs in the group
  1937. * must be represented by the same rcu_node structure (which need not be a
  1938. * leaf rcu_node structure, though it often will be). The gps parameter
  1939. * is the grace-period snapshot, which means that the quiescent states
  1940. * are valid only if rnp->gp_seq is equal to gps. That structure's lock
  1941. * must be held upon entry, and it is released before return.
  1942. *
  1943. * As a special case, if mask is zero, the bit-already-cleared check is
  1944. * disabled. This allows propagating quiescent state due to resumed tasks
  1945. * during grace-period initialization.
  1946. */
  1947. static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
  1948. unsigned long gps, unsigned long flags)
  1949. __releases(rnp->lock)
  1950. {
  1951. unsigned long oldmask = 0;
  1952. struct rcu_node *rnp_c;
  1953. raw_lockdep_assert_held_rcu_node(rnp);
  1954. /* Walk up the rcu_node hierarchy. */
  1955. for (;;) {
  1956. if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) {
  1957. /*
  1958. * Our bit has already been cleared, or the
  1959. * relevant grace period is already over, so done.
  1960. */
  1961. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1962. return;
  1963. }
  1964. WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
  1965. WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
  1966. rcu_preempt_blocked_readers_cgp(rnp));
  1967. WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
  1968. trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
  1969. mask, rnp->qsmask, rnp->level,
  1970. rnp->grplo, rnp->grphi,
  1971. !!rnp->gp_tasks);
  1972. if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
  1973. /* Other bits still set at this level, so done. */
  1974. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1975. return;
  1976. }
  1977. rnp->completedqs = rnp->gp_seq;
  1978. mask = rnp->grpmask;
  1979. if (rnp->parent == NULL) {
  1980. /* No more levels. Exit loop holding root lock. */
  1981. break;
  1982. }
  1983. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  1984. rnp_c = rnp;
  1985. rnp = rnp->parent;
  1986. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  1987. oldmask = READ_ONCE(rnp_c->qsmask);
  1988. }
  1989. /*
  1990. * Get here if we are the last CPU to pass through a quiescent
  1991. * state for this grace period. Invoke rcu_report_qs_rsp()
  1992. * to clean up and start the next grace period if one is needed.
  1993. */
  1994. rcu_report_qs_rsp(flags); /* releases rnp->lock. */
  1995. }
  1996. /*
  1997. * Record a quiescent state for all tasks that were previously queued
  1998. * on the specified rcu_node structure and that were blocking the current
  1999. * RCU grace period. The caller must hold the corresponding rnp->lock with
  2000. * irqs disabled, and this lock is released upon return, but irqs remain
  2001. * disabled.
  2002. */
  2003. static void __maybe_unused
  2004. rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
  2005. __releases(rnp->lock)
  2006. {
  2007. unsigned long gps;
  2008. unsigned long mask;
  2009. struct rcu_node *rnp_p;
  2010. raw_lockdep_assert_held_rcu_node(rnp);
  2011. if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) ||
  2012. WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) ||
  2013. rnp->qsmask != 0) {
  2014. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  2015. return; /* Still need more quiescent states! */
  2016. }
  2017. rnp->completedqs = rnp->gp_seq;
  2018. rnp_p = rnp->parent;
  2019. if (rnp_p == NULL) {
  2020. /*
  2021. * Only one rcu_node structure in the tree, so don't
  2022. * try to report up to its nonexistent parent!
  2023. */
  2024. rcu_report_qs_rsp(flags);
  2025. return;
  2026. }
  2027. /* Report up the rest of the hierarchy, tracking current ->gp_seq. */
  2028. gps = rnp->gp_seq;
  2029. mask = rnp->grpmask;
  2030. raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
  2031. raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
  2032. rcu_report_qs_rnp(mask, rnp_p, gps, flags);
  2033. }
  2034. /*
  2035. * Record a quiescent state for the specified CPU to that CPU's rcu_data
  2036. * structure. This must be called from the specified CPU.
  2037. */
  2038. static void
  2039. rcu_report_qs_rdp(struct rcu_data *rdp)
  2040. {
  2041. unsigned long flags;
  2042. unsigned long mask;
  2043. bool needwake = false;
  2044. const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  2045. rcu_segcblist_is_offloaded(&rdp->cblist);
  2046. struct rcu_node *rnp;
  2047. WARN_ON_ONCE(rdp->cpu != smp_processor_id());
  2048. rnp = rdp->mynode;
  2049. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  2050. if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq ||
  2051. rdp->gpwrap) {
  2052. /*
  2053. * The grace period in which this quiescent state was
  2054. * recorded has ended, so don't report it upwards.
  2055. * We will instead need a new quiescent state that lies
  2056. * within the current grace period.
  2057. */
  2058. rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
  2059. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  2060. return;
  2061. }
  2062. mask = rdp->grpmask;
  2063. rdp->core_needs_qs = false;
  2064. if ((rnp->qsmask & mask) == 0) {
  2065. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  2066. } else {
  2067. /*
  2068. * This GP can't end until cpu checks in, so all of our
  2069. * callbacks can be processed during the next GP.
  2070. */
  2071. if (!offloaded)
  2072. needwake = rcu_accelerate_cbs(rnp, rdp);
  2073. rcu_disable_urgency_upon_qs(rdp);
  2074. rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
  2075. /* ^^^ Released rnp->lock */
  2076. if (needwake)
  2077. rcu_gp_kthread_wake();
  2078. }
  2079. }
  2080. /*
  2081. * Check to see if there is a new grace period of which this CPU
  2082. * is not yet aware, and if so, set up local rcu_data state for it.
  2083. * Otherwise, see if this CPU has just passed through its first
  2084. * quiescent state for this grace period, and record that fact if so.
  2085. */
  2086. static void
  2087. rcu_check_quiescent_state(struct rcu_data *rdp)
  2088. {
  2089. /* Check for grace-period ends and beginnings. */
  2090. note_gp_changes(rdp);
  2091. /*
  2092. * Does this CPU still need to do its part for current grace period?
  2093. * If no, return and let the other CPUs do their part as well.
  2094. */
  2095. if (!rdp->core_needs_qs)
  2096. return;
  2097. /*
  2098. * Was there a quiescent state since the beginning of the grace
  2099. * period? If no, then exit and wait for the next call.
  2100. */
  2101. if (rdp->cpu_no_qs.b.norm)
  2102. return;
  2103. /*
  2104. * Tell RCU we are done (but rcu_report_qs_rdp() will be the
  2105. * judge of that).
  2106. */
  2107. rcu_report_qs_rdp(rdp);
  2108. }
  2109. /*
  2110. * Near the end of the offline process. Trace the fact that this CPU
  2111. * is going offline.
  2112. */
  2113. int rcutree_dying_cpu(unsigned int cpu)
  2114. {
  2115. bool blkd;
  2116. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  2117. struct rcu_node *rnp = rdp->mynode;
  2118. if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
  2119. return 0;
  2120. blkd = !!(rnp->qsmask & rdp->grpmask);
  2121. trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
  2122. blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
  2123. return 0;
  2124. }
  2125. /*
  2126. * All CPUs for the specified rcu_node structure have gone offline,
  2127. * and all tasks that were preempted within an RCU read-side critical
  2128. * section while running on one of those CPUs have since exited their RCU
  2129. * read-side critical section. Some other CPU is reporting this fact with
  2130. * the specified rcu_node structure's ->lock held and interrupts disabled.
  2131. * This function therefore goes up the tree of rcu_node structures,
  2132. * clearing the corresponding bits in the ->qsmaskinit fields. Note that
  2133. * the leaf rcu_node structure's ->qsmaskinit field has already been
  2134. * updated.
  2135. *
  2136. * This function does check that the specified rcu_node structure has
  2137. * all CPUs offline and no blocked tasks, so it is OK to invoke it
  2138. * prematurely. That said, invoking it after the fact will cost you
  2139. * a needless lock acquisition. So once it has done its work, don't
  2140. * invoke it again.
  2141. */
  2142. static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
  2143. {
  2144. long mask;
  2145. struct rcu_node *rnp = rnp_leaf;
  2146. raw_lockdep_assert_held_rcu_node(rnp_leaf);
  2147. if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
  2148. WARN_ON_ONCE(rnp_leaf->qsmaskinit) ||
  2149. WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf)))
  2150. return;
  2151. for (;;) {
  2152. mask = rnp->grpmask;
  2153. rnp = rnp->parent;
  2154. if (!rnp)
  2155. break;
  2156. raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
  2157. rnp->qsmaskinit &= ~mask;
  2158. /* Between grace periods, so better already be zero! */
  2159. WARN_ON_ONCE(rnp->qsmask);
  2160. if (rnp->qsmaskinit) {
  2161. raw_spin_unlock_rcu_node(rnp);
  2162. /* irqs remain disabled. */
  2163. return;
  2164. }
  2165. raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
  2166. }
  2167. }
  2168. /*
  2169. * The CPU has been completely removed, and some other CPU is reporting
  2170. * this fact from process context. Do the remainder of the cleanup.
  2171. * There can only be one CPU hotplug operation at a time, so no need for
  2172. * explicit locking.
  2173. */
  2174. int rcutree_dead_cpu(unsigned int cpu)
  2175. {
  2176. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  2177. struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
  2178. if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
  2179. return 0;
  2180. /* Adjust any no-longer-needed kthreads. */
  2181. rcu_boost_kthread_setaffinity(rnp, -1);
  2182. /* Do any needed no-CB deferred wakeups from this CPU. */
  2183. do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
  2184. // Stop-machine done, so allow nohz_full to disable tick.
  2185. tick_dep_clear(TICK_DEP_BIT_RCU);
  2186. return 0;
  2187. }
  2188. /*
  2189. * Invoke any RCU callbacks that have made it to the end of their grace
  2190. * period. Thottle as specified by rdp->blimit.
  2191. */
  2192. static void rcu_do_batch(struct rcu_data *rdp)
  2193. {
  2194. int div;
  2195. unsigned long flags;
  2196. const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  2197. rcu_segcblist_is_offloaded(&rdp->cblist);
  2198. struct rcu_head *rhp;
  2199. struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
  2200. long bl, count;
  2201. long pending, tlimit = 0;
  2202. /* If no callbacks are ready, just return. */
  2203. if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
  2204. trace_rcu_batch_start(rcu_state.name,
  2205. rcu_segcblist_n_cbs(&rdp->cblist), 0);
  2206. trace_rcu_batch_end(rcu_state.name, 0,
  2207. !rcu_segcblist_empty(&rdp->cblist),
  2208. need_resched(), is_idle_task(current),
  2209. rcu_is_callbacks_kthread());
  2210. return;
  2211. }
  2212. /*
  2213. * Extract the list of ready callbacks, disabling to prevent
  2214. * races with call_rcu() from interrupt handlers. Leave the
  2215. * callback counts, as rcu_barrier() needs to be conservative.
  2216. */
  2217. local_irq_save(flags);
  2218. rcu_nocb_lock(rdp);
  2219. WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
  2220. pending = rcu_segcblist_n_cbs(&rdp->cblist);
  2221. div = READ_ONCE(rcu_divisor);
  2222. div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
  2223. bl = max(rdp->blimit, pending >> div);
  2224. if (unlikely(bl > 100)) {
  2225. long rrn = READ_ONCE(rcu_resched_ns);
  2226. rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
  2227. tlimit = local_clock() + rrn;
  2228. }
  2229. trace_rcu_batch_start(rcu_state.name,
  2230. rcu_segcblist_n_cbs(&rdp->cblist), bl);
  2231. rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
  2232. if (offloaded)
  2233. rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
  2234. rcu_nocb_unlock_irqrestore(rdp, flags);
  2235. /* Invoke callbacks. */
  2236. tick_dep_set_task(current, TICK_DEP_BIT_RCU);
  2237. rhp = rcu_cblist_dequeue(&rcl);
  2238. for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
  2239. rcu_callback_t f;
  2240. debug_rcu_head_unqueue(rhp);
  2241. rcu_lock_acquire(&rcu_callback_map);
  2242. trace_rcu_invoke_callback(rcu_state.name, rhp);
  2243. f = rhp->func;
  2244. WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
  2245. f(rhp);
  2246. rcu_lock_release(&rcu_callback_map);
  2247. /*
  2248. * Stop only if limit reached and CPU has something to do.
  2249. * Note: The rcl structure counts down from zero.
  2250. */
  2251. if (-rcl.len >= bl && !offloaded &&
  2252. (need_resched() ||
  2253. (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
  2254. break;
  2255. if (unlikely(tlimit)) {
  2256. /* only call local_clock() every 32 callbacks */
  2257. if (likely((-rcl.len & 31) || local_clock() < tlimit))
  2258. continue;
  2259. /* Exceeded the time limit, so leave. */
  2260. break;
  2261. }
  2262. if (offloaded) {
  2263. WARN_ON_ONCE(in_serving_softirq());
  2264. local_bh_enable();
  2265. lockdep_assert_irqs_enabled();
  2266. cond_resched_tasks_rcu_qs();
  2267. lockdep_assert_irqs_enabled();
  2268. local_bh_disable();
  2269. }
  2270. }
  2271. local_irq_save(flags);
  2272. rcu_nocb_lock(rdp);
  2273. count = -rcl.len;
  2274. rdp->n_cbs_invoked += count;
  2275. trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
  2276. is_idle_task(current), rcu_is_callbacks_kthread());
  2277. /* Update counts and requeue any remaining callbacks. */
  2278. rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);
  2279. smp_mb(); /* List handling before counting for rcu_barrier(). */
  2280. rcu_segcblist_insert_count(&rdp->cblist, &rcl);
  2281. /* Reinstate batch limit if we have worked down the excess. */
  2282. count = rcu_segcblist_n_cbs(&rdp->cblist);
  2283. if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
  2284. rdp->blimit = blimit;
  2285. /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
  2286. if (count == 0 && rdp->qlen_last_fqs_check != 0) {
  2287. rdp->qlen_last_fqs_check = 0;
  2288. rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
  2289. } else if (count < rdp->qlen_last_fqs_check - qhimark)
  2290. rdp->qlen_last_fqs_check = count;
  2291. /*
  2292. * The following usually indicates a double call_rcu(). To track
  2293. * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
  2294. */
  2295. WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
  2296. WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  2297. count != 0 && rcu_segcblist_empty(&rdp->cblist));
  2298. rcu_nocb_unlock_irqrestore(rdp, flags);
  2299. /* Re-invoke RCU core processing if there are callbacks remaining. */
  2300. if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
  2301. invoke_rcu_core();
  2302. tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
  2303. }
  2304. /*
  2305. * This function is invoked from each scheduling-clock interrupt,
  2306. * and checks to see if this CPU is in a non-context-switch quiescent
  2307. * state, for example, user mode or idle loop. It also schedules RCU
  2308. * core processing. If the current grace period has gone on too long,
  2309. * it will ask the scheduler to manufacture a context switch for the sole
  2310. * purpose of providing a providing the needed quiescent state.
  2311. */
  2312. void rcu_sched_clock_irq(int user)
  2313. {
  2314. trace_rcu_utilization(TPS("Start scheduler-tick"));
  2315. lockdep_assert_irqs_disabled();
  2316. raw_cpu_inc(rcu_data.ticks_this_gp);
  2317. /* The load-acquire pairs with the store-release setting to true. */
  2318. if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
  2319. /* Idle and userspace execution already are quiescent states. */
  2320. if (!rcu_is_cpu_rrupt_from_idle() && !user) {
  2321. set_tsk_need_resched(current);
  2322. set_preempt_need_resched();
  2323. }
  2324. __this_cpu_write(rcu_data.rcu_urgent_qs, false);
  2325. }
  2326. rcu_flavor_sched_clock_irq(user);
  2327. if (rcu_pending(user))
  2328. invoke_rcu_core();
  2329. lockdep_assert_irqs_disabled();
  2330. trace_rcu_utilization(TPS("End scheduler-tick"));
  2331. }
  2332. /*
  2333. * Scan the leaf rcu_node structures. For each structure on which all
  2334. * CPUs have reported a quiescent state and on which there are tasks
  2335. * blocking the current grace period, initiate RCU priority boosting.
  2336. * Otherwise, invoke the specified function to check dyntick state for
  2337. * each CPU that has not yet reported a quiescent state.
  2338. */
  2339. static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
  2340. {
  2341. int cpu;
  2342. unsigned long flags;
  2343. unsigned long mask;
  2344. struct rcu_data *rdp;
  2345. struct rcu_node *rnp;
  2346. rcu_state.cbovld = rcu_state.cbovldnext;
  2347. rcu_state.cbovldnext = false;
  2348. rcu_for_each_leaf_node(rnp) {
  2349. cond_resched_tasks_rcu_qs();
  2350. mask = 0;
  2351. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  2352. rcu_state.cbovldnext |= !!rnp->cbovldmask;
  2353. if (rnp->qsmask == 0) {
  2354. if (rcu_preempt_blocked_readers_cgp(rnp)) {
  2355. /*
  2356. * No point in scanning bits because they
  2357. * are all zero. But we might need to
  2358. * priority-boost blocked readers.
  2359. */
  2360. rcu_initiate_boost(rnp, flags);
  2361. /* rcu_initiate_boost() releases rnp->lock */
  2362. continue;
  2363. }
  2364. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  2365. continue;
  2366. }
  2367. for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
  2368. rdp = per_cpu_ptr(&rcu_data, cpu);
  2369. if (f(rdp)) {
  2370. mask |= rdp->grpmask;
  2371. rcu_disable_urgency_upon_qs(rdp);
  2372. }
  2373. }
  2374. if (mask != 0) {
  2375. /* Idle/offline CPUs, report (releases rnp->lock). */
  2376. rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
  2377. } else {
  2378. /* Nothing to do here, so just drop the lock. */
  2379. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  2380. }
  2381. }
  2382. }
  2383. /*
  2384. * Force quiescent states on reluctant CPUs, and also detect which
  2385. * CPUs are in dyntick-idle mode.
  2386. */
  2387. void rcu_force_quiescent_state(void)
  2388. {
  2389. unsigned long flags;
  2390. bool ret;
  2391. struct rcu_node *rnp;
  2392. struct rcu_node *rnp_old = NULL;
  2393. /* Funnel through hierarchy to reduce memory contention. */
  2394. rnp = __this_cpu_read(rcu_data.mynode);
  2395. for (; rnp != NULL; rnp = rnp->parent) {
  2396. ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
  2397. !raw_spin_trylock(&rnp->fqslock);
  2398. if (rnp_old != NULL)
  2399. raw_spin_unlock(&rnp_old->fqslock);
  2400. if (ret)
  2401. return;
  2402. rnp_old = rnp;
  2403. }
  2404. /* rnp_old == rcu_get_root(), rnp == NULL. */
  2405. /* Reached the root of the rcu_node tree, acquire lock. */
  2406. raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
  2407. raw_spin_unlock(&rnp_old->fqslock);
  2408. if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
  2409. raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
  2410. return; /* Someone beat us to it. */
  2411. }
  2412. WRITE_ONCE(rcu_state.gp_flags,
  2413. READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
  2414. raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
  2415. rcu_gp_kthread_wake();
  2416. }
  2417. EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
  2418. // Workqueue handler for an RCU reader for kernels enforcing struct RCU
  2419. // grace periods.
  2420. static void strict_work_handler(struct work_struct *work)
  2421. {
  2422. rcu_read_lock();
  2423. rcu_read_unlock();
  2424. }
  2425. /* Perform RCU core processing work for the current CPU. */
  2426. static __latent_entropy void rcu_core(void)
  2427. {
  2428. unsigned long flags;
  2429. struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
  2430. struct rcu_node *rnp = rdp->mynode;
  2431. const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  2432. rcu_segcblist_is_offloaded(&rdp->cblist);
  2433. if (cpu_is_offline(smp_processor_id()))
  2434. return;
  2435. trace_rcu_utilization(TPS("Start RCU core"));
  2436. WARN_ON_ONCE(!rdp->beenonline);
  2437. /* Report any deferred quiescent states if preemption enabled. */
  2438. if (!(preempt_count() & PREEMPT_MASK)) {
  2439. rcu_preempt_deferred_qs(current);
  2440. } else if (rcu_preempt_need_deferred_qs(current)) {
  2441. set_tsk_need_resched(current);
  2442. set_preempt_need_resched();
  2443. }
  2444. /* Update RCU state based on any recent quiescent states. */
  2445. rcu_check_quiescent_state(rdp);
  2446. /* No grace period and unregistered callbacks? */
  2447. if (!rcu_gp_in_progress() &&
  2448. rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
  2449. local_irq_save(flags);
  2450. if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
  2451. rcu_accelerate_cbs_unlocked(rnp, rdp);
  2452. local_irq_restore(flags);
  2453. }
  2454. rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
  2455. /* If there are callbacks ready, invoke them. */
  2456. if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
  2457. likely(READ_ONCE(rcu_scheduler_fully_active)))
  2458. rcu_do_batch(rdp);
  2459. /* Do any needed deferred wakeups of rcuo kthreads. */
  2460. do_nocb_deferred_wakeup(rdp);
  2461. trace_rcu_utilization(TPS("End RCU core"));
  2462. // If strict GPs, schedule an RCU reader in a clean environment.
  2463. if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
  2464. queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work);
  2465. }
  2466. static void rcu_core_si(struct softirq_action *h)
  2467. {
  2468. rcu_core();
  2469. }
  2470. static void rcu_wake_cond(struct task_struct *t, int status)
  2471. {
  2472. /*
  2473. * If the thread is yielding, only wake it when this
  2474. * is invoked from idle
  2475. */
  2476. if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
  2477. wake_up_process(t);
  2478. }
  2479. static void invoke_rcu_core_kthread(void)
  2480. {
  2481. struct task_struct *t;
  2482. unsigned long flags;
  2483. local_irq_save(flags);
  2484. __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
  2485. t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
  2486. if (t != NULL && t != current)
  2487. rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
  2488. local_irq_restore(flags);
  2489. }
  2490. /*
  2491. * Wake up this CPU's rcuc kthread to do RCU core processing.
  2492. */
  2493. static void invoke_rcu_core(void)
  2494. {
  2495. if (!cpu_online(smp_processor_id()))
  2496. return;
  2497. if (use_softirq)
  2498. raise_softirq(RCU_SOFTIRQ);
  2499. else
  2500. invoke_rcu_core_kthread();
  2501. }
  2502. static void rcu_cpu_kthread_park(unsigned int cpu)
  2503. {
  2504. per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
  2505. }
  2506. static int rcu_cpu_kthread_should_run(unsigned int cpu)
  2507. {
  2508. return __this_cpu_read(rcu_data.rcu_cpu_has_work);
  2509. }
  2510. /*
  2511. * Per-CPU kernel thread that invokes RCU callbacks. This replaces
  2512. * the RCU softirq used in configurations of RCU that do not support RCU
  2513. * priority boosting.
  2514. */
  2515. static void rcu_cpu_kthread(unsigned int cpu)
  2516. {
  2517. unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
  2518. char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
  2519. int spincnt;
  2520. trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
  2521. for (spincnt = 0; spincnt < 10; spincnt++) {
  2522. local_bh_disable();
  2523. *statusp = RCU_KTHREAD_RUNNING;
  2524. local_irq_disable();
  2525. work = *workp;
  2526. *workp = 0;
  2527. local_irq_enable();
  2528. if (work)
  2529. rcu_core();
  2530. local_bh_enable();
  2531. if (*workp == 0) {
  2532. trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
  2533. *statusp = RCU_KTHREAD_WAITING;
  2534. return;
  2535. }
  2536. }
  2537. *statusp = RCU_KTHREAD_YIELDING;
  2538. trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
  2539. schedule_timeout_idle(2);
  2540. trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
  2541. *statusp = RCU_KTHREAD_WAITING;
  2542. }
  2543. static struct smp_hotplug_thread rcu_cpu_thread_spec = {
  2544. .store = &rcu_data.rcu_cpu_kthread_task,
  2545. .thread_should_run = rcu_cpu_kthread_should_run,
  2546. .thread_fn = rcu_cpu_kthread,
  2547. .thread_comm = "rcuc/%u",
  2548. .setup = rcu_cpu_kthread_setup,
  2549. .park = rcu_cpu_kthread_park,
  2550. };
  2551. /*
  2552. * Spawn per-CPU RCU core processing kthreads.
  2553. */
  2554. static int __init rcu_spawn_core_kthreads(void)
  2555. {
  2556. int cpu;
  2557. for_each_possible_cpu(cpu)
  2558. per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
  2559. if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
  2560. return 0;
  2561. WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
  2562. "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
  2563. return 0;
  2564. }
  2565. /*
  2566. * Handle any core-RCU processing required by a call_rcu() invocation.
  2567. */
  2568. static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head,
  2569. unsigned long flags)
  2570. {
  2571. /*
  2572. * If called from an extended quiescent state, invoke the RCU
  2573. * core in order to force a re-evaluation of RCU's idleness.
  2574. */
  2575. if (!rcu_is_watching())
  2576. invoke_rcu_core();
  2577. /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
  2578. if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))
  2579. return;
  2580. /*
  2581. * Force the grace period if too many callbacks or too long waiting.
  2582. * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
  2583. * if some other CPU has recently done so. Also, don't bother
  2584. * invoking rcu_force_quiescent_state() if the newly enqueued callback
  2585. * is the only one waiting for a grace period to complete.
  2586. */
  2587. if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
  2588. rdp->qlen_last_fqs_check + qhimark)) {
  2589. /* Are we ignoring a completed grace period? */
  2590. note_gp_changes(rdp);
  2591. /* Start a new grace period if one not already started. */
  2592. if (!rcu_gp_in_progress()) {
  2593. rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
  2594. } else {
  2595. /* Give the grace period a kick. */
  2596. rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
  2597. if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
  2598. rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
  2599. rcu_force_quiescent_state();
  2600. rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
  2601. rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
  2602. }
  2603. }
  2604. }
  2605. /*
  2606. * RCU callback function to leak a callback.
  2607. */
  2608. static void rcu_leak_callback(struct rcu_head *rhp)
  2609. {
  2610. }
  2611. /*
  2612. * Check and if necessary update the leaf rcu_node structure's
  2613. * ->cbovldmask bit corresponding to the current CPU based on that CPU's
  2614. * number of queued RCU callbacks. The caller must hold the leaf rcu_node
  2615. * structure's ->lock.
  2616. */
  2617. static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp)
  2618. {
  2619. raw_lockdep_assert_held_rcu_node(rnp);
  2620. if (qovld_calc <= 0)
  2621. return; // Early boot and wildcard value set.
  2622. if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
  2623. WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask | rdp->grpmask);
  2624. else
  2625. WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
  2626. }
  2627. /*
  2628. * Check and if necessary update the leaf rcu_node structure's
  2629. * ->cbovldmask bit corresponding to the current CPU based on that CPU's
  2630. * number of queued RCU callbacks. No locks need be held, but the
  2631. * caller must have disabled interrupts.
  2632. *
  2633. * Note that this function ignores the possibility that there are a lot
  2634. * of callbacks all of which have already seen the end of their respective
  2635. * grace periods. This omission is due to the need for no-CBs CPUs to
  2636. * be holding ->nocb_lock to do this check, which is too heavy for a
  2637. * common-case operation.
  2638. */
  2639. static void check_cb_ovld(struct rcu_data *rdp)
  2640. {
  2641. struct rcu_node *const rnp = rdp->mynode;
  2642. if (qovld_calc <= 0 ||
  2643. ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
  2644. !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
  2645. return; // Early boot wildcard value or already set correctly.
  2646. raw_spin_lock_rcu_node(rnp);
  2647. check_cb_ovld_locked(rdp, rnp);
  2648. raw_spin_unlock_rcu_node(rnp);
  2649. }
  2650. /* Helper function for call_rcu() and friends. */
  2651. static void
  2652. __call_rcu(struct rcu_head *head, rcu_callback_t func)
  2653. {
  2654. unsigned long flags;
  2655. struct rcu_data *rdp;
  2656. bool was_alldone;
  2657. /* Misaligned rcu_head! */
  2658. WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
  2659. if (debug_rcu_head_queue(head)) {
  2660. /*
  2661. * Probable double call_rcu(), so leak the callback.
  2662. * Use rcu:rcu_callback trace event to find the previous
  2663. * time callback was passed to __call_rcu().
  2664. */
  2665. WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
  2666. head, head->func);
  2667. WRITE_ONCE(head->func, rcu_leak_callback);
  2668. return;
  2669. }
  2670. head->func = func;
  2671. head->next = NULL;
  2672. local_irq_save(flags);
  2673. kasan_record_aux_stack(head);
  2674. rdp = this_cpu_ptr(&rcu_data);
  2675. /* Add the callback to our list. */
  2676. if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
  2677. // This can trigger due to call_rcu() from offline CPU:
  2678. WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
  2679. WARN_ON_ONCE(!rcu_is_watching());
  2680. // Very early boot, before rcu_init(). Initialize if needed
  2681. // and then drop through to queue the callback.
  2682. if (rcu_segcblist_empty(&rdp->cblist))
  2683. rcu_segcblist_init(&rdp->cblist);
  2684. }
  2685. check_cb_ovld(rdp);
  2686. if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
  2687. return; // Enqueued onto ->nocb_bypass, so just leave.
  2688. // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
  2689. rcu_segcblist_enqueue(&rdp->cblist, head);
  2690. if (__is_kvfree_rcu_offset((unsigned long)func))
  2691. trace_rcu_kvfree_callback(rcu_state.name, head,
  2692. (unsigned long)func,
  2693. rcu_segcblist_n_cbs(&rdp->cblist));
  2694. else
  2695. trace_rcu_callback(rcu_state.name, head,
  2696. rcu_segcblist_n_cbs(&rdp->cblist));
  2697. /* Go handle any RCU core processing required. */
  2698. if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  2699. unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
  2700. __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
  2701. } else {
  2702. __call_rcu_core(rdp, head, flags);
  2703. local_irq_restore(flags);
  2704. }
  2705. }
  2706. /**
  2707. * call_rcu() - Queue an RCU callback for invocation after a grace period.
  2708. * @head: structure to be used for queueing the RCU updates.
  2709. * @func: actual callback function to be invoked after the grace period
  2710. *
  2711. * The callback function will be invoked some time after a full grace
  2712. * period elapses, in other words after all pre-existing RCU read-side
  2713. * critical sections have completed. However, the callback function
  2714. * might well execute concurrently with RCU read-side critical sections
  2715. * that started after call_rcu() was invoked. RCU read-side critical
  2716. * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
  2717. * may be nested. In addition, regions of code across which interrupts,
  2718. * preemption, or softirqs have been disabled also serve as RCU read-side
  2719. * critical sections. This includes hardware interrupt handlers, softirq
  2720. * handlers, and NMI handlers.
  2721. *
  2722. * Note that all CPUs must agree that the grace period extended beyond
  2723. * all pre-existing RCU read-side critical section. On systems with more
  2724. * than one CPU, this means that when "func()" is invoked, each CPU is
  2725. * guaranteed to have executed a full memory barrier since the end of its
  2726. * last RCU read-side critical section whose beginning preceded the call
  2727. * to call_rcu(). It also means that each CPU executing an RCU read-side
  2728. * critical section that continues beyond the start of "func()" must have
  2729. * executed a memory barrier after the call_rcu() but before the beginning
  2730. * of that RCU read-side critical section. Note that these guarantees
  2731. * include CPUs that are offline, idle, or executing in user mode, as
  2732. * well as CPUs that are executing in the kernel.
  2733. *
  2734. * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
  2735. * resulting RCU callback function "func()", then both CPU A and CPU B are
  2736. * guaranteed to execute a full memory barrier during the time interval
  2737. * between the call to call_rcu() and the invocation of "func()" -- even
  2738. * if CPU A and CPU B are the same CPU (but again only if the system has
  2739. * more than one CPU).
  2740. */
  2741. void call_rcu(struct rcu_head *head, rcu_callback_t func)
  2742. {
  2743. __call_rcu(head, func);
  2744. }
  2745. EXPORT_SYMBOL_GPL(call_rcu);
  2746. /* Maximum number of jiffies to wait before draining a batch. */
  2747. #define KFREE_DRAIN_JIFFIES (HZ / 50)
  2748. #define KFREE_N_BATCHES 2
  2749. #define FREE_N_CHANNELS 2
  2750. /**
  2751. * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers
  2752. * @nr_records: Number of active pointers in the array
  2753. * @next: Next bulk object in the block chain
  2754. * @records: Array of the kvfree_rcu() pointers
  2755. */
  2756. struct kvfree_rcu_bulk_data {
  2757. unsigned long nr_records;
  2758. struct kvfree_rcu_bulk_data *next;
  2759. void *records[];
  2760. };
  2761. /*
  2762. * This macro defines how many entries the "records" array
  2763. * will contain. It is based on the fact that the size of
  2764. * kvfree_rcu_bulk_data structure becomes exactly one page.
  2765. */
  2766. #define KVFREE_BULK_MAX_ENTR \
  2767. ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *))
  2768. /**
  2769. * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
  2770. * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
  2771. * @head_free: List of kfree_rcu() objects waiting for a grace period
  2772. * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
  2773. * @krcp: Pointer to @kfree_rcu_cpu structure
  2774. */
  2775. struct kfree_rcu_cpu_work {
  2776. struct rcu_work rcu_work;
  2777. struct rcu_head *head_free;
  2778. struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS];
  2779. struct kfree_rcu_cpu *krcp;
  2780. };
  2781. /**
  2782. * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
  2783. * @head: List of kfree_rcu() objects not yet waiting for a grace period
  2784. * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
  2785. * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
  2786. * @lock: Synchronize access to this structure
  2787. * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
  2788. * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
  2789. * @initialized: The @rcu_work fields have been initialized
  2790. * @count: Number of objects for which GP not started
  2791. * @bkvcache:
  2792. * A simple cache list that contains objects for reuse purpose.
  2793. * In order to save some per-cpu space the list is singular.
  2794. * Even though it is lockless an access has to be protected by the
  2795. * per-cpu lock.
  2796. * @page_cache_work: A work to refill the cache when it is empty
  2797. * @work_in_progress: Indicates that page_cache_work is running
  2798. * @hrtimer: A hrtimer for scheduling a page_cache_work
  2799. * @nr_bkv_objs: number of allocated objects at @bkvcache.
  2800. *
  2801. * This is a per-CPU structure. The reason that it is not included in
  2802. * the rcu_data structure is to permit this code to be extracted from
  2803. * the RCU files. Such extraction could allow further optimization of
  2804. * the interactions with the slab allocators.
  2805. */
  2806. struct kfree_rcu_cpu {
  2807. struct rcu_head *head;
  2808. struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS];
  2809. struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
  2810. raw_spinlock_t lock;
  2811. struct delayed_work monitor_work;
  2812. bool monitor_todo;
  2813. bool initialized;
  2814. int count;
  2815. struct work_struct page_cache_work;
  2816. atomic_t work_in_progress;
  2817. struct hrtimer hrtimer;
  2818. struct llist_head bkvcache;
  2819. int nr_bkv_objs;
  2820. };
  2821. static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
  2822. .lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
  2823. };
  2824. static __always_inline void
  2825. debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
  2826. {
  2827. #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
  2828. int i;
  2829. for (i = 0; i < bhead->nr_records; i++)
  2830. debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
  2831. #endif
  2832. }
  2833. static inline struct kfree_rcu_cpu *
  2834. krc_this_cpu_lock(unsigned long *flags)
  2835. {
  2836. struct kfree_rcu_cpu *krcp;
  2837. local_irq_save(*flags); // For safely calling this_cpu_ptr().
  2838. krcp = this_cpu_ptr(&krc);
  2839. raw_spin_lock(&krcp->lock);
  2840. return krcp;
  2841. }
  2842. static inline void
  2843. krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
  2844. {
  2845. raw_spin_unlock(&krcp->lock);
  2846. local_irq_restore(flags);
  2847. }
  2848. static inline struct kvfree_rcu_bulk_data *
  2849. get_cached_bnode(struct kfree_rcu_cpu *krcp)
  2850. {
  2851. if (!krcp->nr_bkv_objs)
  2852. return NULL;
  2853. krcp->nr_bkv_objs--;
  2854. return (struct kvfree_rcu_bulk_data *)
  2855. llist_del_first(&krcp->bkvcache);
  2856. }
  2857. static inline bool
  2858. put_cached_bnode(struct kfree_rcu_cpu *krcp,
  2859. struct kvfree_rcu_bulk_data *bnode)
  2860. {
  2861. // Check the limit.
  2862. if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
  2863. return false;
  2864. llist_add((struct llist_node *) bnode, &krcp->bkvcache);
  2865. krcp->nr_bkv_objs++;
  2866. return true;
  2867. }
  2868. /*
  2869. * This function is invoked in workqueue context after a grace period.
  2870. * It frees all the objects queued on ->bhead_free or ->head_free.
  2871. */
  2872. static void kfree_rcu_work(struct work_struct *work)
  2873. {
  2874. unsigned long flags;
  2875. struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS], *bnext;
  2876. struct rcu_head *head, *next;
  2877. struct kfree_rcu_cpu *krcp;
  2878. struct kfree_rcu_cpu_work *krwp;
  2879. int i, j;
  2880. krwp = container_of(to_rcu_work(work),
  2881. struct kfree_rcu_cpu_work, rcu_work);
  2882. krcp = krwp->krcp;
  2883. raw_spin_lock_irqsave(&krcp->lock, flags);
  2884. // Channels 1 and 2.
  2885. for (i = 0; i < FREE_N_CHANNELS; i++) {
  2886. bkvhead[i] = krwp->bkvhead_free[i];
  2887. krwp->bkvhead_free[i] = NULL;
  2888. }
  2889. // Channel 3.
  2890. head = krwp->head_free;
  2891. krwp->head_free = NULL;
  2892. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  2893. // Handle two first channels.
  2894. for (i = 0; i < FREE_N_CHANNELS; i++) {
  2895. for (; bkvhead[i]; bkvhead[i] = bnext) {
  2896. bnext = bkvhead[i]->next;
  2897. debug_rcu_bhead_unqueue(bkvhead[i]);
  2898. rcu_lock_acquire(&rcu_callback_map);
  2899. if (i == 0) { // kmalloc() / kfree().
  2900. trace_rcu_invoke_kfree_bulk_callback(
  2901. rcu_state.name, bkvhead[i]->nr_records,
  2902. bkvhead[i]->records);
  2903. kfree_bulk(bkvhead[i]->nr_records,
  2904. bkvhead[i]->records);
  2905. } else { // vmalloc() / vfree().
  2906. for (j = 0; j < bkvhead[i]->nr_records; j++) {
  2907. trace_rcu_invoke_kvfree_callback(
  2908. rcu_state.name,
  2909. bkvhead[i]->records[j], 0);
  2910. vfree(bkvhead[i]->records[j]);
  2911. }
  2912. }
  2913. rcu_lock_release(&rcu_callback_map);
  2914. raw_spin_lock_irqsave(&krcp->lock, flags);
  2915. if (put_cached_bnode(krcp, bkvhead[i]))
  2916. bkvhead[i] = NULL;
  2917. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  2918. if (bkvhead[i])
  2919. free_page((unsigned long) bkvhead[i]);
  2920. cond_resched_tasks_rcu_qs();
  2921. }
  2922. }
  2923. /*
  2924. * Emergency case only. It can happen under low memory
  2925. * condition when an allocation gets failed, so the "bulk"
  2926. * path can not be temporary maintained.
  2927. */
  2928. for (; head; head = next) {
  2929. unsigned long offset = (unsigned long)head->func;
  2930. void *ptr = (void *)head - offset;
  2931. next = head->next;
  2932. debug_rcu_head_unqueue((struct rcu_head *)ptr);
  2933. rcu_lock_acquire(&rcu_callback_map);
  2934. trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);
  2935. if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset)))
  2936. kvfree(ptr);
  2937. rcu_lock_release(&rcu_callback_map);
  2938. cond_resched_tasks_rcu_qs();
  2939. }
  2940. }
  2941. /*
  2942. * Schedule the kfree batch RCU work to run in workqueue context after a GP.
  2943. *
  2944. * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
  2945. * timeout has been reached.
  2946. */
  2947. static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
  2948. {
  2949. struct kfree_rcu_cpu_work *krwp;
  2950. bool repeat = false;
  2951. int i, j;
  2952. lockdep_assert_held(&krcp->lock);
  2953. for (i = 0; i < KFREE_N_BATCHES; i++) {
  2954. krwp = &(krcp->krw_arr[i]);
  2955. /*
  2956. * Try to detach bkvhead or head and attach it over any
  2957. * available corresponding free channel. It can be that
  2958. * a previous RCU batch is in progress, it means that
  2959. * immediately to queue another one is not possible so
  2960. * return false to tell caller to retry.
  2961. */
  2962. if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) ||
  2963. (krcp->bkvhead[1] && !krwp->bkvhead_free[1]) ||
  2964. (krcp->head && !krwp->head_free)) {
  2965. // Channel 1 corresponds to SLAB ptrs.
  2966. // Channel 2 corresponds to vmalloc ptrs.
  2967. for (j = 0; j < FREE_N_CHANNELS; j++) {
  2968. if (!krwp->bkvhead_free[j]) {
  2969. krwp->bkvhead_free[j] = krcp->bkvhead[j];
  2970. krcp->bkvhead[j] = NULL;
  2971. }
  2972. }
  2973. // Channel 3 corresponds to emergency path.
  2974. if (!krwp->head_free) {
  2975. krwp->head_free = krcp->head;
  2976. krcp->head = NULL;
  2977. }
  2978. WRITE_ONCE(krcp->count, 0);
  2979. /*
  2980. * One work is per one batch, so there are three
  2981. * "free channels", the batch can handle. It can
  2982. * be that the work is in the pending state when
  2983. * channels have been detached following by each
  2984. * other.
  2985. */
  2986. queue_rcu_work(system_wq, &krwp->rcu_work);
  2987. }
  2988. // Repeat if any "free" corresponding channel is still busy.
  2989. if (krcp->bkvhead[0] || krcp->bkvhead[1] || krcp->head)
  2990. repeat = true;
  2991. }
  2992. return !repeat;
  2993. }
  2994. static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
  2995. unsigned long flags)
  2996. {
  2997. // Attempt to start a new batch.
  2998. krcp->monitor_todo = false;
  2999. if (queue_kfree_rcu_work(krcp)) {
  3000. // Success! Our job is done here.
  3001. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3002. return;
  3003. }
  3004. // Previous RCU batch still in progress, try again later.
  3005. krcp->monitor_todo = true;
  3006. schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
  3007. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3008. }
  3009. /*
  3010. * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
  3011. * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
  3012. */
  3013. static void kfree_rcu_monitor(struct work_struct *work)
  3014. {
  3015. unsigned long flags;
  3016. struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu,
  3017. monitor_work.work);
  3018. raw_spin_lock_irqsave(&krcp->lock, flags);
  3019. if (krcp->monitor_todo)
  3020. kfree_rcu_drain_unlock(krcp, flags);
  3021. else
  3022. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3023. }
  3024. static enum hrtimer_restart
  3025. schedule_page_work_fn(struct hrtimer *t)
  3026. {
  3027. struct kfree_rcu_cpu *krcp =
  3028. container_of(t, struct kfree_rcu_cpu, hrtimer);
  3029. queue_work(system_highpri_wq, &krcp->page_cache_work);
  3030. return HRTIMER_NORESTART;
  3031. }
  3032. static void fill_page_cache_func(struct work_struct *work)
  3033. {
  3034. struct kvfree_rcu_bulk_data *bnode;
  3035. struct kfree_rcu_cpu *krcp =
  3036. container_of(work, struct kfree_rcu_cpu,
  3037. page_cache_work);
  3038. unsigned long flags;
  3039. bool pushed;
  3040. int i;
  3041. for (i = 0; i < rcu_min_cached_objs; i++) {
  3042. bnode = (struct kvfree_rcu_bulk_data *)
  3043. __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
  3044. if (bnode) {
  3045. raw_spin_lock_irqsave(&krcp->lock, flags);
  3046. pushed = put_cached_bnode(krcp, bnode);
  3047. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3048. if (!pushed) {
  3049. free_page((unsigned long) bnode);
  3050. break;
  3051. }
  3052. }
  3053. }
  3054. atomic_set(&krcp->work_in_progress, 0);
  3055. }
  3056. static void
  3057. run_page_cache_worker(struct kfree_rcu_cpu *krcp)
  3058. {
  3059. if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
  3060. !atomic_xchg(&krcp->work_in_progress, 1)) {
  3061. hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
  3062. HRTIMER_MODE_REL);
  3063. krcp->hrtimer.function = schedule_page_work_fn;
  3064. hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
  3065. }
  3066. }
  3067. static inline bool
  3068. kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
  3069. {
  3070. struct kvfree_rcu_bulk_data *bnode;
  3071. int idx;
  3072. if (unlikely(!krcp->initialized))
  3073. return false;
  3074. lockdep_assert_held(&krcp->lock);
  3075. idx = !!is_vmalloc_addr(ptr);
  3076. /* Check if a new block is required. */
  3077. if (!krcp->bkvhead[idx] ||
  3078. krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
  3079. bnode = get_cached_bnode(krcp);
  3080. /* Switch to emergency path. */
  3081. if (!bnode)
  3082. return false;
  3083. /* Initialize the new block. */
  3084. bnode->nr_records = 0;
  3085. bnode->next = krcp->bkvhead[idx];
  3086. /* Attach it to the head. */
  3087. krcp->bkvhead[idx] = bnode;
  3088. }
  3089. /* Finally insert. */
  3090. krcp->bkvhead[idx]->records
  3091. [krcp->bkvhead[idx]->nr_records++] = ptr;
  3092. return true;
  3093. }
  3094. /*
  3095. * Queue a request for lazy invocation of appropriate free routine after a
  3096. * grace period. Please note there are three paths are maintained, two are the
  3097. * main ones that use array of pointers interface and third one is emergency
  3098. * one, that is used only when the main path can not be maintained temporary,
  3099. * due to memory pressure.
  3100. *
  3101. * Each kvfree_call_rcu() request is added to a batch. The batch will be drained
  3102. * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
  3103. * be free'd in workqueue context. This allows us to: batch requests together to
  3104. * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
  3105. */
  3106. void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
  3107. {
  3108. unsigned long flags;
  3109. struct kfree_rcu_cpu *krcp;
  3110. bool success;
  3111. void *ptr;
  3112. if (head) {
  3113. ptr = (void *) head - (unsigned long) func;
  3114. } else {
  3115. /*
  3116. * Please note there is a limitation for the head-less
  3117. * variant, that is why there is a clear rule for such
  3118. * objects: it can be used from might_sleep() context
  3119. * only. For other places please embed an rcu_head to
  3120. * your data.
  3121. */
  3122. might_sleep();
  3123. ptr = (unsigned long *) func;
  3124. }
  3125. krcp = krc_this_cpu_lock(&flags);
  3126. // Queue the object but don't yet schedule the batch.
  3127. if (debug_rcu_head_queue(ptr)) {
  3128. // Probable double kfree_rcu(), just leak.
  3129. WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
  3130. __func__, head);
  3131. // Mark as success and leave.
  3132. success = true;
  3133. goto unlock_return;
  3134. }
  3135. success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
  3136. if (!success) {
  3137. run_page_cache_worker(krcp);
  3138. if (head == NULL)
  3139. // Inline if kvfree_rcu(one_arg) call.
  3140. goto unlock_return;
  3141. head->func = func;
  3142. head->next = krcp->head;
  3143. krcp->head = head;
  3144. success = true;
  3145. }
  3146. WRITE_ONCE(krcp->count, krcp->count + 1);
  3147. // Set timer to drain after KFREE_DRAIN_JIFFIES.
  3148. if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
  3149. !krcp->monitor_todo) {
  3150. krcp->monitor_todo = true;
  3151. schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
  3152. }
  3153. unlock_return:
  3154. krc_this_cpu_unlock(krcp, flags);
  3155. /*
  3156. * Inline kvfree() after synchronize_rcu(). We can do
  3157. * it from might_sleep() context only, so the current
  3158. * CPU can pass the QS state.
  3159. */
  3160. if (!success) {
  3161. debug_rcu_head_unqueue((struct rcu_head *) ptr);
  3162. synchronize_rcu();
  3163. kvfree(ptr);
  3164. }
  3165. }
  3166. EXPORT_SYMBOL_GPL(kvfree_call_rcu);
  3167. static unsigned long
  3168. kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
  3169. {
  3170. int cpu;
  3171. unsigned long count = 0;
  3172. /* Snapshot count of all CPUs */
  3173. for_each_possible_cpu(cpu) {
  3174. struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
  3175. count += READ_ONCE(krcp->count);
  3176. }
  3177. return count;
  3178. }
  3179. static unsigned long
  3180. kfree_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
  3181. {
  3182. int cpu, freed = 0;
  3183. unsigned long flags;
  3184. for_each_possible_cpu(cpu) {
  3185. int count;
  3186. struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
  3187. count = krcp->count;
  3188. raw_spin_lock_irqsave(&krcp->lock, flags);
  3189. if (krcp->monitor_todo)
  3190. kfree_rcu_drain_unlock(krcp, flags);
  3191. else
  3192. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3193. sc->nr_to_scan -= count;
  3194. freed += count;
  3195. if (sc->nr_to_scan <= 0)
  3196. break;
  3197. }
  3198. return freed == 0 ? SHRINK_STOP : freed;
  3199. }
  3200. static struct shrinker kfree_rcu_shrinker = {
  3201. .count_objects = kfree_rcu_shrink_count,
  3202. .scan_objects = kfree_rcu_shrink_scan,
  3203. .batch = 0,
  3204. .seeks = DEFAULT_SEEKS,
  3205. };
  3206. void __init kfree_rcu_scheduler_running(void)
  3207. {
  3208. int cpu;
  3209. unsigned long flags;
  3210. for_each_possible_cpu(cpu) {
  3211. struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
  3212. raw_spin_lock_irqsave(&krcp->lock, flags);
  3213. if (!krcp->head || krcp->monitor_todo) {
  3214. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3215. continue;
  3216. }
  3217. krcp->monitor_todo = true;
  3218. schedule_delayed_work_on(cpu, &krcp->monitor_work,
  3219. KFREE_DRAIN_JIFFIES);
  3220. raw_spin_unlock_irqrestore(&krcp->lock, flags);
  3221. }
  3222. }
  3223. /*
  3224. * During early boot, any blocking grace-period wait automatically
  3225. * implies a grace period. Later on, this is never the case for PREEMPTION.
  3226. *
  3227. * Howevr, because a context switch is a grace period for !PREEMPTION, any
  3228. * blocking grace-period wait automatically implies a grace period if
  3229. * there is only one CPU online at any point time during execution of
  3230. * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
  3231. * occasionally incorrectly indicate that there are multiple CPUs online
  3232. * when there was in fact only one the whole time, as this just adds some
  3233. * overhead: RCU still operates correctly.
  3234. */
  3235. static int rcu_blocking_is_gp(void)
  3236. {
  3237. int ret;
  3238. if (IS_ENABLED(CONFIG_PREEMPTION))
  3239. return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
  3240. might_sleep(); /* Check for RCU read-side critical section. */
  3241. preempt_disable();
  3242. ret = num_online_cpus() <= 1;
  3243. preempt_enable();
  3244. return ret;
  3245. }
  3246. /**
  3247. * synchronize_rcu - wait until a grace period has elapsed.
  3248. *
  3249. * Control will return to the caller some time after a full grace
  3250. * period has elapsed, in other words after all currently executing RCU
  3251. * read-side critical sections have completed. Note, however, that
  3252. * upon return from synchronize_rcu(), the caller might well be executing
  3253. * concurrently with new RCU read-side critical sections that began while
  3254. * synchronize_rcu() was waiting. RCU read-side critical sections are
  3255. * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
  3256. * In addition, regions of code across which interrupts, preemption, or
  3257. * softirqs have been disabled also serve as RCU read-side critical
  3258. * sections. This includes hardware interrupt handlers, softirq handlers,
  3259. * and NMI handlers.
  3260. *
  3261. * Note that this guarantee implies further memory-ordering guarantees.
  3262. * On systems with more than one CPU, when synchronize_rcu() returns,
  3263. * each CPU is guaranteed to have executed a full memory barrier since
  3264. * the end of its last RCU read-side critical section whose beginning
  3265. * preceded the call to synchronize_rcu(). In addition, each CPU having
  3266. * an RCU read-side critical section that extends beyond the return from
  3267. * synchronize_rcu() is guaranteed to have executed a full memory barrier
  3268. * after the beginning of synchronize_rcu() and before the beginning of
  3269. * that RCU read-side critical section. Note that these guarantees include
  3270. * CPUs that are offline, idle, or executing in user mode, as well as CPUs
  3271. * that are executing in the kernel.
  3272. *
  3273. * Furthermore, if CPU A invoked synchronize_rcu(), which returned
  3274. * to its caller on CPU B, then both CPU A and CPU B are guaranteed
  3275. * to have executed a full memory barrier during the execution of
  3276. * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
  3277. * again only if the system has more than one CPU).
  3278. */
  3279. void synchronize_rcu(void)
  3280. {
  3281. RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
  3282. lock_is_held(&rcu_lock_map) ||
  3283. lock_is_held(&rcu_sched_lock_map),
  3284. "Illegal synchronize_rcu() in RCU read-side critical section");
  3285. if (rcu_blocking_is_gp())
  3286. return;
  3287. if (rcu_gp_is_expedited())
  3288. synchronize_rcu_expedited();
  3289. else
  3290. wait_rcu_gp(call_rcu);
  3291. }
  3292. EXPORT_SYMBOL_GPL(synchronize_rcu);
  3293. /**
  3294. * get_state_synchronize_rcu - Snapshot current RCU state
  3295. *
  3296. * Returns a cookie that is used by a later call to cond_synchronize_rcu()
  3297. * to determine whether or not a full grace period has elapsed in the
  3298. * meantime.
  3299. */
  3300. unsigned long get_state_synchronize_rcu(void)
  3301. {
  3302. /*
  3303. * Any prior manipulation of RCU-protected data must happen
  3304. * before the load from ->gp_seq.
  3305. */
  3306. smp_mb(); /* ^^^ */
  3307. return rcu_seq_snap(&rcu_state.gp_seq);
  3308. }
  3309. EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
  3310. /**
  3311. * cond_synchronize_rcu - Conditionally wait for an RCU grace period
  3312. *
  3313. * @oldstate: return value from earlier call to get_state_synchronize_rcu()
  3314. *
  3315. * If a full RCU grace period has elapsed since the earlier call to
  3316. * get_state_synchronize_rcu(), just return. Otherwise, invoke
  3317. * synchronize_rcu() to wait for a full grace period.
  3318. *
  3319. * Yes, this function does not take counter wrap into account. But
  3320. * counter wrap is harmless. If the counter wraps, we have waited for
  3321. * more than 2 billion grace periods (and way more on a 64-bit system!),
  3322. * so waiting for one additional grace period should be just fine.
  3323. */
  3324. void cond_synchronize_rcu(unsigned long oldstate)
  3325. {
  3326. if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
  3327. synchronize_rcu();
  3328. else
  3329. smp_mb(); /* Ensure GP ends before subsequent accesses. */
  3330. }
  3331. EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
  3332. /*
  3333. * Check to see if there is any immediate RCU-related work to be done by
  3334. * the current CPU, returning 1 if so and zero otherwise. The checks are
  3335. * in order of increasing expense: checks that can be carried out against
  3336. * CPU-local state are performed first. However, we must check for CPU
  3337. * stalls first, else we might not get a chance.
  3338. */
  3339. static int rcu_pending(int user)
  3340. {
  3341. bool gp_in_progress;
  3342. struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
  3343. struct rcu_node *rnp = rdp->mynode;
  3344. lockdep_assert_irqs_disabled();
  3345. /* Check for CPU stalls, if enabled. */
  3346. check_cpu_stall(rdp);
  3347. /* Does this CPU need a deferred NOCB wakeup? */
  3348. if (rcu_nocb_need_deferred_wakeup(rdp))
  3349. return 1;
  3350. /* Is this a nohz_full CPU in userspace or idle? (Ignore RCU if so.) */
  3351. if ((user || rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
  3352. return 0;
  3353. /* Is the RCU core waiting for a quiescent state from this CPU? */
  3354. gp_in_progress = rcu_gp_in_progress();
  3355. if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress)
  3356. return 1;
  3357. /* Does this CPU have callbacks ready to invoke? */
  3358. if (rcu_segcblist_ready_cbs(&rdp->cblist))
  3359. return 1;
  3360. /* Has RCU gone idle with this CPU needing another grace period? */
  3361. if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) &&
  3362. (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) ||
  3363. !rcu_segcblist_is_offloaded(&rdp->cblist)) &&
  3364. !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
  3365. return 1;
  3366. /* Have RCU grace period completed or started? */
  3367. if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq ||
  3368. unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
  3369. return 1;
  3370. /* nothing to do */
  3371. return 0;
  3372. }
  3373. /*
  3374. * Helper function for rcu_barrier() tracing. If tracing is disabled,
  3375. * the compiler is expected to optimize this away.
  3376. */
  3377. static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
  3378. {
  3379. trace_rcu_barrier(rcu_state.name, s, cpu,
  3380. atomic_read(&rcu_state.barrier_cpu_count), done);
  3381. }
  3382. /*
  3383. * RCU callback function for rcu_barrier(). If we are last, wake
  3384. * up the task executing rcu_barrier().
  3385. *
  3386. * Note that the value of rcu_state.barrier_sequence must be captured
  3387. * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
  3388. * other CPUs might count the value down to zero before this CPU gets
  3389. * around to invoking rcu_barrier_trace(), which might result in bogus
  3390. * data from the next instance of rcu_barrier().
  3391. */
  3392. static void rcu_barrier_callback(struct rcu_head *rhp)
  3393. {
  3394. unsigned long __maybe_unused s = rcu_state.barrier_sequence;
  3395. if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
  3396. rcu_barrier_trace(TPS("LastCB"), -1, s);
  3397. complete(&rcu_state.barrier_completion);
  3398. } else {
  3399. rcu_barrier_trace(TPS("CB"), -1, s);
  3400. }
  3401. }
  3402. /*
  3403. * Called with preemption disabled, and from cross-cpu IRQ context.
  3404. */
  3405. static void rcu_barrier_func(void *cpu_in)
  3406. {
  3407. uintptr_t cpu = (uintptr_t)cpu_in;
  3408. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  3409. rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
  3410. rdp->barrier_head.func = rcu_barrier_callback;
  3411. debug_rcu_head_queue(&rdp->barrier_head);
  3412. rcu_nocb_lock(rdp);
  3413. WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
  3414. if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
  3415. atomic_inc(&rcu_state.barrier_cpu_count);
  3416. } else {
  3417. debug_rcu_head_unqueue(&rdp->barrier_head);
  3418. rcu_barrier_trace(TPS("IRQNQ"), -1,
  3419. rcu_state.barrier_sequence);
  3420. }
  3421. rcu_nocb_unlock(rdp);
  3422. }
  3423. /**
  3424. * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
  3425. *
  3426. * Note that this primitive does not necessarily wait for an RCU grace period
  3427. * to complete. For example, if there are no RCU callbacks queued anywhere
  3428. * in the system, then rcu_barrier() is within its rights to return
  3429. * immediately, without waiting for anything, much less an RCU grace period.
  3430. */
  3431. void rcu_barrier(void)
  3432. {
  3433. uintptr_t cpu;
  3434. struct rcu_data *rdp;
  3435. unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
  3436. rcu_barrier_trace(TPS("Begin"), -1, s);
  3437. /* Take mutex to serialize concurrent rcu_barrier() requests. */
  3438. mutex_lock(&rcu_state.barrier_mutex);
  3439. /* Did someone else do our work for us? */
  3440. if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
  3441. rcu_barrier_trace(TPS("EarlyExit"), -1,
  3442. rcu_state.barrier_sequence);
  3443. smp_mb(); /* caller's subsequent code after above check. */
  3444. mutex_unlock(&rcu_state.barrier_mutex);
  3445. return;
  3446. }
  3447. /* Mark the start of the barrier operation. */
  3448. rcu_seq_start(&rcu_state.barrier_sequence);
  3449. rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
  3450. /*
  3451. * Initialize the count to two rather than to zero in order
  3452. * to avoid a too-soon return to zero in case of an immediate
  3453. * invocation of the just-enqueued callback (or preemption of
  3454. * this task). Exclude CPU-hotplug operations to ensure that no
  3455. * offline non-offloaded CPU has callbacks queued.
  3456. */
  3457. init_completion(&rcu_state.barrier_completion);
  3458. atomic_set(&rcu_state.barrier_cpu_count, 2);
  3459. get_online_cpus();
  3460. /*
  3461. * Force each CPU with callbacks to register a new callback.
  3462. * When that callback is invoked, we will know that all of the
  3463. * corresponding CPU's preceding callbacks have been invoked.
  3464. */
  3465. for_each_possible_cpu(cpu) {
  3466. rdp = per_cpu_ptr(&rcu_data, cpu);
  3467. if (cpu_is_offline(cpu) &&
  3468. !rcu_segcblist_is_offloaded(&rdp->cblist))
  3469. continue;
  3470. if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
  3471. rcu_barrier_trace(TPS("OnlineQ"), cpu,
  3472. rcu_state.barrier_sequence);
  3473. smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
  3474. } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
  3475. cpu_is_offline(cpu)) {
  3476. rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
  3477. rcu_state.barrier_sequence);
  3478. local_irq_disable();
  3479. rcu_barrier_func((void *)cpu);
  3480. local_irq_enable();
  3481. } else if (cpu_is_offline(cpu)) {
  3482. rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
  3483. rcu_state.barrier_sequence);
  3484. } else {
  3485. rcu_barrier_trace(TPS("OnlineNQ"), cpu,
  3486. rcu_state.barrier_sequence);
  3487. }
  3488. }
  3489. put_online_cpus();
  3490. /*
  3491. * Now that we have an rcu_barrier_callback() callback on each
  3492. * CPU, and thus each counted, remove the initial count.
  3493. */
  3494. if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
  3495. complete(&rcu_state.barrier_completion);
  3496. /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
  3497. wait_for_completion(&rcu_state.barrier_completion);
  3498. /* Mark the end of the barrier operation. */
  3499. rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
  3500. rcu_seq_end(&rcu_state.barrier_sequence);
  3501. /* Other rcu_barrier() invocations can now safely proceed. */
  3502. mutex_unlock(&rcu_state.barrier_mutex);
  3503. }
  3504. EXPORT_SYMBOL_GPL(rcu_barrier);
  3505. /*
  3506. * Propagate ->qsinitmask bits up the rcu_node tree to account for the
  3507. * first CPU in a given leaf rcu_node structure coming online. The caller
  3508. * must hold the corresponding leaf rcu_node ->lock with interrrupts
  3509. * disabled.
  3510. */
  3511. static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
  3512. {
  3513. long mask;
  3514. long oldmask;
  3515. struct rcu_node *rnp = rnp_leaf;
  3516. raw_lockdep_assert_held_rcu_node(rnp_leaf);
  3517. WARN_ON_ONCE(rnp->wait_blkd_tasks);
  3518. for (;;) {
  3519. mask = rnp->grpmask;
  3520. rnp = rnp->parent;
  3521. if (rnp == NULL)
  3522. return;
  3523. raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */
  3524. oldmask = rnp->qsmaskinit;
  3525. rnp->qsmaskinit |= mask;
  3526. raw_spin_unlock_rcu_node(rnp); /* Interrupts remain disabled. */
  3527. if (oldmask)
  3528. return;
  3529. }
  3530. }
  3531. /*
  3532. * Do boot-time initialization of a CPU's per-CPU RCU data.
  3533. */
  3534. static void __init
  3535. rcu_boot_init_percpu_data(int cpu)
  3536. {
  3537. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  3538. /* Set up local state, ensuring consistent view of global state. */
  3539. rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
  3540. INIT_WORK(&rdp->strict_work, strict_work_handler);
  3541. WARN_ON_ONCE(rdp->dynticks_nesting != 1);
  3542. WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
  3543. rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
  3544. rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
  3545. rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
  3546. rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
  3547. rdp->cpu = cpu;
  3548. rcu_boot_init_nocb_percpu_data(rdp);
  3549. }
  3550. /*
  3551. * Invoked early in the CPU-online process, when pretty much all services
  3552. * are available. The incoming CPU is not present.
  3553. *
  3554. * Initializes a CPU's per-CPU RCU data. Note that only one online or
  3555. * offline event can be happening at a given time. Note also that we can
  3556. * accept some slop in the rsp->gp_seq access due to the fact that this
  3557. * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
  3558. * And any offloaded callbacks are being numbered elsewhere.
  3559. */
  3560. int rcutree_prepare_cpu(unsigned int cpu)
  3561. {
  3562. unsigned long flags;
  3563. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  3564. struct rcu_node *rnp = rcu_get_root();
  3565. /* Set up local state, ensuring consistent view of global state. */
  3566. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  3567. rdp->qlen_last_fqs_check = 0;
  3568. rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
  3569. rdp->blimit = blimit;
  3570. if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
  3571. !rcu_segcblist_is_offloaded(&rdp->cblist))
  3572. rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */
  3573. rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
  3574. rcu_dynticks_eqs_online();
  3575. raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
  3576. /*
  3577. * Add CPU to leaf rcu_node pending-online bitmask. Any needed
  3578. * propagation up the rcu_node tree will happen at the beginning
  3579. * of the next grace period.
  3580. */
  3581. rnp = rdp->mynode;
  3582. raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
  3583. rdp->beenonline = true; /* We have now been online. */
  3584. rdp->gp_seq = READ_ONCE(rnp->gp_seq);
  3585. rdp->gp_seq_needed = rdp->gp_seq;
  3586. rdp->cpu_no_qs.b.norm = true;
  3587. rdp->core_needs_qs = false;
  3588. rdp->rcu_iw_pending = false;
  3589. rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
  3590. trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
  3591. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  3592. rcu_prepare_kthreads(cpu);
  3593. rcu_spawn_cpu_nocb_kthread(cpu);
  3594. return 0;
  3595. }
  3596. /*
  3597. * Update RCU priority boot kthread affinity for CPU-hotplug changes.
  3598. */
  3599. static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
  3600. {
  3601. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  3602. rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
  3603. }
  3604. /*
  3605. * Near the end of the CPU-online process. Pretty much all services
  3606. * enabled, and the CPU is now very much alive.
  3607. */
  3608. int rcutree_online_cpu(unsigned int cpu)
  3609. {
  3610. unsigned long flags;
  3611. struct rcu_data *rdp;
  3612. struct rcu_node *rnp;
  3613. rdp = per_cpu_ptr(&rcu_data, cpu);
  3614. rnp = rdp->mynode;
  3615. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  3616. rnp->ffmask |= rdp->grpmask;
  3617. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  3618. if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
  3619. return 0; /* Too early in boot for scheduler work. */
  3620. sync_sched_exp_online_cleanup(cpu);
  3621. rcutree_affinity_setting(cpu, -1);
  3622. // Stop-machine done, so allow nohz_full to disable tick.
  3623. tick_dep_clear(TICK_DEP_BIT_RCU);
  3624. return 0;
  3625. }
  3626. /*
  3627. * Near the beginning of the process. The CPU is still very much alive
  3628. * with pretty much all services enabled.
  3629. */
  3630. int rcutree_offline_cpu(unsigned int cpu)
  3631. {
  3632. unsigned long flags;
  3633. struct rcu_data *rdp;
  3634. struct rcu_node *rnp;
  3635. rdp = per_cpu_ptr(&rcu_data, cpu);
  3636. rnp = rdp->mynode;
  3637. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  3638. rnp->ffmask &= ~rdp->grpmask;
  3639. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  3640. rcutree_affinity_setting(cpu, cpu);
  3641. // nohz_full CPUs need the tick for stop-machine to work quickly
  3642. tick_dep_set(TICK_DEP_BIT_RCU);
  3643. return 0;
  3644. }
  3645. /*
  3646. * Mark the specified CPU as being online so that subsequent grace periods
  3647. * (both expedited and normal) will wait on it. Note that this means that
  3648. * incoming CPUs are not allowed to use RCU read-side critical sections
  3649. * until this function is called. Failing to observe this restriction
  3650. * will result in lockdep splats.
  3651. *
  3652. * Note that this function is special in that it is invoked directly
  3653. * from the incoming CPU rather than from the cpuhp_step mechanism.
  3654. * This is because this function must be invoked at a precise location.
  3655. */
  3656. void rcu_cpu_starting(unsigned int cpu)
  3657. {
  3658. unsigned long flags;
  3659. unsigned long mask;
  3660. struct rcu_data *rdp;
  3661. struct rcu_node *rnp;
  3662. bool newcpu;
  3663. rdp = per_cpu_ptr(&rcu_data, cpu);
  3664. if (rdp->cpu_started)
  3665. return;
  3666. rdp->cpu_started = true;
  3667. rnp = rdp->mynode;
  3668. mask = rdp->grpmask;
  3669. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  3670. WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask);
  3671. newcpu = !(rnp->expmaskinitnext & mask);
  3672. rnp->expmaskinitnext |= mask;
  3673. /* Allow lockless access for expedited grace periods. */
  3674. smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */
  3675. ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);
  3676. rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
  3677. rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
  3678. rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
  3679. if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
  3680. rcu_disable_urgency_upon_qs(rdp);
  3681. /* Report QS -after- changing ->qsmaskinitnext! */
  3682. rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
  3683. } else {
  3684. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  3685. }
  3686. smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
  3687. }
  3688. /*
  3689. * The outgoing function has no further need of RCU, so remove it from
  3690. * the rcu_node tree's ->qsmaskinitnext bit masks.
  3691. *
  3692. * Note that this function is special in that it is invoked directly
  3693. * from the outgoing CPU rather than from the cpuhp_step mechanism.
  3694. * This is because this function must be invoked at a precise location.
  3695. */
  3696. void rcu_report_dead(unsigned int cpu)
  3697. {
  3698. unsigned long flags;
  3699. unsigned long mask;
  3700. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  3701. struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
  3702. /* QS for any half-done expedited grace period. */
  3703. preempt_disable();
  3704. rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
  3705. preempt_enable();
  3706. rcu_preempt_deferred_qs(current);
  3707. /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
  3708. mask = rdp->grpmask;
  3709. raw_spin_lock(&rcu_state.ofl_lock);
  3710. raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
  3711. rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
  3712. rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
  3713. if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
  3714. /* Report quiescent state -before- changing ->qsmaskinitnext! */
  3715. rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
  3716. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  3717. }
  3718. WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
  3719. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  3720. raw_spin_unlock(&rcu_state.ofl_lock);
  3721. rdp->cpu_started = false;
  3722. }
  3723. #ifdef CONFIG_HOTPLUG_CPU
  3724. /*
  3725. * The outgoing CPU has just passed through the dying-idle state, and we
  3726. * are being invoked from the CPU that was IPIed to continue the offline
  3727. * operation. Migrate the outgoing CPU's callbacks to the current CPU.
  3728. */
  3729. void rcutree_migrate_callbacks(int cpu)
  3730. {
  3731. unsigned long flags;
  3732. struct rcu_data *my_rdp;
  3733. struct rcu_node *my_rnp;
  3734. struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
  3735. bool needwake;
  3736. if (rcu_segcblist_is_offloaded(&rdp->cblist) ||
  3737. rcu_segcblist_empty(&rdp->cblist))
  3738. return; /* No callbacks to migrate. */
  3739. local_irq_save(flags);
  3740. my_rdp = this_cpu_ptr(&rcu_data);
  3741. my_rnp = my_rdp->mynode;
  3742. rcu_nocb_lock(my_rdp); /* irqs already disabled. */
  3743. WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
  3744. raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
  3745. /* Leverage recent GPs and set GP for new callbacks. */
  3746. needwake = rcu_advance_cbs(my_rnp, rdp) ||
  3747. rcu_advance_cbs(my_rnp, my_rdp);
  3748. rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
  3749. needwake = needwake || rcu_advance_cbs(my_rnp, my_rdp);
  3750. rcu_segcblist_disable(&rdp->cblist);
  3751. WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
  3752. !rcu_segcblist_n_cbs(&my_rdp->cblist));
  3753. if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
  3754. raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
  3755. __call_rcu_nocb_wake(my_rdp, true, flags);
  3756. } else {
  3757. rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
  3758. raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
  3759. }
  3760. if (needwake)
  3761. rcu_gp_kthread_wake();
  3762. lockdep_assert_irqs_enabled();
  3763. WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 ||
  3764. !rcu_segcblist_empty(&rdp->cblist),
  3765. "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
  3766. cpu, rcu_segcblist_n_cbs(&rdp->cblist),
  3767. rcu_segcblist_first_cb(&rdp->cblist));
  3768. }
  3769. #endif
  3770. /*
  3771. * On non-huge systems, use expedited RCU grace periods to make suspend
  3772. * and hibernation run faster.
  3773. */
  3774. static int rcu_pm_notify(struct notifier_block *self,
  3775. unsigned long action, void *hcpu)
  3776. {
  3777. switch (action) {
  3778. case PM_HIBERNATION_PREPARE:
  3779. case PM_SUSPEND_PREPARE:
  3780. rcu_expedite_gp();
  3781. break;
  3782. case PM_POST_HIBERNATION:
  3783. case PM_POST_SUSPEND:
  3784. rcu_unexpedite_gp();
  3785. break;
  3786. default:
  3787. break;
  3788. }
  3789. return NOTIFY_OK;
  3790. }
  3791. /*
  3792. * Spawn the kthreads that handle RCU's grace periods.
  3793. */
  3794. static int __init rcu_spawn_gp_kthread(void)
  3795. {
  3796. unsigned long flags;
  3797. int kthread_prio_in = kthread_prio;
  3798. struct rcu_node *rnp;
  3799. struct sched_param sp;
  3800. struct task_struct *t;
  3801. /* Force priority into range. */
  3802. if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 2
  3803. && IS_BUILTIN(CONFIG_RCU_TORTURE_TEST))
  3804. kthread_prio = 2;
  3805. else if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1)
  3806. kthread_prio = 1;
  3807. else if (kthread_prio < 0)
  3808. kthread_prio = 0;
  3809. else if (kthread_prio > 99)
  3810. kthread_prio = 99;
  3811. if (kthread_prio != kthread_prio_in)
  3812. pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
  3813. kthread_prio, kthread_prio_in);
  3814. rcu_scheduler_fully_active = 1;
  3815. t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
  3816. if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
  3817. return 0;
  3818. if (kthread_prio) {
  3819. sp.sched_priority = kthread_prio;
  3820. sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
  3821. }
  3822. rnp = rcu_get_root();
  3823. raw_spin_lock_irqsave_rcu_node(rnp, flags);
  3824. WRITE_ONCE(rcu_state.gp_activity, jiffies);
  3825. WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
  3826. // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
  3827. smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
  3828. raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
  3829. wake_up_process(t);
  3830. rcu_spawn_nocb_kthreads();
  3831. rcu_spawn_boost_kthreads();
  3832. rcu_spawn_core_kthreads();
  3833. return 0;
  3834. }
  3835. early_initcall(rcu_spawn_gp_kthread);
  3836. /*
  3837. * This function is invoked towards the end of the scheduler's
  3838. * initialization process. Before this is called, the idle task might
  3839. * contain synchronous grace-period primitives (during which time, this idle
  3840. * task is booting the system, and such primitives are no-ops). After this
  3841. * function is called, any synchronous grace-period primitives are run as
  3842. * expedited, with the requesting task driving the grace period forward.
  3843. * A later core_initcall() rcu_set_runtime_mode() will switch to full
  3844. * runtime RCU functionality.
  3845. */
  3846. void rcu_scheduler_starting(void)
  3847. {
  3848. WARN_ON(num_online_cpus() != 1);
  3849. WARN_ON(nr_context_switches() > 0);
  3850. rcu_test_sync_prims();
  3851. rcu_scheduler_active = RCU_SCHEDULER_INIT;
  3852. rcu_test_sync_prims();
  3853. }
  3854. /*
  3855. * Helper function for rcu_init() that initializes the rcu_state structure.
  3856. */
  3857. static void __init rcu_init_one(void)
  3858. {
  3859. static const char * const buf[] = RCU_NODE_NAME_INIT;
  3860. static const char * const fqs[] = RCU_FQS_NAME_INIT;
  3861. static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
  3862. static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
  3863. int levelspread[RCU_NUM_LVLS]; /* kids/node in each level. */
  3864. int cpustride = 1;
  3865. int i;
  3866. int j;
  3867. struct rcu_node *rnp;
  3868. BUILD_BUG_ON(RCU_NUM_LVLS > ARRAY_SIZE(buf)); /* Fix buf[] init! */
  3869. /* Silence gcc 4.8 false positive about array index out of range. */
  3870. if (rcu_num_lvls <= 0 || rcu_num_lvls > RCU_NUM_LVLS)
  3871. panic("rcu_init_one: rcu_num_lvls out of range");
  3872. /* Initialize the level-tracking arrays. */
  3873. for (i = 1; i < rcu_num_lvls; i++)
  3874. rcu_state.level[i] =
  3875. rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
  3876. rcu_init_levelspread(levelspread, num_rcu_lvl);
  3877. /* Initialize the elements themselves, starting from the leaves. */
  3878. for (i = rcu_num_lvls - 1; i >= 0; i--) {
  3879. cpustride *= levelspread[i];
  3880. rnp = rcu_state.level[i];
  3881. for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
  3882. raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
  3883. lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
  3884. &rcu_node_class[i], buf[i]);
  3885. raw_spin_lock_init(&rnp->fqslock);
  3886. lockdep_set_class_and_name(&rnp->fqslock,
  3887. &rcu_fqs_class[i], fqs[i]);
  3888. rnp->gp_seq = rcu_state.gp_seq;
  3889. rnp->gp_seq_needed = rcu_state.gp_seq;
  3890. rnp->completedqs = rcu_state.gp_seq;
  3891. rnp->qsmask = 0;
  3892. rnp->qsmaskinit = 0;
  3893. rnp->grplo = j * cpustride;
  3894. rnp->grphi = (j + 1) * cpustride - 1;
  3895. if (rnp->grphi >= nr_cpu_ids)
  3896. rnp->grphi = nr_cpu_ids - 1;
  3897. if (i == 0) {
  3898. rnp->grpnum = 0;
  3899. rnp->grpmask = 0;
  3900. rnp->parent = NULL;
  3901. } else {
  3902. rnp->grpnum = j % levelspread[i - 1];
  3903. rnp->grpmask = BIT(rnp->grpnum);
  3904. rnp->parent = rcu_state.level[i - 1] +
  3905. j / levelspread[i - 1];
  3906. }
  3907. rnp->level = i;
  3908. INIT_LIST_HEAD(&rnp->blkd_tasks);
  3909. rcu_init_one_nocb(rnp);
  3910. init_waitqueue_head(&rnp->exp_wq[0]);
  3911. init_waitqueue_head(&rnp->exp_wq[1]);
  3912. init_waitqueue_head(&rnp->exp_wq[2]);
  3913. init_waitqueue_head(&rnp->exp_wq[3]);
  3914. spin_lock_init(&rnp->exp_lock);
  3915. }
  3916. }
  3917. init_swait_queue_head(&rcu_state.gp_wq);
  3918. init_swait_queue_head(&rcu_state.expedited_wq);
  3919. rnp = rcu_first_leaf_node();
  3920. for_each_possible_cpu(i) {
  3921. while (i > rnp->grphi)
  3922. rnp++;
  3923. per_cpu_ptr(&rcu_data, i)->mynode = rnp;
  3924. rcu_boot_init_percpu_data(i);
  3925. }
  3926. }
  3927. /*
  3928. * Compute the rcu_node tree geometry from kernel parameters. This cannot
  3929. * replace the definitions in tree.h because those are needed to size
  3930. * the ->node array in the rcu_state structure.
  3931. */
  3932. void rcu_init_geometry(void)
  3933. {
  3934. ulong d;
  3935. int i;
  3936. static unsigned long old_nr_cpu_ids;
  3937. int rcu_capacity[RCU_NUM_LVLS];
  3938. static bool initialized;
  3939. if (initialized) {
  3940. /*
  3941. * Warn if setup_nr_cpu_ids() had not yet been invoked,
  3942. * unless nr_cpus_ids == NR_CPUS, in which case who cares?
  3943. */
  3944. WARN_ON_ONCE(old_nr_cpu_ids != nr_cpu_ids);
  3945. return;
  3946. }
  3947. old_nr_cpu_ids = nr_cpu_ids;
  3948. initialized = true;
  3949. /*
  3950. * Initialize any unspecified boot parameters.
  3951. * The default values of jiffies_till_first_fqs and
  3952. * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS
  3953. * value, which is a function of HZ, then adding one for each
  3954. * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
  3955. */
  3956. d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
  3957. if (jiffies_till_first_fqs == ULONG_MAX)
  3958. jiffies_till_first_fqs = d;
  3959. if (jiffies_till_next_fqs == ULONG_MAX)
  3960. jiffies_till_next_fqs = d;
  3961. adjust_jiffies_till_sched_qs();
  3962. /* If the compile-time values are accurate, just leave. */
  3963. if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
  3964. nr_cpu_ids == NR_CPUS)
  3965. return;
  3966. pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
  3967. rcu_fanout_leaf, nr_cpu_ids);
  3968. /*
  3969. * The boot-time rcu_fanout_leaf parameter must be at least two
  3970. * and cannot exceed the number of bits in the rcu_node masks.
  3971. * Complain and fall back to the compile-time values if this
  3972. * limit is exceeded.
  3973. */
  3974. if (rcu_fanout_leaf < 2 ||
  3975. rcu_fanout_leaf > sizeof(unsigned long) * 8) {
  3976. rcu_fanout_leaf = RCU_FANOUT_LEAF;
  3977. WARN_ON(1);
  3978. return;
  3979. }
  3980. /*
  3981. * Compute number of nodes that can be handled an rcu_node tree
  3982. * with the given number of levels.
  3983. */
  3984. rcu_capacity[0] = rcu_fanout_leaf;
  3985. for (i = 1; i < RCU_NUM_LVLS; i++)
  3986. rcu_capacity[i] = rcu_capacity[i - 1] * RCU_FANOUT;
  3987. /*
  3988. * The tree must be able to accommodate the configured number of CPUs.
  3989. * If this limit is exceeded, fall back to the compile-time values.
  3990. */
  3991. if (nr_cpu_ids > rcu_capacity[RCU_NUM_LVLS - 1]) {
  3992. rcu_fanout_leaf = RCU_FANOUT_LEAF;
  3993. WARN_ON(1);
  3994. return;
  3995. }
  3996. /* Calculate the number of levels in the tree. */
  3997. for (i = 0; nr_cpu_ids > rcu_capacity[i]; i++) {
  3998. }
  3999. rcu_num_lvls = i + 1;
  4000. /* Calculate the number of rcu_nodes at each level of the tree. */
  4001. for (i = 0; i < rcu_num_lvls; i++) {
  4002. int cap = rcu_capacity[(rcu_num_lvls - 1) - i];
  4003. num_rcu_lvl[i] = DIV_ROUND_UP(nr_cpu_ids, cap);
  4004. }
  4005. /* Calculate the total number of rcu_node structures. */
  4006. rcu_num_nodes = 0;
  4007. for (i = 0; i < rcu_num_lvls; i++)
  4008. rcu_num_nodes += num_rcu_lvl[i];
  4009. }
  4010. /*
  4011. * Dump out the structure of the rcu_node combining tree associated
  4012. * with the rcu_state structure.
  4013. */
  4014. static void __init rcu_dump_rcu_node_tree(void)
  4015. {
  4016. int level = 0;
  4017. struct rcu_node *rnp;
  4018. pr_info("rcu_node tree layout dump\n");
  4019. pr_info(" ");
  4020. rcu_for_each_node_breadth_first(rnp) {
  4021. if (rnp->level != level) {
  4022. pr_cont("\n");
  4023. pr_info(" ");
  4024. level = rnp->level;
  4025. }
  4026. pr_cont("%d:%d ^%d ", rnp->grplo, rnp->grphi, rnp->grpnum);
  4027. }
  4028. pr_cont("\n");
  4029. }
  4030. struct workqueue_struct *rcu_gp_wq;
  4031. struct workqueue_struct *rcu_par_gp_wq;
  4032. static void __init kfree_rcu_batch_init(void)
  4033. {
  4034. int cpu;
  4035. int i;
  4036. for_each_possible_cpu(cpu) {
  4037. struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
  4038. for (i = 0; i < KFREE_N_BATCHES; i++) {
  4039. INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
  4040. krcp->krw_arr[i].krcp = krcp;
  4041. }
  4042. INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
  4043. INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
  4044. krcp->initialized = true;
  4045. }
  4046. if (register_shrinker(&kfree_rcu_shrinker))
  4047. pr_err("Failed to register kfree_rcu() shrinker!\n");
  4048. }
  4049. void __init rcu_init(void)
  4050. {
  4051. int cpu;
  4052. rcu_early_boot_tests();
  4053. kfree_rcu_batch_init();
  4054. rcu_bootup_announce();
  4055. rcu_init_geometry();
  4056. rcu_init_one();
  4057. if (dump_tree)
  4058. rcu_dump_rcu_node_tree();
  4059. if (use_softirq)
  4060. open_softirq(RCU_SOFTIRQ, rcu_core_si);
  4061. /*
  4062. * We don't need protection against CPU-hotplug here because
  4063. * this is called early in boot, before either interrupts
  4064. * or the scheduler are operational.
  4065. */
  4066. pm_notifier(rcu_pm_notify, 0);
  4067. for_each_online_cpu(cpu) {
  4068. rcutree_prepare_cpu(cpu);
  4069. rcu_cpu_starting(cpu);
  4070. rcutree_online_cpu(cpu);
  4071. }
  4072. /* Create workqueue for expedited GPs and for Tree SRCU. */
  4073. rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
  4074. WARN_ON(!rcu_gp_wq);
  4075. rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
  4076. WARN_ON(!rcu_par_gp_wq);
  4077. srcu_init();
  4078. /* Fill in default value for rcutree.qovld boot parameter. */
  4079. /* -After- the rcu_node ->lock fields are initialized! */
  4080. if (qovld < 0)
  4081. qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
  4082. else
  4083. qovld_calc = qovld;
  4084. }
  4085. #include "tree_stall.h"
  4086. #include "tree_exp.h"
  4087. #include "tree_plugin.h"