workqueue.c 167 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kernel/workqueue.c - generic async execution with shared worker pool
  4. *
  5. * Copyright (C) 2002 Ingo Molnar
  6. *
  7. * Derived from the taskqueue/keventd code by:
  8. * David Woodhouse <dwmw2@infradead.org>
  9. * Andrew Morton
  10. * Kai Petzke <wpp@marie.physik.tu-berlin.de>
  11. * Theodore Ts'o <tytso@mit.edu>
  12. *
  13. * Made to use alloc_percpu by Christoph Lameter.
  14. *
  15. * Copyright (C) 2010 SUSE Linux Products GmbH
  16. * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
  17. *
  18. * This is the generic async execution mechanism. Work items as are
  19. * executed in process context. The worker pool is shared and
  20. * automatically managed. There are two worker pools for each CPU (one for
  21. * normal work items and the other for high priority ones) and some extra
  22. * pools for workqueues which are not bound to any specific CPU - the
  23. * number of these backing pools is dynamic.
  24. *
  25. * Please read Documentation/core-api/workqueue.rst for details.
  26. */
  27. #include <linux/export.h>
  28. #include <linux/kernel.h>
  29. #include <linux/sched.h>
  30. #include <linux/init.h>
  31. #include <linux/signal.h>
  32. #include <linux/completion.h>
  33. #include <linux/workqueue.h>
  34. #include <linux/slab.h>
  35. #include <linux/cpu.h>
  36. #include <linux/notifier.h>
  37. #include <linux/kthread.h>
  38. #include <linux/hardirq.h>
  39. #include <linux/mempolicy.h>
  40. #include <linux/freezer.h>
  41. #include <linux/debug_locks.h>
  42. #include <linux/lockdep.h>
  43. #include <linux/idr.h>
  44. #include <linux/jhash.h>
  45. #include <linux/hashtable.h>
  46. #include <linux/rculist.h>
  47. #include <linux/nodemask.h>
  48. #include <linux/moduleparam.h>
  49. #include <linux/uaccess.h>
  50. #include <linux/sched/isolation.h>
  51. #include <linux/nmi.h>
  52. #include <linux/kvm_para.h>
  53. #include "workqueue_internal.h"
  54. #include <trace/hooks/wqlockup.h>
  55. /* events/workqueue.h uses default TRACE_INCLUDE_PATH */
  56. #undef TRACE_INCLUDE_PATH
  57. enum {
  58. /*
  59. * worker_pool flags
  60. *
  61. * A bound pool is either associated or disassociated with its CPU.
  62. * While associated (!DISASSOCIATED), all workers are bound to the
  63. * CPU and none has %WORKER_UNBOUND set and concurrency management
  64. * is in effect.
  65. *
  66. * While DISASSOCIATED, the cpu may be offline and all workers have
  67. * %WORKER_UNBOUND set and concurrency management disabled, and may
  68. * be executing on any CPU. The pool behaves as an unbound one.
  69. *
  70. * Note that DISASSOCIATED should be flipped only while holding
  71. * wq_pool_attach_mutex to avoid changing binding state while
  72. * worker_attach_to_pool() is in progress.
  73. */
  74. POOL_MANAGER_ACTIVE = 1 << 0, /* being managed */
  75. POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
  76. /* worker flags */
  77. WORKER_DIE = 1 << 1, /* die die die */
  78. WORKER_IDLE = 1 << 2, /* is idle */
  79. WORKER_PREP = 1 << 3, /* preparing to run works */
  80. WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
  81. WORKER_UNBOUND = 1 << 7, /* worker is unbound */
  82. WORKER_REBOUND = 1 << 8, /* worker was rebound */
  83. WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
  84. WORKER_UNBOUND | WORKER_REBOUND,
  85. NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
  86. UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
  87. BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
  88. MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
  89. IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
  90. MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
  91. /* call for help after 10ms
  92. (min two ticks) */
  93. MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
  94. CREATE_COOLDOWN = HZ, /* time to breath after fail */
  95. /*
  96. * Rescue workers are used only on emergencies and shared by
  97. * all cpus. Give MIN_NICE.
  98. */
  99. RESCUER_NICE_LEVEL = MIN_NICE,
  100. HIGHPRI_NICE_LEVEL = MIN_NICE,
  101. WQ_NAME_LEN = 24,
  102. };
  103. /*
  104. * Structure fields follow one of the following exclusion rules.
  105. *
  106. * I: Modifiable by initialization/destruction paths and read-only for
  107. * everyone else.
  108. *
  109. * P: Preemption protected. Disabling preemption is enough and should
  110. * only be modified and accessed from the local cpu.
  111. *
  112. * L: pool->lock protected. Access with pool->lock held.
  113. *
  114. * X: During normal operation, modification requires pool->lock and should
  115. * be done only from local cpu. Either disabling preemption on local
  116. * cpu or grabbing pool->lock is enough for read access. If
  117. * POOL_DISASSOCIATED is set, it's identical to L.
  118. *
  119. * A: wq_pool_attach_mutex protected.
  120. *
  121. * PL: wq_pool_mutex protected.
  122. *
  123. * PR: wq_pool_mutex protected for writes. RCU protected for reads.
  124. *
  125. * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
  126. *
  127. * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
  128. * RCU for reads.
  129. *
  130. * WQ: wq->mutex protected.
  131. *
  132. * WR: wq->mutex protected for writes. RCU protected for reads.
  133. *
  134. * MD: wq_mayday_lock protected.
  135. */
  136. /* struct worker is defined in workqueue_internal.h */
  137. struct worker_pool {
  138. raw_spinlock_t lock; /* the pool lock */
  139. int cpu; /* I: the associated cpu */
  140. int node; /* I: the associated node ID */
  141. int id; /* I: pool ID */
  142. unsigned int flags; /* X: flags */
  143. unsigned long watchdog_ts; /* L: watchdog timestamp */
  144. struct list_head worklist; /* L: list of pending works */
  145. int nr_workers; /* L: total number of workers */
  146. int nr_idle; /* L: currently idle workers */
  147. struct list_head idle_list; /* X: list of idle workers */
  148. struct timer_list idle_timer; /* L: worker idle timeout */
  149. struct timer_list mayday_timer; /* L: SOS timer for workers */
  150. /* a workers is either on busy_hash or idle_list, or the manager */
  151. DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
  152. /* L: hash of busy workers */
  153. struct worker *manager; /* L: purely informational */
  154. struct list_head workers; /* A: attached workers */
  155. struct completion *detach_completion; /* all workers detached */
  156. struct ida worker_ida; /* worker IDs for task name */
  157. struct workqueue_attrs *attrs; /* I: worker attributes */
  158. struct hlist_node hash_node; /* PL: unbound_pool_hash node */
  159. int refcnt; /* PL: refcnt for unbound pools */
  160. /*
  161. * The current concurrency level. As it's likely to be accessed
  162. * from other CPUs during try_to_wake_up(), put it in a separate
  163. * cacheline.
  164. */
  165. atomic_t nr_running ____cacheline_aligned_in_smp;
  166. /*
  167. * Destruction of pool is RCU protected to allow dereferences
  168. * from get_work_pool().
  169. */
  170. struct rcu_head rcu;
  171. } ____cacheline_aligned_in_smp;
  172. /*
  173. * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
  174. * of work_struct->data are used for flags and the remaining high bits
  175. * point to the pwq; thus, pwqs need to be aligned at two's power of the
  176. * number of flag bits.
  177. */
  178. struct pool_workqueue {
  179. struct worker_pool *pool; /* I: the associated pool */
  180. struct workqueue_struct *wq; /* I: the owning workqueue */
  181. int work_color; /* L: current color */
  182. int flush_color; /* L: flushing color */
  183. int refcnt; /* L: reference count */
  184. int nr_in_flight[WORK_NR_COLORS];
  185. /* L: nr of in_flight works */
  186. int nr_active; /* L: nr of active works */
  187. int max_active; /* L: max active works */
  188. struct list_head delayed_works; /* L: delayed works */
  189. struct list_head pwqs_node; /* WR: node on wq->pwqs */
  190. struct list_head mayday_node; /* MD: node on wq->maydays */
  191. /*
  192. * Release of unbound pwq is punted to system_wq. See put_pwq()
  193. * and pwq_unbound_release_workfn() for details. pool_workqueue
  194. * itself is also RCU protected so that the first pwq can be
  195. * determined without grabbing wq->mutex.
  196. */
  197. struct work_struct unbound_release_work;
  198. struct rcu_head rcu;
  199. } __aligned(1 << WORK_STRUCT_FLAG_BITS);
  200. /*
  201. * Structure used to wait for workqueue flush.
  202. */
  203. struct wq_flusher {
  204. struct list_head list; /* WQ: list of flushers */
  205. int flush_color; /* WQ: flush color waiting for */
  206. struct completion done; /* flush completion */
  207. };
  208. struct wq_device;
  209. /*
  210. * The externally visible workqueue. It relays the issued work items to
  211. * the appropriate worker_pool through its pool_workqueues.
  212. */
  213. struct workqueue_struct {
  214. struct list_head pwqs; /* WR: all pwqs of this wq */
  215. struct list_head list; /* PR: list of all workqueues */
  216. struct mutex mutex; /* protects this wq */
  217. int work_color; /* WQ: current work color */
  218. int flush_color; /* WQ: current flush color */
  219. atomic_t nr_pwqs_to_flush; /* flush in progress */
  220. struct wq_flusher *first_flusher; /* WQ: first flusher */
  221. struct list_head flusher_queue; /* WQ: flush waiters */
  222. struct list_head flusher_overflow; /* WQ: flush overflow list */
  223. struct list_head maydays; /* MD: pwqs requesting rescue */
  224. struct worker *rescuer; /* MD: rescue worker */
  225. int nr_drainers; /* WQ: drain in progress */
  226. int saved_max_active; /* WQ: saved pwq max_active */
  227. struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
  228. struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
  229. #ifdef CONFIG_SYSFS
  230. struct wq_device *wq_dev; /* I: for sysfs interface */
  231. #endif
  232. #ifdef CONFIG_LOCKDEP
  233. char *lock_name;
  234. struct lock_class_key key;
  235. struct lockdep_map lockdep_map;
  236. #endif
  237. char name[WQ_NAME_LEN]; /* I: workqueue name */
  238. /*
  239. * Destruction of workqueue_struct is RCU protected to allow walking
  240. * the workqueues list without grabbing wq_pool_mutex.
  241. * This is used to dump all workqueues from sysrq.
  242. */
  243. struct rcu_head rcu;
  244. /* hot fields used during command issue, aligned to cacheline */
  245. unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
  246. struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
  247. struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
  248. };
  249. static struct kmem_cache *pwq_cache;
  250. static cpumask_var_t *wq_numa_possible_cpumask;
  251. /* possible CPUs of each node */
  252. static bool wq_disable_numa;
  253. module_param_named(disable_numa, wq_disable_numa, bool, 0444);
  254. /* see the comment above the definition of WQ_POWER_EFFICIENT */
  255. static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
  256. module_param_named(power_efficient, wq_power_efficient, bool, 0444);
  257. static bool wq_online; /* can kworkers be created yet? */
  258. static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
  259. /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
  260. static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
  261. static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
  262. static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
  263. static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
  264. /* wait for manager to go away */
  265. static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
  266. static LIST_HEAD(workqueues); /* PR: list of all workqueues */
  267. static bool workqueue_freezing; /* PL: have wqs started freezing? */
  268. /* PL: allowable cpus for unbound wqs and work items */
  269. static cpumask_var_t wq_unbound_cpumask;
  270. /* CPU where unbound work was last round robin scheduled from this CPU */
  271. static DEFINE_PER_CPU(int, wq_rr_cpu_last);
  272. /*
  273. * Local execution of unbound work items is no longer guaranteed. The
  274. * following always forces round-robin CPU selection on unbound work items
  275. * to uncover usages which depend on it.
  276. */
  277. #ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
  278. static bool wq_debug_force_rr_cpu = true;
  279. #else
  280. static bool wq_debug_force_rr_cpu = false;
  281. #endif
  282. module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
  283. /* the per-cpu worker pools */
  284. static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
  285. static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
  286. /* PL: hash of all unbound pools keyed by pool->attrs */
  287. static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
  288. /* I: attributes used when instantiating standard unbound pools on demand */
  289. static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
  290. /* I: attributes used when instantiating ordered pools on demand */
  291. static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
  292. struct workqueue_struct *system_wq __read_mostly;
  293. EXPORT_SYMBOL(system_wq);
  294. struct workqueue_struct *system_highpri_wq __read_mostly;
  295. EXPORT_SYMBOL_GPL(system_highpri_wq);
  296. struct workqueue_struct *system_long_wq __read_mostly;
  297. EXPORT_SYMBOL_GPL(system_long_wq);
  298. struct workqueue_struct *system_unbound_wq __read_mostly;
  299. EXPORT_SYMBOL_GPL(system_unbound_wq);
  300. struct workqueue_struct *system_freezable_wq __read_mostly;
  301. EXPORT_SYMBOL_GPL(system_freezable_wq);
  302. struct workqueue_struct *system_power_efficient_wq __read_mostly;
  303. EXPORT_SYMBOL_GPL(system_power_efficient_wq);
  304. struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
  305. EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
  306. static int worker_thread(void *__worker);
  307. static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
  308. static void show_pwq(struct pool_workqueue *pwq);
  309. #define CREATE_TRACE_POINTS
  310. #include <trace/events/workqueue.h>
  311. EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_start);
  312. EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_end);
  313. #define assert_rcu_or_pool_mutex() \
  314. RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  315. !lockdep_is_held(&wq_pool_mutex), \
  316. "RCU or wq_pool_mutex should be held")
  317. #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
  318. RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  319. !lockdep_is_held(&wq->mutex) && \
  320. !lockdep_is_held(&wq_pool_mutex), \
  321. "RCU, wq->mutex or wq_pool_mutex should be held")
  322. #define for_each_cpu_worker_pool(pool, cpu) \
  323. for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
  324. (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
  325. (pool)++)
  326. /**
  327. * for_each_pool - iterate through all worker_pools in the system
  328. * @pool: iteration cursor
  329. * @pi: integer used for iteration
  330. *
  331. * This must be called either with wq_pool_mutex held or RCU read
  332. * locked. If the pool needs to be used beyond the locking in effect, the
  333. * caller is responsible for guaranteeing that the pool stays online.
  334. *
  335. * The if/else clause exists only for the lockdep assertion and can be
  336. * ignored.
  337. */
  338. #define for_each_pool(pool, pi) \
  339. idr_for_each_entry(&worker_pool_idr, pool, pi) \
  340. if (({ assert_rcu_or_pool_mutex(); false; })) { } \
  341. else
  342. /**
  343. * for_each_pool_worker - iterate through all workers of a worker_pool
  344. * @worker: iteration cursor
  345. * @pool: worker_pool to iterate workers of
  346. *
  347. * This must be called with wq_pool_attach_mutex.
  348. *
  349. * The if/else clause exists only for the lockdep assertion and can be
  350. * ignored.
  351. */
  352. #define for_each_pool_worker(worker, pool) \
  353. list_for_each_entry((worker), &(pool)->workers, node) \
  354. if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
  355. else
  356. /**
  357. * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
  358. * @pwq: iteration cursor
  359. * @wq: the target workqueue
  360. *
  361. * This must be called either with wq->mutex held or RCU read locked.
  362. * If the pwq needs to be used beyond the locking in effect, the caller is
  363. * responsible for guaranteeing that the pwq stays online.
  364. *
  365. * The if/else clause exists only for the lockdep assertion and can be
  366. * ignored.
  367. */
  368. #define for_each_pwq(pwq, wq) \
  369. list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
  370. lockdep_is_held(&(wq->mutex)))
  371. #ifdef CONFIG_DEBUG_OBJECTS_WORK
  372. static const struct debug_obj_descr work_debug_descr;
  373. static void *work_debug_hint(void *addr)
  374. {
  375. return ((struct work_struct *) addr)->func;
  376. }
  377. static bool work_is_static_object(void *addr)
  378. {
  379. struct work_struct *work = addr;
  380. return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
  381. }
  382. /*
  383. * fixup_init is called when:
  384. * - an active object is initialized
  385. */
  386. static bool work_fixup_init(void *addr, enum debug_obj_state state)
  387. {
  388. struct work_struct *work = addr;
  389. switch (state) {
  390. case ODEBUG_STATE_ACTIVE:
  391. cancel_work_sync(work);
  392. debug_object_init(work, &work_debug_descr);
  393. return true;
  394. default:
  395. return false;
  396. }
  397. }
  398. /*
  399. * fixup_free is called when:
  400. * - an active object is freed
  401. */
  402. static bool work_fixup_free(void *addr, enum debug_obj_state state)
  403. {
  404. struct work_struct *work = addr;
  405. switch (state) {
  406. case ODEBUG_STATE_ACTIVE:
  407. cancel_work_sync(work);
  408. debug_object_free(work, &work_debug_descr);
  409. return true;
  410. default:
  411. return false;
  412. }
  413. }
  414. static const struct debug_obj_descr work_debug_descr = {
  415. .name = "work_struct",
  416. .debug_hint = work_debug_hint,
  417. .is_static_object = work_is_static_object,
  418. .fixup_init = work_fixup_init,
  419. .fixup_free = work_fixup_free,
  420. };
  421. static inline void debug_work_activate(struct work_struct *work)
  422. {
  423. debug_object_activate(work, &work_debug_descr);
  424. }
  425. static inline void debug_work_deactivate(struct work_struct *work)
  426. {
  427. debug_object_deactivate(work, &work_debug_descr);
  428. }
  429. void __init_work(struct work_struct *work, int onstack)
  430. {
  431. if (onstack)
  432. debug_object_init_on_stack(work, &work_debug_descr);
  433. else
  434. debug_object_init(work, &work_debug_descr);
  435. }
  436. EXPORT_SYMBOL_GPL(__init_work);
  437. void destroy_work_on_stack(struct work_struct *work)
  438. {
  439. debug_object_free(work, &work_debug_descr);
  440. }
  441. EXPORT_SYMBOL_GPL(destroy_work_on_stack);
  442. void destroy_delayed_work_on_stack(struct delayed_work *work)
  443. {
  444. destroy_timer_on_stack(&work->timer);
  445. debug_object_free(&work->work, &work_debug_descr);
  446. }
  447. EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
  448. #else
  449. static inline void debug_work_activate(struct work_struct *work) { }
  450. static inline void debug_work_deactivate(struct work_struct *work) { }
  451. #endif
  452. /**
  453. * worker_pool_assign_id - allocate ID and assing it to @pool
  454. * @pool: the pool pointer of interest
  455. *
  456. * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
  457. * successfully, -errno on failure.
  458. */
  459. static int worker_pool_assign_id(struct worker_pool *pool)
  460. {
  461. int ret;
  462. lockdep_assert_held(&wq_pool_mutex);
  463. ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
  464. GFP_KERNEL);
  465. if (ret >= 0) {
  466. pool->id = ret;
  467. return 0;
  468. }
  469. return ret;
  470. }
  471. /**
  472. * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
  473. * @wq: the target workqueue
  474. * @node: the node ID
  475. *
  476. * This must be called with any of wq_pool_mutex, wq->mutex or RCU
  477. * read locked.
  478. * If the pwq needs to be used beyond the locking in effect, the caller is
  479. * responsible for guaranteeing that the pwq stays online.
  480. *
  481. * Return: The unbound pool_workqueue for @node.
  482. */
  483. static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
  484. int node)
  485. {
  486. assert_rcu_or_wq_mutex_or_pool_mutex(wq);
  487. /*
  488. * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
  489. * delayed item is pending. The plan is to keep CPU -> NODE
  490. * mapping valid and stable across CPU on/offlines. Once that
  491. * happens, this workaround can be removed.
  492. */
  493. if (unlikely(node == NUMA_NO_NODE))
  494. return wq->dfl_pwq;
  495. return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
  496. }
  497. static unsigned int work_color_to_flags(int color)
  498. {
  499. return color << WORK_STRUCT_COLOR_SHIFT;
  500. }
  501. static int get_work_color(struct work_struct *work)
  502. {
  503. return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
  504. ((1 << WORK_STRUCT_COLOR_BITS) - 1);
  505. }
  506. static int work_next_color(int color)
  507. {
  508. return (color + 1) % WORK_NR_COLORS;
  509. }
  510. /*
  511. * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
  512. * contain the pointer to the queued pwq. Once execution starts, the flag
  513. * is cleared and the high bits contain OFFQ flags and pool ID.
  514. *
  515. * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
  516. * and clear_work_data() can be used to set the pwq, pool or clear
  517. * work->data. These functions should only be called while the work is
  518. * owned - ie. while the PENDING bit is set.
  519. *
  520. * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
  521. * corresponding to a work. Pool is available once the work has been
  522. * queued anywhere after initialization until it is sync canceled. pwq is
  523. * available only while the work item is queued.
  524. *
  525. * %WORK_OFFQ_CANCELING is used to mark a work item which is being
  526. * canceled. While being canceled, a work item may have its PENDING set
  527. * but stay off timer and worklist for arbitrarily long and nobody should
  528. * try to steal the PENDING bit.
  529. */
  530. static inline void set_work_data(struct work_struct *work, unsigned long data,
  531. unsigned long flags)
  532. {
  533. WARN_ON_ONCE(!work_pending(work));
  534. atomic_long_set(&work->data, data | flags | work_static(work));
  535. }
  536. static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
  537. unsigned long extra_flags)
  538. {
  539. set_work_data(work, (unsigned long)pwq,
  540. WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
  541. }
  542. static void set_work_pool_and_keep_pending(struct work_struct *work,
  543. int pool_id)
  544. {
  545. set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
  546. WORK_STRUCT_PENDING);
  547. }
  548. static void set_work_pool_and_clear_pending(struct work_struct *work,
  549. int pool_id)
  550. {
  551. /*
  552. * The following wmb is paired with the implied mb in
  553. * test_and_set_bit(PENDING) and ensures all updates to @work made
  554. * here are visible to and precede any updates by the next PENDING
  555. * owner.
  556. */
  557. smp_wmb();
  558. set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
  559. /*
  560. * The following mb guarantees that previous clear of a PENDING bit
  561. * will not be reordered with any speculative LOADS or STORES from
  562. * work->current_func, which is executed afterwards. This possible
  563. * reordering can lead to a missed execution on attempt to queue
  564. * the same @work. E.g. consider this case:
  565. *
  566. * CPU#0 CPU#1
  567. * ---------------------------- --------------------------------
  568. *
  569. * 1 STORE event_indicated
  570. * 2 queue_work_on() {
  571. * 3 test_and_set_bit(PENDING)
  572. * 4 } set_..._and_clear_pending() {
  573. * 5 set_work_data() # clear bit
  574. * 6 smp_mb()
  575. * 7 work->current_func() {
  576. * 8 LOAD event_indicated
  577. * }
  578. *
  579. * Without an explicit full barrier speculative LOAD on line 8 can
  580. * be executed before CPU#0 does STORE on line 1. If that happens,
  581. * CPU#0 observes the PENDING bit is still set and new execution of
  582. * a @work is not queued in a hope, that CPU#1 will eventually
  583. * finish the queued @work. Meanwhile CPU#1 does not see
  584. * event_indicated is set, because speculative LOAD was executed
  585. * before actual STORE.
  586. */
  587. smp_mb();
  588. }
  589. static void clear_work_data(struct work_struct *work)
  590. {
  591. smp_wmb(); /* see set_work_pool_and_clear_pending() */
  592. set_work_data(work, WORK_STRUCT_NO_POOL, 0);
  593. }
  594. static struct pool_workqueue *get_work_pwq(struct work_struct *work)
  595. {
  596. unsigned long data = atomic_long_read(&work->data);
  597. if (data & WORK_STRUCT_PWQ)
  598. return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
  599. else
  600. return NULL;
  601. }
  602. /**
  603. * get_work_pool - return the worker_pool a given work was associated with
  604. * @work: the work item of interest
  605. *
  606. * Pools are created and destroyed under wq_pool_mutex, and allows read
  607. * access under RCU read lock. As such, this function should be
  608. * called under wq_pool_mutex or inside of a rcu_read_lock() region.
  609. *
  610. * All fields of the returned pool are accessible as long as the above
  611. * mentioned locking is in effect. If the returned pool needs to be used
  612. * beyond the critical section, the caller is responsible for ensuring the
  613. * returned pool is and stays online.
  614. *
  615. * Return: The worker_pool @work was last associated with. %NULL if none.
  616. */
  617. static struct worker_pool *get_work_pool(struct work_struct *work)
  618. {
  619. unsigned long data = atomic_long_read(&work->data);
  620. int pool_id;
  621. assert_rcu_or_pool_mutex();
  622. if (data & WORK_STRUCT_PWQ)
  623. return ((struct pool_workqueue *)
  624. (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
  625. pool_id = data >> WORK_OFFQ_POOL_SHIFT;
  626. if (pool_id == WORK_OFFQ_POOL_NONE)
  627. return NULL;
  628. return idr_find(&worker_pool_idr, pool_id);
  629. }
  630. /**
  631. * get_work_pool_id - return the worker pool ID a given work is associated with
  632. * @work: the work item of interest
  633. *
  634. * Return: The worker_pool ID @work was last associated with.
  635. * %WORK_OFFQ_POOL_NONE if none.
  636. */
  637. static int get_work_pool_id(struct work_struct *work)
  638. {
  639. unsigned long data = atomic_long_read(&work->data);
  640. if (data & WORK_STRUCT_PWQ)
  641. return ((struct pool_workqueue *)
  642. (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
  643. return data >> WORK_OFFQ_POOL_SHIFT;
  644. }
  645. static void mark_work_canceling(struct work_struct *work)
  646. {
  647. unsigned long pool_id = get_work_pool_id(work);
  648. pool_id <<= WORK_OFFQ_POOL_SHIFT;
  649. set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
  650. }
  651. static bool work_is_canceling(struct work_struct *work)
  652. {
  653. unsigned long data = atomic_long_read(&work->data);
  654. return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
  655. }
  656. /*
  657. * Policy functions. These define the policies on how the global worker
  658. * pools are managed. Unless noted otherwise, these functions assume that
  659. * they're being called with pool->lock held.
  660. */
  661. static bool __need_more_worker(struct worker_pool *pool)
  662. {
  663. return !atomic_read(&pool->nr_running);
  664. }
  665. /*
  666. * Need to wake up a worker? Called from anything but currently
  667. * running workers.
  668. *
  669. * Note that, because unbound workers never contribute to nr_running, this
  670. * function will always return %true for unbound pools as long as the
  671. * worklist isn't empty.
  672. */
  673. static bool need_more_worker(struct worker_pool *pool)
  674. {
  675. return !list_empty(&pool->worklist) && __need_more_worker(pool);
  676. }
  677. /* Can I start working? Called from busy but !running workers. */
  678. static bool may_start_working(struct worker_pool *pool)
  679. {
  680. return pool->nr_idle;
  681. }
  682. /* Do I need to keep working? Called from currently running workers. */
  683. static bool keep_working(struct worker_pool *pool)
  684. {
  685. return !list_empty(&pool->worklist) &&
  686. atomic_read(&pool->nr_running) <= 1;
  687. }
  688. /* Do we need a new worker? Called from manager. */
  689. static bool need_to_create_worker(struct worker_pool *pool)
  690. {
  691. return need_more_worker(pool) && !may_start_working(pool);
  692. }
  693. /* Do we have too many workers and should some go away? */
  694. static bool too_many_workers(struct worker_pool *pool)
  695. {
  696. bool managing = pool->flags & POOL_MANAGER_ACTIVE;
  697. int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
  698. int nr_busy = pool->nr_workers - nr_idle;
  699. return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
  700. }
  701. /*
  702. * Wake up functions.
  703. */
  704. /* Return the first idle worker. Safe with preemption disabled */
  705. static struct worker *first_idle_worker(struct worker_pool *pool)
  706. {
  707. if (unlikely(list_empty(&pool->idle_list)))
  708. return NULL;
  709. return list_first_entry(&pool->idle_list, struct worker, entry);
  710. }
  711. /**
  712. * wake_up_worker - wake up an idle worker
  713. * @pool: worker pool to wake worker from
  714. *
  715. * Wake up the first idle worker of @pool.
  716. *
  717. * CONTEXT:
  718. * raw_spin_lock_irq(pool->lock).
  719. */
  720. static void wake_up_worker(struct worker_pool *pool)
  721. {
  722. struct worker *worker = first_idle_worker(pool);
  723. if (likely(worker))
  724. wake_up_process(worker->task);
  725. }
  726. /**
  727. * wq_worker_running - a worker is running again
  728. * @task: task waking up
  729. *
  730. * This function is called when a worker returns from schedule()
  731. */
  732. void wq_worker_running(struct task_struct *task)
  733. {
  734. struct worker *worker = kthread_data(task);
  735. if (!worker->sleeping)
  736. return;
  737. /*
  738. * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
  739. * and the nr_running increment below, we may ruin the nr_running reset
  740. * and leave with an unexpected pool->nr_running == 1 on the newly unbound
  741. * pool. Protect against such race.
  742. */
  743. preempt_disable();
  744. if (!(worker->flags & WORKER_NOT_RUNNING))
  745. atomic_inc(&worker->pool->nr_running);
  746. preempt_enable();
  747. worker->sleeping = 0;
  748. }
  749. /**
  750. * wq_worker_sleeping - a worker is going to sleep
  751. * @task: task going to sleep
  752. *
  753. * This function is called from schedule() when a busy worker is
  754. * going to sleep. Preemption needs to be disabled to protect ->sleeping
  755. * assignment.
  756. */
  757. void wq_worker_sleeping(struct task_struct *task)
  758. {
  759. struct worker *next, *worker = kthread_data(task);
  760. struct worker_pool *pool;
  761. /*
  762. * Rescuers, which may not have all the fields set up like normal
  763. * workers, also reach here, let's not access anything before
  764. * checking NOT_RUNNING.
  765. */
  766. if (worker->flags & WORKER_NOT_RUNNING)
  767. return;
  768. pool = worker->pool;
  769. /* Return if preempted before wq_worker_running() was reached */
  770. if (worker->sleeping)
  771. return;
  772. worker->sleeping = 1;
  773. raw_spin_lock_irq(&pool->lock);
  774. /*
  775. * The counterpart of the following dec_and_test, implied mb,
  776. * worklist not empty test sequence is in insert_work().
  777. * Please read comment there.
  778. *
  779. * NOT_RUNNING is clear. This means that we're bound to and
  780. * running on the local cpu w/ rq lock held and preemption
  781. * disabled, which in turn means that none else could be
  782. * manipulating idle_list, so dereferencing idle_list without pool
  783. * lock is safe.
  784. */
  785. if (atomic_dec_and_test(&pool->nr_running) &&
  786. !list_empty(&pool->worklist)) {
  787. next = first_idle_worker(pool);
  788. if (next)
  789. wake_up_process(next->task);
  790. }
  791. raw_spin_unlock_irq(&pool->lock);
  792. }
  793. /**
  794. * wq_worker_last_func - retrieve worker's last work function
  795. * @task: Task to retrieve last work function of.
  796. *
  797. * Determine the last function a worker executed. This is called from
  798. * the scheduler to get a worker's last known identity.
  799. *
  800. * CONTEXT:
  801. * raw_spin_lock_irq(rq->lock)
  802. *
  803. * This function is called during schedule() when a kworker is going
  804. * to sleep. It's used by psi to identify aggregation workers during
  805. * dequeuing, to allow periodic aggregation to shut-off when that
  806. * worker is the last task in the system or cgroup to go to sleep.
  807. *
  808. * As this function doesn't involve any workqueue-related locking, it
  809. * only returns stable values when called from inside the scheduler's
  810. * queuing and dequeuing paths, when @task, which must be a kworker,
  811. * is guaranteed to not be processing any works.
  812. *
  813. * Return:
  814. * The last work function %current executed as a worker, NULL if it
  815. * hasn't executed any work yet.
  816. */
  817. work_func_t wq_worker_last_func(struct task_struct *task)
  818. {
  819. struct worker *worker = kthread_data(task);
  820. return worker->last_func;
  821. }
  822. /**
  823. * worker_set_flags - set worker flags and adjust nr_running accordingly
  824. * @worker: self
  825. * @flags: flags to set
  826. *
  827. * Set @flags in @worker->flags and adjust nr_running accordingly.
  828. *
  829. * CONTEXT:
  830. * raw_spin_lock_irq(pool->lock)
  831. */
  832. static inline void worker_set_flags(struct worker *worker, unsigned int flags)
  833. {
  834. struct worker_pool *pool = worker->pool;
  835. WARN_ON_ONCE(worker->task != current);
  836. /* If transitioning into NOT_RUNNING, adjust nr_running. */
  837. if ((flags & WORKER_NOT_RUNNING) &&
  838. !(worker->flags & WORKER_NOT_RUNNING)) {
  839. atomic_dec(&pool->nr_running);
  840. }
  841. worker->flags |= flags;
  842. }
  843. /**
  844. * worker_clr_flags - clear worker flags and adjust nr_running accordingly
  845. * @worker: self
  846. * @flags: flags to clear
  847. *
  848. * Clear @flags in @worker->flags and adjust nr_running accordingly.
  849. *
  850. * CONTEXT:
  851. * raw_spin_lock_irq(pool->lock)
  852. */
  853. static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
  854. {
  855. struct worker_pool *pool = worker->pool;
  856. unsigned int oflags = worker->flags;
  857. WARN_ON_ONCE(worker->task != current);
  858. worker->flags &= ~flags;
  859. /*
  860. * If transitioning out of NOT_RUNNING, increment nr_running. Note
  861. * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
  862. * of multiple flags, not a single flag.
  863. */
  864. if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
  865. if (!(worker->flags & WORKER_NOT_RUNNING))
  866. atomic_inc(&pool->nr_running);
  867. }
  868. /**
  869. * find_worker_executing_work - find worker which is executing a work
  870. * @pool: pool of interest
  871. * @work: work to find worker for
  872. *
  873. * Find a worker which is executing @work on @pool by searching
  874. * @pool->busy_hash which is keyed by the address of @work. For a worker
  875. * to match, its current execution should match the address of @work and
  876. * its work function. This is to avoid unwanted dependency between
  877. * unrelated work executions through a work item being recycled while still
  878. * being executed.
  879. *
  880. * This is a bit tricky. A work item may be freed once its execution
  881. * starts and nothing prevents the freed area from being recycled for
  882. * another work item. If the same work item address ends up being reused
  883. * before the original execution finishes, workqueue will identify the
  884. * recycled work item as currently executing and make it wait until the
  885. * current execution finishes, introducing an unwanted dependency.
  886. *
  887. * This function checks the work item address and work function to avoid
  888. * false positives. Note that this isn't complete as one may construct a
  889. * work function which can introduce dependency onto itself through a
  890. * recycled work item. Well, if somebody wants to shoot oneself in the
  891. * foot that badly, there's only so much we can do, and if such deadlock
  892. * actually occurs, it should be easy to locate the culprit work function.
  893. *
  894. * CONTEXT:
  895. * raw_spin_lock_irq(pool->lock).
  896. *
  897. * Return:
  898. * Pointer to worker which is executing @work if found, %NULL
  899. * otherwise.
  900. */
  901. static struct worker *find_worker_executing_work(struct worker_pool *pool,
  902. struct work_struct *work)
  903. {
  904. struct worker *worker;
  905. hash_for_each_possible(pool->busy_hash, worker, hentry,
  906. (unsigned long)work)
  907. if (worker->current_work == work &&
  908. worker->current_func == work->func)
  909. return worker;
  910. return NULL;
  911. }
  912. /**
  913. * move_linked_works - move linked works to a list
  914. * @work: start of series of works to be scheduled
  915. * @head: target list to append @work to
  916. * @nextp: out parameter for nested worklist walking
  917. *
  918. * Schedule linked works starting from @work to @head. Work series to
  919. * be scheduled starts at @work and includes any consecutive work with
  920. * WORK_STRUCT_LINKED set in its predecessor.
  921. *
  922. * If @nextp is not NULL, it's updated to point to the next work of
  923. * the last scheduled work. This allows move_linked_works() to be
  924. * nested inside outer list_for_each_entry_safe().
  925. *
  926. * CONTEXT:
  927. * raw_spin_lock_irq(pool->lock).
  928. */
  929. static void move_linked_works(struct work_struct *work, struct list_head *head,
  930. struct work_struct **nextp)
  931. {
  932. struct work_struct *n;
  933. /*
  934. * Linked worklist will always end before the end of the list,
  935. * use NULL for list head.
  936. */
  937. list_for_each_entry_safe_from(work, n, NULL, entry) {
  938. list_move_tail(&work->entry, head);
  939. if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
  940. break;
  941. }
  942. /*
  943. * If we're already inside safe list traversal and have moved
  944. * multiple works to the scheduled queue, the next position
  945. * needs to be updated.
  946. */
  947. if (nextp)
  948. *nextp = n;
  949. }
  950. /**
  951. * get_pwq - get an extra reference on the specified pool_workqueue
  952. * @pwq: pool_workqueue to get
  953. *
  954. * Obtain an extra reference on @pwq. The caller should guarantee that
  955. * @pwq has positive refcnt and be holding the matching pool->lock.
  956. */
  957. static void get_pwq(struct pool_workqueue *pwq)
  958. {
  959. lockdep_assert_held(&pwq->pool->lock);
  960. WARN_ON_ONCE(pwq->refcnt <= 0);
  961. pwq->refcnt++;
  962. }
  963. /**
  964. * put_pwq - put a pool_workqueue reference
  965. * @pwq: pool_workqueue to put
  966. *
  967. * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
  968. * destruction. The caller should be holding the matching pool->lock.
  969. */
  970. static void put_pwq(struct pool_workqueue *pwq)
  971. {
  972. lockdep_assert_held(&pwq->pool->lock);
  973. if (likely(--pwq->refcnt))
  974. return;
  975. if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
  976. return;
  977. /*
  978. * @pwq can't be released under pool->lock, bounce to
  979. * pwq_unbound_release_workfn(). This never recurses on the same
  980. * pool->lock as this path is taken only for unbound workqueues and
  981. * the release work item is scheduled on a per-cpu workqueue. To
  982. * avoid lockdep warning, unbound pool->locks are given lockdep
  983. * subclass of 1 in get_unbound_pool().
  984. */
  985. schedule_work(&pwq->unbound_release_work);
  986. }
  987. /**
  988. * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
  989. * @pwq: pool_workqueue to put (can be %NULL)
  990. *
  991. * put_pwq() with locking. This function also allows %NULL @pwq.
  992. */
  993. static void put_pwq_unlocked(struct pool_workqueue *pwq)
  994. {
  995. if (pwq) {
  996. /*
  997. * As both pwqs and pools are RCU protected, the
  998. * following lock operations are safe.
  999. */
  1000. raw_spin_lock_irq(&pwq->pool->lock);
  1001. put_pwq(pwq);
  1002. raw_spin_unlock_irq(&pwq->pool->lock);
  1003. }
  1004. }
  1005. static void pwq_activate_delayed_work(struct work_struct *work)
  1006. {
  1007. struct pool_workqueue *pwq = get_work_pwq(work);
  1008. trace_workqueue_activate_work(work);
  1009. if (list_empty(&pwq->pool->worklist))
  1010. pwq->pool->watchdog_ts = jiffies;
  1011. move_linked_works(work, &pwq->pool->worklist, NULL);
  1012. __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
  1013. pwq->nr_active++;
  1014. }
  1015. static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
  1016. {
  1017. struct work_struct *work = list_first_entry(&pwq->delayed_works,
  1018. struct work_struct, entry);
  1019. pwq_activate_delayed_work(work);
  1020. }
  1021. /**
  1022. * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
  1023. * @pwq: pwq of interest
  1024. * @color: color of work which left the queue
  1025. *
  1026. * A work either has completed or is removed from pending queue,
  1027. * decrement nr_in_flight of its pwq and handle workqueue flushing.
  1028. *
  1029. * CONTEXT:
  1030. * raw_spin_lock_irq(pool->lock).
  1031. */
  1032. static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
  1033. {
  1034. /* uncolored work items don't participate in flushing or nr_active */
  1035. if (color == WORK_NO_COLOR)
  1036. goto out_put;
  1037. pwq->nr_in_flight[color]--;
  1038. pwq->nr_active--;
  1039. if (!list_empty(&pwq->delayed_works)) {
  1040. /* one down, submit a delayed one */
  1041. if (pwq->nr_active < pwq->max_active)
  1042. pwq_activate_first_delayed(pwq);
  1043. }
  1044. /* is flush in progress and are we at the flushing tip? */
  1045. if (likely(pwq->flush_color != color))
  1046. goto out_put;
  1047. /* are there still in-flight works? */
  1048. if (pwq->nr_in_flight[color])
  1049. goto out_put;
  1050. /* this pwq is done, clear flush_color */
  1051. pwq->flush_color = -1;
  1052. /*
  1053. * If this was the last pwq, wake up the first flusher. It
  1054. * will handle the rest.
  1055. */
  1056. if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
  1057. complete(&pwq->wq->first_flusher->done);
  1058. out_put:
  1059. put_pwq(pwq);
  1060. }
  1061. /**
  1062. * try_to_grab_pending - steal work item from worklist and disable irq
  1063. * @work: work item to steal
  1064. * @is_dwork: @work is a delayed_work
  1065. * @flags: place to store irq state
  1066. *
  1067. * Try to grab PENDING bit of @work. This function can handle @work in any
  1068. * stable state - idle, on timer or on worklist.
  1069. *
  1070. * Return:
  1071. *
  1072. * ======== ================================================================
  1073. * 1 if @work was pending and we successfully stole PENDING
  1074. * 0 if @work was idle and we claimed PENDING
  1075. * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
  1076. * -ENOENT if someone else is canceling @work, this state may persist
  1077. * for arbitrarily long
  1078. * ======== ================================================================
  1079. *
  1080. * Note:
  1081. * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
  1082. * interrupted while holding PENDING and @work off queue, irq must be
  1083. * disabled on entry. This, combined with delayed_work->timer being
  1084. * irqsafe, ensures that we return -EAGAIN for finite short period of time.
  1085. *
  1086. * On successful return, >= 0, irq is disabled and the caller is
  1087. * responsible for releasing it using local_irq_restore(*@flags).
  1088. *
  1089. * This function is safe to call from any context including IRQ handler.
  1090. */
  1091. static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
  1092. unsigned long *flags)
  1093. {
  1094. struct worker_pool *pool;
  1095. struct pool_workqueue *pwq;
  1096. local_irq_save(*flags);
  1097. /* try to steal the timer if it exists */
  1098. if (is_dwork) {
  1099. struct delayed_work *dwork = to_delayed_work(work);
  1100. /*
  1101. * dwork->timer is irqsafe. If del_timer() fails, it's
  1102. * guaranteed that the timer is not queued anywhere and not
  1103. * running on the local CPU.
  1104. */
  1105. if (likely(del_timer(&dwork->timer)))
  1106. return 1;
  1107. }
  1108. /* try to claim PENDING the normal way */
  1109. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
  1110. return 0;
  1111. rcu_read_lock();
  1112. /*
  1113. * The queueing is in progress, or it is already queued. Try to
  1114. * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
  1115. */
  1116. pool = get_work_pool(work);
  1117. if (!pool)
  1118. goto fail;
  1119. raw_spin_lock(&pool->lock);
  1120. /*
  1121. * work->data is guaranteed to point to pwq only while the work
  1122. * item is queued on pwq->wq, and both updating work->data to point
  1123. * to pwq on queueing and to pool on dequeueing are done under
  1124. * pwq->pool->lock. This in turn guarantees that, if work->data
  1125. * points to pwq which is associated with a locked pool, the work
  1126. * item is currently queued on that pool.
  1127. */
  1128. pwq = get_work_pwq(work);
  1129. if (pwq && pwq->pool == pool) {
  1130. debug_work_deactivate(work);
  1131. /*
  1132. * A delayed work item cannot be grabbed directly because
  1133. * it might have linked NO_COLOR work items which, if left
  1134. * on the delayed_list, will confuse pwq->nr_active
  1135. * management later on and cause stall. Make sure the work
  1136. * item is activated before grabbing.
  1137. */
  1138. if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
  1139. pwq_activate_delayed_work(work);
  1140. list_del_init(&work->entry);
  1141. pwq_dec_nr_in_flight(pwq, get_work_color(work));
  1142. /* work->data points to pwq iff queued, point to pool */
  1143. set_work_pool_and_keep_pending(work, pool->id);
  1144. raw_spin_unlock(&pool->lock);
  1145. rcu_read_unlock();
  1146. return 1;
  1147. }
  1148. raw_spin_unlock(&pool->lock);
  1149. fail:
  1150. rcu_read_unlock();
  1151. local_irq_restore(*flags);
  1152. if (work_is_canceling(work))
  1153. return -ENOENT;
  1154. cpu_relax();
  1155. return -EAGAIN;
  1156. }
  1157. /**
  1158. * insert_work - insert a work into a pool
  1159. * @pwq: pwq @work belongs to
  1160. * @work: work to insert
  1161. * @head: insertion point
  1162. * @extra_flags: extra WORK_STRUCT_* flags to set
  1163. *
  1164. * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
  1165. * work_struct flags.
  1166. *
  1167. * CONTEXT:
  1168. * raw_spin_lock_irq(pool->lock).
  1169. */
  1170. static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
  1171. struct list_head *head, unsigned int extra_flags)
  1172. {
  1173. struct worker_pool *pool = pwq->pool;
  1174. /* record the work call stack in order to print it in KASAN reports */
  1175. kasan_record_aux_stack(work);
  1176. /* we own @work, set data and link */
  1177. set_work_pwq(work, pwq, extra_flags);
  1178. list_add_tail(&work->entry, head);
  1179. get_pwq(pwq);
  1180. /*
  1181. * Ensure either wq_worker_sleeping() sees the above
  1182. * list_add_tail() or we see zero nr_running to avoid workers lying
  1183. * around lazily while there are works to be processed.
  1184. */
  1185. smp_mb();
  1186. if (__need_more_worker(pool))
  1187. wake_up_worker(pool);
  1188. }
  1189. /*
  1190. * Test whether @work is being queued from another work executing on the
  1191. * same workqueue.
  1192. */
  1193. static bool is_chained_work(struct workqueue_struct *wq)
  1194. {
  1195. struct worker *worker;
  1196. worker = current_wq_worker();
  1197. /*
  1198. * Return %true iff I'm a worker executing a work item on @wq. If
  1199. * I'm @worker, it's safe to dereference it without locking.
  1200. */
  1201. return worker && worker->current_pwq->wq == wq;
  1202. }
  1203. /*
  1204. * When queueing an unbound work item to a wq, prefer local CPU if allowed
  1205. * by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
  1206. * avoid perturbing sensitive tasks.
  1207. */
  1208. static int wq_select_unbound_cpu(int cpu)
  1209. {
  1210. static bool printed_dbg_warning;
  1211. int new_cpu;
  1212. if (likely(!wq_debug_force_rr_cpu)) {
  1213. if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
  1214. return cpu;
  1215. } else if (!printed_dbg_warning) {
  1216. pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n");
  1217. printed_dbg_warning = true;
  1218. }
  1219. if (cpumask_empty(wq_unbound_cpumask))
  1220. return cpu;
  1221. new_cpu = __this_cpu_read(wq_rr_cpu_last);
  1222. new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
  1223. if (unlikely(new_cpu >= nr_cpu_ids)) {
  1224. new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
  1225. if (unlikely(new_cpu >= nr_cpu_ids))
  1226. return cpu;
  1227. }
  1228. __this_cpu_write(wq_rr_cpu_last, new_cpu);
  1229. return new_cpu;
  1230. }
  1231. static void __queue_work(int cpu, struct workqueue_struct *wq,
  1232. struct work_struct *work)
  1233. {
  1234. struct pool_workqueue *pwq;
  1235. struct worker_pool *last_pool;
  1236. struct list_head *worklist;
  1237. unsigned int work_flags;
  1238. unsigned int req_cpu = cpu;
  1239. /*
  1240. * While a work item is PENDING && off queue, a task trying to
  1241. * steal the PENDING will busy-loop waiting for it to either get
  1242. * queued or lose PENDING. Grabbing PENDING and queueing should
  1243. * happen with IRQ disabled.
  1244. */
  1245. lockdep_assert_irqs_disabled();
  1246. /* if draining, only works from the same workqueue are allowed */
  1247. if (unlikely(wq->flags & __WQ_DRAINING) &&
  1248. WARN_ON_ONCE(!is_chained_work(wq)))
  1249. return;
  1250. rcu_read_lock();
  1251. retry:
  1252. /* pwq which will be used unless @work is executing elsewhere */
  1253. if (wq->flags & WQ_UNBOUND) {
  1254. if (req_cpu == WORK_CPU_UNBOUND)
  1255. cpu = wq_select_unbound_cpu(raw_smp_processor_id());
  1256. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  1257. } else {
  1258. if (req_cpu == WORK_CPU_UNBOUND)
  1259. cpu = raw_smp_processor_id();
  1260. pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
  1261. }
  1262. /*
  1263. * If @work was previously on a different pool, it might still be
  1264. * running there, in which case the work needs to be queued on that
  1265. * pool to guarantee non-reentrancy.
  1266. */
  1267. last_pool = get_work_pool(work);
  1268. if (last_pool && last_pool != pwq->pool) {
  1269. struct worker *worker;
  1270. raw_spin_lock(&last_pool->lock);
  1271. worker = find_worker_executing_work(last_pool, work);
  1272. if (worker && worker->current_pwq->wq == wq) {
  1273. pwq = worker->current_pwq;
  1274. } else {
  1275. /* meh... not running there, queue here */
  1276. raw_spin_unlock(&last_pool->lock);
  1277. raw_spin_lock(&pwq->pool->lock);
  1278. }
  1279. } else {
  1280. raw_spin_lock(&pwq->pool->lock);
  1281. }
  1282. /*
  1283. * pwq is determined and locked. For unbound pools, we could have
  1284. * raced with pwq release and it could already be dead. If its
  1285. * refcnt is zero, repeat pwq selection. Note that pwqs never die
  1286. * without another pwq replacing it in the numa_pwq_tbl or while
  1287. * work items are executing on it, so the retrying is guaranteed to
  1288. * make forward-progress.
  1289. */
  1290. if (unlikely(!pwq->refcnt)) {
  1291. if (wq->flags & WQ_UNBOUND) {
  1292. raw_spin_unlock(&pwq->pool->lock);
  1293. cpu_relax();
  1294. goto retry;
  1295. }
  1296. /* oops */
  1297. WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
  1298. wq->name, cpu);
  1299. }
  1300. /* pwq determined, queue */
  1301. trace_workqueue_queue_work(req_cpu, pwq, work);
  1302. if (WARN_ON(!list_empty(&work->entry)))
  1303. goto out;
  1304. pwq->nr_in_flight[pwq->work_color]++;
  1305. work_flags = work_color_to_flags(pwq->work_color);
  1306. if (likely(pwq->nr_active < pwq->max_active)) {
  1307. trace_workqueue_activate_work(work);
  1308. pwq->nr_active++;
  1309. worklist = &pwq->pool->worklist;
  1310. if (list_empty(worklist))
  1311. pwq->pool->watchdog_ts = jiffies;
  1312. } else {
  1313. work_flags |= WORK_STRUCT_DELAYED;
  1314. worklist = &pwq->delayed_works;
  1315. }
  1316. debug_work_activate(work);
  1317. insert_work(pwq, work, worklist, work_flags);
  1318. out:
  1319. raw_spin_unlock(&pwq->pool->lock);
  1320. rcu_read_unlock();
  1321. }
  1322. /**
  1323. * queue_work_on - queue work on specific cpu
  1324. * @cpu: CPU number to execute work on
  1325. * @wq: workqueue to use
  1326. * @work: work to queue
  1327. *
  1328. * We queue the work to a specific CPU, the caller must ensure it
  1329. * can't go away.
  1330. *
  1331. * Return: %false if @work was already on a queue, %true otherwise.
  1332. */
  1333. bool queue_work_on(int cpu, struct workqueue_struct *wq,
  1334. struct work_struct *work)
  1335. {
  1336. bool ret = false;
  1337. unsigned long flags;
  1338. local_irq_save(flags);
  1339. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1340. __queue_work(cpu, wq, work);
  1341. ret = true;
  1342. }
  1343. local_irq_restore(flags);
  1344. return ret;
  1345. }
  1346. EXPORT_SYMBOL(queue_work_on);
  1347. /**
  1348. * workqueue_select_cpu_near - Select a CPU based on NUMA node
  1349. * @node: NUMA node ID that we want to select a CPU from
  1350. *
  1351. * This function will attempt to find a "random" cpu available on a given
  1352. * node. If there are no CPUs available on the given node it will return
  1353. * WORK_CPU_UNBOUND indicating that we should just schedule to any
  1354. * available CPU if we need to schedule this work.
  1355. */
  1356. static int workqueue_select_cpu_near(int node)
  1357. {
  1358. int cpu;
  1359. /* No point in doing this if NUMA isn't enabled for workqueues */
  1360. if (!wq_numa_enabled)
  1361. return WORK_CPU_UNBOUND;
  1362. /* Delay binding to CPU if node is not valid or online */
  1363. if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
  1364. return WORK_CPU_UNBOUND;
  1365. /* Use local node/cpu if we are already there */
  1366. cpu = raw_smp_processor_id();
  1367. if (node == cpu_to_node(cpu))
  1368. return cpu;
  1369. /* Use "random" otherwise know as "first" online CPU of node */
  1370. cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
  1371. /* If CPU is valid return that, otherwise just defer */
  1372. return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
  1373. }
  1374. /**
  1375. * queue_work_node - queue work on a "random" cpu for a given NUMA node
  1376. * @node: NUMA node that we are targeting the work for
  1377. * @wq: workqueue to use
  1378. * @work: work to queue
  1379. *
  1380. * We queue the work to a "random" CPU within a given NUMA node. The basic
  1381. * idea here is to provide a way to somehow associate work with a given
  1382. * NUMA node.
  1383. *
  1384. * This function will only make a best effort attempt at getting this onto
  1385. * the right NUMA node. If no node is requested or the requested node is
  1386. * offline then we just fall back to standard queue_work behavior.
  1387. *
  1388. * Currently the "random" CPU ends up being the first available CPU in the
  1389. * intersection of cpu_online_mask and the cpumask of the node, unless we
  1390. * are running on the node. In that case we just use the current CPU.
  1391. *
  1392. * Return: %false if @work was already on a queue, %true otherwise.
  1393. */
  1394. bool queue_work_node(int node, struct workqueue_struct *wq,
  1395. struct work_struct *work)
  1396. {
  1397. unsigned long flags;
  1398. bool ret = false;
  1399. /*
  1400. * This current implementation is specific to unbound workqueues.
  1401. * Specifically we only return the first available CPU for a given
  1402. * node instead of cycling through individual CPUs within the node.
  1403. *
  1404. * If this is used with a per-cpu workqueue then the logic in
  1405. * workqueue_select_cpu_near would need to be updated to allow for
  1406. * some round robin type logic.
  1407. */
  1408. WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
  1409. local_irq_save(flags);
  1410. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1411. int cpu = workqueue_select_cpu_near(node);
  1412. __queue_work(cpu, wq, work);
  1413. ret = true;
  1414. }
  1415. local_irq_restore(flags);
  1416. return ret;
  1417. }
  1418. EXPORT_SYMBOL_GPL(queue_work_node);
  1419. void delayed_work_timer_fn(struct timer_list *t)
  1420. {
  1421. struct delayed_work *dwork = from_timer(dwork, t, timer);
  1422. /* should have been called from irqsafe timer with irq already off */
  1423. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  1424. }
  1425. EXPORT_SYMBOL(delayed_work_timer_fn);
  1426. static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
  1427. struct delayed_work *dwork, unsigned long delay)
  1428. {
  1429. struct timer_list *timer = &dwork->timer;
  1430. struct work_struct *work = &dwork->work;
  1431. WARN_ON_ONCE(!wq);
  1432. /*
  1433. * With CFI, timer->function can point to a jump table entry in a module,
  1434. * which fails the comparison. Disable the warning if CFI and modules are
  1435. * both enabled.
  1436. */
  1437. if (!IS_ENABLED(CONFIG_CFI_CLANG) || !IS_ENABLED(CONFIG_MODULES))
  1438. WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
  1439. WARN_ON_ONCE(timer_pending(timer));
  1440. WARN_ON_ONCE(!list_empty(&work->entry));
  1441. /*
  1442. * If @delay is 0, queue @dwork->work immediately. This is for
  1443. * both optimization and correctness. The earliest @timer can
  1444. * expire is on the closest next tick and delayed_work users depend
  1445. * on that there's no such delay when @delay is 0.
  1446. */
  1447. if (!delay) {
  1448. __queue_work(cpu, wq, &dwork->work);
  1449. return;
  1450. }
  1451. dwork->wq = wq;
  1452. dwork->cpu = cpu;
  1453. timer->expires = jiffies + delay;
  1454. if (unlikely(cpu != WORK_CPU_UNBOUND))
  1455. add_timer_on(timer, cpu);
  1456. else
  1457. add_timer(timer);
  1458. }
  1459. /**
  1460. * queue_delayed_work_on - queue work on specific CPU after delay
  1461. * @cpu: CPU number to execute work on
  1462. * @wq: workqueue to use
  1463. * @dwork: work to queue
  1464. * @delay: number of jiffies to wait before queueing
  1465. *
  1466. * Return: %false if @work was already on a queue, %true otherwise. If
  1467. * @delay is zero and @dwork is idle, it will be scheduled for immediate
  1468. * execution.
  1469. */
  1470. bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
  1471. struct delayed_work *dwork, unsigned long delay)
  1472. {
  1473. struct work_struct *work = &dwork->work;
  1474. bool ret = false;
  1475. unsigned long flags;
  1476. /* read the comment in __queue_work() */
  1477. local_irq_save(flags);
  1478. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1479. __queue_delayed_work(cpu, wq, dwork, delay);
  1480. ret = true;
  1481. }
  1482. local_irq_restore(flags);
  1483. return ret;
  1484. }
  1485. EXPORT_SYMBOL(queue_delayed_work_on);
  1486. /**
  1487. * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
  1488. * @cpu: CPU number to execute work on
  1489. * @wq: workqueue to use
  1490. * @dwork: work to queue
  1491. * @delay: number of jiffies to wait before queueing
  1492. *
  1493. * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
  1494. * modify @dwork's timer so that it expires after @delay. If @delay is
  1495. * zero, @work is guaranteed to be scheduled immediately regardless of its
  1496. * current state.
  1497. *
  1498. * Return: %false if @dwork was idle and queued, %true if @dwork was
  1499. * pending and its timer was modified.
  1500. *
  1501. * This function is safe to call from any context including IRQ handler.
  1502. * See try_to_grab_pending() for details.
  1503. */
  1504. bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
  1505. struct delayed_work *dwork, unsigned long delay)
  1506. {
  1507. unsigned long flags;
  1508. int ret;
  1509. do {
  1510. ret = try_to_grab_pending(&dwork->work, true, &flags);
  1511. } while (unlikely(ret == -EAGAIN));
  1512. if (likely(ret >= 0)) {
  1513. __queue_delayed_work(cpu, wq, dwork, delay);
  1514. local_irq_restore(flags);
  1515. }
  1516. /* -ENOENT from try_to_grab_pending() becomes %true */
  1517. return ret;
  1518. }
  1519. EXPORT_SYMBOL_GPL(mod_delayed_work_on);
  1520. static void rcu_work_rcufn(struct rcu_head *rcu)
  1521. {
  1522. struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
  1523. /* read the comment in __queue_work() */
  1524. local_irq_disable();
  1525. __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
  1526. local_irq_enable();
  1527. }
  1528. /**
  1529. * queue_rcu_work - queue work after a RCU grace period
  1530. * @wq: workqueue to use
  1531. * @rwork: work to queue
  1532. *
  1533. * Return: %false if @rwork was already pending, %true otherwise. Note
  1534. * that a full RCU grace period is guaranteed only after a %true return.
  1535. * While @rwork is guaranteed to be executed after a %false return, the
  1536. * execution may happen before a full RCU grace period has passed.
  1537. */
  1538. bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
  1539. {
  1540. struct work_struct *work = &rwork->work;
  1541. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1542. rwork->wq = wq;
  1543. call_rcu(&rwork->rcu, rcu_work_rcufn);
  1544. return true;
  1545. }
  1546. return false;
  1547. }
  1548. EXPORT_SYMBOL(queue_rcu_work);
  1549. /**
  1550. * worker_enter_idle - enter idle state
  1551. * @worker: worker which is entering idle state
  1552. *
  1553. * @worker is entering idle state. Update stats and idle timer if
  1554. * necessary.
  1555. *
  1556. * LOCKING:
  1557. * raw_spin_lock_irq(pool->lock).
  1558. */
  1559. static void worker_enter_idle(struct worker *worker)
  1560. {
  1561. struct worker_pool *pool = worker->pool;
  1562. if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
  1563. WARN_ON_ONCE(!list_empty(&worker->entry) &&
  1564. (worker->hentry.next || worker->hentry.pprev)))
  1565. return;
  1566. /* can't use worker_set_flags(), also called from create_worker() */
  1567. worker->flags |= WORKER_IDLE;
  1568. pool->nr_idle++;
  1569. worker->last_active = jiffies;
  1570. /* idle_list is LIFO */
  1571. list_add(&worker->entry, &pool->idle_list);
  1572. if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
  1573. mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
  1574. /*
  1575. * Sanity check nr_running. Because unbind_workers() releases
  1576. * pool->lock between setting %WORKER_UNBOUND and zapping
  1577. * nr_running, the warning may trigger spuriously. Check iff
  1578. * unbind is not in progress.
  1579. */
  1580. WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
  1581. pool->nr_workers == pool->nr_idle &&
  1582. atomic_read(&pool->nr_running));
  1583. }
  1584. /**
  1585. * worker_leave_idle - leave idle state
  1586. * @worker: worker which is leaving idle state
  1587. *
  1588. * @worker is leaving idle state. Update stats.
  1589. *
  1590. * LOCKING:
  1591. * raw_spin_lock_irq(pool->lock).
  1592. */
  1593. static void worker_leave_idle(struct worker *worker)
  1594. {
  1595. struct worker_pool *pool = worker->pool;
  1596. if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
  1597. return;
  1598. worker_clr_flags(worker, WORKER_IDLE);
  1599. pool->nr_idle--;
  1600. list_del_init(&worker->entry);
  1601. }
  1602. static struct worker *alloc_worker(int node)
  1603. {
  1604. struct worker *worker;
  1605. worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
  1606. if (worker) {
  1607. INIT_LIST_HEAD(&worker->entry);
  1608. INIT_LIST_HEAD(&worker->scheduled);
  1609. INIT_LIST_HEAD(&worker->node);
  1610. /* on creation a worker is in !idle && prep state */
  1611. worker->flags = WORKER_PREP;
  1612. }
  1613. return worker;
  1614. }
  1615. /**
  1616. * worker_attach_to_pool() - attach a worker to a pool
  1617. * @worker: worker to be attached
  1618. * @pool: the target pool
  1619. *
  1620. * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
  1621. * cpu-binding of @worker are kept coordinated with the pool across
  1622. * cpu-[un]hotplugs.
  1623. */
  1624. static void worker_attach_to_pool(struct worker *worker,
  1625. struct worker_pool *pool)
  1626. {
  1627. mutex_lock(&wq_pool_attach_mutex);
  1628. /*
  1629. * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains
  1630. * stable across this function. See the comments above the flag
  1631. * definition for details.
  1632. */
  1633. if (pool->flags & POOL_DISASSOCIATED)
  1634. worker->flags |= WORKER_UNBOUND;
  1635. if (worker->rescue_wq)
  1636. set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
  1637. list_add_tail(&worker->node, &pool->workers);
  1638. worker->pool = pool;
  1639. mutex_unlock(&wq_pool_attach_mutex);
  1640. }
  1641. /**
  1642. * worker_detach_from_pool() - detach a worker from its pool
  1643. * @worker: worker which is attached to its pool
  1644. *
  1645. * Undo the attaching which had been done in worker_attach_to_pool(). The
  1646. * caller worker shouldn't access to the pool after detached except it has
  1647. * other reference to the pool.
  1648. */
  1649. static void worker_detach_from_pool(struct worker *worker)
  1650. {
  1651. struct worker_pool *pool = worker->pool;
  1652. struct completion *detach_completion = NULL;
  1653. mutex_lock(&wq_pool_attach_mutex);
  1654. list_del(&worker->node);
  1655. worker->pool = NULL;
  1656. if (list_empty(&pool->workers))
  1657. detach_completion = pool->detach_completion;
  1658. mutex_unlock(&wq_pool_attach_mutex);
  1659. /* clear leftover flags without pool->lock after it is detached */
  1660. worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
  1661. if (detach_completion)
  1662. complete(detach_completion);
  1663. }
  1664. /**
  1665. * create_worker - create a new workqueue worker
  1666. * @pool: pool the new worker will belong to
  1667. *
  1668. * Create and start a new worker which is attached to @pool.
  1669. *
  1670. * CONTEXT:
  1671. * Might sleep. Does GFP_KERNEL allocations.
  1672. *
  1673. * Return:
  1674. * Pointer to the newly created worker.
  1675. */
  1676. static struct worker *create_worker(struct worker_pool *pool)
  1677. {
  1678. struct worker *worker = NULL;
  1679. int id = -1;
  1680. char id_buf[16];
  1681. /* ID is needed to determine kthread name */
  1682. id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);
  1683. if (id < 0)
  1684. goto fail;
  1685. worker = alloc_worker(pool->node);
  1686. if (!worker)
  1687. goto fail;
  1688. worker->id = id;
  1689. if (pool->cpu >= 0)
  1690. snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
  1691. pool->attrs->nice < 0 ? "H" : "");
  1692. else
  1693. snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
  1694. worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
  1695. "kworker/%s", id_buf);
  1696. if (IS_ERR(worker->task))
  1697. goto fail;
  1698. set_user_nice(worker->task, pool->attrs->nice);
  1699. kthread_bind_mask(worker->task, pool->attrs->cpumask);
  1700. /* successful, attach the worker to the pool */
  1701. worker_attach_to_pool(worker, pool);
  1702. /* start the newly created worker */
  1703. raw_spin_lock_irq(&pool->lock);
  1704. worker->pool->nr_workers++;
  1705. worker_enter_idle(worker);
  1706. wake_up_process(worker->task);
  1707. raw_spin_unlock_irq(&pool->lock);
  1708. return worker;
  1709. fail:
  1710. if (id >= 0)
  1711. ida_simple_remove(&pool->worker_ida, id);
  1712. kfree(worker);
  1713. return NULL;
  1714. }
  1715. /**
  1716. * destroy_worker - destroy a workqueue worker
  1717. * @worker: worker to be destroyed
  1718. *
  1719. * Destroy @worker and adjust @pool stats accordingly. The worker should
  1720. * be idle.
  1721. *
  1722. * CONTEXT:
  1723. * raw_spin_lock_irq(pool->lock).
  1724. */
  1725. static void destroy_worker(struct worker *worker)
  1726. {
  1727. struct worker_pool *pool = worker->pool;
  1728. lockdep_assert_held(&pool->lock);
  1729. /* sanity check frenzy */
  1730. if (WARN_ON(worker->current_work) ||
  1731. WARN_ON(!list_empty(&worker->scheduled)) ||
  1732. WARN_ON(!(worker->flags & WORKER_IDLE)))
  1733. return;
  1734. pool->nr_workers--;
  1735. pool->nr_idle--;
  1736. list_del_init(&worker->entry);
  1737. worker->flags |= WORKER_DIE;
  1738. wake_up_process(worker->task);
  1739. }
  1740. static void idle_worker_timeout(struct timer_list *t)
  1741. {
  1742. struct worker_pool *pool = from_timer(pool, t, idle_timer);
  1743. raw_spin_lock_irq(&pool->lock);
  1744. while (too_many_workers(pool)) {
  1745. struct worker *worker;
  1746. unsigned long expires;
  1747. /* idle_list is kept in LIFO order, check the last one */
  1748. worker = list_entry(pool->idle_list.prev, struct worker, entry);
  1749. expires = worker->last_active + IDLE_WORKER_TIMEOUT;
  1750. if (time_before(jiffies, expires)) {
  1751. mod_timer(&pool->idle_timer, expires);
  1752. break;
  1753. }
  1754. destroy_worker(worker);
  1755. }
  1756. raw_spin_unlock_irq(&pool->lock);
  1757. }
  1758. static void send_mayday(struct work_struct *work)
  1759. {
  1760. struct pool_workqueue *pwq = get_work_pwq(work);
  1761. struct workqueue_struct *wq = pwq->wq;
  1762. lockdep_assert_held(&wq_mayday_lock);
  1763. if (!wq->rescuer)
  1764. return;
  1765. /* mayday mayday mayday */
  1766. if (list_empty(&pwq->mayday_node)) {
  1767. /*
  1768. * If @pwq is for an unbound wq, its base ref may be put at
  1769. * any time due to an attribute change. Pin @pwq until the
  1770. * rescuer is done with it.
  1771. */
  1772. get_pwq(pwq);
  1773. list_add_tail(&pwq->mayday_node, &wq->maydays);
  1774. wake_up_process(wq->rescuer->task);
  1775. }
  1776. }
  1777. static void pool_mayday_timeout(struct timer_list *t)
  1778. {
  1779. struct worker_pool *pool = from_timer(pool, t, mayday_timer);
  1780. struct work_struct *work;
  1781. raw_spin_lock_irq(&pool->lock);
  1782. raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
  1783. if (need_to_create_worker(pool)) {
  1784. /*
  1785. * We've been trying to create a new worker but
  1786. * haven't been successful. We might be hitting an
  1787. * allocation deadlock. Send distress signals to
  1788. * rescuers.
  1789. */
  1790. list_for_each_entry(work, &pool->worklist, entry)
  1791. send_mayday(work);
  1792. }
  1793. raw_spin_unlock(&wq_mayday_lock);
  1794. raw_spin_unlock_irq(&pool->lock);
  1795. mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
  1796. }
  1797. /**
  1798. * maybe_create_worker - create a new worker if necessary
  1799. * @pool: pool to create a new worker for
  1800. *
  1801. * Create a new worker for @pool if necessary. @pool is guaranteed to
  1802. * have at least one idle worker on return from this function. If
  1803. * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
  1804. * sent to all rescuers with works scheduled on @pool to resolve
  1805. * possible allocation deadlock.
  1806. *
  1807. * On return, need_to_create_worker() is guaranteed to be %false and
  1808. * may_start_working() %true.
  1809. *
  1810. * LOCKING:
  1811. * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
  1812. * multiple times. Does GFP_KERNEL allocations. Called only from
  1813. * manager.
  1814. */
  1815. static void maybe_create_worker(struct worker_pool *pool)
  1816. __releases(&pool->lock)
  1817. __acquires(&pool->lock)
  1818. {
  1819. restart:
  1820. raw_spin_unlock_irq(&pool->lock);
  1821. /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
  1822. mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
  1823. while (true) {
  1824. if (create_worker(pool) || !need_to_create_worker(pool))
  1825. break;
  1826. schedule_timeout_interruptible(CREATE_COOLDOWN);
  1827. if (!need_to_create_worker(pool))
  1828. break;
  1829. }
  1830. del_timer_sync(&pool->mayday_timer);
  1831. raw_spin_lock_irq(&pool->lock);
  1832. /*
  1833. * This is necessary even after a new worker was just successfully
  1834. * created as @pool->lock was dropped and the new worker might have
  1835. * already become busy.
  1836. */
  1837. if (need_to_create_worker(pool))
  1838. goto restart;
  1839. }
  1840. /**
  1841. * manage_workers - manage worker pool
  1842. * @worker: self
  1843. *
  1844. * Assume the manager role and manage the worker pool @worker belongs
  1845. * to. At any given time, there can be only zero or one manager per
  1846. * pool. The exclusion is handled automatically by this function.
  1847. *
  1848. * The caller can safely start processing works on false return. On
  1849. * true return, it's guaranteed that need_to_create_worker() is false
  1850. * and may_start_working() is true.
  1851. *
  1852. * CONTEXT:
  1853. * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
  1854. * multiple times. Does GFP_KERNEL allocations.
  1855. *
  1856. * Return:
  1857. * %false if the pool doesn't need management and the caller can safely
  1858. * start processing works, %true if management function was performed and
  1859. * the conditions that the caller verified before calling the function may
  1860. * no longer be true.
  1861. */
  1862. static bool manage_workers(struct worker *worker)
  1863. {
  1864. struct worker_pool *pool = worker->pool;
  1865. if (pool->flags & POOL_MANAGER_ACTIVE)
  1866. return false;
  1867. pool->flags |= POOL_MANAGER_ACTIVE;
  1868. pool->manager = worker;
  1869. maybe_create_worker(pool);
  1870. pool->manager = NULL;
  1871. pool->flags &= ~POOL_MANAGER_ACTIVE;
  1872. rcuwait_wake_up(&manager_wait);
  1873. return true;
  1874. }
  1875. /**
  1876. * process_one_work - process single work
  1877. * @worker: self
  1878. * @work: work to process
  1879. *
  1880. * Process @work. This function contains all the logics necessary to
  1881. * process a single work including synchronization against and
  1882. * interaction with other workers on the same cpu, queueing and
  1883. * flushing. As long as context requirement is met, any worker can
  1884. * call this function to process a work.
  1885. *
  1886. * CONTEXT:
  1887. * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
  1888. */
  1889. static void process_one_work(struct worker *worker, struct work_struct *work)
  1890. __releases(&pool->lock)
  1891. __acquires(&pool->lock)
  1892. {
  1893. struct pool_workqueue *pwq = get_work_pwq(work);
  1894. struct worker_pool *pool = worker->pool;
  1895. bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
  1896. int work_color;
  1897. struct worker *collision;
  1898. #ifdef CONFIG_LOCKDEP
  1899. /*
  1900. * It is permissible to free the struct work_struct from
  1901. * inside the function that is called from it, this we need to
  1902. * take into account for lockdep too. To avoid bogus "held
  1903. * lock freed" warnings as well as problems when looking into
  1904. * work->lockdep_map, make a copy and use that here.
  1905. */
  1906. struct lockdep_map lockdep_map;
  1907. lockdep_copy_map(&lockdep_map, &work->lockdep_map);
  1908. #endif
  1909. /* ensure we're on the correct CPU */
  1910. WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
  1911. raw_smp_processor_id() != pool->cpu);
  1912. /*
  1913. * A single work shouldn't be executed concurrently by
  1914. * multiple workers on a single cpu. Check whether anyone is
  1915. * already processing the work. If so, defer the work to the
  1916. * currently executing one.
  1917. */
  1918. collision = find_worker_executing_work(pool, work);
  1919. if (unlikely(collision)) {
  1920. move_linked_works(work, &collision->scheduled, NULL);
  1921. return;
  1922. }
  1923. /* claim and dequeue */
  1924. debug_work_deactivate(work);
  1925. hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
  1926. worker->current_work = work;
  1927. worker->current_func = work->func;
  1928. worker->current_pwq = pwq;
  1929. work_color = get_work_color(work);
  1930. /*
  1931. * Record wq name for cmdline and debug reporting, may get
  1932. * overridden through set_worker_desc().
  1933. */
  1934. strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
  1935. list_del_init(&work->entry);
  1936. /*
  1937. * CPU intensive works don't participate in concurrency management.
  1938. * They're the scheduler's responsibility. This takes @worker out
  1939. * of concurrency management and the next code block will chain
  1940. * execution of the pending work items.
  1941. */
  1942. if (unlikely(cpu_intensive))
  1943. worker_set_flags(worker, WORKER_CPU_INTENSIVE);
  1944. /*
  1945. * Wake up another worker if necessary. The condition is always
  1946. * false for normal per-cpu workers since nr_running would always
  1947. * be >= 1 at this point. This is used to chain execution of the
  1948. * pending work items for WORKER_NOT_RUNNING workers such as the
  1949. * UNBOUND and CPU_INTENSIVE ones.
  1950. */
  1951. if (need_more_worker(pool))
  1952. wake_up_worker(pool);
  1953. /*
  1954. * Record the last pool and clear PENDING which should be the last
  1955. * update to @work. Also, do this inside @pool->lock so that
  1956. * PENDING and queued state changes happen together while IRQ is
  1957. * disabled.
  1958. */
  1959. set_work_pool_and_clear_pending(work, pool->id);
  1960. raw_spin_unlock_irq(&pool->lock);
  1961. lock_map_acquire(&pwq->wq->lockdep_map);
  1962. lock_map_acquire(&lockdep_map);
  1963. /*
  1964. * Strictly speaking we should mark the invariant state without holding
  1965. * any locks, that is, before these two lock_map_acquire()'s.
  1966. *
  1967. * However, that would result in:
  1968. *
  1969. * A(W1)
  1970. * WFC(C)
  1971. * A(W1)
  1972. * C(C)
  1973. *
  1974. * Which would create W1->C->W1 dependencies, even though there is no
  1975. * actual deadlock possible. There are two solutions, using a
  1976. * read-recursive acquire on the work(queue) 'locks', but this will then
  1977. * hit the lockdep limitation on recursive locks, or simply discard
  1978. * these locks.
  1979. *
  1980. * AFAICT there is no possible deadlock scenario between the
  1981. * flush_work() and complete() primitives (except for single-threaded
  1982. * workqueues), so hiding them isn't a problem.
  1983. */
  1984. lockdep_invariant_state(true);
  1985. trace_workqueue_execute_start(work);
  1986. worker->current_func(work);
  1987. /*
  1988. * While we must be careful to not use "work" after this, the trace
  1989. * point will only record its address.
  1990. */
  1991. trace_workqueue_execute_end(work, worker->current_func);
  1992. lock_map_release(&lockdep_map);
  1993. lock_map_release(&pwq->wq->lockdep_map);
  1994. if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
  1995. pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
  1996. " last function: %ps\n",
  1997. current->comm, preempt_count(), task_pid_nr(current),
  1998. worker->current_func);
  1999. debug_show_held_locks(current);
  2000. dump_stack();
  2001. }
  2002. /*
  2003. * The following prevents a kworker from hogging CPU on !PREEMPTION
  2004. * kernels, where a requeueing work item waiting for something to
  2005. * happen could deadlock with stop_machine as such work item could
  2006. * indefinitely requeue itself while all other CPUs are trapped in
  2007. * stop_machine. At the same time, report a quiescent RCU state so
  2008. * the same condition doesn't freeze RCU.
  2009. */
  2010. cond_resched();
  2011. raw_spin_lock_irq(&pool->lock);
  2012. /* clear cpu intensive status */
  2013. if (unlikely(cpu_intensive))
  2014. worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
  2015. /* tag the worker for identification in schedule() */
  2016. worker->last_func = worker->current_func;
  2017. /* we're done with it, release */
  2018. hash_del(&worker->hentry);
  2019. worker->current_work = NULL;
  2020. worker->current_func = NULL;
  2021. worker->current_pwq = NULL;
  2022. pwq_dec_nr_in_flight(pwq, work_color);
  2023. }
  2024. /**
  2025. * process_scheduled_works - process scheduled works
  2026. * @worker: self
  2027. *
  2028. * Process all scheduled works. Please note that the scheduled list
  2029. * may change while processing a work, so this function repeatedly
  2030. * fetches a work from the top and executes it.
  2031. *
  2032. * CONTEXT:
  2033. * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
  2034. * multiple times.
  2035. */
  2036. static void process_scheduled_works(struct worker *worker)
  2037. {
  2038. while (!list_empty(&worker->scheduled)) {
  2039. struct work_struct *work = list_first_entry(&worker->scheduled,
  2040. struct work_struct, entry);
  2041. process_one_work(worker, work);
  2042. }
  2043. }
  2044. static void set_pf_worker(bool val)
  2045. {
  2046. mutex_lock(&wq_pool_attach_mutex);
  2047. if (val)
  2048. current->flags |= PF_WQ_WORKER;
  2049. else
  2050. current->flags &= ~PF_WQ_WORKER;
  2051. mutex_unlock(&wq_pool_attach_mutex);
  2052. }
  2053. /**
  2054. * worker_thread - the worker thread function
  2055. * @__worker: self
  2056. *
  2057. * The worker thread function. All workers belong to a worker_pool -
  2058. * either a per-cpu one or dynamic unbound one. These workers process all
  2059. * work items regardless of their specific target workqueue. The only
  2060. * exception is work items which belong to workqueues with a rescuer which
  2061. * will be explained in rescuer_thread().
  2062. *
  2063. * Return: 0
  2064. */
  2065. static int worker_thread(void *__worker)
  2066. {
  2067. struct worker *worker = __worker;
  2068. struct worker_pool *pool = worker->pool;
  2069. /* tell the scheduler that this is a workqueue worker */
  2070. set_pf_worker(true);
  2071. woke_up:
  2072. raw_spin_lock_irq(&pool->lock);
  2073. /* am I supposed to die? */
  2074. if (unlikely(worker->flags & WORKER_DIE)) {
  2075. raw_spin_unlock_irq(&pool->lock);
  2076. WARN_ON_ONCE(!list_empty(&worker->entry));
  2077. set_pf_worker(false);
  2078. set_task_comm(worker->task, "kworker/dying");
  2079. ida_simple_remove(&pool->worker_ida, worker->id);
  2080. worker_detach_from_pool(worker);
  2081. kfree(worker);
  2082. return 0;
  2083. }
  2084. worker_leave_idle(worker);
  2085. recheck:
  2086. /* no more worker necessary? */
  2087. if (!need_more_worker(pool))
  2088. goto sleep;
  2089. /* do we need to manage? */
  2090. if (unlikely(!may_start_working(pool)) && manage_workers(worker))
  2091. goto recheck;
  2092. /*
  2093. * ->scheduled list can only be filled while a worker is
  2094. * preparing to process a work or actually processing it.
  2095. * Make sure nobody diddled with it while I was sleeping.
  2096. */
  2097. WARN_ON_ONCE(!list_empty(&worker->scheduled));
  2098. /*
  2099. * Finish PREP stage. We're guaranteed to have at least one idle
  2100. * worker or that someone else has already assumed the manager
  2101. * role. This is where @worker starts participating in concurrency
  2102. * management if applicable and concurrency management is restored
  2103. * after being rebound. See rebind_workers() for details.
  2104. */
  2105. worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
  2106. do {
  2107. struct work_struct *work =
  2108. list_first_entry(&pool->worklist,
  2109. struct work_struct, entry);
  2110. pool->watchdog_ts = jiffies;
  2111. if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
  2112. /* optimization path, not strictly necessary */
  2113. process_one_work(worker, work);
  2114. if (unlikely(!list_empty(&worker->scheduled)))
  2115. process_scheduled_works(worker);
  2116. } else {
  2117. move_linked_works(work, &worker->scheduled, NULL);
  2118. process_scheduled_works(worker);
  2119. }
  2120. } while (keep_working(pool));
  2121. worker_set_flags(worker, WORKER_PREP);
  2122. sleep:
  2123. /*
  2124. * pool->lock is held and there's no work to process and no need to
  2125. * manage, sleep. Workers are woken up only while holding
  2126. * pool->lock or from local cpu, so setting the current state
  2127. * before releasing pool->lock is enough to prevent losing any
  2128. * event.
  2129. */
  2130. worker_enter_idle(worker);
  2131. __set_current_state(TASK_IDLE);
  2132. raw_spin_unlock_irq(&pool->lock);
  2133. schedule();
  2134. goto woke_up;
  2135. }
  2136. /**
  2137. * rescuer_thread - the rescuer thread function
  2138. * @__rescuer: self
  2139. *
  2140. * Workqueue rescuer thread function. There's one rescuer for each
  2141. * workqueue which has WQ_MEM_RECLAIM set.
  2142. *
  2143. * Regular work processing on a pool may block trying to create a new
  2144. * worker which uses GFP_KERNEL allocation which has slight chance of
  2145. * developing into deadlock if some works currently on the same queue
  2146. * need to be processed to satisfy the GFP_KERNEL allocation. This is
  2147. * the problem rescuer solves.
  2148. *
  2149. * When such condition is possible, the pool summons rescuers of all
  2150. * workqueues which have works queued on the pool and let them process
  2151. * those works so that forward progress can be guaranteed.
  2152. *
  2153. * This should happen rarely.
  2154. *
  2155. * Return: 0
  2156. */
  2157. static int rescuer_thread(void *__rescuer)
  2158. {
  2159. struct worker *rescuer = __rescuer;
  2160. struct workqueue_struct *wq = rescuer->rescue_wq;
  2161. struct list_head *scheduled = &rescuer->scheduled;
  2162. bool should_stop;
  2163. set_user_nice(current, RESCUER_NICE_LEVEL);
  2164. /*
  2165. * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
  2166. * doesn't participate in concurrency management.
  2167. */
  2168. set_pf_worker(true);
  2169. repeat:
  2170. set_current_state(TASK_IDLE);
  2171. /*
  2172. * By the time the rescuer is requested to stop, the workqueue
  2173. * shouldn't have any work pending, but @wq->maydays may still have
  2174. * pwq(s) queued. This can happen by non-rescuer workers consuming
  2175. * all the work items before the rescuer got to them. Go through
  2176. * @wq->maydays processing before acting on should_stop so that the
  2177. * list is always empty on exit.
  2178. */
  2179. should_stop = kthread_should_stop();
  2180. /* see whether any pwq is asking for help */
  2181. raw_spin_lock_irq(&wq_mayday_lock);
  2182. while (!list_empty(&wq->maydays)) {
  2183. struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
  2184. struct pool_workqueue, mayday_node);
  2185. struct worker_pool *pool = pwq->pool;
  2186. struct work_struct *work, *n;
  2187. bool first = true;
  2188. __set_current_state(TASK_RUNNING);
  2189. list_del_init(&pwq->mayday_node);
  2190. raw_spin_unlock_irq(&wq_mayday_lock);
  2191. worker_attach_to_pool(rescuer, pool);
  2192. raw_spin_lock_irq(&pool->lock);
  2193. /*
  2194. * Slurp in all works issued via this workqueue and
  2195. * process'em.
  2196. */
  2197. WARN_ON_ONCE(!list_empty(scheduled));
  2198. list_for_each_entry_safe(work, n, &pool->worklist, entry) {
  2199. if (get_work_pwq(work) == pwq) {
  2200. if (first)
  2201. pool->watchdog_ts = jiffies;
  2202. move_linked_works(work, scheduled, &n);
  2203. }
  2204. first = false;
  2205. }
  2206. if (!list_empty(scheduled)) {
  2207. process_scheduled_works(rescuer);
  2208. /*
  2209. * The above execution of rescued work items could
  2210. * have created more to rescue through
  2211. * pwq_activate_first_delayed() or chained
  2212. * queueing. Let's put @pwq back on mayday list so
  2213. * that such back-to-back work items, which may be
  2214. * being used to relieve memory pressure, don't
  2215. * incur MAYDAY_INTERVAL delay inbetween.
  2216. */
  2217. if (pwq->nr_active && need_to_create_worker(pool)) {
  2218. raw_spin_lock(&wq_mayday_lock);
  2219. /*
  2220. * Queue iff we aren't racing destruction
  2221. * and somebody else hasn't queued it already.
  2222. */
  2223. if (wq->rescuer && list_empty(&pwq->mayday_node)) {
  2224. get_pwq(pwq);
  2225. list_add_tail(&pwq->mayday_node, &wq->maydays);
  2226. }
  2227. raw_spin_unlock(&wq_mayday_lock);
  2228. }
  2229. }
  2230. /*
  2231. * Put the reference grabbed by send_mayday(). @pool won't
  2232. * go away while we're still attached to it.
  2233. */
  2234. put_pwq(pwq);
  2235. /*
  2236. * Leave this pool. If need_more_worker() is %true, notify a
  2237. * regular worker; otherwise, we end up with 0 concurrency
  2238. * and stalling the execution.
  2239. */
  2240. if (need_more_worker(pool))
  2241. wake_up_worker(pool);
  2242. raw_spin_unlock_irq(&pool->lock);
  2243. worker_detach_from_pool(rescuer);
  2244. raw_spin_lock_irq(&wq_mayday_lock);
  2245. }
  2246. raw_spin_unlock_irq(&wq_mayday_lock);
  2247. if (should_stop) {
  2248. __set_current_state(TASK_RUNNING);
  2249. set_pf_worker(false);
  2250. return 0;
  2251. }
  2252. /* rescuers should never participate in concurrency management */
  2253. WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
  2254. schedule();
  2255. goto repeat;
  2256. }
  2257. /**
  2258. * check_flush_dependency - check for flush dependency sanity
  2259. * @target_wq: workqueue being flushed
  2260. * @target_work: work item being flushed (NULL for workqueue flushes)
  2261. *
  2262. * %current is trying to flush the whole @target_wq or @target_work on it.
  2263. * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
  2264. * reclaiming memory or running on a workqueue which doesn't have
  2265. * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
  2266. * a deadlock.
  2267. */
  2268. static void check_flush_dependency(struct workqueue_struct *target_wq,
  2269. struct work_struct *target_work)
  2270. {
  2271. work_func_t target_func = target_work ? target_work->func : NULL;
  2272. struct worker *worker;
  2273. if (target_wq->flags & WQ_MEM_RECLAIM)
  2274. return;
  2275. worker = current_wq_worker();
  2276. WARN_ONCE(current->flags & PF_MEMALLOC,
  2277. "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
  2278. current->pid, current->comm, target_wq->name, target_func);
  2279. WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
  2280. (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
  2281. "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
  2282. worker->current_pwq->wq->name, worker->current_func,
  2283. target_wq->name, target_func);
  2284. }
  2285. struct wq_barrier {
  2286. struct work_struct work;
  2287. struct completion done;
  2288. struct task_struct *task; /* purely informational */
  2289. };
  2290. static void wq_barrier_func(struct work_struct *work)
  2291. {
  2292. struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
  2293. complete(&barr->done);
  2294. }
  2295. /**
  2296. * insert_wq_barrier - insert a barrier work
  2297. * @pwq: pwq to insert barrier into
  2298. * @barr: wq_barrier to insert
  2299. * @target: target work to attach @barr to
  2300. * @worker: worker currently executing @target, NULL if @target is not executing
  2301. *
  2302. * @barr is linked to @target such that @barr is completed only after
  2303. * @target finishes execution. Please note that the ordering
  2304. * guarantee is observed only with respect to @target and on the local
  2305. * cpu.
  2306. *
  2307. * Currently, a queued barrier can't be canceled. This is because
  2308. * try_to_grab_pending() can't determine whether the work to be
  2309. * grabbed is at the head of the queue and thus can't clear LINKED
  2310. * flag of the previous work while there must be a valid next work
  2311. * after a work with LINKED flag set.
  2312. *
  2313. * Note that when @worker is non-NULL, @target may be modified
  2314. * underneath us, so we can't reliably determine pwq from @target.
  2315. *
  2316. * CONTEXT:
  2317. * raw_spin_lock_irq(pool->lock).
  2318. */
  2319. static void insert_wq_barrier(struct pool_workqueue *pwq,
  2320. struct wq_barrier *barr,
  2321. struct work_struct *target, struct worker *worker)
  2322. {
  2323. struct list_head *head;
  2324. unsigned int linked = 0;
  2325. /*
  2326. * debugobject calls are safe here even with pool->lock locked
  2327. * as we know for sure that this will not trigger any of the
  2328. * checks and call back into the fixup functions where we
  2329. * might deadlock.
  2330. */
  2331. INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
  2332. __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
  2333. init_completion_map(&barr->done, &target->lockdep_map);
  2334. barr->task = current;
  2335. /*
  2336. * If @target is currently being executed, schedule the
  2337. * barrier to the worker; otherwise, put it after @target.
  2338. */
  2339. if (worker)
  2340. head = worker->scheduled.next;
  2341. else {
  2342. unsigned long *bits = work_data_bits(target);
  2343. head = target->entry.next;
  2344. /* there can already be other linked works, inherit and set */
  2345. linked = *bits & WORK_STRUCT_LINKED;
  2346. __set_bit(WORK_STRUCT_LINKED_BIT, bits);
  2347. }
  2348. debug_work_activate(&barr->work);
  2349. insert_work(pwq, &barr->work, head,
  2350. work_color_to_flags(WORK_NO_COLOR) | linked);
  2351. }
  2352. /**
  2353. * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
  2354. * @wq: workqueue being flushed
  2355. * @flush_color: new flush color, < 0 for no-op
  2356. * @work_color: new work color, < 0 for no-op
  2357. *
  2358. * Prepare pwqs for workqueue flushing.
  2359. *
  2360. * If @flush_color is non-negative, flush_color on all pwqs should be
  2361. * -1. If no pwq has in-flight commands at the specified color, all
  2362. * pwq->flush_color's stay at -1 and %false is returned. If any pwq
  2363. * has in flight commands, its pwq->flush_color is set to
  2364. * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
  2365. * wakeup logic is armed and %true is returned.
  2366. *
  2367. * The caller should have initialized @wq->first_flusher prior to
  2368. * calling this function with non-negative @flush_color. If
  2369. * @flush_color is negative, no flush color update is done and %false
  2370. * is returned.
  2371. *
  2372. * If @work_color is non-negative, all pwqs should have the same
  2373. * work_color which is previous to @work_color and all will be
  2374. * advanced to @work_color.
  2375. *
  2376. * CONTEXT:
  2377. * mutex_lock(wq->mutex).
  2378. *
  2379. * Return:
  2380. * %true if @flush_color >= 0 and there's something to flush. %false
  2381. * otherwise.
  2382. */
  2383. static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
  2384. int flush_color, int work_color)
  2385. {
  2386. bool wait = false;
  2387. struct pool_workqueue *pwq;
  2388. if (flush_color >= 0) {
  2389. WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
  2390. atomic_set(&wq->nr_pwqs_to_flush, 1);
  2391. }
  2392. for_each_pwq(pwq, wq) {
  2393. struct worker_pool *pool = pwq->pool;
  2394. raw_spin_lock_irq(&pool->lock);
  2395. if (flush_color >= 0) {
  2396. WARN_ON_ONCE(pwq->flush_color != -1);
  2397. if (pwq->nr_in_flight[flush_color]) {
  2398. pwq->flush_color = flush_color;
  2399. atomic_inc(&wq->nr_pwqs_to_flush);
  2400. wait = true;
  2401. }
  2402. }
  2403. if (work_color >= 0) {
  2404. WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
  2405. pwq->work_color = work_color;
  2406. }
  2407. raw_spin_unlock_irq(&pool->lock);
  2408. }
  2409. if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
  2410. complete(&wq->first_flusher->done);
  2411. return wait;
  2412. }
  2413. /**
  2414. * flush_workqueue - ensure that any scheduled work has run to completion.
  2415. * @wq: workqueue to flush
  2416. *
  2417. * This function sleeps until all work items which were queued on entry
  2418. * have finished execution, but it is not livelocked by new incoming ones.
  2419. */
  2420. void flush_workqueue(struct workqueue_struct *wq)
  2421. {
  2422. struct wq_flusher this_flusher = {
  2423. .list = LIST_HEAD_INIT(this_flusher.list),
  2424. .flush_color = -1,
  2425. .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
  2426. };
  2427. int next_color;
  2428. if (WARN_ON(!wq_online))
  2429. return;
  2430. lock_map_acquire(&wq->lockdep_map);
  2431. lock_map_release(&wq->lockdep_map);
  2432. mutex_lock(&wq->mutex);
  2433. /*
  2434. * Start-to-wait phase
  2435. */
  2436. next_color = work_next_color(wq->work_color);
  2437. if (next_color != wq->flush_color) {
  2438. /*
  2439. * Color space is not full. The current work_color
  2440. * becomes our flush_color and work_color is advanced
  2441. * by one.
  2442. */
  2443. WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
  2444. this_flusher.flush_color = wq->work_color;
  2445. wq->work_color = next_color;
  2446. if (!wq->first_flusher) {
  2447. /* no flush in progress, become the first flusher */
  2448. WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
  2449. wq->first_flusher = &this_flusher;
  2450. if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
  2451. wq->work_color)) {
  2452. /* nothing to flush, done */
  2453. wq->flush_color = next_color;
  2454. wq->first_flusher = NULL;
  2455. goto out_unlock;
  2456. }
  2457. } else {
  2458. /* wait in queue */
  2459. WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
  2460. list_add_tail(&this_flusher.list, &wq->flusher_queue);
  2461. flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
  2462. }
  2463. } else {
  2464. /*
  2465. * Oops, color space is full, wait on overflow queue.
  2466. * The next flush completion will assign us
  2467. * flush_color and transfer to flusher_queue.
  2468. */
  2469. list_add_tail(&this_flusher.list, &wq->flusher_overflow);
  2470. }
  2471. check_flush_dependency(wq, NULL);
  2472. mutex_unlock(&wq->mutex);
  2473. wait_for_completion(&this_flusher.done);
  2474. /*
  2475. * Wake-up-and-cascade phase
  2476. *
  2477. * First flushers are responsible for cascading flushes and
  2478. * handling overflow. Non-first flushers can simply return.
  2479. */
  2480. if (READ_ONCE(wq->first_flusher) != &this_flusher)
  2481. return;
  2482. mutex_lock(&wq->mutex);
  2483. /* we might have raced, check again with mutex held */
  2484. if (wq->first_flusher != &this_flusher)
  2485. goto out_unlock;
  2486. WRITE_ONCE(wq->first_flusher, NULL);
  2487. WARN_ON_ONCE(!list_empty(&this_flusher.list));
  2488. WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
  2489. while (true) {
  2490. struct wq_flusher *next, *tmp;
  2491. /* complete all the flushers sharing the current flush color */
  2492. list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
  2493. if (next->flush_color != wq->flush_color)
  2494. break;
  2495. list_del_init(&next->list);
  2496. complete(&next->done);
  2497. }
  2498. WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
  2499. wq->flush_color != work_next_color(wq->work_color));
  2500. /* this flush_color is finished, advance by one */
  2501. wq->flush_color = work_next_color(wq->flush_color);
  2502. /* one color has been freed, handle overflow queue */
  2503. if (!list_empty(&wq->flusher_overflow)) {
  2504. /*
  2505. * Assign the same color to all overflowed
  2506. * flushers, advance work_color and append to
  2507. * flusher_queue. This is the start-to-wait
  2508. * phase for these overflowed flushers.
  2509. */
  2510. list_for_each_entry(tmp, &wq->flusher_overflow, list)
  2511. tmp->flush_color = wq->work_color;
  2512. wq->work_color = work_next_color(wq->work_color);
  2513. list_splice_tail_init(&wq->flusher_overflow,
  2514. &wq->flusher_queue);
  2515. flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
  2516. }
  2517. if (list_empty(&wq->flusher_queue)) {
  2518. WARN_ON_ONCE(wq->flush_color != wq->work_color);
  2519. break;
  2520. }
  2521. /*
  2522. * Need to flush more colors. Make the next flusher
  2523. * the new first flusher and arm pwqs.
  2524. */
  2525. WARN_ON_ONCE(wq->flush_color == wq->work_color);
  2526. WARN_ON_ONCE(wq->flush_color != next->flush_color);
  2527. list_del_init(&next->list);
  2528. wq->first_flusher = next;
  2529. if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
  2530. break;
  2531. /*
  2532. * Meh... this color is already done, clear first
  2533. * flusher and repeat cascading.
  2534. */
  2535. wq->first_flusher = NULL;
  2536. }
  2537. out_unlock:
  2538. mutex_unlock(&wq->mutex);
  2539. }
  2540. EXPORT_SYMBOL(flush_workqueue);
  2541. /**
  2542. * drain_workqueue - drain a workqueue
  2543. * @wq: workqueue to drain
  2544. *
  2545. * Wait until the workqueue becomes empty. While draining is in progress,
  2546. * only chain queueing is allowed. IOW, only currently pending or running
  2547. * work items on @wq can queue further work items on it. @wq is flushed
  2548. * repeatedly until it becomes empty. The number of flushing is determined
  2549. * by the depth of chaining and should be relatively short. Whine if it
  2550. * takes too long.
  2551. */
  2552. void drain_workqueue(struct workqueue_struct *wq)
  2553. {
  2554. unsigned int flush_cnt = 0;
  2555. struct pool_workqueue *pwq;
  2556. /*
  2557. * __queue_work() needs to test whether there are drainers, is much
  2558. * hotter than drain_workqueue() and already looks at @wq->flags.
  2559. * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
  2560. */
  2561. mutex_lock(&wq->mutex);
  2562. if (!wq->nr_drainers++)
  2563. wq->flags |= __WQ_DRAINING;
  2564. mutex_unlock(&wq->mutex);
  2565. reflush:
  2566. flush_workqueue(wq);
  2567. mutex_lock(&wq->mutex);
  2568. for_each_pwq(pwq, wq) {
  2569. bool drained;
  2570. raw_spin_lock_irq(&pwq->pool->lock);
  2571. drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
  2572. raw_spin_unlock_irq(&pwq->pool->lock);
  2573. if (drained)
  2574. continue;
  2575. if (++flush_cnt == 10 ||
  2576. (flush_cnt % 100 == 0 && flush_cnt <= 1000))
  2577. pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
  2578. wq->name, flush_cnt);
  2579. mutex_unlock(&wq->mutex);
  2580. goto reflush;
  2581. }
  2582. if (!--wq->nr_drainers)
  2583. wq->flags &= ~__WQ_DRAINING;
  2584. mutex_unlock(&wq->mutex);
  2585. }
  2586. EXPORT_SYMBOL_GPL(drain_workqueue);
  2587. static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
  2588. bool from_cancel)
  2589. {
  2590. struct worker *worker = NULL;
  2591. struct worker_pool *pool;
  2592. struct pool_workqueue *pwq;
  2593. might_sleep();
  2594. rcu_read_lock();
  2595. pool = get_work_pool(work);
  2596. if (!pool) {
  2597. rcu_read_unlock();
  2598. return false;
  2599. }
  2600. raw_spin_lock_irq(&pool->lock);
  2601. /* see the comment in try_to_grab_pending() with the same code */
  2602. pwq = get_work_pwq(work);
  2603. if (pwq) {
  2604. if (unlikely(pwq->pool != pool))
  2605. goto already_gone;
  2606. } else {
  2607. worker = find_worker_executing_work(pool, work);
  2608. if (!worker)
  2609. goto already_gone;
  2610. pwq = worker->current_pwq;
  2611. }
  2612. check_flush_dependency(pwq->wq, work);
  2613. insert_wq_barrier(pwq, barr, work, worker);
  2614. raw_spin_unlock_irq(&pool->lock);
  2615. /*
  2616. * Force a lock recursion deadlock when using flush_work() inside a
  2617. * single-threaded or rescuer equipped workqueue.
  2618. *
  2619. * For single threaded workqueues the deadlock happens when the work
  2620. * is after the work issuing the flush_work(). For rescuer equipped
  2621. * workqueues the deadlock happens when the rescuer stalls, blocking
  2622. * forward progress.
  2623. */
  2624. if (!from_cancel &&
  2625. (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {
  2626. lock_map_acquire(&pwq->wq->lockdep_map);
  2627. lock_map_release(&pwq->wq->lockdep_map);
  2628. }
  2629. rcu_read_unlock();
  2630. return true;
  2631. already_gone:
  2632. raw_spin_unlock_irq(&pool->lock);
  2633. rcu_read_unlock();
  2634. return false;
  2635. }
  2636. static bool __flush_work(struct work_struct *work, bool from_cancel)
  2637. {
  2638. struct wq_barrier barr;
  2639. if (WARN_ON(!wq_online))
  2640. return false;
  2641. if (WARN_ON(!work->func))
  2642. return false;
  2643. if (!from_cancel) {
  2644. lock_map_acquire(&work->lockdep_map);
  2645. lock_map_release(&work->lockdep_map);
  2646. }
  2647. if (start_flush_work(work, &barr, from_cancel)) {
  2648. wait_for_completion(&barr.done);
  2649. destroy_work_on_stack(&barr.work);
  2650. return true;
  2651. } else {
  2652. return false;
  2653. }
  2654. }
  2655. /**
  2656. * flush_work - wait for a work to finish executing the last queueing instance
  2657. * @work: the work to flush
  2658. *
  2659. * Wait until @work has finished execution. @work is guaranteed to be idle
  2660. * on return if it hasn't been requeued since flush started.
  2661. *
  2662. * Return:
  2663. * %true if flush_work() waited for the work to finish execution,
  2664. * %false if it was already idle.
  2665. */
  2666. bool flush_work(struct work_struct *work)
  2667. {
  2668. return __flush_work(work, false);
  2669. }
  2670. EXPORT_SYMBOL_GPL(flush_work);
  2671. struct cwt_wait {
  2672. wait_queue_entry_t wait;
  2673. struct work_struct *work;
  2674. };
  2675. static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  2676. {
  2677. struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
  2678. if (cwait->work != key)
  2679. return 0;
  2680. return autoremove_wake_function(wait, mode, sync, key);
  2681. }
  2682. static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
  2683. {
  2684. static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
  2685. unsigned long flags;
  2686. int ret;
  2687. do {
  2688. ret = try_to_grab_pending(work, is_dwork, &flags);
  2689. /*
  2690. * If someone else is already canceling, wait for it to
  2691. * finish. flush_work() doesn't work for PREEMPT_NONE
  2692. * because we may get scheduled between @work's completion
  2693. * and the other canceling task resuming and clearing
  2694. * CANCELING - flush_work() will return false immediately
  2695. * as @work is no longer busy, try_to_grab_pending() will
  2696. * return -ENOENT as @work is still being canceled and the
  2697. * other canceling task won't be able to clear CANCELING as
  2698. * we're hogging the CPU.
  2699. *
  2700. * Let's wait for completion using a waitqueue. As this
  2701. * may lead to the thundering herd problem, use a custom
  2702. * wake function which matches @work along with exclusive
  2703. * wait and wakeup.
  2704. */
  2705. if (unlikely(ret == -ENOENT)) {
  2706. struct cwt_wait cwait;
  2707. init_wait(&cwait.wait);
  2708. cwait.wait.func = cwt_wakefn;
  2709. cwait.work = work;
  2710. prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
  2711. TASK_UNINTERRUPTIBLE);
  2712. if (work_is_canceling(work))
  2713. schedule();
  2714. finish_wait(&cancel_waitq, &cwait.wait);
  2715. }
  2716. } while (unlikely(ret < 0));
  2717. /* tell other tasks trying to grab @work to back off */
  2718. mark_work_canceling(work);
  2719. local_irq_restore(flags);
  2720. /*
  2721. * This allows canceling during early boot. We know that @work
  2722. * isn't executing.
  2723. */
  2724. if (wq_online)
  2725. __flush_work(work, true);
  2726. clear_work_data(work);
  2727. /*
  2728. * Paired with prepare_to_wait() above so that either
  2729. * waitqueue_active() is visible here or !work_is_canceling() is
  2730. * visible there.
  2731. */
  2732. smp_mb();
  2733. if (waitqueue_active(&cancel_waitq))
  2734. __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
  2735. return ret;
  2736. }
  2737. /**
  2738. * cancel_work_sync - cancel a work and wait for it to finish
  2739. * @work: the work to cancel
  2740. *
  2741. * Cancel @work and wait for its execution to finish. This function
  2742. * can be used even if the work re-queues itself or migrates to
  2743. * another workqueue. On return from this function, @work is
  2744. * guaranteed to be not pending or executing on any CPU.
  2745. *
  2746. * cancel_work_sync(&delayed_work->work) must not be used for
  2747. * delayed_work's. Use cancel_delayed_work_sync() instead.
  2748. *
  2749. * The caller must ensure that the workqueue on which @work was last
  2750. * queued can't be destroyed before this function returns.
  2751. *
  2752. * Return:
  2753. * %true if @work was pending, %false otherwise.
  2754. */
  2755. bool cancel_work_sync(struct work_struct *work)
  2756. {
  2757. return __cancel_work_timer(work, false);
  2758. }
  2759. EXPORT_SYMBOL_GPL(cancel_work_sync);
  2760. /**
  2761. * flush_delayed_work - wait for a dwork to finish executing the last queueing
  2762. * @dwork: the delayed work to flush
  2763. *
  2764. * Delayed timer is cancelled and the pending work is queued for
  2765. * immediate execution. Like flush_work(), this function only
  2766. * considers the last queueing instance of @dwork.
  2767. *
  2768. * Return:
  2769. * %true if flush_work() waited for the work to finish execution,
  2770. * %false if it was already idle.
  2771. */
  2772. bool flush_delayed_work(struct delayed_work *dwork)
  2773. {
  2774. local_irq_disable();
  2775. if (del_timer_sync(&dwork->timer))
  2776. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  2777. local_irq_enable();
  2778. return flush_work(&dwork->work);
  2779. }
  2780. EXPORT_SYMBOL(flush_delayed_work);
  2781. /**
  2782. * flush_rcu_work - wait for a rwork to finish executing the last queueing
  2783. * @rwork: the rcu work to flush
  2784. *
  2785. * Return:
  2786. * %true if flush_rcu_work() waited for the work to finish execution,
  2787. * %false if it was already idle.
  2788. */
  2789. bool flush_rcu_work(struct rcu_work *rwork)
  2790. {
  2791. if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
  2792. rcu_barrier();
  2793. flush_work(&rwork->work);
  2794. return true;
  2795. } else {
  2796. return flush_work(&rwork->work);
  2797. }
  2798. }
  2799. EXPORT_SYMBOL(flush_rcu_work);
  2800. static bool __cancel_work(struct work_struct *work, bool is_dwork)
  2801. {
  2802. unsigned long flags;
  2803. int ret;
  2804. do {
  2805. ret = try_to_grab_pending(work, is_dwork, &flags);
  2806. } while (unlikely(ret == -EAGAIN));
  2807. if (unlikely(ret < 0))
  2808. return false;
  2809. set_work_pool_and_clear_pending(work, get_work_pool_id(work));
  2810. local_irq_restore(flags);
  2811. return ret;
  2812. }
  2813. /**
  2814. * cancel_delayed_work - cancel a delayed work
  2815. * @dwork: delayed_work to cancel
  2816. *
  2817. * Kill off a pending delayed_work.
  2818. *
  2819. * Return: %true if @dwork was pending and canceled; %false if it wasn't
  2820. * pending.
  2821. *
  2822. * Note:
  2823. * The work callback function may still be running on return, unless
  2824. * it returns %true and the work doesn't re-arm itself. Explicitly flush or
  2825. * use cancel_delayed_work_sync() to wait on it.
  2826. *
  2827. * This function is safe to call from any context including IRQ handler.
  2828. */
  2829. bool cancel_delayed_work(struct delayed_work *dwork)
  2830. {
  2831. return __cancel_work(&dwork->work, true);
  2832. }
  2833. EXPORT_SYMBOL(cancel_delayed_work);
  2834. /**
  2835. * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
  2836. * @dwork: the delayed work cancel
  2837. *
  2838. * This is cancel_work_sync() for delayed works.
  2839. *
  2840. * Return:
  2841. * %true if @dwork was pending, %false otherwise.
  2842. */
  2843. bool cancel_delayed_work_sync(struct delayed_work *dwork)
  2844. {
  2845. return __cancel_work_timer(&dwork->work, true);
  2846. }
  2847. EXPORT_SYMBOL(cancel_delayed_work_sync);
  2848. /**
  2849. * schedule_on_each_cpu - execute a function synchronously on each online CPU
  2850. * @func: the function to call
  2851. *
  2852. * schedule_on_each_cpu() executes @func on each online CPU using the
  2853. * system workqueue and blocks until all CPUs have completed.
  2854. * schedule_on_each_cpu() is very slow.
  2855. *
  2856. * Return:
  2857. * 0 on success, -errno on failure.
  2858. */
  2859. int schedule_on_each_cpu(work_func_t func)
  2860. {
  2861. int cpu;
  2862. struct work_struct __percpu *works;
  2863. works = alloc_percpu(struct work_struct);
  2864. if (!works)
  2865. return -ENOMEM;
  2866. get_online_cpus();
  2867. for_each_online_cpu(cpu) {
  2868. struct work_struct *work = per_cpu_ptr(works, cpu);
  2869. INIT_WORK(work, func);
  2870. schedule_work_on(cpu, work);
  2871. }
  2872. for_each_online_cpu(cpu)
  2873. flush_work(per_cpu_ptr(works, cpu));
  2874. put_online_cpus();
  2875. free_percpu(works);
  2876. return 0;
  2877. }
  2878. /**
  2879. * execute_in_process_context - reliably execute the routine with user context
  2880. * @fn: the function to execute
  2881. * @ew: guaranteed storage for the execute work structure (must
  2882. * be available when the work executes)
  2883. *
  2884. * Executes the function immediately if process context is available,
  2885. * otherwise schedules the function for delayed execution.
  2886. *
  2887. * Return: 0 - function was executed
  2888. * 1 - function was scheduled for execution
  2889. */
  2890. int execute_in_process_context(work_func_t fn, struct execute_work *ew)
  2891. {
  2892. if (!in_interrupt()) {
  2893. fn(&ew->work);
  2894. return 0;
  2895. }
  2896. INIT_WORK(&ew->work, fn);
  2897. schedule_work(&ew->work);
  2898. return 1;
  2899. }
  2900. EXPORT_SYMBOL_GPL(execute_in_process_context);
  2901. /**
  2902. * free_workqueue_attrs - free a workqueue_attrs
  2903. * @attrs: workqueue_attrs to free
  2904. *
  2905. * Undo alloc_workqueue_attrs().
  2906. */
  2907. void free_workqueue_attrs(struct workqueue_attrs *attrs)
  2908. {
  2909. if (attrs) {
  2910. free_cpumask_var(attrs->cpumask);
  2911. kfree(attrs);
  2912. }
  2913. }
  2914. /**
  2915. * alloc_workqueue_attrs - allocate a workqueue_attrs
  2916. *
  2917. * Allocate a new workqueue_attrs, initialize with default settings and
  2918. * return it.
  2919. *
  2920. * Return: The allocated new workqueue_attr on success. %NULL on failure.
  2921. */
  2922. struct workqueue_attrs *alloc_workqueue_attrs(void)
  2923. {
  2924. struct workqueue_attrs *attrs;
  2925. attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
  2926. if (!attrs)
  2927. goto fail;
  2928. if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
  2929. goto fail;
  2930. cpumask_copy(attrs->cpumask, cpu_possible_mask);
  2931. return attrs;
  2932. fail:
  2933. free_workqueue_attrs(attrs);
  2934. return NULL;
  2935. }
  2936. static void copy_workqueue_attrs(struct workqueue_attrs *to,
  2937. const struct workqueue_attrs *from)
  2938. {
  2939. to->nice = from->nice;
  2940. cpumask_copy(to->cpumask, from->cpumask);
  2941. /*
  2942. * Unlike hash and equality test, this function doesn't ignore
  2943. * ->no_numa as it is used for both pool and wq attrs. Instead,
  2944. * get_unbound_pool() explicitly clears ->no_numa after copying.
  2945. */
  2946. to->no_numa = from->no_numa;
  2947. }
  2948. /* hash value of the content of @attr */
  2949. static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
  2950. {
  2951. u32 hash = 0;
  2952. hash = jhash_1word(attrs->nice, hash);
  2953. hash = jhash(cpumask_bits(attrs->cpumask),
  2954. BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
  2955. return hash;
  2956. }
  2957. /* content equality test */
  2958. static bool wqattrs_equal(const struct workqueue_attrs *a,
  2959. const struct workqueue_attrs *b)
  2960. {
  2961. if (a->nice != b->nice)
  2962. return false;
  2963. if (!cpumask_equal(a->cpumask, b->cpumask))
  2964. return false;
  2965. return true;
  2966. }
  2967. /**
  2968. * init_worker_pool - initialize a newly zalloc'd worker_pool
  2969. * @pool: worker_pool to initialize
  2970. *
  2971. * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
  2972. *
  2973. * Return: 0 on success, -errno on failure. Even on failure, all fields
  2974. * inside @pool proper are initialized and put_unbound_pool() can be called
  2975. * on @pool safely to release it.
  2976. */
  2977. static int init_worker_pool(struct worker_pool *pool)
  2978. {
  2979. raw_spin_lock_init(&pool->lock);
  2980. pool->id = -1;
  2981. pool->cpu = -1;
  2982. pool->node = NUMA_NO_NODE;
  2983. pool->flags |= POOL_DISASSOCIATED;
  2984. pool->watchdog_ts = jiffies;
  2985. INIT_LIST_HEAD(&pool->worklist);
  2986. INIT_LIST_HEAD(&pool->idle_list);
  2987. hash_init(pool->busy_hash);
  2988. timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
  2989. timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
  2990. INIT_LIST_HEAD(&pool->workers);
  2991. ida_init(&pool->worker_ida);
  2992. INIT_HLIST_NODE(&pool->hash_node);
  2993. pool->refcnt = 1;
  2994. /* shouldn't fail above this point */
  2995. pool->attrs = alloc_workqueue_attrs();
  2996. if (!pool->attrs)
  2997. return -ENOMEM;
  2998. return 0;
  2999. }
  3000. #ifdef CONFIG_LOCKDEP
  3001. static void wq_init_lockdep(struct workqueue_struct *wq)
  3002. {
  3003. char *lock_name;
  3004. lockdep_register_key(&wq->key);
  3005. lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
  3006. if (!lock_name)
  3007. lock_name = wq->name;
  3008. wq->lock_name = lock_name;
  3009. lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
  3010. }
  3011. static void wq_unregister_lockdep(struct workqueue_struct *wq)
  3012. {
  3013. lockdep_unregister_key(&wq->key);
  3014. }
  3015. static void wq_free_lockdep(struct workqueue_struct *wq)
  3016. {
  3017. if (wq->lock_name != wq->name)
  3018. kfree(wq->lock_name);
  3019. }
  3020. #else
  3021. static void wq_init_lockdep(struct workqueue_struct *wq)
  3022. {
  3023. }
  3024. static void wq_unregister_lockdep(struct workqueue_struct *wq)
  3025. {
  3026. }
  3027. static void wq_free_lockdep(struct workqueue_struct *wq)
  3028. {
  3029. }
  3030. #endif
  3031. static void rcu_free_wq(struct rcu_head *rcu)
  3032. {
  3033. struct workqueue_struct *wq =
  3034. container_of(rcu, struct workqueue_struct, rcu);
  3035. wq_free_lockdep(wq);
  3036. if (!(wq->flags & WQ_UNBOUND))
  3037. free_percpu(wq->cpu_pwqs);
  3038. else
  3039. free_workqueue_attrs(wq->unbound_attrs);
  3040. kfree(wq);
  3041. }
  3042. static void rcu_free_pool(struct rcu_head *rcu)
  3043. {
  3044. struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
  3045. ida_destroy(&pool->worker_ida);
  3046. free_workqueue_attrs(pool->attrs);
  3047. kfree(pool);
  3048. }
  3049. /* This returns with the lock held on success (pool manager is inactive). */
  3050. static bool wq_manager_inactive(struct worker_pool *pool)
  3051. {
  3052. raw_spin_lock_irq(&pool->lock);
  3053. if (pool->flags & POOL_MANAGER_ACTIVE) {
  3054. raw_spin_unlock_irq(&pool->lock);
  3055. return false;
  3056. }
  3057. return true;
  3058. }
  3059. /**
  3060. * put_unbound_pool - put a worker_pool
  3061. * @pool: worker_pool to put
  3062. *
  3063. * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
  3064. * safe manner. get_unbound_pool() calls this function on its failure path
  3065. * and this function should be able to release pools which went through,
  3066. * successfully or not, init_worker_pool().
  3067. *
  3068. * Should be called with wq_pool_mutex held.
  3069. */
  3070. static void put_unbound_pool(struct worker_pool *pool)
  3071. {
  3072. DECLARE_COMPLETION_ONSTACK(detach_completion);
  3073. struct worker *worker;
  3074. lockdep_assert_held(&wq_pool_mutex);
  3075. if (--pool->refcnt)
  3076. return;
  3077. /* sanity checks */
  3078. if (WARN_ON(!(pool->cpu < 0)) ||
  3079. WARN_ON(!list_empty(&pool->worklist)))
  3080. return;
  3081. /* release id and unhash */
  3082. if (pool->id >= 0)
  3083. idr_remove(&worker_pool_idr, pool->id);
  3084. hash_del(&pool->hash_node);
  3085. /*
  3086. * Become the manager and destroy all workers. This prevents
  3087. * @pool's workers from blocking on attach_mutex. We're the last
  3088. * manager and @pool gets freed with the flag set.
  3089. * Because of how wq_manager_inactive() works, we will hold the
  3090. * spinlock after a successful wait.
  3091. */
  3092. rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
  3093. TASK_UNINTERRUPTIBLE);
  3094. pool->flags |= POOL_MANAGER_ACTIVE;
  3095. while ((worker = first_idle_worker(pool)))
  3096. destroy_worker(worker);
  3097. WARN_ON(pool->nr_workers || pool->nr_idle);
  3098. raw_spin_unlock_irq(&pool->lock);
  3099. mutex_lock(&wq_pool_attach_mutex);
  3100. if (!list_empty(&pool->workers))
  3101. pool->detach_completion = &detach_completion;
  3102. mutex_unlock(&wq_pool_attach_mutex);
  3103. if (pool->detach_completion)
  3104. wait_for_completion(pool->detach_completion);
  3105. /* shut down the timers */
  3106. del_timer_sync(&pool->idle_timer);
  3107. del_timer_sync(&pool->mayday_timer);
  3108. /* RCU protected to allow dereferences from get_work_pool() */
  3109. call_rcu(&pool->rcu, rcu_free_pool);
  3110. }
  3111. /**
  3112. * get_unbound_pool - get a worker_pool with the specified attributes
  3113. * @attrs: the attributes of the worker_pool to get
  3114. *
  3115. * Obtain a worker_pool which has the same attributes as @attrs, bump the
  3116. * reference count and return it. If there already is a matching
  3117. * worker_pool, it will be used; otherwise, this function attempts to
  3118. * create a new one.
  3119. *
  3120. * Should be called with wq_pool_mutex held.
  3121. *
  3122. * Return: On success, a worker_pool with the same attributes as @attrs.
  3123. * On failure, %NULL.
  3124. */
  3125. static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
  3126. {
  3127. u32 hash = wqattrs_hash(attrs);
  3128. struct worker_pool *pool;
  3129. int node;
  3130. int target_node = NUMA_NO_NODE;
  3131. lockdep_assert_held(&wq_pool_mutex);
  3132. /* do we already have a matching pool? */
  3133. hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
  3134. if (wqattrs_equal(pool->attrs, attrs)) {
  3135. pool->refcnt++;
  3136. return pool;
  3137. }
  3138. }
  3139. /* if cpumask is contained inside a NUMA node, we belong to that node */
  3140. if (wq_numa_enabled) {
  3141. for_each_node(node) {
  3142. if (cpumask_subset(attrs->cpumask,
  3143. wq_numa_possible_cpumask[node])) {
  3144. target_node = node;
  3145. break;
  3146. }
  3147. }
  3148. }
  3149. /* nope, create a new one */
  3150. pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
  3151. if (!pool || init_worker_pool(pool) < 0)
  3152. goto fail;
  3153. lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
  3154. copy_workqueue_attrs(pool->attrs, attrs);
  3155. pool->node = target_node;
  3156. /*
  3157. * no_numa isn't a worker_pool attribute, always clear it. See
  3158. * 'struct workqueue_attrs' comments for detail.
  3159. */
  3160. pool->attrs->no_numa = false;
  3161. if (worker_pool_assign_id(pool) < 0)
  3162. goto fail;
  3163. /* create and start the initial worker */
  3164. if (wq_online && !create_worker(pool))
  3165. goto fail;
  3166. /* install */
  3167. hash_add(unbound_pool_hash, &pool->hash_node, hash);
  3168. return pool;
  3169. fail:
  3170. if (pool)
  3171. put_unbound_pool(pool);
  3172. return NULL;
  3173. }
  3174. static void rcu_free_pwq(struct rcu_head *rcu)
  3175. {
  3176. kmem_cache_free(pwq_cache,
  3177. container_of(rcu, struct pool_workqueue, rcu));
  3178. }
  3179. /*
  3180. * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
  3181. * and needs to be destroyed.
  3182. */
  3183. static void pwq_unbound_release_workfn(struct work_struct *work)
  3184. {
  3185. struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
  3186. unbound_release_work);
  3187. struct workqueue_struct *wq = pwq->wq;
  3188. struct worker_pool *pool = pwq->pool;
  3189. bool is_last = false;
  3190. /*
  3191. * when @pwq is not linked, it doesn't hold any reference to the
  3192. * @wq, and @wq is invalid to access.
  3193. */
  3194. if (!list_empty(&pwq->pwqs_node)) {
  3195. if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
  3196. return;
  3197. mutex_lock(&wq->mutex);
  3198. list_del_rcu(&pwq->pwqs_node);
  3199. is_last = list_empty(&wq->pwqs);
  3200. mutex_unlock(&wq->mutex);
  3201. }
  3202. mutex_lock(&wq_pool_mutex);
  3203. put_unbound_pool(pool);
  3204. mutex_unlock(&wq_pool_mutex);
  3205. call_rcu(&pwq->rcu, rcu_free_pwq);
  3206. /*
  3207. * If we're the last pwq going away, @wq is already dead and no one
  3208. * is gonna access it anymore. Schedule RCU free.
  3209. */
  3210. if (is_last) {
  3211. wq_unregister_lockdep(wq);
  3212. call_rcu(&wq->rcu, rcu_free_wq);
  3213. }
  3214. }
  3215. /**
  3216. * pwq_adjust_max_active - update a pwq's max_active to the current setting
  3217. * @pwq: target pool_workqueue
  3218. *
  3219. * If @pwq isn't freezing, set @pwq->max_active to the associated
  3220. * workqueue's saved_max_active and activate delayed work items
  3221. * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
  3222. */
  3223. static void pwq_adjust_max_active(struct pool_workqueue *pwq)
  3224. {
  3225. struct workqueue_struct *wq = pwq->wq;
  3226. bool freezable = wq->flags & WQ_FREEZABLE;
  3227. unsigned long flags;
  3228. /* for @wq->saved_max_active */
  3229. lockdep_assert_held(&wq->mutex);
  3230. /* fast exit for non-freezable wqs */
  3231. if (!freezable && pwq->max_active == wq->saved_max_active)
  3232. return;
  3233. /* this function can be called during early boot w/ irq disabled */
  3234. raw_spin_lock_irqsave(&pwq->pool->lock, flags);
  3235. /*
  3236. * During [un]freezing, the caller is responsible for ensuring that
  3237. * this function is called at least once after @workqueue_freezing
  3238. * is updated and visible.
  3239. */
  3240. if (!freezable || !workqueue_freezing) {
  3241. bool kick = false;
  3242. pwq->max_active = wq->saved_max_active;
  3243. while (!list_empty(&pwq->delayed_works) &&
  3244. pwq->nr_active < pwq->max_active) {
  3245. pwq_activate_first_delayed(pwq);
  3246. kick = true;
  3247. }
  3248. /*
  3249. * Need to kick a worker after thawed or an unbound wq's
  3250. * max_active is bumped. In realtime scenarios, always kicking a
  3251. * worker will cause interference on the isolated cpu cores, so
  3252. * let's kick iff work items were activated.
  3253. */
  3254. if (kick)
  3255. wake_up_worker(pwq->pool);
  3256. } else {
  3257. pwq->max_active = 0;
  3258. }
  3259. raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
  3260. }
  3261. /* initialize newly alloced @pwq which is associated with @wq and @pool */
  3262. static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
  3263. struct worker_pool *pool)
  3264. {
  3265. BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
  3266. memset(pwq, 0, sizeof(*pwq));
  3267. pwq->pool = pool;
  3268. pwq->wq = wq;
  3269. pwq->flush_color = -1;
  3270. pwq->refcnt = 1;
  3271. INIT_LIST_HEAD(&pwq->delayed_works);
  3272. INIT_LIST_HEAD(&pwq->pwqs_node);
  3273. INIT_LIST_HEAD(&pwq->mayday_node);
  3274. INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
  3275. }
  3276. /* sync @pwq with the current state of its associated wq and link it */
  3277. static void link_pwq(struct pool_workqueue *pwq)
  3278. {
  3279. struct workqueue_struct *wq = pwq->wq;
  3280. lockdep_assert_held(&wq->mutex);
  3281. /* may be called multiple times, ignore if already linked */
  3282. if (!list_empty(&pwq->pwqs_node))
  3283. return;
  3284. /* set the matching work_color */
  3285. pwq->work_color = wq->work_color;
  3286. /* sync max_active to the current setting */
  3287. pwq_adjust_max_active(pwq);
  3288. /* link in @pwq */
  3289. list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
  3290. }
  3291. /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
  3292. static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
  3293. const struct workqueue_attrs *attrs)
  3294. {
  3295. struct worker_pool *pool;
  3296. struct pool_workqueue *pwq;
  3297. lockdep_assert_held(&wq_pool_mutex);
  3298. pool = get_unbound_pool(attrs);
  3299. if (!pool)
  3300. return NULL;
  3301. pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
  3302. if (!pwq) {
  3303. put_unbound_pool(pool);
  3304. return NULL;
  3305. }
  3306. init_pwq(pwq, wq, pool);
  3307. return pwq;
  3308. }
  3309. /**
  3310. * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
  3311. * @attrs: the wq_attrs of the default pwq of the target workqueue
  3312. * @node: the target NUMA node
  3313. * @cpu_going_down: if >= 0, the CPU to consider as offline
  3314. * @cpumask: outarg, the resulting cpumask
  3315. *
  3316. * Calculate the cpumask a workqueue with @attrs should use on @node. If
  3317. * @cpu_going_down is >= 0, that cpu is considered offline during
  3318. * calculation. The result is stored in @cpumask.
  3319. *
  3320. * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
  3321. * enabled and @node has online CPUs requested by @attrs, the returned
  3322. * cpumask is the intersection of the possible CPUs of @node and
  3323. * @attrs->cpumask.
  3324. *
  3325. * The caller is responsible for ensuring that the cpumask of @node stays
  3326. * stable.
  3327. *
  3328. * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
  3329. * %false if equal.
  3330. */
  3331. static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
  3332. int cpu_going_down, cpumask_t *cpumask)
  3333. {
  3334. if (!wq_numa_enabled || attrs->no_numa)
  3335. goto use_dfl;
  3336. /* does @node have any online CPUs @attrs wants? */
  3337. cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
  3338. if (cpu_going_down >= 0)
  3339. cpumask_clear_cpu(cpu_going_down, cpumask);
  3340. if (cpumask_empty(cpumask))
  3341. goto use_dfl;
  3342. /* yeap, return possible CPUs in @node that @attrs wants */
  3343. cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
  3344. if (cpumask_empty(cpumask)) {
  3345. pr_warn_once("WARNING: workqueue cpumask: online intersect > "
  3346. "possible intersect\n");
  3347. return false;
  3348. }
  3349. return !cpumask_equal(cpumask, attrs->cpumask);
  3350. use_dfl:
  3351. cpumask_copy(cpumask, attrs->cpumask);
  3352. return false;
  3353. }
  3354. /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
  3355. static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
  3356. int node,
  3357. struct pool_workqueue *pwq)
  3358. {
  3359. struct pool_workqueue *old_pwq;
  3360. lockdep_assert_held(&wq_pool_mutex);
  3361. lockdep_assert_held(&wq->mutex);
  3362. /* link_pwq() can handle duplicate calls */
  3363. link_pwq(pwq);
  3364. old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
  3365. rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
  3366. return old_pwq;
  3367. }
  3368. /* context to store the prepared attrs & pwqs before applying */
  3369. struct apply_wqattrs_ctx {
  3370. struct workqueue_struct *wq; /* target workqueue */
  3371. struct workqueue_attrs *attrs; /* attrs to apply */
  3372. struct list_head list; /* queued for batching commit */
  3373. struct pool_workqueue *dfl_pwq;
  3374. struct pool_workqueue *pwq_tbl[];
  3375. };
  3376. /* free the resources after success or abort */
  3377. static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
  3378. {
  3379. if (ctx) {
  3380. int node;
  3381. for_each_node(node)
  3382. put_pwq_unlocked(ctx->pwq_tbl[node]);
  3383. put_pwq_unlocked(ctx->dfl_pwq);
  3384. free_workqueue_attrs(ctx->attrs);
  3385. kfree(ctx);
  3386. }
  3387. }
  3388. /* allocate the attrs and pwqs for later installation */
  3389. static struct apply_wqattrs_ctx *
  3390. apply_wqattrs_prepare(struct workqueue_struct *wq,
  3391. const struct workqueue_attrs *attrs)
  3392. {
  3393. struct apply_wqattrs_ctx *ctx;
  3394. struct workqueue_attrs *new_attrs, *tmp_attrs;
  3395. int node;
  3396. lockdep_assert_held(&wq_pool_mutex);
  3397. ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
  3398. new_attrs = alloc_workqueue_attrs();
  3399. tmp_attrs = alloc_workqueue_attrs();
  3400. if (!ctx || !new_attrs || !tmp_attrs)
  3401. goto out_free;
  3402. /*
  3403. * Calculate the attrs of the default pwq.
  3404. * If the user configured cpumask doesn't overlap with the
  3405. * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
  3406. */
  3407. copy_workqueue_attrs(new_attrs, attrs);
  3408. cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);
  3409. if (unlikely(cpumask_empty(new_attrs->cpumask)))
  3410. cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask);
  3411. /*
  3412. * We may create multiple pwqs with differing cpumasks. Make a
  3413. * copy of @new_attrs which will be modified and used to obtain
  3414. * pools.
  3415. */
  3416. copy_workqueue_attrs(tmp_attrs, new_attrs);
  3417. /*
  3418. * If something goes wrong during CPU up/down, we'll fall back to
  3419. * the default pwq covering whole @attrs->cpumask. Always create
  3420. * it even if we don't use it immediately.
  3421. */
  3422. ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
  3423. if (!ctx->dfl_pwq)
  3424. goto out_free;
  3425. for_each_node(node) {
  3426. if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
  3427. ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
  3428. if (!ctx->pwq_tbl[node])
  3429. goto out_free;
  3430. } else {
  3431. ctx->dfl_pwq->refcnt++;
  3432. ctx->pwq_tbl[node] = ctx->dfl_pwq;
  3433. }
  3434. }
  3435. /* save the user configured attrs and sanitize it. */
  3436. copy_workqueue_attrs(new_attrs, attrs);
  3437. cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
  3438. ctx->attrs = new_attrs;
  3439. ctx->wq = wq;
  3440. free_workqueue_attrs(tmp_attrs);
  3441. return ctx;
  3442. out_free:
  3443. free_workqueue_attrs(tmp_attrs);
  3444. free_workqueue_attrs(new_attrs);
  3445. apply_wqattrs_cleanup(ctx);
  3446. return NULL;
  3447. }
  3448. /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
  3449. static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
  3450. {
  3451. int node;
  3452. /* all pwqs have been created successfully, let's install'em */
  3453. mutex_lock(&ctx->wq->mutex);
  3454. copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
  3455. /* save the previous pwq and install the new one */
  3456. for_each_node(node)
  3457. ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
  3458. ctx->pwq_tbl[node]);
  3459. /* @dfl_pwq might not have been used, ensure it's linked */
  3460. link_pwq(ctx->dfl_pwq);
  3461. swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
  3462. mutex_unlock(&ctx->wq->mutex);
  3463. }
  3464. static void apply_wqattrs_lock(void)
  3465. {
  3466. /* CPUs should stay stable across pwq creations and installations */
  3467. get_online_cpus();
  3468. mutex_lock(&wq_pool_mutex);
  3469. }
  3470. static void apply_wqattrs_unlock(void)
  3471. {
  3472. mutex_unlock(&wq_pool_mutex);
  3473. put_online_cpus();
  3474. }
  3475. static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
  3476. const struct workqueue_attrs *attrs)
  3477. {
  3478. struct apply_wqattrs_ctx *ctx;
  3479. /* only unbound workqueues can change attributes */
  3480. if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
  3481. return -EINVAL;
  3482. /* creating multiple pwqs breaks ordering guarantee */
  3483. if (!list_empty(&wq->pwqs)) {
  3484. if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
  3485. return -EINVAL;
  3486. wq->flags &= ~__WQ_ORDERED;
  3487. }
  3488. ctx = apply_wqattrs_prepare(wq, attrs);
  3489. if (!ctx)
  3490. return -ENOMEM;
  3491. /* the ctx has been prepared successfully, let's commit it */
  3492. apply_wqattrs_commit(ctx);
  3493. apply_wqattrs_cleanup(ctx);
  3494. return 0;
  3495. }
  3496. /**
  3497. * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
  3498. * @wq: the target workqueue
  3499. * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
  3500. *
  3501. * Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
  3502. * machines, this function maps a separate pwq to each NUMA node with
  3503. * possibles CPUs in @attrs->cpumask so that work items are affine to the
  3504. * NUMA node it was issued on. Older pwqs are released as in-flight work
  3505. * items finish. Note that a work item which repeatedly requeues itself
  3506. * back-to-back will stay on its current pwq.
  3507. *
  3508. * Performs GFP_KERNEL allocations.
  3509. *
  3510. * Assumes caller has CPU hotplug read exclusion, i.e. get_online_cpus().
  3511. *
  3512. * Return: 0 on success and -errno on failure.
  3513. */
  3514. int apply_workqueue_attrs(struct workqueue_struct *wq,
  3515. const struct workqueue_attrs *attrs)
  3516. {
  3517. int ret;
  3518. lockdep_assert_cpus_held();
  3519. mutex_lock(&wq_pool_mutex);
  3520. ret = apply_workqueue_attrs_locked(wq, attrs);
  3521. mutex_unlock(&wq_pool_mutex);
  3522. return ret;
  3523. }
  3524. /**
  3525. * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
  3526. * @wq: the target workqueue
  3527. * @cpu: the CPU coming up or going down
  3528. * @online: whether @cpu is coming up or going down
  3529. *
  3530. * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
  3531. * %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
  3532. * @wq accordingly.
  3533. *
  3534. * If NUMA affinity can't be adjusted due to memory allocation failure, it
  3535. * falls back to @wq->dfl_pwq which may not be optimal but is always
  3536. * correct.
  3537. *
  3538. * Note that when the last allowed CPU of a NUMA node goes offline for a
  3539. * workqueue with a cpumask spanning multiple nodes, the workers which were
  3540. * already executing the work items for the workqueue will lose their CPU
  3541. * affinity and may execute on any CPU. This is similar to how per-cpu
  3542. * workqueues behave on CPU_DOWN. If a workqueue user wants strict
  3543. * affinity, it's the user's responsibility to flush the work item from
  3544. * CPU_DOWN_PREPARE.
  3545. */
  3546. static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
  3547. bool online)
  3548. {
  3549. int node = cpu_to_node(cpu);
  3550. int cpu_off = online ? -1 : cpu;
  3551. struct pool_workqueue *old_pwq = NULL, *pwq;
  3552. struct workqueue_attrs *target_attrs;
  3553. cpumask_t *cpumask;
  3554. lockdep_assert_held(&wq_pool_mutex);
  3555. if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||
  3556. wq->unbound_attrs->no_numa)
  3557. return;
  3558. /*
  3559. * We don't wanna alloc/free wq_attrs for each wq for each CPU.
  3560. * Let's use a preallocated one. The following buf is protected by
  3561. * CPU hotplug exclusion.
  3562. */
  3563. target_attrs = wq_update_unbound_numa_attrs_buf;
  3564. cpumask = target_attrs->cpumask;
  3565. copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
  3566. pwq = unbound_pwq_by_node(wq, node);
  3567. /*
  3568. * Let's determine what needs to be done. If the target cpumask is
  3569. * different from the default pwq's, we need to compare it to @pwq's
  3570. * and create a new one if they don't match. If the target cpumask
  3571. * equals the default pwq's, the default pwq should be used.
  3572. */
  3573. if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
  3574. if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
  3575. return;
  3576. } else {
  3577. goto use_dfl_pwq;
  3578. }
  3579. /* create a new pwq */
  3580. pwq = alloc_unbound_pwq(wq, target_attrs);
  3581. if (!pwq) {
  3582. pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
  3583. wq->name);
  3584. goto use_dfl_pwq;
  3585. }
  3586. /* Install the new pwq. */
  3587. mutex_lock(&wq->mutex);
  3588. old_pwq = numa_pwq_tbl_install(wq, node, pwq);
  3589. goto out_unlock;
  3590. use_dfl_pwq:
  3591. mutex_lock(&wq->mutex);
  3592. raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
  3593. get_pwq(wq->dfl_pwq);
  3594. raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
  3595. old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
  3596. out_unlock:
  3597. mutex_unlock(&wq->mutex);
  3598. put_pwq_unlocked(old_pwq);
  3599. }
  3600. static int alloc_and_link_pwqs(struct workqueue_struct *wq)
  3601. {
  3602. bool highpri = wq->flags & WQ_HIGHPRI;
  3603. int cpu, ret;
  3604. if (!(wq->flags & WQ_UNBOUND)) {
  3605. wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
  3606. if (!wq->cpu_pwqs)
  3607. return -ENOMEM;
  3608. for_each_possible_cpu(cpu) {
  3609. struct pool_workqueue *pwq =
  3610. per_cpu_ptr(wq->cpu_pwqs, cpu);
  3611. struct worker_pool *cpu_pools =
  3612. per_cpu(cpu_worker_pools, cpu);
  3613. init_pwq(pwq, wq, &cpu_pools[highpri]);
  3614. mutex_lock(&wq->mutex);
  3615. link_pwq(pwq);
  3616. mutex_unlock(&wq->mutex);
  3617. }
  3618. return 0;
  3619. }
  3620. get_online_cpus();
  3621. if (wq->flags & __WQ_ORDERED) {
  3622. ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
  3623. /* there should only be single pwq for ordering guarantee */
  3624. WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
  3625. wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
  3626. "ordering guarantee broken for workqueue %s\n", wq->name);
  3627. } else {
  3628. ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
  3629. }
  3630. put_online_cpus();
  3631. return ret;
  3632. }
  3633. static int wq_clamp_max_active(int max_active, unsigned int flags,
  3634. const char *name)
  3635. {
  3636. int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
  3637. if (max_active < 1 || max_active > lim)
  3638. pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
  3639. max_active, name, 1, lim);
  3640. return clamp_val(max_active, 1, lim);
  3641. }
  3642. /*
  3643. * Workqueues which may be used during memory reclaim should have a rescuer
  3644. * to guarantee forward progress.
  3645. */
  3646. static int init_rescuer(struct workqueue_struct *wq)
  3647. {
  3648. struct worker *rescuer;
  3649. int ret;
  3650. if (!(wq->flags & WQ_MEM_RECLAIM))
  3651. return 0;
  3652. rescuer = alloc_worker(NUMA_NO_NODE);
  3653. if (!rescuer)
  3654. return -ENOMEM;
  3655. rescuer->rescue_wq = wq;
  3656. rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
  3657. if (IS_ERR(rescuer->task)) {
  3658. ret = PTR_ERR(rescuer->task);
  3659. kfree(rescuer);
  3660. return ret;
  3661. }
  3662. wq->rescuer = rescuer;
  3663. kthread_bind_mask(rescuer->task, cpu_possible_mask);
  3664. wake_up_process(rescuer->task);
  3665. return 0;
  3666. }
  3667. __printf(1, 4)
  3668. struct workqueue_struct *alloc_workqueue(const char *fmt,
  3669. unsigned int flags,
  3670. int max_active, ...)
  3671. {
  3672. size_t tbl_size = 0;
  3673. va_list args;
  3674. struct workqueue_struct *wq;
  3675. struct pool_workqueue *pwq;
  3676. /*
  3677. * Unbound && max_active == 1 used to imply ordered, which is no
  3678. * longer the case on NUMA machines due to per-node pools. While
  3679. * alloc_ordered_workqueue() is the right way to create an ordered
  3680. * workqueue, keep the previous behavior to avoid subtle breakages
  3681. * on NUMA.
  3682. */
  3683. if ((flags & WQ_UNBOUND) && max_active == 1)
  3684. flags |= __WQ_ORDERED;
  3685. /* see the comment above the definition of WQ_POWER_EFFICIENT */
  3686. if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
  3687. flags |= WQ_UNBOUND;
  3688. /* allocate wq and format name */
  3689. if (flags & WQ_UNBOUND)
  3690. tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
  3691. wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
  3692. if (!wq)
  3693. return NULL;
  3694. if (flags & WQ_UNBOUND) {
  3695. wq->unbound_attrs = alloc_workqueue_attrs();
  3696. if (!wq->unbound_attrs)
  3697. goto err_free_wq;
  3698. }
  3699. va_start(args, max_active);
  3700. vsnprintf(wq->name, sizeof(wq->name), fmt, args);
  3701. va_end(args);
  3702. max_active = max_active ?: WQ_DFL_ACTIVE;
  3703. max_active = wq_clamp_max_active(max_active, flags, wq->name);
  3704. /* init wq */
  3705. wq->flags = flags;
  3706. wq->saved_max_active = max_active;
  3707. mutex_init(&wq->mutex);
  3708. atomic_set(&wq->nr_pwqs_to_flush, 0);
  3709. INIT_LIST_HEAD(&wq->pwqs);
  3710. INIT_LIST_HEAD(&wq->flusher_queue);
  3711. INIT_LIST_HEAD(&wq->flusher_overflow);
  3712. INIT_LIST_HEAD(&wq->maydays);
  3713. wq_init_lockdep(wq);
  3714. INIT_LIST_HEAD(&wq->list);
  3715. if (alloc_and_link_pwqs(wq) < 0)
  3716. goto err_unreg_lockdep;
  3717. if (wq_online && init_rescuer(wq) < 0)
  3718. goto err_destroy;
  3719. if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
  3720. goto err_destroy;
  3721. /*
  3722. * wq_pool_mutex protects global freeze state and workqueues list.
  3723. * Grab it, adjust max_active and add the new @wq to workqueues
  3724. * list.
  3725. */
  3726. mutex_lock(&wq_pool_mutex);
  3727. mutex_lock(&wq->mutex);
  3728. for_each_pwq(pwq, wq)
  3729. pwq_adjust_max_active(pwq);
  3730. mutex_unlock(&wq->mutex);
  3731. list_add_tail_rcu(&wq->list, &workqueues);
  3732. mutex_unlock(&wq_pool_mutex);
  3733. return wq;
  3734. err_unreg_lockdep:
  3735. wq_unregister_lockdep(wq);
  3736. wq_free_lockdep(wq);
  3737. err_free_wq:
  3738. free_workqueue_attrs(wq->unbound_attrs);
  3739. kfree(wq);
  3740. return NULL;
  3741. err_destroy:
  3742. destroy_workqueue(wq);
  3743. return NULL;
  3744. }
  3745. EXPORT_SYMBOL_GPL(alloc_workqueue);
  3746. static bool pwq_busy(struct pool_workqueue *pwq)
  3747. {
  3748. int i;
  3749. for (i = 0; i < WORK_NR_COLORS; i++)
  3750. if (pwq->nr_in_flight[i])
  3751. return true;
  3752. if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
  3753. return true;
  3754. if (pwq->nr_active || !list_empty(&pwq->delayed_works))
  3755. return true;
  3756. return false;
  3757. }
  3758. /**
  3759. * destroy_workqueue - safely terminate a workqueue
  3760. * @wq: target workqueue
  3761. *
  3762. * Safely destroy a workqueue. All work currently pending will be done first.
  3763. */
  3764. void destroy_workqueue(struct workqueue_struct *wq)
  3765. {
  3766. struct pool_workqueue *pwq;
  3767. int node;
  3768. /*
  3769. * Remove it from sysfs first so that sanity check failure doesn't
  3770. * lead to sysfs name conflicts.
  3771. */
  3772. workqueue_sysfs_unregister(wq);
  3773. /* drain it before proceeding with destruction */
  3774. drain_workqueue(wq);
  3775. /* kill rescuer, if sanity checks fail, leave it w/o rescuer */
  3776. if (wq->rescuer) {
  3777. struct worker *rescuer = wq->rescuer;
  3778. /* this prevents new queueing */
  3779. raw_spin_lock_irq(&wq_mayday_lock);
  3780. wq->rescuer = NULL;
  3781. raw_spin_unlock_irq(&wq_mayday_lock);
  3782. /* rescuer will empty maydays list before exiting */
  3783. kthread_stop(rescuer->task);
  3784. kfree(rescuer);
  3785. }
  3786. /*
  3787. * Sanity checks - grab all the locks so that we wait for all
  3788. * in-flight operations which may do put_pwq().
  3789. */
  3790. mutex_lock(&wq_pool_mutex);
  3791. mutex_lock(&wq->mutex);
  3792. for_each_pwq(pwq, wq) {
  3793. raw_spin_lock_irq(&pwq->pool->lock);
  3794. if (WARN_ON(pwq_busy(pwq))) {
  3795. pr_warn("%s: %s has the following busy pwq\n",
  3796. __func__, wq->name);
  3797. show_pwq(pwq);
  3798. raw_spin_unlock_irq(&pwq->pool->lock);
  3799. mutex_unlock(&wq->mutex);
  3800. mutex_unlock(&wq_pool_mutex);
  3801. show_workqueue_state();
  3802. return;
  3803. }
  3804. raw_spin_unlock_irq(&pwq->pool->lock);
  3805. }
  3806. mutex_unlock(&wq->mutex);
  3807. /*
  3808. * wq list is used to freeze wq, remove from list after
  3809. * flushing is complete in case freeze races us.
  3810. */
  3811. list_del_rcu(&wq->list);
  3812. mutex_unlock(&wq_pool_mutex);
  3813. if (!(wq->flags & WQ_UNBOUND)) {
  3814. wq_unregister_lockdep(wq);
  3815. /*
  3816. * The base ref is never dropped on per-cpu pwqs. Directly
  3817. * schedule RCU free.
  3818. */
  3819. call_rcu(&wq->rcu, rcu_free_wq);
  3820. } else {
  3821. /*
  3822. * We're the sole accessor of @wq at this point. Directly
  3823. * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
  3824. * @wq will be freed when the last pwq is released.
  3825. */
  3826. for_each_node(node) {
  3827. pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
  3828. RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
  3829. put_pwq_unlocked(pwq);
  3830. }
  3831. /*
  3832. * Put dfl_pwq. @wq may be freed any time after dfl_pwq is
  3833. * put. Don't access it afterwards.
  3834. */
  3835. pwq = wq->dfl_pwq;
  3836. wq->dfl_pwq = NULL;
  3837. put_pwq_unlocked(pwq);
  3838. }
  3839. }
  3840. EXPORT_SYMBOL_GPL(destroy_workqueue);
  3841. /**
  3842. * workqueue_set_max_active - adjust max_active of a workqueue
  3843. * @wq: target workqueue
  3844. * @max_active: new max_active value.
  3845. *
  3846. * Set max_active of @wq to @max_active.
  3847. *
  3848. * CONTEXT:
  3849. * Don't call from IRQ context.
  3850. */
  3851. void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
  3852. {
  3853. struct pool_workqueue *pwq;
  3854. /* disallow meddling with max_active for ordered workqueues */
  3855. if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
  3856. return;
  3857. max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
  3858. mutex_lock(&wq->mutex);
  3859. wq->flags &= ~__WQ_ORDERED;
  3860. wq->saved_max_active = max_active;
  3861. for_each_pwq(pwq, wq)
  3862. pwq_adjust_max_active(pwq);
  3863. mutex_unlock(&wq->mutex);
  3864. }
  3865. EXPORT_SYMBOL_GPL(workqueue_set_max_active);
  3866. /**
  3867. * current_work - retrieve %current task's work struct
  3868. *
  3869. * Determine if %current task is a workqueue worker and what it's working on.
  3870. * Useful to find out the context that the %current task is running in.
  3871. *
  3872. * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
  3873. */
  3874. struct work_struct *current_work(void)
  3875. {
  3876. struct worker *worker = current_wq_worker();
  3877. return worker ? worker->current_work : NULL;
  3878. }
  3879. EXPORT_SYMBOL(current_work);
  3880. /**
  3881. * current_is_workqueue_rescuer - is %current workqueue rescuer?
  3882. *
  3883. * Determine whether %current is a workqueue rescuer. Can be used from
  3884. * work functions to determine whether it's being run off the rescuer task.
  3885. *
  3886. * Return: %true if %current is a workqueue rescuer. %false otherwise.
  3887. */
  3888. bool current_is_workqueue_rescuer(void)
  3889. {
  3890. struct worker *worker = current_wq_worker();
  3891. return worker && worker->rescue_wq;
  3892. }
  3893. /**
  3894. * workqueue_congested - test whether a workqueue is congested
  3895. * @cpu: CPU in question
  3896. * @wq: target workqueue
  3897. *
  3898. * Test whether @wq's cpu workqueue for @cpu is congested. There is
  3899. * no synchronization around this function and the test result is
  3900. * unreliable and only useful as advisory hints or for debugging.
  3901. *
  3902. * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
  3903. * Note that both per-cpu and unbound workqueues may be associated with
  3904. * multiple pool_workqueues which have separate congested states. A
  3905. * workqueue being congested on one CPU doesn't mean the workqueue is also
  3906. * contested on other CPUs / NUMA nodes.
  3907. *
  3908. * Return:
  3909. * %true if congested, %false otherwise.
  3910. */
  3911. bool workqueue_congested(int cpu, struct workqueue_struct *wq)
  3912. {
  3913. struct pool_workqueue *pwq;
  3914. bool ret;
  3915. rcu_read_lock();
  3916. preempt_disable();
  3917. if (cpu == WORK_CPU_UNBOUND)
  3918. cpu = smp_processor_id();
  3919. if (!(wq->flags & WQ_UNBOUND))
  3920. pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
  3921. else
  3922. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  3923. ret = !list_empty(&pwq->delayed_works);
  3924. preempt_enable();
  3925. rcu_read_unlock();
  3926. return ret;
  3927. }
  3928. EXPORT_SYMBOL_GPL(workqueue_congested);
  3929. /**
  3930. * work_busy - test whether a work is currently pending or running
  3931. * @work: the work to be tested
  3932. *
  3933. * Test whether @work is currently pending or running. There is no
  3934. * synchronization around this function and the test result is
  3935. * unreliable and only useful as advisory hints or for debugging.
  3936. *
  3937. * Return:
  3938. * OR'd bitmask of WORK_BUSY_* bits.
  3939. */
  3940. unsigned int work_busy(struct work_struct *work)
  3941. {
  3942. struct worker_pool *pool;
  3943. unsigned long flags;
  3944. unsigned int ret = 0;
  3945. if (work_pending(work))
  3946. ret |= WORK_BUSY_PENDING;
  3947. rcu_read_lock();
  3948. pool = get_work_pool(work);
  3949. if (pool) {
  3950. raw_spin_lock_irqsave(&pool->lock, flags);
  3951. if (find_worker_executing_work(pool, work))
  3952. ret |= WORK_BUSY_RUNNING;
  3953. raw_spin_unlock_irqrestore(&pool->lock, flags);
  3954. }
  3955. rcu_read_unlock();
  3956. return ret;
  3957. }
  3958. EXPORT_SYMBOL_GPL(work_busy);
  3959. /**
  3960. * set_worker_desc - set description for the current work item
  3961. * @fmt: printf-style format string
  3962. * @...: arguments for the format string
  3963. *
  3964. * This function can be called by a running work function to describe what
  3965. * the work item is about. If the worker task gets dumped, this
  3966. * information will be printed out together to help debugging. The
  3967. * description can be at most WORKER_DESC_LEN including the trailing '\0'.
  3968. */
  3969. void set_worker_desc(const char *fmt, ...)
  3970. {
  3971. struct worker *worker = current_wq_worker();
  3972. va_list args;
  3973. if (worker) {
  3974. va_start(args, fmt);
  3975. vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
  3976. va_end(args);
  3977. }
  3978. }
  3979. EXPORT_SYMBOL_GPL(set_worker_desc);
  3980. /**
  3981. * print_worker_info - print out worker information and description
  3982. * @log_lvl: the log level to use when printing
  3983. * @task: target task
  3984. *
  3985. * If @task is a worker and currently executing a work item, print out the
  3986. * name of the workqueue being serviced and worker description set with
  3987. * set_worker_desc() by the currently executing work item.
  3988. *
  3989. * This function can be safely called on any task as long as the
  3990. * task_struct itself is accessible. While safe, this function isn't
  3991. * synchronized and may print out mixups or garbages of limited length.
  3992. */
  3993. void print_worker_info(const char *log_lvl, struct task_struct *task)
  3994. {
  3995. work_func_t *fn = NULL;
  3996. char name[WQ_NAME_LEN] = { };
  3997. char desc[WORKER_DESC_LEN] = { };
  3998. struct pool_workqueue *pwq = NULL;
  3999. struct workqueue_struct *wq = NULL;
  4000. struct worker *worker;
  4001. if (!(task->flags & PF_WQ_WORKER))
  4002. return;
  4003. /*
  4004. * This function is called without any synchronization and @task
  4005. * could be in any state. Be careful with dereferences.
  4006. */
  4007. worker = kthread_probe_data(task);
  4008. /*
  4009. * Carefully copy the associated workqueue's workfn, name and desc.
  4010. * Keep the original last '\0' in case the original is garbage.
  4011. */
  4012. copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
  4013. copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
  4014. copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
  4015. copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
  4016. copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
  4017. if (fn || name[0] || desc[0]) {
  4018. printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
  4019. if (strcmp(name, desc))
  4020. pr_cont(" (%s)", desc);
  4021. pr_cont("\n");
  4022. }
  4023. }
  4024. static void pr_cont_pool_info(struct worker_pool *pool)
  4025. {
  4026. pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
  4027. if (pool->node != NUMA_NO_NODE)
  4028. pr_cont(" node=%d", pool->node);
  4029. pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
  4030. }
  4031. static void pr_cont_work(bool comma, struct work_struct *work)
  4032. {
  4033. if (work->func == wq_barrier_func) {
  4034. struct wq_barrier *barr;
  4035. barr = container_of(work, struct wq_barrier, work);
  4036. pr_cont("%s BAR(%d)", comma ? "," : "",
  4037. task_pid_nr(barr->task));
  4038. } else {
  4039. pr_cont("%s %ps", comma ? "," : "", work->func);
  4040. }
  4041. }
  4042. static void show_pwq(struct pool_workqueue *pwq)
  4043. {
  4044. struct worker_pool *pool = pwq->pool;
  4045. struct work_struct *work;
  4046. struct worker *worker;
  4047. bool has_in_flight = false, has_pending = false;
  4048. int bkt;
  4049. pr_info(" pwq %d:", pool->id);
  4050. pr_cont_pool_info(pool);
  4051. pr_cont(" active=%d/%d refcnt=%d%s\n",
  4052. pwq->nr_active, pwq->max_active, pwq->refcnt,
  4053. !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
  4054. hash_for_each(pool->busy_hash, bkt, worker, hentry) {
  4055. if (worker->current_pwq == pwq) {
  4056. has_in_flight = true;
  4057. break;
  4058. }
  4059. }
  4060. if (has_in_flight) {
  4061. bool comma = false;
  4062. pr_info(" in-flight:");
  4063. hash_for_each(pool->busy_hash, bkt, worker, hentry) {
  4064. if (worker->current_pwq != pwq)
  4065. continue;
  4066. pr_cont("%s %d%s:%ps", comma ? "," : "",
  4067. task_pid_nr(worker->task),
  4068. worker->rescue_wq ? "(RESCUER)" : "",
  4069. worker->current_func);
  4070. list_for_each_entry(work, &worker->scheduled, entry)
  4071. pr_cont_work(false, work);
  4072. comma = true;
  4073. }
  4074. pr_cont("\n");
  4075. }
  4076. list_for_each_entry(work, &pool->worklist, entry) {
  4077. if (get_work_pwq(work) == pwq) {
  4078. has_pending = true;
  4079. break;
  4080. }
  4081. }
  4082. if (has_pending) {
  4083. bool comma = false;
  4084. pr_info(" pending:");
  4085. list_for_each_entry(work, &pool->worklist, entry) {
  4086. if (get_work_pwq(work) != pwq)
  4087. continue;
  4088. pr_cont_work(comma, work);
  4089. comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
  4090. }
  4091. pr_cont("\n");
  4092. }
  4093. if (!list_empty(&pwq->delayed_works)) {
  4094. bool comma = false;
  4095. pr_info(" delayed:");
  4096. list_for_each_entry(work, &pwq->delayed_works, entry) {
  4097. pr_cont_work(comma, work);
  4098. comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
  4099. }
  4100. pr_cont("\n");
  4101. }
  4102. }
  4103. /**
  4104. * show_workqueue_state - dump workqueue state
  4105. *
  4106. * Called from a sysrq handler or try_to_freeze_tasks() and prints out
  4107. * all busy workqueues and pools.
  4108. */
  4109. void show_workqueue_state(void)
  4110. {
  4111. struct workqueue_struct *wq;
  4112. struct worker_pool *pool;
  4113. unsigned long flags;
  4114. int pi;
  4115. rcu_read_lock();
  4116. pr_info("Showing busy workqueues and worker pools:\n");
  4117. list_for_each_entry_rcu(wq, &workqueues, list) {
  4118. struct pool_workqueue *pwq;
  4119. bool idle = true;
  4120. for_each_pwq(pwq, wq) {
  4121. if (pwq->nr_active || !list_empty(&pwq->delayed_works)) {
  4122. idle = false;
  4123. break;
  4124. }
  4125. }
  4126. if (idle)
  4127. continue;
  4128. pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
  4129. for_each_pwq(pwq, wq) {
  4130. raw_spin_lock_irqsave(&pwq->pool->lock, flags);
  4131. if (pwq->nr_active || !list_empty(&pwq->delayed_works))
  4132. show_pwq(pwq);
  4133. raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
  4134. /*
  4135. * We could be printing a lot from atomic context, e.g.
  4136. * sysrq-t -> show_workqueue_state(). Avoid triggering
  4137. * hard lockup.
  4138. */
  4139. touch_nmi_watchdog();
  4140. }
  4141. }
  4142. for_each_pool(pool, pi) {
  4143. struct worker *worker;
  4144. bool first = true;
  4145. raw_spin_lock_irqsave(&pool->lock, flags);
  4146. if (pool->nr_workers == pool->nr_idle)
  4147. goto next_pool;
  4148. pr_info("pool %d:", pool->id);
  4149. pr_cont_pool_info(pool);
  4150. pr_cont(" hung=%us workers=%d",
  4151. jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,
  4152. pool->nr_workers);
  4153. if (pool->manager)
  4154. pr_cont(" manager: %d",
  4155. task_pid_nr(pool->manager->task));
  4156. list_for_each_entry(worker, &pool->idle_list, entry) {
  4157. pr_cont(" %s%d", first ? "idle: " : "",
  4158. task_pid_nr(worker->task));
  4159. first = false;
  4160. }
  4161. pr_cont("\n");
  4162. next_pool:
  4163. raw_spin_unlock_irqrestore(&pool->lock, flags);
  4164. /*
  4165. * We could be printing a lot from atomic context, e.g.
  4166. * sysrq-t -> show_workqueue_state(). Avoid triggering
  4167. * hard lockup.
  4168. */
  4169. touch_nmi_watchdog();
  4170. }
  4171. rcu_read_unlock();
  4172. }
  4173. /* used to show worker information through /proc/PID/{comm,stat,status} */
  4174. void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
  4175. {
  4176. int off;
  4177. /* always show the actual comm */
  4178. off = strscpy(buf, task->comm, size);
  4179. if (off < 0)
  4180. return;
  4181. /* stabilize PF_WQ_WORKER and worker pool association */
  4182. mutex_lock(&wq_pool_attach_mutex);
  4183. if (task->flags & PF_WQ_WORKER) {
  4184. struct worker *worker = kthread_data(task);
  4185. struct worker_pool *pool = worker->pool;
  4186. if (pool) {
  4187. raw_spin_lock_irq(&pool->lock);
  4188. /*
  4189. * ->desc tracks information (wq name or
  4190. * set_worker_desc()) for the latest execution. If
  4191. * current, prepend '+', otherwise '-'.
  4192. */
  4193. if (worker->desc[0] != '\0') {
  4194. if (worker->current_work)
  4195. scnprintf(buf + off, size - off, "+%s",
  4196. worker->desc);
  4197. else
  4198. scnprintf(buf + off, size - off, "-%s",
  4199. worker->desc);
  4200. }
  4201. raw_spin_unlock_irq(&pool->lock);
  4202. }
  4203. }
  4204. mutex_unlock(&wq_pool_attach_mutex);
  4205. }
  4206. EXPORT_SYMBOL_GPL(wq_worker_comm);
  4207. #ifdef CONFIG_SMP
  4208. /*
  4209. * CPU hotplug.
  4210. *
  4211. * There are two challenges in supporting CPU hotplug. Firstly, there
  4212. * are a lot of assumptions on strong associations among work, pwq and
  4213. * pool which make migrating pending and scheduled works very
  4214. * difficult to implement without impacting hot paths. Secondly,
  4215. * worker pools serve mix of short, long and very long running works making
  4216. * blocked draining impractical.
  4217. *
  4218. * This is solved by allowing the pools to be disassociated from the CPU
  4219. * running as an unbound one and allowing it to be reattached later if the
  4220. * cpu comes back online.
  4221. */
  4222. static void unbind_workers(int cpu)
  4223. {
  4224. struct worker_pool *pool;
  4225. struct worker *worker;
  4226. for_each_cpu_worker_pool(pool, cpu) {
  4227. mutex_lock(&wq_pool_attach_mutex);
  4228. raw_spin_lock_irq(&pool->lock);
  4229. /*
  4230. * We've blocked all attach/detach operations. Make all workers
  4231. * unbound and set DISASSOCIATED. Before this, all workers
  4232. * except for the ones which are still executing works from
  4233. * before the last CPU down must be on the cpu. After
  4234. * this, they may become diasporas.
  4235. */
  4236. for_each_pool_worker(worker, pool)
  4237. worker->flags |= WORKER_UNBOUND;
  4238. pool->flags |= POOL_DISASSOCIATED;
  4239. raw_spin_unlock_irq(&pool->lock);
  4240. mutex_unlock(&wq_pool_attach_mutex);
  4241. /*
  4242. * Call schedule() so that we cross rq->lock and thus can
  4243. * guarantee sched callbacks see the %WORKER_UNBOUND flag.
  4244. * This is necessary as scheduler callbacks may be invoked
  4245. * from other cpus.
  4246. */
  4247. schedule();
  4248. /*
  4249. * Sched callbacks are disabled now. Zap nr_running.
  4250. * After this, nr_running stays zero and need_more_worker()
  4251. * and keep_working() are always true as long as the
  4252. * worklist is not empty. This pool now behaves as an
  4253. * unbound (in terms of concurrency management) pool which
  4254. * are served by workers tied to the pool.
  4255. */
  4256. atomic_set(&pool->nr_running, 0);
  4257. /*
  4258. * With concurrency management just turned off, a busy
  4259. * worker blocking could lead to lengthy stalls. Kick off
  4260. * unbound chain execution of currently pending work items.
  4261. */
  4262. raw_spin_lock_irq(&pool->lock);
  4263. wake_up_worker(pool);
  4264. raw_spin_unlock_irq(&pool->lock);
  4265. }
  4266. }
  4267. /**
  4268. * rebind_workers - rebind all workers of a pool to the associated CPU
  4269. * @pool: pool of interest
  4270. *
  4271. * @pool->cpu is coming online. Rebind all workers to the CPU.
  4272. */
  4273. static void rebind_workers(struct worker_pool *pool)
  4274. {
  4275. struct worker *worker;
  4276. lockdep_assert_held(&wq_pool_attach_mutex);
  4277. /*
  4278. * Restore CPU affinity of all workers. As all idle workers should
  4279. * be on the run-queue of the associated CPU before any local
  4280. * wake-ups for concurrency management happen, restore CPU affinity
  4281. * of all workers first and then clear UNBOUND. As we're called
  4282. * from CPU_ONLINE, the following shouldn't fail.
  4283. */
  4284. for_each_pool_worker(worker, pool)
  4285. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
  4286. pool->attrs->cpumask) < 0);
  4287. raw_spin_lock_irq(&pool->lock);
  4288. pool->flags &= ~POOL_DISASSOCIATED;
  4289. for_each_pool_worker(worker, pool) {
  4290. unsigned int worker_flags = worker->flags;
  4291. /*
  4292. * A bound idle worker should actually be on the runqueue
  4293. * of the associated CPU for local wake-ups targeting it to
  4294. * work. Kick all idle workers so that they migrate to the
  4295. * associated CPU. Doing this in the same loop as
  4296. * replacing UNBOUND with REBOUND is safe as no worker will
  4297. * be bound before @pool->lock is released.
  4298. */
  4299. if (worker_flags & WORKER_IDLE)
  4300. wake_up_process(worker->task);
  4301. /*
  4302. * We want to clear UNBOUND but can't directly call
  4303. * worker_clr_flags() or adjust nr_running. Atomically
  4304. * replace UNBOUND with another NOT_RUNNING flag REBOUND.
  4305. * @worker will clear REBOUND using worker_clr_flags() when
  4306. * it initiates the next execution cycle thus restoring
  4307. * concurrency management. Note that when or whether
  4308. * @worker clears REBOUND doesn't affect correctness.
  4309. *
  4310. * WRITE_ONCE() is necessary because @worker->flags may be
  4311. * tested without holding any lock in
  4312. * wq_worker_running(). Without it, NOT_RUNNING test may
  4313. * fail incorrectly leading to premature concurrency
  4314. * management operations.
  4315. */
  4316. WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
  4317. worker_flags |= WORKER_REBOUND;
  4318. worker_flags &= ~WORKER_UNBOUND;
  4319. WRITE_ONCE(worker->flags, worker_flags);
  4320. }
  4321. raw_spin_unlock_irq(&pool->lock);
  4322. }
  4323. /**
  4324. * restore_unbound_workers_cpumask - restore cpumask of unbound workers
  4325. * @pool: unbound pool of interest
  4326. * @cpu: the CPU which is coming up
  4327. *
  4328. * An unbound pool may end up with a cpumask which doesn't have any online
  4329. * CPUs. When a worker of such pool get scheduled, the scheduler resets
  4330. * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
  4331. * online CPU before, cpus_allowed of all its workers should be restored.
  4332. */
  4333. static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
  4334. {
  4335. static cpumask_t cpumask;
  4336. struct worker *worker;
  4337. lockdep_assert_held(&wq_pool_attach_mutex);
  4338. /* is @cpu allowed for @pool? */
  4339. if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
  4340. return;
  4341. cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
  4342. /* as we're called from CPU_ONLINE, the following shouldn't fail */
  4343. for_each_pool_worker(worker, pool)
  4344. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
  4345. }
  4346. int workqueue_prepare_cpu(unsigned int cpu)
  4347. {
  4348. struct worker_pool *pool;
  4349. for_each_cpu_worker_pool(pool, cpu) {
  4350. if (pool->nr_workers)
  4351. continue;
  4352. if (!create_worker(pool))
  4353. return -ENOMEM;
  4354. }
  4355. return 0;
  4356. }
  4357. int workqueue_online_cpu(unsigned int cpu)
  4358. {
  4359. struct worker_pool *pool;
  4360. struct workqueue_struct *wq;
  4361. int pi;
  4362. mutex_lock(&wq_pool_mutex);
  4363. for_each_pool(pool, pi) {
  4364. mutex_lock(&wq_pool_attach_mutex);
  4365. if (pool->cpu == cpu)
  4366. rebind_workers(pool);
  4367. else if (pool->cpu < 0)
  4368. restore_unbound_workers_cpumask(pool, cpu);
  4369. mutex_unlock(&wq_pool_attach_mutex);
  4370. }
  4371. /* update NUMA affinity of unbound workqueues */
  4372. list_for_each_entry(wq, &workqueues, list)
  4373. wq_update_unbound_numa(wq, cpu, true);
  4374. mutex_unlock(&wq_pool_mutex);
  4375. return 0;
  4376. }
  4377. int workqueue_offline_cpu(unsigned int cpu)
  4378. {
  4379. struct workqueue_struct *wq;
  4380. /* unbinding per-cpu workers should happen on the local CPU */
  4381. if (WARN_ON(cpu != smp_processor_id()))
  4382. return -1;
  4383. unbind_workers(cpu);
  4384. /* update NUMA affinity of unbound workqueues */
  4385. mutex_lock(&wq_pool_mutex);
  4386. list_for_each_entry(wq, &workqueues, list)
  4387. wq_update_unbound_numa(wq, cpu, false);
  4388. mutex_unlock(&wq_pool_mutex);
  4389. return 0;
  4390. }
  4391. struct work_for_cpu {
  4392. struct work_struct work;
  4393. long (*fn)(void *);
  4394. void *arg;
  4395. long ret;
  4396. };
  4397. static void work_for_cpu_fn(struct work_struct *work)
  4398. {
  4399. struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
  4400. wfc->ret = wfc->fn(wfc->arg);
  4401. }
  4402. /**
  4403. * work_on_cpu - run a function in thread context on a particular cpu
  4404. * @cpu: the cpu to run on
  4405. * @fn: the function to run
  4406. * @arg: the function arg
  4407. *
  4408. * It is up to the caller to ensure that the cpu doesn't go offline.
  4409. * The caller must not hold any locks which would prevent @fn from completing.
  4410. *
  4411. * Return: The value @fn returns.
  4412. */
  4413. long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
  4414. {
  4415. struct work_for_cpu wfc = { .fn = fn, .arg = arg };
  4416. INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
  4417. schedule_work_on(cpu, &wfc.work);
  4418. flush_work(&wfc.work);
  4419. destroy_work_on_stack(&wfc.work);
  4420. return wfc.ret;
  4421. }
  4422. EXPORT_SYMBOL_GPL(work_on_cpu);
  4423. /**
  4424. * work_on_cpu_safe - run a function in thread context on a particular cpu
  4425. * @cpu: the cpu to run on
  4426. * @fn: the function to run
  4427. * @arg: the function argument
  4428. *
  4429. * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
  4430. * any locks which would prevent @fn from completing.
  4431. *
  4432. * Return: The value @fn returns.
  4433. */
  4434. long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg)
  4435. {
  4436. long ret = -ENODEV;
  4437. get_online_cpus();
  4438. if (cpu_online(cpu))
  4439. ret = work_on_cpu(cpu, fn, arg);
  4440. put_online_cpus();
  4441. return ret;
  4442. }
  4443. EXPORT_SYMBOL_GPL(work_on_cpu_safe);
  4444. #endif /* CONFIG_SMP */
  4445. #ifdef CONFIG_FREEZER
  4446. /**
  4447. * freeze_workqueues_begin - begin freezing workqueues
  4448. *
  4449. * Start freezing workqueues. After this function returns, all freezable
  4450. * workqueues will queue new works to their delayed_works list instead of
  4451. * pool->worklist.
  4452. *
  4453. * CONTEXT:
  4454. * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
  4455. */
  4456. void freeze_workqueues_begin(void)
  4457. {
  4458. struct workqueue_struct *wq;
  4459. struct pool_workqueue *pwq;
  4460. mutex_lock(&wq_pool_mutex);
  4461. WARN_ON_ONCE(workqueue_freezing);
  4462. workqueue_freezing = true;
  4463. list_for_each_entry(wq, &workqueues, list) {
  4464. mutex_lock(&wq->mutex);
  4465. for_each_pwq(pwq, wq)
  4466. pwq_adjust_max_active(pwq);
  4467. mutex_unlock(&wq->mutex);
  4468. }
  4469. mutex_unlock(&wq_pool_mutex);
  4470. }
  4471. /**
  4472. * freeze_workqueues_busy - are freezable workqueues still busy?
  4473. *
  4474. * Check whether freezing is complete. This function must be called
  4475. * between freeze_workqueues_begin() and thaw_workqueues().
  4476. *
  4477. * CONTEXT:
  4478. * Grabs and releases wq_pool_mutex.
  4479. *
  4480. * Return:
  4481. * %true if some freezable workqueues are still busy. %false if freezing
  4482. * is complete.
  4483. */
  4484. bool freeze_workqueues_busy(void)
  4485. {
  4486. bool busy = false;
  4487. struct workqueue_struct *wq;
  4488. struct pool_workqueue *pwq;
  4489. mutex_lock(&wq_pool_mutex);
  4490. WARN_ON_ONCE(!workqueue_freezing);
  4491. list_for_each_entry(wq, &workqueues, list) {
  4492. if (!(wq->flags & WQ_FREEZABLE))
  4493. continue;
  4494. /*
  4495. * nr_active is monotonically decreasing. It's safe
  4496. * to peek without lock.
  4497. */
  4498. rcu_read_lock();
  4499. for_each_pwq(pwq, wq) {
  4500. WARN_ON_ONCE(pwq->nr_active < 0);
  4501. if (pwq->nr_active) {
  4502. busy = true;
  4503. rcu_read_unlock();
  4504. goto out_unlock;
  4505. }
  4506. }
  4507. rcu_read_unlock();
  4508. }
  4509. out_unlock:
  4510. mutex_unlock(&wq_pool_mutex);
  4511. return busy;
  4512. }
  4513. /**
  4514. * thaw_workqueues - thaw workqueues
  4515. *
  4516. * Thaw workqueues. Normal queueing is restored and all collected
  4517. * frozen works are transferred to their respective pool worklists.
  4518. *
  4519. * CONTEXT:
  4520. * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
  4521. */
  4522. void thaw_workqueues(void)
  4523. {
  4524. struct workqueue_struct *wq;
  4525. struct pool_workqueue *pwq;
  4526. mutex_lock(&wq_pool_mutex);
  4527. if (!workqueue_freezing)
  4528. goto out_unlock;
  4529. workqueue_freezing = false;
  4530. /* restore max_active and repopulate worklist */
  4531. list_for_each_entry(wq, &workqueues, list) {
  4532. mutex_lock(&wq->mutex);
  4533. for_each_pwq(pwq, wq)
  4534. pwq_adjust_max_active(pwq);
  4535. mutex_unlock(&wq->mutex);
  4536. }
  4537. out_unlock:
  4538. mutex_unlock(&wq_pool_mutex);
  4539. }
  4540. #endif /* CONFIG_FREEZER */
  4541. static int workqueue_apply_unbound_cpumask(void)
  4542. {
  4543. LIST_HEAD(ctxs);
  4544. int ret = 0;
  4545. struct workqueue_struct *wq;
  4546. struct apply_wqattrs_ctx *ctx, *n;
  4547. lockdep_assert_held(&wq_pool_mutex);
  4548. list_for_each_entry(wq, &workqueues, list) {
  4549. if (!(wq->flags & WQ_UNBOUND))
  4550. continue;
  4551. /* creating multiple pwqs breaks ordering guarantee */
  4552. if (wq->flags & __WQ_ORDERED)
  4553. continue;
  4554. ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
  4555. if (!ctx) {
  4556. ret = -ENOMEM;
  4557. break;
  4558. }
  4559. list_add_tail(&ctx->list, &ctxs);
  4560. }
  4561. list_for_each_entry_safe(ctx, n, &ctxs, list) {
  4562. if (!ret)
  4563. apply_wqattrs_commit(ctx);
  4564. apply_wqattrs_cleanup(ctx);
  4565. }
  4566. return ret;
  4567. }
  4568. /**
  4569. * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
  4570. * @cpumask: the cpumask to set
  4571. *
  4572. * The low-level workqueues cpumask is a global cpumask that limits
  4573. * the affinity of all unbound workqueues. This function check the @cpumask
  4574. * and apply it to all unbound workqueues and updates all pwqs of them.
  4575. *
  4576. * Retun: 0 - Success
  4577. * -EINVAL - Invalid @cpumask
  4578. * -ENOMEM - Failed to allocate memory for attrs or pwqs.
  4579. */
  4580. int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
  4581. {
  4582. int ret = -EINVAL;
  4583. cpumask_var_t saved_cpumask;
  4584. /*
  4585. * Not excluding isolated cpus on purpose.
  4586. * If the user wishes to include them, we allow that.
  4587. */
  4588. cpumask_and(cpumask, cpumask, cpu_possible_mask);
  4589. if (!cpumask_empty(cpumask)) {
  4590. apply_wqattrs_lock();
  4591. if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
  4592. ret = 0;
  4593. goto out_unlock;
  4594. }
  4595. if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL)) {
  4596. ret = -ENOMEM;
  4597. goto out_unlock;
  4598. }
  4599. /* save the old wq_unbound_cpumask. */
  4600. cpumask_copy(saved_cpumask, wq_unbound_cpumask);
  4601. /* update wq_unbound_cpumask at first and apply it to wqs. */
  4602. cpumask_copy(wq_unbound_cpumask, cpumask);
  4603. ret = workqueue_apply_unbound_cpumask();
  4604. /* restore the wq_unbound_cpumask when failed. */
  4605. if (ret < 0)
  4606. cpumask_copy(wq_unbound_cpumask, saved_cpumask);
  4607. free_cpumask_var(saved_cpumask);
  4608. out_unlock:
  4609. apply_wqattrs_unlock();
  4610. }
  4611. return ret;
  4612. }
  4613. #ifdef CONFIG_SYSFS
  4614. /*
  4615. * Workqueues with WQ_SYSFS flag set is visible to userland via
  4616. * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
  4617. * following attributes.
  4618. *
  4619. * per_cpu RO bool : whether the workqueue is per-cpu or unbound
  4620. * max_active RW int : maximum number of in-flight work items
  4621. *
  4622. * Unbound workqueues have the following extra attributes.
  4623. *
  4624. * pool_ids RO int : the associated pool IDs for each node
  4625. * nice RW int : nice value of the workers
  4626. * cpumask RW mask : bitmask of allowed CPUs for the workers
  4627. * numa RW bool : whether enable NUMA affinity
  4628. */
  4629. struct wq_device {
  4630. struct workqueue_struct *wq;
  4631. struct device dev;
  4632. };
  4633. static struct workqueue_struct *dev_to_wq(struct device *dev)
  4634. {
  4635. struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
  4636. return wq_dev->wq;
  4637. }
  4638. static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
  4639. char *buf)
  4640. {
  4641. struct workqueue_struct *wq = dev_to_wq(dev);
  4642. return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
  4643. }
  4644. static DEVICE_ATTR_RO(per_cpu);
  4645. static ssize_t max_active_show(struct device *dev,
  4646. struct device_attribute *attr, char *buf)
  4647. {
  4648. struct workqueue_struct *wq = dev_to_wq(dev);
  4649. return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
  4650. }
  4651. static ssize_t max_active_store(struct device *dev,
  4652. struct device_attribute *attr, const char *buf,
  4653. size_t count)
  4654. {
  4655. struct workqueue_struct *wq = dev_to_wq(dev);
  4656. int val;
  4657. if (sscanf(buf, "%d", &val) != 1 || val <= 0)
  4658. return -EINVAL;
  4659. workqueue_set_max_active(wq, val);
  4660. return count;
  4661. }
  4662. static DEVICE_ATTR_RW(max_active);
  4663. static struct attribute *wq_sysfs_attrs[] = {
  4664. &dev_attr_per_cpu.attr,
  4665. &dev_attr_max_active.attr,
  4666. NULL,
  4667. };
  4668. ATTRIBUTE_GROUPS(wq_sysfs);
  4669. static ssize_t wq_pool_ids_show(struct device *dev,
  4670. struct device_attribute *attr, char *buf)
  4671. {
  4672. struct workqueue_struct *wq = dev_to_wq(dev);
  4673. const char *delim = "";
  4674. int node, written = 0;
  4675. get_online_cpus();
  4676. rcu_read_lock();
  4677. for_each_node(node) {
  4678. written += scnprintf(buf + written, PAGE_SIZE - written,
  4679. "%s%d:%d", delim, node,
  4680. unbound_pwq_by_node(wq, node)->pool->id);
  4681. delim = " ";
  4682. }
  4683. written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
  4684. rcu_read_unlock();
  4685. put_online_cpus();
  4686. return written;
  4687. }
  4688. static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
  4689. char *buf)
  4690. {
  4691. struct workqueue_struct *wq = dev_to_wq(dev);
  4692. int written;
  4693. mutex_lock(&wq->mutex);
  4694. written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
  4695. mutex_unlock(&wq->mutex);
  4696. return written;
  4697. }
  4698. /* prepare workqueue_attrs for sysfs store operations */
  4699. static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
  4700. {
  4701. struct workqueue_attrs *attrs;
  4702. lockdep_assert_held(&wq_pool_mutex);
  4703. attrs = alloc_workqueue_attrs();
  4704. if (!attrs)
  4705. return NULL;
  4706. copy_workqueue_attrs(attrs, wq->unbound_attrs);
  4707. return attrs;
  4708. }
  4709. static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
  4710. const char *buf, size_t count)
  4711. {
  4712. struct workqueue_struct *wq = dev_to_wq(dev);
  4713. struct workqueue_attrs *attrs;
  4714. int ret = -ENOMEM;
  4715. apply_wqattrs_lock();
  4716. attrs = wq_sysfs_prep_attrs(wq);
  4717. if (!attrs)
  4718. goto out_unlock;
  4719. if (sscanf(buf, "%d", &attrs->nice) == 1 &&
  4720. attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
  4721. ret = apply_workqueue_attrs_locked(wq, attrs);
  4722. else
  4723. ret = -EINVAL;
  4724. out_unlock:
  4725. apply_wqattrs_unlock();
  4726. free_workqueue_attrs(attrs);
  4727. return ret ?: count;
  4728. }
  4729. static ssize_t wq_cpumask_show(struct device *dev,
  4730. struct device_attribute *attr, char *buf)
  4731. {
  4732. struct workqueue_struct *wq = dev_to_wq(dev);
  4733. int written;
  4734. mutex_lock(&wq->mutex);
  4735. written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
  4736. cpumask_pr_args(wq->unbound_attrs->cpumask));
  4737. mutex_unlock(&wq->mutex);
  4738. return written;
  4739. }
  4740. static ssize_t wq_cpumask_store(struct device *dev,
  4741. struct device_attribute *attr,
  4742. const char *buf, size_t count)
  4743. {
  4744. struct workqueue_struct *wq = dev_to_wq(dev);
  4745. struct workqueue_attrs *attrs;
  4746. int ret = -ENOMEM;
  4747. apply_wqattrs_lock();
  4748. attrs = wq_sysfs_prep_attrs(wq);
  4749. if (!attrs)
  4750. goto out_unlock;
  4751. ret = cpumask_parse(buf, attrs->cpumask);
  4752. if (!ret)
  4753. ret = apply_workqueue_attrs_locked(wq, attrs);
  4754. out_unlock:
  4755. apply_wqattrs_unlock();
  4756. free_workqueue_attrs(attrs);
  4757. return ret ?: count;
  4758. }
  4759. static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
  4760. char *buf)
  4761. {
  4762. struct workqueue_struct *wq = dev_to_wq(dev);
  4763. int written;
  4764. mutex_lock(&wq->mutex);
  4765. written = scnprintf(buf, PAGE_SIZE, "%d\n",
  4766. !wq->unbound_attrs->no_numa);
  4767. mutex_unlock(&wq->mutex);
  4768. return written;
  4769. }
  4770. static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
  4771. const char *buf, size_t count)
  4772. {
  4773. struct workqueue_struct *wq = dev_to_wq(dev);
  4774. struct workqueue_attrs *attrs;
  4775. int v, ret = -ENOMEM;
  4776. apply_wqattrs_lock();
  4777. attrs = wq_sysfs_prep_attrs(wq);
  4778. if (!attrs)
  4779. goto out_unlock;
  4780. ret = -EINVAL;
  4781. if (sscanf(buf, "%d", &v) == 1) {
  4782. attrs->no_numa = !v;
  4783. ret = apply_workqueue_attrs_locked(wq, attrs);
  4784. }
  4785. out_unlock:
  4786. apply_wqattrs_unlock();
  4787. free_workqueue_attrs(attrs);
  4788. return ret ?: count;
  4789. }
  4790. static struct device_attribute wq_sysfs_unbound_attrs[] = {
  4791. __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
  4792. __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
  4793. __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
  4794. __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
  4795. __ATTR_NULL,
  4796. };
  4797. static struct bus_type wq_subsys = {
  4798. .name = "workqueue",
  4799. .dev_groups = wq_sysfs_groups,
  4800. };
  4801. static ssize_t wq_unbound_cpumask_show(struct device *dev,
  4802. struct device_attribute *attr, char *buf)
  4803. {
  4804. int written;
  4805. mutex_lock(&wq_pool_mutex);
  4806. written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
  4807. cpumask_pr_args(wq_unbound_cpumask));
  4808. mutex_unlock(&wq_pool_mutex);
  4809. return written;
  4810. }
  4811. static ssize_t wq_unbound_cpumask_store(struct device *dev,
  4812. struct device_attribute *attr, const char *buf, size_t count)
  4813. {
  4814. cpumask_var_t cpumask;
  4815. int ret;
  4816. if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
  4817. return -ENOMEM;
  4818. ret = cpumask_parse(buf, cpumask);
  4819. if (!ret)
  4820. ret = workqueue_set_unbound_cpumask(cpumask);
  4821. free_cpumask_var(cpumask);
  4822. return ret ? ret : count;
  4823. }
  4824. static struct device_attribute wq_sysfs_cpumask_attr =
  4825. __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
  4826. wq_unbound_cpumask_store);
  4827. static int __init wq_sysfs_init(void)
  4828. {
  4829. int err;
  4830. err = subsys_virtual_register(&wq_subsys, NULL);
  4831. if (err)
  4832. return err;
  4833. return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
  4834. }
  4835. core_initcall(wq_sysfs_init);
  4836. static void wq_device_release(struct device *dev)
  4837. {
  4838. struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
  4839. kfree(wq_dev);
  4840. }
  4841. /**
  4842. * workqueue_sysfs_register - make a workqueue visible in sysfs
  4843. * @wq: the workqueue to register
  4844. *
  4845. * Expose @wq in sysfs under /sys/bus/workqueue/devices.
  4846. * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
  4847. * which is the preferred method.
  4848. *
  4849. * Workqueue user should use this function directly iff it wants to apply
  4850. * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
  4851. * apply_workqueue_attrs() may race against userland updating the
  4852. * attributes.
  4853. *
  4854. * Return: 0 on success, -errno on failure.
  4855. */
  4856. int workqueue_sysfs_register(struct workqueue_struct *wq)
  4857. {
  4858. struct wq_device *wq_dev;
  4859. int ret;
  4860. /*
  4861. * Adjusting max_active or creating new pwqs by applying
  4862. * attributes breaks ordering guarantee. Disallow exposing ordered
  4863. * workqueues.
  4864. */
  4865. if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
  4866. return -EINVAL;
  4867. wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
  4868. if (!wq_dev)
  4869. return -ENOMEM;
  4870. wq_dev->wq = wq;
  4871. wq_dev->dev.bus = &wq_subsys;
  4872. wq_dev->dev.release = wq_device_release;
  4873. dev_set_name(&wq_dev->dev, "%s", wq->name);
  4874. /*
  4875. * unbound_attrs are created separately. Suppress uevent until
  4876. * everything is ready.
  4877. */
  4878. dev_set_uevent_suppress(&wq_dev->dev, true);
  4879. ret = device_register(&wq_dev->dev);
  4880. if (ret) {
  4881. put_device(&wq_dev->dev);
  4882. wq->wq_dev = NULL;
  4883. return ret;
  4884. }
  4885. if (wq->flags & WQ_UNBOUND) {
  4886. struct device_attribute *attr;
  4887. for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
  4888. ret = device_create_file(&wq_dev->dev, attr);
  4889. if (ret) {
  4890. device_unregister(&wq_dev->dev);
  4891. wq->wq_dev = NULL;
  4892. return ret;
  4893. }
  4894. }
  4895. }
  4896. dev_set_uevent_suppress(&wq_dev->dev, false);
  4897. kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
  4898. return 0;
  4899. }
  4900. /**
  4901. * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
  4902. * @wq: the workqueue to unregister
  4903. *
  4904. * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
  4905. */
  4906. static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
  4907. {
  4908. struct wq_device *wq_dev = wq->wq_dev;
  4909. if (!wq->wq_dev)
  4910. return;
  4911. wq->wq_dev = NULL;
  4912. device_unregister(&wq_dev->dev);
  4913. }
  4914. #else /* CONFIG_SYSFS */
  4915. static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
  4916. #endif /* CONFIG_SYSFS */
  4917. /*
  4918. * Workqueue watchdog.
  4919. *
  4920. * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
  4921. * flush dependency, a concurrency managed work item which stays RUNNING
  4922. * indefinitely. Workqueue stalls can be very difficult to debug as the
  4923. * usual warning mechanisms don't trigger and internal workqueue state is
  4924. * largely opaque.
  4925. *
  4926. * Workqueue watchdog monitors all worker pools periodically and dumps
  4927. * state if some pools failed to make forward progress for a while where
  4928. * forward progress is defined as the first item on ->worklist changing.
  4929. *
  4930. * This mechanism is controlled through the kernel parameter
  4931. * "workqueue.watchdog_thresh" which can be updated at runtime through the
  4932. * corresponding sysfs parameter file.
  4933. */
  4934. #ifdef CONFIG_WQ_WATCHDOG
  4935. static unsigned long wq_watchdog_thresh = 30;
  4936. static struct timer_list wq_watchdog_timer;
  4937. static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
  4938. static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
  4939. static void wq_watchdog_reset_touched(void)
  4940. {
  4941. int cpu;
  4942. wq_watchdog_touched = jiffies;
  4943. for_each_possible_cpu(cpu)
  4944. per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
  4945. }
  4946. static void wq_watchdog_timer_fn(struct timer_list *unused)
  4947. {
  4948. unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
  4949. bool lockup_detected = false;
  4950. unsigned long now = jiffies;
  4951. struct worker_pool *pool;
  4952. int pi;
  4953. if (!thresh)
  4954. return;
  4955. rcu_read_lock();
  4956. for_each_pool(pool, pi) {
  4957. unsigned long pool_ts, touched, ts;
  4958. if (list_empty(&pool->worklist))
  4959. continue;
  4960. /*
  4961. * If a virtual machine is stopped by the host it can look to
  4962. * the watchdog like a stall.
  4963. */
  4964. kvm_check_and_clear_guest_paused();
  4965. /* get the latest of pool and touched timestamps */
  4966. pool_ts = READ_ONCE(pool->watchdog_ts);
  4967. touched = READ_ONCE(wq_watchdog_touched);
  4968. if (time_after(pool_ts, touched))
  4969. ts = pool_ts;
  4970. else
  4971. ts = touched;
  4972. if (pool->cpu >= 0) {
  4973. unsigned long cpu_touched =
  4974. READ_ONCE(per_cpu(wq_watchdog_touched_cpu,
  4975. pool->cpu));
  4976. if (time_after(cpu_touched, ts))
  4977. ts = cpu_touched;
  4978. }
  4979. /* did we stall? */
  4980. if (time_after(now, ts + thresh)) {
  4981. lockup_detected = true;
  4982. pr_emerg("BUG: workqueue lockup - pool");
  4983. pr_cont_pool_info(pool);
  4984. pr_cont(" stuck for %us!\n",
  4985. jiffies_to_msecs(now - pool_ts) / 1000);
  4986. trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts);
  4987. }
  4988. }
  4989. rcu_read_unlock();
  4990. if (lockup_detected)
  4991. show_workqueue_state();
  4992. wq_watchdog_reset_touched();
  4993. mod_timer(&wq_watchdog_timer, jiffies + thresh);
  4994. }
  4995. notrace void wq_watchdog_touch(int cpu)
  4996. {
  4997. if (cpu >= 0)
  4998. per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
  4999. else
  5000. wq_watchdog_touched = jiffies;
  5001. }
  5002. static void wq_watchdog_set_thresh(unsigned long thresh)
  5003. {
  5004. wq_watchdog_thresh = 0;
  5005. del_timer_sync(&wq_watchdog_timer);
  5006. if (thresh) {
  5007. wq_watchdog_thresh = thresh;
  5008. wq_watchdog_reset_touched();
  5009. mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
  5010. }
  5011. }
  5012. static int wq_watchdog_param_set_thresh(const char *val,
  5013. const struct kernel_param *kp)
  5014. {
  5015. unsigned long thresh;
  5016. int ret;
  5017. ret = kstrtoul(val, 0, &thresh);
  5018. if (ret)
  5019. return ret;
  5020. if (system_wq)
  5021. wq_watchdog_set_thresh(thresh);
  5022. else
  5023. wq_watchdog_thresh = thresh;
  5024. return 0;
  5025. }
  5026. static const struct kernel_param_ops wq_watchdog_thresh_ops = {
  5027. .set = wq_watchdog_param_set_thresh,
  5028. .get = param_get_ulong,
  5029. };
  5030. module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
  5031. 0644);
  5032. static void wq_watchdog_init(void)
  5033. {
  5034. timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
  5035. wq_watchdog_set_thresh(wq_watchdog_thresh);
  5036. }
  5037. #else /* CONFIG_WQ_WATCHDOG */
  5038. static inline void wq_watchdog_init(void) { }
  5039. #endif /* CONFIG_WQ_WATCHDOG */
  5040. static void __init wq_numa_init(void)
  5041. {
  5042. cpumask_var_t *tbl;
  5043. int node, cpu;
  5044. if (num_possible_nodes() <= 1)
  5045. return;
  5046. if (wq_disable_numa) {
  5047. pr_info("workqueue: NUMA affinity support disabled\n");
  5048. return;
  5049. }
  5050. for_each_possible_cpu(cpu) {
  5051. if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
  5052. pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
  5053. return;
  5054. }
  5055. }
  5056. wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
  5057. BUG_ON(!wq_update_unbound_numa_attrs_buf);
  5058. /*
  5059. * We want masks of possible CPUs of each node which isn't readily
  5060. * available. Build one from cpu_to_node() which should have been
  5061. * fully initialized by now.
  5062. */
  5063. tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);
  5064. BUG_ON(!tbl);
  5065. for_each_node(node)
  5066. BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
  5067. node_online(node) ? node : NUMA_NO_NODE));
  5068. for_each_possible_cpu(cpu) {
  5069. node = cpu_to_node(cpu);
  5070. cpumask_set_cpu(cpu, tbl[node]);
  5071. }
  5072. wq_numa_possible_cpumask = tbl;
  5073. wq_numa_enabled = true;
  5074. }
  5075. /**
  5076. * workqueue_init_early - early init for workqueue subsystem
  5077. *
  5078. * This is the first half of two-staged workqueue subsystem initialization
  5079. * and invoked as soon as the bare basics - memory allocation, cpumasks and
  5080. * idr are up. It sets up all the data structures and system workqueues
  5081. * and allows early boot code to create workqueues and queue/cancel work
  5082. * items. Actual work item execution starts only after kthreads can be
  5083. * created and scheduled right before early initcalls.
  5084. */
  5085. void __init workqueue_init_early(void)
  5086. {
  5087. int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
  5088. int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;
  5089. int i, cpu;
  5090. BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
  5091. BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
  5092. cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));
  5093. pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
  5094. /* initialize CPU pools */
  5095. for_each_possible_cpu(cpu) {
  5096. struct worker_pool *pool;
  5097. i = 0;
  5098. for_each_cpu_worker_pool(pool, cpu) {
  5099. BUG_ON(init_worker_pool(pool));
  5100. pool->cpu = cpu;
  5101. cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
  5102. pool->attrs->nice = std_nice[i++];
  5103. pool->node = cpu_to_node(cpu);
  5104. /* alloc pool ID */
  5105. mutex_lock(&wq_pool_mutex);
  5106. BUG_ON(worker_pool_assign_id(pool));
  5107. mutex_unlock(&wq_pool_mutex);
  5108. }
  5109. }
  5110. /* create default unbound and ordered wq attrs */
  5111. for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
  5112. struct workqueue_attrs *attrs;
  5113. BUG_ON(!(attrs = alloc_workqueue_attrs()));
  5114. attrs->nice = std_nice[i];
  5115. unbound_std_wq_attrs[i] = attrs;
  5116. /*
  5117. * An ordered wq should have only one pwq as ordering is
  5118. * guaranteed by max_active which is enforced by pwqs.
  5119. * Turn off NUMA so that dfl_pwq is used for all nodes.
  5120. */
  5121. BUG_ON(!(attrs = alloc_workqueue_attrs()));
  5122. attrs->nice = std_nice[i];
  5123. attrs->no_numa = true;
  5124. ordered_wq_attrs[i] = attrs;
  5125. }
  5126. system_wq = alloc_workqueue("events", 0, 0);
  5127. system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
  5128. system_long_wq = alloc_workqueue("events_long", 0, 0);
  5129. system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
  5130. WQ_UNBOUND_MAX_ACTIVE);
  5131. system_freezable_wq = alloc_workqueue("events_freezable",
  5132. WQ_FREEZABLE, 0);
  5133. system_power_efficient_wq = alloc_workqueue("events_power_efficient",
  5134. WQ_POWER_EFFICIENT, 0);
  5135. system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
  5136. WQ_FREEZABLE | WQ_POWER_EFFICIENT,
  5137. 0);
  5138. BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
  5139. !system_unbound_wq || !system_freezable_wq ||
  5140. !system_power_efficient_wq ||
  5141. !system_freezable_power_efficient_wq);
  5142. }
  5143. /**
  5144. * workqueue_init - bring workqueue subsystem fully online
  5145. *
  5146. * This is the latter half of two-staged workqueue subsystem initialization
  5147. * and invoked as soon as kthreads can be created and scheduled.
  5148. * Workqueues have been created and work items queued on them, but there
  5149. * are no kworkers executing the work items yet. Populate the worker pools
  5150. * with the initial workers and enable future kworker creations.
  5151. */
  5152. void __init workqueue_init(void)
  5153. {
  5154. struct workqueue_struct *wq;
  5155. struct worker_pool *pool;
  5156. int cpu, bkt;
  5157. /*
  5158. * It'd be simpler to initialize NUMA in workqueue_init_early() but
  5159. * CPU to node mapping may not be available that early on some
  5160. * archs such as power and arm64. As per-cpu pools created
  5161. * previously could be missing node hint and unbound pools NUMA
  5162. * affinity, fix them up.
  5163. *
  5164. * Also, while iterating workqueues, create rescuers if requested.
  5165. */
  5166. wq_numa_init();
  5167. mutex_lock(&wq_pool_mutex);
  5168. for_each_possible_cpu(cpu) {
  5169. for_each_cpu_worker_pool(pool, cpu) {
  5170. pool->node = cpu_to_node(cpu);
  5171. }
  5172. }
  5173. list_for_each_entry(wq, &workqueues, list) {
  5174. wq_update_unbound_numa(wq, smp_processor_id(), true);
  5175. WARN(init_rescuer(wq),
  5176. "workqueue: failed to create early rescuer for %s",
  5177. wq->name);
  5178. }
  5179. mutex_unlock(&wq_pool_mutex);
  5180. /* create the initial workers */
  5181. for_each_online_cpu(cpu) {
  5182. for_each_cpu_worker_pool(pool, cpu) {
  5183. pool->flags &= ~POOL_DISASSOCIATED;
  5184. BUG_ON(!create_worker(pool));
  5185. }
  5186. }
  5187. hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
  5188. BUG_ON(!create_worker(pool));
  5189. wq_online = true;
  5190. wq_watchdog_init();
  5191. }