cpuset.c 103 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715
  1. /*
  2. * kernel/cpuset.c
  3. *
  4. * Processor and Memory placement constraints for sets of tasks.
  5. *
  6. * Copyright (C) 2003 BULL SA.
  7. * Copyright (C) 2004-2007 Silicon Graphics, Inc.
  8. * Copyright (C) 2006 Google, Inc
  9. *
  10. * Portions derived from Patrick Mochel's sysfs code.
  11. * sysfs is Copyright (c) 2001-3 Patrick Mochel
  12. *
  13. * 2003-10-10 Written by Simon Derr.
  14. * 2003-10-22 Updates by Stephen Hemminger.
  15. * 2004 May-July Rework by Paul Jackson.
  16. * 2006 Rework by Paul Menage to use generic cgroups
  17. * 2008 Rework of the scheduler domains and CPU hotplug handling
  18. * by Max Krasnyansky
  19. *
  20. * This file is subject to the terms and conditions of the GNU General Public
  21. * License. See the file COPYING in the main directory of the Linux
  22. * distribution for more details.
  23. */
  24. #include <linux/cpu.h>
  25. #include <linux/cpumask.h>
  26. #include <linux/cpuset.h>
  27. #include <linux/err.h>
  28. #include <linux/errno.h>
  29. #include <linux/file.h>
  30. #include <linux/fs.h>
  31. #include <linux/init.h>
  32. #include <linux/interrupt.h>
  33. #include <linux/kernel.h>
  34. #include <linux/kmod.h>
  35. #include <linux/list.h>
  36. #include <linux/mempolicy.h>
  37. #include <linux/mm.h>
  38. #include <linux/memory.h>
  39. #include <linux/export.h>
  40. #include <linux/mount.h>
  41. #include <linux/fs_context.h>
  42. #include <linux/namei.h>
  43. #include <linux/pagemap.h>
  44. #include <linux/proc_fs.h>
  45. #include <linux/rcupdate.h>
  46. #include <linux/sched.h>
  47. #include <linux/sched/deadline.h>
  48. #include <linux/sched/mm.h>
  49. #include <linux/sched/task.h>
  50. #include <linux/seq_file.h>
  51. #include <linux/security.h>
  52. #include <linux/slab.h>
  53. #include <linux/spinlock.h>
  54. #include <linux/stat.h>
  55. #include <linux/string.h>
  56. #include <linux/time.h>
  57. #include <linux/time64.h>
  58. #include <linux/backing-dev.h>
  59. #include <linux/sort.h>
  60. #include <linux/oom.h>
  61. #include <linux/sched/isolation.h>
  62. #include <linux/uaccess.h>
  63. #include <linux/atomic.h>
  64. #include <linux/mutex.h>
  65. #include <linux/cgroup.h>
  66. #include <linux/wait.h>
  67. #include <trace/hooks/sched.h>
  68. #include <trace/hooks/cgroup.h>
  69. DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
  70. DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
  71. /* See "Frequency meter" comments, below. */
  72. struct fmeter {
  73. int cnt; /* unprocessed events count */
  74. int val; /* most recent output value */
  75. time64_t time; /* clock (secs) when val computed */
  76. spinlock_t lock; /* guards read or write of above */
  77. };
  78. struct cpuset {
  79. struct cgroup_subsys_state css;
  80. unsigned long flags; /* "unsigned long" so bitops work */
  81. /*
  82. * On default hierarchy:
  83. *
  84. * The user-configured masks can only be changed by writing to
  85. * cpuset.cpus and cpuset.mems, and won't be limited by the
  86. * parent masks.
  87. *
  88. * The effective masks is the real masks that apply to the tasks
  89. * in the cpuset. They may be changed if the configured masks are
  90. * changed or hotplug happens.
  91. *
  92. * effective_mask == configured_mask & parent's effective_mask,
  93. * and if it ends up empty, it will inherit the parent's mask.
  94. *
  95. *
  96. * On legacy hierachy:
  97. *
  98. * The user-configured masks are always the same with effective masks.
  99. */
  100. /* user-configured CPUs and Memory Nodes allow to tasks */
  101. cpumask_var_t cpus_allowed;
  102. cpumask_var_t cpus_requested;
  103. nodemask_t mems_allowed;
  104. /* effective CPUs and Memory Nodes allow to tasks */
  105. cpumask_var_t effective_cpus;
  106. nodemask_t effective_mems;
  107. /*
  108. * CPUs allocated to child sub-partitions (default hierarchy only)
  109. * - CPUs granted by the parent = effective_cpus U subparts_cpus
  110. * - effective_cpus and subparts_cpus are mutually exclusive.
  111. *
  112. * effective_cpus contains only onlined CPUs, but subparts_cpus
  113. * may have offlined ones.
  114. */
  115. cpumask_var_t subparts_cpus;
  116. /*
  117. * This is old Memory Nodes tasks took on.
  118. *
  119. * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
  120. * - A new cpuset's old_mems_allowed is initialized when some
  121. * task is moved into it.
  122. * - old_mems_allowed is used in cpuset_migrate_mm() when we change
  123. * cpuset.mems_allowed and have tasks' nodemask updated, and
  124. * then old_mems_allowed is updated to mems_allowed.
  125. */
  126. nodemask_t old_mems_allowed;
  127. struct fmeter fmeter; /* memory_pressure filter */
  128. /*
  129. * Tasks are being attached to this cpuset. Used to prevent
  130. * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
  131. */
  132. int attach_in_progress;
  133. /* partition number for rebuild_sched_domains() */
  134. int pn;
  135. /* for custom sched domain */
  136. int relax_domain_level;
  137. /* number of CPUs in subparts_cpus */
  138. int nr_subparts_cpus;
  139. /* partition root state */
  140. int partition_root_state;
  141. /*
  142. * Default hierarchy only:
  143. * use_parent_ecpus - set if using parent's effective_cpus
  144. * child_ecpus_count - # of children with use_parent_ecpus set
  145. */
  146. int use_parent_ecpus;
  147. int child_ecpus_count;
  148. };
  149. /*
  150. * Partition root states:
  151. *
  152. * 0 - not a partition root
  153. *
  154. * 1 - partition root
  155. *
  156. * -1 - invalid partition root
  157. * None of the cpus in cpus_allowed can be put into the parent's
  158. * subparts_cpus. In this case, the cpuset is not a real partition
  159. * root anymore. However, the CPU_EXCLUSIVE bit will still be set
  160. * and the cpuset can be restored back to a partition root if the
  161. * parent cpuset can give more CPUs back to this child cpuset.
  162. */
  163. #define PRS_DISABLED 0
  164. #define PRS_ENABLED 1
  165. #define PRS_ERROR -1
  166. /*
  167. * Temporary cpumasks for working with partitions that are passed among
  168. * functions to avoid memory allocation in inner functions.
  169. */
  170. struct tmpmasks {
  171. cpumask_var_t addmask, delmask; /* For partition root */
  172. cpumask_var_t new_cpus; /* For update_cpumasks_hier() */
  173. };
  174. static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
  175. {
  176. return css ? container_of(css, struct cpuset, css) : NULL;
  177. }
  178. /* Retrieve the cpuset for a task */
  179. static inline struct cpuset *task_cs(struct task_struct *task)
  180. {
  181. return css_cs(task_css(task, cpuset_cgrp_id));
  182. }
  183. static inline struct cpuset *parent_cs(struct cpuset *cs)
  184. {
  185. return css_cs(cs->css.parent);
  186. }
  187. /* bits in struct cpuset flags field */
  188. typedef enum {
  189. CS_ONLINE,
  190. CS_CPU_EXCLUSIVE,
  191. CS_MEM_EXCLUSIVE,
  192. CS_MEM_HARDWALL,
  193. CS_MEMORY_MIGRATE,
  194. CS_SCHED_LOAD_BALANCE,
  195. CS_SPREAD_PAGE,
  196. CS_SPREAD_SLAB,
  197. } cpuset_flagbits_t;
  198. /* convenient tests for these bits */
  199. static inline bool is_cpuset_online(struct cpuset *cs)
  200. {
  201. return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css);
  202. }
  203. static inline int is_cpu_exclusive(const struct cpuset *cs)
  204. {
  205. return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
  206. }
  207. static inline int is_mem_exclusive(const struct cpuset *cs)
  208. {
  209. return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
  210. }
  211. static inline int is_mem_hardwall(const struct cpuset *cs)
  212. {
  213. return test_bit(CS_MEM_HARDWALL, &cs->flags);
  214. }
  215. static inline int is_sched_load_balance(const struct cpuset *cs)
  216. {
  217. return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
  218. }
  219. static inline int is_memory_migrate(const struct cpuset *cs)
  220. {
  221. return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
  222. }
  223. static inline int is_spread_page(const struct cpuset *cs)
  224. {
  225. return test_bit(CS_SPREAD_PAGE, &cs->flags);
  226. }
  227. static inline int is_spread_slab(const struct cpuset *cs)
  228. {
  229. return test_bit(CS_SPREAD_SLAB, &cs->flags);
  230. }
  231. static inline int is_partition_root(const struct cpuset *cs)
  232. {
  233. return cs->partition_root_state > 0;
  234. }
  235. static struct cpuset top_cpuset = {
  236. .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
  237. (1 << CS_MEM_EXCLUSIVE)),
  238. .partition_root_state = PRS_ENABLED,
  239. };
  240. /**
  241. * cpuset_for_each_child - traverse online children of a cpuset
  242. * @child_cs: loop cursor pointing to the current child
  243. * @pos_css: used for iteration
  244. * @parent_cs: target cpuset to walk children of
  245. *
  246. * Walk @child_cs through the online children of @parent_cs. Must be used
  247. * with RCU read locked.
  248. */
  249. #define cpuset_for_each_child(child_cs, pos_css, parent_cs) \
  250. css_for_each_child((pos_css), &(parent_cs)->css) \
  251. if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
  252. /**
  253. * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
  254. * @des_cs: loop cursor pointing to the current descendant
  255. * @pos_css: used for iteration
  256. * @root_cs: target cpuset to walk ancestor of
  257. *
  258. * Walk @des_cs through the online descendants of @root_cs. Must be used
  259. * with RCU read locked. The caller may modify @pos_css by calling
  260. * css_rightmost_descendant() to skip subtree. @root_cs is included in the
  261. * iteration and the first node to be visited.
  262. */
  263. #define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \
  264. css_for_each_descendant_pre((pos_css), &(root_cs)->css) \
  265. if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
  266. /*
  267. * There are two global locks guarding cpuset structures - cpuset_mutex and
  268. * callback_lock. We also require taking task_lock() when dereferencing a
  269. * task's cpuset pointer. See "The task_lock() exception", at the end of this
  270. * comment.
  271. *
  272. * A task must hold both locks to modify cpusets. If a task holds
  273. * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it
  274. * is the only task able to also acquire callback_lock and be able to
  275. * modify cpusets. It can perform various checks on the cpuset structure
  276. * first, knowing nothing will change. It can also allocate memory while
  277. * just holding cpuset_mutex. While it is performing these checks, various
  278. * callback routines can briefly acquire callback_lock to query cpusets.
  279. * Once it is ready to make the changes, it takes callback_lock, blocking
  280. * everyone else.
  281. *
  282. * Calls to the kernel memory allocator can not be made while holding
  283. * callback_lock, as that would risk double tripping on callback_lock
  284. * from one of the callbacks into the cpuset code from within
  285. * __alloc_pages().
  286. *
  287. * If a task is only holding callback_lock, then it has read-only
  288. * access to cpusets.
  289. *
  290. * Now, the task_struct fields mems_allowed and mempolicy may be changed
  291. * by other task, we use alloc_lock in the task_struct fields to protect
  292. * them.
  293. *
  294. * The cpuset_common_file_read() handlers only hold callback_lock across
  295. * small pieces of code, such as when reading out possibly multi-word
  296. * cpumasks and nodemasks.
  297. *
  298. * Accessing a task's cpuset should be done in accordance with the
  299. * guidelines for accessing subsystem state in kernel/cgroup.c
  300. */
  301. static DEFINE_MUTEX(cpuset_mutex);
  302. static DEFINE_SPINLOCK(callback_lock);
  303. static struct workqueue_struct *cpuset_migrate_mm_wq;
  304. /*
  305. * CPU / memory hotplug is handled asynchronously
  306. * for hotplug, synchronously for resume_cpus
  307. */
  308. static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);
  309. static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
  310. /*
  311. * Cgroup v2 behavior is used on the "cpus" and "mems" control files when
  312. * on default hierarchy or when the cpuset_v2_mode flag is set by mounting
  313. * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option.
  314. * With v2 behavior, "cpus" and "mems" are always what the users have
  315. * requested and won't be changed by hotplug events. Only the effective
  316. * cpus or mems will be affected.
  317. */
  318. static inline bool is_in_v2_mode(void)
  319. {
  320. return cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
  321. (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE);
  322. }
  323. /*
  324. * Return in pmask the portion of a task's cpusets's cpus_allowed that
  325. * are online and are capable of running the task. If none are found,
  326. * walk up the cpuset hierarchy until we find one that does have some
  327. * appropriate cpus.
  328. *
  329. * One way or another, we guarantee to return some non-empty subset
  330. * of cpu_active_mask.
  331. *
  332. * Call with callback_lock or cpuset_mutex held.
  333. */
  334. static void guarantee_online_cpus(struct task_struct *tsk,
  335. struct cpumask *pmask)
  336. {
  337. const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
  338. struct cpuset *cs;
  339. if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_active_mask)))
  340. cpumask_copy(pmask, cpu_active_mask);
  341. rcu_read_lock();
  342. cs = task_cs(tsk);
  343. while (!cpumask_intersects(cs->effective_cpus, pmask)) {
  344. cs = parent_cs(cs);
  345. if (unlikely(!cs)) {
  346. /*
  347. * The top cpuset doesn't have any online cpu as a
  348. * consequence of a race between cpuset_hotplug_work
  349. * and cpu hotplug notifier. But we know the top
  350. * cpuset's effective_cpus is on its way to be
  351. * identical to cpu_online_mask.
  352. */
  353. goto out_unlock;
  354. }
  355. }
  356. cpumask_and(pmask, pmask, cs->effective_cpus);
  357. out_unlock:
  358. rcu_read_unlock();
  359. }
  360. /*
  361. * Return in *pmask the portion of a cpusets's mems_allowed that
  362. * are online, with memory. If none are online with memory, walk
  363. * up the cpuset hierarchy until we find one that does have some
  364. * online mems. The top cpuset always has some mems online.
  365. *
  366. * One way or another, we guarantee to return some non-empty subset
  367. * of node_states[N_MEMORY].
  368. *
  369. * Call with callback_lock or cpuset_mutex held.
  370. */
  371. static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
  372. {
  373. while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
  374. cs = parent_cs(cs);
  375. nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
  376. }
  377. /*
  378. * update task's spread flag if cpuset's page/slab spread flag is set
  379. *
  380. * Call with callback_lock or cpuset_mutex held.
  381. */
  382. static void cpuset_update_task_spread_flag(struct cpuset *cs,
  383. struct task_struct *tsk)
  384. {
  385. if (is_spread_page(cs))
  386. task_set_spread_page(tsk);
  387. else
  388. task_clear_spread_page(tsk);
  389. if (is_spread_slab(cs))
  390. task_set_spread_slab(tsk);
  391. else
  392. task_clear_spread_slab(tsk);
  393. }
  394. /*
  395. * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
  396. *
  397. * One cpuset is a subset of another if all its allowed CPUs and
  398. * Memory Nodes are a subset of the other, and its exclusive flags
  399. * are only set if the other's are set. Call holding cpuset_mutex.
  400. */
  401. static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
  402. {
  403. return cpumask_subset(p->cpus_requested, q->cpus_requested) &&
  404. nodes_subset(p->mems_allowed, q->mems_allowed) &&
  405. is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
  406. is_mem_exclusive(p) <= is_mem_exclusive(q);
  407. }
  408. /**
  409. * alloc_cpumasks - allocate three cpumasks for cpuset
  410. * @cs: the cpuset that have cpumasks to be allocated.
  411. * @tmp: the tmpmasks structure pointer
  412. * Return: 0 if successful, -ENOMEM otherwise.
  413. *
  414. * Only one of the two input arguments should be non-NULL.
  415. */
  416. static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
  417. {
  418. cpumask_var_t *pmask1, *pmask2, *pmask3;
  419. if (cs) {
  420. pmask1 = &cs->cpus_allowed;
  421. pmask2 = &cs->effective_cpus;
  422. pmask3 = &cs->subparts_cpus;
  423. } else {
  424. pmask1 = &tmp->new_cpus;
  425. pmask2 = &tmp->addmask;
  426. pmask3 = &tmp->delmask;
  427. }
  428. if (!zalloc_cpumask_var(pmask1, GFP_KERNEL))
  429. return -ENOMEM;
  430. if (!zalloc_cpumask_var(pmask2, GFP_KERNEL))
  431. goto free_one;
  432. if (!zalloc_cpumask_var(pmask3, GFP_KERNEL))
  433. goto free_two;
  434. if (cs && !zalloc_cpumask_var(&cs->cpus_requested, GFP_KERNEL))
  435. goto free_three;
  436. return 0;
  437. free_three:
  438. free_cpumask_var(*pmask3);
  439. free_two:
  440. free_cpumask_var(*pmask2);
  441. free_one:
  442. free_cpumask_var(*pmask1);
  443. return -ENOMEM;
  444. }
  445. /**
  446. * free_cpumasks - free cpumasks in a tmpmasks structure
  447. * @cs: the cpuset that have cpumasks to be free.
  448. * @tmp: the tmpmasks structure pointer
  449. */
  450. static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
  451. {
  452. if (cs) {
  453. free_cpumask_var(cs->cpus_allowed);
  454. free_cpumask_var(cs->cpus_requested);
  455. free_cpumask_var(cs->effective_cpus);
  456. free_cpumask_var(cs->subparts_cpus);
  457. }
  458. if (tmp) {
  459. free_cpumask_var(tmp->new_cpus);
  460. free_cpumask_var(tmp->addmask);
  461. free_cpumask_var(tmp->delmask);
  462. }
  463. }
  464. /**
  465. * alloc_trial_cpuset - allocate a trial cpuset
  466. * @cs: the cpuset that the trial cpuset duplicates
  467. */
  468. static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
  469. {
  470. struct cpuset *trial;
  471. trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
  472. if (!trial)
  473. return NULL;
  474. if (alloc_cpumasks(trial, NULL)) {
  475. kfree(trial);
  476. return NULL;
  477. }
  478. cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
  479. cpumask_copy(trial->cpus_requested, cs->cpus_requested);
  480. cpumask_copy(trial->effective_cpus, cs->effective_cpus);
  481. return trial;
  482. }
  483. /**
  484. * free_cpuset - free the cpuset
  485. * @cs: the cpuset to be freed
  486. */
  487. static inline void free_cpuset(struct cpuset *cs)
  488. {
  489. free_cpumasks(cs, NULL);
  490. kfree(cs);
  491. }
  492. /*
  493. * validate_change() - Used to validate that any proposed cpuset change
  494. * follows the structural rules for cpusets.
  495. *
  496. * If we replaced the flag and mask values of the current cpuset
  497. * (cur) with those values in the trial cpuset (trial), would
  498. * our various subset and exclusive rules still be valid? Presumes
  499. * cpuset_mutex held.
  500. *
  501. * 'cur' is the address of an actual, in-use cpuset. Operations
  502. * such as list traversal that depend on the actual address of the
  503. * cpuset in the list must use cur below, not trial.
  504. *
  505. * 'trial' is the address of bulk structure copy of cur, with
  506. * perhaps one or more of the fields cpus_allowed, mems_allowed,
  507. * or flags changed to new, trial values.
  508. *
  509. * Return 0 if valid, -errno if not.
  510. */
  511. static int validate_change(struct cpuset *cur, struct cpuset *trial)
  512. {
  513. struct cgroup_subsys_state *css;
  514. struct cpuset *c, *par;
  515. int ret;
  516. rcu_read_lock();
  517. /* Each of our child cpusets must be a subset of us */
  518. ret = -EBUSY;
  519. cpuset_for_each_child(c, css, cur)
  520. if (!is_cpuset_subset(c, trial))
  521. goto out;
  522. /* Remaining checks don't apply to root cpuset */
  523. ret = 0;
  524. if (cur == &top_cpuset)
  525. goto out;
  526. par = parent_cs(cur);
  527. /* On legacy hiearchy, we must be a subset of our parent cpuset. */
  528. ret = -EACCES;
  529. if (!is_in_v2_mode() && !is_cpuset_subset(trial, par))
  530. goto out;
  531. /*
  532. * If either I or some sibling (!= me) is exclusive, we can't
  533. * overlap
  534. */
  535. ret = -EINVAL;
  536. cpuset_for_each_child(c, css, par) {
  537. if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
  538. c != cur &&
  539. cpumask_intersects(trial->cpus_requested, c->cpus_requested))
  540. goto out;
  541. if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
  542. c != cur &&
  543. nodes_intersects(trial->mems_allowed, c->mems_allowed))
  544. goto out;
  545. }
  546. /*
  547. * Cpusets with tasks - existing or newly being attached - can't
  548. * be changed to have empty cpus_allowed or mems_allowed.
  549. */
  550. ret = -ENOSPC;
  551. if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) {
  552. if (!cpumask_empty(cur->cpus_allowed) &&
  553. cpumask_empty(trial->cpus_allowed))
  554. goto out;
  555. if (!nodes_empty(cur->mems_allowed) &&
  556. nodes_empty(trial->mems_allowed))
  557. goto out;
  558. }
  559. /*
  560. * We can't shrink if we won't have enough room for SCHED_DEADLINE
  561. * tasks.
  562. */
  563. ret = -EBUSY;
  564. if (is_cpu_exclusive(cur) &&
  565. !cpuset_cpumask_can_shrink(cur->cpus_allowed,
  566. trial->cpus_allowed))
  567. goto out;
  568. ret = 0;
  569. out:
  570. rcu_read_unlock();
  571. return ret;
  572. }
  573. #ifdef CONFIG_SMP
  574. /*
  575. * Helper routine for generate_sched_domains().
  576. * Do cpusets a, b have overlapping effective cpus_allowed masks?
  577. */
  578. static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
  579. {
  580. return cpumask_intersects(a->effective_cpus, b->effective_cpus);
  581. }
  582. static void
  583. update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
  584. {
  585. if (dattr->relax_domain_level < c->relax_domain_level)
  586. dattr->relax_domain_level = c->relax_domain_level;
  587. return;
  588. }
  589. static void update_domain_attr_tree(struct sched_domain_attr *dattr,
  590. struct cpuset *root_cs)
  591. {
  592. struct cpuset *cp;
  593. struct cgroup_subsys_state *pos_css;
  594. rcu_read_lock();
  595. cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
  596. /* skip the whole subtree if @cp doesn't have any CPU */
  597. if (cpumask_empty(cp->cpus_allowed)) {
  598. pos_css = css_rightmost_descendant(pos_css);
  599. continue;
  600. }
  601. if (is_sched_load_balance(cp))
  602. update_domain_attr(dattr, cp);
  603. }
  604. rcu_read_unlock();
  605. }
  606. /* Must be called with cpuset_mutex held. */
  607. static inline int nr_cpusets(void)
  608. {
  609. /* jump label reference count + the top-level cpuset */
  610. return static_key_count(&cpusets_enabled_key.key) + 1;
  611. }
  612. /*
  613. * generate_sched_domains()
  614. *
  615. * This function builds a partial partition of the systems CPUs
  616. * A 'partial partition' is a set of non-overlapping subsets whose
  617. * union is a subset of that set.
  618. * The output of this function needs to be passed to kernel/sched/core.c
  619. * partition_sched_domains() routine, which will rebuild the scheduler's
  620. * load balancing domains (sched domains) as specified by that partial
  621. * partition.
  622. *
  623. * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
  624. * for a background explanation of this.
  625. *
  626. * Does not return errors, on the theory that the callers of this
  627. * routine would rather not worry about failures to rebuild sched
  628. * domains when operating in the severe memory shortage situations
  629. * that could cause allocation failures below.
  630. *
  631. * Must be called with cpuset_mutex held.
  632. *
  633. * The three key local variables below are:
  634. * cp - cpuset pointer, used (together with pos_css) to perform a
  635. * top-down scan of all cpusets. For our purposes, rebuilding
  636. * the schedulers sched domains, we can ignore !is_sched_load_
  637. * balance cpusets.
  638. * csa - (for CpuSet Array) Array of pointers to all the cpusets
  639. * that need to be load balanced, for convenient iterative
  640. * access by the subsequent code that finds the best partition,
  641. * i.e the set of domains (subsets) of CPUs such that the
  642. * cpus_allowed of every cpuset marked is_sched_load_balance
  643. * is a subset of one of these domains, while there are as
  644. * many such domains as possible, each as small as possible.
  645. * doms - Conversion of 'csa' to an array of cpumasks, for passing to
  646. * the kernel/sched/core.c routine partition_sched_domains() in a
  647. * convenient format, that can be easily compared to the prior
  648. * value to determine what partition elements (sched domains)
  649. * were changed (added or removed.)
  650. *
  651. * Finding the best partition (set of domains):
  652. * The triple nested loops below over i, j, k scan over the
  653. * load balanced cpusets (using the array of cpuset pointers in
  654. * csa[]) looking for pairs of cpusets that have overlapping
  655. * cpus_allowed, but which don't have the same 'pn' partition
  656. * number and gives them in the same partition number. It keeps
  657. * looping on the 'restart' label until it can no longer find
  658. * any such pairs.
  659. *
  660. * The union of the cpus_allowed masks from the set of
  661. * all cpusets having the same 'pn' value then form the one
  662. * element of the partition (one sched domain) to be passed to
  663. * partition_sched_domains().
  664. */
  665. static int generate_sched_domains(cpumask_var_t **domains,
  666. struct sched_domain_attr **attributes)
  667. {
  668. struct cpuset *cp; /* top-down scan of cpusets */
  669. struct cpuset **csa; /* array of all cpuset ptrs */
  670. int csn; /* how many cpuset ptrs in csa so far */
  671. int i, j, k; /* indices for partition finding loops */
  672. cpumask_var_t *doms; /* resulting partition; i.e. sched domains */
  673. struct sched_domain_attr *dattr; /* attributes for custom domains */
  674. int ndoms = 0; /* number of sched domains in result */
  675. int nslot; /* next empty doms[] struct cpumask slot */
  676. struct cgroup_subsys_state *pos_css;
  677. bool root_load_balance = is_sched_load_balance(&top_cpuset);
  678. doms = NULL;
  679. dattr = NULL;
  680. csa = NULL;
  681. /* Special case for the 99% of systems with one, full, sched domain */
  682. if (root_load_balance && !top_cpuset.nr_subparts_cpus) {
  683. ndoms = 1;
  684. doms = alloc_sched_domains(ndoms);
  685. if (!doms)
  686. goto done;
  687. dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
  688. if (dattr) {
  689. *dattr = SD_ATTR_INIT;
  690. update_domain_attr_tree(dattr, &top_cpuset);
  691. }
  692. cpumask_and(doms[0], top_cpuset.effective_cpus,
  693. housekeeping_cpumask(HK_FLAG_DOMAIN));
  694. goto done;
  695. }
  696. csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL);
  697. if (!csa)
  698. goto done;
  699. csn = 0;
  700. rcu_read_lock();
  701. if (root_load_balance)
  702. csa[csn++] = &top_cpuset;
  703. cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
  704. if (cp == &top_cpuset)
  705. continue;
  706. /*
  707. * Continue traversing beyond @cp iff @cp has some CPUs and
  708. * isn't load balancing. The former is obvious. The
  709. * latter: All child cpusets contain a subset of the
  710. * parent's cpus, so just skip them, and then we call
  711. * update_domain_attr_tree() to calc relax_domain_level of
  712. * the corresponding sched domain.
  713. *
  714. * If root is load-balancing, we can skip @cp if it
  715. * is a subset of the root's effective_cpus.
  716. */
  717. if (!cpumask_empty(cp->cpus_allowed) &&
  718. !(is_sched_load_balance(cp) &&
  719. cpumask_intersects(cp->cpus_allowed,
  720. housekeeping_cpumask(HK_FLAG_DOMAIN))))
  721. continue;
  722. if (root_load_balance &&
  723. cpumask_subset(cp->cpus_allowed, top_cpuset.effective_cpus))
  724. continue;
  725. if (is_sched_load_balance(cp) &&
  726. !cpumask_empty(cp->effective_cpus))
  727. csa[csn++] = cp;
  728. /* skip @cp's subtree if not a partition root */
  729. if (!is_partition_root(cp))
  730. pos_css = css_rightmost_descendant(pos_css);
  731. }
  732. rcu_read_unlock();
  733. for (i = 0; i < csn; i++)
  734. csa[i]->pn = i;
  735. ndoms = csn;
  736. restart:
  737. /* Find the best partition (set of sched domains) */
  738. for (i = 0; i < csn; i++) {
  739. struct cpuset *a = csa[i];
  740. int apn = a->pn;
  741. for (j = 0; j < csn; j++) {
  742. struct cpuset *b = csa[j];
  743. int bpn = b->pn;
  744. if (apn != bpn && cpusets_overlap(a, b)) {
  745. for (k = 0; k < csn; k++) {
  746. struct cpuset *c = csa[k];
  747. if (c->pn == bpn)
  748. c->pn = apn;
  749. }
  750. ndoms--; /* one less element */
  751. goto restart;
  752. }
  753. }
  754. }
  755. /*
  756. * Now we know how many domains to create.
  757. * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
  758. */
  759. doms = alloc_sched_domains(ndoms);
  760. if (!doms)
  761. goto done;
  762. /*
  763. * The rest of the code, including the scheduler, can deal with
  764. * dattr==NULL case. No need to abort if alloc fails.
  765. */
  766. dattr = kmalloc_array(ndoms, sizeof(struct sched_domain_attr),
  767. GFP_KERNEL);
  768. for (nslot = 0, i = 0; i < csn; i++) {
  769. struct cpuset *a = csa[i];
  770. struct cpumask *dp;
  771. int apn = a->pn;
  772. if (apn < 0) {
  773. /* Skip completed partitions */
  774. continue;
  775. }
  776. dp = doms[nslot];
  777. if (nslot == ndoms) {
  778. static int warnings = 10;
  779. if (warnings) {
  780. pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n",
  781. nslot, ndoms, csn, i, apn);
  782. warnings--;
  783. }
  784. continue;
  785. }
  786. cpumask_clear(dp);
  787. if (dattr)
  788. *(dattr + nslot) = SD_ATTR_INIT;
  789. for (j = i; j < csn; j++) {
  790. struct cpuset *b = csa[j];
  791. if (apn == b->pn) {
  792. cpumask_or(dp, dp, b->effective_cpus);
  793. cpumask_and(dp, dp, housekeeping_cpumask(HK_FLAG_DOMAIN));
  794. if (dattr)
  795. update_domain_attr_tree(dattr + nslot, b);
  796. /* Done with this partition */
  797. b->pn = -1;
  798. }
  799. }
  800. nslot++;
  801. }
  802. BUG_ON(nslot != ndoms);
  803. done:
  804. kfree(csa);
  805. /*
  806. * Fallback to the default domain if kmalloc() failed.
  807. * See comments in partition_sched_domains().
  808. */
  809. if (doms == NULL)
  810. ndoms = 1;
  811. *domains = doms;
  812. *attributes = dattr;
  813. return ndoms;
  814. }
  815. static void update_tasks_root_domain(struct cpuset *cs)
  816. {
  817. struct css_task_iter it;
  818. struct task_struct *task;
  819. css_task_iter_start(&cs->css, 0, &it);
  820. while ((task = css_task_iter_next(&it)))
  821. dl_add_task_root_domain(task);
  822. css_task_iter_end(&it);
  823. }
  824. static void rebuild_root_domains(void)
  825. {
  826. struct cpuset *cs = NULL;
  827. struct cgroup_subsys_state *pos_css;
  828. lockdep_assert_held(&cpuset_mutex);
  829. lockdep_assert_cpus_held();
  830. lockdep_assert_held(&sched_domains_mutex);
  831. rcu_read_lock();
  832. /*
  833. * Clear default root domain DL accounting, it will be computed again
  834. * if a task belongs to it.
  835. */
  836. dl_clear_root_domain(&def_root_domain);
  837. cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
  838. if (cpumask_empty(cs->effective_cpus)) {
  839. pos_css = css_rightmost_descendant(pos_css);
  840. continue;
  841. }
  842. css_get(&cs->css);
  843. rcu_read_unlock();
  844. update_tasks_root_domain(cs);
  845. rcu_read_lock();
  846. css_put(&cs->css);
  847. }
  848. rcu_read_unlock();
  849. }
  850. static void
  851. partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
  852. struct sched_domain_attr *dattr_new)
  853. {
  854. mutex_lock(&sched_domains_mutex);
  855. partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
  856. rebuild_root_domains();
  857. mutex_unlock(&sched_domains_mutex);
  858. }
  859. /*
  860. * Rebuild scheduler domains.
  861. *
  862. * If the flag 'sched_load_balance' of any cpuset with non-empty
  863. * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
  864. * which has that flag enabled, or if any cpuset with a non-empty
  865. * 'cpus' is removed, then call this routine to rebuild the
  866. * scheduler's dynamic sched domains.
  867. *
  868. * Call with cpuset_mutex held. Takes get_online_cpus().
  869. */
  870. static void rebuild_sched_domains_locked(void)
  871. {
  872. struct cgroup_subsys_state *pos_css;
  873. struct sched_domain_attr *attr;
  874. cpumask_var_t *doms;
  875. struct cpuset *cs;
  876. int ndoms;
  877. lockdep_assert_held(&cpuset_mutex);
  878. /*
  879. * If we have raced with CPU hotplug, return early to avoid
  880. * passing doms with offlined cpu to partition_sched_domains().
  881. * Anyways, cpuset_hotplug_workfn() will rebuild sched domains.
  882. *
  883. * With no CPUs in any subpartitions, top_cpuset's effective CPUs
  884. * should be the same as the active CPUs, so checking only top_cpuset
  885. * is enough to detect racing CPU offlines.
  886. */
  887. if (!top_cpuset.nr_subparts_cpus &&
  888. !cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask))
  889. return;
  890. /*
  891. * With subpartition CPUs, however, the effective CPUs of a partition
  892. * root should be only a subset of the active CPUs. Since a CPU in any
  893. * partition root could be offlined, all must be checked.
  894. */
  895. if (top_cpuset.nr_subparts_cpus) {
  896. rcu_read_lock();
  897. cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
  898. if (!is_partition_root(cs)) {
  899. pos_css = css_rightmost_descendant(pos_css);
  900. continue;
  901. }
  902. if (!cpumask_subset(cs->effective_cpus,
  903. cpu_active_mask)) {
  904. rcu_read_unlock();
  905. return;
  906. }
  907. }
  908. rcu_read_unlock();
  909. }
  910. /* Generate domain masks and attrs */
  911. ndoms = generate_sched_domains(&doms, &attr);
  912. /* Have scheduler rebuild the domains */
  913. partition_and_rebuild_sched_domains(ndoms, doms, attr);
  914. }
  915. #else /* !CONFIG_SMP */
  916. static void rebuild_sched_domains_locked(void)
  917. {
  918. }
  919. #endif /* CONFIG_SMP */
  920. void rebuild_sched_domains(void)
  921. {
  922. get_online_cpus();
  923. mutex_lock(&cpuset_mutex);
  924. rebuild_sched_domains_locked();
  925. mutex_unlock(&cpuset_mutex);
  926. put_online_cpus();
  927. }
  928. static int update_cpus_allowed(struct cpuset *cs, struct task_struct *p,
  929. const struct cpumask *new_mask)
  930. {
  931. int ret = -EINVAL;
  932. trace_android_rvh_update_cpus_allowed(p, cs->cpus_requested, new_mask, &ret);
  933. if (!ret)
  934. return ret;
  935. return set_cpus_allowed_ptr(p, new_mask);
  936. }
  937. /**
  938. * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
  939. * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
  940. *
  941. * Iterate through each task of @cs updating its cpus_allowed to the
  942. * effective cpuset's. As this function is called with cpuset_mutex held,
  943. * cpuset membership stays stable.
  944. */
  945. static void update_tasks_cpumask(struct cpuset *cs)
  946. {
  947. struct css_task_iter it;
  948. struct task_struct *task;
  949. css_task_iter_start(&cs->css, 0, &it);
  950. while ((task = css_task_iter_next(&it)))
  951. update_cpus_allowed(cs, task, cs->effective_cpus);
  952. css_task_iter_end(&it);
  953. }
  954. /**
  955. * compute_effective_cpumask - Compute the effective cpumask of the cpuset
  956. * @new_cpus: the temp variable for the new effective_cpus mask
  957. * @cs: the cpuset the need to recompute the new effective_cpus mask
  958. * @parent: the parent cpuset
  959. *
  960. * If the parent has subpartition CPUs, include them in the list of
  961. * allowable CPUs in computing the new effective_cpus mask. Since offlined
  962. * CPUs are not removed from subparts_cpus, we have to use cpu_active_mask
  963. * to mask those out.
  964. */
  965. static void compute_effective_cpumask(struct cpumask *new_cpus,
  966. struct cpuset *cs, struct cpuset *parent)
  967. {
  968. if (parent->nr_subparts_cpus) {
  969. cpumask_or(new_cpus, parent->effective_cpus,
  970. parent->subparts_cpus);
  971. cpumask_and(new_cpus, new_cpus, cs->cpus_requested);
  972. cpumask_and(new_cpus, new_cpus, cpu_active_mask);
  973. } else {
  974. cpumask_and(new_cpus, cs->cpus_requested, parent_cs(cs)->effective_cpus);
  975. }
  976. }
  977. /*
  978. * Commands for update_parent_subparts_cpumask
  979. */
  980. enum subparts_cmd {
  981. partcmd_enable, /* Enable partition root */
  982. partcmd_disable, /* Disable partition root */
  983. partcmd_update, /* Update parent's subparts_cpus */
  984. };
  985. /**
  986. * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset
  987. * @cpuset: The cpuset that requests change in partition root state
  988. * @cmd: Partition root state change command
  989. * @newmask: Optional new cpumask for partcmd_update
  990. * @tmp: Temporary addmask and delmask
  991. * Return: 0, 1 or an error code
  992. *
  993. * For partcmd_enable, the cpuset is being transformed from a non-partition
  994. * root to a partition root. The cpus_allowed mask of the given cpuset will
  995. * be put into parent's subparts_cpus and taken away from parent's
  996. * effective_cpus. The function will return 0 if all the CPUs listed in
  997. * cpus_allowed can be granted or an error code will be returned.
  998. *
  999. * For partcmd_disable, the cpuset is being transofrmed from a partition
  1000. * root back to a non-partition root. Any CPUs in cpus_allowed that are in
  1001. * parent's subparts_cpus will be taken away from that cpumask and put back
  1002. * into parent's effective_cpus. 0 should always be returned.
  1003. *
  1004. * For partcmd_update, if the optional newmask is specified, the cpu
  1005. * list is to be changed from cpus_allowed to newmask. Otherwise,
  1006. * cpus_allowed is assumed to remain the same. The cpuset should either
  1007. * be a partition root or an invalid partition root. The partition root
  1008. * state may change if newmask is NULL and none of the requested CPUs can
  1009. * be granted by the parent. The function will return 1 if changes to
  1010. * parent's subparts_cpus and effective_cpus happen or 0 otherwise.
  1011. * Error code should only be returned when newmask is non-NULL.
  1012. *
  1013. * The partcmd_enable and partcmd_disable commands are used by
  1014. * update_prstate(). The partcmd_update command is used by
  1015. * update_cpumasks_hier() with newmask NULL and update_cpumask() with
  1016. * newmask set.
  1017. *
  1018. * The checking is more strict when enabling partition root than the
  1019. * other two commands.
  1020. *
  1021. * Because of the implicit cpu exclusive nature of a partition root,
  1022. * cpumask changes that violates the cpu exclusivity rule will not be
  1023. * permitted when checked by validate_change(). The validate_change()
  1024. * function will also prevent any changes to the cpu list if it is not
  1025. * a superset of children's cpu lists.
  1026. */
  1027. static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd,
  1028. struct cpumask *newmask,
  1029. struct tmpmasks *tmp)
  1030. {
  1031. struct cpuset *parent = parent_cs(cpuset);
  1032. int adding; /* Moving cpus from effective_cpus to subparts_cpus */
  1033. int deleting; /* Moving cpus from subparts_cpus to effective_cpus */
  1034. int new_prs;
  1035. bool part_error = false; /* Partition error? */
  1036. lockdep_assert_held(&cpuset_mutex);
  1037. /*
  1038. * The parent must be a partition root.
  1039. * The new cpumask, if present, or the current cpus_allowed must
  1040. * not be empty.
  1041. */
  1042. if (!is_partition_root(parent) ||
  1043. (newmask && cpumask_empty(newmask)) ||
  1044. (!newmask && cpumask_empty(cpuset->cpus_allowed)))
  1045. return -EINVAL;
  1046. /*
  1047. * Enabling/disabling partition root is not allowed if there are
  1048. * online children.
  1049. */
  1050. if ((cmd != partcmd_update) && css_has_online_children(&cpuset->css))
  1051. return -EBUSY;
  1052. /*
  1053. * Enabling partition root is not allowed if not all the CPUs
  1054. * can be granted from parent's effective_cpus or at least one
  1055. * CPU will be left after that.
  1056. */
  1057. if ((cmd == partcmd_enable) &&
  1058. (!cpumask_subset(cpuset->cpus_allowed, parent->effective_cpus) ||
  1059. cpumask_equal(cpuset->cpus_allowed, parent->effective_cpus)))
  1060. return -EINVAL;
  1061. /*
  1062. * A cpumask update cannot make parent's effective_cpus become empty.
  1063. */
  1064. adding = deleting = false;
  1065. new_prs = cpuset->partition_root_state;
  1066. if (cmd == partcmd_enable) {
  1067. cpumask_copy(tmp->addmask, cpuset->cpus_allowed);
  1068. adding = true;
  1069. } else if (cmd == partcmd_disable) {
  1070. deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed,
  1071. parent->subparts_cpus);
  1072. } else if (newmask) {
  1073. /*
  1074. * partcmd_update with newmask:
  1075. *
  1076. * delmask = cpus_allowed & ~newmask & parent->subparts_cpus
  1077. * addmask = newmask & parent->effective_cpus
  1078. * & ~parent->subparts_cpus
  1079. */
  1080. cpumask_andnot(tmp->delmask, cpuset->cpus_allowed, newmask);
  1081. deleting = cpumask_and(tmp->delmask, tmp->delmask,
  1082. parent->subparts_cpus);
  1083. cpumask_and(tmp->addmask, newmask, parent->effective_cpus);
  1084. adding = cpumask_andnot(tmp->addmask, tmp->addmask,
  1085. parent->subparts_cpus);
  1086. /*
  1087. * Return error if the new effective_cpus could become empty.
  1088. */
  1089. if (adding &&
  1090. cpumask_equal(parent->effective_cpus, tmp->addmask)) {
  1091. if (!deleting)
  1092. return -EINVAL;
  1093. /*
  1094. * As some of the CPUs in subparts_cpus might have
  1095. * been offlined, we need to compute the real delmask
  1096. * to confirm that.
  1097. */
  1098. if (!cpumask_and(tmp->addmask, tmp->delmask,
  1099. cpu_active_mask))
  1100. return -EINVAL;
  1101. cpumask_copy(tmp->addmask, parent->effective_cpus);
  1102. }
  1103. } else {
  1104. /*
  1105. * partcmd_update w/o newmask:
  1106. *
  1107. * addmask = cpus_allowed & parent->effective_cpus
  1108. *
  1109. * Note that parent's subparts_cpus may have been
  1110. * pre-shrunk in case there is a change in the cpu list.
  1111. * So no deletion is needed.
  1112. */
  1113. adding = cpumask_and(tmp->addmask, cpuset->cpus_allowed,
  1114. parent->effective_cpus);
  1115. part_error = cpumask_equal(tmp->addmask,
  1116. parent->effective_cpus);
  1117. }
  1118. if (cmd == partcmd_update) {
  1119. int prev_prs = cpuset->partition_root_state;
  1120. /*
  1121. * Check for possible transition between PRS_ENABLED
  1122. * and PRS_ERROR.
  1123. */
  1124. switch (cpuset->partition_root_state) {
  1125. case PRS_ENABLED:
  1126. if (part_error)
  1127. new_prs = PRS_ERROR;
  1128. break;
  1129. case PRS_ERROR:
  1130. if (!part_error)
  1131. new_prs = PRS_ENABLED;
  1132. break;
  1133. }
  1134. /*
  1135. * Set part_error if previously in invalid state.
  1136. */
  1137. part_error = (prev_prs == PRS_ERROR);
  1138. }
  1139. if (!part_error && (new_prs == PRS_ERROR))
  1140. return 0; /* Nothing need to be done */
  1141. if (new_prs == PRS_ERROR) {
  1142. /*
  1143. * Remove all its cpus from parent's subparts_cpus.
  1144. */
  1145. adding = false;
  1146. deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed,
  1147. parent->subparts_cpus);
  1148. }
  1149. if (!adding && !deleting && (new_prs == cpuset->partition_root_state))
  1150. return 0;
  1151. /*
  1152. * Change the parent's subparts_cpus.
  1153. * Newly added CPUs will be removed from effective_cpus and
  1154. * newly deleted ones will be added back to effective_cpus.
  1155. */
  1156. spin_lock_irq(&callback_lock);
  1157. if (adding) {
  1158. cpumask_or(parent->subparts_cpus,
  1159. parent->subparts_cpus, tmp->addmask);
  1160. cpumask_andnot(parent->effective_cpus,
  1161. parent->effective_cpus, tmp->addmask);
  1162. }
  1163. if (deleting) {
  1164. cpumask_andnot(parent->subparts_cpus,
  1165. parent->subparts_cpus, tmp->delmask);
  1166. /*
  1167. * Some of the CPUs in subparts_cpus might have been offlined.
  1168. */
  1169. cpumask_and(tmp->delmask, tmp->delmask, cpu_active_mask);
  1170. cpumask_or(parent->effective_cpus,
  1171. parent->effective_cpus, tmp->delmask);
  1172. }
  1173. parent->nr_subparts_cpus = cpumask_weight(parent->subparts_cpus);
  1174. if (cpuset->partition_root_state != new_prs)
  1175. cpuset->partition_root_state = new_prs;
  1176. spin_unlock_irq(&callback_lock);
  1177. return cmd == partcmd_update;
  1178. }
  1179. /*
  1180. * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
  1181. * @cs: the cpuset to consider
  1182. * @tmp: temp variables for calculating effective_cpus & partition setup
  1183. *
  1184. * When congifured cpumask is changed, the effective cpumasks of this cpuset
  1185. * and all its descendants need to be updated.
  1186. *
  1187. * On legacy hierachy, effective_cpus will be the same with cpu_allowed.
  1188. *
  1189. * Called with cpuset_mutex held
  1190. */
  1191. static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp)
  1192. {
  1193. struct cpuset *cp;
  1194. struct cgroup_subsys_state *pos_css;
  1195. bool need_rebuild_sched_domains = false;
  1196. int new_prs;
  1197. rcu_read_lock();
  1198. cpuset_for_each_descendant_pre(cp, pos_css, cs) {
  1199. struct cpuset *parent = parent_cs(cp);
  1200. compute_effective_cpumask(tmp->new_cpus, cp, parent);
  1201. /*
  1202. * If it becomes empty, inherit the effective mask of the
  1203. * parent, which is guaranteed to have some CPUs.
  1204. */
  1205. if (is_in_v2_mode() && cpumask_empty(tmp->new_cpus)) {
  1206. cpumask_copy(tmp->new_cpus, parent->effective_cpus);
  1207. if (!cp->use_parent_ecpus) {
  1208. cp->use_parent_ecpus = true;
  1209. parent->child_ecpus_count++;
  1210. }
  1211. } else if (cp->use_parent_ecpus) {
  1212. cp->use_parent_ecpus = false;
  1213. WARN_ON_ONCE(!parent->child_ecpus_count);
  1214. parent->child_ecpus_count--;
  1215. }
  1216. /*
  1217. * Skip the whole subtree if the cpumask remains the same
  1218. * and has no partition root state.
  1219. */
  1220. if (!cp->partition_root_state &&
  1221. cpumask_equal(tmp->new_cpus, cp->effective_cpus)) {
  1222. pos_css = css_rightmost_descendant(pos_css);
  1223. continue;
  1224. }
  1225. /*
  1226. * update_parent_subparts_cpumask() should have been called
  1227. * for cs already in update_cpumask(). We should also call
  1228. * update_tasks_cpumask() again for tasks in the parent
  1229. * cpuset if the parent's subparts_cpus changes.
  1230. */
  1231. new_prs = cp->partition_root_state;
  1232. if ((cp != cs) && new_prs) {
  1233. switch (parent->partition_root_state) {
  1234. case PRS_DISABLED:
  1235. /*
  1236. * If parent is not a partition root or an
  1237. * invalid partition root, clear its state
  1238. * and its CS_CPU_EXCLUSIVE flag.
  1239. */
  1240. WARN_ON_ONCE(cp->partition_root_state
  1241. != PRS_ERROR);
  1242. new_prs = PRS_DISABLED;
  1243. /*
  1244. * clear_bit() is an atomic operation and
  1245. * readers aren't interested in the state
  1246. * of CS_CPU_EXCLUSIVE anyway. So we can
  1247. * just update the flag without holding
  1248. * the callback_lock.
  1249. */
  1250. clear_bit(CS_CPU_EXCLUSIVE, &cp->flags);
  1251. break;
  1252. case PRS_ENABLED:
  1253. if (update_parent_subparts_cpumask(cp, partcmd_update, NULL, tmp))
  1254. update_tasks_cpumask(parent);
  1255. break;
  1256. case PRS_ERROR:
  1257. /*
  1258. * When parent is invalid, it has to be too.
  1259. */
  1260. new_prs = PRS_ERROR;
  1261. break;
  1262. }
  1263. }
  1264. if (!css_tryget_online(&cp->css))
  1265. continue;
  1266. rcu_read_unlock();
  1267. spin_lock_irq(&callback_lock);
  1268. cpumask_copy(cp->effective_cpus, tmp->new_cpus);
  1269. if (cp->nr_subparts_cpus && (new_prs != PRS_ENABLED)) {
  1270. cp->nr_subparts_cpus = 0;
  1271. cpumask_clear(cp->subparts_cpus);
  1272. } else if (cp->nr_subparts_cpus) {
  1273. /*
  1274. * Make sure that effective_cpus & subparts_cpus
  1275. * are mutually exclusive.
  1276. *
  1277. * In the unlikely event that effective_cpus
  1278. * becomes empty. we clear cp->nr_subparts_cpus and
  1279. * let its child partition roots to compete for
  1280. * CPUs again.
  1281. */
  1282. cpumask_andnot(cp->effective_cpus, cp->effective_cpus,
  1283. cp->subparts_cpus);
  1284. if (cpumask_empty(cp->effective_cpus)) {
  1285. cpumask_copy(cp->effective_cpus, tmp->new_cpus);
  1286. cpumask_clear(cp->subparts_cpus);
  1287. cp->nr_subparts_cpus = 0;
  1288. } else if (!cpumask_subset(cp->subparts_cpus,
  1289. tmp->new_cpus)) {
  1290. cpumask_andnot(cp->subparts_cpus,
  1291. cp->subparts_cpus, tmp->new_cpus);
  1292. cp->nr_subparts_cpus
  1293. = cpumask_weight(cp->subparts_cpus);
  1294. }
  1295. }
  1296. if (new_prs != cp->partition_root_state)
  1297. cp->partition_root_state = new_prs;
  1298. spin_unlock_irq(&callback_lock);
  1299. WARN_ON(!is_in_v2_mode() &&
  1300. !cpumask_equal(cp->cpus_allowed, cp->effective_cpus));
  1301. update_tasks_cpumask(cp);
  1302. /*
  1303. * On legacy hierarchy, if the effective cpumask of any non-
  1304. * empty cpuset is changed, we need to rebuild sched domains.
  1305. * On default hierarchy, the cpuset needs to be a partition
  1306. * root as well.
  1307. */
  1308. if (!cpumask_empty(cp->cpus_allowed) &&
  1309. is_sched_load_balance(cp) &&
  1310. (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
  1311. is_partition_root(cp)))
  1312. need_rebuild_sched_domains = true;
  1313. rcu_read_lock();
  1314. css_put(&cp->css);
  1315. }
  1316. rcu_read_unlock();
  1317. if (need_rebuild_sched_domains)
  1318. rebuild_sched_domains_locked();
  1319. }
  1320. /**
  1321. * update_sibling_cpumasks - Update siblings cpumasks
  1322. * @parent: Parent cpuset
  1323. * @cs: Current cpuset
  1324. * @tmp: Temp variables
  1325. */
  1326. static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
  1327. struct tmpmasks *tmp)
  1328. {
  1329. struct cpuset *sibling;
  1330. struct cgroup_subsys_state *pos_css;
  1331. lockdep_assert_held(&cpuset_mutex);
  1332. /*
  1333. * Check all its siblings and call update_cpumasks_hier()
  1334. * if their use_parent_ecpus flag is set in order for them
  1335. * to use the right effective_cpus value.
  1336. *
  1337. * The update_cpumasks_hier() function may sleep. So we have to
  1338. * release the RCU read lock before calling it.
  1339. */
  1340. rcu_read_lock();
  1341. cpuset_for_each_child(sibling, pos_css, parent) {
  1342. if (sibling == cs)
  1343. continue;
  1344. if (!sibling->use_parent_ecpus)
  1345. continue;
  1346. if (!css_tryget_online(&sibling->css))
  1347. continue;
  1348. rcu_read_unlock();
  1349. update_cpumasks_hier(sibling, tmp);
  1350. rcu_read_lock();
  1351. css_put(&sibling->css);
  1352. }
  1353. rcu_read_unlock();
  1354. }
  1355. /**
  1356. * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
  1357. * @cs: the cpuset to consider
  1358. * @trialcs: trial cpuset
  1359. * @buf: buffer of cpu numbers written to this cpuset
  1360. */
  1361. static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
  1362. const char *buf)
  1363. {
  1364. int retval;
  1365. struct tmpmasks tmp;
  1366. /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
  1367. if (cs == &top_cpuset)
  1368. return -EACCES;
  1369. /*
  1370. * An empty cpus_requested is ok only if the cpuset has no tasks.
  1371. * Since cpulist_parse() fails on an empty mask, we special case
  1372. * that parsing. The validate_change() call ensures that cpusets
  1373. * with tasks have cpus.
  1374. */
  1375. if (!*buf) {
  1376. cpumask_clear(trialcs->cpus_requested);
  1377. } else {
  1378. retval = cpulist_parse(buf, trialcs->cpus_requested);
  1379. if (retval < 0)
  1380. return retval;
  1381. }
  1382. if (!cpumask_subset(trialcs->cpus_requested, cpu_present_mask))
  1383. return -EINVAL;
  1384. cpumask_and(trialcs->cpus_allowed, trialcs->cpus_requested, cpu_active_mask);
  1385. /* Nothing to do if the cpus didn't change */
  1386. if (cpumask_equal(cs->cpus_requested, trialcs->cpus_requested))
  1387. return 0;
  1388. retval = validate_change(cs, trialcs);
  1389. if (retval < 0)
  1390. return retval;
  1391. #ifdef CONFIG_CPUMASK_OFFSTACK
  1392. /*
  1393. * Use the cpumasks in trialcs for tmpmasks when they are pointers
  1394. * to allocated cpumasks.
  1395. */
  1396. tmp.addmask = trialcs->subparts_cpus;
  1397. tmp.delmask = trialcs->effective_cpus;
  1398. tmp.new_cpus = trialcs->cpus_allowed;
  1399. #endif
  1400. if (cs->partition_root_state) {
  1401. /* Cpumask of a partition root cannot be empty */
  1402. if (cpumask_empty(trialcs->cpus_allowed))
  1403. return -EINVAL;
  1404. if (update_parent_subparts_cpumask(cs, partcmd_update,
  1405. trialcs->cpus_allowed, &tmp) < 0)
  1406. return -EINVAL;
  1407. }
  1408. spin_lock_irq(&callback_lock);
  1409. cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
  1410. cpumask_copy(cs->cpus_requested, trialcs->cpus_requested);
  1411. /*
  1412. * Make sure that subparts_cpus is a subset of cpus_allowed.
  1413. */
  1414. if (cs->nr_subparts_cpus) {
  1415. cpumask_and(cs->subparts_cpus, cs->subparts_cpus, cs->cpus_allowed);
  1416. cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus);
  1417. }
  1418. spin_unlock_irq(&callback_lock);
  1419. update_cpumasks_hier(cs, &tmp);
  1420. if (cs->partition_root_state) {
  1421. struct cpuset *parent = parent_cs(cs);
  1422. /*
  1423. * For partition root, update the cpumasks of sibling
  1424. * cpusets if they use parent's effective_cpus.
  1425. */
  1426. if (parent->child_ecpus_count)
  1427. update_sibling_cpumasks(parent, cs, &tmp);
  1428. }
  1429. return 0;
  1430. }
  1431. /*
  1432. * Migrate memory region from one set of nodes to another. This is
  1433. * performed asynchronously as it can be called from process migration path
  1434. * holding locks involved in process management. All mm migrations are
  1435. * performed in the queued order and can be waited for by flushing
  1436. * cpuset_migrate_mm_wq.
  1437. */
  1438. struct cpuset_migrate_mm_work {
  1439. struct work_struct work;
  1440. struct mm_struct *mm;
  1441. nodemask_t from;
  1442. nodemask_t to;
  1443. };
  1444. static void cpuset_migrate_mm_workfn(struct work_struct *work)
  1445. {
  1446. struct cpuset_migrate_mm_work *mwork =
  1447. container_of(work, struct cpuset_migrate_mm_work, work);
  1448. /* on a wq worker, no need to worry about %current's mems_allowed */
  1449. do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL);
  1450. mmput(mwork->mm);
  1451. kfree(mwork);
  1452. }
  1453. static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
  1454. const nodemask_t *to)
  1455. {
  1456. struct cpuset_migrate_mm_work *mwork;
  1457. mwork = kzalloc(sizeof(*mwork), GFP_KERNEL);
  1458. if (mwork) {
  1459. mwork->mm = mm;
  1460. mwork->from = *from;
  1461. mwork->to = *to;
  1462. INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn);
  1463. queue_work(cpuset_migrate_mm_wq, &mwork->work);
  1464. } else {
  1465. mmput(mm);
  1466. }
  1467. }
  1468. static void cpuset_post_attach(void)
  1469. {
  1470. flush_workqueue(cpuset_migrate_mm_wq);
  1471. }
  1472. /*
  1473. * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
  1474. * @tsk: the task to change
  1475. * @newmems: new nodes that the task will be set
  1476. *
  1477. * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
  1478. * and rebind an eventual tasks' mempolicy. If the task is allocating in
  1479. * parallel, it might temporarily see an empty intersection, which results in
  1480. * a seqlock check and retry before OOM or allocation failure.
  1481. */
  1482. static void cpuset_change_task_nodemask(struct task_struct *tsk,
  1483. nodemask_t *newmems)
  1484. {
  1485. task_lock(tsk);
  1486. local_irq_disable();
  1487. write_seqcount_begin(&tsk->mems_allowed_seq);
  1488. nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
  1489. mpol_rebind_task(tsk, newmems);
  1490. tsk->mems_allowed = *newmems;
  1491. write_seqcount_end(&tsk->mems_allowed_seq);
  1492. local_irq_enable();
  1493. task_unlock(tsk);
  1494. }
  1495. static void *cpuset_being_rebound;
  1496. /**
  1497. * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
  1498. * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
  1499. *
  1500. * Iterate through each task of @cs updating its mems_allowed to the
  1501. * effective cpuset's. As this function is called with cpuset_mutex held,
  1502. * cpuset membership stays stable.
  1503. */
  1504. static void update_tasks_nodemask(struct cpuset *cs)
  1505. {
  1506. static nodemask_t newmems; /* protected by cpuset_mutex */
  1507. struct css_task_iter it;
  1508. struct task_struct *task;
  1509. cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
  1510. guarantee_online_mems(cs, &newmems);
  1511. /*
  1512. * The mpol_rebind_mm() call takes mmap_lock, which we couldn't
  1513. * take while holding tasklist_lock. Forks can happen - the
  1514. * mpol_dup() cpuset_being_rebound check will catch such forks,
  1515. * and rebind their vma mempolicies too. Because we still hold
  1516. * the global cpuset_mutex, we know that no other rebind effort
  1517. * will be contending for the global variable cpuset_being_rebound.
  1518. * It's ok if we rebind the same mm twice; mpol_rebind_mm()
  1519. * is idempotent. Also migrate pages in each mm to new nodes.
  1520. */
  1521. css_task_iter_start(&cs->css, 0, &it);
  1522. while ((task = css_task_iter_next(&it))) {
  1523. struct mm_struct *mm;
  1524. bool migrate;
  1525. cpuset_change_task_nodemask(task, &newmems);
  1526. mm = get_task_mm(task);
  1527. if (!mm)
  1528. continue;
  1529. migrate = is_memory_migrate(cs);
  1530. mpol_rebind_mm(mm, &cs->mems_allowed);
  1531. if (migrate)
  1532. cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
  1533. else
  1534. mmput(mm);
  1535. }
  1536. css_task_iter_end(&it);
  1537. /*
  1538. * All the tasks' nodemasks have been updated, update
  1539. * cs->old_mems_allowed.
  1540. */
  1541. cs->old_mems_allowed = newmems;
  1542. /* We're done rebinding vmas to this cpuset's new mems_allowed. */
  1543. cpuset_being_rebound = NULL;
  1544. }
  1545. /*
  1546. * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
  1547. * @cs: the cpuset to consider
  1548. * @new_mems: a temp variable for calculating new effective_mems
  1549. *
  1550. * When configured nodemask is changed, the effective nodemasks of this cpuset
  1551. * and all its descendants need to be updated.
  1552. *
  1553. * On legacy hiearchy, effective_mems will be the same with mems_allowed.
  1554. *
  1555. * Called with cpuset_mutex held
  1556. */
  1557. static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
  1558. {
  1559. struct cpuset *cp;
  1560. struct cgroup_subsys_state *pos_css;
  1561. rcu_read_lock();
  1562. cpuset_for_each_descendant_pre(cp, pos_css, cs) {
  1563. struct cpuset *parent = parent_cs(cp);
  1564. nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems);
  1565. /*
  1566. * If it becomes empty, inherit the effective mask of the
  1567. * parent, which is guaranteed to have some MEMs.
  1568. */
  1569. if (is_in_v2_mode() && nodes_empty(*new_mems))
  1570. *new_mems = parent->effective_mems;
  1571. /* Skip the whole subtree if the nodemask remains the same. */
  1572. if (nodes_equal(*new_mems, cp->effective_mems)) {
  1573. pos_css = css_rightmost_descendant(pos_css);
  1574. continue;
  1575. }
  1576. if (!css_tryget_online(&cp->css))
  1577. continue;
  1578. rcu_read_unlock();
  1579. spin_lock_irq(&callback_lock);
  1580. cp->effective_mems = *new_mems;
  1581. spin_unlock_irq(&callback_lock);
  1582. WARN_ON(!is_in_v2_mode() &&
  1583. !nodes_equal(cp->mems_allowed, cp->effective_mems));
  1584. update_tasks_nodemask(cp);
  1585. rcu_read_lock();
  1586. css_put(&cp->css);
  1587. }
  1588. rcu_read_unlock();
  1589. }
  1590. /*
  1591. * Handle user request to change the 'mems' memory placement
  1592. * of a cpuset. Needs to validate the request, update the
  1593. * cpusets mems_allowed, and for each task in the cpuset,
  1594. * update mems_allowed and rebind task's mempolicy and any vma
  1595. * mempolicies and if the cpuset is marked 'memory_migrate',
  1596. * migrate the tasks pages to the new memory.
  1597. *
  1598. * Call with cpuset_mutex held. May take callback_lock during call.
  1599. * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
  1600. * lock each such tasks mm->mmap_lock, scan its vma's and rebind
  1601. * their mempolicies to the cpusets new mems_allowed.
  1602. */
  1603. static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
  1604. const char *buf)
  1605. {
  1606. int retval;
  1607. /*
  1608. * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
  1609. * it's read-only
  1610. */
  1611. if (cs == &top_cpuset) {
  1612. retval = -EACCES;
  1613. goto done;
  1614. }
  1615. /*
  1616. * An empty mems_allowed is ok iff there are no tasks in the cpuset.
  1617. * Since nodelist_parse() fails on an empty mask, we special case
  1618. * that parsing. The validate_change() call ensures that cpusets
  1619. * with tasks have memory.
  1620. */
  1621. if (!*buf) {
  1622. nodes_clear(trialcs->mems_allowed);
  1623. } else {
  1624. retval = nodelist_parse(buf, trialcs->mems_allowed);
  1625. if (retval < 0)
  1626. goto done;
  1627. if (!nodes_subset(trialcs->mems_allowed,
  1628. top_cpuset.mems_allowed)) {
  1629. retval = -EINVAL;
  1630. goto done;
  1631. }
  1632. }
  1633. if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
  1634. retval = 0; /* Too easy - nothing to do */
  1635. goto done;
  1636. }
  1637. retval = validate_change(cs, trialcs);
  1638. if (retval < 0)
  1639. goto done;
  1640. spin_lock_irq(&callback_lock);
  1641. cs->mems_allowed = trialcs->mems_allowed;
  1642. spin_unlock_irq(&callback_lock);
  1643. /* use trialcs->mems_allowed as a temp variable */
  1644. update_nodemasks_hier(cs, &trialcs->mems_allowed);
  1645. done:
  1646. return retval;
  1647. }
  1648. bool current_cpuset_is_being_rebound(void)
  1649. {
  1650. bool ret;
  1651. rcu_read_lock();
  1652. ret = task_cs(current) == cpuset_being_rebound;
  1653. rcu_read_unlock();
  1654. return ret;
  1655. }
  1656. static int update_relax_domain_level(struct cpuset *cs, s64 val)
  1657. {
  1658. #ifdef CONFIG_SMP
  1659. if (val < -1 || val >= sched_domain_level_max)
  1660. return -EINVAL;
  1661. #endif
  1662. if (val != cs->relax_domain_level) {
  1663. cs->relax_domain_level = val;
  1664. if (!cpumask_empty(cs->cpus_allowed) &&
  1665. is_sched_load_balance(cs))
  1666. rebuild_sched_domains_locked();
  1667. }
  1668. return 0;
  1669. }
  1670. /**
  1671. * update_tasks_flags - update the spread flags of tasks in the cpuset.
  1672. * @cs: the cpuset in which each task's spread flags needs to be changed
  1673. *
  1674. * Iterate through each task of @cs updating its spread flags. As this
  1675. * function is called with cpuset_mutex held, cpuset membership stays
  1676. * stable.
  1677. */
  1678. static void update_tasks_flags(struct cpuset *cs)
  1679. {
  1680. struct css_task_iter it;
  1681. struct task_struct *task;
  1682. css_task_iter_start(&cs->css, 0, &it);
  1683. while ((task = css_task_iter_next(&it)))
  1684. cpuset_update_task_spread_flag(cs, task);
  1685. css_task_iter_end(&it);
  1686. }
  1687. /*
  1688. * update_flag - read a 0 or a 1 in a file and update associated flag
  1689. * bit: the bit to update (see cpuset_flagbits_t)
  1690. * cs: the cpuset to update
  1691. * turning_on: whether the flag is being set or cleared
  1692. *
  1693. * Call with cpuset_mutex held.
  1694. */
  1695. static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
  1696. int turning_on)
  1697. {
  1698. struct cpuset *trialcs;
  1699. int balance_flag_changed;
  1700. int spread_flag_changed;
  1701. int err;
  1702. trialcs = alloc_trial_cpuset(cs);
  1703. if (!trialcs)
  1704. return -ENOMEM;
  1705. if (turning_on)
  1706. set_bit(bit, &trialcs->flags);
  1707. else
  1708. clear_bit(bit, &trialcs->flags);
  1709. err = validate_change(cs, trialcs);
  1710. if (err < 0)
  1711. goto out;
  1712. balance_flag_changed = (is_sched_load_balance(cs) !=
  1713. is_sched_load_balance(trialcs));
  1714. spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
  1715. || (is_spread_page(cs) != is_spread_page(trialcs)));
  1716. spin_lock_irq(&callback_lock);
  1717. cs->flags = trialcs->flags;
  1718. spin_unlock_irq(&callback_lock);
  1719. if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
  1720. rebuild_sched_domains_locked();
  1721. if (spread_flag_changed)
  1722. update_tasks_flags(cs);
  1723. out:
  1724. free_cpuset(trialcs);
  1725. return err;
  1726. }
  1727. /*
  1728. * update_prstate - update partititon_root_state
  1729. * cs: the cpuset to update
  1730. * new_prs: new partition root state
  1731. *
  1732. * Call with cpuset_mutex held.
  1733. */
  1734. static int update_prstate(struct cpuset *cs, int new_prs)
  1735. {
  1736. int err, old_prs = cs->partition_root_state;
  1737. struct cpuset *parent = parent_cs(cs);
  1738. struct tmpmasks tmpmask;
  1739. if (old_prs == new_prs)
  1740. return 0;
  1741. /*
  1742. * Cannot force a partial or invalid partition root to a full
  1743. * partition root.
  1744. */
  1745. if (new_prs && (old_prs == PRS_ERROR))
  1746. return -EINVAL;
  1747. if (alloc_cpumasks(NULL, &tmpmask))
  1748. return -ENOMEM;
  1749. err = -EINVAL;
  1750. if (!old_prs) {
  1751. /*
  1752. * Turning on partition root requires setting the
  1753. * CS_CPU_EXCLUSIVE bit implicitly as well and cpus_allowed
  1754. * cannot be NULL.
  1755. */
  1756. if (cpumask_empty(cs->cpus_allowed))
  1757. goto out;
  1758. err = update_flag(CS_CPU_EXCLUSIVE, cs, 1);
  1759. if (err)
  1760. goto out;
  1761. err = update_parent_subparts_cpumask(cs, partcmd_enable,
  1762. NULL, &tmpmask);
  1763. if (err) {
  1764. update_flag(CS_CPU_EXCLUSIVE, cs, 0);
  1765. goto out;
  1766. }
  1767. } else {
  1768. /*
  1769. * Turning off partition root will clear the
  1770. * CS_CPU_EXCLUSIVE bit.
  1771. */
  1772. if (old_prs == PRS_ERROR) {
  1773. update_flag(CS_CPU_EXCLUSIVE, cs, 0);
  1774. err = 0;
  1775. goto out;
  1776. }
  1777. err = update_parent_subparts_cpumask(cs, partcmd_disable,
  1778. NULL, &tmpmask);
  1779. if (err)
  1780. goto out;
  1781. /* Turning off CS_CPU_EXCLUSIVE will not return error */
  1782. update_flag(CS_CPU_EXCLUSIVE, cs, 0);
  1783. }
  1784. /*
  1785. * Update cpumask of parent's tasks except when it is the top
  1786. * cpuset as some system daemons cannot be mapped to other CPUs.
  1787. */
  1788. if (parent != &top_cpuset)
  1789. update_tasks_cpumask(parent);
  1790. if (parent->child_ecpus_count)
  1791. update_sibling_cpumasks(parent, cs, &tmpmask);
  1792. rebuild_sched_domains_locked();
  1793. out:
  1794. if (!err) {
  1795. spin_lock_irq(&callback_lock);
  1796. cs->partition_root_state = new_prs;
  1797. spin_unlock_irq(&callback_lock);
  1798. }
  1799. free_cpumasks(NULL, &tmpmask);
  1800. return err;
  1801. }
  1802. /*
  1803. * Frequency meter - How fast is some event occurring?
  1804. *
  1805. * These routines manage a digitally filtered, constant time based,
  1806. * event frequency meter. There are four routines:
  1807. * fmeter_init() - initialize a frequency meter.
  1808. * fmeter_markevent() - called each time the event happens.
  1809. * fmeter_getrate() - returns the recent rate of such events.
  1810. * fmeter_update() - internal routine used to update fmeter.
  1811. *
  1812. * A common data structure is passed to each of these routines,
  1813. * which is used to keep track of the state required to manage the
  1814. * frequency meter and its digital filter.
  1815. *
  1816. * The filter works on the number of events marked per unit time.
  1817. * The filter is single-pole low-pass recursive (IIR). The time unit
  1818. * is 1 second. Arithmetic is done using 32-bit integers scaled to
  1819. * simulate 3 decimal digits of precision (multiplied by 1000).
  1820. *
  1821. * With an FM_COEF of 933, and a time base of 1 second, the filter
  1822. * has a half-life of 10 seconds, meaning that if the events quit
  1823. * happening, then the rate returned from the fmeter_getrate()
  1824. * will be cut in half each 10 seconds, until it converges to zero.
  1825. *
  1826. * It is not worth doing a real infinitely recursive filter. If more
  1827. * than FM_MAXTICKS ticks have elapsed since the last filter event,
  1828. * just compute FM_MAXTICKS ticks worth, by which point the level
  1829. * will be stable.
  1830. *
  1831. * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
  1832. * arithmetic overflow in the fmeter_update() routine.
  1833. *
  1834. * Given the simple 32 bit integer arithmetic used, this meter works
  1835. * best for reporting rates between one per millisecond (msec) and
  1836. * one per 32 (approx) seconds. At constant rates faster than one
  1837. * per msec it maxes out at values just under 1,000,000. At constant
  1838. * rates between one per msec, and one per second it will stabilize
  1839. * to a value N*1000, where N is the rate of events per second.
  1840. * At constant rates between one per second and one per 32 seconds,
  1841. * it will be choppy, moving up on the seconds that have an event,
  1842. * and then decaying until the next event. At rates slower than
  1843. * about one in 32 seconds, it decays all the way back to zero between
  1844. * each event.
  1845. */
  1846. #define FM_COEF 933 /* coefficient for half-life of 10 secs */
  1847. #define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */
  1848. #define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
  1849. #define FM_SCALE 1000 /* faux fixed point scale */
  1850. /* Initialize a frequency meter */
  1851. static void fmeter_init(struct fmeter *fmp)
  1852. {
  1853. fmp->cnt = 0;
  1854. fmp->val = 0;
  1855. fmp->time = 0;
  1856. spin_lock_init(&fmp->lock);
  1857. }
  1858. /* Internal meter update - process cnt events and update value */
  1859. static void fmeter_update(struct fmeter *fmp)
  1860. {
  1861. time64_t now;
  1862. u32 ticks;
  1863. now = ktime_get_seconds();
  1864. ticks = now - fmp->time;
  1865. if (ticks == 0)
  1866. return;
  1867. ticks = min(FM_MAXTICKS, ticks);
  1868. while (ticks-- > 0)
  1869. fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
  1870. fmp->time = now;
  1871. fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
  1872. fmp->cnt = 0;
  1873. }
  1874. /* Process any previous ticks, then bump cnt by one (times scale). */
  1875. static void fmeter_markevent(struct fmeter *fmp)
  1876. {
  1877. spin_lock(&fmp->lock);
  1878. fmeter_update(fmp);
  1879. fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
  1880. spin_unlock(&fmp->lock);
  1881. }
  1882. /* Process any previous ticks, then return current value. */
  1883. static int fmeter_getrate(struct fmeter *fmp)
  1884. {
  1885. int val;
  1886. spin_lock(&fmp->lock);
  1887. fmeter_update(fmp);
  1888. val = fmp->val;
  1889. spin_unlock(&fmp->lock);
  1890. return val;
  1891. }
  1892. static struct cpuset *cpuset_attach_old_cs;
  1893. /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
  1894. static int cpuset_can_attach(struct cgroup_taskset *tset)
  1895. {
  1896. struct cgroup_subsys_state *css;
  1897. struct cpuset *cs;
  1898. struct task_struct *task;
  1899. int ret;
  1900. /* used later by cpuset_attach() */
  1901. cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
  1902. cs = css_cs(css);
  1903. mutex_lock(&cpuset_mutex);
  1904. /* allow moving tasks into an empty cpuset if on default hierarchy */
  1905. ret = -ENOSPC;
  1906. if (!is_in_v2_mode() &&
  1907. (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
  1908. goto out_unlock;
  1909. cgroup_taskset_for_each(task, css, tset) {
  1910. ret = task_can_attach(task, cs->cpus_allowed);
  1911. if (ret)
  1912. goto out_unlock;
  1913. ret = security_task_setscheduler(task);
  1914. if (ret)
  1915. goto out_unlock;
  1916. }
  1917. /*
  1918. * Mark attach is in progress. This makes validate_change() fail
  1919. * changes which zero cpus/mems_allowed.
  1920. */
  1921. cs->attach_in_progress++;
  1922. ret = 0;
  1923. out_unlock:
  1924. mutex_unlock(&cpuset_mutex);
  1925. return ret;
  1926. }
  1927. static void cpuset_cancel_attach(struct cgroup_taskset *tset)
  1928. {
  1929. struct cgroup_subsys_state *css;
  1930. cgroup_taskset_first(tset, &css);
  1931. mutex_lock(&cpuset_mutex);
  1932. css_cs(css)->attach_in_progress--;
  1933. mutex_unlock(&cpuset_mutex);
  1934. }
  1935. /*
  1936. * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach()
  1937. * but we can't allocate it dynamically there. Define it global and
  1938. * allocate from cpuset_init().
  1939. */
  1940. static cpumask_var_t cpus_attach;
  1941. static void cpuset_attach(struct cgroup_taskset *tset)
  1942. {
  1943. /* static buf protected by cpuset_mutex */
  1944. static nodemask_t cpuset_attach_nodemask_to;
  1945. struct task_struct *task;
  1946. struct task_struct *leader;
  1947. struct cgroup_subsys_state *css;
  1948. struct cpuset *cs;
  1949. struct cpuset *oldcs = cpuset_attach_old_cs;
  1950. cgroup_taskset_first(tset, &css);
  1951. cs = css_cs(css);
  1952. cpus_read_lock();
  1953. mutex_lock(&cpuset_mutex);
  1954. guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
  1955. cgroup_taskset_for_each(task, css, tset) {
  1956. if (cs != &top_cpuset)
  1957. guarantee_online_cpus(task, cpus_attach);
  1958. else
  1959. cpumask_copy(cpus_attach, task_cpu_possible_mask(task));
  1960. /*
  1961. * can_attach beforehand should guarantee that this doesn't
  1962. * fail. TODO: have a better way to handle failure here
  1963. */
  1964. WARN_ON_ONCE(update_cpus_allowed(cs, task, cpus_attach));
  1965. cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
  1966. cpuset_update_task_spread_flag(cs, task);
  1967. }
  1968. /*
  1969. * Change mm for all threadgroup leaders. This is expensive and may
  1970. * sleep and should be moved outside migration path proper.
  1971. */
  1972. cpuset_attach_nodemask_to = cs->effective_mems;
  1973. cgroup_taskset_for_each_leader(leader, css, tset) {
  1974. struct mm_struct *mm = get_task_mm(leader);
  1975. if (mm) {
  1976. mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
  1977. /*
  1978. * old_mems_allowed is the same with mems_allowed
  1979. * here, except if this task is being moved
  1980. * automatically due to hotplug. In that case
  1981. * @mems_allowed has been updated and is empty, so
  1982. * @old_mems_allowed is the right nodesets that we
  1983. * migrate mm from.
  1984. */
  1985. if (is_memory_migrate(cs))
  1986. cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
  1987. &cpuset_attach_nodemask_to);
  1988. else
  1989. mmput(mm);
  1990. }
  1991. }
  1992. cs->old_mems_allowed = cpuset_attach_nodemask_to;
  1993. cs->attach_in_progress--;
  1994. if (!cs->attach_in_progress)
  1995. wake_up(&cpuset_attach_wq);
  1996. mutex_unlock(&cpuset_mutex);
  1997. cpus_read_unlock();
  1998. }
  1999. /* The various types of files and directories in a cpuset file system */
  2000. typedef enum {
  2001. FILE_MEMORY_MIGRATE,
  2002. FILE_CPULIST,
  2003. FILE_MEMLIST,
  2004. FILE_EFFECTIVE_CPULIST,
  2005. FILE_EFFECTIVE_MEMLIST,
  2006. FILE_SUBPARTS_CPULIST,
  2007. FILE_CPU_EXCLUSIVE,
  2008. FILE_MEM_EXCLUSIVE,
  2009. FILE_MEM_HARDWALL,
  2010. FILE_SCHED_LOAD_BALANCE,
  2011. FILE_PARTITION_ROOT,
  2012. FILE_SCHED_RELAX_DOMAIN_LEVEL,
  2013. FILE_MEMORY_PRESSURE_ENABLED,
  2014. FILE_MEMORY_PRESSURE,
  2015. FILE_SPREAD_PAGE,
  2016. FILE_SPREAD_SLAB,
  2017. } cpuset_filetype_t;
  2018. static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
  2019. u64 val)
  2020. {
  2021. struct cpuset *cs = css_cs(css);
  2022. cpuset_filetype_t type = cft->private;
  2023. int retval = 0;
  2024. get_online_cpus();
  2025. mutex_lock(&cpuset_mutex);
  2026. if (!is_cpuset_online(cs)) {
  2027. retval = -ENODEV;
  2028. goto out_unlock;
  2029. }
  2030. switch (type) {
  2031. case FILE_CPU_EXCLUSIVE:
  2032. retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
  2033. break;
  2034. case FILE_MEM_EXCLUSIVE:
  2035. retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
  2036. break;
  2037. case FILE_MEM_HARDWALL:
  2038. retval = update_flag(CS_MEM_HARDWALL, cs, val);
  2039. break;
  2040. case FILE_SCHED_LOAD_BALANCE:
  2041. retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
  2042. break;
  2043. case FILE_MEMORY_MIGRATE:
  2044. retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
  2045. break;
  2046. case FILE_MEMORY_PRESSURE_ENABLED:
  2047. cpuset_memory_pressure_enabled = !!val;
  2048. break;
  2049. case FILE_SPREAD_PAGE:
  2050. retval = update_flag(CS_SPREAD_PAGE, cs, val);
  2051. break;
  2052. case FILE_SPREAD_SLAB:
  2053. retval = update_flag(CS_SPREAD_SLAB, cs, val);
  2054. break;
  2055. default:
  2056. retval = -EINVAL;
  2057. break;
  2058. }
  2059. out_unlock:
  2060. mutex_unlock(&cpuset_mutex);
  2061. put_online_cpus();
  2062. return retval;
  2063. }
  2064. static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
  2065. s64 val)
  2066. {
  2067. struct cpuset *cs = css_cs(css);
  2068. cpuset_filetype_t type = cft->private;
  2069. int retval = -ENODEV;
  2070. get_online_cpus();
  2071. mutex_lock(&cpuset_mutex);
  2072. if (!is_cpuset_online(cs))
  2073. goto out_unlock;
  2074. switch (type) {
  2075. case FILE_SCHED_RELAX_DOMAIN_LEVEL:
  2076. retval = update_relax_domain_level(cs, val);
  2077. break;
  2078. default:
  2079. retval = -EINVAL;
  2080. break;
  2081. }
  2082. out_unlock:
  2083. mutex_unlock(&cpuset_mutex);
  2084. put_online_cpus();
  2085. return retval;
  2086. }
  2087. /*
  2088. * Common handling for a write to a "cpus" or "mems" file.
  2089. */
  2090. static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
  2091. char *buf, size_t nbytes, loff_t off)
  2092. {
  2093. struct cpuset *cs = css_cs(of_css(of));
  2094. struct cpuset *trialcs;
  2095. int retval = -ENODEV;
  2096. buf = strstrip(buf);
  2097. /*
  2098. * CPU or memory hotunplug may leave @cs w/o any execution
  2099. * resources, in which case the hotplug code asynchronously updates
  2100. * configuration and transfers all tasks to the nearest ancestor
  2101. * which can execute.
  2102. *
  2103. * As writes to "cpus" or "mems" may restore @cs's execution
  2104. * resources, wait for the previously scheduled operations before
  2105. * proceeding, so that we don't end up keep removing tasks added
  2106. * after execution capability is restored.
  2107. *
  2108. * cpuset_hotplug_work calls back into cgroup core via
  2109. * cgroup_transfer_tasks() and waiting for it from a cgroupfs
  2110. * operation like this one can lead to a deadlock through kernfs
  2111. * active_ref protection. Let's break the protection. Losing the
  2112. * protection is okay as we check whether @cs is online after
  2113. * grabbing cpuset_mutex anyway. This only happens on the legacy
  2114. * hierarchies.
  2115. */
  2116. css_get(&cs->css);
  2117. kernfs_break_active_protection(of->kn);
  2118. flush_work(&cpuset_hotplug_work);
  2119. get_online_cpus();
  2120. mutex_lock(&cpuset_mutex);
  2121. if (!is_cpuset_online(cs))
  2122. goto out_unlock;
  2123. trialcs = alloc_trial_cpuset(cs);
  2124. if (!trialcs) {
  2125. retval = -ENOMEM;
  2126. goto out_unlock;
  2127. }
  2128. switch (of_cft(of)->private) {
  2129. case FILE_CPULIST:
  2130. retval = update_cpumask(cs, trialcs, buf);
  2131. break;
  2132. case FILE_MEMLIST:
  2133. retval = update_nodemask(cs, trialcs, buf);
  2134. break;
  2135. default:
  2136. retval = -EINVAL;
  2137. break;
  2138. }
  2139. free_cpuset(trialcs);
  2140. out_unlock:
  2141. mutex_unlock(&cpuset_mutex);
  2142. put_online_cpus();
  2143. kernfs_unbreak_active_protection(of->kn);
  2144. css_put(&cs->css);
  2145. flush_workqueue(cpuset_migrate_mm_wq);
  2146. return retval ?: nbytes;
  2147. }
  2148. /*
  2149. * These ascii lists should be read in a single call, by using a user
  2150. * buffer large enough to hold the entire map. If read in smaller
  2151. * chunks, there is no guarantee of atomicity. Since the display format
  2152. * used, list of ranges of sequential numbers, is variable length,
  2153. * and since these maps can change value dynamically, one could read
  2154. * gibberish by doing partial reads while a list was changing.
  2155. */
  2156. static int cpuset_common_seq_show(struct seq_file *sf, void *v)
  2157. {
  2158. struct cpuset *cs = css_cs(seq_css(sf));
  2159. cpuset_filetype_t type = seq_cft(sf)->private;
  2160. int ret = 0;
  2161. spin_lock_irq(&callback_lock);
  2162. switch (type) {
  2163. case FILE_CPULIST:
  2164. seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_requested));
  2165. break;
  2166. case FILE_MEMLIST:
  2167. seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed));
  2168. break;
  2169. case FILE_EFFECTIVE_CPULIST:
  2170. seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus));
  2171. break;
  2172. case FILE_EFFECTIVE_MEMLIST:
  2173. seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems));
  2174. break;
  2175. case FILE_SUBPARTS_CPULIST:
  2176. seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->subparts_cpus));
  2177. break;
  2178. default:
  2179. ret = -EINVAL;
  2180. }
  2181. spin_unlock_irq(&callback_lock);
  2182. return ret;
  2183. }
  2184. static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
  2185. {
  2186. struct cpuset *cs = css_cs(css);
  2187. cpuset_filetype_t type = cft->private;
  2188. switch (type) {
  2189. case FILE_CPU_EXCLUSIVE:
  2190. return is_cpu_exclusive(cs);
  2191. case FILE_MEM_EXCLUSIVE:
  2192. return is_mem_exclusive(cs);
  2193. case FILE_MEM_HARDWALL:
  2194. return is_mem_hardwall(cs);
  2195. case FILE_SCHED_LOAD_BALANCE:
  2196. return is_sched_load_balance(cs);
  2197. case FILE_MEMORY_MIGRATE:
  2198. return is_memory_migrate(cs);
  2199. case FILE_MEMORY_PRESSURE_ENABLED:
  2200. return cpuset_memory_pressure_enabled;
  2201. case FILE_MEMORY_PRESSURE:
  2202. return fmeter_getrate(&cs->fmeter);
  2203. case FILE_SPREAD_PAGE:
  2204. return is_spread_page(cs);
  2205. case FILE_SPREAD_SLAB:
  2206. return is_spread_slab(cs);
  2207. default:
  2208. BUG();
  2209. }
  2210. /* Unreachable but makes gcc happy */
  2211. return 0;
  2212. }
  2213. static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
  2214. {
  2215. struct cpuset *cs = css_cs(css);
  2216. cpuset_filetype_t type = cft->private;
  2217. switch (type) {
  2218. case FILE_SCHED_RELAX_DOMAIN_LEVEL:
  2219. return cs->relax_domain_level;
  2220. default:
  2221. BUG();
  2222. }
  2223. /* Unrechable but makes gcc happy */
  2224. return 0;
  2225. }
  2226. static int sched_partition_show(struct seq_file *seq, void *v)
  2227. {
  2228. struct cpuset *cs = css_cs(seq_css(seq));
  2229. switch (cs->partition_root_state) {
  2230. case PRS_ENABLED:
  2231. seq_puts(seq, "root\n");
  2232. break;
  2233. case PRS_DISABLED:
  2234. seq_puts(seq, "member\n");
  2235. break;
  2236. case PRS_ERROR:
  2237. seq_puts(seq, "root invalid\n");
  2238. break;
  2239. }
  2240. return 0;
  2241. }
  2242. static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf,
  2243. size_t nbytes, loff_t off)
  2244. {
  2245. struct cpuset *cs = css_cs(of_css(of));
  2246. int val;
  2247. int retval = -ENODEV;
  2248. buf = strstrip(buf);
  2249. /*
  2250. * Convert "root" to ENABLED, and convert "member" to DISABLED.
  2251. */
  2252. if (!strcmp(buf, "root"))
  2253. val = PRS_ENABLED;
  2254. else if (!strcmp(buf, "member"))
  2255. val = PRS_DISABLED;
  2256. else
  2257. return -EINVAL;
  2258. css_get(&cs->css);
  2259. get_online_cpus();
  2260. mutex_lock(&cpuset_mutex);
  2261. if (!is_cpuset_online(cs))
  2262. goto out_unlock;
  2263. retval = update_prstate(cs, val);
  2264. out_unlock:
  2265. mutex_unlock(&cpuset_mutex);
  2266. put_online_cpus();
  2267. css_put(&cs->css);
  2268. return retval ?: nbytes;
  2269. }
  2270. /*
  2271. * for the common functions, 'private' gives the type of file
  2272. */
  2273. static struct cftype legacy_files[] = {
  2274. {
  2275. .name = "cpus",
  2276. .seq_show = cpuset_common_seq_show,
  2277. .write = cpuset_write_resmask,
  2278. .max_write_len = (100U + 6 * NR_CPUS),
  2279. .private = FILE_CPULIST,
  2280. },
  2281. {
  2282. .name = "mems",
  2283. .seq_show = cpuset_common_seq_show,
  2284. .write = cpuset_write_resmask,
  2285. .max_write_len = (100U + 6 * MAX_NUMNODES),
  2286. .private = FILE_MEMLIST,
  2287. },
  2288. {
  2289. .name = "effective_cpus",
  2290. .seq_show = cpuset_common_seq_show,
  2291. .private = FILE_EFFECTIVE_CPULIST,
  2292. },
  2293. {
  2294. .name = "effective_mems",
  2295. .seq_show = cpuset_common_seq_show,
  2296. .private = FILE_EFFECTIVE_MEMLIST,
  2297. },
  2298. {
  2299. .name = "cpu_exclusive",
  2300. .read_u64 = cpuset_read_u64,
  2301. .write_u64 = cpuset_write_u64,
  2302. .private = FILE_CPU_EXCLUSIVE,
  2303. },
  2304. {
  2305. .name = "mem_exclusive",
  2306. .read_u64 = cpuset_read_u64,
  2307. .write_u64 = cpuset_write_u64,
  2308. .private = FILE_MEM_EXCLUSIVE,
  2309. },
  2310. {
  2311. .name = "mem_hardwall",
  2312. .read_u64 = cpuset_read_u64,
  2313. .write_u64 = cpuset_write_u64,
  2314. .private = FILE_MEM_HARDWALL,
  2315. },
  2316. {
  2317. .name = "sched_load_balance",
  2318. .read_u64 = cpuset_read_u64,
  2319. .write_u64 = cpuset_write_u64,
  2320. .private = FILE_SCHED_LOAD_BALANCE,
  2321. },
  2322. {
  2323. .name = "sched_relax_domain_level",
  2324. .read_s64 = cpuset_read_s64,
  2325. .write_s64 = cpuset_write_s64,
  2326. .private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
  2327. },
  2328. {
  2329. .name = "memory_migrate",
  2330. .read_u64 = cpuset_read_u64,
  2331. .write_u64 = cpuset_write_u64,
  2332. .private = FILE_MEMORY_MIGRATE,
  2333. },
  2334. {
  2335. .name = "memory_pressure",
  2336. .read_u64 = cpuset_read_u64,
  2337. .private = FILE_MEMORY_PRESSURE,
  2338. },
  2339. {
  2340. .name = "memory_spread_page",
  2341. .read_u64 = cpuset_read_u64,
  2342. .write_u64 = cpuset_write_u64,
  2343. .private = FILE_SPREAD_PAGE,
  2344. },
  2345. {
  2346. .name = "memory_spread_slab",
  2347. .read_u64 = cpuset_read_u64,
  2348. .write_u64 = cpuset_write_u64,
  2349. .private = FILE_SPREAD_SLAB,
  2350. },
  2351. {
  2352. .name = "memory_pressure_enabled",
  2353. .flags = CFTYPE_ONLY_ON_ROOT,
  2354. .read_u64 = cpuset_read_u64,
  2355. .write_u64 = cpuset_write_u64,
  2356. .private = FILE_MEMORY_PRESSURE_ENABLED,
  2357. },
  2358. { } /* terminate */
  2359. };
  2360. /*
  2361. * This is currently a minimal set for the default hierarchy. It can be
  2362. * expanded later on by migrating more features and control files from v1.
  2363. */
  2364. static struct cftype dfl_files[] = {
  2365. {
  2366. .name = "cpus",
  2367. .seq_show = cpuset_common_seq_show,
  2368. .write = cpuset_write_resmask,
  2369. .max_write_len = (100U + 6 * NR_CPUS),
  2370. .private = FILE_CPULIST,
  2371. .flags = CFTYPE_NOT_ON_ROOT,
  2372. },
  2373. {
  2374. .name = "mems",
  2375. .seq_show = cpuset_common_seq_show,
  2376. .write = cpuset_write_resmask,
  2377. .max_write_len = (100U + 6 * MAX_NUMNODES),
  2378. .private = FILE_MEMLIST,
  2379. .flags = CFTYPE_NOT_ON_ROOT,
  2380. },
  2381. {
  2382. .name = "cpus.effective",
  2383. .seq_show = cpuset_common_seq_show,
  2384. .private = FILE_EFFECTIVE_CPULIST,
  2385. },
  2386. {
  2387. .name = "mems.effective",
  2388. .seq_show = cpuset_common_seq_show,
  2389. .private = FILE_EFFECTIVE_MEMLIST,
  2390. },
  2391. {
  2392. .name = "cpus.partition",
  2393. .seq_show = sched_partition_show,
  2394. .write = sched_partition_write,
  2395. .private = FILE_PARTITION_ROOT,
  2396. .flags = CFTYPE_NOT_ON_ROOT,
  2397. },
  2398. {
  2399. .name = "cpus.subpartitions",
  2400. .seq_show = cpuset_common_seq_show,
  2401. .private = FILE_SUBPARTS_CPULIST,
  2402. .flags = CFTYPE_DEBUG,
  2403. },
  2404. { } /* terminate */
  2405. };
  2406. /*
  2407. * cpuset_css_alloc - allocate a cpuset css
  2408. * cgrp: control group that the new cpuset will be part of
  2409. */
  2410. static struct cgroup_subsys_state *
  2411. cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
  2412. {
  2413. struct cpuset *cs;
  2414. if (!parent_css)
  2415. return &top_cpuset.css;
  2416. cs = kzalloc(sizeof(*cs), GFP_KERNEL);
  2417. if (!cs)
  2418. return ERR_PTR(-ENOMEM);
  2419. if (alloc_cpumasks(cs, NULL)) {
  2420. kfree(cs);
  2421. return ERR_PTR(-ENOMEM);
  2422. }
  2423. set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
  2424. nodes_clear(cs->mems_allowed);
  2425. nodes_clear(cs->effective_mems);
  2426. fmeter_init(&cs->fmeter);
  2427. cs->relax_domain_level = -1;
  2428. return &cs->css;
  2429. }
  2430. static int cpuset_css_online(struct cgroup_subsys_state *css)
  2431. {
  2432. struct cpuset *cs = css_cs(css);
  2433. struct cpuset *parent = parent_cs(cs);
  2434. struct cpuset *tmp_cs;
  2435. struct cgroup_subsys_state *pos_css;
  2436. if (!parent)
  2437. return 0;
  2438. get_online_cpus();
  2439. mutex_lock(&cpuset_mutex);
  2440. set_bit(CS_ONLINE, &cs->flags);
  2441. if (is_spread_page(parent))
  2442. set_bit(CS_SPREAD_PAGE, &cs->flags);
  2443. if (is_spread_slab(parent))
  2444. set_bit(CS_SPREAD_SLAB, &cs->flags);
  2445. cpuset_inc();
  2446. spin_lock_irq(&callback_lock);
  2447. if (is_in_v2_mode()) {
  2448. cpumask_copy(cs->effective_cpus, parent->effective_cpus);
  2449. cs->effective_mems = parent->effective_mems;
  2450. cs->use_parent_ecpus = true;
  2451. parent->child_ecpus_count++;
  2452. }
  2453. spin_unlock_irq(&callback_lock);
  2454. if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
  2455. goto out_unlock;
  2456. /*
  2457. * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
  2458. * set. This flag handling is implemented in cgroup core for
  2459. * histrical reasons - the flag may be specified during mount.
  2460. *
  2461. * Currently, if any sibling cpusets have exclusive cpus or mem, we
  2462. * refuse to clone the configuration - thereby refusing the task to
  2463. * be entered, and as a result refusing the sys_unshare() or
  2464. * clone() which initiated it. If this becomes a problem for some
  2465. * users who wish to allow that scenario, then this could be
  2466. * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
  2467. * (and likewise for mems) to the new cgroup.
  2468. */
  2469. rcu_read_lock();
  2470. cpuset_for_each_child(tmp_cs, pos_css, parent) {
  2471. if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
  2472. rcu_read_unlock();
  2473. goto out_unlock;
  2474. }
  2475. }
  2476. rcu_read_unlock();
  2477. spin_lock_irq(&callback_lock);
  2478. cs->mems_allowed = parent->mems_allowed;
  2479. cs->effective_mems = parent->mems_allowed;
  2480. cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
  2481. cpumask_copy(cs->cpus_requested, parent->cpus_requested);
  2482. cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
  2483. spin_unlock_irq(&callback_lock);
  2484. out_unlock:
  2485. mutex_unlock(&cpuset_mutex);
  2486. put_online_cpus();
  2487. return 0;
  2488. }
  2489. /*
  2490. * If the cpuset being removed has its flag 'sched_load_balance'
  2491. * enabled, then simulate turning sched_load_balance off, which
  2492. * will call rebuild_sched_domains_locked(). That is not needed
  2493. * in the default hierarchy where only changes in partition
  2494. * will cause repartitioning.
  2495. *
  2496. * If the cpuset has the 'sched.partition' flag enabled, simulate
  2497. * turning 'sched.partition" off.
  2498. */
  2499. static void cpuset_css_offline(struct cgroup_subsys_state *css)
  2500. {
  2501. struct cpuset *cs = css_cs(css);
  2502. get_online_cpus();
  2503. mutex_lock(&cpuset_mutex);
  2504. if (is_partition_root(cs))
  2505. update_prstate(cs, 0);
  2506. if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
  2507. is_sched_load_balance(cs))
  2508. update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
  2509. if (cs->use_parent_ecpus) {
  2510. struct cpuset *parent = parent_cs(cs);
  2511. cs->use_parent_ecpus = false;
  2512. parent->child_ecpus_count--;
  2513. }
  2514. cpuset_dec();
  2515. clear_bit(CS_ONLINE, &cs->flags);
  2516. mutex_unlock(&cpuset_mutex);
  2517. put_online_cpus();
  2518. }
  2519. static void cpuset_css_free(struct cgroup_subsys_state *css)
  2520. {
  2521. struct cpuset *cs = css_cs(css);
  2522. free_cpuset(cs);
  2523. }
  2524. static void cpuset_bind(struct cgroup_subsys_state *root_css)
  2525. {
  2526. mutex_lock(&cpuset_mutex);
  2527. spin_lock_irq(&callback_lock);
  2528. if (is_in_v2_mode()) {
  2529. cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
  2530. top_cpuset.mems_allowed = node_possible_map;
  2531. } else {
  2532. cpumask_copy(top_cpuset.cpus_allowed,
  2533. top_cpuset.effective_cpus);
  2534. top_cpuset.mems_allowed = top_cpuset.effective_mems;
  2535. }
  2536. spin_unlock_irq(&callback_lock);
  2537. mutex_unlock(&cpuset_mutex);
  2538. }
  2539. /*
  2540. * Make sure the new task conform to the current state of its parent,
  2541. * which could have been changed by cpuset just after it inherits the
  2542. * state from the parent and before it sits on the cgroup's task list.
  2543. */
  2544. static void cpuset_fork(struct task_struct *task)
  2545. {
  2546. int inherit_cpus = 0;
  2547. if (task_css_is_root(task, cpuset_cgrp_id))
  2548. return;
  2549. trace_android_rvh_cpuset_fork(task, &inherit_cpus);
  2550. if (!inherit_cpus)
  2551. set_cpus_allowed_ptr(task, current->cpus_ptr);
  2552. task->mems_allowed = current->mems_allowed;
  2553. }
  2554. struct cgroup_subsys cpuset_cgrp_subsys = {
  2555. .css_alloc = cpuset_css_alloc,
  2556. .css_online = cpuset_css_online,
  2557. .css_offline = cpuset_css_offline,
  2558. .css_free = cpuset_css_free,
  2559. .can_attach = cpuset_can_attach,
  2560. .cancel_attach = cpuset_cancel_attach,
  2561. .attach = cpuset_attach,
  2562. .post_attach = cpuset_post_attach,
  2563. .bind = cpuset_bind,
  2564. .fork = cpuset_fork,
  2565. .legacy_cftypes = legacy_files,
  2566. .dfl_cftypes = dfl_files,
  2567. .early_init = true,
  2568. .threaded = true,
  2569. };
  2570. /**
  2571. * cpuset_init - initialize cpusets at system boot
  2572. *
  2573. * Description: Initialize top_cpuset
  2574. **/
  2575. int __init cpuset_init(void)
  2576. {
  2577. BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
  2578. BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
  2579. BUG_ON(!zalloc_cpumask_var(&top_cpuset.subparts_cpus, GFP_KERNEL));
  2580. BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_requested, GFP_KERNEL));
  2581. cpumask_setall(top_cpuset.cpus_allowed);
  2582. cpumask_setall(top_cpuset.cpus_requested);
  2583. nodes_setall(top_cpuset.mems_allowed);
  2584. cpumask_setall(top_cpuset.effective_cpus);
  2585. nodes_setall(top_cpuset.effective_mems);
  2586. fmeter_init(&top_cpuset.fmeter);
  2587. set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
  2588. top_cpuset.relax_domain_level = -1;
  2589. BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL));
  2590. return 0;
  2591. }
  2592. /*
  2593. * If CPU and/or memory hotplug handlers, below, unplug any CPUs
  2594. * or memory nodes, we need to walk over the cpuset hierarchy,
  2595. * removing that CPU or node from all cpusets. If this removes the
  2596. * last CPU or node from a cpuset, then move the tasks in the empty
  2597. * cpuset to its next-highest non-empty parent.
  2598. */
  2599. static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
  2600. {
  2601. struct cpuset *parent;
  2602. /*
  2603. * Find its next-highest non-empty parent, (top cpuset
  2604. * has online cpus, so can't be empty).
  2605. */
  2606. parent = parent_cs(cs);
  2607. while (cpumask_empty(parent->cpus_allowed) ||
  2608. nodes_empty(parent->mems_allowed))
  2609. parent = parent_cs(parent);
  2610. if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
  2611. pr_err("cpuset: failed to transfer tasks out of empty cpuset ");
  2612. pr_cont_cgroup_name(cs->css.cgroup);
  2613. pr_cont("\n");
  2614. }
  2615. }
  2616. static void
  2617. hotplug_update_tasks_legacy(struct cpuset *cs,
  2618. struct cpumask *new_cpus, nodemask_t *new_mems,
  2619. bool cpus_updated, bool mems_updated)
  2620. {
  2621. bool is_empty;
  2622. spin_lock_irq(&callback_lock);
  2623. cpumask_copy(cs->cpus_allowed, new_cpus);
  2624. cpumask_copy(cs->effective_cpus, new_cpus);
  2625. cs->mems_allowed = *new_mems;
  2626. cs->effective_mems = *new_mems;
  2627. spin_unlock_irq(&callback_lock);
  2628. /*
  2629. * Don't call update_tasks_cpumask() if the cpuset becomes empty,
  2630. * as the tasks will be migratecd to an ancestor.
  2631. */
  2632. if (cpus_updated && !cpumask_empty(cs->cpus_allowed))
  2633. update_tasks_cpumask(cs);
  2634. if (mems_updated && !nodes_empty(cs->mems_allowed))
  2635. update_tasks_nodemask(cs);
  2636. is_empty = cpumask_empty(cs->cpus_allowed) ||
  2637. nodes_empty(cs->mems_allowed);
  2638. mutex_unlock(&cpuset_mutex);
  2639. /*
  2640. * Move tasks to the nearest ancestor with execution resources,
  2641. * This is full cgroup operation which will also call back into
  2642. * cpuset. Should be done outside any lock.
  2643. */
  2644. if (is_empty)
  2645. remove_tasks_in_empty_cpuset(cs);
  2646. mutex_lock(&cpuset_mutex);
  2647. }
  2648. static void
  2649. hotplug_update_tasks(struct cpuset *cs,
  2650. struct cpumask *new_cpus, nodemask_t *new_mems,
  2651. bool cpus_updated, bool mems_updated)
  2652. {
  2653. if (cpumask_empty(new_cpus))
  2654. cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus);
  2655. if (nodes_empty(*new_mems))
  2656. *new_mems = parent_cs(cs)->effective_mems;
  2657. spin_lock_irq(&callback_lock);
  2658. cpumask_copy(cs->effective_cpus, new_cpus);
  2659. cs->effective_mems = *new_mems;
  2660. spin_unlock_irq(&callback_lock);
  2661. if (cpus_updated)
  2662. update_tasks_cpumask(cs);
  2663. if (mems_updated)
  2664. update_tasks_nodemask(cs);
  2665. }
  2666. static bool force_rebuild;
  2667. void cpuset_force_rebuild(void)
  2668. {
  2669. force_rebuild = true;
  2670. }
  2671. /**
  2672. * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
  2673. * @cs: cpuset in interest
  2674. * @tmp: the tmpmasks structure pointer
  2675. *
  2676. * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
  2677. * offline, update @cs accordingly. If @cs ends up with no CPU or memory,
  2678. * all its tasks are moved to the nearest ancestor with both resources.
  2679. */
  2680. static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
  2681. {
  2682. static cpumask_t new_cpus;
  2683. static nodemask_t new_mems;
  2684. bool cpus_updated;
  2685. bool mems_updated;
  2686. struct cpuset *parent;
  2687. retry:
  2688. wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
  2689. mutex_lock(&cpuset_mutex);
  2690. /*
  2691. * We have raced with task attaching. We wait until attaching
  2692. * is finished, so we won't attach a task to an empty cpuset.
  2693. */
  2694. if (cs->attach_in_progress) {
  2695. mutex_unlock(&cpuset_mutex);
  2696. goto retry;
  2697. }
  2698. parent = parent_cs(cs);
  2699. compute_effective_cpumask(&new_cpus, cs, parent);
  2700. nodes_and(new_mems, cs->mems_allowed, parent->effective_mems);
  2701. if (cs->nr_subparts_cpus)
  2702. /*
  2703. * Make sure that CPUs allocated to child partitions
  2704. * do not show up in effective_cpus.
  2705. */
  2706. cpumask_andnot(&new_cpus, &new_cpus, cs->subparts_cpus);
  2707. if (!tmp || !cs->partition_root_state)
  2708. goto update_tasks;
  2709. /*
  2710. * In the unlikely event that a partition root has empty
  2711. * effective_cpus or its parent becomes erroneous, we have to
  2712. * transition it to the erroneous state.
  2713. */
  2714. if (is_partition_root(cs) && (cpumask_empty(&new_cpus) ||
  2715. (parent->partition_root_state == PRS_ERROR))) {
  2716. if (cs->nr_subparts_cpus) {
  2717. spin_lock_irq(&callback_lock);
  2718. cs->nr_subparts_cpus = 0;
  2719. cpumask_clear(cs->subparts_cpus);
  2720. spin_unlock_irq(&callback_lock);
  2721. compute_effective_cpumask(&new_cpus, cs, parent);
  2722. }
  2723. /*
  2724. * If the effective_cpus is empty because the child
  2725. * partitions take away all the CPUs, we can keep
  2726. * the current partition and let the child partitions
  2727. * fight for available CPUs.
  2728. */
  2729. if ((parent->partition_root_state == PRS_ERROR) ||
  2730. cpumask_empty(&new_cpus)) {
  2731. update_parent_subparts_cpumask(cs, partcmd_disable,
  2732. NULL, tmp);
  2733. spin_lock_irq(&callback_lock);
  2734. cs->partition_root_state = PRS_ERROR;
  2735. spin_unlock_irq(&callback_lock);
  2736. }
  2737. cpuset_force_rebuild();
  2738. }
  2739. /*
  2740. * On the other hand, an erroneous partition root may be transitioned
  2741. * back to a regular one or a partition root with no CPU allocated
  2742. * from the parent may change to erroneous.
  2743. */
  2744. if (is_partition_root(parent) &&
  2745. ((cs->partition_root_state == PRS_ERROR) ||
  2746. !cpumask_intersects(&new_cpus, parent->subparts_cpus)) &&
  2747. update_parent_subparts_cpumask(cs, partcmd_update, NULL, tmp))
  2748. cpuset_force_rebuild();
  2749. update_tasks:
  2750. cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus);
  2751. mems_updated = !nodes_equal(new_mems, cs->effective_mems);
  2752. if (is_in_v2_mode())
  2753. hotplug_update_tasks(cs, &new_cpus, &new_mems,
  2754. cpus_updated, mems_updated);
  2755. else
  2756. hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems,
  2757. cpus_updated, mems_updated);
  2758. mutex_unlock(&cpuset_mutex);
  2759. }
  2760. /**
  2761. * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
  2762. *
  2763. * This function is called after either CPU or memory configuration has
  2764. * changed and updates cpuset accordingly. The top_cpuset is always
  2765. * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
  2766. * order to make cpusets transparent (of no affect) on systems that are
  2767. * actively using CPU hotplug but making no active use of cpusets.
  2768. *
  2769. * Non-root cpusets are only affected by offlining. If any CPUs or memory
  2770. * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
  2771. * all descendants.
  2772. *
  2773. * Note that CPU offlining during suspend is ignored. We don't modify
  2774. * cpusets across suspend/resume cycles at all.
  2775. */
  2776. void cpuset_hotplug_workfn(struct work_struct *work)
  2777. {
  2778. static cpumask_t new_cpus;
  2779. static nodemask_t new_mems;
  2780. bool cpus_updated, mems_updated;
  2781. bool on_dfl = is_in_v2_mode();
  2782. struct tmpmasks tmp, *ptmp = NULL;
  2783. if (on_dfl && !alloc_cpumasks(NULL, &tmp))
  2784. ptmp = &tmp;
  2785. mutex_lock(&cpuset_mutex);
  2786. /* fetch the available cpus/mems and find out which changed how */
  2787. cpumask_copy(&new_cpus, cpu_active_mask);
  2788. new_mems = node_states[N_MEMORY];
  2789. /*
  2790. * If subparts_cpus is populated, it is likely that the check below
  2791. * will produce a false positive on cpus_updated when the cpu list
  2792. * isn't changed. It is extra work, but it is better to be safe.
  2793. */
  2794. cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus);
  2795. mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);
  2796. /*
  2797. * In the rare case that hotplug removes all the cpus in subparts_cpus,
  2798. * we assumed that cpus are updated.
  2799. */
  2800. if (!cpus_updated && top_cpuset.nr_subparts_cpus)
  2801. cpus_updated = true;
  2802. /* synchronize cpus_allowed to cpu_active_mask */
  2803. if (cpus_updated) {
  2804. spin_lock_irq(&callback_lock);
  2805. if (!on_dfl)
  2806. cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
  2807. /*
  2808. * Make sure that CPUs allocated to child partitions
  2809. * do not show up in effective_cpus. If no CPU is left,
  2810. * we clear the subparts_cpus & let the child partitions
  2811. * fight for the CPUs again.
  2812. */
  2813. if (top_cpuset.nr_subparts_cpus) {
  2814. if (cpumask_subset(&new_cpus,
  2815. top_cpuset.subparts_cpus)) {
  2816. top_cpuset.nr_subparts_cpus = 0;
  2817. cpumask_clear(top_cpuset.subparts_cpus);
  2818. } else {
  2819. cpumask_andnot(&new_cpus, &new_cpus,
  2820. top_cpuset.subparts_cpus);
  2821. }
  2822. }
  2823. cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
  2824. spin_unlock_irq(&callback_lock);
  2825. /* we don't mess with cpumasks of tasks in top_cpuset */
  2826. }
  2827. /* synchronize mems_allowed to N_MEMORY */
  2828. if (mems_updated) {
  2829. spin_lock_irq(&callback_lock);
  2830. if (!on_dfl)
  2831. top_cpuset.mems_allowed = new_mems;
  2832. top_cpuset.effective_mems = new_mems;
  2833. spin_unlock_irq(&callback_lock);
  2834. update_tasks_nodemask(&top_cpuset);
  2835. }
  2836. mutex_unlock(&cpuset_mutex);
  2837. /* if cpus or mems changed, we need to propagate to descendants */
  2838. if (cpus_updated || mems_updated) {
  2839. struct cpuset *cs;
  2840. struct cgroup_subsys_state *pos_css;
  2841. rcu_read_lock();
  2842. cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
  2843. if (cs == &top_cpuset || !css_tryget_online(&cs->css))
  2844. continue;
  2845. rcu_read_unlock();
  2846. cpuset_hotplug_update_tasks(cs, ptmp);
  2847. rcu_read_lock();
  2848. css_put(&cs->css);
  2849. }
  2850. rcu_read_unlock();
  2851. }
  2852. /* rebuild sched domains if cpus_allowed has changed */
  2853. if (cpus_updated || force_rebuild) {
  2854. force_rebuild = false;
  2855. rebuild_sched_domains();
  2856. }
  2857. free_cpumasks(NULL, ptmp);
  2858. }
  2859. void cpuset_update_active_cpus(void)
  2860. {
  2861. /*
  2862. * We're inside cpu hotplug critical region which usually nests
  2863. * inside cgroup synchronization. Bounce actual hotplug processing
  2864. * to a work item to avoid reverse locking order.
  2865. */
  2866. schedule_work(&cpuset_hotplug_work);
  2867. }
  2868. void cpuset_update_active_cpus_affine(int cpu)
  2869. {
  2870. schedule_work_on(cpu, &cpuset_hotplug_work);
  2871. }
  2872. void cpuset_wait_for_hotplug(void)
  2873. {
  2874. flush_work(&cpuset_hotplug_work);
  2875. }
  2876. /*
  2877. * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
  2878. * Call this routine anytime after node_states[N_MEMORY] changes.
  2879. * See cpuset_update_active_cpus() for CPU hotplug handling.
  2880. */
  2881. static int cpuset_track_online_nodes(struct notifier_block *self,
  2882. unsigned long action, void *arg)
  2883. {
  2884. schedule_work(&cpuset_hotplug_work);
  2885. return NOTIFY_OK;
  2886. }
  2887. static struct notifier_block cpuset_track_online_nodes_nb = {
  2888. .notifier_call = cpuset_track_online_nodes,
  2889. .priority = 10, /* ??! */
  2890. };
  2891. /**
  2892. * cpuset_init_smp - initialize cpus_allowed
  2893. *
  2894. * Description: Finish top cpuset after cpu, node maps are initialized
  2895. */
  2896. void __init cpuset_init_smp(void)
  2897. {
  2898. cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
  2899. top_cpuset.mems_allowed = node_states[N_MEMORY];
  2900. top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
  2901. cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
  2902. top_cpuset.effective_mems = node_states[N_MEMORY];
  2903. register_hotmemory_notifier(&cpuset_track_online_nodes_nb);
  2904. cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
  2905. BUG_ON(!cpuset_migrate_mm_wq);
  2906. }
  2907. /**
  2908. * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
  2909. * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
  2910. * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
  2911. *
  2912. * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
  2913. * attached to the specified @tsk. Guaranteed to return some non-empty
  2914. * subset of cpu_online_mask, even if this means going outside the
  2915. * tasks cpuset.
  2916. **/
  2917. void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
  2918. {
  2919. unsigned long flags;
  2920. spin_lock_irqsave(&callback_lock, flags);
  2921. rcu_read_lock();
  2922. guarantee_online_cpus(tsk, pmask);
  2923. rcu_read_unlock();
  2924. spin_unlock_irqrestore(&callback_lock, flags);
  2925. }
  2926. EXPORT_SYMBOL_GPL(cpuset_cpus_allowed);
  2927. /**
  2928. * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
  2929. * @tsk: pointer to task_struct with which the scheduler is struggling
  2930. *
  2931. * Description: In the case that the scheduler cannot find an allowed cpu in
  2932. * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
  2933. * mode however, this value is the same as task_cs(tsk)->effective_cpus,
  2934. * which will not contain a sane cpumask during cases such as cpu hotplugging.
  2935. * This is the absolute last resort for the scheduler and it is only used if
  2936. * _every_ other avenue has been traveled.
  2937. **/
  2938. void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
  2939. {
  2940. const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
  2941. const struct cpumask *cs_mask;
  2942. rcu_read_lock();
  2943. cs_mask = task_cs(tsk)->cpus_allowed;
  2944. if (!is_in_v2_mode() || !cpumask_subset(cs_mask, possible_mask))
  2945. goto unlock; /* select_fallback_rq will try harder */
  2946. do_set_cpus_allowed(tsk, cs_mask);
  2947. unlock:
  2948. rcu_read_unlock();
  2949. /*
  2950. * We own tsk->cpus_allowed, nobody can change it under us.
  2951. *
  2952. * But we used cs && cs->cpus_allowed lockless and thus can
  2953. * race with cgroup_attach_task() or update_cpumask() and get
  2954. * the wrong tsk->cpus_allowed. However, both cases imply the
  2955. * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
  2956. * which takes task_rq_lock().
  2957. *
  2958. * If we are called after it dropped the lock we must see all
  2959. * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
  2960. * set any mask even if it is not right from task_cs() pov,
  2961. * the pending set_cpus_allowed_ptr() will fix things.
  2962. *
  2963. * select_fallback_rq() will fix things ups and set cpu_possible_mask
  2964. * if required.
  2965. */
  2966. }
  2967. void __init cpuset_init_current_mems_allowed(void)
  2968. {
  2969. nodes_setall(current->mems_allowed);
  2970. }
  2971. /**
  2972. * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
  2973. * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
  2974. *
  2975. * Description: Returns the nodemask_t mems_allowed of the cpuset
  2976. * attached to the specified @tsk. Guaranteed to return some non-empty
  2977. * subset of node_states[N_MEMORY], even if this means going outside the
  2978. * tasks cpuset.
  2979. **/
  2980. nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
  2981. {
  2982. nodemask_t mask;
  2983. unsigned long flags;
  2984. spin_lock_irqsave(&callback_lock, flags);
  2985. rcu_read_lock();
  2986. guarantee_online_mems(task_cs(tsk), &mask);
  2987. rcu_read_unlock();
  2988. spin_unlock_irqrestore(&callback_lock, flags);
  2989. return mask;
  2990. }
  2991. /**
  2992. * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
  2993. * @nodemask: the nodemask to be checked
  2994. *
  2995. * Are any of the nodes in the nodemask allowed in current->mems_allowed?
  2996. */
  2997. int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
  2998. {
  2999. return nodes_intersects(*nodemask, current->mems_allowed);
  3000. }
  3001. /*
  3002. * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
  3003. * mem_hardwall ancestor to the specified cpuset. Call holding
  3004. * callback_lock. If no ancestor is mem_exclusive or mem_hardwall
  3005. * (an unusual configuration), then returns the root cpuset.
  3006. */
  3007. static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
  3008. {
  3009. while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
  3010. cs = parent_cs(cs);
  3011. return cs;
  3012. }
  3013. /**
  3014. * cpuset_node_allowed - Can we allocate on a memory node?
  3015. * @node: is this an allowed node?
  3016. * @gfp_mask: memory allocation flags
  3017. *
  3018. * If we're in interrupt, yes, we can always allocate. If @node is set in
  3019. * current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this
  3020. * node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
  3021. * yes. If current has access to memory reserves as an oom victim, yes.
  3022. * Otherwise, no.
  3023. *
  3024. * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
  3025. * and do not allow allocations outside the current tasks cpuset
  3026. * unless the task has been OOM killed.
  3027. * GFP_KERNEL allocations are not so marked, so can escape to the
  3028. * nearest enclosing hardwalled ancestor cpuset.
  3029. *
  3030. * Scanning up parent cpusets requires callback_lock. The
  3031. * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
  3032. * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
  3033. * current tasks mems_allowed came up empty on the first pass over
  3034. * the zonelist. So only GFP_KERNEL allocations, if all nodes in the
  3035. * cpuset are short of memory, might require taking the callback_lock.
  3036. *
  3037. * The first call here from mm/page_alloc:get_page_from_freelist()
  3038. * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
  3039. * so no allocation on a node outside the cpuset is allowed (unless
  3040. * in interrupt, of course).
  3041. *
  3042. * The second pass through get_page_from_freelist() doesn't even call
  3043. * here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
  3044. * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
  3045. * in alloc_flags. That logic and the checks below have the combined
  3046. * affect that:
  3047. * in_interrupt - any node ok (current task context irrelevant)
  3048. * GFP_ATOMIC - any node ok
  3049. * tsk_is_oom_victim - any node ok
  3050. * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
  3051. * GFP_USER - only nodes in current tasks mems allowed ok.
  3052. */
  3053. bool __cpuset_node_allowed(int node, gfp_t gfp_mask)
  3054. {
  3055. struct cpuset *cs; /* current cpuset ancestors */
  3056. int allowed; /* is allocation in zone z allowed? */
  3057. unsigned long flags;
  3058. if (in_interrupt())
  3059. return true;
  3060. if (node_isset(node, current->mems_allowed))
  3061. return true;
  3062. /*
  3063. * Allow tasks that have access to memory reserves because they have
  3064. * been OOM killed to get memory anywhere.
  3065. */
  3066. if (unlikely(tsk_is_oom_victim(current)))
  3067. return true;
  3068. if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
  3069. return false;
  3070. if (current->flags & PF_EXITING) /* Let dying task have memory */
  3071. return true;
  3072. /* Not hardwall and node outside mems_allowed: scan up cpusets */
  3073. spin_lock_irqsave(&callback_lock, flags);
  3074. rcu_read_lock();
  3075. cs = nearest_hardwall_ancestor(task_cs(current));
  3076. allowed = node_isset(node, cs->mems_allowed);
  3077. rcu_read_unlock();
  3078. spin_unlock_irqrestore(&callback_lock, flags);
  3079. return allowed;
  3080. }
  3081. /**
  3082. * cpuset_mem_spread_node() - On which node to begin search for a file page
  3083. * cpuset_slab_spread_node() - On which node to begin search for a slab page
  3084. *
  3085. * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
  3086. * tasks in a cpuset with is_spread_page or is_spread_slab set),
  3087. * and if the memory allocation used cpuset_mem_spread_node()
  3088. * to determine on which node to start looking, as it will for
  3089. * certain page cache or slab cache pages such as used for file
  3090. * system buffers and inode caches, then instead of starting on the
  3091. * local node to look for a free page, rather spread the starting
  3092. * node around the tasks mems_allowed nodes.
  3093. *
  3094. * We don't have to worry about the returned node being offline
  3095. * because "it can't happen", and even if it did, it would be ok.
  3096. *
  3097. * The routines calling guarantee_online_mems() are careful to
  3098. * only set nodes in task->mems_allowed that are online. So it
  3099. * should not be possible for the following code to return an
  3100. * offline node. But if it did, that would be ok, as this routine
  3101. * is not returning the node where the allocation must be, only
  3102. * the node where the search should start. The zonelist passed to
  3103. * __alloc_pages() will include all nodes. If the slab allocator
  3104. * is passed an offline node, it will fall back to the local node.
  3105. * See kmem_cache_alloc_node().
  3106. */
  3107. static int cpuset_spread_node(int *rotor)
  3108. {
  3109. return *rotor = next_node_in(*rotor, current->mems_allowed);
  3110. }
  3111. int cpuset_mem_spread_node(void)
  3112. {
  3113. if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
  3114. current->cpuset_mem_spread_rotor =
  3115. node_random(&current->mems_allowed);
  3116. return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
  3117. }
  3118. int cpuset_slab_spread_node(void)
  3119. {
  3120. if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
  3121. current->cpuset_slab_spread_rotor =
  3122. node_random(&current->mems_allowed);
  3123. return cpuset_spread_node(&current->cpuset_slab_spread_rotor);
  3124. }
  3125. EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
  3126. /**
  3127. * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
  3128. * @tsk1: pointer to task_struct of some task.
  3129. * @tsk2: pointer to task_struct of some other task.
  3130. *
  3131. * Description: Return true if @tsk1's mems_allowed intersects the
  3132. * mems_allowed of @tsk2. Used by the OOM killer to determine if
  3133. * one of the task's memory usage might impact the memory available
  3134. * to the other.
  3135. **/
  3136. int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
  3137. const struct task_struct *tsk2)
  3138. {
  3139. return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
  3140. }
  3141. /**
  3142. * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
  3143. *
  3144. * Description: Prints current's name, cpuset name, and cached copy of its
  3145. * mems_allowed to the kernel log.
  3146. */
  3147. void cpuset_print_current_mems_allowed(void)
  3148. {
  3149. struct cgroup *cgrp;
  3150. rcu_read_lock();
  3151. cgrp = task_cs(current)->css.cgroup;
  3152. pr_cont(",cpuset=");
  3153. pr_cont_cgroup_name(cgrp);
  3154. pr_cont(",mems_allowed=%*pbl",
  3155. nodemask_pr_args(&current->mems_allowed));
  3156. rcu_read_unlock();
  3157. }
  3158. /*
  3159. * Collection of memory_pressure is suppressed unless
  3160. * this flag is enabled by writing "1" to the special
  3161. * cpuset file 'memory_pressure_enabled' in the root cpuset.
  3162. */
  3163. int cpuset_memory_pressure_enabled __read_mostly;
  3164. /**
  3165. * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
  3166. *
  3167. * Keep a running average of the rate of synchronous (direct)
  3168. * page reclaim efforts initiated by tasks in each cpuset.
  3169. *
  3170. * This represents the rate at which some task in the cpuset
  3171. * ran low on memory on all nodes it was allowed to use, and
  3172. * had to enter the kernels page reclaim code in an effort to
  3173. * create more free memory by tossing clean pages or swapping
  3174. * or writing dirty pages.
  3175. *
  3176. * Display to user space in the per-cpuset read-only file
  3177. * "memory_pressure". Value displayed is an integer
  3178. * representing the recent rate of entry into the synchronous
  3179. * (direct) page reclaim by any task attached to the cpuset.
  3180. **/
  3181. void __cpuset_memory_pressure_bump(void)
  3182. {
  3183. rcu_read_lock();
  3184. fmeter_markevent(&task_cs(current)->fmeter);
  3185. rcu_read_unlock();
  3186. }
  3187. #ifdef CONFIG_PROC_PID_CPUSET
  3188. /*
  3189. * proc_cpuset_show()
  3190. * - Print tasks cpuset path into seq_file.
  3191. * - Used for /proc/<pid>/cpuset.
  3192. * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
  3193. * doesn't really matter if tsk->cpuset changes after we read it,
  3194. * and we take cpuset_mutex, keeping cpuset_attach() from changing it
  3195. * anyway.
  3196. */
  3197. int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns,
  3198. struct pid *pid, struct task_struct *tsk)
  3199. {
  3200. char *buf;
  3201. struct cgroup_subsys_state *css;
  3202. int retval;
  3203. retval = -ENOMEM;
  3204. buf = kmalloc(PATH_MAX, GFP_KERNEL);
  3205. if (!buf)
  3206. goto out;
  3207. css = task_get_css(tsk, cpuset_cgrp_id);
  3208. retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX,
  3209. current->nsproxy->cgroup_ns);
  3210. css_put(css);
  3211. if (retval >= PATH_MAX)
  3212. retval = -ENAMETOOLONG;
  3213. if (retval < 0)
  3214. goto out_free;
  3215. seq_puts(m, buf);
  3216. seq_putc(m, '\n');
  3217. retval = 0;
  3218. out_free:
  3219. kfree(buf);
  3220. out:
  3221. return retval;
  3222. }
  3223. #endif /* CONFIG_PROC_PID_CPUSET */
  3224. /* Display task mems_allowed in /proc/<pid>/status file. */
  3225. void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
  3226. {
  3227. seq_printf(m, "Mems_allowed:\t%*pb\n",
  3228. nodemask_pr_args(&task->mems_allowed));
  3229. seq_printf(m, "Mems_allowed_list:\t%*pbl\n",
  3230. nodemask_pr_args(&task->mems_allowed));
  3231. }