sched.h 71 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Scheduler internal types and methods:
  4. */
  5. #include <linux/sched.h>
  6. #include <linux/sched/autogroup.h>
  7. #include <linux/sched/clock.h>
  8. #include <linux/sched/coredump.h>
  9. #include <linux/sched/cpufreq.h>
  10. #include <linux/sched/cputime.h>
  11. #include <linux/sched/deadline.h>
  12. #include <linux/sched/debug.h>
  13. #include <linux/sched/hotplug.h>
  14. #include <linux/sched/idle.h>
  15. #include <linux/sched/init.h>
  16. #include <linux/sched/isolation.h>
  17. #include <linux/sched/jobctl.h>
  18. #include <linux/sched/loadavg.h>
  19. #include <linux/sched/mm.h>
  20. #include <linux/sched/nohz.h>
  21. #include <linux/sched/numa_balancing.h>
  22. #include <linux/sched/prio.h>
  23. #include <linux/sched/rt.h>
  24. #include <linux/sched/signal.h>
  25. #include <linux/sched/smt.h>
  26. #include <linux/sched/stat.h>
  27. #include <linux/sched/sysctl.h>
  28. #include <linux/sched/task.h>
  29. #include <linux/sched/task_stack.h>
  30. #include <linux/sched/topology.h>
  31. #include <linux/sched/user.h>
  32. #include <linux/sched/wake_q.h>
  33. #include <linux/sched/xacct.h>
  34. #include <uapi/linux/sched/types.h>
  35. #include <linux/binfmts.h>
  36. #include <linux/blkdev.h>
  37. #include <linux/compat.h>
  38. #include <linux/context_tracking.h>
  39. #include <linux/cpufreq.h>
  40. #include <linux/cpuidle.h>
  41. #include <linux/cpuset.h>
  42. #include <linux/ctype.h>
  43. #include <linux/debugfs.h>
  44. #include <linux/delayacct.h>
  45. #include <linux/energy_model.h>
  46. #include <linux/init_task.h>
  47. #include <linux/kprobes.h>
  48. #include <linux/kthread.h>
  49. #include <linux/membarrier.h>
  50. #include <linux/migrate.h>
  51. #include <linux/mmu_context.h>
  52. #include <linux/nmi.h>
  53. #include <linux/proc_fs.h>
  54. #include <linux/prefetch.h>
  55. #include <linux/profile.h>
  56. #include <linux/psi.h>
  57. #include <linux/rcupdate_wait.h>
  58. #include <linux/security.h>
  59. #include <linux/stop_machine.h>
  60. #include <linux/suspend.h>
  61. #include <linux/swait.h>
  62. #include <linux/syscalls.h>
  63. #include <linux/task_work.h>
  64. #include <linux/tsacct_kern.h>
  65. #include <linux/android_vendor.h>
  66. #include <linux/android_kabi.h>
  67. #include <asm/tlb.h>
  68. #include <asm-generic/vmlinux.lds.h>
  69. #ifdef CONFIG_PARAVIRT
  70. # include <asm/paravirt.h>
  71. #endif
  72. #include "cpupri.h"
  73. #include "cpudeadline.h"
  74. #include <trace/events/sched.h>
  75. #ifdef CONFIG_SCHED_DEBUG
  76. # define SCHED_WARN_ON(x) WARN_ONCE(x, #x)
  77. #else
  78. # define SCHED_WARN_ON(x) ({ (void)(x), 0; })
  79. #endif
  80. struct rq;
  81. struct cpuidle_state;
  82. /* task_struct::on_rq states: */
  83. #define TASK_ON_RQ_QUEUED 1
  84. #define TASK_ON_RQ_MIGRATING 2
  85. extern __read_mostly int scheduler_running;
  86. extern unsigned long calc_load_update;
  87. extern atomic_long_t calc_load_tasks;
  88. extern void calc_global_load_tick(struct rq *this_rq);
  89. extern long calc_load_fold_active(struct rq *this_rq, long adjust);
  90. extern void call_trace_sched_update_nr_running(struct rq *rq, int count);
  91. /*
  92. * Helpers for converting nanosecond timing to jiffy resolution
  93. */
  94. #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
  95. /*
  96. * Increase resolution of nice-level calculations for 64-bit architectures.
  97. * The extra resolution improves shares distribution and load balancing of
  98. * low-weight task groups (eg. nice +19 on an autogroup), deeper taskgroup
  99. * hierarchies, especially on larger systems. This is not a user-visible change
  100. * and does not change the user-interface for setting shares/weights.
  101. *
  102. * We increase resolution only if we have enough bits to allow this increased
  103. * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit
  104. * are pretty high and the returns do not justify the increased costs.
  105. *
  106. * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to
  107. * increase coverage and consistency always enable it on 64-bit platforms.
  108. */
  109. #ifdef CONFIG_64BIT
  110. # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)
  111. # define scale_load(w) ((w) << SCHED_FIXEDPOINT_SHIFT)
  112. # define scale_load_down(w) \
  113. ({ \
  114. unsigned long __w = (w); \
  115. if (__w) \
  116. __w = max(2UL, __w >> SCHED_FIXEDPOINT_SHIFT); \
  117. __w; \
  118. })
  119. #else
  120. # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT)
  121. # define scale_load(w) (w)
  122. # define scale_load_down(w) (w)
  123. #endif
  124. /*
  125. * Task weight (visible to users) and its load (invisible to users) have
  126. * independent resolution, but they should be well calibrated. We use
  127. * scale_load() and scale_load_down(w) to convert between them. The
  128. * following must be true:
  129. *
  130. * scale_load(sched_prio_to_weight[USER_PRIO(NICE_TO_PRIO(0))]) == NICE_0_LOAD
  131. *
  132. */
  133. #define NICE_0_LOAD (1L << NICE_0_LOAD_SHIFT)
  134. /*
  135. * Single value that decides SCHED_DEADLINE internal math precision.
  136. * 10 -> just above 1us
  137. * 9 -> just above 0.5us
  138. */
  139. #define DL_SCALE 10
  140. /*
  141. * Single value that denotes runtime == period, ie unlimited time.
  142. */
  143. #define RUNTIME_INF ((u64)~0ULL)
  144. static inline int idle_policy(int policy)
  145. {
  146. return policy == SCHED_IDLE;
  147. }
  148. static inline int fair_policy(int policy)
  149. {
  150. return policy == SCHED_NORMAL || policy == SCHED_BATCH;
  151. }
  152. static inline int rt_policy(int policy)
  153. {
  154. return policy == SCHED_FIFO || policy == SCHED_RR;
  155. }
  156. static inline int dl_policy(int policy)
  157. {
  158. return policy == SCHED_DEADLINE;
  159. }
  160. static inline bool valid_policy(int policy)
  161. {
  162. return idle_policy(policy) || fair_policy(policy) ||
  163. rt_policy(policy) || dl_policy(policy);
  164. }
  165. static inline int task_has_idle_policy(struct task_struct *p)
  166. {
  167. return idle_policy(p->policy);
  168. }
  169. static inline int task_has_rt_policy(struct task_struct *p)
  170. {
  171. return rt_policy(p->policy);
  172. }
  173. static inline int task_has_dl_policy(struct task_struct *p)
  174. {
  175. return dl_policy(p->policy);
  176. }
  177. #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT)
  178. static inline void update_avg(u64 *avg, u64 sample)
  179. {
  180. s64 diff = sample - *avg;
  181. *avg += diff / 8;
  182. }
  183. /*
  184. * Shifting a value by an exponent greater *or equal* to the size of said value
  185. * is UB; cap at size-1.
  186. */
  187. #define shr_bound(val, shift) \
  188. (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1))
  189. /*
  190. * !! For sched_setattr_nocheck() (kernel) only !!
  191. *
  192. * This is actually gross. :(
  193. *
  194. * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE
  195. * tasks, but still be able to sleep. We need this on platforms that cannot
  196. * atomically change clock frequency. Remove once fast switching will be
  197. * available on such platforms.
  198. *
  199. * SUGOV stands for SchedUtil GOVernor.
  200. */
  201. #define SCHED_FLAG_SUGOV 0x10000000
  202. #define SCHED_DL_FLAGS (SCHED_FLAG_RECLAIM | SCHED_FLAG_DL_OVERRUN | SCHED_FLAG_SUGOV)
  203. static inline bool dl_entity_is_special(struct sched_dl_entity *dl_se)
  204. {
  205. #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
  206. return unlikely(dl_se->flags & SCHED_FLAG_SUGOV);
  207. #else
  208. return false;
  209. #endif
  210. }
  211. /*
  212. * Tells if entity @a should preempt entity @b.
  213. */
  214. static inline bool
  215. dl_entity_preempt(struct sched_dl_entity *a, struct sched_dl_entity *b)
  216. {
  217. return dl_entity_is_special(a) ||
  218. dl_time_before(a->deadline, b->deadline);
  219. }
  220. /*
  221. * This is the priority-queue data structure of the RT scheduling class:
  222. */
  223. struct rt_prio_array {
  224. DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
  225. struct list_head queue[MAX_RT_PRIO];
  226. };
  227. struct rt_bandwidth {
  228. /* nests inside the rq lock: */
  229. raw_spinlock_t rt_runtime_lock;
  230. ktime_t rt_period;
  231. u64 rt_runtime;
  232. struct hrtimer rt_period_timer;
  233. unsigned int rt_period_active;
  234. };
  235. void __dl_clear_params(struct task_struct *p);
  236. struct dl_bandwidth {
  237. raw_spinlock_t dl_runtime_lock;
  238. u64 dl_runtime;
  239. u64 dl_period;
  240. };
  241. static inline int dl_bandwidth_enabled(void)
  242. {
  243. return sysctl_sched_rt_runtime >= 0;
  244. }
  245. /*
  246. * To keep the bandwidth of -deadline tasks under control
  247. * we need some place where:
  248. * - store the maximum -deadline bandwidth of each cpu;
  249. * - cache the fraction of bandwidth that is currently allocated in
  250. * each root domain;
  251. *
  252. * This is all done in the data structure below. It is similar to the
  253. * one used for RT-throttling (rt_bandwidth), with the main difference
  254. * that, since here we are only interested in admission control, we
  255. * do not decrease any runtime while the group "executes", neither we
  256. * need a timer to replenish it.
  257. *
  258. * With respect to SMP, bandwidth is given on a per root domain basis,
  259. * meaning that:
  260. * - bw (< 100%) is the deadline bandwidth of each CPU;
  261. * - total_bw is the currently allocated bandwidth in each root domain;
  262. */
  263. struct dl_bw {
  264. raw_spinlock_t lock;
  265. u64 bw;
  266. u64 total_bw;
  267. };
  268. static inline void __dl_update(struct dl_bw *dl_b, s64 bw);
  269. static inline
  270. void __dl_sub(struct dl_bw *dl_b, u64 tsk_bw, int cpus)
  271. {
  272. dl_b->total_bw -= tsk_bw;
  273. __dl_update(dl_b, (s32)tsk_bw / cpus);
  274. }
  275. static inline
  276. void __dl_add(struct dl_bw *dl_b, u64 tsk_bw, int cpus)
  277. {
  278. dl_b->total_bw += tsk_bw;
  279. __dl_update(dl_b, -((s32)tsk_bw / cpus));
  280. }
  281. static inline bool __dl_overflow(struct dl_bw *dl_b, unsigned long cap,
  282. u64 old_bw, u64 new_bw)
  283. {
  284. return dl_b->bw != -1 &&
  285. cap_scale(dl_b->bw, cap) < dl_b->total_bw - old_bw + new_bw;
  286. }
  287. /*
  288. * Verify the fitness of task @p to run on @cpu taking into account the
  289. * CPU original capacity and the runtime/deadline ratio of the task.
  290. *
  291. * The function will return true if the CPU original capacity of the
  292. * @cpu scaled by SCHED_CAPACITY_SCALE >= runtime/deadline ratio of the
  293. * task and false otherwise.
  294. */
  295. static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu)
  296. {
  297. unsigned long cap = arch_scale_cpu_capacity(cpu);
  298. return cap_scale(p->dl.dl_deadline, cap) >= p->dl.dl_runtime;
  299. }
  300. extern void init_dl_bw(struct dl_bw *dl_b);
  301. extern int sched_dl_global_validate(void);
  302. extern void sched_dl_do_global(void);
  303. extern int sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr);
  304. extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr);
  305. extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr);
  306. extern bool __checkparam_dl(const struct sched_attr *attr);
  307. extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
  308. extern int dl_task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
  309. extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
  310. extern bool dl_cpu_busy(unsigned int cpu);
  311. #ifdef CONFIG_CGROUP_SCHED
  312. #include <linux/cgroup.h>
  313. #include <linux/psi.h>
  314. struct cfs_rq;
  315. struct rt_rq;
  316. extern struct list_head task_groups;
  317. struct cfs_bandwidth {
  318. #ifdef CONFIG_CFS_BANDWIDTH
  319. raw_spinlock_t lock;
  320. ktime_t period;
  321. u64 quota;
  322. u64 runtime;
  323. s64 hierarchical_quota;
  324. u8 idle;
  325. u8 period_active;
  326. u8 slack_started;
  327. struct hrtimer period_timer;
  328. struct hrtimer slack_timer;
  329. struct list_head throttled_cfs_rq;
  330. /* Statistics: */
  331. int nr_periods;
  332. int nr_throttled;
  333. u64 throttled_time;
  334. #endif
  335. };
  336. /* Task group related information */
  337. struct task_group {
  338. struct cgroup_subsys_state css;
  339. #ifdef CONFIG_FAIR_GROUP_SCHED
  340. /* schedulable entities of this group on each CPU */
  341. struct sched_entity **se;
  342. /* runqueue "owned" by this group on each CPU */
  343. struct cfs_rq **cfs_rq;
  344. unsigned long shares;
  345. #ifdef CONFIG_SMP
  346. /*
  347. * load_avg can be heavily contended at clock tick time, so put
  348. * it in its own cacheline separated from the fields above which
  349. * will also be accessed at each tick.
  350. */
  351. atomic_long_t load_avg ____cacheline_aligned;
  352. #endif
  353. #endif
  354. #ifdef CONFIG_RT_GROUP_SCHED
  355. struct sched_rt_entity **rt_se;
  356. struct rt_rq **rt_rq;
  357. struct rt_bandwidth rt_bandwidth;
  358. #endif
  359. struct rcu_head rcu;
  360. struct list_head list;
  361. struct task_group *parent;
  362. struct list_head siblings;
  363. struct list_head children;
  364. #ifdef CONFIG_SCHED_AUTOGROUP
  365. struct autogroup *autogroup;
  366. #endif
  367. struct cfs_bandwidth cfs_bandwidth;
  368. #ifdef CONFIG_UCLAMP_TASK_GROUP
  369. /* The two decimal precision [%] value requested from user-space */
  370. unsigned int uclamp_pct[UCLAMP_CNT];
  371. /* Clamp values requested for a task group */
  372. struct uclamp_se uclamp_req[UCLAMP_CNT];
  373. /* Effective clamp values used for a task group */
  374. struct uclamp_se uclamp[UCLAMP_CNT];
  375. /* Latency-sensitive flag used for a task group */
  376. unsigned int latency_sensitive;
  377. ANDROID_VENDOR_DATA_ARRAY(1, 4);
  378. #endif
  379. ANDROID_KABI_RESERVE(1);
  380. ANDROID_KABI_RESERVE(2);
  381. ANDROID_KABI_RESERVE(3);
  382. ANDROID_KABI_RESERVE(4);
  383. };
  384. #ifdef CONFIG_FAIR_GROUP_SCHED
  385. #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD
  386. /*
  387. * A weight of 0 or 1 can cause arithmetics problems.
  388. * A weight of a cfs_rq is the sum of weights of which entities
  389. * are queued on this cfs_rq, so a weight of a entity should not be
  390. * too large, so as the shares value of a task group.
  391. * (The default weight is 1024 - so there's no practical
  392. * limitation from this.)
  393. */
  394. #define MIN_SHARES (1UL << 1)
  395. #define MAX_SHARES (1UL << 18)
  396. #endif
  397. typedef int (*tg_visitor)(struct task_group *, void *);
  398. extern int walk_tg_tree_from(struct task_group *from,
  399. tg_visitor down, tg_visitor up, void *data);
  400. /*
  401. * Iterate the full tree, calling @down when first entering a node and @up when
  402. * leaving it for the final time.
  403. *
  404. * Caller must hold rcu_lock or sufficient equivalent.
  405. */
  406. static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
  407. {
  408. return walk_tg_tree_from(&root_task_group, down, up, data);
  409. }
  410. extern int tg_nop(struct task_group *tg, void *data);
  411. extern void free_fair_sched_group(struct task_group *tg);
  412. extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent);
  413. extern void online_fair_sched_group(struct task_group *tg);
  414. extern void unregister_fair_sched_group(struct task_group *tg);
  415. extern void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
  416. struct sched_entity *se, int cpu,
  417. struct sched_entity *parent);
  418. extern void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
  419. extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b);
  420. extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
  421. extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);
  422. extern void free_rt_sched_group(struct task_group *tg);
  423. extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
  424. extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
  425. struct sched_rt_entity *rt_se, int cpu,
  426. struct sched_rt_entity *parent);
  427. extern int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us);
  428. extern int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us);
  429. extern long sched_group_rt_runtime(struct task_group *tg);
  430. extern long sched_group_rt_period(struct task_group *tg);
  431. extern int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk);
  432. extern struct task_group *sched_create_group(struct task_group *parent);
  433. extern void sched_online_group(struct task_group *tg,
  434. struct task_group *parent);
  435. extern void sched_destroy_group(struct task_group *tg);
  436. extern void sched_offline_group(struct task_group *tg);
  437. extern void sched_move_task(struct task_struct *tsk);
  438. #ifdef CONFIG_FAIR_GROUP_SCHED
  439. extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);
  440. #ifdef CONFIG_SMP
  441. extern void set_task_rq_fair(struct sched_entity *se,
  442. struct cfs_rq *prev, struct cfs_rq *next);
  443. #else /* !CONFIG_SMP */
  444. static inline void set_task_rq_fair(struct sched_entity *se,
  445. struct cfs_rq *prev, struct cfs_rq *next) { }
  446. #endif /* CONFIG_SMP */
  447. #endif /* CONFIG_FAIR_GROUP_SCHED */
  448. #else /* CONFIG_CGROUP_SCHED */
  449. struct cfs_bandwidth { };
  450. #endif /* CONFIG_CGROUP_SCHED */
  451. /* CFS-related fields in a runqueue */
  452. struct cfs_rq {
  453. struct load_weight load;
  454. unsigned int nr_running;
  455. unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */
  456. unsigned int idle_h_nr_running; /* SCHED_IDLE */
  457. u64 exec_clock;
  458. u64 min_vruntime;
  459. #ifndef CONFIG_64BIT
  460. u64 min_vruntime_copy;
  461. #endif
  462. struct rb_root_cached tasks_timeline;
  463. /*
  464. * 'curr' points to currently running entity on this cfs_rq.
  465. * It is set to NULL otherwise (i.e when none are currently running).
  466. */
  467. struct sched_entity *curr;
  468. struct sched_entity *next;
  469. struct sched_entity *last;
  470. struct sched_entity *skip;
  471. #ifdef CONFIG_SCHED_DEBUG
  472. unsigned int nr_spread_over;
  473. #endif
  474. #ifdef CONFIG_SMP
  475. /*
  476. * CFS load tracking
  477. */
  478. struct sched_avg avg;
  479. #ifndef CONFIG_64BIT
  480. u64 load_last_update_time_copy;
  481. #endif
  482. struct {
  483. raw_spinlock_t lock ____cacheline_aligned;
  484. int nr;
  485. unsigned long load_avg;
  486. unsigned long util_avg;
  487. unsigned long runnable_avg;
  488. } removed;
  489. #ifdef CONFIG_FAIR_GROUP_SCHED
  490. unsigned long tg_load_avg_contrib;
  491. long propagate;
  492. long prop_runnable_sum;
  493. /*
  494. * h_load = weight * f(tg)
  495. *
  496. * Where f(tg) is the recursive weight fraction assigned to
  497. * this group.
  498. */
  499. unsigned long h_load;
  500. u64 last_h_load_update;
  501. struct sched_entity *h_load_next;
  502. #endif /* CONFIG_FAIR_GROUP_SCHED */
  503. #endif /* CONFIG_SMP */
  504. #ifdef CONFIG_FAIR_GROUP_SCHED
  505. struct rq *rq; /* CPU runqueue to which this cfs_rq is attached */
  506. /*
  507. * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
  508. * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
  509. * (like users, containers etc.)
  510. *
  511. * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a CPU.
  512. * This list is used during load balance.
  513. */
  514. int on_list;
  515. struct list_head leaf_cfs_rq_list;
  516. struct task_group *tg; /* group that "owns" this runqueue */
  517. #ifdef CONFIG_CFS_BANDWIDTH
  518. int runtime_enabled;
  519. s64 runtime_remaining;
  520. u64 throttled_clock;
  521. u64 throttled_clock_task;
  522. u64 throttled_clock_task_time;
  523. int throttled;
  524. int throttle_count;
  525. struct list_head throttled_list;
  526. #endif /* CONFIG_CFS_BANDWIDTH */
  527. ANDROID_VENDOR_DATA_ARRAY(1, 16);
  528. #endif /* CONFIG_FAIR_GROUP_SCHED */
  529. };
  530. static inline int rt_bandwidth_enabled(void)
  531. {
  532. return sysctl_sched_rt_runtime >= 0;
  533. }
  534. /* RT IPI pull logic requires IRQ_WORK */
  535. #if defined(CONFIG_IRQ_WORK) && defined(CONFIG_SMP)
  536. # define HAVE_RT_PUSH_IPI
  537. #endif
  538. /* Real-Time classes' related field in a runqueue: */
  539. struct rt_rq {
  540. struct rt_prio_array active;
  541. unsigned int rt_nr_running;
  542. unsigned int rr_nr_running;
  543. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  544. struct {
  545. int curr; /* highest queued rt task prio */
  546. #ifdef CONFIG_SMP
  547. int next; /* next highest */
  548. #endif
  549. } highest_prio;
  550. #endif
  551. #ifdef CONFIG_SMP
  552. unsigned long rt_nr_migratory;
  553. unsigned long rt_nr_total;
  554. int overloaded;
  555. struct plist_head pushable_tasks;
  556. #endif /* CONFIG_SMP */
  557. int rt_queued;
  558. int rt_throttled;
  559. u64 rt_time;
  560. u64 rt_runtime;
  561. /* Nests inside the rq lock: */
  562. raw_spinlock_t rt_runtime_lock;
  563. #ifdef CONFIG_RT_GROUP_SCHED
  564. unsigned long rt_nr_boosted;
  565. struct rq *rq;
  566. struct task_group *tg;
  567. #endif
  568. };
  569. static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq)
  570. {
  571. return rt_rq->rt_queued && rt_rq->rt_nr_running;
  572. }
  573. /* Deadline class' related fields in a runqueue */
  574. struct dl_rq {
  575. /* runqueue is an rbtree, ordered by deadline */
  576. struct rb_root_cached root;
  577. unsigned long dl_nr_running;
  578. #ifdef CONFIG_SMP
  579. /*
  580. * Deadline values of the currently executing and the
  581. * earliest ready task on this rq. Caching these facilitates
  582. * the decision whether or not a ready but not running task
  583. * should migrate somewhere else.
  584. */
  585. struct {
  586. u64 curr;
  587. u64 next;
  588. } earliest_dl;
  589. unsigned long dl_nr_migratory;
  590. int overloaded;
  591. /*
  592. * Tasks on this rq that can be pushed away. They are kept in
  593. * an rb-tree, ordered by tasks' deadlines, with caching
  594. * of the leftmost (earliest deadline) element.
  595. */
  596. struct rb_root_cached pushable_dl_tasks_root;
  597. #else
  598. struct dl_bw dl_bw;
  599. #endif
  600. /*
  601. * "Active utilization" for this runqueue: increased when a
  602. * task wakes up (becomes TASK_RUNNING) and decreased when a
  603. * task blocks
  604. */
  605. u64 running_bw;
  606. /*
  607. * Utilization of the tasks "assigned" to this runqueue (including
  608. * the tasks that are in runqueue and the tasks that executed on this
  609. * CPU and blocked). Increased when a task moves to this runqueue, and
  610. * decreased when the task moves away (migrates, changes scheduling
  611. * policy, or terminates).
  612. * This is needed to compute the "inactive utilization" for the
  613. * runqueue (inactive utilization = this_bw - running_bw).
  614. */
  615. u64 this_bw;
  616. u64 extra_bw;
  617. /*
  618. * Inverse of the fraction of CPU utilization that can be reclaimed
  619. * by the GRUB algorithm.
  620. */
  621. u64 bw_ratio;
  622. };
  623. #ifdef CONFIG_FAIR_GROUP_SCHED
  624. /* An entity is a task if it doesn't "own" a runqueue */
  625. #define entity_is_task(se) (!se->my_q)
  626. static inline void se_update_runnable(struct sched_entity *se)
  627. {
  628. if (!entity_is_task(se))
  629. se->runnable_weight = se->my_q->h_nr_running;
  630. }
  631. static inline long se_runnable(struct sched_entity *se)
  632. {
  633. if (entity_is_task(se))
  634. return !!se->on_rq;
  635. else
  636. return se->runnable_weight;
  637. }
  638. #else
  639. #define entity_is_task(se) 1
  640. static inline void se_update_runnable(struct sched_entity *se) {}
  641. static inline long se_runnable(struct sched_entity *se)
  642. {
  643. return !!se->on_rq;
  644. }
  645. #endif
  646. #ifdef CONFIG_SMP
  647. /*
  648. * XXX we want to get rid of these helpers and use the full load resolution.
  649. */
  650. static inline long se_weight(struct sched_entity *se)
  651. {
  652. return scale_load_down(se->load.weight);
  653. }
  654. static inline bool sched_asym_prefer(int a, int b)
  655. {
  656. return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b);
  657. }
  658. struct perf_domain {
  659. struct em_perf_domain *em_pd;
  660. struct perf_domain *next;
  661. struct rcu_head rcu;
  662. };
  663. /* Scheduling group status flags */
  664. #define SG_OVERLOAD 0x1 /* More than one runnable task on a CPU. */
  665. #define SG_OVERUTILIZED 0x2 /* One or more CPUs are over-utilized. */
  666. /*
  667. * We add the notion of a root-domain which will be used to define per-domain
  668. * variables. Each exclusive cpuset essentially defines an island domain by
  669. * fully partitioning the member CPUs from any other cpuset. Whenever a new
  670. * exclusive cpuset is created, we also create and attach a new root-domain
  671. * object.
  672. *
  673. */
  674. struct root_domain {
  675. atomic_t refcount;
  676. atomic_t rto_count;
  677. struct rcu_head rcu;
  678. cpumask_var_t span;
  679. cpumask_var_t online;
  680. /*
  681. * Indicate pullable load on at least one CPU, e.g:
  682. * - More than one runnable task
  683. * - Running task is misfit
  684. */
  685. int overload;
  686. /* Indicate one or more cpus over-utilized (tipping point) */
  687. int overutilized;
  688. /*
  689. * The bit corresponding to a CPU gets set here if such CPU has more
  690. * than one runnable -deadline task (as it is below for RT tasks).
  691. */
  692. cpumask_var_t dlo_mask;
  693. atomic_t dlo_count;
  694. struct dl_bw dl_bw;
  695. struct cpudl cpudl;
  696. #ifdef HAVE_RT_PUSH_IPI
  697. /*
  698. * For IPI pull requests, loop across the rto_mask.
  699. */
  700. struct irq_work rto_push_work;
  701. raw_spinlock_t rto_lock;
  702. /* These are only updated and read within rto_lock */
  703. int rto_loop;
  704. int rto_cpu;
  705. /* These atomics are updated outside of a lock */
  706. atomic_t rto_loop_next;
  707. atomic_t rto_loop_start;
  708. #endif
  709. /*
  710. * The "RT overload" flag: it gets set if a CPU has more than
  711. * one runnable RT task.
  712. */
  713. cpumask_var_t rto_mask;
  714. struct cpupri cpupri;
  715. unsigned long max_cpu_capacity;
  716. /*
  717. * NULL-terminated list of performance domains intersecting with the
  718. * CPUs of the rd. Protected by RCU.
  719. */
  720. struct perf_domain __rcu *pd;
  721. ANDROID_VENDOR_DATA_ARRAY(1, 4);
  722. ANDROID_KABI_RESERVE(1);
  723. ANDROID_KABI_RESERVE(2);
  724. ANDROID_KABI_RESERVE(3);
  725. ANDROID_KABI_RESERVE(4);
  726. };
  727. extern void init_defrootdomain(void);
  728. extern int sched_init_domains(const struct cpumask *cpu_map);
  729. extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
  730. extern void sched_get_rd(struct root_domain *rd);
  731. extern void sched_put_rd(struct root_domain *rd);
  732. #ifdef HAVE_RT_PUSH_IPI
  733. extern void rto_push_irq_work_func(struct irq_work *work);
  734. #endif
  735. extern struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu);
  736. #endif /* CONFIG_SMP */
  737. #ifdef CONFIG_UCLAMP_TASK
  738. /*
  739. * struct uclamp_bucket - Utilization clamp bucket
  740. * @value: utilization clamp value for tasks on this clamp bucket
  741. * @tasks: number of RUNNABLE tasks on this clamp bucket
  742. *
  743. * Keep track of how many tasks are RUNNABLE for a given utilization
  744. * clamp value.
  745. */
  746. struct uclamp_bucket {
  747. unsigned long value : bits_per(SCHED_CAPACITY_SCALE);
  748. unsigned long tasks : BITS_PER_LONG - bits_per(SCHED_CAPACITY_SCALE);
  749. };
  750. /*
  751. * struct uclamp_rq - rq's utilization clamp
  752. * @value: currently active clamp values for a rq
  753. * @bucket: utilization clamp buckets affecting a rq
  754. *
  755. * Keep track of RUNNABLE tasks on a rq to aggregate their clamp values.
  756. * A clamp value is affecting a rq when there is at least one task RUNNABLE
  757. * (or actually running) with that value.
  758. *
  759. * There are up to UCLAMP_CNT possible different clamp values, currently there
  760. * are only two: minimum utilization and maximum utilization.
  761. *
  762. * All utilization clamping values are MAX aggregated, since:
  763. * - for util_min: we want to run the CPU at least at the max of the minimum
  764. * utilization required by its currently RUNNABLE tasks.
  765. * - for util_max: we want to allow the CPU to run up to the max of the
  766. * maximum utilization allowed by its currently RUNNABLE tasks.
  767. *
  768. * Since on each system we expect only a limited number of different
  769. * utilization clamp values (UCLAMP_BUCKETS), use a simple array to track
  770. * the metrics required to compute all the per-rq utilization clamp values.
  771. */
  772. struct uclamp_rq {
  773. unsigned int value;
  774. struct uclamp_bucket bucket[UCLAMP_BUCKETS];
  775. };
  776. DECLARE_STATIC_KEY_FALSE(sched_uclamp_used);
  777. #endif /* CONFIG_UCLAMP_TASK */
  778. /*
  779. * This is the main, per-CPU runqueue data structure.
  780. *
  781. * Locking rule: those places that want to lock multiple runqueues
  782. * (such as the load balancing or the thread migration code), lock
  783. * acquire operations must be ordered by ascending &runqueue.
  784. */
  785. struct rq {
  786. /* runqueue lock: */
  787. raw_spinlock_t lock;
  788. /*
  789. * nr_running and cpu_load should be in the same cacheline because
  790. * remote CPUs use both these fields when doing load calculation.
  791. */
  792. unsigned int nr_running;
  793. #ifdef CONFIG_NUMA_BALANCING
  794. unsigned int nr_numa_running;
  795. unsigned int nr_preferred_running;
  796. unsigned int numa_migrate_on;
  797. #endif
  798. #ifdef CONFIG_NO_HZ_COMMON
  799. #ifdef CONFIG_SMP
  800. unsigned long last_blocked_load_update_tick;
  801. unsigned int has_blocked_load;
  802. call_single_data_t nohz_csd;
  803. #endif /* CONFIG_SMP */
  804. unsigned int nohz_tick_stopped;
  805. atomic_t nohz_flags;
  806. #endif /* CONFIG_NO_HZ_COMMON */
  807. #ifdef CONFIG_SMP
  808. unsigned int ttwu_pending;
  809. #endif
  810. u64 nr_switches;
  811. #ifdef CONFIG_UCLAMP_TASK
  812. /* Utilization clamp values based on CPU's RUNNABLE tasks */
  813. struct uclamp_rq uclamp[UCLAMP_CNT] ____cacheline_aligned;
  814. unsigned int uclamp_flags;
  815. #define UCLAMP_FLAG_IDLE 0x01
  816. #endif
  817. struct cfs_rq cfs;
  818. struct rt_rq rt;
  819. struct dl_rq dl;
  820. #ifdef CONFIG_FAIR_GROUP_SCHED
  821. /* list of leaf cfs_rq on this CPU: */
  822. struct list_head leaf_cfs_rq_list;
  823. struct list_head *tmp_alone_branch;
  824. #endif /* CONFIG_FAIR_GROUP_SCHED */
  825. /*
  826. * This is part of a global counter where only the total sum
  827. * over all CPUs matters. A task can increase this counter on
  828. * one CPU and if it got migrated afterwards it may decrease
  829. * it on another CPU. Always updated under the runqueue lock:
  830. */
  831. unsigned long nr_uninterruptible;
  832. struct task_struct __rcu *curr;
  833. struct task_struct *idle;
  834. struct task_struct *stop;
  835. unsigned long next_balance;
  836. struct mm_struct *prev_mm;
  837. unsigned int clock_update_flags;
  838. u64 clock;
  839. /* Ensure that all clocks are in the same cache line */
  840. u64 clock_task ____cacheline_aligned;
  841. u64 clock_pelt;
  842. unsigned long lost_idle_time;
  843. atomic_t nr_iowait;
  844. #ifdef CONFIG_MEMBARRIER
  845. int membarrier_state;
  846. #endif
  847. #ifdef CONFIG_SMP
  848. struct root_domain *rd;
  849. struct sched_domain __rcu *sd;
  850. unsigned long cpu_capacity;
  851. unsigned long cpu_capacity_orig;
  852. struct callback_head *balance_callback;
  853. unsigned char nohz_idle_balance;
  854. unsigned char idle_balance;
  855. unsigned long misfit_task_load;
  856. /* For active balancing */
  857. int active_balance;
  858. int push_cpu;
  859. struct cpu_stop_work active_balance_work;
  860. /* CPU of this runqueue: */
  861. int cpu;
  862. int online;
  863. struct list_head cfs_tasks;
  864. struct sched_avg avg_rt;
  865. struct sched_avg avg_dl;
  866. #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
  867. struct sched_avg avg_irq;
  868. #endif
  869. #ifdef CONFIG_SCHED_THERMAL_PRESSURE
  870. struct sched_avg avg_thermal;
  871. #endif
  872. u64 idle_stamp;
  873. u64 avg_idle;
  874. /* This is used to determine avg_idle's max value */
  875. u64 max_idle_balance_cost;
  876. #endif /* CONFIG_SMP */
  877. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  878. u64 prev_irq_time;
  879. #endif
  880. #ifdef CONFIG_PARAVIRT
  881. u64 prev_steal_time;
  882. #endif
  883. #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
  884. u64 prev_steal_time_rq;
  885. #endif
  886. /* calc_load related fields */
  887. unsigned long calc_load_update;
  888. long calc_load_active;
  889. #ifdef CONFIG_SCHED_HRTICK
  890. #ifdef CONFIG_SMP
  891. call_single_data_t hrtick_csd;
  892. #endif
  893. struct hrtimer hrtick_timer;
  894. ktime_t hrtick_time;
  895. #endif
  896. #ifdef CONFIG_SCHEDSTATS
  897. /* latency stats */
  898. struct sched_info rq_sched_info;
  899. unsigned long long rq_cpu_time;
  900. /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
  901. /* sys_sched_yield() stats */
  902. unsigned int yld_count;
  903. /* schedule() stats */
  904. unsigned int sched_count;
  905. unsigned int sched_goidle;
  906. /* try_to_wake_up() stats */
  907. unsigned int ttwu_count;
  908. unsigned int ttwu_local;
  909. #endif
  910. #ifdef CONFIG_HOTPLUG_CPU
  911. struct cpu_stop_work drain;
  912. struct cpu_stop_done drain_done;
  913. #endif
  914. #ifdef CONFIG_CPU_IDLE
  915. /* Must be inspected within a rcu lock section */
  916. struct cpuidle_state *idle_state;
  917. #endif
  918. ANDROID_VENDOR_DATA_ARRAY(1, 96);
  919. ANDROID_OEM_DATA_ARRAY(1, 16);
  920. ANDROID_KABI_RESERVE(1);
  921. ANDROID_KABI_RESERVE(2);
  922. ANDROID_KABI_RESERVE(3);
  923. ANDROID_KABI_RESERVE(4);
  924. };
  925. #ifdef CONFIG_FAIR_GROUP_SCHED
  926. /* CPU runqueue to which this cfs_rq is attached */
  927. static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
  928. {
  929. return cfs_rq->rq;
  930. }
  931. #else
  932. static inline struct rq *rq_of(struct cfs_rq *cfs_rq)
  933. {
  934. return container_of(cfs_rq, struct rq, cfs);
  935. }
  936. #endif
  937. static inline int cpu_of(struct rq *rq)
  938. {
  939. #ifdef CONFIG_SMP
  940. return rq->cpu;
  941. #else
  942. return 0;
  943. #endif
  944. }
  945. #ifdef CONFIG_SCHED_SMT
  946. extern void __update_idle_core(struct rq *rq);
  947. static inline void update_idle_core(struct rq *rq)
  948. {
  949. if (static_branch_unlikely(&sched_smt_present))
  950. __update_idle_core(rq);
  951. }
  952. #else
  953. static inline void update_idle_core(struct rq *rq) { }
  954. #endif
  955. DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
  956. #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
  957. #define this_rq() this_cpu_ptr(&runqueues)
  958. #define task_rq(p) cpu_rq(task_cpu(p))
  959. #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
  960. #define raw_rq() raw_cpu_ptr(&runqueues)
  961. extern void update_rq_clock(struct rq *rq);
  962. static inline u64 __rq_clock_broken(struct rq *rq)
  963. {
  964. return READ_ONCE(rq->clock);
  965. }
  966. /*
  967. * rq::clock_update_flags bits
  968. *
  969. * %RQCF_REQ_SKIP - will request skipping of clock update on the next
  970. * call to __schedule(). This is an optimisation to avoid
  971. * neighbouring rq clock updates.
  972. *
  973. * %RQCF_ACT_SKIP - is set from inside of __schedule() when skipping is
  974. * in effect and calls to update_rq_clock() are being ignored.
  975. *
  976. * %RQCF_UPDATED - is a debug flag that indicates whether a call has been
  977. * made to update_rq_clock() since the last time rq::lock was pinned.
  978. *
  979. * If inside of __schedule(), clock_update_flags will have been
  980. * shifted left (a left shift is a cheap operation for the fast path
  981. * to promote %RQCF_REQ_SKIP to %RQCF_ACT_SKIP), so you must use,
  982. *
  983. * if (rq-clock_update_flags >= RQCF_UPDATED)
  984. *
  985. * to check if %RQCF_UPADTED is set. It'll never be shifted more than
  986. * one position though, because the next rq_unpin_lock() will shift it
  987. * back.
  988. */
  989. #define RQCF_REQ_SKIP 0x01
  990. #define RQCF_ACT_SKIP 0x02
  991. #define RQCF_UPDATED 0x04
  992. static inline void assert_clock_updated(struct rq *rq)
  993. {
  994. /*
  995. * The only reason for not seeing a clock update since the
  996. * last rq_pin_lock() is if we're currently skipping updates.
  997. */
  998. SCHED_WARN_ON(rq->clock_update_flags < RQCF_ACT_SKIP);
  999. }
  1000. static inline u64 rq_clock(struct rq *rq)
  1001. {
  1002. lockdep_assert_held(&rq->lock);
  1003. assert_clock_updated(rq);
  1004. return rq->clock;
  1005. }
  1006. static inline u64 rq_clock_task(struct rq *rq)
  1007. {
  1008. lockdep_assert_held(&rq->lock);
  1009. assert_clock_updated(rq);
  1010. return rq->clock_task;
  1011. }
  1012. /**
  1013. * By default the decay is the default pelt decay period.
  1014. * The decay shift can change the decay period in
  1015. * multiples of 32.
  1016. * Decay shift Decay period(ms)
  1017. * 0 32
  1018. * 1 64
  1019. * 2 128
  1020. * 3 256
  1021. * 4 512
  1022. */
  1023. extern int sched_thermal_decay_shift;
  1024. static inline u64 rq_clock_thermal(struct rq *rq)
  1025. {
  1026. return rq_clock_task(rq) >> sched_thermal_decay_shift;
  1027. }
  1028. static inline void rq_clock_skip_update(struct rq *rq)
  1029. {
  1030. lockdep_assert_held(&rq->lock);
  1031. rq->clock_update_flags |= RQCF_REQ_SKIP;
  1032. }
  1033. /*
  1034. * See rt task throttling, which is the only time a skip
  1035. * request is cancelled.
  1036. */
  1037. static inline void rq_clock_cancel_skipupdate(struct rq *rq)
  1038. {
  1039. lockdep_assert_held(&rq->lock);
  1040. rq->clock_update_flags &= ~RQCF_REQ_SKIP;
  1041. }
  1042. struct rq_flags {
  1043. unsigned long flags;
  1044. struct pin_cookie cookie;
  1045. #ifdef CONFIG_SCHED_DEBUG
  1046. /*
  1047. * A copy of (rq::clock_update_flags & RQCF_UPDATED) for the
  1048. * current pin context is stashed here in case it needs to be
  1049. * restored in rq_repin_lock().
  1050. */
  1051. unsigned int clock_update_flags;
  1052. #endif
  1053. };
  1054. /*
  1055. * Lockdep annotation that avoids accidental unlocks; it's like a
  1056. * sticky/continuous lockdep_assert_held().
  1057. *
  1058. * This avoids code that has access to 'struct rq *rq' (basically everything in
  1059. * the scheduler) from accidentally unlocking the rq if they do not also have a
  1060. * copy of the (on-stack) 'struct rq_flags rf'.
  1061. *
  1062. * Also see Documentation/locking/lockdep-design.rst.
  1063. */
  1064. static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf)
  1065. {
  1066. rf->cookie = lockdep_pin_lock(&rq->lock);
  1067. #ifdef CONFIG_SCHED_DEBUG
  1068. rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
  1069. rf->clock_update_flags = 0;
  1070. #endif
  1071. }
  1072. static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)
  1073. {
  1074. #ifdef CONFIG_SCHED_DEBUG
  1075. if (rq->clock_update_flags > RQCF_ACT_SKIP)
  1076. rf->clock_update_flags = RQCF_UPDATED;
  1077. #endif
  1078. lockdep_unpin_lock(&rq->lock, rf->cookie);
  1079. }
  1080. static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf)
  1081. {
  1082. lockdep_repin_lock(&rq->lock, rf->cookie);
  1083. #ifdef CONFIG_SCHED_DEBUG
  1084. /*
  1085. * Restore the value we stashed in @rf for this pin context.
  1086. */
  1087. rq->clock_update_flags |= rf->clock_update_flags;
  1088. #endif
  1089. }
  1090. struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
  1091. __acquires(rq->lock);
  1092. struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
  1093. __acquires(p->pi_lock)
  1094. __acquires(rq->lock);
  1095. static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf)
  1096. __releases(rq->lock)
  1097. {
  1098. rq_unpin_lock(rq, rf);
  1099. raw_spin_unlock(&rq->lock);
  1100. }
  1101. static inline void
  1102. task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
  1103. __releases(rq->lock)
  1104. __releases(p->pi_lock)
  1105. {
  1106. rq_unpin_lock(rq, rf);
  1107. raw_spin_unlock(&rq->lock);
  1108. raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
  1109. }
  1110. static inline void
  1111. rq_lock_irqsave(struct rq *rq, struct rq_flags *rf)
  1112. __acquires(rq->lock)
  1113. {
  1114. raw_spin_lock_irqsave(&rq->lock, rf->flags);
  1115. rq_pin_lock(rq, rf);
  1116. }
  1117. static inline void
  1118. rq_lock_irq(struct rq *rq, struct rq_flags *rf)
  1119. __acquires(rq->lock)
  1120. {
  1121. raw_spin_lock_irq(&rq->lock);
  1122. rq_pin_lock(rq, rf);
  1123. }
  1124. static inline void
  1125. rq_lock(struct rq *rq, struct rq_flags *rf)
  1126. __acquires(rq->lock)
  1127. {
  1128. raw_spin_lock(&rq->lock);
  1129. rq_pin_lock(rq, rf);
  1130. }
  1131. static inline void
  1132. rq_relock(struct rq *rq, struct rq_flags *rf)
  1133. __acquires(rq->lock)
  1134. {
  1135. raw_spin_lock(&rq->lock);
  1136. rq_repin_lock(rq, rf);
  1137. }
  1138. static inline void
  1139. rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf)
  1140. __releases(rq->lock)
  1141. {
  1142. rq_unpin_lock(rq, rf);
  1143. raw_spin_unlock_irqrestore(&rq->lock, rf->flags);
  1144. }
  1145. static inline void
  1146. rq_unlock_irq(struct rq *rq, struct rq_flags *rf)
  1147. __releases(rq->lock)
  1148. {
  1149. rq_unpin_lock(rq, rf);
  1150. raw_spin_unlock_irq(&rq->lock);
  1151. }
  1152. static inline void
  1153. rq_unlock(struct rq *rq, struct rq_flags *rf)
  1154. __releases(rq->lock)
  1155. {
  1156. rq_unpin_lock(rq, rf);
  1157. raw_spin_unlock(&rq->lock);
  1158. }
  1159. static inline struct rq *
  1160. this_rq_lock_irq(struct rq_flags *rf)
  1161. __acquires(rq->lock)
  1162. {
  1163. struct rq *rq;
  1164. local_irq_disable();
  1165. rq = this_rq();
  1166. rq_lock(rq, rf);
  1167. return rq;
  1168. }
  1169. #ifdef CONFIG_NUMA
  1170. enum numa_topology_type {
  1171. NUMA_DIRECT,
  1172. NUMA_GLUELESS_MESH,
  1173. NUMA_BACKPLANE,
  1174. };
  1175. extern enum numa_topology_type sched_numa_topology_type;
  1176. extern int sched_max_numa_distance;
  1177. extern bool find_numa_distance(int distance);
  1178. extern void sched_init_numa(void);
  1179. extern void sched_domains_numa_masks_set(unsigned int cpu);
  1180. extern void sched_domains_numa_masks_clear(unsigned int cpu);
  1181. extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu);
  1182. #else
  1183. static inline void sched_init_numa(void) { }
  1184. static inline void sched_domains_numa_masks_set(unsigned int cpu) { }
  1185. static inline void sched_domains_numa_masks_clear(unsigned int cpu) { }
  1186. static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
  1187. {
  1188. return nr_cpu_ids;
  1189. }
  1190. #endif
  1191. #ifdef CONFIG_NUMA_BALANCING
  1192. /* The regions in numa_faults array from task_struct */
  1193. enum numa_faults_stats {
  1194. NUMA_MEM = 0,
  1195. NUMA_CPU,
  1196. NUMA_MEMBUF,
  1197. NUMA_CPUBUF
  1198. };
  1199. extern void sched_setnuma(struct task_struct *p, int node);
  1200. extern int migrate_task_to(struct task_struct *p, int cpu);
  1201. extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p);
  1202. #else
  1203. static inline void
  1204. init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
  1205. {
  1206. }
  1207. #endif /* CONFIG_NUMA_BALANCING */
  1208. #ifdef CONFIG_SMP
  1209. extern int migrate_swap(struct task_struct *p, struct task_struct *t,
  1210. int cpu, int scpu);
  1211. static inline void
  1212. queue_balance_callback(struct rq *rq,
  1213. struct callback_head *head,
  1214. void (*func)(struct rq *rq))
  1215. {
  1216. lockdep_assert_held(&rq->lock);
  1217. if (unlikely(head->next))
  1218. return;
  1219. head->func = (void (*)(struct callback_head *))func;
  1220. head->next = rq->balance_callback;
  1221. rq->balance_callback = head;
  1222. }
  1223. #define rcu_dereference_check_sched_domain(p) \
  1224. rcu_dereference_check((p), \
  1225. lockdep_is_held(&sched_domains_mutex))
  1226. /*
  1227. * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
  1228. * See destroy_sched_domains: call_rcu for details.
  1229. *
  1230. * The domain tree of any CPU may only be accessed from within
  1231. * preempt-disabled sections.
  1232. */
  1233. #define for_each_domain(cpu, __sd) \
  1234. for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); \
  1235. __sd; __sd = __sd->parent)
  1236. /**
  1237. * highest_flag_domain - Return highest sched_domain containing flag.
  1238. * @cpu: The CPU whose highest level of sched domain is to
  1239. * be returned.
  1240. * @flag: The flag to check for the highest sched_domain
  1241. * for the given CPU.
  1242. *
  1243. * Returns the highest sched_domain of a CPU which contains the given flag.
  1244. */
  1245. static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
  1246. {
  1247. struct sched_domain *sd, *hsd = NULL;
  1248. for_each_domain(cpu, sd) {
  1249. if (!(sd->flags & flag))
  1250. break;
  1251. hsd = sd;
  1252. }
  1253. return hsd;
  1254. }
  1255. static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
  1256. {
  1257. struct sched_domain *sd;
  1258. for_each_domain(cpu, sd) {
  1259. if (sd->flags & flag)
  1260. break;
  1261. }
  1262. return sd;
  1263. }
  1264. DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);
  1265. DECLARE_PER_CPU(int, sd_llc_size);
  1266. DECLARE_PER_CPU(int, sd_llc_id);
  1267. DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
  1268. DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
  1269. DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
  1270. DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
  1271. extern struct static_key_false sched_asym_cpucapacity;
  1272. struct sched_group_capacity {
  1273. atomic_t ref;
  1274. /*
  1275. * CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity
  1276. * for a single CPU.
  1277. */
  1278. unsigned long capacity;
  1279. unsigned long min_capacity; /* Min per-CPU capacity in group */
  1280. unsigned long max_capacity; /* Max per-CPU capacity in group */
  1281. unsigned long next_update;
  1282. int imbalance; /* XXX unrelated to capacity but shared group state */
  1283. #ifdef CONFIG_SCHED_DEBUG
  1284. int id;
  1285. #endif
  1286. unsigned long cpumask[]; /* Balance mask */
  1287. };
  1288. struct sched_group {
  1289. struct sched_group *next; /* Must be a circular list */
  1290. atomic_t ref;
  1291. unsigned int group_weight;
  1292. struct sched_group_capacity *sgc;
  1293. int asym_prefer_cpu; /* CPU of highest priority in group */
  1294. /*
  1295. * The CPUs this group covers.
  1296. *
  1297. * NOTE: this field is variable length. (Allocated dynamically
  1298. * by attaching extra space to the end of the structure,
  1299. * depending on how many CPUs the kernel has booted up with)
  1300. */
  1301. unsigned long cpumask[];
  1302. };
  1303. static inline struct cpumask *sched_group_span(struct sched_group *sg)
  1304. {
  1305. return to_cpumask(sg->cpumask);
  1306. }
  1307. /*
  1308. * See build_balance_mask().
  1309. */
  1310. static inline struct cpumask *group_balance_mask(struct sched_group *sg)
  1311. {
  1312. return to_cpumask(sg->sgc->cpumask);
  1313. }
  1314. /**
  1315. * group_first_cpu - Returns the first CPU in the cpumask of a sched_group.
  1316. * @group: The group whose first CPU is to be returned.
  1317. */
  1318. static inline unsigned int group_first_cpu(struct sched_group *group)
  1319. {
  1320. return cpumask_first(sched_group_span(group));
  1321. }
  1322. extern int group_balance_cpu(struct sched_group *sg);
  1323. #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
  1324. void register_sched_domain_sysctl(void);
  1325. void dirty_sched_domain_sysctl(int cpu);
  1326. void unregister_sched_domain_sysctl(void);
  1327. #else
  1328. static inline void register_sched_domain_sysctl(void)
  1329. {
  1330. }
  1331. static inline void dirty_sched_domain_sysctl(int cpu)
  1332. {
  1333. }
  1334. static inline void unregister_sched_domain_sysctl(void)
  1335. {
  1336. }
  1337. #endif
  1338. extern void flush_smp_call_function_from_idle(void);
  1339. #else /* !CONFIG_SMP: */
  1340. static inline void flush_smp_call_function_from_idle(void) { }
  1341. #endif
  1342. #include "stats.h"
  1343. #include "autogroup.h"
  1344. #ifdef CONFIG_CGROUP_SCHED
  1345. /*
  1346. * Return the group to which this tasks belongs.
  1347. *
  1348. * We cannot use task_css() and friends because the cgroup subsystem
  1349. * changes that value before the cgroup_subsys::attach() method is called,
  1350. * therefore we cannot pin it and might observe the wrong value.
  1351. *
  1352. * The same is true for autogroup's p->signal->autogroup->tg, the autogroup
  1353. * core changes this before calling sched_move_task().
  1354. *
  1355. * Instead we use a 'copy' which is updated from sched_move_task() while
  1356. * holding both task_struct::pi_lock and rq::lock.
  1357. */
  1358. static inline struct task_group *task_group(struct task_struct *p)
  1359. {
  1360. return p->sched_task_group;
  1361. }
  1362. /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
  1363. static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
  1364. {
  1365. #if defined(CONFIG_FAIR_GROUP_SCHED) || defined(CONFIG_RT_GROUP_SCHED)
  1366. struct task_group *tg = task_group(p);
  1367. #endif
  1368. #ifdef CONFIG_FAIR_GROUP_SCHED
  1369. set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
  1370. p->se.cfs_rq = tg->cfs_rq[cpu];
  1371. p->se.parent = tg->se[cpu];
  1372. #endif
  1373. #ifdef CONFIG_RT_GROUP_SCHED
  1374. p->rt.rt_rq = tg->rt_rq[cpu];
  1375. p->rt.parent = tg->rt_se[cpu];
  1376. #endif
  1377. }
  1378. #else /* CONFIG_CGROUP_SCHED */
  1379. static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
  1380. static inline struct task_group *task_group(struct task_struct *p)
  1381. {
  1382. return NULL;
  1383. }
  1384. #endif /* CONFIG_CGROUP_SCHED */
  1385. static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  1386. {
  1387. set_task_rq(p, cpu);
  1388. #ifdef CONFIG_SMP
  1389. /*
  1390. * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
  1391. * successfully executed on another CPU. We must ensure that updates of
  1392. * per-task data have been completed by this moment.
  1393. */
  1394. smp_wmb();
  1395. #ifdef CONFIG_THREAD_INFO_IN_TASK
  1396. WRITE_ONCE(p->cpu, cpu);
  1397. #else
  1398. WRITE_ONCE(task_thread_info(p)->cpu, cpu);
  1399. #endif
  1400. p->wake_cpu = cpu;
  1401. #endif
  1402. }
  1403. /*
  1404. * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  1405. */
  1406. #ifdef CONFIG_SCHED_DEBUG
  1407. # include <linux/static_key.h>
  1408. # define const_debug __read_mostly
  1409. #else
  1410. # define const_debug const
  1411. #endif
  1412. #define SCHED_FEAT(name, enabled) \
  1413. __SCHED_FEAT_##name ,
  1414. enum {
  1415. #include "features.h"
  1416. __SCHED_FEAT_NR,
  1417. };
  1418. #undef SCHED_FEAT
  1419. #ifdef CONFIG_SCHED_DEBUG
  1420. /*
  1421. * To support run-time toggling of sched features, all the translation units
  1422. * (but core.c) reference the sysctl_sched_features defined in core.c.
  1423. */
  1424. extern const_debug unsigned int sysctl_sched_features;
  1425. #ifdef CONFIG_JUMP_LABEL
  1426. #define SCHED_FEAT(name, enabled) \
  1427. static __always_inline bool static_branch_##name(struct static_key *key) \
  1428. { \
  1429. return static_key_##enabled(key); \
  1430. }
  1431. #include "features.h"
  1432. #undef SCHED_FEAT
  1433. extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
  1434. extern const char * const sched_feat_names[__SCHED_FEAT_NR];
  1435. #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
  1436. #else /* !CONFIG_JUMP_LABEL */
  1437. #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
  1438. #endif /* CONFIG_JUMP_LABEL */
  1439. #else /* !SCHED_DEBUG */
  1440. /*
  1441. * Each translation unit has its own copy of sysctl_sched_features to allow
  1442. * constants propagation at compile time and compiler optimization based on
  1443. * features default.
  1444. */
  1445. #define SCHED_FEAT(name, enabled) \
  1446. (1UL << __SCHED_FEAT_##name) * enabled |
  1447. static const_debug __maybe_unused unsigned int sysctl_sched_features =
  1448. #include "features.h"
  1449. 0;
  1450. #undef SCHED_FEAT
  1451. #define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
  1452. #endif /* SCHED_DEBUG */
  1453. extern struct static_key_false sched_numa_balancing;
  1454. extern struct static_key_false sched_schedstats;
  1455. static inline u64 global_rt_period(void)
  1456. {
  1457. return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
  1458. }
  1459. static inline u64 global_rt_runtime(void)
  1460. {
  1461. if (sysctl_sched_rt_runtime < 0)
  1462. return RUNTIME_INF;
  1463. return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
  1464. }
  1465. static inline int task_current(struct rq *rq, struct task_struct *p)
  1466. {
  1467. return rq->curr == p;
  1468. }
  1469. static inline int task_running(struct rq *rq, struct task_struct *p)
  1470. {
  1471. #ifdef CONFIG_SMP
  1472. return p->on_cpu;
  1473. #else
  1474. return task_current(rq, p);
  1475. #endif
  1476. }
  1477. static inline int task_on_rq_queued(struct task_struct *p)
  1478. {
  1479. return p->on_rq == TASK_ON_RQ_QUEUED;
  1480. }
  1481. static inline int task_on_rq_migrating(struct task_struct *p)
  1482. {
  1483. return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING;
  1484. }
  1485. /*
  1486. * wake flags
  1487. */
  1488. #define WF_SYNC 0x01 /* Waker goes to sleep after wakeup */
  1489. #define WF_FORK 0x02 /* Child wakeup after fork */
  1490. #define WF_MIGRATED 0x04 /* Internal use, task got migrated */
  1491. #define WF_ON_CPU 0x08 /* Wakee is on_cpu */
  1492. #define WF_ANDROID_VENDOR 0x1000 /* Vendor specific for Android */
  1493. /*
  1494. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  1495. * of tasks with abnormal "nice" values across CPUs the contribution that
  1496. * each task makes to its run queue's load is weighted according to its
  1497. * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
  1498. * scaled version of the new time slice allocation that they receive on time
  1499. * slice expiry etc.
  1500. */
  1501. #define WEIGHT_IDLEPRIO 3
  1502. #define WMULT_IDLEPRIO 1431655765
  1503. extern const int sched_prio_to_weight[40];
  1504. extern const u32 sched_prio_to_wmult[40];
  1505. /*
  1506. * {de,en}queue flags:
  1507. *
  1508. * DEQUEUE_SLEEP - task is no longer runnable
  1509. * ENQUEUE_WAKEUP - task just became runnable
  1510. *
  1511. * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks
  1512. * are in a known state which allows modification. Such pairs
  1513. * should preserve as much state as possible.
  1514. *
  1515. * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location
  1516. * in the runqueue.
  1517. *
  1518. * ENQUEUE_HEAD - place at front of runqueue (tail if not specified)
  1519. * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline)
  1520. * ENQUEUE_MIGRATED - the task was migrated during wakeup
  1521. *
  1522. */
  1523. #define DEQUEUE_SLEEP 0x01
  1524. #define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */
  1525. #define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */
  1526. #define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */
  1527. #define ENQUEUE_WAKEUP 0x01
  1528. #define ENQUEUE_RESTORE 0x02
  1529. #define ENQUEUE_MOVE 0x04
  1530. #define ENQUEUE_NOCLOCK 0x08
  1531. #define ENQUEUE_HEAD 0x10
  1532. #define ENQUEUE_REPLENISH 0x20
  1533. #ifdef CONFIG_SMP
  1534. #define ENQUEUE_MIGRATED 0x40
  1535. #else
  1536. #define ENQUEUE_MIGRATED 0x00
  1537. #endif
  1538. #define ENQUEUE_WAKEUP_SYNC 0x80
  1539. #define RETRY_TASK ((void *)-1UL)
  1540. struct sched_class {
  1541. #ifdef CONFIG_UCLAMP_TASK
  1542. int uclamp_enabled;
  1543. #endif
  1544. void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags);
  1545. void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags);
  1546. void (*yield_task) (struct rq *rq);
  1547. bool (*yield_to_task)(struct rq *rq, struct task_struct *p);
  1548. void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int flags);
  1549. struct task_struct *(*pick_next_task)(struct rq *rq);
  1550. void (*put_prev_task)(struct rq *rq, struct task_struct *p);
  1551. void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first);
  1552. #ifdef CONFIG_SMP
  1553. int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
  1554. int (*select_task_rq)(struct task_struct *p, int task_cpu, int sd_flag, int flags);
  1555. void (*migrate_task_rq)(struct task_struct *p, int new_cpu);
  1556. void (*task_woken)(struct rq *this_rq, struct task_struct *task);
  1557. void (*set_cpus_allowed)(struct task_struct *p,
  1558. const struct cpumask *newmask);
  1559. void (*rq_online)(struct rq *rq);
  1560. void (*rq_offline)(struct rq *rq);
  1561. #endif
  1562. void (*task_tick)(struct rq *rq, struct task_struct *p, int queued);
  1563. void (*task_fork)(struct task_struct *p);
  1564. void (*task_dead)(struct task_struct *p);
  1565. /*
  1566. * The switched_from() call is allowed to drop rq->lock, therefore we
  1567. * cannot assume the switched_from/switched_to pair is serliazed by
  1568. * rq->lock. They are however serialized by p->pi_lock.
  1569. */
  1570. void (*switched_from)(struct rq *this_rq, struct task_struct *task);
  1571. void (*switched_to) (struct rq *this_rq, struct task_struct *task);
  1572. void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
  1573. int oldprio);
  1574. unsigned int (*get_rr_interval)(struct rq *rq,
  1575. struct task_struct *task);
  1576. void (*update_curr)(struct rq *rq);
  1577. #define TASK_SET_GROUP 0
  1578. #define TASK_MOVE_GROUP 1
  1579. #ifdef CONFIG_FAIR_GROUP_SCHED
  1580. void (*task_change_group)(struct task_struct *p, int type);
  1581. #endif
  1582. } __aligned(STRUCT_ALIGNMENT); /* STRUCT_ALIGN(), vmlinux.lds.h */
  1583. static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
  1584. {
  1585. WARN_ON_ONCE(rq->curr != prev);
  1586. prev->sched_class->put_prev_task(rq, prev);
  1587. }
  1588. static inline void set_next_task(struct rq *rq, struct task_struct *next)
  1589. {
  1590. WARN_ON_ONCE(rq->curr != next);
  1591. next->sched_class->set_next_task(rq, next, false);
  1592. }
  1593. /* Defined in include/asm-generic/vmlinux.lds.h */
  1594. extern struct sched_class __begin_sched_classes[];
  1595. extern struct sched_class __end_sched_classes[];
  1596. #define sched_class_highest (__end_sched_classes - 1)
  1597. #define sched_class_lowest (__begin_sched_classes - 1)
  1598. #define for_class_range(class, _from, _to) \
  1599. for (class = (_from); class != (_to); class--)
  1600. #define for_each_class(class) \
  1601. for_class_range(class, sched_class_highest, sched_class_lowest)
  1602. extern const struct sched_class stop_sched_class;
  1603. extern const struct sched_class dl_sched_class;
  1604. extern const struct sched_class rt_sched_class;
  1605. extern const struct sched_class fair_sched_class;
  1606. extern const struct sched_class idle_sched_class;
  1607. static inline bool sched_stop_runnable(struct rq *rq)
  1608. {
  1609. return rq->stop && task_on_rq_queued(rq->stop);
  1610. }
  1611. static inline bool sched_dl_runnable(struct rq *rq)
  1612. {
  1613. return rq->dl.dl_nr_running > 0;
  1614. }
  1615. static inline bool sched_rt_runnable(struct rq *rq)
  1616. {
  1617. return rq->rt.rt_queued > 0;
  1618. }
  1619. static inline bool sched_fair_runnable(struct rq *rq)
  1620. {
  1621. return rq->cfs.nr_running > 0;
  1622. }
  1623. extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
  1624. extern struct task_struct *pick_next_task_idle(struct rq *rq);
  1625. #ifdef CONFIG_SMP
  1626. extern void update_group_capacity(struct sched_domain *sd, int cpu);
  1627. extern void trigger_load_balance(struct rq *rq);
  1628. extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask);
  1629. extern unsigned long __read_mostly max_load_balance_interval;
  1630. #endif
  1631. #ifdef CONFIG_CPU_IDLE
  1632. static inline void idle_set_state(struct rq *rq,
  1633. struct cpuidle_state *idle_state)
  1634. {
  1635. rq->idle_state = idle_state;
  1636. }
  1637. static inline struct cpuidle_state *idle_get_state(struct rq *rq)
  1638. {
  1639. SCHED_WARN_ON(!rcu_read_lock_held());
  1640. return rq->idle_state;
  1641. }
  1642. #else
  1643. static inline void idle_set_state(struct rq *rq,
  1644. struct cpuidle_state *idle_state)
  1645. {
  1646. }
  1647. static inline struct cpuidle_state *idle_get_state(struct rq *rq)
  1648. {
  1649. return NULL;
  1650. }
  1651. #endif
  1652. extern void schedule_idle(void);
  1653. extern void sysrq_sched_debug_show(void);
  1654. extern void sched_init_granularity(void);
  1655. extern void update_max_interval(void);
  1656. extern void init_sched_dl_class(void);
  1657. extern void init_sched_rt_class(void);
  1658. extern void init_sched_fair_class(void);
  1659. extern void reweight_task(struct task_struct *p, int prio);
  1660. extern void resched_curr(struct rq *rq);
  1661. extern void resched_cpu(int cpu);
  1662. extern struct rt_bandwidth def_rt_bandwidth;
  1663. extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
  1664. extern struct dl_bandwidth def_dl_bandwidth;
  1665. extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime);
  1666. extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
  1667. extern void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se);
  1668. #define BW_SHIFT 20
  1669. #define BW_UNIT (1 << BW_SHIFT)
  1670. #define RATIO_SHIFT 8
  1671. #define MAX_BW_BITS (64 - BW_SHIFT)
  1672. #define MAX_BW ((1ULL << MAX_BW_BITS) - 1)
  1673. unsigned long to_ratio(u64 period, u64 runtime);
  1674. extern void init_entity_runnable_average(struct sched_entity *se);
  1675. extern void post_init_entity_util_avg(struct task_struct *p);
  1676. #ifdef CONFIG_NO_HZ_FULL
  1677. extern bool sched_can_stop_tick(struct rq *rq);
  1678. extern int __init sched_tick_offload_init(void);
  1679. /*
  1680. * Tick may be needed by tasks in the runqueue depending on their policy and
  1681. * requirements. If tick is needed, lets send the target an IPI to kick it out of
  1682. * nohz mode if necessary.
  1683. */
  1684. static inline void sched_update_tick_dependency(struct rq *rq)
  1685. {
  1686. int cpu = cpu_of(rq);
  1687. if (!tick_nohz_full_cpu(cpu))
  1688. return;
  1689. if (sched_can_stop_tick(rq))
  1690. tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED);
  1691. else
  1692. tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
  1693. }
  1694. #else
  1695. static inline int sched_tick_offload_init(void) { return 0; }
  1696. static inline void sched_update_tick_dependency(struct rq *rq) { }
  1697. #endif
  1698. static inline void add_nr_running(struct rq *rq, unsigned count)
  1699. {
  1700. unsigned prev_nr = rq->nr_running;
  1701. rq->nr_running = prev_nr + count;
  1702. if (trace_sched_update_nr_running_tp_enabled()) {
  1703. call_trace_sched_update_nr_running(rq, count);
  1704. }
  1705. #ifdef CONFIG_SMP
  1706. if (prev_nr < 2 && rq->nr_running >= 2) {
  1707. if (!READ_ONCE(rq->rd->overload))
  1708. WRITE_ONCE(rq->rd->overload, 1);
  1709. }
  1710. #endif
  1711. sched_update_tick_dependency(rq);
  1712. }
  1713. static inline void sub_nr_running(struct rq *rq, unsigned count)
  1714. {
  1715. rq->nr_running -= count;
  1716. if (trace_sched_update_nr_running_tp_enabled()) {
  1717. call_trace_sched_update_nr_running(rq, -count);
  1718. }
  1719. /* Check if we still need preemption */
  1720. sched_update_tick_dependency(rq);
  1721. }
  1722. extern void activate_task(struct rq *rq, struct task_struct *p, int flags);
  1723. extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
  1724. extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
  1725. extern const_debug unsigned int sysctl_sched_nr_migrate;
  1726. extern const_debug unsigned int sysctl_sched_migration_cost;
  1727. #ifdef CONFIG_SCHED_HRTICK
  1728. /*
  1729. * Use hrtick when:
  1730. * - enabled by features
  1731. * - hrtimer is actually high res
  1732. */
  1733. static inline int hrtick_enabled(struct rq *rq)
  1734. {
  1735. if (!sched_feat(HRTICK))
  1736. return 0;
  1737. if (!cpu_active(cpu_of(rq)))
  1738. return 0;
  1739. return hrtimer_is_hres_active(&rq->hrtick_timer);
  1740. }
  1741. void hrtick_start(struct rq *rq, u64 delay);
  1742. #else
  1743. static inline int hrtick_enabled(struct rq *rq)
  1744. {
  1745. return 0;
  1746. }
  1747. #endif /* CONFIG_SCHED_HRTICK */
  1748. #ifndef arch_scale_freq_tick
  1749. static __always_inline
  1750. void arch_scale_freq_tick(void)
  1751. {
  1752. }
  1753. #endif
  1754. #ifndef arch_scale_freq_capacity
  1755. /**
  1756. * arch_scale_freq_capacity - get the frequency scale factor of a given CPU.
  1757. * @cpu: the CPU in question.
  1758. *
  1759. * Return: the frequency scale factor normalized against SCHED_CAPACITY_SCALE, i.e.
  1760. *
  1761. * f_curr
  1762. * ------ * SCHED_CAPACITY_SCALE
  1763. * f_max
  1764. */
  1765. static __always_inline
  1766. unsigned long arch_scale_freq_capacity(int cpu)
  1767. {
  1768. return SCHED_CAPACITY_SCALE;
  1769. }
  1770. #endif
  1771. #ifdef CONFIG_SMP
  1772. #ifdef CONFIG_PREEMPTION
  1773. static inline void double_rq_lock(struct rq *rq1, struct rq *rq2);
  1774. /*
  1775. * fair double_lock_balance: Safely acquires both rq->locks in a fair
  1776. * way at the expense of forcing extra atomic operations in all
  1777. * invocations. This assures that the double_lock is acquired using the
  1778. * same underlying policy as the spinlock_t on this architecture, which
  1779. * reduces latency compared to the unfair variant below. However, it
  1780. * also adds more overhead and therefore may reduce throughput.
  1781. */
  1782. static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1783. __releases(this_rq->lock)
  1784. __acquires(busiest->lock)
  1785. __acquires(this_rq->lock)
  1786. {
  1787. raw_spin_unlock(&this_rq->lock);
  1788. double_rq_lock(this_rq, busiest);
  1789. return 1;
  1790. }
  1791. #else
  1792. /*
  1793. * Unfair double_lock_balance: Optimizes throughput at the expense of
  1794. * latency by eliminating extra atomic operations when the locks are
  1795. * already in proper order on entry. This favors lower CPU-ids and will
  1796. * grant the double lock to lower CPUs over higher ids under contention,
  1797. * regardless of entry order into the function.
  1798. */
  1799. static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1800. __releases(this_rq->lock)
  1801. __acquires(busiest->lock)
  1802. __acquires(this_rq->lock)
  1803. {
  1804. int ret = 0;
  1805. if (unlikely(!raw_spin_trylock(&busiest->lock))) {
  1806. if (busiest < this_rq) {
  1807. raw_spin_unlock(&this_rq->lock);
  1808. raw_spin_lock(&busiest->lock);
  1809. raw_spin_lock_nested(&this_rq->lock,
  1810. SINGLE_DEPTH_NESTING);
  1811. ret = 1;
  1812. } else
  1813. raw_spin_lock_nested(&busiest->lock,
  1814. SINGLE_DEPTH_NESTING);
  1815. }
  1816. return ret;
  1817. }
  1818. #endif /* CONFIG_PREEMPTION */
  1819. /*
  1820. * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
  1821. */
  1822. static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1823. {
  1824. if (unlikely(!irqs_disabled())) {
  1825. /* printk() doesn't work well under rq->lock */
  1826. raw_spin_unlock(&this_rq->lock);
  1827. BUG_ON(1);
  1828. }
  1829. return _double_lock_balance(this_rq, busiest);
  1830. }
  1831. static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
  1832. __releases(busiest->lock)
  1833. {
  1834. raw_spin_unlock(&busiest->lock);
  1835. lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
  1836. }
  1837. static inline void double_lock(spinlock_t *l1, spinlock_t *l2)
  1838. {
  1839. if (l1 > l2)
  1840. swap(l1, l2);
  1841. spin_lock(l1);
  1842. spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
  1843. }
  1844. static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2)
  1845. {
  1846. if (l1 > l2)
  1847. swap(l1, l2);
  1848. spin_lock_irq(l1);
  1849. spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
  1850. }
  1851. static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2)
  1852. {
  1853. if (l1 > l2)
  1854. swap(l1, l2);
  1855. raw_spin_lock(l1);
  1856. raw_spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
  1857. }
  1858. /*
  1859. * double_rq_lock - safely lock two runqueues
  1860. *
  1861. * Note this does not disable interrupts like task_rq_lock,
  1862. * you need to do so manually before calling.
  1863. */
  1864. static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)
  1865. __acquires(rq1->lock)
  1866. __acquires(rq2->lock)
  1867. {
  1868. BUG_ON(!irqs_disabled());
  1869. if (rq1 == rq2) {
  1870. raw_spin_lock(&rq1->lock);
  1871. __acquire(rq2->lock); /* Fake it out ;) */
  1872. } else {
  1873. if (rq1 < rq2) {
  1874. raw_spin_lock(&rq1->lock);
  1875. raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
  1876. } else {
  1877. raw_spin_lock(&rq2->lock);
  1878. raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
  1879. }
  1880. }
  1881. }
  1882. /*
  1883. * double_rq_unlock - safely unlock two runqueues
  1884. *
  1885. * Note this does not restore interrupts like task_rq_unlock,
  1886. * you need to do so manually after calling.
  1887. */
  1888. static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  1889. __releases(rq1->lock)
  1890. __releases(rq2->lock)
  1891. {
  1892. raw_spin_unlock(&rq1->lock);
  1893. if (rq1 != rq2)
  1894. raw_spin_unlock(&rq2->lock);
  1895. else
  1896. __release(rq2->lock);
  1897. }
  1898. extern void set_rq_online (struct rq *rq);
  1899. extern void set_rq_offline(struct rq *rq);
  1900. extern bool sched_smp_initialized;
  1901. #else /* CONFIG_SMP */
  1902. /*
  1903. * double_rq_lock - safely lock two runqueues
  1904. *
  1905. * Note this does not disable interrupts like task_rq_lock,
  1906. * you need to do so manually before calling.
  1907. */
  1908. static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)
  1909. __acquires(rq1->lock)
  1910. __acquires(rq2->lock)
  1911. {
  1912. BUG_ON(!irqs_disabled());
  1913. BUG_ON(rq1 != rq2);
  1914. raw_spin_lock(&rq1->lock);
  1915. __acquire(rq2->lock); /* Fake it out ;) */
  1916. }
  1917. /*
  1918. * double_rq_unlock - safely unlock two runqueues
  1919. *
  1920. * Note this does not restore interrupts like task_rq_unlock,
  1921. * you need to do so manually after calling.
  1922. */
  1923. static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  1924. __releases(rq1->lock)
  1925. __releases(rq2->lock)
  1926. {
  1927. BUG_ON(rq1 != rq2);
  1928. raw_spin_unlock(&rq1->lock);
  1929. __release(rq2->lock);
  1930. }
  1931. #endif
  1932. extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq);
  1933. extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq);
  1934. #ifdef CONFIG_SCHED_DEBUG
  1935. extern bool sched_debug_enabled;
  1936. extern void print_cfs_stats(struct seq_file *m, int cpu);
  1937. extern void print_rt_stats(struct seq_file *m, int cpu);
  1938. extern void print_dl_stats(struct seq_file *m, int cpu);
  1939. extern void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq);
  1940. extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
  1941. extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq);
  1942. #ifdef CONFIG_NUMA_BALANCING
  1943. extern void
  1944. show_numa_stats(struct task_struct *p, struct seq_file *m);
  1945. extern void
  1946. print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
  1947. unsigned long tpf, unsigned long gsf, unsigned long gpf);
  1948. #endif /* CONFIG_NUMA_BALANCING */
  1949. #endif /* CONFIG_SCHED_DEBUG */
  1950. extern void init_cfs_rq(struct cfs_rq *cfs_rq);
  1951. extern void init_rt_rq(struct rt_rq *rt_rq);
  1952. extern void init_dl_rq(struct dl_rq *dl_rq);
  1953. extern void cfs_bandwidth_usage_inc(void);
  1954. extern void cfs_bandwidth_usage_dec(void);
  1955. #ifdef CONFIG_NO_HZ_COMMON
  1956. #define NOHZ_BALANCE_KICK_BIT 0
  1957. #define NOHZ_STATS_KICK_BIT 1
  1958. #define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
  1959. #define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT)
  1960. #define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
  1961. #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
  1962. extern void nohz_balance_exit_idle(struct rq *rq);
  1963. #else
  1964. static inline void nohz_balance_exit_idle(struct rq *rq) { }
  1965. #endif
  1966. #ifdef CONFIG_SMP
  1967. static inline
  1968. void __dl_update(struct dl_bw *dl_b, s64 bw)
  1969. {
  1970. struct root_domain *rd = container_of(dl_b, struct root_domain, dl_bw);
  1971. int i;
  1972. RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
  1973. "sched RCU must be held");
  1974. for_each_cpu_and(i, rd->span, cpu_active_mask) {
  1975. struct rq *rq = cpu_rq(i);
  1976. rq->dl.extra_bw += bw;
  1977. }
  1978. }
  1979. #else
  1980. static inline
  1981. void __dl_update(struct dl_bw *dl_b, s64 bw)
  1982. {
  1983. struct dl_rq *dl = container_of(dl_b, struct dl_rq, dl_bw);
  1984. dl->extra_bw += bw;
  1985. }
  1986. #endif
  1987. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  1988. struct irqtime {
  1989. u64 total;
  1990. u64 tick_delta;
  1991. u64 irq_start_time;
  1992. struct u64_stats_sync sync;
  1993. };
  1994. DECLARE_PER_CPU(struct irqtime, cpu_irqtime);
  1995. /*
  1996. * Returns the irqtime minus the softirq time computed by ksoftirqd.
  1997. * Otherwise ksoftirqd's sum_exec_runtime is substracted its own runtime
  1998. * and never move forward.
  1999. */
  2000. static inline u64 irq_time_read(int cpu)
  2001. {
  2002. struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu);
  2003. unsigned int seq;
  2004. u64 total;
  2005. do {
  2006. seq = __u64_stats_fetch_begin(&irqtime->sync);
  2007. total = irqtime->total;
  2008. } while (__u64_stats_fetch_retry(&irqtime->sync, seq));
  2009. return total;
  2010. }
  2011. #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
  2012. #ifdef CONFIG_CPU_FREQ
  2013. DECLARE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data);
  2014. /**
  2015. * cpufreq_update_util - Take a note about CPU utilization changes.
  2016. * @rq: Runqueue to carry out the update for.
  2017. * @flags: Update reason flags.
  2018. *
  2019. * This function is called by the scheduler on the CPU whose utilization is
  2020. * being updated.
  2021. *
  2022. * It can only be called from RCU-sched read-side critical sections.
  2023. *
  2024. * The way cpufreq is currently arranged requires it to evaluate the CPU
  2025. * performance state (frequency/voltage) on a regular basis to prevent it from
  2026. * being stuck in a completely inadequate performance level for too long.
  2027. * That is not guaranteed to happen if the updates are only triggered from CFS
  2028. * and DL, though, because they may not be coming in if only RT tasks are
  2029. * active all the time (or there are RT tasks only).
  2030. *
  2031. * As a workaround for that issue, this function is called periodically by the
  2032. * RT sched class to trigger extra cpufreq updates to prevent it from stalling,
  2033. * but that really is a band-aid. Going forward it should be replaced with
  2034. * solutions targeted more specifically at RT tasks.
  2035. */
  2036. static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
  2037. {
  2038. struct update_util_data *data;
  2039. data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data,
  2040. cpu_of(rq)));
  2041. if (data)
  2042. data->func(data, rq_clock(rq), flags);
  2043. }
  2044. #else
  2045. static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
  2046. #endif /* CONFIG_CPU_FREQ */
  2047. #ifdef CONFIG_UCLAMP_TASK
  2048. unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id);
  2049. /**
  2050. * uclamp_rq_util_with - clamp @util with @rq and @p effective uclamp values.
  2051. * @rq: The rq to clamp against. Must not be NULL.
  2052. * @util: The util value to clamp.
  2053. * @p: The task to clamp against. Can be NULL if you want to clamp
  2054. * against @rq only.
  2055. *
  2056. * Clamps the passed @util to the max(@rq, @p) effective uclamp values.
  2057. *
  2058. * If sched_uclamp_used static key is disabled, then just return the util
  2059. * without any clamping since uclamp aggregation at the rq level in the fast
  2060. * path is disabled, rendering this operation a NOP.
  2061. *
  2062. * Use uclamp_eff_value() if you don't care about uclamp values at rq level. It
  2063. * will return the correct effective uclamp value of the task even if the
  2064. * static key is disabled.
  2065. */
  2066. static __always_inline
  2067. unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
  2068. struct task_struct *p)
  2069. {
  2070. unsigned long min_util = 0;
  2071. unsigned long max_util = 0;
  2072. if (!static_branch_likely(&sched_uclamp_used))
  2073. return util;
  2074. if (p) {
  2075. min_util = uclamp_eff_value(p, UCLAMP_MIN);
  2076. max_util = uclamp_eff_value(p, UCLAMP_MAX);
  2077. /*
  2078. * Ignore last runnable task's max clamp, as this task will
  2079. * reset it. Similarly, no need to read the rq's min clamp.
  2080. */
  2081. if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
  2082. goto out;
  2083. }
  2084. min_util = max_t(unsigned long, min_util, READ_ONCE(rq->uclamp[UCLAMP_MIN].value));
  2085. max_util = max_t(unsigned long, max_util, READ_ONCE(rq->uclamp[UCLAMP_MAX].value));
  2086. out:
  2087. /*
  2088. * Since CPU's {min,max}_util clamps are MAX aggregated considering
  2089. * RUNNABLE tasks with _different_ clamps, we can end up with an
  2090. * inversion. Fix it now when the clamps are applied.
  2091. */
  2092. if (unlikely(min_util >= max_util))
  2093. return min_util;
  2094. return clamp(util, min_util, max_util);
  2095. }
  2096. static inline bool uclamp_boosted(struct task_struct *p)
  2097. {
  2098. return uclamp_eff_value(p, UCLAMP_MIN) > 0;
  2099. }
  2100. /*
  2101. * When uclamp is compiled in, the aggregation at rq level is 'turned off'
  2102. * by default in the fast path and only gets turned on once userspace performs
  2103. * an operation that requires it.
  2104. *
  2105. * Returns true if userspace opted-in to use uclamp and aggregation at rq level
  2106. * hence is active.
  2107. */
  2108. static inline bool uclamp_is_used(void)
  2109. {
  2110. return static_branch_likely(&sched_uclamp_used);
  2111. }
  2112. #else /* CONFIG_UCLAMP_TASK */
  2113. static inline
  2114. unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
  2115. struct task_struct *p)
  2116. {
  2117. return util;
  2118. }
  2119. static inline bool uclamp_boosted(struct task_struct *p)
  2120. {
  2121. return false;
  2122. }
  2123. static inline bool uclamp_is_used(void)
  2124. {
  2125. return false;
  2126. }
  2127. #endif /* CONFIG_UCLAMP_TASK */
  2128. #ifdef CONFIG_UCLAMP_TASK_GROUP
  2129. static inline bool uclamp_latency_sensitive(struct task_struct *p)
  2130. {
  2131. struct cgroup_subsys_state *css = task_css(p, cpu_cgrp_id);
  2132. struct task_group *tg;
  2133. if (!css)
  2134. return false;
  2135. tg = container_of(css, struct task_group, css);
  2136. return tg->latency_sensitive;
  2137. }
  2138. #else
  2139. static inline bool uclamp_latency_sensitive(struct task_struct *p)
  2140. {
  2141. return false;
  2142. }
  2143. #endif /* CONFIG_UCLAMP_TASK_GROUP */
  2144. #ifdef arch_scale_freq_capacity
  2145. # ifndef arch_scale_freq_invariant
  2146. # define arch_scale_freq_invariant() true
  2147. # endif
  2148. #else
  2149. # define arch_scale_freq_invariant() false
  2150. #endif
  2151. #ifdef CONFIG_SMP
  2152. static inline unsigned long capacity_orig_of(int cpu)
  2153. {
  2154. return cpu_rq(cpu)->cpu_capacity_orig;
  2155. }
  2156. #endif
  2157. /**
  2158. * enum schedutil_type - CPU utilization type
  2159. * @FREQUENCY_UTIL: Utilization used to select frequency
  2160. * @ENERGY_UTIL: Utilization used during energy calculation
  2161. *
  2162. * The utilization signals of all scheduling classes (CFS/RT/DL) and IRQ time
  2163. * need to be aggregated differently depending on the usage made of them. This
  2164. * enum is used within schedutil_freq_util() to differentiate the types of
  2165. * utilization expected by the callers, and adjust the aggregation accordingly.
  2166. */
  2167. enum schedutil_type {
  2168. FREQUENCY_UTIL,
  2169. ENERGY_UTIL,
  2170. };
  2171. #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
  2172. unsigned long schedutil_cpu_util(int cpu, unsigned long util_cfs,
  2173. unsigned long max, enum schedutil_type type,
  2174. struct task_struct *p);
  2175. static inline unsigned long cpu_bw_dl(struct rq *rq)
  2176. {
  2177. return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT;
  2178. }
  2179. static inline unsigned long cpu_util_dl(struct rq *rq)
  2180. {
  2181. return READ_ONCE(rq->avg_dl.util_avg);
  2182. }
  2183. static inline unsigned long cpu_util_cfs(struct rq *rq)
  2184. {
  2185. unsigned long util = READ_ONCE(rq->cfs.avg.util_avg);
  2186. if (sched_feat(UTIL_EST)) {
  2187. util = max_t(unsigned long, util,
  2188. READ_ONCE(rq->cfs.avg.util_est.enqueued));
  2189. }
  2190. return util;
  2191. }
  2192. static inline unsigned long cpu_util_rt(struct rq *rq)
  2193. {
  2194. return READ_ONCE(rq->avg_rt.util_avg);
  2195. }
  2196. #else /* CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
  2197. static inline unsigned long schedutil_cpu_util(int cpu, unsigned long util_cfs,
  2198. unsigned long max, enum schedutil_type type,
  2199. struct task_struct *p)
  2200. {
  2201. return 0;
  2202. }
  2203. #endif /* CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
  2204. #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
  2205. static inline unsigned long cpu_util_irq(struct rq *rq)
  2206. {
  2207. return rq->avg_irq.util_avg;
  2208. }
  2209. static inline
  2210. unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)
  2211. {
  2212. util *= (max - irq);
  2213. util /= max;
  2214. return util;
  2215. }
  2216. #else
  2217. static inline unsigned long cpu_util_irq(struct rq *rq)
  2218. {
  2219. return 0;
  2220. }
  2221. static inline
  2222. unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)
  2223. {
  2224. return util;
  2225. }
  2226. #endif
  2227. #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
  2228. #define perf_domain_span(pd) (to_cpumask(((pd)->em_pd->cpus)))
  2229. DECLARE_STATIC_KEY_FALSE(sched_energy_present);
  2230. static inline bool sched_energy_enabled(void)
  2231. {
  2232. return static_branch_unlikely(&sched_energy_present);
  2233. }
  2234. #else /* ! (CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */
  2235. #define perf_domain_span(pd) NULL
  2236. static inline bool sched_energy_enabled(void) { return false; }
  2237. #endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
  2238. #ifdef CONFIG_MEMBARRIER
  2239. /*
  2240. * The scheduler provides memory barriers required by membarrier between:
  2241. * - prior user-space memory accesses and store to rq->membarrier_state,
  2242. * - store to rq->membarrier_state and following user-space memory accesses.
  2243. * In the same way it provides those guarantees around store to rq->curr.
  2244. */
  2245. static inline void membarrier_switch_mm(struct rq *rq,
  2246. struct mm_struct *prev_mm,
  2247. struct mm_struct *next_mm)
  2248. {
  2249. int membarrier_state;
  2250. if (prev_mm == next_mm)
  2251. return;
  2252. membarrier_state = atomic_read(&next_mm->membarrier_state);
  2253. if (READ_ONCE(rq->membarrier_state) == membarrier_state)
  2254. return;
  2255. WRITE_ONCE(rq->membarrier_state, membarrier_state);
  2256. }
  2257. #else
  2258. static inline void membarrier_switch_mm(struct rq *rq,
  2259. struct mm_struct *prev_mm,
  2260. struct mm_struct *next_mm)
  2261. {
  2262. }
  2263. #endif
  2264. #ifdef CONFIG_SMP
  2265. static inline bool is_per_cpu_kthread(struct task_struct *p)
  2266. {
  2267. if (!(p->flags & PF_KTHREAD))
  2268. return false;
  2269. if (p->nr_cpus_allowed != 1)
  2270. return false;
  2271. return true;
  2272. }
  2273. #endif
  2274. void swake_up_all_locked(struct swait_queue_head *q);
  2275. void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
  2276. /*
  2277. * task_may_not_preempt - check whether a task may not be preemptible soon
  2278. */
  2279. #ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
  2280. extern bool task_may_not_preempt(struct task_struct *task, int cpu);
  2281. #else
  2282. static inline bool task_may_not_preempt(struct task_struct *task, int cpu)
  2283. {
  2284. return false;
  2285. }
  2286. #endif /* CONFIG_RT_SOFTINT_OPTIMIZATION */