vmstat.c 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/mm/vmstat.c
  4. *
  5. * Manages VM statistics
  6. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  7. *
  8. * zoned VM statistics
  9. * Copyright (C) 2006 Silicon Graphics, Inc.,
  10. * Christoph Lameter <christoph@lameter.com>
  11. * Copyright (C) 2008-2014 Christoph Lameter
  12. */
  13. #include <linux/fs.h>
  14. #include <linux/mm.h>
  15. #include <linux/err.h>
  16. #include <linux/module.h>
  17. #include <linux/slab.h>
  18. #include <linux/cpu.h>
  19. #include <linux/cpumask.h>
  20. #include <linux/vmstat.h>
  21. #include <linux/proc_fs.h>
  22. #include <linux/seq_file.h>
  23. #include <linux/debugfs.h>
  24. #include <linux/sched.h>
  25. #include <linux/math64.h>
  26. #include <linux/writeback.h>
  27. #include <linux/compaction.h>
  28. #include <linux/mm_inline.h>
  29. #include <linux/page_ext.h>
  30. #include <linux/page_owner.h>
  31. #include "internal.h"
  32. #define NUMA_STATS_THRESHOLD (U16_MAX - 2)
  33. #ifdef CONFIG_NUMA
  34. int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
  35. /* zero numa counters within a zone */
  36. static void zero_zone_numa_counters(struct zone *zone)
  37. {
  38. int item, cpu;
  39. for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) {
  40. atomic_long_set(&zone->vm_numa_stat[item], 0);
  41. for_each_online_cpu(cpu)
  42. per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item]
  43. = 0;
  44. }
  45. }
  46. /* zero numa counters of all the populated zones */
  47. static void zero_zones_numa_counters(void)
  48. {
  49. struct zone *zone;
  50. for_each_populated_zone(zone)
  51. zero_zone_numa_counters(zone);
  52. }
  53. /* zero global numa counters */
  54. static void zero_global_numa_counters(void)
  55. {
  56. int item;
  57. for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++)
  58. atomic_long_set(&vm_numa_stat[item], 0);
  59. }
  60. static void invalid_numa_statistics(void)
  61. {
  62. zero_zones_numa_counters();
  63. zero_global_numa_counters();
  64. }
  65. static DEFINE_MUTEX(vm_numa_stat_lock);
  66. int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
  67. void *buffer, size_t *length, loff_t *ppos)
  68. {
  69. int ret, oldval;
  70. mutex_lock(&vm_numa_stat_lock);
  71. if (write)
  72. oldval = sysctl_vm_numa_stat;
  73. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  74. if (ret || !write)
  75. goto out;
  76. if (oldval == sysctl_vm_numa_stat)
  77. goto out;
  78. else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
  79. static_branch_enable(&vm_numa_stat_key);
  80. pr_info("enable numa statistics\n");
  81. } else {
  82. static_branch_disable(&vm_numa_stat_key);
  83. invalid_numa_statistics();
  84. pr_info("disable numa statistics, and clear numa counters\n");
  85. }
  86. out:
  87. mutex_unlock(&vm_numa_stat_lock);
  88. return ret;
  89. }
  90. #endif
  91. #ifdef CONFIG_VM_EVENT_COUNTERS
  92. DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
  93. EXPORT_PER_CPU_SYMBOL(vm_event_states);
  94. static void sum_vm_events(unsigned long *ret)
  95. {
  96. int cpu;
  97. int i;
  98. memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
  99. for_each_online_cpu(cpu) {
  100. struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
  101. for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
  102. ret[i] += this->event[i];
  103. }
  104. }
  105. /*
  106. * Accumulate the vm event counters across all CPUs.
  107. * The result is unavoidably approximate - it can change
  108. * during and after execution of this function.
  109. */
  110. void all_vm_events(unsigned long *ret)
  111. {
  112. get_online_cpus();
  113. sum_vm_events(ret);
  114. put_online_cpus();
  115. }
  116. EXPORT_SYMBOL_GPL(all_vm_events);
  117. /*
  118. * Fold the foreign cpu events into our own.
  119. *
  120. * This is adding to the events on one processor
  121. * but keeps the global counts constant.
  122. */
  123. void vm_events_fold_cpu(int cpu)
  124. {
  125. struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
  126. int i;
  127. for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
  128. count_vm_events(i, fold_state->event[i]);
  129. fold_state->event[i] = 0;
  130. }
  131. }
  132. #endif /* CONFIG_VM_EVENT_COUNTERS */
  133. /*
  134. * Manage combined zone based / global counters
  135. *
  136. * vm_stat contains the global counters
  137. */
  138. atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
  139. atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS] __cacheline_aligned_in_smp;
  140. atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
  141. EXPORT_SYMBOL(vm_zone_stat);
  142. EXPORT_SYMBOL(vm_numa_stat);
  143. EXPORT_SYMBOL(vm_node_stat);
  144. #ifdef CONFIG_SMP
  145. int calculate_pressure_threshold(struct zone *zone)
  146. {
  147. int threshold;
  148. int watermark_distance;
  149. /*
  150. * As vmstats are not up to date, there is drift between the estimated
  151. * and real values. For high thresholds and a high number of CPUs, it
  152. * is possible for the min watermark to be breached while the estimated
  153. * value looks fine. The pressure threshold is a reduced value such
  154. * that even the maximum amount of drift will not accidentally breach
  155. * the min watermark
  156. */
  157. watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
  158. threshold = max(1, (int)(watermark_distance / num_online_cpus()));
  159. /*
  160. * Maximum threshold is 125
  161. */
  162. threshold = min(125, threshold);
  163. return threshold;
  164. }
  165. int calculate_normal_threshold(struct zone *zone)
  166. {
  167. int threshold;
  168. int mem; /* memory in 128 MB units */
  169. /*
  170. * The threshold scales with the number of processors and the amount
  171. * of memory per zone. More memory means that we can defer updates for
  172. * longer, more processors could lead to more contention.
  173. * fls() is used to have a cheap way of logarithmic scaling.
  174. *
  175. * Some sample thresholds:
  176. *
  177. * Threshold Processors (fls) Zonesize fls(mem+1)
  178. * ------------------------------------------------------------------
  179. * 8 1 1 0.9-1 GB 4
  180. * 16 2 2 0.9-1 GB 4
  181. * 20 2 2 1-2 GB 5
  182. * 24 2 2 2-4 GB 6
  183. * 28 2 2 4-8 GB 7
  184. * 32 2 2 8-16 GB 8
  185. * 4 2 2 <128M 1
  186. * 30 4 3 2-4 GB 5
  187. * 48 4 3 8-16 GB 8
  188. * 32 8 4 1-2 GB 4
  189. * 32 8 4 0.9-1GB 4
  190. * 10 16 5 <128M 1
  191. * 40 16 5 900M 4
  192. * 70 64 7 2-4 GB 5
  193. * 84 64 7 4-8 GB 6
  194. * 108 512 9 4-8 GB 6
  195. * 125 1024 10 8-16 GB 8
  196. * 125 1024 10 16-32 GB 9
  197. */
  198. mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);
  199. threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
  200. /*
  201. * Maximum threshold is 125
  202. */
  203. threshold = min(125, threshold);
  204. return threshold;
  205. }
  206. /*
  207. * Refresh the thresholds for each zone.
  208. */
  209. void refresh_zone_stat_thresholds(void)
  210. {
  211. struct pglist_data *pgdat;
  212. struct zone *zone;
  213. int cpu;
  214. int threshold;
  215. /* Zero current pgdat thresholds */
  216. for_each_online_pgdat(pgdat) {
  217. for_each_online_cpu(cpu) {
  218. per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
  219. }
  220. }
  221. for_each_populated_zone(zone) {
  222. struct pglist_data *pgdat = zone->zone_pgdat;
  223. unsigned long max_drift, tolerate_drift;
  224. threshold = calculate_normal_threshold(zone);
  225. for_each_online_cpu(cpu) {
  226. int pgdat_threshold;
  227. per_cpu_ptr(zone->pageset, cpu)->stat_threshold
  228. = threshold;
  229. /* Base nodestat threshold on the largest populated zone. */
  230. pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
  231. per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
  232. = max(threshold, pgdat_threshold);
  233. }
  234. /*
  235. * Only set percpu_drift_mark if there is a danger that
  236. * NR_FREE_PAGES reports the low watermark is ok when in fact
  237. * the min watermark could be breached by an allocation
  238. */
  239. tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
  240. max_drift = num_online_cpus() * threshold;
  241. if (max_drift > tolerate_drift)
  242. zone->percpu_drift_mark = high_wmark_pages(zone) +
  243. max_drift;
  244. }
  245. }
  246. void set_pgdat_percpu_threshold(pg_data_t *pgdat,
  247. int (*calculate_pressure)(struct zone *))
  248. {
  249. struct zone *zone;
  250. int cpu;
  251. int threshold;
  252. int i;
  253. for (i = 0; i < pgdat->nr_zones; i++) {
  254. zone = &pgdat->node_zones[i];
  255. if (!zone->percpu_drift_mark)
  256. continue;
  257. threshold = (*calculate_pressure)(zone);
  258. for_each_online_cpu(cpu)
  259. per_cpu_ptr(zone->pageset, cpu)->stat_threshold
  260. = threshold;
  261. }
  262. }
  263. /*
  264. * For use when we know that interrupts are disabled,
  265. * or when we know that preemption is disabled and that
  266. * particular counter cannot be updated from interrupt context.
  267. */
  268. void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
  269. long delta)
  270. {
  271. struct per_cpu_pageset __percpu *pcp = zone->pageset;
  272. s8 __percpu *p = pcp->vm_stat_diff + item;
  273. long x;
  274. long t;
  275. x = delta + __this_cpu_read(*p);
  276. t = __this_cpu_read(pcp->stat_threshold);
  277. if (unlikely(abs(x) > t)) {
  278. zone_page_state_add(x, zone, item);
  279. x = 0;
  280. }
  281. __this_cpu_write(*p, x);
  282. }
  283. EXPORT_SYMBOL(__mod_zone_page_state);
  284. void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
  285. long delta)
  286. {
  287. struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
  288. s8 __percpu *p = pcp->vm_node_stat_diff + item;
  289. long x;
  290. long t;
  291. if (vmstat_item_in_bytes(item)) {
  292. VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
  293. delta >>= PAGE_SHIFT;
  294. }
  295. x = delta + __this_cpu_read(*p);
  296. t = __this_cpu_read(pcp->stat_threshold);
  297. if (unlikely(abs(x) > t)) {
  298. node_page_state_add(x, pgdat, item);
  299. x = 0;
  300. }
  301. __this_cpu_write(*p, x);
  302. }
  303. EXPORT_SYMBOL(__mod_node_page_state);
  304. /*
  305. * Optimized increment and decrement functions.
  306. *
  307. * These are only for a single page and therefore can take a struct page *
  308. * argument instead of struct zone *. This allows the inclusion of the code
  309. * generated for page_zone(page) into the optimized functions.
  310. *
  311. * No overflow check is necessary and therefore the differential can be
  312. * incremented or decremented in place which may allow the compilers to
  313. * generate better code.
  314. * The increment or decrement is known and therefore one boundary check can
  315. * be omitted.
  316. *
  317. * NOTE: These functions are very performance sensitive. Change only
  318. * with care.
  319. *
  320. * Some processors have inc/dec instructions that are atomic vs an interrupt.
  321. * However, the code must first determine the differential location in a zone
  322. * based on the processor number and then inc/dec the counter. There is no
  323. * guarantee without disabling preemption that the processor will not change
  324. * in between and therefore the atomicity vs. interrupt cannot be exploited
  325. * in a useful way here.
  326. */
  327. void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
  328. {
  329. struct per_cpu_pageset __percpu *pcp = zone->pageset;
  330. s8 __percpu *p = pcp->vm_stat_diff + item;
  331. s8 v, t;
  332. v = __this_cpu_inc_return(*p);
  333. t = __this_cpu_read(pcp->stat_threshold);
  334. if (unlikely(v > t)) {
  335. s8 overstep = t >> 1;
  336. zone_page_state_add(v + overstep, zone, item);
  337. __this_cpu_write(*p, -overstep);
  338. }
  339. }
  340. void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
  341. {
  342. struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
  343. s8 __percpu *p = pcp->vm_node_stat_diff + item;
  344. s8 v, t;
  345. VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
  346. v = __this_cpu_inc_return(*p);
  347. t = __this_cpu_read(pcp->stat_threshold);
  348. if (unlikely(v > t)) {
  349. s8 overstep = t >> 1;
  350. node_page_state_add(v + overstep, pgdat, item);
  351. __this_cpu_write(*p, -overstep);
  352. }
  353. }
  354. void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
  355. {
  356. __inc_zone_state(page_zone(page), item);
  357. }
  358. EXPORT_SYMBOL(__inc_zone_page_state);
  359. void __inc_node_page_state(struct page *page, enum node_stat_item item)
  360. {
  361. __inc_node_state(page_pgdat(page), item);
  362. }
  363. EXPORT_SYMBOL(__inc_node_page_state);
  364. void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
  365. {
  366. struct per_cpu_pageset __percpu *pcp = zone->pageset;
  367. s8 __percpu *p = pcp->vm_stat_diff + item;
  368. s8 v, t;
  369. v = __this_cpu_dec_return(*p);
  370. t = __this_cpu_read(pcp->stat_threshold);
  371. if (unlikely(v < - t)) {
  372. s8 overstep = t >> 1;
  373. zone_page_state_add(v - overstep, zone, item);
  374. __this_cpu_write(*p, overstep);
  375. }
  376. }
  377. void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
  378. {
  379. struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
  380. s8 __percpu *p = pcp->vm_node_stat_diff + item;
  381. s8 v, t;
  382. VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
  383. v = __this_cpu_dec_return(*p);
  384. t = __this_cpu_read(pcp->stat_threshold);
  385. if (unlikely(v < - t)) {
  386. s8 overstep = t >> 1;
  387. node_page_state_add(v - overstep, pgdat, item);
  388. __this_cpu_write(*p, overstep);
  389. }
  390. }
  391. void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
  392. {
  393. __dec_zone_state(page_zone(page), item);
  394. }
  395. EXPORT_SYMBOL(__dec_zone_page_state);
  396. void __dec_node_page_state(struct page *page, enum node_stat_item item)
  397. {
  398. __dec_node_state(page_pgdat(page), item);
  399. }
  400. EXPORT_SYMBOL(__dec_node_page_state);
  401. #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
  402. /*
  403. * If we have cmpxchg_local support then we do not need to incur the overhead
  404. * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
  405. *
  406. * mod_state() modifies the zone counter state through atomic per cpu
  407. * operations.
  408. *
  409. * Overstep mode specifies how overstep should handled:
  410. * 0 No overstepping
  411. * 1 Overstepping half of threshold
  412. * -1 Overstepping minus half of threshold
  413. */
  414. static inline void mod_zone_state(struct zone *zone,
  415. enum zone_stat_item item, long delta, int overstep_mode)
  416. {
  417. struct per_cpu_pageset __percpu *pcp = zone->pageset;
  418. s8 __percpu *p = pcp->vm_stat_diff + item;
  419. long o, n, t, z;
  420. do {
  421. z = 0; /* overflow to zone counters */
  422. /*
  423. * The fetching of the stat_threshold is racy. We may apply
  424. * a counter threshold to the wrong the cpu if we get
  425. * rescheduled while executing here. However, the next
  426. * counter update will apply the threshold again and
  427. * therefore bring the counter under the threshold again.
  428. *
  429. * Most of the time the thresholds are the same anyways
  430. * for all cpus in a zone.
  431. */
  432. t = this_cpu_read(pcp->stat_threshold);
  433. o = this_cpu_read(*p);
  434. n = delta + o;
  435. if (abs(n) > t) {
  436. int os = overstep_mode * (t >> 1) ;
  437. /* Overflow must be added to zone counters */
  438. z = n + os;
  439. n = -os;
  440. }
  441. } while (this_cpu_cmpxchg(*p, o, n) != o);
  442. if (z)
  443. zone_page_state_add(z, zone, item);
  444. }
  445. void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
  446. long delta)
  447. {
  448. mod_zone_state(zone, item, delta, 0);
  449. }
  450. EXPORT_SYMBOL(mod_zone_page_state);
  451. void inc_zone_page_state(struct page *page, enum zone_stat_item item)
  452. {
  453. mod_zone_state(page_zone(page), item, 1, 1);
  454. }
  455. EXPORT_SYMBOL(inc_zone_page_state);
  456. void dec_zone_page_state(struct page *page, enum zone_stat_item item)
  457. {
  458. mod_zone_state(page_zone(page), item, -1, -1);
  459. }
  460. EXPORT_SYMBOL(dec_zone_page_state);
  461. static inline void mod_node_state(struct pglist_data *pgdat,
  462. enum node_stat_item item, int delta, int overstep_mode)
  463. {
  464. struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
  465. s8 __percpu *p = pcp->vm_node_stat_diff + item;
  466. long o, n, t, z;
  467. if (vmstat_item_in_bytes(item)) {
  468. VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
  469. delta >>= PAGE_SHIFT;
  470. }
  471. do {
  472. z = 0; /* overflow to node counters */
  473. /*
  474. * The fetching of the stat_threshold is racy. We may apply
  475. * a counter threshold to the wrong the cpu if we get
  476. * rescheduled while executing here. However, the next
  477. * counter update will apply the threshold again and
  478. * therefore bring the counter under the threshold again.
  479. *
  480. * Most of the time the thresholds are the same anyways
  481. * for all cpus in a node.
  482. */
  483. t = this_cpu_read(pcp->stat_threshold);
  484. o = this_cpu_read(*p);
  485. n = delta + o;
  486. if (abs(n) > t) {
  487. int os = overstep_mode * (t >> 1) ;
  488. /* Overflow must be added to node counters */
  489. z = n + os;
  490. n = -os;
  491. }
  492. } while (this_cpu_cmpxchg(*p, o, n) != o);
  493. if (z)
  494. node_page_state_add(z, pgdat, item);
  495. }
  496. void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
  497. long delta)
  498. {
  499. mod_node_state(pgdat, item, delta, 0);
  500. }
  501. EXPORT_SYMBOL(mod_node_page_state);
  502. void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
  503. {
  504. mod_node_state(pgdat, item, 1, 1);
  505. }
  506. void inc_node_page_state(struct page *page, enum node_stat_item item)
  507. {
  508. mod_node_state(page_pgdat(page), item, 1, 1);
  509. }
  510. EXPORT_SYMBOL(inc_node_page_state);
  511. void dec_node_page_state(struct page *page, enum node_stat_item item)
  512. {
  513. mod_node_state(page_pgdat(page), item, -1, -1);
  514. }
  515. EXPORT_SYMBOL(dec_node_page_state);
  516. #else
  517. /*
  518. * Use interrupt disable to serialize counter updates
  519. */
  520. void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
  521. long delta)
  522. {
  523. unsigned long flags;
  524. local_irq_save(flags);
  525. __mod_zone_page_state(zone, item, delta);
  526. local_irq_restore(flags);
  527. }
  528. EXPORT_SYMBOL(mod_zone_page_state);
  529. void inc_zone_page_state(struct page *page, enum zone_stat_item item)
  530. {
  531. unsigned long flags;
  532. struct zone *zone;
  533. zone = page_zone(page);
  534. local_irq_save(flags);
  535. __inc_zone_state(zone, item);
  536. local_irq_restore(flags);
  537. }
  538. EXPORT_SYMBOL(inc_zone_page_state);
  539. void dec_zone_page_state(struct page *page, enum zone_stat_item item)
  540. {
  541. unsigned long flags;
  542. local_irq_save(flags);
  543. __dec_zone_page_state(page, item);
  544. local_irq_restore(flags);
  545. }
  546. EXPORT_SYMBOL(dec_zone_page_state);
  547. void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
  548. {
  549. unsigned long flags;
  550. local_irq_save(flags);
  551. __inc_node_state(pgdat, item);
  552. local_irq_restore(flags);
  553. }
  554. EXPORT_SYMBOL(inc_node_state);
  555. void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
  556. long delta)
  557. {
  558. unsigned long flags;
  559. local_irq_save(flags);
  560. __mod_node_page_state(pgdat, item, delta);
  561. local_irq_restore(flags);
  562. }
  563. EXPORT_SYMBOL(mod_node_page_state);
  564. void inc_node_page_state(struct page *page, enum node_stat_item item)
  565. {
  566. unsigned long flags;
  567. struct pglist_data *pgdat;
  568. pgdat = page_pgdat(page);
  569. local_irq_save(flags);
  570. __inc_node_state(pgdat, item);
  571. local_irq_restore(flags);
  572. }
  573. EXPORT_SYMBOL(inc_node_page_state);
  574. void dec_node_page_state(struct page *page, enum node_stat_item item)
  575. {
  576. unsigned long flags;
  577. local_irq_save(flags);
  578. __dec_node_page_state(page, item);
  579. local_irq_restore(flags);
  580. }
  581. EXPORT_SYMBOL(dec_node_page_state);
  582. #endif
  583. /*
  584. * Fold a differential into the global counters.
  585. * Returns the number of counters updated.
  586. */
  587. #ifdef CONFIG_NUMA
  588. static int fold_diff(int *zone_diff, int *numa_diff, int *node_diff)
  589. {
  590. int i;
  591. int changes = 0;
  592. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  593. if (zone_diff[i]) {
  594. atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
  595. changes++;
  596. }
  597. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
  598. if (numa_diff[i]) {
  599. atomic_long_add(numa_diff[i], &vm_numa_stat[i]);
  600. changes++;
  601. }
  602. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  603. if (node_diff[i]) {
  604. atomic_long_add(node_diff[i], &vm_node_stat[i]);
  605. changes++;
  606. }
  607. return changes;
  608. }
  609. #else
  610. static int fold_diff(int *zone_diff, int *node_diff)
  611. {
  612. int i;
  613. int changes = 0;
  614. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  615. if (zone_diff[i]) {
  616. atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
  617. changes++;
  618. }
  619. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  620. if (node_diff[i]) {
  621. atomic_long_add(node_diff[i], &vm_node_stat[i]);
  622. changes++;
  623. }
  624. return changes;
  625. }
  626. #endif /* CONFIG_NUMA */
  627. /*
  628. * Update the zone counters for the current cpu.
  629. *
  630. * Note that refresh_cpu_vm_stats strives to only access
  631. * node local memory. The per cpu pagesets on remote zones are placed
  632. * in the memory local to the processor using that pageset. So the
  633. * loop over all zones will access a series of cachelines local to
  634. * the processor.
  635. *
  636. * The call to zone_page_state_add updates the cachelines with the
  637. * statistics in the remote zone struct as well as the global cachelines
  638. * with the global counters. These could cause remote node cache line
  639. * bouncing and will have to be only done when necessary.
  640. *
  641. * The function returns the number of global counters updated.
  642. */
  643. static int refresh_cpu_vm_stats(bool do_pagesets)
  644. {
  645. struct pglist_data *pgdat;
  646. struct zone *zone;
  647. int i;
  648. int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
  649. #ifdef CONFIG_NUMA
  650. int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, };
  651. #endif
  652. int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
  653. int changes = 0;
  654. for_each_populated_zone(zone) {
  655. struct per_cpu_pageset __percpu *p = zone->pageset;
  656. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
  657. int v;
  658. v = this_cpu_xchg(p->vm_stat_diff[i], 0);
  659. if (v) {
  660. atomic_long_add(v, &zone->vm_stat[i]);
  661. global_zone_diff[i] += v;
  662. #ifdef CONFIG_NUMA
  663. /* 3 seconds idle till flush */
  664. __this_cpu_write(p->expire, 3);
  665. #endif
  666. }
  667. }
  668. #ifdef CONFIG_NUMA
  669. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) {
  670. int v;
  671. v = this_cpu_xchg(p->vm_numa_stat_diff[i], 0);
  672. if (v) {
  673. atomic_long_add(v, &zone->vm_numa_stat[i]);
  674. global_numa_diff[i] += v;
  675. __this_cpu_write(p->expire, 3);
  676. }
  677. }
  678. if (do_pagesets) {
  679. cond_resched();
  680. /*
  681. * Deal with draining the remote pageset of this
  682. * processor
  683. *
  684. * Check if there are pages remaining in this pageset
  685. * if not then there is nothing to expire.
  686. */
  687. if (!__this_cpu_read(p->expire) ||
  688. !__this_cpu_read(p->pcp.count))
  689. continue;
  690. /*
  691. * We never drain zones local to this processor.
  692. */
  693. if (zone_to_nid(zone) == numa_node_id()) {
  694. __this_cpu_write(p->expire, 0);
  695. continue;
  696. }
  697. if (__this_cpu_dec_return(p->expire))
  698. continue;
  699. if (__this_cpu_read(p->pcp.count)) {
  700. drain_zone_pages(zone, this_cpu_ptr(&p->pcp));
  701. changes++;
  702. }
  703. }
  704. #endif
  705. }
  706. for_each_online_pgdat(pgdat) {
  707. struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
  708. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
  709. int v;
  710. v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
  711. if (v) {
  712. atomic_long_add(v, &pgdat->vm_stat[i]);
  713. global_node_diff[i] += v;
  714. }
  715. }
  716. }
  717. #ifdef CONFIG_NUMA
  718. changes += fold_diff(global_zone_diff, global_numa_diff,
  719. global_node_diff);
  720. #else
  721. changes += fold_diff(global_zone_diff, global_node_diff);
  722. #endif
  723. return changes;
  724. }
  725. /*
  726. * Fold the data for an offline cpu into the global array.
  727. * There cannot be any access by the offline cpu and therefore
  728. * synchronization is simplified.
  729. */
  730. void cpu_vm_stats_fold(int cpu)
  731. {
  732. struct pglist_data *pgdat;
  733. struct zone *zone;
  734. int i;
  735. int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
  736. #ifdef CONFIG_NUMA
  737. int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, };
  738. #endif
  739. int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
  740. for_each_populated_zone(zone) {
  741. struct per_cpu_pageset *p;
  742. p = per_cpu_ptr(zone->pageset, cpu);
  743. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  744. if (p->vm_stat_diff[i]) {
  745. int v;
  746. v = p->vm_stat_diff[i];
  747. p->vm_stat_diff[i] = 0;
  748. atomic_long_add(v, &zone->vm_stat[i]);
  749. global_zone_diff[i] += v;
  750. }
  751. #ifdef CONFIG_NUMA
  752. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
  753. if (p->vm_numa_stat_diff[i]) {
  754. int v;
  755. v = p->vm_numa_stat_diff[i];
  756. p->vm_numa_stat_diff[i] = 0;
  757. atomic_long_add(v, &zone->vm_numa_stat[i]);
  758. global_numa_diff[i] += v;
  759. }
  760. #endif
  761. }
  762. for_each_online_pgdat(pgdat) {
  763. struct per_cpu_nodestat *p;
  764. p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
  765. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  766. if (p->vm_node_stat_diff[i]) {
  767. int v;
  768. v = p->vm_node_stat_diff[i];
  769. p->vm_node_stat_diff[i] = 0;
  770. atomic_long_add(v, &pgdat->vm_stat[i]);
  771. global_node_diff[i] += v;
  772. }
  773. }
  774. #ifdef CONFIG_NUMA
  775. fold_diff(global_zone_diff, global_numa_diff, global_node_diff);
  776. #else
  777. fold_diff(global_zone_diff, global_node_diff);
  778. #endif
  779. }
  780. /*
  781. * this is only called if !populated_zone(zone), which implies no other users of
  782. * pset->vm_stat_diff[] exsist.
  783. */
  784. void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset)
  785. {
  786. int i;
  787. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  788. if (pset->vm_stat_diff[i]) {
  789. int v = pset->vm_stat_diff[i];
  790. pset->vm_stat_diff[i] = 0;
  791. atomic_long_add(v, &zone->vm_stat[i]);
  792. atomic_long_add(v, &vm_zone_stat[i]);
  793. }
  794. #ifdef CONFIG_NUMA
  795. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
  796. if (pset->vm_numa_stat_diff[i]) {
  797. int v = pset->vm_numa_stat_diff[i];
  798. pset->vm_numa_stat_diff[i] = 0;
  799. atomic_long_add(v, &zone->vm_numa_stat[i]);
  800. atomic_long_add(v, &vm_numa_stat[i]);
  801. }
  802. #endif
  803. }
  804. #endif
  805. #ifdef CONFIG_NUMA
  806. void __inc_numa_state(struct zone *zone,
  807. enum numa_stat_item item)
  808. {
  809. struct per_cpu_pageset __percpu *pcp = zone->pageset;
  810. u16 __percpu *p = pcp->vm_numa_stat_diff + item;
  811. u16 v;
  812. v = __this_cpu_inc_return(*p);
  813. if (unlikely(v > NUMA_STATS_THRESHOLD)) {
  814. zone_numa_state_add(v, zone, item);
  815. __this_cpu_write(*p, 0);
  816. }
  817. }
  818. /*
  819. * Determine the per node value of a stat item. This function
  820. * is called frequently in a NUMA machine, so try to be as
  821. * frugal as possible.
  822. */
  823. unsigned long sum_zone_node_page_state(int node,
  824. enum zone_stat_item item)
  825. {
  826. struct zone *zones = NODE_DATA(node)->node_zones;
  827. int i;
  828. unsigned long count = 0;
  829. for (i = 0; i < MAX_NR_ZONES; i++)
  830. count += zone_page_state(zones + i, item);
  831. return count;
  832. }
  833. /*
  834. * Determine the per node value of a numa stat item. To avoid deviation,
  835. * the per cpu stat number in vm_numa_stat_diff[] is also included.
  836. */
  837. unsigned long sum_zone_numa_state(int node,
  838. enum numa_stat_item item)
  839. {
  840. struct zone *zones = NODE_DATA(node)->node_zones;
  841. int i;
  842. unsigned long count = 0;
  843. for (i = 0; i < MAX_NR_ZONES; i++)
  844. count += zone_numa_state_snapshot(zones + i, item);
  845. return count;
  846. }
  847. /*
  848. * Determine the per node value of a stat item.
  849. */
  850. unsigned long node_page_state_pages(struct pglist_data *pgdat,
  851. enum node_stat_item item)
  852. {
  853. long x = atomic_long_read(&pgdat->vm_stat[item]);
  854. #ifdef CONFIG_SMP
  855. if (x < 0)
  856. x = 0;
  857. #endif
  858. return x;
  859. }
  860. unsigned long node_page_state(struct pglist_data *pgdat,
  861. enum node_stat_item item)
  862. {
  863. VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
  864. return node_page_state_pages(pgdat, item);
  865. }
  866. #endif
  867. #ifdef CONFIG_COMPACTION
  868. struct contig_page_info {
  869. unsigned long free_pages;
  870. unsigned long free_blocks_total;
  871. unsigned long free_blocks_suitable;
  872. };
  873. /*
  874. * Calculate the number of free pages in a zone, how many contiguous
  875. * pages are free and how many are large enough to satisfy an allocation of
  876. * the target size. Note that this function makes no attempt to estimate
  877. * how many suitable free blocks there *might* be if MOVABLE pages were
  878. * migrated. Calculating that is possible, but expensive and can be
  879. * figured out from userspace
  880. */
  881. static void fill_contig_page_info(struct zone *zone,
  882. unsigned int suitable_order,
  883. struct contig_page_info *info)
  884. {
  885. unsigned int order;
  886. info->free_pages = 0;
  887. info->free_blocks_total = 0;
  888. info->free_blocks_suitable = 0;
  889. for (order = 0; order < MAX_ORDER; order++) {
  890. unsigned long blocks;
  891. /* Count number of free blocks */
  892. blocks = zone->free_area[order].nr_free;
  893. info->free_blocks_total += blocks;
  894. /* Count free base pages */
  895. info->free_pages += blocks << order;
  896. /* Count the suitable free blocks */
  897. if (order >= suitable_order)
  898. info->free_blocks_suitable += blocks <<
  899. (order - suitable_order);
  900. }
  901. }
  902. /*
  903. * A fragmentation index only makes sense if an allocation of a requested
  904. * size would fail. If that is true, the fragmentation index indicates
  905. * whether external fragmentation or a lack of memory was the problem.
  906. * The value can be used to determine if page reclaim or compaction
  907. * should be used
  908. */
  909. static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
  910. {
  911. unsigned long requested = 1UL << order;
  912. if (WARN_ON_ONCE(order >= MAX_ORDER))
  913. return 0;
  914. if (!info->free_blocks_total)
  915. return 0;
  916. /* Fragmentation index only makes sense when a request would fail */
  917. if (info->free_blocks_suitable)
  918. return -1000;
  919. /*
  920. * Index is between 0 and 1 so return within 3 decimal places
  921. *
  922. * 0 => allocation would fail due to lack of memory
  923. * 1 => allocation would fail due to fragmentation
  924. */
  925. return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
  926. }
  927. /*
  928. * Calculates external fragmentation within a zone wrt the given order.
  929. * It is defined as the percentage of pages found in blocks of size
  930. * less than 1 << order. It returns values in range [0, 100].
  931. */
  932. unsigned int extfrag_for_order(struct zone *zone, unsigned int order)
  933. {
  934. struct contig_page_info info;
  935. fill_contig_page_info(zone, order, &info);
  936. if (info.free_pages == 0)
  937. return 0;
  938. return div_u64((info.free_pages -
  939. (info.free_blocks_suitable << order)) * 100,
  940. info.free_pages);
  941. }
  942. /* Same as __fragmentation index but allocs contig_page_info on stack */
  943. int fragmentation_index(struct zone *zone, unsigned int order)
  944. {
  945. struct contig_page_info info;
  946. fill_contig_page_info(zone, order, &info);
  947. return __fragmentation_index(order, &info);
  948. }
  949. #endif
  950. #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \
  951. defined(CONFIG_NUMA) || defined(CONFIG_MEMCG)
  952. #ifdef CONFIG_ZONE_DMA
  953. #define TEXT_FOR_DMA(xx) xx "_dma",
  954. #else
  955. #define TEXT_FOR_DMA(xx)
  956. #endif
  957. #ifdef CONFIG_ZONE_DMA32
  958. #define TEXT_FOR_DMA32(xx) xx "_dma32",
  959. #else
  960. #define TEXT_FOR_DMA32(xx)
  961. #endif
  962. #ifdef CONFIG_HIGHMEM
  963. #define TEXT_FOR_HIGHMEM(xx) xx "_high",
  964. #else
  965. #define TEXT_FOR_HIGHMEM(xx)
  966. #endif
  967. #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
  968. TEXT_FOR_HIGHMEM(xx) xx "_movable",
  969. const char * const vmstat_text[] = {
  970. /* enum zone_stat_item counters */
  971. "nr_free_pages",
  972. "nr_zone_inactive_anon",
  973. "nr_zone_active_anon",
  974. "nr_zone_inactive_file",
  975. "nr_zone_active_file",
  976. "nr_zone_unevictable",
  977. "nr_zone_write_pending",
  978. "nr_mlock",
  979. "nr_page_table_pages",
  980. "nr_bounce",
  981. "nr_zspages",
  982. "nr_free_cma",
  983. /* enum numa_stat_item counters */
  984. #ifdef CONFIG_NUMA
  985. "numa_hit",
  986. "numa_miss",
  987. "numa_foreign",
  988. "numa_interleave",
  989. "numa_local",
  990. "numa_other",
  991. #endif
  992. /* enum node_stat_item counters */
  993. "nr_inactive_anon",
  994. "nr_active_anon",
  995. "nr_inactive_file",
  996. "nr_active_file",
  997. "nr_unevictable",
  998. "nr_slab_reclaimable",
  999. "nr_slab_unreclaimable",
  1000. "nr_isolated_anon",
  1001. "nr_isolated_file",
  1002. "workingset_nodes",
  1003. "workingset_refault_anon",
  1004. "workingset_refault_file",
  1005. "workingset_activate_anon",
  1006. "workingset_activate_file",
  1007. "workingset_restore_anon",
  1008. "workingset_restore_file",
  1009. "workingset_nodereclaim",
  1010. "nr_anon_pages",
  1011. "nr_mapped",
  1012. "nr_file_pages",
  1013. "nr_dirty",
  1014. "nr_writeback",
  1015. "nr_writeback_temp",
  1016. "nr_shmem",
  1017. "nr_shmem_hugepages",
  1018. "nr_shmem_pmdmapped",
  1019. "nr_file_hugepages",
  1020. "nr_file_pmdmapped",
  1021. "nr_anon_transparent_hugepages",
  1022. "nr_vmscan_write",
  1023. "nr_vmscan_immediate_reclaim",
  1024. "nr_dirtied",
  1025. "nr_written",
  1026. "nr_kernel_misc_reclaimable",
  1027. "nr_foll_pin_acquired",
  1028. "nr_foll_pin_released",
  1029. "nr_kernel_stack",
  1030. #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
  1031. "nr_shadow_call_stack",
  1032. #endif
  1033. /* enum writeback_stat_item counters */
  1034. "nr_dirty_threshold",
  1035. "nr_dirty_background_threshold",
  1036. #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
  1037. /* enum vm_event_item counters */
  1038. "pgpgin",
  1039. "pgpgout",
  1040. "pswpin",
  1041. "pswpout",
  1042. TEXTS_FOR_ZONES("pgalloc")
  1043. TEXTS_FOR_ZONES("allocstall")
  1044. TEXTS_FOR_ZONES("pgskip")
  1045. "pgfree",
  1046. "pgactivate",
  1047. "pgdeactivate",
  1048. "pglazyfree",
  1049. "pgfault",
  1050. "pgmajfault",
  1051. "pglazyfreed",
  1052. "pgrefill",
  1053. "pgreuse",
  1054. "pgsteal_kswapd",
  1055. "pgsteal_direct",
  1056. "pgscan_kswapd",
  1057. "pgscan_direct",
  1058. "pgscan_direct_throttle",
  1059. "pgscan_anon",
  1060. "pgscan_file",
  1061. "pgsteal_anon",
  1062. "pgsteal_file",
  1063. #ifdef CONFIG_NUMA
  1064. "zone_reclaim_failed",
  1065. #endif
  1066. "pginodesteal",
  1067. "slabs_scanned",
  1068. "kswapd_inodesteal",
  1069. "kswapd_low_wmark_hit_quickly",
  1070. "kswapd_high_wmark_hit_quickly",
  1071. "pageoutrun",
  1072. "pgrotated",
  1073. "drop_pagecache",
  1074. "drop_slab",
  1075. "oom_kill",
  1076. #ifdef CONFIG_NUMA_BALANCING
  1077. "numa_pte_updates",
  1078. "numa_huge_pte_updates",
  1079. "numa_hint_faults",
  1080. "numa_hint_faults_local",
  1081. "numa_pages_migrated",
  1082. #endif
  1083. #ifdef CONFIG_MIGRATION
  1084. "pgmigrate_success",
  1085. "pgmigrate_fail",
  1086. "thp_migration_success",
  1087. "thp_migration_fail",
  1088. "thp_migration_split",
  1089. #endif
  1090. #ifdef CONFIG_COMPACTION
  1091. "compact_migrate_scanned",
  1092. "compact_free_scanned",
  1093. "compact_isolated",
  1094. "compact_stall",
  1095. "compact_fail",
  1096. "compact_success",
  1097. "compact_daemon_wake",
  1098. "compact_daemon_migrate_scanned",
  1099. "compact_daemon_free_scanned",
  1100. #endif
  1101. #ifdef CONFIG_HUGETLB_PAGE
  1102. "htlb_buddy_alloc_success",
  1103. "htlb_buddy_alloc_fail",
  1104. #endif
  1105. #ifdef CONFIG_CMA
  1106. "cma_alloc_success",
  1107. "cma_alloc_fail",
  1108. #endif
  1109. "unevictable_pgs_culled",
  1110. "unevictable_pgs_scanned",
  1111. "unevictable_pgs_rescued",
  1112. "unevictable_pgs_mlocked",
  1113. "unevictable_pgs_munlocked",
  1114. "unevictable_pgs_cleared",
  1115. "unevictable_pgs_stranded",
  1116. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1117. "thp_fault_alloc",
  1118. "thp_fault_fallback",
  1119. "thp_fault_fallback_charge",
  1120. "thp_collapse_alloc",
  1121. "thp_collapse_alloc_failed",
  1122. "thp_file_alloc",
  1123. "thp_file_fallback",
  1124. "thp_file_fallback_charge",
  1125. "thp_file_mapped",
  1126. "thp_split_page",
  1127. "thp_split_page_failed",
  1128. "thp_deferred_split_page",
  1129. "thp_split_pmd",
  1130. #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
  1131. "thp_split_pud",
  1132. #endif
  1133. "thp_zero_page_alloc",
  1134. "thp_zero_page_alloc_failed",
  1135. "thp_swpout",
  1136. "thp_swpout_fallback",
  1137. #endif
  1138. #ifdef CONFIG_MEMORY_BALLOON
  1139. "balloon_inflate",
  1140. "balloon_deflate",
  1141. #ifdef CONFIG_BALLOON_COMPACTION
  1142. "balloon_migrate",
  1143. #endif
  1144. #endif /* CONFIG_MEMORY_BALLOON */
  1145. #ifdef CONFIG_DEBUG_TLBFLUSH
  1146. "nr_tlb_remote_flush",
  1147. "nr_tlb_remote_flush_received",
  1148. "nr_tlb_local_flush_all",
  1149. "nr_tlb_local_flush_one",
  1150. #endif /* CONFIG_DEBUG_TLBFLUSH */
  1151. #ifdef CONFIG_DEBUG_VM_VMACACHE
  1152. "vmacache_find_calls",
  1153. "vmacache_find_hits",
  1154. #endif
  1155. #ifdef CONFIG_SWAP
  1156. "swap_ra",
  1157. "swap_ra_hit",
  1158. #endif
  1159. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  1160. "speculative_pgfault",
  1161. "speculative_pgfault_file"
  1162. #endif
  1163. #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
  1164. };
  1165. #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
  1166. #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
  1167. defined(CONFIG_PROC_FS)
  1168. static void *frag_start(struct seq_file *m, loff_t *pos)
  1169. {
  1170. pg_data_t *pgdat;
  1171. loff_t node = *pos;
  1172. for (pgdat = first_online_pgdat();
  1173. pgdat && node;
  1174. pgdat = next_online_pgdat(pgdat))
  1175. --node;
  1176. return pgdat;
  1177. }
  1178. static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
  1179. {
  1180. pg_data_t *pgdat = (pg_data_t *)arg;
  1181. (*pos)++;
  1182. return next_online_pgdat(pgdat);
  1183. }
  1184. static void frag_stop(struct seq_file *m, void *arg)
  1185. {
  1186. }
  1187. /*
  1188. * Walk zones in a node and print using a callback.
  1189. * If @assert_populated is true, only use callback for zones that are populated.
  1190. */
  1191. static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
  1192. bool assert_populated, bool nolock,
  1193. void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
  1194. {
  1195. struct zone *zone;
  1196. struct zone *node_zones = pgdat->node_zones;
  1197. unsigned long flags;
  1198. for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
  1199. if (assert_populated && !populated_zone(zone))
  1200. continue;
  1201. if (!nolock)
  1202. spin_lock_irqsave(&zone->lock, flags);
  1203. print(m, pgdat, zone);
  1204. if (!nolock)
  1205. spin_unlock_irqrestore(&zone->lock, flags);
  1206. }
  1207. }
  1208. #endif
  1209. #ifdef CONFIG_PROC_FS
  1210. static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
  1211. struct zone *zone)
  1212. {
  1213. int order;
  1214. seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
  1215. for (order = 0; order < MAX_ORDER; ++order)
  1216. seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
  1217. seq_putc(m, '\n');
  1218. }
  1219. /*
  1220. * This walks the free areas for each zone.
  1221. */
  1222. static int frag_show(struct seq_file *m, void *arg)
  1223. {
  1224. pg_data_t *pgdat = (pg_data_t *)arg;
  1225. walk_zones_in_node(m, pgdat, true, false, frag_show_print);
  1226. return 0;
  1227. }
  1228. static void pagetypeinfo_showfree_print(struct seq_file *m,
  1229. pg_data_t *pgdat, struct zone *zone)
  1230. {
  1231. int order, mtype;
  1232. for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
  1233. seq_printf(m, "Node %4d, zone %8s, type %12s ",
  1234. pgdat->node_id,
  1235. zone->name,
  1236. migratetype_names[mtype]);
  1237. for (order = 0; order < MAX_ORDER; ++order) {
  1238. unsigned long freecount = 0;
  1239. struct free_area *area;
  1240. struct list_head *curr;
  1241. bool overflow = false;
  1242. area = &(zone->free_area[order]);
  1243. list_for_each(curr, &area->free_list[mtype]) {
  1244. /*
  1245. * Cap the free_list iteration because it might
  1246. * be really large and we are under a spinlock
  1247. * so a long time spent here could trigger a
  1248. * hard lockup detector. Anyway this is a
  1249. * debugging tool so knowing there is a handful
  1250. * of pages of this order should be more than
  1251. * sufficient.
  1252. */
  1253. if (++freecount >= 100000) {
  1254. overflow = true;
  1255. break;
  1256. }
  1257. }
  1258. seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount);
  1259. spin_unlock_irq(&zone->lock);
  1260. cond_resched();
  1261. spin_lock_irq(&zone->lock);
  1262. }
  1263. seq_putc(m, '\n');
  1264. }
  1265. }
  1266. /* Print out the free pages at each order for each migatetype */
  1267. static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
  1268. {
  1269. int order;
  1270. pg_data_t *pgdat = (pg_data_t *)arg;
  1271. /* Print header */
  1272. seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
  1273. for (order = 0; order < MAX_ORDER; ++order)
  1274. seq_printf(m, "%6d ", order);
  1275. seq_putc(m, '\n');
  1276. walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print);
  1277. return 0;
  1278. }
  1279. static void pagetypeinfo_showblockcount_print(struct seq_file *m,
  1280. pg_data_t *pgdat, struct zone *zone)
  1281. {
  1282. int mtype;
  1283. unsigned long pfn;
  1284. unsigned long start_pfn = zone->zone_start_pfn;
  1285. unsigned long end_pfn = zone_end_pfn(zone);
  1286. unsigned long count[MIGRATE_TYPES] = { 0, };
  1287. for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
  1288. struct page *page;
  1289. page = pfn_to_online_page(pfn);
  1290. if (!page)
  1291. continue;
  1292. /* Watch for unexpected holes punched in the memmap */
  1293. if (!memmap_valid_within(pfn, page, zone))
  1294. continue;
  1295. if (page_zone(page) != zone)
  1296. continue;
  1297. mtype = get_pageblock_migratetype(page);
  1298. if (mtype < MIGRATE_TYPES)
  1299. count[mtype]++;
  1300. }
  1301. /* Print counts */
  1302. seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
  1303. for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
  1304. seq_printf(m, "%12lu ", count[mtype]);
  1305. seq_putc(m, '\n');
  1306. }
  1307. /* Print out the number of pageblocks for each migratetype */
  1308. static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
  1309. {
  1310. int mtype;
  1311. pg_data_t *pgdat = (pg_data_t *)arg;
  1312. seq_printf(m, "\n%-23s", "Number of blocks type ");
  1313. for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
  1314. seq_printf(m, "%12s ", migratetype_names[mtype]);
  1315. seq_putc(m, '\n');
  1316. walk_zones_in_node(m, pgdat, true, false,
  1317. pagetypeinfo_showblockcount_print);
  1318. return 0;
  1319. }
  1320. /*
  1321. * Print out the number of pageblocks for each migratetype that contain pages
  1322. * of other types. This gives an indication of how well fallbacks are being
  1323. * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
  1324. * to determine what is going on
  1325. */
  1326. static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
  1327. {
  1328. #ifdef CONFIG_PAGE_OWNER
  1329. int mtype;
  1330. if (!static_branch_unlikely(&page_owner_inited))
  1331. return;
  1332. drain_all_pages(NULL);
  1333. seq_printf(m, "\n%-23s", "Number of mixed blocks ");
  1334. for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
  1335. seq_printf(m, "%12s ", migratetype_names[mtype]);
  1336. seq_putc(m, '\n');
  1337. walk_zones_in_node(m, pgdat, true, true,
  1338. pagetypeinfo_showmixedcount_print);
  1339. #endif /* CONFIG_PAGE_OWNER */
  1340. }
  1341. /*
  1342. * This prints out statistics in relation to grouping pages by mobility.
  1343. * It is expensive to collect so do not constantly read the file.
  1344. */
  1345. static int pagetypeinfo_show(struct seq_file *m, void *arg)
  1346. {
  1347. pg_data_t *pgdat = (pg_data_t *)arg;
  1348. /* check memoryless node */
  1349. if (!node_state(pgdat->node_id, N_MEMORY))
  1350. return 0;
  1351. seq_printf(m, "Page block order: %d\n", pageblock_order);
  1352. seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
  1353. seq_putc(m, '\n');
  1354. pagetypeinfo_showfree(m, pgdat);
  1355. pagetypeinfo_showblockcount(m, pgdat);
  1356. pagetypeinfo_showmixedcount(m, pgdat);
  1357. return 0;
  1358. }
  1359. static const struct seq_operations fragmentation_op = {
  1360. .start = frag_start,
  1361. .next = frag_next,
  1362. .stop = frag_stop,
  1363. .show = frag_show,
  1364. };
  1365. static const struct seq_operations pagetypeinfo_op = {
  1366. .start = frag_start,
  1367. .next = frag_next,
  1368. .stop = frag_stop,
  1369. .show = pagetypeinfo_show,
  1370. };
  1371. static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
  1372. {
  1373. int zid;
  1374. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  1375. struct zone *compare = &pgdat->node_zones[zid];
  1376. if (populated_zone(compare))
  1377. return zone == compare;
  1378. }
  1379. return false;
  1380. }
  1381. static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
  1382. struct zone *zone)
  1383. {
  1384. int i;
  1385. seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
  1386. if (is_zone_first_populated(pgdat, zone)) {
  1387. seq_printf(m, "\n per-node stats");
  1388. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
  1389. seq_printf(m, "\n %-12s %lu", node_stat_name(i),
  1390. node_page_state_pages(pgdat, i));
  1391. }
  1392. }
  1393. seq_printf(m,
  1394. "\n pages free %lu"
  1395. "\n min %lu"
  1396. "\n low %lu"
  1397. "\n high %lu"
  1398. "\n spanned %lu"
  1399. "\n present %lu"
  1400. "\n managed %lu"
  1401. "\n cma %lu",
  1402. zone_page_state(zone, NR_FREE_PAGES),
  1403. min_wmark_pages(zone),
  1404. low_wmark_pages(zone),
  1405. high_wmark_pages(zone),
  1406. zone->spanned_pages,
  1407. zone->present_pages,
  1408. zone_managed_pages(zone),
  1409. zone_cma_pages(zone));
  1410. seq_printf(m,
  1411. "\n protection: (%ld",
  1412. zone->lowmem_reserve[0]);
  1413. for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
  1414. seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
  1415. seq_putc(m, ')');
  1416. /* If unpopulated, no other information is useful */
  1417. if (!populated_zone(zone)) {
  1418. seq_putc(m, '\n');
  1419. return;
  1420. }
  1421. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  1422. seq_printf(m, "\n %-12s %lu", zone_stat_name(i),
  1423. zone_page_state(zone, i));
  1424. #ifdef CONFIG_NUMA
  1425. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
  1426. seq_printf(m, "\n %-12s %lu", numa_stat_name(i),
  1427. zone_numa_state_snapshot(zone, i));
  1428. #endif
  1429. seq_printf(m, "\n pagesets");
  1430. for_each_online_cpu(i) {
  1431. struct per_cpu_pageset *pageset;
  1432. pageset = per_cpu_ptr(zone->pageset, i);
  1433. seq_printf(m,
  1434. "\n cpu: %i"
  1435. "\n count: %i"
  1436. "\n high: %i"
  1437. "\n batch: %i",
  1438. i,
  1439. pageset->pcp.count,
  1440. pageset->pcp.high,
  1441. pageset->pcp.batch);
  1442. #ifdef CONFIG_SMP
  1443. seq_printf(m, "\n vm stats threshold: %d",
  1444. pageset->stat_threshold);
  1445. #endif
  1446. }
  1447. seq_printf(m,
  1448. "\n node_unreclaimable: %u"
  1449. "\n start_pfn: %lu",
  1450. pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES,
  1451. zone->zone_start_pfn);
  1452. seq_putc(m, '\n');
  1453. }
  1454. /*
  1455. * Output information about zones in @pgdat. All zones are printed regardless
  1456. * of whether they are populated or not: lowmem_reserve_ratio operates on the
  1457. * set of all zones and userspace would not be aware of such zones if they are
  1458. * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio).
  1459. */
  1460. static int zoneinfo_show(struct seq_file *m, void *arg)
  1461. {
  1462. pg_data_t *pgdat = (pg_data_t *)arg;
  1463. walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print);
  1464. return 0;
  1465. }
  1466. static const struct seq_operations zoneinfo_op = {
  1467. .start = frag_start, /* iterate over all zones. The same as in
  1468. * fragmentation. */
  1469. .next = frag_next,
  1470. .stop = frag_stop,
  1471. .show = zoneinfo_show,
  1472. };
  1473. #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
  1474. NR_VM_NUMA_STAT_ITEMS + \
  1475. NR_VM_NODE_STAT_ITEMS + \
  1476. NR_VM_WRITEBACK_STAT_ITEMS + \
  1477. (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
  1478. NR_VM_EVENT_ITEMS : 0))
  1479. static void *vmstat_start(struct seq_file *m, loff_t *pos)
  1480. {
  1481. unsigned long *v;
  1482. int i;
  1483. if (*pos >= NR_VMSTAT_ITEMS)
  1484. return NULL;
  1485. BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) < NR_VMSTAT_ITEMS);
  1486. v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL);
  1487. m->private = v;
  1488. if (!v)
  1489. return ERR_PTR(-ENOMEM);
  1490. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
  1491. v[i] = global_zone_page_state(i);
  1492. v += NR_VM_ZONE_STAT_ITEMS;
  1493. #ifdef CONFIG_NUMA
  1494. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
  1495. v[i] = global_numa_state(i);
  1496. v += NR_VM_NUMA_STAT_ITEMS;
  1497. #endif
  1498. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  1499. v[i] = global_node_page_state_pages(i);
  1500. v += NR_VM_NODE_STAT_ITEMS;
  1501. global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
  1502. v + NR_DIRTY_THRESHOLD);
  1503. v += NR_VM_WRITEBACK_STAT_ITEMS;
  1504. #ifdef CONFIG_VM_EVENT_COUNTERS
  1505. all_vm_events(v);
  1506. v[PGPGIN] /= 2; /* sectors -> kbytes */
  1507. v[PGPGOUT] /= 2;
  1508. #endif
  1509. return (unsigned long *)m->private + *pos;
  1510. }
  1511. static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
  1512. {
  1513. (*pos)++;
  1514. if (*pos >= NR_VMSTAT_ITEMS)
  1515. return NULL;
  1516. return (unsigned long *)m->private + *pos;
  1517. }
  1518. static int vmstat_show(struct seq_file *m, void *arg)
  1519. {
  1520. unsigned long *l = arg;
  1521. unsigned long off = l - (unsigned long *)m->private;
  1522. seq_puts(m, vmstat_text[off]);
  1523. seq_put_decimal_ull(m, " ", *l);
  1524. seq_putc(m, '\n');
  1525. if (off == NR_VMSTAT_ITEMS - 1) {
  1526. /*
  1527. * We've come to the end - add any deprecated counters to avoid
  1528. * breaking userspace which might depend on them being present.
  1529. */
  1530. seq_puts(m, "nr_unstable 0\n");
  1531. }
  1532. return 0;
  1533. }
  1534. static void vmstat_stop(struct seq_file *m, void *arg)
  1535. {
  1536. kfree(m->private);
  1537. m->private = NULL;
  1538. }
  1539. static const struct seq_operations vmstat_op = {
  1540. .start = vmstat_start,
  1541. .next = vmstat_next,
  1542. .stop = vmstat_stop,
  1543. .show = vmstat_show,
  1544. };
  1545. #endif /* CONFIG_PROC_FS */
  1546. #ifdef CONFIG_SMP
  1547. static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
  1548. int sysctl_stat_interval __read_mostly = HZ;
  1549. #ifdef CONFIG_PROC_FS
  1550. static void refresh_vm_stats(struct work_struct *work)
  1551. {
  1552. refresh_cpu_vm_stats(true);
  1553. }
  1554. int vmstat_refresh(struct ctl_table *table, int write,
  1555. void *buffer, size_t *lenp, loff_t *ppos)
  1556. {
  1557. long val;
  1558. int err;
  1559. int i;
  1560. /*
  1561. * The regular update, every sysctl_stat_interval, may come later
  1562. * than expected: leaving a significant amount in per_cpu buckets.
  1563. * This is particularly misleading when checking a quantity of HUGE
  1564. * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
  1565. * which can equally be echo'ed to or cat'ted from (by root),
  1566. * can be used to update the stats just before reading them.
  1567. *
  1568. * Oh, and since global_zone_page_state() etc. are so careful to hide
  1569. * transiently negative values, report an error here if any of
  1570. * the stats is negative, so we know to go looking for imbalance.
  1571. */
  1572. err = schedule_on_each_cpu(refresh_vm_stats);
  1573. if (err)
  1574. return err;
  1575. for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
  1576. val = atomic_long_read(&vm_zone_stat[i]);
  1577. if (val < 0) {
  1578. pr_warn("%s: %s %ld\n",
  1579. __func__, zone_stat_name(i), val);
  1580. err = -EINVAL;
  1581. }
  1582. }
  1583. #ifdef CONFIG_NUMA
  1584. for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) {
  1585. val = atomic_long_read(&vm_numa_stat[i]);
  1586. if (val < 0) {
  1587. pr_warn("%s: %s %ld\n",
  1588. __func__, numa_stat_name(i), val);
  1589. err = -EINVAL;
  1590. }
  1591. }
  1592. #endif
  1593. if (err)
  1594. return err;
  1595. if (write)
  1596. *ppos += *lenp;
  1597. else
  1598. *lenp = 0;
  1599. return 0;
  1600. }
  1601. #endif /* CONFIG_PROC_FS */
  1602. static void vmstat_update(struct work_struct *w)
  1603. {
  1604. if (refresh_cpu_vm_stats(true)) {
  1605. /*
  1606. * Counters were updated so we expect more updates
  1607. * to occur in the future. Keep on running the
  1608. * update worker thread.
  1609. */
  1610. queue_delayed_work_on(smp_processor_id(), mm_percpu_wq,
  1611. this_cpu_ptr(&vmstat_work),
  1612. round_jiffies_relative(sysctl_stat_interval));
  1613. }
  1614. }
  1615. /*
  1616. * Switch off vmstat processing and then fold all the remaining differentials
  1617. * until the diffs stay at zero. The function is used by NOHZ and can only be
  1618. * invoked when tick processing is not active.
  1619. */
  1620. /*
  1621. * Check if the diffs for a certain cpu indicate that
  1622. * an update is needed.
  1623. */
  1624. static bool need_update(int cpu)
  1625. {
  1626. struct zone *zone;
  1627. for_each_populated_zone(zone) {
  1628. struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu);
  1629. BUILD_BUG_ON(sizeof(p->vm_stat_diff[0]) != 1);
  1630. #ifdef CONFIG_NUMA
  1631. BUILD_BUG_ON(sizeof(p->vm_numa_stat_diff[0]) != 2);
  1632. #endif
  1633. /*
  1634. * The fast way of checking if there are any vmstat diffs.
  1635. */
  1636. if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS *
  1637. sizeof(p->vm_stat_diff[0])))
  1638. return true;
  1639. #ifdef CONFIG_NUMA
  1640. if (memchr_inv(p->vm_numa_stat_diff, 0, NR_VM_NUMA_STAT_ITEMS *
  1641. sizeof(p->vm_numa_stat_diff[0])))
  1642. return true;
  1643. #endif
  1644. }
  1645. return false;
  1646. }
  1647. /*
  1648. * Switch off vmstat processing and then fold all the remaining differentials
  1649. * until the diffs stay at zero. The function is used by NOHZ and can only be
  1650. * invoked when tick processing is not active.
  1651. */
  1652. void quiet_vmstat(void)
  1653. {
  1654. if (system_state != SYSTEM_RUNNING)
  1655. return;
  1656. if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
  1657. return;
  1658. if (!need_update(smp_processor_id()))
  1659. return;
  1660. /*
  1661. * Just refresh counters and do not care about the pending delayed
  1662. * vmstat_update. It doesn't fire that often to matter and canceling
  1663. * it would be too expensive from this path.
  1664. * vmstat_shepherd will take care about that for us.
  1665. */
  1666. refresh_cpu_vm_stats(false);
  1667. }
  1668. /*
  1669. * Shepherd worker thread that checks the
  1670. * differentials of processors that have their worker
  1671. * threads for vm statistics updates disabled because of
  1672. * inactivity.
  1673. */
  1674. static void vmstat_shepherd(struct work_struct *w);
  1675. static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
  1676. static void vmstat_shepherd(struct work_struct *w)
  1677. {
  1678. int cpu;
  1679. get_online_cpus();
  1680. /* Check processors whose vmstat worker threads have been disabled */
  1681. for_each_online_cpu(cpu) {
  1682. struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
  1683. if (!delayed_work_pending(dw) && need_update(cpu))
  1684. queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0);
  1685. }
  1686. put_online_cpus();
  1687. schedule_delayed_work(&shepherd,
  1688. round_jiffies_relative(sysctl_stat_interval));
  1689. }
  1690. static void __init start_shepherd_timer(void)
  1691. {
  1692. int cpu;
  1693. for_each_possible_cpu(cpu)
  1694. INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
  1695. vmstat_update);
  1696. schedule_delayed_work(&shepherd,
  1697. round_jiffies_relative(sysctl_stat_interval));
  1698. }
  1699. static void __init init_cpu_node_state(void)
  1700. {
  1701. int node;
  1702. for_each_online_node(node) {
  1703. if (cpumask_weight(cpumask_of_node(node)) > 0)
  1704. node_set_state(node, N_CPU);
  1705. }
  1706. }
  1707. static int vmstat_cpu_online(unsigned int cpu)
  1708. {
  1709. refresh_zone_stat_thresholds();
  1710. node_set_state(cpu_to_node(cpu), N_CPU);
  1711. return 0;
  1712. }
  1713. static int vmstat_cpu_down_prep(unsigned int cpu)
  1714. {
  1715. cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu));
  1716. return 0;
  1717. }
  1718. static int vmstat_cpu_dead(unsigned int cpu)
  1719. {
  1720. const struct cpumask *node_cpus;
  1721. int node;
  1722. node = cpu_to_node(cpu);
  1723. refresh_zone_stat_thresholds();
  1724. node_cpus = cpumask_of_node(node);
  1725. if (cpumask_weight(node_cpus) > 0)
  1726. return 0;
  1727. node_clear_state(node, N_CPU);
  1728. return 0;
  1729. }
  1730. #endif
  1731. struct workqueue_struct *mm_percpu_wq;
  1732. void __init init_mm_internals(void)
  1733. {
  1734. int ret __maybe_unused;
  1735. mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0);
  1736. #ifdef CONFIG_SMP
  1737. ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead",
  1738. NULL, vmstat_cpu_dead);
  1739. if (ret < 0)
  1740. pr_err("vmstat: failed to register 'dead' hotplug state\n");
  1741. ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online",
  1742. vmstat_cpu_online,
  1743. vmstat_cpu_down_prep);
  1744. if (ret < 0)
  1745. pr_err("vmstat: failed to register 'online' hotplug state\n");
  1746. get_online_cpus();
  1747. init_cpu_node_state();
  1748. put_online_cpus();
  1749. start_shepherd_timer();
  1750. #endif
  1751. #ifdef CONFIG_PROC_FS
  1752. proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op);
  1753. proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op);
  1754. proc_create_seq("vmstat", 0444, NULL, &vmstat_op);
  1755. proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op);
  1756. #endif
  1757. }
  1758. #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
  1759. /*
  1760. * Return an index indicating how much of the available free memory is
  1761. * unusable for an allocation of the requested size.
  1762. */
  1763. static int unusable_free_index(unsigned int order,
  1764. struct contig_page_info *info)
  1765. {
  1766. /* No free memory is interpreted as all free memory is unusable */
  1767. if (info->free_pages == 0)
  1768. return 1000;
  1769. /*
  1770. * Index should be a value between 0 and 1. Return a value to 3
  1771. * decimal places.
  1772. *
  1773. * 0 => no fragmentation
  1774. * 1 => high fragmentation
  1775. */
  1776. return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
  1777. }
  1778. static void unusable_show_print(struct seq_file *m,
  1779. pg_data_t *pgdat, struct zone *zone)
  1780. {
  1781. unsigned int order;
  1782. int index;
  1783. struct contig_page_info info;
  1784. seq_printf(m, "Node %d, zone %8s ",
  1785. pgdat->node_id,
  1786. zone->name);
  1787. for (order = 0; order < MAX_ORDER; ++order) {
  1788. fill_contig_page_info(zone, order, &info);
  1789. index = unusable_free_index(order, &info);
  1790. seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
  1791. }
  1792. seq_putc(m, '\n');
  1793. }
  1794. /*
  1795. * Display unusable free space index
  1796. *
  1797. * The unusable free space index measures how much of the available free
  1798. * memory cannot be used to satisfy an allocation of a given size and is a
  1799. * value between 0 and 1. The higher the value, the more of free memory is
  1800. * unusable and by implication, the worse the external fragmentation is. This
  1801. * can be expressed as a percentage by multiplying by 100.
  1802. */
  1803. static int unusable_show(struct seq_file *m, void *arg)
  1804. {
  1805. pg_data_t *pgdat = (pg_data_t *)arg;
  1806. /* check memoryless node */
  1807. if (!node_state(pgdat->node_id, N_MEMORY))
  1808. return 0;
  1809. walk_zones_in_node(m, pgdat, true, false, unusable_show_print);
  1810. return 0;
  1811. }
  1812. static const struct seq_operations unusable_sops = {
  1813. .start = frag_start,
  1814. .next = frag_next,
  1815. .stop = frag_stop,
  1816. .show = unusable_show,
  1817. };
  1818. DEFINE_SEQ_ATTRIBUTE(unusable);
  1819. static void extfrag_show_print(struct seq_file *m,
  1820. pg_data_t *pgdat, struct zone *zone)
  1821. {
  1822. unsigned int order;
  1823. int index;
  1824. /* Alloc on stack as interrupts are disabled for zone walk */
  1825. struct contig_page_info info;
  1826. seq_printf(m, "Node %d, zone %8s ",
  1827. pgdat->node_id,
  1828. zone->name);
  1829. for (order = 0; order < MAX_ORDER; ++order) {
  1830. fill_contig_page_info(zone, order, &info);
  1831. index = __fragmentation_index(order, &info);
  1832. seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
  1833. }
  1834. seq_putc(m, '\n');
  1835. }
  1836. /*
  1837. * Display fragmentation index for orders that allocations would fail for
  1838. */
  1839. static int extfrag_show(struct seq_file *m, void *arg)
  1840. {
  1841. pg_data_t *pgdat = (pg_data_t *)arg;
  1842. walk_zones_in_node(m, pgdat, true, false, extfrag_show_print);
  1843. return 0;
  1844. }
  1845. static const struct seq_operations extfrag_sops = {
  1846. .start = frag_start,
  1847. .next = frag_next,
  1848. .stop = frag_stop,
  1849. .show = extfrag_show,
  1850. };
  1851. DEFINE_SEQ_ATTRIBUTE(extfrag);
  1852. static int __init extfrag_debug_init(void)
  1853. {
  1854. struct dentry *extfrag_debug_root;
  1855. extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
  1856. debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL,
  1857. &unusable_fops);
  1858. debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL,
  1859. &extfrag_fops);
  1860. return 0;
  1861. }
  1862. module_init(extfrag_debug_init);
  1863. #endif