task_mmu.c 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/pagewalk.h>
  3. #include <linux/vmacache.h>
  4. #include <linux/hugetlb.h>
  5. #include <linux/huge_mm.h>
  6. #include <linux/mount.h>
  7. #include <linux/seq_file.h>
  8. #include <linux/highmem.h>
  9. #include <linux/ptrace.h>
  10. #include <linux/slab.h>
  11. #include <linux/pagemap.h>
  12. #include <linux/mempolicy.h>
  13. #include <linux/rmap.h>
  14. #include <linux/swap.h>
  15. #include <linux/sched/mm.h>
  16. #include <linux/swapops.h>
  17. #include <linux/mmu_notifier.h>
  18. #include <linux/page_idle.h>
  19. #include <linux/shmem_fs.h>
  20. #include <linux/uaccess.h>
  21. #include <linux/pkeys.h>
  22. #include <asm/elf.h>
  23. #include <asm/tlb.h>
  24. #include <asm/tlbflush.h>
  25. #include "internal.h"
  26. #define SEQ_PUT_DEC(str, val) \
  27. seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
  28. void task_mem(struct seq_file *m, struct mm_struct *mm)
  29. {
  30. unsigned long text, lib, swap, anon, file, shmem;
  31. unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
  32. anon = get_mm_counter(mm, MM_ANONPAGES);
  33. file = get_mm_counter(mm, MM_FILEPAGES);
  34. shmem = get_mm_counter(mm, MM_SHMEMPAGES);
  35. /*
  36. * Note: to minimize their overhead, mm maintains hiwater_vm and
  37. * hiwater_rss only when about to *lower* total_vm or rss. Any
  38. * collector of these hiwater stats must therefore get total_vm
  39. * and rss too, which will usually be the higher. Barriers? not
  40. * worth the effort, such snapshots can always be inconsistent.
  41. */
  42. hiwater_vm = total_vm = mm->total_vm;
  43. if (hiwater_vm < mm->hiwater_vm)
  44. hiwater_vm = mm->hiwater_vm;
  45. hiwater_rss = total_rss = anon + file + shmem;
  46. if (hiwater_rss < mm->hiwater_rss)
  47. hiwater_rss = mm->hiwater_rss;
  48. /* split executable areas between text and lib */
  49. text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
  50. text = min(text, mm->exec_vm << PAGE_SHIFT);
  51. lib = (mm->exec_vm << PAGE_SHIFT) - text;
  52. swap = get_mm_counter(mm, MM_SWAPENTS);
  53. SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
  54. SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
  55. SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
  56. SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
  57. SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
  58. SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
  59. SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
  60. SEQ_PUT_DEC(" kB\nRssFile:\t", file);
  61. SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
  62. SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
  63. SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
  64. seq_put_decimal_ull_width(m,
  65. " kB\nVmExe:\t", text >> 10, 8);
  66. seq_put_decimal_ull_width(m,
  67. " kB\nVmLib:\t", lib >> 10, 8);
  68. seq_put_decimal_ull_width(m,
  69. " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
  70. SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
  71. seq_puts(m, " kB\n");
  72. hugetlb_report_usage(m, mm);
  73. }
  74. #undef SEQ_PUT_DEC
  75. unsigned long task_vsize(struct mm_struct *mm)
  76. {
  77. return PAGE_SIZE * mm->total_vm;
  78. }
  79. unsigned long task_statm(struct mm_struct *mm,
  80. unsigned long *shared, unsigned long *text,
  81. unsigned long *data, unsigned long *resident)
  82. {
  83. *shared = get_mm_counter(mm, MM_FILEPAGES) +
  84. get_mm_counter(mm, MM_SHMEMPAGES);
  85. *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
  86. >> PAGE_SHIFT;
  87. *data = mm->data_vm + mm->stack_vm;
  88. *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
  89. return mm->total_vm;
  90. }
  91. #ifdef CONFIG_NUMA
  92. /*
  93. * Save get_task_policy() for show_numa_map().
  94. */
  95. static void hold_task_mempolicy(struct proc_maps_private *priv)
  96. {
  97. struct task_struct *task = priv->task;
  98. task_lock(task);
  99. priv->task_mempolicy = get_task_policy(task);
  100. mpol_get(priv->task_mempolicy);
  101. task_unlock(task);
  102. }
  103. static void release_task_mempolicy(struct proc_maps_private *priv)
  104. {
  105. mpol_put(priv->task_mempolicy);
  106. }
  107. #else
  108. static void hold_task_mempolicy(struct proc_maps_private *priv)
  109. {
  110. }
  111. static void release_task_mempolicy(struct proc_maps_private *priv)
  112. {
  113. }
  114. #endif
  115. static void seq_print_vma_name(struct seq_file *m, struct vm_area_struct *vma)
  116. {
  117. const char __user *name = vma_get_anon_name(vma);
  118. struct mm_struct *mm = vma->vm_mm;
  119. unsigned long page_start_vaddr;
  120. unsigned long page_offset;
  121. unsigned long num_pages;
  122. unsigned long max_len = NAME_MAX;
  123. int i;
  124. page_start_vaddr = (unsigned long)name & PAGE_MASK;
  125. page_offset = (unsigned long)name - page_start_vaddr;
  126. num_pages = DIV_ROUND_UP(page_offset + max_len, PAGE_SIZE);
  127. seq_puts(m, "[anon:");
  128. for (i = 0; i < num_pages; i++) {
  129. int len;
  130. int write_len;
  131. const char *kaddr;
  132. long pages_pinned;
  133. struct page *page;
  134. pages_pinned = get_user_pages_remote(mm, page_start_vaddr, 1, 0,
  135. &page, NULL, NULL);
  136. if (pages_pinned < 1) {
  137. seq_puts(m, "<fault>]");
  138. return;
  139. }
  140. kaddr = (const char *)kmap(page);
  141. len = min(max_len, PAGE_SIZE - page_offset);
  142. write_len = strnlen(kaddr + page_offset, len);
  143. seq_write(m, kaddr + page_offset, write_len);
  144. kunmap(page);
  145. put_user_page(page);
  146. /* if strnlen hit a null terminator then we're done */
  147. if (write_len != len)
  148. break;
  149. max_len -= len;
  150. page_offset = 0;
  151. page_start_vaddr += PAGE_SIZE;
  152. }
  153. seq_putc(m, ']');
  154. }
  155. static void *m_start(struct seq_file *m, loff_t *ppos)
  156. {
  157. struct proc_maps_private *priv = m->private;
  158. unsigned long last_addr = *ppos;
  159. struct mm_struct *mm;
  160. struct vm_area_struct *vma;
  161. /* See m_next(). Zero at the start or after lseek. */
  162. if (last_addr == -1UL)
  163. return NULL;
  164. priv->task = get_proc_task(priv->inode);
  165. if (!priv->task)
  166. return ERR_PTR(-ESRCH);
  167. mm = priv->mm;
  168. if (!mm || !mmget_not_zero(mm)) {
  169. put_task_struct(priv->task);
  170. priv->task = NULL;
  171. return NULL;
  172. }
  173. if (mmap_read_lock_killable(mm)) {
  174. mmput(mm);
  175. put_task_struct(priv->task);
  176. priv->task = NULL;
  177. return ERR_PTR(-EINTR);
  178. }
  179. hold_task_mempolicy(priv);
  180. priv->tail_vma = get_gate_vma(mm);
  181. vma = find_vma(mm, last_addr);
  182. if (vma)
  183. return vma;
  184. return priv->tail_vma;
  185. }
  186. static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
  187. {
  188. struct proc_maps_private *priv = m->private;
  189. struct vm_area_struct *next, *vma = v;
  190. if (vma == priv->tail_vma)
  191. next = NULL;
  192. else if (vma->vm_next)
  193. next = vma->vm_next;
  194. else
  195. next = priv->tail_vma;
  196. *ppos = next ? next->vm_start : -1UL;
  197. return next;
  198. }
  199. static void m_stop(struct seq_file *m, void *v)
  200. {
  201. struct proc_maps_private *priv = m->private;
  202. struct mm_struct *mm = priv->mm;
  203. if (!priv->task)
  204. return;
  205. release_task_mempolicy(priv);
  206. mmap_read_unlock(mm);
  207. mmput(mm);
  208. put_task_struct(priv->task);
  209. priv->task = NULL;
  210. }
  211. static int proc_maps_open(struct inode *inode, struct file *file,
  212. const struct seq_operations *ops, int psize)
  213. {
  214. struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
  215. if (!priv)
  216. return -ENOMEM;
  217. priv->inode = inode;
  218. priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
  219. if (IS_ERR(priv->mm)) {
  220. int err = PTR_ERR(priv->mm);
  221. seq_release_private(inode, file);
  222. return err;
  223. }
  224. return 0;
  225. }
  226. static int proc_map_release(struct inode *inode, struct file *file)
  227. {
  228. struct seq_file *seq = file->private_data;
  229. struct proc_maps_private *priv = seq->private;
  230. if (priv->mm)
  231. mmdrop(priv->mm);
  232. return seq_release_private(inode, file);
  233. }
  234. static int do_maps_open(struct inode *inode, struct file *file,
  235. const struct seq_operations *ops)
  236. {
  237. return proc_maps_open(inode, file, ops,
  238. sizeof(struct proc_maps_private));
  239. }
  240. /*
  241. * Indicate if the VMA is a stack for the given task; for
  242. * /proc/PID/maps that is the stack of the main task.
  243. */
  244. static int is_stack(struct vm_area_struct *vma)
  245. {
  246. /*
  247. * We make no effort to guess what a given thread considers to be
  248. * its "stack". It's not even well-defined for programs written
  249. * languages like Go.
  250. */
  251. return vma->vm_start <= vma->vm_mm->start_stack &&
  252. vma->vm_end >= vma->vm_mm->start_stack;
  253. }
  254. static void show_vma_header_prefix(struct seq_file *m,
  255. unsigned long start, unsigned long end,
  256. vm_flags_t flags, unsigned long long pgoff,
  257. dev_t dev, unsigned long ino)
  258. {
  259. seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
  260. seq_put_hex_ll(m, NULL, start, 8);
  261. seq_put_hex_ll(m, "-", end, 8);
  262. seq_putc(m, ' ');
  263. seq_putc(m, flags & VM_READ ? 'r' : '-');
  264. seq_putc(m, flags & VM_WRITE ? 'w' : '-');
  265. seq_putc(m, flags & VM_EXEC ? 'x' : '-');
  266. seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
  267. seq_put_hex_ll(m, " ", pgoff, 8);
  268. seq_put_hex_ll(m, " ", MAJOR(dev), 2);
  269. seq_put_hex_ll(m, ":", MINOR(dev), 2);
  270. seq_put_decimal_ull(m, " ", ino);
  271. seq_putc(m, ' ');
  272. }
  273. static void
  274. show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
  275. {
  276. struct mm_struct *mm = vma->vm_mm;
  277. struct file *file = vma->vm_file;
  278. vm_flags_t flags = vma->vm_flags;
  279. unsigned long ino = 0;
  280. unsigned long long pgoff = 0;
  281. unsigned long start, end;
  282. dev_t dev = 0;
  283. const char *name = NULL;
  284. if (file) {
  285. struct inode *inode = file_inode(vma->vm_file);
  286. dev = inode->i_sb->s_dev;
  287. ino = inode->i_ino;
  288. pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
  289. }
  290. start = vma->vm_start;
  291. end = vma->vm_end;
  292. show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
  293. /*
  294. * Print the dentry name for named mappings, and a
  295. * special [heap] marker for the heap:
  296. */
  297. if (file) {
  298. seq_pad(m, ' ');
  299. seq_file_path(m, file, "\n");
  300. goto done;
  301. }
  302. if (vma->vm_ops && vma->vm_ops->name) {
  303. name = vma->vm_ops->name(vma);
  304. if (name)
  305. goto done;
  306. }
  307. name = arch_vma_name(vma);
  308. if (!name) {
  309. if (!mm) {
  310. name = "[vdso]";
  311. goto done;
  312. }
  313. if (vma->vm_start <= mm->brk &&
  314. vma->vm_end >= mm->start_brk) {
  315. name = "[heap]";
  316. goto done;
  317. }
  318. if (is_stack(vma)) {
  319. name = "[stack]";
  320. goto done;
  321. }
  322. if (vma_get_anon_name(vma)) {
  323. seq_pad(m, ' ');
  324. seq_print_vma_name(m, vma);
  325. }
  326. }
  327. done:
  328. if (name) {
  329. seq_pad(m, ' ');
  330. seq_puts(m, name);
  331. }
  332. seq_putc(m, '\n');
  333. }
  334. static int show_map(struct seq_file *m, void *v)
  335. {
  336. show_map_vma(m, v);
  337. return 0;
  338. }
  339. static const struct seq_operations proc_pid_maps_op = {
  340. .start = m_start,
  341. .next = m_next,
  342. .stop = m_stop,
  343. .show = show_map
  344. };
  345. static int pid_maps_open(struct inode *inode, struct file *file)
  346. {
  347. return do_maps_open(inode, file, &proc_pid_maps_op);
  348. }
  349. const struct file_operations proc_pid_maps_operations = {
  350. .open = pid_maps_open,
  351. .read = seq_read,
  352. .llseek = seq_lseek,
  353. .release = proc_map_release,
  354. };
  355. /*
  356. * Proportional Set Size(PSS): my share of RSS.
  357. *
  358. * PSS of a process is the count of pages it has in memory, where each
  359. * page is divided by the number of processes sharing it. So if a
  360. * process has 1000 pages all to itself, and 1000 shared with one other
  361. * process, its PSS will be 1500.
  362. *
  363. * To keep (accumulated) division errors low, we adopt a 64bit
  364. * fixed-point pss counter to minimize division errors. So (pss >>
  365. * PSS_SHIFT) would be the real byte count.
  366. *
  367. * A shift of 12 before division means (assuming 4K page size):
  368. * - 1M 3-user-pages add up to 8KB errors;
  369. * - supports mapcount up to 2^24, or 16M;
  370. * - supports PSS up to 2^52 bytes, or 4PB.
  371. */
  372. #define PSS_SHIFT 12
  373. #ifdef CONFIG_PROC_PAGE_MONITOR
  374. struct mem_size_stats {
  375. unsigned long resident;
  376. unsigned long shared_clean;
  377. unsigned long shared_dirty;
  378. unsigned long private_clean;
  379. unsigned long private_dirty;
  380. unsigned long referenced;
  381. unsigned long anonymous;
  382. unsigned long lazyfree;
  383. unsigned long anonymous_thp;
  384. unsigned long shmem_thp;
  385. unsigned long file_thp;
  386. unsigned long swap;
  387. unsigned long shared_hugetlb;
  388. unsigned long private_hugetlb;
  389. u64 pss;
  390. u64 pss_anon;
  391. u64 pss_file;
  392. u64 pss_shmem;
  393. u64 pss_locked;
  394. u64 swap_pss;
  395. bool check_shmem_swap;
  396. };
  397. static void smaps_page_accumulate(struct mem_size_stats *mss,
  398. struct page *page, unsigned long size, unsigned long pss,
  399. bool dirty, bool locked, bool private)
  400. {
  401. mss->pss += pss;
  402. if (PageAnon(page))
  403. mss->pss_anon += pss;
  404. else if (PageSwapBacked(page))
  405. mss->pss_shmem += pss;
  406. else
  407. mss->pss_file += pss;
  408. if (locked)
  409. mss->pss_locked += pss;
  410. if (dirty || PageDirty(page)) {
  411. if (private)
  412. mss->private_dirty += size;
  413. else
  414. mss->shared_dirty += size;
  415. } else {
  416. if (private)
  417. mss->private_clean += size;
  418. else
  419. mss->shared_clean += size;
  420. }
  421. }
  422. static void smaps_account(struct mem_size_stats *mss, struct page *page,
  423. bool compound, bool young, bool dirty, bool locked,
  424. bool migration)
  425. {
  426. int i, nr = compound ? compound_nr(page) : 1;
  427. unsigned long size = nr * PAGE_SIZE;
  428. /*
  429. * First accumulate quantities that depend only on |size| and the type
  430. * of the compound page.
  431. */
  432. if (PageAnon(page)) {
  433. mss->anonymous += size;
  434. if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
  435. mss->lazyfree += size;
  436. }
  437. mss->resident += size;
  438. /* Accumulate the size in pages that have been accessed. */
  439. if (young || page_is_young(page) || PageReferenced(page))
  440. mss->referenced += size;
  441. /*
  442. * Then accumulate quantities that may depend on sharing, or that may
  443. * differ page-by-page.
  444. *
  445. * page_count(page) == 1 guarantees the page is mapped exactly once.
  446. * If any subpage of the compound page mapped with PTE it would elevate
  447. * page_count().
  448. *
  449. * The page_mapcount() is called to get a snapshot of the mapcount.
  450. * Without holding the page lock this snapshot can be slightly wrong as
  451. * we cannot always read the mapcount atomically. It is not safe to
  452. * call page_mapcount() even with PTL held if the page is not mapped,
  453. * especially for migration entries. Treat regular migration entries
  454. * as mapcount == 1.
  455. */
  456. if ((page_count(page) == 1) || migration) {
  457. smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
  458. locked, true);
  459. return;
  460. }
  461. for (i = 0; i < nr; i++, page++) {
  462. int mapcount = page_mapcount(page);
  463. unsigned long pss = PAGE_SIZE << PSS_SHIFT;
  464. if (mapcount >= 2)
  465. pss /= mapcount;
  466. smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked,
  467. mapcount < 2);
  468. }
  469. }
  470. #ifdef CONFIG_SHMEM
  471. static int smaps_pte_hole(unsigned long addr, unsigned long end,
  472. __always_unused int depth, struct mm_walk *walk)
  473. {
  474. struct mem_size_stats *mss = walk->private;
  475. mss->swap += shmem_partial_swap_usage(
  476. walk->vma->vm_file->f_mapping, addr, end);
  477. return 0;
  478. }
  479. #else
  480. #define smaps_pte_hole NULL
  481. #endif /* CONFIG_SHMEM */
  482. static void smaps_pte_entry(pte_t *pte, unsigned long addr,
  483. struct mm_walk *walk)
  484. {
  485. struct mem_size_stats *mss = walk->private;
  486. struct vm_area_struct *vma = walk->vma;
  487. bool locked = !!(vma->vm_flags & VM_LOCKED);
  488. struct page *page = NULL;
  489. bool migration = false;
  490. if (pte_present(*pte)) {
  491. page = vm_normal_page(vma, addr, *pte);
  492. } else if (is_swap_pte(*pte)) {
  493. swp_entry_t swpent = pte_to_swp_entry(*pte);
  494. if (!non_swap_entry(swpent)) {
  495. int mapcount;
  496. mss->swap += PAGE_SIZE;
  497. mapcount = swp_swapcount(swpent);
  498. if (mapcount >= 2) {
  499. u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
  500. do_div(pss_delta, mapcount);
  501. mss->swap_pss += pss_delta;
  502. } else {
  503. mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
  504. }
  505. } else if (is_migration_entry(swpent)) {
  506. migration = true;
  507. page = migration_entry_to_page(swpent);
  508. } else if (is_device_private_entry(swpent))
  509. page = device_private_entry_to_page(swpent);
  510. } else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
  511. && pte_none(*pte))) {
  512. page = xa_load(&vma->vm_file->f_mapping->i_pages,
  513. linear_page_index(vma, addr));
  514. if (xa_is_value(page))
  515. mss->swap += PAGE_SIZE;
  516. return;
  517. }
  518. if (!page)
  519. return;
  520. smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte),
  521. locked, migration);
  522. }
  523. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  524. static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
  525. struct mm_walk *walk)
  526. {
  527. struct mem_size_stats *mss = walk->private;
  528. struct vm_area_struct *vma = walk->vma;
  529. bool locked = !!(vma->vm_flags & VM_LOCKED);
  530. struct page *page = NULL;
  531. bool migration = false;
  532. if (pmd_present(*pmd)) {
  533. /* FOLL_DUMP will return -EFAULT on huge zero page */
  534. page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
  535. } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
  536. swp_entry_t entry = pmd_to_swp_entry(*pmd);
  537. if (is_migration_entry(entry)) {
  538. migration = true;
  539. page = migration_entry_to_page(entry);
  540. }
  541. }
  542. if (IS_ERR_OR_NULL(page))
  543. return;
  544. if (PageAnon(page))
  545. mss->anonymous_thp += HPAGE_PMD_SIZE;
  546. else if (PageSwapBacked(page))
  547. mss->shmem_thp += HPAGE_PMD_SIZE;
  548. else if (is_zone_device_page(page))
  549. /* pass */;
  550. else
  551. mss->file_thp += HPAGE_PMD_SIZE;
  552. smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
  553. locked, migration);
  554. }
  555. #else
  556. static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
  557. struct mm_walk *walk)
  558. {
  559. }
  560. #endif
  561. static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
  562. struct mm_walk *walk)
  563. {
  564. struct vm_area_struct *vma = walk->vma;
  565. pte_t *pte;
  566. spinlock_t *ptl;
  567. ptl = pmd_trans_huge_lock(pmd, vma);
  568. if (ptl) {
  569. smaps_pmd_entry(pmd, addr, walk);
  570. spin_unlock(ptl);
  571. goto out;
  572. }
  573. if (pmd_trans_unstable(pmd))
  574. goto out;
  575. /*
  576. * The mmap_lock held all the way back in m_start() is what
  577. * keeps khugepaged out of here and from collapsing things
  578. * in here.
  579. */
  580. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  581. for (; addr != end; pte++, addr += PAGE_SIZE)
  582. smaps_pte_entry(pte, addr, walk);
  583. pte_unmap_unlock(pte - 1, ptl);
  584. out:
  585. cond_resched();
  586. return 0;
  587. }
  588. static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
  589. {
  590. /*
  591. * Don't forget to update Documentation/ on changes.
  592. */
  593. static const char mnemonics[BITS_PER_LONG][2] = {
  594. /*
  595. * In case if we meet a flag we don't know about.
  596. */
  597. [0 ... (BITS_PER_LONG-1)] = "??",
  598. [ilog2(VM_READ)] = "rd",
  599. [ilog2(VM_WRITE)] = "wr",
  600. [ilog2(VM_EXEC)] = "ex",
  601. [ilog2(VM_SHARED)] = "sh",
  602. [ilog2(VM_MAYREAD)] = "mr",
  603. [ilog2(VM_MAYWRITE)] = "mw",
  604. [ilog2(VM_MAYEXEC)] = "me",
  605. [ilog2(VM_MAYSHARE)] = "ms",
  606. [ilog2(VM_GROWSDOWN)] = "gd",
  607. [ilog2(VM_PFNMAP)] = "pf",
  608. [ilog2(VM_DENYWRITE)] = "dw",
  609. [ilog2(VM_LOCKED)] = "lo",
  610. [ilog2(VM_IO)] = "io",
  611. [ilog2(VM_SEQ_READ)] = "sr",
  612. [ilog2(VM_RAND_READ)] = "rr",
  613. [ilog2(VM_DONTCOPY)] = "dc",
  614. [ilog2(VM_DONTEXPAND)] = "de",
  615. [ilog2(VM_ACCOUNT)] = "ac",
  616. [ilog2(VM_NORESERVE)] = "nr",
  617. [ilog2(VM_HUGETLB)] = "ht",
  618. [ilog2(VM_SYNC)] = "sf",
  619. [ilog2(VM_ARCH_1)] = "ar",
  620. [ilog2(VM_WIPEONFORK)] = "wf",
  621. [ilog2(VM_DONTDUMP)] = "dd",
  622. #ifdef CONFIG_ARM64_BTI
  623. [ilog2(VM_ARM64_BTI)] = "bt",
  624. #endif
  625. #ifdef CONFIG_MEM_SOFT_DIRTY
  626. [ilog2(VM_SOFTDIRTY)] = "sd",
  627. #endif
  628. [ilog2(VM_MIXEDMAP)] = "mm",
  629. [ilog2(VM_HUGEPAGE)] = "hg",
  630. [ilog2(VM_NOHUGEPAGE)] = "nh",
  631. [ilog2(VM_MERGEABLE)] = "mg",
  632. [ilog2(VM_UFFD_MISSING)]= "um",
  633. [ilog2(VM_UFFD_WP)] = "uw",
  634. #ifdef CONFIG_ARM64_MTE
  635. [ilog2(VM_MTE)] = "mt",
  636. [ilog2(VM_MTE_ALLOWED)] = "",
  637. #endif
  638. #ifdef CONFIG_ARCH_HAS_PKEYS
  639. /* These come out via ProtectionKey: */
  640. [ilog2(VM_PKEY_BIT0)] = "",
  641. [ilog2(VM_PKEY_BIT1)] = "",
  642. [ilog2(VM_PKEY_BIT2)] = "",
  643. [ilog2(VM_PKEY_BIT3)] = "",
  644. #if VM_PKEY_BIT4
  645. [ilog2(VM_PKEY_BIT4)] = "",
  646. #endif
  647. #endif /* CONFIG_ARCH_HAS_PKEYS */
  648. #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
  649. [ilog2(VM_UFFD_MINOR)] = "ui",
  650. #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
  651. };
  652. size_t i;
  653. seq_puts(m, "VmFlags: ");
  654. for (i = 0; i < BITS_PER_LONG; i++) {
  655. if (!mnemonics[i][0])
  656. continue;
  657. if (vma->vm_flags & (1UL << i)) {
  658. seq_putc(m, mnemonics[i][0]);
  659. seq_putc(m, mnemonics[i][1]);
  660. seq_putc(m, ' ');
  661. }
  662. }
  663. seq_putc(m, '\n');
  664. }
  665. #ifdef CONFIG_HUGETLB_PAGE
  666. static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
  667. unsigned long addr, unsigned long end,
  668. struct mm_walk *walk)
  669. {
  670. struct mem_size_stats *mss = walk->private;
  671. struct vm_area_struct *vma = walk->vma;
  672. struct page *page = NULL;
  673. if (pte_present(*pte)) {
  674. page = vm_normal_page(vma, addr, *pte);
  675. } else if (is_swap_pte(*pte)) {
  676. swp_entry_t swpent = pte_to_swp_entry(*pte);
  677. if (is_migration_entry(swpent))
  678. page = migration_entry_to_page(swpent);
  679. else if (is_device_private_entry(swpent))
  680. page = device_private_entry_to_page(swpent);
  681. }
  682. if (page) {
  683. int mapcount = page_mapcount(page);
  684. if (mapcount >= 2)
  685. mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
  686. else
  687. mss->private_hugetlb += huge_page_size(hstate_vma(vma));
  688. }
  689. return 0;
  690. }
  691. #else
  692. #define smaps_hugetlb_range NULL
  693. #endif /* HUGETLB_PAGE */
  694. static const struct mm_walk_ops smaps_walk_ops = {
  695. .pmd_entry = smaps_pte_range,
  696. .hugetlb_entry = smaps_hugetlb_range,
  697. };
  698. static const struct mm_walk_ops smaps_shmem_walk_ops = {
  699. .pmd_entry = smaps_pte_range,
  700. .hugetlb_entry = smaps_hugetlb_range,
  701. .pte_hole = smaps_pte_hole,
  702. };
  703. /*
  704. * Gather mem stats from @vma with the indicated beginning
  705. * address @start, and keep them in @mss.
  706. *
  707. * Use vm_start of @vma as the beginning address if @start is 0.
  708. */
  709. static void smap_gather_stats(struct vm_area_struct *vma,
  710. struct mem_size_stats *mss, unsigned long start)
  711. {
  712. const struct mm_walk_ops *ops = &smaps_walk_ops;
  713. /* Invalid start */
  714. if (start >= vma->vm_end)
  715. return;
  716. #ifdef CONFIG_SHMEM
  717. /* In case of smaps_rollup, reset the value from previous vma */
  718. mss->check_shmem_swap = false;
  719. if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
  720. /*
  721. * For shared or readonly shmem mappings we know that all
  722. * swapped out pages belong to the shmem object, and we can
  723. * obtain the swap value much more efficiently. For private
  724. * writable mappings, we might have COW pages that are
  725. * not affected by the parent swapped out pages of the shmem
  726. * object, so we have to distinguish them during the page walk.
  727. * Unless we know that the shmem object (or the part mapped by
  728. * our VMA) has no swapped out pages at all.
  729. */
  730. unsigned long shmem_swapped = shmem_swap_usage(vma);
  731. if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
  732. !(vma->vm_flags & VM_WRITE))) {
  733. mss->swap += shmem_swapped;
  734. } else {
  735. mss->check_shmem_swap = true;
  736. ops = &smaps_shmem_walk_ops;
  737. }
  738. }
  739. #endif
  740. /* mmap_lock is held in m_start */
  741. if (!start)
  742. walk_page_vma(vma, ops, mss);
  743. else
  744. walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
  745. }
  746. #define SEQ_PUT_DEC(str, val) \
  747. seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
  748. /* Show the contents common for smaps and smaps_rollup */
  749. static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
  750. bool rollup_mode)
  751. {
  752. SEQ_PUT_DEC("Rss: ", mss->resident);
  753. SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
  754. if (rollup_mode) {
  755. /*
  756. * These are meaningful only for smaps_rollup, otherwise two of
  757. * them are zero, and the other one is the same as Pss.
  758. */
  759. SEQ_PUT_DEC(" kB\nPss_Anon: ",
  760. mss->pss_anon >> PSS_SHIFT);
  761. SEQ_PUT_DEC(" kB\nPss_File: ",
  762. mss->pss_file >> PSS_SHIFT);
  763. SEQ_PUT_DEC(" kB\nPss_Shmem: ",
  764. mss->pss_shmem >> PSS_SHIFT);
  765. }
  766. SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
  767. SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
  768. SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
  769. SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
  770. SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
  771. SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
  772. SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
  773. SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
  774. SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
  775. SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
  776. SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
  777. seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
  778. mss->private_hugetlb >> 10, 7);
  779. SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
  780. SEQ_PUT_DEC(" kB\nSwapPss: ",
  781. mss->swap_pss >> PSS_SHIFT);
  782. SEQ_PUT_DEC(" kB\nLocked: ",
  783. mss->pss_locked >> PSS_SHIFT);
  784. seq_puts(m, " kB\n");
  785. }
  786. static int show_smap(struct seq_file *m, void *v)
  787. {
  788. struct vm_area_struct *vma = v;
  789. struct mem_size_stats mss;
  790. memset(&mss, 0, sizeof(mss));
  791. smap_gather_stats(vma, &mss, 0);
  792. show_map_vma(m, vma);
  793. if (vma_get_anon_name(vma)) {
  794. seq_puts(m, "Name: ");
  795. seq_print_vma_name(m, vma);
  796. seq_putc(m, '\n');
  797. }
  798. SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
  799. SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
  800. SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
  801. seq_puts(m, " kB\n");
  802. __show_smap(m, &mss, false);
  803. seq_printf(m, "THPeligible: %d\n",
  804. transparent_hugepage_active(vma));
  805. if (arch_pkeys_enabled())
  806. seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
  807. show_smap_vma_flags(m, vma);
  808. return 0;
  809. }
  810. static int show_smaps_rollup(struct seq_file *m, void *v)
  811. {
  812. struct proc_maps_private *priv = m->private;
  813. struct mem_size_stats mss;
  814. struct mm_struct *mm;
  815. struct vm_area_struct *vma;
  816. unsigned long last_vma_end = 0;
  817. int ret = 0;
  818. priv->task = get_proc_task(priv->inode);
  819. if (!priv->task)
  820. return -ESRCH;
  821. mm = priv->mm;
  822. if (!mm || !mmget_not_zero(mm)) {
  823. ret = -ESRCH;
  824. goto out_put_task;
  825. }
  826. memset(&mss, 0, sizeof(mss));
  827. ret = mmap_read_lock_killable(mm);
  828. if (ret)
  829. goto out_put_mm;
  830. hold_task_mempolicy(priv);
  831. for (vma = priv->mm->mmap; vma;) {
  832. smap_gather_stats(vma, &mss, 0);
  833. last_vma_end = vma->vm_end;
  834. /*
  835. * Release mmap_lock temporarily if someone wants to
  836. * access it for write request.
  837. */
  838. if (mmap_lock_is_contended(mm)) {
  839. mmap_read_unlock(mm);
  840. ret = mmap_read_lock_killable(mm);
  841. if (ret) {
  842. release_task_mempolicy(priv);
  843. goto out_put_mm;
  844. }
  845. /*
  846. * After dropping the lock, there are four cases to
  847. * consider. See the following example for explanation.
  848. *
  849. * +------+------+-----------+
  850. * | VMA1 | VMA2 | VMA3 |
  851. * +------+------+-----------+
  852. * | | | |
  853. * 4k 8k 16k 400k
  854. *
  855. * Suppose we drop the lock after reading VMA2 due to
  856. * contention, then we get:
  857. *
  858. * last_vma_end = 16k
  859. *
  860. * 1) VMA2 is freed, but VMA3 exists:
  861. *
  862. * find_vma(mm, 16k - 1) will return VMA3.
  863. * In this case, just continue from VMA3.
  864. *
  865. * 2) VMA2 still exists:
  866. *
  867. * find_vma(mm, 16k - 1) will return VMA2.
  868. * Iterate the loop like the original one.
  869. *
  870. * 3) No more VMAs can be found:
  871. *
  872. * find_vma(mm, 16k - 1) will return NULL.
  873. * No more things to do, just break.
  874. *
  875. * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
  876. *
  877. * find_vma(mm, 16k - 1) will return VMA' whose range
  878. * contains last_vma_end.
  879. * Iterate VMA' from last_vma_end.
  880. */
  881. vma = find_vma(mm, last_vma_end - 1);
  882. /* Case 3 above */
  883. if (!vma)
  884. break;
  885. /* Case 1 above */
  886. if (vma->vm_start >= last_vma_end)
  887. continue;
  888. /* Case 4 above */
  889. if (vma->vm_end > last_vma_end)
  890. smap_gather_stats(vma, &mss, last_vma_end);
  891. }
  892. /* Case 2 above */
  893. vma = vma->vm_next;
  894. }
  895. show_vma_header_prefix(m, priv->mm->mmap->vm_start,
  896. last_vma_end, 0, 0, 0, 0);
  897. seq_pad(m, ' ');
  898. seq_puts(m, "[rollup]\n");
  899. __show_smap(m, &mss, true);
  900. release_task_mempolicy(priv);
  901. mmap_read_unlock(mm);
  902. out_put_mm:
  903. mmput(mm);
  904. out_put_task:
  905. put_task_struct(priv->task);
  906. priv->task = NULL;
  907. return ret;
  908. }
  909. #undef SEQ_PUT_DEC
  910. static const struct seq_operations proc_pid_smaps_op = {
  911. .start = m_start,
  912. .next = m_next,
  913. .stop = m_stop,
  914. .show = show_smap
  915. };
  916. static int pid_smaps_open(struct inode *inode, struct file *file)
  917. {
  918. return do_maps_open(inode, file, &proc_pid_smaps_op);
  919. }
  920. static int smaps_rollup_open(struct inode *inode, struct file *file)
  921. {
  922. int ret;
  923. struct proc_maps_private *priv;
  924. priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
  925. if (!priv)
  926. return -ENOMEM;
  927. ret = single_open(file, show_smaps_rollup, priv);
  928. if (ret)
  929. goto out_free;
  930. priv->inode = inode;
  931. priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
  932. if (IS_ERR(priv->mm)) {
  933. ret = PTR_ERR(priv->mm);
  934. single_release(inode, file);
  935. goto out_free;
  936. }
  937. return 0;
  938. out_free:
  939. kfree(priv);
  940. return ret;
  941. }
  942. static int smaps_rollup_release(struct inode *inode, struct file *file)
  943. {
  944. struct seq_file *seq = file->private_data;
  945. struct proc_maps_private *priv = seq->private;
  946. if (priv->mm)
  947. mmdrop(priv->mm);
  948. kfree(priv);
  949. return single_release(inode, file);
  950. }
  951. const struct file_operations proc_pid_smaps_operations = {
  952. .open = pid_smaps_open,
  953. .read = seq_read,
  954. .llseek = seq_lseek,
  955. .release = proc_map_release,
  956. };
  957. const struct file_operations proc_pid_smaps_rollup_operations = {
  958. .open = smaps_rollup_open,
  959. .read = seq_read,
  960. .llseek = seq_lseek,
  961. .release = smaps_rollup_release,
  962. };
  963. enum clear_refs_types {
  964. CLEAR_REFS_ALL = 1,
  965. CLEAR_REFS_ANON,
  966. CLEAR_REFS_MAPPED,
  967. CLEAR_REFS_SOFT_DIRTY,
  968. CLEAR_REFS_MM_HIWATER_RSS,
  969. CLEAR_REFS_LAST,
  970. };
  971. struct clear_refs_private {
  972. enum clear_refs_types type;
  973. };
  974. #ifdef CONFIG_MEM_SOFT_DIRTY
  975. #define is_cow_mapping(flags) (((flags) & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE)
  976. static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
  977. {
  978. struct page *page;
  979. if (!pte_write(pte))
  980. return false;
  981. if (!is_cow_mapping(vma->vm_flags))
  982. return false;
  983. if (likely(!atomic_read(&vma->vm_mm->has_pinned)))
  984. return false;
  985. page = vm_normal_page(vma, addr, pte);
  986. if (!page)
  987. return false;
  988. return page_maybe_dma_pinned(page);
  989. }
  990. static inline void clear_soft_dirty(struct vm_area_struct *vma,
  991. unsigned long addr, pte_t *pte)
  992. {
  993. /*
  994. * The soft-dirty tracker uses #PF-s to catch writes
  995. * to pages, so write-protect the pte as well. See the
  996. * Documentation/admin-guide/mm/soft-dirty.rst for full description
  997. * of how soft-dirty works.
  998. */
  999. pte_t ptent = *pte;
  1000. if (pte_present(ptent)) {
  1001. pte_t old_pte;
  1002. if (pte_is_pinned(vma, addr, ptent))
  1003. return;
  1004. old_pte = ptep_modify_prot_start(vma, addr, pte);
  1005. ptent = pte_wrprotect(old_pte);
  1006. ptent = pte_clear_soft_dirty(ptent);
  1007. ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
  1008. } else if (is_swap_pte(ptent)) {
  1009. ptent = pte_swp_clear_soft_dirty(ptent);
  1010. set_pte_at(vma->vm_mm, addr, pte, ptent);
  1011. }
  1012. }
  1013. #else
  1014. static inline void clear_soft_dirty(struct vm_area_struct *vma,
  1015. unsigned long addr, pte_t *pte)
  1016. {
  1017. }
  1018. #endif
  1019. #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
  1020. static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
  1021. unsigned long addr, pmd_t *pmdp)
  1022. {
  1023. pmd_t old, pmd = *pmdp;
  1024. if (pmd_present(pmd)) {
  1025. /* See comment in change_huge_pmd() */
  1026. old = pmdp_invalidate(vma, addr, pmdp);
  1027. if (pmd_dirty(old))
  1028. pmd = pmd_mkdirty(pmd);
  1029. if (pmd_young(old))
  1030. pmd = pmd_mkyoung(pmd);
  1031. pmd = pmd_wrprotect(pmd);
  1032. pmd = pmd_clear_soft_dirty(pmd);
  1033. set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
  1034. } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
  1035. pmd = pmd_swp_clear_soft_dirty(pmd);
  1036. set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
  1037. }
  1038. }
  1039. #else
  1040. static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
  1041. unsigned long addr, pmd_t *pmdp)
  1042. {
  1043. }
  1044. #endif
  1045. static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
  1046. unsigned long end, struct mm_walk *walk)
  1047. {
  1048. struct clear_refs_private *cp = walk->private;
  1049. struct vm_area_struct *vma = walk->vma;
  1050. pte_t *pte, ptent;
  1051. spinlock_t *ptl;
  1052. struct page *page;
  1053. ptl = pmd_trans_huge_lock(pmd, vma);
  1054. if (ptl) {
  1055. if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
  1056. clear_soft_dirty_pmd(vma, addr, pmd);
  1057. goto out;
  1058. }
  1059. if (!pmd_present(*pmd))
  1060. goto out;
  1061. page = pmd_page(*pmd);
  1062. /* Clear accessed and referenced bits. */
  1063. pmdp_test_and_clear_young(vma, addr, pmd);
  1064. test_and_clear_page_young(page);
  1065. ClearPageReferenced(page);
  1066. out:
  1067. spin_unlock(ptl);
  1068. return 0;
  1069. }
  1070. if (pmd_trans_unstable(pmd))
  1071. return 0;
  1072. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  1073. for (; addr != end; pte++, addr += PAGE_SIZE) {
  1074. ptent = *pte;
  1075. if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
  1076. clear_soft_dirty(vma, addr, pte);
  1077. continue;
  1078. }
  1079. if (!pte_present(ptent))
  1080. continue;
  1081. page = vm_normal_page(vma, addr, ptent);
  1082. if (!page)
  1083. continue;
  1084. /* Clear accessed and referenced bits. */
  1085. ptep_test_and_clear_young(vma, addr, pte);
  1086. test_and_clear_page_young(page);
  1087. ClearPageReferenced(page);
  1088. }
  1089. pte_unmap_unlock(pte - 1, ptl);
  1090. cond_resched();
  1091. return 0;
  1092. }
  1093. static int clear_refs_test_walk(unsigned long start, unsigned long end,
  1094. struct mm_walk *walk)
  1095. {
  1096. struct clear_refs_private *cp = walk->private;
  1097. struct vm_area_struct *vma = walk->vma;
  1098. if (vma->vm_flags & VM_PFNMAP)
  1099. return 1;
  1100. /*
  1101. * Writing 1 to /proc/pid/clear_refs affects all pages.
  1102. * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
  1103. * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
  1104. * Writing 4 to /proc/pid/clear_refs affects all pages.
  1105. */
  1106. if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
  1107. return 1;
  1108. if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
  1109. return 1;
  1110. return 0;
  1111. }
  1112. static const struct mm_walk_ops clear_refs_walk_ops = {
  1113. .pmd_entry = clear_refs_pte_range,
  1114. .test_walk = clear_refs_test_walk,
  1115. };
  1116. static ssize_t clear_refs_write(struct file *file, const char __user *buf,
  1117. size_t count, loff_t *ppos)
  1118. {
  1119. struct task_struct *task;
  1120. char buffer[PROC_NUMBUF];
  1121. struct mm_struct *mm;
  1122. struct vm_area_struct *vma;
  1123. enum clear_refs_types type;
  1124. int itype;
  1125. int rv;
  1126. memset(buffer, 0, sizeof(buffer));
  1127. if (count > sizeof(buffer) - 1)
  1128. count = sizeof(buffer) - 1;
  1129. if (copy_from_user(buffer, buf, count))
  1130. return -EFAULT;
  1131. rv = kstrtoint(strstrip(buffer), 10, &itype);
  1132. if (rv < 0)
  1133. return rv;
  1134. type = (enum clear_refs_types)itype;
  1135. if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
  1136. return -EINVAL;
  1137. task = get_proc_task(file_inode(file));
  1138. if (!task)
  1139. return -ESRCH;
  1140. mm = get_task_mm(task);
  1141. if (mm) {
  1142. struct mmu_notifier_range range;
  1143. struct clear_refs_private cp = {
  1144. .type = type,
  1145. };
  1146. if (mmap_write_lock_killable(mm)) {
  1147. count = -EINTR;
  1148. goto out_mm;
  1149. }
  1150. if (type == CLEAR_REFS_MM_HIWATER_RSS) {
  1151. /*
  1152. * Writing 5 to /proc/pid/clear_refs resets the peak
  1153. * resident set size to this mm's current rss value.
  1154. */
  1155. reset_mm_hiwater_rss(mm);
  1156. goto out_unlock;
  1157. }
  1158. if (type == CLEAR_REFS_SOFT_DIRTY) {
  1159. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  1160. if (!(vma->vm_flags & VM_SOFTDIRTY))
  1161. continue;
  1162. vm_write_begin(vma);
  1163. WRITE_ONCE(vma->vm_flags,
  1164. vma->vm_flags & ~VM_SOFTDIRTY);
  1165. vma_set_page_prot(vma);
  1166. vm_write_end(vma);
  1167. }
  1168. inc_tlb_flush_pending(mm);
  1169. mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
  1170. 0, NULL, mm, 0, -1UL);
  1171. mmu_notifier_invalidate_range_start(&range);
  1172. }
  1173. walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops,
  1174. &cp);
  1175. if (type == CLEAR_REFS_SOFT_DIRTY) {
  1176. mmu_notifier_invalidate_range_end(&range);
  1177. flush_tlb_mm(mm);
  1178. dec_tlb_flush_pending(mm);
  1179. }
  1180. out_unlock:
  1181. mmap_write_unlock(mm);
  1182. out_mm:
  1183. mmput(mm);
  1184. }
  1185. put_task_struct(task);
  1186. return count;
  1187. }
  1188. const struct file_operations proc_clear_refs_operations = {
  1189. .write = clear_refs_write,
  1190. .llseek = noop_llseek,
  1191. };
  1192. typedef struct {
  1193. u64 pme;
  1194. } pagemap_entry_t;
  1195. struct pagemapread {
  1196. int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
  1197. pagemap_entry_t *buffer;
  1198. bool show_pfn;
  1199. };
  1200. #define PAGEMAP_WALK_SIZE (PMD_SIZE)
  1201. #define PAGEMAP_WALK_MASK (PMD_MASK)
  1202. #define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
  1203. #define PM_PFRAME_BITS 55
  1204. #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
  1205. #define PM_SOFT_DIRTY BIT_ULL(55)
  1206. #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
  1207. #define PM_FILE BIT_ULL(61)
  1208. #define PM_SWAP BIT_ULL(62)
  1209. #define PM_PRESENT BIT_ULL(63)
  1210. #define PM_END_OF_BUFFER 1
  1211. static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
  1212. {
  1213. return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
  1214. }
  1215. static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
  1216. struct pagemapread *pm)
  1217. {
  1218. pm->buffer[pm->pos++] = *pme;
  1219. if (pm->pos >= pm->len)
  1220. return PM_END_OF_BUFFER;
  1221. return 0;
  1222. }
  1223. static int pagemap_pte_hole(unsigned long start, unsigned long end,
  1224. __always_unused int depth, struct mm_walk *walk)
  1225. {
  1226. struct pagemapread *pm = walk->private;
  1227. unsigned long addr = start;
  1228. int err = 0;
  1229. while (addr < end) {
  1230. struct vm_area_struct *vma = find_vma(walk->mm, addr);
  1231. pagemap_entry_t pme = make_pme(0, 0);
  1232. /* End of address space hole, which we mark as non-present. */
  1233. unsigned long hole_end;
  1234. if (vma)
  1235. hole_end = min(end, vma->vm_start);
  1236. else
  1237. hole_end = end;
  1238. for (; addr < hole_end; addr += PAGE_SIZE) {
  1239. err = add_to_pagemap(addr, &pme, pm);
  1240. if (err)
  1241. goto out;
  1242. }
  1243. if (!vma)
  1244. break;
  1245. /* Addresses in the VMA. */
  1246. if (vma->vm_flags & VM_SOFTDIRTY)
  1247. pme = make_pme(0, PM_SOFT_DIRTY);
  1248. for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
  1249. err = add_to_pagemap(addr, &pme, pm);
  1250. if (err)
  1251. goto out;
  1252. }
  1253. }
  1254. out:
  1255. return err;
  1256. }
  1257. static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
  1258. struct vm_area_struct *vma, unsigned long addr, pte_t pte)
  1259. {
  1260. u64 frame = 0, flags = 0;
  1261. struct page *page = NULL;
  1262. bool migration = false;
  1263. if (pte_present(pte)) {
  1264. if (pm->show_pfn)
  1265. frame = pte_pfn(pte);
  1266. flags |= PM_PRESENT;
  1267. page = vm_normal_page(vma, addr, pte);
  1268. if (pte_soft_dirty(pte))
  1269. flags |= PM_SOFT_DIRTY;
  1270. } else if (is_swap_pte(pte)) {
  1271. swp_entry_t entry;
  1272. if (pte_swp_soft_dirty(pte))
  1273. flags |= PM_SOFT_DIRTY;
  1274. entry = pte_to_swp_entry(pte);
  1275. if (pm->show_pfn)
  1276. frame = swp_type(entry) |
  1277. (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
  1278. flags |= PM_SWAP;
  1279. if (is_migration_entry(entry)) {
  1280. migration = true;
  1281. page = migration_entry_to_page(entry);
  1282. }
  1283. if (is_device_private_entry(entry))
  1284. page = device_private_entry_to_page(entry);
  1285. }
  1286. if (page && !PageAnon(page))
  1287. flags |= PM_FILE;
  1288. if (page && !migration && page_mapcount(page) == 1)
  1289. flags |= PM_MMAP_EXCLUSIVE;
  1290. if (vma->vm_flags & VM_SOFTDIRTY)
  1291. flags |= PM_SOFT_DIRTY;
  1292. return make_pme(frame, flags);
  1293. }
  1294. static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
  1295. struct mm_walk *walk)
  1296. {
  1297. struct vm_area_struct *vma = walk->vma;
  1298. struct pagemapread *pm = walk->private;
  1299. spinlock_t *ptl;
  1300. pte_t *pte, *orig_pte;
  1301. int err = 0;
  1302. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1303. bool migration = false;
  1304. ptl = pmd_trans_huge_lock(pmdp, vma);
  1305. if (ptl) {
  1306. u64 flags = 0, frame = 0;
  1307. pmd_t pmd = *pmdp;
  1308. struct page *page = NULL;
  1309. if (vma->vm_flags & VM_SOFTDIRTY)
  1310. flags |= PM_SOFT_DIRTY;
  1311. if (pmd_present(pmd)) {
  1312. page = pmd_page(pmd);
  1313. flags |= PM_PRESENT;
  1314. if (pmd_soft_dirty(pmd))
  1315. flags |= PM_SOFT_DIRTY;
  1316. if (pm->show_pfn)
  1317. frame = pmd_pfn(pmd) +
  1318. ((addr & ~PMD_MASK) >> PAGE_SHIFT);
  1319. }
  1320. #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
  1321. else if (is_swap_pmd(pmd)) {
  1322. swp_entry_t entry = pmd_to_swp_entry(pmd);
  1323. unsigned long offset;
  1324. if (pm->show_pfn) {
  1325. offset = swp_offset(entry) +
  1326. ((addr & ~PMD_MASK) >> PAGE_SHIFT);
  1327. frame = swp_type(entry) |
  1328. (offset << MAX_SWAPFILES_SHIFT);
  1329. }
  1330. flags |= PM_SWAP;
  1331. if (pmd_swp_soft_dirty(pmd))
  1332. flags |= PM_SOFT_DIRTY;
  1333. VM_BUG_ON(!is_pmd_migration_entry(pmd));
  1334. migration = is_migration_entry(entry);
  1335. page = migration_entry_to_page(entry);
  1336. }
  1337. #endif
  1338. if (page && !migration && page_mapcount(page) == 1)
  1339. flags |= PM_MMAP_EXCLUSIVE;
  1340. for (; addr != end; addr += PAGE_SIZE) {
  1341. pagemap_entry_t pme = make_pme(frame, flags);
  1342. err = add_to_pagemap(addr, &pme, pm);
  1343. if (err)
  1344. break;
  1345. if (pm->show_pfn) {
  1346. if (flags & PM_PRESENT)
  1347. frame++;
  1348. else if (flags & PM_SWAP)
  1349. frame += (1 << MAX_SWAPFILES_SHIFT);
  1350. }
  1351. }
  1352. spin_unlock(ptl);
  1353. return err;
  1354. }
  1355. if (pmd_trans_unstable(pmdp))
  1356. return 0;
  1357. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  1358. /*
  1359. * We can assume that @vma always points to a valid one and @end never
  1360. * goes beyond vma->vm_end.
  1361. */
  1362. orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
  1363. for (; addr < end; pte++, addr += PAGE_SIZE) {
  1364. pagemap_entry_t pme;
  1365. pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
  1366. err = add_to_pagemap(addr, &pme, pm);
  1367. if (err)
  1368. break;
  1369. }
  1370. pte_unmap_unlock(orig_pte, ptl);
  1371. cond_resched();
  1372. return err;
  1373. }
  1374. #ifdef CONFIG_HUGETLB_PAGE
  1375. /* This function walks within one hugetlb entry in the single call */
  1376. static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
  1377. unsigned long addr, unsigned long end,
  1378. struct mm_walk *walk)
  1379. {
  1380. struct pagemapread *pm = walk->private;
  1381. struct vm_area_struct *vma = walk->vma;
  1382. u64 flags = 0, frame = 0;
  1383. int err = 0;
  1384. pte_t pte;
  1385. if (vma->vm_flags & VM_SOFTDIRTY)
  1386. flags |= PM_SOFT_DIRTY;
  1387. pte = huge_ptep_get(ptep);
  1388. if (pte_present(pte)) {
  1389. struct page *page = pte_page(pte);
  1390. if (!PageAnon(page))
  1391. flags |= PM_FILE;
  1392. if (page_mapcount(page) == 1)
  1393. flags |= PM_MMAP_EXCLUSIVE;
  1394. flags |= PM_PRESENT;
  1395. if (pm->show_pfn)
  1396. frame = pte_pfn(pte) +
  1397. ((addr & ~hmask) >> PAGE_SHIFT);
  1398. }
  1399. for (; addr != end; addr += PAGE_SIZE) {
  1400. pagemap_entry_t pme = make_pme(frame, flags);
  1401. err = add_to_pagemap(addr, &pme, pm);
  1402. if (err)
  1403. return err;
  1404. if (pm->show_pfn && (flags & PM_PRESENT))
  1405. frame++;
  1406. }
  1407. cond_resched();
  1408. return err;
  1409. }
  1410. #else
  1411. #define pagemap_hugetlb_range NULL
  1412. #endif /* HUGETLB_PAGE */
  1413. static const struct mm_walk_ops pagemap_ops = {
  1414. .pmd_entry = pagemap_pmd_range,
  1415. .pte_hole = pagemap_pte_hole,
  1416. .hugetlb_entry = pagemap_hugetlb_range,
  1417. };
  1418. /*
  1419. * /proc/pid/pagemap - an array mapping virtual pages to pfns
  1420. *
  1421. * For each page in the address space, this file contains one 64-bit entry
  1422. * consisting of the following:
  1423. *
  1424. * Bits 0-54 page frame number (PFN) if present
  1425. * Bits 0-4 swap type if swapped
  1426. * Bits 5-54 swap offset if swapped
  1427. * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
  1428. * Bit 56 page exclusively mapped
  1429. * Bits 57-60 zero
  1430. * Bit 61 page is file-page or shared-anon
  1431. * Bit 62 page swapped
  1432. * Bit 63 page present
  1433. *
  1434. * If the page is not present but in swap, then the PFN contains an
  1435. * encoding of the swap file number and the page's offset into the
  1436. * swap. Unmapped pages return a null PFN. This allows determining
  1437. * precisely which pages are mapped (or in swap) and comparing mapped
  1438. * pages between processes.
  1439. *
  1440. * Efficient users of this interface will use /proc/pid/maps to
  1441. * determine which areas of memory are actually mapped and llseek to
  1442. * skip over unmapped regions.
  1443. */
  1444. static ssize_t pagemap_read(struct file *file, char __user *buf,
  1445. size_t count, loff_t *ppos)
  1446. {
  1447. struct mm_struct *mm = file->private_data;
  1448. struct pagemapread pm;
  1449. unsigned long src;
  1450. unsigned long svpfn;
  1451. unsigned long start_vaddr;
  1452. unsigned long end_vaddr;
  1453. int ret = 0, copied = 0;
  1454. if (!mm || !mmget_not_zero(mm))
  1455. goto out;
  1456. ret = -EINVAL;
  1457. /* file position must be aligned */
  1458. if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
  1459. goto out_mm;
  1460. ret = 0;
  1461. if (!count)
  1462. goto out_mm;
  1463. /* do not disclose physical addresses: attack vector */
  1464. pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
  1465. pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
  1466. pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
  1467. ret = -ENOMEM;
  1468. if (!pm.buffer)
  1469. goto out_mm;
  1470. src = *ppos;
  1471. svpfn = src / PM_ENTRY_BYTES;
  1472. end_vaddr = mm->task_size;
  1473. /* watch out for wraparound */
  1474. start_vaddr = end_vaddr;
  1475. if (svpfn <= (ULONG_MAX >> PAGE_SHIFT))
  1476. start_vaddr = untagged_addr(svpfn << PAGE_SHIFT);
  1477. /* Ensure the address is inside the task */
  1478. if (start_vaddr > mm->task_size)
  1479. start_vaddr = end_vaddr;
  1480. /*
  1481. * The odds are that this will stop walking way
  1482. * before end_vaddr, because the length of the
  1483. * user buffer is tracked in "pm", and the walk
  1484. * will stop when we hit the end of the buffer.
  1485. */
  1486. ret = 0;
  1487. while (count && (start_vaddr < end_vaddr)) {
  1488. int len;
  1489. unsigned long end;
  1490. pm.pos = 0;
  1491. end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
  1492. /* overflow ? */
  1493. if (end < start_vaddr || end > end_vaddr)
  1494. end = end_vaddr;
  1495. ret = mmap_read_lock_killable(mm);
  1496. if (ret)
  1497. goto out_free;
  1498. ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
  1499. mmap_read_unlock(mm);
  1500. start_vaddr = end;
  1501. len = min(count, PM_ENTRY_BYTES * pm.pos);
  1502. if (copy_to_user(buf, pm.buffer, len)) {
  1503. ret = -EFAULT;
  1504. goto out_free;
  1505. }
  1506. copied += len;
  1507. buf += len;
  1508. count -= len;
  1509. }
  1510. *ppos += copied;
  1511. if (!ret || ret == PM_END_OF_BUFFER)
  1512. ret = copied;
  1513. out_free:
  1514. kfree(pm.buffer);
  1515. out_mm:
  1516. mmput(mm);
  1517. out:
  1518. return ret;
  1519. }
  1520. static int pagemap_open(struct inode *inode, struct file *file)
  1521. {
  1522. struct mm_struct *mm;
  1523. mm = proc_mem_open(inode, PTRACE_MODE_READ);
  1524. if (IS_ERR(mm))
  1525. return PTR_ERR(mm);
  1526. file->private_data = mm;
  1527. return 0;
  1528. }
  1529. static int pagemap_release(struct inode *inode, struct file *file)
  1530. {
  1531. struct mm_struct *mm = file->private_data;
  1532. if (mm)
  1533. mmdrop(mm);
  1534. return 0;
  1535. }
  1536. const struct file_operations proc_pagemap_operations = {
  1537. .llseek = mem_lseek, /* borrow this */
  1538. .read = pagemap_read,
  1539. .open = pagemap_open,
  1540. .release = pagemap_release,
  1541. };
  1542. #endif /* CONFIG_PROC_PAGE_MONITOR */
  1543. #ifdef CONFIG_NUMA
  1544. struct numa_maps {
  1545. unsigned long pages;
  1546. unsigned long anon;
  1547. unsigned long active;
  1548. unsigned long writeback;
  1549. unsigned long mapcount_max;
  1550. unsigned long dirty;
  1551. unsigned long swapcache;
  1552. unsigned long node[MAX_NUMNODES];
  1553. };
  1554. struct numa_maps_private {
  1555. struct proc_maps_private proc_maps;
  1556. struct numa_maps md;
  1557. };
  1558. static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
  1559. unsigned long nr_pages)
  1560. {
  1561. int count = page_mapcount(page);
  1562. md->pages += nr_pages;
  1563. if (pte_dirty || PageDirty(page))
  1564. md->dirty += nr_pages;
  1565. if (PageSwapCache(page))
  1566. md->swapcache += nr_pages;
  1567. if (PageActive(page) || PageUnevictable(page))
  1568. md->active += nr_pages;
  1569. if (PageWriteback(page))
  1570. md->writeback += nr_pages;
  1571. if (PageAnon(page))
  1572. md->anon += nr_pages;
  1573. if (count > md->mapcount_max)
  1574. md->mapcount_max = count;
  1575. md->node[page_to_nid(page)] += nr_pages;
  1576. }
  1577. static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
  1578. unsigned long addr)
  1579. {
  1580. struct page *page;
  1581. int nid;
  1582. if (!pte_present(pte))
  1583. return NULL;
  1584. page = vm_normal_page(vma, addr, pte);
  1585. if (!page)
  1586. return NULL;
  1587. if (PageReserved(page))
  1588. return NULL;
  1589. nid = page_to_nid(page);
  1590. if (!node_isset(nid, node_states[N_MEMORY]))
  1591. return NULL;
  1592. return page;
  1593. }
  1594. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1595. static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
  1596. struct vm_area_struct *vma,
  1597. unsigned long addr)
  1598. {
  1599. struct page *page;
  1600. int nid;
  1601. if (!pmd_present(pmd))
  1602. return NULL;
  1603. page = vm_normal_page_pmd(vma, addr, pmd);
  1604. if (!page)
  1605. return NULL;
  1606. if (PageReserved(page))
  1607. return NULL;
  1608. nid = page_to_nid(page);
  1609. if (!node_isset(nid, node_states[N_MEMORY]))
  1610. return NULL;
  1611. return page;
  1612. }
  1613. #endif
  1614. static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
  1615. unsigned long end, struct mm_walk *walk)
  1616. {
  1617. struct numa_maps *md = walk->private;
  1618. struct vm_area_struct *vma = walk->vma;
  1619. spinlock_t *ptl;
  1620. pte_t *orig_pte;
  1621. pte_t *pte;
  1622. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1623. ptl = pmd_trans_huge_lock(pmd, vma);
  1624. if (ptl) {
  1625. struct page *page;
  1626. page = can_gather_numa_stats_pmd(*pmd, vma, addr);
  1627. if (page)
  1628. gather_stats(page, md, pmd_dirty(*pmd),
  1629. HPAGE_PMD_SIZE/PAGE_SIZE);
  1630. spin_unlock(ptl);
  1631. return 0;
  1632. }
  1633. if (pmd_trans_unstable(pmd))
  1634. return 0;
  1635. #endif
  1636. orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
  1637. do {
  1638. struct page *page = can_gather_numa_stats(*pte, vma, addr);
  1639. if (!page)
  1640. continue;
  1641. gather_stats(page, md, pte_dirty(*pte), 1);
  1642. } while (pte++, addr += PAGE_SIZE, addr != end);
  1643. pte_unmap_unlock(orig_pte, ptl);
  1644. cond_resched();
  1645. return 0;
  1646. }
  1647. #ifdef CONFIG_HUGETLB_PAGE
  1648. static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
  1649. unsigned long addr, unsigned long end, struct mm_walk *walk)
  1650. {
  1651. pte_t huge_pte = huge_ptep_get(pte);
  1652. struct numa_maps *md;
  1653. struct page *page;
  1654. if (!pte_present(huge_pte))
  1655. return 0;
  1656. page = pte_page(huge_pte);
  1657. if (!page)
  1658. return 0;
  1659. md = walk->private;
  1660. gather_stats(page, md, pte_dirty(huge_pte), 1);
  1661. return 0;
  1662. }
  1663. #else
  1664. static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
  1665. unsigned long addr, unsigned long end, struct mm_walk *walk)
  1666. {
  1667. return 0;
  1668. }
  1669. #endif
  1670. static const struct mm_walk_ops show_numa_ops = {
  1671. .hugetlb_entry = gather_hugetlb_stats,
  1672. .pmd_entry = gather_pte_stats,
  1673. };
  1674. /*
  1675. * Display pages allocated per node and memory policy via /proc.
  1676. */
  1677. static int show_numa_map(struct seq_file *m, void *v)
  1678. {
  1679. struct numa_maps_private *numa_priv = m->private;
  1680. struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
  1681. struct vm_area_struct *vma = v;
  1682. struct numa_maps *md = &numa_priv->md;
  1683. struct file *file = vma->vm_file;
  1684. struct mm_struct *mm = vma->vm_mm;
  1685. struct mempolicy *pol;
  1686. char buffer[64];
  1687. int nid;
  1688. if (!mm)
  1689. return 0;
  1690. /* Ensure we start with an empty set of numa_maps statistics. */
  1691. memset(md, 0, sizeof(*md));
  1692. pol = __get_vma_policy(vma, vma->vm_start);
  1693. if (pol) {
  1694. mpol_to_str(buffer, sizeof(buffer), pol);
  1695. mpol_cond_put(pol);
  1696. } else {
  1697. mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
  1698. }
  1699. seq_printf(m, "%08lx %s", vma->vm_start, buffer);
  1700. if (file) {
  1701. seq_puts(m, " file=");
  1702. seq_file_path(m, file, "\n\t= ");
  1703. } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
  1704. seq_puts(m, " heap");
  1705. } else if (is_stack(vma)) {
  1706. seq_puts(m, " stack");
  1707. }
  1708. if (is_vm_hugetlb_page(vma))
  1709. seq_puts(m, " huge");
  1710. /* mmap_lock is held by m_start */
  1711. walk_page_vma(vma, &show_numa_ops, md);
  1712. if (!md->pages)
  1713. goto out;
  1714. if (md->anon)
  1715. seq_printf(m, " anon=%lu", md->anon);
  1716. if (md->dirty)
  1717. seq_printf(m, " dirty=%lu", md->dirty);
  1718. if (md->pages != md->anon && md->pages != md->dirty)
  1719. seq_printf(m, " mapped=%lu", md->pages);
  1720. if (md->mapcount_max > 1)
  1721. seq_printf(m, " mapmax=%lu", md->mapcount_max);
  1722. if (md->swapcache)
  1723. seq_printf(m, " swapcache=%lu", md->swapcache);
  1724. if (md->active < md->pages && !is_vm_hugetlb_page(vma))
  1725. seq_printf(m, " active=%lu", md->active);
  1726. if (md->writeback)
  1727. seq_printf(m, " writeback=%lu", md->writeback);
  1728. for_each_node_state(nid, N_MEMORY)
  1729. if (md->node[nid])
  1730. seq_printf(m, " N%d=%lu", nid, md->node[nid]);
  1731. seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
  1732. out:
  1733. seq_putc(m, '\n');
  1734. return 0;
  1735. }
  1736. static const struct seq_operations proc_pid_numa_maps_op = {
  1737. .start = m_start,
  1738. .next = m_next,
  1739. .stop = m_stop,
  1740. .show = show_numa_map,
  1741. };
  1742. static int pid_numa_maps_open(struct inode *inode, struct file *file)
  1743. {
  1744. return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
  1745. sizeof(struct numa_maps_private));
  1746. }
  1747. const struct file_operations proc_pid_numa_maps_operations = {
  1748. .open = pid_numa_maps_open,
  1749. .read = seq_read,
  1750. .llseek = seq_lseek,
  1751. .release = proc_map_release,
  1752. };
  1753. #endif /* CONFIG_NUMA */