madvise.c 31 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/mm/madvise.c
  4. *
  5. * Copyright (C) 1999 Linus Torvalds
  6. * Copyright (C) 2002 Christoph Hellwig
  7. */
  8. #include <linux/mman.h>
  9. #include <linux/pagemap.h>
  10. #include <linux/syscalls.h>
  11. #include <linux/mempolicy.h>
  12. #include <linux/page-isolation.h>
  13. #include <linux/page_idle.h>
  14. #include <linux/userfaultfd_k.h>
  15. #include <linux/hugetlb.h>
  16. #include <linux/falloc.h>
  17. #include <linux/fadvise.h>
  18. #include <linux/sched.h>
  19. #include <linux/sched/mm.h>
  20. #include <linux/uio.h>
  21. #include <linux/ksm.h>
  22. #include <linux/fs.h>
  23. #include <linux/file.h>
  24. #include <linux/blkdev.h>
  25. #include <linux/backing-dev.h>
  26. #include <linux/pagewalk.h>
  27. #include <linux/swap.h>
  28. #include <linux/swapops.h>
  29. #include <linux/shmem_fs.h>
  30. #include <linux/mmu_notifier.h>
  31. #include <asm/tlb.h>
  32. #include "internal.h"
  33. struct madvise_walk_private {
  34. struct mmu_gather *tlb;
  35. bool pageout;
  36. };
  37. /*
  38. * Any behaviour which results in changes to the vma->vm_flags needs to
  39. * take mmap_lock for writing. Others, which simply traverse vmas, need
  40. * to only take it for reading.
  41. */
  42. static int madvise_need_mmap_write(int behavior)
  43. {
  44. switch (behavior) {
  45. case MADV_REMOVE:
  46. case MADV_WILLNEED:
  47. case MADV_DONTNEED:
  48. case MADV_COLD:
  49. case MADV_PAGEOUT:
  50. case MADV_FREE:
  51. return 0;
  52. default:
  53. /* be safe, default to 1. list exceptions explicitly */
  54. return 1;
  55. }
  56. }
  57. /*
  58. * We can potentially split a vm area into separate
  59. * areas, each area with its own behavior.
  60. */
  61. static long madvise_behavior(struct vm_area_struct *vma,
  62. struct vm_area_struct **prev,
  63. unsigned long start, unsigned long end, int behavior)
  64. {
  65. struct mm_struct *mm = vma->vm_mm;
  66. int error = 0;
  67. pgoff_t pgoff;
  68. unsigned long new_flags = vma->vm_flags;
  69. switch (behavior) {
  70. case MADV_NORMAL:
  71. new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
  72. break;
  73. case MADV_SEQUENTIAL:
  74. new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
  75. break;
  76. case MADV_RANDOM:
  77. new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
  78. break;
  79. case MADV_DONTFORK:
  80. new_flags |= VM_DONTCOPY;
  81. break;
  82. case MADV_DOFORK:
  83. if (vma->vm_flags & VM_IO) {
  84. error = -EINVAL;
  85. goto out;
  86. }
  87. new_flags &= ~VM_DONTCOPY;
  88. break;
  89. case MADV_WIPEONFORK:
  90. /* MADV_WIPEONFORK is only supported on anonymous memory. */
  91. if (vma->vm_file || vma->vm_flags & VM_SHARED) {
  92. error = -EINVAL;
  93. goto out;
  94. }
  95. new_flags |= VM_WIPEONFORK;
  96. break;
  97. case MADV_KEEPONFORK:
  98. new_flags &= ~VM_WIPEONFORK;
  99. break;
  100. case MADV_DONTDUMP:
  101. new_flags |= VM_DONTDUMP;
  102. break;
  103. case MADV_DODUMP:
  104. if (!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL) {
  105. error = -EINVAL;
  106. goto out;
  107. }
  108. new_flags &= ~VM_DONTDUMP;
  109. break;
  110. case MADV_MERGEABLE:
  111. case MADV_UNMERGEABLE:
  112. error = ksm_madvise(vma, start, end, behavior, &new_flags);
  113. if (error)
  114. goto out_convert_errno;
  115. break;
  116. case MADV_HUGEPAGE:
  117. case MADV_NOHUGEPAGE:
  118. error = hugepage_madvise(vma, &new_flags, behavior);
  119. if (error)
  120. goto out_convert_errno;
  121. break;
  122. }
  123. if (new_flags == vma->vm_flags) {
  124. *prev = vma;
  125. goto out;
  126. }
  127. pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
  128. *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
  129. vma->vm_file, pgoff, vma_policy(vma),
  130. vma->vm_userfaultfd_ctx, vma_get_anon_name(vma));
  131. if (*prev) {
  132. vma = *prev;
  133. goto success;
  134. }
  135. *prev = vma;
  136. if (start != vma->vm_start) {
  137. if (unlikely(mm->map_count >= sysctl_max_map_count)) {
  138. error = -ENOMEM;
  139. goto out;
  140. }
  141. error = __split_vma(mm, vma, start, 1);
  142. if (error)
  143. goto out_convert_errno;
  144. }
  145. if (end != vma->vm_end) {
  146. if (unlikely(mm->map_count >= sysctl_max_map_count)) {
  147. error = -ENOMEM;
  148. goto out;
  149. }
  150. error = __split_vma(mm, vma, end, 0);
  151. if (error)
  152. goto out_convert_errno;
  153. }
  154. success:
  155. /*
  156. * vm_flags is protected by the mmap_lock held in write mode.
  157. */
  158. vm_write_begin(vma);
  159. WRITE_ONCE(vma->vm_flags, new_flags);
  160. vm_write_end(vma);
  161. out_convert_errno:
  162. /*
  163. * madvise() returns EAGAIN if kernel resources, such as
  164. * slab, are temporarily unavailable.
  165. */
  166. if (error == -ENOMEM)
  167. error = -EAGAIN;
  168. out:
  169. return error;
  170. }
  171. #ifdef CONFIG_SWAP
  172. static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
  173. unsigned long end, struct mm_walk *walk)
  174. {
  175. pte_t *orig_pte;
  176. struct vm_area_struct *vma = walk->private;
  177. unsigned long index;
  178. if (pmd_none_or_trans_huge_or_clear_bad(pmd))
  179. return 0;
  180. for (index = start; index != end; index += PAGE_SIZE) {
  181. pte_t pte;
  182. swp_entry_t entry;
  183. struct page *page;
  184. spinlock_t *ptl;
  185. orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
  186. pte = *(orig_pte + ((index - start) / PAGE_SIZE));
  187. pte_unmap_unlock(orig_pte, ptl);
  188. if (pte_present(pte) || pte_none(pte))
  189. continue;
  190. entry = pte_to_swp_entry(pte);
  191. if (unlikely(non_swap_entry(entry)))
  192. continue;
  193. page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
  194. vma, index, false);
  195. if (page)
  196. put_page(page);
  197. }
  198. return 0;
  199. }
  200. static const struct mm_walk_ops swapin_walk_ops = {
  201. .pmd_entry = swapin_walk_pmd_entry,
  202. };
  203. static void force_shm_swapin_readahead(struct vm_area_struct *vma,
  204. unsigned long start, unsigned long end,
  205. struct address_space *mapping)
  206. {
  207. XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
  208. pgoff_t end_index = linear_page_index(vma, end + PAGE_SIZE - 1);
  209. struct page *page;
  210. rcu_read_lock();
  211. xas_for_each(&xas, page, end_index) {
  212. swp_entry_t swap;
  213. if (!xa_is_value(page))
  214. continue;
  215. xas_pause(&xas);
  216. rcu_read_unlock();
  217. swap = radix_to_swp_entry(page);
  218. page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
  219. NULL, 0, false);
  220. if (page)
  221. put_page(page);
  222. rcu_read_lock();
  223. }
  224. rcu_read_unlock();
  225. lru_add_drain(); /* Push any new pages onto the LRU now */
  226. }
  227. #endif /* CONFIG_SWAP */
  228. /*
  229. * Schedule all required I/O operations. Do not wait for completion.
  230. */
  231. static long madvise_willneed(struct vm_area_struct *vma,
  232. struct vm_area_struct **prev,
  233. unsigned long start, unsigned long end)
  234. {
  235. struct mm_struct *mm = vma->vm_mm;
  236. struct file *file = vma->vm_file;
  237. loff_t offset;
  238. *prev = vma;
  239. #ifdef CONFIG_SWAP
  240. if (!file) {
  241. walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
  242. lru_add_drain(); /* Push any new pages onto the LRU now */
  243. return 0;
  244. }
  245. if (shmem_mapping(file->f_mapping)) {
  246. force_shm_swapin_readahead(vma, start, end,
  247. file->f_mapping);
  248. return 0;
  249. }
  250. #else
  251. if (!file)
  252. return -EBADF;
  253. #endif
  254. if (IS_DAX(file_inode(file))) {
  255. /* no bad return value, but ignore advice */
  256. return 0;
  257. }
  258. /*
  259. * Filesystem's fadvise may need to take various locks. We need to
  260. * explicitly grab a reference because the vma (and hence the
  261. * vma's reference to the file) can go away as soon as we drop
  262. * mmap_lock.
  263. */
  264. *prev = NULL; /* tell sys_madvise we drop mmap_lock */
  265. get_file(file);
  266. offset = (loff_t)(start - vma->vm_start)
  267. + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
  268. mmap_read_unlock(mm);
  269. vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
  270. fput(file);
  271. mmap_read_lock(mm);
  272. return 0;
  273. }
  274. static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
  275. unsigned long addr, unsigned long end,
  276. struct mm_walk *walk)
  277. {
  278. struct madvise_walk_private *private = walk->private;
  279. struct mmu_gather *tlb = private->tlb;
  280. bool pageout = private->pageout;
  281. struct mm_struct *mm = tlb->mm;
  282. struct vm_area_struct *vma = walk->vma;
  283. pte_t *orig_pte, *pte, ptent;
  284. spinlock_t *ptl;
  285. struct page *page = NULL;
  286. LIST_HEAD(page_list);
  287. if (fatal_signal_pending(current))
  288. return -EINTR;
  289. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  290. if (pmd_trans_huge(*pmd)) {
  291. pmd_t orig_pmd;
  292. unsigned long next = pmd_addr_end(addr, end);
  293. tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
  294. ptl = pmd_trans_huge_lock(pmd, vma);
  295. if (!ptl)
  296. return 0;
  297. orig_pmd = *pmd;
  298. if (is_huge_zero_pmd(orig_pmd))
  299. goto huge_unlock;
  300. if (unlikely(!pmd_present(orig_pmd))) {
  301. VM_BUG_ON(thp_migration_supported() &&
  302. !is_pmd_migration_entry(orig_pmd));
  303. goto huge_unlock;
  304. }
  305. page = pmd_page(orig_pmd);
  306. /* Do not interfere with other mappings of this page */
  307. if (page_mapcount(page) != 1)
  308. goto huge_unlock;
  309. if (next - addr != HPAGE_PMD_SIZE) {
  310. int err;
  311. get_page(page);
  312. spin_unlock(ptl);
  313. lock_page(page);
  314. err = split_huge_page(page);
  315. unlock_page(page);
  316. put_page(page);
  317. if (!err)
  318. goto regular_page;
  319. return 0;
  320. }
  321. if (pmd_young(orig_pmd)) {
  322. pmdp_invalidate(vma, addr, pmd);
  323. orig_pmd = pmd_mkold(orig_pmd);
  324. set_pmd_at(mm, addr, pmd, orig_pmd);
  325. tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
  326. }
  327. ClearPageReferenced(page);
  328. test_and_clear_page_young(page);
  329. if (pageout) {
  330. if (!isolate_lru_page(page)) {
  331. if (PageUnevictable(page))
  332. putback_lru_page(page);
  333. else
  334. list_add(&page->lru, &page_list);
  335. }
  336. } else
  337. deactivate_page(page);
  338. huge_unlock:
  339. spin_unlock(ptl);
  340. if (pageout)
  341. reclaim_pages(&page_list);
  342. return 0;
  343. }
  344. regular_page:
  345. if (pmd_trans_unstable(pmd))
  346. return 0;
  347. #endif
  348. tlb_change_page_size(tlb, PAGE_SIZE);
  349. orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  350. flush_tlb_batched_pending(mm);
  351. arch_enter_lazy_mmu_mode();
  352. for (; addr < end; pte++, addr += PAGE_SIZE) {
  353. ptent = *pte;
  354. if (pte_none(ptent))
  355. continue;
  356. if (!pte_present(ptent))
  357. continue;
  358. page = vm_normal_page(vma, addr, ptent);
  359. if (!page)
  360. continue;
  361. /*
  362. * Creating a THP page is expensive so split it only if we
  363. * are sure it's worth. Split it if we are only owner.
  364. */
  365. if (PageTransCompound(page)) {
  366. if (page_mapcount(page) != 1)
  367. break;
  368. get_page(page);
  369. if (!trylock_page(page)) {
  370. put_page(page);
  371. break;
  372. }
  373. pte_unmap_unlock(orig_pte, ptl);
  374. if (split_huge_page(page)) {
  375. unlock_page(page);
  376. put_page(page);
  377. pte_offset_map_lock(mm, pmd, addr, &ptl);
  378. break;
  379. }
  380. unlock_page(page);
  381. put_page(page);
  382. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  383. pte--;
  384. addr -= PAGE_SIZE;
  385. continue;
  386. }
  387. /* Do not interfere with other mappings of this page */
  388. if (page_mapcount(page) != 1)
  389. continue;
  390. VM_BUG_ON_PAGE(PageTransCompound(page), page);
  391. if (pte_young(ptent)) {
  392. ptent = ptep_get_and_clear_full(mm, addr, pte,
  393. tlb->fullmm);
  394. ptent = pte_mkold(ptent);
  395. set_pte_at(mm, addr, pte, ptent);
  396. tlb_remove_tlb_entry(tlb, pte, addr);
  397. }
  398. /*
  399. * We are deactivating a page for accelerating reclaiming.
  400. * VM couldn't reclaim the page unless we clear PG_young.
  401. * As a side effect, it makes confuse idle-page tracking
  402. * because they will miss recent referenced history.
  403. */
  404. ClearPageReferenced(page);
  405. test_and_clear_page_young(page);
  406. if (pageout) {
  407. if (!isolate_lru_page(page)) {
  408. if (PageUnevictable(page))
  409. putback_lru_page(page);
  410. else
  411. list_add(&page->lru, &page_list);
  412. }
  413. } else
  414. deactivate_page(page);
  415. }
  416. arch_leave_lazy_mmu_mode();
  417. pte_unmap_unlock(orig_pte, ptl);
  418. if (pageout)
  419. reclaim_pages(&page_list);
  420. cond_resched();
  421. return 0;
  422. }
  423. static const struct mm_walk_ops cold_walk_ops = {
  424. .pmd_entry = madvise_cold_or_pageout_pte_range,
  425. };
  426. static void madvise_cold_page_range(struct mmu_gather *tlb,
  427. struct vm_area_struct *vma,
  428. unsigned long addr, unsigned long end)
  429. {
  430. struct madvise_walk_private walk_private = {
  431. .pageout = false,
  432. .tlb = tlb,
  433. };
  434. vm_write_begin(vma);
  435. tlb_start_vma(tlb, vma);
  436. walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
  437. tlb_end_vma(tlb, vma);
  438. vm_write_end(vma);
  439. }
  440. static long madvise_cold(struct vm_area_struct *vma,
  441. struct vm_area_struct **prev,
  442. unsigned long start_addr, unsigned long end_addr)
  443. {
  444. struct mm_struct *mm = vma->vm_mm;
  445. struct mmu_gather tlb;
  446. *prev = vma;
  447. if (!can_madv_lru_vma(vma))
  448. return -EINVAL;
  449. lru_add_drain();
  450. tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
  451. madvise_cold_page_range(&tlb, vma, start_addr, end_addr);
  452. tlb_finish_mmu(&tlb, start_addr, end_addr);
  453. return 0;
  454. }
  455. static void madvise_pageout_page_range(struct mmu_gather *tlb,
  456. struct vm_area_struct *vma,
  457. unsigned long addr, unsigned long end)
  458. {
  459. struct madvise_walk_private walk_private = {
  460. .pageout = true,
  461. .tlb = tlb,
  462. };
  463. vm_write_begin(vma);
  464. tlb_start_vma(tlb, vma);
  465. walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
  466. tlb_end_vma(tlb, vma);
  467. vm_write_end(vma);
  468. }
  469. static inline bool can_do_pageout(struct vm_area_struct *vma)
  470. {
  471. if (vma_is_anonymous(vma))
  472. return true;
  473. if (!vma->vm_file)
  474. return false;
  475. /*
  476. * paging out pagecache only for non-anonymous mappings that correspond
  477. * to the files the calling process could (if tried) open for writing;
  478. * otherwise we'd be including shared non-exclusive mappings, which
  479. * opens a side channel.
  480. */
  481. return inode_owner_or_capable(file_inode(vma->vm_file)) ||
  482. inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0;
  483. }
  484. static long madvise_pageout(struct vm_area_struct *vma,
  485. struct vm_area_struct **prev,
  486. unsigned long start_addr, unsigned long end_addr)
  487. {
  488. struct mm_struct *mm = vma->vm_mm;
  489. struct mmu_gather tlb;
  490. *prev = vma;
  491. if (!can_madv_lru_vma(vma))
  492. return -EINVAL;
  493. if (!can_do_pageout(vma))
  494. return 0;
  495. lru_add_drain();
  496. tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
  497. madvise_pageout_page_range(&tlb, vma, start_addr, end_addr);
  498. tlb_finish_mmu(&tlb, start_addr, end_addr);
  499. return 0;
  500. }
  501. static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
  502. unsigned long end, struct mm_walk *walk)
  503. {
  504. struct mmu_gather *tlb = walk->private;
  505. struct mm_struct *mm = tlb->mm;
  506. struct vm_area_struct *vma = walk->vma;
  507. spinlock_t *ptl;
  508. pte_t *orig_pte, *pte, ptent;
  509. struct page *page;
  510. int nr_swap = 0;
  511. unsigned long next;
  512. next = pmd_addr_end(addr, end);
  513. if (pmd_trans_huge(*pmd))
  514. if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
  515. goto next;
  516. if (pmd_trans_unstable(pmd))
  517. return 0;
  518. tlb_change_page_size(tlb, PAGE_SIZE);
  519. orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  520. flush_tlb_batched_pending(mm);
  521. arch_enter_lazy_mmu_mode();
  522. for (; addr != end; pte++, addr += PAGE_SIZE) {
  523. ptent = *pte;
  524. if (pte_none(ptent))
  525. continue;
  526. /*
  527. * If the pte has swp_entry, just clear page table to
  528. * prevent swap-in which is more expensive rather than
  529. * (page allocation + zeroing).
  530. */
  531. if (!pte_present(ptent)) {
  532. swp_entry_t entry;
  533. entry = pte_to_swp_entry(ptent);
  534. if (non_swap_entry(entry))
  535. continue;
  536. nr_swap--;
  537. free_swap_and_cache(entry);
  538. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  539. continue;
  540. }
  541. page = vm_normal_page(vma, addr, ptent);
  542. if (!page)
  543. continue;
  544. /*
  545. * If pmd isn't transhuge but the page is THP and
  546. * is owned by only this process, split it and
  547. * deactivate all pages.
  548. */
  549. if (PageTransCompound(page)) {
  550. if (page_mapcount(page) != 1)
  551. goto out;
  552. get_page(page);
  553. if (!trylock_page(page)) {
  554. put_page(page);
  555. goto out;
  556. }
  557. pte_unmap_unlock(orig_pte, ptl);
  558. if (split_huge_page(page)) {
  559. unlock_page(page);
  560. put_page(page);
  561. pte_offset_map_lock(mm, pmd, addr, &ptl);
  562. goto out;
  563. }
  564. unlock_page(page);
  565. put_page(page);
  566. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  567. pte--;
  568. addr -= PAGE_SIZE;
  569. continue;
  570. }
  571. VM_BUG_ON_PAGE(PageTransCompound(page), page);
  572. if (PageSwapCache(page) || PageDirty(page)) {
  573. if (!trylock_page(page))
  574. continue;
  575. /*
  576. * If page is shared with others, we couldn't clear
  577. * PG_dirty of the page.
  578. */
  579. if (page_mapcount(page) != 1) {
  580. unlock_page(page);
  581. continue;
  582. }
  583. if (PageSwapCache(page) && !try_to_free_swap(page)) {
  584. unlock_page(page);
  585. continue;
  586. }
  587. ClearPageDirty(page);
  588. unlock_page(page);
  589. }
  590. if (pte_young(ptent) || pte_dirty(ptent)) {
  591. /*
  592. * Some of architecture(ex, PPC) don't update TLB
  593. * with set_pte_at and tlb_remove_tlb_entry so for
  594. * the portability, remap the pte with old|clean
  595. * after pte clearing.
  596. */
  597. ptent = ptep_get_and_clear_full(mm, addr, pte,
  598. tlb->fullmm);
  599. ptent = pte_mkold(ptent);
  600. ptent = pte_mkclean(ptent);
  601. set_pte_at(mm, addr, pte, ptent);
  602. tlb_remove_tlb_entry(tlb, pte, addr);
  603. }
  604. mark_page_lazyfree(page);
  605. }
  606. out:
  607. if (nr_swap) {
  608. if (current->mm == mm)
  609. sync_mm_rss(mm);
  610. add_mm_counter(mm, MM_SWAPENTS, nr_swap);
  611. }
  612. arch_leave_lazy_mmu_mode();
  613. pte_unmap_unlock(orig_pte, ptl);
  614. cond_resched();
  615. next:
  616. return 0;
  617. }
  618. static const struct mm_walk_ops madvise_free_walk_ops = {
  619. .pmd_entry = madvise_free_pte_range,
  620. };
  621. static int madvise_free_single_vma(struct vm_area_struct *vma,
  622. unsigned long start_addr, unsigned long end_addr)
  623. {
  624. struct mm_struct *mm = vma->vm_mm;
  625. struct mmu_notifier_range range;
  626. struct mmu_gather tlb;
  627. /* MADV_FREE works for only anon vma at the moment */
  628. if (!vma_is_anonymous(vma))
  629. return -EINVAL;
  630. range.start = max(vma->vm_start, start_addr);
  631. if (range.start >= vma->vm_end)
  632. return -EINVAL;
  633. range.end = min(vma->vm_end, end_addr);
  634. if (range.end <= vma->vm_start)
  635. return -EINVAL;
  636. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
  637. range.start, range.end);
  638. lru_add_drain();
  639. tlb_gather_mmu(&tlb, mm, range.start, range.end);
  640. update_hiwater_rss(mm);
  641. mmu_notifier_invalidate_range_start(&range);
  642. vm_write_begin(vma);
  643. tlb_start_vma(&tlb, vma);
  644. walk_page_range(vma->vm_mm, range.start, range.end,
  645. &madvise_free_walk_ops, &tlb);
  646. tlb_end_vma(&tlb, vma);
  647. vm_write_end(vma);
  648. mmu_notifier_invalidate_range_end(&range);
  649. tlb_finish_mmu(&tlb, range.start, range.end);
  650. return 0;
  651. }
  652. /*
  653. * Application no longer needs these pages. If the pages are dirty,
  654. * it's OK to just throw them away. The app will be more careful about
  655. * data it wants to keep. Be sure to free swap resources too. The
  656. * zap_page_range call sets things up for shrink_active_list to actually free
  657. * these pages later if no one else has touched them in the meantime,
  658. * although we could add these pages to a global reuse list for
  659. * shrink_active_list to pick up before reclaiming other pages.
  660. *
  661. * NB: This interface discards data rather than pushes it out to swap,
  662. * as some implementations do. This has performance implications for
  663. * applications like large transactional databases which want to discard
  664. * pages in anonymous maps after committing to backing store the data
  665. * that was kept in them. There is no reason to write this data out to
  666. * the swap area if the application is discarding it.
  667. *
  668. * An interface that causes the system to free clean pages and flush
  669. * dirty pages is already available as msync(MS_INVALIDATE).
  670. */
  671. static long madvise_dontneed_single_vma(struct vm_area_struct *vma,
  672. unsigned long start, unsigned long end)
  673. {
  674. zap_page_range(vma, start, end - start);
  675. return 0;
  676. }
  677. static long madvise_dontneed_free(struct vm_area_struct *vma,
  678. struct vm_area_struct **prev,
  679. unsigned long start, unsigned long end,
  680. int behavior)
  681. {
  682. struct mm_struct *mm = vma->vm_mm;
  683. *prev = vma;
  684. if (!can_madv_lru_vma(vma))
  685. return -EINVAL;
  686. if (!userfaultfd_remove(vma, start, end)) {
  687. *prev = NULL; /* mmap_lock has been dropped, prev is stale */
  688. mmap_read_lock(mm);
  689. vma = find_vma(mm, start);
  690. if (!vma)
  691. return -ENOMEM;
  692. if (start < vma->vm_start) {
  693. /*
  694. * This "vma" under revalidation is the one
  695. * with the lowest vma->vm_start where start
  696. * is also < vma->vm_end. If start <
  697. * vma->vm_start it means an hole materialized
  698. * in the user address space within the
  699. * virtual range passed to MADV_DONTNEED
  700. * or MADV_FREE.
  701. */
  702. return -ENOMEM;
  703. }
  704. if (!can_madv_lru_vma(vma))
  705. return -EINVAL;
  706. if (end > vma->vm_end) {
  707. /*
  708. * Don't fail if end > vma->vm_end. If the old
  709. * vma was splitted while the mmap_lock was
  710. * released the effect of the concurrent
  711. * operation may not cause madvise() to
  712. * have an undefined result. There may be an
  713. * adjacent next vma that we'll walk
  714. * next. userfaultfd_remove() will generate an
  715. * UFFD_EVENT_REMOVE repetition on the
  716. * end-vma->vm_end range, but the manager can
  717. * handle a repetition fine.
  718. */
  719. end = vma->vm_end;
  720. }
  721. VM_WARN_ON(start >= end);
  722. }
  723. if (behavior == MADV_DONTNEED)
  724. return madvise_dontneed_single_vma(vma, start, end);
  725. else if (behavior == MADV_FREE)
  726. return madvise_free_single_vma(vma, start, end);
  727. else
  728. return -EINVAL;
  729. }
  730. /*
  731. * Application wants to free up the pages and associated backing store.
  732. * This is effectively punching a hole into the middle of a file.
  733. */
  734. static long madvise_remove(struct vm_area_struct *vma,
  735. struct vm_area_struct **prev,
  736. unsigned long start, unsigned long end)
  737. {
  738. loff_t offset;
  739. int error;
  740. struct file *f;
  741. struct mm_struct *mm = vma->vm_mm;
  742. *prev = NULL; /* tell sys_madvise we drop mmap_lock */
  743. if (vma->vm_flags & VM_LOCKED)
  744. return -EINVAL;
  745. f = vma->vm_file;
  746. if (!f || !f->f_mapping || !f->f_mapping->host) {
  747. return -EINVAL;
  748. }
  749. if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
  750. return -EACCES;
  751. offset = (loff_t)(start - vma->vm_start)
  752. + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
  753. /*
  754. * Filesystem's fallocate may need to take i_mutex. We need to
  755. * explicitly grab a reference because the vma (and hence the
  756. * vma's reference to the file) can go away as soon as we drop
  757. * mmap_lock.
  758. */
  759. get_file(f);
  760. if (userfaultfd_remove(vma, start, end)) {
  761. /* mmap_lock was not released by userfaultfd_remove() */
  762. mmap_read_unlock(mm);
  763. }
  764. error = vfs_fallocate(f,
  765. FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
  766. offset, end - start);
  767. fput(f);
  768. mmap_read_lock(mm);
  769. return error;
  770. }
  771. #ifdef CONFIG_MEMORY_FAILURE
  772. /*
  773. * Error injection support for memory error handling.
  774. */
  775. static int madvise_inject_error(int behavior,
  776. unsigned long start, unsigned long end)
  777. {
  778. struct zone *zone;
  779. unsigned long size;
  780. if (!capable(CAP_SYS_ADMIN))
  781. return -EPERM;
  782. for (; start < end; start += size) {
  783. unsigned long pfn;
  784. struct page *page;
  785. int ret;
  786. ret = get_user_pages_fast(start, 1, 0, &page);
  787. if (ret != 1)
  788. return ret;
  789. pfn = page_to_pfn(page);
  790. /*
  791. * When soft offlining hugepages, after migrating the page
  792. * we dissolve it, therefore in the second loop "page" will
  793. * no longer be a compound page.
  794. */
  795. size = page_size(compound_head(page));
  796. if (behavior == MADV_SOFT_OFFLINE) {
  797. pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
  798. pfn, start);
  799. ret = soft_offline_page(pfn, MF_COUNT_INCREASED);
  800. } else {
  801. pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n",
  802. pfn, start);
  803. ret = memory_failure(pfn, MF_COUNT_INCREASED);
  804. }
  805. if (ret)
  806. return ret;
  807. }
  808. /* Ensure that all poisoned pages are removed from per-cpu lists */
  809. for_each_populated_zone(zone)
  810. drain_all_pages(zone);
  811. return 0;
  812. }
  813. #endif
  814. static long
  815. madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
  816. unsigned long start, unsigned long end, int behavior)
  817. {
  818. switch (behavior) {
  819. case MADV_REMOVE:
  820. return madvise_remove(vma, prev, start, end);
  821. case MADV_WILLNEED:
  822. return madvise_willneed(vma, prev, start, end);
  823. case MADV_COLD:
  824. return madvise_cold(vma, prev, start, end);
  825. case MADV_PAGEOUT:
  826. return madvise_pageout(vma, prev, start, end);
  827. case MADV_FREE:
  828. case MADV_DONTNEED:
  829. return madvise_dontneed_free(vma, prev, start, end, behavior);
  830. default:
  831. return madvise_behavior(vma, prev, start, end, behavior);
  832. }
  833. }
  834. static bool
  835. madvise_behavior_valid(int behavior)
  836. {
  837. switch (behavior) {
  838. case MADV_DOFORK:
  839. case MADV_DONTFORK:
  840. case MADV_NORMAL:
  841. case MADV_SEQUENTIAL:
  842. case MADV_RANDOM:
  843. case MADV_REMOVE:
  844. case MADV_WILLNEED:
  845. case MADV_DONTNEED:
  846. case MADV_FREE:
  847. case MADV_COLD:
  848. case MADV_PAGEOUT:
  849. #ifdef CONFIG_KSM
  850. case MADV_MERGEABLE:
  851. case MADV_UNMERGEABLE:
  852. #endif
  853. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  854. case MADV_HUGEPAGE:
  855. case MADV_NOHUGEPAGE:
  856. #endif
  857. case MADV_DONTDUMP:
  858. case MADV_DODUMP:
  859. case MADV_WIPEONFORK:
  860. case MADV_KEEPONFORK:
  861. #ifdef CONFIG_MEMORY_FAILURE
  862. case MADV_SOFT_OFFLINE:
  863. case MADV_HWPOISON:
  864. #endif
  865. return true;
  866. default:
  867. return false;
  868. }
  869. }
  870. static bool
  871. process_madvise_behavior_valid(int behavior)
  872. {
  873. switch (behavior) {
  874. case MADV_COLD:
  875. case MADV_PAGEOUT:
  876. case MADV_WILLNEED:
  877. return true;
  878. default:
  879. return false;
  880. }
  881. }
  882. /*
  883. * The madvise(2) system call.
  884. *
  885. * Applications can use madvise() to advise the kernel how it should
  886. * handle paging I/O in this VM area. The idea is to help the kernel
  887. * use appropriate read-ahead and caching techniques. The information
  888. * provided is advisory only, and can be safely disregarded by the
  889. * kernel without affecting the correct operation of the application.
  890. *
  891. * behavior values:
  892. * MADV_NORMAL - the default behavior is to read clusters. This
  893. * results in some read-ahead and read-behind.
  894. * MADV_RANDOM - the system should read the minimum amount of data
  895. * on any access, since it is unlikely that the appli-
  896. * cation will need more than what it asks for.
  897. * MADV_SEQUENTIAL - pages in the given range will probably be accessed
  898. * once, so they can be aggressively read ahead, and
  899. * can be freed soon after they are accessed.
  900. * MADV_WILLNEED - the application is notifying the system to read
  901. * some pages ahead.
  902. * MADV_DONTNEED - the application is finished with the given range,
  903. * so the kernel can free resources associated with it.
  904. * MADV_FREE - the application marks pages in the given range as lazy free,
  905. * where actual purges are postponed until memory pressure happens.
  906. * MADV_REMOVE - the application wants to free up the given range of
  907. * pages and associated backing store.
  908. * MADV_DONTFORK - omit this area from child's address space when forking:
  909. * typically, to avoid COWing pages pinned by get_user_pages().
  910. * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
  911. * MADV_WIPEONFORK - present the child process with zero-filled memory in this
  912. * range after a fork.
  913. * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
  914. * MADV_HWPOISON - trigger memory error handler as if the given memory range
  915. * were corrupted by unrecoverable hardware memory failure.
  916. * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
  917. * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
  918. * this area with pages of identical content from other such areas.
  919. * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
  920. * MADV_HUGEPAGE - the application wants to back the given range by transparent
  921. * huge pages in the future. Existing pages might be coalesced and
  922. * new pages might be allocated as THP.
  923. * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
  924. * transparent huge pages so the existing pages will not be
  925. * coalesced into THP and new pages will not be allocated as THP.
  926. * MADV_DONTDUMP - the application wants to prevent pages in the given range
  927. * from being included in its core dump.
  928. * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
  929. * MADV_COLD - the application is not expected to use this memory soon,
  930. * deactivate pages in this range so that they can be reclaimed
  931. * easily if memory pressure hanppens.
  932. * MADV_PAGEOUT - the application is not expected to use this memory soon,
  933. * page out the pages in this range immediately.
  934. *
  935. * return values:
  936. * zero - success
  937. * -EINVAL - start + len < 0, start is not page-aligned,
  938. * "behavior" is not a valid value, or application
  939. * is attempting to release locked or shared pages,
  940. * or the specified address range includes file, Huge TLB,
  941. * MAP_SHARED or VMPFNMAP range.
  942. * -ENOMEM - addresses in the specified range are not currently
  943. * mapped, or are outside the AS of the process.
  944. * -EIO - an I/O error occurred while paging in data.
  945. * -EBADF - map exists, but area maps something that isn't a file.
  946. * -EAGAIN - a kernel resource was temporarily unavailable.
  947. */
  948. int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior)
  949. {
  950. unsigned long end, tmp;
  951. struct vm_area_struct *vma, *prev;
  952. int unmapped_error = 0;
  953. int error = -EINVAL;
  954. int write;
  955. size_t len;
  956. struct blk_plug plug;
  957. start = untagged_addr(start);
  958. if (!madvise_behavior_valid(behavior))
  959. return error;
  960. if (!PAGE_ALIGNED(start))
  961. return error;
  962. len = PAGE_ALIGN(len_in);
  963. /* Check to see whether len was rounded up from small -ve to zero */
  964. if (len_in && !len)
  965. return error;
  966. end = start + len;
  967. if (end < start)
  968. return error;
  969. error = 0;
  970. if (end == start)
  971. return error;
  972. #ifdef CONFIG_MEMORY_FAILURE
  973. if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
  974. return madvise_inject_error(behavior, start, start + len_in);
  975. #endif
  976. write = madvise_need_mmap_write(behavior);
  977. if (write) {
  978. if (mmap_write_lock_killable(mm))
  979. return -EINTR;
  980. } else {
  981. mmap_read_lock(mm);
  982. }
  983. /*
  984. * If the interval [start,end) covers some unmapped address
  985. * ranges, just ignore them, but return -ENOMEM at the end.
  986. * - different from the way of handling in mlock etc.
  987. */
  988. vma = find_vma_prev(mm, start, &prev);
  989. if (vma && start > vma->vm_start)
  990. prev = vma;
  991. blk_start_plug(&plug);
  992. for (;;) {
  993. /* Still start < end. */
  994. error = -ENOMEM;
  995. if (!vma)
  996. goto out;
  997. /* Here start < (end|vma->vm_end). */
  998. if (start < vma->vm_start) {
  999. unmapped_error = -ENOMEM;
  1000. start = vma->vm_start;
  1001. if (start >= end)
  1002. goto out;
  1003. }
  1004. /* Here vma->vm_start <= start < (end|vma->vm_end) */
  1005. tmp = vma->vm_end;
  1006. if (end < tmp)
  1007. tmp = end;
  1008. /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
  1009. error = madvise_vma(vma, &prev, start, tmp, behavior);
  1010. if (error)
  1011. goto out;
  1012. start = tmp;
  1013. if (prev && start < prev->vm_end)
  1014. start = prev->vm_end;
  1015. error = unmapped_error;
  1016. if (start >= end)
  1017. goto out;
  1018. if (prev)
  1019. vma = prev->vm_next;
  1020. else /* madvise_remove dropped mmap_lock */
  1021. vma = find_vma(mm, start);
  1022. }
  1023. out:
  1024. blk_finish_plug(&plug);
  1025. if (write)
  1026. mmap_write_unlock(mm);
  1027. else
  1028. mmap_read_unlock(mm);
  1029. return error;
  1030. }
  1031. SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
  1032. {
  1033. return do_madvise(current->mm, start, len_in, behavior);
  1034. }
  1035. SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
  1036. size_t, vlen, int, behavior, unsigned int, flags)
  1037. {
  1038. ssize_t ret;
  1039. struct iovec iovstack[UIO_FASTIOV], iovec;
  1040. struct iovec *iov = iovstack;
  1041. struct iov_iter iter;
  1042. struct pid *pid;
  1043. struct task_struct *task;
  1044. struct mm_struct *mm;
  1045. size_t total_len;
  1046. unsigned int f_flags;
  1047. if (flags != 0) {
  1048. ret = -EINVAL;
  1049. goto out;
  1050. }
  1051. ret = import_iovec(READ, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
  1052. if (ret < 0)
  1053. goto out;
  1054. pid = pidfd_get_pid(pidfd, &f_flags);
  1055. if (IS_ERR(pid)) {
  1056. ret = PTR_ERR(pid);
  1057. goto free_iov;
  1058. }
  1059. task = get_pid_task(pid, PIDTYPE_PID);
  1060. if (!task) {
  1061. ret = -ESRCH;
  1062. goto put_pid;
  1063. }
  1064. if (!process_madvise_behavior_valid(behavior)) {
  1065. ret = -EINVAL;
  1066. goto release_task;
  1067. }
  1068. /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */
  1069. mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
  1070. if (IS_ERR_OR_NULL(mm)) {
  1071. ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
  1072. goto release_task;
  1073. }
  1074. /*
  1075. * Require CAP_SYS_NICE for influencing process performance. Note that
  1076. * only non-destructive hints are currently supported.
  1077. */
  1078. if (!capable(CAP_SYS_NICE)) {
  1079. ret = -EPERM;
  1080. goto release_mm;
  1081. }
  1082. total_len = iov_iter_count(&iter);
  1083. while (iov_iter_count(&iter)) {
  1084. iovec = iov_iter_iovec(&iter);
  1085. ret = do_madvise(mm, (unsigned long)iovec.iov_base,
  1086. iovec.iov_len, behavior);
  1087. if (ret < 0)
  1088. break;
  1089. iov_iter_advance(&iter, iovec.iov_len);
  1090. }
  1091. ret = (total_len - iov_iter_count(&iter)) ? : ret;
  1092. release_mm:
  1093. mmput(mm);
  1094. release_task:
  1095. put_task_struct(task);
  1096. put_pid:
  1097. put_pid(pid);
  1098. free_iov:
  1099. kfree(iov);
  1100. out:
  1101. return ret;
  1102. }