rmap.c 56 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018
  1. /*
  2. * mm/rmap.c - physical to virtual reverse mappings
  3. *
  4. * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
  5. * Released under the General Public License (GPL).
  6. *
  7. * Simple, low overhead reverse mapping scheme.
  8. * Please try to keep this thing as modular as possible.
  9. *
  10. * Provides methods for unmapping each kind of mapped page:
  11. * the anon methods track anonymous pages, and
  12. * the file methods track pages belonging to an inode.
  13. *
  14. * Original design by Rik van Riel <riel@conectiva.com.br> 2001
  15. * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
  16. * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
  17. * Contributions by Hugh Dickins 2003, 2004
  18. */
  19. /*
  20. * Lock ordering in mm:
  21. *
  22. * inode->i_mutex (while writing or truncating, not reading or faulting)
  23. * mm->mmap_lock
  24. * page->flags PG_locked (lock_page) * (see huegtlbfs below)
  25. * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
  26. * mapping->i_mmap_rwsem
  27. * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
  28. * anon_vma->rwsem
  29. * mm->page_table_lock or pte_lock
  30. * pgdat->lru_lock (in mark_page_accessed, isolate_lru_page)
  31. * swap_lock (in swap_duplicate, swap_info_get)
  32. * mmlist_lock (in mmput, drain_mmlist and others)
  33. * mapping->private_lock (in __set_page_dirty_buffers)
  34. * mem_cgroup_{begin,end}_page_stat (memcg->move_lock)
  35. * i_pages lock (widely used)
  36. * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
  37. * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
  38. * sb_lock (within inode_lock in fs/fs-writeback.c)
  39. * i_pages lock (widely used, in set_page_dirty,
  40. * in arch-dependent flush_dcache_mmap_lock,
  41. * within bdi.wb->list_lock in __sync_single_inode)
  42. *
  43. * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon)
  44. * ->tasklist_lock
  45. * pte map lock
  46. *
  47. * * hugetlbfs PageHuge() pages take locks in this order:
  48. * mapping->i_mmap_rwsem
  49. * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
  50. * page->flags PG_locked (lock_page)
  51. */
  52. #include <linux/mm.h>
  53. #include <linux/sched/mm.h>
  54. #include <linux/sched/task.h>
  55. #include <linux/pagemap.h>
  56. #include <linux/swap.h>
  57. #include <linux/swapops.h>
  58. #include <linux/slab.h>
  59. #include <linux/init.h>
  60. #include <linux/ksm.h>
  61. #include <linux/rmap.h>
  62. #include <linux/rcupdate.h>
  63. #include <linux/export.h>
  64. #include <linux/memcontrol.h>
  65. #include <linux/mmu_notifier.h>
  66. #include <linux/migrate.h>
  67. #include <linux/hugetlb.h>
  68. #include <linux/huge_mm.h>
  69. #include <linux/backing-dev.h>
  70. #include <linux/page_idle.h>
  71. #include <linux/memremap.h>
  72. #include <linux/userfaultfd_k.h>
  73. #include <asm/tlbflush.h>
  74. #include <trace/events/tlb.h>
  75. #include <trace/hooks/mm.h>
  76. #include "internal.h"
  77. static struct kmem_cache *anon_vma_cachep;
  78. static struct kmem_cache *anon_vma_chain_cachep;
  79. static inline struct anon_vma *anon_vma_alloc(void)
  80. {
  81. struct anon_vma *anon_vma;
  82. anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
  83. if (anon_vma) {
  84. atomic_set(&anon_vma->refcount, 1);
  85. anon_vma->degree = 1; /* Reference for first vma */
  86. anon_vma->parent = anon_vma;
  87. /*
  88. * Initialise the anon_vma root to point to itself. If called
  89. * from fork, the root will be reset to the parents anon_vma.
  90. */
  91. anon_vma->root = anon_vma;
  92. }
  93. return anon_vma;
  94. }
  95. static inline void anon_vma_free(struct anon_vma *anon_vma)
  96. {
  97. VM_BUG_ON(atomic_read(&anon_vma->refcount));
  98. /*
  99. * Synchronize against page_lock_anon_vma_read() such that
  100. * we can safely hold the lock without the anon_vma getting
  101. * freed.
  102. *
  103. * Relies on the full mb implied by the atomic_dec_and_test() from
  104. * put_anon_vma() against the acquire barrier implied by
  105. * down_read_trylock() from page_lock_anon_vma_read(). This orders:
  106. *
  107. * page_lock_anon_vma_read() VS put_anon_vma()
  108. * down_read_trylock() atomic_dec_and_test()
  109. * LOCK MB
  110. * atomic_read() rwsem_is_locked()
  111. *
  112. * LOCK should suffice since the actual taking of the lock must
  113. * happen _before_ what follows.
  114. */
  115. might_sleep();
  116. if (rwsem_is_locked(&anon_vma->root->rwsem)) {
  117. anon_vma_lock_write(anon_vma);
  118. anon_vma_unlock_write(anon_vma);
  119. }
  120. kmem_cache_free(anon_vma_cachep, anon_vma);
  121. }
  122. static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
  123. {
  124. return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
  125. }
  126. static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
  127. {
  128. kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
  129. }
  130. static void anon_vma_chain_link(struct vm_area_struct *vma,
  131. struct anon_vma_chain *avc,
  132. struct anon_vma *anon_vma)
  133. {
  134. avc->vma = vma;
  135. avc->anon_vma = anon_vma;
  136. list_add(&avc->same_vma, &vma->anon_vma_chain);
  137. anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
  138. }
  139. /**
  140. * __anon_vma_prepare - attach an anon_vma to a memory region
  141. * @vma: the memory region in question
  142. *
  143. * This makes sure the memory mapping described by 'vma' has
  144. * an 'anon_vma' attached to it, so that we can associate the
  145. * anonymous pages mapped into it with that anon_vma.
  146. *
  147. * The common case will be that we already have one, which
  148. * is handled inline by anon_vma_prepare(). But if
  149. * not we either need to find an adjacent mapping that we
  150. * can re-use the anon_vma from (very common when the only
  151. * reason for splitting a vma has been mprotect()), or we
  152. * allocate a new one.
  153. *
  154. * Anon-vma allocations are very subtle, because we may have
  155. * optimistically looked up an anon_vma in page_lock_anon_vma_read()
  156. * and that may actually touch the spinlock even in the newly
  157. * allocated vma (it depends on RCU to make sure that the
  158. * anon_vma isn't actually destroyed).
  159. *
  160. * As a result, we need to do proper anon_vma locking even
  161. * for the new allocation. At the same time, we do not want
  162. * to do any locking for the common case of already having
  163. * an anon_vma.
  164. *
  165. * This must be called with the mmap_lock held for reading.
  166. */
  167. int __anon_vma_prepare(struct vm_area_struct *vma)
  168. {
  169. struct mm_struct *mm = vma->vm_mm;
  170. struct anon_vma *anon_vma, *allocated;
  171. struct anon_vma_chain *avc;
  172. might_sleep();
  173. avc = anon_vma_chain_alloc(GFP_KERNEL);
  174. if (!avc)
  175. goto out_enomem;
  176. anon_vma = find_mergeable_anon_vma(vma);
  177. allocated = NULL;
  178. if (!anon_vma) {
  179. anon_vma = anon_vma_alloc();
  180. if (unlikely(!anon_vma))
  181. goto out_enomem_free_avc;
  182. allocated = anon_vma;
  183. }
  184. anon_vma_lock_write(anon_vma);
  185. /* page_table_lock to protect against threads */
  186. spin_lock(&mm->page_table_lock);
  187. if (likely(!vma->anon_vma)) {
  188. vma->anon_vma = anon_vma;
  189. anon_vma_chain_link(vma, avc, anon_vma);
  190. /* vma reference or self-parent link for new root */
  191. anon_vma->degree++;
  192. allocated = NULL;
  193. avc = NULL;
  194. }
  195. spin_unlock(&mm->page_table_lock);
  196. anon_vma_unlock_write(anon_vma);
  197. if (unlikely(allocated))
  198. put_anon_vma(allocated);
  199. if (unlikely(avc))
  200. anon_vma_chain_free(avc);
  201. return 0;
  202. out_enomem_free_avc:
  203. anon_vma_chain_free(avc);
  204. out_enomem:
  205. return -ENOMEM;
  206. }
  207. /*
  208. * This is a useful helper function for locking the anon_vma root as
  209. * we traverse the vma->anon_vma_chain, looping over anon_vma's that
  210. * have the same vma.
  211. *
  212. * Such anon_vma's should have the same root, so you'd expect to see
  213. * just a single mutex_lock for the whole traversal.
  214. */
  215. static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
  216. {
  217. struct anon_vma *new_root = anon_vma->root;
  218. if (new_root != root) {
  219. if (WARN_ON_ONCE(root))
  220. up_write(&root->rwsem);
  221. root = new_root;
  222. down_write(&root->rwsem);
  223. }
  224. return root;
  225. }
  226. static inline void unlock_anon_vma_root(struct anon_vma *root)
  227. {
  228. if (root)
  229. up_write(&root->rwsem);
  230. }
  231. /*
  232. * Attach the anon_vmas from src to dst.
  233. * Returns 0 on success, -ENOMEM on failure.
  234. *
  235. * anon_vma_clone() is called by __vma_split(), __split_vma(), copy_vma() and
  236. * anon_vma_fork(). The first three want an exact copy of src, while the last
  237. * one, anon_vma_fork(), may try to reuse an existing anon_vma to prevent
  238. * endless growth of anon_vma. Since dst->anon_vma is set to NULL before call,
  239. * we can identify this case by checking (!dst->anon_vma && src->anon_vma).
  240. *
  241. * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
  242. * and reuse existing anon_vma which has no vmas and only one child anon_vma.
  243. * This prevents degradation of anon_vma hierarchy to endless linear chain in
  244. * case of constantly forking task. On the other hand, an anon_vma with more
  245. * than one child isn't reused even if there was no alive vma, thus rmap
  246. * walker has a good chance of avoiding scanning the whole hierarchy when it
  247. * searches where page is mapped.
  248. */
  249. int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
  250. {
  251. struct anon_vma_chain *avc, *pavc;
  252. struct anon_vma *root = NULL;
  253. list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
  254. struct anon_vma *anon_vma;
  255. avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
  256. if (unlikely(!avc)) {
  257. unlock_anon_vma_root(root);
  258. root = NULL;
  259. avc = anon_vma_chain_alloc(GFP_KERNEL);
  260. if (!avc)
  261. goto enomem_failure;
  262. }
  263. anon_vma = pavc->anon_vma;
  264. root = lock_anon_vma_root(root, anon_vma);
  265. anon_vma_chain_link(dst, avc, anon_vma);
  266. /*
  267. * Reuse existing anon_vma if its degree lower than two,
  268. * that means it has no vma and only one anon_vma child.
  269. *
  270. * Do not chose parent anon_vma, otherwise first child
  271. * will always reuse it. Root anon_vma is never reused:
  272. * it has self-parent reference and at least one child.
  273. */
  274. if (!dst->anon_vma && src->anon_vma &&
  275. anon_vma != src->anon_vma && anon_vma->degree < 2)
  276. dst->anon_vma = anon_vma;
  277. }
  278. if (dst->anon_vma)
  279. dst->anon_vma->degree++;
  280. unlock_anon_vma_root(root);
  281. return 0;
  282. enomem_failure:
  283. /*
  284. * dst->anon_vma is dropped here otherwise its degree can be incorrectly
  285. * decremented in unlink_anon_vmas().
  286. * We can safely do this because callers of anon_vma_clone() don't care
  287. * about dst->anon_vma if anon_vma_clone() failed.
  288. */
  289. dst->anon_vma = NULL;
  290. unlink_anon_vmas(dst);
  291. return -ENOMEM;
  292. }
  293. /*
  294. * Attach vma to its own anon_vma, as well as to the anon_vmas that
  295. * the corresponding VMA in the parent process is attached to.
  296. * Returns 0 on success, non-zero on failure.
  297. */
  298. int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
  299. {
  300. struct anon_vma_chain *avc;
  301. struct anon_vma *anon_vma;
  302. int error;
  303. /* Don't bother if the parent process has no anon_vma here. */
  304. if (!pvma->anon_vma)
  305. return 0;
  306. /* Drop inherited anon_vma, we'll reuse existing or allocate new. */
  307. vma->anon_vma = NULL;
  308. /*
  309. * First, attach the new VMA to the parent VMA's anon_vmas,
  310. * so rmap can find non-COWed pages in child processes.
  311. */
  312. error = anon_vma_clone(vma, pvma);
  313. if (error)
  314. return error;
  315. /* An existing anon_vma has been reused, all done then. */
  316. if (vma->anon_vma)
  317. return 0;
  318. /* Then add our own anon_vma. */
  319. anon_vma = anon_vma_alloc();
  320. if (!anon_vma)
  321. goto out_error;
  322. avc = anon_vma_chain_alloc(GFP_KERNEL);
  323. if (!avc)
  324. goto out_error_free_anon_vma;
  325. /*
  326. * The root anon_vma's spinlock is the lock actually used when we
  327. * lock any of the anon_vmas in this anon_vma tree.
  328. */
  329. anon_vma->root = pvma->anon_vma->root;
  330. anon_vma->parent = pvma->anon_vma;
  331. /*
  332. * With refcounts, an anon_vma can stay around longer than the
  333. * process it belongs to. The root anon_vma needs to be pinned until
  334. * this anon_vma is freed, because the lock lives in the root.
  335. */
  336. get_anon_vma(anon_vma->root);
  337. /* Mark this anon_vma as the one where our new (COWed) pages go. */
  338. vma->anon_vma = anon_vma;
  339. anon_vma_lock_write(anon_vma);
  340. anon_vma_chain_link(vma, avc, anon_vma);
  341. anon_vma->parent->degree++;
  342. anon_vma_unlock_write(anon_vma);
  343. return 0;
  344. out_error_free_anon_vma:
  345. put_anon_vma(anon_vma);
  346. out_error:
  347. unlink_anon_vmas(vma);
  348. return -ENOMEM;
  349. }
  350. void unlink_anon_vmas(struct vm_area_struct *vma)
  351. {
  352. struct anon_vma_chain *avc, *next;
  353. struct anon_vma *root = NULL;
  354. /*
  355. * Unlink each anon_vma chained to the VMA. This list is ordered
  356. * from newest to oldest, ensuring the root anon_vma gets freed last.
  357. */
  358. list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
  359. struct anon_vma *anon_vma = avc->anon_vma;
  360. root = lock_anon_vma_root(root, anon_vma);
  361. anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
  362. /*
  363. * Leave empty anon_vmas on the list - we'll need
  364. * to free them outside the lock.
  365. */
  366. if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
  367. anon_vma->parent->degree--;
  368. continue;
  369. }
  370. list_del(&avc->same_vma);
  371. anon_vma_chain_free(avc);
  372. }
  373. if (vma->anon_vma)
  374. vma->anon_vma->degree--;
  375. unlock_anon_vma_root(root);
  376. /*
  377. * Iterate the list once more, it now only contains empty and unlinked
  378. * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
  379. * needing to write-acquire the anon_vma->root->rwsem.
  380. */
  381. list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
  382. struct anon_vma *anon_vma = avc->anon_vma;
  383. VM_WARN_ON(anon_vma->degree);
  384. put_anon_vma(anon_vma);
  385. list_del(&avc->same_vma);
  386. anon_vma_chain_free(avc);
  387. }
  388. }
  389. static void anon_vma_ctor(void *data)
  390. {
  391. struct anon_vma *anon_vma = data;
  392. init_rwsem(&anon_vma->rwsem);
  393. atomic_set(&anon_vma->refcount, 0);
  394. anon_vma->rb_root = RB_ROOT_CACHED;
  395. }
  396. void __init anon_vma_init(void)
  397. {
  398. anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
  399. 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
  400. anon_vma_ctor);
  401. anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
  402. SLAB_PANIC|SLAB_ACCOUNT);
  403. }
  404. /*
  405. * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
  406. *
  407. * Since there is no serialization what so ever against page_remove_rmap()
  408. * the best this function can do is return a locked anon_vma that might
  409. * have been relevant to this page.
  410. *
  411. * The page might have been remapped to a different anon_vma or the anon_vma
  412. * returned may already be freed (and even reused).
  413. *
  414. * In case it was remapped to a different anon_vma, the new anon_vma will be a
  415. * child of the old anon_vma, and the anon_vma lifetime rules will therefore
  416. * ensure that any anon_vma obtained from the page will still be valid for as
  417. * long as we observe page_mapped() [ hence all those page_mapped() tests ].
  418. *
  419. * All users of this function must be very careful when walking the anon_vma
  420. * chain and verify that the page in question is indeed mapped in it
  421. * [ something equivalent to page_mapped_in_vma() ].
  422. *
  423. * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
  424. * page_remove_rmap() that the anon_vma pointer from page->mapping is valid
  425. * if there is a mapcount, we can dereference the anon_vma after observing
  426. * those.
  427. */
  428. struct anon_vma *page_get_anon_vma(struct page *page)
  429. {
  430. struct anon_vma *anon_vma = NULL;
  431. unsigned long anon_mapping;
  432. rcu_read_lock();
  433. anon_mapping = (unsigned long)READ_ONCE(page->mapping);
  434. if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
  435. goto out;
  436. if (!page_mapped(page))
  437. goto out;
  438. anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
  439. if (!atomic_inc_not_zero(&anon_vma->refcount)) {
  440. anon_vma = NULL;
  441. goto out;
  442. }
  443. /*
  444. * If this page is still mapped, then its anon_vma cannot have been
  445. * freed. But if it has been unmapped, we have no security against the
  446. * anon_vma structure being freed and reused (for another anon_vma:
  447. * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
  448. * above cannot corrupt).
  449. */
  450. if (!page_mapped(page)) {
  451. rcu_read_unlock();
  452. put_anon_vma(anon_vma);
  453. return NULL;
  454. }
  455. out:
  456. rcu_read_unlock();
  457. return anon_vma;
  458. }
  459. /*
  460. * Similar to page_get_anon_vma() except it locks the anon_vma.
  461. *
  462. * Its a little more complex as it tries to keep the fast path to a single
  463. * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
  464. * reference like with page_get_anon_vma() and then block on the mutex.
  465. */
  466. struct anon_vma *page_lock_anon_vma_read(struct page *page)
  467. {
  468. struct anon_vma *anon_vma = NULL;
  469. struct anon_vma *root_anon_vma;
  470. unsigned long anon_mapping;
  471. rcu_read_lock();
  472. anon_mapping = (unsigned long)READ_ONCE(page->mapping);
  473. if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
  474. goto out;
  475. if (!page_mapped(page))
  476. goto out;
  477. anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
  478. root_anon_vma = READ_ONCE(anon_vma->root);
  479. if (down_read_trylock(&root_anon_vma->rwsem)) {
  480. /*
  481. * If the page is still mapped, then this anon_vma is still
  482. * its anon_vma, and holding the mutex ensures that it will
  483. * not go away, see anon_vma_free().
  484. */
  485. if (!page_mapped(page)) {
  486. up_read(&root_anon_vma->rwsem);
  487. anon_vma = NULL;
  488. }
  489. goto out;
  490. }
  491. /* trylock failed, we got to sleep */
  492. if (!atomic_inc_not_zero(&anon_vma->refcount)) {
  493. anon_vma = NULL;
  494. goto out;
  495. }
  496. if (!page_mapped(page)) {
  497. rcu_read_unlock();
  498. put_anon_vma(anon_vma);
  499. return NULL;
  500. }
  501. /* we pinned the anon_vma, its safe to sleep */
  502. rcu_read_unlock();
  503. anon_vma_lock_read(anon_vma);
  504. if (atomic_dec_and_test(&anon_vma->refcount)) {
  505. /*
  506. * Oops, we held the last refcount, release the lock
  507. * and bail -- can't simply use put_anon_vma() because
  508. * we'll deadlock on the anon_vma_lock_write() recursion.
  509. */
  510. anon_vma_unlock_read(anon_vma);
  511. __put_anon_vma(anon_vma);
  512. anon_vma = NULL;
  513. }
  514. return anon_vma;
  515. out:
  516. rcu_read_unlock();
  517. return anon_vma;
  518. }
  519. void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
  520. {
  521. anon_vma_unlock_read(anon_vma);
  522. }
  523. #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
  524. /*
  525. * Flush TLB entries for recently unmapped pages from remote CPUs. It is
  526. * important if a PTE was dirty when it was unmapped that it's flushed
  527. * before any IO is initiated on the page to prevent lost writes. Similarly,
  528. * it must be flushed before freeing to prevent data leakage.
  529. */
  530. void try_to_unmap_flush(void)
  531. {
  532. struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
  533. if (!tlb_ubc->flush_required)
  534. return;
  535. arch_tlbbatch_flush(&tlb_ubc->arch);
  536. tlb_ubc->flush_required = false;
  537. tlb_ubc->writable = false;
  538. }
  539. /* Flush iff there are potentially writable TLB entries that can race with IO */
  540. void try_to_unmap_flush_dirty(void)
  541. {
  542. struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
  543. if (tlb_ubc->writable)
  544. try_to_unmap_flush();
  545. }
  546. static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
  547. {
  548. struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
  549. arch_tlbbatch_add_mm(&tlb_ubc->arch, mm);
  550. tlb_ubc->flush_required = true;
  551. /*
  552. * Ensure compiler does not re-order the setting of tlb_flush_batched
  553. * before the PTE is cleared.
  554. */
  555. barrier();
  556. mm->tlb_flush_batched = true;
  557. /*
  558. * If the PTE was dirty then it's best to assume it's writable. The
  559. * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
  560. * before the page is queued for IO.
  561. */
  562. if (writable)
  563. tlb_ubc->writable = true;
  564. }
  565. /*
  566. * Returns true if the TLB flush should be deferred to the end of a batch of
  567. * unmap operations to reduce IPIs.
  568. */
  569. static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
  570. {
  571. bool should_defer = false;
  572. if (!(flags & TTU_BATCH_FLUSH))
  573. return false;
  574. /* If remote CPUs need to be flushed then defer batch the flush */
  575. if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids)
  576. should_defer = true;
  577. put_cpu();
  578. return should_defer;
  579. }
  580. /*
  581. * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
  582. * releasing the PTL if TLB flushes are batched. It's possible for a parallel
  583. * operation such as mprotect or munmap to race between reclaim unmapping
  584. * the page and flushing the page. If this race occurs, it potentially allows
  585. * access to data via a stale TLB entry. Tracking all mm's that have TLB
  586. * batching in flight would be expensive during reclaim so instead track
  587. * whether TLB batching occurred in the past and if so then do a flush here
  588. * if required. This will cost one additional flush per reclaim cycle paid
  589. * by the first operation at risk such as mprotect and mumap.
  590. *
  591. * This must be called under the PTL so that an access to tlb_flush_batched
  592. * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
  593. * via the PTL.
  594. */
  595. void flush_tlb_batched_pending(struct mm_struct *mm)
  596. {
  597. if (data_race(mm->tlb_flush_batched)) {
  598. flush_tlb_mm(mm);
  599. /*
  600. * Do not allow the compiler to re-order the clearing of
  601. * tlb_flush_batched before the tlb is flushed.
  602. */
  603. barrier();
  604. mm->tlb_flush_batched = false;
  605. }
  606. }
  607. #else
  608. static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
  609. {
  610. }
  611. static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
  612. {
  613. return false;
  614. }
  615. #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
  616. /*
  617. * At what user virtual address is page expected in vma?
  618. * Caller should check the page is actually part of the vma.
  619. */
  620. unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
  621. {
  622. if (PageAnon(page)) {
  623. struct anon_vma *page__anon_vma = page_anon_vma(page);
  624. /*
  625. * Note: swapoff's unuse_vma() is more efficient with this
  626. * check, and needs it to match anon_vma when KSM is active.
  627. */
  628. if (!vma->anon_vma || !page__anon_vma ||
  629. vma->anon_vma->root != page__anon_vma->root)
  630. return -EFAULT;
  631. } else if (!vma->vm_file) {
  632. return -EFAULT;
  633. } else if (vma->vm_file->f_mapping != compound_head(page)->mapping) {
  634. return -EFAULT;
  635. }
  636. return vma_address(page, vma);
  637. }
  638. pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
  639. {
  640. pgd_t *pgd;
  641. p4d_t *p4d;
  642. pud_t *pud;
  643. pmd_t *pmd = NULL;
  644. pmd_t pmde;
  645. pgd = pgd_offset(mm, address);
  646. if (!pgd_present(*pgd))
  647. goto out;
  648. p4d = p4d_offset(pgd, address);
  649. if (!p4d_present(*p4d))
  650. goto out;
  651. pud = pud_offset(p4d, address);
  652. if (!pud_present(*pud))
  653. goto out;
  654. pmd = pmd_offset(pud, address);
  655. /*
  656. * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
  657. * without holding anon_vma lock for write. So when looking for a
  658. * genuine pmde (in which to find pte), test present and !THP together.
  659. */
  660. pmde = *pmd;
  661. barrier();
  662. if (!pmd_present(pmde) || pmd_trans_huge(pmde))
  663. pmd = NULL;
  664. out:
  665. return pmd;
  666. }
  667. struct page_referenced_arg {
  668. int mapcount;
  669. int referenced;
  670. unsigned long vm_flags;
  671. struct mem_cgroup *memcg;
  672. };
  673. /*
  674. * arg: page_referenced_arg will be passed
  675. */
  676. static bool page_referenced_one(struct page *page, struct vm_area_struct *vma,
  677. unsigned long address, void *arg)
  678. {
  679. struct page_referenced_arg *pra = arg;
  680. struct page_vma_mapped_walk pvmw = {
  681. .page = page,
  682. .vma = vma,
  683. .address = address,
  684. };
  685. int referenced = 0;
  686. while (page_vma_mapped_walk(&pvmw)) {
  687. address = pvmw.address;
  688. if (vma->vm_flags & VM_LOCKED) {
  689. page_vma_mapped_walk_done(&pvmw);
  690. pra->vm_flags |= VM_LOCKED;
  691. return false; /* To break the loop */
  692. }
  693. if (pvmw.pte) {
  694. if (ptep_clear_flush_young_notify(vma, address,
  695. pvmw.pte)) {
  696. /*
  697. * Don't treat a reference through
  698. * a sequentially read mapping as such.
  699. * If the page has been used in another mapping,
  700. * we will catch it; if this other mapping is
  701. * already gone, the unmap path will have set
  702. * PG_referenced or activated the page.
  703. */
  704. if (likely(!(vma->vm_flags & VM_SEQ_READ)))
  705. referenced++;
  706. }
  707. } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
  708. if (pmdp_clear_flush_young_notify(vma, address,
  709. pvmw.pmd))
  710. referenced++;
  711. } else {
  712. /* unexpected pmd-mapped page? */
  713. WARN_ON_ONCE(1);
  714. }
  715. pra->mapcount--;
  716. }
  717. if (referenced)
  718. clear_page_idle(page);
  719. if (test_and_clear_page_young(page))
  720. referenced++;
  721. if (referenced) {
  722. pra->referenced++;
  723. pra->vm_flags |= vma->vm_flags;
  724. }
  725. if (!pra->mapcount)
  726. return false; /* To break the loop */
  727. return true;
  728. }
  729. static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
  730. {
  731. struct page_referenced_arg *pra = arg;
  732. struct mem_cgroup *memcg = pra->memcg;
  733. if (!mm_match_cgroup(vma->vm_mm, memcg))
  734. return true;
  735. return false;
  736. }
  737. /**
  738. * page_referenced - test if the page was referenced
  739. * @page: the page to test
  740. * @is_locked: caller holds lock on the page
  741. * @memcg: target memory cgroup
  742. * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
  743. *
  744. * Quick test_and_clear_referenced for all mappings to a page,
  745. * returns the number of ptes which referenced the page.
  746. */
  747. int page_referenced(struct page *page,
  748. int is_locked,
  749. struct mem_cgroup *memcg,
  750. unsigned long *vm_flags)
  751. {
  752. int we_locked = 0;
  753. struct page_referenced_arg pra = {
  754. .mapcount = total_mapcount(page),
  755. .memcg = memcg,
  756. };
  757. struct rmap_walk_control rwc = {
  758. .rmap_one = page_referenced_one,
  759. .arg = (void *)&pra,
  760. .anon_lock = page_lock_anon_vma_read,
  761. };
  762. *vm_flags = 0;
  763. if (!pra.mapcount)
  764. return 0;
  765. if (!page_rmapping(page))
  766. return 0;
  767. if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
  768. we_locked = trylock_page(page);
  769. if (!we_locked)
  770. return 1;
  771. }
  772. /*
  773. * If we are reclaiming on behalf of a cgroup, skip
  774. * counting on behalf of references from different
  775. * cgroups
  776. */
  777. if (memcg) {
  778. rwc.invalid_vma = invalid_page_referenced_vma;
  779. }
  780. rmap_walk(page, &rwc);
  781. *vm_flags = pra.vm_flags;
  782. if (we_locked)
  783. unlock_page(page);
  784. return pra.referenced;
  785. }
  786. static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
  787. unsigned long address, void *arg)
  788. {
  789. struct page_vma_mapped_walk pvmw = {
  790. .page = page,
  791. .vma = vma,
  792. .address = address,
  793. .flags = PVMW_SYNC,
  794. };
  795. struct mmu_notifier_range range;
  796. int *cleaned = arg;
  797. /*
  798. * We have to assume the worse case ie pmd for invalidation. Note that
  799. * the page can not be free from this function.
  800. */
  801. mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
  802. 0, vma, vma->vm_mm, address,
  803. vma_address_end(page, vma));
  804. mmu_notifier_invalidate_range_start(&range);
  805. while (page_vma_mapped_walk(&pvmw)) {
  806. int ret = 0;
  807. address = pvmw.address;
  808. if (pvmw.pte) {
  809. pte_t entry;
  810. pte_t *pte = pvmw.pte;
  811. if (!pte_dirty(*pte) && !pte_write(*pte))
  812. continue;
  813. flush_cache_page(vma, address, pte_pfn(*pte));
  814. entry = ptep_clear_flush(vma, address, pte);
  815. entry = pte_wrprotect(entry);
  816. entry = pte_mkclean(entry);
  817. set_pte_at(vma->vm_mm, address, pte, entry);
  818. ret = 1;
  819. } else {
  820. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  821. pmd_t *pmd = pvmw.pmd;
  822. pmd_t entry;
  823. if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
  824. continue;
  825. flush_cache_page(vma, address, page_to_pfn(page));
  826. entry = pmdp_invalidate(vma, address, pmd);
  827. entry = pmd_wrprotect(entry);
  828. entry = pmd_mkclean(entry);
  829. set_pmd_at(vma->vm_mm, address, pmd, entry);
  830. ret = 1;
  831. #else
  832. /* unexpected pmd-mapped page? */
  833. WARN_ON_ONCE(1);
  834. #endif
  835. }
  836. /*
  837. * No need to call mmu_notifier_invalidate_range() as we are
  838. * downgrading page table protection not changing it to point
  839. * to a new page.
  840. *
  841. * See Documentation/vm/mmu_notifier.rst
  842. */
  843. if (ret)
  844. (*cleaned)++;
  845. }
  846. mmu_notifier_invalidate_range_end(&range);
  847. return true;
  848. }
  849. static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
  850. {
  851. if (vma->vm_flags & VM_SHARED)
  852. return false;
  853. return true;
  854. }
  855. int page_mkclean(struct page *page)
  856. {
  857. int cleaned = 0;
  858. struct address_space *mapping;
  859. struct rmap_walk_control rwc = {
  860. .arg = (void *)&cleaned,
  861. .rmap_one = page_mkclean_one,
  862. .invalid_vma = invalid_mkclean_vma,
  863. };
  864. BUG_ON(!PageLocked(page));
  865. if (!page_mapped(page))
  866. return 0;
  867. mapping = page_mapping(page);
  868. if (!mapping)
  869. return 0;
  870. rmap_walk(page, &rwc);
  871. return cleaned;
  872. }
  873. EXPORT_SYMBOL_GPL(page_mkclean);
  874. /**
  875. * page_move_anon_rmap - move a page to our anon_vma
  876. * @page: the page to move to our anon_vma
  877. * @vma: the vma the page belongs to
  878. *
  879. * When a page belongs exclusively to one process after a COW event,
  880. * that page can be moved into the anon_vma that belongs to just that
  881. * process, so the rmap code will not search the parent or sibling
  882. * processes.
  883. */
  884. void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
  885. {
  886. struct anon_vma *anon_vma = vma->anon_vma;
  887. page = compound_head(page);
  888. VM_BUG_ON_PAGE(!PageLocked(page), page);
  889. VM_BUG_ON_VMA(!anon_vma, vma);
  890. anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
  891. /*
  892. * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
  893. * simultaneously, so a concurrent reader (eg page_referenced()'s
  894. * PageAnon()) will not see one without the other.
  895. */
  896. WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
  897. }
  898. /**
  899. * __page_set_anon_rmap - set up new anonymous rmap
  900. * @page: Page or Hugepage to add to rmap
  901. * @vma: VM area to add page to.
  902. * @address: User virtual address of the mapping
  903. * @exclusive: the page is exclusively owned by the current process
  904. */
  905. static void __page_set_anon_rmap(struct page *page,
  906. struct vm_area_struct *vma, unsigned long address, int exclusive)
  907. {
  908. struct anon_vma *anon_vma = vma->anon_vma;
  909. BUG_ON(!anon_vma);
  910. if (PageAnon(page))
  911. return;
  912. /*
  913. * If the page isn't exclusively mapped into this vma,
  914. * we must use the _oldest_ possible anon_vma for the
  915. * page mapping!
  916. */
  917. if (!exclusive)
  918. anon_vma = anon_vma->root;
  919. anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
  920. page->mapping = (struct address_space *) anon_vma;
  921. page->index = linear_page_index(vma, address);
  922. }
  923. /**
  924. * __page_check_anon_rmap - sanity check anonymous rmap addition
  925. * @page: the page to add the mapping to
  926. * @vma: the vm area in which the mapping is added
  927. * @address: the user virtual address mapped
  928. */
  929. static void __page_check_anon_rmap(struct page *page,
  930. struct vm_area_struct *vma, unsigned long address)
  931. {
  932. /*
  933. * The page's anon-rmap details (mapping and index) are guaranteed to
  934. * be set up correctly at this point.
  935. *
  936. * We have exclusion against page_add_anon_rmap because the caller
  937. * always holds the page locked, except if called from page_dup_rmap,
  938. * in which case the page is already known to be setup.
  939. *
  940. * We have exclusion against page_add_new_anon_rmap because those pages
  941. * are initially only visible via the pagetables, and the pte is locked
  942. * over the call to page_add_new_anon_rmap.
  943. */
  944. VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page);
  945. VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
  946. page);
  947. }
  948. /**
  949. * page_add_anon_rmap - add pte mapping to an anonymous page
  950. * @page: the page to add the mapping to
  951. * @vma: the vm area in which the mapping is added
  952. * @address: the user virtual address mapped
  953. * @compound: charge the page as compound or small page
  954. *
  955. * The caller needs to hold the pte lock, and the page must be locked in
  956. * the anon_vma case: to serialize mapping,index checking after setting,
  957. * and to ensure that PageAnon is not being upgraded racily to PageKsm
  958. * (but PageKsm is never downgraded to PageAnon).
  959. */
  960. void page_add_anon_rmap(struct page *page,
  961. struct vm_area_struct *vma, unsigned long address, bool compound)
  962. {
  963. do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0);
  964. }
  965. /*
  966. * Special version of the above for do_swap_page, which often runs
  967. * into pages that are exclusively owned by the current process.
  968. * Everybody else should continue to use page_add_anon_rmap above.
  969. */
  970. void do_page_add_anon_rmap(struct page *page,
  971. struct vm_area_struct *vma, unsigned long address, int flags)
  972. {
  973. bool compound = flags & RMAP_COMPOUND;
  974. bool first;
  975. if (unlikely(PageKsm(page)))
  976. lock_page_memcg(page);
  977. else
  978. VM_BUG_ON_PAGE(!PageLocked(page), page);
  979. if (compound) {
  980. atomic_t *mapcount;
  981. VM_BUG_ON_PAGE(!PageLocked(page), page);
  982. VM_BUG_ON_PAGE(!PageTransHuge(page), page);
  983. mapcount = compound_mapcount_ptr(page);
  984. first = atomic_inc_and_test(mapcount);
  985. } else {
  986. first = atomic_inc_and_test(&page->_mapcount);
  987. }
  988. if (first) {
  989. int nr = compound ? thp_nr_pages(page) : 1;
  990. /*
  991. * We use the irq-unsafe __{inc|mod}_zone_page_stat because
  992. * these counters are not modified in interrupt context, and
  993. * pte lock(a spinlock) is held, which implies preemption
  994. * disabled.
  995. */
  996. if (compound)
  997. __inc_lruvec_page_state(page, NR_ANON_THPS);
  998. __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);
  999. }
  1000. if (unlikely(PageKsm(page))) {
  1001. unlock_page_memcg(page);
  1002. return;
  1003. }
  1004. /* address might be in next vma when migration races vma_adjust */
  1005. if (first)
  1006. __page_set_anon_rmap(page, vma, address,
  1007. flags & RMAP_EXCLUSIVE);
  1008. else
  1009. __page_check_anon_rmap(page, vma, address);
  1010. }
  1011. /**
  1012. * __page_add_new_anon_rmap - add pte mapping to a new anonymous page
  1013. * @page: the page to add the mapping to
  1014. * @vma: the vm area in which the mapping is added
  1015. * @address: the user virtual address mapped
  1016. * @compound: charge the page as compound or small page
  1017. *
  1018. * Same as page_add_anon_rmap but must only be called on *new* pages.
  1019. * This means the inc-and-test can be bypassed.
  1020. * Page does not have to be locked.
  1021. */
  1022. void __page_add_new_anon_rmap(struct page *page,
  1023. struct vm_area_struct *vma, unsigned long address, bool compound)
  1024. {
  1025. int nr = compound ? thp_nr_pages(page) : 1;
  1026. __SetPageSwapBacked(page);
  1027. if (compound) {
  1028. VM_BUG_ON_PAGE(!PageTransHuge(page), page);
  1029. /* increment count (starts at -1) */
  1030. atomic_set(compound_mapcount_ptr(page), 0);
  1031. if (hpage_pincount_available(page))
  1032. atomic_set(compound_pincount_ptr(page), 0);
  1033. __inc_lruvec_page_state(page, NR_ANON_THPS);
  1034. } else {
  1035. /* Anon THP always mapped first with PMD */
  1036. VM_BUG_ON_PAGE(PageTransCompound(page), page);
  1037. /* increment count (starts at -1) */
  1038. atomic_set(&page->_mapcount, 0);
  1039. }
  1040. __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);
  1041. __page_set_anon_rmap(page, vma, address, 1);
  1042. }
  1043. /**
  1044. * page_add_file_rmap - add pte mapping to a file page
  1045. * @page: the page to add the mapping to
  1046. * @compound: charge the page as compound or small page
  1047. *
  1048. * The caller needs to hold the pte lock.
  1049. */
  1050. void page_add_file_rmap(struct page *page, bool compound)
  1051. {
  1052. int i, nr = 1;
  1053. VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
  1054. lock_page_memcg(page);
  1055. if (compound && PageTransHuge(page)) {
  1056. for (i = 0, nr = 0; i < thp_nr_pages(page); i++) {
  1057. if (atomic_inc_and_test(&page[i]._mapcount))
  1058. nr++;
  1059. }
  1060. if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
  1061. goto out;
  1062. if (PageSwapBacked(page))
  1063. __inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
  1064. else
  1065. __inc_node_page_state(page, NR_FILE_PMDMAPPED);
  1066. } else {
  1067. if (PageTransCompound(page) && page_mapping(page)) {
  1068. VM_WARN_ON_ONCE(!PageLocked(page));
  1069. SetPageDoubleMap(compound_head(page));
  1070. if (PageMlocked(page))
  1071. clear_page_mlock(compound_head(page));
  1072. }
  1073. if (!atomic_inc_and_test(&page->_mapcount))
  1074. goto out;
  1075. }
  1076. __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);
  1077. out:
  1078. unlock_page_memcg(page);
  1079. }
  1080. static void page_remove_file_rmap(struct page *page, bool compound)
  1081. {
  1082. int i, nr = 1;
  1083. VM_BUG_ON_PAGE(compound && !PageHead(page), page);
  1084. /* Hugepages are not counted in NR_FILE_MAPPED for now. */
  1085. if (unlikely(PageHuge(page))) {
  1086. /* hugetlb pages are always mapped with pmds */
  1087. atomic_dec(compound_mapcount_ptr(page));
  1088. return;
  1089. }
  1090. /* page still mapped by someone else? */
  1091. if (compound && PageTransHuge(page)) {
  1092. for (i = 0, nr = 0; i < thp_nr_pages(page); i++) {
  1093. if (atomic_add_negative(-1, &page[i]._mapcount))
  1094. nr++;
  1095. }
  1096. if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
  1097. return;
  1098. if (PageSwapBacked(page))
  1099. __dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
  1100. else
  1101. __dec_node_page_state(page, NR_FILE_PMDMAPPED);
  1102. } else {
  1103. if (!atomic_add_negative(-1, &page->_mapcount))
  1104. return;
  1105. }
  1106. /*
  1107. * We use the irq-unsafe __{inc|mod}_lruvec_page_state because
  1108. * these counters are not modified in interrupt context, and
  1109. * pte lock(a spinlock) is held, which implies preemption disabled.
  1110. */
  1111. __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr);
  1112. if (unlikely(PageMlocked(page)))
  1113. clear_page_mlock(page);
  1114. }
  1115. static void page_remove_anon_compound_rmap(struct page *page)
  1116. {
  1117. int i, nr;
  1118. if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
  1119. return;
  1120. /* Hugepages are not counted in NR_ANON_PAGES for now. */
  1121. if (unlikely(PageHuge(page)))
  1122. return;
  1123. if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
  1124. return;
  1125. __dec_lruvec_page_state(page, NR_ANON_THPS);
  1126. if (TestClearPageDoubleMap(page)) {
  1127. /*
  1128. * Subpages can be mapped with PTEs too. Check how many of
  1129. * them are still mapped.
  1130. */
  1131. for (i = 0, nr = 0; i < thp_nr_pages(page); i++) {
  1132. if (atomic_add_negative(-1, &page[i]._mapcount))
  1133. nr++;
  1134. }
  1135. /*
  1136. * Queue the page for deferred split if at least one small
  1137. * page of the compound page is unmapped, but at least one
  1138. * small page is still mapped.
  1139. */
  1140. if (nr && nr < thp_nr_pages(page))
  1141. deferred_split_huge_page(page);
  1142. } else {
  1143. nr = thp_nr_pages(page);
  1144. }
  1145. if (unlikely(PageMlocked(page)))
  1146. clear_page_mlock(page);
  1147. if (nr)
  1148. __mod_lruvec_page_state(page, NR_ANON_MAPPED, -nr);
  1149. }
  1150. /**
  1151. * page_remove_rmap - take down pte mapping from a page
  1152. * @page: page to remove mapping from
  1153. * @compound: uncharge the page as compound or small page
  1154. *
  1155. * The caller needs to hold the pte lock.
  1156. */
  1157. void page_remove_rmap(struct page *page, bool compound)
  1158. {
  1159. lock_page_memcg(page);
  1160. if (!PageAnon(page)) {
  1161. page_remove_file_rmap(page, compound);
  1162. goto out;
  1163. }
  1164. if (compound) {
  1165. page_remove_anon_compound_rmap(page);
  1166. goto out;
  1167. }
  1168. /* page still mapped by someone else? */
  1169. if (!atomic_add_negative(-1, &page->_mapcount))
  1170. goto out;
  1171. /*
  1172. * We use the irq-unsafe __{inc|mod}_zone_page_stat because
  1173. * these counters are not modified in interrupt context, and
  1174. * pte lock(a spinlock) is held, which implies preemption disabled.
  1175. */
  1176. __dec_lruvec_page_state(page, NR_ANON_MAPPED);
  1177. if (unlikely(PageMlocked(page)))
  1178. clear_page_mlock(page);
  1179. if (PageTransCompound(page))
  1180. deferred_split_huge_page(compound_head(page));
  1181. /*
  1182. * It would be tidy to reset the PageAnon mapping here,
  1183. * but that might overwrite a racing page_add_anon_rmap
  1184. * which increments mapcount after us but sets mapping
  1185. * before us: so leave the reset to free_unref_page,
  1186. * and remember that it's only reliable while mapped.
  1187. * Leaving it set also helps swapoff to reinstate ptes
  1188. * faster for those pages still in swapcache.
  1189. */
  1190. out:
  1191. unlock_page_memcg(page);
  1192. }
  1193. /*
  1194. * @arg: enum ttu_flags will be passed to this argument
  1195. */
  1196. static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
  1197. unsigned long address, void *arg)
  1198. {
  1199. struct mm_struct *mm = vma->vm_mm;
  1200. struct page_vma_mapped_walk pvmw = {
  1201. .page = page,
  1202. .vma = vma,
  1203. .address = address,
  1204. };
  1205. pte_t pteval;
  1206. struct page *subpage;
  1207. bool ret = true;
  1208. struct mmu_notifier_range range;
  1209. enum ttu_flags flags = (enum ttu_flags)(long)arg;
  1210. /*
  1211. * When racing against e.g. zap_pte_range() on another cpu,
  1212. * in between its ptep_get_and_clear_full() and page_remove_rmap(),
  1213. * try_to_unmap() may return false when it is about to become true,
  1214. * if page table locking is skipped: use TTU_SYNC to wait for that.
  1215. */
  1216. if (flags & TTU_SYNC)
  1217. pvmw.flags = PVMW_SYNC;
  1218. /* munlock has nothing to gain from examining un-locked vmas */
  1219. if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
  1220. return true;
  1221. if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION) &&
  1222. is_zone_device_page(page) && !is_device_private_page(page))
  1223. return true;
  1224. if (flags & TTU_SPLIT_HUGE_PMD) {
  1225. split_huge_pmd_address(vma, address,
  1226. flags & TTU_SPLIT_FREEZE, page);
  1227. }
  1228. /*
  1229. * For THP, we have to assume the worse case ie pmd for invalidation.
  1230. * For hugetlb, it could be much worse if we need to do pud
  1231. * invalidation in the case of pmd sharing.
  1232. *
  1233. * Note that the page can not be free in this function as call of
  1234. * try_to_unmap() must hold a reference on the page.
  1235. */
  1236. range.end = PageKsm(page) ?
  1237. address + PAGE_SIZE : vma_address_end(page, vma);
  1238. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
  1239. address, range.end);
  1240. if (PageHuge(page)) {
  1241. /*
  1242. * If sharing is possible, start and end will be adjusted
  1243. * accordingly.
  1244. */
  1245. adjust_range_if_pmd_sharing_possible(vma, &range.start,
  1246. &range.end);
  1247. }
  1248. mmu_notifier_invalidate_range_start(&range);
  1249. while (page_vma_mapped_walk(&pvmw)) {
  1250. #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
  1251. /* PMD-mapped THP migration entry */
  1252. if (!pvmw.pte && (flags & TTU_MIGRATION)) {
  1253. VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
  1254. set_pmd_migration_entry(&pvmw, page);
  1255. continue;
  1256. }
  1257. #endif
  1258. /*
  1259. * If the page is mlock()d, we cannot swap it out.
  1260. * If it's recently referenced (perhaps page_referenced
  1261. * skipped over this mm) then we should reactivate it.
  1262. */
  1263. if (!(flags & TTU_IGNORE_MLOCK)) {
  1264. if (vma->vm_flags & VM_LOCKED) {
  1265. /* PTE-mapped THP are never mlocked */
  1266. if (!PageTransCompound(page)) {
  1267. /*
  1268. * Holding pte lock, we do *not* need
  1269. * mmap_lock here
  1270. */
  1271. mlock_vma_page(page);
  1272. }
  1273. ret = false;
  1274. page_vma_mapped_walk_done(&pvmw);
  1275. break;
  1276. }
  1277. if (flags & TTU_MUNLOCK)
  1278. continue;
  1279. }
  1280. /* Unexpected PMD-mapped THP? */
  1281. VM_BUG_ON_PAGE(!pvmw.pte, page);
  1282. subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
  1283. address = pvmw.address;
  1284. if (PageHuge(page) && !PageAnon(page)) {
  1285. /*
  1286. * To call huge_pmd_unshare, i_mmap_rwsem must be
  1287. * held in write mode. Caller needs to explicitly
  1288. * do this outside rmap routines.
  1289. */
  1290. VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
  1291. if (huge_pmd_unshare(mm, vma, &address, pvmw.pte)) {
  1292. /*
  1293. * huge_pmd_unshare unmapped an entire PMD
  1294. * page. There is no way of knowing exactly
  1295. * which PMDs may be cached for this mm, so
  1296. * we must flush them all. start/end were
  1297. * already adjusted above to cover this range.
  1298. */
  1299. flush_cache_range(vma, range.start, range.end);
  1300. flush_tlb_range(vma, range.start, range.end);
  1301. mmu_notifier_invalidate_range(mm, range.start,
  1302. range.end);
  1303. /*
  1304. * The ref count of the PMD page was dropped
  1305. * which is part of the way map counting
  1306. * is done for shared PMDs. Return 'true'
  1307. * here. When there is no other sharing,
  1308. * huge_pmd_unshare returns false and we will
  1309. * unmap the actual page and drop map count
  1310. * to zero.
  1311. */
  1312. page_vma_mapped_walk_done(&pvmw);
  1313. break;
  1314. }
  1315. }
  1316. if (IS_ENABLED(CONFIG_MIGRATION) &&
  1317. (flags & TTU_MIGRATION) &&
  1318. is_zone_device_page(page)) {
  1319. swp_entry_t entry;
  1320. pte_t swp_pte;
  1321. pteval = ptep_get_and_clear(mm, pvmw.address, pvmw.pte);
  1322. /*
  1323. * Store the pfn of the page in a special migration
  1324. * pte. do_swap_page() will wait until the migration
  1325. * pte is removed and then restart fault handling.
  1326. */
  1327. entry = make_migration_entry(page, 0);
  1328. swp_pte = swp_entry_to_pte(entry);
  1329. /*
  1330. * pteval maps a zone device page and is therefore
  1331. * a swap pte.
  1332. */
  1333. if (pte_swp_soft_dirty(pteval))
  1334. swp_pte = pte_swp_mksoft_dirty(swp_pte);
  1335. if (pte_swp_uffd_wp(pteval))
  1336. swp_pte = pte_swp_mkuffd_wp(swp_pte);
  1337. set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
  1338. /*
  1339. * No need to invalidate here it will synchronize on
  1340. * against the special swap migration pte.
  1341. *
  1342. * The assignment to subpage above was computed from a
  1343. * swap PTE which results in an invalid pointer.
  1344. * Since only PAGE_SIZE pages can currently be
  1345. * migrated, just set it to page. This will need to be
  1346. * changed when hugepage migrations to device private
  1347. * memory are supported.
  1348. */
  1349. subpage = page;
  1350. goto discard;
  1351. }
  1352. /* Nuke the page table entry. */
  1353. flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
  1354. if (should_defer_flush(mm, flags)) {
  1355. /*
  1356. * We clear the PTE but do not flush so potentially
  1357. * a remote CPU could still be writing to the page.
  1358. * If the entry was previously clean then the
  1359. * architecture must guarantee that a clear->dirty
  1360. * transition on a cached TLB entry is written through
  1361. * and traps if the PTE is unmapped.
  1362. */
  1363. pteval = ptep_get_and_clear(mm, address, pvmw.pte);
  1364. set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
  1365. } else {
  1366. pteval = ptep_clear_flush(vma, address, pvmw.pte);
  1367. }
  1368. /* Move the dirty bit to the page. Now the pte is gone. */
  1369. if (pte_dirty(pteval))
  1370. set_page_dirty(page);
  1371. /* Update high watermark before we lower rss */
  1372. update_hiwater_rss(mm);
  1373. if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
  1374. pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
  1375. if (PageHuge(page)) {
  1376. hugetlb_count_sub(compound_nr(page), mm);
  1377. set_huge_swap_pte_at(mm, address,
  1378. pvmw.pte, pteval,
  1379. vma_mmu_pagesize(vma));
  1380. } else {
  1381. dec_mm_counter(mm, mm_counter(page));
  1382. set_pte_at(mm, address, pvmw.pte, pteval);
  1383. }
  1384. } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
  1385. /*
  1386. * The guest indicated that the page content is of no
  1387. * interest anymore. Simply discard the pte, vmscan
  1388. * will take care of the rest.
  1389. * A future reference will then fault in a new zero
  1390. * page. When userfaultfd is active, we must not drop
  1391. * this page though, as its main user (postcopy
  1392. * migration) will not expect userfaults on already
  1393. * copied pages.
  1394. */
  1395. dec_mm_counter(mm, mm_counter(page));
  1396. /* We have to invalidate as we cleared the pte */
  1397. mmu_notifier_invalidate_range(mm, address,
  1398. address + PAGE_SIZE);
  1399. } else if (IS_ENABLED(CONFIG_MIGRATION) &&
  1400. (flags & (TTU_MIGRATION|TTU_SPLIT_FREEZE))) {
  1401. swp_entry_t entry;
  1402. pte_t swp_pte;
  1403. if (arch_unmap_one(mm, vma, address, pteval) < 0) {
  1404. set_pte_at(mm, address, pvmw.pte, pteval);
  1405. ret = false;
  1406. page_vma_mapped_walk_done(&pvmw);
  1407. break;
  1408. }
  1409. /*
  1410. * Store the pfn of the page in a special migration
  1411. * pte. do_swap_page() will wait until the migration
  1412. * pte is removed and then restart fault handling.
  1413. */
  1414. entry = make_migration_entry(subpage,
  1415. pte_write(pteval));
  1416. swp_pte = swp_entry_to_pte(entry);
  1417. if (pte_soft_dirty(pteval))
  1418. swp_pte = pte_swp_mksoft_dirty(swp_pte);
  1419. if (pte_uffd_wp(pteval))
  1420. swp_pte = pte_swp_mkuffd_wp(swp_pte);
  1421. set_pte_at(mm, address, pvmw.pte, swp_pte);
  1422. /*
  1423. * No need to invalidate here it will synchronize on
  1424. * against the special swap migration pte.
  1425. */
  1426. } else if (PageAnon(page)) {
  1427. swp_entry_t entry = { .val = page_private(subpage) };
  1428. pte_t swp_pte;
  1429. /*
  1430. * Store the swap location in the pte.
  1431. * See handle_pte_fault() ...
  1432. */
  1433. if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) {
  1434. WARN_ON_ONCE(1);
  1435. ret = false;
  1436. /* We have to invalidate as we cleared the pte */
  1437. mmu_notifier_invalidate_range(mm, address,
  1438. address + PAGE_SIZE);
  1439. page_vma_mapped_walk_done(&pvmw);
  1440. break;
  1441. }
  1442. /* MADV_FREE page check */
  1443. if (!PageSwapBacked(page)) {
  1444. int ref_count, map_count;
  1445. /*
  1446. * Synchronize with gup_pte_range():
  1447. * - clear PTE; barrier; read refcount
  1448. * - inc refcount; barrier; read PTE
  1449. */
  1450. smp_mb();
  1451. ref_count = page_ref_count(page);
  1452. map_count = page_mapcount(page);
  1453. /*
  1454. * Order reads for page refcount and dirty flag
  1455. * (see comments in __remove_mapping()).
  1456. */
  1457. smp_rmb();
  1458. /*
  1459. * The only page refs must be one from isolation
  1460. * plus the rmap(s) (dropped by discard:).
  1461. */
  1462. if (ref_count == 1 + map_count &&
  1463. !PageDirty(page)) {
  1464. /* Invalidate as we cleared the pte */
  1465. mmu_notifier_invalidate_range(mm,
  1466. address, address + PAGE_SIZE);
  1467. dec_mm_counter(mm, MM_ANONPAGES);
  1468. goto discard;
  1469. }
  1470. /*
  1471. * If the page was redirtied, it cannot be
  1472. * discarded. Remap the page to page table.
  1473. */
  1474. set_pte_at(mm, address, pvmw.pte, pteval);
  1475. SetPageSwapBacked(page);
  1476. ret = false;
  1477. page_vma_mapped_walk_done(&pvmw);
  1478. break;
  1479. }
  1480. if (swap_duplicate(entry) < 0) {
  1481. set_pte_at(mm, address, pvmw.pte, pteval);
  1482. ret = false;
  1483. page_vma_mapped_walk_done(&pvmw);
  1484. break;
  1485. }
  1486. if (arch_unmap_one(mm, vma, address, pteval) < 0) {
  1487. set_pte_at(mm, address, pvmw.pte, pteval);
  1488. ret = false;
  1489. page_vma_mapped_walk_done(&pvmw);
  1490. break;
  1491. }
  1492. if (list_empty(&mm->mmlist)) {
  1493. spin_lock(&mmlist_lock);
  1494. if (list_empty(&mm->mmlist))
  1495. list_add(&mm->mmlist, &init_mm.mmlist);
  1496. spin_unlock(&mmlist_lock);
  1497. }
  1498. dec_mm_counter(mm, MM_ANONPAGES);
  1499. inc_mm_counter(mm, MM_SWAPENTS);
  1500. swp_pte = swp_entry_to_pte(entry);
  1501. if (pte_soft_dirty(pteval))
  1502. swp_pte = pte_swp_mksoft_dirty(swp_pte);
  1503. if (pte_uffd_wp(pteval))
  1504. swp_pte = pte_swp_mkuffd_wp(swp_pte);
  1505. set_pte_at(mm, address, pvmw.pte, swp_pte);
  1506. /* Invalidate as we cleared the pte */
  1507. mmu_notifier_invalidate_range(mm, address,
  1508. address + PAGE_SIZE);
  1509. } else {
  1510. /*
  1511. * This is a locked file-backed page, thus it cannot
  1512. * be removed from the page cache and replaced by a new
  1513. * page before mmu_notifier_invalidate_range_end, so no
  1514. * concurrent thread might update its page table to
  1515. * point at new page while a device still is using this
  1516. * page.
  1517. *
  1518. * See Documentation/vm/mmu_notifier.rst
  1519. */
  1520. dec_mm_counter(mm, mm_counter_file(page));
  1521. }
  1522. discard:
  1523. /*
  1524. * No need to call mmu_notifier_invalidate_range() it has be
  1525. * done above for all cases requiring it to happen under page
  1526. * table lock before mmu_notifier_invalidate_range_end()
  1527. *
  1528. * See Documentation/vm/mmu_notifier.rst
  1529. */
  1530. page_remove_rmap(subpage, PageHuge(page));
  1531. put_page(page);
  1532. }
  1533. mmu_notifier_invalidate_range_end(&range);
  1534. trace_android_vh_try_to_unmap_one(vma, page, address, ret);
  1535. return ret;
  1536. }
  1537. static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
  1538. {
  1539. return vma_is_temporary_stack(vma);
  1540. }
  1541. static int page_not_mapped(struct page *page)
  1542. {
  1543. return !page_mapped(page);
  1544. }
  1545. /**
  1546. * try_to_unmap - try to remove all page table mappings to a page
  1547. * @page: the page to get unmapped
  1548. * @flags: action and flags
  1549. *
  1550. * Tries to remove all the page table entries which are mapping this
  1551. * page, used in the pageout path. Caller must hold the page lock.
  1552. *
  1553. * If unmap is successful, return true. Otherwise, false.
  1554. */
  1555. bool try_to_unmap(struct page *page, enum ttu_flags flags)
  1556. {
  1557. struct rmap_walk_control rwc = {
  1558. .rmap_one = try_to_unmap_one,
  1559. .arg = (void *)flags,
  1560. .done = page_not_mapped,
  1561. .anon_lock = page_lock_anon_vma_read,
  1562. };
  1563. /*
  1564. * During exec, a temporary VMA is setup and later moved.
  1565. * The VMA is moved under the anon_vma lock but not the
  1566. * page tables leading to a race where migration cannot
  1567. * find the migration ptes. Rather than increasing the
  1568. * locking requirements of exec(), migration skips
  1569. * temporary VMAs until after exec() completes.
  1570. */
  1571. if ((flags & (TTU_MIGRATION|TTU_SPLIT_FREEZE))
  1572. && !PageKsm(page) && PageAnon(page))
  1573. rwc.invalid_vma = invalid_migration_vma;
  1574. if (flags & TTU_RMAP_LOCKED)
  1575. rmap_walk_locked(page, &rwc);
  1576. else
  1577. rmap_walk(page, &rwc);
  1578. /*
  1579. * When racing against e.g. zap_pte_range() on another cpu,
  1580. * in between its ptep_get_and_clear_full() and page_remove_rmap(),
  1581. * try_to_unmap() may return false when it is about to become true,
  1582. * if page table locking is skipped: use TTU_SYNC to wait for that.
  1583. */
  1584. return !page_mapcount(page);
  1585. }
  1586. /**
  1587. * try_to_munlock - try to munlock a page
  1588. * @page: the page to be munlocked
  1589. *
  1590. * Called from munlock code. Checks all of the VMAs mapping the page
  1591. * to make sure nobody else has this page mlocked. The page will be
  1592. * returned with PG_mlocked cleared if no other vmas have it mlocked.
  1593. */
  1594. void try_to_munlock(struct page *page)
  1595. {
  1596. struct rmap_walk_control rwc = {
  1597. .rmap_one = try_to_unmap_one,
  1598. .arg = (void *)TTU_MUNLOCK,
  1599. .done = page_not_mapped,
  1600. .anon_lock = page_lock_anon_vma_read,
  1601. };
  1602. VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
  1603. VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
  1604. rmap_walk(page, &rwc);
  1605. }
  1606. void __put_anon_vma(struct anon_vma *anon_vma)
  1607. {
  1608. struct anon_vma *root = anon_vma->root;
  1609. anon_vma_free(anon_vma);
  1610. if (root != anon_vma && atomic_dec_and_test(&root->refcount))
  1611. anon_vma_free(root);
  1612. }
  1613. static struct anon_vma *rmap_walk_anon_lock(struct page *page,
  1614. struct rmap_walk_control *rwc)
  1615. {
  1616. struct anon_vma *anon_vma;
  1617. if (rwc->anon_lock)
  1618. return rwc->anon_lock(page);
  1619. /*
  1620. * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()
  1621. * because that depends on page_mapped(); but not all its usages
  1622. * are holding mmap_lock. Users without mmap_lock are required to
  1623. * take a reference count to prevent the anon_vma disappearing
  1624. */
  1625. anon_vma = page_anon_vma(page);
  1626. if (!anon_vma)
  1627. return NULL;
  1628. anon_vma_lock_read(anon_vma);
  1629. return anon_vma;
  1630. }
  1631. /*
  1632. * rmap_walk_anon - do something to anonymous page using the object-based
  1633. * rmap method
  1634. * @page: the page to be handled
  1635. * @rwc: control variable according to each walk type
  1636. *
  1637. * Find all the mappings of a page using the mapping pointer and the vma chains
  1638. * contained in the anon_vma struct it points to.
  1639. *
  1640. * When called from try_to_munlock(), the mmap_lock of the mm containing the vma
  1641. * where the page was found will be held for write. So, we won't recheck
  1642. * vm_flags for that VMA. That should be OK, because that vma shouldn't be
  1643. * LOCKED.
  1644. */
  1645. static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
  1646. bool locked)
  1647. {
  1648. struct anon_vma *anon_vma;
  1649. pgoff_t pgoff_start, pgoff_end;
  1650. struct anon_vma_chain *avc;
  1651. if (locked) {
  1652. anon_vma = page_anon_vma(page);
  1653. /* anon_vma disappear under us? */
  1654. VM_BUG_ON_PAGE(!anon_vma, page);
  1655. } else {
  1656. anon_vma = rmap_walk_anon_lock(page, rwc);
  1657. }
  1658. if (!anon_vma)
  1659. return;
  1660. pgoff_start = page_to_pgoff(page);
  1661. pgoff_end = pgoff_start + thp_nr_pages(page) - 1;
  1662. anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
  1663. pgoff_start, pgoff_end) {
  1664. struct vm_area_struct *vma = avc->vma;
  1665. unsigned long address = vma_address(page, vma);
  1666. VM_BUG_ON_VMA(address == -EFAULT, vma);
  1667. cond_resched();
  1668. if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
  1669. continue;
  1670. if (!rwc->rmap_one(page, vma, address, rwc->arg))
  1671. break;
  1672. if (rwc->done && rwc->done(page))
  1673. break;
  1674. }
  1675. if (!locked)
  1676. anon_vma_unlock_read(anon_vma);
  1677. }
  1678. /*
  1679. * rmap_walk_file - do something to file page using the object-based rmap method
  1680. * @page: the page to be handled
  1681. * @rwc: control variable according to each walk type
  1682. *
  1683. * Find all the mappings of a page using the mapping pointer and the vma chains
  1684. * contained in the address_space struct it points to.
  1685. *
  1686. * When called from try_to_munlock(), the mmap_lock of the mm containing the vma
  1687. * where the page was found will be held for write. So, we won't recheck
  1688. * vm_flags for that VMA. That should be OK, because that vma shouldn't be
  1689. * LOCKED.
  1690. */
  1691. static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
  1692. bool locked)
  1693. {
  1694. struct address_space *mapping = page_mapping(page);
  1695. pgoff_t pgoff_start, pgoff_end;
  1696. struct vm_area_struct *vma;
  1697. /*
  1698. * The page lock not only makes sure that page->mapping cannot
  1699. * suddenly be NULLified by truncation, it makes sure that the
  1700. * structure at mapping cannot be freed and reused yet,
  1701. * so we can safely take mapping->i_mmap_rwsem.
  1702. */
  1703. VM_BUG_ON_PAGE(!PageLocked(page), page);
  1704. if (!mapping)
  1705. return;
  1706. pgoff_start = page_to_pgoff(page);
  1707. pgoff_end = pgoff_start + thp_nr_pages(page) - 1;
  1708. if (!locked)
  1709. i_mmap_lock_read(mapping);
  1710. vma_interval_tree_foreach(vma, &mapping->i_mmap,
  1711. pgoff_start, pgoff_end) {
  1712. unsigned long address = vma_address(page, vma);
  1713. VM_BUG_ON_VMA(address == -EFAULT, vma);
  1714. cond_resched();
  1715. if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
  1716. continue;
  1717. if (!rwc->rmap_one(page, vma, address, rwc->arg))
  1718. goto done;
  1719. if (rwc->done && rwc->done(page))
  1720. goto done;
  1721. }
  1722. done:
  1723. if (!locked)
  1724. i_mmap_unlock_read(mapping);
  1725. }
  1726. void rmap_walk(struct page *page, struct rmap_walk_control *rwc)
  1727. {
  1728. if (unlikely(PageKsm(page)))
  1729. rmap_walk_ksm(page, rwc);
  1730. else if (PageAnon(page))
  1731. rmap_walk_anon(page, rwc, false);
  1732. else
  1733. rmap_walk_file(page, rwc, false);
  1734. }
  1735. /* Like rmap_walk, but caller holds relevant rmap lock */
  1736. void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
  1737. {
  1738. /* no ksm support for now */
  1739. VM_BUG_ON_PAGE(PageKsm(page), page);
  1740. if (PageAnon(page))
  1741. rmap_walk_anon(page, rwc, true);
  1742. else
  1743. rmap_walk_file(page, rwc, true);
  1744. }
  1745. #ifdef CONFIG_HUGETLB_PAGE
  1746. /*
  1747. * The following two functions are for anonymous (private mapped) hugepages.
  1748. * Unlike common anonymous pages, anonymous hugepages have no accounting code
  1749. * and no lru code, because we handle hugepages differently from common pages.
  1750. */
  1751. void hugepage_add_anon_rmap(struct page *page,
  1752. struct vm_area_struct *vma, unsigned long address)
  1753. {
  1754. struct anon_vma *anon_vma = vma->anon_vma;
  1755. int first;
  1756. BUG_ON(!PageLocked(page));
  1757. BUG_ON(!anon_vma);
  1758. /* address might be in next vma when migration races vma_adjust */
  1759. first = atomic_inc_and_test(compound_mapcount_ptr(page));
  1760. if (first)
  1761. __page_set_anon_rmap(page, vma, address, 0);
  1762. }
  1763. void hugepage_add_new_anon_rmap(struct page *page,
  1764. struct vm_area_struct *vma, unsigned long address)
  1765. {
  1766. BUG_ON(address < vma->vm_start || address >= vma->vm_end);
  1767. atomic_set(compound_mapcount_ptr(page), 0);
  1768. if (hpage_pincount_available(page))
  1769. atomic_set(compound_pincount_ptr(page), 0);
  1770. __page_set_anon_rmap(page, vma, address, 1);
  1771. }
  1772. #endif /* CONFIG_HUGETLB_PAGE */