rmap.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941
  1. /*
  2. * mm/rmap.c - physical to virtual reverse mappings
  3. *
  4. * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
  5. * Released under the General Public License (GPL).
  6. *
  7. * Simple, low overhead reverse mapping scheme.
  8. * Please try to keep this thing as modular as possible.
  9. *
  10. * Provides methods for unmapping each kind of mapped page:
  11. * the anon methods track anonymous pages, and
  12. * the file methods track pages belonging to an inode.
  13. *
  14. * Original design by Rik van Riel <riel@conectiva.com.br> 2001
  15. * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
  16. * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
  17. * Contributions by Hugh Dickins <hugh@veritas.com> 2003, 2004
  18. */
  19. /*
  20. * Lock ordering in mm:
  21. *
  22. * inode->i_mutex (while writing or truncating, not reading or faulting)
  23. * inode->i_alloc_sem (vmtruncate_range)
  24. * mm->mmap_sem
  25. * page->flags PG_locked (lock_page)
  26. * mapping->i_mmap_lock
  27. * anon_vma->lock
  28. * mm->page_table_lock or pte_lock
  29. * zone->lru_lock (in mark_page_accessed, isolate_lru_page)
  30. * swap_lock (in swap_duplicate, swap_info_get)
  31. * mmlist_lock (in mmput, drain_mmlist and others)
  32. * mapping->private_lock (in __set_page_dirty_buffers)
  33. * inode_lock (in set_page_dirty's __mark_inode_dirty)
  34. * sb_lock (within inode_lock in fs/fs-writeback.c)
  35. * mapping->tree_lock (widely used, in set_page_dirty,
  36. * in arch-dependent flush_dcache_mmap_lock,
  37. * within inode_lock in __sync_single_inode)
  38. */
  39. #include <linux/mm.h>
  40. #include <linux/pagemap.h>
  41. #include <linux/swap.h>
  42. #include <linux/swapops.h>
  43. #include <linux/slab.h>
  44. #include <linux/init.h>
  45. #include <linux/rmap.h>
  46. #include <linux/rcupdate.h>
  47. #include <linux/module.h>
  48. #include <linux/kallsyms.h>
  49. #include <asm/tlbflush.h>
  50. struct kmem_cache *anon_vma_cachep;
  51. static inline void validate_anon_vma(struct vm_area_struct *find_vma)
  52. {
  53. #ifdef CONFIG_DEBUG_VM
  54. struct anon_vma *anon_vma = find_vma->anon_vma;
  55. struct vm_area_struct *vma;
  56. unsigned int mapcount = 0;
  57. int found = 0;
  58. list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
  59. mapcount++;
  60. BUG_ON(mapcount > 100000);
  61. if (vma == find_vma)
  62. found = 1;
  63. }
  64. BUG_ON(!found);
  65. #endif
  66. }
  67. /* This must be called under the mmap_sem. */
  68. int anon_vma_prepare(struct vm_area_struct *vma)
  69. {
  70. struct anon_vma *anon_vma = vma->anon_vma;
  71. might_sleep();
  72. if (unlikely(!anon_vma)) {
  73. struct mm_struct *mm = vma->vm_mm;
  74. struct anon_vma *allocated, *locked;
  75. anon_vma = find_mergeable_anon_vma(vma);
  76. if (anon_vma) {
  77. allocated = NULL;
  78. locked = anon_vma;
  79. spin_lock(&locked->lock);
  80. } else {
  81. anon_vma = anon_vma_alloc();
  82. if (unlikely(!anon_vma))
  83. return -ENOMEM;
  84. allocated = anon_vma;
  85. locked = NULL;
  86. }
  87. /* page_table_lock to protect against threads */
  88. spin_lock(&mm->page_table_lock);
  89. if (likely(!vma->anon_vma)) {
  90. vma->anon_vma = anon_vma;
  91. list_add_tail(&vma->anon_vma_node, &anon_vma->head);
  92. allocated = NULL;
  93. }
  94. spin_unlock(&mm->page_table_lock);
  95. if (locked)
  96. spin_unlock(&locked->lock);
  97. if (unlikely(allocated))
  98. anon_vma_free(allocated);
  99. }
  100. return 0;
  101. }
  102. void __anon_vma_merge(struct vm_area_struct *vma, struct vm_area_struct *next)
  103. {
  104. BUG_ON(vma->anon_vma != next->anon_vma);
  105. list_del(&next->anon_vma_node);
  106. }
  107. void __anon_vma_link(struct vm_area_struct *vma)
  108. {
  109. struct anon_vma *anon_vma = vma->anon_vma;
  110. if (anon_vma) {
  111. list_add_tail(&vma->anon_vma_node, &anon_vma->head);
  112. validate_anon_vma(vma);
  113. }
  114. }
  115. void anon_vma_link(struct vm_area_struct *vma)
  116. {
  117. struct anon_vma *anon_vma = vma->anon_vma;
  118. if (anon_vma) {
  119. spin_lock(&anon_vma->lock);
  120. list_add_tail(&vma->anon_vma_node, &anon_vma->head);
  121. validate_anon_vma(vma);
  122. spin_unlock(&anon_vma->lock);
  123. }
  124. }
  125. void anon_vma_unlink(struct vm_area_struct *vma)
  126. {
  127. struct anon_vma *anon_vma = vma->anon_vma;
  128. int empty;
  129. if (!anon_vma)
  130. return;
  131. spin_lock(&anon_vma->lock);
  132. validate_anon_vma(vma);
  133. list_del(&vma->anon_vma_node);
  134. /* We must garbage collect the anon_vma if it's empty */
  135. empty = list_empty(&anon_vma->head);
  136. spin_unlock(&anon_vma->lock);
  137. if (empty)
  138. anon_vma_free(anon_vma);
  139. }
  140. static void anon_vma_ctor(void *data, struct kmem_cache *cachep,
  141. unsigned long flags)
  142. {
  143. if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
  144. SLAB_CTOR_CONSTRUCTOR) {
  145. struct anon_vma *anon_vma = data;
  146. spin_lock_init(&anon_vma->lock);
  147. INIT_LIST_HEAD(&anon_vma->head);
  148. }
  149. }
  150. void __init anon_vma_init(void)
  151. {
  152. anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
  153. 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor, NULL);
  154. }
  155. /*
  156. * Getting a lock on a stable anon_vma from a page off the LRU is
  157. * tricky: page_lock_anon_vma rely on RCU to guard against the races.
  158. */
  159. static struct anon_vma *page_lock_anon_vma(struct page *page)
  160. {
  161. struct anon_vma *anon_vma;
  162. unsigned long anon_mapping;
  163. rcu_read_lock();
  164. anon_mapping = (unsigned long) page->mapping;
  165. if (!(anon_mapping & PAGE_MAPPING_ANON))
  166. goto out;
  167. if (!page_mapped(page))
  168. goto out;
  169. anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
  170. spin_lock(&anon_vma->lock);
  171. return anon_vma;
  172. out:
  173. rcu_read_unlock();
  174. return NULL;
  175. }
  176. static void page_unlock_anon_vma(struct anon_vma *anon_vma)
  177. {
  178. spin_unlock(&anon_vma->lock);
  179. rcu_read_unlock();
  180. }
  181. /*
  182. * At what user virtual address is page expected in vma?
  183. */
  184. static inline unsigned long
  185. vma_address(struct page *page, struct vm_area_struct *vma)
  186. {
  187. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  188. unsigned long address;
  189. address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
  190. if (unlikely(address < vma->vm_start || address >= vma->vm_end)) {
  191. /* page should be within any vma from prio_tree_next */
  192. BUG_ON(!PageAnon(page));
  193. return -EFAULT;
  194. }
  195. return address;
  196. }
  197. /*
  198. * At what user virtual address is page expected in vma? checking that the
  199. * page matches the vma: currently only used on anon pages, by unuse_vma;
  200. */
  201. unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
  202. {
  203. if (PageAnon(page)) {
  204. if ((void *)vma->anon_vma !=
  205. (void *)page->mapping - PAGE_MAPPING_ANON)
  206. return -EFAULT;
  207. } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
  208. if (!vma->vm_file ||
  209. vma->vm_file->f_mapping != page->mapping)
  210. return -EFAULT;
  211. } else
  212. return -EFAULT;
  213. return vma_address(page, vma);
  214. }
  215. /*
  216. * Check that @page is mapped at @address into @mm.
  217. *
  218. * On success returns with pte mapped and locked.
  219. */
  220. pte_t *page_check_address(struct page *page, struct mm_struct *mm,
  221. unsigned long address, spinlock_t **ptlp)
  222. {
  223. pgd_t *pgd;
  224. pud_t *pud;
  225. pmd_t *pmd;
  226. pte_t *pte;
  227. spinlock_t *ptl;
  228. pgd = pgd_offset(mm, address);
  229. if (!pgd_present(*pgd))
  230. return NULL;
  231. pud = pud_offset(pgd, address);
  232. if (!pud_present(*pud))
  233. return NULL;
  234. pmd = pmd_offset(pud, address);
  235. if (!pmd_present(*pmd))
  236. return NULL;
  237. pte = pte_offset_map(pmd, address);
  238. /* Make a quick check before getting the lock */
  239. if (!pte_present(*pte)) {
  240. pte_unmap(pte);
  241. return NULL;
  242. }
  243. ptl = pte_lockptr(mm, pmd);
  244. spin_lock(ptl);
  245. if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) {
  246. *ptlp = ptl;
  247. return pte;
  248. }
  249. pte_unmap_unlock(pte, ptl);
  250. return NULL;
  251. }
  252. /*
  253. * Subfunctions of page_referenced: page_referenced_one called
  254. * repeatedly from either page_referenced_anon or page_referenced_file.
  255. */
  256. static int page_referenced_one(struct page *page,
  257. struct vm_area_struct *vma, unsigned int *mapcount)
  258. {
  259. struct mm_struct *mm = vma->vm_mm;
  260. unsigned long address;
  261. pte_t *pte;
  262. spinlock_t *ptl;
  263. int referenced = 0;
  264. address = vma_address(page, vma);
  265. if (address == -EFAULT)
  266. goto out;
  267. pte = page_check_address(page, mm, address, &ptl);
  268. if (!pte)
  269. goto out;
  270. if (ptep_clear_flush_young(vma, address, pte))
  271. referenced++;
  272. /* Pretend the page is referenced if the task has the
  273. swap token and is in the middle of a page fault. */
  274. if (mm != current->mm && has_swap_token(mm) &&
  275. rwsem_is_locked(&mm->mmap_sem))
  276. referenced++;
  277. (*mapcount)--;
  278. pte_unmap_unlock(pte, ptl);
  279. out:
  280. return referenced;
  281. }
  282. static int page_referenced_anon(struct page *page)
  283. {
  284. unsigned int mapcount;
  285. struct anon_vma *anon_vma;
  286. struct vm_area_struct *vma;
  287. int referenced = 0;
  288. anon_vma = page_lock_anon_vma(page);
  289. if (!anon_vma)
  290. return referenced;
  291. mapcount = page_mapcount(page);
  292. list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
  293. referenced += page_referenced_one(page, vma, &mapcount);
  294. if (!mapcount)
  295. break;
  296. }
  297. page_unlock_anon_vma(anon_vma);
  298. return referenced;
  299. }
  300. /**
  301. * page_referenced_file - referenced check for object-based rmap
  302. * @page: the page we're checking references on.
  303. *
  304. * For an object-based mapped page, find all the places it is mapped and
  305. * check/clear the referenced flag. This is done by following the page->mapping
  306. * pointer, then walking the chain of vmas it holds. It returns the number
  307. * of references it found.
  308. *
  309. * This function is only called from page_referenced for object-based pages.
  310. */
  311. static int page_referenced_file(struct page *page)
  312. {
  313. unsigned int mapcount;
  314. struct address_space *mapping = page->mapping;
  315. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  316. struct vm_area_struct *vma;
  317. struct prio_tree_iter iter;
  318. int referenced = 0;
  319. /*
  320. * The caller's checks on page->mapping and !PageAnon have made
  321. * sure that this is a file page: the check for page->mapping
  322. * excludes the case just before it gets set on an anon page.
  323. */
  324. BUG_ON(PageAnon(page));
  325. /*
  326. * The page lock not only makes sure that page->mapping cannot
  327. * suddenly be NULLified by truncation, it makes sure that the
  328. * structure at mapping cannot be freed and reused yet,
  329. * so we can safely take mapping->i_mmap_lock.
  330. */
  331. BUG_ON(!PageLocked(page));
  332. spin_lock(&mapping->i_mmap_lock);
  333. /*
  334. * i_mmap_lock does not stabilize mapcount at all, but mapcount
  335. * is more likely to be accurate if we note it after spinning.
  336. */
  337. mapcount = page_mapcount(page);
  338. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
  339. if ((vma->vm_flags & (VM_LOCKED|VM_MAYSHARE))
  340. == (VM_LOCKED|VM_MAYSHARE)) {
  341. referenced++;
  342. break;
  343. }
  344. referenced += page_referenced_one(page, vma, &mapcount);
  345. if (!mapcount)
  346. break;
  347. }
  348. spin_unlock(&mapping->i_mmap_lock);
  349. return referenced;
  350. }
  351. /**
  352. * page_referenced - test if the page was referenced
  353. * @page: the page to test
  354. * @is_locked: caller holds lock on the page
  355. *
  356. * Quick test_and_clear_referenced for all mappings to a page,
  357. * returns the number of ptes which referenced the page.
  358. */
  359. int page_referenced(struct page *page, int is_locked)
  360. {
  361. int referenced = 0;
  362. if (page_test_and_clear_young(page))
  363. referenced++;
  364. if (TestClearPageReferenced(page))
  365. referenced++;
  366. if (page_mapped(page) && page->mapping) {
  367. if (PageAnon(page))
  368. referenced += page_referenced_anon(page);
  369. else if (is_locked)
  370. referenced += page_referenced_file(page);
  371. else if (TestSetPageLocked(page))
  372. referenced++;
  373. else {
  374. if (page->mapping)
  375. referenced += page_referenced_file(page);
  376. unlock_page(page);
  377. }
  378. }
  379. return referenced;
  380. }
  381. static int page_mkclean_one(struct page *page, struct vm_area_struct *vma)
  382. {
  383. struct mm_struct *mm = vma->vm_mm;
  384. unsigned long address;
  385. pte_t *pte;
  386. spinlock_t *ptl;
  387. int ret = 0;
  388. address = vma_address(page, vma);
  389. if (address == -EFAULT)
  390. goto out;
  391. pte = page_check_address(page, mm, address, &ptl);
  392. if (!pte)
  393. goto out;
  394. if (pte_dirty(*pte) || pte_write(*pte)) {
  395. pte_t entry;
  396. flush_cache_page(vma, address, pte_pfn(*pte));
  397. entry = ptep_clear_flush(vma, address, pte);
  398. entry = pte_wrprotect(entry);
  399. entry = pte_mkclean(entry);
  400. set_pte_at(mm, address, pte, entry);
  401. lazy_mmu_prot_update(entry);
  402. ret = 1;
  403. }
  404. pte_unmap_unlock(pte, ptl);
  405. out:
  406. return ret;
  407. }
  408. static int page_mkclean_file(struct address_space *mapping, struct page *page)
  409. {
  410. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  411. struct vm_area_struct *vma;
  412. struct prio_tree_iter iter;
  413. int ret = 0;
  414. BUG_ON(PageAnon(page));
  415. spin_lock(&mapping->i_mmap_lock);
  416. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
  417. if (vma->vm_flags & VM_SHARED)
  418. ret += page_mkclean_one(page, vma);
  419. }
  420. spin_unlock(&mapping->i_mmap_lock);
  421. return ret;
  422. }
  423. int page_mkclean(struct page *page)
  424. {
  425. int ret = 0;
  426. BUG_ON(!PageLocked(page));
  427. if (page_mapped(page)) {
  428. struct address_space *mapping = page_mapping(page);
  429. if (mapping)
  430. ret = page_mkclean_file(mapping, page);
  431. if (page_test_and_clear_dirty(page))
  432. ret = 1;
  433. }
  434. return ret;
  435. }
  436. /**
  437. * page_set_anon_rmap - setup new anonymous rmap
  438. * @page: the page to add the mapping to
  439. * @vma: the vm area in which the mapping is added
  440. * @address: the user virtual address mapped
  441. */
  442. static void __page_set_anon_rmap(struct page *page,
  443. struct vm_area_struct *vma, unsigned long address)
  444. {
  445. struct anon_vma *anon_vma = vma->anon_vma;
  446. BUG_ON(!anon_vma);
  447. anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
  448. page->mapping = (struct address_space *) anon_vma;
  449. page->index = linear_page_index(vma, address);
  450. /*
  451. * nr_mapped state can be updated without turning off
  452. * interrupts because it is not modified via interrupt.
  453. */
  454. __inc_zone_page_state(page, NR_ANON_PAGES);
  455. }
  456. /**
  457. * page_add_anon_rmap - add pte mapping to an anonymous page
  458. * @page: the page to add the mapping to
  459. * @vma: the vm area in which the mapping is added
  460. * @address: the user virtual address mapped
  461. *
  462. * The caller needs to hold the pte lock.
  463. */
  464. void page_add_anon_rmap(struct page *page,
  465. struct vm_area_struct *vma, unsigned long address)
  466. {
  467. if (atomic_inc_and_test(&page->_mapcount))
  468. __page_set_anon_rmap(page, vma, address);
  469. /* else checking page index and mapping is racy */
  470. }
  471. /*
  472. * page_add_new_anon_rmap - add pte mapping to a new anonymous page
  473. * @page: the page to add the mapping to
  474. * @vma: the vm area in which the mapping is added
  475. * @address: the user virtual address mapped
  476. *
  477. * Same as page_add_anon_rmap but must only be called on *new* pages.
  478. * This means the inc-and-test can be bypassed.
  479. */
  480. void page_add_new_anon_rmap(struct page *page,
  481. struct vm_area_struct *vma, unsigned long address)
  482. {
  483. atomic_set(&page->_mapcount, 0); /* elevate count by 1 (starts at -1) */
  484. __page_set_anon_rmap(page, vma, address);
  485. }
  486. /**
  487. * page_add_file_rmap - add pte mapping to a file page
  488. * @page: the page to add the mapping to
  489. *
  490. * The caller needs to hold the pte lock.
  491. */
  492. void page_add_file_rmap(struct page *page)
  493. {
  494. if (atomic_inc_and_test(&page->_mapcount))
  495. __inc_zone_page_state(page, NR_FILE_MAPPED);
  496. }
  497. /**
  498. * page_remove_rmap - take down pte mapping from a page
  499. * @page: page to remove mapping from
  500. *
  501. * The caller needs to hold the pte lock.
  502. */
  503. void page_remove_rmap(struct page *page, struct vm_area_struct *vma)
  504. {
  505. if (atomic_add_negative(-1, &page->_mapcount)) {
  506. if (unlikely(page_mapcount(page) < 0)) {
  507. printk (KERN_EMERG "Eeek! page_mapcount(page) went negative! (%d)\n", page_mapcount(page));
  508. printk (KERN_EMERG " page pfn = %lx\n", page_to_pfn(page));
  509. printk (KERN_EMERG " page->flags = %lx\n", page->flags);
  510. printk (KERN_EMERG " page->count = %x\n", page_count(page));
  511. printk (KERN_EMERG " page->mapping = %p\n", page->mapping);
  512. print_symbol (KERN_EMERG " vma->vm_ops = %s\n", (unsigned long)vma->vm_ops);
  513. if (vma->vm_ops)
  514. print_symbol (KERN_EMERG " vma->vm_ops->nopage = %s\n", (unsigned long)vma->vm_ops->nopage);
  515. if (vma->vm_file && vma->vm_file->f_op)
  516. print_symbol (KERN_EMERG " vma->vm_file->f_op->mmap = %s\n", (unsigned long)vma->vm_file->f_op->mmap);
  517. BUG();
  518. }
  519. /*
  520. * It would be tidy to reset the PageAnon mapping here,
  521. * but that might overwrite a racing page_add_anon_rmap
  522. * which increments mapcount after us but sets mapping
  523. * before us: so leave the reset to free_hot_cold_page,
  524. * and remember that it's only reliable while mapped.
  525. * Leaving it set also helps swapoff to reinstate ptes
  526. * faster for those pages still in swapcache.
  527. */
  528. if (page_test_and_clear_dirty(page))
  529. set_page_dirty(page);
  530. __dec_zone_page_state(page,
  531. PageAnon(page) ? NR_ANON_PAGES : NR_FILE_MAPPED);
  532. }
  533. }
  534. /*
  535. * Subfunctions of try_to_unmap: try_to_unmap_one called
  536. * repeatedly from either try_to_unmap_anon or try_to_unmap_file.
  537. */
  538. static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
  539. int migration)
  540. {
  541. struct mm_struct *mm = vma->vm_mm;
  542. unsigned long address;
  543. pte_t *pte;
  544. pte_t pteval;
  545. spinlock_t *ptl;
  546. int ret = SWAP_AGAIN;
  547. address = vma_address(page, vma);
  548. if (address == -EFAULT)
  549. goto out;
  550. pte = page_check_address(page, mm, address, &ptl);
  551. if (!pte)
  552. goto out;
  553. /*
  554. * If the page is mlock()d, we cannot swap it out.
  555. * If it's recently referenced (perhaps page_referenced
  556. * skipped over this mm) then we should reactivate it.
  557. */
  558. if (!migration && ((vma->vm_flags & VM_LOCKED) ||
  559. (ptep_clear_flush_young(vma, address, pte)))) {
  560. ret = SWAP_FAIL;
  561. goto out_unmap;
  562. }
  563. /* Nuke the page table entry. */
  564. flush_cache_page(vma, address, page_to_pfn(page));
  565. pteval = ptep_clear_flush(vma, address, pte);
  566. /* Move the dirty bit to the physical page now the pte is gone. */
  567. if (pte_dirty(pteval))
  568. set_page_dirty(page);
  569. /* Update high watermark before we lower rss */
  570. update_hiwater_rss(mm);
  571. if (PageAnon(page)) {
  572. swp_entry_t entry = { .val = page_private(page) };
  573. if (PageSwapCache(page)) {
  574. /*
  575. * Store the swap location in the pte.
  576. * See handle_pte_fault() ...
  577. */
  578. swap_duplicate(entry);
  579. if (list_empty(&mm->mmlist)) {
  580. spin_lock(&mmlist_lock);
  581. if (list_empty(&mm->mmlist))
  582. list_add(&mm->mmlist, &init_mm.mmlist);
  583. spin_unlock(&mmlist_lock);
  584. }
  585. dec_mm_counter(mm, anon_rss);
  586. #ifdef CONFIG_MIGRATION
  587. } else {
  588. /*
  589. * Store the pfn of the page in a special migration
  590. * pte. do_swap_page() will wait until the migration
  591. * pte is removed and then restart fault handling.
  592. */
  593. BUG_ON(!migration);
  594. entry = make_migration_entry(page, pte_write(pteval));
  595. #endif
  596. }
  597. set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
  598. BUG_ON(pte_file(*pte));
  599. } else
  600. #ifdef CONFIG_MIGRATION
  601. if (migration) {
  602. /* Establish migration entry for a file page */
  603. swp_entry_t entry;
  604. entry = make_migration_entry(page, pte_write(pteval));
  605. set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
  606. } else
  607. #endif
  608. dec_mm_counter(mm, file_rss);
  609. page_remove_rmap(page, vma);
  610. page_cache_release(page);
  611. out_unmap:
  612. pte_unmap_unlock(pte, ptl);
  613. out:
  614. return ret;
  615. }
  616. /*
  617. * objrmap doesn't work for nonlinear VMAs because the assumption that
  618. * offset-into-file correlates with offset-into-virtual-addresses does not hold.
  619. * Consequently, given a particular page and its ->index, we cannot locate the
  620. * ptes which are mapping that page without an exhaustive linear search.
  621. *
  622. * So what this code does is a mini "virtual scan" of each nonlinear VMA which
  623. * maps the file to which the target page belongs. The ->vm_private_data field
  624. * holds the current cursor into that scan. Successive searches will circulate
  625. * around the vma's virtual address space.
  626. *
  627. * So as more replacement pressure is applied to the pages in a nonlinear VMA,
  628. * more scanning pressure is placed against them as well. Eventually pages
  629. * will become fully unmapped and are eligible for eviction.
  630. *
  631. * For very sparsely populated VMAs this is a little inefficient - chances are
  632. * there there won't be many ptes located within the scan cluster. In this case
  633. * maybe we could scan further - to the end of the pte page, perhaps.
  634. */
  635. #define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE)
  636. #define CLUSTER_MASK (~(CLUSTER_SIZE - 1))
  637. static void try_to_unmap_cluster(unsigned long cursor,
  638. unsigned int *mapcount, struct vm_area_struct *vma)
  639. {
  640. struct mm_struct *mm = vma->vm_mm;
  641. pgd_t *pgd;
  642. pud_t *pud;
  643. pmd_t *pmd;
  644. pte_t *pte;
  645. pte_t pteval;
  646. spinlock_t *ptl;
  647. struct page *page;
  648. unsigned long address;
  649. unsigned long end;
  650. address = (vma->vm_start + cursor) & CLUSTER_MASK;
  651. end = address + CLUSTER_SIZE;
  652. if (address < vma->vm_start)
  653. address = vma->vm_start;
  654. if (end > vma->vm_end)
  655. end = vma->vm_end;
  656. pgd = pgd_offset(mm, address);
  657. if (!pgd_present(*pgd))
  658. return;
  659. pud = pud_offset(pgd, address);
  660. if (!pud_present(*pud))
  661. return;
  662. pmd = pmd_offset(pud, address);
  663. if (!pmd_present(*pmd))
  664. return;
  665. pte = pte_offset_map_lock(mm, pmd, address, &ptl);
  666. /* Update high watermark before we lower rss */
  667. update_hiwater_rss(mm);
  668. for (; address < end; pte++, address += PAGE_SIZE) {
  669. if (!pte_present(*pte))
  670. continue;
  671. page = vm_normal_page(vma, address, *pte);
  672. BUG_ON(!page || PageAnon(page));
  673. if (ptep_clear_flush_young(vma, address, pte))
  674. continue;
  675. /* Nuke the page table entry. */
  676. flush_cache_page(vma, address, pte_pfn(*pte));
  677. pteval = ptep_clear_flush(vma, address, pte);
  678. /* If nonlinear, store the file page offset in the pte. */
  679. if (page->index != linear_page_index(vma, address))
  680. set_pte_at(mm, address, pte, pgoff_to_pte(page->index));
  681. /* Move the dirty bit to the physical page now the pte is gone. */
  682. if (pte_dirty(pteval))
  683. set_page_dirty(page);
  684. page_remove_rmap(page, vma);
  685. page_cache_release(page);
  686. dec_mm_counter(mm, file_rss);
  687. (*mapcount)--;
  688. }
  689. pte_unmap_unlock(pte - 1, ptl);
  690. }
  691. static int try_to_unmap_anon(struct page *page, int migration)
  692. {
  693. struct anon_vma *anon_vma;
  694. struct vm_area_struct *vma;
  695. int ret = SWAP_AGAIN;
  696. anon_vma = page_lock_anon_vma(page);
  697. if (!anon_vma)
  698. return ret;
  699. list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
  700. ret = try_to_unmap_one(page, vma, migration);
  701. if (ret == SWAP_FAIL || !page_mapped(page))
  702. break;
  703. }
  704. page_unlock_anon_vma(anon_vma);
  705. return ret;
  706. }
  707. /**
  708. * try_to_unmap_file - unmap file page using the object-based rmap method
  709. * @page: the page to unmap
  710. *
  711. * Find all the mappings of a page using the mapping pointer and the vma chains
  712. * contained in the address_space struct it points to.
  713. *
  714. * This function is only called from try_to_unmap for object-based pages.
  715. */
  716. static int try_to_unmap_file(struct page *page, int migration)
  717. {
  718. struct address_space *mapping = page->mapping;
  719. pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  720. struct vm_area_struct *vma;
  721. struct prio_tree_iter iter;
  722. int ret = SWAP_AGAIN;
  723. unsigned long cursor;
  724. unsigned long max_nl_cursor = 0;
  725. unsigned long max_nl_size = 0;
  726. unsigned int mapcount;
  727. spin_lock(&mapping->i_mmap_lock);
  728. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
  729. ret = try_to_unmap_one(page, vma, migration);
  730. if (ret == SWAP_FAIL || !page_mapped(page))
  731. goto out;
  732. }
  733. if (list_empty(&mapping->i_mmap_nonlinear))
  734. goto out;
  735. list_for_each_entry(vma, &mapping->i_mmap_nonlinear,
  736. shared.vm_set.list) {
  737. if ((vma->vm_flags & VM_LOCKED) && !migration)
  738. continue;
  739. cursor = (unsigned long) vma->vm_private_data;
  740. if (cursor > max_nl_cursor)
  741. max_nl_cursor = cursor;
  742. cursor = vma->vm_end - vma->vm_start;
  743. if (cursor > max_nl_size)
  744. max_nl_size = cursor;
  745. }
  746. if (max_nl_size == 0) { /* any nonlinears locked or reserved */
  747. ret = SWAP_FAIL;
  748. goto out;
  749. }
  750. /*
  751. * We don't try to search for this page in the nonlinear vmas,
  752. * and page_referenced wouldn't have found it anyway. Instead
  753. * just walk the nonlinear vmas trying to age and unmap some.
  754. * The mapcount of the page we came in with is irrelevant,
  755. * but even so use it as a guide to how hard we should try?
  756. */
  757. mapcount = page_mapcount(page);
  758. if (!mapcount)
  759. goto out;
  760. cond_resched_lock(&mapping->i_mmap_lock);
  761. max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK;
  762. if (max_nl_cursor == 0)
  763. max_nl_cursor = CLUSTER_SIZE;
  764. do {
  765. list_for_each_entry(vma, &mapping->i_mmap_nonlinear,
  766. shared.vm_set.list) {
  767. if ((vma->vm_flags & VM_LOCKED) && !migration)
  768. continue;
  769. cursor = (unsigned long) vma->vm_private_data;
  770. while ( cursor < max_nl_cursor &&
  771. cursor < vma->vm_end - vma->vm_start) {
  772. try_to_unmap_cluster(cursor, &mapcount, vma);
  773. cursor += CLUSTER_SIZE;
  774. vma->vm_private_data = (void *) cursor;
  775. if ((int)mapcount <= 0)
  776. goto out;
  777. }
  778. vma->vm_private_data = (void *) max_nl_cursor;
  779. }
  780. cond_resched_lock(&mapping->i_mmap_lock);
  781. max_nl_cursor += CLUSTER_SIZE;
  782. } while (max_nl_cursor <= max_nl_size);
  783. /*
  784. * Don't loop forever (perhaps all the remaining pages are
  785. * in locked vmas). Reset cursor on all unreserved nonlinear
  786. * vmas, now forgetting on which ones it had fallen behind.
  787. */
  788. list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
  789. vma->vm_private_data = NULL;
  790. out:
  791. spin_unlock(&mapping->i_mmap_lock);
  792. return ret;
  793. }
  794. /**
  795. * try_to_unmap - try to remove all page table mappings to a page
  796. * @page: the page to get unmapped
  797. *
  798. * Tries to remove all the page table entries which are mapping this
  799. * page, used in the pageout path. Caller must hold the page lock.
  800. * Return values are:
  801. *
  802. * SWAP_SUCCESS - we succeeded in removing all mappings
  803. * SWAP_AGAIN - we missed a mapping, try again later
  804. * SWAP_FAIL - the page is unswappable
  805. */
  806. int try_to_unmap(struct page *page, int migration)
  807. {
  808. int ret;
  809. BUG_ON(!PageLocked(page));
  810. if (PageAnon(page))
  811. ret = try_to_unmap_anon(page, migration);
  812. else
  813. ret = try_to_unmap_file(page, migration);
  814. if (!page_mapped(page))
  815. ret = SWAP_SUCCESS;
  816. return ret;
  817. }