rmap.h 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_RMAP_H
  3. #define _LINUX_RMAP_H
  4. /*
  5. * Declarations for Reverse Mapping functions in mm/rmap.c
  6. */
  7. #include <linux/list.h>
  8. #include <linux/slab.h>
  9. #include <linux/mm.h>
  10. #include <linux/rwsem.h>
  11. #include <linux/memcontrol.h>
  12. #include <linux/highmem.h>
  13. /*
  14. * The anon_vma heads a list of private "related" vmas, to scan if
  15. * an anonymous page pointing to this anon_vma needs to be unmapped:
  16. * the vmas on the list will be related by forking, or by splitting.
  17. *
  18. * Since vmas come and go as they are split and merged (particularly
  19. * in mprotect), the mapping field of an anonymous page cannot point
  20. * directly to a vma: instead it points to an anon_vma, on whose list
  21. * the related vmas can be easily linked or unlinked.
  22. *
  23. * After unlinking the last vma on the list, we must garbage collect
  24. * the anon_vma object itself: we're guaranteed no page can be
  25. * pointing to this anon_vma once its vma list is empty.
  26. */
  27. struct anon_vma {
  28. struct anon_vma *root; /* Root of this anon_vma tree */
  29. struct rw_semaphore rwsem; /* W: modification, R: walking the list */
  30. /*
  31. * The refcount is taken on an anon_vma when there is no
  32. * guarantee that the vma of page tables will exist for
  33. * the duration of the operation. A caller that takes
  34. * the reference is responsible for clearing up the
  35. * anon_vma if they are the last user on release
  36. */
  37. atomic_t refcount;
  38. /*
  39. * Count of child anon_vmas and VMAs which points to this anon_vma.
  40. *
  41. * This counter is used for making decision about reusing anon_vma
  42. * instead of forking new one. See comments in function anon_vma_clone.
  43. */
  44. unsigned degree;
  45. struct anon_vma *parent; /* Parent of this anon_vma */
  46. /*
  47. * NOTE: the LSB of the rb_root.rb_node is set by
  48. * mm_take_all_locks() _after_ taking the above lock. So the
  49. * rb_root must only be read/written after taking the above lock
  50. * to be sure to see a valid next pointer. The LSB bit itself
  51. * is serialized by a system wide lock only visible to
  52. * mm_take_all_locks() (mm_all_locks_mutex).
  53. */
  54. /* Interval tree of private "related" vmas */
  55. struct rb_root_cached rb_root;
  56. };
  57. /*
  58. * The copy-on-write semantics of fork mean that an anon_vma
  59. * can become associated with multiple processes. Furthermore,
  60. * each child process will have its own anon_vma, where new
  61. * pages for that process are instantiated.
  62. *
  63. * This structure allows us to find the anon_vmas associated
  64. * with a VMA, or the VMAs associated with an anon_vma.
  65. * The "same_vma" list contains the anon_vma_chains linking
  66. * all the anon_vmas associated with this VMA.
  67. * The "rb" field indexes on an interval tree the anon_vma_chains
  68. * which link all the VMAs associated with this anon_vma.
  69. */
  70. struct anon_vma_chain {
  71. struct vm_area_struct *vma;
  72. struct anon_vma *anon_vma;
  73. struct list_head same_vma; /* locked by mmap_lock & page_table_lock */
  74. struct rb_node rb; /* locked by anon_vma->rwsem */
  75. unsigned long rb_subtree_last;
  76. #ifdef CONFIG_DEBUG_VM_RB
  77. unsigned long cached_vma_start, cached_vma_last;
  78. #endif
  79. };
  80. enum ttu_flags {
  81. TTU_MIGRATION = 0x1, /* migration mode */
  82. TTU_MUNLOCK = 0x2, /* munlock mode */
  83. TTU_SPLIT_HUGE_PMD = 0x4, /* split huge PMD if any */
  84. TTU_IGNORE_MLOCK = 0x8, /* ignore mlock */
  85. TTU_SYNC = 0x10, /* avoid racy checks with PVMW_SYNC */
  86. TTU_IGNORE_HWPOISON = 0x20, /* corrupted page is recoverable */
  87. TTU_BATCH_FLUSH = 0x40, /* Batch TLB flushes where possible
  88. * and caller guarantees they will
  89. * do a final flush if necessary */
  90. TTU_RMAP_LOCKED = 0x80, /* do not grab rmap lock:
  91. * caller holds it */
  92. TTU_SPLIT_FREEZE = 0x100, /* freeze pte under splitting thp */
  93. };
  94. #ifdef CONFIG_MMU
  95. static inline void get_anon_vma(struct anon_vma *anon_vma)
  96. {
  97. atomic_inc(&anon_vma->refcount);
  98. }
  99. void __put_anon_vma(struct anon_vma *anon_vma);
  100. static inline void put_anon_vma(struct anon_vma *anon_vma)
  101. {
  102. if (atomic_dec_and_test(&anon_vma->refcount))
  103. __put_anon_vma(anon_vma);
  104. }
  105. static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
  106. {
  107. down_write(&anon_vma->root->rwsem);
  108. }
  109. static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
  110. {
  111. up_write(&anon_vma->root->rwsem);
  112. }
  113. static inline void anon_vma_lock_read(struct anon_vma *anon_vma)
  114. {
  115. down_read(&anon_vma->root->rwsem);
  116. }
  117. static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
  118. {
  119. up_read(&anon_vma->root->rwsem);
  120. }
  121. /*
  122. * anon_vma helper functions.
  123. */
  124. void anon_vma_init(void); /* create anon_vma_cachep */
  125. int __anon_vma_prepare(struct vm_area_struct *);
  126. void unlink_anon_vmas(struct vm_area_struct *);
  127. int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
  128. int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
  129. static inline int anon_vma_prepare(struct vm_area_struct *vma)
  130. {
  131. if (likely(vma->anon_vma))
  132. return 0;
  133. return __anon_vma_prepare(vma);
  134. }
  135. static inline void anon_vma_merge(struct vm_area_struct *vma,
  136. struct vm_area_struct *next)
  137. {
  138. VM_BUG_ON_VMA(vma->anon_vma != next->anon_vma, vma);
  139. unlink_anon_vmas(next);
  140. }
  141. struct anon_vma *page_get_anon_vma(struct page *page);
  142. /* bitflags for do_page_add_anon_rmap() */
  143. #define RMAP_EXCLUSIVE 0x01
  144. #define RMAP_COMPOUND 0x02
  145. /*
  146. * rmap interfaces called when adding or removing pte of page
  147. */
  148. void page_move_anon_rmap(struct page *, struct vm_area_struct *);
  149. void page_add_anon_rmap(struct page *, struct vm_area_struct *,
  150. unsigned long, bool);
  151. void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
  152. unsigned long, int);
  153. void __page_add_new_anon_rmap(struct page *page, struct vm_area_struct *vma,
  154. unsigned long address, bool compound);
  155. static inline void page_add_new_anon_rmap(struct page *page,
  156. struct vm_area_struct *vma,
  157. unsigned long address, bool compound)
  158. {
  159. VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
  160. __page_add_new_anon_rmap(page, vma, address, compound);
  161. }
  162. void page_add_file_rmap(struct page *, bool);
  163. void page_remove_rmap(struct page *, bool);
  164. void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
  165. unsigned long);
  166. void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,
  167. unsigned long);
  168. static inline void page_dup_rmap(struct page *page, bool compound)
  169. {
  170. atomic_inc(compound ? compound_mapcount_ptr(page) : &page->_mapcount);
  171. }
  172. /*
  173. * Called from mm/vmscan.c to handle paging out
  174. */
  175. int page_referenced(struct page *, int is_locked,
  176. struct mem_cgroup *memcg, unsigned long *vm_flags);
  177. bool try_to_unmap(struct page *, enum ttu_flags flags);
  178. /* Avoid racy checks */
  179. #define PVMW_SYNC (1 << 0)
  180. /* Look for migarion entries rather than present PTEs */
  181. #define PVMW_MIGRATION (1 << 1)
  182. struct page_vma_mapped_walk {
  183. struct page *page;
  184. struct vm_area_struct *vma;
  185. unsigned long address;
  186. pmd_t *pmd;
  187. pte_t *pte;
  188. spinlock_t *ptl;
  189. unsigned int flags;
  190. };
  191. static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
  192. {
  193. /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
  194. if (pvmw->pte && !PageHuge(pvmw->page))
  195. pte_unmap(pvmw->pte);
  196. if (pvmw->ptl)
  197. spin_unlock(pvmw->ptl);
  198. }
  199. bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw);
  200. /*
  201. * Used by swapoff to help locate where page is expected in vma.
  202. */
  203. unsigned long page_address_in_vma(struct page *, struct vm_area_struct *);
  204. /*
  205. * Cleans the PTEs of shared mappings.
  206. * (and since clean PTEs should also be readonly, write protects them too)
  207. *
  208. * returns the number of cleaned PTEs.
  209. */
  210. int page_mkclean(struct page *);
  211. /*
  212. * called in munlock()/munmap() path to check for other vmas holding
  213. * the page mlocked.
  214. */
  215. void try_to_munlock(struct page *);
  216. void remove_migration_ptes(struct page *old, struct page *new, bool locked);
  217. /*
  218. * Called by memory-failure.c to kill processes.
  219. */
  220. struct anon_vma *page_lock_anon_vma_read(struct page *page);
  221. void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
  222. int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
  223. /*
  224. * rmap_walk_control: To control rmap traversing for specific needs
  225. *
  226. * arg: passed to rmap_one() and invalid_vma()
  227. * rmap_one: executed on each vma where page is mapped
  228. * done: for checking traversing termination condition
  229. * anon_lock: for getting anon_lock by optimized way rather than default
  230. * invalid_vma: for skipping uninterested vma
  231. */
  232. struct rmap_walk_control {
  233. void *arg;
  234. /*
  235. * Return false if page table scanning in rmap_walk should be stopped.
  236. * Otherwise, return true.
  237. */
  238. bool (*rmap_one)(struct page *page, struct vm_area_struct *vma,
  239. unsigned long addr, void *arg);
  240. int (*done)(struct page *page);
  241. struct anon_vma *(*anon_lock)(struct page *page);
  242. bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
  243. };
  244. void rmap_walk(struct page *page, struct rmap_walk_control *rwc);
  245. void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);
  246. #else /* !CONFIG_MMU */
  247. #define anon_vma_init() do {} while (0)
  248. #define anon_vma_prepare(vma) (0)
  249. #define anon_vma_link(vma) do {} while (0)
  250. static inline int page_referenced(struct page *page, int is_locked,
  251. struct mem_cgroup *memcg,
  252. unsigned long *vm_flags)
  253. {
  254. *vm_flags = 0;
  255. return 0;
  256. }
  257. #define try_to_unmap(page, refs) false
  258. static inline int page_mkclean(struct page *page)
  259. {
  260. return 0;
  261. }
  262. #endif /* CONFIG_MMU */
  263. #endif /* _LINUX_RMAP_H */