mapping_dirty_helpers.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/pagewalk.h>
  3. #include <linux/hugetlb.h>
  4. #include <linux/bitops.h>
  5. #include <linux/mmu_notifier.h>
  6. #include <asm/cacheflush.h>
  7. #include <asm/tlbflush.h>
  8. /**
  9. * struct wp_walk - Private struct for pagetable walk callbacks
  10. * @range: Range for mmu notifiers
  11. * @tlbflush_start: Address of first modified pte
  12. * @tlbflush_end: Address of last modified pte + 1
  13. * @total: Total number of modified ptes
  14. */
  15. struct wp_walk {
  16. struct mmu_notifier_range range;
  17. unsigned long tlbflush_start;
  18. unsigned long tlbflush_end;
  19. unsigned long total;
  20. };
  21. /**
  22. * wp_pte - Write-protect a pte
  23. * @pte: Pointer to the pte
  24. * @addr: The virtual page address
  25. * @walk: pagetable walk callback argument
  26. *
  27. * The function write-protects a pte and records the range in
  28. * virtual address space of touched ptes for efficient range TLB flushes.
  29. */
  30. static int wp_pte(pte_t *pte, unsigned long addr, unsigned long end,
  31. struct mm_walk *walk)
  32. {
  33. struct wp_walk *wpwalk = walk->private;
  34. pte_t ptent = *pte;
  35. if (pte_write(ptent)) {
  36. pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);
  37. ptent = pte_wrprotect(old_pte);
  38. ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent);
  39. wpwalk->total++;
  40. wpwalk->tlbflush_start = min(wpwalk->tlbflush_start, addr);
  41. wpwalk->tlbflush_end = max(wpwalk->tlbflush_end,
  42. addr + PAGE_SIZE);
  43. }
  44. return 0;
  45. }
  46. /**
  47. * struct clean_walk - Private struct for the clean_record_pte function.
  48. * @base: struct wp_walk we derive from
  49. * @bitmap_pgoff: Address_space Page offset of the first bit in @bitmap
  50. * @bitmap: Bitmap with one bit for each page offset in the address_space range
  51. * covered.
  52. * @start: Address_space page offset of first modified pte relative
  53. * to @bitmap_pgoff
  54. * @end: Address_space page offset of last modified pte relative
  55. * to @bitmap_pgoff
  56. */
  57. struct clean_walk {
  58. struct wp_walk base;
  59. pgoff_t bitmap_pgoff;
  60. unsigned long *bitmap;
  61. pgoff_t start;
  62. pgoff_t end;
  63. };
  64. #define to_clean_walk(_wpwalk) container_of(_wpwalk, struct clean_walk, base)
  65. /**
  66. * clean_record_pte - Clean a pte and record its address space offset in a
  67. * bitmap
  68. * @pte: Pointer to the pte
  69. * @addr: The virtual page address
  70. * @walk: pagetable walk callback argument
  71. *
  72. * The function cleans a pte and records the range in
  73. * virtual address space of touched ptes for efficient TLB flushes.
  74. * It also records dirty ptes in a bitmap representing page offsets
  75. * in the address_space, as well as the first and last of the bits
  76. * touched.
  77. */
  78. static int clean_record_pte(pte_t *pte, unsigned long addr,
  79. unsigned long end, struct mm_walk *walk)
  80. {
  81. struct wp_walk *wpwalk = walk->private;
  82. struct clean_walk *cwalk = to_clean_walk(wpwalk);
  83. pte_t ptent = *pte;
  84. if (pte_dirty(ptent)) {
  85. pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) +
  86. walk->vma->vm_pgoff - cwalk->bitmap_pgoff;
  87. pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);
  88. ptent = pte_mkclean(old_pte);
  89. ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent);
  90. wpwalk->total++;
  91. wpwalk->tlbflush_start = min(wpwalk->tlbflush_start, addr);
  92. wpwalk->tlbflush_end = max(wpwalk->tlbflush_end,
  93. addr + PAGE_SIZE);
  94. __set_bit(pgoff, cwalk->bitmap);
  95. cwalk->start = min(cwalk->start, pgoff);
  96. cwalk->end = max(cwalk->end, pgoff + 1);
  97. }
  98. return 0;
  99. }
  100. /*
  101. * wp_clean_pmd_entry - The pagewalk pmd callback.
  102. *
  103. * Dirty-tracking should take place on the PTE level, so
  104. * WARN() if encountering a dirty huge pmd.
  105. * Furthermore, never split huge pmds, since that currently
  106. * causes dirty info loss. The pagefault handler should do
  107. * that if needed.
  108. */
  109. static int wp_clean_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long end,
  110. struct mm_walk *walk)
  111. {
  112. pmd_t pmdval = pmd_read_atomic(pmd);
  113. if (!pmd_trans_unstable(&pmdval))
  114. return 0;
  115. if (pmd_none(pmdval)) {
  116. walk->action = ACTION_AGAIN;
  117. return 0;
  118. }
  119. /* Huge pmd, present or migrated */
  120. walk->action = ACTION_CONTINUE;
  121. if (pmd_trans_huge(pmdval) || pmd_devmap(pmdval))
  122. WARN_ON(pmd_write(pmdval) || pmd_dirty(pmdval));
  123. return 0;
  124. }
  125. /*
  126. * wp_clean_pud_entry - The pagewalk pud callback.
  127. *
  128. * Dirty-tracking should take place on the PTE level, so
  129. * WARN() if encountering a dirty huge puds.
  130. * Furthermore, never split huge puds, since that currently
  131. * causes dirty info loss. The pagefault handler should do
  132. * that if needed.
  133. */
  134. static int wp_clean_pud_entry(pud_t *pud, unsigned long addr, unsigned long end,
  135. struct mm_walk *walk)
  136. {
  137. pud_t pudval = READ_ONCE(*pud);
  138. if (!pud_trans_unstable(&pudval))
  139. return 0;
  140. if (pud_none(pudval)) {
  141. walk->action = ACTION_AGAIN;
  142. return 0;
  143. }
  144. /* Huge pud */
  145. walk->action = ACTION_CONTINUE;
  146. if (pud_trans_huge(pudval) || pud_devmap(pudval))
  147. WARN_ON(pud_write(pudval) || pud_dirty(pudval));
  148. return 0;
  149. }
  150. /*
  151. * wp_clean_pre_vma - The pagewalk pre_vma callback.
  152. *
  153. * The pre_vma callback performs the cache flush, stages the tlb flush
  154. * and calls the necessary mmu notifiers.
  155. */
  156. static int wp_clean_pre_vma(unsigned long start, unsigned long end,
  157. struct mm_walk *walk)
  158. {
  159. struct wp_walk *wpwalk = walk->private;
  160. wpwalk->tlbflush_start = end;
  161. wpwalk->tlbflush_end = start;
  162. mmu_notifier_range_init(&wpwalk->range, MMU_NOTIFY_PROTECTION_PAGE, 0,
  163. walk->vma, walk->mm, start, end);
  164. mmu_notifier_invalidate_range_start(&wpwalk->range);
  165. flush_cache_range(walk->vma, start, end);
  166. /*
  167. * We're not using tlb_gather_mmu() since typically
  168. * only a small subrange of PTEs are affected, whereas
  169. * tlb_gather_mmu() records the full range.
  170. */
  171. inc_tlb_flush_pending(walk->mm);
  172. return 0;
  173. }
  174. /*
  175. * wp_clean_post_vma - The pagewalk post_vma callback.
  176. *
  177. * The post_vma callback performs the tlb flush and calls necessary mmu
  178. * notifiers.
  179. */
  180. static void wp_clean_post_vma(struct mm_walk *walk)
  181. {
  182. struct wp_walk *wpwalk = walk->private;
  183. if (mm_tlb_flush_nested(walk->mm))
  184. flush_tlb_range(walk->vma, wpwalk->range.start,
  185. wpwalk->range.end);
  186. else if (wpwalk->tlbflush_end > wpwalk->tlbflush_start)
  187. flush_tlb_range(walk->vma, wpwalk->tlbflush_start,
  188. wpwalk->tlbflush_end);
  189. mmu_notifier_invalidate_range_end(&wpwalk->range);
  190. dec_tlb_flush_pending(walk->mm);
  191. }
  192. /*
  193. * wp_clean_test_walk - The pagewalk test_walk callback.
  194. *
  195. * Won't perform dirty-tracking on COW, read-only or HUGETLB vmas.
  196. */
  197. static int wp_clean_test_walk(unsigned long start, unsigned long end,
  198. struct mm_walk *walk)
  199. {
  200. unsigned long vm_flags = READ_ONCE(walk->vma->vm_flags);
  201. /* Skip non-applicable VMAs */
  202. if ((vm_flags & (VM_SHARED | VM_MAYWRITE | VM_HUGETLB)) !=
  203. (VM_SHARED | VM_MAYWRITE))
  204. return 1;
  205. return 0;
  206. }
  207. static const struct mm_walk_ops clean_walk_ops = {
  208. .pte_entry = clean_record_pte,
  209. .pmd_entry = wp_clean_pmd_entry,
  210. .pud_entry = wp_clean_pud_entry,
  211. .test_walk = wp_clean_test_walk,
  212. .pre_vma = wp_clean_pre_vma,
  213. .post_vma = wp_clean_post_vma
  214. };
  215. static const struct mm_walk_ops wp_walk_ops = {
  216. .pte_entry = wp_pte,
  217. .pmd_entry = wp_clean_pmd_entry,
  218. .pud_entry = wp_clean_pud_entry,
  219. .test_walk = wp_clean_test_walk,
  220. .pre_vma = wp_clean_pre_vma,
  221. .post_vma = wp_clean_post_vma
  222. };
  223. /**
  224. * wp_shared_mapping_range - Write-protect all ptes in an address space range
  225. * @mapping: The address_space we want to write protect
  226. * @first_index: The first page offset in the range
  227. * @nr: Number of incremental page offsets to cover
  228. *
  229. * Note: This function currently skips transhuge page-table entries, since
  230. * it's intended for dirty-tracking on the PTE level. It will warn on
  231. * encountering transhuge write-enabled entries, though, and can easily be
  232. * extended to handle them as well.
  233. *
  234. * Return: The number of ptes actually write-protected. Note that
  235. * already write-protected ptes are not counted.
  236. */
  237. unsigned long wp_shared_mapping_range(struct address_space *mapping,
  238. pgoff_t first_index, pgoff_t nr)
  239. {
  240. struct wp_walk wpwalk = { .total = 0 };
  241. i_mmap_lock_read(mapping);
  242. WARN_ON(walk_page_mapping(mapping, first_index, nr, &wp_walk_ops,
  243. &wpwalk));
  244. i_mmap_unlock_read(mapping);
  245. return wpwalk.total;
  246. }
  247. EXPORT_SYMBOL_GPL(wp_shared_mapping_range);
  248. /**
  249. * clean_record_shared_mapping_range - Clean and record all ptes in an
  250. * address space range
  251. * @mapping: The address_space we want to clean
  252. * @first_index: The first page offset in the range
  253. * @nr: Number of incremental page offsets to cover
  254. * @bitmap_pgoff: The page offset of the first bit in @bitmap
  255. * @bitmap: Pointer to a bitmap of at least @nr bits. The bitmap needs to
  256. * cover the whole range @first_index..@first_index + @nr.
  257. * @start: Pointer to number of the first set bit in @bitmap.
  258. * is modified as new bits are set by the function.
  259. * @end: Pointer to the number of the last set bit in @bitmap.
  260. * none set. The value is modified as new bits are set by the function.
  261. *
  262. * Note: When this function returns there is no guarantee that a CPU has
  263. * not already dirtied new ptes. However it will not clean any ptes not
  264. * reported in the bitmap. The guarantees are as follows:
  265. * a) All ptes dirty when the function starts executing will end up recorded
  266. * in the bitmap.
  267. * b) All ptes dirtied after that will either remain dirty, be recorded in the
  268. * bitmap or both.
  269. *
  270. * If a caller needs to make sure all dirty ptes are picked up and none
  271. * additional are added, it first needs to write-protect the address-space
  272. * range and make sure new writers are blocked in page_mkwrite() or
  273. * pfn_mkwrite(). And then after a TLB flush following the write-protection
  274. * pick up all dirty bits.
  275. *
  276. * Note: This function currently skips transhuge page-table entries, since
  277. * it's intended for dirty-tracking on the PTE level. It will warn on
  278. * encountering transhuge dirty entries, though, and can easily be extended
  279. * to handle them as well.
  280. *
  281. * Return: The number of dirty ptes actually cleaned.
  282. */
  283. unsigned long clean_record_shared_mapping_range(struct address_space *mapping,
  284. pgoff_t first_index, pgoff_t nr,
  285. pgoff_t bitmap_pgoff,
  286. unsigned long *bitmap,
  287. pgoff_t *start,
  288. pgoff_t *end)
  289. {
  290. bool none_set = (*start >= *end);
  291. struct clean_walk cwalk = {
  292. .base = { .total = 0 },
  293. .bitmap_pgoff = bitmap_pgoff,
  294. .bitmap = bitmap,
  295. .start = none_set ? nr : *start,
  296. .end = none_set ? 0 : *end,
  297. };
  298. i_mmap_lock_read(mapping);
  299. WARN_ON(walk_page_mapping(mapping, first_index, nr, &clean_walk_ops,
  300. &cwalk.base));
  301. i_mmap_unlock_read(mapping);
  302. *start = cwalk.start;
  303. *end = cwalk.end;
  304. return cwalk.base.total;
  305. }
  306. EXPORT_SYMBOL_GPL(clean_record_shared_mapping_range);