page_idle.c 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/init.h>
  3. #include <linux/memblock.h>
  4. #include <linux/fs.h>
  5. #include <linux/sysfs.h>
  6. #include <linux/kobject.h>
  7. #include <linux/memory_hotplug.h>
  8. #include <linux/mm.h>
  9. #include <linux/mmzone.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/rmap.h>
  12. #include <linux/mmu_notifier.h>
  13. #include <linux/page_ext.h>
  14. #include <linux/page_idle.h>
  15. #define BITMAP_CHUNK_SIZE sizeof(u64)
  16. #define BITMAP_CHUNK_BITS (BITMAP_CHUNK_SIZE * BITS_PER_BYTE)
  17. /*
  18. * Idle page tracking only considers user memory pages, for other types of
  19. * pages the idle flag is always unset and an attempt to set it is silently
  20. * ignored.
  21. *
  22. * We treat a page as a user memory page if it is on an LRU list, because it is
  23. * always safe to pass such a page to rmap_walk(), which is essential for idle
  24. * page tracking. With such an indicator of user pages we can skip isolated
  25. * pages, but since there are not usually many of them, it will hardly affect
  26. * the overall result.
  27. *
  28. * This function tries to get a user memory page by pfn as described above.
  29. */
  30. static struct page *page_idle_get_page(unsigned long pfn)
  31. {
  32. struct page *page = pfn_to_online_page(pfn);
  33. pg_data_t *pgdat;
  34. if (!page || !PageLRU(page) ||
  35. !get_page_unless_zero(page))
  36. return NULL;
  37. pgdat = page_pgdat(page);
  38. spin_lock_irq(&pgdat->lru_lock);
  39. if (unlikely(!PageLRU(page))) {
  40. put_page(page);
  41. page = NULL;
  42. }
  43. spin_unlock_irq(&pgdat->lru_lock);
  44. return page;
  45. }
  46. static bool page_idle_clear_pte_refs_one(struct page *page,
  47. struct vm_area_struct *vma,
  48. unsigned long addr, void *arg)
  49. {
  50. struct page_vma_mapped_walk pvmw = {
  51. .page = page,
  52. .vma = vma,
  53. .address = addr,
  54. };
  55. bool referenced = false;
  56. while (page_vma_mapped_walk(&pvmw)) {
  57. addr = pvmw.address;
  58. if (pvmw.pte) {
  59. /*
  60. * For PTE-mapped THP, one sub page is referenced,
  61. * the whole THP is referenced.
  62. */
  63. if (ptep_clear_young_notify(vma, addr, pvmw.pte))
  64. referenced = true;
  65. } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
  66. if (pmdp_clear_young_notify(vma, addr, pvmw.pmd))
  67. referenced = true;
  68. } else {
  69. /* unexpected pmd-mapped page? */
  70. WARN_ON_ONCE(1);
  71. }
  72. }
  73. if (referenced) {
  74. clear_page_idle(page);
  75. /*
  76. * We cleared the referenced bit in a mapping to this page. To
  77. * avoid interference with page reclaim, mark it young so that
  78. * page_referenced() will return > 0.
  79. */
  80. set_page_young(page);
  81. }
  82. return true;
  83. }
  84. static void page_idle_clear_pte_refs(struct page *page)
  85. {
  86. /*
  87. * Since rwc.arg is unused, rwc is effectively immutable, so we
  88. * can make it static const to save some cycles and stack.
  89. */
  90. static const struct rmap_walk_control rwc = {
  91. .rmap_one = page_idle_clear_pte_refs_one,
  92. .anon_lock = page_lock_anon_vma_read,
  93. };
  94. bool need_lock;
  95. if (!page_mapped(page) ||
  96. !page_rmapping(page))
  97. return;
  98. need_lock = !PageAnon(page) || PageKsm(page);
  99. if (need_lock && !trylock_page(page))
  100. return;
  101. rmap_walk(page, (struct rmap_walk_control *)&rwc);
  102. if (need_lock)
  103. unlock_page(page);
  104. }
  105. static ssize_t page_idle_bitmap_read(struct file *file, struct kobject *kobj,
  106. struct bin_attribute *attr, char *buf,
  107. loff_t pos, size_t count)
  108. {
  109. u64 *out = (u64 *)buf;
  110. struct page *page;
  111. unsigned long pfn, end_pfn;
  112. int bit;
  113. if (pos % BITMAP_CHUNK_SIZE || count % BITMAP_CHUNK_SIZE)
  114. return -EINVAL;
  115. pfn = pos * BITS_PER_BYTE;
  116. if (pfn >= max_pfn)
  117. return 0;
  118. end_pfn = pfn + count * BITS_PER_BYTE;
  119. if (end_pfn > max_pfn)
  120. end_pfn = max_pfn;
  121. for (; pfn < end_pfn; pfn++) {
  122. bit = pfn % BITMAP_CHUNK_BITS;
  123. if (!bit)
  124. *out = 0ULL;
  125. page = page_idle_get_page(pfn);
  126. if (page) {
  127. if (page_is_idle(page)) {
  128. /*
  129. * The page might have been referenced via a
  130. * pte, in which case it is not idle. Clear
  131. * refs and recheck.
  132. */
  133. page_idle_clear_pte_refs(page);
  134. if (page_is_idle(page))
  135. *out |= 1ULL << bit;
  136. }
  137. put_page(page);
  138. }
  139. if (bit == BITMAP_CHUNK_BITS - 1)
  140. out++;
  141. cond_resched();
  142. }
  143. return (char *)out - buf;
  144. }
  145. static ssize_t page_idle_bitmap_write(struct file *file, struct kobject *kobj,
  146. struct bin_attribute *attr, char *buf,
  147. loff_t pos, size_t count)
  148. {
  149. const u64 *in = (u64 *)buf;
  150. struct page *page;
  151. unsigned long pfn, end_pfn;
  152. int bit;
  153. if (pos % BITMAP_CHUNK_SIZE || count % BITMAP_CHUNK_SIZE)
  154. return -EINVAL;
  155. pfn = pos * BITS_PER_BYTE;
  156. if (pfn >= max_pfn)
  157. return -ENXIO;
  158. end_pfn = pfn + count * BITS_PER_BYTE;
  159. if (end_pfn > max_pfn)
  160. end_pfn = max_pfn;
  161. for (; pfn < end_pfn; pfn++) {
  162. bit = pfn % BITMAP_CHUNK_BITS;
  163. if ((*in >> bit) & 1) {
  164. page = page_idle_get_page(pfn);
  165. if (page) {
  166. page_idle_clear_pte_refs(page);
  167. set_page_idle(page);
  168. put_page(page);
  169. }
  170. }
  171. if (bit == BITMAP_CHUNK_BITS - 1)
  172. in++;
  173. cond_resched();
  174. }
  175. return (char *)in - buf;
  176. }
  177. static struct bin_attribute page_idle_bitmap_attr =
  178. __BIN_ATTR(bitmap, 0600,
  179. page_idle_bitmap_read, page_idle_bitmap_write, 0);
  180. static struct bin_attribute *page_idle_bin_attrs[] = {
  181. &page_idle_bitmap_attr,
  182. NULL,
  183. };
  184. static const struct attribute_group page_idle_attr_group = {
  185. .bin_attrs = page_idle_bin_attrs,
  186. .name = "page_idle",
  187. };
  188. static int __init page_idle_init(void)
  189. {
  190. int err;
  191. err = sysfs_create_group(mm_kobj, &page_idle_attr_group);
  192. if (err) {
  193. pr_err("page_idle: register sysfs failed\n");
  194. return err;
  195. }
  196. return 0;
  197. }
  198. subsys_initcall(page_idle_init);