mmu_gather.c 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. #include <linux/gfp.h>
  2. #include <linux/highmem.h>
  3. #include <linux/kernel.h>
  4. #include <linux/mmdebug.h>
  5. #include <linux/mm_types.h>
  6. #include <linux/pagemap.h>
  7. #include <linux/rcupdate.h>
  8. #include <linux/smp.h>
  9. #include <linux/swap.h>
  10. #include <asm/pgalloc.h>
  11. #include <asm/tlb.h>
  12. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  13. static bool tlb_next_batch(struct mmu_gather *tlb)
  14. {
  15. struct mmu_gather_batch *batch;
  16. batch = tlb->active;
  17. if (batch->next) {
  18. tlb->active = batch->next;
  19. return true;
  20. }
  21. if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
  22. return false;
  23. batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
  24. if (!batch)
  25. return false;
  26. tlb->batch_count++;
  27. batch->next = NULL;
  28. batch->nr = 0;
  29. batch->max = MAX_GATHER_BATCH;
  30. tlb->active->next = batch;
  31. tlb->active = batch;
  32. return true;
  33. }
  34. static void tlb_batch_pages_flush(struct mmu_gather *tlb)
  35. {
  36. struct mmu_gather_batch *batch;
  37. for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
  38. free_pages_and_swap_cache(batch->pages, batch->nr);
  39. batch->nr = 0;
  40. }
  41. tlb->active = &tlb->local;
  42. }
  43. static void tlb_batch_list_free(struct mmu_gather *tlb)
  44. {
  45. struct mmu_gather_batch *batch, *next;
  46. for (batch = tlb->local.next; batch; batch = next) {
  47. next = batch->next;
  48. free_pages((unsigned long)batch, 0);
  49. }
  50. tlb->local.next = NULL;
  51. }
  52. bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
  53. {
  54. struct mmu_gather_batch *batch;
  55. VM_BUG_ON(!tlb->end);
  56. #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
  57. VM_WARN_ON(tlb->page_size != page_size);
  58. #endif
  59. batch = tlb->active;
  60. /*
  61. * Add the page and check if we are full. If so
  62. * force a flush.
  63. */
  64. batch->pages[batch->nr++] = page;
  65. if (batch->nr == batch->max) {
  66. if (!tlb_next_batch(tlb))
  67. return true;
  68. batch = tlb->active;
  69. }
  70. VM_BUG_ON_PAGE(batch->nr > batch->max, page);
  71. return false;
  72. }
  73. #endif /* MMU_GATHER_NO_GATHER */
  74. #ifdef CONFIG_MMU_GATHER_TABLE_FREE
  75. static void __tlb_remove_table_free(struct mmu_table_batch *batch)
  76. {
  77. int i;
  78. for (i = 0; i < batch->nr; i++)
  79. __tlb_remove_table(batch->tables[i]);
  80. free_page((unsigned long)batch);
  81. }
  82. #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
  83. /*
  84. * Semi RCU freeing of the page directories.
  85. *
  86. * This is needed by some architectures to implement software pagetable walkers.
  87. *
  88. * gup_fast() and other software pagetable walkers do a lockless page-table
  89. * walk and therefore needs some synchronization with the freeing of the page
  90. * directories. The chosen means to accomplish that is by disabling IRQs over
  91. * the walk.
  92. *
  93. * Architectures that use IPIs to flush TLBs will then automagically DTRT,
  94. * since we unlink the page, flush TLBs, free the page. Since the disabling of
  95. * IRQs delays the completion of the TLB flush we can never observe an already
  96. * freed page.
  97. *
  98. * Architectures that do not have this (PPC) need to delay the freeing by some
  99. * other means, this is that means.
  100. *
  101. * What we do is batch the freed directory pages (tables) and RCU free them.
  102. * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
  103. * holds off grace periods.
  104. *
  105. * However, in order to batch these pages we need to allocate storage, this
  106. * allocation is deep inside the MM code and can thus easily fail on memory
  107. * pressure. To guarantee progress we fall back to single table freeing, see
  108. * the implementation of tlb_remove_table_one().
  109. *
  110. */
  111. static void tlb_remove_table_smp_sync(void *arg)
  112. {
  113. /* Simply deliver the interrupt */
  114. }
  115. static void tlb_remove_table_sync_one(void)
  116. {
  117. /*
  118. * This isn't an RCU grace period and hence the page-tables cannot be
  119. * assumed to be actually RCU-freed.
  120. *
  121. * It is however sufficient for software page-table walkers that rely on
  122. * IRQ disabling.
  123. */
  124. smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
  125. }
  126. static void tlb_remove_table_rcu(struct rcu_head *head)
  127. {
  128. __tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
  129. }
  130. static void tlb_remove_table_free(struct mmu_table_batch *batch)
  131. {
  132. call_rcu(&batch->rcu, tlb_remove_table_rcu);
  133. }
  134. #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
  135. static void tlb_remove_table_sync_one(void) { }
  136. static void tlb_remove_table_free(struct mmu_table_batch *batch)
  137. {
  138. __tlb_remove_table_free(batch);
  139. }
  140. #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
  141. /*
  142. * If we want tlb_remove_table() to imply TLB invalidates.
  143. */
  144. static inline void tlb_table_invalidate(struct mmu_gather *tlb)
  145. {
  146. if (tlb_needs_table_invalidate()) {
  147. /*
  148. * Invalidate page-table caches used by hardware walkers. Then
  149. * we still need to RCU-sched wait while freeing the pages
  150. * because software walkers can still be in-flight.
  151. */
  152. tlb_flush_mmu_tlbonly(tlb);
  153. }
  154. }
  155. static void tlb_remove_table_one(void *table)
  156. {
  157. tlb_remove_table_sync_one();
  158. __tlb_remove_table(table);
  159. }
  160. static void tlb_table_flush(struct mmu_gather *tlb)
  161. {
  162. struct mmu_table_batch **batch = &tlb->batch;
  163. if (*batch) {
  164. tlb_table_invalidate(tlb);
  165. tlb_remove_table_free(*batch);
  166. *batch = NULL;
  167. }
  168. }
  169. void tlb_remove_table(struct mmu_gather *tlb, void *table)
  170. {
  171. struct mmu_table_batch **batch = &tlb->batch;
  172. if (*batch == NULL) {
  173. *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
  174. if (*batch == NULL) {
  175. tlb_table_invalidate(tlb);
  176. tlb_remove_table_one(table);
  177. return;
  178. }
  179. (*batch)->nr = 0;
  180. }
  181. (*batch)->tables[(*batch)->nr++] = table;
  182. if ((*batch)->nr == MAX_TABLE_BATCH)
  183. tlb_table_flush(tlb);
  184. }
  185. static inline void tlb_table_init(struct mmu_gather *tlb)
  186. {
  187. tlb->batch = NULL;
  188. }
  189. #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
  190. static inline void tlb_table_flush(struct mmu_gather *tlb) { }
  191. static inline void tlb_table_init(struct mmu_gather *tlb) { }
  192. #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
  193. static void tlb_flush_mmu_free(struct mmu_gather *tlb)
  194. {
  195. tlb_table_flush(tlb);
  196. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  197. tlb_batch_pages_flush(tlb);
  198. #endif
  199. }
  200. void tlb_flush_mmu(struct mmu_gather *tlb)
  201. {
  202. tlb_flush_mmu_tlbonly(tlb);
  203. tlb_flush_mmu_free(tlb);
  204. }
  205. /**
  206. * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
  207. * @tlb: the mmu_gather structure to initialize
  208. * @mm: the mm_struct of the target address space
  209. * @start: start of the region that will be removed from the page-table
  210. * @end: end of the region that will be removed from the page-table
  211. *
  212. * Called to initialize an (on-stack) mmu_gather structure for page-table
  213. * tear-down from @mm. The @start and @end are set to 0 and -1
  214. * respectively when @mm is without users and we're going to destroy
  215. * the full address space (exit/execve).
  216. */
  217. void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
  218. unsigned long start, unsigned long end)
  219. {
  220. tlb->mm = mm;
  221. /* Is it from 0 to ~0? */
  222. tlb->fullmm = !(start | (end+1));
  223. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  224. tlb->need_flush_all = 0;
  225. tlb->local.next = NULL;
  226. tlb->local.nr = 0;
  227. tlb->local.max = ARRAY_SIZE(tlb->__pages);
  228. tlb->active = &tlb->local;
  229. tlb->batch_count = 0;
  230. #endif
  231. tlb_table_init(tlb);
  232. #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
  233. tlb->page_size = 0;
  234. #endif
  235. __tlb_reset_range(tlb);
  236. inc_tlb_flush_pending(tlb->mm);
  237. }
  238. /**
  239. * tlb_finish_mmu - finish an mmu_gather structure
  240. * @tlb: the mmu_gather structure to finish
  241. * @start: start of the region that will be removed from the page-table
  242. * @end: end of the region that will be removed from the page-table
  243. *
  244. * Called at the end of the shootdown operation to free up any resources that
  245. * were required.
  246. */
  247. void tlb_finish_mmu(struct mmu_gather *tlb,
  248. unsigned long start, unsigned long end)
  249. {
  250. /*
  251. * If there are parallel threads are doing PTE changes on same range
  252. * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
  253. * flush by batching, one thread may end up seeing inconsistent PTEs
  254. * and result in having stale TLB entries. So flush TLB forcefully
  255. * if we detect parallel PTE batching threads.
  256. *
  257. * However, some syscalls, e.g. munmap(), may free page tables, this
  258. * needs force flush everything in the given range. Otherwise this
  259. * may result in having stale TLB entries for some architectures,
  260. * e.g. aarch64, that could specify flush what level TLB.
  261. */
  262. if (mm_tlb_flush_nested(tlb->mm)) {
  263. /*
  264. * The aarch64 yields better performance with fullmm by
  265. * avoiding multiple CPUs spamming TLBI messages at the
  266. * same time.
  267. *
  268. * On x86 non-fullmm doesn't yield significant difference
  269. * against fullmm.
  270. */
  271. tlb->fullmm = 1;
  272. __tlb_reset_range(tlb);
  273. tlb->freed_tables = 1;
  274. }
  275. tlb_flush_mmu(tlb);
  276. #ifndef CONFIG_MMU_GATHER_NO_GATHER
  277. tlb_batch_list_free(tlb);
  278. #endif
  279. dec_tlb_flush_pending(tlb->mm);
  280. }