flush.c 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. /*
  2. * linux/arch/arm/mm/flush.c
  3. *
  4. * Copyright (C) 1995-2002 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/mm.h>
  12. #include <linux/pagemap.h>
  13. #include <asm/cacheflush.h>
  14. #include <asm/system.h>
  15. #include <asm/tlbflush.h>
  16. #include "mm.h"
  17. #ifdef CONFIG_CPU_CACHE_VIPT
  18. #define ALIAS_FLUSH_START 0xffff4000
  19. static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
  20. {
  21. unsigned long to = ALIAS_FLUSH_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  22. const int zero = 0;
  23. set_pte_ext(TOP_PTE(to), pfn_pte(pfn, PAGE_KERNEL), 0);
  24. flush_tlb_kernel_page(to);
  25. asm( "mcrr p15, 0, %1, %0, c14\n"
  26. " mcr p15, 0, %2, c7, c10, 4\n"
  27. " mcr p15, 0, %2, c7, c5, 0\n"
  28. :
  29. : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES), "r" (zero)
  30. : "cc");
  31. }
  32. void flush_cache_mm(struct mm_struct *mm)
  33. {
  34. if (cache_is_vivt()) {
  35. if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
  36. __cpuc_flush_user_all();
  37. return;
  38. }
  39. if (cache_is_vipt_aliasing()) {
  40. asm( "mcr p15, 0, %0, c7, c14, 0\n"
  41. " mcr p15, 0, %0, c7, c5, 0\n"
  42. " mcr p15, 0, %0, c7, c10, 4"
  43. :
  44. : "r" (0)
  45. : "cc");
  46. }
  47. }
  48. void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  49. {
  50. if (cache_is_vivt()) {
  51. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
  52. __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
  53. vma->vm_flags);
  54. return;
  55. }
  56. if (cache_is_vipt_aliasing()) {
  57. asm( "mcr p15, 0, %0, c7, c14, 0\n"
  58. " mcr p15, 0, %0, c7, c5, 0\n"
  59. " mcr p15, 0, %0, c7, c10, 4"
  60. :
  61. : "r" (0)
  62. : "cc");
  63. }
  64. }
  65. void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  66. {
  67. if (cache_is_vivt()) {
  68. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
  69. unsigned long addr = user_addr & PAGE_MASK;
  70. __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
  71. }
  72. return;
  73. }
  74. if (cache_is_vipt_aliasing())
  75. flush_pfn_alias(pfn, user_addr);
  76. }
  77. void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
  78. unsigned long uaddr, void *kaddr,
  79. unsigned long len, int write)
  80. {
  81. if (cache_is_vivt()) {
  82. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
  83. unsigned long addr = (unsigned long)kaddr;
  84. __cpuc_coherent_kern_range(addr, addr + len);
  85. }
  86. return;
  87. }
  88. if (cache_is_vipt_aliasing()) {
  89. flush_pfn_alias(page_to_pfn(page), uaddr);
  90. return;
  91. }
  92. /* VIPT non-aliasing cache */
  93. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask) &&
  94. vma->vm_flags & VM_EXEC) {
  95. unsigned long addr = (unsigned long)kaddr;
  96. /* only flushing the kernel mapping on non-aliasing VIPT */
  97. __cpuc_coherent_kern_range(addr, addr + len);
  98. }
  99. }
  100. #else
  101. #define flush_pfn_alias(pfn,vaddr) do { } while (0)
  102. #endif
  103. void __flush_dcache_page(struct address_space *mapping, struct page *page)
  104. {
  105. /*
  106. * Writeback any data associated with the kernel mapping of this
  107. * page. This ensures that data in the physical page is mutually
  108. * coherent with the kernels mapping.
  109. */
  110. __cpuc_flush_dcache_page(page_address(page));
  111. /*
  112. * If this is a page cache page, and we have an aliasing VIPT cache,
  113. * we only need to do one flush - which would be at the relevant
  114. * userspace colour, which is congruent with page->index.
  115. */
  116. if (mapping && cache_is_vipt_aliasing())
  117. flush_pfn_alias(page_to_pfn(page),
  118. page->index << PAGE_CACHE_SHIFT);
  119. }
  120. static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
  121. {
  122. struct mm_struct *mm = current->active_mm;
  123. struct vm_area_struct *mpnt;
  124. struct prio_tree_iter iter;
  125. pgoff_t pgoff;
  126. /*
  127. * There are possible user space mappings of this page:
  128. * - VIVT cache: we need to also write back and invalidate all user
  129. * data in the current VM view associated with this page.
  130. * - aliasing VIPT: we only need to find one mapping of this page.
  131. */
  132. pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  133. flush_dcache_mmap_lock(mapping);
  134. vma_prio_tree_foreach(mpnt, &iter, &mapping->i_mmap, pgoff, pgoff) {
  135. unsigned long offset;
  136. /*
  137. * If this VMA is not in our MM, we can ignore it.
  138. */
  139. if (mpnt->vm_mm != mm)
  140. continue;
  141. if (!(mpnt->vm_flags & VM_MAYSHARE))
  142. continue;
  143. offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
  144. flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
  145. }
  146. flush_dcache_mmap_unlock(mapping);
  147. }
  148. /*
  149. * Ensure cache coherency between kernel mapping and userspace mapping
  150. * of this page.
  151. *
  152. * We have three cases to consider:
  153. * - VIPT non-aliasing cache: fully coherent so nothing required.
  154. * - VIVT: fully aliasing, so we need to handle every alias in our
  155. * current VM view.
  156. * - VIPT aliasing: need to handle one alias in our current VM view.
  157. *
  158. * If we need to handle aliasing:
  159. * If the page only exists in the page cache and there are no user
  160. * space mappings, we can be lazy and remember that we may have dirty
  161. * kernel cache lines for later. Otherwise, we assume we have
  162. * aliasing mappings.
  163. *
  164. * Note that we disable the lazy flush for SMP.
  165. */
  166. void flush_dcache_page(struct page *page)
  167. {
  168. struct address_space *mapping = page_mapping(page);
  169. #ifndef CONFIG_SMP
  170. if (mapping && !mapping_mapped(mapping))
  171. set_bit(PG_dcache_dirty, &page->flags);
  172. else
  173. #endif
  174. {
  175. __flush_dcache_page(mapping, page);
  176. if (mapping && cache_is_vivt())
  177. __flush_dcache_aliases(mapping, page);
  178. }
  179. }
  180. EXPORT_SYMBOL(flush_dcache_page);
  181. /*
  182. * Flush an anonymous page so that users of get_user_pages()
  183. * can safely access the data. The expected sequence is:
  184. *
  185. * get_user_pages()
  186. * -> flush_anon_page
  187. * memcpy() to/from page
  188. * if written to page, flush_dcache_page()
  189. */
  190. void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
  191. {
  192. unsigned long pfn;
  193. /* VIPT non-aliasing caches need do nothing */
  194. if (cache_is_vipt_nonaliasing())
  195. return;
  196. /*
  197. * Write back and invalidate userspace mapping.
  198. */
  199. pfn = page_to_pfn(page);
  200. if (cache_is_vivt()) {
  201. flush_cache_page(vma, vmaddr, pfn);
  202. } else {
  203. /*
  204. * For aliasing VIPT, we can flush an alias of the
  205. * userspace address only.
  206. */
  207. flush_pfn_alias(pfn, vmaddr);
  208. }
  209. /*
  210. * Invalidate kernel mapping. No data should be contained
  211. * in this mapping of the page. FIXME: this is overkill
  212. * since we actually ask for a write-back and invalidate.
  213. */
  214. __cpuc_flush_dcache_page(page_address(page));
  215. }