cacheflush.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. /*
  2. * linux/include/asm-arm/cacheflush.h
  3. *
  4. * Copyright (C) 1999-2002 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #ifndef _ASMARM_CACHEFLUSH_H
  11. #define _ASMARM_CACHEFLUSH_H
  12. #include <linux/sched.h>
  13. #include <linux/mm.h>
  14. #include <asm/glue.h>
  15. #include <asm/shmparam.h>
  16. #define CACHE_COLOUR(vaddr) ((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT)
  17. /*
  18. * Cache Model
  19. * ===========
  20. */
  21. #undef _CACHE
  22. #undef MULTI_CACHE
  23. #if defined(CONFIG_CPU_CACHE_V3)
  24. # ifdef _CACHE
  25. # define MULTI_CACHE 1
  26. # else
  27. # define _CACHE v3
  28. # endif
  29. #endif
  30. #if defined(CONFIG_CPU_CACHE_V4)
  31. # ifdef _CACHE
  32. # define MULTI_CACHE 1
  33. # else
  34. # define _CACHE v4
  35. # endif
  36. #endif
  37. #if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
  38. defined(CONFIG_CPU_ARM925T) || defined(CONFIG_CPU_ARM1020)
  39. # define MULTI_CACHE 1
  40. #endif
  41. #if defined(CONFIG_CPU_ARM926T)
  42. # ifdef _CACHE
  43. # define MULTI_CACHE 1
  44. # else
  45. # define _CACHE arm926
  46. # endif
  47. #endif
  48. #if defined(CONFIG_CPU_ARM940T)
  49. # ifdef _CACHE
  50. # define MULTI_CACHE 1
  51. # else
  52. # define _CACHE arm940
  53. # endif
  54. #endif
  55. #if defined(CONFIG_CPU_ARM946E)
  56. # ifdef _CACHE
  57. # define MULTI_CACHE 1
  58. # else
  59. # define _CACHE arm946
  60. # endif
  61. #endif
  62. #if defined(CONFIG_CPU_CACHE_V4WB)
  63. # ifdef _CACHE
  64. # define MULTI_CACHE 1
  65. # else
  66. # define _CACHE v4wb
  67. # endif
  68. #endif
  69. #if defined(CONFIG_CPU_XSCALE)
  70. # ifdef _CACHE
  71. # define MULTI_CACHE 1
  72. # else
  73. # define _CACHE xscale
  74. # endif
  75. #endif
  76. #if defined(CONFIG_CPU_XSC3)
  77. # ifdef _CACHE
  78. # define MULTI_CACHE 1
  79. # else
  80. # define _CACHE xsc3
  81. # endif
  82. #endif
  83. #if defined(CONFIG_CPU_V6)
  84. //# ifdef _CACHE
  85. # define MULTI_CACHE 1
  86. //# else
  87. //# define _CACHE v6
  88. //# endif
  89. #endif
  90. #if !defined(_CACHE) && !defined(MULTI_CACHE)
  91. #error Unknown cache maintainence model
  92. #endif
  93. /*
  94. * This flag is used to indicate that the page pointed to by a pte
  95. * is dirty and requires cleaning before returning it to the user.
  96. */
  97. #define PG_dcache_dirty PG_arch_1
  98. /*
  99. * MM Cache Management
  100. * ===================
  101. *
  102. * The arch/arm/mm/cache-*.S and arch/arm/mm/proc-*.S files
  103. * implement these methods.
  104. *
  105. * Start addresses are inclusive and end addresses are exclusive;
  106. * start addresses should be rounded down, end addresses up.
  107. *
  108. * See Documentation/cachetlb.txt for more information.
  109. * Please note that the implementation of these, and the required
  110. * effects are cache-type (VIVT/VIPT/PIPT) specific.
  111. *
  112. * flush_cache_kern_all()
  113. *
  114. * Unconditionally clean and invalidate the entire cache.
  115. *
  116. * flush_cache_user_mm(mm)
  117. *
  118. * Clean and invalidate all user space cache entries
  119. * before a change of page tables.
  120. *
  121. * flush_cache_user_range(start, end, flags)
  122. *
  123. * Clean and invalidate a range of cache entries in the
  124. * specified address space before a change of page tables.
  125. * - start - user start address (inclusive, page aligned)
  126. * - end - user end address (exclusive, page aligned)
  127. * - flags - vma->vm_flags field
  128. *
  129. * coherent_kern_range(start, end)
  130. *
  131. * Ensure coherency between the Icache and the Dcache in the
  132. * region described by start, end. If you have non-snooping
  133. * Harvard caches, you need to implement this function.
  134. * - start - virtual start address
  135. * - end - virtual end address
  136. *
  137. * DMA Cache Coherency
  138. * ===================
  139. *
  140. * dma_inv_range(start, end)
  141. *
  142. * Invalidate (discard) the specified virtual address range.
  143. * May not write back any entries. If 'start' or 'end'
  144. * are not cache line aligned, those lines must be written
  145. * back.
  146. * - start - virtual start address
  147. * - end - virtual end address
  148. *
  149. * dma_clean_range(start, end)
  150. *
  151. * Clean (write back) the specified virtual address range.
  152. * - start - virtual start address
  153. * - end - virtual end address
  154. *
  155. * dma_flush_range(start, end)
  156. *
  157. * Clean and invalidate the specified virtual address range.
  158. * - start - virtual start address
  159. * - end - virtual end address
  160. */
  161. struct cpu_cache_fns {
  162. void (*flush_kern_all)(void);
  163. void (*flush_user_all)(void);
  164. void (*flush_user_range)(unsigned long, unsigned long, unsigned int);
  165. void (*coherent_kern_range)(unsigned long, unsigned long);
  166. void (*coherent_user_range)(unsigned long, unsigned long);
  167. void (*flush_kern_dcache_page)(void *);
  168. void (*dma_inv_range)(const void *, const void *);
  169. void (*dma_clean_range)(const void *, const void *);
  170. void (*dma_flush_range)(const void *, const void *);
  171. };
  172. struct outer_cache_fns {
  173. void (*inv_range)(unsigned long, unsigned long);
  174. void (*clean_range)(unsigned long, unsigned long);
  175. void (*flush_range)(unsigned long, unsigned long);
  176. };
  177. /*
  178. * Select the calling method
  179. */
  180. #ifdef MULTI_CACHE
  181. extern struct cpu_cache_fns cpu_cache;
  182. #define __cpuc_flush_kern_all cpu_cache.flush_kern_all
  183. #define __cpuc_flush_user_all cpu_cache.flush_user_all
  184. #define __cpuc_flush_user_range cpu_cache.flush_user_range
  185. #define __cpuc_coherent_kern_range cpu_cache.coherent_kern_range
  186. #define __cpuc_coherent_user_range cpu_cache.coherent_user_range
  187. #define __cpuc_flush_dcache_page cpu_cache.flush_kern_dcache_page
  188. /*
  189. * These are private to the dma-mapping API. Do not use directly.
  190. * Their sole purpose is to ensure that data held in the cache
  191. * is visible to DMA, or data written by DMA to system memory is
  192. * visible to the CPU.
  193. */
  194. #define dmac_inv_range cpu_cache.dma_inv_range
  195. #define dmac_clean_range cpu_cache.dma_clean_range
  196. #define dmac_flush_range cpu_cache.dma_flush_range
  197. #else
  198. #define __cpuc_flush_kern_all __glue(_CACHE,_flush_kern_cache_all)
  199. #define __cpuc_flush_user_all __glue(_CACHE,_flush_user_cache_all)
  200. #define __cpuc_flush_user_range __glue(_CACHE,_flush_user_cache_range)
  201. #define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range)
  202. #define __cpuc_coherent_user_range __glue(_CACHE,_coherent_user_range)
  203. #define __cpuc_flush_dcache_page __glue(_CACHE,_flush_kern_dcache_page)
  204. extern void __cpuc_flush_kern_all(void);
  205. extern void __cpuc_flush_user_all(void);
  206. extern void __cpuc_flush_user_range(unsigned long, unsigned long, unsigned int);
  207. extern void __cpuc_coherent_kern_range(unsigned long, unsigned long);
  208. extern void __cpuc_coherent_user_range(unsigned long, unsigned long);
  209. extern void __cpuc_flush_dcache_page(void *);
  210. /*
  211. * These are private to the dma-mapping API. Do not use directly.
  212. * Their sole purpose is to ensure that data held in the cache
  213. * is visible to DMA, or data written by DMA to system memory is
  214. * visible to the CPU.
  215. */
  216. #define dmac_inv_range __glue(_CACHE,_dma_inv_range)
  217. #define dmac_clean_range __glue(_CACHE,_dma_clean_range)
  218. #define dmac_flush_range __glue(_CACHE,_dma_flush_range)
  219. extern void dmac_inv_range(const void *, const void *);
  220. extern void dmac_clean_range(const void *, const void *);
  221. extern void dmac_flush_range(const void *, const void *);
  222. #endif
  223. #ifdef CONFIG_OUTER_CACHE
  224. extern struct outer_cache_fns outer_cache;
  225. static inline void outer_inv_range(unsigned long start, unsigned long end)
  226. {
  227. if (outer_cache.inv_range)
  228. outer_cache.inv_range(start, end);
  229. }
  230. static inline void outer_clean_range(unsigned long start, unsigned long end)
  231. {
  232. if (outer_cache.clean_range)
  233. outer_cache.clean_range(start, end);
  234. }
  235. static inline void outer_flush_range(unsigned long start, unsigned long end)
  236. {
  237. if (outer_cache.flush_range)
  238. outer_cache.flush_range(start, end);
  239. }
  240. #else
  241. static inline void outer_inv_range(unsigned long start, unsigned long end)
  242. { }
  243. static inline void outer_clean_range(unsigned long start, unsigned long end)
  244. { }
  245. static inline void outer_flush_range(unsigned long start, unsigned long end)
  246. { }
  247. #endif
  248. /*
  249. * flush_cache_vmap() is used when creating mappings (eg, via vmap,
  250. * vmalloc, ioremap etc) in kernel space for pages. Since the
  251. * direct-mappings of these pages may contain cached data, we need
  252. * to do a full cache flush to ensure that writebacks don't corrupt
  253. * data placed into these pages via the new mappings.
  254. */
  255. #define flush_cache_vmap(start, end) flush_cache_all()
  256. #define flush_cache_vunmap(start, end) flush_cache_all()
  257. /*
  258. * Copy user data from/to a page which is mapped into a different
  259. * processes address space. Really, we want to allow our "user
  260. * space" model to handle this.
  261. */
  262. #define copy_to_user_page(vma, page, vaddr, dst, src, len) \
  263. do { \
  264. memcpy(dst, src, len); \
  265. flush_ptrace_access(vma, page, vaddr, dst, len, 1);\
  266. } while (0)
  267. #define copy_from_user_page(vma, page, vaddr, dst, src, len) \
  268. do { \
  269. memcpy(dst, src, len); \
  270. } while (0)
  271. /*
  272. * Convert calls to our calling convention.
  273. */
  274. #define flush_cache_all() __cpuc_flush_kern_all()
  275. #ifndef CONFIG_CPU_CACHE_VIPT
  276. static inline void flush_cache_mm(struct mm_struct *mm)
  277. {
  278. if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
  279. __cpuc_flush_user_all();
  280. }
  281. static inline void
  282. flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  283. {
  284. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
  285. __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end),
  286. vma->vm_flags);
  287. }
  288. static inline void
  289. flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  290. {
  291. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
  292. unsigned long addr = user_addr & PAGE_MASK;
  293. __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags);
  294. }
  295. }
  296. static inline void
  297. flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
  298. unsigned long uaddr, void *kaddr,
  299. unsigned long len, int write)
  300. {
  301. if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) {
  302. unsigned long addr = (unsigned long)kaddr;
  303. __cpuc_coherent_kern_range(addr, addr + len);
  304. }
  305. }
  306. #else
  307. extern void flush_cache_mm(struct mm_struct *mm);
  308. extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
  309. extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn);
  310. extern void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
  311. unsigned long uaddr, void *kaddr,
  312. unsigned long len, int write);
  313. #endif
  314. #define flush_cache_dup_mm(mm) flush_cache_mm(mm)
  315. /*
  316. * flush_cache_user_range is used when we want to ensure that the
  317. * Harvard caches are synchronised for the user space address range.
  318. * This is used for the ARM private sys_cacheflush system call.
  319. */
  320. #define flush_cache_user_range(vma,start,end) \
  321. __cpuc_coherent_user_range((start) & PAGE_MASK, PAGE_ALIGN(end))
  322. /*
  323. * Perform necessary cache operations to ensure that data previously
  324. * stored within this range of addresses can be executed by the CPU.
  325. */
  326. #define flush_icache_range(s,e) __cpuc_coherent_kern_range(s,e)
  327. /*
  328. * Perform necessary cache operations to ensure that the TLB will
  329. * see data written in the specified area.
  330. */
  331. #define clean_dcache_area(start,size) cpu_dcache_clean_area(start, size)
  332. /*
  333. * flush_dcache_page is used when the kernel has written to the page
  334. * cache page at virtual address page->virtual.
  335. *
  336. * If this page isn't mapped (ie, page_mapping == NULL), or it might
  337. * have userspace mappings, then we _must_ always clean + invalidate
  338. * the dcache entries associated with the kernel mapping.
  339. *
  340. * Otherwise we can defer the operation, and clean the cache when we are
  341. * about to change to user space. This is the same method as used on SPARC64.
  342. * See update_mmu_cache for the user space part.
  343. */
  344. extern void flush_dcache_page(struct page *);
  345. extern void __flush_dcache_page(struct address_space *mapping, struct page *page);
  346. #define ARCH_HAS_FLUSH_ANON_PAGE
  347. static inline void flush_anon_page(struct vm_area_struct *vma,
  348. struct page *page, unsigned long vmaddr)
  349. {
  350. extern void __flush_anon_page(struct vm_area_struct *vma,
  351. struct page *, unsigned long);
  352. if (PageAnon(page))
  353. __flush_anon_page(vma, page, vmaddr);
  354. }
  355. #define flush_dcache_mmap_lock(mapping) \
  356. write_lock_irq(&(mapping)->tree_lock)
  357. #define flush_dcache_mmap_unlock(mapping) \
  358. write_unlock_irq(&(mapping)->tree_lock)
  359. #define flush_icache_user_range(vma,page,addr,len) \
  360. flush_dcache_page(page)
  361. /*
  362. * We don't appear to need to do anything here. In fact, if we did, we'd
  363. * duplicate cache flushing elsewhere performed by flush_dcache_page().
  364. */
  365. #define flush_icache_page(vma,page) do { } while (0)
  366. #define __cacheid_present(val) (val != read_cpuid(CPUID_ID))
  367. #define __cacheid_vivt(val) ((val & (15 << 25)) != (14 << 25))
  368. #define __cacheid_vipt(val) ((val & (15 << 25)) == (14 << 25))
  369. #define __cacheid_vipt_nonaliasing(val) ((val & (15 << 25 | 1 << 23)) == (14 << 25))
  370. #define __cacheid_vipt_aliasing(val) ((val & (15 << 25 | 1 << 23)) == (14 << 25 | 1 << 23))
  371. #if defined(CONFIG_CPU_CACHE_VIVT) && !defined(CONFIG_CPU_CACHE_VIPT)
  372. #define cache_is_vivt() 1
  373. #define cache_is_vipt() 0
  374. #define cache_is_vipt_nonaliasing() 0
  375. #define cache_is_vipt_aliasing() 0
  376. #elif defined(CONFIG_CPU_CACHE_VIPT)
  377. #define cache_is_vivt() 0
  378. #define cache_is_vipt() 1
  379. #define cache_is_vipt_nonaliasing() \
  380. ({ \
  381. unsigned int __val = read_cpuid(CPUID_CACHETYPE); \
  382. __cacheid_vipt_nonaliasing(__val); \
  383. })
  384. #define cache_is_vipt_aliasing() \
  385. ({ \
  386. unsigned int __val = read_cpuid(CPUID_CACHETYPE); \
  387. __cacheid_vipt_aliasing(__val); \
  388. })
  389. #else
  390. #define cache_is_vivt() \
  391. ({ \
  392. unsigned int __val = read_cpuid(CPUID_CACHETYPE); \
  393. (!__cacheid_present(__val)) || __cacheid_vivt(__val); \
  394. })
  395. #define cache_is_vipt() \
  396. ({ \
  397. unsigned int __val = read_cpuid(CPUID_CACHETYPE); \
  398. __cacheid_present(__val) && __cacheid_vipt(__val); \
  399. })
  400. #define cache_is_vipt_nonaliasing() \
  401. ({ \
  402. unsigned int __val = read_cpuid(CPUID_CACHETYPE); \
  403. __cacheid_present(__val) && \
  404. __cacheid_vipt_nonaliasing(__val); \
  405. })
  406. #define cache_is_vipt_aliasing() \
  407. ({ \
  408. unsigned int __val = read_cpuid(CPUID_CACHETYPE); \
  409. __cacheid_present(__val) && \
  410. __cacheid_vipt_aliasing(__val); \
  411. })
  412. #endif
  413. #endif