smp_tlb.c 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/arch/arm/kernel/smp_tlb.c
  4. *
  5. * Copyright (C) 2002 ARM Limited, All Rights Reserved.
  6. */
  7. #include <linux/preempt.h>
  8. #include <linux/smp.h>
  9. #include <linux/uaccess.h>
  10. #include <asm/smp_plat.h>
  11. #include <asm/tlbflush.h>
  12. #include <asm/mmu_context.h>
  13. /**********************************************************************/
  14. /*
  15. * TLB operations
  16. */
  17. struct tlb_args {
  18. struct vm_area_struct *ta_vma;
  19. unsigned long ta_start;
  20. unsigned long ta_end;
  21. };
  22. static inline void ipi_flush_tlb_all(void *ignored)
  23. {
  24. local_flush_tlb_all();
  25. }
  26. static inline void ipi_flush_tlb_mm(void *arg)
  27. {
  28. struct mm_struct *mm = (struct mm_struct *)arg;
  29. local_flush_tlb_mm(mm);
  30. }
  31. static inline void ipi_flush_tlb_page(void *arg)
  32. {
  33. struct tlb_args *ta = (struct tlb_args *)arg;
  34. unsigned int __ua_flags = uaccess_save_and_enable();
  35. local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  36. uaccess_restore(__ua_flags);
  37. }
  38. static inline void ipi_flush_tlb_kernel_page(void *arg)
  39. {
  40. struct tlb_args *ta = (struct tlb_args *)arg;
  41. local_flush_tlb_kernel_page(ta->ta_start);
  42. }
  43. static inline void ipi_flush_tlb_range(void *arg)
  44. {
  45. struct tlb_args *ta = (struct tlb_args *)arg;
  46. unsigned int __ua_flags = uaccess_save_and_enable();
  47. local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  48. uaccess_restore(__ua_flags);
  49. }
  50. static inline void ipi_flush_tlb_kernel_range(void *arg)
  51. {
  52. struct tlb_args *ta = (struct tlb_args *)arg;
  53. local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  54. }
  55. static inline void ipi_flush_bp_all(void *ignored)
  56. {
  57. local_flush_bp_all();
  58. }
  59. #ifdef CONFIG_ARM_ERRATA_798181
  60. bool (*erratum_a15_798181_handler)(void);
  61. static bool erratum_a15_798181_partial(void)
  62. {
  63. asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
  64. dsb(ish);
  65. return false;
  66. }
  67. static bool erratum_a15_798181_broadcast(void)
  68. {
  69. asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
  70. dsb(ish);
  71. return true;
  72. }
  73. void erratum_a15_798181_init(void)
  74. {
  75. unsigned int midr = read_cpuid_id();
  76. unsigned int revidr = read_cpuid(CPUID_REVIDR);
  77. /* Brahma-B15 r0p0..r0p2 affected
  78. * Cortex-A15 r0p0..r3p3 w/o ECO fix affected
  79. * Fixes applied to A15 with respect to the revision and revidr are:
  80. *
  81. * r0p0-r2p1: No fixes applied
  82. * r2p2,r2p3:
  83. * REVIDR[4]: 798181 Moving a virtual page that is being accessed
  84. * by an active process can lead to unexpected behavior
  85. * REVIDR[9]: Not defined
  86. * r2p4,r3p0,r3p1,r3p2:
  87. * REVIDR[4]: 798181 Moving a virtual page that is being accessed
  88. * by an active process can lead to unexpected behavior
  89. * REVIDR[9]: 798181 Moving a virtual page that is being accessed
  90. * by an active process can lead to unexpected behavior
  91. * - This is an update to a previously released ECO.
  92. * r3p3:
  93. * REVIDR[4]: Reserved
  94. * REVIDR[9]: 798181 Moving a virtual page that is being accessed
  95. * by an active process can lead to unexpected behavior
  96. * - This is an update to a previously released ECO.
  97. *
  98. * Handling:
  99. * REVIDR[9] set -> No WA
  100. * REVIDR[4] set, REVIDR[9] cleared -> Partial WA
  101. * Both cleared -> Full WA
  102. */
  103. if ((midr & 0xff0ffff0) == 0x420f00f0 && midr <= 0x420f00f2) {
  104. erratum_a15_798181_handler = erratum_a15_798181_broadcast;
  105. } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f2) {
  106. erratum_a15_798181_handler = erratum_a15_798181_broadcast;
  107. } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x412fc0f4) {
  108. if (revidr & 0x10)
  109. erratum_a15_798181_handler =
  110. erratum_a15_798181_partial;
  111. else
  112. erratum_a15_798181_handler =
  113. erratum_a15_798181_broadcast;
  114. } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x413fc0f3) {
  115. if ((revidr & 0x210) == 0)
  116. erratum_a15_798181_handler =
  117. erratum_a15_798181_broadcast;
  118. else if (revidr & 0x10)
  119. erratum_a15_798181_handler =
  120. erratum_a15_798181_partial;
  121. } else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr < 0x414fc0f0) {
  122. if ((revidr & 0x200) == 0)
  123. erratum_a15_798181_handler =
  124. erratum_a15_798181_partial;
  125. }
  126. }
  127. #endif
  128. static void ipi_flush_tlb_a15_erratum(void *arg)
  129. {
  130. dmb();
  131. }
  132. static void broadcast_tlb_a15_erratum(void)
  133. {
  134. if (!erratum_a15_798181())
  135. return;
  136. smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
  137. }
  138. static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
  139. {
  140. int this_cpu;
  141. cpumask_t mask = { CPU_BITS_NONE };
  142. if (!erratum_a15_798181())
  143. return;
  144. this_cpu = get_cpu();
  145. a15_erratum_get_cpumask(this_cpu, mm, &mask);
  146. smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
  147. put_cpu();
  148. }
  149. void flush_tlb_all(void)
  150. {
  151. if (tlb_ops_need_broadcast())
  152. on_each_cpu(ipi_flush_tlb_all, NULL, 1);
  153. else
  154. __flush_tlb_all();
  155. broadcast_tlb_a15_erratum();
  156. }
  157. void flush_tlb_mm(struct mm_struct *mm)
  158. {
  159. if (tlb_ops_need_broadcast())
  160. on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
  161. else
  162. __flush_tlb_mm(mm);
  163. broadcast_tlb_mm_a15_erratum(mm);
  164. }
  165. void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
  166. {
  167. if (tlb_ops_need_broadcast()) {
  168. struct tlb_args ta;
  169. ta.ta_vma = vma;
  170. ta.ta_start = uaddr;
  171. on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
  172. &ta, 1);
  173. } else
  174. __flush_tlb_page(vma, uaddr);
  175. broadcast_tlb_mm_a15_erratum(vma->vm_mm);
  176. }
  177. void flush_tlb_kernel_page(unsigned long kaddr)
  178. {
  179. if (tlb_ops_need_broadcast()) {
  180. struct tlb_args ta;
  181. ta.ta_start = kaddr;
  182. on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
  183. } else
  184. __flush_tlb_kernel_page(kaddr);
  185. broadcast_tlb_a15_erratum();
  186. }
  187. void flush_tlb_range(struct vm_area_struct *vma,
  188. unsigned long start, unsigned long end)
  189. {
  190. if (tlb_ops_need_broadcast()) {
  191. struct tlb_args ta;
  192. ta.ta_vma = vma;
  193. ta.ta_start = start;
  194. ta.ta_end = end;
  195. on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
  196. &ta, 1);
  197. } else
  198. local_flush_tlb_range(vma, start, end);
  199. broadcast_tlb_mm_a15_erratum(vma->vm_mm);
  200. }
  201. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  202. {
  203. if (tlb_ops_need_broadcast()) {
  204. struct tlb_args ta;
  205. ta.ta_start = start;
  206. ta.ta_end = end;
  207. on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
  208. } else
  209. local_flush_tlb_kernel_range(start, end);
  210. broadcast_tlb_a15_erratum();
  211. }
  212. void flush_bp_all(void)
  213. {
  214. if (tlb_ops_need_broadcast())
  215. on_each_cpu(ipi_flush_bp_all, NULL, 1);
  216. else
  217. __flush_bp_all();
  218. }