alternative.c 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * alternative runtime patching
  4. * inspired by the x86 version
  5. *
  6. * Copyright (C) 2014 ARM Ltd.
  7. */
  8. #define pr_fmt(fmt) "alternatives: " fmt
  9. #include <linux/init.h>
  10. #include <linux/cpu.h>
  11. #include <asm/cacheflush.h>
  12. #include <asm/alternative.h>
  13. #include <asm/cpufeature.h>
  14. #include <asm/insn.h>
  15. #include <asm/sections.h>
  16. #include <linux/stop_machine.h>
  17. #define __ALT_PTR(a,f) ((void *)&(a)->f + (a)->f)
  18. #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset)
  19. #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset)
  20. /* Volatile, as we may be patching the guts of READ_ONCE() */
  21. static volatile int all_alternatives_applied;
  22. static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS);
  23. struct alt_region {
  24. struct alt_instr *begin;
  25. struct alt_instr *end;
  26. };
  27. bool alternative_is_applied(u16 cpufeature)
  28. {
  29. if (WARN_ON(cpufeature >= ARM64_NCAPS))
  30. return false;
  31. return test_bit(cpufeature, applied_alternatives);
  32. }
  33. /*
  34. * Check if the target PC is within an alternative block.
  35. */
  36. static __always_inline bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
  37. {
  38. unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt);
  39. return !(pc >= replptr && pc <= (replptr + alt->alt_len));
  40. }
  41. #define align_down(x, a) ((unsigned long)(x) & ~(((unsigned long)(a)) - 1))
  42. static __always_inline u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)
  43. {
  44. u32 insn;
  45. insn = le32_to_cpu(*altinsnptr);
  46. if (aarch64_insn_is_branch_imm(insn)) {
  47. s32 offset = aarch64_get_branch_offset(insn);
  48. unsigned long target;
  49. target = (unsigned long)altinsnptr + offset;
  50. /*
  51. * If we're branching inside the alternate sequence,
  52. * do not rewrite the instruction, as it is already
  53. * correct. Otherwise, generate the new instruction.
  54. */
  55. if (branch_insn_requires_update(alt, target)) {
  56. offset = target - (unsigned long)insnptr;
  57. insn = aarch64_set_branch_offset(insn, offset);
  58. }
  59. } else if (aarch64_insn_is_adrp(insn)) {
  60. s32 orig_offset, new_offset;
  61. unsigned long target;
  62. /*
  63. * If we're replacing an adrp instruction, which uses PC-relative
  64. * immediate addressing, adjust the offset to reflect the new
  65. * PC. adrp operates on 4K aligned addresses.
  66. */
  67. orig_offset = aarch64_insn_adrp_get_offset(insn);
  68. target = align_down(altinsnptr, SZ_4K) + orig_offset;
  69. new_offset = target - align_down(insnptr, SZ_4K);
  70. insn = aarch64_insn_adrp_set_offset(insn, new_offset);
  71. } else if (aarch64_insn_uses_literal(insn)) {
  72. /*
  73. * Disallow patching unhandled instructions using PC relative
  74. * literal addresses
  75. */
  76. BUG();
  77. }
  78. return insn;
  79. }
  80. static noinstr void patch_alternative(struct alt_instr *alt,
  81. __le32 *origptr, __le32 *updptr, int nr_inst)
  82. {
  83. __le32 *replptr;
  84. int i;
  85. replptr = ALT_REPL_PTR(alt);
  86. for (i = 0; i < nr_inst; i++) {
  87. u32 insn;
  88. insn = get_alt_insn(alt, origptr + i, replptr + i);
  89. updptr[i] = cpu_to_le32(insn);
  90. }
  91. }
  92. /*
  93. * We provide our own, private D-cache cleaning function so that we don't
  94. * accidentally call into the cache.S code, which is patched by us at
  95. * runtime.
  96. */
  97. static void clean_dcache_range_nopatch(u64 start, u64 end)
  98. {
  99. u64 cur, d_size, ctr_el0;
  100. ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
  101. d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0,
  102. CTR_DMINLINE_SHIFT);
  103. cur = start & ~(d_size - 1);
  104. do {
  105. /*
  106. * We must clean+invalidate to the PoC in order to avoid
  107. * Cortex-A53 errata 826319, 827319, 824069 and 819472
  108. * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE)
  109. */
  110. asm volatile("dc civac, %0" : : "r" (cur) : "memory");
  111. } while (cur += d_size, cur < end);
  112. }
  113. static void __nocfi __apply_alternatives(void *alt_region, bool is_module,
  114. unsigned long *feature_mask)
  115. {
  116. struct alt_instr *alt;
  117. struct alt_region *region = alt_region;
  118. __le32 *origptr, *updptr;
  119. alternative_cb_t alt_cb;
  120. for (alt = region->begin; alt < region->end; alt++) {
  121. int nr_inst;
  122. if (!test_bit(alt->cpufeature, feature_mask))
  123. continue;
  124. /* Use ARM64_CB_PATCH as an unconditional patch */
  125. if (alt->cpufeature < ARM64_CB_PATCH &&
  126. !cpus_have_cap(alt->cpufeature))
  127. continue;
  128. if (alt->cpufeature == ARM64_CB_PATCH)
  129. BUG_ON(alt->alt_len != 0);
  130. else
  131. BUG_ON(alt->alt_len != alt->orig_len);
  132. pr_info_once("patching kernel code\n");
  133. origptr = ALT_ORIG_PTR(alt);
  134. updptr = is_module ? origptr : lm_alias(origptr);
  135. nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
  136. if (alt->cpufeature < ARM64_CB_PATCH)
  137. alt_cb = patch_alternative;
  138. else
  139. alt_cb = ALT_REPL_PTR(alt);
  140. alt_cb(alt, origptr, updptr, nr_inst);
  141. if (!is_module) {
  142. clean_dcache_range_nopatch((u64)origptr,
  143. (u64)(origptr + nr_inst));
  144. }
  145. }
  146. /*
  147. * The core module code takes care of cache maintenance in
  148. * flush_module_icache().
  149. */
  150. if (!is_module) {
  151. dsb(ish);
  152. __flush_icache_all();
  153. isb();
  154. /* Ignore ARM64_CB bit from feature mask */
  155. bitmap_or(applied_alternatives, applied_alternatives,
  156. feature_mask, ARM64_NCAPS);
  157. bitmap_and(applied_alternatives, applied_alternatives,
  158. cpu_hwcaps, ARM64_NCAPS);
  159. }
  160. }
  161. /*
  162. * We might be patching the stop_machine state machine, so implement a
  163. * really simple polling protocol here.
  164. */
  165. static int __apply_alternatives_multi_stop(void *unused)
  166. {
  167. struct alt_region region = {
  168. .begin = (struct alt_instr *)__alt_instructions,
  169. .end = (struct alt_instr *)__alt_instructions_end,
  170. };
  171. /* We always have a CPU 0 at this point (__init) */
  172. if (smp_processor_id()) {
  173. while (!all_alternatives_applied)
  174. cpu_relax();
  175. isb();
  176. } else {
  177. DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);
  178. bitmap_complement(remaining_capabilities, boot_capabilities,
  179. ARM64_NPATCHABLE);
  180. BUG_ON(all_alternatives_applied);
  181. __apply_alternatives(&region, false, remaining_capabilities);
  182. /* Barriers provided by the cache flushing */
  183. all_alternatives_applied = 1;
  184. }
  185. return 0;
  186. }
  187. void __init apply_alternatives_all(void)
  188. {
  189. /* better not try code patching on a live SMP system */
  190. stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
  191. }
  192. /*
  193. * This is called very early in the boot process (directly after we run
  194. * a feature detect on the boot CPU). No need to worry about other CPUs
  195. * here.
  196. */
  197. void __init apply_boot_alternatives(void)
  198. {
  199. struct alt_region region = {
  200. .begin = (struct alt_instr *)__alt_instructions,
  201. .end = (struct alt_instr *)__alt_instructions_end,
  202. };
  203. /* If called on non-boot cpu things could go wrong */
  204. WARN_ON(smp_processor_id() != 0);
  205. __apply_alternatives(&region, false, &boot_capabilities[0]);
  206. }
  207. #ifdef CONFIG_MODULES
  208. void apply_alternatives_module(void *start, size_t length)
  209. {
  210. struct alt_region region = {
  211. .begin = start,
  212. .end = start + length,
  213. };
  214. DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
  215. bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
  216. __apply_alternatives(&region, true, &all_capabilities[0]);
  217. }
  218. #endif