alternative.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #define pr_fmt(fmt) "SMP alternatives: " fmt
  3. #include <linux/module.h>
  4. #include <linux/sched.h>
  5. #include <linux/perf_event.h>
  6. #include <linux/mutex.h>
  7. #include <linux/list.h>
  8. #include <linux/stringify.h>
  9. #include <linux/highmem.h>
  10. #include <linux/mm.h>
  11. #include <linux/vmalloc.h>
  12. #include <linux/memory.h>
  13. #include <linux/stop_machine.h>
  14. #include <linux/slab.h>
  15. #include <linux/kdebug.h>
  16. #include <linux/kprobes.h>
  17. #include <linux/mmu_context.h>
  18. #include <linux/bsearch.h>
  19. #include <linux/sync_core.h>
  20. #include <asm/text-patching.h>
  21. #include <asm/alternative.h>
  22. #include <asm/sections.h>
  23. #include <asm/mce.h>
  24. #include <asm/nmi.h>
  25. #include <asm/cacheflush.h>
  26. #include <asm/tlbflush.h>
  27. #include <asm/insn.h>
  28. #include <asm/io.h>
  29. #include <asm/fixmap.h>
  30. int __read_mostly alternatives_patched;
  31. EXPORT_SYMBOL_GPL(alternatives_patched);
  32. #define MAX_PATCH_LEN (255-1)
  33. static int __initdata_or_module debug_alternative;
  34. static int __init debug_alt(char *str)
  35. {
  36. debug_alternative = 1;
  37. return 1;
  38. }
  39. __setup("debug-alternative", debug_alt);
  40. static int noreplace_smp;
  41. static int __init setup_noreplace_smp(char *str)
  42. {
  43. noreplace_smp = 1;
  44. return 1;
  45. }
  46. __setup("noreplace-smp", setup_noreplace_smp);
  47. #define DPRINTK(fmt, args...) \
  48. do { \
  49. if (debug_alternative) \
  50. printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \
  51. } while (0)
  52. #define DUMP_BYTES(buf, len, fmt, args...) \
  53. do { \
  54. if (unlikely(debug_alternative)) { \
  55. int j; \
  56. \
  57. if (!(len)) \
  58. break; \
  59. \
  60. printk(KERN_DEBUG pr_fmt(fmt), ##args); \
  61. for (j = 0; j < (len) - 1; j++) \
  62. printk(KERN_CONT "%02hhx ", buf[j]); \
  63. printk(KERN_CONT "%02hhx\n", buf[j]); \
  64. } \
  65. } while (0)
  66. /*
  67. * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes
  68. * that correspond to that nop. Getting from one nop to the next, we
  69. * add to the array the offset that is equal to the sum of all sizes of
  70. * nops preceding the one we are after.
  71. *
  72. * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the
  73. * nice symmetry of sizes of the previous nops.
  74. */
  75. #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
  76. static const unsigned char intelnops[] =
  77. {
  78. GENERIC_NOP1,
  79. GENERIC_NOP2,
  80. GENERIC_NOP3,
  81. GENERIC_NOP4,
  82. GENERIC_NOP5,
  83. GENERIC_NOP6,
  84. GENERIC_NOP7,
  85. GENERIC_NOP8,
  86. GENERIC_NOP5_ATOMIC
  87. };
  88. static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
  89. {
  90. NULL,
  91. intelnops,
  92. intelnops + 1,
  93. intelnops + 1 + 2,
  94. intelnops + 1 + 2 + 3,
  95. intelnops + 1 + 2 + 3 + 4,
  96. intelnops + 1 + 2 + 3 + 4 + 5,
  97. intelnops + 1 + 2 + 3 + 4 + 5 + 6,
  98. intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  99. intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
  100. };
  101. #endif
  102. #ifdef K8_NOP1
  103. static const unsigned char k8nops[] =
  104. {
  105. K8_NOP1,
  106. K8_NOP2,
  107. K8_NOP3,
  108. K8_NOP4,
  109. K8_NOP5,
  110. K8_NOP6,
  111. K8_NOP7,
  112. K8_NOP8,
  113. K8_NOP5_ATOMIC
  114. };
  115. static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
  116. {
  117. NULL,
  118. k8nops,
  119. k8nops + 1,
  120. k8nops + 1 + 2,
  121. k8nops + 1 + 2 + 3,
  122. k8nops + 1 + 2 + 3 + 4,
  123. k8nops + 1 + 2 + 3 + 4 + 5,
  124. k8nops + 1 + 2 + 3 + 4 + 5 + 6,
  125. k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  126. k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
  127. };
  128. #endif
  129. #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
  130. static const unsigned char k7nops[] =
  131. {
  132. K7_NOP1,
  133. K7_NOP2,
  134. K7_NOP3,
  135. K7_NOP4,
  136. K7_NOP5,
  137. K7_NOP6,
  138. K7_NOP7,
  139. K7_NOP8,
  140. K7_NOP5_ATOMIC
  141. };
  142. static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
  143. {
  144. NULL,
  145. k7nops,
  146. k7nops + 1,
  147. k7nops + 1 + 2,
  148. k7nops + 1 + 2 + 3,
  149. k7nops + 1 + 2 + 3 + 4,
  150. k7nops + 1 + 2 + 3 + 4 + 5,
  151. k7nops + 1 + 2 + 3 + 4 + 5 + 6,
  152. k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  153. k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
  154. };
  155. #endif
  156. #ifdef P6_NOP1
  157. static const unsigned char p6nops[] =
  158. {
  159. P6_NOP1,
  160. P6_NOP2,
  161. P6_NOP3,
  162. P6_NOP4,
  163. P6_NOP5,
  164. P6_NOP6,
  165. P6_NOP7,
  166. P6_NOP8,
  167. P6_NOP5_ATOMIC
  168. };
  169. static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
  170. {
  171. NULL,
  172. p6nops,
  173. p6nops + 1,
  174. p6nops + 1 + 2,
  175. p6nops + 1 + 2 + 3,
  176. p6nops + 1 + 2 + 3 + 4,
  177. p6nops + 1 + 2 + 3 + 4 + 5,
  178. p6nops + 1 + 2 + 3 + 4 + 5 + 6,
  179. p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  180. p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
  181. };
  182. #endif
  183. /* Initialize these to a safe default */
  184. #ifdef CONFIG_X86_64
  185. const unsigned char * const *ideal_nops = p6_nops;
  186. #else
  187. const unsigned char * const *ideal_nops = intel_nops;
  188. #endif
  189. void __init arch_init_ideal_nops(void)
  190. {
  191. switch (boot_cpu_data.x86_vendor) {
  192. case X86_VENDOR_INTEL:
  193. /*
  194. * Due to a decoder implementation quirk, some
  195. * specific Intel CPUs actually perform better with
  196. * the "k8_nops" than with the SDM-recommended NOPs.
  197. */
  198. if (boot_cpu_data.x86 == 6 &&
  199. boot_cpu_data.x86_model >= 0x0f &&
  200. boot_cpu_data.x86_model != 0x1c &&
  201. boot_cpu_data.x86_model != 0x26 &&
  202. boot_cpu_data.x86_model != 0x27 &&
  203. boot_cpu_data.x86_model < 0x30) {
  204. ideal_nops = k8_nops;
  205. } else if (boot_cpu_has(X86_FEATURE_NOPL)) {
  206. ideal_nops = p6_nops;
  207. } else {
  208. #ifdef CONFIG_X86_64
  209. ideal_nops = k8_nops;
  210. #else
  211. ideal_nops = intel_nops;
  212. #endif
  213. }
  214. break;
  215. case X86_VENDOR_HYGON:
  216. ideal_nops = p6_nops;
  217. return;
  218. case X86_VENDOR_AMD:
  219. if (boot_cpu_data.x86 > 0xf) {
  220. ideal_nops = p6_nops;
  221. return;
  222. }
  223. fallthrough;
  224. default:
  225. #ifdef CONFIG_X86_64
  226. ideal_nops = k8_nops;
  227. #else
  228. if (boot_cpu_has(X86_FEATURE_K8))
  229. ideal_nops = k8_nops;
  230. else if (boot_cpu_has(X86_FEATURE_K7))
  231. ideal_nops = k7_nops;
  232. else
  233. ideal_nops = intel_nops;
  234. #endif
  235. }
  236. }
  237. /* Use this to add nops to a buffer, then text_poke the whole buffer. */
  238. static void __init_or_module add_nops(void *insns, unsigned int len)
  239. {
  240. while (len > 0) {
  241. unsigned int noplen = len;
  242. if (noplen > ASM_NOP_MAX)
  243. noplen = ASM_NOP_MAX;
  244. memcpy(insns, ideal_nops[noplen], noplen);
  245. insns += noplen;
  246. len -= noplen;
  247. }
  248. }
  249. extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
  250. extern s32 __smp_locks[], __smp_locks_end[];
  251. void text_poke_early(void *addr, const void *opcode, size_t len);
  252. /*
  253. * Are we looking at a near JMP with a 1 or 4-byte displacement.
  254. */
  255. static inline bool is_jmp(const u8 opcode)
  256. {
  257. return opcode == 0xeb || opcode == 0xe9;
  258. }
  259. static void __init_or_module
  260. recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff)
  261. {
  262. u8 *next_rip, *tgt_rip;
  263. s32 n_dspl, o_dspl;
  264. int repl_len;
  265. if (a->replacementlen != 5)
  266. return;
  267. o_dspl = *(s32 *)(insn_buff + 1);
  268. /* next_rip of the replacement JMP */
  269. next_rip = repl_insn + a->replacementlen;
  270. /* target rip of the replacement JMP */
  271. tgt_rip = next_rip + o_dspl;
  272. n_dspl = tgt_rip - orig_insn;
  273. DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);
  274. if (tgt_rip - orig_insn >= 0) {
  275. if (n_dspl - 2 <= 127)
  276. goto two_byte_jmp;
  277. else
  278. goto five_byte_jmp;
  279. /* negative offset */
  280. } else {
  281. if (((n_dspl - 2) & 0xff) == (n_dspl - 2))
  282. goto two_byte_jmp;
  283. else
  284. goto five_byte_jmp;
  285. }
  286. two_byte_jmp:
  287. n_dspl -= 2;
  288. insn_buff[0] = 0xeb;
  289. insn_buff[1] = (s8)n_dspl;
  290. add_nops(insn_buff + 2, 3);
  291. repl_len = 2;
  292. goto done;
  293. five_byte_jmp:
  294. n_dspl -= 5;
  295. insn_buff[0] = 0xe9;
  296. *(s32 *)&insn_buff[1] = n_dspl;
  297. repl_len = 5;
  298. done:
  299. DPRINTK("final displ: 0x%08x, JMP 0x%lx",
  300. n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
  301. }
  302. /*
  303. * "noinline" to cause control flow change and thus invalidate I$ and
  304. * cause refetch after modification.
  305. */
  306. static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr)
  307. {
  308. unsigned long flags;
  309. int i;
  310. for (i = 0; i < a->padlen; i++) {
  311. if (instr[i] != 0x90)
  312. return;
  313. }
  314. local_irq_save(flags);
  315. add_nops(instr + (a->instrlen - a->padlen), a->padlen);
  316. local_irq_restore(flags);
  317. DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",
  318. instr, a->instrlen - a->padlen, a->padlen);
  319. }
  320. /*
  321. * Replace instructions with better alternatives for this CPU type. This runs
  322. * before SMP is initialized to avoid SMP problems with self modifying code.
  323. * This implies that asymmetric systems where APs have less capabilities than
  324. * the boot processor are not handled. Tough. Make sure you disable such
  325. * features by hand.
  326. *
  327. * Marked "noinline" to cause control flow change and thus insn cache
  328. * to refetch changed I$ lines.
  329. */
  330. void __init_or_module noinline apply_alternatives(struct alt_instr *start,
  331. struct alt_instr *end)
  332. {
  333. struct alt_instr *a;
  334. u8 *instr, *replacement;
  335. u8 insn_buff[MAX_PATCH_LEN];
  336. DPRINTK("alt table %px, -> %px", start, end);
  337. /*
  338. * The scan order should be from start to end. A later scanned
  339. * alternative code can overwrite previously scanned alternative code.
  340. * Some kernel functions (e.g. memcpy, memset, etc) use this order to
  341. * patch code.
  342. *
  343. * So be careful if you want to change the scan order to any other
  344. * order.
  345. */
  346. for (a = start; a < end; a++) {
  347. int insn_buff_sz = 0;
  348. instr = (u8 *)&a->instr_offset + a->instr_offset;
  349. replacement = (u8 *)&a->repl_offset + a->repl_offset;
  350. BUG_ON(a->instrlen > sizeof(insn_buff));
  351. BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
  352. if (!boot_cpu_has(a->cpuid)) {
  353. if (a->padlen > 1)
  354. optimize_nops(a, instr);
  355. continue;
  356. }
  357. DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
  358. a->cpuid >> 5,
  359. a->cpuid & 0x1f,
  360. instr, instr, a->instrlen,
  361. replacement, a->replacementlen, a->padlen);
  362. DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
  363. DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
  364. memcpy(insn_buff, replacement, a->replacementlen);
  365. insn_buff_sz = a->replacementlen;
  366. /*
  367. * 0xe8 is a relative jump; fix the offset.
  368. *
  369. * Instruction length is checked before the opcode to avoid
  370. * accessing uninitialized bytes for zero-length replacements.
  371. */
  372. if (a->replacementlen == 5 && *insn_buff == 0xe8) {
  373. *(s32 *)(insn_buff + 1) += replacement - instr;
  374. DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
  375. *(s32 *)(insn_buff + 1),
  376. (unsigned long)instr + *(s32 *)(insn_buff + 1) + 5);
  377. }
  378. if (a->replacementlen && is_jmp(replacement[0]))
  379. recompute_jump(a, instr, replacement, insn_buff);
  380. if (a->instrlen > a->replacementlen) {
  381. add_nops(insn_buff + a->replacementlen,
  382. a->instrlen - a->replacementlen);
  383. insn_buff_sz += a->instrlen - a->replacementlen;
  384. }
  385. DUMP_BYTES(insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
  386. text_poke_early(instr, insn_buff, insn_buff_sz);
  387. }
  388. }
  389. #ifdef CONFIG_SMP
  390. static void alternatives_smp_lock(const s32 *start, const s32 *end,
  391. u8 *text, u8 *text_end)
  392. {
  393. const s32 *poff;
  394. for (poff = start; poff < end; poff++) {
  395. u8 *ptr = (u8 *)poff + *poff;
  396. if (!*poff || ptr < text || ptr >= text_end)
  397. continue;
  398. /* turn DS segment override prefix into lock prefix */
  399. if (*ptr == 0x3e)
  400. text_poke(ptr, ((unsigned char []){0xf0}), 1);
  401. }
  402. }
  403. static void alternatives_smp_unlock(const s32 *start, const s32 *end,
  404. u8 *text, u8 *text_end)
  405. {
  406. const s32 *poff;
  407. for (poff = start; poff < end; poff++) {
  408. u8 *ptr = (u8 *)poff + *poff;
  409. if (!*poff || ptr < text || ptr >= text_end)
  410. continue;
  411. /* turn lock prefix into DS segment override prefix */
  412. if (*ptr == 0xf0)
  413. text_poke(ptr, ((unsigned char []){0x3E}), 1);
  414. }
  415. }
  416. struct smp_alt_module {
  417. /* what is this ??? */
  418. struct module *mod;
  419. char *name;
  420. /* ptrs to lock prefixes */
  421. const s32 *locks;
  422. const s32 *locks_end;
  423. /* .text segment, needed to avoid patching init code ;) */
  424. u8 *text;
  425. u8 *text_end;
  426. struct list_head next;
  427. };
  428. static LIST_HEAD(smp_alt_modules);
  429. static bool uniproc_patched = false; /* protected by text_mutex */
  430. void __init_or_module alternatives_smp_module_add(struct module *mod,
  431. char *name,
  432. void *locks, void *locks_end,
  433. void *text, void *text_end)
  434. {
  435. struct smp_alt_module *smp;
  436. mutex_lock(&text_mutex);
  437. if (!uniproc_patched)
  438. goto unlock;
  439. if (num_possible_cpus() == 1)
  440. /* Don't bother remembering, we'll never have to undo it. */
  441. goto smp_unlock;
  442. smp = kzalloc(sizeof(*smp), GFP_KERNEL);
  443. if (NULL == smp)
  444. /* we'll run the (safe but slow) SMP code then ... */
  445. goto unlock;
  446. smp->mod = mod;
  447. smp->name = name;
  448. smp->locks = locks;
  449. smp->locks_end = locks_end;
  450. smp->text = text;
  451. smp->text_end = text_end;
  452. DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",
  453. smp->locks, smp->locks_end,
  454. smp->text, smp->text_end, smp->name);
  455. list_add_tail(&smp->next, &smp_alt_modules);
  456. smp_unlock:
  457. alternatives_smp_unlock(locks, locks_end, text, text_end);
  458. unlock:
  459. mutex_unlock(&text_mutex);
  460. }
  461. void __init_or_module alternatives_smp_module_del(struct module *mod)
  462. {
  463. struct smp_alt_module *item;
  464. mutex_lock(&text_mutex);
  465. list_for_each_entry(item, &smp_alt_modules, next) {
  466. if (mod != item->mod)
  467. continue;
  468. list_del(&item->next);
  469. kfree(item);
  470. break;
  471. }
  472. mutex_unlock(&text_mutex);
  473. }
  474. void alternatives_enable_smp(void)
  475. {
  476. struct smp_alt_module *mod;
  477. /* Why bother if there are no other CPUs? */
  478. BUG_ON(num_possible_cpus() == 1);
  479. mutex_lock(&text_mutex);
  480. if (uniproc_patched) {
  481. pr_info("switching to SMP code\n");
  482. BUG_ON(num_online_cpus() != 1);
  483. clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
  484. clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
  485. list_for_each_entry(mod, &smp_alt_modules, next)
  486. alternatives_smp_lock(mod->locks, mod->locks_end,
  487. mod->text, mod->text_end);
  488. uniproc_patched = false;
  489. }
  490. mutex_unlock(&text_mutex);
  491. }
  492. /*
  493. * Return 1 if the address range is reserved for SMP-alternatives.
  494. * Must hold text_mutex.
  495. */
  496. int alternatives_text_reserved(void *start, void *end)
  497. {
  498. struct smp_alt_module *mod;
  499. const s32 *poff;
  500. u8 *text_start = start;
  501. u8 *text_end = end;
  502. lockdep_assert_held(&text_mutex);
  503. list_for_each_entry(mod, &smp_alt_modules, next) {
  504. if (mod->text > text_end || mod->text_end < text_start)
  505. continue;
  506. for (poff = mod->locks; poff < mod->locks_end; poff++) {
  507. const u8 *ptr = (const u8 *)poff + *poff;
  508. if (text_start <= ptr && text_end > ptr)
  509. return 1;
  510. }
  511. }
  512. return 0;
  513. }
  514. #endif /* CONFIG_SMP */
  515. #ifdef CONFIG_PARAVIRT
  516. void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
  517. struct paravirt_patch_site *end)
  518. {
  519. struct paravirt_patch_site *p;
  520. char insn_buff[MAX_PATCH_LEN];
  521. for (p = start; p < end; p++) {
  522. unsigned int used;
  523. BUG_ON(p->len > MAX_PATCH_LEN);
  524. /* prep the buffer with the original instructions */
  525. memcpy(insn_buff, p->instr, p->len);
  526. used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);
  527. BUG_ON(used > p->len);
  528. /* Pad the rest with nops */
  529. add_nops(insn_buff + used, p->len - used);
  530. text_poke_early(p->instr, insn_buff, p->len);
  531. }
  532. }
  533. extern struct paravirt_patch_site __start_parainstructions[],
  534. __stop_parainstructions[];
  535. #endif /* CONFIG_PARAVIRT */
  536. /*
  537. * Self-test for the INT3 based CALL emulation code.
  538. *
  539. * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
  540. * properly and that there is a stack gap between the INT3 frame and the
  541. * previous context. Without this gap doing a virtual PUSH on the interrupted
  542. * stack would corrupt the INT3 IRET frame.
  543. *
  544. * See entry_{32,64}.S for more details.
  545. */
  546. static void __init __no_sanitize_address notrace int3_magic(unsigned int *ptr)
  547. {
  548. *ptr = 1;
  549. }
  550. extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
  551. static int __init
  552. int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
  553. {
  554. struct die_args *args = data;
  555. struct pt_regs *regs = args->regs;
  556. if (!regs || user_mode(regs))
  557. return NOTIFY_DONE;
  558. if (val != DIE_INT3)
  559. return NOTIFY_DONE;
  560. if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
  561. return NOTIFY_DONE;
  562. int3_emulate_call(regs, (unsigned long)&int3_magic);
  563. return NOTIFY_STOP;
  564. }
  565. static void __init int3_selftest(void)
  566. {
  567. static __initdata struct notifier_block int3_exception_nb = {
  568. .notifier_call = int3_exception_notify,
  569. .priority = INT_MAX-1, /* last */
  570. };
  571. unsigned int val = 0;
  572. BUG_ON(register_die_notifier(&int3_exception_nb));
  573. /*
  574. * Basically: int3_magic(&val); but really complicated :-)
  575. *
  576. * Stick the address of the INT3 instruction into int3_selftest_ip,
  577. * then trigger the INT3, padded with NOPs to match a CALL instruction
  578. * length.
  579. */
  580. asm volatile ("1: int3; nop; nop; nop; nop\n\t"
  581. ".pushsection .init.data,\"aw\"\n\t"
  582. ".align " __ASM_SEL(4, 8) "\n\t"
  583. ".type int3_selftest_ip, @object\n\t"
  584. ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
  585. "int3_selftest_ip:\n\t"
  586. __ASM_SEL(.long, .quad) " 1b\n\t"
  587. ".popsection\n\t"
  588. : ASM_CALL_CONSTRAINT
  589. : __ASM_SEL_RAW(a, D) (&val)
  590. : "memory");
  591. BUG_ON(val != 1);
  592. unregister_die_notifier(&int3_exception_nb);
  593. }
  594. void __init alternative_instructions(void)
  595. {
  596. int3_selftest();
  597. /*
  598. * The patching is not fully atomic, so try to avoid local
  599. * interruptions that might execute the to be patched code.
  600. * Other CPUs are not running.
  601. */
  602. stop_nmi();
  603. /*
  604. * Don't stop machine check exceptions while patching.
  605. * MCEs only happen when something got corrupted and in this
  606. * case we must do something about the corruption.
  607. * Ignoring it is worse than an unlikely patching race.
  608. * Also machine checks tend to be broadcast and if one CPU
  609. * goes into machine check the others follow quickly, so we don't
  610. * expect a machine check to cause undue problems during to code
  611. * patching.
  612. */
  613. apply_alternatives(__alt_instructions, __alt_instructions_end);
  614. #ifdef CONFIG_SMP
  615. /* Patch to UP if other cpus not imminent. */
  616. if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
  617. uniproc_patched = true;
  618. alternatives_smp_module_add(NULL, "core kernel",
  619. __smp_locks, __smp_locks_end,
  620. _text, _etext);
  621. }
  622. if (!uniproc_patched || num_possible_cpus() == 1) {
  623. free_init_pages("SMP alternatives",
  624. (unsigned long)__smp_locks,
  625. (unsigned long)__smp_locks_end);
  626. }
  627. #endif
  628. apply_paravirt(__parainstructions, __parainstructions_end);
  629. restart_nmi();
  630. alternatives_patched = 1;
  631. }
  632. /**
  633. * text_poke_early - Update instructions on a live kernel at boot time
  634. * @addr: address to modify
  635. * @opcode: source of the copy
  636. * @len: length to copy
  637. *
  638. * When you use this code to patch more than one byte of an instruction
  639. * you need to make sure that other CPUs cannot execute this code in parallel.
  640. * Also no thread must be currently preempted in the middle of these
  641. * instructions. And on the local CPU you need to be protected against NMI or
  642. * MCE handlers seeing an inconsistent instruction while you patch.
  643. */
  644. void __init_or_module text_poke_early(void *addr, const void *opcode,
  645. size_t len)
  646. {
  647. unsigned long flags;
  648. if (boot_cpu_has(X86_FEATURE_NX) &&
  649. is_module_text_address((unsigned long)addr)) {
  650. /*
  651. * Modules text is marked initially as non-executable, so the
  652. * code cannot be running and speculative code-fetches are
  653. * prevented. Just change the code.
  654. */
  655. memcpy(addr, opcode, len);
  656. } else {
  657. local_irq_save(flags);
  658. memcpy(addr, opcode, len);
  659. local_irq_restore(flags);
  660. sync_core();
  661. /*
  662. * Could also do a CLFLUSH here to speed up CPU recovery; but
  663. * that causes hangs on some VIA CPUs.
  664. */
  665. }
  666. }
  667. typedef struct {
  668. struct mm_struct *mm;
  669. } temp_mm_state_t;
  670. /*
  671. * Using a temporary mm allows to set temporary mappings that are not accessible
  672. * by other CPUs. Such mappings are needed to perform sensitive memory writes
  673. * that override the kernel memory protections (e.g., W^X), without exposing the
  674. * temporary page-table mappings that are required for these write operations to
  675. * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
  676. * mapping is torn down.
  677. *
  678. * Context: The temporary mm needs to be used exclusively by a single core. To
  679. * harden security IRQs must be disabled while the temporary mm is
  680. * loaded, thereby preventing interrupt handler bugs from overriding
  681. * the kernel memory protection.
  682. */
  683. static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
  684. {
  685. temp_mm_state_t temp_state;
  686. lockdep_assert_irqs_disabled();
  687. /*
  688. * Make sure not to be in TLB lazy mode, as otherwise we'll end up
  689. * with a stale address space WITHOUT being in lazy mode after
  690. * restoring the previous mm.
  691. */
  692. if (this_cpu_read(cpu_tlbstate.is_lazy))
  693. leave_mm(smp_processor_id());
  694. temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm);
  695. switch_mm_irqs_off(NULL, mm, current);
  696. /*
  697. * If breakpoints are enabled, disable them while the temporary mm is
  698. * used. Userspace might set up watchpoints on addresses that are used
  699. * in the temporary mm, which would lead to wrong signals being sent or
  700. * crashes.
  701. *
  702. * Note that breakpoints are not disabled selectively, which also causes
  703. * kernel breakpoints (e.g., perf's) to be disabled. This might be
  704. * undesirable, but still seems reasonable as the code that runs in the
  705. * temporary mm should be short.
  706. */
  707. if (hw_breakpoint_active())
  708. hw_breakpoint_disable();
  709. return temp_state;
  710. }
  711. static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
  712. {
  713. lockdep_assert_irqs_disabled();
  714. switch_mm_irqs_off(NULL, prev_state.mm, current);
  715. /*
  716. * Restore the breakpoints if they were disabled before the temporary mm
  717. * was loaded.
  718. */
  719. if (hw_breakpoint_active())
  720. hw_breakpoint_restore();
  721. }
  722. __ro_after_init struct mm_struct *poking_mm;
  723. __ro_after_init unsigned long poking_addr;
  724. static void *__text_poke(void *addr, const void *opcode, size_t len)
  725. {
  726. bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
  727. struct page *pages[2] = {NULL};
  728. temp_mm_state_t prev;
  729. unsigned long flags;
  730. pte_t pte, *ptep;
  731. spinlock_t *ptl;
  732. pgprot_t pgprot;
  733. /*
  734. * While boot memory allocator is running we cannot use struct pages as
  735. * they are not yet initialized. There is no way to recover.
  736. */
  737. BUG_ON(!after_bootmem);
  738. if (!core_kernel_text((unsigned long)addr)) {
  739. pages[0] = vmalloc_to_page(addr);
  740. if (cross_page_boundary)
  741. pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
  742. } else {
  743. pages[0] = virt_to_page(addr);
  744. WARN_ON(!PageReserved(pages[0]));
  745. if (cross_page_boundary)
  746. pages[1] = virt_to_page(addr + PAGE_SIZE);
  747. }
  748. /*
  749. * If something went wrong, crash and burn since recovery paths are not
  750. * implemented.
  751. */
  752. BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));
  753. /*
  754. * Map the page without the global bit, as TLB flushing is done with
  755. * flush_tlb_mm_range(), which is intended for non-global PTEs.
  756. */
  757. pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL);
  758. /*
  759. * The lock is not really needed, but this allows to avoid open-coding.
  760. */
  761. ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
  762. /*
  763. * This must not fail; preallocated in poking_init().
  764. */
  765. VM_BUG_ON(!ptep);
  766. local_irq_save(flags);
  767. pte = mk_pte(pages[0], pgprot);
  768. set_pte_at(poking_mm, poking_addr, ptep, pte);
  769. if (cross_page_boundary) {
  770. pte = mk_pte(pages[1], pgprot);
  771. set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte);
  772. }
  773. /*
  774. * Loading the temporary mm behaves as a compiler barrier, which
  775. * guarantees that the PTE will be set at the time memcpy() is done.
  776. */
  777. prev = use_temporary_mm(poking_mm);
  778. kasan_disable_current();
  779. memcpy((u8 *)poking_addr + offset_in_page(addr), opcode, len);
  780. kasan_enable_current();
  781. /*
  782. * Ensure that the PTE is only cleared after the instructions of memcpy
  783. * were issued by using a compiler barrier.
  784. */
  785. barrier();
  786. pte_clear(poking_mm, poking_addr, ptep);
  787. if (cross_page_boundary)
  788. pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1);
  789. /*
  790. * Loading the previous page-table hierarchy requires a serializing
  791. * instruction that already allows the core to see the updated version.
  792. * Xen-PV is assumed to serialize execution in a similar manner.
  793. */
  794. unuse_temporary_mm(prev);
  795. /*
  796. * Flushing the TLB might involve IPIs, which would require enabled
  797. * IRQs, but not if the mm is not used, as it is in this point.
  798. */
  799. flush_tlb_mm_range(poking_mm, poking_addr, poking_addr +
  800. (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
  801. PAGE_SHIFT, false);
  802. /*
  803. * If the text does not match what we just wrote then something is
  804. * fundamentally screwy; there's nothing we can really do about that.
  805. */
  806. BUG_ON(memcmp(addr, opcode, len));
  807. local_irq_restore(flags);
  808. pte_unmap_unlock(ptep, ptl);
  809. return addr;
  810. }
  811. /**
  812. * text_poke - Update instructions on a live kernel
  813. * @addr: address to modify
  814. * @opcode: source of the copy
  815. * @len: length to copy
  816. *
  817. * Only atomic text poke/set should be allowed when not doing early patching.
  818. * It means the size must be writable atomically and the address must be aligned
  819. * in a way that permits an atomic write. It also makes sure we fit on a single
  820. * page.
  821. *
  822. * Note that the caller must ensure that if the modified code is part of a
  823. * module, the module would not be removed during poking. This can be achieved
  824. * by registering a module notifier, and ordering module removal and patching
  825. * trough a mutex.
  826. */
  827. void *text_poke(void *addr, const void *opcode, size_t len)
  828. {
  829. lockdep_assert_held(&text_mutex);
  830. return __text_poke(addr, opcode, len);
  831. }
  832. /**
  833. * text_poke_kgdb - Update instructions on a live kernel by kgdb
  834. * @addr: address to modify
  835. * @opcode: source of the copy
  836. * @len: length to copy
  837. *
  838. * Only atomic text poke/set should be allowed when not doing early patching.
  839. * It means the size must be writable atomically and the address must be aligned
  840. * in a way that permits an atomic write. It also makes sure we fit on a single
  841. * page.
  842. *
  843. * Context: should only be used by kgdb, which ensures no other core is running,
  844. * despite the fact it does not hold the text_mutex.
  845. */
  846. void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
  847. {
  848. return __text_poke(addr, opcode, len);
  849. }
  850. static void do_sync_core(void *info)
  851. {
  852. sync_core();
  853. }
  854. void text_poke_sync(void)
  855. {
  856. on_each_cpu(do_sync_core, NULL, 1);
  857. }
  858. struct text_poke_loc {
  859. s32 rel_addr; /* addr := _stext + rel_addr */
  860. s32 rel32;
  861. u8 opcode;
  862. const u8 text[POKE_MAX_OPCODE_SIZE];
  863. u8 old;
  864. };
  865. struct bp_patching_desc {
  866. struct text_poke_loc *vec;
  867. int nr_entries;
  868. atomic_t refs;
  869. };
  870. static struct bp_patching_desc *bp_desc;
  871. static __always_inline
  872. struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp)
  873. {
  874. struct bp_patching_desc *desc = __READ_ONCE(*descp); /* rcu_dereference */
  875. if (!desc || !arch_atomic_inc_not_zero(&desc->refs))
  876. return NULL;
  877. return desc;
  878. }
  879. static __always_inline void put_desc(struct bp_patching_desc *desc)
  880. {
  881. smp_mb__before_atomic();
  882. arch_atomic_dec(&desc->refs);
  883. }
  884. static __always_inline void *text_poke_addr(struct text_poke_loc *tp)
  885. {
  886. return _stext + tp->rel_addr;
  887. }
  888. static __always_inline int patch_cmp(const void *key, const void *elt)
  889. {
  890. struct text_poke_loc *tp = (struct text_poke_loc *) elt;
  891. if (key < text_poke_addr(tp))
  892. return -1;
  893. if (key > text_poke_addr(tp))
  894. return 1;
  895. return 0;
  896. }
  897. noinstr int poke_int3_handler(struct pt_regs *regs)
  898. {
  899. struct bp_patching_desc *desc;
  900. struct text_poke_loc *tp;
  901. int len, ret = 0;
  902. void *ip;
  903. if (user_mode(regs))
  904. return 0;
  905. /*
  906. * Having observed our INT3 instruction, we now must observe
  907. * bp_desc:
  908. *
  909. * bp_desc = desc INT3
  910. * WMB RMB
  911. * write INT3 if (desc)
  912. */
  913. smp_rmb();
  914. desc = try_get_desc(&bp_desc);
  915. if (!desc)
  916. return 0;
  917. /*
  918. * Discount the INT3. See text_poke_bp_batch().
  919. */
  920. ip = (void *) regs->ip - INT3_INSN_SIZE;
  921. /*
  922. * Skip the binary search if there is a single member in the vector.
  923. */
  924. if (unlikely(desc->nr_entries > 1)) {
  925. tp = __inline_bsearch(ip, desc->vec, desc->nr_entries,
  926. sizeof(struct text_poke_loc),
  927. patch_cmp);
  928. if (!tp)
  929. goto out_put;
  930. } else {
  931. tp = desc->vec;
  932. if (text_poke_addr(tp) != ip)
  933. goto out_put;
  934. }
  935. len = text_opcode_size(tp->opcode);
  936. ip += len;
  937. switch (tp->opcode) {
  938. case INT3_INSN_OPCODE:
  939. /*
  940. * Someone poked an explicit INT3, they'll want to handle it,
  941. * do not consume.
  942. */
  943. goto out_put;
  944. case RET_INSN_OPCODE:
  945. int3_emulate_ret(regs);
  946. break;
  947. case CALL_INSN_OPCODE:
  948. int3_emulate_call(regs, (long)ip + tp->rel32);
  949. break;
  950. case JMP32_INSN_OPCODE:
  951. case JMP8_INSN_OPCODE:
  952. int3_emulate_jmp(regs, (long)ip + tp->rel32);
  953. break;
  954. default:
  955. BUG();
  956. }
  957. ret = 1;
  958. out_put:
  959. put_desc(desc);
  960. return ret;
  961. }
  962. #define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
  963. static struct text_poke_loc tp_vec[TP_VEC_MAX];
  964. static int tp_vec_nr;
  965. /**
  966. * text_poke_bp_batch() -- update instructions on live kernel on SMP
  967. * @tp: vector of instructions to patch
  968. * @nr_entries: number of entries in the vector
  969. *
  970. * Modify multi-byte instruction by using int3 breakpoint on SMP.
  971. * We completely avoid stop_machine() here, and achieve the
  972. * synchronization using int3 breakpoint.
  973. *
  974. * The way it is done:
  975. * - For each entry in the vector:
  976. * - add a int3 trap to the address that will be patched
  977. * - sync cores
  978. * - For each entry in the vector:
  979. * - update all but the first byte of the patched range
  980. * - sync cores
  981. * - For each entry in the vector:
  982. * - replace the first byte (int3) by the first byte of
  983. * replacing opcode
  984. * - sync cores
  985. */
  986. static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
  987. {
  988. struct bp_patching_desc desc = {
  989. .vec = tp,
  990. .nr_entries = nr_entries,
  991. .refs = ATOMIC_INIT(1),
  992. };
  993. unsigned char int3 = INT3_INSN_OPCODE;
  994. unsigned int i;
  995. int do_sync;
  996. lockdep_assert_held(&text_mutex);
  997. smp_store_release(&bp_desc, &desc); /* rcu_assign_pointer */
  998. /*
  999. * Corresponding read barrier in int3 notifier for making sure the
  1000. * nr_entries and handler are correctly ordered wrt. patching.
  1001. */
  1002. smp_wmb();
  1003. /*
  1004. * First step: add a int3 trap to the address that will be patched.
  1005. */
  1006. for (i = 0; i < nr_entries; i++) {
  1007. tp[i].old = *(u8 *)text_poke_addr(&tp[i]);
  1008. text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE);
  1009. }
  1010. text_poke_sync();
  1011. /*
  1012. * Second step: update all but the first byte of the patched range.
  1013. */
  1014. for (do_sync = 0, i = 0; i < nr_entries; i++) {
  1015. u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, };
  1016. int len = text_opcode_size(tp[i].opcode);
  1017. if (len - INT3_INSN_SIZE > 0) {
  1018. memcpy(old + INT3_INSN_SIZE,
  1019. text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
  1020. len - INT3_INSN_SIZE);
  1021. text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
  1022. (const char *)tp[i].text + INT3_INSN_SIZE,
  1023. len - INT3_INSN_SIZE);
  1024. do_sync++;
  1025. }
  1026. /*
  1027. * Emit a perf event to record the text poke, primarily to
  1028. * support Intel PT decoding which must walk the executable code
  1029. * to reconstruct the trace. The flow up to here is:
  1030. * - write INT3 byte
  1031. * - IPI-SYNC
  1032. * - write instruction tail
  1033. * At this point the actual control flow will be through the
  1034. * INT3 and handler and not hit the old or new instruction.
  1035. * Intel PT outputs FUP/TIP packets for the INT3, so the flow
  1036. * can still be decoded. Subsequently:
  1037. * - emit RECORD_TEXT_POKE with the new instruction
  1038. * - IPI-SYNC
  1039. * - write first byte
  1040. * - IPI-SYNC
  1041. * So before the text poke event timestamp, the decoder will see
  1042. * either the old instruction flow or FUP/TIP of INT3. After the
  1043. * text poke event timestamp, the decoder will see either the
  1044. * new instruction flow or FUP/TIP of INT3. Thus decoders can
  1045. * use the timestamp as the point at which to modify the
  1046. * executable code.
  1047. * The old instruction is recorded so that the event can be
  1048. * processed forwards or backwards.
  1049. */
  1050. perf_event_text_poke(text_poke_addr(&tp[i]), old, len,
  1051. tp[i].text, len);
  1052. }
  1053. if (do_sync) {
  1054. /*
  1055. * According to Intel, this core syncing is very likely
  1056. * not necessary and we'd be safe even without it. But
  1057. * better safe than sorry (plus there's not only Intel).
  1058. */
  1059. text_poke_sync();
  1060. }
  1061. /*
  1062. * Third step: replace the first byte (int3) by the first byte of
  1063. * replacing opcode.
  1064. */
  1065. for (do_sync = 0, i = 0; i < nr_entries; i++) {
  1066. if (tp[i].text[0] == INT3_INSN_OPCODE)
  1067. continue;
  1068. text_poke(text_poke_addr(&tp[i]), tp[i].text, INT3_INSN_SIZE);
  1069. do_sync++;
  1070. }
  1071. if (do_sync)
  1072. text_poke_sync();
  1073. /*
  1074. * Remove and synchronize_rcu(), except we have a very primitive
  1075. * refcount based completion.
  1076. */
  1077. WRITE_ONCE(bp_desc, NULL); /* RCU_INIT_POINTER */
  1078. if (!atomic_dec_and_test(&desc.refs))
  1079. atomic_cond_read_acquire(&desc.refs, !VAL);
  1080. }
  1081. static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
  1082. const void *opcode, size_t len, const void *emulate)
  1083. {
  1084. struct insn insn;
  1085. memcpy((void *)tp->text, opcode, len);
  1086. if (!emulate)
  1087. emulate = opcode;
  1088. kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
  1089. insn_get_length(&insn);
  1090. BUG_ON(!insn_complete(&insn));
  1091. BUG_ON(len != insn.length);
  1092. tp->rel_addr = addr - (void *)_stext;
  1093. tp->opcode = insn.opcode.bytes[0];
  1094. switch (tp->opcode) {
  1095. case INT3_INSN_OPCODE:
  1096. case RET_INSN_OPCODE:
  1097. break;
  1098. case CALL_INSN_OPCODE:
  1099. case JMP32_INSN_OPCODE:
  1100. case JMP8_INSN_OPCODE:
  1101. tp->rel32 = insn.immediate.value;
  1102. break;
  1103. default: /* assume NOP */
  1104. switch (len) {
  1105. case 2: /* NOP2 -- emulate as JMP8+0 */
  1106. BUG_ON(memcmp(emulate, ideal_nops[len], len));
  1107. tp->opcode = JMP8_INSN_OPCODE;
  1108. tp->rel32 = 0;
  1109. break;
  1110. case 5: /* NOP5 -- emulate as JMP32+0 */
  1111. BUG_ON(memcmp(emulate, ideal_nops[NOP_ATOMIC5], len));
  1112. tp->opcode = JMP32_INSN_OPCODE;
  1113. tp->rel32 = 0;
  1114. break;
  1115. default: /* unknown instruction */
  1116. BUG();
  1117. }
  1118. break;
  1119. }
  1120. }
  1121. /*
  1122. * We hard rely on the tp_vec being ordered; ensure this is so by flushing
  1123. * early if needed.
  1124. */
  1125. static bool tp_order_fail(void *addr)
  1126. {
  1127. struct text_poke_loc *tp;
  1128. if (!tp_vec_nr)
  1129. return false;
  1130. if (!addr) /* force */
  1131. return true;
  1132. tp = &tp_vec[tp_vec_nr - 1];
  1133. if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr)
  1134. return true;
  1135. return false;
  1136. }
  1137. static void text_poke_flush(void *addr)
  1138. {
  1139. if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) {
  1140. text_poke_bp_batch(tp_vec, tp_vec_nr);
  1141. tp_vec_nr = 0;
  1142. }
  1143. }
  1144. void text_poke_finish(void)
  1145. {
  1146. text_poke_flush(NULL);
  1147. }
  1148. void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate)
  1149. {
  1150. struct text_poke_loc *tp;
  1151. if (unlikely(system_state == SYSTEM_BOOTING)) {
  1152. text_poke_early(addr, opcode, len);
  1153. return;
  1154. }
  1155. text_poke_flush(addr);
  1156. tp = &tp_vec[tp_vec_nr++];
  1157. text_poke_loc_init(tp, addr, opcode, len, emulate);
  1158. }
  1159. /**
  1160. * text_poke_bp() -- update instructions on live kernel on SMP
  1161. * @addr: address to patch
  1162. * @opcode: opcode of new instruction
  1163. * @len: length to copy
  1164. * @handler: address to jump to when the temporary breakpoint is hit
  1165. *
  1166. * Update a single instruction with the vector in the stack, avoiding
  1167. * dynamically allocated memory. This function should be used when it is
  1168. * not possible to allocate memory.
  1169. */
  1170. void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
  1171. {
  1172. struct text_poke_loc tp;
  1173. if (unlikely(system_state == SYSTEM_BOOTING)) {
  1174. text_poke_early(addr, opcode, len);
  1175. return;
  1176. }
  1177. text_poke_loc_init(&tp, addr, opcode, len, emulate);
  1178. text_poke_bp_batch(&tp, 1);
  1179. }