jump_label.c 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * jump label x86 support
  4. *
  5. * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
  6. *
  7. */
  8. #include <linux/jump_label.h>
  9. #include <linux/memory.h>
  10. #include <linux/uaccess.h>
  11. #include <linux/module.h>
  12. #include <linux/list.h>
  13. #include <linux/jhash.h>
  14. #include <linux/cpu.h>
  15. #include <asm/kprobes.h>
  16. #include <asm/alternative.h>
  17. #include <asm/text-patching.h>
  18. static void bug_at(const void *ip, int line)
  19. {
  20. /*
  21. * The location is not an op that we were expecting.
  22. * Something went wrong. Crash the box, as something could be
  23. * corrupting the kernel.
  24. */
  25. pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line);
  26. BUG();
  27. }
  28. static const void *
  29. __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init)
  30. {
  31. const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
  32. const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
  33. const void *expect, *code;
  34. const void *addr, *dest;
  35. int line;
  36. addr = (void *)jump_entry_code(entry);
  37. dest = (void *)jump_entry_target(entry);
  38. code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
  39. if (init) {
  40. expect = default_nop; line = __LINE__;
  41. } else if (type == JUMP_LABEL_JMP) {
  42. expect = ideal_nop; line = __LINE__;
  43. } else {
  44. expect = code; line = __LINE__;
  45. }
  46. if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE))
  47. bug_at(addr, line);
  48. if (type == JUMP_LABEL_NOP)
  49. code = ideal_nop;
  50. return code;
  51. }
  52. static inline void __jump_label_transform(struct jump_entry *entry,
  53. enum jump_label_type type,
  54. int init)
  55. {
  56. const void *opcode = __jump_label_set_jump_code(entry, type, init);
  57. /*
  58. * As long as only a single processor is running and the code is still
  59. * not marked as RO, text_poke_early() can be used; Checking that
  60. * system_state is SYSTEM_BOOTING guarantees it. It will be set to
  61. * SYSTEM_SCHEDULING before other cores are awaken and before the
  62. * code is write-protected.
  63. *
  64. * At the time the change is being done, just ignore whether we
  65. * are doing nop -> jump or jump -> nop transition, and assume
  66. * always nop being the 'currently valid' instruction
  67. */
  68. if (init || system_state == SYSTEM_BOOTING) {
  69. text_poke_early((void *)jump_entry_code(entry), opcode,
  70. JUMP_LABEL_NOP_SIZE);
  71. return;
  72. }
  73. text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL);
  74. }
  75. static void __ref jump_label_transform(struct jump_entry *entry,
  76. enum jump_label_type type,
  77. int init)
  78. {
  79. mutex_lock(&text_mutex);
  80. __jump_label_transform(entry, type, init);
  81. mutex_unlock(&text_mutex);
  82. }
  83. void arch_jump_label_transform(struct jump_entry *entry,
  84. enum jump_label_type type)
  85. {
  86. jump_label_transform(entry, type, 0);
  87. }
  88. bool arch_jump_label_transform_queue(struct jump_entry *entry,
  89. enum jump_label_type type)
  90. {
  91. const void *opcode;
  92. if (system_state == SYSTEM_BOOTING) {
  93. /*
  94. * Fallback to the non-batching mode.
  95. */
  96. arch_jump_label_transform(entry, type);
  97. return true;
  98. }
  99. mutex_lock(&text_mutex);
  100. opcode = __jump_label_set_jump_code(entry, type, 0);
  101. text_poke_queue((void *)jump_entry_code(entry),
  102. opcode, JUMP_LABEL_NOP_SIZE, NULL);
  103. mutex_unlock(&text_mutex);
  104. return true;
  105. }
  106. void arch_jump_label_transform_apply(void)
  107. {
  108. mutex_lock(&text_mutex);
  109. text_poke_finish();
  110. mutex_unlock(&text_mutex);
  111. }
  112. static enum {
  113. JL_STATE_START,
  114. JL_STATE_NO_UPDATE,
  115. JL_STATE_UPDATE,
  116. } jlstate __initdata_or_module = JL_STATE_START;
  117. __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
  118. enum jump_label_type type)
  119. {
  120. /*
  121. * This function is called at boot up and when modules are
  122. * first loaded. Check if the default nop, the one that is
  123. * inserted at compile time, is the ideal nop. If it is, then
  124. * we do not need to update the nop, and we can leave it as is.
  125. * If it is not, then we need to update the nop to the ideal nop.
  126. */
  127. if (jlstate == JL_STATE_START) {
  128. const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
  129. const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
  130. if (memcmp(ideal_nop, default_nop, 5) != 0)
  131. jlstate = JL_STATE_UPDATE;
  132. else
  133. jlstate = JL_STATE_NO_UPDATE;
  134. }
  135. if (jlstate == JL_STATE_UPDATE)
  136. jump_label_transform(entry, type, 1);
  137. }