static_call.c 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/static_call.h>
  3. #include <linux/memory.h>
  4. #include <linux/bug.h>
  5. #include <asm/text-patching.h>
  6. enum insn_type {
  7. CALL = 0, /* site call */
  8. NOP = 1, /* site cond-call */
  9. JMP = 2, /* tramp / site tail-call */
  10. RET = 3, /* tramp / site cond-tail-call */
  11. };
  12. static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
  13. {
  14. int size = CALL_INSN_SIZE;
  15. const void *code;
  16. switch (type) {
  17. case CALL:
  18. code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
  19. break;
  20. case NOP:
  21. code = ideal_nops[NOP_ATOMIC5];
  22. break;
  23. case JMP:
  24. code = text_gen_insn(JMP32_INSN_OPCODE, insn, func);
  25. break;
  26. case RET:
  27. code = text_gen_insn(RET_INSN_OPCODE, insn, func);
  28. size = RET_INSN_SIZE;
  29. break;
  30. }
  31. if (memcmp(insn, code, size) == 0)
  32. return;
  33. if (unlikely(system_state == SYSTEM_BOOTING))
  34. return text_poke_early(insn, code, size);
  35. text_poke_bp(insn, code, size, NULL);
  36. }
  37. static void __static_call_validate(void *insn, bool tail)
  38. {
  39. u8 opcode = *(u8 *)insn;
  40. if (tail) {
  41. if (opcode == JMP32_INSN_OPCODE ||
  42. opcode == RET_INSN_OPCODE)
  43. return;
  44. } else {
  45. if (opcode == CALL_INSN_OPCODE ||
  46. !memcmp(insn, ideal_nops[NOP_ATOMIC5], 5))
  47. return;
  48. }
  49. /*
  50. * If we ever trigger this, our text is corrupt, we'll probably not live long.
  51. */
  52. WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);
  53. }
  54. static inline enum insn_type __sc_insn(bool null, bool tail)
  55. {
  56. /*
  57. * Encode the following table without branches:
  58. *
  59. * tail null insn
  60. * -----+-------+------
  61. * 0 | 0 | CALL
  62. * 0 | 1 | NOP
  63. * 1 | 0 | JMP
  64. * 1 | 1 | RET
  65. */
  66. return 2*tail + null;
  67. }
  68. void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
  69. {
  70. mutex_lock(&text_mutex);
  71. if (tramp) {
  72. __static_call_validate(tramp, true);
  73. __static_call_transform(tramp, __sc_insn(!func, true), func);
  74. }
  75. if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) {
  76. __static_call_validate(site, tail);
  77. __static_call_transform(site, __sc_insn(!func, tail), func);
  78. }
  79. mutex_unlock(&text_mutex);
  80. }
  81. EXPORT_SYMBOL_GPL(arch_static_call_transform);