jump_label.c 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/kernel.h>
  3. #include <linux/jump_label.h>
  4. #include "asm/cacheflush.h"
  5. #define JUMPLABEL_ERR "ARC: jump_label: ERROR: "
  6. /* Halt system on fatal error to make debug easier */
  7. #define arc_jl_fatal(format...) \
  8. ({ \
  9. pr_err(JUMPLABEL_ERR format); \
  10. BUG(); \
  11. })
  12. static inline u32 arc_gen_nop(void)
  13. {
  14. /* 1x 32bit NOP in middle endian */
  15. return 0x7000264a;
  16. }
  17. /*
  18. * Atomic update of patched instruction is only available if this
  19. * instruction doesn't cross L1 cache line boundary. You can read about
  20. * the way we achieve this in arc/include/asm/jump_label.h
  21. */
  22. static inline void instruction_align_assert(void *addr, int len)
  23. {
  24. unsigned long a = (unsigned long)addr;
  25. if ((a >> L1_CACHE_SHIFT) != ((a + len - 1) >> L1_CACHE_SHIFT))
  26. arc_jl_fatal("instruction (addr %px) cross L1 cache line border",
  27. addr);
  28. }
  29. /*
  30. * ARCv2 'Branch unconditionally' instruction:
  31. * 00000ssssssssss1SSSSSSSSSSNRtttt
  32. * s S[n:0] lower bits signed immediate (number is bitfield size)
  33. * S S[m:n+1] upper bits signed immediate (number is bitfield size)
  34. * t S[24:21] upper bits signed immediate (branch unconditionally far)
  35. * N N <.d> delay slot mode
  36. * R R Reserved
  37. */
  38. static inline u32 arc_gen_branch(jump_label_t pc, jump_label_t target)
  39. {
  40. u32 instruction_l, instruction_r;
  41. u32 pcl = pc & GENMASK(31, 2);
  42. u32 u_offset = target - pcl;
  43. u32 s, S, t;
  44. /*
  45. * Offset in 32-bit branch instruction must to fit into s25.
  46. * Something is terribly broken if we get such huge offset within one
  47. * function.
  48. */
  49. if ((s32)u_offset < -16777216 || (s32)u_offset > 16777214)
  50. arc_jl_fatal("gen branch with offset (%d) not fit in s25",
  51. (s32)u_offset);
  52. /*
  53. * All instructions are aligned by 2 bytes so we should never get offset
  54. * here which is not 2 bytes aligned.
  55. */
  56. if (u_offset & 0x1)
  57. arc_jl_fatal("gen branch with offset (%d) unaligned to 2 bytes",
  58. (s32)u_offset);
  59. s = (u_offset >> 1) & GENMASK(9, 0);
  60. S = (u_offset >> 11) & GENMASK(9, 0);
  61. t = (u_offset >> 21) & GENMASK(3, 0);
  62. /* 00000ssssssssss1 */
  63. instruction_l = (s << 1) | 0x1;
  64. /* SSSSSSSSSSNRtttt */
  65. instruction_r = (S << 6) | t;
  66. return (instruction_r << 16) | (instruction_l & GENMASK(15, 0));
  67. }
  68. void arch_jump_label_transform(struct jump_entry *entry,
  69. enum jump_label_type type)
  70. {
  71. jump_label_t *instr_addr = (jump_label_t *)entry->code;
  72. u32 instr;
  73. instruction_align_assert(instr_addr, JUMP_LABEL_NOP_SIZE);
  74. if (type == JUMP_LABEL_JMP)
  75. instr = arc_gen_branch(entry->code, entry->target);
  76. else
  77. instr = arc_gen_nop();
  78. WRITE_ONCE(*instr_addr, instr);
  79. flush_icache_range(entry->code, entry->code + JUMP_LABEL_NOP_SIZE);
  80. }
  81. void arch_jump_label_transform_static(struct jump_entry *entry,
  82. enum jump_label_type type)
  83. {
  84. /*
  85. * We use only one NOP type (1x, 4 byte) in arch_static_branch, so
  86. * there's no need to patch an identical NOP over the top of it here.
  87. * The generic code calls 'arch_jump_label_transform' if the NOP needs
  88. * to be replaced by a branch, so 'arch_jump_label_transform_static' is
  89. * never called with type other than JUMP_LABEL_NOP.
  90. */
  91. BUG_ON(type != JUMP_LABEL_NOP);
  92. }
  93. #ifdef CONFIG_ARC_DBG_JUMP_LABEL
  94. #define SELFTEST_MSG "ARC: instruction generation self-test: "
  95. struct arc_gen_branch_testdata {
  96. jump_label_t pc;
  97. jump_label_t target_address;
  98. u32 expected_instr;
  99. };
  100. static __init int branch_gen_test(const struct arc_gen_branch_testdata *test)
  101. {
  102. u32 instr_got;
  103. instr_got = arc_gen_branch(test->pc, test->target_address);
  104. if (instr_got == test->expected_instr)
  105. return 0;
  106. pr_err(SELFTEST_MSG "FAIL:\n arc_gen_branch(0x%08x, 0x%08x) != 0x%08x, got 0x%08x\n",
  107. test->pc, test->target_address,
  108. test->expected_instr, instr_got);
  109. return -EFAULT;
  110. }
  111. /*
  112. * Offset field in branch instruction is not continuous. Test all
  113. * available offset field and sign combinations. Test data is generated
  114. * from real working code.
  115. */
  116. static const struct arc_gen_branch_testdata arcgenbr_test_data[] __initconst = {
  117. {0x90007548, 0x90007514, 0xffcf07cd}, /* tiny (-52) offs */
  118. {0x9000c9c0, 0x9000c782, 0xffcf05c3}, /* tiny (-574) offs */
  119. {0x9000cc1c, 0x9000c782, 0xffcf0367}, /* tiny (-1178) offs */
  120. {0x9009dce0, 0x9009d106, 0xff8f0427}, /* small (-3034) offs */
  121. {0x9000f5de, 0x90007d30, 0xfc0f0755}, /* big (-30892) offs */
  122. {0x900a2444, 0x90035f64, 0xc9cf0321}, /* huge (-443616) offs */
  123. {0x90007514, 0x9000752c, 0x00000019}, /* tiny (+24) offs */
  124. {0x9001a578, 0x9001a77a, 0x00000203}, /* tiny (+514) offs */
  125. {0x90031ed8, 0x90032634, 0x0000075d}, /* tiny (+1884) offs */
  126. {0x9008c7f2, 0x9008d3f0, 0x00400401}, /* small (+3072) offs */
  127. {0x9000bb38, 0x9003b340, 0x17c00009}, /* big (+194568) offs */
  128. {0x90008f44, 0x90578d80, 0xb7c2063d} /* huge (+5701180) offs */
  129. };
  130. static __init int instr_gen_test(void)
  131. {
  132. int i;
  133. for (i = 0; i < ARRAY_SIZE(arcgenbr_test_data); i++)
  134. if (branch_gen_test(&arcgenbr_test_data[i]))
  135. return -EFAULT;
  136. pr_info(SELFTEST_MSG "OK\n");
  137. return 0;
  138. }
  139. early_initcall(instr_gen_test);
  140. #endif /* CONFIG_ARC_DBG_JUMP_LABEL */