compiler.h 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef __LINUX_COMPILER_H
  3. #define __LINUX_COMPILER_H
  4. #include <linux/compiler_types.h>
  5. #ifndef __ASSEMBLY__
  6. #ifdef __KERNEL__
  7. /*
  8. * Note: DISABLE_BRANCH_PROFILING can be used by special lowlevel code
  9. * to disable branch tracing on a per file basis.
  10. */
  11. #if defined(CONFIG_TRACE_BRANCH_PROFILING) \
  12. && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
  13. void ftrace_likely_update(struct ftrace_likely_data *f, int val,
  14. int expect, int is_constant);
  15. #define likely_notrace(x) __builtin_expect(!!(x), 1)
  16. #define unlikely_notrace(x) __builtin_expect(!!(x), 0)
  17. #define __branch_check__(x, expect, is_constant) ({ \
  18. long ______r; \
  19. static struct ftrace_likely_data \
  20. __aligned(4) \
  21. __section("_ftrace_annotated_branch") \
  22. ______f = { \
  23. .data.func = __func__, \
  24. .data.file = __FILE__, \
  25. .data.line = __LINE__, \
  26. }; \
  27. ______r = __builtin_expect(!!(x), expect); \
  28. ftrace_likely_update(&______f, ______r, \
  29. expect, is_constant); \
  30. ______r; \
  31. })
  32. /*
  33. * Using __builtin_constant_p(x) to ignore cases where the return
  34. * value is always the same. This idea is taken from a similar patch
  35. * written by Daniel Walker.
  36. */
  37. # ifndef likely
  38. # define likely(x) (__branch_check__(x, 1, __builtin_constant_p(x)))
  39. # endif
  40. # ifndef unlikely
  41. # define unlikely(x) (__branch_check__(x, 0, __builtin_constant_p(x)))
  42. # endif
  43. #ifdef CONFIG_PROFILE_ALL_BRANCHES
  44. /*
  45. * "Define 'is'", Bill Clinton
  46. * "Define 'if'", Steven Rostedt
  47. */
  48. #define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )
  49. #define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))
  50. #define __trace_if_value(cond) ({ \
  51. static struct ftrace_branch_data \
  52. __aligned(4) \
  53. __section("_ftrace_branch") \
  54. __if_trace = { \
  55. .func = __func__, \
  56. .file = __FILE__, \
  57. .line = __LINE__, \
  58. }; \
  59. (cond) ? \
  60. (__if_trace.miss_hit[1]++,1) : \
  61. (__if_trace.miss_hit[0]++,0); \
  62. })
  63. #endif /* CONFIG_PROFILE_ALL_BRANCHES */
  64. #else
  65. # define likely(x) __builtin_expect(!!(x), 1)
  66. # define unlikely(x) __builtin_expect(!!(x), 0)
  67. #endif
  68. /* Optimization barrier */
  69. #ifndef barrier
  70. /* The "volatile" is due to gcc bugs */
  71. # define barrier() __asm__ __volatile__("": : :"memory")
  72. #endif
  73. #ifndef barrier_data
  74. /*
  75. * This version is i.e. to prevent dead stores elimination on @ptr
  76. * where gcc and llvm may behave differently when otherwise using
  77. * normal barrier(): while gcc behavior gets along with a normal
  78. * barrier(), llvm needs an explicit input variable to be assumed
  79. * clobbered. The issue is as follows: while the inline asm might
  80. * access any memory it wants, the compiler could have fit all of
  81. * @ptr into memory registers instead, and since @ptr never escaped
  82. * from that, it proved that the inline asm wasn't touching any of
  83. * it. This version works well with both compilers, i.e. we're telling
  84. * the compiler that the inline asm absolutely may see the contents
  85. * of @ptr. See also: https://llvm.org/bugs/show_bug.cgi?id=15495
  86. */
  87. # define barrier_data(ptr) __asm__ __volatile__("": :"r"(ptr) :"memory")
  88. #endif
  89. /* workaround for GCC PR82365 if needed */
  90. #ifndef barrier_before_unreachable
  91. # define barrier_before_unreachable() do { } while (0)
  92. #endif
  93. /* Unreachable code */
  94. #ifdef CONFIG_STACK_VALIDATION
  95. /*
  96. * These macros help objtool understand GCC code flow for unreachable code.
  97. * The __COUNTER__ based labels are a hack to make each instance of the macros
  98. * unique, to convince GCC not to merge duplicate inline asm statements.
  99. */
  100. #define annotate_reachable() ({ \
  101. asm volatile("%c0:\n\t" \
  102. ".pushsection .discard.reachable\n\t" \
  103. ".long %c0b - .\n\t" \
  104. ".popsection\n\t" : : "i" (__COUNTER__)); \
  105. })
  106. #define annotate_unreachable() ({ \
  107. asm volatile("%c0:\n\t" \
  108. ".pushsection .discard.unreachable\n\t" \
  109. ".long %c0b - .\n\t" \
  110. ".popsection\n\t" : : "i" (__COUNTER__)); \
  111. })
  112. #define ASM_UNREACHABLE \
  113. "999:\n\t" \
  114. ".pushsection .discard.unreachable\n\t" \
  115. ".long 999b - .\n\t" \
  116. ".popsection\n\t"
  117. /* Annotate a C jump table to allow objtool to follow the code flow */
  118. #define __annotate_jump_table __section(".rodata..c_jump_table")
  119. #else
  120. #define annotate_reachable()
  121. #define annotate_unreachable()
  122. #define __annotate_jump_table
  123. #endif
  124. #ifndef ASM_UNREACHABLE
  125. # define ASM_UNREACHABLE
  126. #endif
  127. #ifndef unreachable
  128. # define unreachable() do { \
  129. annotate_unreachable(); \
  130. __builtin_unreachable(); \
  131. } while (0)
  132. #endif
  133. /*
  134. * KENTRY - kernel entry point
  135. * This can be used to annotate symbols (functions or data) that are used
  136. * without their linker symbol being referenced explicitly. For example,
  137. * interrupt vector handlers, or functions in the kernel image that are found
  138. * programatically.
  139. *
  140. * Not required for symbols exported with EXPORT_SYMBOL, or initcalls. Those
  141. * are handled in their own way (with KEEP() in linker scripts).
  142. *
  143. * KENTRY can be avoided if the symbols in question are marked as KEEP() in the
  144. * linker script. For example an architecture could KEEP() its entire
  145. * boot/exception vector code rather than annotate each function and data.
  146. */
  147. #ifndef KENTRY
  148. # define KENTRY(sym) \
  149. extern typeof(sym) sym; \
  150. static const unsigned long __kentry_##sym \
  151. __used \
  152. __attribute__((__section__("___kentry+" #sym))) \
  153. = (unsigned long)&sym;
  154. #endif
  155. #ifndef RELOC_HIDE
  156. # define RELOC_HIDE(ptr, off) \
  157. ({ unsigned long __ptr; \
  158. __ptr = (unsigned long) (ptr); \
  159. (typeof(ptr)) (__ptr + (off)); })
  160. #endif
  161. #define absolute_pointer(val) RELOC_HIDE((void *)(val), 0)
  162. #ifndef OPTIMIZER_HIDE_VAR
  163. /* Make the optimizer believe the variable can be manipulated arbitrarily. */
  164. #define OPTIMIZER_HIDE_VAR(var) \
  165. __asm__ ("" : "=r" (var) : "0" (var))
  166. #endif
  167. /* Not-quite-unique ID. */
  168. #ifndef __UNIQUE_ID
  169. # define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
  170. #endif
  171. /**
  172. * data_race - mark an expression as containing intentional data races
  173. *
  174. * This data_race() macro is useful for situations in which data races
  175. * should be forgiven. One example is diagnostic code that accesses
  176. * shared variables but is not a part of the core synchronization design.
  177. *
  178. * This macro *does not* affect normal code generation, but is a hint
  179. * to tooling that data races here are to be ignored.
  180. */
  181. #define data_race(expr) \
  182. ({ \
  183. __unqual_scalar_typeof(({ expr; })) __v = ({ \
  184. __kcsan_disable_current(); \
  185. expr; \
  186. }); \
  187. __kcsan_enable_current(); \
  188. __v; \
  189. })
  190. #endif /* __KERNEL__ */
  191. /*
  192. * Force the compiler to emit 'sym' as a symbol, so that we can reference
  193. * it from inline assembler. Necessary in case 'sym' could be inlined
  194. * otherwise, or eliminated entirely due to lack of references that are
  195. * visible to the compiler.
  196. */
  197. #define __ADDRESSABLE(sym) \
  198. static void * __section(".discard.addressable") __used \
  199. __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
  200. /**
  201. * offset_to_ptr - convert a relative memory offset to an absolute pointer
  202. * @off: the address of the 32-bit offset value
  203. */
  204. static inline void *offset_to_ptr(const int *off)
  205. {
  206. return (void *)((unsigned long)off + *off);
  207. }
  208. #endif /* __ASSEMBLY__ */
  209. /* &a[0] degrades to a pointer: a different type from an array */
  210. #define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
  211. /*
  212. * This is needed in functions which generate the stack canary, see
  213. * arch/x86/kernel/smpboot.c::start_secondary() for an example.
  214. */
  215. #define prevent_tail_call_optimization() mb()
  216. #include <asm/rwonce.h>
  217. #endif /* __LINUX_COMPILER_H */