compiler.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef __LINUX_COMPILER_H
  3. #define __LINUX_COMPILER_H
  4. #include <linux/compiler_types.h>
  5. #ifndef __ASSEMBLY__
  6. #ifdef __KERNEL__
  7. /*
  8. * Note: DISABLE_BRANCH_PROFILING can be used by special lowlevel code
  9. * to disable branch tracing on a per file basis.
  10. */
  11. #if defined(CONFIG_TRACE_BRANCH_PROFILING) \
  12. && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
  13. void ftrace_likely_update(struct ftrace_likely_data *f, int val,
  14. int expect, int is_constant);
  15. #define likely_notrace(x) __builtin_expect(!!(x), 1)
  16. #define unlikely_notrace(x) __builtin_expect(!!(x), 0)
  17. #define __branch_check__(x, expect, is_constant) ({ \
  18. long ______r; \
  19. static struct ftrace_likely_data \
  20. __aligned(4) \
  21. __section(_ftrace_annotated_branch) \
  22. ______f = { \
  23. .data.func = __func__, \
  24. .data.file = __FILE__, \
  25. .data.line = __LINE__, \
  26. }; \
  27. ______r = __builtin_expect(!!(x), expect); \
  28. ftrace_likely_update(&______f, ______r, \
  29. expect, is_constant); \
  30. ______r; \
  31. })
  32. /*
  33. * Using __builtin_constant_p(x) to ignore cases where the return
  34. * value is always the same. This idea is taken from a similar patch
  35. * written by Daniel Walker.
  36. */
  37. # ifndef likely
  38. # define likely(x) (__branch_check__(x, 1, __builtin_constant_p(x)))
  39. # endif
  40. # ifndef unlikely
  41. # define unlikely(x) (__branch_check__(x, 0, __builtin_constant_p(x)))
  42. # endif
  43. #ifdef CONFIG_PROFILE_ALL_BRANCHES
  44. /*
  45. * "Define 'is'", Bill Clinton
  46. * "Define 'if'", Steven Rostedt
  47. */
  48. #define if(cond, ...) if ( __trace_if_var( !!(cond , ## __VA_ARGS__) ) )
  49. #define __trace_if_var(cond) (__builtin_constant_p(cond) ? (cond) : __trace_if_value(cond))
  50. #define __trace_if_value(cond) ({ \
  51. static struct ftrace_branch_data \
  52. __aligned(4) \
  53. __section(_ftrace_branch) \
  54. __if_trace = { \
  55. .func = __func__, \
  56. .file = __FILE__, \
  57. .line = __LINE__, \
  58. }; \
  59. (cond) ? \
  60. (__if_trace.miss_hit[1]++,1) : \
  61. (__if_trace.miss_hit[0]++,0); \
  62. })
  63. #endif /* CONFIG_PROFILE_ALL_BRANCHES */
  64. #else
  65. # define likely(x) __builtin_expect(!!(x), 1)
  66. # define unlikely(x) __builtin_expect(!!(x), 0)
  67. #endif
  68. /* Optimization barrier */
  69. #ifndef barrier
  70. # define barrier() __memory_barrier()
  71. #endif
  72. #ifndef barrier_data
  73. # define barrier_data(ptr) barrier()
  74. #endif
  75. /* workaround for GCC PR82365 if needed */
  76. #ifndef barrier_before_unreachable
  77. # define barrier_before_unreachable() do { } while (0)
  78. #endif
  79. /* Unreachable code */
  80. #ifdef CONFIG_STACK_VALIDATION
  81. /*
  82. * These macros help objtool understand GCC code flow for unreachable code.
  83. * The __COUNTER__ based labels are a hack to make each instance of the macros
  84. * unique, to convince GCC not to merge duplicate inline asm statements.
  85. */
  86. #define annotate_reachable() ({ \
  87. asm volatile("%c0:\n\t" \
  88. ".pushsection .discard.reachable\n\t" \
  89. ".long %c0b - .\n\t" \
  90. ".popsection\n\t" : : "i" (__COUNTER__)); \
  91. })
  92. #define annotate_unreachable() ({ \
  93. asm volatile("%c0:\n\t" \
  94. ".pushsection .discard.unreachable\n\t" \
  95. ".long %c0b - .\n\t" \
  96. ".popsection\n\t" : : "i" (__COUNTER__)); \
  97. })
  98. #define ASM_UNREACHABLE \
  99. "999:\n\t" \
  100. ".pushsection .discard.unreachable\n\t" \
  101. ".long 999b - .\n\t" \
  102. ".popsection\n\t"
  103. /* Annotate a C jump table to allow objtool to follow the code flow */
  104. #define __annotate_jump_table __section(.rodata..c_jump_table)
  105. #else
  106. #define annotate_reachable()
  107. #define annotate_unreachable()
  108. #define __annotate_jump_table
  109. #endif
  110. #ifndef ASM_UNREACHABLE
  111. # define ASM_UNREACHABLE
  112. #endif
  113. #ifndef unreachable
  114. # define unreachable() do { \
  115. annotate_unreachable(); \
  116. __builtin_unreachable(); \
  117. } while (0)
  118. #endif
  119. /*
  120. * KENTRY - kernel entry point
  121. * This can be used to annotate symbols (functions or data) that are used
  122. * without their linker symbol being referenced explicitly. For example,
  123. * interrupt vector handlers, or functions in the kernel image that are found
  124. * programatically.
  125. *
  126. * Not required for symbols exported with EXPORT_SYMBOL, or initcalls. Those
  127. * are handled in their own way (with KEEP() in linker scripts).
  128. *
  129. * KENTRY can be avoided if the symbols in question are marked as KEEP() in the
  130. * linker script. For example an architecture could KEEP() its entire
  131. * boot/exception vector code rather than annotate each function and data.
  132. */
  133. #ifndef KENTRY
  134. # define KENTRY(sym) \
  135. extern typeof(sym) sym; \
  136. static const unsigned long __kentry_##sym \
  137. __used \
  138. __section("___kentry" "+" #sym ) \
  139. = (unsigned long)&sym;
  140. #endif
  141. #ifndef RELOC_HIDE
  142. # define RELOC_HIDE(ptr, off) \
  143. ({ unsigned long __ptr; \
  144. __ptr = (unsigned long) (ptr); \
  145. (typeof(ptr)) (__ptr + (off)); })
  146. #endif
  147. #ifndef OPTIMIZER_HIDE_VAR
  148. /* Make the optimizer believe the variable can be manipulated arbitrarily. */
  149. #define OPTIMIZER_HIDE_VAR(var) \
  150. __asm__ ("" : "=r" (var) : "0" (var))
  151. #endif
  152. /* Not-quite-unique ID. */
  153. #ifndef __UNIQUE_ID
  154. # define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
  155. #endif
  156. #include <linux/types.h>
  157. #define __READ_ONCE_SIZE \
  158. ({ \
  159. switch (size) { \
  160. case 1: *(__u8 *)res = *(volatile __u8 *)p; break; \
  161. case 2: *(__u16 *)res = *(volatile __u16 *)p; break; \
  162. case 4: *(__u32 *)res = *(volatile __u32 *)p; break; \
  163. case 8: *(__u64 *)res = *(volatile __u64 *)p; break; \
  164. default: \
  165. barrier(); \
  166. __builtin_memcpy((void *)res, (const void *)p, size); \
  167. barrier(); \
  168. } \
  169. })
  170. static __always_inline
  171. void __read_once_size(const volatile void *p, void *res, int size)
  172. {
  173. __READ_ONCE_SIZE;
  174. }
  175. #ifdef CONFIG_KASAN
  176. /*
  177. * We can't declare function 'inline' because __no_sanitize_address confilcts
  178. * with inlining. Attempt to inline it may cause a build failure.
  179. * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368
  180. * '__maybe_unused' allows us to avoid defined-but-not-used warnings.
  181. */
  182. # define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused
  183. #else
  184. # define __no_kasan_or_inline __always_inline
  185. #endif
  186. static __no_kasan_or_inline
  187. void __read_once_size_nocheck(const volatile void *p, void *res, int size)
  188. {
  189. __READ_ONCE_SIZE;
  190. }
  191. static __always_inline void __write_once_size(volatile void *p, void *res, int size)
  192. {
  193. switch (size) {
  194. case 1: *(volatile __u8 *)p = *(__u8 *)res; break;
  195. case 2: *(volatile __u16 *)p = *(__u16 *)res; break;
  196. case 4: *(volatile __u32 *)p = *(__u32 *)res; break;
  197. case 8: *(volatile __u64 *)p = *(__u64 *)res; break;
  198. default:
  199. barrier();
  200. __builtin_memcpy((void *)p, (const void *)res, size);
  201. barrier();
  202. }
  203. }
  204. /*
  205. * Prevent the compiler from merging or refetching reads or writes. The
  206. * compiler is also forbidden from reordering successive instances of
  207. * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
  208. * particular ordering. One way to make the compiler aware of ordering is to
  209. * put the two invocations of READ_ONCE or WRITE_ONCE in different C
  210. * statements.
  211. *
  212. * These two macros will also work on aggregate data types like structs or
  213. * unions. If the size of the accessed data type exceeds the word size of
  214. * the machine (e.g., 32 bits or 64 bits) READ_ONCE() and WRITE_ONCE() will
  215. * fall back to memcpy(). There's at least two memcpy()s: one for the
  216. * __builtin_memcpy() and then one for the macro doing the copy of variable
  217. * - '__u' allocated on the stack.
  218. *
  219. * Their two major use cases are: (1) Mediating communication between
  220. * process-level code and irq/NMI handlers, all running on the same CPU,
  221. * and (2) Ensuring that the compiler does not fold, spindle, or otherwise
  222. * mutilate accesses that either do not require ordering or that interact
  223. * with an explicit memory barrier or atomic instruction that provides the
  224. * required ordering.
  225. */
  226. #define __READ_ONCE(x, check) \
  227. ({ \
  228. union { typeof(x) __val; char __c[1]; } __u; \
  229. if (check) \
  230. __read_once_size(&(x), __u.__c, sizeof(x)); \
  231. else \
  232. __read_once_size_nocheck(&(x), __u.__c, sizeof(x)); \
  233. __u.__val; \
  234. })
  235. #define READ_ONCE(x) __READ_ONCE(x, 1)
  236. /*
  237. * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need
  238. * to hide memory access from KASAN.
  239. */
  240. #define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0)
  241. static __no_kasan_or_inline
  242. unsigned long read_word_at_a_time(const void *addr)
  243. {
  244. return *(unsigned long *)addr;
  245. }
  246. #define WRITE_ONCE(x, val) \
  247. ({ \
  248. union { typeof(x) __val; char __c[1]; } __u = \
  249. { .__val = (__force typeof(x)) (val) }; \
  250. __write_once_size(&(x), __u.__c, sizeof(x)); \
  251. __u.__val; \
  252. })
  253. #endif /* __KERNEL__ */
  254. /*
  255. * Force the compiler to emit 'sym' as a symbol, so that we can reference
  256. * it from inline assembler. Necessary in case 'sym' could be inlined
  257. * otherwise, or eliminated entirely due to lack of references that are
  258. * visible to the compiler.
  259. */
  260. #define __ADDRESSABLE(sym) \
  261. static void * __section(.discard.addressable) __used \
  262. __PASTE(__addressable_##sym, __LINE__) = (void *)&sym;
  263. /**
  264. * offset_to_ptr - convert a relative memory offset to an absolute pointer
  265. * @off: the address of the 32-bit offset value
  266. */
  267. static inline void *offset_to_ptr(const int *off)
  268. {
  269. return (void *)((unsigned long)off + *off);
  270. }
  271. #endif /* __ASSEMBLY__ */
  272. /* Compile time object size, -1 for unknown */
  273. #ifndef __compiletime_object_size
  274. # define __compiletime_object_size(obj) -1
  275. #endif
  276. #ifndef __compiletime_warning
  277. # define __compiletime_warning(message)
  278. #endif
  279. #ifndef __compiletime_error
  280. # define __compiletime_error(message)
  281. #endif
  282. #ifdef __OPTIMIZE__
  283. # define __compiletime_assert(condition, msg, prefix, suffix) \
  284. do { \
  285. extern void prefix ## suffix(void) __compiletime_error(msg); \
  286. if (!(condition)) \
  287. prefix ## suffix(); \
  288. } while (0)
  289. #else
  290. # define __compiletime_assert(condition, msg, prefix, suffix) do { } while (0)
  291. #endif
  292. #define _compiletime_assert(condition, msg, prefix, suffix) \
  293. __compiletime_assert(condition, msg, prefix, suffix)
  294. /**
  295. * compiletime_assert - break build and emit msg if condition is false
  296. * @condition: a compile-time constant condition to check
  297. * @msg: a message to emit if condition is false
  298. *
  299. * In tradition of POSIX assert, this macro will break the build if the
  300. * supplied condition is *false*, emitting the supplied error message if the
  301. * compiler has support to do so.
  302. */
  303. #define compiletime_assert(condition, msg) \
  304. _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
  305. #define compiletime_assert_atomic_type(t) \
  306. compiletime_assert(__native_word(t), \
  307. "Need native word sized stores/loads for atomicity.")
  308. /* &a[0] degrades to a pointer: a different type from an array */
  309. #define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
  310. #endif /* __LINUX_COMPILER_H */