rwonce.h 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Prevent the compiler from merging or refetching reads or writes. The
  4. * compiler is also forbidden from reordering successive instances of
  5. * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
  6. * particular ordering. One way to make the compiler aware of ordering is to
  7. * put the two invocations of READ_ONCE or WRITE_ONCE in different C
  8. * statements.
  9. *
  10. * These two macros will also work on aggregate data types like structs or
  11. * unions.
  12. *
  13. * Their two major use cases are: (1) Mediating communication between
  14. * process-level code and irq/NMI handlers, all running on the same CPU,
  15. * and (2) Ensuring that the compiler does not fold, spindle, or otherwise
  16. * mutilate accesses that either do not require ordering or that interact
  17. * with an explicit memory barrier or atomic instruction that provides the
  18. * required ordering.
  19. */
  20. #ifndef __ASM_GENERIC_RWONCE_H
  21. #define __ASM_GENERIC_RWONCE_H
  22. #ifndef __ASSEMBLY__
  23. #include <linux/compiler_types.h>
  24. #include <linux/kasan-checks.h>
  25. #include <linux/kcsan-checks.h>
  26. /*
  27. * Yes, this permits 64-bit accesses on 32-bit architectures. These will
  28. * actually be atomic in some cases (namely Armv7 + LPAE), but for others we
  29. * rely on the access being split into 2x32-bit accesses for a 32-bit quantity
  30. * (e.g. a virtual address) and a strong prevailing wind.
  31. */
  32. #define compiletime_assert_rwonce_type(t) \
  33. compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
  34. "Unsupported access size for {READ,WRITE}_ONCE().")
  35. /*
  36. * Use __READ_ONCE() instead of READ_ONCE() if you do not require any
  37. * atomicity. Note that this may result in tears!
  38. */
  39. #ifndef __READ_ONCE
  40. #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
  41. #endif
  42. #define READ_ONCE(x) \
  43. ({ \
  44. compiletime_assert_rwonce_type(x); \
  45. __READ_ONCE(x); \
  46. })
  47. #define __WRITE_ONCE(x, val) \
  48. do { \
  49. *(volatile typeof(x) *)&(x) = (val); \
  50. } while (0)
  51. #define WRITE_ONCE(x, val) \
  52. do { \
  53. compiletime_assert_rwonce_type(x); \
  54. __WRITE_ONCE(x, val); \
  55. } while (0)
  56. static __no_sanitize_or_inline
  57. unsigned long __read_once_word_nocheck(const void *addr)
  58. {
  59. return __READ_ONCE(*(unsigned long *)addr);
  60. }
  61. /*
  62. * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a
  63. * word from memory atomically but without telling KASAN/KCSAN. This is
  64. * usually used by unwinding code when walking the stack of a running process.
  65. */
  66. #define READ_ONCE_NOCHECK(x) \
  67. ({ \
  68. compiletime_assert(sizeof(x) == sizeof(unsigned long), \
  69. "Unsupported access size for READ_ONCE_NOCHECK()."); \
  70. (typeof(x))__read_once_word_nocheck(&(x)); \
  71. })
  72. static __no_kasan_or_inline
  73. unsigned long read_word_at_a_time(const void *addr)
  74. {
  75. kasan_check_read(addr, 1);
  76. return *(unsigned long *)addr;
  77. }
  78. #endif /* __ASSEMBLY__ */
  79. #endif /* __ASM_GENERIC_RWONCE_H */