atomic.h 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
  4. * Copyright (C) 2012 Regents of the University of California
  5. * Copyright (C) 2017 SiFive
  6. */
  7. #ifndef _ASM_RISCV_ATOMIC_H
  8. #define _ASM_RISCV_ATOMIC_H
  9. #ifdef CONFIG_GENERIC_ATOMIC64
  10. # include <asm-generic/atomic64.h>
  11. #else
  12. # if (__riscv_xlen < 64)
  13. # error "64-bit atomics require XLEN to be at least 64"
  14. # endif
  15. #endif
  16. #include <asm/cmpxchg.h>
  17. #include <asm/barrier.h>
  18. #define __atomic_acquire_fence() \
  19. __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
  20. #define __atomic_release_fence() \
  21. __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
  22. static __always_inline int atomic_read(const atomic_t *v)
  23. {
  24. return READ_ONCE(v->counter);
  25. }
  26. static __always_inline void atomic_set(atomic_t *v, int i)
  27. {
  28. WRITE_ONCE(v->counter, i);
  29. }
  30. #ifndef CONFIG_GENERIC_ATOMIC64
  31. #define ATOMIC64_INIT(i) { (i) }
  32. static __always_inline s64 atomic64_read(const atomic64_t *v)
  33. {
  34. return READ_ONCE(v->counter);
  35. }
  36. static __always_inline void atomic64_set(atomic64_t *v, s64 i)
  37. {
  38. WRITE_ONCE(v->counter, i);
  39. }
  40. #endif
  41. /*
  42. * First, the atomic ops that have no ordering constraints and therefor don't
  43. * have the AQ or RL bits set. These don't return anything, so there's only
  44. * one version to worry about.
  45. */
  46. #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
  47. static __always_inline \
  48. void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
  49. { \
  50. __asm__ __volatile__ ( \
  51. " amo" #asm_op "." #asm_type " zero, %1, %0" \
  52. : "+A" (v->counter) \
  53. : "r" (I) \
  54. : "memory"); \
  55. } \
  56. #ifdef CONFIG_GENERIC_ATOMIC64
  57. #define ATOMIC_OPS(op, asm_op, I) \
  58. ATOMIC_OP (op, asm_op, I, w, int, )
  59. #else
  60. #define ATOMIC_OPS(op, asm_op, I) \
  61. ATOMIC_OP (op, asm_op, I, w, int, ) \
  62. ATOMIC_OP (op, asm_op, I, d, s64, 64)
  63. #endif
  64. ATOMIC_OPS(add, add, i)
  65. ATOMIC_OPS(sub, add, -i)
  66. ATOMIC_OPS(and, and, i)
  67. ATOMIC_OPS( or, or, i)
  68. ATOMIC_OPS(xor, xor, i)
  69. #undef ATOMIC_OP
  70. #undef ATOMIC_OPS
  71. /*
  72. * Atomic ops that have ordered, relaxed, acquire, and release variants.
  73. * There's two flavors of these: the arithmatic ops have both fetch and return
  74. * versions, while the logical ops only have fetch versions.
  75. */
  76. #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
  77. static __always_inline \
  78. c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \
  79. atomic##prefix##_t *v) \
  80. { \
  81. register c_type ret; \
  82. __asm__ __volatile__ ( \
  83. " amo" #asm_op "." #asm_type " %1, %2, %0" \
  84. : "+A" (v->counter), "=r" (ret) \
  85. : "r" (I) \
  86. : "memory"); \
  87. return ret; \
  88. } \
  89. static __always_inline \
  90. c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
  91. { \
  92. register c_type ret; \
  93. __asm__ __volatile__ ( \
  94. " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \
  95. : "+A" (v->counter), "=r" (ret) \
  96. : "r" (I) \
  97. : "memory"); \
  98. return ret; \
  99. }
  100. #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
  101. static __always_inline \
  102. c_type atomic##prefix##_##op##_return_relaxed(c_type i, \
  103. atomic##prefix##_t *v) \
  104. { \
  105. return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \
  106. } \
  107. static __always_inline \
  108. c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \
  109. { \
  110. return atomic##prefix##_fetch_##op(i, v) c_op I; \
  111. }
  112. #ifdef CONFIG_GENERIC_ATOMIC64
  113. #define ATOMIC_OPS(op, asm_op, c_op, I) \
  114. ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
  115. ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, )
  116. #else
  117. #define ATOMIC_OPS(op, asm_op, c_op, I) \
  118. ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
  119. ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \
  120. ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \
  121. ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64)
  122. #endif
  123. ATOMIC_OPS(add, add, +, i)
  124. ATOMIC_OPS(sub, add, +, -i)
  125. #define atomic_add_return_relaxed atomic_add_return_relaxed
  126. #define atomic_sub_return_relaxed atomic_sub_return_relaxed
  127. #define atomic_add_return atomic_add_return
  128. #define atomic_sub_return atomic_sub_return
  129. #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
  130. #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
  131. #define atomic_fetch_add atomic_fetch_add
  132. #define atomic_fetch_sub atomic_fetch_sub
  133. #ifndef CONFIG_GENERIC_ATOMIC64
  134. #define atomic64_add_return_relaxed atomic64_add_return_relaxed
  135. #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
  136. #define atomic64_add_return atomic64_add_return
  137. #define atomic64_sub_return atomic64_sub_return
  138. #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
  139. #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
  140. #define atomic64_fetch_add atomic64_fetch_add
  141. #define atomic64_fetch_sub atomic64_fetch_sub
  142. #endif
  143. #undef ATOMIC_OPS
  144. #ifdef CONFIG_GENERIC_ATOMIC64
  145. #define ATOMIC_OPS(op, asm_op, I) \
  146. ATOMIC_FETCH_OP(op, asm_op, I, w, int, )
  147. #else
  148. #define ATOMIC_OPS(op, asm_op, I) \
  149. ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \
  150. ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64)
  151. #endif
  152. ATOMIC_OPS(and, and, i)
  153. ATOMIC_OPS( or, or, i)
  154. ATOMIC_OPS(xor, xor, i)
  155. #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
  156. #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
  157. #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
  158. #define atomic_fetch_and atomic_fetch_and
  159. #define atomic_fetch_or atomic_fetch_or
  160. #define atomic_fetch_xor atomic_fetch_xor
  161. #ifndef CONFIG_GENERIC_ATOMIC64
  162. #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
  163. #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
  164. #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
  165. #define atomic64_fetch_and atomic64_fetch_and
  166. #define atomic64_fetch_or atomic64_fetch_or
  167. #define atomic64_fetch_xor atomic64_fetch_xor
  168. #endif
  169. #undef ATOMIC_OPS
  170. #undef ATOMIC_FETCH_OP
  171. #undef ATOMIC_OP_RETURN
  172. /* This is required to provide a full barrier on success. */
  173. static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
  174. {
  175. int prev, rc;
  176. __asm__ __volatile__ (
  177. "0: lr.w %[p], %[c]\n"
  178. " beq %[p], %[u], 1f\n"
  179. " add %[rc], %[p], %[a]\n"
  180. " sc.w.rl %[rc], %[rc], %[c]\n"
  181. " bnez %[rc], 0b\n"
  182. " fence rw, rw\n"
  183. "1:\n"
  184. : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
  185. : [a]"r" (a), [u]"r" (u)
  186. : "memory");
  187. return prev;
  188. }
  189. #define atomic_fetch_add_unless atomic_fetch_add_unless
  190. #ifndef CONFIG_GENERIC_ATOMIC64
  191. static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
  192. {
  193. s64 prev;
  194. long rc;
  195. __asm__ __volatile__ (
  196. "0: lr.d %[p], %[c]\n"
  197. " beq %[p], %[u], 1f\n"
  198. " add %[rc], %[p], %[a]\n"
  199. " sc.d.rl %[rc], %[rc], %[c]\n"
  200. " bnez %[rc], 0b\n"
  201. " fence rw, rw\n"
  202. "1:\n"
  203. : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
  204. : [a]"r" (a), [u]"r" (u)
  205. : "memory");
  206. return prev;
  207. }
  208. #define atomic64_fetch_add_unless atomic64_fetch_add_unless
  209. #endif
  210. /*
  211. * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
  212. * {cmp,}xchg and the operations that return, so they need a full barrier.
  213. */
  214. #define ATOMIC_OP(c_t, prefix, size) \
  215. static __always_inline \
  216. c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \
  217. { \
  218. return __xchg_relaxed(&(v->counter), n, size); \
  219. } \
  220. static __always_inline \
  221. c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \
  222. { \
  223. return __xchg_acquire(&(v->counter), n, size); \
  224. } \
  225. static __always_inline \
  226. c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \
  227. { \
  228. return __xchg_release(&(v->counter), n, size); \
  229. } \
  230. static __always_inline \
  231. c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \
  232. { \
  233. return __xchg(&(v->counter), n, size); \
  234. } \
  235. static __always_inline \
  236. c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \
  237. c_t o, c_t n) \
  238. { \
  239. return __cmpxchg_relaxed(&(v->counter), o, n, size); \
  240. } \
  241. static __always_inline \
  242. c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \
  243. c_t o, c_t n) \
  244. { \
  245. return __cmpxchg_acquire(&(v->counter), o, n, size); \
  246. } \
  247. static __always_inline \
  248. c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \
  249. c_t o, c_t n) \
  250. { \
  251. return __cmpxchg_release(&(v->counter), o, n, size); \
  252. } \
  253. static __always_inline \
  254. c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \
  255. { \
  256. return __cmpxchg(&(v->counter), o, n, size); \
  257. }
  258. #ifdef CONFIG_GENERIC_ATOMIC64
  259. #define ATOMIC_OPS() \
  260. ATOMIC_OP(int, , 4)
  261. #else
  262. #define ATOMIC_OPS() \
  263. ATOMIC_OP(int, , 4) \
  264. ATOMIC_OP(s64, 64, 8)
  265. #endif
  266. ATOMIC_OPS()
  267. #define atomic_xchg_relaxed atomic_xchg_relaxed
  268. #define atomic_xchg_acquire atomic_xchg_acquire
  269. #define atomic_xchg_release atomic_xchg_release
  270. #define atomic_xchg atomic_xchg
  271. #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed
  272. #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
  273. #define atomic_cmpxchg_release atomic_cmpxchg_release
  274. #define atomic_cmpxchg atomic_cmpxchg
  275. #undef ATOMIC_OPS
  276. #undef ATOMIC_OP
  277. static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
  278. {
  279. int prev, rc;
  280. __asm__ __volatile__ (
  281. "0: lr.w %[p], %[c]\n"
  282. " sub %[rc], %[p], %[o]\n"
  283. " bltz %[rc], 1f\n"
  284. " sc.w.rl %[rc], %[rc], %[c]\n"
  285. " bnez %[rc], 0b\n"
  286. " fence rw, rw\n"
  287. "1:\n"
  288. : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
  289. : [o]"r" (offset)
  290. : "memory");
  291. return prev - offset;
  292. }
  293. #define atomic_dec_if_positive(v) atomic_sub_if_positive(v, 1)
  294. #ifndef CONFIG_GENERIC_ATOMIC64
  295. static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset)
  296. {
  297. s64 prev;
  298. long rc;
  299. __asm__ __volatile__ (
  300. "0: lr.d %[p], %[c]\n"
  301. " sub %[rc], %[p], %[o]\n"
  302. " bltz %[rc], 1f\n"
  303. " sc.d.rl %[rc], %[rc], %[c]\n"
  304. " bnez %[rc], 0b\n"
  305. " fence rw, rw\n"
  306. "1:\n"
  307. : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
  308. : [o]"r" (offset)
  309. : "memory");
  310. return prev - offset;
  311. }
  312. #define atomic64_dec_if_positive(v) atomic64_sub_if_positive(v, 1)
  313. #endif
  314. #endif /* _ASM_RISCV_ATOMIC_H */