cmpxchg.h 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (C) 2014 Regents of the University of California
  4. */
  5. #ifndef _ASM_RISCV_CMPXCHG_H
  6. #define _ASM_RISCV_CMPXCHG_H
  7. #include <linux/bug.h>
  8. #include <asm/barrier.h>
  9. #include <asm/fence.h>
  10. #define __xchg_relaxed(ptr, new, size) \
  11. ({ \
  12. __typeof__(ptr) __ptr = (ptr); \
  13. __typeof__(new) __new = (new); \
  14. __typeof__(*(ptr)) __ret; \
  15. switch (size) { \
  16. case 4: \
  17. __asm__ __volatile__ ( \
  18. " amoswap.w %0, %2, %1\n" \
  19. : "=r" (__ret), "+A" (*__ptr) \
  20. : "r" (__new) \
  21. : "memory"); \
  22. break; \
  23. case 8: \
  24. __asm__ __volatile__ ( \
  25. " amoswap.d %0, %2, %1\n" \
  26. : "=r" (__ret), "+A" (*__ptr) \
  27. : "r" (__new) \
  28. : "memory"); \
  29. break; \
  30. default: \
  31. BUILD_BUG(); \
  32. } \
  33. __ret; \
  34. })
  35. #define xchg_relaxed(ptr, x) \
  36. ({ \
  37. __typeof__(*(ptr)) _x_ = (x); \
  38. (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
  39. _x_, sizeof(*(ptr))); \
  40. })
  41. #define __xchg_acquire(ptr, new, size) \
  42. ({ \
  43. __typeof__(ptr) __ptr = (ptr); \
  44. __typeof__(new) __new = (new); \
  45. __typeof__(*(ptr)) __ret; \
  46. switch (size) { \
  47. case 4: \
  48. __asm__ __volatile__ ( \
  49. " amoswap.w %0, %2, %1\n" \
  50. RISCV_ACQUIRE_BARRIER \
  51. : "=r" (__ret), "+A" (*__ptr) \
  52. : "r" (__new) \
  53. : "memory"); \
  54. break; \
  55. case 8: \
  56. __asm__ __volatile__ ( \
  57. " amoswap.d %0, %2, %1\n" \
  58. RISCV_ACQUIRE_BARRIER \
  59. : "=r" (__ret), "+A" (*__ptr) \
  60. : "r" (__new) \
  61. : "memory"); \
  62. break; \
  63. default: \
  64. BUILD_BUG(); \
  65. } \
  66. __ret; \
  67. })
  68. #define xchg_acquire(ptr, x) \
  69. ({ \
  70. __typeof__(*(ptr)) _x_ = (x); \
  71. (__typeof__(*(ptr))) __xchg_acquire((ptr), \
  72. _x_, sizeof(*(ptr))); \
  73. })
  74. #define __xchg_release(ptr, new, size) \
  75. ({ \
  76. __typeof__(ptr) __ptr = (ptr); \
  77. __typeof__(new) __new = (new); \
  78. __typeof__(*(ptr)) __ret; \
  79. switch (size) { \
  80. case 4: \
  81. __asm__ __volatile__ ( \
  82. RISCV_RELEASE_BARRIER \
  83. " amoswap.w %0, %2, %1\n" \
  84. : "=r" (__ret), "+A" (*__ptr) \
  85. : "r" (__new) \
  86. : "memory"); \
  87. break; \
  88. case 8: \
  89. __asm__ __volatile__ ( \
  90. RISCV_RELEASE_BARRIER \
  91. " amoswap.d %0, %2, %1\n" \
  92. : "=r" (__ret), "+A" (*__ptr) \
  93. : "r" (__new) \
  94. : "memory"); \
  95. break; \
  96. default: \
  97. BUILD_BUG(); \
  98. } \
  99. __ret; \
  100. })
  101. #define xchg_release(ptr, x) \
  102. ({ \
  103. __typeof__(*(ptr)) _x_ = (x); \
  104. (__typeof__(*(ptr))) __xchg_release((ptr), \
  105. _x_, sizeof(*(ptr))); \
  106. })
  107. #define __xchg(ptr, new, size) \
  108. ({ \
  109. __typeof__(ptr) __ptr = (ptr); \
  110. __typeof__(new) __new = (new); \
  111. __typeof__(*(ptr)) __ret; \
  112. switch (size) { \
  113. case 4: \
  114. __asm__ __volatile__ ( \
  115. " amoswap.w.aqrl %0, %2, %1\n" \
  116. : "=r" (__ret), "+A" (*__ptr) \
  117. : "r" (__new) \
  118. : "memory"); \
  119. break; \
  120. case 8: \
  121. __asm__ __volatile__ ( \
  122. " amoswap.d.aqrl %0, %2, %1\n" \
  123. : "=r" (__ret), "+A" (*__ptr) \
  124. : "r" (__new) \
  125. : "memory"); \
  126. break; \
  127. default: \
  128. BUILD_BUG(); \
  129. } \
  130. __ret; \
  131. })
  132. #define xchg(ptr, x) \
  133. ({ \
  134. __typeof__(*(ptr)) _x_ = (x); \
  135. (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
  136. })
  137. #define xchg32(ptr, x) \
  138. ({ \
  139. BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
  140. xchg((ptr), (x)); \
  141. })
  142. #define xchg64(ptr, x) \
  143. ({ \
  144. BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
  145. xchg((ptr), (x)); \
  146. })
  147. /*
  148. * Atomic compare and exchange. Compare OLD with MEM, if identical,
  149. * store NEW in MEM. Return the initial value in MEM. Success is
  150. * indicated by comparing RETURN with OLD.
  151. */
  152. #define __cmpxchg_relaxed(ptr, old, new, size) \
  153. ({ \
  154. __typeof__(ptr) __ptr = (ptr); \
  155. __typeof__(*(ptr)) __old = (old); \
  156. __typeof__(*(ptr)) __new = (new); \
  157. __typeof__(*(ptr)) __ret; \
  158. register unsigned int __rc; \
  159. switch (size) { \
  160. case 4: \
  161. __asm__ __volatile__ ( \
  162. "0: lr.w %0, %2\n" \
  163. " bne %0, %z3, 1f\n" \
  164. " sc.w %1, %z4, %2\n" \
  165. " bnez %1, 0b\n" \
  166. "1:\n" \
  167. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  168. : "rJ" ((long)__old), "rJ" (__new) \
  169. : "memory"); \
  170. break; \
  171. case 8: \
  172. __asm__ __volatile__ ( \
  173. "0: lr.d %0, %2\n" \
  174. " bne %0, %z3, 1f\n" \
  175. " sc.d %1, %z4, %2\n" \
  176. " bnez %1, 0b\n" \
  177. "1:\n" \
  178. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  179. : "rJ" (__old), "rJ" (__new) \
  180. : "memory"); \
  181. break; \
  182. default: \
  183. BUILD_BUG(); \
  184. } \
  185. __ret; \
  186. })
  187. #define cmpxchg_relaxed(ptr, o, n) \
  188. ({ \
  189. __typeof__(*(ptr)) _o_ = (o); \
  190. __typeof__(*(ptr)) _n_ = (n); \
  191. (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
  192. _o_, _n_, sizeof(*(ptr))); \
  193. })
  194. #define __cmpxchg_acquire(ptr, old, new, size) \
  195. ({ \
  196. __typeof__(ptr) __ptr = (ptr); \
  197. __typeof__(*(ptr)) __old = (old); \
  198. __typeof__(*(ptr)) __new = (new); \
  199. __typeof__(*(ptr)) __ret; \
  200. register unsigned int __rc; \
  201. switch (size) { \
  202. case 4: \
  203. __asm__ __volatile__ ( \
  204. "0: lr.w %0, %2\n" \
  205. " bne %0, %z3, 1f\n" \
  206. " sc.w %1, %z4, %2\n" \
  207. " bnez %1, 0b\n" \
  208. RISCV_ACQUIRE_BARRIER \
  209. "1:\n" \
  210. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  211. : "rJ" ((long)__old), "rJ" (__new) \
  212. : "memory"); \
  213. break; \
  214. case 8: \
  215. __asm__ __volatile__ ( \
  216. "0: lr.d %0, %2\n" \
  217. " bne %0, %z3, 1f\n" \
  218. " sc.d %1, %z4, %2\n" \
  219. " bnez %1, 0b\n" \
  220. RISCV_ACQUIRE_BARRIER \
  221. "1:\n" \
  222. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  223. : "rJ" (__old), "rJ" (__new) \
  224. : "memory"); \
  225. break; \
  226. default: \
  227. BUILD_BUG(); \
  228. } \
  229. __ret; \
  230. })
  231. #define cmpxchg_acquire(ptr, o, n) \
  232. ({ \
  233. __typeof__(*(ptr)) _o_ = (o); \
  234. __typeof__(*(ptr)) _n_ = (n); \
  235. (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
  236. _o_, _n_, sizeof(*(ptr))); \
  237. })
  238. #define __cmpxchg_release(ptr, old, new, size) \
  239. ({ \
  240. __typeof__(ptr) __ptr = (ptr); \
  241. __typeof__(*(ptr)) __old = (old); \
  242. __typeof__(*(ptr)) __new = (new); \
  243. __typeof__(*(ptr)) __ret; \
  244. register unsigned int __rc; \
  245. switch (size) { \
  246. case 4: \
  247. __asm__ __volatile__ ( \
  248. RISCV_RELEASE_BARRIER \
  249. "0: lr.w %0, %2\n" \
  250. " bne %0, %z3, 1f\n" \
  251. " sc.w %1, %z4, %2\n" \
  252. " bnez %1, 0b\n" \
  253. "1:\n" \
  254. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  255. : "rJ" ((long)__old), "rJ" (__new) \
  256. : "memory"); \
  257. break; \
  258. case 8: \
  259. __asm__ __volatile__ ( \
  260. RISCV_RELEASE_BARRIER \
  261. "0: lr.d %0, %2\n" \
  262. " bne %0, %z3, 1f\n" \
  263. " sc.d %1, %z4, %2\n" \
  264. " bnez %1, 0b\n" \
  265. "1:\n" \
  266. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  267. : "rJ" (__old), "rJ" (__new) \
  268. : "memory"); \
  269. break; \
  270. default: \
  271. BUILD_BUG(); \
  272. } \
  273. __ret; \
  274. })
  275. #define cmpxchg_release(ptr, o, n) \
  276. ({ \
  277. __typeof__(*(ptr)) _o_ = (o); \
  278. __typeof__(*(ptr)) _n_ = (n); \
  279. (__typeof__(*(ptr))) __cmpxchg_release((ptr), \
  280. _o_, _n_, sizeof(*(ptr))); \
  281. })
  282. #define __cmpxchg(ptr, old, new, size) \
  283. ({ \
  284. __typeof__(ptr) __ptr = (ptr); \
  285. __typeof__(*(ptr)) __old = (old); \
  286. __typeof__(*(ptr)) __new = (new); \
  287. __typeof__(*(ptr)) __ret; \
  288. register unsigned int __rc; \
  289. switch (size) { \
  290. case 4: \
  291. __asm__ __volatile__ ( \
  292. "0: lr.w %0, %2\n" \
  293. " bne %0, %z3, 1f\n" \
  294. " sc.w.rl %1, %z4, %2\n" \
  295. " bnez %1, 0b\n" \
  296. " fence rw, rw\n" \
  297. "1:\n" \
  298. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  299. : "rJ" ((long)__old), "rJ" (__new) \
  300. : "memory"); \
  301. break; \
  302. case 8: \
  303. __asm__ __volatile__ ( \
  304. "0: lr.d %0, %2\n" \
  305. " bne %0, %z3, 1f\n" \
  306. " sc.d.rl %1, %z4, %2\n" \
  307. " bnez %1, 0b\n" \
  308. " fence rw, rw\n" \
  309. "1:\n" \
  310. : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
  311. : "rJ" (__old), "rJ" (__new) \
  312. : "memory"); \
  313. break; \
  314. default: \
  315. BUILD_BUG(); \
  316. } \
  317. __ret; \
  318. })
  319. #define cmpxchg(ptr, o, n) \
  320. ({ \
  321. __typeof__(*(ptr)) _o_ = (o); \
  322. __typeof__(*(ptr)) _n_ = (n); \
  323. (__typeof__(*(ptr))) __cmpxchg((ptr), \
  324. _o_, _n_, sizeof(*(ptr))); \
  325. })
  326. #define cmpxchg_local(ptr, o, n) \
  327. (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
  328. #define cmpxchg32(ptr, o, n) \
  329. ({ \
  330. BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
  331. cmpxchg((ptr), (o), (n)); \
  332. })
  333. #define cmpxchg32_local(ptr, o, n) \
  334. ({ \
  335. BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
  336. cmpxchg_relaxed((ptr), (o), (n)) \
  337. })
  338. #define cmpxchg64(ptr, o, n) \
  339. ({ \
  340. BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
  341. cmpxchg((ptr), (o), (n)); \
  342. })
  343. #define cmpxchg64_local(ptr, o, n) \
  344. ({ \
  345. BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
  346. cmpxchg_relaxed((ptr), (o), (n)); \
  347. })
  348. #endif /* _ASM_RISCV_CMPXCHG_H */