lockref.c 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/export.h>
  3. #include <linux/lockref.h>
  4. #if USE_CMPXCHG_LOCKREF
  5. /*
  6. * Note that the "cmpxchg()" reloads the "old" value for the
  7. * failure case.
  8. */
  9. #define CMPXCHG_LOOP(CODE, SUCCESS) do { \
  10. int retry = 100; \
  11. struct lockref old; \
  12. BUILD_BUG_ON(sizeof(old) != 8); \
  13. old.lock_count = READ_ONCE(lockref->lock_count); \
  14. while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
  15. struct lockref new = old, prev = old; \
  16. CODE \
  17. old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
  18. old.lock_count, \
  19. new.lock_count); \
  20. if (likely(old.lock_count == prev.lock_count)) { \
  21. SUCCESS; \
  22. } \
  23. if (!--retry) \
  24. break; \
  25. cpu_relax(); \
  26. } \
  27. } while (0)
  28. #else
  29. #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
  30. #endif
  31. /**
  32. * lockref_get - Increments reference count unconditionally
  33. * @lockref: pointer to lockref structure
  34. *
  35. * This operation is only valid if you already hold a reference
  36. * to the object, so you know the count cannot be zero.
  37. */
  38. void lockref_get(struct lockref *lockref)
  39. {
  40. CMPXCHG_LOOP(
  41. new.count++;
  42. ,
  43. return;
  44. );
  45. spin_lock(&lockref->lock);
  46. lockref->count++;
  47. spin_unlock(&lockref->lock);
  48. }
  49. EXPORT_SYMBOL(lockref_get);
  50. /**
  51. * lockref_get_not_zero - Increments count unless the count is 0 or dead
  52. * @lockref: pointer to lockref structure
  53. * Return: 1 if count updated successfully or 0 if count was zero
  54. */
  55. int lockref_get_not_zero(struct lockref *lockref)
  56. {
  57. int retval;
  58. CMPXCHG_LOOP(
  59. new.count++;
  60. if (old.count <= 0)
  61. return 0;
  62. ,
  63. return 1;
  64. );
  65. spin_lock(&lockref->lock);
  66. retval = 0;
  67. if (lockref->count > 0) {
  68. lockref->count++;
  69. retval = 1;
  70. }
  71. spin_unlock(&lockref->lock);
  72. return retval;
  73. }
  74. EXPORT_SYMBOL(lockref_get_not_zero);
  75. /**
  76. * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
  77. * @lockref: pointer to lockref structure
  78. * Return: 1 if count updated successfully or 0 if count would become zero
  79. */
  80. int lockref_put_not_zero(struct lockref *lockref)
  81. {
  82. int retval;
  83. CMPXCHG_LOOP(
  84. new.count--;
  85. if (old.count <= 1)
  86. return 0;
  87. ,
  88. return 1;
  89. );
  90. spin_lock(&lockref->lock);
  91. retval = 0;
  92. if (lockref->count > 1) {
  93. lockref->count--;
  94. retval = 1;
  95. }
  96. spin_unlock(&lockref->lock);
  97. return retval;
  98. }
  99. EXPORT_SYMBOL(lockref_put_not_zero);
  100. /**
  101. * lockref_get_or_lock - Increments count unless the count is 0 or dead
  102. * @lockref: pointer to lockref structure
  103. * Return: 1 if count updated successfully or 0 if count was zero
  104. * and we got the lock instead.
  105. */
  106. int lockref_get_or_lock(struct lockref *lockref)
  107. {
  108. CMPXCHG_LOOP(
  109. new.count++;
  110. if (old.count <= 0)
  111. break;
  112. ,
  113. return 1;
  114. );
  115. spin_lock(&lockref->lock);
  116. if (lockref->count <= 0)
  117. return 0;
  118. lockref->count++;
  119. spin_unlock(&lockref->lock);
  120. return 1;
  121. }
  122. EXPORT_SYMBOL(lockref_get_or_lock);
  123. /**
  124. * lockref_put_return - Decrement reference count if possible
  125. * @lockref: pointer to lockref structure
  126. *
  127. * Decrement the reference count and return the new value.
  128. * If the lockref was dead or locked, return an error.
  129. */
  130. int lockref_put_return(struct lockref *lockref)
  131. {
  132. CMPXCHG_LOOP(
  133. new.count--;
  134. if (old.count <= 0)
  135. return -1;
  136. ,
  137. return new.count;
  138. );
  139. return -1;
  140. }
  141. EXPORT_SYMBOL(lockref_put_return);
  142. /**
  143. * lockref_put_or_lock - decrements count unless count <= 1 before decrement
  144. * @lockref: pointer to lockref structure
  145. * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
  146. */
  147. int lockref_put_or_lock(struct lockref *lockref)
  148. {
  149. CMPXCHG_LOOP(
  150. new.count--;
  151. if (old.count <= 1)
  152. break;
  153. ,
  154. return 1;
  155. );
  156. spin_lock(&lockref->lock);
  157. if (lockref->count <= 1)
  158. return 0;
  159. lockref->count--;
  160. spin_unlock(&lockref->lock);
  161. return 1;
  162. }
  163. EXPORT_SYMBOL(lockref_put_or_lock);
  164. /**
  165. * lockref_mark_dead - mark lockref dead
  166. * @lockref: pointer to lockref structure
  167. */
  168. void lockref_mark_dead(struct lockref *lockref)
  169. {
  170. assert_spin_locked(&lockref->lock);
  171. lockref->count = -128;
  172. }
  173. EXPORT_SYMBOL(lockref_mark_dead);
  174. /**
  175. * lockref_get_not_dead - Increments count unless the ref is dead
  176. * @lockref: pointer to lockref structure
  177. * Return: 1 if count updated successfully or 0 if lockref was dead
  178. */
  179. int lockref_get_not_dead(struct lockref *lockref)
  180. {
  181. int retval;
  182. CMPXCHG_LOOP(
  183. new.count++;
  184. if (old.count < 0)
  185. return 0;
  186. ,
  187. return 1;
  188. );
  189. spin_lock(&lockref->lock);
  190. retval = 0;
  191. if (lockref->count >= 0) {
  192. lockref->count++;
  193. retval = 1;
  194. }
  195. spin_unlock(&lockref->lock);
  196. return retval;
  197. }
  198. EXPORT_SYMBOL(lockref_get_not_dead);