percpu-refcount.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Percpu refcounts:
  4. * (C) 2012 Google, Inc.
  5. * Author: Kent Overstreet <koverstreet@google.com>
  6. *
  7. * This implements a refcount with similar semantics to atomic_t - atomic_inc(),
  8. * atomic_dec_and_test() - but percpu.
  9. *
  10. * There's one important difference between percpu refs and normal atomic_t
  11. * refcounts; you have to keep track of your initial refcount, and then when you
  12. * start shutting down you call percpu_ref_kill() _before_ dropping the initial
  13. * refcount.
  14. *
  15. * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
  16. * than an atomic_t - this is because of the way shutdown works, see
  17. * percpu_ref_kill()/PERCPU_COUNT_BIAS.
  18. *
  19. * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
  20. * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
  21. * puts the ref back in single atomic_t mode, collecting the per cpu refs and
  22. * issuing the appropriate barriers, and then marks the ref as shutting down so
  23. * that percpu_ref_put() will check for the ref hitting 0. After it returns,
  24. * it's safe to drop the initial ref.
  25. *
  26. * USAGE:
  27. *
  28. * See fs/aio.c for some example usage; it's used there for struct kioctx, which
  29. * is created when userspaces calls io_setup(), and destroyed when userspace
  30. * calls io_destroy() or the process exits.
  31. *
  32. * In the aio code, kill_ioctx() is called when we wish to destroy a kioctx; it
  33. * removes the kioctx from the proccess's table of kioctxs and kills percpu_ref.
  34. * After that, there can't be any new users of the kioctx (from lookup_ioctx())
  35. * and it's then safe to drop the initial ref with percpu_ref_put().
  36. *
  37. * Note that the free path, free_ioctx(), needs to go through explicit call_rcu()
  38. * to synchronize with RCU protected lookup_ioctx(). percpu_ref operations don't
  39. * imply RCU grace periods of any kind and if a user wants to combine percpu_ref
  40. * with RCU protection, it must be done explicitly.
  41. *
  42. * Code that does a two stage shutdown like this often needs some kind of
  43. * explicit synchronization to ensure the initial refcount can only be dropped
  44. * once - percpu_ref_kill() does this for you, it returns true once and false if
  45. * someone else already called it. The aio code uses it this way, but it's not
  46. * necessary if the code has some other mechanism to synchronize teardown.
  47. * around.
  48. */
  49. #ifndef _LINUX_PERCPU_REFCOUNT_H
  50. #define _LINUX_PERCPU_REFCOUNT_H
  51. #include <linux/atomic.h>
  52. #include <linux/kernel.h>
  53. #include <linux/percpu.h>
  54. #include <linux/rcupdate.h>
  55. #include <linux/gfp.h>
  56. struct percpu_ref;
  57. typedef void (percpu_ref_func_t)(struct percpu_ref *);
  58. /* flags set in the lower bits of percpu_ref->percpu_count_ptr */
  59. enum {
  60. __PERCPU_REF_ATOMIC = 1LU << 0, /* operating in atomic mode */
  61. __PERCPU_REF_DEAD = 1LU << 1, /* (being) killed */
  62. __PERCPU_REF_ATOMIC_DEAD = __PERCPU_REF_ATOMIC | __PERCPU_REF_DEAD,
  63. __PERCPU_REF_FLAG_BITS = 2,
  64. };
  65. /* @flags for percpu_ref_init() */
  66. enum {
  67. /*
  68. * Start w/ ref == 1 in atomic mode. Can be switched to percpu
  69. * operation using percpu_ref_switch_to_percpu(). If initialized
  70. * with this flag, the ref will stay in atomic mode until
  71. * percpu_ref_switch_to_percpu() is invoked on it.
  72. * Implies ALLOW_REINIT.
  73. */
  74. PERCPU_REF_INIT_ATOMIC = 1 << 0,
  75. /*
  76. * Start dead w/ ref == 0 in atomic mode. Must be revived with
  77. * percpu_ref_reinit() before used. Implies INIT_ATOMIC and
  78. * ALLOW_REINIT.
  79. */
  80. PERCPU_REF_INIT_DEAD = 1 << 1,
  81. /*
  82. * Allow switching from atomic mode to percpu mode.
  83. */
  84. PERCPU_REF_ALLOW_REINIT = 1 << 2,
  85. };
  86. struct percpu_ref_data {
  87. atomic_long_t count;
  88. percpu_ref_func_t *release;
  89. percpu_ref_func_t *confirm_switch;
  90. bool force_atomic:1;
  91. bool allow_reinit:1;
  92. struct rcu_head rcu;
  93. struct percpu_ref *ref;
  94. };
  95. struct percpu_ref {
  96. /*
  97. * The low bit of the pointer indicates whether the ref is in percpu
  98. * mode; if set, then get/put will manipulate the atomic_t.
  99. */
  100. unsigned long percpu_count_ptr;
  101. /*
  102. * 'percpu_ref' is often embedded into user structure, and only
  103. * 'percpu_count_ptr' is required in fast path, move other fields
  104. * into 'percpu_ref_data', so we can reduce memory footprint in
  105. * fast path.
  106. */
  107. struct percpu_ref_data *data;
  108. };
  109. int __must_check percpu_ref_init(struct percpu_ref *ref,
  110. percpu_ref_func_t *release, unsigned int flags,
  111. gfp_t gfp);
  112. void percpu_ref_exit(struct percpu_ref *ref);
  113. void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
  114. percpu_ref_func_t *confirm_switch);
  115. void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref);
  116. void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
  117. void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
  118. percpu_ref_func_t *confirm_kill);
  119. void percpu_ref_resurrect(struct percpu_ref *ref);
  120. void percpu_ref_reinit(struct percpu_ref *ref);
  121. bool percpu_ref_is_zero(struct percpu_ref *ref);
  122. /**
  123. * percpu_ref_kill - drop the initial ref
  124. * @ref: percpu_ref to kill
  125. *
  126. * Must be used to drop the initial ref on a percpu refcount; must be called
  127. * precisely once before shutdown.
  128. *
  129. * Switches @ref into atomic mode before gathering up the percpu counters
  130. * and dropping the initial ref.
  131. *
  132. * There are no implied RCU grace periods between kill and release.
  133. */
  134. static inline void percpu_ref_kill(struct percpu_ref *ref)
  135. {
  136. percpu_ref_kill_and_confirm(ref, NULL);
  137. }
  138. /*
  139. * Internal helper. Don't use outside percpu-refcount proper. The
  140. * function doesn't return the pointer and let the caller test it for NULL
  141. * because doing so forces the compiler to generate two conditional
  142. * branches as it can't assume that @ref->percpu_count is not NULL.
  143. */
  144. static inline bool __ref_is_percpu(struct percpu_ref *ref,
  145. unsigned long __percpu **percpu_countp)
  146. {
  147. unsigned long percpu_ptr;
  148. /*
  149. * The value of @ref->percpu_count_ptr is tested for
  150. * !__PERCPU_REF_ATOMIC, which may be set asynchronously, and then
  151. * used as a pointer. If the compiler generates a separate fetch
  152. * when using it as a pointer, __PERCPU_REF_ATOMIC may be set in
  153. * between contaminating the pointer value, meaning that
  154. * READ_ONCE() is required when fetching it.
  155. *
  156. * The dependency ordering from the READ_ONCE() pairs
  157. * with smp_store_release() in __percpu_ref_switch_to_percpu().
  158. */
  159. percpu_ptr = READ_ONCE(ref->percpu_count_ptr);
  160. /*
  161. * Theoretically, the following could test just ATOMIC; however,
  162. * then we'd have to mask off DEAD separately as DEAD may be
  163. * visible without ATOMIC if we race with percpu_ref_kill(). DEAD
  164. * implies ATOMIC anyway. Test them together.
  165. */
  166. if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD))
  167. return false;
  168. *percpu_countp = (unsigned long __percpu *)percpu_ptr;
  169. return true;
  170. }
  171. /**
  172. * percpu_ref_get_many - increment a percpu refcount
  173. * @ref: percpu_ref to get
  174. * @nr: number of references to get
  175. *
  176. * Analogous to atomic_long_add().
  177. *
  178. * This function is safe to call as long as @ref is between init and exit.
  179. */
  180. static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
  181. {
  182. unsigned long __percpu *percpu_count;
  183. rcu_read_lock();
  184. if (__ref_is_percpu(ref, &percpu_count))
  185. this_cpu_add(*percpu_count, nr);
  186. else
  187. atomic_long_add(nr, &ref->data->count);
  188. rcu_read_unlock();
  189. }
  190. /**
  191. * percpu_ref_get - increment a percpu refcount
  192. * @ref: percpu_ref to get
  193. *
  194. * Analagous to atomic_long_inc().
  195. *
  196. * This function is safe to call as long as @ref is between init and exit.
  197. */
  198. static inline void percpu_ref_get(struct percpu_ref *ref)
  199. {
  200. percpu_ref_get_many(ref, 1);
  201. }
  202. /**
  203. * percpu_ref_tryget_many - try to increment a percpu refcount
  204. * @ref: percpu_ref to try-get
  205. * @nr: number of references to get
  206. *
  207. * Increment a percpu refcount by @nr unless its count already reached zero.
  208. * Returns %true on success; %false on failure.
  209. *
  210. * This function is safe to call as long as @ref is between init and exit.
  211. */
  212. static inline bool percpu_ref_tryget_many(struct percpu_ref *ref,
  213. unsigned long nr)
  214. {
  215. unsigned long __percpu *percpu_count;
  216. bool ret;
  217. rcu_read_lock();
  218. if (__ref_is_percpu(ref, &percpu_count)) {
  219. this_cpu_add(*percpu_count, nr);
  220. ret = true;
  221. } else {
  222. ret = atomic_long_add_unless(&ref->data->count, nr, 0);
  223. }
  224. rcu_read_unlock();
  225. return ret;
  226. }
  227. /**
  228. * percpu_ref_tryget - try to increment a percpu refcount
  229. * @ref: percpu_ref to try-get
  230. *
  231. * Increment a percpu refcount unless its count already reached zero.
  232. * Returns %true on success; %false on failure.
  233. *
  234. * This function is safe to call as long as @ref is between init and exit.
  235. */
  236. static inline bool percpu_ref_tryget(struct percpu_ref *ref)
  237. {
  238. return percpu_ref_tryget_many(ref, 1);
  239. }
  240. /**
  241. * percpu_ref_tryget_live - try to increment a live percpu refcount
  242. * @ref: percpu_ref to try-get
  243. *
  244. * Increment a percpu refcount unless it has already been killed. Returns
  245. * %true on success; %false on failure.
  246. *
  247. * Completion of percpu_ref_kill() in itself doesn't guarantee that this
  248. * function will fail. For such guarantee, percpu_ref_kill_and_confirm()
  249. * should be used. After the confirm_kill callback is invoked, it's
  250. * guaranteed that no new reference will be given out by
  251. * percpu_ref_tryget_live().
  252. *
  253. * This function is safe to call as long as @ref is between init and exit.
  254. */
  255. static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
  256. {
  257. unsigned long __percpu *percpu_count;
  258. bool ret = false;
  259. rcu_read_lock();
  260. if (__ref_is_percpu(ref, &percpu_count)) {
  261. this_cpu_inc(*percpu_count);
  262. ret = true;
  263. } else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) {
  264. ret = atomic_long_inc_not_zero(&ref->data->count);
  265. }
  266. rcu_read_unlock();
  267. return ret;
  268. }
  269. /**
  270. * percpu_ref_put_many - decrement a percpu refcount
  271. * @ref: percpu_ref to put
  272. * @nr: number of references to put
  273. *
  274. * Decrement the refcount, and if 0, call the release function (which was passed
  275. * to percpu_ref_init())
  276. *
  277. * This function is safe to call as long as @ref is between init and exit.
  278. */
  279. static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
  280. {
  281. unsigned long __percpu *percpu_count;
  282. rcu_read_lock();
  283. if (__ref_is_percpu(ref, &percpu_count))
  284. this_cpu_sub(*percpu_count, nr);
  285. else if (unlikely(atomic_long_sub_and_test(nr, &ref->data->count)))
  286. ref->data->release(ref);
  287. rcu_read_unlock();
  288. }
  289. /**
  290. * percpu_ref_put - decrement a percpu refcount
  291. * @ref: percpu_ref to put
  292. *
  293. * Decrement the refcount, and if 0, call the release function (which was passed
  294. * to percpu_ref_init())
  295. *
  296. * This function is safe to call as long as @ref is between init and exit.
  297. */
  298. static inline void percpu_ref_put(struct percpu_ref *ref)
  299. {
  300. percpu_ref_put_many(ref, 1);
  301. }
  302. /**
  303. * percpu_ref_is_dying - test whether a percpu refcount is dying or dead
  304. * @ref: percpu_ref to test
  305. *
  306. * Returns %true if @ref is dying or dead.
  307. *
  308. * This function is safe to call as long as @ref is between init and exit
  309. * and the caller is responsible for synchronizing against state changes.
  310. */
  311. static inline bool percpu_ref_is_dying(struct percpu_ref *ref)
  312. {
  313. return ref->percpu_count_ptr & __PERCPU_REF_DEAD;
  314. }
  315. #endif