rcu_segcblist.h 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. /* SPDX-License-Identifier: GPL-2.0+ */
  2. /*
  3. * RCU segmented callback lists, internal-to-rcu header file
  4. *
  5. * Copyright IBM Corporation, 2017
  6. *
  7. * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8. */
  9. #include <linux/rcu_segcblist.h>
  10. /* Return number of callbacks in the specified callback list. */
  11. static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
  12. {
  13. return READ_ONCE(rclp->len);
  14. }
  15. void rcu_cblist_init(struct rcu_cblist *rclp);
  16. void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
  17. void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
  18. struct rcu_cblist *srclp,
  19. struct rcu_head *rhp);
  20. struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
  21. /*
  22. * Is the specified rcu_segcblist structure empty?
  23. *
  24. * But careful! The fact that the ->head field is NULL does not
  25. * necessarily imply that there are no callbacks associated with
  26. * this structure. When callbacks are being invoked, they are
  27. * removed as a group. If callback invocation must be preempted,
  28. * the remaining callbacks will be added back to the list. Either
  29. * way, the counts are updated later.
  30. *
  31. * So it is often the case that rcu_segcblist_n_cbs() should be used
  32. * instead.
  33. */
  34. static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
  35. {
  36. return !READ_ONCE(rsclp->head);
  37. }
  38. /* Return number of callbacks in segmented callback list. */
  39. static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
  40. {
  41. #ifdef CONFIG_RCU_NOCB_CPU
  42. return atomic_long_read(&rsclp->len);
  43. #else
  44. return READ_ONCE(rsclp->len);
  45. #endif
  46. }
  47. /*
  48. * Is the specified rcu_segcblist enabled, for example, not corresponding
  49. * to an offline CPU?
  50. */
  51. static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
  52. {
  53. return rsclp->enabled;
  54. }
  55. /* Is the specified rcu_segcblist offloaded? */
  56. static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
  57. {
  58. return rsclp->offloaded;
  59. }
  60. /*
  61. * Are all segments following the specified segment of the specified
  62. * rcu_segcblist structure empty of callbacks? (The specified
  63. * segment might well contain callbacks.)
  64. */
  65. static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
  66. {
  67. return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
  68. }
  69. void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
  70. void rcu_segcblist_init(struct rcu_segcblist *rsclp);
  71. void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
  72. void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
  73. bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
  74. bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
  75. struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
  76. struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
  77. bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
  78. void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
  79. struct rcu_head *rhp);
  80. bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
  81. struct rcu_head *rhp);
  82. void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
  83. struct rcu_cblist *rclp);
  84. void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
  85. struct rcu_cblist *rclp);
  86. void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
  87. struct rcu_cblist *rclp);
  88. void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
  89. struct rcu_cblist *rclp);
  90. void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
  91. struct rcu_cblist *rclp);
  92. void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
  93. struct rcu_cblist *rclp);
  94. void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
  95. bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
  96. void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
  97. struct rcu_segcblist *src_rsclp);