percpu_counter.h 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_PERCPU_COUNTER_H
  3. #define _LINUX_PERCPU_COUNTER_H
  4. /*
  5. * A simple "approximate counter" for use in ext2 and ext3 superblocks.
  6. *
  7. * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4.
  8. */
  9. #include <linux/spinlock.h>
  10. #include <linux/smp.h>
  11. #include <linux/list.h>
  12. #include <linux/threads.h>
  13. #include <linux/percpu.h>
  14. #include <linux/types.h>
  15. #include <linux/gfp.h>
  16. #ifdef CONFIG_SMP
  17. struct percpu_counter {
  18. raw_spinlock_t lock;
  19. s64 count;
  20. #ifdef CONFIG_HOTPLUG_CPU
  21. struct list_head list; /* All percpu_counters are on a list */
  22. #endif
  23. s32 __percpu *counters;
  24. };
  25. extern int percpu_counter_batch;
  26. int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,
  27. struct lock_class_key *key);
  28. #define percpu_counter_init(fbc, value, gfp) \
  29. ({ \
  30. static struct lock_class_key __key; \
  31. \
  32. __percpu_counter_init(fbc, value, gfp, &__key); \
  33. })
  34. void percpu_counter_destroy(struct percpu_counter *fbc);
  35. void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
  36. void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount,
  37. s32 batch);
  38. s64 __percpu_counter_sum(struct percpu_counter *fbc);
  39. int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch);
  40. void percpu_counter_sync(struct percpu_counter *fbc);
  41. static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
  42. {
  43. return __percpu_counter_compare(fbc, rhs, percpu_counter_batch);
  44. }
  45. static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
  46. {
  47. percpu_counter_add_batch(fbc, amount, percpu_counter_batch);
  48. }
  49. static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
  50. {
  51. s64 ret = __percpu_counter_sum(fbc);
  52. return ret < 0 ? 0 : ret;
  53. }
  54. static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
  55. {
  56. return __percpu_counter_sum(fbc);
  57. }
  58. static inline s64 percpu_counter_read(struct percpu_counter *fbc)
  59. {
  60. return fbc->count;
  61. }
  62. /*
  63. * It is possible for the percpu_counter_read() to return a small negative
  64. * number for some counter which should never be negative.
  65. *
  66. */
  67. static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
  68. {
  69. /* Prevent reloads of fbc->count */
  70. s64 ret = READ_ONCE(fbc->count);
  71. if (ret >= 0)
  72. return ret;
  73. return 0;
  74. }
  75. static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
  76. {
  77. return (fbc->counters != NULL);
  78. }
  79. #else /* !CONFIG_SMP */
  80. struct percpu_counter {
  81. s64 count;
  82. };
  83. static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount,
  84. gfp_t gfp)
  85. {
  86. fbc->count = amount;
  87. return 0;
  88. }
  89. static inline void percpu_counter_destroy(struct percpu_counter *fbc)
  90. {
  91. }
  92. static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
  93. {
  94. fbc->count = amount;
  95. }
  96. static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
  97. {
  98. if (fbc->count > rhs)
  99. return 1;
  100. else if (fbc->count < rhs)
  101. return -1;
  102. else
  103. return 0;
  104. }
  105. static inline int
  106. __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
  107. {
  108. return percpu_counter_compare(fbc, rhs);
  109. }
  110. static inline void
  111. percpu_counter_add(struct percpu_counter *fbc, s64 amount)
  112. {
  113. preempt_disable();
  114. fbc->count += amount;
  115. preempt_enable();
  116. }
  117. static inline void
  118. percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
  119. {
  120. percpu_counter_add(fbc, amount);
  121. }
  122. static inline s64 percpu_counter_read(struct percpu_counter *fbc)
  123. {
  124. return fbc->count;
  125. }
  126. /*
  127. * percpu_counter is intended to track positive numbers. In the UP case the
  128. * number should never be negative.
  129. */
  130. static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
  131. {
  132. return fbc->count;
  133. }
  134. static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
  135. {
  136. return percpu_counter_read_positive(fbc);
  137. }
  138. static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
  139. {
  140. return percpu_counter_read(fbc);
  141. }
  142. static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
  143. {
  144. return true;
  145. }
  146. static inline void percpu_counter_sync(struct percpu_counter *fbc)
  147. {
  148. }
  149. #endif /* CONFIG_SMP */
  150. static inline void percpu_counter_inc(struct percpu_counter *fbc)
  151. {
  152. percpu_counter_add(fbc, 1);
  153. }
  154. static inline void percpu_counter_dec(struct percpu_counter *fbc)
  155. {
  156. percpu_counter_add(fbc, -1);
  157. }
  158. static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
  159. {
  160. percpu_counter_add(fbc, -amount);
  161. }
  162. #endif /* _LINUX_PERCPU_COUNTER_H */