blk-cgroup-rwstat.h 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. /* SPDX-License-Identifier: GPL-2.0
  2. *
  3. * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
  4. * Do not use in new code.
  5. */
  6. #ifndef _BLK_CGROUP_RWSTAT_H
  7. #define _BLK_CGROUP_RWSTAT_H
  8. #include <linux/blk-cgroup.h>
  9. enum blkg_rwstat_type {
  10. BLKG_RWSTAT_READ,
  11. BLKG_RWSTAT_WRITE,
  12. BLKG_RWSTAT_SYNC,
  13. BLKG_RWSTAT_ASYNC,
  14. BLKG_RWSTAT_DISCARD,
  15. BLKG_RWSTAT_NR,
  16. BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
  17. };
  18. /*
  19. * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
  20. * recursive. Used to carry stats of dead children.
  21. */
  22. struct blkg_rwstat {
  23. struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
  24. atomic64_t aux_cnt[BLKG_RWSTAT_NR];
  25. };
  26. struct blkg_rwstat_sample {
  27. u64 cnt[BLKG_RWSTAT_NR];
  28. };
  29. static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
  30. unsigned int idx)
  31. {
  32. return atomic64_read(&rwstat->aux_cnt[idx]) +
  33. percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
  34. }
  35. int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
  36. void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
  37. u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
  38. const struct blkg_rwstat_sample *rwstat);
  39. u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
  40. int off);
  41. void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
  42. int off, struct blkg_rwstat_sample *sum);
  43. /**
  44. * blkg_rwstat_add - add a value to a blkg_rwstat
  45. * @rwstat: target blkg_rwstat
  46. * @op: REQ_OP and flags
  47. * @val: value to add
  48. *
  49. * Add @val to @rwstat. The counters are chosen according to @rw. The
  50. * caller is responsible for synchronizing calls to this function.
  51. */
  52. static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
  53. unsigned int op, uint64_t val)
  54. {
  55. struct percpu_counter *cnt;
  56. if (op_is_discard(op))
  57. cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
  58. else if (op_is_write(op))
  59. cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
  60. else
  61. cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
  62. percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
  63. if (op_is_sync(op))
  64. cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
  65. else
  66. cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
  67. percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
  68. }
  69. /**
  70. * blkg_rwstat_read - read the current values of a blkg_rwstat
  71. * @rwstat: blkg_rwstat to read
  72. *
  73. * Read the current snapshot of @rwstat and return it in the aux counts.
  74. */
  75. static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
  76. struct blkg_rwstat_sample *result)
  77. {
  78. int i;
  79. for (i = 0; i < BLKG_RWSTAT_NR; i++)
  80. result->cnt[i] =
  81. percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
  82. }
  83. /**
  84. * blkg_rwstat_total - read the total count of a blkg_rwstat
  85. * @rwstat: blkg_rwstat to read
  86. *
  87. * Return the total count of @rwstat regardless of the IO direction. This
  88. * function can be called without synchronization and takes care of u64
  89. * atomicity.
  90. */
  91. static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
  92. {
  93. struct blkg_rwstat_sample tmp = { };
  94. blkg_rwstat_read(rwstat, &tmp);
  95. return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
  96. }
  97. /**
  98. * blkg_rwstat_reset - reset a blkg_rwstat
  99. * @rwstat: blkg_rwstat to reset
  100. */
  101. static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
  102. {
  103. int i;
  104. for (i = 0; i < BLKG_RWSTAT_NR; i++) {
  105. percpu_counter_set(&rwstat->cpu_cnt[i], 0);
  106. atomic64_set(&rwstat->aux_cnt[i], 0);
  107. }
  108. }
  109. /**
  110. * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
  111. * @to: the destination blkg_rwstat
  112. * @from: the source
  113. *
  114. * Add @from's count including the aux one to @to's aux count.
  115. */
  116. static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
  117. struct blkg_rwstat *from)
  118. {
  119. u64 sum[BLKG_RWSTAT_NR];
  120. int i;
  121. for (i = 0; i < BLKG_RWSTAT_NR; i++)
  122. sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
  123. for (i = 0; i < BLKG_RWSTAT_NR; i++)
  124. atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
  125. &to->aux_cnt[i]);
  126. }
  127. #endif /* _BLK_CGROUP_RWSTAT_H */