iocost_monitor.py 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. #!/usr/bin/env drgn
  2. #
  3. # Copyright (C) 2019 Tejun Heo <tj@kernel.org>
  4. # Copyright (C) 2019 Facebook
  5. desc = """
  6. This is a drgn script to monitor the blk-iocost cgroup controller.
  7. See the comment at the top of block/blk-iocost.c for more details.
  8. For drgn, visit https://github.com/osandov/drgn.
  9. """
  10. import sys
  11. import re
  12. import time
  13. import json
  14. import math
  15. import drgn
  16. from drgn import container_of
  17. from drgn.helpers.linux.list import list_for_each_entry,list_empty
  18. from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup
  19. import argparse
  20. parser = argparse.ArgumentParser(description=desc,
  21. formatter_class=argparse.RawTextHelpFormatter)
  22. parser.add_argument('devname', metavar='DEV',
  23. help='Target block device name (e.g. sda)')
  24. parser.add_argument('--cgroup', action='append', metavar='REGEX',
  25. help='Regex for target cgroups, ')
  26. parser.add_argument('--interval', '-i', metavar='SECONDS', type=float, default=1,
  27. help='Monitoring interval in seconds (0 exits immediately '
  28. 'after checking requirements)')
  29. parser.add_argument('--json', action='store_true',
  30. help='Output in json')
  31. args = parser.parse_args()
  32. def err(s):
  33. print(s, file=sys.stderr, flush=True)
  34. sys.exit(1)
  35. try:
  36. blkcg_root = prog['blkcg_root']
  37. plid = prog['blkcg_policy_iocost'].plid.value_()
  38. except:
  39. err('The kernel does not have iocost enabled')
  40. IOC_RUNNING = prog['IOC_RUNNING'].value_()
  41. WEIGHT_ONE = prog['WEIGHT_ONE'].value_()
  42. VTIME_PER_SEC = prog['VTIME_PER_SEC'].value_()
  43. VTIME_PER_USEC = prog['VTIME_PER_USEC'].value_()
  44. AUTOP_SSD_FAST = prog['AUTOP_SSD_FAST'].value_()
  45. AUTOP_SSD_DFL = prog['AUTOP_SSD_DFL'].value_()
  46. AUTOP_SSD_QD1 = prog['AUTOP_SSD_QD1'].value_()
  47. AUTOP_HDD = prog['AUTOP_HDD'].value_()
  48. autop_names = {
  49. AUTOP_SSD_FAST: 'ssd_fast',
  50. AUTOP_SSD_DFL: 'ssd_dfl',
  51. AUTOP_SSD_QD1: 'ssd_qd1',
  52. AUTOP_HDD: 'hdd',
  53. }
  54. class BlkgIterator:
  55. def blkcg_name(blkcg):
  56. return blkcg.css.cgroup.kn.name.string_().decode('utf-8')
  57. def walk(self, blkcg, q_id, parent_path):
  58. if not self.include_dying and \
  59. not (blkcg.css.flags.value_() & prog['CSS_ONLINE'].value_()):
  60. return
  61. name = BlkgIterator.blkcg_name(blkcg)
  62. path = parent_path + '/' + name if parent_path else name
  63. blkg = drgn.Object(prog, 'struct blkcg_gq',
  64. address=radix_tree_lookup(blkcg.blkg_tree.address_of_(), q_id))
  65. if not blkg.address_:
  66. return
  67. self.blkgs.append((path if path else '/', blkg))
  68. for c in list_for_each_entry('struct blkcg',
  69. blkcg.css.children.address_of_(), 'css.sibling'):
  70. self.walk(c, q_id, path)
  71. def __init__(self, root_blkcg, q_id, include_dying=False):
  72. self.include_dying = include_dying
  73. self.blkgs = []
  74. self.walk(root_blkcg, q_id, '')
  75. def __iter__(self):
  76. return iter(self.blkgs)
  77. class IocStat:
  78. def __init__(self, ioc):
  79. global autop_names
  80. self.enabled = ioc.enabled.value_()
  81. self.running = ioc.running.value_() == IOC_RUNNING
  82. self.period_ms = ioc.period_us.value_() / 1_000
  83. self.period_at = ioc.period_at.value_() / 1_000_000
  84. self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC
  85. self.vrate_pct = ioc.vtime_base_rate.value_() * 100 / VTIME_PER_USEC
  86. self.busy_level = ioc.busy_level.value_()
  87. self.autop_idx = ioc.autop_idx.value_()
  88. self.user_cost_model = ioc.user_cost_model.value_()
  89. self.user_qos_params = ioc.user_qos_params.value_()
  90. if self.autop_idx in autop_names:
  91. self.autop_name = autop_names[self.autop_idx]
  92. else:
  93. self.autop_name = '?'
  94. def dict(self, now):
  95. return { 'device' : devname,
  96. 'timestamp' : now,
  97. 'enabled' : self.enabled,
  98. 'running' : self.running,
  99. 'period_ms' : self.period_ms,
  100. 'period_at' : self.period_at,
  101. 'period_vtime_at' : self.vperiod_at,
  102. 'busy_level' : self.busy_level,
  103. 'vrate_pct' : self.vrate_pct, }
  104. def table_preamble_str(self):
  105. state = ('RUN' if self.running else 'IDLE') if self.enabled else 'OFF'
  106. output = f'{devname} {state:4} ' \
  107. f'per={self.period_ms}ms ' \
  108. f'cur_per={self.period_at:.3f}:v{self.vperiod_at:.3f} ' \
  109. f'busy={self.busy_level:+3} ' \
  110. f'vrate={self.vrate_pct:6.2f}% ' \
  111. f'params={self.autop_name}'
  112. if self.user_cost_model or self.user_qos_params:
  113. output += f'({"C" if self.user_cost_model else ""}{"Q" if self.user_qos_params else ""})'
  114. return output
  115. def table_header_str(self):
  116. return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \
  117. f'{"debt":>7} {"delay":>7} {"usage%"}'
  118. class IocgStat:
  119. def __init__(self, iocg):
  120. ioc = iocg.ioc
  121. blkg = iocg.pd.blkg
  122. self.is_active = not list_empty(iocg.active_list.address_of_())
  123. self.weight = iocg.weight.value_() / WEIGHT_ONE
  124. self.active = iocg.active.value_() / WEIGHT_ONE
  125. self.inuse = iocg.inuse.value_() / WEIGHT_ONE
  126. self.hwa_pct = iocg.hweight_active.value_() * 100 / WEIGHT_ONE
  127. self.hwi_pct = iocg.hweight_inuse.value_() * 100 / WEIGHT_ONE
  128. self.address = iocg.value_()
  129. vdone = iocg.done_vtime.counter.value_()
  130. vtime = iocg.vtime.counter.value_()
  131. vrate = ioc.vtime_rate.counter.value_()
  132. period_vtime = ioc.period_us.value_() * vrate
  133. if period_vtime:
  134. self.inflight_pct = (vtime - vdone) * 100 / period_vtime
  135. else:
  136. self.inflight_pct = 0
  137. self.usage = (100 * iocg.usage_delta_us.value_() /
  138. ioc.period_us.value_()) if self.active else 0
  139. self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
  140. if blkg.use_delay.counter.value_() != 0:
  141. self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
  142. else:
  143. self.delay_ms = 0
  144. def dict(self, now, path):
  145. out = { 'cgroup' : path,
  146. 'timestamp' : now,
  147. 'is_active' : self.is_active,
  148. 'weight' : self.weight,
  149. 'weight_active' : self.active,
  150. 'weight_inuse' : self.inuse,
  151. 'hweight_active_pct' : self.hwa_pct,
  152. 'hweight_inuse_pct' : self.hwi_pct,
  153. 'inflight_pct' : self.inflight_pct,
  154. 'debt_ms' : self.debt_ms,
  155. 'delay_ms' : self.delay_ms,
  156. 'usage_pct' : self.usage,
  157. 'address' : self.address }
  158. return out
  159. def table_row_str(self, path):
  160. out = f'{path[-28:]:28} ' \
  161. f'{"*" if self.is_active else " "} ' \
  162. f'{round(self.inuse):5}/{round(self.active):5} ' \
  163. f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \
  164. f'{self.inflight_pct:6.2f} ' \
  165. f'{self.debt_ms:7.2f} ' \
  166. f'{self.delay_ms:7.2f} '\
  167. f'{min(self.usage, 999):6.2f}'
  168. out = out.rstrip(':')
  169. return out
  170. # handle args
  171. table_fmt = not args.json
  172. interval = args.interval
  173. devname = args.devname
  174. if args.json:
  175. table_fmt = False
  176. re_str = None
  177. if args.cgroup:
  178. for r in args.cgroup:
  179. if re_str is None:
  180. re_str = r
  181. else:
  182. re_str += '|' + r
  183. filter_re = re.compile(re_str) if re_str else None
  184. # Locate the roots
  185. q_id = None
  186. root_iocg = None
  187. ioc = None
  188. for i, ptr in radix_tree_for_each(blkcg_root.blkg_tree.address_of_()):
  189. blkg = drgn.Object(prog, 'struct blkcg_gq', address=ptr)
  190. try:
  191. if devname == blkg.q.kobj.parent.name.string_().decode('utf-8'):
  192. q_id = blkg.q.id.value_()
  193. if blkg.pd[plid]:
  194. root_iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
  195. ioc = root_iocg.ioc
  196. break
  197. except:
  198. pass
  199. if ioc is None:
  200. err(f'Could not find ioc for {devname}');
  201. if interval == 0:
  202. sys.exit(0)
  203. # Keep printing
  204. while True:
  205. now = time.time()
  206. iocstat = IocStat(ioc)
  207. output = ''
  208. if table_fmt:
  209. output += '\n' + iocstat.table_preamble_str()
  210. output += '\n' + iocstat.table_header_str()
  211. else:
  212. output += json.dumps(iocstat.dict(now))
  213. for path, blkg in BlkgIterator(blkcg_root, q_id):
  214. if filter_re and not filter_re.match(path):
  215. continue
  216. if not blkg.pd[plid]:
  217. continue
  218. iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
  219. iocg_stat = IocgStat(iocg)
  220. if not filter_re and not iocg_stat.is_active:
  221. continue
  222. if table_fmt:
  223. output += '\n' + iocg_stat.table_row_str(path)
  224. else:
  225. output += '\n' + json.dumps(iocg_stat.dict(now, path))
  226. print(output)
  227. sys.stdout.flush()
  228. time.sleep(interval)