progress.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307
  1. """
  2. BitBake progress handling code
  3. """
  4. # Copyright (C) 2016 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. import re
  9. import time
  10. import inspect
  11. import bb.event
  12. import bb.build
  13. from bb.build import StdoutNoopContextManager
  14. # from https://stackoverflow.com/a/14693789/221061
  15. ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
  16. def filter_color(string):
  17. """
  18. Filter ANSI escape codes out of |string|, return new string
  19. """
  20. return ANSI_ESCAPE_REGEX.sub('', string)
  21. def filter_color_n(string):
  22. """
  23. Filter ANSI escape codes out of |string|, returns tuple of
  24. (new string, # of ANSI codes removed)
  25. """
  26. return ANSI_ESCAPE_REGEX.subn('', string)
  27. class ProgressHandler:
  28. """
  29. Base class that can pretend to be a file object well enough to be
  30. used to build objects to intercept console output and determine the
  31. progress of some operation.
  32. """
  33. def __init__(self, d, outfile=None):
  34. self._progress = 0
  35. self._data = d
  36. self._lastevent = 0
  37. if outfile:
  38. self._outfile = outfile
  39. else:
  40. self._outfile = StdoutNoopContextManager()
  41. def __enter__(self):
  42. self._outfile.__enter__()
  43. return self
  44. def __exit__(self, *excinfo):
  45. self._outfile.__exit__(*excinfo)
  46. def _fire_progress(self, taskprogress, rate=None):
  47. """Internal function to fire the progress event"""
  48. bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
  49. def write(self, string):
  50. self._outfile.write(string)
  51. def flush(self):
  52. self._outfile.flush()
  53. def update(self, progress, rate=None):
  54. ts = time.time()
  55. if progress > 100:
  56. progress = 100
  57. if progress != self._progress or self._lastevent + 1 < ts:
  58. self._fire_progress(progress, rate)
  59. self._lastevent = ts
  60. self._progress = progress
  61. class LineFilterProgressHandler(ProgressHandler):
  62. """
  63. A ProgressHandler variant that provides the ability to filter out
  64. the lines if they contain progress information. Additionally, it
  65. filters out anything before the last line feed on a line. This can
  66. be used to keep the logs clean of output that we've only enabled for
  67. getting progress, assuming that that can be done on a per-line
  68. basis.
  69. """
  70. def __init__(self, d, outfile=None):
  71. self._linebuffer = ''
  72. super().__init__(d, outfile)
  73. def write(self, string):
  74. self._linebuffer += string
  75. while True:
  76. breakpos = self._linebuffer.find('\n') + 1
  77. if breakpos == 0:
  78. break
  79. line = self._linebuffer[:breakpos]
  80. self._linebuffer = self._linebuffer[breakpos:]
  81. # Drop any line feeds and anything that precedes them
  82. lbreakpos = line.rfind('\r') + 1
  83. if lbreakpos:
  84. line = line[lbreakpos:]
  85. if self.writeline(filter_color(line)):
  86. super().write(line)
  87. def writeline(self, line):
  88. return True
  89. class BasicProgressHandler(ProgressHandler):
  90. def __init__(self, d, regex=r'(\d+)%', outfile=None):
  91. super().__init__(d, outfile)
  92. self._regex = re.compile(regex)
  93. # Send an initial progress event so the bar gets shown
  94. self._fire_progress(0)
  95. def write(self, string):
  96. percs = self._regex.findall(filter_color(string))
  97. if percs:
  98. progress = int(percs[-1])
  99. self.update(progress)
  100. super().write(string)
  101. class OutOfProgressHandler(ProgressHandler):
  102. def __init__(self, d, regex, outfile=None):
  103. super().__init__(d, outfile)
  104. self._regex = re.compile(regex)
  105. # Send an initial progress event so the bar gets shown
  106. self._fire_progress(0)
  107. def write(self, string):
  108. nums = self._regex.findall(filter_color(string))
  109. if nums:
  110. progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
  111. self.update(progress)
  112. super().write(string)
  113. class MultiStageProgressReporter:
  114. """
  115. Class which allows reporting progress without the caller
  116. having to know where they are in the overall sequence. Useful
  117. for tasks made up of python code spread across multiple
  118. classes / functions - the progress reporter object can
  119. be passed around or stored at the object level and calls
  120. to next_stage() and update() made whereever needed.
  121. """
  122. def __init__(self, d, stage_weights, debug=False):
  123. """
  124. Initialise the progress reporter.
  125. Parameters:
  126. * d: the datastore (needed for firing the events)
  127. * stage_weights: a list of weight values, one for each stage.
  128. The value is scaled internally so you only need to specify
  129. values relative to other values in the list, so if there
  130. are two stages and the first takes 2s and the second takes
  131. 10s you would specify [2, 10] (or [1, 5], it doesn't matter).
  132. * debug: specify True (and ensure you call finish() at the end)
  133. in order to show a printout of the calculated stage weights
  134. based on timing each stage. Use this to determine what the
  135. weights should be when you're not sure.
  136. """
  137. self._data = d
  138. total = sum(stage_weights)
  139. self._stage_weights = [float(x)/total for x in stage_weights]
  140. self._stage = -1
  141. self._base_progress = 0
  142. # Send an initial progress event so the bar gets shown
  143. self._fire_progress(0)
  144. self._debug = debug
  145. self._finished = False
  146. if self._debug:
  147. self._last_time = time.time()
  148. self._stage_times = []
  149. self._stage_total = None
  150. self._callers = []
  151. def __enter__(self):
  152. return self
  153. def __exit__(self, *excinfo):
  154. pass
  155. def _fire_progress(self, taskprogress):
  156. bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
  157. def next_stage(self, stage_total=None):
  158. """
  159. Move to the next stage.
  160. Parameters:
  161. * stage_total: optional total for progress within the stage,
  162. see update() for details
  163. NOTE: you need to call this before the first stage.
  164. """
  165. self._stage += 1
  166. self._stage_total = stage_total
  167. if self._stage == 0:
  168. # First stage
  169. if self._debug:
  170. self._last_time = time.time()
  171. else:
  172. if self._stage < len(self._stage_weights):
  173. self._base_progress = sum(self._stage_weights[:self._stage]) * 100
  174. if self._debug:
  175. currtime = time.time()
  176. self._stage_times.append(currtime - self._last_time)
  177. self._last_time = currtime
  178. self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
  179. elif not self._debug:
  180. bb.warn('ProgressReporter: current stage beyond declared number of stages')
  181. self._base_progress = 100
  182. self._fire_progress(self._base_progress)
  183. def update(self, stage_progress):
  184. """
  185. Update progress within the current stage.
  186. Parameters:
  187. * stage_progress: progress value within the stage. If stage_total
  188. was specified when next_stage() was last called, then this
  189. value is considered to be out of stage_total, otherwise it should
  190. be a percentage value from 0 to 100.
  191. """
  192. progress = None
  193. if self._stage_total:
  194. stage_progress = (float(stage_progress) / self._stage_total) * 100
  195. if self._stage < 0:
  196. bb.warn('ProgressReporter: update called before first call to next_stage()')
  197. elif self._stage < len(self._stage_weights):
  198. progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
  199. else:
  200. progress = self._base_progress
  201. if progress:
  202. if progress > 100:
  203. progress = 100
  204. self._fire_progress(progress)
  205. def finish(self):
  206. if self._finished:
  207. return
  208. self._finished = True
  209. if self._debug:
  210. import math
  211. self._stage_times.append(time.time() - self._last_time)
  212. mintime = max(min(self._stage_times), 0.01)
  213. self._callers.append(None)
  214. stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
  215. bb.warn('Stage weights: %s' % stage_weights)
  216. out = []
  217. for stage_weight, caller in zip(stage_weights, self._callers):
  218. if caller:
  219. out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
  220. else:
  221. out.append('Up to finish: %d' % stage_weight)
  222. bb.warn('Stage times:\n %s' % '\n '.join(out))
  223. class MultiStageProcessProgressReporter(MultiStageProgressReporter):
  224. """
  225. Version of MultiStageProgressReporter intended for use with
  226. standalone processes (such as preparing the runqueue)
  227. """
  228. def __init__(self, d, processname, stage_weights, debug=False):
  229. self._processname = processname
  230. self._started = False
  231. super().__init__(d, stage_weights, debug)
  232. def start(self):
  233. if not self._started:
  234. bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
  235. self._started = True
  236. def _fire_progress(self, taskprogress):
  237. if taskprogress == 0:
  238. self.start()
  239. return
  240. bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
  241. def finish(self):
  242. MultiStageProgressReporter.finish(self)
  243. bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
  244. class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
  245. """
  246. MultiStageProcessProgressReporter that takes the calls and does nothing
  247. with them (to avoid a bunch of "if progress_reporter:" checks)
  248. """
  249. def __init__(self):
  250. super().__init__(None, [])
  251. def _fire_progress(self, taskprogress, rate=None):
  252. pass
  253. def start(self):
  254. pass
  255. def next_stage(self, stage_total=None):
  256. pass
  257. def update(self, stage_progress):
  258. pass
  259. def finish(self):
  260. pass