progress.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. """
  2. BitBake progress handling code
  3. """
  4. # Copyright (C) 2016 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. import sys
  9. import re
  10. import time
  11. import inspect
  12. import bb.event
  13. import bb.build
  14. class ProgressHandler(object):
  15. """
  16. Base class that can pretend to be a file object well enough to be
  17. used to build objects to intercept console output and determine the
  18. progress of some operation.
  19. """
  20. def __init__(self, d, outfile=None):
  21. self._progress = 0
  22. self._data = d
  23. self._lastevent = 0
  24. if outfile:
  25. self._outfile = outfile
  26. else:
  27. self._outfile = sys.stdout
  28. def _fire_progress(self, taskprogress, rate=None):
  29. """Internal function to fire the progress event"""
  30. bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
  31. def write(self, string):
  32. self._outfile.write(string)
  33. def flush(self):
  34. self._outfile.flush()
  35. def update(self, progress, rate=None):
  36. ts = time.time()
  37. if progress > 100:
  38. progress = 100
  39. if progress != self._progress or self._lastevent + 1 < ts:
  40. self._fire_progress(progress, rate)
  41. self._lastevent = ts
  42. self._progress = progress
  43. class LineFilterProgressHandler(ProgressHandler):
  44. """
  45. A ProgressHandler variant that provides the ability to filter out
  46. the lines if they contain progress information. Additionally, it
  47. filters out anything before the last line feed on a line. This can
  48. be used to keep the logs clean of output that we've only enabled for
  49. getting progress, assuming that that can be done on a per-line
  50. basis.
  51. """
  52. def __init__(self, d, outfile=None):
  53. self._linebuffer = ''
  54. super(LineFilterProgressHandler, self).__init__(d, outfile)
  55. def write(self, string):
  56. self._linebuffer += string
  57. while True:
  58. breakpos = self._linebuffer.find('\n') + 1
  59. if breakpos == 0:
  60. break
  61. line = self._linebuffer[:breakpos]
  62. self._linebuffer = self._linebuffer[breakpos:]
  63. # Drop any line feeds and anything that precedes them
  64. lbreakpos = line.rfind('\r') + 1
  65. if lbreakpos:
  66. line = line[lbreakpos:]
  67. if self.writeline(line):
  68. super(LineFilterProgressHandler, self).write(line)
  69. def writeline(self, line):
  70. return True
  71. class BasicProgressHandler(ProgressHandler):
  72. def __init__(self, d, regex=r'(\d+)%', outfile=None):
  73. super(BasicProgressHandler, self).__init__(d, outfile)
  74. self._regex = re.compile(regex)
  75. # Send an initial progress event so the bar gets shown
  76. self._fire_progress(0)
  77. def write(self, string):
  78. percs = self._regex.findall(string)
  79. if percs:
  80. progress = int(percs[-1])
  81. self.update(progress)
  82. super(BasicProgressHandler, self).write(string)
  83. class OutOfProgressHandler(ProgressHandler):
  84. def __init__(self, d, regex, outfile=None):
  85. super(OutOfProgressHandler, self).__init__(d, outfile)
  86. self._regex = re.compile(regex)
  87. # Send an initial progress event so the bar gets shown
  88. self._fire_progress(0)
  89. def write(self, string):
  90. nums = self._regex.findall(string)
  91. if nums:
  92. progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
  93. self.update(progress)
  94. super(OutOfProgressHandler, self).write(string)
  95. class MultiStageProgressReporter(object):
  96. """
  97. Class which allows reporting progress without the caller
  98. having to know where they are in the overall sequence. Useful
  99. for tasks made up of python code spread across multiple
  100. classes / functions - the progress reporter object can
  101. be passed around or stored at the object level and calls
  102. to next_stage() and update() made whereever needed.
  103. """
  104. def __init__(self, d, stage_weights, debug=False):
  105. """
  106. Initialise the progress reporter.
  107. Parameters:
  108. * d: the datastore (needed for firing the events)
  109. * stage_weights: a list of weight values, one for each stage.
  110. The value is scaled internally so you only need to specify
  111. values relative to other values in the list, so if there
  112. are two stages and the first takes 2s and the second takes
  113. 10s you would specify [2, 10] (or [1, 5], it doesn't matter).
  114. * debug: specify True (and ensure you call finish() at the end)
  115. in order to show a printout of the calculated stage weights
  116. based on timing each stage. Use this to determine what the
  117. weights should be when you're not sure.
  118. """
  119. self._data = d
  120. total = sum(stage_weights)
  121. self._stage_weights = [float(x)/total for x in stage_weights]
  122. self._stage = -1
  123. self._base_progress = 0
  124. # Send an initial progress event so the bar gets shown
  125. self._fire_progress(0)
  126. self._debug = debug
  127. self._finished = False
  128. if self._debug:
  129. self._last_time = time.time()
  130. self._stage_times = []
  131. self._stage_total = None
  132. self._callers = []
  133. def _fire_progress(self, taskprogress):
  134. bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
  135. def next_stage(self, stage_total=None):
  136. """
  137. Move to the next stage.
  138. Parameters:
  139. * stage_total: optional total for progress within the stage,
  140. see update() for details
  141. NOTE: you need to call this before the first stage.
  142. """
  143. self._stage += 1
  144. self._stage_total = stage_total
  145. if self._stage == 0:
  146. # First stage
  147. if self._debug:
  148. self._last_time = time.time()
  149. else:
  150. if self._stage < len(self._stage_weights):
  151. self._base_progress = sum(self._stage_weights[:self._stage]) * 100
  152. if self._debug:
  153. currtime = time.time()
  154. self._stage_times.append(currtime - self._last_time)
  155. self._last_time = currtime
  156. self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
  157. elif not self._debug:
  158. bb.warn('ProgressReporter: current stage beyond declared number of stages')
  159. self._base_progress = 100
  160. self._fire_progress(self._base_progress)
  161. def update(self, stage_progress):
  162. """
  163. Update progress within the current stage.
  164. Parameters:
  165. * stage_progress: progress value within the stage. If stage_total
  166. was specified when next_stage() was last called, then this
  167. value is considered to be out of stage_total, otherwise it should
  168. be a percentage value from 0 to 100.
  169. """
  170. if self._stage_total:
  171. stage_progress = (float(stage_progress) / self._stage_total) * 100
  172. if self._stage < 0:
  173. bb.warn('ProgressReporter: update called before first call to next_stage()')
  174. elif self._stage < len(self._stage_weights):
  175. progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
  176. else:
  177. progress = self._base_progress
  178. if progress > 100:
  179. progress = 100
  180. self._fire_progress(progress)
  181. def finish(self):
  182. if self._finished:
  183. return
  184. self._finished = True
  185. if self._debug:
  186. import math
  187. self._stage_times.append(time.time() - self._last_time)
  188. mintime = max(min(self._stage_times), 0.01)
  189. self._callers.append(None)
  190. stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
  191. bb.warn('Stage weights: %s' % stage_weights)
  192. out = []
  193. for stage_weight, caller in zip(stage_weights, self._callers):
  194. if caller:
  195. out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
  196. else:
  197. out.append('Up to finish: %d' % stage_weight)
  198. bb.warn('Stage times:\n %s' % '\n '.join(out))
  199. class MultiStageProcessProgressReporter(MultiStageProgressReporter):
  200. """
  201. Version of MultiStageProgressReporter intended for use with
  202. standalone processes (such as preparing the runqueue)
  203. """
  204. def __init__(self, d, processname, stage_weights, debug=False):
  205. self._processname = processname
  206. self._started = False
  207. MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
  208. def start(self):
  209. if not self._started:
  210. bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
  211. self._started = True
  212. def _fire_progress(self, taskprogress):
  213. if taskprogress == 0:
  214. self.start()
  215. return
  216. bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
  217. def finish(self):
  218. MultiStageProgressReporter.finish(self)
  219. bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
  220. class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
  221. """
  222. MultiStageProcessProgressReporter that takes the calls and does nothing
  223. with them (to avoid a bunch of "if progress_reporter:" checks)
  224. """
  225. def __init__(self):
  226. MultiStageProcessProgressReporter.__init__(self, "", None, [])
  227. def _fire_progress(self, taskprogress, rate=None):
  228. pass
  229. def start(self):
  230. pass
  231. def next_stage(self, stage_total=None):
  232. pass
  233. def update(self, stage_progress):
  234. pass
  235. def finish(self):
  236. pass