progress.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. """
  2. BitBake progress handling code
  3. """
  4. # Copyright (C) 2016 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # This program is free software; you can redistribute it and/or modify
  9. # it under the terms of the GNU General Public License version 2 as
  10. # published by the Free Software Foundation.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License along
  18. # with this program; if not, write to the Free Software Foundation, Inc.,
  19. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20. import sys
  21. import re
  22. import time
  23. import inspect
  24. import bb.event
  25. import bb.build
  26. class ProgressHandler(object):
  27. """
  28. Base class that can pretend to be a file object well enough to be
  29. used to build objects to intercept console output and determine the
  30. progress of some operation.
  31. """
  32. def __init__(self, d, outfile=None):
  33. self._progress = 0
  34. self._data = d
  35. self._lastevent = 0
  36. if outfile:
  37. self._outfile = outfile
  38. else:
  39. self._outfile = sys.stdout
  40. def _fire_progress(self, taskprogress, rate=None):
  41. """Internal function to fire the progress event"""
  42. bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
  43. def write(self, string):
  44. self._outfile.write(string)
  45. def flush(self):
  46. self._outfile.flush()
  47. def update(self, progress, rate=None):
  48. ts = time.time()
  49. if progress > 100:
  50. progress = 100
  51. if progress != self._progress or self._lastevent + 1 < ts:
  52. self._fire_progress(progress, rate)
  53. self._lastevent = ts
  54. self._progress = progress
  55. class LineFilterProgressHandler(ProgressHandler):
  56. """
  57. A ProgressHandler variant that provides the ability to filter out
  58. the lines if they contain progress information. Additionally, it
  59. filters out anything before the last line feed on a line. This can
  60. be used to keep the logs clean of output that we've only enabled for
  61. getting progress, assuming that that can be done on a per-line
  62. basis.
  63. """
  64. def __init__(self, d, outfile=None):
  65. self._linebuffer = ''
  66. super(LineFilterProgressHandler, self).__init__(d, outfile)
  67. def write(self, string):
  68. self._linebuffer += string
  69. while True:
  70. breakpos = self._linebuffer.find('\n') + 1
  71. if breakpos == 0:
  72. break
  73. line = self._linebuffer[:breakpos]
  74. self._linebuffer = self._linebuffer[breakpos:]
  75. # Drop any line feeds and anything that precedes them
  76. lbreakpos = line.rfind('\r') + 1
  77. if lbreakpos:
  78. line = line[lbreakpos:]
  79. if self.writeline(line):
  80. super(LineFilterProgressHandler, self).write(line)
  81. def writeline(self, line):
  82. return True
  83. class BasicProgressHandler(ProgressHandler):
  84. def __init__(self, d, regex=r'(\d+)%', outfile=None):
  85. super(BasicProgressHandler, self).__init__(d, outfile)
  86. self._regex = re.compile(regex)
  87. # Send an initial progress event so the bar gets shown
  88. self._fire_progress(0)
  89. def write(self, string):
  90. percs = self._regex.findall(string)
  91. if percs:
  92. progress = int(percs[-1])
  93. self.update(progress)
  94. super(BasicProgressHandler, self).write(string)
  95. class OutOfProgressHandler(ProgressHandler):
  96. def __init__(self, d, regex, outfile=None):
  97. super(OutOfProgressHandler, self).__init__(d, outfile)
  98. self._regex = re.compile(regex)
  99. # Send an initial progress event so the bar gets shown
  100. self._fire_progress(0)
  101. def write(self, string):
  102. nums = self._regex.findall(string)
  103. if nums:
  104. progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
  105. self.update(progress)
  106. super(OutOfProgressHandler, self).write(string)
  107. class MultiStageProgressReporter(object):
  108. """
  109. Class which allows reporting progress without the caller
  110. having to know where they are in the overall sequence. Useful
  111. for tasks made up of python code spread across multiple
  112. classes / functions - the progress reporter object can
  113. be passed around or stored at the object level and calls
  114. to next_stage() and update() made whereever needed.
  115. """
  116. def __init__(self, d, stage_weights, debug=False):
  117. """
  118. Initialise the progress reporter.
  119. Parameters:
  120. * d: the datastore (needed for firing the events)
  121. * stage_weights: a list of weight values, one for each stage.
  122. The value is scaled internally so you only need to specify
  123. values relative to other values in the list, so if there
  124. are two stages and the first takes 2s and the second takes
  125. 10s you would specify [2, 10] (or [1, 5], it doesn't matter).
  126. * debug: specify True (and ensure you call finish() at the end)
  127. in order to show a printout of the calculated stage weights
  128. based on timing each stage. Use this to determine what the
  129. weights should be when you're not sure.
  130. """
  131. self._data = d
  132. total = sum(stage_weights)
  133. self._stage_weights = [float(x)/total for x in stage_weights]
  134. self._stage = -1
  135. self._base_progress = 0
  136. # Send an initial progress event so the bar gets shown
  137. self._fire_progress(0)
  138. self._debug = debug
  139. self._finished = False
  140. if self._debug:
  141. self._last_time = time.time()
  142. self._stage_times = []
  143. self._stage_total = None
  144. self._callers = []
  145. def _fire_progress(self, taskprogress):
  146. bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
  147. def next_stage(self, stage_total=None):
  148. """
  149. Move to the next stage.
  150. Parameters:
  151. * stage_total: optional total for progress within the stage,
  152. see update() for details
  153. NOTE: you need to call this before the first stage.
  154. """
  155. self._stage += 1
  156. self._stage_total = stage_total
  157. if self._stage == 0:
  158. # First stage
  159. if self._debug:
  160. self._last_time = time.time()
  161. else:
  162. if self._stage < len(self._stage_weights):
  163. self._base_progress = sum(self._stage_weights[:self._stage]) * 100
  164. if self._debug:
  165. currtime = time.time()
  166. self._stage_times.append(currtime - self._last_time)
  167. self._last_time = currtime
  168. self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
  169. elif not self._debug:
  170. bb.warn('ProgressReporter: current stage beyond declared number of stages')
  171. self._base_progress = 100
  172. self._fire_progress(self._base_progress)
  173. def update(self, stage_progress):
  174. """
  175. Update progress within the current stage.
  176. Parameters:
  177. * stage_progress: progress value within the stage. If stage_total
  178. was specified when next_stage() was last called, then this
  179. value is considered to be out of stage_total, otherwise it should
  180. be a percentage value from 0 to 100.
  181. """
  182. if self._stage_total:
  183. stage_progress = (float(stage_progress) / self._stage_total) * 100
  184. if self._stage < 0:
  185. bb.warn('ProgressReporter: update called before first call to next_stage()')
  186. elif self._stage < len(self._stage_weights):
  187. progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
  188. else:
  189. progress = self._base_progress
  190. if progress > 100:
  191. progress = 100
  192. self._fire_progress(progress)
  193. def finish(self):
  194. if self._finished:
  195. return
  196. self._finished = True
  197. if self._debug:
  198. import math
  199. self._stage_times.append(time.time() - self._last_time)
  200. mintime = max(min(self._stage_times), 0.01)
  201. self._callers.append(None)
  202. stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
  203. bb.warn('Stage weights: %s' % stage_weights)
  204. out = []
  205. for stage_weight, caller in zip(stage_weights, self._callers):
  206. if caller:
  207. out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
  208. else:
  209. out.append('Up to finish: %d' % stage_weight)
  210. bb.warn('Stage times:\n %s' % '\n '.join(out))
  211. class MultiStageProcessProgressReporter(MultiStageProgressReporter):
  212. """
  213. Version of MultiStageProgressReporter intended for use with
  214. standalone processes (such as preparing the runqueue)
  215. """
  216. def __init__(self, d, processname, stage_weights, debug=False):
  217. self._processname = processname
  218. self._started = False
  219. MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
  220. def start(self):
  221. if not self._started:
  222. bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
  223. self._started = True
  224. def _fire_progress(self, taskprogress):
  225. if taskprogress == 0:
  226. self.start()
  227. return
  228. bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
  229. def finish(self):
  230. MultiStageProgressReporter.finish(self)
  231. bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
  232. class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
  233. """
  234. MultiStageProcessProgressReporter that takes the calls and does nothing
  235. with them (to avoid a bunch of "if progress_reporter:" checks)
  236. """
  237. def __init__(self):
  238. MultiStageProcessProgressReporter.__init__(self, "", None, [])
  239. def _fire_progress(self, taskprogress, rate=None):
  240. pass
  241. def start(self):
  242. pass
  243. def next_stage(self, stage_total=None):
  244. pass
  245. def update(self, stage_progress):
  246. pass
  247. def finish(self):
  248. pass