123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266 |
- """
- BitBake progress handling code
- """
- # Copyright (C) 2016 Intel Corporation
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- import sys
- import re
- import time
- import inspect
- import bb.event
- import bb.build
- class ProgressHandler(object):
- """
- Base class that can pretend to be a file object well enough to be
- used to build objects to intercept console output and determine the
- progress of some operation.
- """
- def __init__(self, d, outfile=None):
- self._progress = 0
- self._data = d
- self._lastevent = 0
- if outfile:
- self._outfile = outfile
- else:
- self._outfile = sys.stdout
- def _fire_progress(self, taskprogress, rate=None):
- """Internal function to fire the progress event"""
- bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
- def write(self, string):
- self._outfile.write(string)
- def flush(self):
- self._outfile.flush()
- def update(self, progress, rate=None):
- ts = time.time()
- if progress > 100:
- progress = 100
- if progress != self._progress or self._lastevent + 1 < ts:
- self._fire_progress(progress, rate)
- self._lastevent = ts
- self._progress = progress
- class LineFilterProgressHandler(ProgressHandler):
- """
- A ProgressHandler variant that provides the ability to filter out
- the lines if they contain progress information. Additionally, it
- filters out anything before the last line feed on a line. This can
- be used to keep the logs clean of output that we've only enabled for
- getting progress, assuming that that can be done on a per-line
- basis.
- """
- def __init__(self, d, outfile=None):
- self._linebuffer = ''
- super(LineFilterProgressHandler, self).__init__(d, outfile)
- def write(self, string):
- self._linebuffer += string
- while True:
- breakpos = self._linebuffer.find('\n') + 1
- if breakpos == 0:
- break
- line = self._linebuffer[:breakpos]
- self._linebuffer = self._linebuffer[breakpos:]
- # Drop any line feeds and anything that precedes them
- lbreakpos = line.rfind('\r') + 1
- if lbreakpos:
- line = line[lbreakpos:]
- if self.writeline(line):
- super(LineFilterProgressHandler, self).write(line)
- def writeline(self, line):
- return True
- class BasicProgressHandler(ProgressHandler):
- def __init__(self, d, regex=r'(\d+)%', outfile=None):
- super(BasicProgressHandler, self).__init__(d, outfile)
- self._regex = re.compile(regex)
- # Send an initial progress event so the bar gets shown
- self._fire_progress(0)
- def write(self, string):
- percs = self._regex.findall(string)
- if percs:
- progress = int(percs[-1])
- self.update(progress)
- super(BasicProgressHandler, self).write(string)
- class OutOfProgressHandler(ProgressHandler):
- def __init__(self, d, regex, outfile=None):
- super(OutOfProgressHandler, self).__init__(d, outfile)
- self._regex = re.compile(regex)
- # Send an initial progress event so the bar gets shown
- self._fire_progress(0)
- def write(self, string):
- nums = self._regex.findall(string)
- if nums:
- progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
- self.update(progress)
- super(OutOfProgressHandler, self).write(string)
- class MultiStageProgressReporter(object):
- """
- Class which allows reporting progress without the caller
- having to know where they are in the overall sequence. Useful
- for tasks made up of python code spread across multiple
- classes / functions - the progress reporter object can
- be passed around or stored at the object level and calls
- to next_stage() and update() made whereever needed.
- """
- def __init__(self, d, stage_weights, debug=False):
- """
- Initialise the progress reporter.
- Parameters:
- * d: the datastore (needed for firing the events)
- * stage_weights: a list of weight values, one for each stage.
- The value is scaled internally so you only need to specify
- values relative to other values in the list, so if there
- are two stages and the first takes 2s and the second takes
- 10s you would specify [2, 10] (or [1, 5], it doesn't matter).
- * debug: specify True (and ensure you call finish() at the end)
- in order to show a printout of the calculated stage weights
- based on timing each stage. Use this to determine what the
- weights should be when you're not sure.
- """
- self._data = d
- total = sum(stage_weights)
- self._stage_weights = [float(x)/total for x in stage_weights]
- self._stage = -1
- self._base_progress = 0
- # Send an initial progress event so the bar gets shown
- self._fire_progress(0)
- self._debug = debug
- self._finished = False
- if self._debug:
- self._last_time = time.time()
- self._stage_times = []
- self._stage_total = None
- self._callers = []
- def _fire_progress(self, taskprogress):
- bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
- def next_stage(self, stage_total=None):
- """
- Move to the next stage.
- Parameters:
- * stage_total: optional total for progress within the stage,
- see update() for details
- NOTE: you need to call this before the first stage.
- """
- self._stage += 1
- self._stage_total = stage_total
- if self._stage == 0:
- # First stage
- if self._debug:
- self._last_time = time.time()
- else:
- if self._stage < len(self._stage_weights):
- self._base_progress = sum(self._stage_weights[:self._stage]) * 100
- if self._debug:
- currtime = time.time()
- self._stage_times.append(currtime - self._last_time)
- self._last_time = currtime
- self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
- elif not self._debug:
- bb.warn('ProgressReporter: current stage beyond declared number of stages')
- self._base_progress = 100
- self._fire_progress(self._base_progress)
- def update(self, stage_progress):
- """
- Update progress within the current stage.
- Parameters:
- * stage_progress: progress value within the stage. If stage_total
- was specified when next_stage() was last called, then this
- value is considered to be out of stage_total, otherwise it should
- be a percentage value from 0 to 100.
- """
- if self._stage_total:
- stage_progress = (float(stage_progress) / self._stage_total) * 100
- if self._stage < 0:
- bb.warn('ProgressReporter: update called before first call to next_stage()')
- elif self._stage < len(self._stage_weights):
- progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
- else:
- progress = self._base_progress
- if progress > 100:
- progress = 100
- self._fire_progress(progress)
- def finish(self):
- if self._finished:
- return
- self._finished = True
- if self._debug:
- import math
- self._stage_times.append(time.time() - self._last_time)
- mintime = max(min(self._stage_times), 0.01)
- self._callers.append(None)
- stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
- bb.warn('Stage weights: %s' % stage_weights)
- out = []
- for stage_weight, caller in zip(stage_weights, self._callers):
- if caller:
- out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
- else:
- out.append('Up to finish: %d' % stage_weight)
- bb.warn('Stage times:\n %s' % '\n '.join(out))
- class MultiStageProcessProgressReporter(MultiStageProgressReporter):
- """
- Version of MultiStageProgressReporter intended for use with
- standalone processes (such as preparing the runqueue)
- """
- def __init__(self, d, processname, stage_weights, debug=False):
- self._processname = processname
- self._started = False
- MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
- def start(self):
- if not self._started:
- bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
- self._started = True
- def _fire_progress(self, taskprogress):
- if taskprogress == 0:
- self.start()
- return
- bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
- def finish(self):
- MultiStageProgressReporter.finish(self)
- bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
- class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
- """
- MultiStageProcessProgressReporter that takes the calls and does nothing
- with them (to avoid a bunch of "if progress_reporter:" checks)
- """
- def __init__(self):
- MultiStageProcessProgressReporter.__init__(self, "", None, [])
- def _fire_progress(self, taskprogress, rate=None):
- pass
- def start(self):
- pass
- def next_stage(self, stage_total=None):
- pass
- def update(self, stage_progress):
- pass
- def finish(self):
- pass
|