Support progress reporting

index 9605c8e..ddd36d5 100644
@@ -287,6 +287,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         bb.msg.addDefaultlogFilter(consolelog)
         consolelog.setFormatter(conlogformat)
         logger.addHandler(consolelog)
+        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
+        bb.utils.remove(loglink)
+        try:
+           os.symlink(os.path.basename(consolelogfile), loglink)
+        except OSError:
+           pass
 
     llevel, debug_domains = bb.msg.constructLogOptions()
     server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])

index ddd36d5..6a6f688 100644
@@ -51,7 +51,7 @@ class BBProgress(progressbar.ProgressBar):
             self._resize_default = None
         progressbar.ProgressBar.__init__(self, maxval, [self.msg + ": "] + widgets, fd=sys.stdout)
 
-    def _handle_resize(self, signum, frame):
+    def _handle_resize(self, signum=None, frame=None):
         progressbar.ProgressBar._handle_resize(self, signum, frame)
         if self._resize_default:
             self._resize_default(signum, frame)

384 lines not on this page · raw message

52 lines not on this page · raw message

49 lines not on this page · raw message

44 lines not on this page · raw message

307 lines not on this page · raw message

355 lines not on this page · raw message

index e016ae3..04bd580 100644
@@ -35,6 +35,7 @@ import stat
 import bb
 import bb.msg
 import bb.process
+import bb.progress
 from bb import data, event, utils
 
 bblogger = logging.getLogger('BitBake')

@@ -137,6 +138,25 @@ class TaskInvalid(TaskBase):
         super(TaskInvalid, self).__init__(task, None, metadata)
         self._message = "No such task '%s'" % task
 
+class TaskProgress(event.Event):
+    """
+    Task made some progress that could be reported to the user, usually in
+    the form of a progress bar or similar.
+    NOTE: this class does not inherit from TaskBase since it doesn't need
+    to - it's fired within the task context itself, so we don't have any of
+    the context information that you do in the case of the other events.
+    The event PID can be used to determine which task it came from.
+    The progress value is normally 0-100, but can also be negative
+    indicating that progress has been made but we aren't able to determine
+    how much.
+    The rate is optional, this is simply an extra string to display to the
+    user if specified.
+    """
+    def __init__(self, progress, rate=None):
+        self.progress = progress
+        self.rate = rate
+        event.Event.__init__(self)
+
 
 class LogTee(object):
     def __init__(self, logger, outfile):

@@ -340,6 +360,20 @@ exit $ret
     else:
         logfile = sys.stdout
 
+    progress = d.getVarFlag(func, 'progress', True)
+    if progress:
+        if progress == 'percent':
+            # Use default regex
+            logfile = bb.progress.BasicProgressHandler(d, outfile=logfile)
+        elif progress.startswith('percent:'):
+            # Use specified regex
+            logfile = bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+        elif progress.startswith('outof:'):
+            # Use specified regex
+            logfile = bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+        else:
+            bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+
     def readfifo(data):
         lines = data.split(b'\0')
         for line in lines:

86 lines not on this page · raw message

123 lines not on this page · raw message

index db70b76..1915e47 100644
@@ -18,6 +18,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import bb.build
+import time
 
 class BBUIHelper:
     def __init__(self):

@@ -31,7 +32,7 @@ class BBUIHelper:
 
     def eventHandler(self, event):
         if isinstance(event, bb.build.TaskStarted):
-            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task) }
+            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
             self.running_pids.append(event.pid)
             self.needUpdate = True
         if isinstance(event, bb.build.TaskSucceeded):

@@ -52,6 +53,10 @@ class BBUIHelper:
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True
+        if isinstance(event, bb.build.TaskProgress):
+            self.running_tasks[event.pid]['progress'] = event.progress
+            self.running_tasks[event.pid]['rate'] = event.rate
+            self.needUpdate = True
 
     def getTasks(self):
         self.needUpdate = False

index 0b9dcf7..2873ad6 100644
@@ -3,6 +3,8 @@
 # progressbar  - Text progress bar library for Python.
 # Copyright (c) 2005 Nilton Volpato
 #
+# (With some small changes after importing into BitBake)
+#
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either

@@ -261,12 +263,14 @@ class ProgressBar(object):
         now = time.time()
         self.seconds_elapsed = now - self.start_time
         self.next_update = self.currval + self.update_interval
-        self.fd.write(self._format_line() + '\r')
+        output = self._format_line()
+        self.fd.write(output + '\r')
         self.fd.flush()
         self.last_update_time = now
+        return output
 
 
-    def start(self):
+    def start(self, update=True):
         """Starts measuring time, and prints the bar at 0%.
 
         It returns self so you can use it like this:

@@ -289,8 +293,12 @@ class ProgressBar(object):
             self.update_interval = self.maxval / self.num_intervals
 
 
-        self.start_time = self.last_update_time = time.time()
-        self.update(0)
+        self.start_time = time.time()
+        if update:
+            self.last_update_time = self.start_time
+            self.update(0)
+        else:
+            self.last_update_time = 0
 
         return self
 

index 6434ad5..77285ca 100644
@@ -353,3 +353,39 @@ class BouncingBar(Bar):
         if not self.fill_left: rpad, lpad = lpad, rpad
 
         return '%s%s%s%s%s' % (left, lpad, marker, rpad, right)
+
+
+class BouncingSlider(Bar):
+    """
+    A slider that bounces back and forth in response to update() calls
+    without reference to the actual value. Based on a combination of
+    BouncingBar from a newer version of this module and RotatingMarker.
+    """
+    def __init__(self, marker='<=>'):
+        self.curmark = -1
+        self.forward = True
+        Bar.__init__(self, marker=marker)
+    def update(self, pbar, width):
+        left, marker, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+        if width < 0:
+            return ''
+
+        if pbar.finished: return '%s%s%s' % (left, width * '=', right)
+
+        self.curmark = self.curmark + 1
+        position = int(self.curmark % (width * 2 - 1))
+        if position + len(marker) > width:
+            self.forward = not self.forward
+            self.curmark = 1
+            position = 1
+        lpad = ' ' * (position - 1)
+        rpad = ' ' * (width - len(marker) - len(lpad))
+
+        if not self.forward:
+            temp = lpad
+            lpad = rpad
+            rpad = temp
+        return '%s%s%s%s%s' % (left, lpad, marker, rpad, right)

index bab8e94..93e42df 100644
@@ -20,6 +20,7 @@ BitBake progress handling code
 import sys
 import re
 import time
+import inspect
 import bb.event
 import bb.build
 

@@ -84,3 +85,113 @@ class OutOfProgressHandler(ProgressHandler):
             progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
             self.update(progress)
         super(OutOfProgressHandler, self).write(string)
+
+class MultiStageProgressReporter(object):
+    """
+    Class which allows reporting progress without the caller
+    having to know where they are in the overall sequence. Useful
+    for tasks made up of python code spread across multiple
+    classes / functions - the progress reporter object can
+    be passed around or stored at the object level and calls
+    to next_stage() and update() made whereever needed.
+    """
+    def __init__(self, d, stage_weights, debug=False):
+        """
+        Initialise the progress reporter.
+
+        Parameters:
+        * d: the datastore (needed for firing the events)
+        * stage_weights: a list of weight values, one for each stage.
+          The value is scaled internally so you only need to specify
+          values relative to other values in the list, so if there
+          are two stages and the first takes 2s and the second takes
+          10s you would specify [2, 10] (or [1, 5], it doesn't matter).
+        * debug: specify True (and ensure you call finish() at the end)
+          in order to show a printout of the calculated stage weights
+          based on timing each stage. Use this to determine what the
+          weights should be when you're not sure.
+        """
+        self._data = d
+        total = sum(stage_weights)
+        self._stage_weights = [float(x)/total for x in stage_weights]
+        self._stage = -1
+        self._base_progress = 0
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+        self._debug = debug
+        self._finished = False
+        if self._debug:
+            self._last_time = time.time()
+            self._stage_times = []
+            self._stage_total = None
+            self._callers = []
+
+    def _fire_progress(self, taskprogress):
+        bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
+
+    def next_stage(self, stage_total=None):
+        """
+        Move to the next stage.
+        Parameters:
+        * stage_total: optional total for progress within the stage,
+          see update() for details
+        NOTE: you need to call this before the first stage.
+        """
+        self._stage += 1
+        self._stage_total = stage_total
+        if self._stage == 0:
+            # First stage
+            if self._debug:
+                self._last_time = time.time()
+        else:
+            if self._stage < len(self._stage_weights):
+                self._base_progress = sum(self._stage_weights[:self._stage]) * 100
+                if self._debug:
+                    currtime = time.time()
+                    self._stage_times.append(currtime - self._last_time)
+                    self._last_time = currtime
+                    self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
+            elif not self._debug:
+                bb.warn('ProgressReporter: current stage beyond declared number of stages')
+                self._base_progress = 100
+            self._fire_progress(self._base_progress)
+
+    def update(self, stage_progress):
+        """
+        Update progress within the current stage.
+        Parameters:
+        * stage_progress: progress value within the stage. If stage_total
+          was specified when next_stage() was last called, then this
+          value is considered to be out of stage_total, otherwise it should
+          be a percentage value from 0 to 100.
+        """
+        if self._stage_total:
+            stage_progress = (float(stage_progress) / self._stage_total) * 100
+        if self._stage < 0:
+            bb.warn('ProgressReporter: update called before first call to next_stage()')
+        elif self._stage < len(self._stage_weights):
+            progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
+        else:
+            progress = self._base_progress
+        if progress > 100:
+            progress = 100
+        self._fire_progress(progress)
+
+    def finish(self):
+        if self._finished:
+            return
+        self._finished = True
+        if self._debug:
+            import math
+            self._stage_times.append(time.time() - self._last_time)
+            mintime = max(min(self._stage_times), 0.01)
+            self._callers.append(None)
+            stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
+            bb.warn('Stage weights: %s' % stage_weights)
+            out = []
+            for stage_weight, caller in zip(stage_weights, self._callers):
+                if caller:
+                    out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
+                else:
+                    out.append('Up to finish: %d' % stage_weight)
+            bb.warn('Stage times:\n  %s' % '\n  '.join(out))

index b6fcaaa..a27512c 100644
@@ -779,7 +779,7 @@ def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
-def runfetchcmd(cmd, d, quiet=False, cleanup=None):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails

@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
     error_message = ""
 
     try:
-        (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE)
+        (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
         success = True
     except bb.process.NotFoundError as e:
         error_message = "Fetch command %s" % (e.command)

79 lines not on this page · raw message

index d688fd9..3a381b7 100644
@@ -31,6 +31,7 @@ import subprocess
 import os
 import logging
 import bb
+import bb.progress
 import urllib.request, urllib.parse, urllib.error
 from   bb import data
 from   bb.fetch2 import FetchMethod

@@ -41,6 +42,27 @@ from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
+class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from wget output.
+    Note: relies on --progress=dot --show-progress being specified on the
+    wget command line.
+    """
+    def __init__(self, d):
+        super(WgetProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
+        if percs:
+            progress = int(percs[-1][0])
+            rate = percs[-1][1] + '/s'
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):

@@ -70,9 +92,11 @@ class Wget(FetchMethod):
 
     def _runwget(self, ud, d, command, quiet):
 
+        progresshandler = WgetProgressHandler(d)
+
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command)
-        runfetchcmd(command, d, quiet)
+        runfetchcmd(command + ' --progress=dot --show-progress', d, quiet, log=progresshandler)
 
     def download(self, ud, d):
         """Fetch urls"""

index 93e42df..1365068 100644
@@ -58,6 +58,37 @@ class ProgressHandler(object):
             self._lastevent = ts
             self._progress = progress
 
+class LineFilterProgressHandler(ProgressHandler):
+    """
+    A ProgressHandler variant that provides the ability to filter out
+    the lines if they contain progress information. Additionally, it
+    filters out anything before the last line feed on a line. This can
+    be used to keep the logs clean of output that we've only enabled for
+    getting progress, assuming that that can be done on a per-line
+    basis.
+    """
+    def __init__(self, d, outfile=None):
+        self._linebuffer = ''
+        super(LineFilterProgressHandler, self).__init__(d, outfile)
+
+    def write(self, string):
+        self._linebuffer += string
+        while True:
+            breakpos = self._linebuffer.find('\n') + 1
+            if breakpos == 0:
+                break
+            line = self._linebuffer[:breakpos]
+            self._linebuffer = self._linebuffer[breakpos:]
+            # Drop any line feeds and anything that precedes them
+            lbreakpos = line.rfind('\r') + 1
+            if lbreakpos:
+                line = line[lbreakpos:]
+            if self.writeline(line):
+                super(LineFilterProgressHandler, self).write(line)
+
+    def writeline(self, line):
+        return True
+
 class BasicProgressHandler(ProgressHandler):
     def __init__(self, d, regex=r'(\d+)%', outfile=None):
         super(BasicProgressHandler, self).__init__(d, outfile)

index 9b4a4f9..a5f026e 100644
@@ -647,6 +647,33 @@ class MetadataEvent(Event):
         self.type = eventtype
         self._localdata = eventdata
 
+class ProcessStarted(Event):
+    """
+    Generic process started event (usually part of the initial startup)
+    where further progress events will be delivered
+    """
+    def __init__(self, processname, total):
+        Event.__init__(self)
+        self.processname = processname
+        self.total = total
+
+class ProcessProgress(Event):
+    """
+    Generic process progress event (usually part of the initial startup)
+    """
+    def __init__(self, processname, progress):
+        Event.__init__(self)
+        self.processname = processname
+        self.progress = progress
+
+class ProcessFinished(Event):
+    """
+    Generic process finished event (usually part of the initial startup)
+    """
+    def __init__(self, processname):
+        Event.__init__(self)
+        self.processname = processname
+
 class SanityCheck(Event):
     """
     Event to run sanity checks, either raise errors or generate events as return status.

index 2513501..6fdaafe 100644
@@ -90,7 +90,7 @@ class NonInteractiveProgress(object):
         self.msg = msg
         self.maxval = maxval
 
-    def start(self):
+    def start(self, update=True):
         self.fobj.write("%s..." % self.msg)
         self.fobj.flush()
         return self

@@ -304,7 +304,7 @@ _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.Lo
               "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted",
               "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed",
               "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent",
-              "bb.build.TaskProgress"]
+              "bb.build.TaskProgress", "bb.event.ProcessStarted", "bb.event.ProcessProgress", "bb.event.ProcessFinished"]
 
 def main(server, eventHandler, params, tf = TerminalFilter):
 

@@ -579,6 +579,17 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, bb.event.DepTreeGenerated):
                 continue
 
+            if isinstance(event, bb.event.ProcessStarted):
+                parseprogress = new_progress(event.processname, event.total)
+                parseprogress.start(False)
+                continue
+            if isinstance(event, bb.event.ProcessProgress):
+                parseprogress.update(event.progress)
+                continue
+            if isinstance(event, bb.event.ProcessFinished):
+                parseprogress.finish()
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.MetadataEvent,

index 6fdaafe..c245c22 100644
@@ -89,6 +89,7 @@ class NonInteractiveProgress(object):
     def __init__(self, msg, maxval):
         self.msg = msg
         self.maxval = maxval
+        self.finished = False
 
     def start(self, update=True):
         self.fobj.write("%s..." % self.msg)

@@ -99,8 +100,11 @@ class NonInteractiveProgress(object):
         pass
 
     def finish(self):
+        if self.finished:
+            return
         self.fobj.write("done.\n")
         self.fobj.flush()
+        self.finished = True
 
 def new_progress(msg, maxval):
     if interactive:

@@ -204,6 +208,8 @@ class TerminalFilter(object):
         console.addFilter(InteractConsoleLogFilter(self, format))
         errconsole.addFilter(InteractConsoleLogFilter(self, format))
 
+        self.main_progress = None
+
     def clearFooter(self):
         if self.footer_present:
             lines = self.footer_present

@@ -246,11 +252,20 @@ class TerminalFilter(object):
 
         if self.main.shutdown:
             content = "Waiting for %s running tasks to finish:" % len(activetasks)
-        elif not len(activetasks):
-            content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+            print(content)
         else:
-            content = "Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
-        print(content)
+            if not len(activetasks):
+                content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+            else:
+                content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
+            maxtask = self.helper.tasknumber_total + 1
+            if not self.main_progress or self.main_progress.maxval != maxtask:
+                widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
+                self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets)
+                self.main_progress.start(False)
+            self.main_progress.setmessage(content)
+            self.main_progress.update(self.helper.tasknumber_current)
+            print('')
         lines = 1 + int(len(content) / (self.columns + 1))
         for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
             if isinstance(task, tuple):

index 283f29b..3fc3ff5 100755
@@ -179,6 +179,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
                           help="Increase the debug level. You can specify this more than once.")
 
+        parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False,
+                          help="Output less log message data to the terminal.")
+
         parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
                           help="Don't execute, just go through the motions.")
 

@@ -279,6 +282,12 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         options, targets = parser.parse_args(argv)
 
+        if options.quiet and options.verbose:
+            parser.error("options --quiet and --verbose are mutually exclusive")
+
+        if options.quiet and options.debug:
+            parser.error("options --quiet and --debug are mutually exclusive")
+
         # use configuration files from environment variables
         if "BBPRECONF" in os.environ:
             options.prefile.append(os.environ["BBPRECONF"])

index 8c3ab47..b7c39fa 100644
@@ -182,9 +182,12 @@ def constructLogOptions():
         debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
     return level, debug_domains
 
-def addDefaultlogFilter(handler, cls = BBLogFilter):
+def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
     level, debug_domains = constructLogOptions()
 
+    if forcelevel is not None:
+        level = forcelevel
+
     cls(handler, level, debug_domains)
 
 #

106 lines not on this page · raw message

index c9b6b84..b62a28a 100644
@@ -240,6 +240,7 @@ class RunQueueData:
 
         self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
         self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+        self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
 
         self.reset()
 

@@ -1596,8 +1597,51 @@ class RunQueueExecuteTasks(RunQueueExecute):
         Run the tasks in a queue prepared by rqdata.prepare()
         """
 
-        self.rq.read_workers()
+        if self.rqdata.setscenewhitelist:
+            # Check tasks that are going to run against the whitelist
+            def check_norun_task(tid, showerror=False):
+                fn = fn_from_tid(tid)
+                taskname = taskname_from_tid(tid)
+                # Ignore covered tasks
+                if tid in self.rq.scenequeue_covered:
+                    return False
+                # Ignore stamped tasks
+                if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
+                    return False
+                # Ignore noexec tasks
+                taskdep = self.rqdata.dataCache.task_deps[fn]
+                if 'noexec' in taskdep and taskname in taskdep['noexec']:
+                    return False
 
+                pn = self.rqdata.dataCache.pkg_fn[fn]
+                if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+                    if showerror:
+                        if tid in self.rqdata.runq_setscene_tids:
+                            logger.error('Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname))
+                        else:
+                            logger.error('Task %s.%s attempted to execute unexpectedly' % (pn, taskname))
+                    return True
+                return False
+            # Look to see if any tasks that we think shouldn't run are going to
+            unexpected = False
+            for tid in self.rqdata.runtaskentries:
+                if check_norun_task(tid):
+                    unexpected = True
+                    break
+            if unexpected:
+                # Run through the tasks in the rough order they'd have executed and print errors
+                # (since the order can be useful - usually missing sstate for the last few tasks
+                # is the cause of the problem)
+                task = self.sched.next()
+                while task is not None:
+                    check_norun_task(task, showerror=True)
+                    self.task_skip(task, 'Setscene enforcement check')
+                    task = self.sched.next()
+
+                self.rq.state = runQueueCleanUp
+                return True
+
+        self.rq.read_workers()
 
         if self.stats.total == 0:
             # nothing to do

@@ -1940,6 +1984,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.scenequeue_covered.add(task)
         self.scenequeue_updatecounters(task)
 
+    def check_taskfail(self, task):
+        if self.rqdata.setscenewhitelist:
+            realtask = task.split('_setscene')[0]
+            fn = fn_from_tid(realtask)
+            taskname = taskname_from_tid(realtask)
+            pn = self.rqdata.dataCache.pkg_fn[fn]
+            if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+                logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
+                self.rq.state = runQueueCleanUp
+
     def task_complete(self, task):
         self.stats.taskCompleted()
         bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)

@@ -1950,6 +2004,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
+        self.check_taskfail(task)
 
     def task_failoutright(self, task):
         self.runq_running.add(task)

@@ -2231,3 +2286,27 @@ class runQueuePipe():
         if len(self.queue) > 0:
             print("Warning, worker left partial message: %s" % self.queue)
         self.input.close()
+
+def get_setscene_enforce_whitelist(d):
+    if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
+        return None
+    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
+    outlist = []
+    for item in whitelist[:]:
+        if item.startswith('%:'):
+            for target in sys.argv[1:]:
+                if not target.startswith('-'):
+                    outlist.append(target.split(':')[0] + ':' + item.split(':')[1])
+        else:
+            outlist.append(item)
+    return outlist
+
+def check_setscene_enforce_whitelist(pn, taskname, whitelist):
+    import fnmatch
+    if whitelist:
+        item = '%s:%s' % (pn, taskname)
+        for whitelist_item in whitelist:
+            if fnmatch.fnmatch(item, whitelist_item):
+                return True
+        return False
+    return True

index 1365068..f8fa692 100644
@@ -226,3 +226,45 @@ class MultiStageProgressReporter(object):
                 else:
                     out.append('Up to finish: %d' % stage_weight)
             bb.warn('Stage times:\n  %s' % '\n  '.join(out))
+
+class MultiStageProcessProgressReporter(MultiStageProgressReporter):
+    """
+    Version of MultiStageProgressReporter intended for use with
+    standalone processes (such as preparing the runqueue)
+    """
+    def __init__(self, d, processname, stage_weights, debug=False):
+        self._processname = processname
+        MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
+
+    def start(self):
+        bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
+
+    def _fire_progress(self, taskprogress):
+        bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
+
+    def finish(self):
+        MultiStageProgressReporter.finish(self)
+        bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
+
+class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
+    """
+    MultiStageProcessProgressReporter that takes the calls and does nothing
+    with them (to avoid a bunch of "if progress_reporter:" checks)
+    """
+    def __init__(self):
+        MultiStageProcessProgressReporter.__init__(self, "", None, [])
+
+    def _fire_progress(self, taskprogress, rate=None):
+        pass
+
+    def start(self):
+        pass
+
+    def next_stage(self, stage_total=None):
+        pass
+
+    def update(self, stage_progress):
+        pass
+
+    def finish(self):
+        pass

249 lines not on this page · raw message

index b6fcaaa..a27512c 100644
@@ -779,7 +779,7 @@ def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
-def runfetchcmd(cmd, d, quiet=False, cleanup=None):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails

@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
     error_message = ""
 
     try:
-        (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE)
+        (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
         success = True
     except bb.process.NotFoundError as e:
         error_message = "Fetch command %s" % (e.command)

79 lines not on this page · raw message

index d688fd9..275cd7d 100644
@@ -31,6 +31,7 @@ import subprocess
 import os
 import logging
 import bb
+import bb.progress
 import urllib.request, urllib.parse, urllib.error
 from   bb import data
 from   bb.fetch2 import FetchMethod

@@ -41,6 +42,27 @@ from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
+class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from wget output.
+    Note: relies on --progress=dot (with -v or without -q/-nv) being
+    specified on the wget command line.
+    """
+    def __init__(self, d):
+        super(WgetProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
+        if percs:
+            progress = int(percs[-1][0])
+            rate = percs[-1][1] + '/s'
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):

@@ -66,13 +88,15 @@ class Wget(FetchMethod):
         if not ud.localfile:
             ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet):
 
+        progresshandler = WgetProgressHandler(d)
+
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command)
-        runfetchcmd(command, d, quiet)
+        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
 
     def download(self, ud, d):
         """Fetch urls"""

index 93e42df..1365068 100644
@@ -58,6 +58,37 @@ class ProgressHandler(object):
             self._lastevent = ts
             self._progress = progress
 
+class LineFilterProgressHandler(ProgressHandler):
+    """
+    A ProgressHandler variant that provides the ability to filter out
+    the lines if they contain progress information. Additionally, it
+    filters out anything before the last line feed on a line. This can
+    be used to keep the logs clean of output that we've only enabled for
+    getting progress, assuming that that can be done on a per-line
+    basis.
+    """
+    def __init__(self, d, outfile=None):
+        self._linebuffer = ''
+        super(LineFilterProgressHandler, self).__init__(d, outfile)
+
+    def write(self, string):
+        self._linebuffer += string
+        while True:
+            breakpos = self._linebuffer.find('\n') + 1
+            if breakpos == 0:
+                break
+            line = self._linebuffer[:breakpos]
+            self._linebuffer = self._linebuffer[breakpos:]
+            # Drop any line feeds and anything that precedes them
+            lbreakpos = line.rfind('\r') + 1
+            if lbreakpos:
+                line = line[lbreakpos:]
+            if self.writeline(line):
+                super(LineFilterProgressHandler, self).write(line)
+
+    def writeline(self, line):
+        return True
+
 class BasicProgressHandler(ProgressHandler):
     def __init__(self, d, regex=r'(\d+)%', outfile=None):
         super(BasicProgressHandler, self).__init__(d, outfile)