per project build directory

index d409554..6a85ae2 100755
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see http://www.gnu.org/licenses/.
 
-# Usage: source toaster [start|stop|restart-bitbake]
+# Usage: source toaster [start|stop]
 #                       [webport=<port>] [noui] [noweb]
 
 # Helper function to kill a background toaster development server

@@ -95,7 +95,6 @@ stop_system()
         kill `cat ${BUILDDIR}/.toasterui.pid` 2>/dev/null
         rm ${BUILDDIR}/.toasterui.pid
     fi
-    stop_bitbake
     webserverKillAll
     # unset exported variables
     unset DATABASE_URL

@@ -106,31 +105,6 @@ stop_system()
     INSTOPSYSTEM=0
 }
 
-start_bitbake() {
-    unset BBSERVER
-    bitbake --read conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0
-    if [ $? -ne 0 ]; then
-        echo "Bitbake server start failed"
-        return 1
-    fi
-    export BBSERVER=0.0.0.0:-1
-    export DATABASE_URL=`$MANAGE get-dburl`
-    # we start the TOASTERUI only if not inhibited
-    if [ $NOTOASTERUI -eq 0 ]; then
-        bitbake --observe-only -u toasterui --remote-server=$BBSERVER -t xmlrpc \
-            >>${BUILDDIR}/toaster_ui.log 2>&1 \
-            & echo $! >${BUILDDIR}/.toasterui.pid
-    fi
-    return 0
-}
-
-stop_bitbake() {
-    BBSERVER=0.0.0.0:-1 bitbake -m
-    unset BBSERVER
-    # force stop any misbehaving bitbake server
-    lsof -t bitbake.lock | awk '{print $2}' | grep "[0-9]\+" | xargs -n1 -r kill
-}
-
 verify_prereq() {
     # Verify Django version
     reqfile=$(python -c "import os; print os.path.realpath('$BBBASEDIR/toaster-requirements.txt')")

@@ -229,15 +203,6 @@ if [ `basename \"$0\"` = `basename \"${TOASTER}\"` ]; then
     return 1
 fi
 
-if [ "$1" = 'restart-bitbake' ] ; then
-    stop_bitbake
-    sleep 1
-    start_bitbake
-    rc=$?
-    sleep 3
-    return $rc
-fi
-
 verify_prereq || return 1
 
 # We make sure we're running in the current shell and in a good environment

@@ -281,16 +246,6 @@ case $CMD in
              $MANAGE checksocket "0.0.0.0:$WEB_PORT" || return 1
         fi
 
-        # Make sure it's safe to start by checking bitbake lock
-        if [ -e $BUILDDIR/bitbake.lock ]; then
-            python -c "import fcntl; fcntl.flock(open(\"$BUILDDIR/bitbake.lock\"), fcntl.LOCK_EX|fcntl.LOCK_NB)" 2>/dev/null
-            if [ $? -ne 0 ] ; then
-                echo "Error: bitbake lock state error. File locks show that the system is on." 1>&2
-                echo "Please wait for the current build to finish, stop and then start the system again." 1>&2
-                return 3
-            fi
-        fi
-
         # kill Toaster web server if it's alive
         if [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
             echo "Warning: bitbake appears to be dead, but the Toaster web server is running." 1>&2

@@ -308,22 +263,12 @@ case $CMD in
             echo "Failed ${CMD}."
             return 4
         fi
-        start_bitbake
-        if [ $? -eq 0 ]; then
-            $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
-            # set fail safe stop system on terminal exit
-            trap stop_system SIGHUP
-            echo "Successful ${CMD}."
-            return 0
-        else
-            # failed start, do stop
-            stop_system
-            echo "Failed ${CMD}."
-            return 1
-        fi
-        # stop system on terminal exit
-        set -o monitor
+        export DATABASE_URL=`$MANAGE get-dburl`
+        $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
+        # set fail safe stop system on terminal exit
         trap stop_system SIGHUP
+        echo "Successful ${CMD}."
+        return 0
     ;;
     stop )
         stop_system

index 6a85ae2..1e6e690 100755
@@ -175,15 +175,11 @@ fi
 # make sure that the toaster.sqlite file doesn't default to `pwd` like it currently does.
 export TOASTER_DIR=`pwd`
 
-NOTOASTERUI=0
 WEBSERVER=1
 WEB_PORT="8000"
 unset CMD
 for param in $*; do
     case $param in
-    noui )
-            NOTOASTERUI=1
-    ;;
     noweb )
             WEBSERVER=0
     ;;

index 1e6e690..5cc77ed 100755
@@ -100,6 +100,7 @@ stop_system()
     unset DATABASE_URL
     unset TOASTER_CONF
     unset TOASTER_DIR
+    unset BITBAKE_UI
     trap - SIGHUP
     #trap - SIGCHLD
     INSTOPSYSTEM=0

@@ -259,6 +260,7 @@ case $CMD in
             echo "Failed ${CMD}."
             return 4
         fi
+        export BITBAKE_UI='toasterui'
         export DATABASE_URL=`$MANAGE get-dburl`
         $MANAGE runbuilds & echo $! >${BUILDDIR}/.runbuilds.pid
         # set fail safe stop system on terminal exit

index 7805086..81fd5de 100644
@@ -837,7 +837,7 @@ class BuildInfoHelper(object):
     # pylint: disable=bad-continuation
     # we do not follow the python conventions for continuation indentation due to long lines here
 
-    def __init__(self, server, has_build_history = False):
+    def __init__(self, server, has_build_history = False, brbe = None):
         self.internal_state = {}
         self.internal_state['taskdata'] = {}
         self.internal_state['targets'] = []

@@ -853,7 +853,7 @@ class BuildInfoHelper(object):
 
         # this is set for Toaster-triggered builds by localhostbecontroller
         # via toasterui
-        self.brbe = None
+        self.brbe = brbe
 
         self.project = None
 

index ee40110..d1f058f 100644
@@ -197,7 +197,8 @@ def main(server, eventHandler, params):
     taskfailures = []
     first = True
 
-    buildinfohelper = BuildInfoHelper(server, build_history_enabled)
+    buildinfohelper = BuildInfoHelper(server, build_history_enabled,
+                                      os.getenv('TOASTER_BRBE'))
 
     # write our own log files into bitbake's log directory;
     # we're only interested in the path to the parent directory of

index 6b479bf..df093c5 100644
@@ -24,7 +24,7 @@ server and queue them for the UI to process. This process must be used to avoid
 client/server deadlocks.
 """
 
-import socket, threading, pickle
+import socket, threading, pickle, collections
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 
 class BBUIEventQueue:

@@ -51,7 +51,13 @@ class BBUIEventQueue:
         # giving up
 
         for count_tries in range(5):
-            self.EventHandle, error = self.BBServer.registerEventHandler(self.host, self.port)
+            ret = self.BBServer.registerEventHandler(self.host, self.port)
+
+            if isinstance(ret, collections.Iterable):
+                self.EventHandle, error = ret
+            else:
+                self.EventHandle = ret
+                error = ""
 
             if self.EventHandle != None:
                 break

index 81fd5de..0e5f2cf 100644
@@ -1230,7 +1230,6 @@ class BuildInfoHelper(object):
         assert 'layer-priorities' in event._depgraph
         assert 'pn' in event._depgraph
         assert 'tdepends' in event._depgraph
-        assert 'providermap' in event._depgraph
 
         errormsg = ""
 

@@ -1313,7 +1312,7 @@ class BuildInfoHelper(object):
                 if dep in assume_provided:
                     continue
                 via = None
-                if dep in event._depgraph['providermap']:
+                if 'providermap' in event._depgraph and dep in event._depgraph['providermap']:
                     deprecipe = event._depgraph['providermap'][dep][0]
                     dependency = self.internal_state['recipes'][deprecipe]
                     via = Provides.objects.get_or_create(name=dep,

index d1f058f..b4d0a39 100644
@@ -374,9 +374,9 @@ def main(server, eventHandler, params):
 
                 # update the build info helper on BuildCompleted, not on CommandXXX
                 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
+
+                brbe = buildinfohelper.brbe
                 buildinfohelper.close(errorcode)
-                # mark the log output; controllers may kill the toasterUI after seeing this log
-                logger.info("ToasterUI build done 1, brbe: %s", buildinfohelper.brbe )
 
                 # we start a new build info
                 logger.debug("ToasterUI prepared for new build")

@@ -385,7 +385,7 @@ def main(server, eventHandler, params):
                 taskfailures = []
                 buildinfohelper = BuildInfoHelper(server, build_history_enabled)
 
-                logger.info("ToasterUI build done 2")
+                logger.info("ToasterUI build done, brbe: %s", brbe)
                 continue
 
             if isinstance(event, (bb.command.CommandCompleted,

index f40103c..535a398 100644
@@ -127,14 +127,6 @@ class BuildEnvironmentController(object):
         bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
         bblayerconffile.close()
 
-    def startBBServer(self):
-        """ Starts a  BB server with Toaster toasterui set up to record the builds, an no controlling UI.
-            After this method executes, self.be bbaddress/bbport MUST point to a running and free server,
-            and the bbstate MUST be  updated to "started".
-        """
-        raise NotImplementedError("FIXME: Must override in order to actually start the BB server")
-
-
     def setLayers(self, bitbake, ls):
         """ Checks-out bitbake executor and layers from git repositories.
             Sets the layer variables in the config file, after validating local layer paths.

@@ -151,7 +143,6 @@ class BuildEnvironmentController(object):
             starts if needed; or reconnects to the server if we can
         """
         if not self.connection:
-            self.startBBServer()
             self.be.lock = BuildEnvironment.LOCK_RUNNING
             self.be.save()
 

index efd82c3..9027291 100644
@@ -70,38 +70,6 @@ class LocalhostBEController(BuildEnvironmentController):
             logger.debug("localhostbecontroller: shellcmd success")
             return out
 
-    def startBBServer(self):
-        assert self.pokydirname and os.path.exists(self.pokydirname)
-        assert self.islayerset
-
-        # find our own toasterui listener/bitbake
-        from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
-
-        toaster = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/toaster"))
-        assert os.path.exists(toaster) and os.path.isfile(toaster)
-
-        # restart bitbake server and toastergui observer
-        self._shellcmd("bash -c 'source %s restart-bitbake'" % toaster, self.be.builddir)
-        logger.debug("localhostbecontroller: restarted bitbake server")
-
-        # read port number from bitbake.lock
-        self.be.bbport = ""
-        bblock = os.path.join(self.be.builddir, 'bitbake.lock')
-        if os.path.exists(bblock):
-            with open(bblock) as fplock:
-                for line in fplock:
-                    if ":" in line:
-                        self.be.bbport = line.split(":")[-1].strip()
-                        logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
-                        break
-
-        if not self.be.bbport:
-            raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)
-
-        self.be.bbaddress = "localhost"
-        self.be.bbstate = BuildEnvironment.SERVER_STARTED
-        self.be.save()
-
     def getGitCloneDirectory(self, url, branch):
         """Construct unique clone directory name out of url and branch."""
         if branch != "HEAD":

index 535a398..2ca2661 100644
@@ -163,12 +163,6 @@ class BuildEnvironmentController(object):
         """
         raise NotImplementedError("Must return the REST URL of the artifact")
 
-    def release(self):
-        """ This stops the server and releases any resources. After this point, all resources
-            are un-available for further reference
-        """
-        raise NotImplementedError("Must override BE release")
-
     def triggerBuild(self, bitbake, layers, variables, targets):
         raise NotImplementedError("Must override BE release")
 

index 9027291..007f3be 100644
@@ -255,12 +255,6 @@ class LocalhostBEController(BuildEnvironmentController):
     def readServerLogFile(self):
         return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
 
-    def release(self):
-        assert self.be.sourcedir and os.path.exists(self.be.builddir)
-        import shutil
-        shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
-        assert not os.path.exists(self.be.builddir)
-
 
     def triggerBuild(self, bitbake, layers, variables, targets):
         # set up the build environment with the needed layers

index 007f3be..6f2ca33 100644
@@ -256,7 +256,7 @@ class LocalhostBEController(BuildEnvironmentController):
         return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
 
 
-    def triggerBuild(self, bitbake, layers, variables, targets):
+    def triggerBuild(self, bitbake, layers, variables, targets, brbe):
         # set up the build environment with the needed layers
         self.setLayers(bitbake, layers, targets)
 

index 0bd5d08..8ba836e 100644
@@ -49,15 +49,13 @@ class Command(NoArgsCommand):
 
             logger.debug("runbuilds: starting build %s, environment %s" % (str(br).decode('utf-8'), bec.be))
 
-            # write the build identification variable
-            BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk))
-
             # let the build request know where it is being executed
             br.environment = bec.be
             br.save()
 
             # this triggers an async build
-            bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all())
+            bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), br.brvariable_set.all(),
+                             br.brtarget_set.all(), "%d:%d" % (br.pk, bec.be.pk))
 
         except Exception as e:
             logger.error("runbuilds: Error launching build %s" % e)

index 2ca2661..058e490 100644
@@ -109,24 +109,6 @@ class BuildEnvironmentController(object):
         self.be = be
         self.connection = None
 
-    @staticmethod
-    def _updateBBLayers(bblayerconf, layerlist):
-        conflines = open(bblayerconf, "r").readlines()
-
-        bblayerconffile = open(bblayerconf, "w")
-        skip = 0
-        for i in xrange(len(conflines)):
-            if skip > 0:
-                skip =- 1
-                continue
-            if conflines[i].startswith("# line added by toaster"):
-                skip = 1
-            else:
-                bblayerconffile.write(conflines[i])
-
-        bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
-        bblayerconffile.close()
-
     def setLayers(self, bitbake, ls):
         """ Checks-out bitbake executor and layers from git repositories.
             Sets the layer variables in the config file, after validating local layer paths.

index 6f2ca33..833c685 100644
@@ -179,11 +179,6 @@ class LocalhostBEController(BuildEnvironmentController):
 
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
-        # 4. update the bblayers.conf
-        bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
-        if not os.path.exists(bblayerconf):
-            raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
-
         # 5. create custom layer and add custom recipes to it
         layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
         if os.path.isdir(layerpath):

@@ -247,10 +242,8 @@ class LocalhostBEController(BuildEnvironmentController):
         if os.path.isdir(layerpath):
             layerlist.append(layerpath)
 
-        BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
-
         self.islayerset = True
-        return True
+        return layerlist
 
     def readServerLogFile(self):
         return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()

index 833c685..2fd3db2 100644
@@ -250,44 +250,74 @@ class LocalhostBEController(BuildEnvironmentController):
 
 
     def triggerBuild(self, bitbake, layers, variables, targets, brbe):
-        # set up the build environment with the needed layers
-        self.setLayers(bitbake, layers, targets)
+        layers = self.setLayers(bitbake, layers, targets)
+
+        # init build environment from the clone
+        builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
+        oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
+        # init build environment
+        self._shellcmd('. %s %s' % (oe_init, builddir), self.be.sourcedir)
+
+        # update bblayers.conf
+        bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
+        conflines = open(bblconfpath, "r").readlines()
+        skip = False
+        with open(bblconfpath, 'w') as bblayers:
+            for line in conflines:
+                if line.startswith("# line added by toaster"):
+                    skip = True
+                    continue
+                if skip:
+                    skip = False
+                else:
+                    bblayers.write(line)
+
+            bblayers.write('# line added by toaster build control\n'
+                           'BBLAYERS = "%s"' % ' '.join(layers))
 
         # write configuration file
-        filepath = os.path.join(self.be.builddir, "conf/toaster.conf")
-        with open(filepath, 'w') as conf:
+        confpath = os.path.join(builddir, 'conf/toaster.conf')
+        with open(confpath, 'w') as conf:
             for var in variables:
                 conf.write('%s="%s"\n' % (var.name, var.value))
             conf.write('INHERIT+="toaster buildhistory"')
 
-        # get the bb server running with the build req id and build env id
-        bbctrl = self.getBBController()
-
-        # set variables; TOASTER_BRBE is not set on the server, as this
-        # causes events from command-line builds to be attached to the last
-        # Toaster-triggered build; instead, TOASTER_BRBE is fired as an event so
-        # that toasterui can set it on the buildinfohelper;
-        # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
-        for var in variables:
-            if var.name == 'TOASTER_BRBE':
-                bbctrl.triggerEvent('bb.event.MetadataEvent("SetBRBE", "%s")' \
-                                     % var.value)
-            else:
-                bbctrl.setVariable(var.name, var.value)
-
-        # Add 'toaster' and 'buildhistory' to INHERIT variable
-        inherit = {item.strip() for item in bbctrl.getVariable('INHERIT').split()}
-        inherit = inherit.union(["toaster", "buildhistory"])
-        bbctrl.setVariable('INHERIT', ' '.join(inherit))
-
-        # trigger the build command
-        task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets))
-        if len(task) == 0:
-            task = None
-
-        bbctrl.build(list(map(lambda x:x.target, targets)), task)
-
-        logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir)
-
-        # disconnect from the server
-        bbctrl.disconnect()
+        # run bitbake server from the clone
+        bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
+        self._shellcmd('. %s %s; BITBAKE_UI="" %s --read %s '
+                       '--server-only -t xmlrpc -B 0.0.0.0:0' % (oe_init, builddir,
+                       bitbake, confpath), self.be.sourcedir)
+
+        # read port number from bitbake.lock
+        self.be.bbport = ""
+        bblock = os.path.join(builddir, 'bitbake.lock')
+        with open(bblock) as fplock:
+            for line in fplock:
+                if ":" in line:
+                    self.be.bbport = line.split(":")[-1].strip()
+                    logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
+                    break
+
+        if not self.be.bbport:
+            raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)
+
+        self.be.bbaddress = "localhost"
+        self.be.bbstate = BuildEnvironment.SERVER_STARTED
+        self.be.lock = BuildEnvironment.LOCK_RUNNING
+        self.be.save()
+
+        bbtargets = ''
+        for target in targets:
+            task = target.task or 'do_build'
+            if not task.startswith('do_'):
+                task = 'do_' + task
+            bbtargets += '%s:%s ' % (target.target, task)
+
+        # run build with local bitbake
+        log = os.path.join(builddir, 'toaster_ui.log')
+        self._shellcmd('TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
+                       '../bitbake/bin/bitbake %s -u toasterui '
+                       '>>%s 2>&1 &' % (brbe, bbtargets, log), builddir)
+
+        logger.debug('localhostbecontroller: Build launched, exiting. '
+                     'Follow build logs at %s' % log)

index 2fd3db2..8291997 100644
@@ -51,12 +51,14 @@ class LocalhostBEController(BuildEnvironmentController):
         self.pokydirname = None
         self.islayerset = False
 
-    def _shellcmd(self, command, cwd = None):
+    def _shellcmd(self, command, cwd=None, nowait=False):
         if cwd is None:
             cwd = self.be.sourcedir
 
         logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
         p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if nowait:
+            return
         (out,err) = p.communicate()
         p.wait()
         if p.returncode:

index 8291997..ca96ae8 100644
@@ -315,11 +315,13 @@ class LocalhostBEController(BuildEnvironmentController):
                 task = 'do_' + task
             bbtargets += '%s:%s ' % (target.target, task)
 
-        # run build with local bitbake
+        # run build with local bitbake. stop the server after the build.
         log = os.path.join(builddir, 'toaster_ui.log')
-        self._shellcmd('TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
-                       '../bitbake/bin/bitbake %s -u toasterui '
-                       '>>%s 2>&1 &' % (brbe, bbtargets, log), builddir)
+        self._shellcmd(['(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
+                        '../bitbake/bin/bitbake %s -u toasterui >>%s 2>&1;'
+                        'BITBAKE_UI="" BBSERVER=0.0.0.0:-1 %s -m)&' \
+                        % (brbe, bbtargets, log, bitbake)], builddir,
+                        nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '
                      'Follow build logs at %s' % log)

index 5cc77ed..93c9145 100755
@@ -253,8 +253,8 @@ case $CMD in
 
         # Create configuration file
         conf=${BUILDDIR}/conf/toaster.conf
-        echo "# Created by toaster start script" > $conf
-        echo "INHERIT+=\"toaster buildhistory\"" >> $conf
+        line='INHERIT+="toaster buildhistory"'
+        grep -q "$line" $conf || echo $line >> $conf
 
         if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
             echo "Failed ${CMD}."