Fixes and refactoring

index 0f99342..84bf46b 100644
@@ -114,12 +114,13 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, featureSet = []):
+    def __init__(self, configuration, featureSet=None):
         self.recipecache = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
-        for f in featureSet:
-            self.featureset.setFeature(f)
+        if featureSet:
+            for f in featureSet:
+                self.featureset.setFeature(f)
 
         self.configuration = configuration
 

@@ -567,12 +568,14 @@ class BBCooker:
 
             logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
 
-    def showEnvironment(self, buildfile = None, pkgs_to_build = []):
+    def showEnvironment(self, buildfile=None, pkgs_to_build=None):
         """
         Show the outer or per-recipe environment
         """
         fn = None
         envdata = None
+        if not pkgs_to_build:
+            pkgs_to_build = []
 
         if buildfile:
             # Parse the configuration here. We need to do it explicitly here since

@@ -1037,13 +1040,13 @@ class BBCooker:
 
         return pkg_list
 
-    def generateTargetsTree(self, klass=None, pkgs=[]):
+    def generateTargetsTree(self, klass=None, pkgs=None):
         """
         Generate a dependency tree of buildable targets
         Generate an event with the result
         """
         # if the caller hasn't specified a pkgs list default to universe
-        if not len(pkgs):
+        if not pkgs:
             pkgs = ['universe']
         # if inherited_class passed ensure all recipes which inherit the
         # specified class are included in pkgs

index 61a7f4a..0e18110 100644
@@ -176,7 +176,7 @@ def fire_from_worker(event, d):
     fire_ui_handlers(event, d)
 
 noop = lambda _: None
-def register(name, handler, mask=[]):
+def register(name, handler, mask=None):
     """Register an Event handler"""
 
     # already registered

@@ -389,7 +389,7 @@ class DiskFull(Event):
 class NoProvider(Event):
     """No Provider for an Event"""
 
-    def __init__(self, item, runtime=False, dependees=None, reasons=[], close_matches=[]):
+    def __init__(self, item, runtime=False, dependees=None, reasons=None, close_matches=None):
         Event.__init__(self)
         self._item = item
         self._runtime = runtime

index 7b4d130..ec0c31a 100644
@@ -777,7 +777,7 @@ def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
-def runfetchcmd(cmd, d, quiet = False, cleanup = []):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails

@@ -802,6 +802,9 @@ def runfetchcmd(cmd, d, quiet = False, cleanup = []):
                   'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
                   'SOCKS5_USER', 'SOCKS5_PASSWD']
 
+    if not cleanup:
+        cleanup = []
+
     for var in exportvars:
         val = d.getVar(var, True)
         if val:

@@ -1267,7 +1270,7 @@ class FetchData(object):
 class FetchMethod(object):
     """Base class for 'fetch'ing data"""
 
-    def __init__(self, urls = []):
+    def __init__(self, urls=None):
         self.urls = []
 
     def supports(self, urldata, d):

@@ -1552,11 +1555,11 @@ class Fetch(object):
 
         return local
 
-    def download(self, urls = []):
+    def download(self, urls=None):
         """
         Fetch all urls
         """
-        if len(urls) == 0:
+        if not urls:
             urls = self.urls
 
         network = self.d.getVar("BB_NO_NETWORK", True)

@@ -1634,12 +1637,12 @@ class Fetch(object):
             finally:
                 bb.utils.unlockfile(lf)
 
-    def checkstatus(self, urls = []):
+    def checkstatus(self, urls=None):
         """
         Check all urls exist upstream
         """
 
-        if len(urls) == 0:
+        if not urls:
             urls = self.urls
 
         for u in urls:

@@ -1662,12 +1665,12 @@ class Fetch(object):
             if not ret:
                 raise FetchError("URL %s doesn't work" % u, u)
 
-    def unpack(self, root, urls = []):
+    def unpack(self, root, urls=None):
         """
         Check all urls exist upstream
         """
 
-        if len(urls) == 0:
+        if not urls:
             urls = self.urls
 
         for u in urls:

@@ -1685,12 +1688,12 @@ class Fetch(object):
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
-    def clean(self, urls = []):
+    def clean(self, urls=None):
         """
         Clean files that the fetcher gets or places
         """
 
-        if len(urls) == 0:
+        if not urls:
             urls = self.urls
 
         for url in urls:

index d79768d..786b5ae 100644
@@ -150,7 +150,7 @@ loggerDefaultVerbose = False
 loggerVerboseLogs = False
 loggerDefaultDomains = []
 
-def init_msgconfig(verbose, debug, debug_domains = []):
+def init_msgconfig(verbose, debug, debug_domains=None):
     """
     Set default verbosity and debug levels config the logger
     """

@@ -158,7 +158,10 @@ def init_msgconfig(verbose, debug, debug_domains = []):
     bb.msg.loggerDefaultVerbose = verbose
     if verbose:
         bb.msg.loggerVerboseLogs = True
-    bb.msg.loggerDefaultDomains = debug_domains
+    if debug_domains:
+        bb.msg.loggerDefaultDomains = debug_domains
+    else:
+        bb.msg.loggerDefaultDomains = []
 
 def constructLogOptions():
     debug = loggerDefaultDebugLevel

index 7c79785..1c07f2d 100644
@@ -128,10 +128,13 @@ def _logged_communicate(pipe, log, input, extrafiles):
         pipe.stderr.close()
     return ''.join(outdata), ''.join(errdata)
 
-def run(cmd, input=None, log=None, extrafiles=[], **options):
+def run(cmd, input=None, log=None, extrafiles=None, **options):
     """Convenience function to run a command and return its output, raising an
     exception when the command fails"""
 
+    if not extrafiles:
+        extrafiles = []
+
     if isinstance(cmd, basestring) and not "shell" in options:
         options["shell"] = True
 

index ef3ee57..3198635 100644
@@ -124,8 +124,10 @@ class ProcessServer(Process, BaseImplServer):
         self.command_channel.close()
         self.cooker.shutdown(True)
 
-    def idle_commands(self, delay, fds = []):
+    def idle_commands(self, delay, fds=None):
         nextsleep = delay
+        if not fds:
+            fds = []
 
         for function, data in self._idlefuns.items():
             try:

index 75ec855..f1a2067 100644
@@ -281,12 +281,15 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.connection_token = token
 
 class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
-    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
+    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None):
         self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
         self.clientinfo = clientinfo
         self.serverImpl = serverImpl
         self.observer_only = observer_only
-        self.featureset = featureset
+        if featureset:
+            self.featureset = featureset
+        else:
+            self.featureset = []
 
     def connect(self, token = None):
         if token is None:

index ca58e17..5fab704 100644
@@ -514,7 +514,7 @@ class TaskData:
             self.add_runtime_target(fn, item)
             self.add_tasks(fn, dataCache)
 
-    def fail_fnid(self, fnid, missing_list = []):
+    def fail_fnid(self, fnid, missing_list=None):
         """
         Mark a file as failed (unbuildable)
         Remove any references from build and runtime provider lists

@@ -523,6 +523,8 @@ class TaskData:
         """
         if fnid in self.failed_fnids:
             return
+        if not missing_list:
+            missing_list = []
         logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
         self.failed_fnids.append(fnid)
         for target in self.build_targets:

@@ -536,7 +538,7 @@ class TaskData:
                 if len(self.run_targets[target]) == 0:
                     self.remove_runtarget(target, missing_list)
 
-    def remove_buildtarget(self, targetid, missing_list = []):
+    def remove_buildtarget(self, targetid, missing_list=None):
         """
         Mark a build target as failed (unbuildable)
         Trigger removal of any files that have this as a dependency

@@ -561,7 +563,7 @@ class TaskData:
             logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
             raise bb.providers.NoProvider(target)
 
-    def remove_runtarget(self, targetid, missing_list = []):
+    def remove_runtarget(self, targetid, missing_list=None):
         """
         Mark a run target as failed (unbuildable)
         Trigger removal of any files that have this as a dependency

index 43edb70..b71fb33 100644
@@ -440,11 +440,17 @@ class HobHandler(gobject.GObject):
         self.commands_async.append(self.SUB_BUILD_RECIPES)
         self.run_next_command(self.GENERATE_PACKAGES)
 
-    def generate_image(self, image, base_image, image_packages=[], toolchain_packages=[], default_task="build"):
+    def generate_image(self, image, base_image, image_packages=None, toolchain_packages=None, default_task="build"):
         self.image = image
         self.base_image = base_image
-        self.package_queue = image_packages
-        self.toolchain_packages = toolchain_packages
+        if image_packages:
+            self.package_queue = image_packages
+        else:
+            self.package_queue = []
+        if toolchain_packages:
+            self.toolchain_packages = toolchain_packages
+        else:
+            self.toolchain_packages = []
         self.default_task = default_task
         self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
         self.commands_async.append(self.SUB_PARSE_CONFIG)

index 889b6c6..d29ddd6 100755
@@ -522,7 +522,9 @@ def builddashboard( request, build_id ):
 
 
 
-def generateCoveredList2( revlist = [] ):
+def generateCoveredList2( revlist = None ):
+    if not revlist:
+        revlist = []
     covered_list =  [ x for x in revlist if x.outcome == Task.OUTCOME_COVERED ]
     while len(covered_list):
         revlist =  [ x for x in revlist if x.outcome != Task.OUTCOME_COVERED ]

index 24ff341..a7cac97 100644
@@ -71,7 +71,7 @@ class Command:
                 result = command_method(self, commandline)
             except CommandError as exc:
                 return None, exc.args[0]
-            except Exception:
+            except (Exception, SystemExit):
                 import traceback
                 return None, traceback.format_exc()
             else:

index 84bf46b..f0f9c66 100644
@@ -126,12 +126,14 @@ class BBCooker:
 
         self.configwatcher = pyinotify.WatchManager()
         self.configwatcher.bbseen = []
+        self.configwatcher.bbwatchedfiles = []
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO 
         self.watcher = pyinotify.WatchManager()
         self.watcher.bbseen = []
+        self.watcher.bbwatchedfiles = []
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
 

@@ -185,6 +187,8 @@ class BBCooker:
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
     def config_notifications(self, event):
+        if not event.pathname in self.configwatcher.bbwatchedfiles:
+            return
         if not event.path in self.inotify_modified_files:
             self.inotify_modified_files.append(event.path)
         self.baseconfig_valid = False

@@ -198,20 +202,27 @@ class BBCooker:
         if not watcher:
             watcher = self.watcher
         for i in deps:
+            watcher.bbwatchedfiles.append(i[0])
             f = os.path.dirname(i[0])
             if f in watcher.bbseen:
                 continue
             watcher.bbseen.append(f)
+            watchtarget = None
             while True:
                 # We try and add watches for files that don't exist but if they did, would influence
                 # the parser. The parent directory of these files may not exist, in which case we need 
                 # to watch any parent that does exist for changes.
                 try:
                     watcher.add_watch(f, self.watchmask, quiet=False)
+                    if watchtarget:
+                        watcher.bbwatchedfiles.append(watchtarget)
                     break
                 except pyinotify.WatchManagerError as e:
                     if 'ENOENT' in str(e):
+                        watchtarget = f
                         f = os.path.dirname(f)
+                        if f in watcher.bbseen:
+                            break
                         watcher.bbseen.append(f)
                         continue
                     if 'ENOSPC' in str(e):

index a7cac97..398c1d6 100644
@@ -68,6 +68,8 @@ class Command:
                 if not hasattr(command_method, 'readonly') or False == getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
+                if getattr(command_method, 'needconfig', False):
+                    self.cooker.updateCacheSync()
                 result = command_method(self, commandline)
             except CommandError as exc:
                 return None, exc.args[0]

@@ -204,6 +206,7 @@ class CommandsSync:
         postfiles = params[1].split()
         command.cooker.configuration.prefile = prefiles
         command.cooker.configuration.postfile = postfiles
+    setPrePostConfFiles.needconfig = False
 
     def getCpuCount(self, command, params):
         """

@@ -211,10 +214,12 @@ class CommandsSync:
         """
         return bb.utils.cpu_count()
     getCpuCount.readonly = True
+    getCpuCount.needconfig = False
 
     def matchFile(self, command, params):
         fMatch = params[0]
         return command.cooker.matchFile(fMatch)
+    matchFile.needconfig = False
 
     def generateNewImage(self, command, params):
         image = params[0]

@@ -228,6 +233,7 @@ class CommandsSync:
     def ensureDir(self, command, params):
         directory = params[0]
         bb.utils.mkdirhier(directory)
+    ensureDir.needconfig = False
 
     def setVarFile(self, command, params):
         """

@@ -238,6 +244,7 @@ class CommandsSync:
         default_file = params[2]
         op = params[3]
         command.cooker.modifyConfigurationVar(var, val, default_file, op)
+    setVarFile.needconfig = False
 
     def removeVarFile(self, command, params):
         """

@@ -245,6 +252,7 @@ class CommandsSync:
         """
         var = params[0]
         command.cooker.removeConfigurationVar(var)
+    removeVarFile.needconfig = False
 
     def createConfigFile(self, command, params):
         """

@@ -252,6 +260,7 @@ class CommandsSync:
         """
         name = params[0]
         command.cooker.createConfigFile(name)
+    createConfigFile.needconfig = False
 
     def setEventMask(self, command, params):
         handlerNum = params[0]

@@ -259,6 +268,7 @@ class CommandsSync:
         debug_domains = params[2]
         mask = params[3]
         return bb.event.set_UIHmask(handlerNum, llevel, debug_domains, mask)
+    setEventMask.needconfig = False
 
     def setFeatures(self, command, params):
         """

@@ -266,7 +276,7 @@ class CommandsSync:
         """
         features = params[0]
         command.cooker.setFeatures(features)
-
+    setFeatures.needconfig = False
     # although we change the internal state of the cooker, this is transparent since
     # we always take and leave the cooker in state.initial
     setFeatures.readonly = True

@@ -275,6 +285,7 @@ class CommandsSync:
         options = params[0]
         environment = params[1]
         command.cooker.updateConfigOpts(options, environment)
+    updateConfig.needconfig = False
 
 class CommandsAsync:
     """

index f0f9c66..fb38289 100644
@@ -1445,6 +1445,21 @@ class BBCooker:
         if timestamp:
             return timestr
 
+    def updateCacheSync(self):
+        if self.state == state.running:
+            return
+
+        # reload files for which we got notifications
+        for p in self.inotify_modified_files:
+            bb.parse.update_cache(p)
+        self.inotify_modified_files = []
+
+        if not self.baseconfig_valid:
+            logger.debug(1, "Reloading base configuration data")
+            self.initConfigurationData()
+            self.baseconfig_valid = True
+            self.parsecache_valid = False
+
     # This is called for all async commands when self.state != running
     def updateCache(self):
         if self.state == state.running:

@@ -1456,17 +1471,7 @@ class BBCooker:
             raise bb.BBHandledException()
 
         if self.state != state.parsing:
-
-            # reload files for which we got notifications
-            for p in self.inotify_modified_files:
-                bb.parse.update_cache(p)
-            self.inotify_modified_files = []
-
-            if not self.baseconfig_valid:
-                logger.debug(1, "Reloading base configuration data")
-                self.initConfigurationData()
-                self.baseconfig_valid = True
-                self.parsecache_valid = False
+            self.updateCacheSync()
 
         if self.state != state.parsing and not self.parsecache_valid:
             self.parseConfiguration ()

index fb38289..a2b0bb5 100644
@@ -527,9 +527,11 @@ class BBCooker:
         self.handleCollections( self.data.getVar("BBFILE_COLLECTIONS", True) )
 
     def updateConfigOpts(self, options, environment):
+        clean = True
         for o in options:
+            if o in ['prefile', 'postfile']:
+                clean = False
             setattr(self.configuration, o, options[o])
-        clean = True
         for k in bb.utils.approved_variables():
             if k in environment and k not in self.configuration.env:
                 logger.debug(1, "Updating environment variable %s to %s" % (k, environment[k]))

index 57fc6bb..b20040c 100644
@@ -73,7 +73,8 @@ class ConfigParameters(object):
         options = {}
         for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp", 
                   "verbose", "debug", "dry_run", "dump_signatures", 
-                  "debug_domains", "extra_assume_provided", "profile"]:
+                  "debug_domains", "extra_assume_provided", "profile",
+                  "prefile", "postfile"]:
             options[o] = getattr(self.options, o)
 
         ret, error = server.runCommand(["updateConfig", options, environment])

index f08a8ef..4d77408 100755
@@ -358,8 +358,6 @@ def bitbake_main(configParams, configuration):
         try:
             server_connection = server.establishConnection(featureset)
         except Exception as e:
-            if configParams.kill_server:
-                return 0
             bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
 
         # Restore the environment in case the UI needs it

index 4d77408..2b720aa 100755
@@ -41,22 +41,45 @@ logger = logging.getLogger("BitBake")
 class BBMainException(Exception):
     pass
 
-def get_ui(config):
-    if not config.ui:
-        # modify 'ui' attribute because it is also read by cooker
-        config.ui = os.environ.get('BITBAKE_UI', 'knotty')
-
-    interface = config.ui
-
+def list_extension_modules(pkg, checkattr):
+    """
+    Lists extension modules in a specific Python package
+    (e.g. UIs, servers)
+    Parameters:
+        pkg: previously imported Python package to list
+        checkattr: attribute to look for in module to determine if it's valid
+            as the type of extension you are looking for
+    """
+    import pkgutil
+    pkgdir = os.path.dirname(pkg.__file__)
+
+    modules = []
+    for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
+        if os.path.isdir(os.path.join(pkgdir, modulename)):
+            # ignore directories
+            continue
+        try:
+            module = __import__(pkg.__name__, fromlist=[modulename])
+        except ImportError:
+            # If we can't import it, it's not valid
+            continue
+        module_if = getattr(module, modulename)
+        if getattr(module_if, 'hidden_extension', False):
+            continue
+        if not checkattr or hasattr(module_if, checkattr):
+            modules.append(modulename)
+    return modules
+
+def import_extension_module(pkg, modulename):
     try:
         # Dynamically load the UI based on the ui name. Although we
         # suggest a fixed set this allows you to have flexibility in which
         # ones are available.
-        module = __import__("bb.ui", fromlist = [interface])
-        return getattr(module, interface)
+        module = __import__(pkg.__name__, fromlist = [modulename])
+        return getattr(module, modulename)
     except AttributeError:
-        raise BBMainException("FATAL: Invalid user interface '%s' specified.\n"
-                 "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
+        raise BBMainException("FATAL: Unable to import extension module %s from %s" % (modulename, pkg.__name__))
+
 
 
 # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""

@@ -146,11 +169,19 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-P", "--profile", help = "Profile the command and save reports.",
                    action = "store_true", dest = "profile", default = False)
 
-        parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).",
-                   action = "store", dest = "ui")
+        env_ui = os.environ.get('BITBAKE_UI', None)
+        valid_uis = list_extension_modules(bb.ui, 'main')
+        default_ui = env_ui or 'knotty'
+        if env_ui and not env_ui in valid_uis:
+            raise BBMainException('Invalid UI "%s" specified in BITBAKE_UI environment variable - valid choices: %s' % (env_ui, ', '.join(valid_uis)))
+        elif not default_ui in valid_uis:
+            raise BBMainException('Default UI "%s" could not be found')
+        parser.add_option("-u", "--ui", help = "The user interface to use (%s)." % ', '.join(valid_uis),
+                   action="store", dest="ui", type="choice", choices=valid_uis, default=default_ui)
 
-        parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
-                   action = "store", dest = "servertype")
+        valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
+        parser.add_option("-t", "--servertype", help = "Choose which server type to use (%s)." % ', '.join(valid_server_types),
+                   action = "store", dest = "servertype", default = "process")
 
         parser.add_option("", "--token", help = "Specify the connection token to be used when connecting to a remote server.",
                    action = "store", dest = "xmlrpctoken")

@@ -279,21 +310,8 @@ def bitbake_main(configParams, configuration):
 
     configuration.setConfigParameters(configParams)
 
-    ui_module = get_ui(configParams)
-
-    # Server type can be xmlrpc or process currently, if nothing is specified,
-    # the default server is process
-    if configParams.servertype:
-        server_type = configParams.servertype
-    else:
-        server_type = 'process'
-
-    try:
-        module = __import__("bb.server", fromlist = [server_type])
-        servermodule = getattr(module, server_type)
-    except AttributeError:
-        raise BBMainException("FATAL: Invalid server type '%s' specified.\n"
-                              "Valid interfaces: xmlrpc, process [default]." % server_type)
+    ui_module = import_extension_module(bb.ui, configParams.ui)
+    servermodule = import_extension_module(bb.server, configParams.servertype)
 
     if configParams.server_only:
         if configParams.servertype != "xmlrpc":

index a2b0bb5..5452deb 100644
@@ -919,15 +919,25 @@ class BBCooker:
         print("}", file=tdepends_file)
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
-    def show_appends_with_no_recipes( self ):
-        appends_without_recipes = [self.collection.appendlist[recipe]
-                                   for recipe in self.collection.appendlist
-                                   if recipe not in self.collection.appliedappendlist]
+    def show_appends_with_no_recipes(self):
+        # Determine which bbappends haven't been applied
+
+        # First get list of recipes, including skipped
+        recipefns = self.recipecache.pkg_fn.keys()
+        recipefns.extend(self.skiplist.keys())
+
+        # Work out list of bbappends that have been applied
+        applied_appends = []
+        for fn in recipefns:
+            applied_appends.extend(self.collection.get_file_appends(fn))
+
+        appends_without_recipes = []
+        for _, appendfn in self.collection.bbappends:
+            if not appendfn in applied_appends:
+                appends_without_recipes.append(appendfn)
+
         if appends_without_recipes:
-            appendlines = ('  %s' % append
-                           for appends in appends_without_recipes
-                           for append in appends)
-            msg = 'No recipes available for:\n%s' % '\n'.join(appendlines)
+            msg = 'No recipes available for:\n  %s' % '\n  '.join(appends_without_recipes)
             warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
                  False) or "no"
             if warn_only.lower() in ("1", "yes", "true"):

@@ -1646,7 +1656,6 @@ class CookerCollectFiles(object):
     def __init__(self, priorities):
         self.appendlist = {}
         self.bbappends = []
-        self.appliedappendlist = []
         self.bbfile_config_priorities = priorities
 
     def calc_bbfile_priority( self, filename, matched = None ):

@@ -1769,7 +1778,6 @@ class CookerCollectFiles(object):
         for b in self.bbappends:
             (bbappend, filename) = b
             if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
-                self.appliedappendlist.append(bbappend)
                 filelist.append(filename)
         return filelist
 

index 5452deb..6dc9f19 100644
@@ -1654,7 +1654,6 @@ class CookerExit(bb.event.Event):
 
 class CookerCollectFiles(object):
     def __init__(self, priorities):
-        self.appendlist = {}
         self.bbappends = []
         self.bbfile_config_priorities = priorities
 

@@ -1750,10 +1749,6 @@ class CookerCollectFiles(object):
         for f in bbappend:
             base = os.path.basename(f).replace('.bbappend', '.bb')
             self.bbappends.append((base, f))
-            if not base in self.appendlist:
-               self.appendlist[base] = []
-            if f not in self.appendlist[base]:
-                self.appendlist[base].append(f)
 
         # Find overlayed recipes
         # bbfiles will be in priority order which makes this easy

index 5116e59..62b51b0 100755
@@ -595,7 +595,7 @@ build results (as the layer priority order has effectively changed).
                     return layerdir
             return None
 
-        appended_recipes = []
+        applied_appends = []
         for layer in layers:
             overlayed = []
             for f in self.bbhandler.cooker.collection.overlayed.iterkeys():

@@ -623,31 +623,27 @@ build results (as the layer priority order has effectively changed).
                                     logger.warn('Overwriting file %s', fdest)
                             bb.utils.copyfile(f1full, fdest)
                             if ext == '.bb':
-                                if f1 in self.bbhandler.cooker.collection.appendlist:
-                                    appends = self.bbhandler.cooker.collection.appendlist[f1]
-                                    if appends:
-                                        logger.plain('  Applying appends to %s' % fdest )
-                                        for appendname in appends:
-                                            if layer_path_match(appendname):
-                                                self.apply_append(appendname, fdest)
-                                    appended_recipes.append(f1)
+                                for append in self.bbhandler.cooker.collection.get_file_appends(f1full):
+                                    if layer_path_match(append):
+                                        logger.plain('  Applying append %s to %s' % (append, fdest))
+                                        self.apply_append(append, fdest)
+                                        applied_appends.append(append)
 
         # Take care of when some layers are excluded and yet we have included bbappends for those recipes
-        for recipename in self.bbhandler.cooker.collection.appendlist.iterkeys():
-            if recipename not in appended_recipes:
-                appends = self.bbhandler.cooker.collection.appendlist[recipename]
+        for b in self.bbhandler.cooker.collection.bbappends:
+            (recipename, appendname) = b
+            if appendname not in applied_appends:
                 first_append = None
-                for appendname in appends:
-                    layer = layer_path_match(appendname)
-                    if layer:
-                        if first_append:
-                            self.apply_append(appendname, first_append)
-                        else:
-                            fdest = appendname[len(layer):]
-                            fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
-                            bb.utils.mkdirhier(os.path.dirname(fdest))
-                            bb.utils.copyfile(appendname, fdest)
-                            first_append = fdest
+                layer = layer_path_match(appendname)
+                if layer:
+                    if first_append:
+                        self.apply_append(appendname, first_append)
+                    else:
+                        fdest = appendname[len(layer):]
+                        fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
+                        bb.utils.mkdirhier(os.path.dirname(fdest))
+                        bb.utils.copyfile(appendname, fdest)
+                        first_append = fdest
 
         # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
         # have come from)

@@ -723,7 +719,7 @@ build results (as the layer priority order has effectively changed).
 Lists recipes with the bbappends that apply to them as subitems.
 """
         self.init_bbhandler()
-        if not self.bbhandler.cooker.collection.appendlist:
+        if not self.bbhandler.cooker.collection.bbappends:
             logger.plain('No append files found')
             return 0
 

index 62b51b0..8cf7196 100755
@@ -719,18 +719,21 @@ build results (as the layer priority order has effectively changed).
 Lists recipes with the bbappends that apply to them as subitems.
 """
         self.init_bbhandler()
-        if not self.bbhandler.cooker.collection.bbappends:
-            logger.plain('No append files found')
-            return 0
 
         logger.plain('=== Appended recipes ===')
 
         pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys())
         pnlist.sort()
+        appends = False
         for pn in pnlist:
-            self.show_appends_for_pn(pn)
+            if self.show_appends_for_pn(pn):
+                appends = True
+
+        if self.show_appends_for_skipped():
+            appends = True
 
-        self.show_appends_for_skipped()
+        if not appends:
+            logger.plain('No append files found')
 
     def show_appends_for_pn(self, pn):
         filenames = self.bbhandler.cooker_data.pkg_pn[pn]

@@ -741,12 +744,12 @@ Lists recipes with the bbappends that apply to them as subitems.
                                              self.bbhandler.cooker_data.pkg_pn)
         best_filename = os.path.basename(best[3])
 
-        self.show_appends_output(filenames, best_filename)
+        return self.show_appends_output(filenames, best_filename)
 
     def show_appends_for_skipped(self):
         filenames = [os.path.basename(f)
                     for f in self.bbhandler.cooker.skiplist.iterkeys()]
-        self.show_appends_output(filenames, None, " (skipped)")
+        return self.show_appends_output(filenames, None, " (skipped)")
 
     def show_appends_output(self, filenames, best_filename, name_suffix = ''):
         appended, missing = self.get_appends_for_files(filenames)

@@ -760,7 +763,9 @@ Lists recipes with the bbappends that apply to them as subitems.
                 if best_filename in missing:
                     logger.warn('%s: missing append for preferred version',
                                 best_filename)
-
+            return True
+        else:
+            return False
 
     def get_appends_for_files(self, filenames):
         appended, notappended = [], []

index 8cf7196..5518c63 100755
@@ -705,13 +705,11 @@ build results (as the layer priority order has effectively changed).
         return os.path.basename(layerdir.rstrip(os.sep))
 
     def apply_append(self, appendname, recipename):
-        appendfile = open(appendname, 'r')
-        recipefile = open(recipename, 'a')
-        recipefile.write('\n')
-        recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
-        recipefile.writelines(appendfile.readlines())
-        recipefile.close()
-        appendfile.close()
+        with open(appendname, 'r') as appendfile:
+            with open(recipename, 'a') as recipefile:
+                recipefile.write('\n')
+                recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
+                recipefile.writelines(appendfile.readlines())
 
     def do_show_appends(self, args):
         """list bbappend files and recipe files they apply to

@@ -879,20 +877,19 @@ NOTE: .bbappend files can impact the dependencies.
 
             # The 'require/include xxx' in the bb file
             pv_re = re.compile(r"\${PV}")
-            fnfile = open(f, 'r')
-            line = fnfile.readline()
-            while line:
-                m, keyword = self.match_require_include(line)
-                # Found the 'require/include xxxx'
-                if m:
-                    needed_file = m.group(1)
-                    # Replace the ${PV} with the real PV
-                    if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
-                        pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
-                        needed_file = re.sub(r"\${PV}", pv, needed_file)
-                    self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
+            with open(f, 'r') as fnfile:
                 line = fnfile.readline()
-            fnfile.close()
+                while line:
+                    m, keyword = self.match_require_include(line)
+                    # Found the 'require/include xxxx'
+                    if m:
+                        needed_file = m.group(1)
+                        # Replace the ${PV} with the real PV
+                        if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
+                            pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
+                            needed_file = re.sub(r"\${PV}", pv, needed_file)
+                        self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
+                    line = fnfile.readline()
 
         # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
         conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")

@@ -906,20 +903,19 @@ NOTE: .bbappend files can impact the dependencies.
                     f = os.path.join(dirpath, name)
                     s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
                     if s:
-                        ffile = open(f, 'r')
-                        line = ffile.readline()
-                        while line:
-                            m, keyword = self.match_require_include(line)
-                            # Only bbclass has the "inherit xxx" here.
-                            bbclass=""
-                            if not m and f.endswith(".bbclass"):
-                                m, keyword = self.match_inherit(line)
-                                bbclass=".bbclass"
-                            # Find a 'require/include xxxx'
-                            if m:
-                                self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
+                        with open(f, 'r') as ffile:
                             line = ffile.readline()
-                        ffile.close()
+                            while line:
+                                m, keyword = self.match_require_include(line)
+                                # Only bbclass has the "inherit xxx" here.
+                                bbclass=""
+                                if not m and f.endswith(".bbclass"):
+                                    m, keyword = self.match_inherit(line)
+                                    bbclass=".bbclass"
+                                # Find a 'require/include xxxx'
+                                if m:
+                                    self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
+                                line = ffile.readline()
 
     def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
         """Print the depends that crosses a layer boundary"""

index 1becaa4..67ec71f 100644
@@ -71,7 +71,12 @@ def cached_mtime_noerror(f):
     return __mtime_cache[f]
 
 def update_mtime(f):
-    __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+    try:
+        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+    except OSError:
+        if f in __mtime_cache:
+            del __mtime_cache[f]
+        return 0
     return __mtime_cache[f]
 
 def update_cache(f):

index 4d77408..c98cf44 100755
@@ -41,22 +41,44 @@ logger = logging.getLogger("BitBake")
 class BBMainException(Exception):
     pass
 
-def get_ui(config):
-    if not config.ui:
-        # modify 'ui' attribute because it is also read by cooker
-        config.ui = os.environ.get('BITBAKE_UI', 'knotty')
-
-    interface = config.ui
-
+def list_extension_modules(pkg, checkattr):
+    """
+    Lists extension modules in a specific Python package
+    (e.g. UIs, servers)
+    Parameters:
+        pkg: previously imported Python package to list
+        checkattr: attribute to look for in module to determine if it's valid
+            as the type of extension you are looking for
+    """
+    import pkgutil
+    pkgdir = os.path.dirname(pkg.__file__)
+
+    modules = []
+    for _, modulename, _ in pkgutil.iter_modules([pkgdir]):
+        if os.path.isdir(os.path.join(pkgdir, modulename)):
+            # ignore directories
+            continue
+        try:
+            module = __import__(pkg.__name__, fromlist=[modulename])
+        except (ImportError, SystemExit):
+            # If we can't import it, it's not valid
+            continue
+        module_if = getattr(module, modulename)
+        if getattr(module_if, 'hidden_extension', False):
+            continue
+        if not checkattr or hasattr(module_if, checkattr):
+            modules.append(modulename)
+    return modules
+
+def import_extension_module(pkg, modulename):
     try:
         # Dynamically load the UI based on the ui name. Although we
         # suggest a fixed set this allows you to have flexibility in which
         # ones are available.
-        module = __import__("bb.ui", fromlist = [interface])
-        return getattr(module, interface)
+        module = __import__(pkg.__name__, fromlist = [modulename])
+        return getattr(module, modulename)
     except AttributeError:
-        raise BBMainException("FATAL: Invalid user interface '%s' specified.\n"
-                 "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
+        raise BBMainException("FATAL: Unable to import extension module %s from %s" % (modulename, pkg.__name__))
 
 
 # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""

@@ -146,11 +168,25 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-P", "--profile", help = "Profile the command and save reports.",
                    action = "store_true", dest = "profile", default = False)
 
-        parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).",
-                   action = "store", dest = "ui")
-
-        parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
-                   action = "store", dest = "servertype")
+        def present_options(optionlist):
+            if len(optionlist) > 1:
+                return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]])
+            else:
+                return optionlist[0]
+
+        env_ui = os.environ.get('BITBAKE_UI', None)
+        valid_uis = list_extension_modules(bb.ui, 'main')
+        default_ui = env_ui or 'knotty'
+        if env_ui and not env_ui in valid_uis:
+            raise BBMainException('Invalid UI "%s" specified in BITBAKE_UI environment variable - valid choices: %s' % (env_ui, present_options(valid_uis)))
+        elif not default_ui in valid_uis:
+            raise BBMainException('Default UI "%s" could not be found')
+        parser.add_option("-u", "--ui", help = "The user interface to use (%s - default %%default)." % present_options(valid_uis),
+                   action="store", dest="ui", type="choice", choices=valid_uis, default=default_ui)
+
+        valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
+        parser.add_option("-t", "--servertype", help = "Choose which server type to use (%s - default %%default)." % present_options(valid_server_types),
+                   action = "store", dest = "servertype", default = "process")
 
         parser.add_option("", "--token", help = "Specify the connection token to be used when connecting to a remote server.",
                    action = "store", dest = "xmlrpctoken")

@@ -279,21 +315,8 @@ def bitbake_main(configParams, configuration):
 
     configuration.setConfigParameters(configParams)
 
-    ui_module = get_ui(configParams)
-
-    # Server type can be xmlrpc or process currently, if nothing is specified,
-    # the default server is process
-    if configParams.servertype:
-        server_type = configParams.servertype
-    else:
-        server_type = 'process'
-
-    try:
-        module = __import__("bb.server", fromlist = [server_type])
-        servermodule = getattr(module, server_type)
-    except AttributeError:
-        raise BBMainException("FATAL: Invalid server type '%s' specified.\n"
-                              "Valid interfaces: xmlrpc, process [default]." % server_type)
+    ui_module = import_extension_module(bb.ui, configParams.ui)
+    servermodule = import_extension_module(bb.server, configParams.servertype)
 
     if configParams.server_only:
         if configParams.servertype != "xmlrpc":