Layer index improvements

index ed36859..c569b61 100644
@@ -97,14 +97,15 @@ def patch_recipe(fn, relpath, values):
 
     with tempfile.NamedTemporaryFile('w', delete=False) as tf:
         def outputvalue(name):
-            rawtext = '%s = "%s"\n' % (name, values[name])
-            if name in nowrap_vars:
-                tf.write(rawtext)
-            else:
-                wrapped = textwrap.wrap(rawtext)
-                for wrapline in wrapped[:-1]:
-                    tf.write('%s \\\n' % wrapline)
-                tf.write('%s\n' % wrapped[-1])
+            if values[name]:
+                rawtext = '%s = "%s"\n' % (name, values[name])
+                if name in nowrap_vars:
+                    tf.write(rawtext)
+                else:
+                    wrapped = textwrap.wrap(rawtext)
+                    for wrapline in wrapped[:-1]:
+                        tf.write('%s \\\n' % wrapline)
+                    tf.write('%s\n' % wrapped[-1])
 
         tfn = tf.name
         with open(fn, 'r') as f:

index 78711e5..60653cf 100644
@@ -189,25 +189,6 @@ class BulkChangeEditForm(forms.ModelForm):
         model = RecipeChange
         fields = ('summary', 'description', 'homepage', 'bugtracker', 'section', 'license')
 
-    def __init__(self, *args, **kwargs):
-        instance = kwargs.get('instance', None)
-        initial = kwargs.get('initial', {})
-        if instance:
-            recipe = instance.recipe
-            if recipe:
-                for fieldname in self._meta.fields:
-                    if not getattr(instance, fieldname):
-                        initial[fieldname] = getattr(recipe, fieldname)
-        kwargs['initial'] = initial
-        super(BulkChangeEditForm, self).__init__(*args, **kwargs)
-
-    def clear_same_values(self):
-        for fieldname in self._meta.fields:
-            oldval = getattr(self.instance.recipe, fieldname)
-            newval = getattr(self.instance, fieldname)
-            if oldval == newval:
-                setattr(self.instance, fieldname, '')
-
 BulkChangeEditFormSet = modelformset_factory(RecipeChange, form=BulkChangeEditForm, extra=0)
 
 

index cde8fe3..e0d85ea 100644
@@ -415,12 +415,16 @@ class RecipeChange(models.Model):
 
     def changed_fields(self, mapped = False):
         res = {}
-        for field in self._meta.fields:
-            if not field.name in ['id', 'changeset', 'recipe']:
-                value = getattr(self, field.name)
-                if value:
-                    if mapped:
-                        res[self.RECIPE_VARIABLE_MAP[field.name]] = value
-                    else:
-                        res[field.name] = value
+        for fieldname in self.RECIPE_VARIABLE_MAP:
+            value = getattr(self, fieldname)
+            origvalue = getattr(self.recipe, fieldname)
+            if value != origvalue:
+                if mapped:
+                    res[self.RECIPE_VARIABLE_MAP[fieldname]] = value
+                else:
+                    res[fieldname] = value
         return res
+
+    def reset_fields(self):
+        for fieldname in self.RECIPE_VARIABLE_MAP:
+            setattr(self, fieldname, getattr(self.recipe, fieldname))

index 898a7c4..d033046 100644
@@ -200,8 +200,6 @@ def bulk_change_edit_view(request, template_name, pk):
     if request.method == 'POST':
         formset = BulkChangeEditFormSet(request.POST, queryset=changeset.recipechange_set.all())
         if formset.is_valid():
-            for form in formset:
-                form.clear_same_values()
             formset.save()
             return HttpResponseRedirect(reverse('bulk_change_review', args=(changeset.id,)))
     else:

@@ -524,6 +522,7 @@ class BulkChangeSearchView(AdvancedRecipeSearchView):
                     change = RecipeChange()
                     change.changeset = changeset
                     change.recipe = recipe
+                    change.reset_fields()
                     change.save()
 
         if 'add_selected' in request.POST:

index c569b61..08d6d78 100644
@@ -70,7 +70,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
         tar = tarfile.open(None, "w:gz", tmptarfile)
         for patch in patches:
             patchfn = os.path.join(tmpoutdir, patch)
-            tar.add(patchfn)
+            tar.add(patchfn, arcname=patch)
         tar.close()
         ret = tmptarname
     elif len(patches) == 1:

index 84edee2..ff0a7a5 100644
@@ -119,11 +119,8 @@ On a regular basis you need to run the update script:
 path/to/layerindex/update.py
 
 This will fetch all of the layer repositories, analyse their contents
-and update the database with the results. Note that if you set up more
-than just the master branch in the database, you will need to run the
-script once for each branch using -b (or --branch) to specify the
-branch name. Run the script with --help for further information on
-available options.
+and update the database with the results. Run the script with --help for
+further information on available options.
 
 
 Maintenance

199 lines not on this page · raw message

index e0d85ea..6c5f65f 100644
@@ -19,6 +19,7 @@ class Branch(models.Model):
     bitbake_branch = models.CharField(max_length=50)
     short_description = models.CharField(max_length=50, blank=True)
     sort_priority = models.IntegerField(blank=True, null=True)
+    updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
 
     updated = models.DateTimeField(auto_now = True, default = datetime.now)
 

91 lines not on this page · raw message

index 2d5a53d..9ab7aa7 100644
@@ -10,7 +10,6 @@ TODO:
 * Make it easy to update people's email addresses
 
 Bugs
-* git recipe shows up all _git bbappends: http://layers.openembedded.org/layerindex/recipe/5542/
 * Duplication of first maintainer when editing to add a second?
 * PROVIDES column appears to be too short for some recipes
 

index d033046..d9480c9 100644
@@ -672,7 +672,7 @@ class RecipeDetailView(DetailView):
             verappendprefix = recipe.filename.split('.bb')[0]
             appendprefix = verappendprefix.split('_')[0]
             #context['verappends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename='%s.bbappend' % verappendprefix)
-            context['appends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename__regex=r'%s(_[^_]*)?\.bbappend' % appendprefix)
+            context['appends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename__regex=r'^%s(_[^_]*)?\.bbappend' % appendprefix)
             verappends = []
             for append in context['appends']:
                 if append.matches_recipe(recipe):

index 9ab7aa7..b5e8974 100644
@@ -11,7 +11,6 @@ TODO:
 
 Bugs
 * Duplication of first maintainer when editing to add a second?
-* PROVIDES column appears to be too short for some recipes
 
 Other
 * Full-text search on layer contents

203 lines not on this page · raw message

index 6c5f65f..8d82846 100644
@@ -238,10 +238,10 @@ class Recipe(models.Model):
     summary = models.CharField(max_length=200, blank=True)
     description = models.TextField(blank=True)
     section = models.CharField(max_length=100, blank=True)
-    license = models.CharField(max_length=100, blank=True)
+    license = models.CharField(max_length=2048, blank=True)
     homepage = models.URLField(blank=True)
     bugtracker = models.URLField(blank=True)
-    provides = models.CharField(max_length=255, blank=True)
+    provides = models.CharField(max_length=2048, blank=True)
     bbclassextend = models.CharField(max_length=100, blank=True)
     inherits = models.CharField(max_length=255, blank=True)
     updated = models.DateTimeField(auto_now = True)

index 7daccef..f7cb25c 100755
@@ -2,23 +2,20 @@
 
 # Fetch layer repositories and update layer index database
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
 # Author: Paul Eggleton <[email protected]>
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
 
 import sys
-import os.path
+import os
 import optparse
 import logging
-from datetime import datetime
-import re
-import tempfile
-import shutil
+import subprocess
+import signal
 from distutils.version import LooseVersion
 import utils
-import recipeparse
 
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)

@@ -33,79 +30,21 @@ except ImportError:
     sys.exit(1)
 
 
-def check_machine_conf(path, subdir_start):
-    subpath = path[len(subdir_start):]
-    res = conf_re.match(subpath)
-    if res:
-        return res.group(1)
-    return None
+def reenable_sigint():
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
 
-def split_recipe_fn(path):
-    splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
-    pn = splitfn[0]
-    if len(splitfn) > 1:
-        pv = splitfn[1]
-    else:
-        pv = "1.0"
-    return (pn, pv)
-
-def update_recipe_file(data, path, recipe, layerdir_start, repodir):
-    fn = str(os.path.join(path, recipe.filename))
+def run_command_interruptible(cmd):
+    """
+    Run a command with output displayed on the console, but ensure any Ctrl+C is
+    processed only by the child process.
+    """
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
     try:
-        logger.debug('Updating recipe %s' % fn)
-        envdata = bb.cache.Cache.loadDataFull(fn, [], data)
-        envdata.setVar('SRCPV', 'X')
-        recipe.pn = envdata.getVar("PN", True)
-        recipe.pv = envdata.getVar("PV", True)
-        recipe.summary = envdata.getVar("SUMMARY", True)
-        recipe.description = envdata.getVar("DESCRIPTION", True)
-        recipe.section = envdata.getVar("SECTION", True)
-        recipe.license = envdata.getVar("LICENSE", True)
-        recipe.homepage = envdata.getVar("HOMEPAGE", True)
-        recipe.bugtracker = envdata.getVar("BUGTRACKER", True) or ""
-        recipe.provides = envdata.getVar("PROVIDES", True) or ""
-        recipe.bbclassextend = envdata.getVar("BBCLASSEXTEND", True) or ""
-        # Handle recipe inherits for this recipe
-        gr = set(data.getVar("__inherit_cache", True) or [])
-        lr = set(envdata.getVar("__inherit_cache", True) or [])
-        recipe.inherits = ' '.join(sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}))
-        recipe.blacklisted = envdata.getVarFlag('PNBLACKLIST', recipe.pn, True) or ""
-        recipe.save()
-
-        # Get file dependencies within this layer
-        deps = envdata.getVar('__depends', True)
-        filedeps = []
-        for depstr, date in deps:
-            found = False
-            if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
-                filedeps.append(os.path.relpath(depstr, repodir))
-        from layerindex.models import RecipeFileDependency
-        RecipeFileDependency.objects.filter(recipe=recipe).delete()
-        for filedep in filedeps:
-            recipedep = RecipeFileDependency()
-            recipedep.layerbranch = recipe.layerbranch
-            recipedep.recipe = recipe
-            recipedep.path = filedep
-            recipedep.save()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        if not recipe.pn:
-            recipe.pn = recipe.filename[:-3].split('_')[0]
-        logger.error("Unable to read %s: %s", fn, str(e))
-
-def update_machine_conf_file(path, machine):
-    logger.debug('Updating machine %s' % path)
-    desc = ""
-    with open(path, 'r') as f:
-        for line in f:
-            if line.startswith('#@NAME:'):
-                desc = line[7:].strip()
-            if line.startswith('#@DESCRIPTION:'):
-                desc = line[14:].strip()
-                desc = re.sub(r'Machine configuration for( running)*( an)*( the)*', '', desc)
-                break
-    machine.description = desc
+        ret = subprocess.call(cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint)
+    finally:
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+    return ret
+
 
 def main():
     if LooseVersion(git.__version__) < '0.3.1':

@@ -151,13 +90,9 @@ def main():
         parser.print_help()
         sys.exit(1)
 
-    if options.fullreload:
-        options.reload = True
-
     utils.setup_django()
     import settings
-    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
-    from django.db import transaction
+    from layerindex.models import LayerItem
 
     logger.setLevel(options.loglevel)
 

@@ -225,397 +160,39 @@ def main():
             else:
                 out = utils.runcmd("git fetch", bitbakepath, logger=logger)
 
-        try:
-            (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
-        except recipeparse.RecipeParseError as e:
-            logger.error(str(e))
-            sys.exit(1)
-
-        # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
-        tinfoil.config_data.setVar('SUMMARY', '')
-        # Clear the default value of DESCRIPTION so that we can see where it's not set
-        tinfoil.config_data.setVar('DESCRIPTION', '')
-        # Clear the default value of HOMEPAGE ('unknown')
-        tinfoil.config_data.setVar('HOMEPAGE', '')
-        # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
-        # why won't they just fix that?!)
-        tinfoil.config_data.setVar('LICENSE', '')
-
         # Process and extract data from each layer
+        # We now do this by calling out to a separate script; doing otherwise turned out to be
+        # unreliable due to leaking memory (we're using bitbake internals in a manner in which
+        # they never get used during normal operation).
         for layer in layerquery:
-            transaction.enter_transaction_management()
-            transaction.managed(True)
-            try:
-                urldir = layer.get_fetch_dir()
-                repodir = os.path.join(fetchdir, urldir)
-                if layer.vcs_url in failedrepos:
-                    logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
-                    transaction.rollback()
-                    continue
-
-                layerbranch = layer.get_layerbranch(options.branch)
-
-                branchname = options.branch
-                branchdesc = options.branch
-                if layerbranch:
-                    if layerbranch.actual_branch:
-                        branchname = layerbranch.actual_branch
-                        branchdesc = "%s (%s)" % (options.branch, branchname)
-
-                # Collect repo info
-                repo = git.Repo(repodir)
-                assert repo.bare == False
-                try:
-                    if options.nocheckout:
-                        topcommit = repo.commit('HEAD')
-                    else:
-                        topcommit = repo.commit('origin/%s' % branchname)
-                except:
-                    if layerbranch:
-                        logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
-                    else:
-                        logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
-                    transaction.rollback()
-                    continue
-
-                newbranch = False
-                if not layerbranch:
-                    # LayerBranch doesn't exist for this branch, create it
-                    newbranch = True
-                    layerbranch = LayerBranch()
-                    layerbranch.layer = layer
-                    layerbranch.branch = branch
-                    layerbranch_source = layer.get_layerbranch('master')
-                    if not layerbranch_source:
-                        layerbranch_source = layer.get_layerbranch(None)
-                    if layerbranch_source:
-                        layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
-                    layerbranch.save()
-                    if layerbranch_source:
-                        for maintainer in layerbranch_source.layermaintainer_set.all():
-                            maintainer.pk = None
-                            maintainer.id = None
-                            maintainer.layerbranch = layerbranch
-                            maintainer.save()
-                        for dep in layerbranch_source.dependencies_set.all():
-                            dep.pk = None
-                            dep.id = None
-                            dep.layerbranch = layerbranch
-                            dep.save()
-
-                if layerbranch.vcs_subdir and not options.nocheckout:
-                    # Find latest commit in subdirectory
-                    # A bit odd to do it this way but apparently there's no other way in the GitPython API
-                    topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
-                    if not topcommit:
-                        # This will error out if the directory is completely invalid or had never existed at this point
-                        # If it previously existed but has since been deleted, you will get the revision where it was
-                        # deleted - so we need to handle that case separately later
-                        if newbranch:
-                            logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
-                        elif layerbranch.vcs_subdir:
-                            logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
-                        else:
-                            logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
-                        transaction.rollback()
-                        continue
-
-                layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
-                layerdir_start = os.path.normpath(layerdir) + os.sep
-                layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
-                layermachines = Machine.objects.filter(layerbranch=layerbranch)
-                layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
-                layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
-                if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
-                    # Check out appropriate branch
-                    if not options.nocheckout:
-                        out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
-                        out = utils.runcmd("git clean -f -x", repodir, logger=logger)
-
-                    if layerbranch.vcs_subdir and not os.path.exists(layerdir):
-                        if newbranch:
-                            logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
-                        else:
-                            logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
-                        transaction.rollback()
-                        continue
-
-                    if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
-                        logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
-                        transaction.rollback()
-                        continue
-
-                    logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
-
-                    try:
-                        config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
-                    except recipeparse.RecipeParseError as e:
-                        logger.error(str(e))
-                        transaction.rollback()
-                        continue
-
-                    if layerbranch.vcs_last_rev and not options.reload:
-                        try:
-                            diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
-                        except Exception as e:
-                            logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
-                            diff = None
-                    else:
-                        diff = None
-
-                    # We handle recipes specially to try to preserve the same id
-                    # when recipe upgrades happen (so that if a user bookmarks a
-                    # recipe page it remains valid)
-                    layerrecipes_delete = []
-                    layerrecipes_add = []
-
-                    # Check if any paths should be ignored because there are layers within this layer
-                    removedirs = []
-                    for root, dirs, files in os.walk(layerdir):
-                        for d in dirs:
-                            if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
-                                removedirs.append(os.path.join(root, d) + os.sep)
-
-                    if diff:
-                        # Apply git changes to existing recipe list
-
-                        if layerbranch.vcs_subdir:
-                            subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
-                        else:
-                            subdir_start = ""
-
-                        updatedrecipes = set()
-                        for d in diff.iter_change_type('D'):
-                            path = d.a_blob.path
-                            if path.startswith(subdir_start):
-                                skip = False
-                                for d in removedirs:
-                                    if path.startswith(d):
-                                        skip = True
-                                        break
-                                if skip:
-                                    continue
-                                (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
-                                if typename == 'recipe':
-                                    values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
-                                    if len(values):
-                                        layerrecipes_delete.append(values[0])
-                                        logger.debug("Mark %s for deletion" % values[0])
-                                        updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
-                                    else:
-                                        logger.warn("Deleted recipe %s could not be found" % path)
-                                elif typename == 'bbappend':
-                                    layerappends.filter(filepath=filepath).filter(filename=filename).delete()
-                                elif typename == 'machine':
-                                    layermachines.filter(name=filename).delete()
-                                elif typename == 'bbclass':
-                                    layerclasses.filter(name=filename).delete()
-
-                        for d in diff.iter_change_type('A'):
-                            path = d.b_blob.path
-                            if path.startswith(subdir_start):
-                                skip = False
-                                for d in removedirs:
-                                    if path.startswith(d):
-                                        skip = True
-                                        break
-                                if skip:
-                                    continue
-                                (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
-                                if typename == 'recipe':
-                                    layerrecipes_add.append(os.path.join(repodir, path))
-                                    logger.debug("Mark %s for addition" % path)
-                                    updatedrecipes.add(os.path.join(filepath, filename))
-                                elif typename == 'bbappend':
-                                    append = BBAppend()
-                                    append.layerbranch = layerbranch
-                                    append.filename = filename
-                                    append.filepath = filepath
-                                    append.save()
-                                elif typename == 'machine':
-                                    machine = Machine()
-                                    machine.layerbranch = layerbranch
-                                    machine.name = filename
-                                    update_machine_conf_file(os.path.join(repodir, path), machine)
-                                    machine.save()
-                                elif typename == 'bbclass':
-                                    bbclass = BBClass()
-                                    bbclass.layerbranch = layerbranch
-                                    bbclass.name = filename
-                                    bbclass.save()
-
-                        dirtyrecipes = set()
-                        for d in diff.iter_change_type('M'):
-                            path = d.a_blob.path
-                            if path.startswith(subdir_start):
-                                skip = False
-                                for d in removedirs:
-                                    if path.startswith(d):
-                                        skip = True
-                                        break
-                                if skip:
-                                    continue
-                                (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
-                                if typename == 'recipe':
-                                    logger.debug("Mark %s for update" % path)
-                                    results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
-                                    if results:
-                                        recipe = results[0]
-                                        update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
-                                        recipe.save()
-                                        updatedrecipes.add(recipe.full_path())
-                                elif typename == 'machine':
-                                    results = layermachines.filter(name=filename)
-                                    if results:
-                                        machine = results[0]
-                                        update_machine_conf_file(os.path.join(repodir, path), machine)
-                                        machine.save()
-
-                                deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
-                                for dep in deps:
-                                    dirtyrecipes.add(dep.recipe)
-
-                        for recipe in dirtyrecipes:
-                            if not recipe.full_path() in updatedrecipes:
-                                update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
-                    else:
-                        # Collect recipe data from scratch
-
-                        layerrecipe_fns = []
-                        if options.fullreload:
-                            layerrecipes.delete()
-                        else:
-                            # First, check which recipes still exist
-                            layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
-                            for v in layerrecipe_values:
-                                root = os.path.join(layerdir, v['filepath'])
-                                fullpath = os.path.join(root, v['filename'])
-                                preserve = True
-                                if os.path.exists(fullpath):
-                                    for d in removedirs:
-                                        if fullpath.startswith(d):
-                                            preserve = False
-                                            break
-                                else:
-                                    preserve = False
-
-                                if preserve:
-                                    # Recipe still exists, update it
-                                    results = layerrecipes.filter(id=v['id'])[:1]
-                                    recipe = results[0]
-                                    update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
-                                else:
-                                    # Recipe no longer exists, mark it for later on
-                                    layerrecipes_delete.append(v)
-                                layerrecipe_fns.append(fullpath)
-
-                        layermachines.delete()
-                        layerappends.delete()
-                        layerclasses.delete()
-                        for root, dirs, files in os.walk(layerdir):
-                            if '.git' in dirs:
-                                dirs.remove('.git')
-                            for d in dirs[:]:
-                                fullpath = os.path.join(root, d) + os.sep
-                                if fullpath in removedirs:
-                                    dirs.remove(d)
-                            for f in files:
-                                fullpath = os.path.join(root, f)
-                                (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
-                                if typename == 'recipe':
-                                    if fullpath not in layerrecipe_fns:
-                                        layerrecipes_add.append(fullpath)
-                                elif typename == 'bbappend':
-                                    append = BBAppend()
-                                    append.layerbranch = layerbranch
-                                    append.filename = f
-                                    append.filepath = os.path.relpath(root, layerdir)
-                                    append.save()
-                                elif typename == 'machine':
-                                    machine = Machine()
-                                    machine.layerbranch = layerbranch
-                                    machine.name = filename
-                                    update_machine_conf_file(fullpath, machine)
-                                    machine.save()
-                                elif typename == 'bbclass':
-                                    bbclass = BBClass()
-                                    bbclass.layerbranch = layerbranch
-                                    bbclass.name = filename
-                                    bbclass.save()
-
-                    for added in layerrecipes_add:
-                        # This is good enough without actually parsing the file
-                        (pn, pv) = split_recipe_fn(added)
-                        oldid = -1
-                        for deleted in layerrecipes_delete:
-                            if deleted['pn'] == pn:
-                                oldid = deleted['id']
-                                layerrecipes_delete.remove(deleted)
-                                break
-                        if oldid > -1:
-                            # Reclaim a record we would have deleted
-                            results = Recipe.objects.filter(id=oldid)[:1]
-                            recipe = results[0]
-                            logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
-                        else:
-                            # Create new record
-                            logger.debug("Add new recipe %s" % added)
-                            recipe = Recipe()
-                        recipe.layerbranch = layerbranch
-                        recipe.filename = os.path.basename(added)
-                        root = os.path.dirname(added)
-                        recipe.filepath = os.path.relpath(root, layerdir)
-                        update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
-                        recipe.save()
-
-                    for deleted in layerrecipes_delete:
-                        logger.debug("Delete %s" % deleted)
-                        results = Recipe.objects.filter(id=deleted['id'])[:1]
-                        recipe = results[0]
-                        recipe.delete()
-
-                    # Save repo info
-                    layerbranch.vcs_last_rev = topcommit.hexsha
-                    layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
-                else:
-                    logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
-
-                layerbranch.vcs_last_fetch = datetime.now()
-                layerbranch.save()
-
-                if options.dryrun:
-                    transaction.rollback()
-                else:
-                    transaction.commit()
-
-                # Slightly hacky way of avoiding memory leaks
-                bb.event.ui_queue = []
-                bb.parse.parse_py.BBHandler.cached_statements = {}
-                bb.codeparser.codeparsercache = bb.codeparser.CodeParserCache()
-                if hasattr(bb.codeparser, 'codecache'):
-                    bb.codeparser.codecache = bb.codeparser.SetCache()
-                bb.fetch._checksum_cache = bb.checksum.FileChecksumCache()
-                bb.fetch.urldata_cache = {}
-                bb.fetch.saved_headrevs = {}
-                bb.parse.__pkgsplit_cache__={}
-                bb.parse.__mtime_cache = {}
-                bb.parse.init_parser(tinfoil.config_data)
-
-            except KeyboardInterrupt:
-                transaction.rollback()
-                logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
+            if layer.vcs_url in failedrepos:
+                logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
+
+            urldir = layer.get_fetch_dir()
+            repodir = os.path.join(fetchdir, urldir)
+
+            cmd = 'python update_layer.py -l %s -b %s' % (layer.name, options.branch)
+            if options.reload:
+                cmd += ' --reload'
+            if options.fullreload:
+                cmd += ' --fullreload'
+            if options.nocheckout:
+                cmd += ' --nocheckout'
+            if options.dryrun:
+                cmd += ' -n'
+            if options.loglevel == logging.DEBUG:
+                cmd += ' -d'
+            elif options.loglevel == logging.ERROR:
+                cmd += ' -q'
+            logger.debug('Running layer update command: %s' % cmd)
+            ret = run_command_interruptible(cmd)
+            if ret == 254:
+                # Interrupted by user, break out of loop
                 break
-            except:
-                import traceback
-                traceback.print_exc()
-                transaction.rollback()
-            finally:
-                transaction.leave_transaction_management()
 
     finally:
         utils.unlock_file(lockfile)
 
-    shutil.rmtree(tempdir)
     sys.exit(0)
 
 

547 lines not on this page · raw message

index d9480c9..1f42039 100644
@@ -630,29 +630,30 @@ def annotate_revision(sender, **kwargs):
     for ver, inst in zip(versions, instances):
         currentVersion = ver.field_dict
         modelmeta = ver.content_type.model_class()._meta
-        if ver.type == reversion.models.VERSION_DELETE:
-            changelist.append("Deleted %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
+        #FIXME modern django-reversion dropped the type field (argh!)
+        #if ver.type == reversion.models.VERSION_DELETE:
+        #    changelist.append("Deleted %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
+        #else:
+        pastver = reversion.get_for_object(inst)
+        if pastver:# and ver.type != reversion.models.VERSION_ADD:
+            pastVersion = pastver[0].field_dict
+            changes = set(currentVersion.items()) - set(pastVersion.items())
+            changedVars = [var[0] for var in changes]
+            fieldchanges = []
+            for field in changedVars:
+                if field not in ignorefields:
+                    modelfield = modelmeta.get_field(field)
+                    newvalue = currentVersion[field]
+                    if modelfield.choices:
+                        for v in modelfield.choices:
+                            if v[0] == newvalue:
+                                newvalue = v[1]
+                                break
+                    fieldchanges.append("%s to '%s'" % (modelfield.verbose_name.lower(), newvalue))
+            if fieldchanges:
+                changelist.append("Changed %s %s %s" % (modelmeta.verbose_name.lower(), ver.object_repr, ", ".join(fieldchanges)))
         else:
-            pastver = reversion.get_for_object(inst)
-            if pastver and ver.type != reversion.models.VERSION_ADD:
-                pastVersion = pastver[0].field_dict
-                changes = set(currentVersion.items()) - set(pastVersion.items())
-                changedVars = [var[0] for var in changes]
-                fieldchanges = []
-                for field in changedVars:
-                    if field not in ignorefields:
-                        modelfield = modelmeta.get_field(field)
-                        newvalue = currentVersion[field]
-                        if modelfield.choices:
-                            for v in modelfield.choices:
-                                if v[0] == newvalue:
-                                    newvalue = v[1]
-                                    break
-                        fieldchanges.append("%s to '%s'" % (modelfield.verbose_name.lower(), newvalue))
-                if fieldchanges:
-                    changelist.append("Changed %s %s %s" % (modelmeta.verbose_name.lower(), ver.object_repr, ", ".join(fieldchanges)))
-            else:
-                changelist.append("Added %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
+            changelist.append("Added %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
     comment = '\n'.join(changelist)
     if not comment:
         comment = 'No changes'

index db788df..b3374c9 100644
@@ -11,6 +11,7 @@ Setup
 
 In order to make use of this application you will need:
 
+* Python 3.4+
 * Django 1.6.x - tested with 1.6.10; newer versions may work, but
   the application has not been tested with 1.7 or newer.
 * For production usage, a web server set up to host Django applications

@@ -32,8 +33,9 @@ In order to make use of this application you will need:
   have to have Django installed, have the same or similar configuration
   in settings.py and have access to the database used by the web
   application):
-  * Python 2.7.3
-  * GitPython (python-git) version 0.3.1 or later
+  * Python 2.7.6+ / Python 3.4+ to match with the version of BitBake
+    for the OpenEmbedded branch being parsed
+  * GitPython (python-git) version 2.0 or later
 
 Setup instructions:
 

index 08d6d78..8102571 100644
@@ -50,7 +50,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
                     outfile = open(os.path.join(tmpoutdir, patchname), 'w')
                     last_layer = layer
                 recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
-                varlist = list(set(fields.keys() + meta_vars))
+                varlist = list(set(list(fields.keys()) + meta_vars))
                 varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
                 filevars = localise_file_vars(recipefile, varfiles, fields.keys())
                 for f, fvars in filevars.items():

@@ -66,7 +66,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
     ret = None
     if len(patches) > 1:
         (tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir)
-        tmptarfile = os.fdopen(tmptarfd, "w")
+        tmptarfile = os.fdopen(tmptarfd, "wb")
         tar = tarfile.open(None, "w:gz", tmptarfile)
         for patch in patches:
             patchfn = os.path.join(tmpoutdir, patch)

@@ -75,7 +75,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
         ret = tmptarname
     elif len(patches) == 1:
         (tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir)
-        tmppatchfile = os.fdopen(tmppatchfd, "w")
+        tmppatchfile = os.fdopen(tmppatchfd, "wb")
         with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile:
             shutil.copyfileobj(patchfile, tmppatchfile)
         tmppatchfile.close()

@@ -93,7 +93,7 @@ def patch_recipe(fn, relpath, values):
     remainingnames = {}
     for k in values.keys():
         remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
-    remainingnames = SortedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))
+    remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
 
     with tempfile.NamedTemporaryFile('w', delete=False) as tf:
         def outputvalue(name):

@@ -234,7 +234,7 @@ def main():
         utils.unlock_file(lockfile)
 
     if outp:
-        print outp
+        print(outp)
     else:
         sys.stderr.write("No changes to write\n")
         sys.exit(1)

index 8d82846..e753fbe 100644
@@ -26,7 +26,7 @@ class Branch(models.Model):
     class Meta:
         verbose_name_plural = "Branches"
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 

@@ -106,7 +106,7 @@ class LayerItem(models.Model):
     def get_absolute_url(self):
         return reverse('layer_item', args=('master',self.name));
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 

@@ -188,7 +188,7 @@ class LayerBranch(models.Model):
                     url = resolveComponents(url)
             return url
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s" % (self.layer.name, self.branch.name)
 
 

@@ -203,7 +203,7 @@ class LayerMaintainer(models.Model):
     responsibility = models.CharField(max_length=200, blank=True, help_text='Specific area(s) this maintainer is responsible for, if not the entire layer')
     status = models.CharField(max_length=1, choices=MAINTAINER_STATUS_CHOICES, default='A')
 
-    def __unicode__(self):
+    def __str__(self):
         respstr = ""
         if self.responsibility:
             respstr = " (%s)" % self.responsibility

@@ -217,7 +217,7 @@ class LayerDependency(models.Model):
     class Meta:
         verbose_name_plural = "Layer dependencies"
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s depends on %s" % (self.layerbranch.layer.name, self.dependency.name)
 
 

@@ -225,7 +225,7 @@ class LayerNote(models.Model):
     layer = models.ForeignKey(LayerItem)
     text = models.TextField()
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s" % (self.layer.name, self.text)
 
 

@@ -266,7 +266,7 @@ class Recipe(models.Model):
         else:
             return self.filename.split('_')[0]
 
-    def __unicode__(self):
+    def __str__(self):
         return os.path.join(self.filepath, self.filename)
 
 

@@ -278,7 +278,7 @@ class RecipeFileDependency(models.Model):
     class Meta:
         verbose_name_plural = "Recipe file dependencies"
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s' % self.path
 
 

@@ -341,7 +341,7 @@ class Machine(models.Model):
         url = self.layerbranch.file_url(os.path.join('conf/machine/%s.conf' % self.name))
         return url or ''
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s (%s)' % (self.name, self.layerbranch.layer.name)
 
 

@@ -367,7 +367,7 @@ class BBAppend(models.Model):
             return fnmatch.fnmatch(recipename, appendname.replace('%', '*'))
         return False
 
-    def __unicode__(self):
+    def __str__(self):
         return os.path.join(self.filepath, self.filename)
 
 

@@ -383,7 +383,7 @@ class BBClass(models.Model):
         url = self.layerbranch.file_url(os.path.join('classes', "%s.bbclass" % self.name))
         return url or ''
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s (%s)' % (self.name, self.layerbranch.layer.name)
 
 

@@ -391,7 +391,7 @@ class RecipeChangeset(models.Model):
     user = models.ForeignKey(User)
     name = models.CharField(max_length=255)
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s' % (self.name)
 
 

index 1925a27..43376cf 100644
@@ -82,10 +82,10 @@ def main():
         layerrecipes = Recipe.objects.filter(layer=layer)
         for recipe in layerrecipes:
             fullpath = str(os.path.join(repodir, layer.vcs_subdir, recipe.filepath, recipe.filename))
-            print fullpath
+            print(fullpath)
             try:
                 envdata = bb.cache.Cache.loadDataFull(fullpath, [], tinfoil.config_data)
-                print "DESCRIPTION = \"%s\"" % envdata.getVar("DESCRIPTION", True)
+                print("DESCRIPTION = \"%s\"" % envdata.getVar("DESCRIPTION", True))
             except Exception as e:
                 logger.info("Unable to read %s: %s", fullpath, str(e))
 

index 58a454a..cfebd41 100644
@@ -23,7 +23,7 @@ def addurlparameter(parser, token):
     from re import split
     bits = split(r'\s+', token.contents, 2)
     if len(bits) < 2:
-        raise TemplateSyntaxError, "'%s' tag requires two arguments" % bits[0]
+        raise TemplateSyntaxError("'%s' tag requires two arguments" % bits[0])
     return AddParameter(bits[1],bits[2])
 
 register.tag('addurlparameter', addurlparameter)

index a0771b4..b472d2a 100755
@@ -164,7 +164,7 @@ def get_github_layerinfo(layer_url, username = None, password = None):
         data = resp.read()
         json_data = json.loads(data)
         #headers = dict((key, value) for key, value in resp.getheaders())
-        #print headers
+        #print(headers)
         owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
         if resp.status in [200, 302]:
             owner_data = owner_resp.read()

index ee4138d..157d46f 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Fetch layer repositories and update layer index database
 #

index e7808e6..b170460 100644
@@ -11,7 +11,7 @@ from django.core.urlresolvers import reverse_lazy
 from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView
 from layerindex.models import LayerItem, Recipe, RecipeChangeset
 from rest_framework import routers
-import restviews
+from . import restviews
 from django.conf.urls import include
 
 router = routers.DefaultRouter()

index ebb89d3..72dc342 100644
@@ -6,7 +6,8 @@
 # Licensed under the MIT license, see COPYING.MIT for details
 
 import sys
-import os.path
+import os
+import tempfile
 import subprocess
 import logging
 import time

@@ -32,10 +33,10 @@ def runcmd(cmd, destdir=None, printerr=True, logger=None):
         return output if succeed
     """
     #logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
-    out = os.tmpfile()
+    out = tempfile.TemporaryFile()
     try:
         subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
-    except subprocess.CalledProcessError,e:
+    except subprocess.CalledProcessError as e:
         out.seek(0)
         if printerr:
             output = out.read()

index 1f42039..dfe4453 100644
@@ -4,6 +4,7 @@
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
+import sys
 from django.shortcuts import get_object_or_404, render
 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404
 from django.core.urlresolvers import reverse, reverse_lazy, resolve

@@ -26,7 +27,7 @@ from django.utils.decorators import method_decorator
 from django.contrib.auth.decorators import login_required
 from django.contrib import messages
 from reversion.models import Revision
-import simplesearch
+from . import simplesearch
 import settings
 from django.dispatch import receiver
 import reversion

@@ -212,14 +213,14 @@ def bulk_change_edit_view(request, template_name, pk):
 def bulk_change_patch_view(request, pk):
     import os
     import os.path
-    import utils
+    from layerindex.utils import runcmd
     changeset = get_object_or_404(RecipeChangeset, pk=pk)
     # FIXME this couples the web server and machine running the update script together,
     # but given that it's a separate script the way is open to decouple them in future
     try:
-        ret = utils.runcmd('python bulkchange.py %d %s' % (int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
+        ret = runcmd('%s bulkchange.py %d %s' % (sys.executable, int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
         if ret:
-            fn = ret.splitlines()[-1]
+            fn = ret.splitlines()[-1].decode('utf-8')
             if os.path.exists(fn):
                 if fn.endswith('.tar.gz'):
                     mimetype = 'application/x-gzip'

index ff0a7a5..db788df 100644
@@ -11,22 +11,22 @@ Setup
 
 In order to make use of this application you will need:
 
-* Django 1.4.x - tested with 1.4.1-1.4.10; newer versions may work, but
-  the application has not been tested with 1.5 or newer.
+* Django 1.6.x - tested with 1.6.10; newer versions may work, but
+  the application has not been tested with 1.7 or newer.
 * For production usage, a web server set up to host Django applications
   (not needed for local-only testing)
 * A database supported by Django (SQLite, MySQL, etc.). Django takes
   care of creating the database itself, you just need to ensure that the
   database server (if not using SQLite) is configured and running.
 * The following third-party Django modules (tested versions listed):
-  * django-south (0.8.4)
+  * django-south (1.0.2)
   * django-registration (1.0)
-  * django-reversion (1.7.1)
-  * django-reversion-compare (0.3.5)
-  * django-simple-captcha (0.4.1)
-  * django-nvd3 (0.6.0)
-  * djangorestframework (2.3.14)
-  * django-cors-headers (0.12)
+  * django-reversion (1.8.7)
+  * django-reversion-compare (0.4.0)
+  * django-simple-captcha (0.4.6)
+  * django-nvd3 (0.9.7)
+  * djangorestframework (3.2.5)
+  * django-cors-headers (1.1.0)
 * On the machine that will run the backend update script (which does not
   have to be the same machine as the web server, however it does still
   have to have Django installed, have the same or similar configuration

index 60653cf..e15dbeb 100644
@@ -6,7 +6,7 @@
 
 from layerindex.models import LayerItem, LayerBranch, LayerMaintainer, LayerNote, RecipeChangeset, RecipeChange, ClassicRecipe
 from django import forms
-from django.core.validators import URLValidator, RegexValidator, email_re
+from django.core.validators import URLValidator, RegexValidator, EmailValidator
 from django.forms.models import inlineformset_factory, modelformset_factory
 from captcha.fields import CaptchaField
 from django.contrib.auth.models import User

@@ -29,9 +29,8 @@ class LayerMaintainerForm(forms.ModelForm):
         if email:
             if len(email) < 7:
                 raise forms.ValidationError('%s is not a valid email address' % email)
-            reg = re.compile(email_re)
-            if not reg.match(email):
-                raise forms.ValidationError('%s is not a valid email address' % email)
+            val = EmailValidator()
+            val(email)
 
         return email
 

@@ -115,21 +114,21 @@ class EditLayerForm(forms.ModelForm):
     def clean_vcs_web_tree_base_url(self):
         url = self.cleaned_data['vcs_web_tree_base_url'].strip()
         if url:
-            val = URLValidator(verify_exists=False)
+            val = URLValidator()
             val(url)
         return url
 
     def clean_vcs_web_file_base_url(self):
         url = self.cleaned_data['vcs_web_file_base_url'].strip()
         if url:
-            val = URLValidator(verify_exists=False)
+            val = URLValidator()
             val(url)
         return url
 
     def clean_usage_url(self):
         usage = self.cleaned_data['usage_url'].strip()
         if usage.startswith('http'):
-            val = URLValidator(verify_exists=False)
+            val = URLValidator()
             val(usage)
         return usage
 

index 61698a9..b33d3d1 100644
@@ -1,6 +1,6 @@
-from models import Branch, LayerItem, LayerNote, LayerBranch, LayerDependency, Recipe, Machine
+from layerindex.models import Branch, LayerItem, LayerNote, LayerBranch, LayerDependency, Recipe, Machine
 from rest_framework import viewsets, serializers
-from querysethelper import params_to_queryset, get_search_tuple
+from layerindex.querysethelper import params_to_queryset, get_search_tuple
 
 class ParametricSearchableModelViewSet(viewsets.ModelViewSet):
     def get_queryset(self):

index 1bd4f0b..e7808e6 100644
@@ -4,9 +4,8 @@
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
-from django.conf.urls.defaults import *
-from django.views.generic import TemplateView, DetailView, ListView
-from django.views.generic.simple import redirect_to
+from django.conf.urls import *
+from django.views.generic import TemplateView, DetailView, ListView, RedirectView
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
 from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView

@@ -24,19 +23,20 @@ router.register(r'recipes', restviews.RecipeViewSet)
 router.register(r'machines', restviews.MachineViewSet)
 
 urlpatterns = patterns('',
-    url(r'^$', redirect_to, {'url' : reverse_lazy('layer_list', args=('master',))},
+    url(r'^$',
+        RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',))),
         name='frontpage'),
 
     url(r'^api/', include(router.urls)),
 
     url(r'^layers/$',
-        redirect_to, {'url' : reverse_lazy('layer_list', args=('master',))}),
+        RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)))),
     url(r'^layer/(?P<slug>[-\w]+)/$',
         RedirectParamsView.as_view(), {'redirect_name': 'layer_item', 'branch':'master'}),
     url(r'^recipes/$',
-        redirect_to, {'url' : reverse_lazy('recipe_search', args=('master',))}),
+        RedirectView.as_view(url=reverse_lazy('recipe_search', args=('master',)))),
     url(r'^machines/$',
-        redirect_to, {'url' : reverse_lazy('machine_search', args=('master',))}),
+        RedirectView.as_view(url=reverse_lazy('machine_search', args=('master',)))),
  
     url(r'^submit/$', edit_layer_view, {'template_name': 'layerindex/submitlayer.html'}, name="submit_layer"),
     url(r'^submit/thanks$',

@@ -107,7 +107,7 @@ urlpatterns = patterns('',
             template_name='layerindex/about.html'),
             name="about"),
     url(r'^oe-classic/$',
-        redirect_to, {'url' : reverse_lazy('classic_recipe_search')},
+        RedirectView.as_view(url=reverse_lazy('classic_recipe_search')),
             name='classic'),
     url(r'^oe-classic/recipes/$',
         ClassicRecipeSearchView.as_view(

index b0f577c..ab5e2d5 100644
@@ -4,8 +4,7 @@
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
-from django.conf.urls.defaults import *
-from django.views.generic.simple import redirect_to
+from django.conf.urls import *
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
 from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView

index 440cc88..ebb89d3 100644
@@ -57,12 +57,6 @@ def setup_django():
     sys.path.append(newpath)
     os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
 
-    from django.core.management import setup_environ
-    from django.conf import settings
-    import settings
-
-    setup_environ(settings)
-
 def logger_create(name):
     logger = logging.getLogger(name)
     loggerhandler = logging.StreamHandler()

index 0b219fb..0a64562 100644
@@ -6,20 +6,14 @@
 #
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
-
+#!/usr/bin/env python
 import os
-
-from django.core.management import execute_manager
-import imp
-try:
-    imp.find_module('settings') # Assumed to be in the same directory.
-except ImportError:
-    import sys
-    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__)
-    sys.exit(1)
-
-import settings
+import sys
 
 if __name__ == "__main__":
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
-    execute_manager(settings)
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)
+

index cde88a0..ea5beee 100644
@@ -1,20 +1,20 @@
-Django>=1.4,<1.5
-GitPython>=0.3.7
-Jinja2==2.7.3
+Django==1.6.11
+django-cors-headers==1.1.0
+django-nvd3==0.9.7
+django-registration==1.0
+django-reversion==1.8.7
+django-reversion-compare==0.4.0
+django-simple-captcha==0.4.6
+djangorestframework==3.2.5
+gitdb==0.6.4
+GitPython==2.0.5
+Jinja2==2.8
 MarkupSafe==0.23
-Pillow>=2.4.0
-South==0.8.4
-Unidecode==0.04.16
-argparse==1.2.1
-awesome-slugify==1.5
-django-cors-headers==0.12
-django-nvd3==0.7.4
-django-registration==0.8
-django-reversion==1.6.6
-django-reversion-compare==0.3.5
-django-simple-captcha==0.4.2
-djangorestframework==2.3.14
-python-nvd3==0.12.2
-regex==2014.06.28
-six==1.7.3
-wsgiref==0.1.2
+mysqlclient==1.3.7
+Pillow==3.2.0
+python-nvd3==0.14.2
+python-slugify==1.1.4
+six==1.10.0
+smmap==0.9.0
+South==1.0.2
+Unidecode==0.4.19

index b21a5b4..b731a6b 100644
@@ -131,7 +131,7 @@ TEMPLATE_DIRS = (
     # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
     # Always use forward slashes, even on Windows.
     # Don't forget to use absolute paths, not relative paths.
-    BASE_DIR + "/templates"
+    BASE_DIR + "/templates",
 )
 
 INSTALLED_APPS = (

index 843d5bc..5bd5f75 100644
@@ -22,7 +22,7 @@
 
 <p>The page you requested was not found.</p>
 
-<p><a href="{% url frontpage %}">Return to the front page</a></p>
+<p><a href="{% url 'frontpage' %}">Return to the front page</a></p>
 
 {% endautoescape %}
 {% endblock %}

index c872383..fedcfe2 100644
@@ -28,13 +28,13 @@
     <div class="navbar navbar-fixed-top">
         <div class="navbar-inner">
             <div class="container-fluid">
-                <a class="brand" href="{% url frontpage %}">{{ site_name }}</a>
+                <a class="brand" href="{% url 'frontpage' %}">{{ site_name }}</a>
 
                 {% if user.is_authenticated %}
                     <div class="btn-group pull-right">
                         {% if perms.layerindex.publish_layer %}
                         {% if unpublished_count > 0 %}
-                        <a class="btn" href="{% url layer_list_review %}?branch=master">
+                        <a class="btn" href="{% url 'layer_list_review' %}?branch=master">
                             <span class="badge badge-warning review-notification">{{ unpublished_count }}</span>
                         </a>
                         {% endif %}

@@ -45,14 +45,14 @@
                             <b class="caret"></b>
                         </button>
                         <ul class="dropdown-menu">
-                            <li><a href="{% url auth_logout %}">{% trans "Log out" %}</a></li>
-                            <li><a href="{% url auth_password_change %}">{% trans "Change password" %}</a></li>
-                            <li><a href="{% url profile %}">{% trans "Edit profile" %}</a></li>
+                            <li><a href="{% url 'auth_logout' %}">{% trans "Log out" %}</a></li>
+                            <li><a href="{% url 'auth_password_change' %}">{% trans "Change password" %}</a></li>
+                            <li><a href="{% url 'profile' %}">{% trans "Edit profile" %}</a></li>
                         </ul>
                     </div>
                 {% else %}
                     <div class="pull-right">
-                        <a class="btn" href="{% url auth_login %}">{% trans "Log in" %}</a>
+                        <a class="btn" href="{% url 'auth_login' %}">{% trans "Log in" %}</a>
                     </div>
                 {% endif %}
                 <ul class="nav pull-right">

@@ -60,7 +60,7 @@
                 </ul>
                 {% block topfunctions %}
                 <div class="pull-right nav-spacer">
-                    <a class="btn btn-info" href="{% url submit_layer %}">Submit layer</a>
+                    <a class="btn btn-info" href="{% url 'submit_layer' %}">Submit layer</a>
                 </div>
                 <ul class="nav pull-right">
                     {% if user.is_authenticated %}

@@ -70,8 +70,8 @@
                         <b class="caret"></b>
                         </a>
                         <ul class="dropdown-menu">
-                            <li><a href="{% url bulk_change %}">Bulk Change</a></li>
-                            <li><a href="{% url duplicates 'master' %}">Duplicates</a></li>
+                            <li><a href="{% url 'bulk_change' %}">Bulk Change</a></li>
+                            <li><a href="{% url 'duplicates' 'master' %}">Duplicates</a></li>
                         </ul>
                     </li>
                     {% endif %}

@@ -96,8 +96,8 @@
         {% block footer %}
         <hr />
         <div class="footer">
-            <a href="{% url history_list %}">change history</a>
-            &bull; <a href="{% url about %}">about this site</a>
+            <a href="{% url 'history_list' %}">change history</a>
+            &bull; <a href="{% url 'about' %}">about this site</a>
             &bull; <a href="http://www.openembedded.org/Layers_FAQ">FAQ</a>
         </div>
 

index d9ea7bb..c86ccc2 100644
@@ -1,7 +1,6 @@
 {% extends "base.html" %}
 {% load i18n %}
 
-{% load url from future %}
 {% comment %}
 
   layerindex-web - top level page template

@@ -56,4 +55,4 @@
         </div>
     </div>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}

index 67c646a..3e98825 100644
@@ -11,7 +11,7 @@
 
 <p>This website indexes layers for the <a href="http://www.openembedded.org">OpenEmbedded</a> build system provided by members of the OpenEmbedded / Yocto Project community, suitable for use on top of OpenEmbedded-Core and compatible systems, providing additional recipes, machine support and/or distro policy configuration.</p>
 
-<p>If you have a layer for use with OpenEmbedded that you wish to share with others, please <a href="{% url submit_layer %}">submit it</a>!</p>
+<p>If you have a layer for use with OpenEmbedded that you wish to share with others, please <a href="{% url 'submit_layer' %}">submit it</a>!</p>
 
 <h3>Technologies</h3>
 

index e7fc270..e8936de 100644
@@ -29,7 +29,7 @@ generate a patch for these changes which can be submitted for merging.</p>
     <h3>Select an existing changeset</h3>
     <ul>
     {% for changeset in changesets %}
-        <li><a href="{% url bulk_change_search changeset.id %}">{{ changeset.name }}</a></li>
+        <li><a href="{% url 'bulk_change_search' changeset.id %}">{{ changeset.name }}</a></li>
     {% endfor %}
     </ul>
 {% endif %}

index 5fb971e..a95bb5d 100644
@@ -37,10 +37,10 @@
     </li>
 {% endfor %}
 </ul>
-<a href="{% url bulk_change_search changeset.id %}" class="btn">Add recipes</a>
-<a href="{% url bulk_change_edit changeset.id %}" class="btn">Edit</a>
-<a href="{% url bulk_change_patches changeset.id %}" class="btn">Get patches</a>
-<a href="{% url bulk_change_delete changeset.id %}?cancel=bulk_change_review" class="btn">Delete</a>
+<a href="{% url 'bulk_change_search' changeset.id %}" class="btn">Add recipes</a>
+<a href="{% url 'bulk_change_edit' changeset.id %}" class="btn">Edit</a>
+<a href="{% url 'bulk_change_patches' changeset.id %}" class="btn">Get patches</a>
+<a href="{% url 'bulk_change_delete' changeset.id %}?cancel=bulk_change_review" class="btn">Delete</a>
 
 {% endautoescape %}
 

index 134f9a3..0cb83ff 100644
@@ -78,10 +78,10 @@
                             {% for recipe in recipe_list %}
                                 <tr>
                                     <td><input type="checkbox" name="selecteditems" value="{{ recipe.id }}"></input></td>
-                                    <td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a></td>
+                                    <td><a href="{% url 'recipe' recipe.id %}">{{ recipe.name }}</a></td>
                                     <td>{{ recipe.pv }}</td>
                                     <td>{{ recipe.short_desc }}</td>
-                                    <td><a href="{% url layer_item current_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
+                                    <td><a href="{% url 'layer_item' current_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
                                 </tr>
                             {% endfor %}
                         </tbody>

@@ -120,9 +120,9 @@
                             </ul>
                         </small>
                         <input type="submit" class="btn" name="remove_all" value="Remove all"></input>
-                        <a href="{% url bulk_change_edit changeset.id %}" class="btn">Edit</a>
+                        <a href="{% url 'bulk_change_edit' changeset.id %}" class="btn">Edit</a>
                         {% endif %}
-                        <a href="{% url bulk_change_delete changeset.id %}?cancel=bulk_change_search" class="btn">Delete</a>
+                        <a href="{% url 'bulk_change_delete' changeset.id %}?cancel=bulk_change_search" class="btn">Delete</a>
                     </div>
                     {% endif %}
                 </form>

index eab56f2..7ba63f5 100644
@@ -24,7 +24,7 @@
                             </a>
                             <ul class="dropdown-menu">
                                 {% for branch in all_branches %}
-                                    <li><a href="{% url layer_list branch.name %}">
+                                    <li><a href="{% url 'layer_list' branch.name %}">
                                         {{ branch.name }}
                                         {% if branch.short_description %}
                                         ({{ branch.short_description }})

@@ -32,7 +32,7 @@
                                     </a></li>
                                 {% endfor %}
                                 <li class="divider"></li>
-                                <li><a href="{% url classic %}"><b>OE-Classic</b></a></li>
+                                <li><a href="{% url 'classic' %}"><b>OE-Classic</b></a></li>
                             </ul>
                         </li>
 {% endautoescape %}

@@ -48,9 +48,9 @@
 
                 <p>OpenEmbedded-Classic (OE-Classic) is the name for the old monolithic version of OpenEmbedded. It contained a number of recipes some of which have not yet been migrated on top of OE-Core. To help people to find and migrate these recipes we provide an index here as well as some statistics to get an idea of the migration.</p>
 
-                <a class="btn btn-large btn-primary" href="{% url classic_recipe_search %}">Recipes</a>
-                <a class="btn btn-large" href="{% url classic_recipe_search %}?q=&cover_status=!">Unmigrated Recipes</a>
-                <a class="btn btn-large btn-primary" href="{% url classic_recipe_stats %}">Stats</a>
+                <a class="btn btn-large btn-primary" href="{% url 'classic_recipe_search' %}">Recipes</a>
+                <a class="btn btn-large" href="{% url 'classic_recipe_search' %}?q=&cover_status=!">Unmigrated Recipes</a>
+                <a class="btn btn-large btn-primary" href="{% url 'classic_recipe_stats' %}">Stats</a>
             </div>
 
         </div>

index 9df9a84..45d845f 100644
@@ -21,7 +21,7 @@
 {% autoescape on %}
 
         <ul class="breadcrumb">
-            <li><a href="{% url classic_recipe_search %}">OE-Classic</a> <span class="divider">&rarr;</span></li>
+            <li><a href="{% url 'classic_recipe_search' %}">OE-Classic</a> <span class="divider">&rarr;</span></li>
             <li class="active">{{ recipe.name }}</li>
         </ul>
 

index 7fb62be..799dbb4 100644
@@ -17,8 +17,8 @@
 
 {% block navs %}
 {% autoescape on %}
-                            <li class="active"><a href="{% url classic_recipe_search %}">Recipes</a></li>
-                            <li><a href="{% url classic_recipe_stats %}">Stats</a></li>
+                            <li class="active"><a href="{% url 'classic_recipe_search' %}">Recipes</a></li>
+                            <li><a href="{% url 'classic_recipe_stats' %}">Stats</a></li>
 {% endautoescape %}
 {% endblock %}
 

@@ -31,7 +31,7 @@
                 <h2>OE-Classic recipes</h2>
 
                 <div class="alert alert-warning">
-                    <b>NOTE:</b> This is the recipe search for OE-Classic, the older monolithic version of OpenEmbedded which is no longer actively developed. <a href="{% url recipe_search 'master' %}">Click here</a> to search current recipes.
+                    <b>NOTE:</b> This is the recipe search for OE-Classic, the older monolithic version of OpenEmbedded which is no longer actively developed. <a href="{% url 'recipe_search' 'master' %}">Click here</a> to search current recipes.
                 </div>
 
                 <div class="row-fluid">

@@ -72,13 +72,13 @@
                     <tbody>
                         {% for recipe in recipe_list %}
                             <tr {% if recipe.preferred_count > 0 %}class="muted"{% endif %}>
-                                <td><a href="{% url classic_recipe recipe.id %}">{{ recipe.name }}</a></td>
+                                <td><a href="{% url 'classic_recipe' recipe.id %}">{{ recipe.name }}</a></td>
                                 <td>{{ recipe.pv }}</td>
                                 <td>{{ recipe.short_desc }}</td>
                                 <td>{{ recipe.get_cover_desc }}</td>
                                 <td>{{ recipe.classic_category }}</td>
                                 {% if multi_classic_layers %}
-                                <td><a href="{% url layer_item recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
+                                <td><a href="{% url 'layer_item' recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
                                 {% endif %}
                             </tr>
                         {% endfor %}

index 2fde01b..63d0b43 100644
@@ -19,8 +19,8 @@
 
 {% block navs %}
 {% autoescape on %}
-                            <li><a href="{% url classic_recipe_search %}">Recipes</a></li>
-                            <li class="active"><a href="{% url classic_recipe_stats %}">Stats</a></li>
+                            <li><a href="{% url 'classic_recipe_search' %}">Recipes</a></li>
+                            <li class="active"><a href="{% url 'classic_recipe_stats' %}">Stats</a></li>
 {% endautoescape %}
 {% endblock %}
 

index feb2c6c..d0d11c0 100644
@@ -22,7 +22,7 @@
 
 {% autoescape on %}
         <ul class="breadcrumb">
-            <li><a href="{% url layer_list url_branch %}">{{ layerbranch.branch.name }}</a> <span class="divider">&rarr;</span></li>
+            <li><a href="{% url 'layer_list' url_branch %}">{{ layerbranch.branch.name }}</a> <span class="divider">&rarr;</span></li>
             <li class="active">{{ layeritem.name }}</li>
         </ul>
 

@@ -36,9 +36,9 @@
                         {% if user.is_authenticated %}
                             <span class="pull-right">
                                 {% if perms.layerindex.publish_layer or useredit %}
-                                    <a href="{% url edit_layer url_branch layeritem.name %}" class="btn">Edit layer</a>
+                                    <a href="{% url 'edit_layer' url_branch layeritem.name %}" class="btn">Edit layer</a>
                                     {% if layeritem.layernote_set.count = 0 %}
-                                        <a href="{% url add_layernote layeritem.name %}" class="btn">Add note</a>
+                                        <a href="{% url 'add_layernote' layeritem.name %}" class="btn">Add note</a>
                                     {% endif %}
                                 {% endif %}
                             </span>

@@ -63,8 +63,8 @@
                         {% if perms.layerindex.publish_layer or useredit %}
                             <br><br>
                             <p>
-                                <a href="{% url edit_layernote layeritem.name note.pk %}" class="btn">Edit note</a>
-                                <a href="{% url delete_layernote layeritem.name note.pk %}" class='btn'>Delete note</a>
+                                <a href="{% url 'edit_layernote' layeritem.name note.pk %}" class="btn">Edit note</a>
+                                <a href="{% url 'delete_layernote' layeritem.name note.pk %}" class='btn'>Delete note</a>
                             </p>
                         {% endif %}
                     </div>

@@ -144,7 +144,7 @@
                             <p>The {{ layeritem.name }} layer depends upon:</p>
                             <ul>
                                 {% for dep in layerbranch.dependencies_set.all %}
-                                    <li><a href="{% url layer_item url_branch dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
+                                    <li><a href="{% url 'layer_item' url_branch dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
                                 {% endfor %}
                             </ul>
                          </div> <!-- end of well -->

@@ -198,7 +198,7 @@
                         <tbody>
                             {% for recipe in layerbranch.sorted_recipes %}
                                 <tr>
-                                    <td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
+                                    <td><a href="{% url 'recipe' recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
                                     <td>{{ recipe.pv }}</td>
                                     <td class="span8">{{ recipe.short_desc }}</td>
                                 </tr>

index 116aab9..0ebfe11 100644
@@ -1,6 +1,5 @@
 {% extends "base_toplevel.html" %}
 {% load i18n %}
-{% load url from future %}
 
 {% comment %}
 

index 1c7693c..abf8299 100644
@@ -25,7 +25,7 @@
     {% csrf_token %}
     {{ form.as_p }}
 <input type="submit" value="Save" class='btn' />
-<a href="{% url layer_item 'master' form.instance.layer.name %}" class='btn'>Cancel</a>
+<a href="{% url 'layer_item' 'master' form.instance.layer.name %}" class='btn'>Cancel</a>
 </form>
 
 {% endautoescape %}

index 4ca43fc..cfc4ebd 100644
@@ -18,9 +18,9 @@
 
 {% block navs %}
 {% autoescape on %}
-                            <li class="active"><a href="{% url layer_list url_branch %}">Layers</a></li>
-                            <li><a href="{% url recipe_search url_branch %}">Recipes</a></li>
-                            <li><a href="{% url machine_search url_branch %}">Machines</a></li>
+                            <li class="active"><a href="{% url 'layer_list' url_branch %}">Layers</a></li>
+                            <li><a href="{% url 'recipe_search' url_branch %}">Recipes</a></li>
+                            <li><a href="{% url 'machine_search' url_branch %}">Machines</a></li>
 {% endautoescape %}
 {% endblock %}
 

@@ -71,7 +71,7 @@
                 <tbody>
                     {% for layerbranch in layerbranch_list %}
                     <tr class="layertype_{{ layerbranch.layer.layer_type }}">
-                        <td><a href="{% url layer_item url_branch layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
+                        <td><a href="{% url 'layer_item' url_branch layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
                         <td>{{ layerbranch.layer.summary }}</td>
                         <td>{{ layerbranch.layer.get_layer_type_display }}</td>
                         <td class="showRollie">

index 1851937..e31433c 100644
@@ -17,9 +17,9 @@
 
 {% block navs %}
 {% autoescape on %}
-                            <li><a href="{% url layer_list url_branch %}">Layers</a></li>
-                            <li><a href="{% url recipe_search url_branch %}">Recipes</a></li>
-                            <li class="active"><a href="{% url machine_search url_branch %}">Machines</a></li>
+                            <li><a href="{% url 'layer_list' url_branch %}">Layers</a></li>
+                            <li><a href="{% url 'recipe_search' url_branch %}">Recipes</a></li>
+                            <li class="active"><a href="{% url 'machine_search' url_branch %}">Machines</a></li>
 {% endautoescape %}
 {% endblock %}
 

@@ -30,7 +30,7 @@
 
                 <div class="row-fluid">
                     <div class="input-append">
-                        <form id="filter-form" action="{% url machine_search url_branch %}" method="get">
+                        <form id="filter-form" action="{% url 'machine_search' url_branch %}" method="get">
                             <input type="text" class="input-xxlarge" id="appendedInputButtons" placeholder="Search machines" name="q" value="{{ search_keyword }}" />
                             <button class="btn" type="submit">search</button>
                         </form>

@@ -52,7 +52,7 @@
                             <tr>
                                 <td><a href="{{ machine.vcs_web_url }}">{{ machine.name }}</a></td>
                                 <td>{{ machine.description }}</td>
-                                <td><a href="{% url layer_item url_branch machine.layerbranch.layer.name %}">{{ machine.layerbranch.layer.name }}</a></td>
+                                <td><a href="{% url 'layer_item' url_branch machine.layerbranch.layer.name %}">{{ machine.layerbranch.layer.name }}</a></td>
                             </tr>
                         {% endfor %}
                     </tbody>

index 8edc451..d6d25ce 100644
@@ -42,7 +42,7 @@
     {% endfor %}
 
   <input type="submit" class="btn" value="{% trans 'Save' %}" />
-  <a class="btn" href="{% url frontpage %}">{% trans 'Cancel' %}</a>
+  <a class="btn" href="{% url 'frontpage' %}">{% trans 'Cancel' %}</a>
   {% csrf_token %}
 </form>
 

index 7dbb362..c0199a1 100644
@@ -21,8 +21,8 @@
 {% autoescape on %}
 
         <ul class="breadcrumb">
-            <li><a href="{% url layer_list recipe.layerbranch.branch.name %}">{{ recipe.layerbranch.branch.name }}</a> <span class="divider">&rarr;</span></li>
-            <li><a href="{% url layer_item recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> <span class="divider">&rarr;</span></li>
+            <li><a href="{% url 'layer_list' recipe.layerbranch.branch.name %}">{{ recipe.layerbranch.branch.name }}</a> <span class="divider">&rarr;</span></li>
+            <li><a href="{% url 'layer_item' recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> <span class="divider">&rarr;</span></li>
             <li class="active">{{ recipe.name }}</li>
         </ul>
 

@@ -36,7 +36,7 @@
             {% if recipe.blacklisted %}
                 <div class="alert">
                     <div class="row-fluid">
-                        <p>This recipe is <strong>blacklisted</strong> by the <a href="{% url layer_item recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> layer. The reason provided is:</p>
+                        <p>This recipe is <strong>blacklisted</strong> by the <a href="{% url 'layer_item' recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> layer. The reason provided is:</p>
                         <blockquote class="span7 warn">
                             <p>{{ recipe.blacklisted }}</p>
                         </blockquote>

@@ -92,7 +92,7 @@
                         </tr>
                         <tr>
                             <th>Layer</th>
-                            <td><a href="{% url layer_item recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> ({{ recipe.layerbranch.branch.name}} branch)</td>
+                            <td><a href="{% url 'layer_item' recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> ({{ recipe.layerbranch.branch.name}} branch)</td>
                         </tr>
                         <tr>
                             <th>Inherits</th>

@@ -116,7 +116,7 @@
                 {% for append in verappends %}
                     <tr>
                         <td>
-                            <a href="{% url layer_item append.layerbranch.branch.name append.layerbranch.layer.name %}">{{ append.layerbranch.layer.name }}</a>
+                            <a href="{% url 'layer_item' append.layerbranch.branch.name append.layerbranch.layer.name %}">{{ append.layerbranch.layer.name }}</a>
                         </td>
                         <td>
                             <a href="{{ append.vcs_web_url }}">{{ append.filename }}</a>

@@ -127,7 +127,7 @@
                     {% if not append in verappends %}
                     <tr>
                         <td>
-                            <a href="{% url layer_item append.layerbranch.branch.name append.layerbranch.layer.name %}" class="muted">{{ append.layerbranch.layer.name }}</a>
+                            <a href="{% url 'layer_item' append.layerbranch.branch.name append.layerbranch.layer.name %}" class="muted">{{ append.layerbranch.layer.name }}</a>
                         </td>
                         <td>
                             <a href="{{ append.vcs_web_url }}" class="muted">{{ append.filename }}</a>

index c2141a4..74f3bb4 100644
@@ -17,9 +17,9 @@
 
 {% block navs %}
 {% autoescape on %}
-                            <li><a href="{% url layer_list url_branch %}">Layers</a></li>
-                            <li class="active"><a href="{% url recipe_search url_branch %}">Recipes</a></li>
-                            <li><a href="{% url machine_search url_branch %}">Machines</a></li>
+                            <li><a href="{% url 'layer_list' url_branch %}">Layers</a></li>
+                            <li class="active"><a href="{% url 'recipe_search' url_branch %}">Recipes</a></li>
+                            <li><a href="{% url 'machine_search' url_branch %}">Machines</a></li>
 {% endautoescape %}
 {% endblock %}
 

@@ -31,7 +31,7 @@
 
                 <div class="row-fluid">
                     <div class="input-append">
-                        <form id="filter-form" action="{% url recipe_search url_branch %}" method="get">
+                        <form id="filter-form" action="{% url 'recipe_search' url_branch %}" method="get">
                             <input type="text" class="input-xxlarge" id="appendedInputButtons" placeholder="Search recipes" name="q" value="{{ search_keyword }}" />
                             <button class="btn" type="submit">search</button>
                         </form>

@@ -52,10 +52,10 @@
                     <tbody>
                         {% for recipe in recipe_list %}
                             <tr {% if recipe.preferred_count > 0 %}class="muted"{% endif %}>
-                                <td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
+                                <td><a href="{% url 'recipe' recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
                                 <td>{{ recipe.pv }}</td>
                                 <td>{{ recipe.short_desc }}</td>
-                                <td><a href="{% url layer_item url_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
+                                <td><a href="{% url 'layer_item' url_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
                             </tr>
                         {% endfor %}
                     </tbody>

index 5840e25..cff1731 100644
@@ -35,14 +35,14 @@
                         {% if user.is_authenticated %}
                             <span class="pull-right">
                                 {% if perms.layerindex.publish_layer or useredit %}
-                                    <a href="{% url edit_layer 'master' layeritem.name %}?returnto=layer_review" class="btn">Edit layer</a>
+                                    <a href="{% url 'edit_layer' 'master' layeritem.name %}?returnto=layer_review" class="btn">Edit layer</a>
                                     {% if layeritem.layernote_set.count = 0 %}
-                                        <a href="{% url add_layernote layeritem.name %}" class="btn">Add note</a>
+                                        <a href="{% url 'add_layernote' layeritem.name %}" class="btn">Add note</a>
                                     {% endif %}
                                 {% endif %}
                                 {% if layeritem.status = "N" and perms.layerindex.publish_layer %}
-                                    <a href="{% url delete_layer layeritem.name %}" class="btn btn-warning">Delete layer</a>
-                                    <a href="{% url publish layeritem.name %}" class="btn btn-primary">Publish layer</a>
+                                    <a href="{% url 'delete_layer' layeritem.name %}" class="btn btn-warning">Delete layer</a>
+                                    <a href="{% url 'publish' layeritem.name %}" class="btn btn-primary">Publish layer</a>
                                 {% endif %}
                             </span>
                         {% endif %}

@@ -58,8 +58,8 @@
                         <p>{{ note.text }}</p>
                         {% if perms.layerindex.publish_layer or useredit %}
                             <p>
-                                <a href="{% url edit_layernote layeritem.name note.pk %}" class="btn">Edit note</a>
-                                <a href="{% url delete_layernote layeritem.name note.pk %}" class='btn'>Delete note</a>
+                                <a href="{% url 'edit_layernote' layeritem.name note.pk %}" class="btn">Edit note</a>
+                                <a href="{% url 'delete_layernote' layeritem.name note.pk %}" class='btn'>Delete note</a>
                             </p>
                         {% endif %}
                     </div>

@@ -163,7 +163,7 @@
                                 <td>
                                     <ul class="unstyled">
                                         {% for dep in layerbranch.dependencies_set.all %}
-                                            <li><a href="{% url layer_item 'master' dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
+                                            <li><a href="{% url 'layer_item' 'master' dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
                                         {% endfor %}
                                     </ul>
                                 </td>

index d20001a..eaa24e4 100644
@@ -36,7 +36,7 @@
                 <tbody>
                     {% for layerbranch in layerbranch_list %}
                     <tr class="layertype_{{ layerbranch.layer.layer_type }}">
-                        <td><a href="{% url layer_review layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
+                        <td><a href="{% url 'layer_review' layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
                         <td>{{ layerbranch.layer.summary }}</td>
                         <td>{{ layerbranch.layer.get_layer_type_display }}</td>
                         <td class="showRollie">

index e85121e..6e24890 100644
@@ -7,7 +7,7 @@
 
 <p>{% trans "Account successfully activated" %}</p>
 
-<p><a href="{% url auth_login %}">{% trans "Log in" %}</a></p>
+<p><a href="{% url "auth_login" %}">{% trans "Log in" %}</a></p>
 
 {% else %}
 

index 0820fce..736ea8a 100644
@@ -7,7 +7,7 @@ link is valid for {{ expiration_days }} days.
 
 {% endblocktrans %}
 
-http://{{ site.domain }}{% url registration_activate activation_key %}
+http://{{ site.domain }}{% url "registration_activate" activation_key %}
 
 {% blocktrans %}
 If you did not make this request, please ignore this message.

index 7e3d615..7f8a153 100644
@@ -10,8 +10,8 @@
   {% csrf_token %}
 </form>
 
-<p>{% trans "Forgot password" %}? <a href="{% url auth_password_reset %}">{% trans "Reset it" %}</a>!</p>
-<p>{% trans "Don't have an account" %}? <a href="{% url registration_register %}">{% trans "Create one now" %}</a>!</p>
+<p>{% trans "Forgot password" %}? <a href="{% url "auth_password_reset" %}">{% trans "Reset it" %}</a>!</p>
+<p>{% trans "Don't have an account" %}? <a href="{% url "registration_register" %}">{% trans "Create one now" %}</a>!</p>
 {% endblock %}
 
 

index ef3637c..d9f3879 100644
@@ -5,6 +5,6 @@
 
 <p>{% trans "Password reset successfully" %}</p>
 
-<p><a href="{% url auth_login %}">{% trans "Log in" %}</a></p>
+<p><a href="{% url "auth_login" %}">{% trans "Log in" %}</a></p>
 
 {% endblock %}

index a55c869..f219c68 100644
@@ -1,5 +1,5 @@
 {% load i18n %}
 {% blocktrans %}Reset password at {{ site_name }}{% endblocktrans %}:
 {% block reset_link %}
-{{ protocol }}://{{ domain }}{% url auth_password_reset_confirm uidb36=uid, token=token %}
+{{ protocol }}://{{ domain }}{% url "auth_password_reset_confirm" uidb36=uid, token=token %}
 {% endblock %}

index aa4f8c5..98fc734 100644
@@ -5,8 +5,8 @@
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
 
-from django.conf.urls.defaults import patterns, include, url
-from django.views.generic.simple import redirect_to
+from django.conf.urls import patterns, include, url
+from django.views.generic import RedirectView
 
 from django.contrib import admin
 admin.autodiscover()

@@ -16,6 +16,6 @@ urlpatterns = patterns('',
     url(r'^admin/', include(admin.site.urls)),
     url(r'^accounts/', include('registration.backends.default.urls')),
     url(r'^captcha/', include('captcha.urls')),
-    url(r'.*', redirect_to, {'url' : '/layerindex/'})
+    url(r'.*', RedirectView.as_view(url='/layerindex/')),
 )
 

index bf1dbb2..ac4ae1e 100644
@@ -312,9 +312,9 @@ def main():
             # Check if any paths should be ignored because there are layers within this layer
             removedirs = []
             for root, dirs, files in os.walk(layerdir):
-                for d in dirs:
-                    if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
-                        removedirs.append(os.path.join(root, d) + os.sep)
+                for diritem in dirs:
+                    if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
+                        removedirs.append(os.path.join(root, diritem) + os.sep)
 
             if diff:
                 # Apply git changes to existing recipe list

@@ -325,12 +325,12 @@ def main():
                     subdir_start = ""
 
                 updatedrecipes = set()
-                for d in diff.iter_change_type('D'):
-                    path = d.a_blob.path
+                for diffitem in diff.iter_change_type('D'):
+                    path = diffitem.a_blob.path
                     if path.startswith(subdir_start):
                         skip = False
-                        for d in removedirs:
-                            if path.startswith(d):
+                        for removedir in removedirs:
+                            if path.startswith(removedir):
                                 skip = True
                                 break
                         if skip:

@@ -351,12 +351,12 @@ def main():
                         elif typename == 'bbclass':
                             layerclasses.filter(name=filename).delete()
 
-                for d in diff.iter_change_type('A'):
-                    path = d.b_blob.path
+                for diffitem in diff.iter_change_type('A'):
+                    path = diffitem.b_blob.path
                     if path.startswith(subdir_start):
                         skip = False
-                        for d in removedirs:
-                            if path.startswith(d):
+                        for removedir in removedirs:
+                            if path.startswith(removedir):
                                 skip = True
                                 break
                         if skip:

@@ -385,12 +385,12 @@ def main():
                             bbclass.save()
 
                 dirtyrecipes = set()
-                for d in diff.iter_change_type('M'):
-                    path = d.a_blob.path
+                for diffitem in diff.iter_change_type('M'):
+                    path = diffitem.a_blob.path
                     if path.startswith(subdir_start):
                         skip = False
-                        for d in removedirs:
-                            if path.startswith(d):
+                        for removedir in removedirs:
+                            if path.startswith(removedir):
                                 skip = True
                                 break
                         if skip:

@@ -432,8 +432,8 @@ def main():
                         fullpath = os.path.join(root, v['filename'])
                         preserve = True
                         if os.path.exists(fullpath):
-                            for d in removedirs:
-                                if fullpath.startswith(d):
+                            for removedir in removedirs:
+                                if fullpath.startswith(removedir):
                                     preserve = False
                                     break
                         else:

@@ -455,10 +455,10 @@ def main():
                 for root, dirs, files in os.walk(layerdir):
                     if '.git' in dirs:
                         dirs.remove('.git')
-                    for d in dirs[:]:
-                        fullpath = os.path.join(root, d) + os.sep
+                    for diritem in dirs[:]:
+                        fullpath = os.path.join(root, diritem) + os.sep
                         if fullpath in removedirs:
-                            dirs.remove(d)
+                            dirs.remove(diritem)
                     for f in files:
                         fullpath = os.path.join(root, f)
                         (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)

index ac4ae1e..bda79cb 100644
@@ -17,6 +17,7 @@ import re
 import tempfile
 import shutil
 from distutils.version import LooseVersion
+import itertools
 import utils
 import recipeparse
 

@@ -325,7 +326,79 @@ def main():
                     subdir_start = ""
 
                 updatedrecipes = set()
-                for diffitem in diff.iter_change_type('D'):
+                dirtyrecipes = set()
+                other_deletes = []
+                other_adds = []
+                for diffitem in diff.iter_change_type('R'):
+                    oldpath = diffitem.a_blob.path
+                    newpath = diffitem.b_blob.path
+                    skip = False
+                    for removedir in removedirs:
+                        # FIXME what about files moved into removedirs?
+                        if oldpath.startswith(removedir):
+                            skip = True
+                            break
+                    if skip:
+                        continue
+                    if oldpath.startswith(subdir_start):
+                        (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
+                        (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
+                        if oldtypename != newtypename:
+                            # This is most likely to be a .inc file renamed to a .bb - and since
+                            # there may be another recipe deleted at the same time we probably want
+                            # to consider that, so just treat it as a delete and an add
+                            logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
+                            other_deletes.append(diffitem)
+                            other_adds.append(diffitem)
+                        elif oldtypename == 'recipe':
+                            results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
+                            if len(results):
+                                recipe = results[0]
+                                logger.debug("Rename recipe %s to %s" % (recipe, newpath))
+                                recipe.filepath = newfilepath
+                                recipe.filename = newfilename
+                                recipe.save()
+                                update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
+                                updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
+                                updatedrecipes.add(os.path.join(newfilepath, newfilename))
+                            else:
+                                logger.warn("Renamed recipe %s could not be found" % oldpath)
+                                other_adds.append(diffitem)
+                        elif oldtypename == 'bbappend':
+                            results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
+                            if len(results):
+                                logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
+                                results[0].filepath = newfilepath
+                                results[0].filename = newfilename
+                                results[0].save()
+                            else:
+                                logger.warn("Renamed bbappend %s could not be found" % oldpath)
+                                other_adds.append(diffitem)
+                        elif oldtypename == 'machine':
+                            results = layermachines.filter(name=oldfilename)
+                            if len(results):
+                                logger.debug("Rename machine %s to %s" % (results[0], newfilename))
+                                results[0].name = newfilename
+                                results[0].save()
+                            else:
+                                logger.warn("Renamed machine %s could not be found" % oldpath)
+                                other_adds.append(diffitem)
+                        elif oldtypename == 'bbclass':
+                            results = layerclasses.filter(name=oldfilename)
+                            if len(results):
+                                logger.debug("Rename class %s to %s" % (results[0], newfilename))
+                                results[0].name = newfilename
+                                results[0].save()
+                            else:
+                                logger.warn("Renamed class %s could not be found" % oldpath)
+                                other_adds.append(diffitem)
+
+                        deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
+                        for dep in deps:
+                            dirtyrecipes.add(dep.recipe)
+
+
+                for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
                     path = diffitem.a_blob.path
                     if path.startswith(subdir_start):
                         skip = False

@@ -351,7 +424,7 @@ def main():
                         elif typename == 'bbclass':
                             layerclasses.filter(name=filename).delete()
 
-                for diffitem in diff.iter_change_type('A'):
+                for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
                     path = diffitem.b_blob.path
                     if path.startswith(subdir_start):
                         skip = False

@@ -384,7 +457,6 @@ def main():
                             bbclass.name = filename
                             bbclass.save()
 
-                dirtyrecipes = set()
                 for diffitem in diff.iter_change_type('M'):
                     path = diffitem.a_blob.path
                     if path.startswith(subdir_start):

index 3a5796d..0ed90a4 100644
@@ -38,7 +38,7 @@ class Branch(models.Model):
     updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
     update_environment = models.ForeignKey(PythonEnvironment, blank=True, null=True, on_delete=models.SET_NULL)
 
-    updated = models.DateTimeField(auto_now = True, default = datetime.now)
+    updated = models.DateTimeField(auto_now=True)
 
     class Meta:
         verbose_name_plural = "Branches"

@@ -73,7 +73,7 @@ class LayerItem(models.Model):
     index_preference = models.IntegerField('Preference', default=0, help_text='Number used to find preferred recipes in recipe search results (higher number is greater preference)')
     classic = models.BooleanField('Classic', default=False, help_text='Is this OE-Classic?')
 
-    updated = models.DateTimeField(auto_now = True)
+    updated = models.DateTimeField(auto_now=True)
 
     class Meta:
         verbose_name = "Layer"

@@ -136,7 +136,7 @@ class LayerBranch(models.Model):
     vcs_last_commit = models.DateTimeField('Last commit date', blank=True, null=True)
     actual_branch = models.CharField('Actual Branch', max_length=80, blank=True, help_text='Name of the actual branch in the repository matching the core branch')
 
-    updated = models.DateTimeField(auto_now = True)
+    updated = models.DateTimeField(auto_now=True)
 
     class Meta:
         verbose_name_plural = "Layer branches"

@@ -261,7 +261,7 @@ class Recipe(models.Model):
     provides = models.CharField(max_length=2048, blank=True)
     bbclassextend = models.CharField(max_length=100, blank=True)
     inherits = models.CharField(max_length=255, blank=True)
-    updated = models.DateTimeField(auto_now = True)
+    updated = models.DateTimeField(auto_now=True)
     blacklisted = models.CharField(max_length=255, blank=True)
 
     def vcs_web_url(self):

@@ -352,7 +352,7 @@ class Machine(models.Model):
     name = models.CharField(max_length=255)
     description = models.CharField(max_length=255)
 
-    updated = models.DateTimeField(auto_now = True)
+    updated = models.DateTimeField(auto_now=True)
 
     def vcs_web_url(self):
         url = self.layerbranch.file_url(os.path.join('conf/machine/%s.conf' % self.name))

index bda79cb..fd8d3d4 100644
@@ -34,6 +34,10 @@ except ImportError:
     sys.exit(1)
 
 
+class DryRunRollbackException(Exception):
+    pass
+
+
 def check_machine_conf(path, subdir_start):
     subpath = path[len(subdir_start):]
     res = conf_re.match(subpath)

@@ -187,429 +191,422 @@ def main():
     # why won't they just fix that?!)
     tinfoil.config_data.setVar('LICENSE', '')
 
-    transaction.enter_transaction_management()
-    transaction.managed(True)
     try:
-        layer = utils.get_layer(options.layer)
-        urldir = layer.get_fetch_dir()
-        repodir = os.path.join(fetchdir, urldir)
-
-        layerbranch = layer.get_layerbranch(options.branch)
-
-        branchname = options.branch
-        branchdesc = options.branch
-        if layerbranch:
-            if layerbranch.actual_branch:
-                branchname = layerbranch.actual_branch
-                branchdesc = "%s (%s)" % (options.branch, branchname)
-
-        # Collect repo info
-        repo = git.Repo(repodir)
-        assert repo.bare == False
-        try:
-            if options.nocheckout:
-                topcommit = repo.commit('HEAD')
-            else:
-                topcommit = repo.commit('origin/%s' % branchname)
-        except:
-            if layerbranch:
-                logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
-            else:
-                logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
-            transaction.rollback()
-            sys.exit(1)
-
-        newbranch = False
-        if not layerbranch:
-            # LayerBranch doesn't exist for this branch, create it
-            newbranch = True
-            layerbranch = LayerBranch()
-            layerbranch.layer = layer
-            layerbranch.branch = branch
-            layerbranch_source = layer.get_layerbranch('master')
-            if not layerbranch_source:
-                layerbranch_source = layer.get_layerbranch(None)
-            if layerbranch_source:
-                layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
-            layerbranch.save()
-            if layerbranch_source:
-                for maintainer in layerbranch_source.layermaintainer_set.all():
-                    maintainer.pk = None
-                    maintainer.id = None
-                    maintainer.layerbranch = layerbranch
-                    maintainer.save()
-                for dep in layerbranch_source.dependencies_set.all():
-                    dep.pk = None
-                    dep.id = None
-                    dep.layerbranch = layerbranch
-                    dep.save()
-
-        if layerbranch.vcs_subdir and not options.nocheckout:
-            # Find latest commit in subdirectory
-            # A bit odd to do it this way but apparently there's no other way in the GitPython API
-            topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
-            if not topcommit:
-                # This will error out if the directory is completely invalid or had never existed at this point
-                # If it previously existed but has since been deleted, you will get the revision where it was
-                # deleted - so we need to handle that case separately later
-                if newbranch:
-                    logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
-                elif layerbranch.vcs_subdir:
-                    logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
-                else:
-                    logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
-                transaction.rollback()
-                sys.exit(1)
-
-        layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
-        layerdir_start = os.path.normpath(layerdir) + os.sep
-        layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
-        layermachines = Machine.objects.filter(layerbranch=layerbranch)
-        layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
-        layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
-        if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
-            # Check out appropriate branch
-            if not options.nocheckout:
-                out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
-                out = utils.runcmd("git clean -f -x", repodir, logger=logger)
-
-            if layerbranch.vcs_subdir and not os.path.exists(layerdir):
-                if newbranch:
-                    logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
-                else:
-                    logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
-                transaction.rollback()
-                sys.exit(1)
+        with transaction.atomic():
+            layer = utils.get_layer(options.layer)
+            urldir = layer.get_fetch_dir()
+            repodir = os.path.join(fetchdir, urldir)
 
-            if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
-                logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
-                transaction.rollback()
-                sys.exit(1)
+            layerbranch = layer.get_layerbranch(options.branch)
 
-            logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+            branchname = options.branch
+            branchdesc = options.branch
+            if layerbranch:
+                if layerbranch.actual_branch:
+                    branchname = layerbranch.actual_branch
+                    branchdesc = "%s (%s)" % (options.branch, branchname)
 
+            # Collect repo info
+            repo = git.Repo(repodir)
+            assert repo.bare == False
             try:
-                config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
-            except recipeparse.RecipeParseError as e:
-                logger.error(str(e))
-                transaction.rollback()
+                if options.nocheckout:
+                    topcommit = repo.commit('HEAD')
+                else:
+                    topcommit = repo.commit('origin/%s' % branchname)
+            except:
+                if layerbranch:
+                    logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
+                else:
+                    logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
                 sys.exit(1)
 
-            if layerbranch.vcs_last_rev and not options.reload:
+            newbranch = False
+            if not layerbranch:
+                # LayerBranch doesn't exist for this branch, create it
+                newbranch = True
+                layerbranch = LayerBranch()
+                layerbranch.layer = layer
+                layerbranch.branch = branch
+                layerbranch_source = layer.get_layerbranch('master')
+                if not layerbranch_source:
+                    layerbranch_source = layer.get_layerbranch(None)
+                if layerbranch_source:
+                    layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
+                layerbranch.save()
+                if layerbranch_source:
+                    for maintainer in layerbranch_source.layermaintainer_set.all():
+                        maintainer.pk = None
+                        maintainer.id = None
+                        maintainer.layerbranch = layerbranch
+                        maintainer.save()
+                    for dep in layerbranch_source.dependencies_set.all():
+                        dep.pk = None
+                        dep.id = None
+                        dep.layerbranch = layerbranch
+                        dep.save()
+
+            if layerbranch.vcs_subdir and not options.nocheckout:
+                # Find latest commit in subdirectory
+                # A bit odd to do it this way but apparently there's no other way in the GitPython API
+                topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
+                if not topcommit:
+                    # This will error out if the directory is completely invalid or had never existed at this point
+                    # If it previously existed but has since been deleted, you will get the revision where it was
+                    # deleted - so we need to handle that case separately later
+                    if newbranch:
+                        logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+                    elif layerbranch.vcs_subdir:
+                        logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+                    else:
+                        logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
+                    sys.exit(1)
+
+            layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+            layerdir_start = os.path.normpath(layerdir) + os.sep
+            layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
+            layermachines = Machine.objects.filter(layerbranch=layerbranch)
+            layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
+            layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
+            if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
+                # Check out appropriate branch
+                if not options.nocheckout:
+                    out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
+                    out = utils.runcmd("git clean -f -x", repodir, logger=logger)
+
+                if layerbranch.vcs_subdir and not os.path.exists(layerdir):
+                    if newbranch:
+                        logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+                    else:
+                        logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+                    sys.exit(1)
+
+                if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
+                    logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
+                    sys.exit(1)
+
+                logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+
                 try:
-                    diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
-                except Exception as e:
-                    logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
-                    diff = None
-            else:
-                diff = None
-
-            # We handle recipes specially to try to preserve the same id
-            # when recipe upgrades happen (so that if a user bookmarks a
-            # recipe page it remains valid)
-            layerrecipes_delete = []
-            layerrecipes_add = []
-
-            # Check if any paths should be ignored because there are layers within this layer
-            removedirs = []
-            for root, dirs, files in os.walk(layerdir):
-                for diritem in dirs:
-                    if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
-                        removedirs.append(os.path.join(root, diritem) + os.sep)
-
-            if diff:
-                # Apply git changes to existing recipe list
-
-                if layerbranch.vcs_subdir:
-                    subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+                    config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
+                except recipeparse.RecipeParseError as e:
+                    logger.error(str(e))
+                    sys.exit(1)
+
+                if layerbranch.vcs_last_rev and not options.reload:
+                    try:
+                        diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
+                    except Exception as e:
+                        logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
+                        diff = None
                 else:
-                    subdir_start = ""
-
-                updatedrecipes = set()
-                dirtyrecipes = set()
-                other_deletes = []
-                other_adds = []
-                for diffitem in diff.iter_change_type('R'):
-                    oldpath = diffitem.a_blob.path
-                    newpath = diffitem.b_blob.path
-                    skip = False
-                    for removedir in removedirs:
-                        # FIXME what about files moved into removedirs?
-                        if oldpath.startswith(removedir):
-                            skip = True
-                            break
-                    if skip:
-                        continue
-                    if oldpath.startswith(subdir_start):
-                        (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
-                        (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
-                        if oldtypename != newtypename:
-                            # This is most likely to be a .inc file renamed to a .bb - and since
-                            # there may be another recipe deleted at the same time we probably want
-                            # to consider that, so just treat it as a delete and an add
-                            logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
-                            other_deletes.append(diffitem)
-                            other_adds.append(diffitem)
-                        elif oldtypename == 'recipe':
-                            results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
-                            if len(results):
-                                recipe = results[0]
-                                logger.debug("Rename recipe %s to %s" % (recipe, newpath))
-                                recipe.filepath = newfilepath
-                                recipe.filename = newfilename
-                                recipe.save()
-                                update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
-                                updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
-                                updatedrecipes.add(os.path.join(newfilepath, newfilename))
-                            else:
-                                logger.warn("Renamed recipe %s could not be found" % oldpath)
-                                other_adds.append(diffitem)
-                        elif oldtypename == 'bbappend':
-                            results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
-                            if len(results):
-                                logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
-                                results[0].filepath = newfilepath
-                                results[0].filename = newfilename
-                                results[0].save()
-                            else:
-                                logger.warn("Renamed bbappend %s could not be found" % oldpath)
-                                other_adds.append(diffitem)
-                        elif oldtypename == 'machine':
-                            results = layermachines.filter(name=oldfilename)
-                            if len(results):
-                                logger.debug("Rename machine %s to %s" % (results[0], newfilename))
-                                results[0].name = newfilename
-                                results[0].save()
-                            else:
-                                logger.warn("Renamed machine %s could not be found" % oldpath)
-                                other_adds.append(diffitem)
-                        elif oldtypename == 'bbclass':
-                            results = layerclasses.filter(name=oldfilename)
-                            if len(results):
-                                logger.debug("Rename class %s to %s" % (results[0], newfilename))
-                                results[0].name = newfilename
-                                results[0].save()
-                            else:
-                                logger.warn("Renamed class %s could not be found" % oldpath)
-                                other_adds.append(diffitem)
-
-                        deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
-                        for dep in deps:
-                            dirtyrecipes.add(dep.recipe)
+                    diff = None
 
+                # We handle recipes specially to try to preserve the same id
+                # when recipe upgrades happen (so that if a user bookmarks a
+                # recipe page it remains valid)
+                layerrecipes_delete = []
+                layerrecipes_add = []
 
-                for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
-                    path = diffitem.a_blob.path
-                    if path.startswith(subdir_start):
+                # Check if any paths should be ignored because there are layers within this layer
+                removedirs = []
+                for root, dirs, files in os.walk(layerdir):
+                    for diritem in dirs:
+                        if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
+                            removedirs.append(os.path.join(root, diritem) + os.sep)
+
+                if diff:
+                    # Apply git changes to existing recipe list
+
+                    if layerbranch.vcs_subdir:
+                        subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+                    else:
+                        subdir_start = ""
+
+                    updatedrecipes = set()
+                    dirtyrecipes = set()
+                    other_deletes = []
+                    other_adds = []
+                    for diffitem in diff.iter_change_type('R'):
+                        oldpath = diffitem.a_blob.path
+                        newpath = diffitem.b_blob.path
                         skip = False
                         for removedir in removedirs:
-                            if path.startswith(removedir):
+                            # FIXME what about files moved into removedirs?
+                            if oldpath.startswith(removedir):
                                 skip = True
                                 break
                         if skip:
                             continue
-                        (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
-                        if typename == 'recipe':
-                            values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
-                            if len(values):
-                                layerrecipes_delete.append(values[0])
-                                logger.debug("Mark %s for deletion" % values[0])
-                                updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+                        if oldpath.startswith(subdir_start):
+                            (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
+                            (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
+                            if oldtypename != newtypename:
+                                # This is most likely to be a .inc file renamed to a .bb - and since
+                                # there may be another recipe deleted at the same time we probably want
+                                # to consider that, so just treat it as a delete and an add
+                                logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
+                                other_deletes.append(diffitem)
+                                other_adds.append(diffitem)
+                            elif oldtypename == 'recipe':
+                                results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
+                                if len(results):
+                                    recipe = results[0]
+                                    logger.debug("Rename recipe %s to %s" % (recipe, newpath))
+                                    recipe.filepath = newfilepath
+                                    recipe.filename = newfilename
+                                    recipe.save()
+                                    update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
+                                    updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
+                                    updatedrecipes.add(os.path.join(newfilepath, newfilename))
+                                else:
+                                    logger.warn("Renamed recipe %s could not be found" % oldpath)
+                                    other_adds.append(diffitem)
+                            elif oldtypename == 'bbappend':
+                                results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
+                                if len(results):
+                                    logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
+                                    results[0].filepath = newfilepath
+                                    results[0].filename = newfilename
+                                    results[0].save()
+                                else:
+                                    logger.warn("Renamed bbappend %s could not be found" % oldpath)
+                                    other_adds.append(diffitem)
+                            elif oldtypename == 'machine':
+                                results = layermachines.filter(name=oldfilename)
+                                if len(results):
+                                    logger.debug("Rename machine %s to %s" % (results[0], newfilename))
+                                    results[0].name = newfilename
+                                    results[0].save()
+                                else:
+                                    logger.warn("Renamed machine %s could not be found" % oldpath)
+                                    other_adds.append(diffitem)
+                            elif oldtypename == 'bbclass':
+                                results = layerclasses.filter(name=oldfilename)
+                                if len(results):
+                                    logger.debug("Rename class %s to %s" % (results[0], newfilename))
+                                    results[0].name = newfilename
+                                    results[0].save()
+                                else:
+                                    logger.warn("Renamed class %s could not be found" % oldpath)
+                                    other_adds.append(diffitem)
+
+                            deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
+                            for dep in deps:
+                                dirtyrecipes.add(dep.recipe)
+
+
+                    for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
+                        path = diffitem.a_blob.path
+                        if path.startswith(subdir_start):
+                            skip = False
+                            for removedir in removedirs:
+                                if path.startswith(removedir):
+                                    skip = True
+                                    break
+                            if skip:
+                                continue
+                            (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+                            if typename == 'recipe':
+                                values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
+                                if len(values):
+                                    layerrecipes_delete.append(values[0])
+                                    logger.debug("Mark %s for deletion" % values[0])
+                                    updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+                                else:
+                                    logger.warn("Deleted recipe %s could not be found" % path)
+                            elif typename == 'bbappend':
+                                layerappends.filter(filepath=filepath).filter(filename=filename).delete()
+                            elif typename == 'machine':
+                                layermachines.filter(name=filename).delete()
+                            elif typename == 'bbclass':
+                                layerclasses.filter(name=filename).delete()
+
+                    for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
+                        path = diffitem.b_blob.path
+                        if path.startswith(subdir_start):
+                            skip = False
+                            for removedir in removedirs:
+                                if path.startswith(removedir):
+                                    skip = True
+                                    break
+                            if skip:
+                                continue
+                            (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+                            if typename == 'recipe':
+                                layerrecipes_add.append(os.path.join(repodir, path))
+                                logger.debug("Mark %s for addition" % path)
+                                updatedrecipes.add(os.path.join(filepath, filename))
+                            elif typename == 'bbappend':
+                                append = BBAppend()
+                                append.layerbranch = layerbranch
+                                append.filename = filename
+                                append.filepath = filepath
+                                append.save()
+                            elif typename == 'machine':
+                                machine = Machine()
+                                machine.layerbranch = layerbranch
+                                machine.name = filename
+                                update_machine_conf_file(os.path.join(repodir, path), machine)
+                                machine.save()
+                            elif typename == 'bbclass':
+                                bbclass = BBClass()
+                                bbclass.layerbranch = layerbranch
+                                bbclass.name = filename
+                                bbclass.save()
+
+                    for diffitem in diff.iter_change_type('M'):
+                        path = diffitem.a_blob.path
+                        if path.startswith(subdir_start):
+                            skip = False
+                            for removedir in removedirs:
+                                if path.startswith(removedir):
+                                    skip = True
+                                    break
+                            if skip:
+                                continue
+                            (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+                            if typename == 'recipe':
+                                logger.debug("Mark %s for update" % path)
+                                results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
+                                if results:
+                                    recipe = results[0]
+                                    update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
+                                    recipe.save()
+                                    updatedrecipes.add(recipe.full_path())
+                            elif typename == 'machine':
+                                results = layermachines.filter(name=filename)
+                                if results:
+                                    machine = results[0]
+                                    update_machine_conf_file(os.path.join(repodir, path), machine)
+                                    machine.save()
+
+                            deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
+                            for dep in deps:
+                                dirtyrecipes.add(dep.recipe)
+
+                    for recipe in dirtyrecipes:
+                        if not recipe.full_path() in updatedrecipes:
+                            update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+                else:
+                    # Collect recipe data from scratch
+
+                    layerrecipe_fns = []
+                    if options.fullreload:
+                        layerrecipes.delete()
+                    else:
+                        # First, check which recipes still exist
+                        layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
+                        for v in layerrecipe_values:
+                            root = os.path.join(layerdir, v['filepath'])
+                            fullpath = os.path.join(root, v['filename'])
+                            preserve = True
+                            if os.path.exists(fullpath):
+                                for removedir in removedirs:
+                                    if fullpath.startswith(removedir):
+                                        preserve = False
+                                        break
                             else:
-                                logger.warn("Deleted recipe %s could not be found" % path)
-                        elif typename == 'bbappend':
-                            layerappends.filter(filepath=filepath).filter(filename=filename).delete()
-                        elif typename == 'machine':
-                            layermachines.filter(name=filename).delete()
-                        elif typename == 'bbclass':
-                            layerclasses.filter(name=filename).delete()
-
-                for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
-                    path = diffitem.b_blob.path
-                    if path.startswith(subdir_start):
-                        skip = False
-                        for removedir in removedirs:
-                            if path.startswith(removedir):
-                                skip = True
-                                break
-                        if skip:
-                            continue
-                        (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
-                        if typename == 'recipe':
-                            layerrecipes_add.append(os.path.join(repodir, path))
-                            logger.debug("Mark %s for addition" % path)
-                            updatedrecipes.add(os.path.join(filepath, filename))
-                        elif typename == 'bbappend':
-                            append = BBAppend()
-                            append.layerbranch = layerbranch
-                            append.filename = filename
-                            append.filepath = filepath
-                            append.save()
-                        elif typename == 'machine':
-                            machine = Machine()
-                            machine.layerbranch = layerbranch
-                            machine.name = filename
-                            update_machine_conf_file(os.path.join(repodir, path), machine)
-                            machine.save()
-                        elif typename == 'bbclass':
-                            bbclass = BBClass()
-                            bbclass.layerbranch = layerbranch
-                            bbclass.name = filename
-                            bbclass.save()
-
-                for diffitem in diff.iter_change_type('M'):
-                    path = diffitem.a_blob.path
-                    if path.startswith(subdir_start):
-                        skip = False
-                        for removedir in removedirs:
-                            if path.startswith(removedir):
-                                skip = True
-                                break
-                        if skip:
-                            continue
-                        (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
-                        if typename == 'recipe':
-                            logger.debug("Mark %s for update" % path)
-                            results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
-                            if results:
+                                preserve = False
+
+                            if preserve:
+                                # Recipe still exists, update it
+                                results = layerrecipes.filter(id=v['id'])[:1]
                                 recipe = results[0]
-                                update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
-                                recipe.save()
-                                updatedrecipes.add(recipe.full_path())
-                        elif typename == 'machine':
-                            results = layermachines.filter(name=filename)
-                            if results:
-                                machine = results[0]
-                                update_machine_conf_file(os.path.join(repodir, path), machine)
+                                update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+                            else:
+                                # Recipe no longer exists, mark it for later on
+                                layerrecipes_delete.append(v)
+                            layerrecipe_fns.append(fullpath)
+
+                    layermachines.delete()
+                    layerappends.delete()
+                    layerclasses.delete()
+                    for root, dirs, files in os.walk(layerdir):
+                        if '.git' in dirs:
+                            dirs.remove('.git')
+                        for diritem in dirs[:]:
+                            fullpath = os.path.join(root, diritem) + os.sep
+                            if fullpath in removedirs:
+                                dirs.remove(diritem)
+                        for f in files:
+                            fullpath = os.path.join(root, f)
+                            (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
+                            if typename == 'recipe':
+                                if fullpath not in layerrecipe_fns:
+                                    layerrecipes_add.append(fullpath)
+                            elif typename == 'bbappend':
+                                append = BBAppend()
+                                append.layerbranch = layerbranch
+                                append.filename = f
+                                append.filepath = os.path.relpath(root, layerdir)
+                                append.save()
+                            elif typename == 'machine':
+                                machine = Machine()
+                                machine.layerbranch = layerbranch
+                                machine.name = filename
+                                update_machine_conf_file(fullpath, machine)
                                 machine.save()
+                            elif typename == 'bbclass':
+                                bbclass = BBClass()
+                                bbclass.layerbranch = layerbranch
+                                bbclass.name = filename
+                                bbclass.save()
+
+                for added in layerrecipes_add:
+                    # This is good enough without actually parsing the file
+                    (pn, pv) = split_recipe_fn(added)
+                    oldid = -1
+                    for deleted in layerrecipes_delete:
+                        if deleted['pn'] == pn:
+                            oldid = deleted['id']
+                            layerrecipes_delete.remove(deleted)
+                            break
+                    if oldid > -1:
+                        # Reclaim a record we would have deleted
+                        results = Recipe.objects.filter(id=oldid)[:1]
+                        recipe = results[0]
+                        logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
+                    else:
+                        # Create new record
+                        logger.debug("Add new recipe %s" % added)
+                        recipe = Recipe()
+                    recipe.layerbranch = layerbranch
+                    recipe.filename = os.path.basename(added)
+                    root = os.path.dirname(added)
+                    recipe.filepath = os.path.relpath(root, layerdir)
+                    update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+                    recipe.save()
 
-                        deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
-                        for dep in deps:
-                            dirtyrecipes.add(dep.recipe)
+                for deleted in layerrecipes_delete:
+                    logger.debug("Delete %s" % deleted)
+                    results = Recipe.objects.filter(id=deleted['id'])[:1]
+                    recipe = results[0]
+                    recipe.delete()
 
-                for recipe in dirtyrecipes:
-                    if not recipe.full_path() in updatedrecipes:
-                        update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+                # Save repo info
+                layerbranch.vcs_last_rev = topcommit.hexsha
+                layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
             else:
-                # Collect recipe data from scratch
+                logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
+
+            layerbranch.vcs_last_fetch = datetime.now()
+            layerbranch.save()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
 
-                layerrecipe_fns = []
-                if options.fullreload:
-                    layerrecipes.delete()
-                else:
-                    # First, check which recipes still exist
-                    layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
-                    for v in layerrecipe_values:
-                        root = os.path.join(layerdir, v['filepath'])
-                        fullpath = os.path.join(root, v['filename'])
-                        preserve = True
-                        if os.path.exists(fullpath):
-                            for removedir in removedirs:
-                                if fullpath.startswith(removedir):
-                                    preserve = False
-                                    break
-                        else:
-                            preserve = False
-
-                        if preserve:
-                            # Recipe still exists, update it
-                            results = layerrecipes.filter(id=v['id'])[:1]
-                            recipe = results[0]
-                            update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
-                        else:
-                            # Recipe no longer exists, mark it for later on
-                            layerrecipes_delete.append(v)
-                        layerrecipe_fns.append(fullpath)
-
-                layermachines.delete()
-                layerappends.delete()
-                layerclasses.delete()
-                for root, dirs, files in os.walk(layerdir):
-                    if '.git' in dirs:
-                        dirs.remove('.git')
-                    for diritem in dirs[:]:
-                        fullpath = os.path.join(root, diritem) + os.sep
-                        if fullpath in removedirs:
-                            dirs.remove(diritem)
-                    for f in files:
-                        fullpath = os.path.join(root, f)
-                        (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
-                        if typename == 'recipe':
-                            if fullpath not in layerrecipe_fns:
-                                layerrecipes_add.append(fullpath)
-                        elif typename == 'bbappend':
-                            append = BBAppend()
-                            append.layerbranch = layerbranch
-                            append.filename = f
-                            append.filepath = os.path.relpath(root, layerdir)
-                            append.save()
-                        elif typename == 'machine':
-                            machine = Machine()
-                            machine.layerbranch = layerbranch
-                            machine.name = filename
-                            update_machine_conf_file(fullpath, machine)
-                            machine.save()
-                        elif typename == 'bbclass':
-                            bbclass = BBClass()
-                            bbclass.layerbranch = layerbranch
-                            bbclass.name = filename
-                            bbclass.save()
-
-            for added in layerrecipes_add:
-                # This is good enough without actually parsing the file
-                (pn, pv) = split_recipe_fn(added)
-                oldid = -1
-                for deleted in layerrecipes_delete:
-                    if deleted['pn'] == pn:
-                        oldid = deleted['id']
-                        layerrecipes_delete.remove(deleted)
-                        break
-                if oldid > -1:
-                    # Reclaim a record we would have deleted
-                    results = Recipe.objects.filter(id=oldid)[:1]
-                    recipe = results[0]
-                    logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
-                else:
-                    # Create new record
-                    logger.debug("Add new recipe %s" % added)
-                    recipe = Recipe()
-                recipe.layerbranch = layerbranch
-                recipe.filename = os.path.basename(added)
-                root = os.path.dirname(added)
-                recipe.filepath = os.path.relpath(root, layerdir)
-                update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
-                recipe.save()
-
-            for deleted in layerrecipes_delete:
-                logger.debug("Delete %s" % deleted)
-                results = Recipe.objects.filter(id=deleted['id'])[:1]
-                recipe = results[0]
-                recipe.delete()
-
-            # Save repo info
-            layerbranch.vcs_last_rev = topcommit.hexsha
-            layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
-        else:
-            logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
-
-        layerbranch.vcs_last_fetch = datetime.now()
-        layerbranch.save()
-
-        if options.dryrun:
-            transaction.rollback()
-        else:
-            transaction.commit()
 
     except KeyboardInterrupt:
-        transaction.rollback()
         logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
         sys.exit(254)
+    except SystemExit:
+        raise
+    except DryRunRollbackException:
+        pass
     except:
         import traceback
         traceback.print_exc()
-        transaction.rollback()
-    finally:
-        transaction.leave_transaction_management()
 
     shutil.rmtree(tempdir)
     sys.exit(0)

index b170460..cbd29ad 100644
@@ -24,19 +24,19 @@ router.register(r'machines', restviews.MachineViewSet)
 
 urlpatterns = patterns('',
     url(r'^$',
-        RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',))),
+        RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)), permanent=False),
         name='frontpage'),
 
     url(r'^api/', include(router.urls)),
 
     url(r'^layers/$',
-        RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)))),
+        RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)), permanent=False)),
     url(r'^layer/(?P<slug>[-\w]+)/$',
-        RedirectParamsView.as_view(), {'redirect_name': 'layer_item', 'branch':'master'}),
+        RedirectParamsView.as_view(permanent=False), {'redirect_name': 'layer_item', 'branch':'master'}),
     url(r'^recipes/$',
-        RedirectView.as_view(url=reverse_lazy('recipe_search', args=('master',)))),
+        RedirectView.as_view(url=reverse_lazy('recipe_search', args=('master',)), permanent=False)),
     url(r'^machines/$',
-        RedirectView.as_view(url=reverse_lazy('machine_search', args=('master',)))),
+        RedirectView.as_view(url=reverse_lazy('machine_search', args=('master',)), permanent=False)),
  
     url(r'^submit/$', edit_layer_view, {'template_name': 'layerindex/submitlayer.html'}, name="submit_layer"),
     url(r'^submit/thanks$',

@@ -107,7 +107,7 @@ urlpatterns = patterns('',
             template_name='layerindex/about.html'),
             name="about"),
     url(r'^oe-classic/$',
-        RedirectView.as_view(url=reverse_lazy('classic_recipe_search')),
+        RedirectView.as_view(url=reverse_lazy('classic_recipe_search'), permanent=False),
             name='classic'),
     url(r'^oe-classic/recipes/$',
         ClassicRecipeSearchView.as_view(

index ab5e2d5..3313290 100644
@@ -11,7 +11,7 @@ from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView,
 
 urlpatterns = patterns('',
     url(r'^$', 
-        RedirectParamsView.as_view(), {'redirect_name': 'layer_list'}),
+        RedirectParamsView.as_view(permanent=False), {'redirect_name': 'layer_list'}),
     url(r'^layers/$',
         LayerListView.as_view(
             template_name='layerindex/layers.html'),

index 98fc734..649635c 100644
@@ -16,6 +16,6 @@ urlpatterns = patterns('',
     url(r'^admin/', include(admin.site.urls)),
     url(r'^accounts/', include('registration.backends.default.urls')),
     url(r'^captcha/', include('captcha.urls')),
-    url(r'.*', RedirectView.as_view(url='/layerindex/')),
+    url(r'.*', RedirectView.as_view(url='/layerindex/', permanent=False)),
 )
 

index b3374c9..fa5c3b2 100644
@@ -99,17 +99,27 @@ Setup instructions:
         next to your login name at the top right, click on the
         newly added layer entry, and then click on "Publish Layer".
 
-5. Set the site name (as displayed in the top bar and page titles) by
+5. If you need to support multiple branches of OpenEmbedded/BitBake
+   where some require Python 2.x and others require Python 3.x, then
+   you will need to set up "Python environment" records through the
+   admin interface to correspond to these so that the right Python
+   version gets used to parse the branch, and then set the
+   "Update environment" field on each branch record to point to the
+   appropriate environment. If you're using virtualenv you will need
+   separate virtual environments set up for Python 2 and 3 which you
+   should point to in the Python environment record.
+
+6. Set the site name (as displayed in the top bar and page titles) by
    going into the admin interface (http://127.0.0.1:8000/admin/),
    clicking on "Sites" at the bottom, and editing the first entry,
    setting "Display name" to the desired name.
 
-6. You may wish to customise some of the page templates to suit your
+7. You may wish to customise some of the page templates to suit your
    installation, in particular:
    * templates/base.html
    * templates/layerindex/about.html
 
-7. To use layerindex-web with Docker containers, refer to docker/README
+8. To use layerindex-web with Docker containers, refer to docker/README
    keeping in mind you'll need to set up Docker properly as part of the
    setup process.
 

@@ -156,7 +166,7 @@ Bundled nv.d3.js is redistributed under the Apache License 2.0.
 
 Bundled d3.js is redistributed under the BSD License.
 
-All other content is copyright (C) 2013 Intel Corporation and licensed
-under the MIT license (unless otherwise noted) - see COPYING.MIT for
-details.
+All other content is copyright (C) 2013-2016 Intel Corporation and
+licensed under the MIT license (unless otherwise noted) - see
+COPYING.MIT for details.
 

index 77f660b..accb954 100644
@@ -115,3 +115,4 @@ admin.site.register(BBAppend, BBAppendAdmin)
 admin.site.register(BBClass, BBClassAdmin)
 admin.site.register(RecipeChangeset, RecipeChangesetAdmin)
 admin.site.register(ClassicRecipe, ClassicRecipeAdmin)
+admin.site.register(PythonEnvironment)

219 lines not on this page · raw message

index e753fbe..3a5796d 100644
@@ -1,6 +1,6 @@
 # layerindex-web - model definitions
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 

@@ -14,12 +14,29 @@ import re
 import posixpath
 
 
+class PythonEnvironment(models.Model):
+    name = models.CharField(max_length=50)
+    python_command = models.CharField(max_length=255, default='python')
+    virtualenv_path = models.CharField(max_length=255, blank=True)
+
+    def get_command(self):
+        if self.virtualenv_path:
+            cmd = '. %s/bin/activate; %s' % (self.virtualenv_path, self.python_command)
+        else:
+            cmd = self.python_command
+        return cmd
+
+    def __str__(self):
+        return self.name
+
+
 class Branch(models.Model):
     name = models.CharField(max_length=50)
     bitbake_branch = models.CharField(max_length=50)
     short_description = models.CharField(max_length=50, blank=True)
     sort_priority = models.IntegerField(blank=True, null=True)
     updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
+    update_environment = models.ForeignKey(PythonEnvironment, blank=True, null=True, on_delete=models.SET_NULL)
 
     updated = models.DateTimeField(auto_now = True, default = datetime.now)
 

index 157d46f..ac7de5e 100755
@@ -177,7 +177,12 @@ def main():
                 urldir = layer.get_fetch_dir()
                 repodir = os.path.join(fetchdir, urldir)
 
-                cmd = 'python update_layer.py -l %s -b %s' % (layer.name, branch)
+                branchobj = utils.get_branch(branch)
+                if branchobj.update_environment:
+                    cmdprefix = branchobj.update_environment.get_command()
+                else:
+                    cmdprefix = 'python'
+                cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch)
                 if options.reload:
                     cmd += ' --reload'
                 if options.fullreload: