Layer index improvements

index a905c347..d02e4f91 100644
@@ -259,38 +259,40 @@ def _check_url_branch(kwargs):
 def publish_view(request, name):
     if not (request.user.is_authenticated() and request.user.has_perm('layerindex.publish_layer')):
         raise PermissionDenied
-    layeritem = get_object_or_404(LayerItem, name=name)
-    layerbranch = get_object_or_404(LayerBranch, layer=layeritem)
-    layer_url = request.build_absolute_uri(reverse('layer_item', args=(layerbranch.branch, layeritem.name)))
-    maintainers = get_list_or_404(LayerMaintainer, layerbranch=layerbranch)
-    from_email = settings.SUBMIT_EMAIL_FROM
-    subjecttext = get_template('layerindex/publishemailsubject.txt')
-    bodytext = get_template('layerindex/publishemail.txt')
-    maintainer_names = [m.name for m in maintainers]
-    # find appropriate help contact
-    help_contact = None
-    for user in User.objects.all():
-        if user.username != 'root' and (user.is_staff or user.is_superuser) and user.is_active:
-            help_contact = user
-            break
-
-    # create subject from subject template
-    d = {
-        'layer_name': layeritem.name,
-        'site_name': request.META['HTTP_HOST'],
-    }
-    subject = subjecttext.render(d).rstrip()
-
-    #create body from body template
-    d = {
-        'maintainers': maintainer_names,
-        'layer_name': layeritem.name,
-        'layer_url': layer_url,
-        'help_contact': help_contact,
-    }
-    body = bodytext.render(d)
-
-    tasks.send_email.apply_async((subject, body, from_email, [m.email for m in maintainers]))
+
+    if getattr(settings, 'SEND_PUBLISH_EMAIL', True):
+        layeritem = get_object_or_404(LayerItem, name=name)
+        layerbranch = get_object_or_404(LayerBranch, layer=layeritem)
+        layer_url = request.build_absolute_uri(reverse('layer_item', args=(layerbranch.branch, layeritem.name)))
+        maintainers = get_list_or_404(LayerMaintainer, layerbranch=layerbranch)
+        from_email = settings.SUBMIT_EMAIL_FROM
+        subjecttext = get_template('layerindex/publishemailsubject.txt')
+        bodytext = get_template('layerindex/publishemail.txt')
+        maintainer_names = [m.name for m in maintainers]
+        # find appropriate help contact
+        help_contact = None
+        for user in User.objects.all():
+            if user.username != 'root' and (user.is_staff or user.is_superuser) and user.is_active:
+                help_contact = user
+                break
+
+        # create subject from subject template
+        d = {
+            'layer_name': layeritem.name,
+            'site_name': request.META['HTTP_HOST'],
+        }
+        subject = subjecttext.render(d).rstrip()
+
+        #create body from body template
+        d = {
+            'maintainers': maintainer_names,
+            'layer_name': layeritem.name,
+            'layer_url': layer_url,
+            'help_contact': help_contact,
+        }
+        body = bodytext.render(d)
+
+        tasks.send_email.apply_async((subject, body, from_email, [m.email for m in maintainers]))
 
     return _statuschange(request, name, 'P')
 

index c3075a5e..3d70aaa5 100644
@@ -227,6 +227,9 @@ FORCE_REVIEW_HTTPS = False
 SUBMIT_EMAIL_FROM = '[email protected]'
 SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
 
+# Send email to maintainer(s) when their layer is published
+SEND_PUBLISH_EMAIL = True
+
 # RabbitMQ settings
 RABBIT_BROKER = 'amqp://'
 RABBIT_BACKEND = 'rpc://'

index bbfaba9a..018d7bcd 100644
@@ -228,7 +228,6 @@ def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir, ski
             recipedep.save()
 
         for filedep in recipedeps_delete:
-            logger.debug('%s: removing %s' % (recipe.layerbranch, filedep))
             recipedeps.filter(path=filedep).delete()
 
     except KeyboardInterrupt:

index 96d4feb8..7d522f84 100644
@@ -10,7 +10,6 @@ TODO:
 
 Bugs
 * Duplication of first maintainer when editing to add a second?
-* GitPython sometimes reports renamed files as type M but we don't handle that properly e.g. OE-Core 59285b324f6d9ed270b0bef209ef5da22a620a83 086308aa2a5e332de6f00ed397c4a55d132f158f
 
 Other
 * Full-text search on layer contents

index 018d7bcd..272fe51a 100644
@@ -636,7 +636,7 @@ def main():
                                 bbclass.save()
 
                     for diffitem in diff.iter_change_type('M'):
-                        path = diffitem.a_blob.path
+                        path = diffitem.b_blob.path
                         if path.startswith(subdir_start):
                             skip = False
                             for removedir in removedirs:

index 3cb59691..312f7a30 100644
@@ -198,3 +198,4 @@ admin.site.register(Patch)
 admin.site.register(RecipeChangeset, RecipeChangesetAdmin)
 admin.site.register(ClassicRecipe, ClassicRecipeAdmin)
 admin.site.register(PythonEnvironment)
+admin.site.register(SiteNotice)

index db8e3fa3..7cf20ede 100644
@@ -1,11 +1,13 @@
 # layerindex-web - custom context processor
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013, 2018 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
-from layerindex.models import Branch, LayerItem
+from layerindex.models import Branch, LayerItem, SiteNotice
 from django.contrib.sites.models import Site
+from django.db.models import Q
+from datetime import datetime
 
 def layerindex_context(request):
     import settings

@@ -20,4 +22,5 @@ def layerindex_context(request):
         'oe_classic': Branch.objects.filter(name='oe-classic'),
         'site_name': site_name,
         'rrs_enabled': 'rrs' in settings.INSTALLED_APPS,
+        'notices': SiteNotice.objects.filter(disabled=False).filter(Q(expires__isnull=True) | Q(expires__gte=datetime.now())),
     }

new file mode 100644
index 00000000..630700cf
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layerindex', '0013_patch'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SiteNotice',
+            fields=[
+                ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)),
+                ('text', models.TextField(help_text='Text to show in the notice. A limited subset of HTML is supported for formatting.')),
+                ('level', models.CharField(choices=[('I', 'Info'), ('S', 'Success'), ('W', 'Warning'), ('E', 'Error')], help_text='Level of notice to display', default='I', max_length=1)),
+                ('disabled', models.BooleanField(verbose_name='Disabled', help_text='Use to temporarily disable this notice', default=False)),
+                ('expires', models.DateTimeField(blank=True, help_text='Optional date/time when this notice will stop showing', null=True)),
+            ],
+        ),
+    ]

index ff164baa..891f5dfb 100644
@@ -1,6 +1,6 @@
 # layerindex-web - model definitions
 #
-# Copyright (C) 2013-2016 Intel Corporation
+# Copyright (C) 2013-2018 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 

@@ -658,3 +658,26 @@ class RecipeChange(models.Model):
     def reset_fields(self):
         for fieldname in self.RECIPE_VARIABLE_MAP:
             setattr(self, fieldname, getattr(self.recipe, fieldname))
+
+class SiteNotice(models.Model):
+    NOTICE_LEVEL_CHOICES = [
+        ('I', 'Info'),
+        ('S', 'Success'),
+        ('W', 'Warning'),
+        ('E', 'Error'),
+    ]
+    text = models.TextField(help_text='Text to show in the notice. A limited subset of HTML is supported for formatting.')
+    level = models.CharField(max_length=1, choices=NOTICE_LEVEL_CHOICES, default='I', help_text='Level of notice to display')
+    disabled = models.BooleanField('Disabled', default=False, help_text='Use to temporarily disable this notice')
+    expires = models.DateTimeField(blank=True, null=True, help_text='Optional date/time when this notice will stop showing')
+
+    def __str__(self):
+        prefix = ''
+        if self.expires and datetime.now() >= self.expires:
+            prefix = '[expired] '
+        elif self.disabled:
+            prefix = '[disabled] '
+        return '%s%s' % (prefix, self.text)
+
+    def text_sanitised(self):
+        return utils.sanitise_html(self.text)

index f8c5fd45..fe994439 100644
@@ -14,6 +14,7 @@ import time
 import fcntl
 import signal
 import codecs
+from bs4 import BeautifulSoup
 
 def get_branch(branchname):
     from layerindex.models import Branch

@@ -376,6 +377,7 @@ def setup_core_layer_sys_path(settings, branchname):
     core_layerdir = os.path.join(core_repodir, core_layerbranch.vcs_subdir)
     sys.path.insert(0, os.path.join(core_layerdir, 'lib'))
 
+
 def run_command_interruptible(cmd):
     """
     Run a command with output displayed on the console, but ensure any Ctrl+C is

@@ -404,3 +406,15 @@ def run_command_interruptible(cmd):
     finally:
         signal.signal(signal.SIGINT, signal.SIG_DFL)
     return process.returncode, buf
+
+
+def sanitise_html(html):
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup.findAll(True):
+        if tag.name not in ['strong', 'em', 'b', 'i', 'p', 'ul', 'ol', 'li', 'br', 'p']:
+            tag.hidden = True
+        elif tag.attrs:
+            tag.attrs = []
+
+    return soup.renderContents()
+

index 5ca7cf6e..7facd2b6 100644
@@ -1,5 +1,6 @@
 amqp==2.2.2
 anyjson==0.3.3
+beautifulsoup4==4.6.0
 billiard==3.5.0.3
 celery==4.1.0
 confusable-homoglyphs==3.0.0

index 38373ab8..a5e97cd5 100644
@@ -89,6 +89,11 @@
     {% endblock %}
 
 {% block contenttag %}<div id="content" class="container top-padded">{% endblock %}
+        {% if notices %}
+            {% for notice in notices %}
+            <div class="alert {% if notice.level == 'I' %}alert-info{% elif notice.level == 'S' %}alert-success{% elif notice.level == 'W' %}{% elif notice.level == 'E' %}alert-error{% endif %}">{{ notice.text_sanitised|safe|urlize }}</div>
+            {% endfor %}
+        {% endif %}
         {% if messages %}
             {% for message in messages %}
             <div{% if message.tags %} class="alert {{ message.tags }}"{% endif %}>{{ message }}</div>

new file mode 100755
index 00000000..b02759ea
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+
+# Import layers from another layer index instance
+#
+# Copyright (C) 2018 Intel Corporation
+# Author: Paul Eggleton <[email protected]>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+import optparse
+import re
+import glob
+import logging
+import subprocess
+import urllib.request
+import json
+
+sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
+
+import utils
+from layerconfparse import LayerConfParse
+
+class DryRunRollbackException(Exception):
+    pass
+
+logger = utils.logger_create('LayerIndexImport')
+
+
+
+def main():
+    valid_layer_name = re.compile('[-\w]+$')
+
+    parser = optparse.OptionParser(
+        usage = """
+    %prog [options] <url>""")
+
+    parser.add_option("-n", "--dry-run",
+            help = "Don't write any data back to the database",
+            action="store_true", dest="dryrun")
+    parser.add_option("-d", "--debug",
+            help = "Enable debug output",
+            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+    parser.add_option("-q", "--quiet",
+            help = "Hide all output except error messages",
+            action="store_const", const=logging.ERROR, dest="loglevel")
+
+    options, args = parser.parse_args(sys.argv)
+
+    if len(args) < 2:
+        print("Please specify URL of the layer index")
+        sys.exit(1)
+
+    layerindex_url = args[1]
+
+    utils.setup_django()
+    import settings
+    from layerindex.models import Branch, LayerItem, LayerBranch, LayerDependency, LayerMaintainer, LayerNote
+    from django.db import transaction
+
+    logger.setLevel(options.loglevel)
+
+    fetchdir = settings.LAYER_FETCH_DIR
+    if not fetchdir:
+        logger.error("Please set LAYER_FETCH_DIR in settings.py")
+        sys.exit(1)
+
+    if not os.path.exists(fetchdir):
+        os.makedirs(fetchdir)
+
+    if not layerindex_url.endswith('/'):
+        layerindex_url += '/'
+    if not '/layerindex/api/' in layerindex_url:
+        layerindex_url += '/layerindex/api/'
+
+    rq = urllib.request.Request(layerindex_url)
+    data = urllib.request.urlopen(rq).read()
+    jsdata = json.loads(data.decode('utf-8'))
+
+    branches_url = jsdata['branches']
+    layers_url = jsdata['layerItems']
+    layerdeps_url = jsdata['layerDependencies']
+    layerbranches_url = jsdata['layerBranches']
+    layermaintainers_url = jsdata.get('layerMaintainers', None)
+    layernotes_url = jsdata.get('layerNotes', None)
+
+    logger.debug('Getting branches')
+
+    # Get branches (we assume the ones we want are already there, so skip any that aren't)
+    rq = urllib.request.Request(branches_url)
+    data = urllib.request.urlopen(rq).read()
+    jsdata = json.loads(data.decode('utf-8'))
+    branch_idmap = {}
+    for branchjs in jsdata:
+        res = Branch.objects.filter(name=branchjs['name'])
+        if res:
+            branch = res.first()
+            branch_idmap[branchjs['id']] = branch
+
+    try:
+        with transaction.atomic():
+            # Get layers
+            logger.debug('Importing layers')
+            rq = urllib.request.Request(layers_url)
+            data = urllib.request.urlopen(rq).read()
+            jsdata = json.loads(data.decode('utf-8'))
+
+            layer_idmap = {}
+            exclude_fields = ['id', 'updated']
+            for layerjs in jsdata:
+                res = LayerItem.objects.filter(name=layerjs['name'])
+                if res:
+                    # Already have this layer
+                    logger.debug('Skipping layer %s, already in database' % layerjs['name'])
+                    layer_idmap[layerjs['id']] = res[0]
+                    continue
+                logger.debug('Adding layer %s' % layerjs['name'])
+                layeritem = LayerItem()
+                for key, value in layerjs.items():
+                    if key in exclude_fields:
+                        continue
+                    setattr(layeritem, key, value)
+                layeritem.save()
+                layer_idmap[layerjs['id']] = layeritem
+
+            # Get layer branches
+            logger.debug('Importing layer branches')
+            rq = urllib.request.Request(layerbranches_url)
+            data = urllib.request.urlopen(rq).read()
+            jsdata = json.loads(data.decode('utf-8'))
+
+            layerbranch_idmap = {}
+            exclude_fields = ['id', 'layer', 'branch', 'vcs_last_fetch', 'vcs_last_rev', 'vcs_last_commit', 'yp_compatible_version', 'updated']
+            for layerbranchjs in jsdata:
+                branch = branch_idmap.get(layerbranchjs['branch'], None)
+                if not branch:
+                    # We don't have this branch, skip it
+                    logger.debug('Skipping layerbranch %s, branch not imported' % layerbranchjs['id'])
+                    continue
+                layer = layer_idmap.get(layerbranchjs['layer'], None)
+                if not layer:
+                    # We didn't import this layer, skip it
+                    logger.debug('Skipping layerbranch %s, layer not imported' % layerbranchjs['id'])
+                    continue
+                res = LayerBranch.objects.filter(layer=layer).filter(branch=branch)
+                if res:
+                    # The layerbranch already exists (this will occur for layers
+                    # that already existed, since we need to have those in layer_idmap
+                    # to be able to import layer dependencies)
+                    logger.debug('Skipping layerbranch %s, already exists' % layerbranchjs['id'])
+                    continue
+
+                layerbranch = LayerBranch()
+                for key, value in layerbranchjs.items():
+                    if key in exclude_fields:
+                        continue
+                    setattr(layerbranch, key, value)
+                layerbranch.branch = branch
+                layerbranch.layer = layer
+                layerbranch.save()
+                layerbranch_idmap[layerbranchjs['id']] = layerbranch
+
+            # Get layer dependencies
+            logger.debug('Importing layer dependencies')
+            rq = urllib.request.Request(layerdeps_url)
+            data = urllib.request.urlopen(rq).read()
+            jsdata = json.loads(data.decode('utf-8'))
+
+            exclude_fields = ['id', 'layerbranch', 'dependency', 'updated']
+            for layerdepjs in jsdata:
+                layerbranch = layerbranch_idmap.get(layerdepjs['layerbranch'], None)
+                if not layerbranch:
+                    # We didn't import this layerbranch, skip it
+                    continue
+                dependency = layer_idmap.get(layerdepjs['dependency'], None)
+                if not dependency:
+                    # We didn't import the dependency, skip it
+                    continue
+
+                layerdep = LayerDependency()
+                for key, value in layerdepjs.items():
+                    if key in exclude_fields:
+                        continue
+                    setattr(layerdep, key, value)
+                layerdep.layerbranch = layerbranch
+                layerdep.dependency = dependency
+                layerdep.save()
+
+            if layermaintainers_url:
+                # Get layer maintainers (only available in latest code)
+                logger.debug('Importing layer maintainers')
+                rq = urllib.request.Request(layermaintainers_url)
+                data = urllib.request.urlopen(rq).read()
+                jsdata = json.loads(data.decode('utf-8'))
+
+                exclude_fields = ['id', 'layerbranch']
+                for layermaintainerjs in jsdata:
+                    layerbranch = layerbranch_idmap.get(layermaintainerjs['layerbranch'], None)
+                    if not layerbranch:
+                        # We didn't import this layerbranch, skip it
+                        continue
+
+                    layermaintainer = LayerMaintainer()
+                    for key, value in layermaintainerjs.items():
+                        if key in exclude_fields:
+                            continue
+                        setattr(layermaintainer, key, value)
+                    layermaintainer.layerbranch = layerbranch
+                    layermaintainer.save()
+
+            if layernotes_url:
+                # Get layer notes (only available in latest code)
+                logger.debug('Importing layer notes')
+                rq = urllib.request.Request(layernotes_url)
+                data = urllib.request.urlopen(rq).read()
+                jsdata = json.loads(data.decode('utf-8'))
+
+                exclude_fields = ['id', 'layer']
+                for layernotejs in jsdata:
+                    layer = layer_idmap.get(layernotejs['layer'], None)
+                    if not layer:
+                        # We didn't import this layer, skip it
+                        continue
+                    res = LayerNote.objects.filter(layer=layer).filter(text=layernotejs['text'])
+                    if res:
+                        # The note already exists (this will occur for layers
+                        # that already existed, since we need to have those in layer_idmap
+                        # to be able to import layer dependencies)
+                        logger.debug('Skipping note %s, already exists' % layernotejs['id'])
+                        continue
+
+                    layernote = LayerNote()
+                    for key, value in layernotejs.items():
+                        if key in exclude_fields:
+                            continue
+                        setattr(layernote, key, value)
+                    layernote.layer = layer
+                    layernote.save()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
+    except DryRunRollbackException:
+        pass
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

index f980e64a..be7807d2 100755
@@ -68,7 +68,7 @@ def main():
     try:
         with transaction.atomic():
             def recipe_pn_query(pn):
-                return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference')
+                return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('-layerbranch__layer__index_preference')
 
             recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N'])
             for recipe in recipequery: